From 7f35ddecd7838f20273968a77c2881cffbd86b59 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Tue, 13 Dec 2022 14:41:43 +0200 Subject: [PATCH 1/4] feat(azuredevops): Create client, fetch project ids --- plugins/source/azuredevops/client/client.go | 29 ++++++++++-- plugins/source/azuredevops/client/spec.go | 6 +++ .../source/azuredevops/docs/tables/README.md | 3 +- .../docs/tables/azuredevops_core_projects.md | 16 +++++++ plugins/source/azuredevops/go.mod | 1 + plugins/source/azuredevops/go.sum | 3 ++ .../azuredevops/resources/plugin/plugin.go | 3 +- .../azuredevops/resources/plugin/tables.go | 10 ++++ .../resources/services/core/projects.go | 46 +++++++++++++++++++ 9 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 plugins/source/azuredevops/client/spec.go create mode 100644 plugins/source/azuredevops/docs/tables/azuredevops_core_projects.md create mode 100644 plugins/source/azuredevops/resources/plugin/tables.go create mode 100644 plugins/source/azuredevops/resources/services/core/projects.go diff --git a/plugins/source/azuredevops/client/client.go b/plugins/source/azuredevops/client/client.go index fe01bf2a83bf29..2d83500ded8907 100644 --- a/plugins/source/azuredevops/client/client.go +++ b/plugins/source/azuredevops/client/client.go @@ -2,24 +2,45 @@ package client import ( "context" + "fmt" "github.com/cloudquery/plugin-sdk/schema" "github.com/cloudquery/plugin-sdk/specs" + "github.com/microsoft/azure-devops-go-api/azuredevops" "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) -type AzureDevOpsClient struct { +type Client struct { + logger zerolog.Logger + Connection *azuredevops.Connection } -func (*AzureDevOpsClient) Logger() *zerolog.Logger { +func (*Client) Logger() *zerolog.Logger { return &log.Logger } -func (*AzureDevOpsClient) ID() string { +func (*Client) ID() string { return "AzureDevOpsClient" } func New(ctx context.Context, logger zerolog.Logger, s specs.Source) (schema.ClientMeta, error) { - return &AzureDevOpsClient{}, nil + var spec Spec + err := s.UnmarshalSpec(&spec) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal GitHub spec: %w", err) + } + + if spec.PersonalAccessToken == "" { + return nil, fmt.Errorf("missing personal access token in configuration") + } + if spec.OrganizationURL == "" { + return nil, fmt.Errorf("missing organization url in configuration") + } + + connection := azuredevops.NewPatConnection(spec.OrganizationURL, spec.PersonalAccessToken) + return &Client{ + logger: logger, + Connection: connection, + }, nil } diff --git a/plugins/source/azuredevops/client/spec.go b/plugins/source/azuredevops/client/spec.go new file mode 100644 index 00000000000000..0268895960abb6 --- /dev/null +++ b/plugins/source/azuredevops/client/spec.go @@ -0,0 +1,6 @@ +package client + +type Spec struct { + PersonalAccessToken string `json:"personal_access_token"` + OrganizationURL string `json:"organization_url"` +} diff --git a/plugins/source/azuredevops/docs/tables/README.md b/plugins/source/azuredevops/docs/tables/README.md index 95e4ad5a1e084f..b0eb70cadf9945 100644 --- a/plugins/source/azuredevops/docs/tables/README.md +++ b/plugins/source/azuredevops/docs/tables/README.md @@ -1,2 +1,3 @@ # Source Plugin: azuredevops -## Tables \ No newline at end of file +## Tables +- [azuredevops_core_projects](azuredevops_core_projects.md) \ No newline at end of file diff --git a/plugins/source/azuredevops/docs/tables/azuredevops_core_projects.md b/plugins/source/azuredevops/docs/tables/azuredevops_core_projects.md new file mode 100644 index 00000000000000..cc9abbe17d5e00 --- /dev/null +++ b/plugins/source/azuredevops/docs/tables/azuredevops_core_projects.md @@ -0,0 +1,16 @@ +# Table: azuredevops_core_projects + + + +The primary key for this table is **_cq_id**. + + + +## Columns +| Name | Type | +| ------------- | ------------- | +|_cq_source_name|String| +|_cq_sync_time|Timestamp| +|_cq_id (PK)|UUID| +|_cq_parent_id|UUID| +|id|UUID| \ No newline at end of file diff --git a/plugins/source/azuredevops/go.mod b/plugins/source/azuredevops/go.mod index fd93bf3b9b551e..472848a97f670d 100644 --- a/plugins/source/azuredevops/go.mod +++ b/plugins/source/azuredevops/go.mod @@ -4,6 +4,7 @@ go 1.19 require ( github.com/cloudquery/plugin-sdk v1.12.0 + github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 github.com/rs/zerolog v1.28.0 ) diff --git a/plugins/source/azuredevops/go.sum b/plugins/source/azuredevops/go.sum index fa85daeee50e02..29aa6763ca3ac7 100644 --- a/plugins/source/azuredevops/go.sum +++ b/plugins/source/azuredevops/go.sum @@ -113,6 +113,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -144,6 +145,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 h1:YH424zrwLTlyHSH/GzLMJeu5zhYVZSx5RQxGKm1h96s= +github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5/go.mod h1:PoGiBqKSQK1vIfQ+yVaFcGjDySHvym6FM1cNYnwzbrY= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= diff --git a/plugins/source/azuredevops/resources/plugin/plugin.go b/plugins/source/azuredevops/resources/plugin/plugin.go index e29d767615e131..2eb048ccb67770 100644 --- a/plugins/source/azuredevops/resources/plugin/plugin.go +++ b/plugins/source/azuredevops/resources/plugin/plugin.go @@ -3,7 +3,6 @@ package plugin import ( "github.com/cloudquery/cloudquery/plugins/source/azuredevops/client" "github.com/cloudquery/plugin-sdk/plugins" - "github.com/cloudquery/plugin-sdk/schema" ) var ( @@ -14,7 +13,7 @@ func Plugin() *plugins.SourcePlugin { return plugins.NewSourcePlugin( "azuredevops", Version, - []*schema.Table{}, + tables(), client.New, ) } diff --git a/plugins/source/azuredevops/resources/plugin/tables.go b/plugins/source/azuredevops/resources/plugin/tables.go new file mode 100644 index 00000000000000..f3df973946c7ba --- /dev/null +++ b/plugins/source/azuredevops/resources/plugin/tables.go @@ -0,0 +1,10 @@ +package plugin + +import ( + "github.com/cloudquery/cloudquery/plugins/source/azuredevops/resources/services/core" + "github.com/cloudquery/plugin-sdk/schema" +) + +func tables() []*schema.Table { + return []*schema.Table{core.Projects()} +} diff --git a/plugins/source/azuredevops/resources/services/core/projects.go b/plugins/source/azuredevops/resources/services/core/projects.go new file mode 100644 index 00000000000000..8a91d0379c2f01 --- /dev/null +++ b/plugins/source/azuredevops/resources/services/core/projects.go @@ -0,0 +1,46 @@ +package core + +import ( + "context" + + "github.com/cloudquery/cloudquery/plugins/source/azuredevops/client" + "github.com/cloudquery/plugin-sdk/schema" + "github.com/microsoft/azure-devops-go-api/azuredevops/core" +) + +func Projects() *schema.Table { + return &schema.Table{ + Name: "azuredevops_core_projects", + Resolver: fetchProjects, + Columns: []schema.Column{ + { + Name: "id", + Type: schema.TypeUUID, + Resolver: schema.PathResolver("Id"), + }, + }, + } +} + +func fetchProjects(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + cl := meta.(*client.Client) + coreClient, err := core.NewClient(ctx, cl.Connection) + if err != nil { + return err + } + + stateFilter := &core.ProjectStateValues.All + projects, err := coreClient.GetProjects(ctx, core.GetProjectsArgs{StateFilter: stateFilter}) + if err != nil { + return err + } + res <- projects.Value + for projects.ContinuationToken != "" { + projects, err = coreClient.GetProjects(ctx, core.GetProjectsArgs{StateFilter: stateFilter, ContinuationToken: &projects.ContinuationToken}) + if err != nil { + return err + } + res <- projects.Value + } + return nil +} From 27323ab43af3261f7d522153912ef810eaf8fce5 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Tue, 13 Dec 2022 15:04:10 +0200 Subject: [PATCH 2/4] refactor: Embed connection --- plugins/source/azuredevops/client/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/source/azuredevops/client/client.go b/plugins/source/azuredevops/client/client.go index 2d83500ded8907..623263661023be 100644 --- a/plugins/source/azuredevops/client/client.go +++ b/plugins/source/azuredevops/client/client.go @@ -12,8 +12,8 @@ import ( ) type Client struct { - logger zerolog.Logger - Connection *azuredevops.Connection + logger zerolog.Logger + *azuredevops.Connection } func (*Client) Logger() *zerolog.Logger { From 5434df748cdaf9964d9597e10985da2fd509c8a8 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Tue, 13 Dec 2022 15:06:50 +0200 Subject: [PATCH 3/4] refactor: Simplify loop --- .../resources/services/core/projects.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/plugins/source/azuredevops/resources/services/core/projects.go b/plugins/source/azuredevops/resources/services/core/projects.go index 8a91d0379c2f01..99067de37b2be8 100644 --- a/plugins/source/azuredevops/resources/services/core/projects.go +++ b/plugins/source/azuredevops/resources/services/core/projects.go @@ -29,18 +29,21 @@ func fetchProjects(ctx context.Context, meta schema.ClientMeta, parent *schema.R return err } - stateFilter := &core.ProjectStateValues.All - projects, err := coreClient.GetProjects(ctx, core.GetProjectsArgs{StateFilter: stateFilter}) - if err != nil { - return err - } - res <- projects.Value - for projects.ContinuationToken != "" { - projects, err = coreClient.GetProjects(ctx, core.GetProjectsArgs{StateFilter: stateFilter, ContinuationToken: &projects.ContinuationToken}) + input := core.GetProjectsArgs{StateFilter: &core.ProjectStateValues.All} + for { + projects, err := coreClient.GetProjects(ctx, input) if err != nil { return err } + res <- projects.Value + + if len(projects.ContinuationToken) == 0 { + break + } + + input.ContinuationToken = &projects.ContinuationToken } + return nil } From 0e33317e24f44fc4e466ef167b661b9147cd47a1 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Tue, 13 Dec 2022 15:28:10 +0200 Subject: [PATCH 4/4] feat: Update to latest SDK --- plugins/source/azuredevops/client/client.go | 2 +- plugins/source/azuredevops/go.mod | 2 +- plugins/source/azuredevops/go.sum | 4 ++-- .../source/azuredevops/resources/services/core/projects.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/source/azuredevops/client/client.go b/plugins/source/azuredevops/client/client.go index 623263661023be..31aafab43730e2 100644 --- a/plugins/source/azuredevops/client/client.go +++ b/plugins/source/azuredevops/client/client.go @@ -6,7 +6,7 @@ import ( "github.com/cloudquery/plugin-sdk/schema" "github.com/cloudquery/plugin-sdk/specs" - "github.com/microsoft/azure-devops-go-api/azuredevops" + "github.com/microsoft/azure-devops-go-api/azuredevops/v6" "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) diff --git a/plugins/source/azuredevops/go.mod b/plugins/source/azuredevops/go.mod index 472848a97f670d..937a32496a5924 100644 --- a/plugins/source/azuredevops/go.mod +++ b/plugins/source/azuredevops/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/cloudquery/plugin-sdk v1.12.0 - github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 + github.com/microsoft/azure-devops-go-api/azuredevops/v6 v6.0.1 github.com/rs/zerolog v1.28.0 ) diff --git a/plugins/source/azuredevops/go.sum b/plugins/source/azuredevops/go.sum index 29aa6763ca3ac7..59bac973680b6a 100644 --- a/plugins/source/azuredevops/go.sum +++ b/plugins/source/azuredevops/go.sum @@ -145,8 +145,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 h1:YH424zrwLTlyHSH/GzLMJeu5zhYVZSx5RQxGKm1h96s= -github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5/go.mod h1:PoGiBqKSQK1vIfQ+yVaFcGjDySHvym6FM1cNYnwzbrY= +github.com/microsoft/azure-devops-go-api/azuredevops/v6 v6.0.1 h1:ACnM5CwgTH6OSQHErzZDrotEG0rffPdJxtF/WOWglAw= +github.com/microsoft/azure-devops-go-api/azuredevops/v6 v6.0.1/go.mod h1:1bdoUWt0f/xMYxDzy6FwSvDBxBzJmw99HV//P7b4cyE= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= diff --git a/plugins/source/azuredevops/resources/services/core/projects.go b/plugins/source/azuredevops/resources/services/core/projects.go index 99067de37b2be8..07a785f7c24616 100644 --- a/plugins/source/azuredevops/resources/services/core/projects.go +++ b/plugins/source/azuredevops/resources/services/core/projects.go @@ -5,7 +5,7 @@ import ( "github.com/cloudquery/cloudquery/plugins/source/azuredevops/client" "github.com/cloudquery/plugin-sdk/schema" - "github.com/microsoft/azure-devops-go-api/azuredevops/core" + "github.com/microsoft/azure-devops-go-api/azuredevops/v6/core" ) func Projects() *schema.Table {