diff --git a/plugins/source/azuredevops/Makefile b/plugins/source/azuredevops/Makefile index a106b265b55523..ad042fd343b003 100644 --- a/plugins/source/azuredevops/Makefile +++ b/plugins/source/azuredevops/Makefile @@ -12,5 +12,10 @@ gen-docs: rm -rf ./docs/tables/* go run main.go doc ./docs/tables +.PHONY: gen-code +gen-code: + grep -rl '// Code generated by codegen; DO NOT EDIT.' resources/services/* | xargs rm + go run codegen/main.go + .PHONY: gen -gen: gen-docs \ No newline at end of file +gen: gen-code gen-docs \ No newline at end of file diff --git a/plugins/source/azuredevops/codegen/main.go b/plugins/source/azuredevops/codegen/main.go new file mode 100644 index 00000000000000..d61f2899a33976 --- /dev/null +++ b/plugins/source/azuredevops/codegen/main.go @@ -0,0 +1,18 @@ +package main + +import ( + "log" + + "github.com/cloudquery/cloudquery/plugins/source/azuredevops/codegen/recipes" +) + +func main() { + for _, resource := range recipes.Resources { + if err := resource.Generate(); err != nil { + log.Fatal(err) + } + } + if err := recipes.GenerateTablesList(recipes.Resources); err != nil { + log.Fatal(err) + } +} diff --git a/plugins/source/azuredevops/codegen/recipes/base.go b/plugins/source/azuredevops/codegen/recipes/base.go new file mode 100644 index 00000000000000..3940f64cc5a0f0 --- /dev/null +++ b/plugins/source/azuredevops/codegen/recipes/base.go @@ -0,0 +1,120 @@ +package recipes + +import ( + "bytes" + "embed" + "fmt" + "go/format" + "os" + "path" + "reflect" + "runtime" + "text/template" + + "github.com/cloudquery/plugin-sdk/codegen" + "github.com/cloudquery/plugin-sdk/schema" + "github.com/google/uuid" + "github.com/iancoleman/strcase" +) + +//go:embed templates/*.go.tpl +var templatesFS embed.FS +var Resources []*Resource + +type Resource struct { + Service string + SubService string + Struct interface{} + Table *codegen.TableDefinition +} + +func writeTemplateContentToFile(dir string, filePath string, buff bytes.Buffer) error { + outputPath := path.Join(dir, "../..", filePath) + outputDir := path.Dir(outputPath) + if err := os.MkdirAll(outputDir, os.ModePerm); err != nil { + return fmt.Errorf("failed to create directory %s: %w", outputDir, err) + } + + content := buff.Bytes() + formattedContent, err := format.Source(content) + if err != nil { + fmt.Printf("failed to format source: %s: %v\n", filePath, err) + } else { + content = formattedContent + } + + if err := os.WriteFile(outputPath, content, 0644); err != nil { + return fmt.Errorf("failed to write file %s: %w", filePath, err) + } + return nil +} + +func renderTemplate(name string, filePath string, data interface{}) error { + _, filename, _, ok := runtime.Caller(0) + if !ok { + return fmt.Errorf("failed to get caller information") + } + dir := path.Dir(filename) + + tpl, err := template.New(name).Funcs(template.FuncMap{"ToCamel": strcase.ToCamel}).ParseFS(templatesFS, "templates/*.go.tpl") + if err != nil { + return fmt.Errorf("failed to parse azureDevops templates: %w", err) + } + tpl, err = tpl.ParseFS(codegen.TemplatesFS, "templates/*.go.tpl") + if err != nil { + return fmt.Errorf("failed to parse sdk template: %w", err) + } + + var buff bytes.Buffer + if err := tpl.Execute(&buff, data); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + + return writeTemplateContentToFile(dir, filePath, buff) +} + +func (resource *Resource) generate() error { + return renderTemplate("resource.go.tpl", path.Join("resources", "services", resource.Service, resource.SubService+".go"), resource) +} + +func isUUID(fieldType reflect.Type) bool { + fieldKind := fieldType.Kind() + + if fieldKind == reflect.Ptr { + return isUUID(fieldType.Elem()) + } + + return fieldType == reflect.TypeOf(uuid.UUID{}) +} + +func typeTransformer(field reflect.StructField) (schema.ValueType, error) { + if isUUID(field.Type) { + return schema.TypeUUID, nil + } + + return codegen.DefaultTypeTransformer(field) +} + +func (resource *Resource) Generate() error { + var err error + resource.Table, err = codegen.NewTableFromStruct( + fmt.Sprintf("azuredevops_%s_%s", resource.Service, resource.SubService), + resource.Struct, + codegen.WithTypeTransformer(typeTransformer), + ) + + if err != nil { + return err + } + + resource.Table.Resolver = "fetch" + strcase.ToCamel(resource.SubService) + if err := resource.generate(); err != nil { + return err + } + + return nil +} + +func GenerateTablesList(resources []*Resource) error { + return renderTemplate("tables.go.tpl", path.Join("resources", "plugin", "tables.go"), resources) +} diff --git a/plugins/source/azuredevops/codegen/recipes/core.go b/plugins/source/azuredevops/codegen/recipes/core.go new file mode 100644 index 00000000000000..14a52eb99fd919 --- /dev/null +++ b/plugins/source/azuredevops/codegen/recipes/core.go @@ -0,0 +1,20 @@ +package recipes + +import ( + "github.com/microsoft/azure-devops-go-api/azuredevops/v6/core" +) + +func init() { + resources := []*Resource{ + { + SubService: "projects", + Struct: &core.TeamProjectReference{}, + }, + } + + for _, resource := range resources { + resource.Service = "core" + } + + Resources = append(Resources, resources...) +} diff --git a/plugins/source/azuredevops/codegen/recipes/templates/resource.go.tpl b/plugins/source/azuredevops/codegen/recipes/templates/resource.go.tpl new file mode 100644 index 00000000000000..a9737d4a6671b7 --- /dev/null +++ b/plugins/source/azuredevops/codegen/recipes/templates/resource.go.tpl @@ -0,0 +1,11 @@ +// Code generated by codegen; DO NOT EDIT. + +package {{.Service}} + +import ( + "github.com/cloudquery/plugin-sdk/schema" +) + +func {{.SubService | ToCamel}}() *schema.Table { + return &schema.Table{{template "table.go.tpl" .Table}} +} \ No newline at end of file diff --git a/plugins/source/azuredevops/codegen/recipes/templates/tables.go.tpl b/plugins/source/azuredevops/codegen/recipes/templates/tables.go.tpl new file mode 100644 index 00000000000000..d63ac30497f762 --- /dev/null +++ b/plugins/source/azuredevops/codegen/recipes/templates/tables.go.tpl @@ -0,0 +1,18 @@ +// Code generated by codegen; DO NOT EDIT. + +package plugin + +import ( + {{- range .}} + "github.com/cloudquery/cloudquery/plugins/source/azuredevops/resources/services/{{.Service}}" + {{- end}} + "github.com/cloudquery/plugin-sdk/schema" +) + +func tables() []*schema.Table { + return []*schema.Table{ + {{- range .}} + {{.Service}}.{{.SubService | ToCamel}}(), + {{- end}} + } +} \ No newline at end of file diff --git a/plugins/source/azuredevops/docs/tables/azuredevops_core_projects.md b/plugins/source/azuredevops/docs/tables/azuredevops_core_projects.md index cc9abbe17d5e00..f5bb6026115801 100644 --- a/plugins/source/azuredevops/docs/tables/azuredevops_core_projects.md +++ b/plugins/source/azuredevops/docs/tables/azuredevops_core_projects.md @@ -13,4 +13,13 @@ The primary key for this table is **_cq_id**. |_cq_sync_time|Timestamp| |_cq_id (PK)|UUID| |_cq_parent_id|UUID| -|id|UUID| \ No newline at end of file +|abbreviation|String| +|default_team_image_url|String| +|description|String| +|id|UUID| +|last_update_time|JSON| +|name|String| +|revision|Int| +|state|String| +|url|String| +|visibility|String| \ No newline at end of file diff --git a/plugins/source/azuredevops/go.mod b/plugins/source/azuredevops/go.mod index 41c15a601dfe5a..e179f64d6a9bbc 100644 --- a/plugins/source/azuredevops/go.mod +++ b/plugins/source/azuredevops/go.mod @@ -4,6 +4,8 @@ go 1.19 require ( github.com/cloudquery/plugin-sdk v1.12.5 + github.com/google/uuid v1.3.0 + github.com/iancoleman/strcase v0.2.0 github.com/microsoft/azure-devops-go-api/azuredevops/v6 v6.0.1 github.com/rs/zerolog v1.28.0 ) @@ -12,7 +14,6 @@ require ( github.com/getsentry/sentry-go v0.15.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect @@ -22,6 +23,7 @@ require ( github.com/spf13/cobra v1.6.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/thoas/go-funk v0.9.2 // indirect + golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 // indirect golang.org/x/net v0.2.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.2.0 // indirect diff --git a/plugins/source/azuredevops/go.sum b/plugins/source/azuredevops/go.sum index 0201fdd9f9c7bc..4da10b6dd27b19 100644 --- a/plugins/source/azuredevops/go.sum +++ b/plugins/source/azuredevops/go.sum @@ -126,6 +126,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 h1:o95KDiV/b1xdkumY5 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3/go.mod h1:hTxjzRcX49ogbTGVJ1sM5mz5s+SSgiGIyL3jjPxl32E= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -202,6 +204,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 h1:yZNXmy+j/JpX19vZkVktWqAo7Gny4PBWYYK3zskGpx4= +golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/plugins/source/azuredevops/resources/plugin/tables.go b/plugins/source/azuredevops/resources/plugin/tables.go index f3df973946c7ba..0e5bb78e021f34 100644 --- a/plugins/source/azuredevops/resources/plugin/tables.go +++ b/plugins/source/azuredevops/resources/plugin/tables.go @@ -1,3 +1,5 @@ +// Code generated by codegen; DO NOT EDIT. + package plugin import ( @@ -6,5 +8,7 @@ import ( ) func tables() []*schema.Table { - return []*schema.Table{core.Projects()} + return []*schema.Table{ + core.Projects(), + } } diff --git a/plugins/source/azuredevops/resources/services/core/projects.go b/plugins/source/azuredevops/resources/services/core/projects.go index 07a785f7c24616..7706ae0edf151a 100644 --- a/plugins/source/azuredevops/resources/services/core/projects.go +++ b/plugins/source/azuredevops/resources/services/core/projects.go @@ -1,11 +1,9 @@ +// Code generated by codegen; DO NOT EDIT. + package core import ( - "context" - - "github.com/cloudquery/cloudquery/plugins/source/azuredevops/client" "github.com/cloudquery/plugin-sdk/schema" - "github.com/microsoft/azure-devops-go-api/azuredevops/v6/core" ) func Projects() *schema.Table { @@ -13,37 +11,56 @@ func Projects() *schema.Table { Name: "azuredevops_core_projects", Resolver: fetchProjects, Columns: []schema.Column{ + { + Name: "abbreviation", + Type: schema.TypeString, + Resolver: schema.PathResolver("Abbreviation"), + }, + { + Name: "default_team_image_url", + Type: schema.TypeString, + Resolver: schema.PathResolver("DefaultTeamImageUrl"), + }, + { + Name: "description", + Type: schema.TypeString, + Resolver: schema.PathResolver("Description"), + }, { Name: "id", Type: schema.TypeUUID, Resolver: schema.PathResolver("Id"), }, + { + Name: "last_update_time", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("LastUpdateTime"), + }, + { + Name: "name", + Type: schema.TypeString, + Resolver: schema.PathResolver("Name"), + }, + { + Name: "revision", + Type: schema.TypeInt, + Resolver: schema.PathResolver("Revision"), + }, + { + Name: "state", + Type: schema.TypeString, + Resolver: schema.PathResolver("State"), + }, + { + Name: "url", + Type: schema.TypeString, + Resolver: schema.PathResolver("Url"), + }, + { + Name: "visibility", + Type: schema.TypeString, + Resolver: schema.PathResolver("Visibility"), + }, }, } } - -func fetchProjects(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { - cl := meta.(*client.Client) - coreClient, err := core.NewClient(ctx, cl.Connection) - if err != nil { - return err - } - - input := core.GetProjectsArgs{StateFilter: &core.ProjectStateValues.All} - for { - projects, err := coreClient.GetProjects(ctx, input) - if err != nil { - return err - } - - res <- projects.Value - - if len(projects.ContinuationToken) == 0 { - break - } - - input.ContinuationToken = &projects.ContinuationToken - } - - return nil -} diff --git a/plugins/source/azuredevops/resources/services/core/projects_fetch.go b/plugins/source/azuredevops/resources/services/core/projects_fetch.go new file mode 100644 index 00000000000000..0e7a3b1255bbb3 --- /dev/null +++ b/plugins/source/azuredevops/resources/services/core/projects_fetch.go @@ -0,0 +1,35 @@ +package core + +import ( + "context" + + "github.com/cloudquery/cloudquery/plugins/source/azuredevops/client" + "github.com/cloudquery/plugin-sdk/schema" + "github.com/microsoft/azure-devops-go-api/azuredevops/v6/core" +) + +func fetchProjects(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + cl := meta.(*client.Client) + coreClient, err := core.NewClient(ctx, cl.Connection) + if err != nil { + return err + } + + input := core.GetProjectsArgs{StateFilter: &core.ProjectStateValues.All} + for { + projects, err := coreClient.GetProjects(ctx, input) + if err != nil { + return err + } + + res <- projects.Value + + if len(projects.ContinuationToken) == 0 { + break + } + + input.ContinuationToken = &projects.ContinuationToken + } + + return nil +}