diff --git a/plugins/source/gcp/codegen/recipes/run.go b/plugins/source/gcp/codegen/recipes/run.go index d0134e5f659a9b..16654ffa4640e8 100644 --- a/plugins/source/gcp/codegen/recipes/run.go +++ b/plugins/source/gcp/codegen/recipes/run.go @@ -1,19 +1,27 @@ package recipes import ( - run "cloud.google.com/go/run/apiv2" - pb "cloud.google.com/go/run/apiv2/runpb" + pbv2 "cloud.google.com/go/run/apiv2/runpb" + runv1 "google.golang.org/api/run/v1" ) func init() { resources := []*Resource{ { - SubService: "services", - Struct: &pb.Service{}, - NewFunction: run.NewServicesClient, - RegisterServer: pb.RegisterServicesServer, - ListFunction: (&pb.UnimplementedServicesServer{}).ListServices, - Description: "https://cloud.google.com/run/docs/reference/rest/v2/projects.locations.services#Service", + SubService: "locations", + Struct: &runv1.Location{}, + SkipFetch: true, + SkipMock: true, + Description: "https://cloud.google.com/api-gateway/docs/reference/rest/v1/projects.locations#Location", + Relations: []string{"Services()"}, + }, + { + SubService: "services", + Struct: &pbv2.Service{}, + Description: "https://cloud.google.com/run/docs/reference/rest/v2/projects.locations.services#Service", + ChildTable: true, + SkipFetch: true, + SkipMock: true, }, } @@ -21,9 +29,6 @@ func init() { resource.Service = "run" resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" - resource.MockImports = []string{"cloud.google.com/go/run/apiv2"} - resource.ProtobufImport = "cloud.google.com/go/run/apiv2/runpb" - resource.RequestStructFields = `Parent: "projects/" + c.ProjectId + "locations/-",` } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/docs/tables/README.md b/plugins/source/gcp/docs/tables/README.md index 0c88386ce446c0..6ce9e5b2caa3cb 100644 --- a/plugins/source/gcp/docs/tables/README.md +++ b/plugins/source/gcp/docs/tables/README.md @@ -46,7 +46,8 @@ - [gcp_resourcemanager_folders](gcp_resourcemanager_folders.md) - [gcp_resourcemanager_projects](gcp_resourcemanager_projects.md) - [gcp_resourcemanager_project_policies](gcp_resourcemanager_project_policies.md) -- [gcp_run_services](gcp_run_services.md) +- [gcp_run_locations](gcp_run_locations.md) + - [gcp_run_services](gcp_run_services.md) - [gcp_secretmanager_secrets](gcp_secretmanager_secrets.md) - [gcp_serviceusage_services](gcp_serviceusage_services.md) - [gcp_sql_instances](gcp_sql_instances.md) diff --git a/plugins/source/gcp/docs/tables/gcp_run_locations.md b/plugins/source/gcp/docs/tables/gcp_run_locations.md new file mode 100644 index 00000000000000..2fd1f15e952680 --- /dev/null +++ b/plugins/source/gcp/docs/tables/gcp_run_locations.md @@ -0,0 +1,25 @@ +# Table: gcp_run_locations + +https://cloud.google.com/api-gateway/docs/reference/rest/v1/projects.locations#Location + +The primary key for this table is **_cq_id**. + +## Relations + +The following tables depend on gcp_run_locations: + - [gcp_run_services](gcp_run_services.md) + +## Columns + +| Name | Type | +| ------------- | ------------- | +|_cq_source_name|String| +|_cq_sync_time|Timestamp| +|_cq_id (PK)|UUID| +|_cq_parent_id|UUID| +|project_id|String| +|display_name|String| +|labels|JSON| +|location_id|String| +|metadata|IntArray| +|name|String| \ No newline at end of file diff --git a/plugins/source/gcp/docs/tables/gcp_run_services.md b/plugins/source/gcp/docs/tables/gcp_run_services.md index c472917451ebef..fd5860c2c8dc17 100644 --- a/plugins/source/gcp/docs/tables/gcp_run_services.md +++ b/plugins/source/gcp/docs/tables/gcp_run_services.md @@ -4,6 +4,10 @@ https://cloud.google.com/run/docs/reference/rest/v2/projects.locations.services# The primary key for this table is **_cq_id**. +## Relations + +This table depends on [gcp_run_locations](gcp_run_locations.md). + ## Columns | Name | Type | diff --git a/plugins/source/gcp/go.mod b/plugins/source/gcp/go.mod index f99e2dc3417d8b..7064551ddda99e 100644 --- a/plugins/source/gcp/go.mod +++ b/plugins/source/gcp/go.mod @@ -32,15 +32,15 @@ require ( github.com/stretchr/testify v1.8.1 github.com/thoas/go-funk v0.9.3-0.20221027085339-5573bc209e28 golang.org/x/sync v0.1.0 - google.golang.org/api v0.103.0 - google.golang.org/genproto v0.0.0-20221205194025-8222ab48f5fc + google.golang.org/api v0.105.0 + google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 google.golang.org/grpc v1.51.0 google.golang.org/protobuf v1.28.1 ) require ( cloud.google.com/go v0.107.0 // indirect - cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/compute/metadata v0.2.2 // indirect cloud.google.com/go/longrunning v0.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/getsentry/sentry-go v0.15.0 // indirect @@ -62,7 +62,7 @@ require ( golang.org/x/net v0.2.0 // indirect golang.org/x/oauth2 v0.2.0 // indirect golang.org/x/sys v0.2.0 // indirect - golang.org/x/text v0.4.0 // indirect + golang.org/x/text v0.5.0 // indirect golang.org/x/tools v0.2.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/plugins/source/gcp/go.sum b/plugins/source/gcp/go.sum index e5e6a98c3356d1..2227d87c0f86ad 100644 --- a/plugins/source/gcp/go.sum +++ b/plugins/source/gcp/go.sum @@ -27,8 +27,8 @@ cloud.google.com/go/billing v1.7.0 h1:Xkii76HWELHwBtkQVZvqmSo9GTr0O+tIbRNnMcGdlg cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= -cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= -cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.2 h1:aWKAjYaBaOSrpKl57+jnS/3fJRQnxL7TvR/u1VVbt6k= +cloud.google.com/go/compute/metadata v0.2.2/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/container v1.8.0 h1:wib6oN0XQT4r2M4RFfEz0AdfslEQc7H5SDLCV8EZ/lU= cloud.google.com/go/container v1.8.0/go.mod h1:1ZHYR+yKrEeDs/iBoUNgwyjWNo+WaPN8PRhVnWVCcuA= cloud.google.com/go/containeranalysis v0.6.0 h1:2824iym832ljKdVpCBnpqm5K94YT/uHTVhNF+dRTXPI= @@ -383,8 +383,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -455,8 +455,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8= +google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -495,8 +495,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20221205194025-8222ab48f5fc h1:nUKKji0AarrQKh6XpFEpG3p1TNztxhe7C8TcUvDgXqw= -google.golang.org/genproto v0.0.0-20221205194025-8222ab48f5fc/go.mod h1:1dOng4TWOomJrDGhpXjfCD35wQC6jnC7HpRmOFRqEV0= +google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 h1:AGXp12e/9rItf6/4QymU7WsAUwCf+ICW75cuR91nJIc= +google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6/go.mod h1:1dOng4TWOomJrDGhpXjfCD35wQC6jnC7HpRmOFRqEV0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/plugins/source/gcp/resources/plugin/autogen_tables.go b/plugins/source/gcp/resources/plugin/autogen_tables.go index c461459f2ff0cb..fbf3b21684ac8d 100644 --- a/plugins/source/gcp/resources/plugin/autogen_tables.go +++ b/plugins/source/gcp/resources/plugin/autogen_tables.go @@ -69,7 +69,7 @@ func PluginAutoGeneratedTables() []*schema.Table { resourcemanager.Folders(), resourcemanager.Projects(), resourcemanager.ProjectPolicies(), - run.Services(), + run.Locations(), secretmanager.Secrets(), serviceusage.Services(), sql.Instances(), diff --git a/plugins/source/gcp/resources/services/run/locations.go b/plugins/source/gcp/resources/services/run/locations.go new file mode 100644 index 00000000000000..b8b7b1c0146825 --- /dev/null +++ b/plugins/source/gcp/resources/services/run/locations.go @@ -0,0 +1,53 @@ +// Code generated by codegen; DO NOT EDIT. + +package run + +import ( + "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugins/source/gcp/client" +) + +func Locations() *schema.Table { + return &schema.Table{ + Name: "gcp_run_locations", + Description: `https://cloud.google.com/api-gateway/docs/reference/rest/v1/projects.locations#Location`, + Resolver: fetchLocations, + Multiplex: client.ProjectMultiplex, + Columns: []schema.Column{ + { + Name: "project_id", + Type: schema.TypeString, + Resolver: client.ResolveProject, + }, + { + Name: "display_name", + Type: schema.TypeString, + Resolver: schema.PathResolver("DisplayName"), + }, + { + Name: "labels", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("Labels"), + }, + { + Name: "location_id", + Type: schema.TypeString, + Resolver: schema.PathResolver("LocationId"), + }, + { + Name: "metadata", + Type: schema.TypeIntArray, + Resolver: schema.PathResolver("Metadata"), + }, + { + Name: "name", + Type: schema.TypeString, + Resolver: schema.PathResolver("Name"), + }, + }, + + Relations: []*schema.Table{ + Services(), + }, + } +} diff --git a/plugins/source/gcp/resources/services/run/locations_fetch.go b/plugins/source/gcp/resources/services/run/locations_fetch.go new file mode 100644 index 00000000000000..788f87c52de2db --- /dev/null +++ b/plugins/source/gcp/resources/services/run/locations_fetch.go @@ -0,0 +1,32 @@ +package run + +import ( + "context" + + "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugins/source/gcp/client" + + "google.golang.org/api/run/v1" +) + +func fetchLocations(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { + c := meta.(*client.Client) + nextPageToken := "" + gcpClient, err := run.NewService(ctx, c.ClientOptions...) + if err != nil { + return err + } + + for { + output, err := gcpClient.Projects.Locations.List("projects/" + c.ProjectId).PageToken(nextPageToken).Context(ctx).Do() + if err != nil { + return err + } + res <- output.Locations + if output.NextPageToken == "" { + break + } + nextPageToken = output.NextPageToken + } + return nil +} diff --git a/plugins/source/gcp/resources/services/run/services.go b/plugins/source/gcp/resources/services/run/services.go index 02baea2962122b..fd72a906f2903d 100644 --- a/plugins/source/gcp/resources/services/run/services.go +++ b/plugins/source/gcp/resources/services/run/services.go @@ -3,15 +3,8 @@ package run import ( - "context" - "google.golang.org/api/iterator" - - pb "cloud.google.com/go/run/apiv2/runpb" - "github.com/cloudquery/plugin-sdk/schema" "github.com/cloudquery/plugins/source/gcp/client" - - "cloud.google.com/go/run/apiv2" ) func Services() *schema.Table { @@ -169,28 +162,3 @@ func Services() *schema.Table { }, } } - -func fetchServices(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { - c := meta.(*client.Client) - req := &pb.ListServicesRequest{ - Parent: "projects/" + c.ProjectId + "locations/-", - } - gcpClient, err := run.NewServicesClient(ctx, c.ClientOptions...) - if err != nil { - return err - } - it := gcpClient.ListServices(ctx, req, c.CallOptions...) - for { - resp, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - return err - } - - res <- resp - - } - return nil -} diff --git a/plugins/source/gcp/resources/services/run/services_fetch.go b/plugins/source/gcp/resources/services/run/services_fetch.go new file mode 100644 index 00000000000000..b8aee8c89e7062 --- /dev/null +++ b/plugins/source/gcp/resources/services/run/services_fetch.go @@ -0,0 +1,39 @@ +package run + +import ( + "context" + + "google.golang.org/api/iterator" + + pb "cloud.google.com/go/run/apiv2/runpb" + + "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugins/source/gcp/client" + + v2 "cloud.google.com/go/run/apiv2" + v1 "google.golang.org/api/run/v1" +) + +func fetchServices(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { + c := meta.(*client.Client) + req := &pb.ListServicesRequest{ + Parent: "projects/" + c.ProjectId + "/locations/" + parent.Item.(*v1.Location).LocationId, + } + gcpClient, err := v2.NewServicesClient(ctx, c.ClientOptions...) + if err != nil { + return err + } + it := gcpClient.ListServices(ctx, req, c.CallOptions...) + for { + resp, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + + res <- resp + } + return nil +} diff --git a/plugins/source/gcp/resources/services/run/services_mock_test.go b/plugins/source/gcp/resources/services/run/services_mock_test.go deleted file mode 100644 index 6d17c35b786d4d..00000000000000 --- a/plugins/source/gcp/resources/services/run/services_mock_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Code generated by codegen; DO NOT EDIT. - -package run - -import ( - "context" - "fmt" - "github.com/cloudquery/plugin-sdk/faker" - "github.com/cloudquery/plugins/source/gcp/client" - "google.golang.org/grpc" - "testing" - - pb "cloud.google.com/go/run/apiv2/runpb" -) - -func createServices(gsrv *grpc.Server) error { - fakeServer := &fakeServicesServer{} - pb.RegisterServicesServer(gsrv, fakeServer) - return nil -} - -type fakeServicesServer struct { - pb.UnimplementedServicesServer -} - -func (f *fakeServicesServer) ListServices(context.Context, *pb.ListServicesRequest) (*pb.ListServicesResponse, error) { - resp := pb.ListServicesResponse{} - if err := faker.FakeObject(&resp); err != nil { - return nil, fmt.Errorf("failed to fake data: %w", err) - } - resp.NextPageToken = "" - return &resp, nil -} - -func TestServices(t *testing.T) { - client.MockTestGrpcHelper(t, Services(), createServices, client.TestOptions{}) -} diff --git a/website/pages/docs/plugins/sources/gcp/tables.md b/website/pages/docs/plugins/sources/gcp/tables.md index 7b609f2f4d7585..70441a15f543f6 100644 --- a/website/pages/docs/plugins/sources/gcp/tables.md +++ b/website/pages/docs/plugins/sources/gcp/tables.md @@ -46,7 +46,8 @@ - [gcp_resourcemanager_folders](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gcp/docs/tables/gcp_resourcemanager_folders.md) - [gcp_resourcemanager_projects](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gcp/docs/tables/gcp_resourcemanager_projects.md) - [gcp_resourcemanager_project_policies](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gcp/docs/tables/gcp_resourcemanager_project_policies.md) -- [gcp_run_services](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gcp/docs/tables/gcp_run_services.md) +- [gcp_run_locations](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gcp/docs/tables/gcp_run_locations.md) + - [gcp_run_services](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gcp/docs/tables/gcp_run_services.md) - [gcp_secretmanager_secrets](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gcp/docs/tables/gcp_secretmanager_secrets.md) - [gcp_serviceusage_services](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gcp/docs/tables/gcp_serviceusage_services.md) - [gcp_sql_instances](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gcp/docs/tables/gcp_sql_instances.md)