diff --git a/plugins/source/gcp/client/client.go b/plugins/source/gcp/client/client.go index 96c27c70fb019a..b4fc459b70cc68 100644 --- a/plugins/source/gcp/client/client.go +++ b/plugins/source/gcp/client/client.go @@ -5,19 +5,25 @@ import ( "encoding/json" "fmt" "strings" + "sync" "time" resourcemanager "cloud.google.com/go/resourcemanager/apiv3" "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb" + serviceusage "cloud.google.com/go/serviceusage/apiv1" + pb "cloud.google.com/go/serviceusage/apiv1/serviceusagepb" "github.com/cloudquery/plugin-sdk/schema" "github.com/cloudquery/plugin-sdk/specs" "github.com/googleapis/gax-go/v2" grpczerolog "github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2" "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" crmv1 "google.golang.org/api/cloudresourcemanager/v1" "google.golang.org/api/iterator" "google.golang.org/api/option" + "google.golang.org/grpc" "google.golang.org/grpc/codes" ) @@ -30,9 +36,12 @@ type Client struct { ClientOptions []option.ClientOption CallOptions []gax.CallOption - // this is set by table client multiplexer + + EnabledServices map[string]map[string]any + // this is set by table client project multiplexer ProjectId string - OrgId string + // this is set by table client Org multiplexer + OrgId string // Logger logger zerolog.Logger } @@ -78,7 +87,8 @@ func (c *Client) Logger() *zerolog.Logger { func New(ctx context.Context, logger zerolog.Logger, s specs.Source) (schema.ClientMeta, error) { var err error c := Client{ - logger: logger, + logger: logger, + EnabledServices: map[string]map[string]any{}, } var gcpSpec Spec if err := s.UnmarshalSpec(&gcpSpec); err != nil { @@ -189,6 +199,12 @@ func New(ctx context.Context, logger zerolog.Logger, s specs.Source) (schema.Cli if len(projects) == 1 { c.ProjectId = projects[0] } + if gcpSpec.EnabledServicesOnly { + if err := c.configureEnabledServices(ctx, s.Concurrency); err != nil { + c.logger.Err(err).Msg("failed to list enabled services") + return nil, err + } + } return &c, nil } @@ -369,3 +385,50 @@ func setUnion(a []string, b []string) []string { } return union } + +func (c *Client) configureEnabledServices(ctx context.Context, concurrency uint64) error { + var esLock sync.Mutex + g, ctx := errgroup.WithContext(ctx) + goroutinesSem := semaphore.NewWeighted(int64(concurrency)) + for _, p := range c.projects { + project := p + if err := goroutinesSem.Acquire(ctx, 1); err != nil { + return err + } + g.Go(func() error { + defer goroutinesSem.Release(1) + cl := c.withProject(project) + svc, err := cl.fetchEnabledServices(ctx) + esLock.Lock() + c.EnabledServices[project] = svc + esLock.Unlock() + return err + }) + } + return g.Wait() +} + +func (c *Client) fetchEnabledServices(ctx context.Context) (map[string]any, error) { + enabled := make(map[string]any) + req := &pb.ListServicesRequest{ + Parent: "projects/" + c.ProjectId, + PageSize: 200, + Filter: "state:ENABLED", + } + gcpClient, err := serviceusage.NewClient(ctx, c.ClientOptions...) + if err != nil { + return nil, err + } + it := gcpClient.ListServices(ctx, req, c.CallOptions...) + for { + resp, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + enabled[resp.GetConfig().Name] = resp + } + return enabled, nil +} diff --git a/plugins/source/gcp/client/multiplex.go b/plugins/source/gcp/client/multiplex.go index db3a3405a82ad3..0b0e8c57455e2d 100644 --- a/plugins/source/gcp/client/multiplex.go +++ b/plugins/source/gcp/client/multiplex.go @@ -14,6 +14,27 @@ func ProjectMultiplex(meta schema.ClientMeta) []schema.ClientMeta { return l } +func ProjectMultiplexEnabledServices(enabledService string) func(schema.ClientMeta) []schema.ClientMeta { + if _, ok := GcpServices[enabledService]; !ok { + panic("unknown service: " + enabledService) + } + + return func(meta schema.ClientMeta) []schema.ClientMeta { + cl := meta.(*Client) + // preallocate all clients just in case + l := make([]schema.ClientMeta, 0, len(cl.projects)) + for _, projectId := range cl.projects { + // This map can only be empty if user has not opted into `EnabledServicesOnly` via the spec + if len(cl.EnabledServices) == 0 { + l = append(l, cl.withProject(projectId)) + } else if cl.EnabledServices[projectId] != nil && cl.EnabledServices[projectId][enabledService] != nil { + l = append(l, cl.withProject(projectId)) + } + } + return l + } +} + func OrgMultiplex(meta schema.ClientMeta) []schema.ClientMeta { client := meta.(*Client) diff --git a/plugins/source/gcp/client/services.go b/plugins/source/gcp/client/services.go new file mode 100644 index 00000000000000..2fc0a354759fd6 --- /dev/null +++ b/plugins/source/gcp/client/services.go @@ -0,0 +1,63 @@ +package client + +var GcpServices = map[string]bool{ + "apikeys.googleapis.com": true, + "appengine.googleapis.com": true, + "artifactregistry.googleapis.com": true, + "bigquery.googleapis.com": true, + "bigquerydatatransfer.googleapis.com": true, + "bigquerymigration.googleapis.com": true, + "bigquerystorage.googleapis.com": true, + "cloudapis.googleapis.com": true, + "cloudbilling.googleapis.com": true, + "cloudbuild.googleapis.com": true, + "clouddebugger.googleapis.com": true, + "cloudfunctions.googleapis.com": true, + "cloudkms.googleapis.com": true, + "cloudresourcemanager.googleapis.com": true, + "cloudscheduler.googleapis.com": true, + "cloudtrace.googleapis.com": true, + "compute.googleapis.com": true, + "container.googleapis.com": true, + "containeranalysis.googleapis.com": true, + "containerfilesystem.googleapis.com": true, + "containerregistry.googleapis.com": true, + "containerscanning.googleapis.com": true, + "datastore.googleapis.com": true, + "datastudio.googleapis.com": true, + "deploymentmanager.googleapis.com": true, + "dns.googleapis.com": true, + "domains.googleapis.com": true, + "fcm.googleapis.com": true, + "firebase.googleapis.com": true, + "firebasedynamiclinks.googleapis.com": true, + "firebasehosting.googleapis.com": true, + "firebaseinstallations.googleapis.com": true, + "firebaseremoteconfig.googleapis.com": true, + "firebaserules.googleapis.com": true, + "firestore.googleapis.com": true, + "googlecloudmessaging.googleapis.com": true, + "iam.googleapis.com": true, + "iamcredentials.googleapis.com": true, + "iap.googleapis.com": true, + "identitytoolkit.googleapis.com": true, + "logging.googleapis.com": true, + "monitoring.googleapis.com": true, + "networkmanagement.googleapis.com": true, + "ondemandscanning.googleapis.com": true, + "oslogin.googleapis.com": true, + "pubsub.googleapis.com": true, + "redis.googleapis.com": true, + "run.googleapis.com": true, + "runtimeconfig.googleapis.com": true, + "secretmanager.googleapis.com": true, + "securetoken.googleapis.com": true, + "servicemanagement.googleapis.com": true, + "servicenetworking.googleapis.com": true, + "serviceusage.googleapis.com": true, + "sql-component.googleapis.com": true, + "sqladmin.googleapis.com": true, + "storage-api.googleapis.com": true, + "storage.googleapis.com": true, + "testing.googleapis.com": true, +} diff --git a/plugins/source/gcp/client/spec.go b/plugins/source/gcp/client/spec.go index 893335dbffe6bc..916e68114e30db 100644 --- a/plugins/source/gcp/client/spec.go +++ b/plugins/source/gcp/client/spec.go @@ -9,6 +9,7 @@ type Spec struct { ProjectFilter string `json:"project_filter"` BackoffDelay int `json:"backoff_delay"` BackoffRetries int `json:"backoff_retries"` + EnabledServicesOnly bool `json:"enabled_services_only"` } func (spec *Spec) setDefaults() { diff --git a/plugins/source/gcp/codegen/main.go b/plugins/source/gcp/codegen/main.go index 82bc9240876371..177dd02e4c9b11 100644 --- a/plugins/source/gcp/codegen/main.go +++ b/plugins/source/gcp/codegen/main.go @@ -15,6 +15,7 @@ import ( "github.com/cloudquery/plugin-sdk/codegen" "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugins/source/gcp/client" "github.com/cloudquery/plugins/source/gcp/codegen/recipes" "github.com/iancoleman/strcase" "google.golang.org/protobuf/reflect/protoreflect" @@ -178,7 +179,10 @@ func generateResource(r recipes.Resource, mock bool) { log.Fatal(fmt.Errorf("failed to create table for %s: %w", r.StructName, err)) } if r.Multiplex == nil { - r.Table.Multiplex = "client.ProjectMultiplex" + if _, ok := client.GcpServices[r.ServiceDNS]; !ok { + panic("unknown service DNS: " + r.ServiceDNS) + } + r.Table.Multiplex = "client.ProjectMultiplexEnabledServices(\"" + r.ServiceDNS + "\")" } else { r.Table.Multiplex = *r.Multiplex } diff --git a/plugins/source/gcp/codegen/recipes/apikeys.go b/plugins/source/gcp/codegen/recipes/apikeys.go index c4ed31b3bc37f7..9033309ab32e04 100644 --- a/plugins/source/gcp/codegen/recipes/apikeys.go +++ b/plugins/source/gcp/codegen/recipes/apikeys.go @@ -26,6 +26,7 @@ func init() { resource.MockImports = []string{"cloud.google.com/go/apikeys/apiv2"} resource.NewFunction = apikeys.NewClient resource.RegisterServer = pb.RegisterApiKeysServer + resource.ServiceDNS = "apikeys.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/base.go b/plugins/source/gcp/codegen/recipes/base.go index ef7a5bdd4e2687..dd45caa5c62f19 100644 --- a/plugins/source/gcp/codegen/recipes/base.go +++ b/plugins/source/gcp/codegen/recipes/base.go @@ -22,6 +22,8 @@ type Resource struct { Struct any // StructName is the name of the Struct because it can't be inferred by reflection StructName string + // Service DNS + ServiceDNS string // Service is the name of the gcp service the struct/api is residing Service string // SubService is the name of the gcp subservice the struct/api is residing (gcp is split into service.subservice.list) diff --git a/plugins/source/gcp/codegen/recipes/bigquery.go b/plugins/source/gcp/codegen/recipes/bigquery.go index eb5fb8bc8de73a..0403cecf6224b4 100644 --- a/plugins/source/gcp/codegen/recipes/bigquery.go +++ b/plugins/source/gcp/codegen/recipes/bigquery.go @@ -29,6 +29,7 @@ func init() { for _, resource := range resources { resource.Service = "bigquery" + resource.ServiceDNS = "bigquery.googleapis.com" resource.Template = "newapi_list" } diff --git a/plugins/source/gcp/codegen/recipes/billing.go b/plugins/source/gcp/codegen/recipes/billing.go index 193a0b42535b10..10b5355f5cc207 100644 --- a/plugins/source/gcp/codegen/recipes/billing.go +++ b/plugins/source/gcp/codegen/recipes/billing.go @@ -5,6 +5,8 @@ import ( pb "cloud.google.com/go/billing/apiv1/billingpb" ) +var ProjectMultiplex = "client.ProjectMultiplex" + func init() { resources := []*Resource{ { @@ -20,10 +22,12 @@ func init() { SubService: "services", Struct: &pb.Service{}, NewFunction: billing.NewCloudCatalogClient, + ResponseStruct: &pb.ListServicesResponse{}, ListFunction: (&billing.CloudCatalogClient{}).ListServices, RegisterServer: pb.RegisterCloudCatalogServer, PrimaryKeys: []string{"name"}, Description: "https://cloud.google.com/billing/docs/reference/rest/v1/services/list#Service", + Multiplex: &ProjectMultiplex, }, } @@ -33,6 +37,7 @@ func init() { resource.MockTemplate = "newapi_list_grpc_mock" resource.MockImports = []string{"cloud.google.com/go/billing/apiv1"} resource.ProtobufImport = "cloud.google.com/go/billing/apiv1/billingpb" + resource.ServiceDNS = "cloudbilling.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/compute.go b/plugins/source/gcp/codegen/recipes/compute.go index 1ca4dddf85a94a..d37dde4eb58623 100644 --- a/plugins/source/gcp/codegen/recipes/compute.go +++ b/plugins/source/gcp/codegen/recipes/compute.go @@ -200,6 +200,8 @@ func init() { if resource.PrimaryKeys == nil { resource.PrimaryKeys = []string{"self_link"} } + resource.ServiceDNS = "compute.googleapis.com" + } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/container.go b/plugins/source/gcp/codegen/recipes/container.go index 7889ad09c1768f..256889d8e404c5 100644 --- a/plugins/source/gcp/codegen/recipes/container.go +++ b/plugins/source/gcp/codegen/recipes/container.go @@ -23,6 +23,7 @@ func init() { resource.ProtobufImport = "google.golang.org/genproto/googleapis/container/v1" resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" + resource.ServiceDNS = "container.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/containeranalysis.go b/plugins/source/gcp/codegen/recipes/containeranalysis.go index 66d9ed7b74b0a7..e1a63e25d0b836 100644 --- a/plugins/source/gcp/codegen/recipes/containeranalysis.go +++ b/plugins/source/gcp/codegen/recipes/containeranalysis.go @@ -27,6 +27,7 @@ func init() { resource.MockTemplate = "newapi_list_grpc_mock" resource.RegisterServer = grafeaspb.RegisterGrafeasV1Beta1Server + resource.ServiceDNS = "containeranalysis.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/dns.go b/plugins/source/gcp/codegen/recipes/dns.go index b7418332f26ff4..6758fc692b821d 100644 --- a/plugins/source/gcp/codegen/recipes/dns.go +++ b/plugins/source/gcp/codegen/recipes/dns.go @@ -32,6 +32,7 @@ func init() { resource.Template = "newapi_list" resource.MockTemplate = "resource_list_mock" resource.OutputField = strcase.ToCamel(resource.SubService) + resource.ServiceDNS = "dns.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/domains.go b/plugins/source/gcp/codegen/recipes/domains.go index 61adc93d60520c..45ea9af7041cb5 100644 --- a/plugins/source/gcp/codegen/recipes/domains.go +++ b/plugins/source/gcp/codegen/recipes/domains.go @@ -25,6 +25,8 @@ func init() { resource.ProtobufImport = "cloud.google.com/go/domains/apiv1beta1/domainspb" resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" + // resource.OutputField = strcase.ToCamel(resource.SubService) + resource.ServiceDNS = "domains.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/functions.go b/plugins/source/gcp/codegen/recipes/functions.go index e029c60182d006..937b842bde6329 100644 --- a/plugins/source/gcp/codegen/recipes/functions.go +++ b/plugins/source/gcp/codegen/recipes/functions.go @@ -25,6 +25,7 @@ func init() { resource.MockTemplate = "newapi_list_grpc_mock" resource.MockImports = []string{"cloud.google.com/go/functions/apiv1"} resource.ProtobufImport = "cloud.google.com/go/functions/apiv1/functionspb" + resource.ServiceDNS = "cloudfunctions.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/iam.go b/plugins/source/gcp/codegen/recipes/iam.go index 749b69287db93c..78f923d2ac07c6 100644 --- a/plugins/source/gcp/codegen/recipes/iam.go +++ b/plugins/source/gcp/codegen/recipes/iam.go @@ -61,6 +61,7 @@ func init() { if resource.OutputField == "" { resource.OutputField = strcase.ToCamel(resource.SubService) } + resource.ServiceDNS = "iam.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/kms.go b/plugins/source/gcp/codegen/recipes/kms.go index e898958f61e556..63ec449ea8b012 100644 --- a/plugins/source/gcp/codegen/recipes/kms.go +++ b/plugins/source/gcp/codegen/recipes/kms.go @@ -42,6 +42,7 @@ func init() { resource.ProtobufImport = "cloud.google.com/go/kms/apiv1/kmspb" resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" + resource.ServiceDNS = "cloudkms.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/logging.go b/plugins/source/gcp/codegen/recipes/logging.go index 88be4e3589721f..62943bcf8f010c 100644 --- a/plugins/source/gcp/codegen/recipes/logging.go +++ b/plugins/source/gcp/codegen/recipes/logging.go @@ -35,6 +35,7 @@ func init() { resource.ProtobufImport = "google.golang.org/genproto/googleapis/logging/v2" resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" + resource.ServiceDNS = "logging.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/monitoring.go b/plugins/source/gcp/codegen/recipes/monitoring.go index 4829a5dc674df3..baab24d9557fa3 100644 --- a/plugins/source/gcp/codegen/recipes/monitoring.go +++ b/plugins/source/gcp/codegen/recipes/monitoring.go @@ -25,6 +25,7 @@ func init() { resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" resource.RequestStructFields = `Name: "projects/" + c.ProjectId,` + resource.ServiceDNS = "monitoring.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/redis.go b/plugins/source/gcp/codegen/recipes/redis.go index 365c005405f4cd..2f0b64c349b9b4 100644 --- a/plugins/source/gcp/codegen/recipes/redis.go +++ b/plugins/source/gcp/codegen/recipes/redis.go @@ -25,6 +25,7 @@ func init() { resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" resource.RequestStructFields = `Parent: "projects/" + c.ProjectId + "/locations/-",` + resource.ServiceDNS = "redis.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/resourcemanager.go b/plugins/source/gcp/codegen/recipes/resourcemanager.go index 0de6706dbda18b..69423a8264840c 100644 --- a/plugins/source/gcp/codegen/recipes/resourcemanager.go +++ b/plugins/source/gcp/codegen/recipes/resourcemanager.go @@ -50,6 +50,7 @@ func init() { resource.ProtobufImport = "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb" resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" + resource.ServiceDNS = "cloudresourcemanager.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/run.go b/plugins/source/gcp/codegen/recipes/run.go index 16654ffa4640e8..3efed0d4b8bb2f 100644 --- a/plugins/source/gcp/codegen/recipes/run.go +++ b/plugins/source/gcp/codegen/recipes/run.go @@ -29,6 +29,7 @@ func init() { resource.Service = "run" resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" + resource.ServiceDNS = "run.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/secretmanager.go b/plugins/source/gcp/codegen/recipes/secretmanager.go index ed31c683b01690..f0506015f2f2ed 100644 --- a/plugins/source/gcp/codegen/recipes/secretmanager.go +++ b/plugins/source/gcp/codegen/recipes/secretmanager.go @@ -26,6 +26,7 @@ func init() { resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" resource.RequestStructFields = `Parent: "projects/" + c.ProjectId,` + resource.ServiceDNS = "secretmanager.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/serviceusage.go b/plugins/source/gcp/codegen/recipes/serviceusage.go index 51ef418b192ed6..c0e2267a35bdff 100644 --- a/plugins/source/gcp/codegen/recipes/serviceusage.go +++ b/plugins/source/gcp/codegen/recipes/serviceusage.go @@ -15,6 +15,8 @@ func init() { ListFunction: (&pb.UnimplementedServiceUsageServer{}).ListServices, PrimaryKeys: []string{"name"}, Description: "https://cloud.google.com/service-usage/docs/reference/rest/v1/services#Service", + SkipFetch: true, + SkipMock: true, }, } @@ -24,9 +26,7 @@ func init() { resource.ProtobufImport = "cloud.google.com/go/serviceusage/apiv1/serviceusagepb" resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_grpc_mock" - resource.RequestStructFields = `Parent: "projects/" + c.ProjectId, - PageSize: 200, - Filter: "state:ENABLED",` + resource.ServiceDNS = "serviceusage.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/sqladmin.go b/plugins/source/gcp/codegen/recipes/sqladmin.go index a056a79d507e76..ebda4145e244ef 100644 --- a/plugins/source/gcp/codegen/recipes/sqladmin.go +++ b/plugins/source/gcp/codegen/recipes/sqladmin.go @@ -30,6 +30,7 @@ func init() { for _, resource := range resources { resource.Service = "sql" resource.Template = "newapi_list" + resource.ServiceDNS = "sqladmin.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/codegen/recipes/storage.go b/plugins/source/gcp/codegen/recipes/storage.go index 73ebd69349d47c..dfb97f9a33fda7 100644 --- a/plugins/source/gcp/codegen/recipes/storage.go +++ b/plugins/source/gcp/codegen/recipes/storage.go @@ -42,6 +42,7 @@ func init() { resource.MockImports = []string{"cloud.google.com/go/storage"} resource.Template = "newapi_list" resource.MockTemplate = "newapi_list_rest_mock" + resource.ServiceDNS = "storage.googleapis.com" } Resources = append(Resources, resources...) diff --git a/plugins/source/gcp/resources/services/apikeys/keys.go b/plugins/source/gcp/resources/services/apikeys/keys.go index a68f8cb5c1a24d..f771ad710d7583 100644 --- a/plugins/source/gcp/resources/services/apikeys/keys.go +++ b/plugins/source/gcp/resources/services/apikeys/keys.go @@ -19,7 +19,7 @@ func Keys() *schema.Table { Name: "gcp_apikeys_keys", Description: `https://cloud.google.com/api-keys/docs/reference/rest/v2/projects.locations.keys#Key`, Resolver: fetchKeys, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("apikeys.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/bigquery/datasets.go b/plugins/source/gcp/resources/services/bigquery/datasets.go index 0881983c9ca361..d27429b6cc9bd7 100644 --- a/plugins/source/gcp/resources/services/bigquery/datasets.go +++ b/plugins/source/gcp/resources/services/bigquery/datasets.go @@ -13,7 +13,7 @@ func Datasets() *schema.Table { Description: `https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#Dataset`, Resolver: fetchDatasets, PreResourceResolver: datasetGet, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("bigquery.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/billing/billing_accounts.go b/plugins/source/gcp/resources/services/billing/billing_accounts.go index cba1e74f2c7220..e6729278af04d7 100644 --- a/plugins/source/gcp/resources/services/billing/billing_accounts.go +++ b/plugins/source/gcp/resources/services/billing/billing_accounts.go @@ -19,7 +19,7 @@ func BillingAccounts() *schema.Table { Name: "gcp_billing_billing_accounts", Description: `https://cloud.google.com/billing/docs/reference/rest/v1/billingAccounts#BillingAccount`, Resolver: fetchBillingAccounts, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("cloudbilling.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/addresses.go b/plugins/source/gcp/resources/services/compute/addresses.go index 5c9564408dcadc..3d3e80f63d0b41 100644 --- a/plugins/source/gcp/resources/services/compute/addresses.go +++ b/plugins/source/gcp/resources/services/compute/addresses.go @@ -19,7 +19,7 @@ func Addresses() *schema.Table { Name: "gcp_compute_addresses", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/addresses#Address`, Resolver: fetchAddresses, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/autoscalers.go b/plugins/source/gcp/resources/services/compute/autoscalers.go index 512b589c46929b..1ecdb334e63455 100644 --- a/plugins/source/gcp/resources/services/compute/autoscalers.go +++ b/plugins/source/gcp/resources/services/compute/autoscalers.go @@ -19,7 +19,7 @@ func Autoscalers() *schema.Table { Name: "gcp_compute_autoscalers", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/autoscalers#Autoscaler`, Resolver: fetchAutoscalers, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/backend_services.go b/plugins/source/gcp/resources/services/compute/backend_services.go index 68bafa1e893f64..6c34ef03453db0 100644 --- a/plugins/source/gcp/resources/services/compute/backend_services.go +++ b/plugins/source/gcp/resources/services/compute/backend_services.go @@ -19,7 +19,7 @@ func BackendServices() *schema.Table { Name: "gcp_compute_backend_services", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/backendServices#BackendService`, Resolver: fetchBackendServices, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/disk_types.go b/plugins/source/gcp/resources/services/compute/disk_types.go index 3fbfc54ca9ba29..5ad513afd6e349 100644 --- a/plugins/source/gcp/resources/services/compute/disk_types.go +++ b/plugins/source/gcp/resources/services/compute/disk_types.go @@ -19,7 +19,7 @@ func DiskTypes() *schema.Table { Name: "gcp_compute_disk_types", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/diskTypes#DiskType`, Resolver: fetchDiskTypes, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/disks.go b/plugins/source/gcp/resources/services/compute/disks.go index 06175e047d0cf4..dc7cd4378778e8 100644 --- a/plugins/source/gcp/resources/services/compute/disks.go +++ b/plugins/source/gcp/resources/services/compute/disks.go @@ -19,7 +19,7 @@ func Disks() *schema.Table { Name: "gcp_compute_disks", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/disks#Disk`, Resolver: fetchDisks, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/firewalls.go b/plugins/source/gcp/resources/services/compute/firewalls.go index b116eea26cb251..7b9640ad9c97a8 100644 --- a/plugins/source/gcp/resources/services/compute/firewalls.go +++ b/plugins/source/gcp/resources/services/compute/firewalls.go @@ -18,7 +18,7 @@ func Firewalls() *schema.Table { return &schema.Table{ Name: "gcp_compute_firewalls", Resolver: fetchFirewalls, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/forwarding_rules.go b/plugins/source/gcp/resources/services/compute/forwarding_rules.go index e49ea994e912c3..38a7abefd61d7f 100644 --- a/plugins/source/gcp/resources/services/compute/forwarding_rules.go +++ b/plugins/source/gcp/resources/services/compute/forwarding_rules.go @@ -19,7 +19,7 @@ func ForwardingRules() *schema.Table { Name: "gcp_compute_forwarding_rules", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/forwardingRules#ForwardingRule`, Resolver: fetchForwardingRules, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/images.go b/plugins/source/gcp/resources/services/compute/images.go index 83b4477a6faf72..97d057c1bfb06d 100644 --- a/plugins/source/gcp/resources/services/compute/images.go +++ b/plugins/source/gcp/resources/services/compute/images.go @@ -18,7 +18,7 @@ func Images() *schema.Table { return &schema.Table{ Name: "gcp_compute_images", Resolver: fetchImages, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/instance_groups.go b/plugins/source/gcp/resources/services/compute/instance_groups.go index 621c6b3377e409..257a4c3e892137 100644 --- a/plugins/source/gcp/resources/services/compute/instance_groups.go +++ b/plugins/source/gcp/resources/services/compute/instance_groups.go @@ -19,7 +19,7 @@ func InstanceGroups() *schema.Table { Name: "gcp_compute_instance_groups", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroups#InstanceGroup`, Resolver: fetchInstanceGroups, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/instances.go b/plugins/source/gcp/resources/services/compute/instances.go index 5bbe2befda47ad..e69c744b1f1554 100644 --- a/plugins/source/gcp/resources/services/compute/instances.go +++ b/plugins/source/gcp/resources/services/compute/instances.go @@ -19,7 +19,7 @@ func Instances() *schema.Table { Name: "gcp_compute_instances", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/instances#Instance`, Resolver: fetchInstances, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/interconnects.go b/plugins/source/gcp/resources/services/compute/interconnects.go index 90375d3b0c69a5..bbd354f47d7f1d 100644 --- a/plugins/source/gcp/resources/services/compute/interconnects.go +++ b/plugins/source/gcp/resources/services/compute/interconnects.go @@ -18,7 +18,7 @@ func Interconnects() *schema.Table { return &schema.Table{ Name: "gcp_compute_interconnects", Resolver: fetchInterconnects, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/networks.go b/plugins/source/gcp/resources/services/compute/networks.go index 65bda275940764..3057726de8b4f5 100644 --- a/plugins/source/gcp/resources/services/compute/networks.go +++ b/plugins/source/gcp/resources/services/compute/networks.go @@ -18,7 +18,7 @@ func Networks() *schema.Table { return &schema.Table{ Name: "gcp_compute_networks", Resolver: fetchNetworks, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/projects.go b/plugins/source/gcp/resources/services/compute/projects.go index 4d9cfa18a172d1..1ad1be6c910f1b 100644 --- a/plugins/source/gcp/resources/services/compute/projects.go +++ b/plugins/source/gcp/resources/services/compute/projects.go @@ -11,7 +11,7 @@ func Projects() *schema.Table { return &schema.Table{ Name: "gcp_compute_projects", Resolver: fetchProjects, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/ssl_certificates.go b/plugins/source/gcp/resources/services/compute/ssl_certificates.go index 44f77c43ec63d8..9c6c88b32421ae 100644 --- a/plugins/source/gcp/resources/services/compute/ssl_certificates.go +++ b/plugins/source/gcp/resources/services/compute/ssl_certificates.go @@ -19,7 +19,7 @@ func SslCertificates() *schema.Table { Name: "gcp_compute_ssl_certificates", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/sslCertificates#SslCertificate`, Resolver: fetchSslCertificates, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/ssl_policies.go b/plugins/source/gcp/resources/services/compute/ssl_policies.go index 3c05ba533ebb02..ce158042b2a17a 100644 --- a/plugins/source/gcp/resources/services/compute/ssl_policies.go +++ b/plugins/source/gcp/resources/services/compute/ssl_policies.go @@ -18,7 +18,7 @@ func SslPolicies() *schema.Table { return &schema.Table{ Name: "gcp_compute_ssl_policies", Resolver: fetchSslPolicies, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/subnetworks.go b/plugins/source/gcp/resources/services/compute/subnetworks.go index 810d3768fe4143..d58b255b39e5fa 100644 --- a/plugins/source/gcp/resources/services/compute/subnetworks.go +++ b/plugins/source/gcp/resources/services/compute/subnetworks.go @@ -19,7 +19,7 @@ func Subnetworks() *schema.Table { Name: "gcp_compute_subnetworks", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/subnetworks#Subnetwork`, Resolver: fetchSubnetworks, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/target_http_proxies.go b/plugins/source/gcp/resources/services/compute/target_http_proxies.go index 0f574d89f74c08..f607449496ae55 100644 --- a/plugins/source/gcp/resources/services/compute/target_http_proxies.go +++ b/plugins/source/gcp/resources/services/compute/target_http_proxies.go @@ -19,7 +19,7 @@ func TargetHttpProxies() *schema.Table { Name: "gcp_compute_target_http_proxies", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/targetHttpProxies#TargetHttpProxy`, Resolver: fetchTargetHttpProxies, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/target_ssl_proxies.go b/plugins/source/gcp/resources/services/compute/target_ssl_proxies.go index ed7d3442b08b88..71b4f7933e8795 100644 --- a/plugins/source/gcp/resources/services/compute/target_ssl_proxies.go +++ b/plugins/source/gcp/resources/services/compute/target_ssl_proxies.go @@ -18,7 +18,7 @@ func TargetSslProxies() *schema.Table { return &schema.Table{ Name: "gcp_compute_target_ssl_proxies", Resolver: fetchTargetSslProxies, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/url_maps.go b/plugins/source/gcp/resources/services/compute/url_maps.go index fe7ea44ef7342a..a32f630c6919e6 100644 --- a/plugins/source/gcp/resources/services/compute/url_maps.go +++ b/plugins/source/gcp/resources/services/compute/url_maps.go @@ -19,7 +19,7 @@ func UrlMaps() *schema.Table { Name: "gcp_compute_url_maps", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/urlMaps#UrlMap`, Resolver: fetchUrlMaps, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/compute/vpn_gateways.go b/plugins/source/gcp/resources/services/compute/vpn_gateways.go index 7d03809da6ebd1..846e6253e87b05 100644 --- a/plugins/source/gcp/resources/services/compute/vpn_gateways.go +++ b/plugins/source/gcp/resources/services/compute/vpn_gateways.go @@ -19,7 +19,7 @@ func VpnGateways() *schema.Table { Name: "gcp_compute_vpn_gateways", Description: `https://cloud.google.com/compute/docs/reference/rest/v1/vpnGateways#VpnGateway`, Resolver: fetchVpnGateways, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("compute.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/container/clusters.go b/plugins/source/gcp/resources/services/container/clusters.go index a563a0178d3cd0..ce9340d865df4f 100644 --- a/plugins/source/gcp/resources/services/container/clusters.go +++ b/plugins/source/gcp/resources/services/container/clusters.go @@ -12,7 +12,7 @@ func Clusters() *schema.Table { Name: "gcp_container_clusters", Description: `https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.locations.clusters#Cluster`, Resolver: fetchClusters, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("container.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/containeranalysis/occurrences.go b/plugins/source/gcp/resources/services/containeranalysis/occurrences.go index 0b1249a5d6b5b3..c2112184ada977 100644 --- a/plugins/source/gcp/resources/services/containeranalysis/occurrences.go +++ b/plugins/source/gcp/resources/services/containeranalysis/occurrences.go @@ -19,7 +19,7 @@ func Occurrences() *schema.Table { Name: "gcp_containeranalysis_occurrences", Description: `https://cloud.google.com/container-analysis/docs/reference/rest/v1beta1/projects.occurrences#Occurrence`, Resolver: fetchOccurrences, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("containeranalysis.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/dns/managed_zones.go b/plugins/source/gcp/resources/services/dns/managed_zones.go index 4cd82cbbb883c9..c025611c3cb016 100644 --- a/plugins/source/gcp/resources/services/dns/managed_zones.go +++ b/plugins/source/gcp/resources/services/dns/managed_zones.go @@ -12,7 +12,7 @@ func ManagedZones() *schema.Table { Name: "gcp_dns_managed_zones", Description: `https://cloud.google.com/dns/docs/reference/v1/managedZones#resource`, Resolver: fetchManagedZones, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("dns.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/dns/policies.go b/plugins/source/gcp/resources/services/dns/policies.go index 619e2d7be8f3c8..52869e843eba2f 100644 --- a/plugins/source/gcp/resources/services/dns/policies.go +++ b/plugins/source/gcp/resources/services/dns/policies.go @@ -12,7 +12,7 @@ func Policies() *schema.Table { Name: "gcp_dns_policies", Description: `https://cloud.google.com/dns/docs/reference/v1/policies#resource`, Resolver: fetchPolicies, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("dns.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/domains/registrations.go b/plugins/source/gcp/resources/services/domains/registrations.go index 128188739c3836..3c1d4042fc5302 100644 --- a/plugins/source/gcp/resources/services/domains/registrations.go +++ b/plugins/source/gcp/resources/services/domains/registrations.go @@ -21,7 +21,7 @@ func Registrations() *schema.Table { Name: "gcp_domains_registrations", Description: `https://cloud.google.com/domains/docs/reference/rest/v1beta1/projects.locations.registrations#Registration`, Resolver: fetchRegistrations, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("domains.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/functions/functions.go b/plugins/source/gcp/resources/services/functions/functions.go index 9b34f3c19b1c90..b03973538e448b 100644 --- a/plugins/source/gcp/resources/services/functions/functions.go +++ b/plugins/source/gcp/resources/services/functions/functions.go @@ -19,7 +19,7 @@ func Functions() *schema.Table { Name: "gcp_functions_functions", Description: `https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions#CloudFunction`, Resolver: fetchFunctions, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("cloudfunctions.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/iam/deny_policies.go b/plugins/source/gcp/resources/services/iam/deny_policies.go index c6fb0a3458f93c..5219d1da6f403d 100644 --- a/plugins/source/gcp/resources/services/iam/deny_policies.go +++ b/plugins/source/gcp/resources/services/iam/deny_policies.go @@ -12,7 +12,7 @@ func DenyPolicies() *schema.Table { Name: "gcp_iam_deny_policies", Description: `https://cloud.google.com/iam/docs/reference/rest/v2beta/policies#Policy`, Resolver: fetchDenyPolicies, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("iam.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/iam/roles.go b/plugins/source/gcp/resources/services/iam/roles.go index 94168d30918ba5..ac75cbf5188e3d 100644 --- a/plugins/source/gcp/resources/services/iam/roles.go +++ b/plugins/source/gcp/resources/services/iam/roles.go @@ -12,7 +12,7 @@ func Roles() *schema.Table { Name: "gcp_iam_roles", Description: `https://cloud.google.com/iam/docs/reference/rest/v1/roles#Role`, Resolver: fetchRoles, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("iam.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/iam/service_account_keys.go b/plugins/source/gcp/resources/services/iam/service_account_keys.go index 253cdc767c2cee..d534448d6d2f1d 100644 --- a/plugins/source/gcp/resources/services/iam/service_account_keys.go +++ b/plugins/source/gcp/resources/services/iam/service_account_keys.go @@ -12,7 +12,7 @@ func ServiceAccountKeys() *schema.Table { Name: "gcp_iam_service_account_keys", Description: `https://cloud.google.com/iam/docs/reference/rest/v1/projects.serviceAccounts.keys#ServiceAccountKey`, Resolver: fetchServiceAccountKeys, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("iam.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/iam/service_accounts.go b/plugins/source/gcp/resources/services/iam/service_accounts.go index f13ffc02cd20a3..dc321b928446f2 100644 --- a/plugins/source/gcp/resources/services/iam/service_accounts.go +++ b/plugins/source/gcp/resources/services/iam/service_accounts.go @@ -12,7 +12,7 @@ func ServiceAccounts() *schema.Table { Name: "gcp_iam_service_accounts", Description: `https://cloud.google.com/iam/docs/reference/rest/v1/projects.serviceAccounts#ServiceAccount`, Resolver: fetchServiceAccounts, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("iam.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/kms/keyrings.go b/plugins/source/gcp/resources/services/kms/keyrings.go index 938ddb89912111..946bf714c15874 100644 --- a/plugins/source/gcp/resources/services/kms/keyrings.go +++ b/plugins/source/gcp/resources/services/kms/keyrings.go @@ -12,7 +12,7 @@ func Keyrings() *schema.Table { Name: "gcp_kms_keyrings", Description: `https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings#KeyRing`, Resolver: fetchKeyrings, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("cloudkms.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/logging/metrics.go b/plugins/source/gcp/resources/services/logging/metrics.go index bb40481ae6997d..3db8435e8f23ab 100644 --- a/plugins/source/gcp/resources/services/logging/metrics.go +++ b/plugins/source/gcp/resources/services/logging/metrics.go @@ -19,7 +19,7 @@ func Metrics() *schema.Table { Name: "gcp_logging_metrics", Description: `https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.metrics#LogMetric`, Resolver: fetchMetrics, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("logging.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/logging/sinks.go b/plugins/source/gcp/resources/services/logging/sinks.go index 6b53853825ce0a..07ac155516f6a6 100644 --- a/plugins/source/gcp/resources/services/logging/sinks.go +++ b/plugins/source/gcp/resources/services/logging/sinks.go @@ -19,7 +19,7 @@ func Sinks() *schema.Table { Name: "gcp_logging_sinks", Description: `https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.sinks#LogSink`, Resolver: fetchSinks, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("logging.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/monitoring/alert_policies.go b/plugins/source/gcp/resources/services/monitoring/alert_policies.go index 9c3a7fdf329626..f397ba690fa013 100644 --- a/plugins/source/gcp/resources/services/monitoring/alert_policies.go +++ b/plugins/source/gcp/resources/services/monitoring/alert_policies.go @@ -19,7 +19,7 @@ func AlertPolicies() *schema.Table { Name: "gcp_monitoring_alert_policies", Description: `https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.alertPolicies#AlertPolicy`, Resolver: fetchAlertPolicies, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("monitoring.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/redis/instances.go b/plugins/source/gcp/resources/services/redis/instances.go index 9e6924a13d6a49..966cc26aeb03e8 100644 --- a/plugins/source/gcp/resources/services/redis/instances.go +++ b/plugins/source/gcp/resources/services/redis/instances.go @@ -19,7 +19,7 @@ func Instances() *schema.Table { Name: "gcp_redis_instances", Description: `https://cloud.google.com/memorystore/docs/redis/reference/rest/v1/projects.locations.instances#Instance`, Resolver: fetchInstances, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("redis.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/resourcemanager/project_policies.go b/plugins/source/gcp/resources/services/resourcemanager/project_policies.go index dcede5687f8daf..9f3600c9405f7b 100644 --- a/plugins/source/gcp/resources/services/resourcemanager/project_policies.go +++ b/plugins/source/gcp/resources/services/resourcemanager/project_policies.go @@ -12,7 +12,7 @@ func ProjectPolicies() *schema.Table { Name: "gcp_resourcemanager_project_policies", Description: `https://cloud.google.com/resource-manager/reference/rest/Shared.Types/Policy`, Resolver: fetchProjectPolicies, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("cloudresourcemanager.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/resourcemanager/projects.go b/plugins/source/gcp/resources/services/resourcemanager/projects.go index f0f1209747deb7..825f222eb03f54 100644 --- a/plugins/source/gcp/resources/services/resourcemanager/projects.go +++ b/plugins/source/gcp/resources/services/resourcemanager/projects.go @@ -12,7 +12,7 @@ func Projects() *schema.Table { Name: "gcp_resourcemanager_projects", Description: `https://cloud.google.com/resource-manager/reference/rest/v3/projects#Project`, Resolver: fetchProjects, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("cloudresourcemanager.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/run/locations.go b/plugins/source/gcp/resources/services/run/locations.go index b8b7b1c0146825..11ba69d66eb5f1 100644 --- a/plugins/source/gcp/resources/services/run/locations.go +++ b/plugins/source/gcp/resources/services/run/locations.go @@ -12,7 +12,7 @@ func Locations() *schema.Table { Name: "gcp_run_locations", Description: `https://cloud.google.com/api-gateway/docs/reference/rest/v1/projects.locations#Location`, Resolver: fetchLocations, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("run.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/run/services.go b/plugins/source/gcp/resources/services/run/services.go index fd72a906f2903d..e8f70dcdedf979 100644 --- a/plugins/source/gcp/resources/services/run/services.go +++ b/plugins/source/gcp/resources/services/run/services.go @@ -12,7 +12,7 @@ func Services() *schema.Table { Name: "gcp_run_services", Description: `https://cloud.google.com/run/docs/reference/rest/v2/projects.locations.services#Service`, Resolver: fetchServices, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("run.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/secretmanager/secrets.go b/plugins/source/gcp/resources/services/secretmanager/secrets.go index 207e94c21e83f1..496ae2119a999a 100644 --- a/plugins/source/gcp/resources/services/secretmanager/secrets.go +++ b/plugins/source/gcp/resources/services/secretmanager/secrets.go @@ -19,7 +19,7 @@ func Secrets() *schema.Table { Name: "gcp_secretmanager_secrets", Description: `https://cloud.google.com/secret-manager/docs/reference/rest/v1/projects.secrets#Secret`, Resolver: fetchSecrets, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("secretmanager.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/serviceusage/services.go b/plugins/source/gcp/resources/services/serviceusage/services.go index 988560d82d6bd3..8c4fbd310f5e8a 100644 --- a/plugins/source/gcp/resources/services/serviceusage/services.go +++ b/plugins/source/gcp/resources/services/serviceusage/services.go @@ -3,15 +3,8 @@ package serviceusage import ( - "context" - "google.golang.org/api/iterator" - - pb "cloud.google.com/go/serviceusage/apiv1/serviceusagepb" - "github.com/cloudquery/plugin-sdk/schema" "github.com/cloudquery/plugins/source/gcp/client" - - "cloud.google.com/go/serviceusage/apiv1" ) func Services() *schema.Table { @@ -19,7 +12,7 @@ func Services() *schema.Table { Name: "gcp_serviceusage_services", Description: `https://cloud.google.com/service-usage/docs/reference/rest/v1/services#Service`, Resolver: fetchServices, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("serviceusage.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", @@ -52,30 +45,3 @@ func Services() *schema.Table { }, } } - -func fetchServices(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { - c := meta.(*client.Client) - req := &pb.ListServicesRequest{ - Parent: "projects/" + c.ProjectId, - PageSize: 200, - Filter: "state:ENABLED", - } - gcpClient, err := serviceusage.NewClient(ctx, c.ClientOptions...) - if err != nil { - return err - } - it := gcpClient.ListServices(ctx, req, c.CallOptions...) - for { - resp, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - return err - } - - res <- resp - - } - return nil -} diff --git a/plugins/source/gcp/resources/services/serviceusage/services_fetch.go b/plugins/source/gcp/resources/services/serviceusage/services_fetch.go new file mode 100644 index 00000000000000..264c163c663965 --- /dev/null +++ b/plugins/source/gcp/resources/services/serviceusage/services_fetch.go @@ -0,0 +1,43 @@ +package serviceusage + +import ( + "context" + + serviceusage "cloud.google.com/go/serviceusage/apiv1" + pb "cloud.google.com/go/serviceusage/apiv1/serviceusagepb" + "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugins/source/gcp/client" + "google.golang.org/api/iterator" +) + +func fetchServices(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { + c := meta.(*client.Client) + if len(c.EnabledServices) > 0 { + for _, svc := range c.EnabledServices[c.ProjectId] { + res <- svc.(*pb.Service) + } + return nil + } + req := &pb.ListServicesRequest{ + Parent: "projects/" + c.ProjectId, + PageSize: 200, + Filter: "state:ENABLED", + } + gcpClient, err := serviceusage.NewClient(ctx, c.ClientOptions...) + if err != nil { + return err + } + it := gcpClient.ListServices(ctx, req, c.CallOptions...) + for { + resp, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + + res <- resp + } + return nil +} diff --git a/plugins/source/gcp/resources/services/serviceusage/services_mock_test.go b/plugins/source/gcp/resources/services/serviceusage/services_mock_test.go index 8163e32e3d7ced..8ff357190804d4 100644 --- a/plugins/source/gcp/resources/services/serviceusage/services_mock_test.go +++ b/plugins/source/gcp/resources/services/serviceusage/services_mock_test.go @@ -1,14 +1,13 @@ -// Code generated by codegen; DO NOT EDIT. - package serviceusage import ( "context" "fmt" + "testing" + "github.com/cloudquery/plugin-sdk/faker" "github.com/cloudquery/plugins/source/gcp/client" "google.golang.org/grpc" - "testing" pb "cloud.google.com/go/serviceusage/apiv1/serviceusagepb" ) @@ -23,7 +22,7 @@ type fakeServicesServer struct { pb.UnimplementedServiceUsageServer } -func (f *fakeServicesServer) ListServices(context.Context, *pb.ListServicesRequest) (*pb.ListServicesResponse, error) { +func (*fakeServicesServer) ListServices(context.Context, *pb.ListServicesRequest) (*pb.ListServicesResponse, error) { resp := pb.ListServicesResponse{} if err := faker.FakeObject(&resp); err != nil { return nil, fmt.Errorf("failed to fake data: %w", err) diff --git a/plugins/source/gcp/resources/services/sql/instances.go b/plugins/source/gcp/resources/services/sql/instances.go index 2ef9585a9277df..1d77409a11dadc 100644 --- a/plugins/source/gcp/resources/services/sql/instances.go +++ b/plugins/source/gcp/resources/services/sql/instances.go @@ -12,7 +12,7 @@ func Instances() *schema.Table { Name: "gcp_sql_instances", Description: `https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1beta4/instances#DatabaseInstance`, Resolver: fetchInstances, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("sqladmin.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/sql/users.go b/plugins/source/gcp/resources/services/sql/users.go index ab00ecfdb7f1eb..6cbee4b4ab9c48 100644 --- a/plugins/source/gcp/resources/services/sql/users.go +++ b/plugins/source/gcp/resources/services/sql/users.go @@ -12,7 +12,7 @@ func Users() *schema.Table { Name: "gcp_sql_users", Description: `https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1beta4/users#User`, Resolver: fetchUsers, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("sqladmin.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/storage/bucket_policies.go b/plugins/source/gcp/resources/services/storage/bucket_policies.go index 3691ce2d6816b5..24ee343861e8fd 100644 --- a/plugins/source/gcp/resources/services/storage/bucket_policies.go +++ b/plugins/source/gcp/resources/services/storage/bucket_policies.go @@ -12,7 +12,7 @@ func BucketPolicies() *schema.Table { Name: "gcp_storage_bucket_policies", Description: `https://cloud.google.com/iam/docs/reference/rest/v1/Policy`, Resolver: fetchBucketPolicies, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("storage.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/plugins/source/gcp/resources/services/storage/buckets.go b/plugins/source/gcp/resources/services/storage/buckets.go index ecff447752461f..7d129cdc15850b 100644 --- a/plugins/source/gcp/resources/services/storage/buckets.go +++ b/plugins/source/gcp/resources/services/storage/buckets.go @@ -12,7 +12,7 @@ func Buckets() *schema.Table { Name: "gcp_storage_buckets", Description: `https://cloud.google.com/storage/docs/json_api/v1/buckets#resource`, Resolver: fetchBuckets, - Multiplex: client.ProjectMultiplex, + Multiplex: client.ProjectMultiplexEnabledServices("storage.googleapis.com"), Columns: []schema.Column{ { Name: "project_id", diff --git a/website/pages/docs/plugins/destinations/bigquery/overview.md b/website/pages/docs/plugins/destinations/bigquery/overview.md index d7d2e0eae8237f..43184b0d161b84 100644 --- a/website/pages/docs/plugins/destinations/bigquery/overview.md +++ b/website/pages/docs/plugins/destinations/bigquery/overview.md @@ -86,7 +86,6 @@ This is the top-level spec used by the BigQuery destination plugin. The time partitioning to use when creating tables. The partition time column used will always be `_cq_sync_time` so that all rows for a sync run will be partitioned on the hour/day the sync started. - - `service_account_key_json` (string) (default: empty). GCP service account key content. This allows for using different service accounts for the GCP source and BigQuery destination. If using service account keys, it is best to use [environment or file variable substitution](/docs/advanced-topics/environment-variable-substitution). diff --git a/website/pages/docs/plugins/sources/gcp/configuration.md b/website/pages/docs/plugins/sources/gcp/configuration.md index 2c520b6bcfb202..b6bf86e3ea9b0f 100644 --- a/website/pages/docs/plugins/sources/gcp/configuration.md +++ b/website/pages/docs/plugins/sources/gcp/configuration.md @@ -54,6 +54,9 @@ This is the (nested) spec used by GCP Source Plugin - `backoff_retries` (int) (default: 0). If specified APIs will be retried with exponential backoff if they are rate limited. This is the max number of retries. + - `enabled_services_only` (bool) (default: false). + If enabled CloudQuery will skip any resources that belong to a service that has been disabled or not been enabled. + ## GCP + Kubernetes (GKE) ```yaml