From f66cb1384d33adb114a8bbbf590a37fa2c88e9b4 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 13 Feb 2023 16:38:06 +0000 Subject: [PATCH 1/4] fix(aws): Fix ARN account ID in aws_directconnect_gateways --- .../services/directconnect/gateways.go | 2 +- .../services/directconnect/gateways_fetch.go | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/plugins/source/aws/resources/services/directconnect/gateways.go b/plugins/source/aws/resources/services/directconnect/gateways.go index c2fafd22e71b8d..f43c47d3eb2802 100644 --- a/plugins/source/aws/resources/services/directconnect/gateways.go +++ b/plugins/source/aws/resources/services/directconnect/gateways.go @@ -20,7 +20,7 @@ func Gateways() *schema.Table { { Name: "arn", Type: schema.TypeString, - Resolver: resolveGatewayARN(), + Resolver: resolveGatewayARN, CreationOptions: schema.ColumnCreationOptions{ PrimaryKey: true, }, diff --git a/plugins/source/aws/resources/services/directconnect/gateways_fetch.go b/plugins/source/aws/resources/services/directconnect/gateways_fetch.go index 0ba64a18cc0334..d7356592cad347 100644 --- a/plugins/source/aws/resources/services/directconnect/gateways_fetch.go +++ b/plugins/source/aws/resources/services/directconnect/gateways_fetch.go @@ -2,8 +2,10 @@ package directconnect import ( "context" + "strings" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/directconnect" "github.com/aws/aws-sdk-go-v2/service/directconnect/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" @@ -66,8 +68,15 @@ func fetchDirectconnectGatewayAttachments(ctx context.Context, meta schema.Clien return nil } -func resolveGatewayARN() schema.ColumnResolver { - return client.ResolveARNWithAccount(client.DirectConnectService, func(resource *schema.Resource) ([]string, error) { - return []string{"dx-gateway", *resource.Item.(types.DirectConnectGateway).DirectConnectGatewayId}, nil - }) +func resolveGatewayARN(_ context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + cl := meta.(*client.Client) + gw := resource.Item.(types.DirectConnectGateway) + + return resource.Set(c.Name, arn.ARN{ + Partition: cl.Partition, + Service: "directconnect", + AccountID: *gw.OwnerAccount, + Region: cl.Region, + Resource: strings.Join([]string{"dx-gateway", *gw.DirectConnectGatewayId}, "/"), + }.String()) } From 49b2c7c3863141615cbfc37a51fe4ba549dcac83 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 13 Feb 2023 16:43:13 +0000 Subject: [PATCH 2/4] Include source/queried account ID in the PKs --- .../docs/tables/aws_directconnect_gateway_associations.md | 6 +++--- .../docs/tables/aws_directconnect_gateway_attachments.md | 6 +++--- .../source/aws/docs/tables/aws_directconnect_gateways.md | 4 ++-- .../services/directconnect/gateway_associations.go | 2 +- .../resources/services/directconnect/gateway_attachments.go | 2 +- .../source/aws/resources/services/directconnect/gateways.go | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/source/aws/docs/tables/aws_directconnect_gateway_associations.md b/plugins/source/aws/docs/tables/aws_directconnect_gateway_associations.md index df5baf24860b63..dc6c3fd6621945 100644 --- a/plugins/source/aws/docs/tables/aws_directconnect_gateway_associations.md +++ b/plugins/source/aws/docs/tables/aws_directconnect_gateway_associations.md @@ -2,7 +2,7 @@ https://docs.aws.amazon.com/directconnect/latest/APIReference/API_DirectConnectGatewayAssociation.html -The primary key for this table is **_cq_id**. +The primary key for this table is **account_id**. ## Relations @@ -14,9 +14,9 @@ This table depends on [aws_directconnect_gateways](aws_directconnect_gateways.md | ------------- | ------------- | |_cq_source_name|String| |_cq_sync_time|Timestamp| -|_cq_id (PK)|UUID| +|_cq_id|UUID| |_cq_parent_id|UUID| -|account_id|String| +|account_id (PK)|String| |region|String| |gateway_arn|String| |gateway_id|String| diff --git a/plugins/source/aws/docs/tables/aws_directconnect_gateway_attachments.md b/plugins/source/aws/docs/tables/aws_directconnect_gateway_attachments.md index 3894361aec6e76..52d302028c4354 100644 --- a/plugins/source/aws/docs/tables/aws_directconnect_gateway_attachments.md +++ b/plugins/source/aws/docs/tables/aws_directconnect_gateway_attachments.md @@ -2,7 +2,7 @@ https://docs.aws.amazon.com/directconnect/latest/APIReference/API_DirectConnectGatewayAttachment.html -The primary key for this table is **_cq_id**. +The primary key for this table is **account_id**. ## Relations @@ -14,9 +14,9 @@ This table depends on [aws_directconnect_gateways](aws_directconnect_gateways.md | ------------- | ------------- | |_cq_source_name|String| |_cq_sync_time|Timestamp| -|_cq_id (PK)|UUID| +|_cq_id|UUID| |_cq_parent_id|UUID| -|account_id|String| +|account_id (PK)|String| |region|String| |gateway_arn|String| |gateway_id|String| diff --git a/plugins/source/aws/docs/tables/aws_directconnect_gateways.md b/plugins/source/aws/docs/tables/aws_directconnect_gateways.md index baa9ee30b1ec5a..09e21a305cbe86 100644 --- a/plugins/source/aws/docs/tables/aws_directconnect_gateways.md +++ b/plugins/source/aws/docs/tables/aws_directconnect_gateways.md @@ -2,7 +2,7 @@ https://docs.aws.amazon.com/directconnect/latest/APIReference/API_DirectConnectGateway.html -The primary key for this table is **arn**. +The composite primary key for this table is (**account_id**, **arn**). ## Relations @@ -18,7 +18,7 @@ The following tables depend on aws_directconnect_gateways: |_cq_sync_time|Timestamp| |_cq_id|UUID| |_cq_parent_id|UUID| -|account_id|String| +|account_id (PK)|String| |region|String| |arn (PK)|String| |id|String| diff --git a/plugins/source/aws/resources/services/directconnect/gateway_associations.go b/plugins/source/aws/resources/services/directconnect/gateway_associations.go index f0fa120076a8fb..f68208046e4086 100644 --- a/plugins/source/aws/resources/services/directconnect/gateway_associations.go +++ b/plugins/source/aws/resources/services/directconnect/gateway_associations.go @@ -15,7 +15,7 @@ func GatewayAssociations() *schema.Table { Multiplex: client.ServiceAccountRegionMultiplexer("directconnect"), Transform: transformers.TransformWithStruct(&types.DirectConnectGatewayAssociation{}), Columns: []schema.Column{ - client.DefaultAccountIDColumn(false), + client.DefaultAccountIDColumn(true), client.DefaultRegionColumn(false), { Name: "gateway_arn", diff --git a/plugins/source/aws/resources/services/directconnect/gateway_attachments.go b/plugins/source/aws/resources/services/directconnect/gateway_attachments.go index 9e67870a7363f2..78d44f08c6b115 100644 --- a/plugins/source/aws/resources/services/directconnect/gateway_attachments.go +++ b/plugins/source/aws/resources/services/directconnect/gateway_attachments.go @@ -15,7 +15,7 @@ func GatewayAttachments() *schema.Table { Multiplex: client.ServiceAccountRegionMultiplexer("directconnect"), Transform: transformers.TransformWithStruct(&types.DirectConnectGatewayAttachment{}), Columns: []schema.Column{ - client.DefaultAccountIDColumn(false), + client.DefaultAccountIDColumn(true), client.DefaultRegionColumn(false), { Name: "gateway_arn", diff --git a/plugins/source/aws/resources/services/directconnect/gateways.go b/plugins/source/aws/resources/services/directconnect/gateways.go index f43c47d3eb2802..0a8df1e77309a7 100644 --- a/plugins/source/aws/resources/services/directconnect/gateways.go +++ b/plugins/source/aws/resources/services/directconnect/gateways.go @@ -15,7 +15,7 @@ func Gateways() *schema.Table { Multiplex: client.ServiceAccountRegionMultiplexer("directconnect"), Transform: transformers.TransformWithStruct(&types.DirectConnectGateway{}), Columns: []schema.Column{ - client.DefaultAccountIDColumn(false), + client.DefaultAccountIDColumn(true), client.DefaultRegionColumn(false), { Name: "arn", From b0c13ea8dbbdf5a27600302534203afe4da83223 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 13 Feb 2023 16:44:48 +0000 Subject: [PATCH 3/4] keep _cq_id as PK in subs --- .../docs/tables/aws_directconnect_gateway_associations.md | 6 +++--- .../docs/tables/aws_directconnect_gateway_attachments.md | 6 +++--- .../services/directconnect/gateway_associations.go | 2 +- .../resources/services/directconnect/gateway_attachments.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/source/aws/docs/tables/aws_directconnect_gateway_associations.md b/plugins/source/aws/docs/tables/aws_directconnect_gateway_associations.md index dc6c3fd6621945..df5baf24860b63 100644 --- a/plugins/source/aws/docs/tables/aws_directconnect_gateway_associations.md +++ b/plugins/source/aws/docs/tables/aws_directconnect_gateway_associations.md @@ -2,7 +2,7 @@ https://docs.aws.amazon.com/directconnect/latest/APIReference/API_DirectConnectGatewayAssociation.html -The primary key for this table is **account_id**. +The primary key for this table is **_cq_id**. ## Relations @@ -14,9 +14,9 @@ This table depends on [aws_directconnect_gateways](aws_directconnect_gateways.md | ------------- | ------------- | |_cq_source_name|String| |_cq_sync_time|Timestamp| -|_cq_id|UUID| +|_cq_id (PK)|UUID| |_cq_parent_id|UUID| -|account_id (PK)|String| +|account_id|String| |region|String| |gateway_arn|String| |gateway_id|String| diff --git a/plugins/source/aws/docs/tables/aws_directconnect_gateway_attachments.md b/plugins/source/aws/docs/tables/aws_directconnect_gateway_attachments.md index 52d302028c4354..3894361aec6e76 100644 --- a/plugins/source/aws/docs/tables/aws_directconnect_gateway_attachments.md +++ b/plugins/source/aws/docs/tables/aws_directconnect_gateway_attachments.md @@ -2,7 +2,7 @@ https://docs.aws.amazon.com/directconnect/latest/APIReference/API_DirectConnectGatewayAttachment.html -The primary key for this table is **account_id**. +The primary key for this table is **_cq_id**. ## Relations @@ -14,9 +14,9 @@ This table depends on [aws_directconnect_gateways](aws_directconnect_gateways.md | ------------- | ------------- | |_cq_source_name|String| |_cq_sync_time|Timestamp| -|_cq_id|UUID| +|_cq_id (PK)|UUID| |_cq_parent_id|UUID| -|account_id (PK)|String| +|account_id|String| |region|String| |gateway_arn|String| |gateway_id|String| diff --git a/plugins/source/aws/resources/services/directconnect/gateway_associations.go b/plugins/source/aws/resources/services/directconnect/gateway_associations.go index f68208046e4086..f0fa120076a8fb 100644 --- a/plugins/source/aws/resources/services/directconnect/gateway_associations.go +++ b/plugins/source/aws/resources/services/directconnect/gateway_associations.go @@ -15,7 +15,7 @@ func GatewayAssociations() *schema.Table { Multiplex: client.ServiceAccountRegionMultiplexer("directconnect"), Transform: transformers.TransformWithStruct(&types.DirectConnectGatewayAssociation{}), Columns: []schema.Column{ - client.DefaultAccountIDColumn(true), + client.DefaultAccountIDColumn(false), client.DefaultRegionColumn(false), { Name: "gateway_arn", diff --git a/plugins/source/aws/resources/services/directconnect/gateway_attachments.go b/plugins/source/aws/resources/services/directconnect/gateway_attachments.go index 78d44f08c6b115..9e67870a7363f2 100644 --- a/plugins/source/aws/resources/services/directconnect/gateway_attachments.go +++ b/plugins/source/aws/resources/services/directconnect/gateway_attachments.go @@ -15,7 +15,7 @@ func GatewayAttachments() *schema.Table { Multiplex: client.ServiceAccountRegionMultiplexer("directconnect"), Transform: transformers.TransformWithStruct(&types.DirectConnectGatewayAttachment{}), Columns: []schema.Column{ - client.DefaultAccountIDColumn(true), + client.DefaultAccountIDColumn(false), client.DefaultRegionColumn(false), { Name: "gateway_arn", From fda10d70ee3deca67293de30d6be13e43b770c33 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 13 Feb 2023 17:13:05 +0000 Subject: [PATCH 4/4] Remove region from ARN, add it as PK --- plugins/source/aws/docs/tables/aws_directconnect_gateways.md | 4 ++-- .../source/aws/resources/services/directconnect/gateways.go | 2 +- .../aws/resources/services/directconnect/gateways_fetch.go | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/source/aws/docs/tables/aws_directconnect_gateways.md b/plugins/source/aws/docs/tables/aws_directconnect_gateways.md index 09e21a305cbe86..a65d8633bf310e 100644 --- a/plugins/source/aws/docs/tables/aws_directconnect_gateways.md +++ b/plugins/source/aws/docs/tables/aws_directconnect_gateways.md @@ -2,7 +2,7 @@ https://docs.aws.amazon.com/directconnect/latest/APIReference/API_DirectConnectGateway.html -The composite primary key for this table is (**account_id**, **arn**). +The composite primary key for this table is (**account_id**, **region**, **arn**). ## Relations @@ -19,7 +19,7 @@ The following tables depend on aws_directconnect_gateways: |_cq_id|UUID| |_cq_parent_id|UUID| |account_id (PK)|String| -|region|String| +|region (PK)|String| |arn (PK)|String| |id|String| |amazon_side_asn|Int| diff --git a/plugins/source/aws/resources/services/directconnect/gateways.go b/plugins/source/aws/resources/services/directconnect/gateways.go index 0a8df1e77309a7..fc10f5c3508b12 100644 --- a/plugins/source/aws/resources/services/directconnect/gateways.go +++ b/plugins/source/aws/resources/services/directconnect/gateways.go @@ -16,7 +16,7 @@ func Gateways() *schema.Table { Transform: transformers.TransformWithStruct(&types.DirectConnectGateway{}), Columns: []schema.Column{ client.DefaultAccountIDColumn(true), - client.DefaultRegionColumn(false), + client.DefaultRegionColumn(true), { Name: "arn", Type: schema.TypeString, diff --git a/plugins/source/aws/resources/services/directconnect/gateways_fetch.go b/plugins/source/aws/resources/services/directconnect/gateways_fetch.go index d7356592cad347..723c351534f121 100644 --- a/plugins/source/aws/resources/services/directconnect/gateways_fetch.go +++ b/plugins/source/aws/resources/services/directconnect/gateways_fetch.go @@ -76,7 +76,6 @@ func resolveGatewayARN(_ context.Context, meta schema.ClientMeta, resource *sche Partition: cl.Partition, Service: "directconnect", AccountID: *gw.OwnerAccount, - Region: cl.Region, Resource: strings.Join([]string{"dx-gateway", *gw.DirectConnectGatewayId}, "/"), }.String()) }