From 381499e4bd27c943765f0bdc441b0096de0172ff Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 23 Dec 2022 11:06:31 -0500 Subject: [PATCH 1/7] Update client.go --- plugins/source/aws/client/client.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/source/aws/client/client.go b/plugins/source/aws/client/client.go index 9cfa59c6adbcf0..2fae8fe9eba031 100644 --- a/plugins/source/aws/client/client.go +++ b/plugins/source/aws/client/client.go @@ -235,6 +235,18 @@ func configureAwsClient(ctx context.Context, logger zerolog.Logger, awsConfig *S }) }), } + if awsConfig.EndpointURL != "" { + configFns = append(configFns, config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc( + func(service, region string, options ...any) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: awsConfig.EndpointURL, + HostnameImmutable: aws.ToBool(awsConfig.HostnameImmutable), + PartitionID: awsConfig.PartitionID, + SigningRegion: awsConfig.SigningRegion, + }, nil + })), + ) + } if account.DefaultRegion != "" { // According to the docs: If multiple WithDefaultRegion calls are made, the last call overrides the previous call values From e98d6a4e25b6fac5f4d044d540553346fcedb21d Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 23 Dec 2022 11:06:33 -0500 Subject: [PATCH 2/7] Update spec.go --- plugins/source/aws/client/spec.go | 33 +++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/plugins/source/aws/client/spec.go b/plugins/source/aws/client/spec.go index 85a83000f37a7b..1e3e1be6862e0c 100644 --- a/plugins/source/aws/client/spec.go +++ b/plugins/source/aws/client/spec.go @@ -1,5 +1,7 @@ package client +import "fmt" + type Account struct { ID string `json:"id"` AccountName string `json:"account_name,omitempty"` @@ -23,10 +25,29 @@ type AwsOrg struct { } type Spec struct { - Regions []string `json:"regions,omitempty"` - Accounts []Account `json:"accounts"` - Organization *AwsOrg `json:"org"` - AWSDebug bool `json:"aws_debug,omitempty"` - MaxRetries *int `json:"max_retries,omitempty"` - MaxBackoff *int `json:"max_backoff,omitempty"` + Regions []string `json:"regions,omitempty"` + Accounts []Account `json:"accounts"` + Organization *AwsOrg `json:"org"` + AWSDebug bool `json:"aws_debug,omitempty"` + MaxRetries *int `json:"max_retries,omitempty"` + MaxBackoff *int `json:"max_backoff,omitempty"` + EndpointURL string `json:"custom_endpoint_url,omitempty"` + HostnameImmutable *bool `json:"custom_endpoint_hostname_immutable,omitempty"` + PartitionID string `json:"custom_endpoint_partition_id,omitempty"` + SigningRegion string `json:"custom_endpoint_signing_region,omitempty"` +} + +func (s *Spec) Validate() error { + if s.EndpointURL != "" { + if s.PartitionID == "" { + return fmt.Errorf("custom_endpoint_partition_id is required when custom_endpoint_url is set") + } + if s.SigningRegion == "" { + return fmt.Errorf("custom_endpoint_signing_region is required when custom_endpoint_url is set") + } + if s.HostnameImmutable == nil { + return fmt.Errorf("custom_endpoint_hostname_immutable is required when custom_endpoint_url is set") + } + } + return nil } From bef0d698bd9a01f1e5e325ec5cfd483e2d0b73ff Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 23 Dec 2022 12:07:44 -0500 Subject: [PATCH 3/7] Update configuration.md --- .../docs/plugins/sources/aws/configuration.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/website/pages/docs/plugins/sources/aws/configuration.md b/website/pages/docs/plugins/sources/aws/configuration.md index dac5a3bf8b28c1..504c387a5c15f8 100644 --- a/website/pages/docs/plugins/sources/aws/configuration.md +++ b/website/pages/docs/plugins/sources/aws/configuration.md @@ -100,6 +100,31 @@ This is the (nested) spec used by the AWS source plugin. If true, will log AWS debug logs, including retries and other request/response metadata +- `max_retries` (int) (default: 10) + + Defines the maximum number of times an API request will be retried + +- `max_retries` (int) (max_backoff: 30) + + Defines the duration between retry attempts + +- `custom_endpoint_url` (string) (default: not used) + + The base URL endpoint the SDK API clients will use to make API calls to. The SDK will suffix URI path and query elements to this endpoint + +- `custom_endpoint_hostname_immutable` (bool) (default: not used) + + Specifies if the endpoint's hostname can be modified by the SDK's API client. When using something like LocalStack make sure to set it equal to `True` + +- `custom_endpoint_partition_id` (string) (default: not used) + + The AWS partition the endpoint belongs to + +- `custom_endpoint_signing_region` (string) (default: not used) + + The region that should be used for signing the request to the endpoint + + ## accounts This is used to specify one or more accounts to extract information from. Note that it should be an array of objects, each with the following fields: From 2b28da5b5a0f6a9daf7306cb61b77545493dce24 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 23 Dec 2022 12:11:39 -0500 Subject: [PATCH 4/7] Update spec.go --- plugins/source/aws/client/spec.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/source/aws/client/spec.go b/plugins/source/aws/client/spec.go index 7a1a59341be0ab..844fe2b74fb770 100644 --- a/plugins/source/aws/client/spec.go +++ b/plugins/source/aws/client/spec.go @@ -57,10 +57,7 @@ func (s *Spec) Validate() error { return fmt.Errorf("custom_endpoint_hostname_immutable is required when custom_endpoint_url is set") } } - return nil -} -func (s *Spec) Validate() error { if s.Organization != nil && len(s.Accounts) > 0 { return errors.New("specifying accounts via both the Accounts and Org properties is not supported. To achieve both, use multiple source configurations") } From e586f1fc5acd3b2011ad3f5547e4697bd4c40e14 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 23 Dec 2022 12:59:59 -0500 Subject: [PATCH 5/7] Create Configuring CloudQuery with LocalStack.md --- .../Configuring CloudQuery with LocalStack.md | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 website/pages/blog/Configuring CloudQuery with LocalStack.md diff --git a/website/pages/blog/Configuring CloudQuery with LocalStack.md b/website/pages/blog/Configuring CloudQuery with LocalStack.md new file mode 100644 index 00000000000000..3dc04934318f6e --- /dev/null +++ b/website/pages/blog/Configuring CloudQuery with LocalStack.md @@ -0,0 +1,82 @@ +--- +title: CloudQuery and LocalStack +tag: integration +date: 2022/12/23 +description: >- + How to setup CloudQuery to work with LocalStack +author: benjamin +--- + +import { BlogHeader } from "../../components/BlogHeader" + + + + +In this tutorial we will walk through how to configure CloudQuery to sync from a LocalStack instance. + + + +## Introduction to LocalStack + +LocalStack describes itself as A fully functional local cloud stack that enables developers to Develop and test their cloud and serverless applications offline! + + + +## Walkthrough + +Before beginning this tutorial make sure you have the following tools installed: +- Docker +- CloudQuery + + +### Step 1 + +Start `localstack` + +```bash +docker run --rm -it \ + -p 4566:4566 \ + -p 4510-4559:4510-4559 \ + -e DEBUG=1 \ + localstack/localstack +``` +## Step 2 + +Configure CloudQuery to use the LocalStack endpoint +```yml +kind: source +spec: + # Source spec section + name: "aws" + registry: "github" + path: "cloudquery/aws" + version: "VERSION_SOURCE_AWS" + destinations: ["postgresql"] + skip_tables: + - aws_route53_delegation_sets + - aws_iam_policies + tables: + - "*" + spec: + regions: + - "us-east-1" + + # Configure the AWS SDK to use the localstack endpoint + custom_endpoint_url: http://localhost:4566 + custom_endpoint_hostname_immutable: true + custom_endpoint_partition_id: "aws" + custom_endpoint_signing_region: "us-east-1" + # There is no reason to retry failed requests to localstack + max_retries: 0 +``` + +Note that it is important to skip `aws_route53_delegation_sets` and `aws_iam_policies` as bugs in LocalStack force CloudQuery into an infinite loop + +### Step 3 + +Run CloudQuery + + +``` bash +cloudquery sync config.yml +``` From 40fb0726662bd96efe7a018b8fa8185501f68d38 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 23 Dec 2022 13:14:29 -0500 Subject: [PATCH 6/7] Update Configuring CloudQuery with LocalStack.md --- website/pages/blog/Configuring CloudQuery with LocalStack.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/pages/blog/Configuring CloudQuery with LocalStack.md b/website/pages/blog/Configuring CloudQuery with LocalStack.md index 3dc04934318f6e..98e2fc27c742b4 100644 --- a/website/pages/blog/Configuring CloudQuery with LocalStack.md +++ b/website/pages/blog/Configuring CloudQuery with LocalStack.md @@ -35,7 +35,7 @@ Start `localstack` ```bash docker run --rm -it \ - -p 4566:4566 \ + -p 4566:4566 \ -p 4510-4559:4510-4559 \ -e DEBUG=1 \ localstack/localstack From a5d36b2ae5c9bbfa5d9db0f548a5679efdd5bab1 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 23 Dec 2022 13:15:09 -0500 Subject: [PATCH 7/7] Delete Configuring CloudQuery with LocalStack.md --- .../Configuring CloudQuery with LocalStack.md | 82 ------------------- 1 file changed, 82 deletions(-) delete mode 100644 website/pages/blog/Configuring CloudQuery with LocalStack.md diff --git a/website/pages/blog/Configuring CloudQuery with LocalStack.md b/website/pages/blog/Configuring CloudQuery with LocalStack.md deleted file mode 100644 index 98e2fc27c742b4..00000000000000 --- a/website/pages/blog/Configuring CloudQuery with LocalStack.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -title: CloudQuery and LocalStack -tag: integration -date: 2022/12/23 -description: >- - How to setup CloudQuery to work with LocalStack -author: benjamin ---- - -import { BlogHeader } from "../../components/BlogHeader" - - - - -In this tutorial we will walk through how to configure CloudQuery to sync from a LocalStack instance. - - - -## Introduction to LocalStack - -LocalStack describes itself as A fully functional local cloud stack that enables developers to Develop and test their cloud and serverless applications offline! - - - -## Walkthrough - -Before beginning this tutorial make sure you have the following tools installed: -- Docker -- CloudQuery - - -### Step 1 - -Start `localstack` - -```bash -docker run --rm -it \ - -p 4566:4566 \ - -p 4510-4559:4510-4559 \ - -e DEBUG=1 \ - localstack/localstack -``` -## Step 2 - -Configure CloudQuery to use the LocalStack endpoint -```yml -kind: source -spec: - # Source spec section - name: "aws" - registry: "github" - path: "cloudquery/aws" - version: "VERSION_SOURCE_AWS" - destinations: ["postgresql"] - skip_tables: - - aws_route53_delegation_sets - - aws_iam_policies - tables: - - "*" - spec: - regions: - - "us-east-1" - - # Configure the AWS SDK to use the localstack endpoint - custom_endpoint_url: http://localhost:4566 - custom_endpoint_hostname_immutable: true - custom_endpoint_partition_id: "aws" - custom_endpoint_signing_region: "us-east-1" - # There is no reason to retry failed requests to localstack - max_retries: 0 -``` - -Note that it is important to skip `aws_route53_delegation_sets` and `aws_iam_policies` as bugs in LocalStack force CloudQuery into an infinite loop - -### Step 3 - -Run CloudQuery - - -``` bash -cloudquery sync config.yml -```