Skip to content

Commit 7d50d29

Browse files
yevgenypatsyevgenypats
authored
feat(aws): Update to SDK V4 (#11650)
1 parent 186f594 commit 7d50d29

1,790 files changed

Lines changed: 2327 additions & 3556 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/source_aws.yml‎

Lines changed: 46 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -123,51 +123,51 @@ jobs:
123123
run: goreleaser release --timeout 50m --snapshot --clean --skip-validate --skip-publish --skip-sign -f ./plugins/source/aws/.goreleaser.yaml
124124
env:
125125
GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
126-
test-policies:
127-
timeout-minutes: 30
128-
needs: [resolve-runner]
129-
runs-on: ${{ needs.resolve-runner.outputs.runner }}
130-
defaults:
131-
run:
132-
working-directory: ./plugins/source/aws
133-
services:
134-
postgres:
135-
image: postgres:11
136-
env:
137-
POSTGRES_PASSWORD: pass
138-
POSTGRES_USER: postgres
139-
POSTGRES_DB: postgres
140-
ports:
141-
- 5432:5432
142-
# Set health checks to wait until postgres has started
143-
options: >-
144-
--health-cmd pg_isready
145-
--health-interval 10s
146-
--health-timeout 5s
147-
--health-retries 5
148-
steps:
149-
- name: Checkout
150-
uses: actions/checkout@v3
151-
- name: Set up Go 1.x
152-
uses: erezrokah/setup-go@feat/add_cache_prefix
153-
with:
154-
go-version-file: plugins/source/aws/go.mod
155-
cache: true
156-
cache-dependency-path: plugins/source/aws/go.sum
157-
cache-key-prefix: policies-cache-
158-
- name: Build
159-
run: go build .
160-
- name: Setup CloudQuery
161-
uses: cloudquery/setup-cloudquery@v3
162-
with:
163-
version: 'v3.5.0'
164-
- name: Migrate DB
165-
run: cloudquery migrate test/policy_cq_config.yml
166-
env:
167-
CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
168-
- name: Run all policies
169-
run: cd policies && psql -h localhost -p 5432 -U postgres -d postgres -w -f ./policy.sql
170-
env:
171-
PGPASSWORD: pass
126+
# test-policies:
127+
# timeout-minutes: 30
128+
# needs: [resolve-runner]
129+
# runs-on: ${{ needs.resolve-runner.outputs.runner }}
130+
# defaults:
131+
# run:
132+
# working-directory: ./plugins/source/aws
133+
# services:
134+
# postgres:
135+
# image: postgres:11
136+
# env:
137+
# POSTGRES_PASSWORD: pass
138+
# POSTGRES_USER: postgres
139+
# POSTGRES_DB: postgres
140+
# ports:
141+
# - 5432:5432
142+
# # Set health checks to wait until postgres has started
143+
# options: >-
144+
# --health-cmd pg_isready
145+
# --health-interval 10s
146+
# --health-timeout 5s
147+
# --health-retries 5
148+
# steps:
149+
# - name: Checkout
150+
# uses: actions/checkout@v3
151+
# - name: Set up Go 1.x
152+
# uses: erezrokah/setup-go@feat/add_cache_prefix
153+
# with:
154+
# go-version-file: plugins/source/aws/go.mod
155+
# cache: true
156+
# cache-dependency-path: plugins/source/aws/go.sum
157+
# cache-key-prefix: policies-cache-
158+
# - name: Build
159+
# run: go build .
160+
# - name: Setup CloudQuery
161+
# uses: cloudquery/setup-cloudquery@v3
162+
# with:
163+
# version: 'v3.5.0'
164+
# - name: Migrate DB
165+
# run: cloudquery migrate test/policy_cq_config.yml
166+
# env:
167+
# CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
168+
# - name: Run all policies
169+
# run: cd policies && psql -h localhost -p 5432 -U postgres -d postgres -w -f ./policy.sql
170+
# env:
171+
# PGPASSWORD: pass
172172

173173

‎plugins/source/aws/client/client.go‎

Lines changed: 14 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,8 @@ import (
1111
"github.com/aws/aws-sdk-go-v2/service/sts"
1212
wafv2types "github.com/aws/aws-sdk-go-v2/service/wafv2/types"
1313
"github.com/aws/smithy-go/logging"
14-
"github.com/cloudquery/plugin-pb-go/specs"
15-
"github.com/cloudquery/plugin-sdk/v3/backend"
16-
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
17-
"github.com/cloudquery/plugin-sdk/v3/schema"
14+
"github.com/cloudquery/plugin-sdk/v4/schema"
15+
"github.com/cloudquery/plugin-sdk/v4/state"
1816
"github.com/rs/zerolog"
1917
"github.com/thoas/go-funk"
2018
"golang.org/x/sync/errgroup"
@@ -32,7 +30,7 @@ type Client struct {
3230
WAFScope wafv2types.Scope
3331
Partition string
3432
LanguageCode string
35-
Backend backend.Backend
33+
Backend state.Client
3634
specificRegions bool
3735
Spec *Spec
3836
// Do not rely on this field, it will be removed once https://github.com/aws/aws-sdk-go-v2/issues/2163 is resolved
@@ -91,9 +89,8 @@ func (s *ServicesManager) InitServicesForPartitionAccount(partition, accountId s
9189
s.services[partition][accountId].Regions = funk.UniqString(append(s.services[partition][accountId].Regions, svcs.Regions...))
9290
}
9391

94-
func NewAwsClient(logger zerolog.Logger, b backend.Backend, spec *Spec) Client {
92+
func NewAwsClient(logger zerolog.Logger, spec *Spec) Client {
9593
return Client{
96-
Backend: b,
9794
ServicesManager: ServicesManager{
9895
services: ServicesPartitionAccountMap{},
9996
},
@@ -122,6 +119,11 @@ func (c *Client) Services() *Services {
122119
return c.ServicesManager.ServicesByPartitionAccount(c.Partition, c.AccountID)
123120
}
124121

122+
func (c *Client) Duplicate() *Client {
123+
duplicateClient := *c
124+
return &duplicateClient
125+
}
126+
125127
func (c *Client) withPartitionAccountIDAndRegion(partition, accountID, region string) *Client {
126128
return &Client{
127129
Partition: partition,
@@ -173,21 +175,13 @@ func (c *Client) withLanguageCode(code string) *Client {
173175
}
174176

175177
// Configure is the entrypoint into configuring the AWS plugin. It is called by the plugin initialization in resources/plugin/aws.go
176-
func Configure(ctx context.Context, logger zerolog.Logger, spec specs.Source, opts source.Options) (schema.ClientMeta, error) {
177-
var awsPluginSpec Spec
178-
err := spec.UnmarshalSpec(&awsPluginSpec)
179-
if err != nil {
180-
return nil, fmt.Errorf("failed to unmarshal spec: %w", err)
181-
}
182-
183-
err = awsPluginSpec.Validate()
184-
if err != nil {
178+
func Configure(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.ClientMeta, error) {
179+
if err := spec.Validate(); err != nil {
185180
return nil, fmt.Errorf("spec validation failed: %w", err)
186181
}
182+
spec.SetDefaults()
187183

188-
awsPluginSpec.SetDefaults()
189-
190-
client := NewAwsClient(logger, opts.Backend, &awsPluginSpec)
184+
client := NewAwsClient(logger, &spec)
191185

192186
var adminAccountSts AssumeRoleAPIClient
193187

@@ -226,8 +220,7 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec specs.Source, op
226220
return nil
227221
})
228222
}
229-
err = errorGroup.Wait()
230-
if err != nil {
223+
if err := errorGroup.Wait(); err != nil {
231224
return nil, err
232225
}
233226

‎plugins/source/aws/client/columns.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package client
22

33
import (
44
"github.com/apache/arrow/go/v13/arrow"
5-
"github.com/cloudquery/plugin-sdk/v3/schema"
5+
"github.com/cloudquery/plugin-sdk/v4/schema"
66
)
77

88
func DefaultAccountIDColumn(pk bool) schema.Column {

‎plugins/source/aws/client/helpers.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ import (
1313

1414
"github.com/aws/aws-sdk-go-v2/aws/arn"
1515
"github.com/aws/smithy-go"
16-
"github.com/cloudquery/plugin-sdk/v3/schema"
17-
"github.com/cloudquery/plugin-sdk/v3/transformers"
16+
"github.com/cloudquery/plugin-sdk/v4/schema"
17+
"github.com/cloudquery/plugin-sdk/v4/transformers"
1818
)
1919

2020
type AWSService string

‎plugins/source/aws/client/helpers_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import (
99
"github.com/aws/aws-sdk-go-v2/aws"
1010
ttypes "github.com/aws/aws-sdk-go-v2/service/acm/types"
1111
"github.com/aws/aws-sdk-go-v2/service/apigateway/types"
12-
"github.com/cloudquery/plugin-sdk/v3/scalar"
13-
"github.com/cloudquery/plugin-sdk/v3/schema"
12+
"github.com/cloudquery/plugin-sdk/v4/scalar"
13+
"github.com/cloudquery/plugin-sdk/v4/schema"
1414
"github.com/stretchr/testify/assert"
1515
"github.com/stretchr/testify/require"
1616
)

‎plugins/source/aws/client/multiplexers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"sort"
55

66
wafv2types "github.com/aws/aws-sdk-go-v2/service/wafv2/types"
7-
"github.com/cloudquery/plugin-sdk/v3/schema"
7+
"github.com/cloudquery/plugin-sdk/v4/schema"
88
)
99

1010
var AllNamespaces = []string{ // this is only used in applicationautoscaling

‎plugins/source/aws/client/resolvers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"fmt"
66
"reflect"
77

8-
"github.com/cloudquery/plugin-sdk/v3/schema"
8+
"github.com/cloudquery/plugin-sdk/v4/schema"
99
"github.com/mitchellh/hashstructure/v2"
1010
)
1111

‎plugins/source/aws/client/resolvers_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@ import (
44
"context"
55
"testing"
66

7-
"github.com/cloudquery/plugin-sdk/v3/scalar"
8-
sdkTypes "github.com/cloudquery/plugin-sdk/v3/types"
7+
"github.com/cloudquery/plugin-sdk/v4/scalar"
8+
sdkTypes "github.com/cloudquery/plugin-sdk/v4/types"
99

1010
"github.com/aws/aws-sdk-go-v2/aws"
1111
types1 "github.com/aws/aws-sdk-go-v2/service/codepipeline/types"
1212
types2 "github.com/aws/aws-sdk-go-v2/service/redshift/types"
13-
"github.com/cloudquery/plugin-sdk/v3/schema"
13+
"github.com/cloudquery/plugin-sdk/v4/schema"
1414
"github.com/stretchr/testify/assert"
1515
)
1616

‎plugins/source/aws/client/spec.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ import (
66
"regexp"
77

88
"github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions"
9+
"github.com/cloudquery/plugin-sdk/v4/scheduler"
10+
)
11+
12+
const (
13+
defaultMaxConcurrency = 50000
914
)
1015

1116
type Account struct {
@@ -46,6 +51,8 @@ type Spec struct {
4651
InitializationConcurrency int `json:"initialization_concurrency"`
4752
UsePaidAPIs bool `json:"use_paid_apis"`
4853
TableOptions *tableoptions.TableOptions `json:"table_options,omitempty"`
54+
Concurrency int `json:"concurrency"`
55+
Scheduler scheduler.Strategy `json:"scheduler,omitempty"`
4956
}
5057

5158
func (s *Spec) Validate() error {
@@ -100,4 +107,7 @@ func (s *Spec) SetDefaults() {
100107
if s.TableOptions == nil {
101108
s.TableOptions = &tableoptions.TableOptions{}
102109
}
110+
if s.Concurrency == 0 {
111+
s.Concurrency = defaultMaxConcurrency
112+
}
103113
}

‎plugins/source/aws/client/tableoptions/accessanalyzer.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66

77
"github.com/aws/aws-sdk-go-v2/aws"
88
"github.com/aws/aws-sdk-go-v2/service/accessanalyzer"
9-
"github.com/cloudquery/plugin-sdk/v3/caser"
9+
"github.com/cloudquery/plugin-sdk/v4/caser"
1010
)
1111

1212
type AccessanalyzerFindings struct {

0 commit comments

Comments
 (0)