From ff8a367ed3765f5913d1b0796c470dd4a289f061 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 20 Dec 2022 18:19:03 -0500 Subject: [PATCH 1/9] Paginate Ec2 Describe Images --- plugins/source/aws/go.mod | 2 +- plugins/source/aws/go.sum | 2 ++ .../resources/services/ec2/images_fetch.go | 27 +++++++++++++------ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/plugins/source/aws/go.mod b/plugins/source/aws/go.mod index adb609271a8c8e..82de103581d521 100644 --- a/plugins/source/aws/go.mod +++ b/plugins/source/aws/go.mod @@ -33,7 +33,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/directconnect v1.17.25 github.com/aws/aws-sdk-go-v2/service/docdb v1.19.18 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.9 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.76.1 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 github.com/aws/aws-sdk-go-v2/service/ecr v1.17.25 github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.13.22 github.com/aws/aws-sdk-go-v2/service/ecs v1.22.0 diff --git a/plugins/source/aws/go.sum b/plugins/source/aws/go.sum index b95d87fcd1012a..a75319c4353449 100644 --- a/plugins/source/aws/go.sum +++ b/plugins/source/aws/go.sum @@ -113,6 +113,8 @@ github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.9 h1:b5IdivLEHiIPErQoNNLAt7s github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.9/go.mod h1:uP2wpt43//qh6NqMFslaRu53A2YbnFStkV4Wn1Ldels= github.com/aws/aws-sdk-go-v2/service/ec2 v1.76.1 h1:1bWBmJpLvYijhZEKnIebuhaFrEKcBuHFWhcS+PFg0Yo= github.com/aws/aws-sdk-go-v2/service/ec2 v1.76.1/go.mod h1:mV0E7631M1eXdB+tlGFIw6JxfsC7Pz7+7Aw15oLVhZw= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 h1:m6HYlpZlTWb9vHuuRHpWRieqPHWlS0mvQ90OJNrG/Nk= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0/go.mod h1:mV0E7631M1eXdB+tlGFIw6JxfsC7Pz7+7Aw15oLVhZw= github.com/aws/aws-sdk-go-v2/service/ecr v1.17.25 h1:1bido4Jtd8CG9JcheRITQMQ820RE6mw+ool5ln9jbtY= github.com/aws/aws-sdk-go-v2/service/ecr v1.17.25/go.mod h1:9yGOFsa2OcdyePojE89xNGtdBusTyc8ocjpiuFtFc0g= github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.13.22 h1:mAfbm9OuaKTQ/KweojB47hTHSRxf1Z3h721bOxLJjNY= diff --git a/plugins/source/aws/resources/services/ec2/images_fetch.go b/plugins/source/aws/resources/services/ec2/images_fetch.go index 51e2218b2fa76c..d92aaf311be43c 100644 --- a/plugins/source/aws/resources/services/ec2/images_fetch.go +++ b/plugins/source/aws/resources/services/ec2/images_fetch.go @@ -19,22 +19,33 @@ func fetchEc2Images(ctx context.Context, meta schema.ClientMeta, parent *schema. g, ctx := errgroup.WithContext(ctx) g.Go(func() error { // fetch ec2.Images owned by this account - response, err := svc.DescribeImages(ctx, &ec2.DescribeImagesInput{Owners: []string{"self"}}) - if err != nil { - return err + pag := ec2.NewDescribeImagesPaginator(svc, &ec2.DescribeImagesInput{ + Owners: []string{"self"}, + }) + for pag.HasMorePages() { + resp, err := pag.NextPage(ctx) + if err != nil { + return err + } + res <- resp.Images } - res <- response.Images return nil }) g.Go(func() error { // fetch ec2.Images that are shared with this account - response, err := svc.DescribeImages(ctx, &ec2.DescribeImagesInput{ExecutableUsers: []string{"self"}}) - if err != nil { - return err + pag := ec2.NewDescribeImagesPaginator(svc, &ec2.DescribeImagesInput{ + ExecutableUsers: []string{"self"}, + }) + for pag.HasMorePages() { + resp, err := pag.NextPage(ctx) + if err != nil { + return err + } + res <- resp.Images } - res <- response.Images return nil + }) return g.Wait() From e5b1151b374a6f03b26aa26f3f61930ad6bd5d90 Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 21 Dec 2022 07:26:20 -0500 Subject: [PATCH 2/9] Update images_fetch.go --- plugins/source/aws/resources/services/ec2/images_fetch.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/source/aws/resources/services/ec2/images_fetch.go b/plugins/source/aws/resources/services/ec2/images_fetch.go index d92aaf311be43c..84f6d71b7d2734 100644 --- a/plugins/source/aws/resources/services/ec2/images_fetch.go +++ b/plugins/source/aws/resources/services/ec2/images_fetch.go @@ -45,7 +45,6 @@ func fetchEc2Images(ctx context.Context, meta schema.ClientMeta, parent *schema. res <- resp.Images } return nil - }) return g.Wait() From 04bdf72679a85ef283d4fe53980a9005be62ef60 Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 21 Dec 2022 07:28:15 -0500 Subject: [PATCH 3/9] Update go.sum --- plugins/source/aws/go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/source/aws/go.sum b/plugins/source/aws/go.sum index 11cc622688dee4..75bbebcc2c4945 100644 --- a/plugins/source/aws/go.sum +++ b/plugins/source/aws/go.sum @@ -112,8 +112,6 @@ github.com/aws/aws-sdk-go-v2/service/docdb v1.19.18 h1:6J0Sf+2jBbbdBmDC5ps5f93LA github.com/aws/aws-sdk-go-v2/service/docdb v1.19.18/go.mod h1:L5e0d7YdvUtAiG6AAQqFi5mM1kHkLz32qkoYvZLm1qU= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.9 h1:b5IdivLEHiIPErQoNNLAt7sECZxnL9BT4Bvp7qxCTwQ= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.9/go.mod h1:uP2wpt43//qh6NqMFslaRu53A2YbnFStkV4Wn1Ldels= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.76.1 h1:1bWBmJpLvYijhZEKnIebuhaFrEKcBuHFWhcS+PFg0Yo= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.76.1/go.mod h1:mV0E7631M1eXdB+tlGFIw6JxfsC7Pz7+7Aw15oLVhZw= github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 h1:m6HYlpZlTWb9vHuuRHpWRieqPHWlS0mvQ90OJNrG/Nk= github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0/go.mod h1:mV0E7631M1eXdB+tlGFIw6JxfsC7Pz7+7Aw15oLVhZw= github.com/aws/aws-sdk-go-v2/service/ecr v1.17.25 h1:1bido4Jtd8CG9JcheRITQMQ820RE6mw+ool5ln9jbtY= From 91a7c20b9f797ab57bfe5f3753e92961a2d08c48 Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 21 Dec 2022 08:27:35 -0500 Subject: [PATCH 4/9] Update source_aws.yml --- .github/workflows/source_aws.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/source_aws.yml b/.github/workflows/source_aws.yml index 81ef024a551ba9..ca4e05515ed11e 100644 --- a/.github/workflows/source_aws.yml +++ b/.github/workflows/source_aws.yml @@ -58,7 +58,7 @@ jobs: with: version: v1.50.1 working-directory: plugins/source/aws - args: "--config ../../.golangci.yml" + args: "--config ../../.golangci.yml --new-from-rev=HEAD~" - name: Get dependencies run: go get -t -d ./... - name: Build From 921682d8af29730763398072116d4c0165b42afc Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 21 Dec 2022 08:45:45 -0500 Subject: [PATCH 5/9] Update images_fetch.go --- plugins/source/aws/resources/services/ec2/images_fetch.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/source/aws/resources/services/ec2/images_fetch.go b/plugins/source/aws/resources/services/ec2/images_fetch.go index 84f6d71b7d2734..d92aaf311be43c 100644 --- a/plugins/source/aws/resources/services/ec2/images_fetch.go +++ b/plugins/source/aws/resources/services/ec2/images_fetch.go @@ -45,6 +45,7 @@ func fetchEc2Images(ctx context.Context, meta schema.ClientMeta, parent *schema. res <- resp.Images } return nil + }) return g.Wait() From 4de4743990da3e74f4d0cdfd9cc9ed98045d4176 Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 21 Dec 2022 09:03:02 -0500 Subject: [PATCH 6/9] update ci linting --- .github/workflows/source_aws.yml | 2 +- plugins/.golangci.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/source_aws.yml b/.github/workflows/source_aws.yml index ca4e05515ed11e..81ef024a551ba9 100644 --- a/.github/workflows/source_aws.yml +++ b/.github/workflows/source_aws.yml @@ -58,7 +58,7 @@ jobs: with: version: v1.50.1 working-directory: plugins/source/aws - args: "--config ../../.golangci.yml --new-from-rev=HEAD~" + args: "--config ../../.golangci.yml" - name: Get dependencies run: go get -t -d ./... - name: Build diff --git a/plugins/.golangci.yml b/plugins/.golangci.yml index e8c3f7a1e8e081..11b91820085c5a 100644 --- a/plugins/.golangci.yml +++ b/plugins/.golangci.yml @@ -95,6 +95,8 @@ linters: - unused issues: + new-from-rev: HEAD~ + whole-files: true exclude-rules: # Exclude some linters from running on tests files. - path: _test\.go From b237c280ba9d745cc81dcaefb75c211a175f2b4b Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 21 Dec 2022 09:10:56 -0500 Subject: [PATCH 7/9] Update source_aws.yml --- .github/workflows/source_aws.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/source_aws.yml b/.github/workflows/source_aws.yml index 81ef024a551ba9..3e2d28d7e260e5 100644 --- a/.github/workflows/source_aws.yml +++ b/.github/workflows/source_aws.yml @@ -58,6 +58,7 @@ jobs: with: version: v1.50.1 working-directory: plugins/source/aws + skip-pkg-cache: true args: "--config ../../.golangci.yml" - name: Get dependencies run: go get -t -d ./... From 7bfdbcf8eeb939cfe8b6c7b06188a3358c192aa2 Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 21 Dec 2022 09:15:12 -0500 Subject: [PATCH 8/9] Update source_aws.yml --- .github/workflows/source_aws.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/source_aws.yml b/.github/workflows/source_aws.yml index 3e2d28d7e260e5..81ef024a551ba9 100644 --- a/.github/workflows/source_aws.yml +++ b/.github/workflows/source_aws.yml @@ -58,7 +58,6 @@ jobs: with: version: v1.50.1 working-directory: plugins/source/aws - skip-pkg-cache: true args: "--config ../../.golangci.yml" - name: Get dependencies run: go get -t -d ./... From fc907cf7e1940e37a838a1762456e6236e0106f9 Mon Sep 17 00:00:00 2001 From: bbernays Date: Wed, 21 Dec 2022 09:49:06 -0500 Subject: [PATCH 9/9] reverse changes --- plugins/.golangci.yml | 2 -- plugins/source/aws/resources/services/ec2/images_fetch.go | 1 - 2 files changed, 3 deletions(-) diff --git a/plugins/.golangci.yml b/plugins/.golangci.yml index 11b91820085c5a..e8c3f7a1e8e081 100644 --- a/plugins/.golangci.yml +++ b/plugins/.golangci.yml @@ -95,8 +95,6 @@ linters: - unused issues: - new-from-rev: HEAD~ - whole-files: true exclude-rules: # Exclude some linters from running on tests files. - path: _test\.go diff --git a/plugins/source/aws/resources/services/ec2/images_fetch.go b/plugins/source/aws/resources/services/ec2/images_fetch.go index d92aaf311be43c..84f6d71b7d2734 100644 --- a/plugins/source/aws/resources/services/ec2/images_fetch.go +++ b/plugins/source/aws/resources/services/ec2/images_fetch.go @@ -45,7 +45,6 @@ func fetchEc2Images(ctx context.Context, meta schema.ClientMeta, parent *schema. res <- resp.Images } return nil - }) return g.Wait()