From ef09a92a6adab78822a2bec55cf6ecdb067a8aea Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 17 Mar 2022 10:12:03 +0000 Subject: [PATCH 1/3] feat: Support timeouts in fetch --- go.mod | 2 +- go.sum | 4 ++-- pkg/client/client.go | 1 + pkg/config/provider.go | 8 ++++++++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 90c8caff814737..2be5657ee610e1 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.8.7 + github.com/cloudquery/cq-provider-sdk v0.8.11-0.20220317100158-31eb8bf61ad9 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 diff --git a/go.sum b/go.sum index 970cb5c392000f..92d3c6ae47c923 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.8.7 h1:lVduh/Im+WvMBD/BeHbPfBXdaH4CwSrBKyczYm3GQC0= -github.com/cloudquery/cq-provider-sdk v0.8.7/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= +github.com/cloudquery/cq-provider-sdk v0.8.11-0.20220317100158-31eb8bf61ad9 h1:CxFkPmmVoXsOIuFt7m7i0WBrvtsj7Y/zFufGuPCaIhs= +github.com/cloudquery/cq-provider-sdk v0.8.11-0.20220317100158-31eb8bf61ad9/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/pkg/client/client.go b/pkg/client/client.go index a3f96fcba33a2c..85ec1b2f3ac25d 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -477,6 +477,7 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes PartialFetchingEnabled: true, ParallelFetchingLimit: providerConfig.MaxParallelResourceFetchLimit, MaxGoroutines: providerConfig.MaxGoroutines, + Timeout: time.Duration(providerConfig.Timeout) * time.Second, Metadata: metadata, }) if err != nil { diff --git a/pkg/config/provider.go b/pkg/config/provider.go index d9dc5d95c0e579..eb7515c8aef2dd 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -18,6 +18,7 @@ type Provider struct { Configuration []byte MaxParallelResourceFetchLimit uint64 `hcl:"max_parallel_resource_fetch_limit"` MaxGoroutines uint64 `hcl:"max_goroutines"` + Timeout uint64 `hcl:"timeout"` } func decodeProviderBlock(block *hcl.Block, ctx *hcl.EvalContext, existingProviders map[string]bool) (*Provider, hcl.Diagnostics) { @@ -83,6 +84,10 @@ func decodeProviderBlock(block *hcl.Block, ctx *hcl.EvalContext, existingProvide valDiags := gohcl.DecodeExpression(attr.Expr, ctx, &provider.MaxGoroutines) diags = append(diags, valDiags...) } + if attr, exists := content.Attributes["timeout"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, ctx, &provider.Timeout) + diags = append(diags, valDiags...) + } for _, block := range content.Blocks { switch block.Type { @@ -128,6 +133,9 @@ var providerBlockSchema = &hcl.BodySchema{ { Name: "max_goroutines", }, + { + Name: "timeout", + }, }, Blocks: []hcl.BlockHeaderSchema{ // _All_ of these are reserved for future expansion. From 7292fabda8555bbb08fcb0d9fd314a2da16ea58c Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 17 Mar 2022 12:21:39 +0000 Subject: [PATCH 2/3] resource_timeout --- pkg/client/client.go | 2 +- pkg/config/provider.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 85ec1b2f3ac25d..743d2d9fa1fb44 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -477,7 +477,7 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes PartialFetchingEnabled: true, ParallelFetchingLimit: providerConfig.MaxParallelResourceFetchLimit, MaxGoroutines: providerConfig.MaxGoroutines, - Timeout: time.Duration(providerConfig.Timeout) * time.Second, + Timeout: time.Duration(providerConfig.ResourceTimeout) * time.Second, Metadata: metadata, }) if err != nil { diff --git a/pkg/config/provider.go b/pkg/config/provider.go index eb7515c8aef2dd..ad310b3aff4f8a 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -18,7 +18,7 @@ type Provider struct { Configuration []byte MaxParallelResourceFetchLimit uint64 `hcl:"max_parallel_resource_fetch_limit"` MaxGoroutines uint64 `hcl:"max_goroutines"` - Timeout uint64 `hcl:"timeout"` + ResourceTimeout uint64 `hcl:"resource_timeout"` } func decodeProviderBlock(block *hcl.Block, ctx *hcl.EvalContext, existingProviders map[string]bool) (*Provider, hcl.Diagnostics) { @@ -84,8 +84,8 @@ func decodeProviderBlock(block *hcl.Block, ctx *hcl.EvalContext, existingProvide valDiags := gohcl.DecodeExpression(attr.Expr, ctx, &provider.MaxGoroutines) diags = append(diags, valDiags...) } - if attr, exists := content.Attributes["timeout"]; exists { - valDiags := gohcl.DecodeExpression(attr.Expr, ctx, &provider.Timeout) + if attr, exists := content.Attributes["resource_timeout"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, ctx, &provider.ResourceTimeout) diags = append(diags, valDiags...) } @@ -134,7 +134,7 @@ var providerBlockSchema = &hcl.BodySchema{ Name: "max_goroutines", }, { - Name: "timeout", + Name: "resource_timeout", }, }, Blocks: []hcl.BlockHeaderSchema{ From a7a267883d984dcd6f8b81bfb4f8ba7a9a9e9c03 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 17 Mar 2022 13:33:31 +0000 Subject: [PATCH 3/3] Use SDK 0.8.11 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2be5657ee610e1..f4d61248305161 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.8.11-0.20220317100158-31eb8bf61ad9 + github.com/cloudquery/cq-provider-sdk v0.8.11 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 diff --git a/go.sum b/go.sum index 92d3c6ae47c923..3006855b831e2f 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.8.11-0.20220317100158-31eb8bf61ad9 h1:CxFkPmmVoXsOIuFt7m7i0WBrvtsj7Y/zFufGuPCaIhs= -github.com/cloudquery/cq-provider-sdk v0.8.11-0.20220317100158-31eb8bf61ad9/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= +github.com/cloudquery/cq-provider-sdk v0.8.11 h1:uWGd3WjhbOjCnPq2zKbitXF15ATG6XnMItKM2bA2UHM= +github.com/cloudquery/cq-provider-sdk v0.8.11/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=