diff --git a/go.mod b/go.mod index ad4d52e9c04ab3..24a56f49d85925 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.7.7 + github.com/cloudquery/cq-provider-sdk v0.7.8 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -105,6 +105,7 @@ require ( github.com/mitchellh/hashstructure v1.1.0 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/oklog/run v1.1.0 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml v1.8.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -140,3 +141,4 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) + diff --git a/go.sum b/go.sum index f3f67f19f6676e..d60735e1720150 100644 --- a/go.sum +++ b/go.sum @@ -209,6 +209,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cloudquery/cq-provider-sdk v0.7.7 h1:pvurkIfsg+IKp25ZnBV8aYwakqYJA5ZLnqIJVAG+t7I= github.com/cloudquery/cq-provider-sdk v0.7.7/go.mod h1:3XIx9D1zpx/IS2MQcbyBTxpvpUDI0xyBwPT3reKo/4k= +github.com/cloudquery/cq-provider-sdk v0.7.8 h1:SNbjAqefrdPdu/KDPWb3vUXYlfT7KtJD9XrPIY6oJHY= +github.com/cloudquery/cq-provider-sdk v0.7.8/go.mod h1:/FOZBtSgzfS3Vl7TfoxyacvE9UI+Vf4hiBZnH7Gtcg8= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -1001,6 +1003,8 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= diff --git a/pkg/client/client.go b/pkg/client/client.go index 7cc53659e844ee..d8e75ffb1b8957 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -470,6 +470,7 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes Resources: providerConfig.Resources, PartialFetchingEnabled: providerConfig.EnablePartialFetch, ParallelFetchingLimit: providerConfig.MaxParallelResourceFetchLimit, + MaxGoroutines: providerConfig.MaxGoroutines, }) if err != nil { return err diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 417f9141ee554a..a1adf6b79b6bc8 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -105,12 +105,11 @@ func TestClient_FailOnFetchWithPartialFetch(t *testing.T) { result, err := c.Fetch(ctx, FetchRequest{ UpdateCallback: nil, Providers: []*config.Provider{{ - Name: "test", - Alias: "test_alias", - EnablePartialFetch: true, - Resources: []string{"slow_resource", "panic_resource", "error_resource", "very_slow_resource"}, - Env: nil, - Configuration: nil, + Name: "test", + Alias: "test_alias", + Resources: []string{"slow_resource", "panic_resource", "error_resource", "very_slow_resource"}, + Env: nil, + Configuration: nil, }, }, }) @@ -121,8 +120,7 @@ func TestClient_FailOnFetchWithPartialFetch(t *testing.T) { testSummary, ok := result.ProviderFetchSummary["test(test_alias)"] assert.True(t, ok) assert.True(t, testSummary.HasErrors()) - assert.Len(t, testSummary.PartialFetchErrors, 2) - assert.Len(t, testSummary.FetchErrors, 0) + assert.Len(t, testSummary.FetchErrors, 2) } func TestClient_FailOnFetch(t *testing.T) { @@ -143,12 +141,11 @@ func TestClient_FailOnFetch(t *testing.T) { result, err := c.Fetch(ctx, FetchRequest{ UpdateCallback: nil, Providers: []*config.Provider{{ - Name: "test", - Alias: "test_alias", - EnablePartialFetch: false, - Resources: []string{"slow_resource", "panic_resource", "error_resource", "very_slow_resource"}, - Env: nil, - Configuration: nil, + Name: "test", + Alias: "test_alias", + Resources: []string{"slow_resource", "panic_resource", "error_resource", "very_slow_resource"}, + Env: nil, + Configuration: nil, }, }, }) @@ -177,19 +174,18 @@ func TestClient_PartialFetch(t *testing.T) { result, err := c.Fetch(ctx, FetchRequest{ UpdateCallback: nil, Providers: []*config.Provider{{ - Name: "test", - Alias: "test_alias", - EnablePartialFetch: true, - Resources: []string{"slow_resource", "panic_resource", "error_resource", "very_slow_resource"}, - Env: nil, - Configuration: nil, + Name: "test", + Alias: "test_alias", + Resources: []string{"slow_resource", "panic_resource", "error_resource", "very_slow_resource"}, + Env: nil, + Configuration: nil, }, }, }) assert.Nil(t, err) testSummary, ok := result.ProviderFetchSummary["test(test_alias)"] assert.True(t, ok) - assert.Len(t, testSummary.PartialFetchErrors, 2) + assert.Len(t, testSummary.FetchErrors, 2) } func TestClient_TestNoDownload(t *testing.T) { diff --git a/pkg/config/provider.go b/pkg/config/provider.go index d181c817f694e4..43200fc594d650 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -18,6 +18,7 @@ type Provider struct { Env []string `hcl:"env,optional"` Configuration []byte MaxParallelResourceFetchLimit uint64 `hcl:"max_parallel_resource_fetch_limit"` + MaxGoroutines uint64 `hcl:"max_goroutines"` } func decodeProviderBlock(block *hcl.Block, ctx *hcl.EvalContext, existingProviders map[string]bool) (*Provider, hcl.Diagnostics) { @@ -70,6 +71,10 @@ func decodeProviderBlock(block *hcl.Block, ctx *hcl.EvalContext, existingProvide valDiags := gohcl.DecodeExpression(attr.Expr, ctx, &provider.MaxParallelResourceFetchLimit) diags = append(diags, valDiags...) } + if attr, exists := content.Attributes["max_goroutines"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, ctx, &provider.MaxGoroutines) + diags = append(diags, valDiags...) + } for _, block := range content.Blocks { switch block.Type { @@ -112,6 +117,9 @@ var providerBlockSchema = &hcl.BodySchema{ { Name: "max_parallel_resource_fetch_limit", }, + { + Name: "max_goroutines", + }, }, Blocks: []hcl.BlockHeaderSchema{ // _All_ of these are reserved for future expansion.