cloudquery · disq · May 16, 2022 · May 12, 2022 · May 12, 2022 · May 12, 2022
@@ -14,6 +14,7 @@ type Provider struct {
 	Name                          string   `hcl:"name,label"`
 	Alias                         string   `hcl:"alias,optional"`
 	Resources                     []string `hcl:"resources,optional"`
+	SkipResources                 []string `hcl:"skip_resources,optional"`
 	Env                           []string `hcl:"env,optional"`
 	Configuration                 []byte
 	MaxParallelResourceFetchLimit uint64 `hcl:"max_parallel_resource_fetch_limit"`
@@ -71,6 +72,10 @@ func decodeProviderBlock(block *hcl.Block, ctx *hcl.EvalContext, existingProvide
 		valDiags := gohcl.DecodeExpression(attr.Expr, ctx, &provider.Resources)
 		diags = append(diags, valDiags...)
 	}
+	if attr, exists := content.Attributes["skip_resources"]; exists {
+		valDiags := gohcl.DecodeExpression(attr.Expr, ctx, &provider.SkipResources)
+		diags = append(diags, valDiags...)
+	}
 	if attr, exists := content.Attributes["env"]; exists {
 		valDiags := gohcl.DecodeExpression(attr.Expr, ctx, &provider.Env)
 		diags = append(diags, valDiags...)
@@ -118,6 +123,9 @@ var providerBlockSchema = &hcl.BodySchema{
 		{
 			Name: "resources",
 		},
+		{
+			Name: "skip_resources",
+		},
 		{
 			Name: "alias",
 		},

@@ -6,26 +6,28 @@ import (
 	"fmt"
 	"io"
 	"math"
-	"sort"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/cloudquery/cloudquery/internal/logging"
+	cqsort "github.com/cloudquery/cloudquery/internal/sort"
 	"github.com/cloudquery/cloudquery/pkg/config"
 	"github.com/cloudquery/cloudquery/pkg/core/database"
 	"github.com/cloudquery/cloudquery/pkg/core/history"
 	"github.com/cloudquery/cloudquery/pkg/core/state"
 	"github.com/cloudquery/cloudquery/pkg/plugin"
 	"github.com/cloudquery/cloudquery/pkg/plugin/registry"
-
 	"github.com/cloudquery/cq-provider-sdk/cqproto"
 	sdkdb "github.com/cloudquery/cq-provider-sdk/database"
 	"github.com/cloudquery/cq-provider-sdk/database/dsn"
 	"github.com/cloudquery/cq-provider-sdk/provider/diag"
 	"github.com/cloudquery/cq-provider-sdk/provider/schema"
+
 	"github.com/google/uuid"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
+	"github.com/thoas/go-funk"
 	gcodes "google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -337,7 +339,7 @@ func executeFetch(ctx context.Context, pLog zerolog.Logger, plugin plugin.Plugin
 	}()
 
 	var resources []string
-	resources, diags = normalizeResources(ctx, plugin, info.Config.Resources)
+	resources, diags = normalizeResources(ctx, plugin, info.Config.Resources, info.Config.SkipResources)
 	if diags.HasErrors() {
 		summary.Status = FetchFailed
 		return summary, diags
@@ -418,43 +420,95 @@ func executeFetch(ctx context.Context, pLog zerolog.Logger, plugin plugin.Plugin
 // * wildcard expansion
 // * verify no unknown resources
 // * verify no duplicate resources
-func normalizeResources(ctx context.Context, provider plugin.Plugin, resources []string) ([]string, diag.Diagnostics) {
+func normalizeResources(ctx context.Context, provider plugin.Plugin, resources, skip []string) ([]string, diag.Diagnostics) {
 	s, err := provider.Provider().GetProviderSchema(ctx, &cqproto.GetProviderSchemaRequest{})
 	if err != nil {
 		return nil, diag.FromError(err, diag.INTERNAL)
 	}
-	return doNormalizeResources(resources, s.ResourceTables)
+
+	return doNormalizeResources(resources, skip, s.ResourceTables)
 }
 
-// doNormalizeResources returns a canonical list of resources given a list of requested and all known resources.
-// It replaces wildcard resource with all resources. Error is returned if:
+// doNormalizeResources matches the given two resource lists to all provider resources and returns the requested resources (excluding skip resources) as another list.
+func doNormalizeResources(resources, skip []string, all map[string]*schema.Table) ([]string, diag.Diagnostics) {
+	useRes, diags := doGlobResources(resources, false, all)
+	skipRes, dd := doGlobResources(skip, true, all)
+	return funk.Subtract(useRes, skipRes).([]string), diags.Add(dd)
+}
+
+// doGlobResources returns a canonical list of resources given a list of requested and all known resources.
+// It replaces wildcard resource with all resources in non-wild mode. Error is returned if:
 //
 // * wildcard is present and other explicit resource is requested;
 // * one of explicitly requested resources is not present in all known;
 // * some resource is specified more than once (duplicate).
-func doNormalizeResources(requested []string, all map[string]*schema.Table) ([]string, diag.Diagnostics) {
-	if len(requested) == 1 && requested[0] == "*" {
+func doGlobResources(requested []string, allowWild bool, all map[string]*schema.Table) ([]string, diag.Diagnostics) {
+	if allowWild {
+		for _, s := range requested {
+			if s == "*" {
+				return nil, diag.FromError(fmt.Errorf("wildcard resource can only be in the requested resources list"), diag.USER, diag.WithDetails("you can only use * in the resources part of the configuration"))
+			}
+		}
+	} else if len(requested) == 1 && requested[0] == "*" {
 		requested = make([]string, 0, len(all))
 		for k := range all {
 			requested = append(requested, k)
 		}
 	}
+
 	result := make([]string, 0, len(requested))
 	seen := make(map[string]struct{})
 	for _, r := range requested {
+		if r == "" {
+			return nil, diag.FromError(errors.New("invalid resource"), diag.USER, diag.WithDetails("empty resource names are not allowed"))
+		}
+
 		if _, ok := seen[r]; ok {
 			return nil, diag.FromError(fmt.Errorf("resource %q is duplicate", r), diag.USER, diag.WithDetails("configuration has duplicate resources"))
 		}
 		seen[r] = struct{}{}
-		if _, ok := all[r]; !ok {
-			if r == "*" {
-				return nil, diag.FromError(fmt.Errorf("wildcard resource must be the only one in the list"), diag.USER, diag.WithDetails("you can only use * or a list of resources in configuration, but not both"))
-			}
+
+		if _, ok := all[r]; ok {
+			result = append(result, r)
+			continue
+		}
+
+		if r == "*" {
+			return nil, diag.FromError(fmt.Errorf("wildcard resource must be the only one in the list"), diag.USER, diag.WithDetails("you can only use * or a list of resources in configuration, but not both"))
+		}
+
+		switch globMatches, diags := matchResourceGlob(r, all); {
+		case diags.HasDiags():
+			return nil, diags
+		case len(globMatches) == 0:
 			return nil, diag.FromError(fmt.Errorf("resource %q does not exist", r), diag.USER, diag.WithDetails("configuration refers to a non-existing resource. Maybe you recently downgraded the provider but kept the config, or a typo perhaps?"))
+		default:
+			result = append(result, globMatches...)
+		}
+	}
+
+	return cqsort.Unique(result), nil
+}
+
+// matchResourceGlob matches pattern to the given resources, returns matched resources or diags
+// pattern should end with .*, exact matches are not handled.
+func matchResourceGlob(pattern string, all map[string]*schema.Table) ([]string, diag.Diagnostics) {
+	var result []string
+	wildPos := strings.Index(pattern, ".*")
+
+	if wildPos > 0 {
+		if wildPos != len(pattern)-2 { // make sure it ends with .*
+			return nil, diag.FromError(errors.New("invalid wildcard syntax"), diag.USER, diag.WithDetails("resource match should end with `.*`"))
 		}
-		result = append(result, r)
+		for k := range all {
+			if strings.HasPrefix(k, pattern[:wildPos+1]) { // include the "." in the match
+				result = append(result, k)
+			}
+		}
+	} else if wildPos == 0 || strings.Contains(pattern, "*") {
+		return nil, diag.FromError(errors.New("invalid wildcard syntax"), diag.USER, diag.WithDetails("you can only use `*` or `resource.*` or full resource name"))
 	}
-	sort.Strings(result)
+
 	return result, nil
 }
 

@@ -2,7 +2,6 @@ package core
 
 import (
 	"context"
-	"reflect"
 	"testing"
 	"time"
 
@@ -261,56 +260,114 @@ func Test_doNormalizeResources(t *testing.T) {
 	tests := []struct {
 		name      string
 		requested []string
+		skip      []string
 		all       map[string]*schema.Table
 		want      []string
 		wantErr   bool
 	}{
 		{
 			"wilcard",
 			[]string{"*"},
+			nil,
 			map[string]*schema.Table{"3": nil, "2": nil, "1": nil},
 			[]string{"1", "2", "3"},
 			false,
 		},
 		{
 			"wilcard with explicit",
 			[]string{"*", "1"},
+			nil,
 			map[string]*schema.Table{"3": nil, "2": nil, "1": nil},
 			nil,
 			true,
 		},
 		{
 			"unknown resource",
 			[]string{"1", "2", "x"},
+			nil,
 			map[string]*schema.Table{"3": nil, "2": nil, "1": nil},
 			nil,
 			true,
 		},
 		{
 			"duplicate resource",
 			[]string{"1", "2", "1"},
+			nil,
 			map[string]*schema.Table{"3": nil, "2": nil, "1": nil},
 			nil,
 			true,
 		},
 		{
 			"ok, all explicit",
 			[]string{"2", "1"},
+			nil,
 			map[string]*schema.Table{"3": nil, "2": nil, "1": nil},
 			[]string{"1", "2"},
 			false,
 		},
+		{
+			"ok, all explicit with ignores",
+			[]string{"2", "1", "3"},
+			[]string{"1"},
+			map[string]*schema.Table{"3": nil, "2": nil, "1": nil},
+			[]string{"2", "3"},
+			false,
+		},
+		{
+			"ok, some globs",
+			[]string{"c1.*", "c2.res4"},
+			nil,
+			map[string]*schema.Table{"c1.res1": nil, "c1.res2": nil, "c2.res3": nil, "c2.res4": nil, "c1a.res5": nil},
+			[]string{"c1.res1", "c1.res2", "c2.res4"},
+			false,
+		},
+		{
+			"ok, some globs with skips",
+			[]string{"c1.*", "c2.res4"},
+			[]string{"c1.res1"},
+			map[string]*schema.Table{"c1.res1": nil, "c1.res2": nil, "c2.res3": nil, "c2.res4": nil, "c1a.res5": nil},
+			[]string{"c1.res2", "c2.res4"},
+			false,
+		},
+		{
+			"invalid glob 1",
+			[]string{"c1*res1"},
+			nil,
+			map[string]*schema.Table{"c1.res1": nil, "c1.res2": nil, "c2.res3": nil, "c2.res4": nil},
+			nil,
+			true,
+		},
+		{
+			"invalid glob 2",
+			[]string{"c1.*res1"},
+			nil,
+			map[string]*schema.Table{"c1.res1": nil, "c1.res2": nil, "c2.res3": nil, "c2.res4": nil},
+			nil,
+			true,
+		},
+		{
+			"invalid glob 3",
+			[]string{"c1.res*"},
+			nil,
+			map[string]*schema.Table{"c1.res1": nil, "c1.res2": nil, "c2.res3": nil, "c2.res4": nil},
+			nil,
+			true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := doNormalizeResources(tt.requested, tt.all)
+			got, err := doNormalizeResources(tt.requested, tt.skip, tt.all)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("doInterpolate() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("doInterpolate() = %v, want %v", got, tt.want)
+			if tt.want == nil {
+				tt.want = []string{}
+			}
+			if got == nil {
+				got = []string{}
 			}
+			assert.EqualValues(t, tt.want, got)
 		})
 	}
 }