diff --git a/cmd/fetch.go b/cmd/fetch.go index 391c4093d11669..89bc430d7c304d 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -30,5 +30,8 @@ func init() { _ = viper.BindPFlag("fail-on-error", fetchCmd.PersistentFlags().Lookup("fail-on-error")) fetchCmd.Flags().Bool("skip-schema-upgrade", false, "skip schema upgrade of provider fetch, disabling this flag might cause issues") _ = viper.BindPFlag("skip-schema-upgrade", fetchCmd.Flags().Lookup("skip-schema-upgrade")) + fetchCmd.Flags().Bool("redact-diags", false, "show redacted diagnostics only") + _ = viper.BindPFlag("redact-diags", fetchCmd.Flags().Lookup("redact-diags")) + _ = fetchCmd.Flags().MarkHidden("redact-diags") rootCmd.AddCommand(fetchCmd) } diff --git a/go.mod b/go.mod index ce18e2461682f1..768121120dac47 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.7.0 + github.com/cloudquery/cq-provider-sdk v0.7.4 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 diff --git a/go.sum b/go.sum index bb425e855d0f73..18592c45aaef38 100644 --- a/go.sum +++ b/go.sum @@ -165,6 +165,7 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.2.1/go.mod h1:VimPFPltQ/920i1X0Sb0VJ github.com/aws/aws-sdk-go-v2/service/sts v1.2.2/go.mod h1:ssRzzJ2RZOVuKj2Vx1YE7ypfil/BIlgmQnCSW4DistU= github.com/aws/aws-sdk-go-v2/service/sts v1.4.1/go.mod h1:G9osDWA52WQ38BDcj65VY1cNmcAQXAXTsE8IWH8j81w= github.com/aws/smithy-go v1.3.1/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.4.0 h1:3rsQpgRe+OoQgJhEwGNpIkosl0fJLdmQqF4gSFRjg+4= github.com/aws/smithy-go v1.4.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -206,8 +207,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.7.0 h1:PWERMQi7oUW0rHn/qKvf7cVcOhMB9xgx/i0W3hvoXOc= -github.com/cloudquery/cq-provider-sdk v0.7.0/go.mod h1:T+ngRXzcjJ6otKDGkWnPrHTsZuHUe3KZKtyhSLcvHCs= +github.com/cloudquery/cq-provider-sdk v0.7.4 h1:WPsoRB2O3zrHq2dKNFkm7g5ghZiAnp5YuMmKsWGtXCo= +github.com/cloudquery/cq-provider-sdk v0.7.4/go.mod h1:3XIx9D1zpx/IS2MQcbyBTxpvpUDI0xyBwPT3reKo/4k= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/internal/telemetry/error.go b/internal/telemetry/error.go index 33b2a71455fdd8..77735ff31842ab 100644 --- a/internal/telemetry/error.go +++ b/internal/telemetry/error.go @@ -6,6 +6,7 @@ import ( "net" "strings" + "github.com/cloudquery/cq-provider-sdk/provider/diag" "github.com/jackc/pgconn" "github.com/lib/pq" "go.opentelemetry.io/otel/codes" @@ -20,6 +21,12 @@ func RecordError(span trace.Span, err error, opts ...trace.EventOption) bool { return false } + if rd, ok := err.(diag.Redactable); ok { + if r := rd.Redacted(); r != nil { + err = r + } + } + if cls := classifyError(err); cls != errNoClass { span.SetStatus(codes.Error, string(cls)) return false diff --git a/pkg/client/client.go b/pkg/client/client.go index 74a8ff51daedc3..a5cdd097415eb0 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -30,8 +30,9 @@ import ( "github.com/cloudquery/cq-provider-sdk/database/dsn" "github.com/cloudquery/cq-provider-sdk/migration" "github.com/cloudquery/cq-provider-sdk/migration/migrator" + "github.com/cloudquery/cq-provider-sdk/provider/diag" + "github.com/cloudquery/cq-provider-sdk/provider/execution" "github.com/cloudquery/cq-provider-sdk/provider/schema" - "github.com/cloudquery/cq-provider-sdk/provider/schema/diag" "github.com/getsentry/sentry-go" "github.com/golang-migrate/migrate/v4" "github.com/google/uuid" @@ -202,7 +203,7 @@ type FetchDoneResult struct { // TableCreator creates tables based on schema received from providers type TableCreator interface { - CreateTable(context.Context, schema.QueryExecer, *schema.Table, *schema.Table) error + CreateTable(context.Context, execution.QueryExecer, *schema.Table, *schema.Table) error } type FetchUpdateCallback func(update FetchUpdate) @@ -1083,6 +1084,12 @@ func reportFetchSummaryErrors(span trace.Span, fetchSummaries map[string]Provide span.SetAttributes(telemetry.MapToAttributes(ps.Metrics())...) for _, e := range ps.Diagnostics() { + if rd, ok := e.(diag.Redactable); ok { + if r := rd.Redacted(); r != nil { + e = r + } + } + if e.Severity() == diag.IGNORE { continue } @@ -1095,8 +1102,11 @@ func reportFetchSummaryErrors(span trace.Span, fetchSummaries map[string]Provide "resource": e.Description().Resource, }) scope.SetExtra("detail", e.Description().Detail) - if e.Severity() == diag.WARNING { + switch e.Severity() { + case diag.WARNING: scope.SetLevel(sentry.LevelWarning) + case diag.PANIC: + scope.SetLevel(sentry.LevelFatal) } sentry.CaptureException(e) }) @@ -1110,7 +1120,7 @@ func reportFetchSummaryErrors(span trace.Span, fetchSummaries map[string]Provide ) } -func createCoreSchema(ctx context.Context, db schema.QueryExecer) error { +func createCoreSchema(ctx context.Context, db execution.QueryExecer) error { return db.Exec(ctx, "CREATE SCHEMA IF NOT EXISTS cloudquery") } diff --git a/pkg/client/fetch.go b/pkg/client/fetch.go index 5b3d5ebfe42d40..cd43185321d626 100644 --- a/pkg/client/fetch.go +++ b/pkg/client/fetch.go @@ -7,7 +7,7 @@ import ( "time" "github.com/cloudquery/cq-provider-sdk/cqproto" - "github.com/cloudquery/cq-provider-sdk/provider/schema/diag" + "github.com/cloudquery/cq-provider-sdk/provider/diag" "github.com/doug-martin/goqu/v9" "github.com/google/uuid" ) diff --git a/pkg/module/drift/drift.go b/pkg/module/drift/drift.go index f3b421455fc197..cfd00966bd9006 100644 --- a/pkg/module/drift/drift.go +++ b/pkg/module/drift/drift.go @@ -7,7 +7,7 @@ import ( "regexp" "strings" - "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/cloudquery/cq-provider-sdk/provider/execution" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" "github.com/georgysavva/scany/pgxscan" @@ -260,7 +260,7 @@ func (d *Drift) run(ctx context.Context, req *module.ExecuteRequest) (*Results, return resList, nil } -func queryIntoResourceList(ctx context.Context, logger hclog.Logger, conn schema.QueryExecer, sel *goqu.SelectDataset) (ResourceList, error) { +func queryIntoResourceList(ctx context.Context, logger hclog.Logger, conn execution.QueryExecer, sel *goqu.SelectDataset) (ResourceList, error) { query, args, err := sel.ToSQL() if err != nil { return nil, fmt.Errorf("goqu build failed: %w", err) diff --git a/pkg/module/drift/terraform.go b/pkg/module/drift/terraform.go index 0d83e742e83415..cb2502bc559d25 100644 --- a/pkg/module/drift/terraform.go +++ b/pkg/module/drift/terraform.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/cloudquery/cloudquery/pkg/module/drift/terraform" + "github.com/cloudquery/cq-provider-sdk/provider/execution" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" @@ -165,7 +166,7 @@ func parseTerraformAttribute(val interface{}, t schema.ValueType) interface{} { } } -func driftTerraform(ctx context.Context, logger hclog.Logger, conn schema.QueryExecer, cloudName string, cloudTable *traversedTable, resName string, resources map[string]*ResourceConfig, iacData *IACConfig, states TFStates, runParams RunParams, accountIDs []string) (*Result, error) { +func driftTerraform(ctx context.Context, logger hclog.Logger, conn execution.QueryExecer, cloudName string, cloudTable *traversedTable, resName string, resources map[string]*ResourceConfig, iacData *IACConfig, states TFStates, runParams RunParams, accountIDs []string) (*Result, error) { res := &Result{ Different: nil, Equal: nil, diff --git a/pkg/module/manager.go b/pkg/module/manager.go index 878673e044aa83..424fd1ebc21a14 100644 --- a/pkg/module/manager.go +++ b/pkg/module/manager.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/cloudquery/cq-provider-sdk/provider/execution" "github.com/hashicorp/go-hclog" "github.com/hashicorp/hcl/v2" ) @@ -15,7 +15,7 @@ type ManagerImpl struct { modOrder []string // Instance of database - pool schema.QueryExecer + pool execution.QueryExecer // Logger instance logger hclog.Logger @@ -35,7 +35,7 @@ type Manager interface { } // NewManager returns a new manager instance. -func NewManager(pool schema.QueryExecer, logger hclog.Logger) *ManagerImpl { +func NewManager(pool execution.QueryExecer, logger hclog.Logger) *ManagerImpl { return &ManagerImpl{ modules: make(map[string]Module), pool: pool, diff --git a/pkg/module/types.go b/pkg/module/types.go index b8f1fb0a84d2a0..b0e286ed7b56a2 100644 --- a/pkg/module/types.go +++ b/pkg/module/types.go @@ -4,7 +4,7 @@ import ( "context" "github.com/cloudquery/cq-provider-sdk/cqproto" - "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/cloudquery/cq-provider-sdk/provider/execution" "github.com/hashicorp/hcl/v2" ) @@ -28,7 +28,7 @@ type ExecuteRequest struct { // Providers is the list of providers to process Providers []*cqproto.GetProviderSchemaResponse // Conn is the db connection to use - Conn schema.QueryExecer + Conn execution.QueryExecer } type ExecutionResult struct { diff --git a/pkg/policy/execute.go b/pkg/policy/execute.go index 590285c06b63ae..ee72bb10d172dc 100644 --- a/pkg/policy/execute.go +++ b/pkg/policy/execute.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/cloudquery/cq-provider-sdk/provider/execution" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-version" "github.com/spf13/afero" @@ -45,7 +45,7 @@ func (f Update) DoneCount() int { // Executor implements the execution framework. type Executor struct { // Connection to the database - conn schema.QueryExecer + conn execution.QueryExecer log hclog.Logger PolicyPath []string @@ -101,7 +101,7 @@ type ExecuteRequest struct { } // NewExecutor creates a new executor. -func NewExecutor(conn schema.QueryExecer, log hclog.Logger, progressUpdate UpdateCallback) *Executor { +func NewExecutor(conn execution.QueryExecer, log hclog.Logger, progressUpdate UpdateCallback) *Executor { return &Executor{ conn: conn, log: log, diff --git a/pkg/policy/execute_test.go b/pkg/policy/execute_test.go index 64cee82806de92..ff77f5b9147428 100644 --- a/pkg/policy/execute_test.go +++ b/pkg/policy/execute_test.go @@ -6,12 +6,12 @@ import ( "testing" sdkdb "github.com/cloudquery/cq-provider-sdk/database" - "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/cloudquery/cq-provider-sdk/provider/execution" "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/assert" ) -func setupPolicyDatabase(t *testing.T, tableName string) (schema.QueryExecer, func(t *testing.T)) { +func setupPolicyDatabase(t *testing.T, tableName string) (execution.QueryExecer, func(t *testing.T)) { conn, err := sdkdb.New(context.Background(), hclog.NewNullLogger(), "postgres://postgres:pass@localhost:5432/postgres") assert.NoError(t, err) diff --git a/pkg/policy/manager.go b/pkg/policy/manager.go index c25507d8d5e601..d2282d5b0d239e 100644 --- a/pkg/policy/manager.go +++ b/pkg/policy/manager.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/cloudquery/cq-provider-sdk/provider/execution" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" @@ -21,7 +21,7 @@ type ManagerImpl struct { policyDirectory string // Instance of a database connection pool - pool schema.QueryExecer + pool execution.QueryExecer // Logger instance logger hclog.Logger @@ -38,7 +38,7 @@ type Manager interface { } // NewManager returns a new manager instance. -func NewManager(policyDir string, pool schema.QueryExecer, logger hclog.Logger) *ManagerImpl { +func NewManager(policyDir string, pool execution.QueryExecer, logger hclog.Logger) *ManagerImpl { return &ManagerImpl{ policyDirectory: policyDir, pool: pool, diff --git a/pkg/ui/console/client.go b/pkg/ui/console/client.go index a13f6cf97b2538..06cbfa9222204a 100644 --- a/pkg/ui/console/client.go +++ b/pkg/ui/console/client.go @@ -134,7 +134,7 @@ func (c Client) Fetch(ctx context.Context, failOnError bool) error { if ui.IsTerminal() && fetchProgress != nil { fetchProgress.MarkAllDone() fetchProgress.Wait() - printFetchResponse(response) + printFetchResponse(response, viper.GetBool("redact-diags")) } if response == nil { diff --git a/pkg/ui/console/fetch.go b/pkg/ui/console/fetch.go index 232eb6a8f9d6c9..72cc1c7e1eec4b 100644 --- a/pkg/ui/console/fetch.go +++ b/pkg/ui/console/fetch.go @@ -5,16 +5,16 @@ import ( "github.com/cloudquery/cloudquery/pkg/client" "github.com/cloudquery/cloudquery/pkg/ui" - "github.com/cloudquery/cq-provider-sdk/provider/schema/diag" + "github.com/cloudquery/cq-provider-sdk/provider/diag" ) -func printFetchResponse(summary *client.FetchResponse) { +func printFetchResponse(summary *client.FetchResponse, redactDiags bool) { if summary == nil { return } for _, pfs := range summary.ProviderFetchSummary { if len(pfs.Diagnostics()) > 0 { - printDiagnostics(pfs.ProviderName, pfs.Diagnostics()) + printDiagnostics(pfs.ProviderName, pfs.Diagnostics(), redactDiags) continue } if len(pfs.PartialFetchErrors) == 0 { @@ -40,11 +40,18 @@ func printFetchResponse(summary *client.FetchResponse) { } } -func printDiagnostics(providerName string, diags diag.Diagnostics) { +func printDiagnostics(providerName string, diags diag.Diagnostics, redactDiags bool) { // sort diagnostics by severity/type sort.Sort(diags) ui.ColorizedOutput(ui.ColorHeader, "Fetch Diagnostics for provider %s:\n\n", providerName) for _, d := range diags { + if redactDiags { + if rd, ok := d.(diag.Redactable); ok { + if r := rd.Redacted(); r != nil { + d = r + } + } + } desc := d.Description() switch d.Severity() { case diag.IGNORE: