Skip to content

feat: Use redacted errors for sentry/telemetry #461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
disq merged 6 commits into from
Feb 3, 2022
Merged

feat: Use redacted errors for sentry/telemetry #461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
24d8915
feat: Use redacted errors for sentry/telemetry
disq Feb 2, 2022
b2a05f6
Update SDK
disq Feb 2, 2022
0bfba0e
Use 0.7.2
disq Feb 2, 2022
abff5c8
cli: Add --redact-diags to fetch
disq Feb 2, 2022
09e08ec
Update SDK
disq Feb 2, 2022
22bfd04
0.7.4
disq Feb 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cmd/fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,8 @@ func init() {
_ = viper.BindPFlag("fail-on-error", fetchCmd.PersistentFlags().Lookup("fail-on-error"))
fetchCmd.Flags().Bool("skip-schema-upgrade", false, "skip schema upgrade of provider fetch, disabling this flag might cause issues")
_ = viper.BindPFlag("skip-schema-upgrade", fetchCmd.Flags().Lookup("skip-schema-upgrade"))
fetchCmd.Flags().Bool("redact-diags", false, "show redacted diagnostics only")
_ = viper.BindPFlag("redact-diags", fetchCmd.Flags().Lookup("redact-diags"))
_ = fetchCmd.Flags().MarkHidden("redact-diags")
rootCmd.AddCommand(fetchCmd)
}
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.17
require (
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/aws/aws-lambda-go v1.23.0
github.com/cloudquery/cq-provider-sdk v0.7.0
github.com/cloudquery/cq-provider-sdk v0.7.4
github.com/fatih/color v1.13.0
github.com/fsnotify/fsnotify v1.4.9
github.com/golang-migrate/migrate/v4 v4.15.0
Expand Down
5 changes: 3 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.2.1/go.mod h1:VimPFPltQ/920i1X0Sb0VJ
github.com/aws/aws-sdk-go-v2/service/sts v1.2.2/go.mod h1:ssRzzJ2RZOVuKj2Vx1YE7ypfil/BIlgmQnCSW4DistU=
github.com/aws/aws-sdk-go-v2/service/sts v1.4.1/go.mod h1:G9osDWA52WQ38BDcj65VY1cNmcAQXAXTsE8IWH8j81w=
github.com/aws/smithy-go v1.3.1/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
github.com/aws/smithy-go v1.4.0 h1:3rsQpgRe+OoQgJhEwGNpIkosl0fJLdmQqF4gSFRjg+4=
github.com/aws/smithy-go v1.4.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
Expand Down Expand Up @@ -206,8 +207,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ
github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/cloudquery/cq-provider-sdk v0.7.0 h1:PWERMQi7oUW0rHn/qKvf7cVcOhMB9xgx/i0W3hvoXOc=
github.com/cloudquery/cq-provider-sdk v0.7.0/go.mod h1:T+ngRXzcjJ6otKDGkWnPrHTsZuHUe3KZKtyhSLcvHCs=
github.com/cloudquery/cq-provider-sdk v0.7.4 h1:WPsoRB2O3zrHq2dKNFkm7g5ghZiAnp5YuMmKsWGtXCo=
github.com/cloudquery/cq-provider-sdk v0.7.4/go.mod h1:3XIx9D1zpx/IS2MQcbyBTxpvpUDI0xyBwPT3reKo/4k=
github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec=
github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
Expand Down
7 changes: 7 additions & 0 deletions internal/telemetry/error.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net"
"strings"

"github.com/cloudquery/cq-provider-sdk/provider/diag"
"github.com/jackc/pgconn"
"github.com/lib/pq"
"go.opentelemetry.io/otel/codes"
Expand All @@ -20,6 +21,12 @@ func RecordError(span trace.Span, err error, opts ...trace.EventOption) bool {
return false
}

if rd, ok := err.(diag.Redactable); ok {
if r := rd.Redacted(); r != nil {
err = r
}
}

if cls := classifyError(err); cls != errNoClass {
span.SetStatus(codes.Error, string(cls))
return false
Expand Down
18 changes: 14 additions & 4 deletions pkg/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ import (
"github.com/cloudquery/cq-provider-sdk/database/dsn"
"github.com/cloudquery/cq-provider-sdk/migration"
"github.com/cloudquery/cq-provider-sdk/migration/migrator"
"github.com/cloudquery/cq-provider-sdk/provider/diag"
"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/cloudquery/cq-provider-sdk/provider/schema"
"github.com/cloudquery/cq-provider-sdk/provider/schema/diag"
"github.com/getsentry/sentry-go"
"github.com/golang-migrate/migrate/v4"
"github.com/google/uuid"
Expand Down Expand Up @@ -202,7 +203,7 @@ type FetchDoneResult struct {

// TableCreator creates tables based on schema received from providers
type TableCreator interface {
CreateTable(context.Context, schema.QueryExecer, *schema.Table, *schema.Table) error
CreateTable(context.Context, execution.QueryExecer, *schema.Table, *schema.Table) error
}

type FetchUpdateCallback func(update FetchUpdate)
Expand Down Expand Up @@ -1083,6 +1084,12 @@ func reportFetchSummaryErrors(span trace.Span, fetchSummaries map[string]Provide
span.SetAttributes(telemetry.MapToAttributes(ps.Metrics())...)

for _, e := range ps.Diagnostics() {
if rd, ok := e.(diag.Redactable); ok {
if r := rd.Redacted(); r != nil {
e = r
}
}

if e.Severity() == diag.IGNORE {
continue
}
Expand All @@ -1095,8 +1102,11 @@ func reportFetchSummaryErrors(span trace.Span, fetchSummaries map[string]Provide
"resource": e.Description().Resource,
})
scope.SetExtra("detail", e.Description().Detail)
if e.Severity() == diag.WARNING {
switch e.Severity() {
case diag.WARNING:
scope.SetLevel(sentry.LevelWarning)
case diag.PANIC:
scope.SetLevel(sentry.LevelFatal)
}
sentry.CaptureException(e)
})
Expand All @@ -1110,7 +1120,7 @@ func reportFetchSummaryErrors(span trace.Span, fetchSummaries map[string]Provide
)
}

func createCoreSchema(ctx context.Context, db schema.QueryExecer) error {
func createCoreSchema(ctx context.Context, db execution.QueryExecer) error {
return db.Exec(ctx, "CREATE SCHEMA IF NOT EXISTS cloudquery")
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/client/fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"time"

"github.com/cloudquery/cq-provider-sdk/cqproto"
"github.com/cloudquery/cq-provider-sdk/provider/schema/diag"
"github.com/cloudquery/cq-provider-sdk/provider/diag"
"github.com/doug-martin/goqu/v9"
"github.com/google/uuid"
)
Expand Down
4 changes: 2 additions & 2 deletions pkg/module/drift/drift.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"regexp"
"strings"

"github.com/cloudquery/cq-provider-sdk/provider/schema"
"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/doug-martin/goqu/v9"
"github.com/doug-martin/goqu/v9/exp"
"github.com/georgysavva/scany/pgxscan"
Expand Down Expand Up @@ -260,7 +260,7 @@ func (d *Drift) run(ctx context.Context, req *module.ExecuteRequest) (*Results,
return resList, nil
}

func queryIntoResourceList(ctx context.Context, logger hclog.Logger, conn schema.QueryExecer, sel *goqu.SelectDataset) (ResourceList, error) {
func queryIntoResourceList(ctx context.Context, logger hclog.Logger, conn execution.QueryExecer, sel *goqu.SelectDataset) (ResourceList, error) {
query, args, err := sel.ToSQL()
if err != nil {
return nil, fmt.Errorf("goqu build failed: %w", err)
Expand Down
3 changes: 2 additions & 1 deletion pkg/module/drift/terraform.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/aws/aws-sdk-go/aws/arn"
"github.com/cloudquery/cloudquery/pkg/module/drift/terraform"
"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/cloudquery/cq-provider-sdk/provider/schema"
"github.com/doug-martin/goqu/v9"
"github.com/doug-martin/goqu/v9/exp"
Expand Down Expand Up @@ -165,7 +166,7 @@ func parseTerraformAttribute(val interface{}, t schema.ValueType) interface{} {
}
}

func driftTerraform(ctx context.Context, logger hclog.Logger, conn schema.QueryExecer, cloudName string, cloudTable *traversedTable, resName string, resources map[string]*ResourceConfig, iacData *IACConfig, states TFStates, runParams RunParams, accountIDs []string) (*Result, error) {
func driftTerraform(ctx context.Context, logger hclog.Logger, conn execution.QueryExecer, cloudName string, cloudTable *traversedTable, resName string, resources map[string]*ResourceConfig, iacData *IACConfig, states TFStates, runParams RunParams, accountIDs []string) (*Result, error) {
res := &Result{
Different: nil,
Equal: nil,
Expand Down
6 changes: 3 additions & 3 deletions pkg/module/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"context"
"fmt"

"github.com/cloudquery/cq-provider-sdk/provider/schema"
"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/hcl/v2"
)
Expand All @@ -15,7 +15,7 @@ type ManagerImpl struct {
modOrder []string

// Instance of database
pool schema.QueryExecer
pool execution.QueryExecer

// Logger instance
logger hclog.Logger
Expand All @@ -35,7 +35,7 @@ type Manager interface {
}

// NewManager returns a new manager instance.
func NewManager(pool schema.QueryExecer, logger hclog.Logger) *ManagerImpl {
func NewManager(pool execution.QueryExecer, logger hclog.Logger) *ManagerImpl {
return &ManagerImpl{
modules: make(map[string]Module),
pool: pool,
Expand Down
4 changes: 2 additions & 2 deletions pkg/module/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"context"

"github.com/cloudquery/cq-provider-sdk/cqproto"
"github.com/cloudquery/cq-provider-sdk/provider/schema"
"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/hashicorp/hcl/v2"
)

Expand All @@ -28,7 +28,7 @@ type ExecuteRequest struct {
// Providers is the list of providers to process
Providers []*cqproto.GetProviderSchemaResponse
// Conn is the db connection to use
Conn schema.QueryExecer
Conn execution.QueryExecer
}

type ExecutionResult struct {
Expand Down
6 changes: 3 additions & 3 deletions pkg/policy/execute.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"strings"
"time"

"github.com/cloudquery/cq-provider-sdk/provider/schema"
"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-version"
"github.com/spf13/afero"
Expand Down Expand Up @@ -45,7 +45,7 @@ func (f Update) DoneCount() int {
// Executor implements the execution framework.
type Executor struct {
// Connection to the database
conn schema.QueryExecer
conn execution.QueryExecer
log hclog.Logger

PolicyPath []string
Expand Down Expand Up @@ -101,7 +101,7 @@ type ExecuteRequest struct {
}

// NewExecutor creates a new executor.
func NewExecutor(conn schema.QueryExecer, log hclog.Logger, progressUpdate UpdateCallback) *Executor {
func NewExecutor(conn execution.QueryExecer, log hclog.Logger, progressUpdate UpdateCallback) *Executor {
return &Executor{
conn: conn,
log: log,
Expand Down
4 changes: 2 additions & 2 deletions pkg/policy/execute_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ import (
"testing"

sdkdb "github.com/cloudquery/cq-provider-sdk/database"
"github.com/cloudquery/cq-provider-sdk/provider/schema"
"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/hashicorp/go-hclog"
"github.com/stretchr/testify/assert"
)

func setupPolicyDatabase(t *testing.T, tableName string) (schema.QueryExecer, func(t *testing.T)) {
func setupPolicyDatabase(t *testing.T, tableName string) (execution.QueryExecer, func(t *testing.T)) {
conn, err := sdkdb.New(context.Background(), hclog.NewNullLogger(), "postgres://postgres:pass@localhost:5432/postgres")
assert.NoError(t, err)

Expand Down
6 changes: 3 additions & 3 deletions pkg/policy/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"context"
"fmt"

"github.com/cloudquery/cq-provider-sdk/provider/schema"
"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"

Expand All @@ -21,7 +21,7 @@ type ManagerImpl struct {
policyDirectory string

// Instance of a database connection pool
pool schema.QueryExecer
pool execution.QueryExecer

// Logger instance
logger hclog.Logger
Expand All @@ -38,7 +38,7 @@ type Manager interface {
}

// NewManager returns a new manager instance.
func NewManager(policyDir string, pool schema.QueryExecer, logger hclog.Logger) *ManagerImpl {
func NewManager(policyDir string, pool execution.QueryExecer, logger hclog.Logger) *ManagerImpl {
return &ManagerImpl{
policyDirectory: policyDir,
pool: pool,
Expand Down
2 changes: 1 addition & 1 deletion pkg/ui/console/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ func (c Client) Fetch(ctx context.Context, failOnError bool) error {
if ui.IsTerminal() && fetchProgress != nil {
fetchProgress.MarkAllDone()
fetchProgress.Wait()
printFetchResponse(response)
printFetchResponse(response, viper.GetBool("redact-diags"))
}

if response == nil {
Expand Down
15 changes: 11 additions & 4 deletions pkg/ui/console/fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,16 @@ import (

"github.com/cloudquery/cloudquery/pkg/client"
"github.com/cloudquery/cloudquery/pkg/ui"
"github.com/cloudquery/cq-provider-sdk/provider/schema/diag"
"github.com/cloudquery/cq-provider-sdk/provider/diag"
)

func printFetchResponse(summary *client.FetchResponse) {
func printFetchResponse(summary *client.FetchResponse, redactDiags bool) {
if summary == nil {
return
}
for _, pfs := range summary.ProviderFetchSummary {
if len(pfs.Diagnostics()) > 0 {
printDiagnostics(pfs.ProviderName, pfs.Diagnostics())
printDiagnostics(pfs.ProviderName, pfs.Diagnostics(), redactDiags)
continue
}
if len(pfs.PartialFetchErrors) == 0 {
Expand All @@ -40,11 +40,18 @@ func printFetchResponse(summary *client.FetchResponse) {
}
}

func printDiagnostics(providerName string, diags diag.Diagnostics) {
func printDiagnostics(providerName string, diags diag.Diagnostics, redactDiags bool) {
// sort diagnostics by severity/type
sort.Sort(diags)
ui.ColorizedOutput(ui.ColorHeader, "Fetch Diagnostics for provider %s:\n\n", providerName)
for _, d := range diags {
if redactDiags {
if rd, ok := d.(diag.Redactable); ok {
if r := rd.Redacted(); r != nil {
d = r
}
}
}
desc := d.Description()
switch d.Severity() {
case diag.IGNORE:
Expand Down