From 963a854fd28952164aa7e15d0d3f3970e177e068 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 8 Feb 2024 11:27:08 +0200 Subject: [PATCH 1/4] feat: Validate plugin spec before init --- cli/cmd/migrate_v3.go | 24 +++----------- cli/cmd/specs.go | 77 +++++++++++++++++++++++++++++++++++++++++++ cli/cmd/sync_v3.go | 34 ++++--------------- cli/cmd/tables_v3.go | 4 +-- 4 files changed, 90 insertions(+), 49 deletions(-) diff --git a/cli/cmd/migrate_v3.go b/cli/cmd/migrate_v3.go index de3868380a3fda..e303011769c367 100644 --- a/cli/cmd/migrate_v3.go +++ b/cli/cmd/migrate_v3.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "encoding/json" "fmt" "time" @@ -45,28 +44,15 @@ func migrateConnectionV3(ctx context.Context, sourceClient *managedplugin.Client } // initialize destinations first, so that their connections may be used as backends by the source - for i := range destinationsClients { - destSpec := destinationSpecs[i] - destSpecBytes, err := json.Marshal(destSpec.Spec) - if err != nil { - return err - } - if _, err := destinationsPbClients[i].Init(ctx, &plugin.Init_Request{ - Spec: destSpecBytes, - }); err != nil { - return err + for i, destinationSpec := range destinationSpecs { + if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false); err != nil { + return fmt.Errorf("failed to init destination %v: %w", destinationSpec.Name, err) } } - specBytes, err := json.Marshal(sourceSpec.Spec) + err := initPlugin(ctx, sourcePbClient, sourceSpec.Spec, true) if err != nil { - return err - } - if _, err := sourcePbClient.Init(ctx, &plugin.Init_Request{ - Spec: specBytes, - NoConnection: true, - }); err != nil { - return err + return fmt.Errorf("failed to init source %v: %w", sourceSpec.Name, err) } writeClients := make([]plugin.Plugin_WriteClient, len(destinationsPbClients)) diff --git a/cli/cmd/specs.go b/cli/cmd/specs.go index e8ea2b3eaf2dc5..768e7f9582e49a 100644 --- a/cli/cmd/specs.go +++ b/cli/cmd/specs.go @@ -1,10 +1,18 @@ package cmd import ( + "context" + "encoding/json" "fmt" + "strings" "github.com/cloudquery/cloudquery/cli/internal/specs/v0" + "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" pbSpecs "github.com/cloudquery/plugin-pb-go/specs" + "github.com/rs/zerolog/log" + "github.com/santhosh-tekuri/jsonschema/v5" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func CLIRegistryToPbRegistry(registry specs.Registry) pbSpecs.Registry { @@ -86,3 +94,72 @@ func CLIDestinationSpecToPbSpec(spec specs.Destination) pbSpecs.Destination { Spec: spec.Spec, } } + +// initPlugin is a simple wrapper that will try to validate the spec before actually passing it to Init. +func initPlugin(ctx context.Context, client plugin.PluginClient, spec any, noConnection bool) error { + if !noConnection { + // perform spec validation + if err := validatePluginSpec(ctx, client, spec); err != nil { + return err + } + } + + specBytes, err := json.Marshal(spec) + if err != nil { + return err + } + + _, err = client.Init(ctx, &plugin.Init_Request{Spec: specBytes, NoConnection: noConnection}) + return err +} + +// validatePluginSpec encompasses spec validation only: +// 1. Get spec schema from the plugin. +// If the call isn't implemented, just skip the validation. +// 2. Validate that the provided JSON schema is valid & can be used for spec validation. +// If the spec is empty (i.e., the plugin didn't supply the schema) just skip. +// 3. If the schema isn't empty but not valid, print the error message & skip the validation. +// 4. Finally, return the validation result. +func validatePluginSpec(ctx context.Context, client plugin.PluginClient, spec any) error { + schema, err := client.GetSpecSchema(ctx, &plugin.GetSpecSchema_Request{}) + if err != nil { + st, ok := status.FromError(err) + if !ok { + // not a gRPC-compatible error + log.Err(err).Msg("failed to get spec schema") + return err + } + if st.Code() != codes.Unimplemented { + // unimplemented is OK, treat as empty schema + log.Err(err).Msg("failed to get spec schema") + return err + } + } + + jsonSchema := schema.GetJsonSchema() + if len(jsonSchema) == 0 { + // This will also be true for Unimplemented response (schema = nil => schema.GetJsonSchema() = "") + log.Info().Msg("empty JSON schema for plugin spec, skipping validation") + return nil + } + + sc, err := parseJSONSchema(jsonSchema) + if err != nil { + log.Err(err).Msg("failed to parse spec schema, skipping validation") + return nil + } + + return sc.Validate(spec) +} + +func parseJSONSchema(jsonSchema string) (*jsonschema.Schema, error) { + c := jsonschema.NewCompiler() + c.Draft = jsonschema.Draft2020 + c.AssertFormat = true + + if err := c.AddResource("schema.json", strings.NewReader(jsonSchema)); err != nil { + return nil, err + } + + return c.Compile("schema.json") +} diff --git a/cli/cmd/sync_v3.go b/cli/cmd/sync_v3.go index 10eb650134afb5..8503e696caf34d 100644 --- a/cli/cmd/sync_v3.go +++ b/cli/cmd/sync_v3.go @@ -126,28 +126,14 @@ func syncConnectionV3(ctx context.Context, source v3source, destinations []v3des } // initialize destinations first, so that their connections may be used as backends by the source - for i := range destinationsClients { - destSpec := destinationSpecs[i] - destSpecBytes, err := json.Marshal(destSpec.Spec) - if err != nil { - return err - } - if _, err := destinationsPbClients[i].Init(ctx, &plugin.Init_Request{ - Spec: destSpecBytes, - }); err != nil { - return fmt.Errorf("failed to init destination %v: %w", destSpec.Name, err) + for i, destinationSpec := range destinationSpecs { + if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false); err != nil { + return fmt.Errorf("failed to init destination %v: %w", destinationSpec.Name, err) } } if backend != nil { - backendSpec := backend.spec - backendSpecBytes, err := json.Marshal(backendSpec.Spec) - if err != nil { - return err - } - if _, err := backendPbClient.Init(ctx, &plugin.Init_Request{ - Spec: backendSpecBytes, - }); err != nil { - return fmt.Errorf("failed to init backend %v: %w", backendSpec.Name, err) + if err := initPlugin(ctx, backendPbClient, backend.spec.Spec, false); err != nil { + return fmt.Errorf("failed to init backend %v: %w", backend.spec.Name, err) } } @@ -165,14 +151,8 @@ func syncConnectionV3(ctx context.Context, source v3source, destinations []v3des return fmt.Errorf("failed to unmarshal source spec JSON after variable replacement: %w", err) } - sourceSpecBytes, err := json.Marshal(sourceSpec.Spec) - if err != nil { - return err - } - if _, err := sourcePbClient.Init(ctx, &plugin.Init_Request{ - Spec: sourceSpecBytes, - }); err != nil { - return err + if err = initPlugin(ctx, sourcePbClient, sourceSpec.Spec, false); err != nil { + return fmt.Errorf("failed to init source %v: %w", sourceSpec.Name, err) } writeClients := make([]plugin.Plugin_WriteClient, len(destinationsPbClients)) diff --git a/cli/cmd/tables_v3.go b/cli/cmd/tables_v3.go index 4b1adc257fbdba..3a05217aca1584 100644 --- a/cli/cmd/tables_v3.go +++ b/cli/cmd/tables_v3.go @@ -20,9 +20,7 @@ func tablesV3(ctx context.Context, sourceClient *managedplugin.Client, path stri return err } sourcePbClient := pluginPb.NewPluginClient(sourceClient.Conn) - if _, err := sourcePbClient.Init(ctx, &pluginPb.Init_Request{ - NoConnection: true, - }); err != nil { + if err := initPlugin(ctx, sourcePbClient, nil, true); err != nil { return fmt.Errorf("failed to init source: %w", err) } getTablesResp, err := sourcePbClient.GetTables(ctx, &pluginPb.GetTables_Request{ From 7f2834a78933fe51d3091a6d13ee5eb21ba6f191 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 8 Feb 2024 12:46:41 +0200 Subject: [PATCH 2/4] prettify compilation error --- cli/cmd/specs.go | 16 +++++++++++++++- cli/go.mod | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cli/cmd/specs.go b/cli/cmd/specs.go index 768e7f9582e49a..47ea5be69bd69f 100644 --- a/cli/cmd/specs.go +++ b/cli/cmd/specs.go @@ -3,6 +3,7 @@ package cmd import ( "context" "encoding/json" + "errors" "fmt" "strings" @@ -161,5 +162,18 @@ func parseJSONSchema(jsonSchema string) (*jsonschema.Schema, error) { return nil, err } - return c.Compile("schema.json") + sc, err := c.Compile("schema.json") + if err != nil { + var se *jsonschema.SchemaError + if errors.As(err, &se); se != nil && se.Err != nil { + // We add resource as `file`, but there's none, actually. + // So, we need to prettify message a bit. + + return nil, fmt.Errorf("jsonschema compilation failed: %w", + errors.New(strings.Replace(se.Err.Error(), "jsonschema: '' ", "jsonschema: ", 1))) + } + return nil, err + } + + return sc, nil } diff --git a/cli/go.mod b/cli/go.mod index 10575f624b691c..adba65fce98521 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -23,6 +23,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/rs/zerolog v1.31.0 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/schollz/progressbar/v3 v3.13.1 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.4 @@ -118,7 +119,6 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect From 604ae84b96e6825d2a6d9afc6bb3b4e62d157f71 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 8 Feb 2024 12:49:11 +0200 Subject: [PATCH 3/4] rm extra line --- cli/cmd/specs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/cmd/specs.go b/cli/cmd/specs.go index 47ea5be69bd69f..f6fa89c1bd9de9 100644 --- a/cli/cmd/specs.go +++ b/cli/cmd/specs.go @@ -168,7 +168,6 @@ func parseJSONSchema(jsonSchema string) (*jsonschema.Schema, error) { if errors.As(err, &se); se != nil && se.Err != nil { // We add resource as `file`, but there's none, actually. // So, we need to prettify message a bit. - return nil, fmt.Errorf("jsonschema compilation failed: %w", errors.New(strings.Replace(se.Err.Error(), "jsonschema: '' ", "jsonschema: ", 1))) } From 38990858b56be67ef091af4df67f59d452825a8d Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 8 Feb 2024 13:19:50 +0200 Subject: [PATCH 4/4] Update cli/cmd/specs.go Co-authored-by: Kemal <223029+disq@users.noreply.github.com> --- cli/cmd/specs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/cmd/specs.go b/cli/cmd/specs.go index f6fa89c1bd9de9..fa10f4329adeda 100644 --- a/cli/cmd/specs.go +++ b/cli/cmd/specs.go @@ -169,7 +169,7 @@ func parseJSONSchema(jsonSchema string) (*jsonschema.Schema, error) { // We add resource as `file`, but there's none, actually. // So, we need to prettify message a bit. return nil, fmt.Errorf("jsonschema compilation failed: %w", - errors.New(strings.Replace(se.Err.Error(), "jsonschema: '' ", "jsonschema: ", 1))) + errors.New(strings.Replace(se.Err.Error(), "jsonschema: '' ", "", 1))) } return nil, err }