From 7b6eaeb2c1a2e16436f14d4e77335c24df77377b Mon Sep 17 00:00:00 2001 From: bbernays Date: Thu, 4 Apr 2024 16:26:46 -0500 Subject: [PATCH 01/14] initial --- cli/cmd/migrate_v3.go | 4 ++-- cli/cmd/specs.go | 9 ++++++++- cli/cmd/sync_v3.go | 6 +++--- cli/cmd/tables_v3.go | 2 +- cli/cmd/test_connection.go | 24 ++++++++++++++++++++---- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/cli/cmd/migrate_v3.go b/cli/cmd/migrate_v3.go index 2438573142519a..366bc800d96276 100644 --- a/cli/cmd/migrate_v3.go +++ b/cli/cmd/migrate_v3.go @@ -45,12 +45,12 @@ func migrateConnectionV3(ctx context.Context, sourceClient *managedplugin.Client // initialize destinations first, so that their connections may be used as backends by the source for i, destinationSpec := range destinationSpecs { - if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false, invocationUUID.String()); err != nil { + if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false, false, invocationUUID.String()); err != nil { return fmt.Errorf("failed to init destination %v: %w", destinationSpec.Name, err) } } - err := initPlugin(ctx, sourcePbClient, sourceSpec.Spec, true, invocationUUID.String()) + err := initPlugin(ctx, sourcePbClient, sourceSpec.Spec, true, false, invocationUUID.String()) if err != nil { return fmt.Errorf("failed to init source %v: %w", sourceSpec.Name, err) } diff --git a/cli/cmd/specs.go b/cli/cmd/specs.go index 3c83ed4b50700f..f5fc661082cc53 100644 --- a/cli/cmd/specs.go +++ b/cli/cmd/specs.go @@ -97,14 +97,21 @@ func CLIDestinationSpecToPbSpec(spec specs.Destination) pbSpecs.Destination { } // initPlugin is a simple wrapper that will try to validate the spec before actually passing it to Init. -func initPlugin(ctx context.Context, client plugin.PluginClient, spec map[string]any, noConnection bool, syncID string) error { +func initPlugin(ctx context.Context, client plugin.PluginClient, spec map[string]any, noConnection bool, noInit bool, syncID string) error { if !noConnection { // perform spec validation if err := validatePluginSpec(ctx, client, spec); err != nil { + if noInit { + return err + } log.Warn().Err(err).Msg("plugin spec validation failed, but continuing with Init") } } + if noInit { + return nil + } + var ( specBytes []byte err error diff --git a/cli/cmd/sync_v3.go b/cli/cmd/sync_v3.go index c1b3383f56c045..07e5a37e1c2944 100644 --- a/cli/cmd/sync_v3.go +++ b/cli/cmd/sync_v3.go @@ -127,12 +127,12 @@ func syncConnectionV3(ctx context.Context, source v3source, destinations []v3des // initialize destinations first, so that their connections may be used as backends by the source for i, destinationSpec := range destinationSpecs { - if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false, uid); err != nil { + if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false, false, uid); err != nil { return fmt.Errorf("failed to init destination %v: %w", destinationSpec.Name, err) } } if backend != nil { - if err := initPlugin(ctx, backendPbClient, backend.spec.Spec, false, uid); err != nil { + if err := initPlugin(ctx, backendPbClient, backend.spec.Spec, false, false, uid); err != nil { return fmt.Errorf("failed to init backend %v: %w", backend.spec.Name, err) } } @@ -151,7 +151,7 @@ func syncConnectionV3(ctx context.Context, source v3source, destinations []v3des return fmt.Errorf("failed to unmarshal source spec JSON after variable replacement: %w", err) } - if err = initPlugin(ctx, sourcePbClient, sourceSpec.Spec, false, uid); err != nil { + if err = initPlugin(ctx, sourcePbClient, sourceSpec.Spec, false, false, uid); err != nil { return fmt.Errorf("failed to init source %v: %w", sourceSpec.Name, err) } diff --git a/cli/cmd/tables_v3.go b/cli/cmd/tables_v3.go index 920d5b4dceb4bb..ea0b9d8c02fb57 100644 --- a/cli/cmd/tables_v3.go +++ b/cli/cmd/tables_v3.go @@ -20,7 +20,7 @@ func tablesV3(ctx context.Context, sourceClient *managedplugin.Client, sourceSpe return err } sourcePbClient := pluginPb.NewPluginClient(sourceClient.Conn) - if err := initPlugin(ctx, sourcePbClient, sourceSpec, true, invocationUUID.String()); err != nil { + if err := initPlugin(ctx, sourcePbClient, sourceSpec, true, false, invocationUUID.String()); err != nil { return fmt.Errorf("failed to init source: %w", err) } getTablesResp, err := sourcePbClient.GetTables(ctx, &pluginPb.GetTables_Request{ diff --git a/cli/cmd/test_connection.go b/cli/cmd/test_connection.go index 69491cc3c27f50..712ab765822ef6 100644 --- a/cli/cmd/test_connection.go +++ b/cli/cmd/test_connection.go @@ -87,6 +87,9 @@ func newCmdTestConnection() *cobra.Command { RunE: testConnection, Hidden: true, } + + cmd.Flags().Bool("no-init", false, "Disable initialization of plugins. Useful for validating spec(s) without connecting to sources and destinations") + return cmd } @@ -96,6 +99,11 @@ func testConnection(cmd *cobra.Command, args []string) error { return err } + noInit, err := cmd.Flags().GetBool("no-init") + if err != nil { + return err + } + ctx := cmd.Context() updateSyncTestConnectionStatus(cmd.Context(), log.Logger, cloudquery_api.SyncTestConnectionStatusStarted) @@ -176,9 +184,13 @@ func testConnection(cmd *cobra.Command, args []string) error { for i, client := range sourceClients { pluginClient := plugin.NewPluginClient(client.Conn) log.Info().Str("source", sources[i].VersionString()).Msg("Initializing source") - err := initPlugin(ctx, pluginClient, sources[i].Spec, false, invocationUUID.String()) + err := initPlugin(ctx, pluginClient, sources[i].Spec, false, noInit, invocationUUID.String()) if err != nil { - initErrors = append(initErrors, fmt.Errorf("failed to init source %v: %w", sources[i].VersionString(), err)) + if noInit { + initErrors = append(initErrors, fmt.Errorf("failed to validate source config %v: %w", sources[i].VersionString(), err)) + } else { + initErrors = append(initErrors, fmt.Errorf("failed to init source %v: %w", sources[i].VersionString(), err)) + } } else { log.Info().Str("source", sources[i].VersionString()).Msg("Initialized source") } @@ -186,9 +198,13 @@ func testConnection(cmd *cobra.Command, args []string) error { for i, client := range destinationClients { pluginClient := plugin.NewPluginClient(client.Conn) log.Info().Str("destination", destinations[i].VersionString()).Msg("Initializing destination") - err := initPlugin(ctx, pluginClient, destinations[i].Spec, false, invocationUUID.String()) + err := initPlugin(ctx, pluginClient, destinations[i].Spec, false, noInit, invocationUUID.String()) if err != nil { - initErrors = append(initErrors, fmt.Errorf("failed to init destination %v: %w", destinations[i].VersionString(), err)) + if noInit { + initErrors = append(initErrors, fmt.Errorf("failed to validate destination config %v: %w", sources[i].VersionString(), err)) + } else { + initErrors = append(initErrors, fmt.Errorf("failed to init destination %v: %w", destinations[i].VersionString(), err)) + } } else { log.Info().Str("destination", destinations[i].VersionString()).Msg("Initialized destination") } From 4f9324aec28f883230158ab0ba1701fed4ed6d13 Mon Sep 17 00:00:00 2001 From: bbernays Date: Thu, 4 Apr 2024 16:36:57 -0500 Subject: [PATCH 02/14] Update specs.go --- cli/cmd/specs.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/cli/cmd/specs.go b/cli/cmd/specs.go index f5fc661082cc53..0c284f87613a40 100644 --- a/cli/cmd/specs.go +++ b/cli/cmd/specs.go @@ -98,20 +98,17 @@ func CLIDestinationSpecToPbSpec(spec specs.Destination) pbSpecs.Destination { // initPlugin is a simple wrapper that will try to validate the spec before actually passing it to Init. func initPlugin(ctx context.Context, client plugin.PluginClient, spec map[string]any, noConnection bool, noInit bool, syncID string) error { + if noInit { + return validatePluginSpec(ctx, client, spec) + } + if !noConnection { // perform spec validation if err := validatePluginSpec(ctx, client, spec); err != nil { - if noInit { - return err - } log.Warn().Err(err).Msg("plugin spec validation failed, but continuing with Init") } } - if noInit { - return nil - } - var ( specBytes []byte err error From ba354d93b4082b4a68776fa746f5b0ba822447a8 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 5 Apr 2024 07:50:50 -0500 Subject: [PATCH 03/14] revert no-init --- cli/cmd/migrate_v3.go | 4 +- cli/cmd/specs.go | 6 +-- cli/cmd/sync_v3.go | 6 +-- cli/cmd/tables_v3.go | 2 +- cli/cmd/test_connection.go | 23 ++------- .../pages/docs/reference/cli/cloudquery.md | 1 + .../cli/cloudquery_validate-config.md | 47 +++++++++++++++++++ 7 files changed, 59 insertions(+), 30 deletions(-) create mode 100644 website/pages/docs/reference/cli/cloudquery_validate-config.md diff --git a/cli/cmd/migrate_v3.go b/cli/cmd/migrate_v3.go index 366bc800d96276..2438573142519a 100644 --- a/cli/cmd/migrate_v3.go +++ b/cli/cmd/migrate_v3.go @@ -45,12 +45,12 @@ func migrateConnectionV3(ctx context.Context, sourceClient *managedplugin.Client // initialize destinations first, so that their connections may be used as backends by the source for i, destinationSpec := range destinationSpecs { - if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false, false, invocationUUID.String()); err != nil { + if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false, invocationUUID.String()); err != nil { return fmt.Errorf("failed to init destination %v: %w", destinationSpec.Name, err) } } - err := initPlugin(ctx, sourcePbClient, sourceSpec.Spec, true, false, invocationUUID.String()) + err := initPlugin(ctx, sourcePbClient, sourceSpec.Spec, true, invocationUUID.String()) if err != nil { return fmt.Errorf("failed to init source %v: %w", sourceSpec.Name, err) } diff --git a/cli/cmd/specs.go b/cli/cmd/specs.go index 0c284f87613a40..3c83ed4b50700f 100644 --- a/cli/cmd/specs.go +++ b/cli/cmd/specs.go @@ -97,11 +97,7 @@ func CLIDestinationSpecToPbSpec(spec specs.Destination) pbSpecs.Destination { } // initPlugin is a simple wrapper that will try to validate the spec before actually passing it to Init. -func initPlugin(ctx context.Context, client plugin.PluginClient, spec map[string]any, noConnection bool, noInit bool, syncID string) error { - if noInit { - return validatePluginSpec(ctx, client, spec) - } - +func initPlugin(ctx context.Context, client plugin.PluginClient, spec map[string]any, noConnection bool, syncID string) error { if !noConnection { // perform spec validation if err := validatePluginSpec(ctx, client, spec); err != nil { diff --git a/cli/cmd/sync_v3.go b/cli/cmd/sync_v3.go index 07e5a37e1c2944..c1b3383f56c045 100644 --- a/cli/cmd/sync_v3.go +++ b/cli/cmd/sync_v3.go @@ -127,12 +127,12 @@ func syncConnectionV3(ctx context.Context, source v3source, destinations []v3des // initialize destinations first, so that their connections may be used as backends by the source for i, destinationSpec := range destinationSpecs { - if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false, false, uid); err != nil { + if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false, uid); err != nil { return fmt.Errorf("failed to init destination %v: %w", destinationSpec.Name, err) } } if backend != nil { - if err := initPlugin(ctx, backendPbClient, backend.spec.Spec, false, false, uid); err != nil { + if err := initPlugin(ctx, backendPbClient, backend.spec.Spec, false, uid); err != nil { return fmt.Errorf("failed to init backend %v: %w", backend.spec.Name, err) } } @@ -151,7 +151,7 @@ func syncConnectionV3(ctx context.Context, source v3source, destinations []v3des return fmt.Errorf("failed to unmarshal source spec JSON after variable replacement: %w", err) } - if err = initPlugin(ctx, sourcePbClient, sourceSpec.Spec, false, false, uid); err != nil { + if err = initPlugin(ctx, sourcePbClient, sourceSpec.Spec, false, uid); err != nil { return fmt.Errorf("failed to init source %v: %w", sourceSpec.Name, err) } diff --git a/cli/cmd/tables_v3.go b/cli/cmd/tables_v3.go index ea0b9d8c02fb57..920d5b4dceb4bb 100644 --- a/cli/cmd/tables_v3.go +++ b/cli/cmd/tables_v3.go @@ -20,7 +20,7 @@ func tablesV3(ctx context.Context, sourceClient *managedplugin.Client, sourceSpe return err } sourcePbClient := pluginPb.NewPluginClient(sourceClient.Conn) - if err := initPlugin(ctx, sourcePbClient, sourceSpec, true, false, invocationUUID.String()); err != nil { + if err := initPlugin(ctx, sourcePbClient, sourceSpec, true, invocationUUID.String()); err != nil { return fmt.Errorf("failed to init source: %w", err) } getTablesResp, err := sourcePbClient.GetTables(ctx, &pluginPb.GetTables_Request{ diff --git a/cli/cmd/test_connection.go b/cli/cmd/test_connection.go index 712ab765822ef6..8088b8692ec5c1 100644 --- a/cli/cmd/test_connection.go +++ b/cli/cmd/test_connection.go @@ -88,8 +88,6 @@ func newCmdTestConnection() *cobra.Command { Hidden: true, } - cmd.Flags().Bool("no-init", false, "Disable initialization of plugins. Useful for validating spec(s) without connecting to sources and destinations") - return cmd } @@ -99,11 +97,6 @@ func testConnection(cmd *cobra.Command, args []string) error { return err } - noInit, err := cmd.Flags().GetBool("no-init") - if err != nil { - return err - } - ctx := cmd.Context() updateSyncTestConnectionStatus(cmd.Context(), log.Logger, cloudquery_api.SyncTestConnectionStatusStarted) @@ -184,13 +177,9 @@ func testConnection(cmd *cobra.Command, args []string) error { for i, client := range sourceClients { pluginClient := plugin.NewPluginClient(client.Conn) log.Info().Str("source", sources[i].VersionString()).Msg("Initializing source") - err := initPlugin(ctx, pluginClient, sources[i].Spec, false, noInit, invocationUUID.String()) + err := initPlugin(ctx, pluginClient, sources[i].Spec, false, invocationUUID.String()) if err != nil { - if noInit { - initErrors = append(initErrors, fmt.Errorf("failed to validate source config %v: %w", sources[i].VersionString(), err)) - } else { - initErrors = append(initErrors, fmt.Errorf("failed to init source %v: %w", sources[i].VersionString(), err)) - } + initErrors = append(initErrors, fmt.Errorf("failed to init source %v: %w", sources[i].VersionString(), err)) } else { log.Info().Str("source", sources[i].VersionString()).Msg("Initialized source") } @@ -198,13 +187,9 @@ func testConnection(cmd *cobra.Command, args []string) error { for i, client := range destinationClients { pluginClient := plugin.NewPluginClient(client.Conn) log.Info().Str("destination", destinations[i].VersionString()).Msg("Initializing destination") - err := initPlugin(ctx, pluginClient, destinations[i].Spec, false, noInit, invocationUUID.String()) + err := initPlugin(ctx, pluginClient, destinations[i].Spec, false, invocationUUID.String()) if err != nil { - if noInit { - initErrors = append(initErrors, fmt.Errorf("failed to validate destination config %v: %w", sources[i].VersionString(), err)) - } else { - initErrors = append(initErrors, fmt.Errorf("failed to init destination %v: %w", destinations[i].VersionString(), err)) - } + initErrors = append(initErrors, fmt.Errorf("failed to init destination %v: %w", destinations[i].VersionString(), err)) } else { log.Info().Str("destination", destinations[i].VersionString()).Msg("Initialized destination") } diff --git a/website/pages/docs/reference/cli/cloudquery.md b/website/pages/docs/reference/cli/cloudquery.md index 3d01496d944617..bad6830fdbb3ce 100644 --- a/website/pages/docs/reference/cli/cloudquery.md +++ b/website/pages/docs/reference/cli/cloudquery.md @@ -37,4 +37,5 @@ Find more information at: * [cloudquery switch](/docs/reference/cli/cloudquery_switch) - Switches between teams. * [cloudquery sync](/docs/reference/cli/cloudquery_sync) - Sync resources from configured source plugins to destinations * [cloudquery tables](/docs/reference/cli/cloudquery_tables) - Generate documentation for all supported tables of source plugins specified in the spec(s) +* [cloudquery validate-config](/docs/reference/cli/cloudquery_validate-config) - Validate configs diff --git a/website/pages/docs/reference/cli/cloudquery_validate-config.md b/website/pages/docs/reference/cli/cloudquery_validate-config.md new file mode 100644 index 00000000000000..bff07df35fca1b --- /dev/null +++ b/website/pages/docs/reference/cli/cloudquery_validate-config.md @@ -0,0 +1,47 @@ +--- +title: "validate-config" +--- +## cloudquery validate-config + +Validate configs + +### Synopsis + +Validate configs without requiring any credentials or connections. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation. + +``` +cloudquery validate-config [files or directories] [flags] +``` + +### Examples + +``` +# Validate configs +cloudquery validate-config ./directory +# Validate configs from directories and files +cloudquery validate-config ./directory ./aws.yml ./pg.yml + +``` + +### Options + +``` + -h, --help help for validate-config +``` + +### Options inherited from parent commands + +``` + --cq-dir string directory to store cloudquery files, such as downloaded plugins (default ".cq") + --log-console enable console logging + --log-file-name string Log filename (default "cloudquery.log") + --log-format string Logging format (json, text) (default "text") + --log-level string Logging level (trace, debug, info, warn, error) (default "info") + --no-log-file Disable logging to file + --telemetry-level string Telemetry level (none, errors, stats, all) (default "all") +``` + +### SEE ALSO + +* [cloudquery](/docs/reference/cli/cloudquery) - CloudQuery CLI + From 3a42b33b6473b000590fa30f00d4421fe4278647 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 5 Apr 2024 07:50:53 -0500 Subject: [PATCH 04/14] Create validate_config.go --- cli/cmd/validate_config.go | 144 +++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 cli/cmd/validate_config.go diff --git a/cli/cmd/validate_config.go b/cli/cmd/validate_config.go new file mode 100644 index 00000000000000..fbebd3eef6dea5 --- /dev/null +++ b/cli/cmd/validate_config.go @@ -0,0 +1,144 @@ +package cmd + +import ( + "errors" + "fmt" + "strings" + + "github.com/cloudquery/cloudquery/cli/internal/auth" + "github.com/cloudquery/cloudquery/cli/internal/specs/v0" + "github.com/cloudquery/plugin-pb-go/managedplugin" + "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +const ( + validateConfigShort = "Validate configs" + validateConfigLong = "Validate configs without requiring any credentials or connections. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation." + validateConfigExample = `# Validate configs +cloudquery validate-config ./directory +# Validate configs from directories and files +cloudquery validate-config ./directory ./aws.yml ./pg.yml +` +) + +func newCmdValidateConfig() *cobra.Command { + cmd := &cobra.Command{ + Use: "validate-config [files or directories]", + Short: validateConfigShort, + Long: validateConfigLong, + Example: validateConfigExample, + Args: cobra.MinimumNArgs(1), + RunE: validateConfig, + Hidden: false, + } + + return cmd +} + +func validateConfig(cmd *cobra.Command, args []string) error { + cqDir, err := cmd.Flags().GetString("cq-dir") + if err != nil { + return err + } + + ctx := cmd.Context() + + log.Info().Strs("args", args).Msg("Loading spec(s)") + fmt.Printf("Loading spec(s) from %s\n", strings.Join(args, ", ")) + specReader, err := specs.NewSpecReader(args) + if err != nil { + return fmt.Errorf("failed to load spec(s) from %s. Error: %w", strings.Join(args, ", "), err) + } + sources := specReader.Sources + destinations := specReader.Destinations + + authToken, err := auth.GetAuthTokenIfNeeded(log.Logger, sources, destinations) + if err != nil { + return fmt.Errorf("failed to get auth token: %w", err) + } + teamName, err := auth.GetTeamForToken(authToken) + if err != nil { + return fmt.Errorf("failed to get team name: %w", err) + } + opts := []managedplugin.Option{ + managedplugin.WithLogger(log.Logger), + managedplugin.WithAuthToken(authToken.Value), + managedplugin.WithTeamName(teamName), + } + if cqDir != "" { + opts = append(opts, managedplugin.WithDirectory(cqDir)) + } + if disableSentry { + opts = append(opts, managedplugin.WithNoSentry()) + } + + sourcePluginConfigs := make([]managedplugin.Config, len(sources)) + sourceRegInferred := make([]bool, len(sources)) + for i, source := range sources { + sourcePluginConfigs[i] = managedplugin.Config{ + Name: source.Name, + Version: source.Version, + Path: source.Path, + Registry: SpecRegistryToPlugin(source.Registry), + DockerAuth: source.DockerRegistryAuthToken, + } + sourceRegInferred[i] = source.RegistryInferred() + } + destinationPluginConfigs := make([]managedplugin.Config, len(destinations)) + destinationRegInferred := make([]bool, len(destinations)) + for i, destination := range destinations { + destinationPluginConfigs[i] = managedplugin.Config{ + Name: destination.Name, + Version: destination.Version, + Path: destination.Path, + Registry: SpecRegistryToPlugin(destination.Registry), + DockerAuth: destination.DockerRegistryAuthToken, + } + destinationRegInferred[i] = destination.RegistryInferred() + } + + sourceClients, err := managedplugin.NewClients(ctx, managedplugin.PluginSource, sourcePluginConfigs, opts...) + if err != nil { + return enrichClientError(sourceClients, sourceRegInferred, err) + } + defer func() { + if err := sourceClients.Terminate(); err != nil { + fmt.Println(err) + } + }() + destinationClients, err := managedplugin.NewClients(ctx, managedplugin.PluginDestination, destinationPluginConfigs, opts...) + if err != nil { + return enrichClientError(destinationClients, destinationRegInferred, err) + } + defer func() { + if err := destinationClients.Terminate(); err != nil { + fmt.Println(err) + } + }() + + var initErrors []error + for i, client := range sourceClients { + pluginClient := plugin.NewPluginClient(client.Conn) + log.Info().Str("source", sources[i].VersionString()).Msg("Initializing source") + err := validatePluginSpec(ctx, pluginClient, sources[i].Spec) + if err != nil { + initErrors = append(initErrors, fmt.Errorf("failed to validate source config %v: %w", sources[i].VersionString(), err)) + } else { + log.Info().Str("source", sources[i].VersionString()).Msg("validated successfully") + } + } + for i, client := range destinationClients { + pluginClient := plugin.NewPluginClient(client.Conn) + log.Info().Str("destination", destinations[i].VersionString()).Msg("Initializing destination") + err = validatePluginSpec(ctx, pluginClient, destinations[i].Spec) + if err != nil { + initErrors = append(initErrors, fmt.Errorf("failed to validate destination config %v: %w", sources[i].VersionString(), err)) + } else { + log.Info().Str("destination", destinations[i].VersionString()).Msg("validated successfully") + } + } + + return errors.Join(initErrors...) +} From fc8e161f6987de454e66a9de50cd1e014c03d717 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 5 Apr 2024 07:50:56 -0500 Subject: [PATCH 05/14] Update root.go --- cli/cmd/root.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/cmd/root.go b/cli/cmd/root.go index a080d5e48fddfb..6584d258dc6f42 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -171,6 +171,7 @@ func NewCmdRoot() *cobra.Command { newCmdLogout(), newCmdSwitch(), newCmdTestConnection(), + newCmdValidateConfig(), newCmdPluginInstall(true), // legacy pluginCmd, addonCmd, From 5321b893e657f19874b8118a336d5de3fdc99e9b Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 5 Apr 2024 08:04:57 -0500 Subject: [PATCH 06/14] tests --- cli/cmd/testdata/validate-config-error.yml | 20 ++++++++ cli/cmd/validate_config.go | 2 +- cli/cmd/validate_config_test.go | 57 ++++++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 cli/cmd/testdata/validate-config-error.yml create mode 100644 cli/cmd/validate_config_test.go diff --git a/cli/cmd/testdata/validate-config-error.yml b/cli/cmd/testdata/validate-config-error.yml new file mode 100644 index 00000000000000..939d4ed4dbc71d --- /dev/null +++ b/cli/cmd/testdata/validate-config-error.yml @@ -0,0 +1,20 @@ +kind: source +spec: + name: aws + path: cloudquery/aws + registry: cloudquery + version: "v25.5.0" + destinations: ["postgresql"] + tables: ["aws_ec2_instances"] + spec: + invalid_key: "invalid_value" +--- +kind: destination +spec: + name: "postgresql" + path: "cloudquery/postgresql" + registry: cloudquery + version: "v7.3.5" + spec: + connection_string: "postgresql://postgres:not-a-real-password@localhost:5432/postgres?sslmode=disable" + invalid_key: "invalid_value" diff --git a/cli/cmd/validate_config.go b/cli/cmd/validate_config.go index fbebd3eef6dea5..217ebd468013fc 100644 --- a/cli/cmd/validate_config.go +++ b/cli/cmd/validate_config.go @@ -134,7 +134,7 @@ func validateConfig(cmd *cobra.Command, args []string) error { log.Info().Str("destination", destinations[i].VersionString()).Msg("Initializing destination") err = validatePluginSpec(ctx, pluginClient, destinations[i].Spec) if err != nil { - initErrors = append(initErrors, fmt.Errorf("failed to validate destination config %v: %w", sources[i].VersionString(), err)) + initErrors = append(initErrors, fmt.Errorf("failed to validate destination config %v: %w", destinations[i].VersionString(), err)) } else { log.Info().Str("destination", destinations[i].VersionString()).Msg("validated successfully") } diff --git a/cli/cmd/validate_config_test.go b/cli/cmd/validate_config_test.go new file mode 100644 index 00000000000000..d8427e1a8186d4 --- /dev/null +++ b/cli/cmd/validate_config_test.go @@ -0,0 +1,57 @@ +package cmd + +import ( + "os" + "path" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateConfig(t *testing.T) { + configs := []struct { + name string + config string + errors []string + }{ + { + name: "multiple test sources should pass validation", + config: "multiple-sources.yml", + }, + { + name: "bad AWS and Postgres auth should fail validation", + config: "validate-config-error.yml", + errors: []string{"failed to validate source config aws", "failed to validate destination config postgresql"}, + }, + } + _, filename, _, _ := runtime.Caller(0) + currentDir := path.Dir(filename) + cqDir := t.TempDir() + defer os.RemoveAll(cqDir) + + for _, tc := range configs { + t.Run(tc.name, func(t *testing.T) { + defer CloseLogFile() + testConfig := path.Join(currentDir, "testdata", tc.config) + logFileName := path.Join(cqDir, "cloudquery.log") + cmd := NewCmdRoot() + cmd.SetArgs([]string{"validate-config", testConfig, "--cq-dir", cqDir, "--log-file-name", logFileName}) + err := cmd.Execute() + if tc.errors != nil { + for _, e := range tc.errors { + assert.Contains(t, err.Error(), e) + } + } else { + assert.NoError(t, err) + } + + // check that log was written and contains some lines from the plugin + b, logFileError := os.ReadFile(path.Join(cqDir, "cloudquery.log")) + logContent := string(b) + require.NoError(t, logFileError, "failed to read cloudquery.log") + require.NotEmpty(t, logContent, "cloudquery.log empty; expected some logs") + }) + } +} From 4fa27a1e191dbd3c63a71b8cdea769722b5f1f00 Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Fri, 5 Apr 2024 16:07:40 +0300 Subject: [PATCH 07/14] Update cli/cmd/test_connection.go --- cli/cmd/test_connection.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/cmd/test_connection.go b/cli/cmd/test_connection.go index 8088b8692ec5c1..69491cc3c27f50 100644 --- a/cli/cmd/test_connection.go +++ b/cli/cmd/test_connection.go @@ -87,7 +87,6 @@ func newCmdTestConnection() *cobra.Command { RunE: testConnection, Hidden: true, } - return cmd } From 844fd2f6b031a3cf75901ce31b9b496e5d7098e3 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 5 Apr 2024 08:25:59 -0500 Subject: [PATCH 08/14] Update validate_config_test.go --- cli/cmd/validate_config_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cli/cmd/validate_config_test.go b/cli/cmd/validate_config_test.go index d8427e1a8186d4..bc3a155f46642b 100644 --- a/cli/cmd/validate_config_test.go +++ b/cli/cmd/validate_config_test.go @@ -28,16 +28,15 @@ func TestValidateConfig(t *testing.T) { } _, filename, _, _ := runtime.Caller(0) currentDir := path.Dir(filename) - cqDir := t.TempDir() - defer os.RemoveAll(cqDir) for _, tc := range configs { t.Run(tc.name, func(t *testing.T) { - defer CloseLogFile() - testConfig := path.Join(currentDir, "testdata", tc.config) - logFileName := path.Join(cqDir, "cloudquery.log") cmd := NewCmdRoot() - cmd.SetArgs([]string{"validate-config", testConfig, "--cq-dir", cqDir, "--log-file-name", logFileName}) + testConfig := path.Join(currentDir, "testdata", tc.config) + baseArgs := testCommandArgs(t) + + args := append([]string{"validate-config", testConfig}, baseArgs...) + cmd.SetArgs(args) err := cmd.Execute() if tc.errors != nil { for _, e := range tc.errors { @@ -48,7 +47,7 @@ func TestValidateConfig(t *testing.T) { } // check that log was written and contains some lines from the plugin - b, logFileError := os.ReadFile(path.Join(cqDir, "cloudquery.log")) + b, logFileError := os.ReadFile(baseArgs[3]) logContent := string(b) require.NoError(t, logFileError, "failed to read cloudquery.log") require.NotEmpty(t, logContent, "cloudquery.log empty; expected some logs") From fe808122bca30af87eca5e1ca677f1eb9b1c4612 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 5 Apr 2024 08:26:02 -0500 Subject: [PATCH 09/14] Update doc_test.go --- cli/cmd/doc_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/cmd/doc_test.go b/cli/cmd/doc_test.go index c0a9850b3ea18a..924f64a29f4f7c 100644 --- a/cli/cmd/doc_test.go +++ b/cli/cmd/doc_test.go @@ -19,6 +19,7 @@ var docFiles = []string{ "cloudquery_migrate.md", "cloudquery_tables.md", "cloudquery_test-connection.md", + "cloudquery_validate-config.md", "cloudquery_plugin.md", "cloudquery_plugin_install.md", "cloudquery_plugin_publish.md", From 4867a4b8111fd81e7c0a15d27360c9b47071a834 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 5 Apr 2024 08:33:57 -0500 Subject: [PATCH 10/14] docs --- cli/cmd/validate_config.go | 4 ++-- website/pages/docs/reference/cli/cloudquery.md | 2 +- .../pages/docs/reference/cli/cloudquery_validate-config.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/cmd/validate_config.go b/cli/cmd/validate_config.go index 217ebd468013fc..f0ad54e378e15b 100644 --- a/cli/cmd/validate_config.go +++ b/cli/cmd/validate_config.go @@ -14,8 +14,8 @@ import ( ) const ( - validateConfigShort = "Validate configs" - validateConfigLong = "Validate configs without requiring any credentials or connections. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation." + validateConfigShort = "Validate config" + validateConfigLong = "Validate configuration without requiring any credentials or connections. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation." validateConfigExample = `# Validate configs cloudquery validate-config ./directory # Validate configs from directories and files diff --git a/website/pages/docs/reference/cli/cloudquery.md b/website/pages/docs/reference/cli/cloudquery.md index 2e50035b527b01..4c7bfd87312921 100644 --- a/website/pages/docs/reference/cli/cloudquery.md +++ b/website/pages/docs/reference/cli/cloudquery.md @@ -38,5 +38,5 @@ Find more information at: * [cloudquery sync](/docs/reference/cli/cloudquery_sync) - Sync resources from configured source plugins to destinations * [cloudquery tables](/docs/reference/cli/cloudquery_tables) - Generate documentation for all supported tables of source plugins specified in the spec(s) * [cloudquery test-connection](/docs/reference/cli/cloudquery_test-connection) - Test plugin connections to sources and destinations -* [cloudquery validate-config](/docs/reference/cli/cloudquery_validate-config) - Validate configs +* [cloudquery validate-config](/docs/reference/cli/cloudquery_validate-config) - Validate config diff --git a/website/pages/docs/reference/cli/cloudquery_validate-config.md b/website/pages/docs/reference/cli/cloudquery_validate-config.md index bff07df35fca1b..e0f17d6af16d0e 100644 --- a/website/pages/docs/reference/cli/cloudquery_validate-config.md +++ b/website/pages/docs/reference/cli/cloudquery_validate-config.md @@ -3,11 +3,11 @@ title: "validate-config" --- ## cloudquery validate-config -Validate configs +Validate config ### Synopsis -Validate configs without requiring any credentials or connections. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation. +Validate configuration without requiring any credentials or connections. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation. ``` cloudquery validate-config [files or directories] [flags] From 38c85d8443bffe1f6153bea3c2e4df029563663a Mon Sep 17 00:00:00 2001 From: erezrokah Date: Fri, 5 Apr 2024 15:55:19 +0100 Subject: [PATCH 11/14] test: Use a version that doesn't require auth --- cli/cmd/testdata/validate-config-error.yml | 8 ++++---- cli/cmd/validate_config_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/cmd/testdata/validate-config-error.yml b/cli/cmd/testdata/validate-config-error.yml index 939d4ed4dbc71d..3e03816f190e1e 100644 --- a/cli/cmd/testdata/validate-config-error.yml +++ b/cli/cmd/testdata/validate-config-error.yml @@ -1,12 +1,12 @@ kind: source spec: - name: aws - path: cloudquery/aws + name: cloudflare + path: cloudquery/cloudflare registry: cloudquery - version: "v25.5.0" + version: "v6.1.2" destinations: ["postgresql"] tables: ["aws_ec2_instances"] - spec: + spec: invalid_key: "invalid_value" --- kind: destination diff --git a/cli/cmd/validate_config_test.go b/cli/cmd/validate_config_test.go index bc3a155f46642b..285754099131f9 100644 --- a/cli/cmd/validate_config_test.go +++ b/cli/cmd/validate_config_test.go @@ -23,7 +23,7 @@ func TestValidateConfig(t *testing.T) { { name: "bad AWS and Postgres auth should fail validation", config: "validate-config-error.yml", - errors: []string{"failed to validate source config aws", "failed to validate destination config postgresql"}, + errors: []string{"failed to validate source config cloudflare", "failed to validate destination config postgresql"}, }, } _, filename, _, _ := runtime.Caller(0) From 0697e089a4f9c9b16fa9e92c9d3c67a369ae9931 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Fri, 5 Apr 2024 15:56:41 +0100 Subject: [PATCH 12/14] test: Update tables --- cli/cmd/testdata/validate-config-error.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/cmd/testdata/validate-config-error.yml b/cli/cmd/testdata/validate-config-error.yml index 3e03816f190e1e..04d4321a6d2458 100644 --- a/cli/cmd/testdata/validate-config-error.yml +++ b/cli/cmd/testdata/validate-config-error.yml @@ -5,7 +5,7 @@ spec: registry: cloudquery version: "v6.1.2" destinations: ["postgresql"] - tables: ["aws_ec2_instances"] + tables: ["*"] spec: invalid_key: "invalid_value" --- From 5ebb55685a85979b755d6d0086e5f8bae82bef46 Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 5 Apr 2024 10:11:56 -0500 Subject: [PATCH 13/14] Update switch_test.go --- cli/cmd/switch_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/cmd/switch_test.go b/cli/cmd/switch_test.go index 375ec189205af9..af3bc621acdba8 100644 --- a/cli/cmd/switch_test.go +++ b/cli/cmd/switch_test.go @@ -18,7 +18,8 @@ import ( ) func TestSwitch(t *testing.T) { - configDir := t.TempDir() + baseArgs := testCommandArgs(t) + configDir := baseArgs[1] ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.URL.Path == "/teams": @@ -39,7 +40,6 @@ func TestSwitch(t *testing.T) { err := config.SetConfigHome(configDir) require.NoError(t, err) - baseArgs := testCommandArgs(t) // calling switch before a team is set should not result in an error cmd := NewCmdRoot() cmd.SetArgs(append([]string{"switch"}, baseArgs...)) From d64ff84f1bcdd3d59da189dcf07416ce1eca977d Mon Sep 17 00:00:00 2001 From: bbernays Date: Fri, 5 Apr 2024 10:13:50 -0500 Subject: [PATCH 14/14] add caveat --- cli/cmd/validate_config.go | 2 +- website/pages/docs/reference/cli/cloudquery_validate-config.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/cmd/validate_config.go b/cli/cmd/validate_config.go index f0ad54e378e15b..bbbf427d955156 100644 --- a/cli/cmd/validate_config.go +++ b/cli/cmd/validate_config.go @@ -15,7 +15,7 @@ import ( const ( validateConfigShort = "Validate config" - validateConfigLong = "Validate configuration without requiring any credentials or connections. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation." + validateConfigLong = "Validate configuration without requiring any credentials or connections. This will not validate the tables specified in the tables list. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation." validateConfigExample = `# Validate configs cloudquery validate-config ./directory # Validate configs from directories and files diff --git a/website/pages/docs/reference/cli/cloudquery_validate-config.md b/website/pages/docs/reference/cli/cloudquery_validate-config.md index e0f17d6af16d0e..d1b9c2a48e0271 100644 --- a/website/pages/docs/reference/cli/cloudquery_validate-config.md +++ b/website/pages/docs/reference/cli/cloudquery_validate-config.md @@ -7,7 +7,7 @@ Validate config ### Synopsis -Validate configuration without requiring any credentials or connections. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation. +Validate configuration without requiring any credentials or connections. This will not validate the tables specified in the tables list. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation. ``` cloudquery validate-config [files or directories] [flags]