diff --git a/cli/cmd/doc_test.go b/cli/cmd/doc_test.go index c0a9850b3ea18a..924f64a29f4f7c 100644 --- a/cli/cmd/doc_test.go +++ b/cli/cmd/doc_test.go @@ -19,6 +19,7 @@ var docFiles = []string{ "cloudquery_migrate.md", "cloudquery_tables.md", "cloudquery_test-connection.md", + "cloudquery_validate-config.md", "cloudquery_plugin.md", "cloudquery_plugin_install.md", "cloudquery_plugin_publish.md", diff --git a/cli/cmd/root.go b/cli/cmd/root.go index a080d5e48fddfb..6584d258dc6f42 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -171,6 +171,7 @@ func NewCmdRoot() *cobra.Command { newCmdLogout(), newCmdSwitch(), newCmdTestConnection(), + newCmdValidateConfig(), newCmdPluginInstall(true), // legacy pluginCmd, addonCmd, diff --git a/cli/cmd/switch_test.go b/cli/cmd/switch_test.go index 375ec189205af9..af3bc621acdba8 100644 --- a/cli/cmd/switch_test.go +++ b/cli/cmd/switch_test.go @@ -18,7 +18,8 @@ import ( ) func TestSwitch(t *testing.T) { - configDir := t.TempDir() + baseArgs := testCommandArgs(t) + configDir := baseArgs[1] ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.URL.Path == "/teams": @@ -39,7 +40,6 @@ func TestSwitch(t *testing.T) { err := config.SetConfigHome(configDir) require.NoError(t, err) - baseArgs := testCommandArgs(t) // calling switch before a team is set should not result in an error cmd := NewCmdRoot() cmd.SetArgs(append([]string{"switch"}, baseArgs...)) diff --git a/cli/cmd/testdata/validate-config-error.yml b/cli/cmd/testdata/validate-config-error.yml new file mode 100644 index 00000000000000..04d4321a6d2458 --- /dev/null +++ b/cli/cmd/testdata/validate-config-error.yml @@ -0,0 +1,20 @@ +kind: source +spec: + name: cloudflare + path: cloudquery/cloudflare + registry: cloudquery + version: "v6.1.2" + destinations: ["postgresql"] + tables: ["*"] + spec: + invalid_key: "invalid_value" +--- +kind: destination +spec: + name: "postgresql" + path: "cloudquery/postgresql" + registry: cloudquery + version: "v7.3.5" + spec: + connection_string: "postgresql://postgres:not-a-real-password@localhost:5432/postgres?sslmode=disable" + invalid_key: "invalid_value" diff --git a/cli/cmd/validate_config.go b/cli/cmd/validate_config.go new file mode 100644 index 00000000000000..bbbf427d955156 --- /dev/null +++ b/cli/cmd/validate_config.go @@ -0,0 +1,144 @@ +package cmd + +import ( + "errors" + "fmt" + "strings" + + "github.com/cloudquery/cloudquery/cli/internal/auth" + "github.com/cloudquery/cloudquery/cli/internal/specs/v0" + "github.com/cloudquery/plugin-pb-go/managedplugin" + "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +const ( + validateConfigShort = "Validate config" + validateConfigLong = "Validate configuration without requiring any credentials or connections. This will not validate the tables specified in the tables list. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation." + validateConfigExample = `# Validate configs +cloudquery validate-config ./directory +# Validate configs from directories and files +cloudquery validate-config ./directory ./aws.yml ./pg.yml +` +) + +func newCmdValidateConfig() *cobra.Command { + cmd := &cobra.Command{ + Use: "validate-config [files or directories]", + Short: validateConfigShort, + Long: validateConfigLong, + Example: validateConfigExample, + Args: cobra.MinimumNArgs(1), + RunE: validateConfig, + Hidden: false, + } + + return cmd +} + +func validateConfig(cmd *cobra.Command, args []string) error { + cqDir, err := cmd.Flags().GetString("cq-dir") + if err != nil { + return err + } + + ctx := cmd.Context() + + log.Info().Strs("args", args).Msg("Loading spec(s)") + fmt.Printf("Loading spec(s) from %s\n", strings.Join(args, ", ")) + specReader, err := specs.NewSpecReader(args) + if err != nil { + return fmt.Errorf("failed to load spec(s) from %s. Error: %w", strings.Join(args, ", "), err) + } + sources := specReader.Sources + destinations := specReader.Destinations + + authToken, err := auth.GetAuthTokenIfNeeded(log.Logger, sources, destinations) + if err != nil { + return fmt.Errorf("failed to get auth token: %w", err) + } + teamName, err := auth.GetTeamForToken(authToken) + if err != nil { + return fmt.Errorf("failed to get team name: %w", err) + } + opts := []managedplugin.Option{ + managedplugin.WithLogger(log.Logger), + managedplugin.WithAuthToken(authToken.Value), + managedplugin.WithTeamName(teamName), + } + if cqDir != "" { + opts = append(opts, managedplugin.WithDirectory(cqDir)) + } + if disableSentry { + opts = append(opts, managedplugin.WithNoSentry()) + } + + sourcePluginConfigs := make([]managedplugin.Config, len(sources)) + sourceRegInferred := make([]bool, len(sources)) + for i, source := range sources { + sourcePluginConfigs[i] = managedplugin.Config{ + Name: source.Name, + Version: source.Version, + Path: source.Path, + Registry: SpecRegistryToPlugin(source.Registry), + DockerAuth: source.DockerRegistryAuthToken, + } + sourceRegInferred[i] = source.RegistryInferred() + } + destinationPluginConfigs := make([]managedplugin.Config, len(destinations)) + destinationRegInferred := make([]bool, len(destinations)) + for i, destination := range destinations { + destinationPluginConfigs[i] = managedplugin.Config{ + Name: destination.Name, + Version: destination.Version, + Path: destination.Path, + Registry: SpecRegistryToPlugin(destination.Registry), + DockerAuth: destination.DockerRegistryAuthToken, + } + destinationRegInferred[i] = destination.RegistryInferred() + } + + sourceClients, err := managedplugin.NewClients(ctx, managedplugin.PluginSource, sourcePluginConfigs, opts...) + if err != nil { + return enrichClientError(sourceClients, sourceRegInferred, err) + } + defer func() { + if err := sourceClients.Terminate(); err != nil { + fmt.Println(err) + } + }() + destinationClients, err := managedplugin.NewClients(ctx, managedplugin.PluginDestination, destinationPluginConfigs, opts...) + if err != nil { + return enrichClientError(destinationClients, destinationRegInferred, err) + } + defer func() { + if err := destinationClients.Terminate(); err != nil { + fmt.Println(err) + } + }() + + var initErrors []error + for i, client := range sourceClients { + pluginClient := plugin.NewPluginClient(client.Conn) + log.Info().Str("source", sources[i].VersionString()).Msg("Initializing source") + err := validatePluginSpec(ctx, pluginClient, sources[i].Spec) + if err != nil { + initErrors = append(initErrors, fmt.Errorf("failed to validate source config %v: %w", sources[i].VersionString(), err)) + } else { + log.Info().Str("source", sources[i].VersionString()).Msg("validated successfully") + } + } + for i, client := range destinationClients { + pluginClient := plugin.NewPluginClient(client.Conn) + log.Info().Str("destination", destinations[i].VersionString()).Msg("Initializing destination") + err = validatePluginSpec(ctx, pluginClient, destinations[i].Spec) + if err != nil { + initErrors = append(initErrors, fmt.Errorf("failed to validate destination config %v: %w", destinations[i].VersionString(), err)) + } else { + log.Info().Str("destination", destinations[i].VersionString()).Msg("validated successfully") + } + } + + return errors.Join(initErrors...) +} diff --git a/cli/cmd/validate_config_test.go b/cli/cmd/validate_config_test.go new file mode 100644 index 00000000000000..285754099131f9 --- /dev/null +++ b/cli/cmd/validate_config_test.go @@ -0,0 +1,56 @@ +package cmd + +import ( + "os" + "path" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateConfig(t *testing.T) { + configs := []struct { + name string + config string + errors []string + }{ + { + name: "multiple test sources should pass validation", + config: "multiple-sources.yml", + }, + { + name: "bad AWS and Postgres auth should fail validation", + config: "validate-config-error.yml", + errors: []string{"failed to validate source config cloudflare", "failed to validate destination config postgresql"}, + }, + } + _, filename, _, _ := runtime.Caller(0) + currentDir := path.Dir(filename) + + for _, tc := range configs { + t.Run(tc.name, func(t *testing.T) { + cmd := NewCmdRoot() + testConfig := path.Join(currentDir, "testdata", tc.config) + baseArgs := testCommandArgs(t) + + args := append([]string{"validate-config", testConfig}, baseArgs...) + cmd.SetArgs(args) + err := cmd.Execute() + if tc.errors != nil { + for _, e := range tc.errors { + assert.Contains(t, err.Error(), e) + } + } else { + assert.NoError(t, err) + } + + // check that log was written and contains some lines from the plugin + b, logFileError := os.ReadFile(baseArgs[3]) + logContent := string(b) + require.NoError(t, logFileError, "failed to read cloudquery.log") + require.NotEmpty(t, logContent, "cloudquery.log empty; expected some logs") + }) + } +} diff --git a/website/pages/docs/reference/cli/cloudquery.md b/website/pages/docs/reference/cli/cloudquery.md index 1e578d7fb6eb75..4c7bfd87312921 100644 --- a/website/pages/docs/reference/cli/cloudquery.md +++ b/website/pages/docs/reference/cli/cloudquery.md @@ -38,4 +38,5 @@ Find more information at: * [cloudquery sync](/docs/reference/cli/cloudquery_sync) - Sync resources from configured source plugins to destinations * [cloudquery tables](/docs/reference/cli/cloudquery_tables) - Generate documentation for all supported tables of source plugins specified in the spec(s) * [cloudquery test-connection](/docs/reference/cli/cloudquery_test-connection) - Test plugin connections to sources and destinations +* [cloudquery validate-config](/docs/reference/cli/cloudquery_validate-config) - Validate config diff --git a/website/pages/docs/reference/cli/cloudquery_validate-config.md b/website/pages/docs/reference/cli/cloudquery_validate-config.md new file mode 100644 index 00000000000000..d1b9c2a48e0271 --- /dev/null +++ b/website/pages/docs/reference/cli/cloudquery_validate-config.md @@ -0,0 +1,47 @@ +--- +title: "validate-config" +--- +## cloudquery validate-config + +Validate config + +### Synopsis + +Validate configuration without requiring any credentials or connections. This will not validate the tables specified in the tables list. This validation is stricter than the validation done during `sync`, but if it passes this validation it will pass the sync validation. + +``` +cloudquery validate-config [files or directories] [flags] +``` + +### Examples + +``` +# Validate configs +cloudquery validate-config ./directory +# Validate configs from directories and files +cloudquery validate-config ./directory ./aws.yml ./pg.yml + +``` + +### Options + +``` + -h, --help help for validate-config +``` + +### Options inherited from parent commands + +``` + --cq-dir string directory to store cloudquery files, such as downloaded plugins (default ".cq") + --log-console enable console logging + --log-file-name string Log filename (default "cloudquery.log") + --log-format string Logging format (json, text) (default "text") + --log-level string Logging level (trace, debug, info, warn, error) (default "info") + --no-log-file Disable logging to file + --telemetry-level string Telemetry level (none, errors, stats, all) (default "all") +``` + +### SEE ALSO + +* [cloudquery](/docs/reference/cli/cloudquery) - CloudQuery CLI +