Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 114 additions & 1 deletion cli/cmd/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cmd

import (
"fmt"
"os"
"path"
"path/filepath"
"strings"
Expand All @@ -14,19 +15,131 @@ const (
docShort = "Generate CLI documentation markdown files"
)

// seeAlsoSections maps generated filename to the "## See Also" content to append.
var seeAlsoSections = map[string]string{
"cloudquery.md": `## See Also

- [Getting Started](/cli/getting-started) - Install and run your first sync
- [Configuration Guide](/cli/core-concepts/configuration) - Configure source and destination integrations
`,
"cloudquery_addon.md": `## See Also

- [Transformations](/cli/core-concepts/transformations) - Official dbt and SQL transformations
- [Dashboards & Visualizations](/cli/core-concepts/dashboards) - Grafana dashboards from the hub
`,
"cloudquery_addon_download.md": `## See Also

- [Transformations](/cli/core-concepts/transformations) - Available transformations and policies
- [Dashboards & Visualizations](/cli/core-concepts/dashboards) - Grafana dashboards from the hub
`,
"cloudquery_addon_publish.md": `## See Also

- [Publishing an Addon](/cli/advanced/publishing-an-addon-to-the-hub) - Full addon publishing guide
- [Transformations](/cli/core-concepts/transformations) - Official transformations and policies
`,
"cloudquery_init.md": `## See Also

- [Getting Started](/cli/getting-started) - Full quickstart guide using the init command
- [Configuration Guide](/cli/core-concepts/configuration) - Understand the generated configuration files
`,
"cloudquery_login.md": `## See Also

- [Generate API Key](/cli/managing-cloudquery/deployments/generate-api-key) - Create API keys for headless authentication
- [Getting Started](/cli/getting-started) - Install and run your first sync
`,
"cloudquery_logout.md": `## See Also

- [Generate API Key](/cli/managing-cloudquery/deployments/generate-api-key) - Manage API keys for authentication
- [Security](/cli/managing-cloudquery/security) - CloudQuery security best practices
`,
"cloudquery_migrate.md": `## See Also

- [Schema Migrations](/cli/managing-cloudquery/migrations) - How CloudQuery handles schema changes
- [Destination Integrations](/cli/integrations/destinations) - Configure migration modes
`,
"cloudquery_plugin.md": `## See Also

- [Integration Concepts](/cli/core-concepts/integrations) - How integrations work
- [Managing Versions](/cli/advanced/managing-versions) - Integration versioning
`,
"cloudquery_plugin_install.md": `## See Also

- [Managing Versions](/cli/advanced/managing-versions) - Understand version management
- [Source Integrations](/cli/integrations/sources) - Available source integrations
`,
"cloudquery_plugin_publish.md": `## See Also

- [Publishing an Integration](/cli/integrations/creating-new-integration/publishing) - Full publishing guide
- [Creating a New Integration](/cli/integrations/creating-new-integration) - Build an integration first
`,
"cloudquery_switch.md": `## See Also

- [Managing Versions](/cli/advanced/managing-versions) - Understand integration versioning
- [Source Integrations](/cli/integrations/sources) - Configure source integration versions
`,
"cloudquery_sync.md": `## See Also

- [Syncs](/cli/core-concepts/syncs) - Understand full and incremental sync modes
- [Configuration Guide](/cli/core-concepts/configuration) - Set up sync configurations
- [Performance Tuning](/cli/advanced/performance-tuning) - Optimize sync performance
`,
"cloudquery_tables.md": `## See Also

- [Source Integrations](/cli/integrations/sources) - Configure which tables to sync
- [Integration Concepts](/cli/core-concepts/integrations) - How source integrations define tables
`,
"cloudquery_test-connection.md": `## See Also

- [Source Integrations](/cli/integrations/sources) - Configure source connections
- [Destination Integrations](/cli/integrations/destinations) - Configure destination connections
- [Troubleshooting](/cli/getting-support/troubleshooting) - Debug connection issues
`,
"cloudquery_validate-config.md": `## See Also

- [Configuration Guide](/cli/core-concepts/configuration) - Configuration format and options
- [Environment Variables](/cli/managing-cloudquery/environment-variables) - Variable substitution in configuration files
`,
}

func newCmdDoc() *cobra.Command {
cmd := &cobra.Command{
Use: "doc [directory_path]",
Short: docShort,
Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
Hidden: true,
RunE: func(cmd *cobra.Command, args []string) error {
return doc.GenMarkdownTreeCustom(cmd.Parent(), args[0], filePrepender, linkHandler)
if err := doc.GenMarkdownTreeCustom(cmd.Parent(), args[0], filePrepender, linkHandler); err != nil {
return err
}
return appendSeeAlsoSections(args[0])
},
}
return cmd
}

// appendSeeAlsoSections appends the "## See Also" section to each generated file that has one defined.
func appendSeeAlsoSections(dir string) error {
for filename, content := range seeAlsoSections {
fpath := filepath.Join(dir, filename)
f, err := os.OpenFile(fpath, os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
if os.IsNotExist(err) {
continue
}
return fmt.Errorf("opening %s: %w", fpath, err)
}
_, writeErr := fmt.Fprintf(f, "\n%s", content)
closeErr := f.Close()
if writeErr != nil {
return fmt.Errorf("writing to %s: %w", fpath, writeErr)
}
if closeErr != nil {
return fmt.Errorf("closing %s: %w", fpath, closeErr)
}
}
return nil
}

func linkHandler(s string) string {
if strings.HasSuffix(s, ".md") {
fileName := strings.TrimSuffix(s, ".md")
Expand Down
25 changes: 23 additions & 2 deletions cli/cmd/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,28 @@ import (
)

const (
initShort = `Generate a configuration file for a sync`
initShort = `Generate a configuration file for a sync`
initLong = `Generate a configuration file for a sync

### Modes

The ` + "`init`" + ` command operates in one of three modes depending on your authentication state and flags:

**AI-assisted mode** (default when logged in)

Activates when you are logged in to a team (` + "`cloudquery login`" + `) and don't specify ` + "`--source`" + ` or ` + "`--destination`" + `. Launches an interactive AI chat session that walks you through the setup process — selecting integrations, generating YAML configuration files, testing connections, and giving you some example queries.

Type ` + "`exit`" + ` or ` + "`quit`" + ` to end the conversation. Use ` + "`--resume-conversation`" + ` to continue a previous session instead of starting a new one.

**Basic interactive mode**

Activates when you pass ` + "`--disable-ai`" + `, or as a fallback if the AI assistant is unavailable. Presents a searchable picker to select source and destination integrations, then generates a configuration file from their default templates.

**Non-interactive mode**

Activates when both ` + "`--source`" + ` and ` + "`--destination`" + ` are specified. Generates the configuration file directly without prompts.

Authentication via ` + "`cloudquery login`" + ` is required for AI-assisted and basic interactive modes.`
initExample = `# Display prompts to select source and destination plugins and generate a configuration file from them
cloudquery init
# Generate a configuration file for a sync from aws to bigquery
Expand All @@ -50,7 +71,7 @@ func newCmdInit() *cobra.Command {
cmd := &cobra.Command{
Use: "init",
Short: initShort,
Long: initShort,
Long: initLong,
Example: initExample,
Args: cobra.ExactArgs(0),
RunE: initCmd,
Expand Down
56 changes: 54 additions & 2 deletions cli/cmd/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,59 @@ import (
)

const (
syncShort = "Sync resources from configured source plugins to destinations"
syncShort = "Sync resources from configured source plugins to destinations"
syncLong = `Sync resources from configured source plugins to destinations

### Flag Details

#### --summary-location

When set, a JSON summary of each sync is appended to the specified file after the sync completes. The file uses JSONL format (one JSON object per line). When syncing to multiple destinations, a separate entry is written for each destination.

Example usage:

` + "```bash\ncloudquery sync config.yml --summary-location summary.jsonl\n```" + `

The summary contains the following fields:

| Field | Type | Description |
| ------------------------ | -------- | ----------------------------------------------------------------- |
| ` + "`cli_version`" + ` | string | CloudQuery CLI version |
| ` + "`sync_id`" + ` | string | Unique sync identifier (from ` + "`--invocation-id`" + ` or auto-generated) |
| ` + "`sync_time`" + ` | string | Sync start time (RFC 3339) |
| ` + "`sync_duration_ms`" + ` | number | Sync duration in milliseconds |
| ` + "`resources`" + ` | number | Total resources synced |
| ` + "`source_name`" + ` | string | Source integration name |
| ` + "`source_path`" + ` | string | Source integration path |
| ` + "`source_version`" + ` | string | Source integration version |
| ` + "`source_errors`" + ` | number | Errors from the source integration |
| ` + "`source_warnings`" + ` | number | Warnings from the source integration |
| ` + "`source_tables`" + ` | string[] | Tables synced from source |
| ` + "`destination_name`" + ` | string | Destination integration name |
| ` + "`destination_path`" + ` | string | Destination integration path |
| ` + "`destination_version`" + ` | string | Destination integration version |
| ` + "`destination_errors`" + ` | number | Errors from the destination integration |
| ` + "`destination_warnings`" + ` | number | Warnings from the destination integration |
| ` + "`sync_group_id`" + ` | string | Rendered sync group ID (if configured) |
| ` + "`shard_num`" + ` | number | Shard number (if using ` + "`--shard`" + `) |
| ` + "`shard_total`" + ` | number | Total shards (if using ` + "`--shard`" + `) |
| ` + "`resources_per_table`" + ` | object | Resource count per table |
| ` + "`errors_per_table`" + ` | object | Error count per table |
| ` + "`durations_per_table_ms`" + ` | object | Duration per table in milliseconds |

#### --invocation-id

A UUID that uniquely identifies a sync invocation. If not provided, a random UUID is automatically generated.

This flag serves three purposes:

1. **OpenTelemetry correlation**: The UUID is attached to all logs and traces, allowing you to correlate CLI activity with integration activity in your [monitoring setup](/cli/managing-cloudquery/monitoring/overview).
2. **Sync summary**: The UUID is stored as the ` + "`sync_id`" + ` field in the [sync summary](#--summary-location).
3. **` + "`sync_group_id`" + ` template**: When a destination's [` + "`sync_group_id`" + `](/cli/integrations/destinations#sync_group_id) uses the ` + "`{{SYNC_ID}}`" + ` placeholder, it is replaced with this UUID at runtime.

Example: using a fixed invocation ID for repeatable tracing:

` + "```bash\ncloudquery sync config.yml --invocation-id 550e8400-e29b-41d4-a716-446655440000\n```"
syncExample = `# Sync resources from configuration in a directory
cloudquery sync ./directory
# Sync resources from directories and files
Expand All @@ -34,7 +86,7 @@ func NewCmdSync() *cobra.Command {
cmd := &cobra.Command{
Use: "sync [files or directories]",
Short: syncShort,
Long: syncShort,
Long: syncLong,
Example: syncExample,
Args: cobra.MinimumNArgs(1),
RunE: sync,
Expand Down
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,8 @@ Find more information at:
* [cloudquery test-connection](/cli/cli-reference/cloudquery_test-connection) - Test plugin connections to sources and/or destinations
* [cloudquery validate-config](/cli/cli-reference/cloudquery_validate-config) - Validate config


## See Also

- [Getting Started](/cli/getting-started) - Install and run your first sync
- [Configuration Guide](/cli/core-concepts/configuration) - Configure source and destination integrations
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery_addon.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,8 @@ Addon commands
* [cloudquery addon download](/cli/cli-reference/cloudquery_addon_download) - Download addon from CloudQuery Hub.
* [cloudquery addon publish](/cli/cli-reference/cloudquery_addon_publish) - Publish to CloudQuery Hub.


## See Also

- [Transformations](/cli/core-concepts/transformations) - Official dbt and SQL transformations
- [Dashboards & Visualizations](/cli/core-concepts/dashboards) - Grafana dashboards from the hub
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery_addon_download.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,8 @@ cloudquery addon download cloudquery/transformation/aws-compliance-premium@v1.9.

* [cloudquery addon](/cli/cli-reference/cloudquery_addon) - Addon commands


## See Also

- [Transformations](/cli/core-concepts/transformations) - Available transformations and policies
- [Dashboards & Visualizations](/cli/core-concepts/dashboards) - Grafana dashboards from the hub
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery_addon_publish.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,8 @@ cloudquery addon publish /path/to/manifest.json v1.0.0

* [cloudquery addon](/cli/cli-reference/cloudquery_addon) - Addon commands


## See Also

- [Publishing an Addon](/cli/advanced/publishing-an-addon-to-the-hub) - Full addon publishing guide
- [Transformations](/cli/core-concepts/transformations) - Official transformations and policies
25 changes: 25 additions & 0 deletions cli/docs/reference/cloudquery_init.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,26 @@ Generate a configuration file for a sync

Generate a configuration file for a sync

### Modes

The `init` command operates in one of three modes depending on your authentication state and flags:

**AI-assisted mode** (default when logged in)

Activates when you are logged in to a team (`cloudquery login`) and don't specify `--source` or `--destination`. Launches an interactive AI chat session that walks you through the setup process — selecting integrations, generating YAML configuration files, testing connections, and giving you some example queries.

Type `exit` or `quit` to end the conversation. Use `--resume-conversation` to continue a previous session instead of starting a new one.

**Basic interactive mode**

Activates when you pass `--disable-ai`, or as a fallback if the AI assistant is unavailable. Presents a searchable picker to select source and destination integrations, then generates a configuration file from their default templates.

**Non-interactive mode**

Activates when both `--source` and `--destination` are specified. Generates the configuration file directly without prompts.

Authentication via `cloudquery login` is required for AI-assisted and basic interactive modes.

```
cloudquery init [flags]
```
Expand Down Expand Up @@ -58,3 +78,8 @@ cloudquery init --yes

* [cloudquery](/cli/cli-reference/cloudquery) - CloudQuery CLI


## See Also

- [Getting Started](/cli/getting-started) - Full quickstart guide using the init command
- [Configuration Guide](/cli/core-concepts/configuration) - Understand the generated configuration files
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery_login.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,8 @@ cloudquery login --team my-team

* [cloudquery](/cli/cli-reference/cloudquery) - CloudQuery CLI


## See Also

- [Generate API Key](/cli/managing-cloudquery/deployments/generate-api-key) - Create API keys for headless authentication
- [Getting Started](/cli/getting-started) - Install and run your first sync
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery_logout.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,8 @@ cloudquery logout [flags]

* [cloudquery](/cli/cli-reference/cloudquery) - CloudQuery CLI


## See Also

- [Generate API Key](/cli/managing-cloudquery/deployments/generate-api-key) - Manage API keys for authentication
- [Security](/cli/managing-cloudquery/security) - CloudQuery security best practices
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery_migrate.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,8 @@ cloudquery migrate ./directory ./aws.yml ./pg.yml

* [cloudquery](/cli/cli-reference/cloudquery) - CloudQuery CLI


## See Also

- [Schema Migrations](/cli/managing-cloudquery/migrations) - How CloudQuery handles schema changes
- [Destination Integrations](/cli/integrations/destinations) - Configure migration modes
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery_plugin.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,8 @@ Plugin commands
* [cloudquery plugin install](/cli/cli-reference/cloudquery_plugin_install) - Install required plugin images from your configuration
* [cloudquery plugin publish](/cli/cli-reference/cloudquery_plugin_publish) - Publish to CloudQuery Hub.


## See Also

- [Integration Concepts](/cli/core-concepts/integrations) - How integrations work
- [Managing Versions](/cli/advanced/managing-versions) - Integration versioning
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery_plugin_install.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,8 @@ cloudquery plugin install ./directory ./aws.yml ./pg.yml

* [cloudquery plugin](/cli/cli-reference/cloudquery_plugin) - Plugin commands


## See Also

- [Managing Versions](/cli/advanced/managing-versions) - Understand version management
- [Source Integrations](/cli/integrations/sources) - Available source integrations
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery_plugin_publish.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,8 @@ cloudquery plugin publish

* [cloudquery plugin](/cli/cli-reference/cloudquery_plugin) - Plugin commands


## See Also

- [Publishing an Integration](/cli/integrations/creating-new-integration/publishing) - Full publishing guide
- [Creating a New Integration](/cli/integrations/creating-new-integration) - Build an integration first
5 changes: 5 additions & 0 deletions cli/docs/reference/cloudquery_switch.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,8 @@ cloudquery switch my-team

* [cloudquery](/cli/cli-reference/cloudquery) - CloudQuery CLI


## See Also

- [Managing Versions](/cli/advanced/managing-versions) - Understand integration versioning
- [Source Integrations](/cli/integrations/sources) - Configure source integration versions
Loading