diff --git a/.github/workflows/manual_commands_triggers.yml b/.github/workflows/manual_commands_triggers.yml index 5db8d80a60d7d5..82a8e38b8e5c1d 100644 --- a/.github/workflows/manual_commands_triggers.yml +++ b/.github/workflows/manual_commands_triggers.yml @@ -32,7 +32,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Set up Go 1.x uses: actions/setup-go@v4 diff --git a/.github/workflows/publish_plugin_to_hub.yml b/.github/workflows/publish_plugin_to_hub.yml index 815fb329230e4d..229cb7704bb746 100644 --- a/.github/workflows/publish_plugin_to_hub.yml +++ b/.github/workflows/publish_plugin_to_hub.yml @@ -132,7 +132,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} @@ -206,7 +206,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} @@ -287,7 +287,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} diff --git a/.github/workflows/publish_plugin_to_hub_duckdb.yml b/.github/workflows/publish_plugin_to_hub_duckdb.yml index 333f576bb63d8f..562523c4bb2a02 100644 --- a/.github/workflows/publish_plugin_to_hub_duckdb.yml +++ b/.github/workflows/publish_plugin_to_hub_duckdb.yml @@ -106,7 +106,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} diff --git a/.github/workflows/publish_plugin_to_hub_snowflake.yml b/.github/workflows/publish_plugin_to_hub_snowflake.yml index 485507d7f6201b..d7fe1da922e688 100644 --- a/.github/workflows/publish_plugin_to_hub_snowflake.yml +++ b/.github/workflows/publish_plugin_to_hub_snowflake.yml @@ -107,7 +107,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} diff --git a/.github/workflows/publish_plugin_to_hub_sqlite.yml b/.github/workflows/publish_plugin_to_hub_sqlite.yml index 77c698d2bca8bd..e69341fb556175 100644 --- a/.github/workflows/publish_plugin_to_hub_sqlite.yml +++ b/.github/workflows/publish_plugin_to_hub_sqlite.yml @@ -106,7 +106,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} diff --git a/.github/workflows/source_alicloud.yml b/.github/workflows/source_alicloud.yml index 2d70d24c77bb71..da4c2b3c827f2f 100644 --- a/.github/workflows/source_alicloud.yml +++ b/.github/workflows/source_alicloud.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_aws.yml b/.github/workflows/source_aws.yml index 2b77f1c38ae278..13a95db0f6af5e 100644 --- a/.github/workflows/source_aws.yml +++ b/.github/workflows/source_aws.yml @@ -49,7 +49,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_awspricing.yml b/.github/workflows/source_awspricing.yml index d6e621837e908b..04f5b760e0881e 100644 --- a/.github/workflows/source_awspricing.yml +++ b/.github/workflows/source_awspricing.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_azure.yml b/.github/workflows/source_azure.yml index 0f441e9994dcd7..7fffa58eee4abf 100644 --- a/.github/workflows/source_azure.yml +++ b/.github/workflows/source_azure.yml @@ -47,7 +47,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_cloudflare.yml b/.github/workflows/source_cloudflare.yml index 89c0364bde5db3..18273397146c0a 100644 --- a/.github/workflows/source_cloudflare.yml +++ b/.github/workflows/source_cloudflare.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_datadog.yml b/.github/workflows/source_datadog.yml index 3fc89e8e230255..6a374359b84181 100644 --- a/.github/workflows/source_datadog.yml +++ b/.github/workflows/source_datadog.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_digitalocean.yml b/.github/workflows/source_digitalocean.yml index ce0c81918f46d8..b1aeda8bbb378e 100644 --- a/.github/workflows/source_digitalocean.yml +++ b/.github/workflows/source_digitalocean.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_gcp.yml b/.github/workflows/source_gcp.yml index 6d84bc4b2f0416..15463564bbda83 100644 --- a/.github/workflows/source_gcp.yml +++ b/.github/workflows/source_gcp.yml @@ -47,7 +47,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen @@ -130,7 +130,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Migrate DB run: cloudquery migrate test/policy_cq_config.yml env: diff --git a/.github/workflows/source_github.yml b/.github/workflows/source_github.yml index c212390243511d..049ff8382dfd5f 100644 --- a/.github/workflows/source_github.yml +++ b/.github/workflows/source_github.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_gitlab.yml b/.github/workflows/source_gitlab.yml index a0212ea43ae438..008e7c8e09d991 100644 --- a/.github/workflows/source_gitlab.yml +++ b/.github/workflows/source_gitlab.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_hackernews.yml b/.github/workflows/source_hackernews.yml index 524ff69e036d09..9bef835664accb 100644 --- a/.github/workflows/source_hackernews.yml +++ b/.github/workflows/source_hackernews.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_hubspot.yml b/.github/workflows/source_hubspot.yml index 5a01cdf29d9916..9a66f119ccca60 100644 --- a/.github/workflows/source_hubspot.yml +++ b/.github/workflows/source_hubspot.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_k8s.yml b/.github/workflows/source_k8s.yml index 18072a6bca07ff..0865ee7790e970 100644 --- a/.github/workflows/source_k8s.yml +++ b/.github/workflows/source_k8s.yml @@ -47,7 +47,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen @@ -130,7 +130,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Migrate DB run: cloudquery migrate test/policy_cq_config.yml env: diff --git a/.github/workflows/source_notion.yml b/.github/workflows/source_notion.yml index 96180b92bb8796..e55484ade93043 100644 --- a/.github/workflows/source_notion.yml +++ b/.github/workflows/source_notion.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_okta.yml b/.github/workflows/source_okta.yml index 136ea22450bd3e..e671d376ffb202 100644 --- a/.github/workflows/source_okta.yml +++ b/.github/workflows/source_okta.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_oracle.yml b/.github/workflows/source_oracle.yml index 5fb0c862194ff7..fab774bd21f05f 100644 --- a/.github/workflows/source_oracle.yml +++ b/.github/workflows/source_oracle.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen-docs if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_pagerduty.yml b/.github/workflows/source_pagerduty.yml index a0709b507261fe..f6fbd64a784873 100644 --- a/.github/workflows/source_pagerduty.yml +++ b/.github/workflows/source_pagerduty.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_test.yml b/.github/workflows/source_test.yml index 057767f5bd38e6..48017475028c60 100644 --- a/.github/workflows/source_test.yml +++ b/.github/workflows/source_test.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 01adf1ca368ff3..ad08b9c08a987d 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,5 +1,5 @@ { - "cli": "5.3.1", + "cli": "5.4.0", "cli+FILLER": "0.0.0", "plugins/source/aws": "22.19.2", "plugins/source/aws+FILLER": "0.0.0", @@ -15,7 +15,7 @@ "plugins/source/github+FILLER": "0.0.0", "plugins/source/k8s": "6.0.1", "plugins/source/k8s+FILLER": "0.0.0", - "plugins/source/okta": "3.2.18", + "plugins/source/okta": "4.0.0", "plugins/source/okta+FILLER": "0.0.0", "plugins/source/terraform": "3.0.12", "plugins/source/terraform+FILLER": "0.0.0", @@ -33,7 +33,7 @@ "plugins/source/datadog+FILLER": "0.0.0", "plugins/destination/bigquery": "3.3.16", "plugins/destination/bigquery+FILLER": "0.0.0", - "plugins/source/pagerduty": "3.1.4", + "plugins/source/pagerduty": "3.2.0", "plugins/source/pagerduty+FILLER": "0.0.0", "plugins/destination/mongodb": "2.2.22", "plugins/destination/mongodb+FILLER": "0.0.0", diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 7d98be7a749edc..d33d6074f7ef98 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to CloudQuery will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.4.0](https://github.com/cloudquery/cloudquery/compare/cli-v5.3.1...cli-v5.4.0) (2024-02-08) + + +### Features + +* Validate plugin spec before init ([#16546](https://github.com/cloudquery/cloudquery/issues/16546)) ([081e1b7](https://github.com/cloudquery/cloudquery/commit/081e1b7840153570c9b3dea2a7ab9144188ced77)) + + +### Bug Fixes + +* **deps:** Update module github.com/cloudquery/plugin-pb-go to v1.17.0 ([#16528](https://github.com/cloudquery/cloudquery/issues/16528)) ([c6b0142](https://github.com/cloudquery/cloudquery/commit/c6b01424871865bc3e79ba0548acf63f0009538e)) +* **deps:** Update module github.com/cloudquery/plugin-pb-go to v1.17.1 ([#16539](https://github.com/cloudquery/cloudquery/issues/16539)) ([844434b](https://github.com/cloudquery/cloudquery/commit/844434be7fa6362de2a3c9dd47477fbb69d02c6c)) + ## [5.3.1](https://github.com/cloudquery/cloudquery/compare/cli-v5.3.0...cli-v5.3.1) (2024-02-06) diff --git a/cli/cmd/install.go b/cli/cmd/install.go index 2787df46042ea7..a904005191af0a 100644 --- a/cli/cmd/install.go +++ b/cli/cmd/install.go @@ -118,7 +118,7 @@ func enrichClientError(clientsList managedplugin.Clients, inferredList []bool, e return fmt.Errorf("%w. Hint: use `cloudquery switch` to set a team", err) } - if !strings.Contains(err.Error(), "not found") { + if !strings.Contains(strings.ToLower(err.Error()), "not found") { return err } l := len(clientsList) diff --git a/cli/cmd/migrate_v3.go b/cli/cmd/migrate_v3.go index de3868380a3fda..e303011769c367 100644 --- a/cli/cmd/migrate_v3.go +++ b/cli/cmd/migrate_v3.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "encoding/json" "fmt" "time" @@ -45,28 +44,15 @@ func migrateConnectionV3(ctx context.Context, sourceClient *managedplugin.Client } // initialize destinations first, so that their connections may be used as backends by the source - for i := range destinationsClients { - destSpec := destinationSpecs[i] - destSpecBytes, err := json.Marshal(destSpec.Spec) - if err != nil { - return err - } - if _, err := destinationsPbClients[i].Init(ctx, &plugin.Init_Request{ - Spec: destSpecBytes, - }); err != nil { - return err + for i, destinationSpec := range destinationSpecs { + if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false); err != nil { + return fmt.Errorf("failed to init destination %v: %w", destinationSpec.Name, err) } } - specBytes, err := json.Marshal(sourceSpec.Spec) + err := initPlugin(ctx, sourcePbClient, sourceSpec.Spec, true) if err != nil { - return err - } - if _, err := sourcePbClient.Init(ctx, &plugin.Init_Request{ - Spec: specBytes, - NoConnection: true, - }); err != nil { - return err + return fmt.Errorf("failed to init source %v: %w", sourceSpec.Name, err) } writeClients := make([]plugin.Plugin_WriteClient, len(destinationsPbClients)) diff --git a/cli/cmd/specs.go b/cli/cmd/specs.go index e8ea2b3eaf2dc5..fa10f4329adeda 100644 --- a/cli/cmd/specs.go +++ b/cli/cmd/specs.go @@ -1,10 +1,19 @@ package cmd import ( + "context" + "encoding/json" + "errors" "fmt" + "strings" "github.com/cloudquery/cloudquery/cli/internal/specs/v0" + "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" pbSpecs "github.com/cloudquery/plugin-pb-go/specs" + "github.com/rs/zerolog/log" + "github.com/santhosh-tekuri/jsonschema/v5" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func CLIRegistryToPbRegistry(registry specs.Registry) pbSpecs.Registry { @@ -86,3 +95,84 @@ func CLIDestinationSpecToPbSpec(spec specs.Destination) pbSpecs.Destination { Spec: spec.Spec, } } + +// initPlugin is a simple wrapper that will try to validate the spec before actually passing it to Init. +func initPlugin(ctx context.Context, client plugin.PluginClient, spec any, noConnection bool) error { + if !noConnection { + // perform spec validation + if err := validatePluginSpec(ctx, client, spec); err != nil { + return err + } + } + + specBytes, err := json.Marshal(spec) + if err != nil { + return err + } + + _, err = client.Init(ctx, &plugin.Init_Request{Spec: specBytes, NoConnection: noConnection}) + return err +} + +// validatePluginSpec encompasses spec validation only: +// 1. Get spec schema from the plugin. +// If the call isn't implemented, just skip the validation. +// 2. Validate that the provided JSON schema is valid & can be used for spec validation. +// If the spec is empty (i.e., the plugin didn't supply the schema) just skip. +// 3. If the schema isn't empty but not valid, print the error message & skip the validation. +// 4. Finally, return the validation result. +func validatePluginSpec(ctx context.Context, client plugin.PluginClient, spec any) error { + schema, err := client.GetSpecSchema(ctx, &plugin.GetSpecSchema_Request{}) + if err != nil { + st, ok := status.FromError(err) + if !ok { + // not a gRPC-compatible error + log.Err(err).Msg("failed to get spec schema") + return err + } + if st.Code() != codes.Unimplemented { + // unimplemented is OK, treat as empty schema + log.Err(err).Msg("failed to get spec schema") + return err + } + } + + jsonSchema := schema.GetJsonSchema() + if len(jsonSchema) == 0 { + // This will also be true for Unimplemented response (schema = nil => schema.GetJsonSchema() = "") + log.Info().Msg("empty JSON schema for plugin spec, skipping validation") + return nil + } + + sc, err := parseJSONSchema(jsonSchema) + if err != nil { + log.Err(err).Msg("failed to parse spec schema, skipping validation") + return nil + } + + return sc.Validate(spec) +} + +func parseJSONSchema(jsonSchema string) (*jsonschema.Schema, error) { + c := jsonschema.NewCompiler() + c.Draft = jsonschema.Draft2020 + c.AssertFormat = true + + if err := c.AddResource("schema.json", strings.NewReader(jsonSchema)); err != nil { + return nil, err + } + + sc, err := c.Compile("schema.json") + if err != nil { + var se *jsonschema.SchemaError + if errors.As(err, &se); se != nil && se.Err != nil { + // We add resource as `file`, but there's none, actually. + // So, we need to prettify message a bit. + return nil, fmt.Errorf("jsonschema compilation failed: %w", + errors.New(strings.Replace(se.Err.Error(), "jsonschema: '' ", "", 1))) + } + return nil, err + } + + return sc, nil +} diff --git a/cli/cmd/sync_v3.go b/cli/cmd/sync_v3.go index 10eb650134afb5..8503e696caf34d 100644 --- a/cli/cmd/sync_v3.go +++ b/cli/cmd/sync_v3.go @@ -126,28 +126,14 @@ func syncConnectionV3(ctx context.Context, source v3source, destinations []v3des } // initialize destinations first, so that their connections may be used as backends by the source - for i := range destinationsClients { - destSpec := destinationSpecs[i] - destSpecBytes, err := json.Marshal(destSpec.Spec) - if err != nil { - return err - } - if _, err := destinationsPbClients[i].Init(ctx, &plugin.Init_Request{ - Spec: destSpecBytes, - }); err != nil { - return fmt.Errorf("failed to init destination %v: %w", destSpec.Name, err) + for i, destinationSpec := range destinationSpecs { + if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false); err != nil { + return fmt.Errorf("failed to init destination %v: %w", destinationSpec.Name, err) } } if backend != nil { - backendSpec := backend.spec - backendSpecBytes, err := json.Marshal(backendSpec.Spec) - if err != nil { - return err - } - if _, err := backendPbClient.Init(ctx, &plugin.Init_Request{ - Spec: backendSpecBytes, - }); err != nil { - return fmt.Errorf("failed to init backend %v: %w", backendSpec.Name, err) + if err := initPlugin(ctx, backendPbClient, backend.spec.Spec, false); err != nil { + return fmt.Errorf("failed to init backend %v: %w", backend.spec.Name, err) } } @@ -165,14 +151,8 @@ func syncConnectionV3(ctx context.Context, source v3source, destinations []v3des return fmt.Errorf("failed to unmarshal source spec JSON after variable replacement: %w", err) } - sourceSpecBytes, err := json.Marshal(sourceSpec.Spec) - if err != nil { - return err - } - if _, err := sourcePbClient.Init(ctx, &plugin.Init_Request{ - Spec: sourceSpecBytes, - }); err != nil { - return err + if err = initPlugin(ctx, sourcePbClient, sourceSpec.Spec, false); err != nil { + return fmt.Errorf("failed to init source %v: %w", sourceSpec.Name, err) } writeClients := make([]plugin.Plugin_WriteClient, len(destinationsPbClients)) diff --git a/cli/cmd/tables_v3.go b/cli/cmd/tables_v3.go index 4b1adc257fbdba..3a05217aca1584 100644 --- a/cli/cmd/tables_v3.go +++ b/cli/cmd/tables_v3.go @@ -20,9 +20,7 @@ func tablesV3(ctx context.Context, sourceClient *managedplugin.Client, path stri return err } sourcePbClient := pluginPb.NewPluginClient(sourceClient.Conn) - if _, err := sourcePbClient.Init(ctx, &pluginPb.Init_Request{ - NoConnection: true, - }); err != nil { + if err := initPlugin(ctx, sourcePbClient, nil, true); err != nil { return fmt.Errorf("failed to init source: %w", err) } getTablesResp, err := sourcePbClient.GetTables(ctx, &pluginPb.GetTables_Request{ diff --git a/cli/go.mod b/cli/go.mod index 88a1a61b69bffa..9ff8099364d154 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -1,6 +1,6 @@ module github.com/cloudquery/cloudquery/cli -go 1.21.4 +go 1.21.5 require ( github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1 @@ -8,7 +8,7 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/cloudquery/cloudquery-api-go v1.7.3 github.com/cloudquery/codegen v0.3.12 - github.com/cloudquery/plugin-pb-go v1.16.8 + github.com/cloudquery/plugin-pb-go v1.17.1 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/distribution/reference v0.5.0 github.com/docker/distribution v2.8.3+incompatible @@ -21,6 +21,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/rs/zerolog v1.31.0 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/schollz/progressbar/v3 v3.13.1 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.4 @@ -116,7 +117,6 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/cli/go.sum b/cli/go.sum index 06335f0592b749..c31cc2fe425566 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -68,8 +68,8 @@ github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1 github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8= github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb h1:/l8fbvLOCNlgkHp8VUKTTL+Tk9gs5y/K3Yx/bRfReNk= github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= -github.com/cloudquery/plugin-pb-go v1.16.8 h1:VKSii3yuDi7XIVNIkgrqMdd2tAWPX1O/V/HtZp9Y9Ak= -github.com/cloudquery/plugin-pb-go v1.16.8/go.mod h1:taeDcOU7nkiPerOmsUwwYzBoZpQpRNe6nyLWegi67Os= +github.com/cloudquery/plugin-pb-go v1.17.1 h1:nOvPjfP8eUxupe8OCCIClZovxpt0dNiIZcWNUR9Nn5c= +github.com/cloudquery/plugin-pb-go v1.17.1/go.mod h1:IHfPXOgPgziUrfKw4tN0+DVYTABM3UaZIWrzXPBjQG0= github.com/cloudquery/plugin-sdk/v4 v4.29.1 h1:vBDrg/e6Zv5lR5DsKETBbVcjS0Q70lRTeUqCM5TzVrk= github.com/cloudquery/plugin-sdk/v4 v4.29.1/go.mod h1:pM5EnbRX7apbgg1e5JC9TZkcJ22eyf4b4Uo1ekDb/GQ= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= diff --git a/plugins/source/aws/client/data/partition_service_region.json b/plugins/source/aws/client/data/partition_service_region.json index fb9a931dd3e0ae..262ded39b9376d 100644 --- a/plugins/source/aws/client/data/partition_service_region.json +++ b/plugins/source/aws/client/data/partition_service_region.json @@ -5094,27 +5094,19 @@ }, "resource-explorer-2": { "regions": { - "af-south-1": {}, - "ap-east-1": {}, "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-northeast-3": {}, "ap-south-1": {}, - "ap-south-2": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ap-southeast-3": {}, - "ap-southeast-4": {}, "ca-central-1": {}, "eu-central-1": {}, - "eu-central-2": {}, "eu-north-1": {}, - "eu-south-1": {}, "eu-west-1": {}, "eu-west-2": {}, "eu-west-3": {}, - "il-central-1": {}, - "me-central-1": {}, "me-south-1": {}, "sa-east-1": {}, "us-east-1": {}, @@ -7137,6 +7129,12 @@ "cn-northwest-1": {} } }, + "inspector2": { + "regions": { + "cn-north-1": {}, + "cn-northwest-1": {} + } + }, "internetmonitor": { "regions": { "cn-north-1": {}, diff --git a/plugins/source/aws/docs/tables/aws_ssm_parameters.md b/plugins/source/aws/docs/tables/aws_ssm_parameters.md index 311dacd2b23b1b..3a81e5b3d6adbc 100644 --- a/plugins/source/aws/docs/tables/aws_ssm_parameters.md +++ b/plugins/source/aws/docs/tables/aws_ssm_parameters.md @@ -16,6 +16,7 @@ The following fields are used to calculate the value of `_cq_id`: (**account_id* |account_id|`utf8`| |region|`utf8`| |name|`utf8`| +|tags|`json`| |allowed_pattern|`utf8`| |data_type|`utf8`| |description|`utf8`| diff --git a/plugins/source/aws/resources/services/ssm/parameters.go b/plugins/source/aws/resources/services/ssm/parameters.go index 1f377ca79114c6..8628dac7e5ba8b 100644 --- a/plugins/source/aws/resources/services/ssm/parameters.go +++ b/plugins/source/aws/resources/services/ssm/parameters.go @@ -9,6 +9,7 @@ import ( "github.com/cloudquery/cloudquery/plugins/source/aws/client" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/transformers" + sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" ) func Parameters() *schema.Table { @@ -28,6 +29,11 @@ func Parameters() *schema.Table { Description: `The parameter name`, PrimaryKeyComponent: true, }, + { + Name: "tags", + Type: sdkTypes.ExtensionTypes.JSON, + Resolver: resolveParameterTags, + }, }, } } @@ -47,3 +53,20 @@ func fetchSsmParameters(ctx context.Context, meta schema.ClientMeta, parent *sch } return nil } +func resolveParameterTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + cl := meta.(*client.Client) + svc := cl.Services(client.AWSServiceSsm).Ssm + pm := resource.Item.(types.ParameterMetadata) + resp, err := svc.ListTagsForResource(ctx, &ssm.ListTagsForResourceInput{ + ResourceId: pm.Name, + ResourceType: types.ResourceTypeForTaggingParameter, + }, func(options *ssm.Options) { + options.Region = cl.Region + }) + if err != nil { + return err + } + tags := make(map[string]string) + client.TagsIntoMap(resp.TagList, tags) + return resource.Set(c.Name, tags) +} diff --git a/plugins/source/aws/resources/services/ssm/parameters_mock_test.go b/plugins/source/aws/resources/services/ssm/parameters_mock_test.go index 348f05b4759265..0de83450fe9d5e 100644 --- a/plugins/source/aws/resources/services/ssm/parameters_mock_test.go +++ b/plugins/source/aws/resources/services/ssm/parameters_mock_test.go @@ -17,6 +17,8 @@ func buildParameters(t *testing.T, ctrl *gomock.Controller) client.Services { var pm types.ParameterMetadata require.NoError(t, faker.FakeObject(&pm)) + var tags []types.Tag + require.NoError(t, faker.FakeObject(&tags)) mock.EXPECT().DescribeParameters( gomock.Any(), &ssm.DescribeParametersInput{}, @@ -25,6 +27,18 @@ func buildParameters(t *testing.T, ctrl *gomock.Controller) client.Services { &ssm.DescribeParametersOutput{Parameters: []types.ParameterMetadata{pm}}, nil, ) + + mock.EXPECT().ListTagsForResource( + gomock.Any(), + &ssm.ListTagsForResourceInput{ + ResourceId: pm.Name, + ResourceType: types.ResourceTypeForTaggingParameter, + }, + gomock.Any(), + ).Return( + &ssm.ListTagsForResourceOutput{TagList: tags}, + nil, + ) return client.Services{Ssm: mock} } diff --git a/plugins/source/gcp/client/client.go b/plugins/source/gcp/client/client.go index 072e70c3549de4..84dec4ee1b0856 100644 --- a/plugins/source/gcp/client/client.go +++ b/plugins/source/gcp/client/client.go @@ -56,6 +56,7 @@ type Client struct { Backend state.Client TestingGRPCEndpoint *string TestingHTTPEndpoint *string + Spec *spec.Spec } func (c *Client) WithBackend(backend state.Client) *Client { @@ -130,6 +131,7 @@ func New(ctx context.Context, logger zerolog.Logger, s *spec.Spec) (schema.Clien c := Client{ logger: logger, EnabledServices: map[string]map[string]any{}, + Spec: s, } projects := s.ProjectIDs @@ -232,7 +234,10 @@ func New(ctx context.Context, logger zerolog.Logger, s *spec.Spec) (schema.Clien projects = setUnion(projects, projectsWithFilter) } - if len(s.OrganizationIDs) == 0 && len(s.OrganizationFilter) == 0 { + switch { + case s.SkipOrganizationResources: + c.logger.Info().Msg("Skipping organization resources") + case len(s.OrganizationIDs) == 0 && len(s.OrganizationFilter) == 0: c.logger.Info().Msg("No organization_ids or organization_filter specified - assuming all organizations") c.logger.Info().Msg("Listing organizations...") @@ -240,7 +245,7 @@ func New(ctx context.Context, logger zerolog.Logger, s *spec.Spec) (schema.Clien if err != nil { c.logger.Err(err).Msg("failed to get organizations") } - } else { + default: if len(s.OrganizationIDs) > 0 { for _, orgID := range s.OrganizationIDs { c.logger.Info().Msgf("Getting spec organization %q...", orgID) diff --git a/plugins/source/gcp/client/spec/schema.json b/plugins/source/gcp/client/spec/schema.json index 198b178cd4a3d5..31a990e938f630 100644 --- a/plugins/source/gcp/client/spec/schema.json +++ b/plugins/source/gcp/client/spec/schema.json @@ -191,6 +191,11 @@ "type": "null" } ] + }, + "skip_organization_resources": { + "type": "boolean", + "description": "If enabled CloudQuery will skip syncing organization level resources", + "default": false } }, "additionalProperties": false, diff --git a/plugins/source/gcp/client/spec/spec.go b/plugins/source/gcp/client/spec/spec.go index d6e84b5be9f903..1f20a717b80c12 100644 --- a/plugins/source/gcp/client/spec/spec.go +++ b/plugins/source/gcp/client/spec/spec.go @@ -105,6 +105,9 @@ type Spec struct { // Service Account impersonation configuration. ServiceAccountImpersonation *CredentialsConfig `json:"service_account_impersonation"` + + // If enabled CloudQuery will skip syncing organization level resources + SkipOrganizationResources bool `json:"skip_organization_resources" jsonschema:"default=false"` } func (s *Spec) Validate() error { diff --git a/plugins/source/gcp/resources/plugin/client.go b/plugins/source/gcp/resources/plugin/client.go index b723a736cbb961..c16d2b3efbe523 100644 --- a/plugins/source/gcp/resources/plugin/client.go +++ b/plugins/source/gcp/resources/plugin/client.go @@ -4,6 +4,9 @@ import ( "context" "encoding/json" "fmt" + "reflect" + "runtime" + "strings" "github.com/cloudquery/cloudquery/plugins/source/gcp/client" "github.com/cloudquery/cloudquery/plugins/source/gcp/client/spec" @@ -69,6 +72,19 @@ func (c *Client) Sync(ctx context.Context, options plugin.SyncOptions, res chan< if err != nil { return err } + var skippedOrgTables []string + for _, table := range tables { + if c.syncClient.Spec.SkipOrganizationResources && table.Multiplex != nil { + // get the name of the multiplexer via reflection + multiplexerName := runtime.FuncForPC(reflect.ValueOf(table.Multiplex).Pointer()).Name() + if strings.Contains(multiplexerName, "plugins/source/gcp/client.OrgMultiplex") { + skippedOrgTables = append(skippedOrgTables, table.Name) + } + } + } + if len(skippedOrgTables) > 0 { + return fmt.Errorf("invalid spec: organizational level tables (%s) included while `SkipOrganizationResources` is enabled", strings.Join(skippedOrgTables, ", ")) + } stateClient := state.Client(&state.NoOpClient{}) if options.BackendOptions != nil { conn, err := grpc.DialContext(ctx, options.BackendOptions.Connection, diff --git a/plugins/source/notion/Makefile b/plugins/source/notion/Makefile index c9205d7966a451..410e9342c54ae8 100644 --- a/plugins/source/notion/Makefile +++ b/plugins/source/notion/Makefile @@ -21,6 +21,10 @@ gen-docs: build cloudquery tables --format markdown --output-dir docs test/config.yml mv docs/$(shell basename $(CURDIR)) docs/tables +.PHONY: gen-spec-schema +gen-spec-schema: + go run client/spec/gen/main.go + # All gen targets .PHONY: gen -gen: gen-docs +gen: gen-spec-schema gen-docs diff --git a/plugins/source/notion/client/schema.json b/plugins/source/notion/client/schema.json new file mode 100644 index 00000000000000..c90df04ddf7c11 --- /dev/null +++ b/plugins/source/notion/client/schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/cloudquery/plugins/source/notion/client/spec", + "$ref": "#/$defs/Spec", + "$defs": { + "Spec": { + "properties": { + "bearer_token": { + "type": "string", + "minLength": 1 + }, + "notion_version": { + "type": "string", + "minLength": 1, + "default": "2022-02-22" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "bearer_token" + ] + } + } +} diff --git a/plugins/source/notion/client/spec.go b/plugins/source/notion/client/spec.go index 30749cc9dab107..7930fd23408411 100644 --- a/plugins/source/notion/client/spec.go +++ b/plugins/source/notion/client/spec.go @@ -1,10 +1,15 @@ package client +import _ "embed" + type Spec struct { - BearerToken string `json:"bearer_token,omitempty"` - NotionVersion string `json:"notion_version,omitempty"` + BearerToken string `json:"bearer_token" jsonschema:"required,minLength=1"` + NotionVersion string `json:"notion_version,omitempty" jsonschema:"minLength=1,default=2022-02-22"` } +//go:embed schema.json +var JSONSchema string + func (s *Spec) SetDefaults() { if len(s.NotionVersion) < 1 { s.NotionVersion = "2022-02-22" diff --git a/plugins/source/notion/client/spec/gen/main.go b/plugins/source/notion/client/spec/gen/main.go new file mode 100644 index 00000000000000..a7cd21c217703c --- /dev/null +++ b/plugins/source/notion/client/spec/gen/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "log" + "path" + "runtime" + + "github.com/cloudquery/cloudquery/plugins/source/notion/client" + cqjsonschema "github.com/cloudquery/codegen/jsonschema" +) + +func main() { + fmt.Println("Generating JSON schema for plugin spec") + cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"), + cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/source/notion/client", path.Join(currDir(), "../..")), + ) +} + +func currDir() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Failed to get caller information") + } + return path.Dir(filename) +} diff --git a/plugins/source/notion/client/spec_test.go b/plugins/source/notion/client/spec_test.go new file mode 100644 index 00000000000000..25bbcf747c105e --- /dev/null +++ b/plugins/source/notion/client/spec_test.go @@ -0,0 +1,40 @@ +package client + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" +) + +func TestJSONSchema(t *testing.T) { + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty spec", + Spec: `{}`, + Err: true, + }, + { + Name: "spec with bearer_token", + Spec: `{"bearer_token": "tok"}`, + }, + { + Name: "spec with bearer_token and empty notion_version", + Spec: `{"bearer_token": "tok", "notion_version": ""}`, + Err: true, + }, + { + Name: "spec with bearer_token and valid notion_version", + Spec: `{"bearer_token": "tok", "notion_version": "2021-01-01"}`, + }, + { + Name: "spec with bearer_token and bool notion_version", + Spec: `{"bearer_token": "tok", "notion_version": true}`, + Err: true, + }, + { + Name: "spec with unknown field", + Spec: `{"bearer_token": "tok", "unknown": "test"}`, + Err: true, + }, + }) +} diff --git a/plugins/source/notion/go.mod b/plugins/source/notion/go.mod index 70487265231397..682990d7869466 100644 --- a/plugins/source/notion/go.mod +++ b/plugins/source/notion/go.mod @@ -3,6 +3,7 @@ module github.com/cloudquery/cloudquery/plugins/source/notion go 1.21.4 require ( + github.com/cloudquery/codegen v0.3.12 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/rs/zerolog v1.31.0 ) @@ -122,3 +123,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// github.com/cloudquery/jsonschema @ cqmain +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb diff --git a/plugins/source/notion/go.sum b/plugins/source/notion/go.sum index f5d263952cffa3..70cdea347b94a7 100644 --- a/plugins/source/notion/go.sum +++ b/plugins/source/notion/go.sum @@ -47,6 +47,10 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/cloudquery/cloudquery-api-go v1.7.2 h1:jpQfeZUxekbV7ASN5ONpGIkrtKIZvC/Y8fOj+tQxLm4= github.com/cloudquery/cloudquery-api-go v1.7.2/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA= +github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE= +github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8= +github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb h1:/l8fbvLOCNlgkHp8VUKTTL+Tk9gs5y/K3Yx/bRfReNk= +github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= github.com/cloudquery/plugin-pb-go v1.16.7 h1:wLx5TFvS6gAvD1dcBZdv5YSskcNCnNpF1JNituka5jM= github.com/cloudquery/plugin-pb-go v1.16.7/go.mod h1:Sd08P8HIjwi3gmfoE0X21qo6HL1NiVdNl/00JrK+DkM= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= @@ -128,8 +132,6 @@ github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/C github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= -github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go= github.com/iris-contrib/httpexpect/v2 v2.15.2/go.mod h1:JLDgIqnFy5loDSUv1OA2j0mb6p/rDhiCqigP22Uq9xE= github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= diff --git a/plugins/source/notion/resources/plugin/plugin.go b/plugins/source/notion/resources/plugin/plugin.go index fc7eb3ffecf651..868230c4255765 100644 --- a/plugins/source/notion/resources/plugin/plugin.go +++ b/plugins/source/notion/resources/plugin/plugin.go @@ -1,6 +1,7 @@ package plugin import ( + "github.com/cloudquery/cloudquery/plugins/source/notion/client" "github.com/cloudquery/plugin-sdk/v4/plugin" ) @@ -18,5 +19,6 @@ func Plugin() *plugin.Plugin { Configure, plugin.WithKind(Kind), plugin.WithTeam(Team), + plugin.WithJSONSchema(client.JSONSchema), ) } diff --git a/plugins/source/okta/CHANGELOG.md b/plugins/source/okta/CHANGELOG.md index 8af588f2798877..3d5f4ffdd0e420 100644 --- a/plugins/source/okta/CHANGELOG.md +++ b/plugins/source/okta/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [4.0.0](https://github.com/cloudquery/cloudquery/compare/plugins-source-okta-v3.2.18...plugins-source-okta-v4.0.0) (2024-02-06) + + +### ⚠ BREAKING CHANGES + +* Remove support for deprecated `OKTA_API_TOKEN` environment variable ([#16498](https://github.com/cloudquery/cloudquery/issues/16498)) + +### Features + +* Add JSON Schema to Okta source plugin ([#16493](https://github.com/cloudquery/cloudquery/issues/16493)) ([75ce0ab](https://github.com/cloudquery/cloudquery/commit/75ce0abd90080dd537342dba17b03a60bc4c2b8a)) + + +### Bug Fixes + +* **deps:** Update golang.org/x/exp digest to 1b97071 ([#16419](https://github.com/cloudquery/cloudquery/issues/16419)) ([6d77cd1](https://github.com/cloudquery/cloudquery/commit/6d77cd19b6fc648a4ddb12025c22127e960036a4)) +* **deps:** Update google.golang.org/genproto/googleapis/api digest to 1f4bbc5 ([#16421](https://github.com/cloudquery/cloudquery/issues/16421)) ([9489931](https://github.com/cloudquery/cloudquery/commit/9489931c1b64bf1f7d5da51997944ee54370215b)) +* **deps:** Update google.golang.org/genproto/googleapis/rpc digest to 1f4bbc5 ([#16422](https://github.com/cloudquery/cloudquery/issues/16422)) ([74e98fc](https://github.com/cloudquery/cloudquery/commit/74e98fcbde6c6e11baf98284aef0341c597d4817)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.0 ([#16237](https://github.com/cloudquery/cloudquery/issues/16237)) ([3fcdab0](https://github.com/cloudquery/cloudquery/commit/3fcdab08816ad9de7bb4eecab59c7be1bda3d00c)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.1 ([#16296](https://github.com/cloudquery/cloudquery/issues/16296)) ([ab4a0da](https://github.com/cloudquery/cloudquery/commit/ab4a0dace0a870755fd22d92c6e9c999351f594e)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.2 ([#16342](https://github.com/cloudquery/cloudquery/issues/16342)) ([f3eb857](https://github.com/cloudquery/cloudquery/commit/f3eb85729e5db16c2530b31d6d276934866d5ef0)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.28.0 ([#16362](https://github.com/cloudquery/cloudquery/issues/16362)) ([9166b6b](https://github.com/cloudquery/cloudquery/commit/9166b6b603d0d56a30c2e5072c4f2da5c0c837b5)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.0 ([#16395](https://github.com/cloudquery/cloudquery/issues/16395)) ([fb1102e](https://github.com/cloudquery/cloudquery/commit/fb1102eac8af4b3722b82b882187fdf322546513)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.1 ([#16430](https://github.com/cloudquery/cloudquery/issues/16430)) ([738e89f](https://github.com/cloudquery/cloudquery/commit/738e89f2c969a8a3f1698a8686aeaddb358e7a23)) +* Remove support for deprecated `OKTA_API_TOKEN` environment variable ([#16498](https://github.com/cloudquery/cloudquery/issues/16498)) ([25303df](https://github.com/cloudquery/cloudquery/commit/25303df6f8aeda463c35043c17abe37e6611bc68)) + ## [3.2.18](https://github.com/cloudquery/cloudquery/compare/plugins-source-okta-v3.2.17...plugins-source-okta-v3.2.18) (2024-01-16) diff --git a/plugins/source/pagerduty/CHANGELOG.md b/plugins/source/pagerduty/CHANGELOG.md index 29bea2cb93c7da..0edb71aad8dbb1 100644 --- a/plugins/source/pagerduty/CHANGELOG.md +++ b/plugins/source/pagerduty/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [3.2.0](https://github.com/cloudquery/cloudquery/compare/plugins-source-pagerduty-v3.1.4...plugins-source-pagerduty-v3.2.0) (2024-02-06) + + +### Features + +* Add JSON schema for pagerduty source plugin ([#16494](https://github.com/cloudquery/cloudquery/issues/16494)) ([e6716aa](https://github.com/cloudquery/cloudquery/commit/e6716aa3f2445959f76344f8f8c48027fe7ad45e)) + + +### Bug Fixes + +* **deps:** Update github.com/cloudquery/jsonschema digest to d771afd ([#16511](https://github.com/cloudquery/cloudquery/issues/16511)) ([d374ba8](https://github.com/cloudquery/cloudquery/commit/d374ba88fcc339e992ed9cd298e301acf0fff108)) +* **deps:** Update golang.org/x/exp digest to 1b97071 ([#16419](https://github.com/cloudquery/cloudquery/issues/16419)) ([6d77cd1](https://github.com/cloudquery/cloudquery/commit/6d77cd19b6fc648a4ddb12025c22127e960036a4)) +* **deps:** Update google.golang.org/genproto/googleapis/api digest to 1f4bbc5 ([#16421](https://github.com/cloudquery/cloudquery/issues/16421)) ([9489931](https://github.com/cloudquery/cloudquery/commit/9489931c1b64bf1f7d5da51997944ee54370215b)) +* **deps:** Update google.golang.org/genproto/googleapis/rpc digest to 1f4bbc5 ([#16422](https://github.com/cloudquery/cloudquery/issues/16422)) ([74e98fc](https://github.com/cloudquery/cloudquery/commit/74e98fcbde6c6e11baf98284aef0341c597d4817)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.0 ([#16237](https://github.com/cloudquery/cloudquery/issues/16237)) ([3fcdab0](https://github.com/cloudquery/cloudquery/commit/3fcdab08816ad9de7bb4eecab59c7be1bda3d00c)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.1 ([#16296](https://github.com/cloudquery/cloudquery/issues/16296)) ([ab4a0da](https://github.com/cloudquery/cloudquery/commit/ab4a0dace0a870755fd22d92c6e9c999351f594e)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.2 ([#16342](https://github.com/cloudquery/cloudquery/issues/16342)) ([f3eb857](https://github.com/cloudquery/cloudquery/commit/f3eb85729e5db16c2530b31d6d276934866d5ef0)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.28.0 ([#16362](https://github.com/cloudquery/cloudquery/issues/16362)) ([9166b6b](https://github.com/cloudquery/cloudquery/commit/9166b6b603d0d56a30c2e5072c4f2da5c0c837b5)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.0 ([#16395](https://github.com/cloudquery/cloudquery/issues/16395)) ([fb1102e](https://github.com/cloudquery/cloudquery/commit/fb1102eac8af4b3722b82b882187fdf322546513)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.1 ([#16430](https://github.com/cloudquery/cloudquery/issues/16430)) ([738e89f](https://github.com/cloudquery/cloudquery/commit/738e89f2c969a8a3f1698a8686aeaddb358e7a23)) + ## [3.1.4](https://github.com/cloudquery/cloudquery/compare/plugins-source-pagerduty-v3.1.3...plugins-source-pagerduty-v3.1.4) (2024-01-16) diff --git a/scaffold/cmd/templates/source/go.mod.tpl b/scaffold/cmd/templates/source/go.mod.tpl index 0d2be8ce862604..e84c72175af244 100644 --- a/scaffold/cmd/templates/source/go.mod.tpl +++ b/scaffold/cmd/templates/source/go.mod.tpl @@ -4,7 +4,7 @@ go 1.21.4 require ( github.com/apache/arrow/go/v15 bcaeaa8c2d97 - github.com/cloudquery/plugin-pb-go v1.16.8 + github.com/cloudquery/plugin-pb-go v1.17.1 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/rs/zerolog v1.29.0 ) diff --git a/website/pages/how-to-guides/open-source-cspm.mdx b/website/pages/how-to-guides/open-source-cspm.mdx index dc2ab1cd9d036e..d13079061f7fbf 100644 --- a/website/pages/how-to-guides/open-source-cspm.mdx +++ b/website/pages/how-to-guides/open-source-cspm.mdx @@ -17,48 +17,137 @@ CSPMs are probably the biggest offenders of yet-another-dashboard syndrome, and - **ETL (Extract-Transform-Load) ingestion layer:** [CloudQuery](https://github.com/cloudquery/cloudquery) - **Datastore:** PostgreSQL -- **Policies:** Standard [SQL Policies](https://hub.cloudquery.io/addons/transformation) to be executed via `psql` +- **Transformations (policies):** dbt [Transformations](https://hub.cloudquery.io/addons/transformation) to be executed via `dbt` - **Data Visualization and Exploration Platform:** Grafana ## What you will get -- **Raw SQL access** to all your cloud asset inventory, open source SQL based policies. +- **Raw SQL access** to all your cloud asset inventory. - **Multi-Cloud Asset Inventory:** Ingest configuration from all your clouds to a single datastore with a unified structure. - **Avoid yet-another-dashboard fatigue:** Reuse your existing BI/Visualization stack (Grafana in this example) to build an open source CSPM. -### Step 1: **Install or Deploy CloudQuery** +## Building the CSPM step by step + +### Step 1: Install or deploy CloudQuery If it’s your first time using CloudQuery we suggest you first run it locally to get familiar with the tool. Take a look at our [quick start guide](/docs/quickstart). -If you are already familiar with CloudQuery, take a look at how to deploy it to AWS on RDS Aurora and EKS at [github.com/cludquery/terraform-aws-cloudquery](https://github.com/cloudquery/terraform-aws-cloudquery) , or GCP and Cloud SQL at [https://github.com/cloudquery/terraform-gcp-cloudquery](https://github.com/cloudquery/terraform-gcp-cloudquery) +If you are already familiar with CloudQuery, take a look at how to deploy it with [Kubernetes](/docs/deployment/kubernetes), on [AWS ECS using Fargate](/docs/deployment/ecs) or with [Google Cloud Run](/docs/deployment/cloud-run). For more deployment guides, see our [Deployment Overview](/docs/deployment/overview). -### Step 2: **Install Grafana** +### Step 2: Install Grafana Grafana is a well-known open source observability and visualization tool. It is open source, so there are a number of ways to deploy it: - **Self-hosted (local, docker, k8s):** [Official guide.](https://grafana.com/docs/grafana/latest/setup-grafana/installation/) - **SaaS/managed:** [Grafana.com](https://grafana.com/) -- AWS Managed Grafana: [https://aws.amazon.com/grafana/](https://aws.amazon.com/grafana/) +- **AWS Managed Grafana**: [https://aws.amazon.com/grafana/](https://aws.amazon.com/grafana/) + +### Step 3: Install dbt + +CloudQuery policies and rules are implemented using [dbt](https://docs.getdbt.com/docs/core/installation-overview), a popular open source tool for data transformation. + +After the installation, you will need to provide dbt with a profile which defines how to connect to relevant databases. By default, it searches for a `profiles.yml` file in the local directory and falls back to `~/.dbt/`. Read more about profiles in the [dbt documentation](https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles). -### Step 3: Run Policies (CSPM - Cloud Security Posture Management) +Your profiles.yml file should look like this: -CloudQuery policies and rules are implemented in pure SQL and they store results in a single table that you can easily query and visualize. Here is a [link](https://hub.cloudquery.io/addons/transformation) to all available policies and compliance frameworks. In this section we will go quickly through how to run multiple benchmarks for AWS. +```yaml +config: + send_anonymous_usage_stats: False + use_colors: True + +aws_compliance: # this should match the profile name in your dbt_project.yml, see step 5. + target: postgres + outputs: + postgres: + type: postgres + host: "your postgres host" + user: "postgres user name" + pass: "postgres password" + port: 5432 + dbname: "database name" + schema: public + threads: 4 -```bash copy -git clone https://github.com/cloudquery/cloudquery.git -cd cloudquery/plugins/source/aws/policies -# change the DSN to your PostgreSQL instance populated by CloudQuery -psql postgres://postgres:pass@localhost:5432/postgres -f policy.sql ``` -This should run all [available](https://hub.cloudquery.io/plugins/source/cloudquery/aws) compliance framework and store the results in [aws_policy_results](https://hub.cloudquery.io/plugins/source/cloudquery/aws). Now you can query the table directly and export in various formats such as CSV or HTML, all with standard `psql` , and of course visualize them in your favorite BI tool. We prepared a pre-built dashboard for Grafana that you can check out [here](https://github.com/cloudquery/cloudquery/tree/main/plugins/source/aws/dashboards): +### Step 4: Configure and run CloudQuery sync + +You will need to configure CloudQuery to sync your cloud assets to your PostgreSQL instance. For AWS assets, you will need the [AWS Plugin](https://hub.cloudquery.io/plugins/source/cloudquery/aws/). Policies are also available for [GCP](https://hub.cloudquery.io/plugins/source/cloudquery/gcp/), [Azure](https://hub.cloudquery.io/plugins/source/cloudquery/azure/), and [Kubernetes](https://hub.cloudquery.io/plugins/source/cloudquery/k8s/). + +To fully define the configuration, you will need to specify what tables to sync. This depends on the policy you want to use. For example, to use [AWS Compliance (free)](https://hub.cloudquery.io/addons/transformation/cloudquery/aws-compliance-free/), you will need to sync the following tables: + +```yaml + - aws_cloudwatch_alarms + - aws_cloudwatchlogs_metric_filters + - aws_ec2_network_acls + - aws_ec2_security_groups + - aws_sns_subscriptions + - aws_iam_credential_reports + - aws_iam_password_policies + - aws_iam_user_access_keys + - aws_iam_users + - aws_autoscaling_groups + - aws_cloudtrail_trail_event_selectors + - aws_cloudtrail_trails + - aws_codebuild_projects + - aws_config_configuration_recorders + - aws_apigateway_rest_api_stages + - aws_apigateway_rest_apis + - aws_apigatewayv2_api_routes + - aws_apigatewayv2_api_stages + - aws_apigatewayv2_apis + - aws_cloudfront_distributions + - aws_efs_access_points + - aws_elasticbeanstalk_environments + - aws_elbv1_load_balancers + - aws_elbv2_load_balancer_attributes + - aws_elbv2_load_balancers + - aws_iam_accounts + - aws_rds_clusters + - aws_s3_accounts +``` -![](/images/blog/open-source-cspm/image0.png) +To see what other transformations are available and what tables are required, visit the individual [Transformations](https://hub.cloudquery.io/addons/transformation) documentation pages. -## Summary +### Step 5: Run policies (CSPM - Cloud Security Posture Management) + +Download the [AWS Compliance (free)](https://hub.cloudquery.io/addons/transformation/cloudquery/aws-compliance-free/) and extract the package. -That’s it! Now you have fully functional CSPM (KSPM, or any other SPM) with those nice bonuses: +Navigate to the extracted directory where the dbt project file (`dbt_project.yml`) resides. Check that the `profile: aws_compliance` matches the profile name in your `profiles.yml` file that you configured in step 3. -1. Access to raw data available and stored in your PostgreSQL. +Before executing the dbt run command, it might be useful to check for any potential issues: + +```shell +dbt compile +``` + +If everything compiles without errors, you can then execute: + +```shell +dbt run +``` + +This command will run all the dbt models and create views in your destination database as defined in the models. + +Now you can query the views directly and export in various formats such as CSV or HTML, all with standard `psql`, and of course visualize them in your favorite BI tool. + +You can repeat this for other cloud providers by adding additional plugins to your syncs and running the relevant transformations. + +### Step 6: Visualize in Grafana + +For AWS Transformations, we offer free Grafana dashboards that you can use as a starting point. + +Download [AWS Compliance Dashboard](https://hub.cloudquery.io/addons/visualization/cloudquery/aws-compliance/) and extract the zip file. Find the `dashboard.json` file in the extracted directory (in `aws_compliance/grafana/postgres`) and [import](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#export-and-import-dashboards) it into your Grafana instance. + +At the top of the dashboard, select the data source to be the PostgreSQL database with data synced by CloudQuery. + +Now you should see a dashboard similar to this: + +![](/images/blog/open-source-cspm/image0.png) + +See our [Hub](https://hub.cloudquery.io/addons/visualization) for additional dashboards. + +## Summary -2. Policies easily customizable and defined in pure SQL. +That’s it! Now you have fully functional CSPM (KSPM, or any other SPM) with access to raw data available and stored in your PostgreSQL. +Don't forget to run the syncs regularly to get fresh data. You only need to run the transformations when you want to update to a newer version. diff --git a/website/versions/cli.json b/website/versions/cli.json index 85423c4966a501..83acf7b428fb3b 100644 --- a/website/versions/cli.json +++ b/website/versions/cli.json @@ -1 +1 @@ -{ "latest": "cli-v5.3.0" } +{ "latest": "cli-v5.3.1" } diff --git a/website/versions/source-okta.json b/website/versions/source-okta.json index ad97ab8deb55ea..2d9175c3527e50 100644 --- a/website/versions/source-okta.json +++ b/website/versions/source-okta.json @@ -1 +1 @@ -{ "latest": "plugins-source-okta-v3.2.18" } +{ "latest": "plugins-source-okta-v4.0.0" } diff --git a/website/versions/source-pagerduty.json b/website/versions/source-pagerduty.json index 90b9a77c2ec9e4..95ffe866e21491 100644 --- a/website/versions/source-pagerduty.json +++ b/website/versions/source-pagerduty.json @@ -1 +1 @@ -{ "latest": "plugins-source-pagerduty-v3.1.4" } +{ "latest": "plugins-source-pagerduty-v3.2.0" }