From 0a6f89734f3a7e3c8d18e2319e89322d498aa16c Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Tue, 6 Feb 2024 17:35:24 +0200 Subject: [PATCH 01/17] chore(main): Release plugins-source-pagerduty v3.2.0 (#16295) :robot: I have created a release *beep* *boop* --- ## [3.2.0](https://github.com/cloudquery/cloudquery/compare/plugins-source-pagerduty-v3.1.4...plugins-source-pagerduty-v3.2.0) (2024-02-06) ### Features * Add JSON schema for pagerduty source plugin ([#16494](https://github.com/cloudquery/cloudquery/issues/16494)) ([e6716aa](https://github.com/cloudquery/cloudquery/commit/e6716aa3f2445959f76344f8f8c48027fe7ad45e)) ### Bug Fixes * **deps:** Update github.com/cloudquery/jsonschema digest to d771afd ([#16511](https://github.com/cloudquery/cloudquery/issues/16511)) ([d374ba8](https://github.com/cloudquery/cloudquery/commit/d374ba88fcc339e992ed9cd298e301acf0fff108)) * **deps:** Update golang.org/x/exp digest to 1b97071 ([#16419](https://github.com/cloudquery/cloudquery/issues/16419)) ([6d77cd1](https://github.com/cloudquery/cloudquery/commit/6d77cd19b6fc648a4ddb12025c22127e960036a4)) * **deps:** Update google.golang.org/genproto/googleapis/api digest to 1f4bbc5 ([#16421](https://github.com/cloudquery/cloudquery/issues/16421)) ([9489931](https://github.com/cloudquery/cloudquery/commit/9489931c1b64bf1f7d5da51997944ee54370215b)) * **deps:** Update google.golang.org/genproto/googleapis/rpc digest to 1f4bbc5 ([#16422](https://github.com/cloudquery/cloudquery/issues/16422)) ([74e98fc](https://github.com/cloudquery/cloudquery/commit/74e98fcbde6c6e11baf98284aef0341c597d4817)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.0 ([#16237](https://github.com/cloudquery/cloudquery/issues/16237)) ([3fcdab0](https://github.com/cloudquery/cloudquery/commit/3fcdab08816ad9de7bb4eecab59c7be1bda3d00c)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.1 ([#16296](https://github.com/cloudquery/cloudquery/issues/16296)) ([ab4a0da](https://github.com/cloudquery/cloudquery/commit/ab4a0dace0a870755fd22d92c6e9c999351f594e)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.2 ([#16342](https://github.com/cloudquery/cloudquery/issues/16342)) ([f3eb857](https://github.com/cloudquery/cloudquery/commit/f3eb85729e5db16c2530b31d6d276934866d5ef0)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.28.0 ([#16362](https://github.com/cloudquery/cloudquery/issues/16362)) ([9166b6b](https://github.com/cloudquery/cloudquery/commit/9166b6b603d0d56a30c2e5072c4f2da5c0c837b5)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.0 ([#16395](https://github.com/cloudquery/cloudquery/issues/16395)) ([fb1102e](https://github.com/cloudquery/cloudquery/commit/fb1102eac8af4b3722b82b882187fdf322546513)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.1 ([#16430](https://github.com/cloudquery/cloudquery/issues/16430)) ([738e89f](https://github.com/cloudquery/cloudquery/commit/738e89f2c969a8a3f1698a8686aeaddb358e7a23)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- .release-please-manifest.json | 2 +- plugins/source/pagerduty/CHANGELOG.md | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 01adf1ca368ff3..87651aa28dd7b8 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -33,7 +33,7 @@ "plugins/source/datadog+FILLER": "0.0.0", "plugins/destination/bigquery": "3.3.16", "plugins/destination/bigquery+FILLER": "0.0.0", - "plugins/source/pagerduty": "3.1.4", + "plugins/source/pagerduty": "3.2.0", "plugins/source/pagerduty+FILLER": "0.0.0", "plugins/destination/mongodb": "2.2.22", "plugins/destination/mongodb+FILLER": "0.0.0", diff --git a/plugins/source/pagerduty/CHANGELOG.md b/plugins/source/pagerduty/CHANGELOG.md index 29bea2cb93c7da..0edb71aad8dbb1 100644 --- a/plugins/source/pagerduty/CHANGELOG.md +++ b/plugins/source/pagerduty/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [3.2.0](https://github.com/cloudquery/cloudquery/compare/plugins-source-pagerduty-v3.1.4...plugins-source-pagerduty-v3.2.0) (2024-02-06) + + +### Features + +* Add JSON schema for pagerduty source plugin ([#16494](https://github.com/cloudquery/cloudquery/issues/16494)) ([e6716aa](https://github.com/cloudquery/cloudquery/commit/e6716aa3f2445959f76344f8f8c48027fe7ad45e)) + + +### Bug Fixes + +* **deps:** Update github.com/cloudquery/jsonschema digest to d771afd ([#16511](https://github.com/cloudquery/cloudquery/issues/16511)) ([d374ba8](https://github.com/cloudquery/cloudquery/commit/d374ba88fcc339e992ed9cd298e301acf0fff108)) +* **deps:** Update golang.org/x/exp digest to 1b97071 ([#16419](https://github.com/cloudquery/cloudquery/issues/16419)) ([6d77cd1](https://github.com/cloudquery/cloudquery/commit/6d77cd19b6fc648a4ddb12025c22127e960036a4)) +* **deps:** Update google.golang.org/genproto/googleapis/api digest to 1f4bbc5 ([#16421](https://github.com/cloudquery/cloudquery/issues/16421)) ([9489931](https://github.com/cloudquery/cloudquery/commit/9489931c1b64bf1f7d5da51997944ee54370215b)) +* **deps:** Update google.golang.org/genproto/googleapis/rpc digest to 1f4bbc5 ([#16422](https://github.com/cloudquery/cloudquery/issues/16422)) ([74e98fc](https://github.com/cloudquery/cloudquery/commit/74e98fcbde6c6e11baf98284aef0341c597d4817)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.0 ([#16237](https://github.com/cloudquery/cloudquery/issues/16237)) ([3fcdab0](https://github.com/cloudquery/cloudquery/commit/3fcdab08816ad9de7bb4eecab59c7be1bda3d00c)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.1 ([#16296](https://github.com/cloudquery/cloudquery/issues/16296)) ([ab4a0da](https://github.com/cloudquery/cloudquery/commit/ab4a0dace0a870755fd22d92c6e9c999351f594e)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.2 ([#16342](https://github.com/cloudquery/cloudquery/issues/16342)) ([f3eb857](https://github.com/cloudquery/cloudquery/commit/f3eb85729e5db16c2530b31d6d276934866d5ef0)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.28.0 ([#16362](https://github.com/cloudquery/cloudquery/issues/16362)) ([9166b6b](https://github.com/cloudquery/cloudquery/commit/9166b6b603d0d56a30c2e5072c4f2da5c0c837b5)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.0 ([#16395](https://github.com/cloudquery/cloudquery/issues/16395)) ([fb1102e](https://github.com/cloudquery/cloudquery/commit/fb1102eac8af4b3722b82b882187fdf322546513)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.1 ([#16430](https://github.com/cloudquery/cloudquery/issues/16430)) ([738e89f](https://github.com/cloudquery/cloudquery/commit/738e89f2c969a8a3f1698a8686aeaddb358e7a23)) + ## [3.1.4](https://github.com/cloudquery/cloudquery/compare/plugins-source-pagerduty-v3.1.3...plugins-source-pagerduty-v3.1.4) (2024-01-16) From c1917343812d473eb365a6cfb33cf91cca1e4dc1 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Tue, 6 Feb 2024 17:41:18 +0200 Subject: [PATCH 02/17] chore(main): Release plugins-source-okta v4.0.0 (#16293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit :robot: I have created a release *beep* *boop* --- ## [4.0.0](https://github.com/cloudquery/cloudquery/compare/plugins-source-okta-v3.2.18...plugins-source-okta-v4.0.0) (2024-02-06) ### ⚠ BREAKING CHANGES * Remove support for deprecated `OKTA_API_TOKEN` environment variable ([#16498](https://github.com/cloudquery/cloudquery/issues/16498)) ### Features * Add JSON Schema to Okta source plugin ([#16493](https://github.com/cloudquery/cloudquery/issues/16493)) ([75ce0ab](https://github.com/cloudquery/cloudquery/commit/75ce0abd90080dd537342dba17b03a60bc4c2b8a)) ### Bug Fixes * **deps:** Update golang.org/x/exp digest to 1b97071 ([#16419](https://github.com/cloudquery/cloudquery/issues/16419)) ([6d77cd1](https://github.com/cloudquery/cloudquery/commit/6d77cd19b6fc648a4ddb12025c22127e960036a4)) * **deps:** Update google.golang.org/genproto/googleapis/api digest to 1f4bbc5 ([#16421](https://github.com/cloudquery/cloudquery/issues/16421)) ([9489931](https://github.com/cloudquery/cloudquery/commit/9489931c1b64bf1f7d5da51997944ee54370215b)) * **deps:** Update google.golang.org/genproto/googleapis/rpc digest to 1f4bbc5 ([#16422](https://github.com/cloudquery/cloudquery/issues/16422)) ([74e98fc](https://github.com/cloudquery/cloudquery/commit/74e98fcbde6c6e11baf98284aef0341c597d4817)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.0 ([#16237](https://github.com/cloudquery/cloudquery/issues/16237)) ([3fcdab0](https://github.com/cloudquery/cloudquery/commit/3fcdab08816ad9de7bb4eecab59c7be1bda3d00c)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.1 ([#16296](https://github.com/cloudquery/cloudquery/issues/16296)) ([ab4a0da](https://github.com/cloudquery/cloudquery/commit/ab4a0dace0a870755fd22d92c6e9c999351f594e)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.2 ([#16342](https://github.com/cloudquery/cloudquery/issues/16342)) ([f3eb857](https://github.com/cloudquery/cloudquery/commit/f3eb85729e5db16c2530b31d6d276934866d5ef0)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.28.0 ([#16362](https://github.com/cloudquery/cloudquery/issues/16362)) ([9166b6b](https://github.com/cloudquery/cloudquery/commit/9166b6b603d0d56a30c2e5072c4f2da5c0c837b5)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.0 ([#16395](https://github.com/cloudquery/cloudquery/issues/16395)) ([fb1102e](https://github.com/cloudquery/cloudquery/commit/fb1102eac8af4b3722b82b882187fdf322546513)) * **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.1 ([#16430](https://github.com/cloudquery/cloudquery/issues/16430)) ([738e89f](https://github.com/cloudquery/cloudquery/commit/738e89f2c969a8a3f1698a8686aeaddb358e7a23)) * Remove support for deprecated `OKTA_API_TOKEN` environment variable ([#16498](https://github.com/cloudquery/cloudquery/issues/16498)) ([25303df](https://github.com/cloudquery/cloudquery/commit/25303df6f8aeda463c35043c17abe37e6611bc68)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- .release-please-manifest.json | 2 +- plugins/source/okta/CHANGELOG.md | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 87651aa28dd7b8..850c3389067e97 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -15,7 +15,7 @@ "plugins/source/github+FILLER": "0.0.0", "plugins/source/k8s": "6.0.1", "plugins/source/k8s+FILLER": "0.0.0", - "plugins/source/okta": "3.2.18", + "plugins/source/okta": "4.0.0", "plugins/source/okta+FILLER": "0.0.0", "plugins/source/terraform": "3.0.12", "plugins/source/terraform+FILLER": "0.0.0", diff --git a/plugins/source/okta/CHANGELOG.md b/plugins/source/okta/CHANGELOG.md index 8af588f2798877..3d5f4ffdd0e420 100644 --- a/plugins/source/okta/CHANGELOG.md +++ b/plugins/source/okta/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [4.0.0](https://github.com/cloudquery/cloudquery/compare/plugins-source-okta-v3.2.18...plugins-source-okta-v4.0.0) (2024-02-06) + + +### ⚠ BREAKING CHANGES + +* Remove support for deprecated `OKTA_API_TOKEN` environment variable ([#16498](https://github.com/cloudquery/cloudquery/issues/16498)) + +### Features + +* Add JSON Schema to Okta source plugin ([#16493](https://github.com/cloudquery/cloudquery/issues/16493)) ([75ce0ab](https://github.com/cloudquery/cloudquery/commit/75ce0abd90080dd537342dba17b03a60bc4c2b8a)) + + +### Bug Fixes + +* **deps:** Update golang.org/x/exp digest to 1b97071 ([#16419](https://github.com/cloudquery/cloudquery/issues/16419)) ([6d77cd1](https://github.com/cloudquery/cloudquery/commit/6d77cd19b6fc648a4ddb12025c22127e960036a4)) +* **deps:** Update google.golang.org/genproto/googleapis/api digest to 1f4bbc5 ([#16421](https://github.com/cloudquery/cloudquery/issues/16421)) ([9489931](https://github.com/cloudquery/cloudquery/commit/9489931c1b64bf1f7d5da51997944ee54370215b)) +* **deps:** Update google.golang.org/genproto/googleapis/rpc digest to 1f4bbc5 ([#16422](https://github.com/cloudquery/cloudquery/issues/16422)) ([74e98fc](https://github.com/cloudquery/cloudquery/commit/74e98fcbde6c6e11baf98284aef0341c597d4817)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.0 ([#16237](https://github.com/cloudquery/cloudquery/issues/16237)) ([3fcdab0](https://github.com/cloudquery/cloudquery/commit/3fcdab08816ad9de7bb4eecab59c7be1bda3d00c)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.1 ([#16296](https://github.com/cloudquery/cloudquery/issues/16296)) ([ab4a0da](https://github.com/cloudquery/cloudquery/commit/ab4a0dace0a870755fd22d92c6e9c999351f594e)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.27.2 ([#16342](https://github.com/cloudquery/cloudquery/issues/16342)) ([f3eb857](https://github.com/cloudquery/cloudquery/commit/f3eb85729e5db16c2530b31d6d276934866d5ef0)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.28.0 ([#16362](https://github.com/cloudquery/cloudquery/issues/16362)) ([9166b6b](https://github.com/cloudquery/cloudquery/commit/9166b6b603d0d56a30c2e5072c4f2da5c0c837b5)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.0 ([#16395](https://github.com/cloudquery/cloudquery/issues/16395)) ([fb1102e](https://github.com/cloudquery/cloudquery/commit/fb1102eac8af4b3722b82b882187fdf322546513)) +* **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.1 ([#16430](https://github.com/cloudquery/cloudquery/issues/16430)) ([738e89f](https://github.com/cloudquery/cloudquery/commit/738e89f2c969a8a3f1698a8686aeaddb358e7a23)) +* Remove support for deprecated `OKTA_API_TOKEN` environment variable ([#16498](https://github.com/cloudquery/cloudquery/issues/16498)) ([25303df](https://github.com/cloudquery/cloudquery/commit/25303df6f8aeda463c35043c17abe37e6611bc68)) + ## [3.2.18](https://github.com/cloudquery/cloudquery/compare/plugins-source-okta-v3.2.17...plugins-source-okta-v3.2.18) (2024-01-16) From 58c8b094e528b7fb6e7745108b1f35893f5bb152 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Tue, 6 Feb 2024 17:42:22 +0200 Subject: [PATCH 03/17] chore: Update CLI version to v5.3.1 (#16532) Updates the CLI latest version to v5.3.1 --- website/versions/cli.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/versions/cli.json b/website/versions/cli.json index 85423c4966a501..83acf7b428fb3b 100644 --- a/website/versions/cli.json +++ b/website/versions/cli.json @@ -1 +1 @@ -{ "latest": "cli-v5.3.0" } +{ "latest": "cli-v5.3.1" } From 31afd39d1ccbd6312ee115146f4befa1e4fabc7c Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Tue, 6 Feb 2024 17:49:47 +0200 Subject: [PATCH 04/17] chore: Update plugin `source-pagerduty` version to v3.2.0 (#16533) Updates the `source-pagerduty` plugin latest version to v3.2.0 --- website/versions/source-pagerduty.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/versions/source-pagerduty.json b/website/versions/source-pagerduty.json index 90b9a77c2ec9e4..95ffe866e21491 100644 --- a/website/versions/source-pagerduty.json +++ b/website/versions/source-pagerduty.json @@ -1 +1 @@ -{ "latest": "plugins-source-pagerduty-v3.1.4" } +{ "latest": "plugins-source-pagerduty-v3.2.0" } From 4371ad79fe2cae905c288d28dfbbb0cd0e715097 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Tue, 6 Feb 2024 17:55:51 +0200 Subject: [PATCH 05/17] chore: Update plugin `source-okta` version to v4.0.0 (#16534) Updates the `source-okta` plugin latest version to v4.0.0 --- website/versions/source-okta.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/versions/source-okta.json b/website/versions/source-okta.json index ad97ab8deb55ea..2d9175c3527e50 100644 --- a/website/versions/source-okta.json +++ b/website/versions/source-okta.json @@ -1 +1 @@ -{ "latest": "plugins-source-okta-v3.2.18" } +{ "latest": "plugins-source-okta-v4.0.0" } From c6b01424871865bc3e79ba0548acf63f0009538e Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Tue, 6 Feb 2024 18:05:59 +0200 Subject: [PATCH 06/17] fix(deps): Update module github.com/cloudquery/plugin-pb-go to v1.17.0 (#16528) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [github.com/cloudquery/plugin-pb-go](https://togithub.com/cloudquery/plugin-pb-go) | require | minor | `v1.16.8` -> `v1.17.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the Dependency Dashboard for more information. --- ### Release Notes
cloudquery/plugin-pb-go (github.com/cloudquery/plugin-pb-go) ### [`v1.17.0`](https://togithub.com/cloudquery/plugin-pb-go/releases/tag/v1.17.0) [Compare Source](https://togithub.com/cloudquery/plugin-pb-go/compare/v1.16.8...v1.17.0) ##### Features - Allow use of CloudQuery registry with docker plugins ([#​235](https://togithub.com/cloudquery/plugin-pb-go/issues/235)) ([58ec077](https://togithub.com/cloudquery/plugin-pb-go/commit/58ec0771f976cb64470b3d1dbab058ffebec45d8)) ##### Bug Fixes - **deps:** Update module github.com/cloudquery/cloudquery-api-go to v1.7.3 ([#​237](https://togithub.com/cloudquery/plugin-pb-go/issues/237)) ([e7a6085](https://togithub.com/cloudquery/plugin-pb-go/commit/e7a6085a508719075954bd370f7182d8d82be894))
--- ### Configuration πŸ“… **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. β™» **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. πŸ”• **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://togithub.com/renovatebot/renovate). --- cli/cmd/install.go | 2 +- cli/go.mod | 2 +- cli/go.sum | 4 ++-- scaffold/cmd/templates/source/go.mod.tpl | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/cmd/install.go b/cli/cmd/install.go index 2787df46042ea7..a904005191af0a 100644 --- a/cli/cmd/install.go +++ b/cli/cmd/install.go @@ -118,7 +118,7 @@ func enrichClientError(clientsList managedplugin.Clients, inferredList []bool, e return fmt.Errorf("%w. Hint: use `cloudquery switch` to set a team", err) } - if !strings.Contains(err.Error(), "not found") { + if !strings.Contains(strings.ToLower(err.Error()), "not found") { return err } l := len(clientsList) diff --git a/cli/go.mod b/cli/go.mod index 88a1a61b69bffa..90002df2c41903 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -8,7 +8,7 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/cloudquery/cloudquery-api-go v1.7.3 github.com/cloudquery/codegen v0.3.12 - github.com/cloudquery/plugin-pb-go v1.16.8 + github.com/cloudquery/plugin-pb-go v1.17.0 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/distribution/reference v0.5.0 github.com/docker/distribution v2.8.3+incompatible diff --git a/cli/go.sum b/cli/go.sum index 06335f0592b749..b356dd350f86e2 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -68,8 +68,8 @@ github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1 github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8= github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb h1:/l8fbvLOCNlgkHp8VUKTTL+Tk9gs5y/K3Yx/bRfReNk= github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= -github.com/cloudquery/plugin-pb-go v1.16.8 h1:VKSii3yuDi7XIVNIkgrqMdd2tAWPX1O/V/HtZp9Y9Ak= -github.com/cloudquery/plugin-pb-go v1.16.8/go.mod h1:taeDcOU7nkiPerOmsUwwYzBoZpQpRNe6nyLWegi67Os= +github.com/cloudquery/plugin-pb-go v1.17.0 h1:zRB3L36MK9eM2iCC6gD7HTB2YRP1JN8f8em7/VbLPw8= +github.com/cloudquery/plugin-pb-go v1.17.0/go.mod h1:mW7K0QCdgAaehGbyWtfewmnTOfsu6QrD1ooGooXK/+M= github.com/cloudquery/plugin-sdk/v4 v4.29.1 h1:vBDrg/e6Zv5lR5DsKETBbVcjS0Q70lRTeUqCM5TzVrk= github.com/cloudquery/plugin-sdk/v4 v4.29.1/go.mod h1:pM5EnbRX7apbgg1e5JC9TZkcJ22eyf4b4Uo1ekDb/GQ= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= diff --git a/scaffold/cmd/templates/source/go.mod.tpl b/scaffold/cmd/templates/source/go.mod.tpl index 0d2be8ce862604..02b47dd77509c4 100644 --- a/scaffold/cmd/templates/source/go.mod.tpl +++ b/scaffold/cmd/templates/source/go.mod.tpl @@ -4,7 +4,7 @@ go 1.21.4 require ( github.com/apache/arrow/go/v15 bcaeaa8c2d97 - github.com/cloudquery/plugin-pb-go v1.16.8 + github.com/cloudquery/plugin-pb-go v1.17.0 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/rs/zerolog v1.29.0 ) From d6cf3035c25178707b4cead4453a9c8bd5ef7501 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Tue, 6 Feb 2024 18:36:34 +0200 Subject: [PATCH 07/17] chore(deps): Update dependency cli to v5.3.1 (#16536) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | [cli](https://togithub.com/cloudquery/cloudquery) | patch | `v5.3.0` -> `v5.3.1` | --- > [!WARNING] > Some dependencies could not be looked up. Check the Dependency Dashboard for more information. --- ### Release Notes
cloudquery/cloudquery (cli) ### [`v5.3.1`](https://togithub.com/cloudquery/cloudquery/releases/tag/cli-v5.3.1) [Compare Source](https://togithub.com/cloudquery/cloudquery/compare/cli-v5.3.0...cli-v5.3.1) ##### Bug Fixes - **deps:** Update github.com/cloudquery/jsonschema digest to [`d771afd`](https://togithub.com/cloudquery/cloudquery/commit/d771afd) ([#​16483](https://togithub.com/cloudquery/cloudquery/issues/16483)) ([dcaa994](https://togithub.com/cloudquery/cloudquery/commit/dcaa9949df43919c0745e05308ce97bf409c4d77)) - **deps:** Update module github.com/cloudquery/cloudquery-api-go to v1.7.3 ([#​16513](https://togithub.com/cloudquery/cloudquery/issues/16513)) ([9463676](https://togithub.com/cloudquery/cloudquery/commit/94636769f67a4c41893742ea1e53b98f3c1614e4)) - **deps:** Update module github.com/cloudquery/plugin-pb-go to v1.16.8 ([#​16467](https://togithub.com/cloudquery/cloudquery/issues/16467)) ([3a68f8f](https://togithub.com/cloudquery/cloudquery/commit/3a68f8fb50aaf30490751c86e1212ae1f790b054)) - **deps:** Update module github.com/cloudquery/plugin-sdk/v4 to v4.29.1 ([#​16430](https://togithub.com/cloudquery/cloudquery/issues/16430)) ([738e89f](https://togithub.com/cloudquery/cloudquery/commit/738e89f2c969a8a3f1698a8686aeaddb358e7a23))
--- ### Configuration πŸ“… **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. β™» **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. πŸ”• **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://togithub.com/renovatebot/renovate). --- .github/workflows/manual_commands_triggers.yml | 2 +- .github/workflows/publish_plugin_to_hub.yml | 6 +++--- .github/workflows/publish_plugin_to_hub_duckdb.yml | 2 +- .github/workflows/publish_plugin_to_hub_snowflake.yml | 2 +- .github/workflows/publish_plugin_to_hub_sqlite.yml | 2 +- .github/workflows/source_alicloud.yml | 2 +- .github/workflows/source_aws.yml | 2 +- .github/workflows/source_awspricing.yml | 2 +- .github/workflows/source_azure.yml | 2 +- .github/workflows/source_cloudflare.yml | 2 +- .github/workflows/source_datadog.yml | 2 +- .github/workflows/source_digitalocean.yml | 2 +- .github/workflows/source_gcp.yml | 4 ++-- .github/workflows/source_github.yml | 2 +- .github/workflows/source_gitlab.yml | 2 +- .github/workflows/source_hackernews.yml | 2 +- .github/workflows/source_hubspot.yml | 2 +- .github/workflows/source_k8s.yml | 4 ++-- .github/workflows/source_notion.yml | 2 +- .github/workflows/source_okta.yml | 2 +- .github/workflows/source_oracle.yml | 2 +- .github/workflows/source_pagerduty.yml | 2 +- .github/workflows/source_test.yml | 2 +- 23 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/manual_commands_triggers.yml b/.github/workflows/manual_commands_triggers.yml index 5db8d80a60d7d5..82a8e38b8e5c1d 100644 --- a/.github/workflows/manual_commands_triggers.yml +++ b/.github/workflows/manual_commands_triggers.yml @@ -32,7 +32,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Set up Go 1.x uses: actions/setup-go@v4 diff --git a/.github/workflows/publish_plugin_to_hub.yml b/.github/workflows/publish_plugin_to_hub.yml index 815fb329230e4d..229cb7704bb746 100644 --- a/.github/workflows/publish_plugin_to_hub.yml +++ b/.github/workflows/publish_plugin_to_hub.yml @@ -132,7 +132,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} @@ -206,7 +206,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} @@ -287,7 +287,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} diff --git a/.github/workflows/publish_plugin_to_hub_duckdb.yml b/.github/workflows/publish_plugin_to_hub_duckdb.yml index 333f576bb63d8f..562523c4bb2a02 100644 --- a/.github/workflows/publish_plugin_to_hub_duckdb.yml +++ b/.github/workflows/publish_plugin_to_hub_duckdb.yml @@ -106,7 +106,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} diff --git a/.github/workflows/publish_plugin_to_hub_snowflake.yml b/.github/workflows/publish_plugin_to_hub_snowflake.yml index 485507d7f6201b..d7fe1da922e688 100644 --- a/.github/workflows/publish_plugin_to_hub_snowflake.yml +++ b/.github/workflows/publish_plugin_to_hub_snowflake.yml @@ -107,7 +107,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} diff --git a/.github/workflows/publish_plugin_to_hub_sqlite.yml b/.github/workflows/publish_plugin_to_hub_sqlite.yml index 77c698d2bca8bd..e69341fb556175 100644 --- a/.github/workflows/publish_plugin_to_hub_sqlite.yml +++ b/.github/workflows/publish_plugin_to_hub_sqlite.yml @@ -106,7 +106,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Publish plugin to hub working-directory: ${{ needs.prepare.outputs.plugin_dir }} diff --git a/.github/workflows/source_alicloud.yml b/.github/workflows/source_alicloud.yml index 2d70d24c77bb71..da4c2b3c827f2f 100644 --- a/.github/workflows/source_alicloud.yml +++ b/.github/workflows/source_alicloud.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_aws.yml b/.github/workflows/source_aws.yml index 2b77f1c38ae278..13a95db0f6af5e 100644 --- a/.github/workflows/source_aws.yml +++ b/.github/workflows/source_aws.yml @@ -49,7 +49,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_awspricing.yml b/.github/workflows/source_awspricing.yml index d6e621837e908b..04f5b760e0881e 100644 --- a/.github/workflows/source_awspricing.yml +++ b/.github/workflows/source_awspricing.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_azure.yml b/.github/workflows/source_azure.yml index 0f441e9994dcd7..7fffa58eee4abf 100644 --- a/.github/workflows/source_azure.yml +++ b/.github/workflows/source_azure.yml @@ -47,7 +47,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_cloudflare.yml b/.github/workflows/source_cloudflare.yml index 89c0364bde5db3..18273397146c0a 100644 --- a/.github/workflows/source_cloudflare.yml +++ b/.github/workflows/source_cloudflare.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_datadog.yml b/.github/workflows/source_datadog.yml index 3fc89e8e230255..6a374359b84181 100644 --- a/.github/workflows/source_datadog.yml +++ b/.github/workflows/source_datadog.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_digitalocean.yml b/.github/workflows/source_digitalocean.yml index ce0c81918f46d8..b1aeda8bbb378e 100644 --- a/.github/workflows/source_digitalocean.yml +++ b/.github/workflows/source_digitalocean.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_gcp.yml b/.github/workflows/source_gcp.yml index 6d84bc4b2f0416..15463564bbda83 100644 --- a/.github/workflows/source_gcp.yml +++ b/.github/workflows/source_gcp.yml @@ -47,7 +47,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen @@ -130,7 +130,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Migrate DB run: cloudquery migrate test/policy_cq_config.yml env: diff --git a/.github/workflows/source_github.yml b/.github/workflows/source_github.yml index c212390243511d..049ff8382dfd5f 100644 --- a/.github/workflows/source_github.yml +++ b/.github/workflows/source_github.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_gitlab.yml b/.github/workflows/source_gitlab.yml index a0212ea43ae438..008e7c8e09d991 100644 --- a/.github/workflows/source_gitlab.yml +++ b/.github/workflows/source_gitlab.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_hackernews.yml b/.github/workflows/source_hackernews.yml index 524ff69e036d09..9bef835664accb 100644 --- a/.github/workflows/source_hackernews.yml +++ b/.github/workflows/source_hackernews.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_hubspot.yml b/.github/workflows/source_hubspot.yml index 5a01cdf29d9916..9a66f119ccca60 100644 --- a/.github/workflows/source_hubspot.yml +++ b/.github/workflows/source_hubspot.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_k8s.yml b/.github/workflows/source_k8s.yml index 18072a6bca07ff..0865ee7790e970 100644 --- a/.github/workflows/source_k8s.yml +++ b/.github/workflows/source_k8s.yml @@ -47,7 +47,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen @@ -130,7 +130,7 @@ jobs: - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: Migrate DB run: cloudquery migrate test/policy_cq_config.yml env: diff --git a/.github/workflows/source_notion.yml b/.github/workflows/source_notion.yml index 96180b92bb8796..e55484ade93043 100644 --- a/.github/workflows/source_notion.yml +++ b/.github/workflows/source_notion.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_okta.yml b/.github/workflows/source_okta.yml index 136ea22450bd3e..e671d376ffb202 100644 --- a/.github/workflows/source_okta.yml +++ b/.github/workflows/source_okta.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_oracle.yml b/.github/workflows/source_oracle.yml index 5fb0c862194ff7..fab774bd21f05f 100644 --- a/.github/workflows/source_oracle.yml +++ b/.github/workflows/source_oracle.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen-docs if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_pagerduty.yml b/.github/workflows/source_pagerduty.yml index a0709b507261fe..f6fbd64a784873 100644 --- a/.github/workflows/source_pagerduty.yml +++ b/.github/workflows/source_pagerduty.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen diff --git a/.github/workflows/source_test.yml b/.github/workflows/source_test.yml index 057767f5bd38e6..48017475028c60 100644 --- a/.github/workflows/source_test.yml +++ b/.github/workflows/source_test.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'pull_request' uses: cloudquery/setup-cloudquery@v3 with: - version: v5.3.0 + version: v5.3.1 - name: gen if: github.event_name == 'pull_request' run: make gen From b4db363f58894f0888ff5a1effa4e2582f58840c Mon Sep 17 00:00:00 2001 From: Michal Brutvan <57617962+pilvikala@users.noreply.github.com> Date: Wed, 7 Feb 2024 15:39:20 +0200 Subject: [PATCH 08/17] doc: Updated how-to for open source CSPM (#16537) This is an update for our Open Source CSPM "How To" guide that reflects the recent changes to policies. --- .../pages/how-to-guides/open-source-cspm.mdx | 127 +++++++++++++++--- 1 file changed, 108 insertions(+), 19 deletions(-) diff --git a/website/pages/how-to-guides/open-source-cspm.mdx b/website/pages/how-to-guides/open-source-cspm.mdx index dc2ab1cd9d036e..d8624fd387a66d 100644 --- a/website/pages/how-to-guides/open-source-cspm.mdx +++ b/website/pages/how-to-guides/open-source-cspm.mdx @@ -17,48 +17,137 @@ CSPMs are probably the biggest offenders of yet-another-dashboard syndrome, and - **ETL (Extract-Transform-Load) ingestion layer:** [CloudQuery](https://github.com/cloudquery/cloudquery) - **Datastore:** PostgreSQL -- **Policies:** Standard [SQL Policies](https://hub.cloudquery.io/addons/transformation) to be executed via `psql` +- **Transformations (policies):** dbt [Transformations](https://hub.cloudquery.io/addons/transformation) to be executed via `dbt` - **Data Visualization and Exploration Platform:** Grafana ## What you will get -- **Raw SQL access** to all your cloud asset inventory, open source SQL based policies. +- **Raw SQL access** to all your cloud asset inventory. - **Multi-Cloud Asset Inventory:** Ingest configuration from all your clouds to a single datastore with a unified structure. - **Avoid yet-another-dashboard fatigue:** Reuse your existing BI/Visualization stack (Grafana in this example) to build an open source CSPM. -### Step 1: **Install or Deploy CloudQuery** +## Building the CSPM step by step + +### Step 1: Install or deploy CloudQuery If it’s your first time using CloudQuery we suggest you first run it locally to get familiar with the tool. Take a look at our [quick start guide](/docs/quickstart). -If you are already familiar with CloudQuery, take a look at how to deploy it to AWS on RDS Aurora and EKS at [github.com/cludquery/terraform-aws-cloudquery](https://github.com/cloudquery/terraform-aws-cloudquery) , or GCP and Cloud SQL at [https://github.com/cloudquery/terraform-gcp-cloudquery](https://github.com/cloudquery/terraform-gcp-cloudquery) +If you are already familiar with CloudQuery, take a look at how to deploy it with [Kubernetes](/docs/deployment/kubernetes), on [AWS ECS using Fargate](/docs/deployment/ecs) or with [Google Cloud Run](/docs/deployment/cloud-run). For more deployment guides, see our [Deployment Overview](/docs/deployment/overview). -### Step 2: **Install Grafana** +### Step 2: Install Grafana Grafana is a well-known open source observability and visualization tool. It is open source, so there are a number of ways to deploy it: - **Self-hosted (local, docker, k8s):** [Official guide.](https://grafana.com/docs/grafana/latest/setup-grafana/installation/) - **SaaS/managed:** [Grafana.com](https://grafana.com/) -- AWS Managed Grafana: [https://aws.amazon.com/grafana/](https://aws.amazon.com/grafana/) +- **AWS Managed Grafana**: [https://aws.amazon.com/grafana/](https://aws.amazon.com/grafana/) + +### Step 3: Install dbt + +CloudQuery policies and rules are implemented using [dbt](https://docs.getdbt.com/docs/installation), a popular open source tool for data transformation. + +After the installation, you will need to provide dbt with a profile which defines how to connect to relevant databases. By default, it searches for a `profiles.yml` file in the local directory and falls back to `~/.dbt/`. Read more about profiles in the [dbt documentation](https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles). -### Step 3: Run Policies (CSPM - Cloud Security Posture Management) +Your profiles.yml file should look like this: -CloudQuery policies and rules are implemented in pure SQL and they store results in a single table that you can easily query and visualize. Here is a [link](https://hub.cloudquery.io/addons/transformation) to all available policies and compliance frameworks. In this section we will go quickly through how to run multiple benchmarks for AWS. +```yaml +config: + send_anonymous_usage_stats: False + use_colors: True + +aws_compliance: # this should match the profile name in your dbt_project.yml, see step 5. + target: postgres + outputs: + postgres: + type: postgres + host: "your postgres host" + user: "postgres user name" + pass: "postgres password" + port: 5432 + dbname: "database name" + schema: public + threads: 4 -```bash copy -git clone https://github.com/cloudquery/cloudquery.git -cd cloudquery/plugins/source/aws/policies -# change the DSN to your PostgreSQL instance populated by CloudQuery -psql postgres://postgres:pass@localhost:5432/postgres -f policy.sql ``` -This should run all [available](https://hub.cloudquery.io/plugins/source/cloudquery/aws) compliance framework and store the results in [aws_policy_results](https://hub.cloudquery.io/plugins/source/cloudquery/aws). Now you can query the table directly and export in various formats such as CSV or HTML, all with standard `psql` , and of course visualize them in your favorite BI tool. We prepared a pre-built dashboard for Grafana that you can check out [here](https://github.com/cloudquery/cloudquery/tree/main/plugins/source/aws/dashboards): +### Step 4: Configure and run CloudQuery sync + +You will need to configure CloudQuery to sync your cloud assets to your PostgreSQL instance. For AWS assets, you will need the [AWS Plugin](https://hub.cloudquery.io/plugins/source/cloudquery/aws/). Policies are also available for [GCP](https://hub.cloudquery.io/plugins/source/cloudquery/gcp/), [Azure](https://hub.cloudquery.io/plugins/source/cloudquery/azure/), and [Kubernetes](https://hub.cloudquery.io/plugins/source/cloudquery/k8s/). + +To fully define the configuration, you will need to specify what tables to sync. This depends on the policy you want to use. For example, to use [AWS Compliance (free)](https://hub.cloudquery.io/addons/transformation/cloudquery/aws-compliance-free/), you will need to sync the following tables: + +```yaml + - aws_cloudwatch_alarms + - aws_cloudwatchlogs_metric_filters + - aws_ec2_network_acls + - aws_ec2_security_groups + - aws_sns_subscriptions + - aws_iam_credential_reports + - aws_iam_password_policies + - aws_iam_user_access_keys + - aws_iam_users + - aws_autoscaling_groups + - aws_cloudtrail_trail_event_selectors + - aws_cloudtrail_trails + - aws_codebuild_projects + - aws_config_configuration_recorders + - aws_apigateway_rest_api_stages + - aws_apigateway_rest_apis + - aws_apigatewayv2_api_routes + - aws_apigatewayv2_api_stages + - aws_apigatewayv2_apis + - aws_cloudfront_distributions + - aws_efs_access_points + - aws_elasticbeanstalk_environments + - aws_elbv1_load_balancers + - aws_elbv2_load_balancer_attributes + - aws_elbv2_load_balancers + - aws_iam_accounts + - aws_rds_clusters + - aws_s3_accounts +``` -![](/images/blog/open-source-cspm/image0.png) +To see what other transformations are available and what tables are required, visit the individual [Transformations](https://hub.cloudquery.io/addons/transformation) documentation pages. -## Summary +### Step 5: Run policies (CSPM - Cloud Security Posture Management) + +Download the [AWS Compliance (free)](https://hub.cloudquery.io/addons/transformation/cloudquery/aws-compliance-free/) and extract the package. -That’s it! Now you have fully functional CSPM (KSPM, or any other SPM) with those nice bonuses: +Navigate to the extracted directory where the dbt project file (`dbt_project.yml`) resides. Check that the `profile: aws_compliance` matches the profile name in your `profiles.yml` file that you configured in step 3. -1. Access to raw data available and stored in your PostgreSQL. +Before executing the dbt run command, it might be useful to check for any potential issues: + +```shell +dbt compile +``` + +If everything compiles without errors, you can then execute: + +```shell +dbt run +``` + +This command will run all the dbt models and create views in your destination database as defined in the models. + +Now you can query the views directly and export in various formats such as CSV or HTML, all with standard `psql`, and of course visualize them in your favorite BI tool. + +You can repeat this for other cloud providers by adding additional plugins to your syncs and running the relevant transformations. + +### Step 6: Visualize in Grafana + +For AWS Transformations, we offer free Grafana dashboards that you can use as a starting point. + +Download [AWS Compliance Dashboard](https://hub.cloudquery.io/addons/visualization/cloudquery/aws-compliance/) and extract the zip file. Find the `dashboard.json` file in the extracted directory (in `aws_compliance/grafana/postgres`) and [import](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#export-and-import-dashboards) it into your Grafana instance. + +At the top of the dashboard, select the data source to be the PostgreSQL database with data synced by CloudQuery. + +Now you should see a dashboard similar to this: + +![](/images/blog/open-source-cspm/image0.png) + +See our [Hub](https://hub.cloudquery.io/addons/visualization) for additional dashboards. + +## Summary -2. Policies easily customizable and defined in pure SQL. +That’s it! Now you have fully functional CSPM (KSPM, or any other SPM) with access to raw data available and stored in your PostgreSQL. +Don't forget to run the syncs regularly to get fresh data. You only need to run the transformations when you want to update to a newer version. From 844434be7fa6362de2a3c9dd47477fbb69d02c6c Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Wed, 7 Feb 2024 18:27:33 +0200 Subject: [PATCH 09/17] fix(deps): Update module github.com/cloudquery/plugin-pb-go to v1.17.1 (#16539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [github.com/cloudquery/plugin-pb-go](https://togithub.com/cloudquery/plugin-pb-go) | require | patch | `v1.17.0` -> `v1.17.1` | --- > [!WARNING] > Some dependencies could not be looked up. Check the Dependency Dashboard for more information. --- ### Release Notes
cloudquery/plugin-pb-go (github.com/cloudquery/plugin-pb-go) ### [`v1.17.1`](https://togithub.com/cloudquery/plugin-pb-go/releases/tag/v1.17.1) [Compare Source](https://togithub.com/cloudquery/plugin-pb-go/compare/v1.17.0...v1.17.1) ##### Bug Fixes - **deps:** Update dependency go to v1.21.5 ([#​206](https://togithub.com/cloudquery/plugin-pb-go/issues/206)) ([4b1add4](https://togithub.com/cloudquery/plugin-pb-go/commit/4b1add42b00c51dce1e3e17a896b4e64076b1393)) - Generate Go Code from `plugin-pb` ([#​239](https://togithub.com/cloudquery/plugin-pb-go/issues/239)) ([08be360](https://togithub.com/cloudquery/plugin-pb-go/commit/08be360d887faaa564fb5c12aef274c70fa62bf4))
--- ### Configuration πŸ“… **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. β™» **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. πŸ”• **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://togithub.com/renovatebot/renovate). --- cli/go.mod | 6 ++++-- cli/go.sum | 4 ++-- scaffold/cmd/templates/source/go.mod.tpl | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cli/go.mod b/cli/go.mod index 90002df2c41903..10575f624b691c 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -1,6 +1,8 @@ module github.com/cloudquery/cloudquery/cli -go 1.21.4 +go 1.21.5 + +toolchain go1.22.0 require ( github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1 @@ -8,7 +10,7 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/cloudquery/cloudquery-api-go v1.7.3 github.com/cloudquery/codegen v0.3.12 - github.com/cloudquery/plugin-pb-go v1.17.0 + github.com/cloudquery/plugin-pb-go v1.17.1 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/distribution/reference v0.5.0 github.com/docker/distribution v2.8.3+incompatible diff --git a/cli/go.sum b/cli/go.sum index b356dd350f86e2..c31cc2fe425566 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -68,8 +68,8 @@ github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1 github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8= github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb h1:/l8fbvLOCNlgkHp8VUKTTL+Tk9gs5y/K3Yx/bRfReNk= github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= -github.com/cloudquery/plugin-pb-go v1.17.0 h1:zRB3L36MK9eM2iCC6gD7HTB2YRP1JN8f8em7/VbLPw8= -github.com/cloudquery/plugin-pb-go v1.17.0/go.mod h1:mW7K0QCdgAaehGbyWtfewmnTOfsu6QrD1ooGooXK/+M= +github.com/cloudquery/plugin-pb-go v1.17.1 h1:nOvPjfP8eUxupe8OCCIClZovxpt0dNiIZcWNUR9Nn5c= +github.com/cloudquery/plugin-pb-go v1.17.1/go.mod h1:IHfPXOgPgziUrfKw4tN0+DVYTABM3UaZIWrzXPBjQG0= github.com/cloudquery/plugin-sdk/v4 v4.29.1 h1:vBDrg/e6Zv5lR5DsKETBbVcjS0Q70lRTeUqCM5TzVrk= github.com/cloudquery/plugin-sdk/v4 v4.29.1/go.mod h1:pM5EnbRX7apbgg1e5JC9TZkcJ22eyf4b4Uo1ekDb/GQ= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= diff --git a/scaffold/cmd/templates/source/go.mod.tpl b/scaffold/cmd/templates/source/go.mod.tpl index 02b47dd77509c4..e84c72175af244 100644 --- a/scaffold/cmd/templates/source/go.mod.tpl +++ b/scaffold/cmd/templates/source/go.mod.tpl @@ -4,7 +4,7 @@ go 1.21.4 require ( github.com/apache/arrow/go/v15 bcaeaa8c2d97 - github.com/cloudquery/plugin-pb-go v1.17.0 + github.com/cloudquery/plugin-pb-go v1.17.1 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/rs/zerolog v1.29.0 ) From 4315ea663f653f610ffc4e73a17b5641dbb0f9ad Mon Sep 17 00:00:00 2001 From: Ben Bernays Date: Wed, 7 Feb 2024 14:07:29 -0600 Subject: [PATCH 10/17] feat: Enable Skipping of Organization Discovery (#16540) #### Summary If a user runs the GCP in parallel it can run into rate limits for discovering organizations. In cases when there are no org resources this allows users to skip those api calls altogether --- plugins/source/gcp/client/client.go | 9 +++++++-- plugins/source/gcp/client/spec/schema.json | 5 +++++ plugins/source/gcp/client/spec/spec.go | 3 +++ plugins/source/gcp/resources/plugin/client.go | 16 ++++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/plugins/source/gcp/client/client.go b/plugins/source/gcp/client/client.go index 072e70c3549de4..84dec4ee1b0856 100644 --- a/plugins/source/gcp/client/client.go +++ b/plugins/source/gcp/client/client.go @@ -56,6 +56,7 @@ type Client struct { Backend state.Client TestingGRPCEndpoint *string TestingHTTPEndpoint *string + Spec *spec.Spec } func (c *Client) WithBackend(backend state.Client) *Client { @@ -130,6 +131,7 @@ func New(ctx context.Context, logger zerolog.Logger, s *spec.Spec) (schema.Clien c := Client{ logger: logger, EnabledServices: map[string]map[string]any{}, + Spec: s, } projects := s.ProjectIDs @@ -232,7 +234,10 @@ func New(ctx context.Context, logger zerolog.Logger, s *spec.Spec) (schema.Clien projects = setUnion(projects, projectsWithFilter) } - if len(s.OrganizationIDs) == 0 && len(s.OrganizationFilter) == 0 { + switch { + case s.SkipOrganizationResources: + c.logger.Info().Msg("Skipping organization resources") + case len(s.OrganizationIDs) == 0 && len(s.OrganizationFilter) == 0: c.logger.Info().Msg("No organization_ids or organization_filter specified - assuming all organizations") c.logger.Info().Msg("Listing organizations...") @@ -240,7 +245,7 @@ func New(ctx context.Context, logger zerolog.Logger, s *spec.Spec) (schema.Clien if err != nil { c.logger.Err(err).Msg("failed to get organizations") } - } else { + default: if len(s.OrganizationIDs) > 0 { for _, orgID := range s.OrganizationIDs { c.logger.Info().Msgf("Getting spec organization %q...", orgID) diff --git a/plugins/source/gcp/client/spec/schema.json b/plugins/source/gcp/client/spec/schema.json index 198b178cd4a3d5..31a990e938f630 100644 --- a/plugins/source/gcp/client/spec/schema.json +++ b/plugins/source/gcp/client/spec/schema.json @@ -191,6 +191,11 @@ "type": "null" } ] + }, + "skip_organization_resources": { + "type": "boolean", + "description": "If enabled CloudQuery will skip syncing organization level resources", + "default": false } }, "additionalProperties": false, diff --git a/plugins/source/gcp/client/spec/spec.go b/plugins/source/gcp/client/spec/spec.go index d6e84b5be9f903..1f20a717b80c12 100644 --- a/plugins/source/gcp/client/spec/spec.go +++ b/plugins/source/gcp/client/spec/spec.go @@ -105,6 +105,9 @@ type Spec struct { // Service Account impersonation configuration. ServiceAccountImpersonation *CredentialsConfig `json:"service_account_impersonation"` + + // If enabled CloudQuery will skip syncing organization level resources + SkipOrganizationResources bool `json:"skip_organization_resources" jsonschema:"default=false"` } func (s *Spec) Validate() error { diff --git a/plugins/source/gcp/resources/plugin/client.go b/plugins/source/gcp/resources/plugin/client.go index b723a736cbb961..c16d2b3efbe523 100644 --- a/plugins/source/gcp/resources/plugin/client.go +++ b/plugins/source/gcp/resources/plugin/client.go @@ -4,6 +4,9 @@ import ( "context" "encoding/json" "fmt" + "reflect" + "runtime" + "strings" "github.com/cloudquery/cloudquery/plugins/source/gcp/client" "github.com/cloudquery/cloudquery/plugins/source/gcp/client/spec" @@ -69,6 +72,19 @@ func (c *Client) Sync(ctx context.Context, options plugin.SyncOptions, res chan< if err != nil { return err } + var skippedOrgTables []string + for _, table := range tables { + if c.syncClient.Spec.SkipOrganizationResources && table.Multiplex != nil { + // get the name of the multiplexer via reflection + multiplexerName := runtime.FuncForPC(reflect.ValueOf(table.Multiplex).Pointer()).Name() + if strings.Contains(multiplexerName, "plugins/source/gcp/client.OrgMultiplex") { + skippedOrgTables = append(skippedOrgTables, table.Name) + } + } + } + if len(skippedOrgTables) > 0 { + return fmt.Errorf("invalid spec: organizational level tables (%s) included while `SkipOrganizationResources` is enabled", strings.Join(skippedOrgTables, ", ")) + } stateClient := state.Client(&state.NoOpClient{}) if options.BackendOptions != nil { conn, err := grpc.DialContext(ctx, options.BackendOptions.Connection, From e0090a38b5ee097359a09698b288888f744c3315 Mon Sep 17 00:00:00 2001 From: Ben Bernays Date: Wed, 7 Feb 2024 14:38:16 -0600 Subject: [PATCH 11/17] feat: Add tags for AWS Systems Manager Parameters (#16542) #### Summary Add tags for AWS Systems Manager Parameters --- .../aws/docs/tables/aws_ssm_parameters.md | 1 + .../aws/resources/services/ssm/parameters.go | 23 +++++++++++++++++++ .../services/ssm/parameters_mock_test.go | 14 +++++++++++ 3 files changed, 38 insertions(+) diff --git a/plugins/source/aws/docs/tables/aws_ssm_parameters.md b/plugins/source/aws/docs/tables/aws_ssm_parameters.md index 311dacd2b23b1b..3a81e5b3d6adbc 100644 --- a/plugins/source/aws/docs/tables/aws_ssm_parameters.md +++ b/plugins/source/aws/docs/tables/aws_ssm_parameters.md @@ -16,6 +16,7 @@ The following fields are used to calculate the value of `_cq_id`: (**account_id* |account_id|`utf8`| |region|`utf8`| |name|`utf8`| +|tags|`json`| |allowed_pattern|`utf8`| |data_type|`utf8`| |description|`utf8`| diff --git a/plugins/source/aws/resources/services/ssm/parameters.go b/plugins/source/aws/resources/services/ssm/parameters.go index 1f377ca79114c6..8628dac7e5ba8b 100644 --- a/plugins/source/aws/resources/services/ssm/parameters.go +++ b/plugins/source/aws/resources/services/ssm/parameters.go @@ -9,6 +9,7 @@ import ( "github.com/cloudquery/cloudquery/plugins/source/aws/client" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/transformers" + sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" ) func Parameters() *schema.Table { @@ -28,6 +29,11 @@ func Parameters() *schema.Table { Description: `The parameter name`, PrimaryKeyComponent: true, }, + { + Name: "tags", + Type: sdkTypes.ExtensionTypes.JSON, + Resolver: resolveParameterTags, + }, }, } } @@ -47,3 +53,20 @@ func fetchSsmParameters(ctx context.Context, meta schema.ClientMeta, parent *sch } return nil } +func resolveParameterTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + cl := meta.(*client.Client) + svc := cl.Services(client.AWSServiceSsm).Ssm + pm := resource.Item.(types.ParameterMetadata) + resp, err := svc.ListTagsForResource(ctx, &ssm.ListTagsForResourceInput{ + ResourceId: pm.Name, + ResourceType: types.ResourceTypeForTaggingParameter, + }, func(options *ssm.Options) { + options.Region = cl.Region + }) + if err != nil { + return err + } + tags := make(map[string]string) + client.TagsIntoMap(resp.TagList, tags) + return resource.Set(c.Name, tags) +} diff --git a/plugins/source/aws/resources/services/ssm/parameters_mock_test.go b/plugins/source/aws/resources/services/ssm/parameters_mock_test.go index 348f05b4759265..0de83450fe9d5e 100644 --- a/plugins/source/aws/resources/services/ssm/parameters_mock_test.go +++ b/plugins/source/aws/resources/services/ssm/parameters_mock_test.go @@ -17,6 +17,8 @@ func buildParameters(t *testing.T, ctrl *gomock.Controller) client.Services { var pm types.ParameterMetadata require.NoError(t, faker.FakeObject(&pm)) + var tags []types.Tag + require.NoError(t, faker.FakeObject(&tags)) mock.EXPECT().DescribeParameters( gomock.Any(), &ssm.DescribeParametersInput{}, @@ -25,6 +27,18 @@ func buildParameters(t *testing.T, ctrl *gomock.Controller) client.Services { &ssm.DescribeParametersOutput{Parameters: []types.ParameterMetadata{pm}}, nil, ) + + mock.EXPECT().ListTagsForResource( + gomock.Any(), + &ssm.ListTagsForResourceInput{ + ResourceId: pm.Name, + ResourceType: types.ResourceTypeForTaggingParameter, + }, + gomock.Any(), + ).Return( + &ssm.ListTagsForResourceOutput{TagList: tags}, + nil, + ) return client.Services{Ssm: mock} } From 6189b49251798c400644b480d0cb591dbf3a5586 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Wed, 7 Feb 2024 23:17:01 +0200 Subject: [PATCH 12/17] feat(services): Support newly added regions (#16543) This PR was created by a scheduled workflow to update supported regions for AWS services --- .../aws/client/data/partition_service_region.json | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/plugins/source/aws/client/data/partition_service_region.json b/plugins/source/aws/client/data/partition_service_region.json index fb9a931dd3e0ae..262ded39b9376d 100644 --- a/plugins/source/aws/client/data/partition_service_region.json +++ b/plugins/source/aws/client/data/partition_service_region.json @@ -5094,27 +5094,19 @@ }, "resource-explorer-2": { "regions": { - "af-south-1": {}, - "ap-east-1": {}, "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-northeast-3": {}, "ap-south-1": {}, - "ap-south-2": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ap-southeast-3": {}, - "ap-southeast-4": {}, "ca-central-1": {}, "eu-central-1": {}, - "eu-central-2": {}, "eu-north-1": {}, - "eu-south-1": {}, "eu-west-1": {}, "eu-west-2": {}, "eu-west-3": {}, - "il-central-1": {}, - "me-central-1": {}, "me-south-1": {}, "sa-east-1": {}, "us-east-1": {}, @@ -7137,6 +7129,12 @@ "cn-northwest-1": {} } }, + "inspector2": { + "regions": { + "cn-north-1": {}, + "cn-northwest-1": {} + } + }, "internetmonitor": { "regions": { "cn-north-1": {}, From b1e1fb6a382a010fe6bd867ee1528edd1f01da0f Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Thu, 8 Feb 2024 14:32:32 +0100 Subject: [PATCH 13/17] chore: Revert toolchain update (#16549) #### Summary Not sure why our bot did this in https://github.com/cloudquery/cloudquery/pull/16539/files, but we don't need it |true| S A{No Connection?} --> |false| G[Get spec schema] subgraph validate G --> |non-empty schema| P[parse schema] G --> |failed call| E[error-out] P --> |OK| V[validate spec] V --> |failed| E end P --> |failed| S G --> |unimplemented| S G --> |empty schema| S V --> |OK| I[init] S[Skip validation] --> I ``` --- cli/cmd/migrate_v3.go | 24 +++--------- cli/cmd/specs.go | 90 +++++++++++++++++++++++++++++++++++++++++++ cli/cmd/sync_v3.go | 34 ++++------------ cli/cmd/tables_v3.go | 4 +- cli/go.mod | 2 +- 5 files changed, 104 insertions(+), 50 deletions(-) diff --git a/cli/cmd/migrate_v3.go b/cli/cmd/migrate_v3.go index de3868380a3fda..e303011769c367 100644 --- a/cli/cmd/migrate_v3.go +++ b/cli/cmd/migrate_v3.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "encoding/json" "fmt" "time" @@ -45,28 +44,15 @@ func migrateConnectionV3(ctx context.Context, sourceClient *managedplugin.Client } // initialize destinations first, so that their connections may be used as backends by the source - for i := range destinationsClients { - destSpec := destinationSpecs[i] - destSpecBytes, err := json.Marshal(destSpec.Spec) - if err != nil { - return err - } - if _, err := destinationsPbClients[i].Init(ctx, &plugin.Init_Request{ - Spec: destSpecBytes, - }); err != nil { - return err + for i, destinationSpec := range destinationSpecs { + if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false); err != nil { + return fmt.Errorf("failed to init destination %v: %w", destinationSpec.Name, err) } } - specBytes, err := json.Marshal(sourceSpec.Spec) + err := initPlugin(ctx, sourcePbClient, sourceSpec.Spec, true) if err != nil { - return err - } - if _, err := sourcePbClient.Init(ctx, &plugin.Init_Request{ - Spec: specBytes, - NoConnection: true, - }); err != nil { - return err + return fmt.Errorf("failed to init source %v: %w", sourceSpec.Name, err) } writeClients := make([]plugin.Plugin_WriteClient, len(destinationsPbClients)) diff --git a/cli/cmd/specs.go b/cli/cmd/specs.go index e8ea2b3eaf2dc5..fa10f4329adeda 100644 --- a/cli/cmd/specs.go +++ b/cli/cmd/specs.go @@ -1,10 +1,19 @@ package cmd import ( + "context" + "encoding/json" + "errors" "fmt" + "strings" "github.com/cloudquery/cloudquery/cli/internal/specs/v0" + "github.com/cloudquery/plugin-pb-go/pb/plugin/v3" pbSpecs "github.com/cloudquery/plugin-pb-go/specs" + "github.com/rs/zerolog/log" + "github.com/santhosh-tekuri/jsonschema/v5" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func CLIRegistryToPbRegistry(registry specs.Registry) pbSpecs.Registry { @@ -86,3 +95,84 @@ func CLIDestinationSpecToPbSpec(spec specs.Destination) pbSpecs.Destination { Spec: spec.Spec, } } + +// initPlugin is a simple wrapper that will try to validate the spec before actually passing it to Init. +func initPlugin(ctx context.Context, client plugin.PluginClient, spec any, noConnection bool) error { + if !noConnection { + // perform spec validation + if err := validatePluginSpec(ctx, client, spec); err != nil { + return err + } + } + + specBytes, err := json.Marshal(spec) + if err != nil { + return err + } + + _, err = client.Init(ctx, &plugin.Init_Request{Spec: specBytes, NoConnection: noConnection}) + return err +} + +// validatePluginSpec encompasses spec validation only: +// 1. Get spec schema from the plugin. +// If the call isn't implemented, just skip the validation. +// 2. Validate that the provided JSON schema is valid & can be used for spec validation. +// If the spec is empty (i.e., the plugin didn't supply the schema) just skip. +// 3. If the schema isn't empty but not valid, print the error message & skip the validation. +// 4. Finally, return the validation result. +func validatePluginSpec(ctx context.Context, client plugin.PluginClient, spec any) error { + schema, err := client.GetSpecSchema(ctx, &plugin.GetSpecSchema_Request{}) + if err != nil { + st, ok := status.FromError(err) + if !ok { + // not a gRPC-compatible error + log.Err(err).Msg("failed to get spec schema") + return err + } + if st.Code() != codes.Unimplemented { + // unimplemented is OK, treat as empty schema + log.Err(err).Msg("failed to get spec schema") + return err + } + } + + jsonSchema := schema.GetJsonSchema() + if len(jsonSchema) == 0 { + // This will also be true for Unimplemented response (schema = nil => schema.GetJsonSchema() = "") + log.Info().Msg("empty JSON schema for plugin spec, skipping validation") + return nil + } + + sc, err := parseJSONSchema(jsonSchema) + if err != nil { + log.Err(err).Msg("failed to parse spec schema, skipping validation") + return nil + } + + return sc.Validate(spec) +} + +func parseJSONSchema(jsonSchema string) (*jsonschema.Schema, error) { + c := jsonschema.NewCompiler() + c.Draft = jsonschema.Draft2020 + c.AssertFormat = true + + if err := c.AddResource("schema.json", strings.NewReader(jsonSchema)); err != nil { + return nil, err + } + + sc, err := c.Compile("schema.json") + if err != nil { + var se *jsonschema.SchemaError + if errors.As(err, &se); se != nil && se.Err != nil { + // We add resource as `file`, but there's none, actually. + // So, we need to prettify message a bit. + return nil, fmt.Errorf("jsonschema compilation failed: %w", + errors.New(strings.Replace(se.Err.Error(), "jsonschema: '' ", "", 1))) + } + return nil, err + } + + return sc, nil +} diff --git a/cli/cmd/sync_v3.go b/cli/cmd/sync_v3.go index 10eb650134afb5..8503e696caf34d 100644 --- a/cli/cmd/sync_v3.go +++ b/cli/cmd/sync_v3.go @@ -126,28 +126,14 @@ func syncConnectionV3(ctx context.Context, source v3source, destinations []v3des } // initialize destinations first, so that their connections may be used as backends by the source - for i := range destinationsClients { - destSpec := destinationSpecs[i] - destSpecBytes, err := json.Marshal(destSpec.Spec) - if err != nil { - return err - } - if _, err := destinationsPbClients[i].Init(ctx, &plugin.Init_Request{ - Spec: destSpecBytes, - }); err != nil { - return fmt.Errorf("failed to init destination %v: %w", destSpec.Name, err) + for i, destinationSpec := range destinationSpecs { + if err := initPlugin(ctx, destinationsPbClients[i], destinationSpec.Spec, false); err != nil { + return fmt.Errorf("failed to init destination %v: %w", destinationSpec.Name, err) } } if backend != nil { - backendSpec := backend.spec - backendSpecBytes, err := json.Marshal(backendSpec.Spec) - if err != nil { - return err - } - if _, err := backendPbClient.Init(ctx, &plugin.Init_Request{ - Spec: backendSpecBytes, - }); err != nil { - return fmt.Errorf("failed to init backend %v: %w", backendSpec.Name, err) + if err := initPlugin(ctx, backendPbClient, backend.spec.Spec, false); err != nil { + return fmt.Errorf("failed to init backend %v: %w", backend.spec.Name, err) } } @@ -165,14 +151,8 @@ func syncConnectionV3(ctx context.Context, source v3source, destinations []v3des return fmt.Errorf("failed to unmarshal source spec JSON after variable replacement: %w", err) } - sourceSpecBytes, err := json.Marshal(sourceSpec.Spec) - if err != nil { - return err - } - if _, err := sourcePbClient.Init(ctx, &plugin.Init_Request{ - Spec: sourceSpecBytes, - }); err != nil { - return err + if err = initPlugin(ctx, sourcePbClient, sourceSpec.Spec, false); err != nil { + return fmt.Errorf("failed to init source %v: %w", sourceSpec.Name, err) } writeClients := make([]plugin.Plugin_WriteClient, len(destinationsPbClients)) diff --git a/cli/cmd/tables_v3.go b/cli/cmd/tables_v3.go index 4b1adc257fbdba..3a05217aca1584 100644 --- a/cli/cmd/tables_v3.go +++ b/cli/cmd/tables_v3.go @@ -20,9 +20,7 @@ func tablesV3(ctx context.Context, sourceClient *managedplugin.Client, path stri return err } sourcePbClient := pluginPb.NewPluginClient(sourceClient.Conn) - if _, err := sourcePbClient.Init(ctx, &pluginPb.Init_Request{ - NoConnection: true, - }); err != nil { + if err := initPlugin(ctx, sourcePbClient, nil, true); err != nil { return fmt.Errorf("failed to init source: %w", err) } getTablesResp, err := sourcePbClient.GetTables(ctx, &pluginPb.GetTables_Request{ diff --git a/cli/go.mod b/cli/go.mod index 53f5d188ffcbc2..9ff8099364d154 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -21,6 +21,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/rs/zerolog v1.31.0 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/schollz/progressbar/v3 v3.13.1 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.4 @@ -116,7 +117,6 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect From 38698faf80f4d92e988ad141be092e048016889b Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Thu, 8 Feb 2024 15:03:53 +0100 Subject: [PATCH 15/17] chore: Use redirect target to avoid the extra hop in dbt installation link (#16548) #### Summary Should fix https://github.com/cloudquery/cloudquery/actions/runs/7829313019