From da2da8787e16f8d8772c3e1687a5ad6638d1af67 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:07:20 +0200 Subject: [PATCH 1/3] chore(deps): Update module github.com/cloudquery/plugin-sdk/v4 to v4.72.1 (#2016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [github.com/cloudquery/plugin-sdk/v4](https://togithub.com/cloudquery/plugin-sdk) | require | patch | `v4.72.0` -> `v4.72.1` | --- ### Release Notes
cloudquery/plugin-sdk (github.com/cloudquery/plugin-sdk/v4) ### [`v4.72.1`](https://togithub.com/cloudquery/plugin-sdk/releases/tag/v4.72.1) [Compare Source](https://togithub.com/cloudquery/plugin-sdk/compare/v4.72.0...v4.72.1) ##### Bug Fixes - **deps:** Update aws-sdk-go-v2 monorepo ([#​2007](https://togithub.com/cloudquery/plugin-sdk/issues/2007)) ([7f3818d](https://togithub.com/cloudquery/plugin-sdk/commit/7f3818d51a2d60bc7dc2a3846ef038a783d984bc)) - **deps:** Update module github.com/cloudquery/cloudquery-api-go to v1.13.5 ([#​2015](https://togithub.com/cloudquery/plugin-sdk/issues/2015)) ([9b6e9f2](https://togithub.com/cloudquery/plugin-sdk/commit/9b6e9f29ac3d165bf5470e933b8638a961b4bd64)) - **deps:** Update module github.com/cloudquery/plugin-pb-go to v1.26.1 ([#​2010](https://togithub.com/cloudquery/plugin-sdk/issues/2010)) ([b12dc10](https://togithub.com/cloudquery/plugin-sdk/commit/b12dc1033a5130629c4ff3eb76c233704df81747)) - **deps:** Update module golang.org/x/net to v0.33.0 \[SECURITY] ([#​2014](https://togithub.com/cloudquery/plugin-sdk/issues/2014)) ([7360bd2](https://togithub.com/cloudquery/plugin-sdk/commit/7360bd26d49e76f48182efdad8d75a07a95e0263)) - **deps:** Update module google.golang.org/grpc to v1.69.0 ([#​2008](https://togithub.com/cloudquery/plugin-sdk/issues/2008)) ([aae018f](https://togithub.com/cloudquery/plugin-sdk/commit/aae018f9838c80c3ff5f10ec6b47c41d809b4694)) - OpenTelemetry schema URL panic ([#​2012](https://togithub.com/cloudquery/plugin-sdk/issues/2012)) ([b616279](https://togithub.com/cloudquery/plugin-sdk/commit/b6162796a417cea0b8bb0efee8074917dce63415))
--- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://togithub.com/renovatebot/renovate). --- examples/simple_plugin/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple_plugin/go.mod b/examples/simple_plugin/go.mod index 153bd7027b..7936892c82 100644 --- a/examples/simple_plugin/go.mod +++ b/examples/simple_plugin/go.mod @@ -4,7 +4,7 @@ go 1.22.7 require ( github.com/apache/arrow-go/v18 v18.0.0 - github.com/cloudquery/plugin-sdk/v4 v4.72.0 + github.com/cloudquery/plugin-sdk/v4 v4.72.1 github.com/rs/zerolog v1.33.0 ) From 7ca8009bec4214928fdeb2473b7c04294ae7952e Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Thu, 19 Dec 2024 17:36:03 +0000 Subject: [PATCH 2/3] fix: Use field name in json type schema if json tag is missing (#2011) I'm working on validating some data (internal issue https://github.com/cloudquery/cloudquery-issues/issues/2971) using the json type schema and noticed a bug (noticeable on AWS where structs don't have json tags). We should not use the name transformer to get the name as it defaults to `ToSnake` when a json tag is missing. Instead we should mimic the JSON marshaling default behavior to use the field name as is. A better approach can be to create an instance of the type, marshal it to JSON, then use that for the names. Open to ideas how to do that using reflection, with a caveat that we might need to initialize it using non zero values otherwise those can be omitted --- --- transformers/name.go | 12 ++++++++++++ transformers/options.go | 8 ++++++++ transformers/struct.go | 14 ++++++++------ transformers/struct_test.go | 19 +++++++++++++++++++ 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/transformers/name.go b/transformers/name.go index 93faad20da..8f47fd2fb8 100644 --- a/transformers/name.go +++ b/transformers/name.go @@ -26,4 +26,16 @@ func DefaultNameTransformer(field reflect.StructField) (string, error) { return defaultCaser.ToSnake(name), nil } +func DefaultJSONColumnSchemaNameTransformer(field reflect.StructField) (string, error) { + name := field.Name + if jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]; len(jsonTag) > 0 { + // return empty string if the field is not related api response + if jsonTag == "-" { + return "", nil + } + return jsonTag, nil + } + return name, nil +} + var _ NameTransformer = DefaultNameTransformer diff --git a/transformers/options.go b/transformers/options.go index 779a5791c0..ffeec142b3 100644 --- a/transformers/options.go +++ b/transformers/options.go @@ -31,6 +31,14 @@ func WithNameTransformer(transformer NameTransformer) StructTransformerOption { } } +// WithJSONSchemaNameTransformer overrides how column name will be determined. +// DefaultJSONColumnSchemaNameTransformer is used as the default. +func WithJSONSchemaNameTransformer(transformer NameTransformer) StructTransformerOption { + return func(t *structTransformer) { + t.jsonSchemaNameTransformer = transformer + } +} + // WithTypeTransformer overrides how column type will be determined. // DefaultTypeTransformer is used as the default. func WithTypeTransformer(transformer TypeTransformer) StructTransformerOption { diff --git a/transformers/struct.go b/transformers/struct.go index a85d22aebf..9435c6098a 100644 --- a/transformers/struct.go +++ b/transformers/struct.go @@ -29,6 +29,7 @@ type structTransformer struct { pkFieldsFound []string pkComponentFields []string pkComponentFieldsFound []string + jsonSchemaNameTransformer NameTransformer maxJSONTypeSchemaDepth int } @@ -194,11 +195,12 @@ func (t *structTransformer) addColumnFromField(field reflect.StructField, parent func TransformWithStruct(st any, opts ...StructTransformerOption) schema.Transform { t := &structTransformer{ - nameTransformer: DefaultNameTransformer, - typeTransformer: DefaultTypeTransformer, - resolverTransformer: DefaultResolverTransformer, - ignoreInTestsTransformer: DefaultIgnoreInTestsTransformer, - maxJSONTypeSchemaDepth: DefaultMaxJSONTypeSchemaDepth, + nameTransformer: DefaultNameTransformer, + typeTransformer: DefaultTypeTransformer, + resolverTransformer: DefaultResolverTransformer, + ignoreInTestsTransformer: DefaultIgnoreInTestsTransformer, + jsonSchemaNameTransformer: DefaultJSONColumnSchemaNameTransformer, + maxJSONTypeSchemaDepth: DefaultMaxJSONTypeSchemaDepth, } for _, opt := range opts { opt(t) @@ -284,7 +286,7 @@ func (t *structTransformer) fieldToJSONSchema(field reflect.StructField, depth i if !structField.IsExported() || isTypeIgnored(structField.Type) { continue } - name, err := t.nameTransformer(structField) + name, err := t.jsonSchemaNameTransformer(structField) if err != nil { continue } diff --git a/transformers/struct_test.go b/transformers/struct_test.go index 81234323ca..281ce34eff 100644 --- a/transformers/struct_test.go +++ b/transformers/struct_test.go @@ -12,6 +12,7 @@ import ( "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/types" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" ) type ( @@ -665,6 +666,23 @@ func TestJSONTypeSchema(t *testing.T) { "item": `{"exported":"utf8"}`, }, }, + { + name: "no json tags", + testStruct: struct { + Tags map[string]string + Item struct { + Name string + Tags map[string]string + FlatItems []string + ComplexItems []struct { + Name string + } + } + }{}, + want: map[string]string{ + "item": `{"ComplexItems":[{"Name":"utf8"}],"FlatItems":["utf8"],"Name":"utf8","Tags":{"utf8":"utf8"}}`, + }, + }, } for _, tt := range tests { @@ -684,6 +702,7 @@ func TestJSONTypeSchema(t *testing.T) { } for col, schema := range tt.want { column := table.Column(col) + require.NotNil(t, column, "column %q not found", col) if diff := cmp.Diff(column.TypeSchema, schema); diff != "" { t.Fatalf("table does not match expected. diff (-got, +want): %v", diff) } From 38f8fdc5d1dfed32bda61dad1012eba8c05443d3 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Thu, 19 Dec 2024 19:41:34 +0200 Subject: [PATCH 3/3] chore(main): Release v4.72.2 (#2017) :robot: I have created a release *beep* *boop* --- ## [4.72.2](https://github.com/cloudquery/plugin-sdk/compare/v4.72.1...v4.72.2) (2024-12-19) ### Bug Fixes * Use field name in json type schema if json tag is missing ([#2011](https://github.com/cloudquery/plugin-sdk/issues/2011)) ([7ca8009](https://github.com/cloudquery/plugin-sdk/commit/7ca8009bec4214928fdeb2473b7c04294ae7952e)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- .release-please-manifest.json | 2 +- CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index c2c361a60a..072c82906d 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "4.72.1" + ".": "4.72.2" } diff --git a/CHANGELOG.md b/CHANGELOG.md index a8a9c692c8..3d1c299172 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [4.72.2](https://github.com/cloudquery/plugin-sdk/compare/v4.72.1...v4.72.2) (2024-12-19) + + +### Bug Fixes + +* Use field name in json type schema if json tag is missing ([#2011](https://github.com/cloudquery/plugin-sdk/issues/2011)) ([7ca8009](https://github.com/cloudquery/plugin-sdk/commit/7ca8009bec4214928fdeb2473b7c04294ae7952e)) + ## [4.72.1](https://github.com/cloudquery/plugin-sdk/compare/v4.72.0...v4.72.1) (2024-12-19)