From 72c2cc0b8a1da4ed5285bcbf651da04ff4dc619f Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Sun, 26 Mar 2023 10:44:22 +0300 Subject: [PATCH 1/3] chore(deps): Update dependency golangci/golangci-lint to v1.52.1 (#736) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | [golangci/golangci-lint](https://togithub.com/golangci/golangci-lint) | minor | `v1.51.2` -> `v1.52.1` | --- ### Release Notes
golangci/golangci-lint ### [`v1.52.1`](https://togithub.com/golangci/golangci-lint/blob/HEAD/CHANGELOG.md#v1521) [Compare Source](https://togithub.com/golangci/golangci-lint/compare/v1.52.0...v1.52.1) 1. misc. - fix: improve panic management - fix: the combination of --fix and --path-prefix ### [`v1.52.0`](https://togithub.com/golangci/golangci-lint/blob/HEAD/CHANGELOG.md#v1520) [Compare Source](https://togithub.com/golangci/golangci-lint/compare/v1.51.2...v1.52.0) 1. updated linters - `asciicheck`: from 0.1.1 to 0.2.0 - `bidichk`: from 0.2.3 to 0.2.4 - `contextcheck`: from 1.1.3 to 1.1.4 - `dupword`: from 0.0.9 to 0.0.11 - `durationcheck`: from 0.0.9 to 0.0.10 - `errchkjson`: from 0.3.0 to 0.3.1 - `errname`: from 0.1.7 to 0.1.9 - `forbidigo`: from 1.4.0 to 1.5.1 - `gci`: from 0.9.1 to 0.10.1 - `ginkgolinter`: from 0.8.1 to 0.9.0 - `go-critic`: from 0.6.7 to 0.7.0 - `go-errorlint`: from 1.1.0 to 1.4.0 - `godox`: bump to HEAD - `lll`: skip go command - `loggercheck`: from 0.9.3 to 0.9.4 - `musttag`: from 0.4.5 to 0.5.0 - `nilnil`: from 0.1.1 to 0.1.3 - `noctx`: from 0.0.1 to 0.0.2 - `revive`: from 1.2.5 to 1.3.1 - `rowserrcheck`: remove limitation related to generics support - `staticcheck`: from 0.4.2 to 0.4.3 - `testpackage`: from 1.1.0 to 1.1.1 - `tparallel`: from 0.2.1 to 0.3.0 - `wastedassign`: remove limitation related to generics support - `wrapcheck`: from 2.8.0 to 2.8.1 2. misc. - Add TeamCity output format - Consider path prefix when matching path patterns - Add Go version to version information 3. Documentation - Add Tekton in Trusted By page - Clarify that custom linters are not enabled by default - Remove description for deprecated "go" option
--- ### Configuration 📅 **Schedule**: Branch creation - "before 3am on the first day of the month" (UTC), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://togithub.com/renovatebot/renovate). --- .github/workflows/lint_golang.yml | 2 +- caser/caser.go | 2 +- helpers/slice.go | 2 +- plugins/destination/plugin.go | 2 +- plugins/source/benchmark_test.go | 6 +++--- schema/array.go | 2 +- schema/cidr_array_test.go | 2 +- schema/inet_array_test.go | 2 +- schema/table_test.go | 2 +- schema/text_array.go | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/lint_golang.yml b/.github/workflows/lint_golang.yml index fcfe9f3d1a..0b9829b3c7 100644 --- a/.github/workflows/lint_golang.yml +++ b/.github/workflows/lint_golang.yml @@ -20,4 +20,4 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: - version: v1.51.2 + version: v1.52.1 diff --git a/caser/caser.go b/caser/caser.go index c52affbe24..84323d08df 100644 --- a/caser/caser.go +++ b/caser/caser.go @@ -106,7 +106,7 @@ func (c *Caser) ToSnake(s string) string { // append the last word if s[lastPos:] != "" { - //handle plurals of initialisms like CDNs, ARNs, IDs + // handle plurals of initialisms like CDNs, ARNs, IDs if w := s[lastPos:]; w == "s" { words[len(words)-1] = words[len(words)-1] + w } else { diff --git a/helpers/slice.go b/helpers/slice.go index 111e6ce6b9..c0fe019638 100644 --- a/helpers/slice.go +++ b/helpers/slice.go @@ -9,7 +9,7 @@ func InterfaceSlice(slice any) []any { return nil } s := reflect.ValueOf(slice) - //handle slice behind pointer + // handle slice behind pointer if s.Kind() == reflect.Ptr && s.Elem().Kind() == reflect.Slice { // Keep the distinction between nil and empty slice input if s.Elem().IsNil() { diff --git a/plugins/destination/plugin.go b/plugins/destination/plugin.go index 0721dcf161..490a8035b1 100644 --- a/plugins/destination/plugin.go +++ b/plugins/destination/plugin.go @@ -198,7 +198,7 @@ func (p *Plugin) readAll(ctx context.Context, table *schema.Table, sourceName st defer close(ch) readErr = p.Read(ctx, table, sourceName, ch) }() - //nolint:prealloc + // nolint:prealloc var resources []schema.CQTypes for resource := range ch { resources = append(resources, resource) diff --git a/plugins/source/benchmark_test.go b/plugins/source/benchmark_test.go index 955d21474b..9df1c7820a 100644 --- a/plugins/source/benchmark_test.go +++ b/plugins/source/benchmark_test.go @@ -222,9 +222,9 @@ func (s *Benchmark) Run() { s.b.ReportMetric(float64(totalResources)/(end.Sub(start).Seconds()), "resources/s") // Enable the below metrics for more verbose information about the scenario: - //s.b.ReportMetric(float64(s.apiCalls.Load())/(end.Sub(start).Seconds()), "api-calls/s") - //s.b.ReportMetric(float64(totalResources), "resources") - //s.b.ReportMetric(float64(s.apiCalls.Load()), "apiCalls") + // s.b.ReportMetric(float64(s.apiCalls.Load())/(end.Sub(start).Seconds()), "api-calls/s") + // s.b.ReportMetric(float64(totalResources), "resources") + // s.b.ReportMetric(float64(s.apiCalls.Load()), "apiCalls") } } diff --git a/schema/array.go b/schema/array.go index 0709c1f584..8486b0bd2a 100644 --- a/schema/array.go +++ b/schema/array.go @@ -17,7 +17,7 @@ type ArrayDimension struct { LowerBound int32 } -//nolint:unparam +// nolint:unparam func findDimensionsFromValue(value reflect.Value, dimensions []ArrayDimension, elementsLength int) ([]ArrayDimension, int, bool) { switch value.Kind() { case reflect.Array: diff --git a/schema/cidr_array_test.go b/schema/cidr_array_test.go index 9d06066708..1aa7a4d690 100644 --- a/schema/cidr_array_test.go +++ b/schema/cidr_array_test.go @@ -1,4 +1,4 @@ -//nolint:dupl +// nolint:dupl package schema import ( diff --git a/schema/inet_array_test.go b/schema/inet_array_test.go index 1ff3242197..3741a841b0 100644 --- a/schema/inet_array_test.go +++ b/schema/inet_array_test.go @@ -1,4 +1,4 @@ -//nolint:dupl +// nolint:dupl package schema import ( diff --git a/schema/table_test.go b/schema/table_test.go index 5b5df67f59..306dac51ca 100644 --- a/schema/table_test.go +++ b/schema/table_test.go @@ -224,7 +224,7 @@ func TestTablesFilterDFS(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { gotTables, err := tt.tables.FilterDfs(tt.configurationTables, tt.configurationSkipTables, tt.skipDependentTables) - //nolint:gocritic + // nolint:gocritic if err != nil && tt.err == "" { t.Errorf("got error %v, want nil", err) } else if err != nil && tt.err != "" && err.Error() != tt.err { diff --git a/schema/text_array.go b/schema/text_array.go index ed84e73fef..e6bb68a0ab 100644 --- a/schema/text_array.go +++ b/schema/text_array.go @@ -1,4 +1,4 @@ -//nolint:gocritic +// nolint:gocritic package schema import ( From d7649d80f1a15cac6b7a29b6d0458a83db68cc76 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 31 Mar 2023 10:37:25 +0300 Subject: [PATCH 2/3] fix(transform): Use path instead of field name for PK options (#739) While using `transformers.WithPrimaryKeys` option there is an odd behavior being observed, where the unwrapped struct field also gets promoted to be PK. Consider the following example: ``` type A struct { ID string B B } type B struct { ID string } ``` Currently, if `transformers.WithPrimaryKeys("ID")` option was used to transform struct `A` we would get the following columns: * `id`: PK, taken from `A` * `b_id`: PK, taken from `B` However, with the current change, the following becomes possible: | `transformers.WithPrimaryKeys` | `id` is PK | `b_id` is PK | | --- | --- | --- | | `"ID"` | `true` | `false` | | `"B.ID"` | `false` | `true` | | `"ID", "B.ID"` | `true` | `true` | --- transformers/struct.go | 7 ++- transformers/struct_test.go | 103 +++++++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/transformers/struct.go b/transformers/struct.go index 0c69dfbe78..6560008233 100644 --- a/transformers/struct.go +++ b/transformers/struct.go @@ -272,9 +272,12 @@ func (t *structTransformer) addColumnFromField(field reflect.StructField, parent } for _, pk := range t.pkFields { - if pk == field.Name { + if pk == path { + // use path to allow the following + // 1. Don't duplicate the PK fields if the unwrapped struct contains a fields with the same name + // 2. Allow specifying the nested unwrapped field as part of the PK. column.CreationOptions.PrimaryKey = true - t.pkFieldsFound = append(t.pkFieldsFound, field.Name) + t.pkFieldsFound = append(t.pkFieldsFound, pk) } } diff --git a/transformers/struct_test.go b/transformers/struct_test.go index 3091bd18b8..751e4f87ec 100644 --- a/transformers/struct_test.go +++ b/transformers/struct_test.go @@ -8,11 +8,13 @@ import ( "github.com/cloudquery/plugin-sdk/schema" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "golang.org/x/exp/slices" ) type ( embeddedStruct struct { EmbeddedString string + IntCol int `json:"int_col,omitempty"` } testStruct struct { @@ -45,10 +47,12 @@ type ( *embeddedStruct } testStructWithEmbeddedStruct struct { + IntCol int `json:"int_col,omitempty"` *testStruct *embeddedStruct } testStructWithNonEmbeddedStruct struct { + IntCol int `json:"int_col,omitempty"` TestStruct *testStruct NonEmbedded *embeddedStruct } @@ -140,16 +144,61 @@ var ( Columns: expectedColumns, } expectedTestTableEmbeddedStruct = schema.Table{ + Name: "test_struct", + Columns: append(expectedColumns, schema.Column{Name: "embedded_string", Type: schema.TypeString}), + } + expectedTestTableEmbeddedStructWithTopLevelPK = schema.Table{ + Name: "test_struct", + Columns: func(base schema.ColumnList) schema.ColumnList { + cols := slices.Clone(base) + cols = append(cols, schema.Column{Name: "embedded_string", Type: schema.TypeString}) + cols[cols.Index("int_col")].CreationOptions.PrimaryKey = true + return cols + }(expectedColumns), + } + expectedTestTableEmbeddedStructWithUnwrappedPK = schema.Table{ Name: "test_struct", Columns: append( expectedColumns, schema.Column{ - Name: "embedded_string", - Type: schema.TypeString, + Name: "embedded_string", + Type: schema.TypeString, + CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true}, }), } expectedTestTableNonEmbeddedStruct = schema.Table{ Name: "test_struct", Columns: schema.ColumnList{ + schema.Column{Name: "int_col", Type: schema.TypeInt}, + // Should not be unwrapped + schema.Column{Name: "test_struct", Type: schema.TypeJSON}, + // Should be unwrapped + schema.Column{Name: "non_embedded_embedded_string", Type: schema.TypeString}, + schema.Column{Name: "non_embedded_int_col", Type: schema.TypeInt}, + }, + } + expectedTestTableNonEmbeddedStructWithTopLevelPK = schema.Table{ + Name: "test_struct", + Columns: schema.ColumnList{ + schema.Column{ + Name: "int_col", + Type: schema.TypeInt, + CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true}, + }, + // Should not be unwrapped + schema.Column{Name: "test_struct", Type: schema.TypeJSON}, + // Should be unwrapped + schema.Column{ + Name: "non_embedded_embedded_string", + Type: schema.TypeString, + }, + schema.Column{Name: "non_embedded_int_col", Type: schema.TypeInt}, + }, + } + expectedTestTableNonEmbeddedStructWithUnwrappedPK = schema.Table{ + Name: "test_struct", + Columns: schema.ColumnList{ + // shouldn't be PK + schema.Column{Name: "int_col", Type: schema.TypeInt}, // Should not be unwrapped schema.Column{Name: "test_struct", Type: schema.TypeJSON}, // Should be unwrapped @@ -157,6 +206,12 @@ var ( Name: "non_embedded_embedded_string", Type: schema.TypeString, }, + // should be PK + schema.Column{ + Name: "non_embedded_int_col", + Type: schema.TypeInt, + CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true}, + }, }, } expectedTestSliceStruct = schema.Table{ @@ -219,6 +274,28 @@ func TestTableFromGoStruct(t *testing.T) { }, want: expectedTestTableEmbeddedStruct, }, + { + name: "should unwrap all embedded structs when option is set and use top-level field as PK", + args: args{ + testStruct: testStructWithEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapAllEmbeddedStructs(), + WithPrimaryKeys("IntCol"), + }, + }, + want: expectedTestTableEmbeddedStructWithTopLevelPK, + }, + { + name: "should unwrap all embedded structs when option is set and use its field as PK", + args: args{ + testStruct: testStructWithEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapAllEmbeddedStructs(), + WithPrimaryKeys("EmbeddedString"), + }, + }, + want: expectedTestTableEmbeddedStructWithUnwrappedPK, + }, { name: "should unwrap specific structs when option is set", args: args{ @@ -229,6 +306,28 @@ func TestTableFromGoStruct(t *testing.T) { }, want: expectedTestTableNonEmbeddedStruct, }, + { + name: "should unwrap specific structs when option is set and use top level field as PK", + args: args{ + testStruct: testStructWithNonEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapStructFields("NonEmbedded"), + WithPrimaryKeys("IntCol"), + }, + }, + want: expectedTestTableNonEmbeddedStructWithTopLevelPK, + }, + { + name: "should unwrap specific structs when option is set and use its field as PK", + args: args{ + testStruct: testStructWithNonEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapStructFields("NonEmbedded"), + WithPrimaryKeys("NonEmbedded.IntCol"), + }, + }, + want: expectedTestTableNonEmbeddedStructWithUnwrappedPK, + }, { name: "should generate table from slice struct", args: args{ From cbe9b27ed8dd72f04641526619c0e56524197087 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Fri, 31 Mar 2023 10:51:44 +0300 Subject: [PATCH 3/3] chore(main): Release v1.44.1 (#743) :robot: I have created a release *beep* *boop* --- ## [1.44.1](https://github.com/cloudquery/plugin-sdk/compare/v1.44.0...v1.44.1) (2023-03-31) ### Bug Fixes * **transform:** Use path instead of field name for PK options ([#739](https://github.com/cloudquery/plugin-sdk/issues/739)) ([d7649d8](https://github.com/cloudquery/plugin-sdk/commit/d7649d80f1a15cac6b7a29b6d0458a83db68cc76)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82033fadbf..76858130da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.44.1](https://github.com/cloudquery/plugin-sdk/compare/v1.44.0...v1.44.1) (2023-03-31) + + +### Bug Fixes + +* **transform:** Use path instead of field name for PK options ([#739](https://github.com/cloudquery/plugin-sdk/issues/739)) ([d7649d8](https://github.com/cloudquery/plugin-sdk/commit/d7649d80f1a15cac6b7a29b6d0458a83db68cc76)) + ## [1.44.0](https://github.com/cloudquery/plugin-sdk/compare/v1.43.0...v1.44.0) (2023-03-17)