diff --git a/.github/workflows/lint_golang.yml b/.github/workflows/lint_golang.yml index fcfe9f3d1a..0b9829b3c7 100644 --- a/.github/workflows/lint_golang.yml +++ b/.github/workflows/lint_golang.yml @@ -20,4 +20,4 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: - version: v1.51.2 + version: v1.52.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 82033fadbf..76858130da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.44.1](https://github.com/cloudquery/plugin-sdk/compare/v1.44.0...v1.44.1) (2023-03-31) + + +### Bug Fixes + +* **transform:** Use path instead of field name for PK options ([#739](https://github.com/cloudquery/plugin-sdk/issues/739)) ([d7649d8](https://github.com/cloudquery/plugin-sdk/commit/d7649d80f1a15cac6b7a29b6d0458a83db68cc76)) + ## [1.44.0](https://github.com/cloudquery/plugin-sdk/compare/v1.43.0...v1.44.0) (2023-03-17) diff --git a/caser/caser.go b/caser/caser.go index c52affbe24..84323d08df 100644 --- a/caser/caser.go +++ b/caser/caser.go @@ -106,7 +106,7 @@ func (c *Caser) ToSnake(s string) string { // append the last word if s[lastPos:] != "" { - //handle plurals of initialisms like CDNs, ARNs, IDs + // handle plurals of initialisms like CDNs, ARNs, IDs if w := s[lastPos:]; w == "s" { words[len(words)-1] = words[len(words)-1] + w } else { diff --git a/helpers/slice.go b/helpers/slice.go index 111e6ce6b9..c0fe019638 100644 --- a/helpers/slice.go +++ b/helpers/slice.go @@ -9,7 +9,7 @@ func InterfaceSlice(slice any) []any { return nil } s := reflect.ValueOf(slice) - //handle slice behind pointer + // handle slice behind pointer if s.Kind() == reflect.Ptr && s.Elem().Kind() == reflect.Slice { // Keep the distinction between nil and empty slice input if s.Elem().IsNil() { diff --git a/plugins/destination/plugin.go b/plugins/destination/plugin.go index 0721dcf161..490a8035b1 100644 --- a/plugins/destination/plugin.go +++ b/plugins/destination/plugin.go @@ -198,7 +198,7 @@ func (p *Plugin) readAll(ctx context.Context, table *schema.Table, sourceName st defer close(ch) readErr = p.Read(ctx, table, sourceName, ch) }() - //nolint:prealloc + // nolint:prealloc var resources []schema.CQTypes for resource := range ch { resources = append(resources, resource) diff --git a/plugins/source/benchmark_test.go b/plugins/source/benchmark_test.go index 955d21474b..9df1c7820a 100644 --- a/plugins/source/benchmark_test.go +++ b/plugins/source/benchmark_test.go @@ -222,9 +222,9 @@ func (s *Benchmark) Run() { s.b.ReportMetric(float64(totalResources)/(end.Sub(start).Seconds()), "resources/s") // Enable the below metrics for more verbose information about the scenario: - //s.b.ReportMetric(float64(s.apiCalls.Load())/(end.Sub(start).Seconds()), "api-calls/s") - //s.b.ReportMetric(float64(totalResources), "resources") - //s.b.ReportMetric(float64(s.apiCalls.Load()), "apiCalls") + // s.b.ReportMetric(float64(s.apiCalls.Load())/(end.Sub(start).Seconds()), "api-calls/s") + // s.b.ReportMetric(float64(totalResources), "resources") + // s.b.ReportMetric(float64(s.apiCalls.Load()), "apiCalls") } } diff --git a/schema/array.go b/schema/array.go index 0709c1f584..8486b0bd2a 100644 --- a/schema/array.go +++ b/schema/array.go @@ -17,7 +17,7 @@ type ArrayDimension struct { LowerBound int32 } -//nolint:unparam +// nolint:unparam func findDimensionsFromValue(value reflect.Value, dimensions []ArrayDimension, elementsLength int) ([]ArrayDimension, int, bool) { switch value.Kind() { case reflect.Array: diff --git a/schema/cidr_array_test.go b/schema/cidr_array_test.go index 9d06066708..1aa7a4d690 100644 --- a/schema/cidr_array_test.go +++ b/schema/cidr_array_test.go @@ -1,4 +1,4 @@ -//nolint:dupl +// nolint:dupl package schema import ( diff --git a/schema/inet_array_test.go b/schema/inet_array_test.go index 1ff3242197..3741a841b0 100644 --- a/schema/inet_array_test.go +++ b/schema/inet_array_test.go @@ -1,4 +1,4 @@ -//nolint:dupl +// nolint:dupl package schema import ( diff --git a/schema/table_test.go b/schema/table_test.go index 5b5df67f59..306dac51ca 100644 --- a/schema/table_test.go +++ b/schema/table_test.go @@ -224,7 +224,7 @@ func TestTablesFilterDFS(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { gotTables, err := tt.tables.FilterDfs(tt.configurationTables, tt.configurationSkipTables, tt.skipDependentTables) - //nolint:gocritic + // nolint:gocritic if err != nil && tt.err == "" { t.Errorf("got error %v, want nil", err) } else if err != nil && tt.err != "" && err.Error() != tt.err { diff --git a/schema/text_array.go b/schema/text_array.go index ed84e73fef..e6bb68a0ab 100644 --- a/schema/text_array.go +++ b/schema/text_array.go @@ -1,4 +1,4 @@ -//nolint:gocritic +// nolint:gocritic package schema import ( diff --git a/transformers/struct.go b/transformers/struct.go index 0c69dfbe78..6560008233 100644 --- a/transformers/struct.go +++ b/transformers/struct.go @@ -272,9 +272,12 @@ func (t *structTransformer) addColumnFromField(field reflect.StructField, parent } for _, pk := range t.pkFields { - if pk == field.Name { + if pk == path { + // use path to allow the following + // 1. Don't duplicate the PK fields if the unwrapped struct contains a fields with the same name + // 2. Allow specifying the nested unwrapped field as part of the PK. column.CreationOptions.PrimaryKey = true - t.pkFieldsFound = append(t.pkFieldsFound, field.Name) + t.pkFieldsFound = append(t.pkFieldsFound, pk) } } diff --git a/transformers/struct_test.go b/transformers/struct_test.go index 3091bd18b8..751e4f87ec 100644 --- a/transformers/struct_test.go +++ b/transformers/struct_test.go @@ -8,11 +8,13 @@ import ( "github.com/cloudquery/plugin-sdk/schema" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "golang.org/x/exp/slices" ) type ( embeddedStruct struct { EmbeddedString string + IntCol int `json:"int_col,omitempty"` } testStruct struct { @@ -45,10 +47,12 @@ type ( *embeddedStruct } testStructWithEmbeddedStruct struct { + IntCol int `json:"int_col,omitempty"` *testStruct *embeddedStruct } testStructWithNonEmbeddedStruct struct { + IntCol int `json:"int_col,omitempty"` TestStruct *testStruct NonEmbedded *embeddedStruct } @@ -140,16 +144,61 @@ var ( Columns: expectedColumns, } expectedTestTableEmbeddedStruct = schema.Table{ + Name: "test_struct", + Columns: append(expectedColumns, schema.Column{Name: "embedded_string", Type: schema.TypeString}), + } + expectedTestTableEmbeddedStructWithTopLevelPK = schema.Table{ + Name: "test_struct", + Columns: func(base schema.ColumnList) schema.ColumnList { + cols := slices.Clone(base) + cols = append(cols, schema.Column{Name: "embedded_string", Type: schema.TypeString}) + cols[cols.Index("int_col")].CreationOptions.PrimaryKey = true + return cols + }(expectedColumns), + } + expectedTestTableEmbeddedStructWithUnwrappedPK = schema.Table{ Name: "test_struct", Columns: append( expectedColumns, schema.Column{ - Name: "embedded_string", - Type: schema.TypeString, + Name: "embedded_string", + Type: schema.TypeString, + CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true}, }), } expectedTestTableNonEmbeddedStruct = schema.Table{ Name: "test_struct", Columns: schema.ColumnList{ + schema.Column{Name: "int_col", Type: schema.TypeInt}, + // Should not be unwrapped + schema.Column{Name: "test_struct", Type: schema.TypeJSON}, + // Should be unwrapped + schema.Column{Name: "non_embedded_embedded_string", Type: schema.TypeString}, + schema.Column{Name: "non_embedded_int_col", Type: schema.TypeInt}, + }, + } + expectedTestTableNonEmbeddedStructWithTopLevelPK = schema.Table{ + Name: "test_struct", + Columns: schema.ColumnList{ + schema.Column{ + Name: "int_col", + Type: schema.TypeInt, + CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true}, + }, + // Should not be unwrapped + schema.Column{Name: "test_struct", Type: schema.TypeJSON}, + // Should be unwrapped + schema.Column{ + Name: "non_embedded_embedded_string", + Type: schema.TypeString, + }, + schema.Column{Name: "non_embedded_int_col", Type: schema.TypeInt}, + }, + } + expectedTestTableNonEmbeddedStructWithUnwrappedPK = schema.Table{ + Name: "test_struct", + Columns: schema.ColumnList{ + // shouldn't be PK + schema.Column{Name: "int_col", Type: schema.TypeInt}, // Should not be unwrapped schema.Column{Name: "test_struct", Type: schema.TypeJSON}, // Should be unwrapped @@ -157,6 +206,12 @@ var ( Name: "non_embedded_embedded_string", Type: schema.TypeString, }, + // should be PK + schema.Column{ + Name: "non_embedded_int_col", + Type: schema.TypeInt, + CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true}, + }, }, } expectedTestSliceStruct = schema.Table{ @@ -219,6 +274,28 @@ func TestTableFromGoStruct(t *testing.T) { }, want: expectedTestTableEmbeddedStruct, }, + { + name: "should unwrap all embedded structs when option is set and use top-level field as PK", + args: args{ + testStruct: testStructWithEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapAllEmbeddedStructs(), + WithPrimaryKeys("IntCol"), + }, + }, + want: expectedTestTableEmbeddedStructWithTopLevelPK, + }, + { + name: "should unwrap all embedded structs when option is set and use its field as PK", + args: args{ + testStruct: testStructWithEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapAllEmbeddedStructs(), + WithPrimaryKeys("EmbeddedString"), + }, + }, + want: expectedTestTableEmbeddedStructWithUnwrappedPK, + }, { name: "should unwrap specific structs when option is set", args: args{ @@ -229,6 +306,28 @@ func TestTableFromGoStruct(t *testing.T) { }, want: expectedTestTableNonEmbeddedStruct, }, + { + name: "should unwrap specific structs when option is set and use top level field as PK", + args: args{ + testStruct: testStructWithNonEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapStructFields("NonEmbedded"), + WithPrimaryKeys("IntCol"), + }, + }, + want: expectedTestTableNonEmbeddedStructWithTopLevelPK, + }, + { + name: "should unwrap specific structs when option is set and use its field as PK", + args: args{ + testStruct: testStructWithNonEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapStructFields("NonEmbedded"), + WithPrimaryKeys("NonEmbedded.IntCol"), + }, + }, + want: expectedTestTableNonEmbeddedStructWithUnwrappedPK, + }, { name: "should generate table from slice struct", args: args{