From 99e2a2efa21aa02eba02a2e2da79f0534d95b82e Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Wed, 29 Mar 2023 12:17:45 +0300 Subject: [PATCH 1/3] use path for PK check --- transformers/struct.go | 7 ++++-- transformers/struct_test.go | 47 +++++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/transformers/struct.go b/transformers/struct.go index 0c69dfbe78..6560008233 100644 --- a/transformers/struct.go +++ b/transformers/struct.go @@ -272,9 +272,12 @@ func (t *structTransformer) addColumnFromField(field reflect.StructField, parent } for _, pk := range t.pkFields { - if pk == field.Name { + if pk == path { + // use path to allow the following + // 1. Don't duplicate the PK fields if the unwrapped struct contains a fields with the same name + // 2. Allow specifying the nested unwrapped field as part of the PK. column.CreationOptions.PrimaryKey = true - t.pkFieldsFound = append(t.pkFieldsFound, field.Name) + t.pkFieldsFound = append(t.pkFieldsFound, pk) } } diff --git a/transformers/struct_test.go b/transformers/struct_test.go index 3091bd18b8..dae4dac890 100644 --- a/transformers/struct_test.go +++ b/transformers/struct_test.go @@ -13,6 +13,7 @@ import ( type ( embeddedStruct struct { EmbeddedString string + IntCol int `json:"int_col,omitempty"` } testStruct struct { @@ -140,14 +141,29 @@ var ( Columns: expectedColumns, } expectedTestTableEmbeddedStruct = schema.Table{ + Name: "test_struct", + Columns: append(expectedColumns, schema.Column{Name: "embedded_string", Type: schema.TypeString}), + } + expectedTestTableEmbeddedStructWithPK = schema.Table{ Name: "test_struct", Columns: append( expectedColumns, schema.Column{ - Name: "embedded_string", - Type: schema.TypeString, + Name: "embedded_string", + Type: schema.TypeString, + CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true}, }), } expectedTestTableNonEmbeddedStruct = schema.Table{ + Name: "test_struct", + Columns: schema.ColumnList{ + // Should not be unwrapped + schema.Column{Name: "test_struct", Type: schema.TypeJSON}, + // Should be unwrapped + schema.Column{Name: "non_embedded_embedded_string", Type: schema.TypeString}, + schema.Column{Name: "non_embedded_int_col", Type: schema.TypeInt}, + }, + } + expectedTestTableNonEmbeddedStructWithPK = schema.Table{ Name: "test_struct", Columns: schema.ColumnList{ // Should not be unwrapped @@ -157,6 +173,11 @@ var ( Name: "non_embedded_embedded_string", Type: schema.TypeString, }, + schema.Column{ + Name: "non_embedded_int_col", + Type: schema.TypeInt, + CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true}, + }, }, } expectedTestSliceStruct = schema.Table{ @@ -219,6 +240,17 @@ func TestTableFromGoStruct(t *testing.T) { }, want: expectedTestTableEmbeddedStruct, }, + { + name: "should unwrap all embedded structs when option is set and use its field as PK", + args: args{ + testStruct: testStructWithEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapAllEmbeddedStructs(), + WithPrimaryKeys("EmbeddedString"), + }, + }, + want: expectedTestTableEmbeddedStructWithPK, + }, { name: "should unwrap specific structs when option is set", args: args{ @@ -229,6 +261,17 @@ func TestTableFromGoStruct(t *testing.T) { }, want: expectedTestTableNonEmbeddedStruct, }, + { + name: "should unwrap specific structs when option is set and use its field as PK", + args: args{ + testStruct: testStructWithNonEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapStructFields("NonEmbedded"), + WithPrimaryKeys("NonEmbedded.IntCol"), + }, + }, + want: expectedTestTableNonEmbeddedStructWithPK, + }, { name: "should generate table from slice struct", args: args{ From e5da956ae595448014bf8924533065ce8b4435bd Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Wed, 29 Mar 2023 18:13:50 +0300 Subject: [PATCH 2/3] extra test --- transformers/struct_test.go | 39 +++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/transformers/struct_test.go b/transformers/struct_test.go index dae4dac890..3f3605feb1 100644 --- a/transformers/struct_test.go +++ b/transformers/struct_test.go @@ -46,10 +46,12 @@ type ( *embeddedStruct } testStructWithEmbeddedStruct struct { + IntCol int `json:"int_col,omitempty"` *testStruct *embeddedStruct } testStructWithNonEmbeddedStruct struct { + IntCol int `json:"int_col,omitempty"` TestStruct *testStruct NonEmbedded *embeddedStruct } @@ -156,6 +158,7 @@ var ( expectedTestTableNonEmbeddedStruct = schema.Table{ Name: "test_struct", Columns: schema.ColumnList{ + schema.Column{Name: "int_col", Type: schema.TypeInt}, // Should not be unwrapped schema.Column{Name: "test_struct", Type: schema.TypeJSON}, // Should be unwrapped @@ -163,9 +166,29 @@ var ( schema.Column{Name: "non_embedded_int_col", Type: schema.TypeInt}, }, } - expectedTestTableNonEmbeddedStructWithPK = schema.Table{ + expectedTestTableNonEmbeddedStructWithTopLevelPK = schema.Table{ Name: "test_struct", Columns: schema.ColumnList{ + schema.Column{ + Name: "int_col", + Type: schema.TypeInt, + CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true}, + }, + // Should not be unwrapped + schema.Column{Name: "test_struct", Type: schema.TypeJSON}, + // Should be unwrapped + schema.Column{ + Name: "non_embedded_embedded_string", + Type: schema.TypeString, + }, + schema.Column{Name: "non_embedded_int_col", Type: schema.TypeInt}, + }, + } + expectedTestTableNonEmbeddedStructWithUnwrappedPK = schema.Table{ + Name: "test_struct", + Columns: schema.ColumnList{ + // shouldn't be PK + schema.Column{Name: "int_col", Type: schema.TypeInt}, // Should not be unwrapped schema.Column{Name: "test_struct", Type: schema.TypeJSON}, // Should be unwrapped @@ -173,6 +196,7 @@ var ( Name: "non_embedded_embedded_string", Type: schema.TypeString, }, + // should be PK schema.Column{ Name: "non_embedded_int_col", Type: schema.TypeInt, @@ -261,6 +285,17 @@ func TestTableFromGoStruct(t *testing.T) { }, want: expectedTestTableNonEmbeddedStruct, }, + { + name: "should unwrap specific structs when option is set and use top level field as PK", + args: args{ + testStruct: testStructWithNonEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapStructFields("NonEmbedded"), + WithPrimaryKeys("IntCol"), + }, + }, + want: expectedTestTableNonEmbeddedStructWithTopLevelPK, + }, { name: "should unwrap specific structs when option is set and use its field as PK", args: args{ @@ -270,7 +305,7 @@ func TestTableFromGoStruct(t *testing.T) { WithPrimaryKeys("NonEmbedded.IntCol"), }, }, - want: expectedTestTableNonEmbeddedStructWithPK, + want: expectedTestTableNonEmbeddedStructWithUnwrappedPK, }, { name: "should generate table from slice struct", From 0157878ce8c8f68d845dff5628f4b5712ed86f72 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Wed, 29 Mar 2023 22:55:56 +0300 Subject: [PATCH 3/3] add embedded col test example --- transformers/struct_test.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/transformers/struct_test.go b/transformers/struct_test.go index 3f3605feb1..751e4f87ec 100644 --- a/transformers/struct_test.go +++ b/transformers/struct_test.go @@ -8,6 +8,7 @@ import ( "github.com/cloudquery/plugin-sdk/schema" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "golang.org/x/exp/slices" ) type ( @@ -146,7 +147,16 @@ var ( Name: "test_struct", Columns: append(expectedColumns, schema.Column{Name: "embedded_string", Type: schema.TypeString}), } - expectedTestTableEmbeddedStructWithPK = schema.Table{ + expectedTestTableEmbeddedStructWithTopLevelPK = schema.Table{ + Name: "test_struct", + Columns: func(base schema.ColumnList) schema.ColumnList { + cols := slices.Clone(base) + cols = append(cols, schema.Column{Name: "embedded_string", Type: schema.TypeString}) + cols[cols.Index("int_col")].CreationOptions.PrimaryKey = true + return cols + }(expectedColumns), + } + expectedTestTableEmbeddedStructWithUnwrappedPK = schema.Table{ Name: "test_struct", Columns: append( expectedColumns, schema.Column{ @@ -264,6 +274,17 @@ func TestTableFromGoStruct(t *testing.T) { }, want: expectedTestTableEmbeddedStruct, }, + { + name: "should unwrap all embedded structs when option is set and use top-level field as PK", + args: args{ + testStruct: testStructWithEmbeddedStruct{}, + options: []StructTransformerOption{ + WithUnwrapAllEmbeddedStructs(), + WithPrimaryKeys("IntCol"), + }, + }, + want: expectedTestTableEmbeddedStructWithTopLevelPK, + }, { name: "should unwrap all embedded structs when option is set and use its field as PK", args: args{ @@ -273,7 +294,7 @@ func TestTableFromGoStruct(t *testing.T) { WithPrimaryKeys("EmbeddedString"), }, }, - want: expectedTestTableEmbeddedStructWithPK, + want: expectedTestTableEmbeddedStructWithUnwrappedPK, }, { name: "should unwrap specific structs when option is set",