Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion plugins/source/mysql/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.20

require (
github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882
github.com/cloudquery/plugin-sdk/v4 v4.5.1
github.com/cloudquery/plugin-sdk/v4 v4.5.3
github.com/go-sql-driver/mysql v1.7.0
github.com/rs/zerolog v1.29.1
github.com/stretchr/testify v1.8.4
Expand Down
4 changes: 2 additions & 2 deletions plugins/source/mysql/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ github.com/cloudquery/plugin-pb-go v1.9.3 h1:A+TwhOHB68ZnjqrQYnVtWAzmUKEDnKleVG6
github.com/cloudquery/plugin-pb-go v1.9.3/go.mod h1:f00zd6V5mWD+8Qw9U0eb4HD8RnAobwV9byBexE7Qa+0=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
github.com/cloudquery/plugin-sdk/v4 v4.5.1 h1:V6EfHkW6kyYVMSX2S/Fn1b+62bU1jegIuv16Koo2f98=
github.com/cloudquery/plugin-sdk/v4 v4.5.1/go.mod h1:WNTjNp8CPDDSpJHOSaho+MUzntKO6czG6E+yvVe++aA=
github.com/cloudquery/plugin-sdk/v4 v4.5.3 h1:hIo2miaWqdfWl46hdq2I7XS0cqsKLerD4EQlQJ/o0sw=
github.com/cloudquery/plugin-sdk/v4 v4.5.3/go.mod h1:/2/9y6iODo8jOT0mrAp8K1eogr7+228cy53jK8aUCjY=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
Expand Down
23 changes: 11 additions & 12 deletions plugins/source/mysql/resources/plugin/plugin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func createTable(ctx context.Context, db *sql.DB, table *schema.Table) error {
return err
}

func insertTable(ctx context.Context, db *sql.DB, table *schema.Table, records []arrow.Record) error {
func insertTable(ctx context.Context, db *sql.DB, table *schema.Table, record arrow.Record) error {
sb := strings.Builder{}
sb.WriteString("INSERT INTO " + client.Identifier(table.Name))
sb.WriteString(" (")
Expand All @@ -99,16 +99,14 @@ func insertTable(ctx context.Context, db *sql.DB, table *schema.Table, records [
sb.WriteString(")")

query := sb.String()
for _, record := range records {
transformedRecords, err := client.TransformRecord(record)
if err != nil {
transformedRecords, err := client.TransformRecord(record)
if err != nil {
return err
}
for _, transformedRecord := range transformedRecords {
if _, err := db.ExecContext(ctx, query, transformedRecord...); err != nil {
return err
}
for _, transformedRecord := range transformedRecords {
if _, err := db.ExecContext(ctx, query, transformedRecord...); err != nil {
return err
}
}
}
return nil
}
Expand Down Expand Up @@ -150,8 +148,8 @@ func TestPlugin(t *testing.T) {
t.Fatal(err)
}
syncTime := time.Now()
expectedRecords := schema.NewTestDataGenerator().Generate(testTable, schema.GenTestDataOptions{MaxRows: 2, SyncTime: syncTime})
if err := insertTable(ctx, db, testTable, expectedRecords); err != nil {
writtenRecord := schema.NewTestDataGenerator().Generate(testTable, schema.GenTestDataOptions{MaxRows: 2, SyncTime: syncTime})
if err := insertTable(ctx, db, testTable, writtenRecord); err != nil {
t.Fatal(err)
}

Expand Down Expand Up @@ -196,7 +194,8 @@ func TestPlugin(t *testing.T) {

sortResults(testTable, actualRecords)

for recordIndex, expectedRecord := range expectedRecords {
for recordIndex := int64(0); recordIndex < writtenRecord.NumRows(); recordIndex++ {
expectedRecord := writtenRecord.NewSlice(recordIndex, recordIndex+1)
actualRecord := actualRecords[recordIndex]
if expectedRecord.NumCols() != actualRecord.NumCols() {
t.Fatalf("expected record %d to have %d columns, got %d", recordIndex, expectedRecord.NumCols(), actualRecord.NumCols())
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/oracledb/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.20

require (
github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882
github.com/cloudquery/plugin-sdk/v4 v4.5.1
github.com/cloudquery/plugin-sdk/v4 v4.5.3
github.com/rs/zerolog v1.29.1
github.com/sijms/go-ora/v2 v2.7.9
github.com/stretchr/testify v1.8.4
Expand Down
4 changes: 2 additions & 2 deletions plugins/source/oracledb/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ github.com/cloudquery/plugin-pb-go v1.9.3 h1:A+TwhOHB68ZnjqrQYnVtWAzmUKEDnKleVG6
github.com/cloudquery/plugin-pb-go v1.9.3/go.mod h1:f00zd6V5mWD+8Qw9U0eb4HD8RnAobwV9byBexE7Qa+0=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
github.com/cloudquery/plugin-sdk/v4 v4.5.1 h1:V6EfHkW6kyYVMSX2S/Fn1b+62bU1jegIuv16Koo2f98=
github.com/cloudquery/plugin-sdk/v4 v4.5.1/go.mod h1:WNTjNp8CPDDSpJHOSaho+MUzntKO6czG6E+yvVe++aA=
github.com/cloudquery/plugin-sdk/v4 v4.5.3 h1:hIo2miaWqdfWl46hdq2I7XS0cqsKLerD4EQlQJ/o0sw=
github.com/cloudquery/plugin-sdk/v4 v4.5.3/go.mod h1:/2/9y6iODo8jOT0mrAp8K1eogr7+228cy53jK8aUCjY=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
Expand Down
23 changes: 11 additions & 12 deletions plugins/source/oracledb/resources/plugin/plugin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func createTable(ctx context.Context, db *sql.DB, table *schema.Table) error {
return err
}

func insertTable(ctx context.Context, db *sql.DB, table *schema.Table, records []arrow.Record) error {
func insertTable(ctx context.Context, db *sql.DB, table *schema.Table, record arrow.Record) error {
builder := strings.Builder{}
builder.WriteString("INSERT INTO " + client.Identifier(table.Name))
builder.WriteString(" (")
Expand All @@ -110,16 +110,14 @@ func insertTable(ctx context.Context, db *sql.DB, table *schema.Table, records [
}
builder.WriteString(")")

for _, record := range records {
transformedRecords, err := client.TransformRecord(record)
if err != nil {
transformedRecords, err := client.TransformRecord(record)
if err != nil {
return err
}
for _, transformedRecord := range transformedRecords {
if _, err := db.ExecContext(ctx, builder.String(), transformedRecord...); err != nil {
return err
}
for _, transformedRecord := range transformedRecords {
if _, err := db.ExecContext(ctx, builder.String(), transformedRecord...); err != nil {
return err
}
}
}
return nil
}
Expand Down Expand Up @@ -171,8 +169,8 @@ func TestPlugin(t *testing.T) {
if err := createTable(ctx, db, testTable); err != nil {
t.Fatal(err)
}
expectedRecords := schema.NewTestDataGenerator().Generate(testTable, schema.GenTestDataOptions{MaxRows: 2})
if err := insertTable(ctx, db, testTable, expectedRecords); err != nil {
writtenRecord := schema.NewTestDataGenerator().Generate(testTable, schema.GenTestDataOptions{MaxRows: 2})
if err := insertTable(ctx, db, testTable, writtenRecord); err != nil {
t.Fatal(err)
}

Expand Down Expand Up @@ -218,7 +216,8 @@ func TestPlugin(t *testing.T) {

sortResults(testTable, actualRecords)

for recordIndex, expectedRecord := range expectedRecords {
for recordIndex := int64(0); recordIndex < writtenRecord.NumRows(); recordIndex++ {
expectedRecord := writtenRecord.NewSlice(recordIndex, recordIndex+1)
actualRecord := actualRecords[recordIndex]
if expectedRecord.NumCols() != actualRecord.NumCols() {
t.Fatalf("expected record %d to have %d columns, got %d", recordIndex, expectedRecord.NumCols(), actualRecord.NumCols())
Expand Down
2 changes: 1 addition & 1 deletion plugins/source/test/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.20

require (
github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882
github.com/cloudquery/plugin-sdk/v4 v4.5.1
github.com/cloudquery/plugin-sdk/v4 v4.5.3
github.com/rs/zerolog v1.29.1
)

Expand Down
4 changes: 2 additions & 2 deletions plugins/source/test/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ github.com/cloudquery/plugin-pb-go v1.9.3 h1:A+TwhOHB68ZnjqrQYnVtWAzmUKEDnKleVG6
github.com/cloudquery/plugin-pb-go v1.9.3/go.mod h1:f00zd6V5mWD+8Qw9U0eb4HD8RnAobwV9byBexE7Qa+0=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
github.com/cloudquery/plugin-sdk/v4 v4.5.1 h1:V6EfHkW6kyYVMSX2S/Fn1b+62bU1jegIuv16Koo2f98=
github.com/cloudquery/plugin-sdk/v4 v4.5.1/go.mod h1:WNTjNp8CPDDSpJHOSaho+MUzntKO6czG6E+yvVe++aA=
github.com/cloudquery/plugin-sdk/v4 v4.5.3 h1:hIo2miaWqdfWl46hdq2I7XS0cqsKLerD4EQlQJ/o0sw=
github.com/cloudquery/plugin-sdk/v4 v4.5.3/go.mod h1:/2/9y6iODo8jOT0mrAp8K1eogr7+228cy53jK8aUCjY=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
Expand Down
6 changes: 3 additions & 3 deletions plugins/source/test/resources/services/testdata.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,13 @@ func TestDataTable() *schema.Table {
data := tg.Generate(table, schema.GenTestDataOptions{
MaxRows: 1,
})
if len(data) != 1 {
if data.NumRows() != 1 {
panic("Expected 1 row of data")
}

dataAsMap := make(map[string]any)
for i, col := range data[0].Columns() {
dataAsMap[data[0].ColumnName(i)] = col.ValueStr(0)
for i, col := range data.Columns() {
dataAsMap[data.ColumnName(i)] = col.ValueStr(0)
}

table.Description = "Testdata table"
Expand Down