diff --git a/.github/pr_labeler.yml b/.github/pr_labeler.yml index da723d5ed7fbab..a00f508d05e1dc 100644 --- a/.github/pr_labeler.yml +++ b/.github/pr_labeler.yml @@ -54,6 +54,8 @@ area/plugin/source/typeform: - plugins/source/typeform/**/* area/plugin/source/xkcd: - plugins/source/xkcd/**/* +area/plugin/transform/basic: + - plugins/transform/basic/**/* area/scaffold: - scaffold/**/* area/website: diff --git a/.github/workflows/transf_basic.yml b/.github/workflows/transf_basic.yml new file mode 100644 index 00000000000000..b1088aba95476d --- /dev/null +++ b/.github/workflows/transf_basic.yml @@ -0,0 +1,47 @@ +name: Transformer Plugin Basic Workflow + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + paths: + - "plugins/transformer/basic/**" + - ".github/workflows/transf_basic.yml" + push: + branches: + - main + paths: + - "plugins/transformer/basic/**" + - ".github/workflows/transf_basic.yml" + +jobs: + plugins-transformer-basic: + timeout-minutes: 30 + name: "plugins/transformer/basic" + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./plugins/transformer/basic + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version-file: plugins/transformer/basic/go.mod + cache: true + cache-dependency-path: plugins/transformer/basic/go.sum + - name: golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: v1.55.2 + working-directory: plugins/transformer/basic + args: "--config ../../.golangci.yml" + - name: Build + run: go build . + - name: Test + run: make test + \ No newline at end of file diff --git a/CODEOWNERS b/CODEOWNERS index 3547fbd856e1b5..1e2971788cc11d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,6 +1,7 @@ * @cloudquery/cloudquery-framework .github/workflows/source_* @cloudquery/cloudquery-plugins .github/workflows/dest_* @cloudquery/cloudquery-plugins +.github/workflows/transf_* @cloudquery/cloudquery-plugins plugins/** @cloudquery/cloudquery-plugins **/go.mod diff --git a/plugins/transformer/basic/.gitignore b/plugins/transformer/basic/.gitignore new file mode 100644 index 00000000000000..15a13db4511332 --- /dev/null +++ b/plugins/transformer/basic/.gitignore @@ -0,0 +1 @@ +basic diff --git a/plugins/transformer/basic/Makefile b/plugins/transformer/basic/Makefile new file mode 100644 index 00000000000000..15799c88362d63 --- /dev/null +++ b/plugins/transformer/basic/Makefile @@ -0,0 +1,8 @@ +.PHONY: test +test: + go test -race -timeout 3m ./... + +.PHONY: lint +lint: + golangci-lint run --config ../../.golangci.yml + diff --git a/plugins/transformer/basic/README.md b/plugins/transformer/basic/README.md new file mode 100644 index 00000000000000..10b2c3c94df4d2 --- /dev/null +++ b/plugins/transformer/basic/README.md @@ -0,0 +1,3 @@ +# CloudQuery Basic Transformer Plugin + +This transformer plugin applies basic transformations to an Arrow record diff --git a/plugins/transformer/basic/client/client.go b/plugins/transformer/basic/client/client.go new file mode 100644 index 00000000000000..4224e489cb49f5 --- /dev/null +++ b/plugins/transformer/basic/client/client.go @@ -0,0 +1,88 @@ +package client + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/cloudquery/cloudquery/plugins/transformer/basic/client/spec" + "github.com/cloudquery/cloudquery/plugins/transformer/basic/client/transformers" + "github.com/cloudquery/plugin-sdk/v4/plugin" + "github.com/rs/zerolog" +) + +type Client struct { + plugin.UnimplementedSource + plugin.UnimplementedDestination + + logger zerolog.Logger + spec spec.Spec + tfs []*transformers.Transformer +} + +func New(_ context.Context, logger zerolog.Logger, s []byte, opts plugin.NewClientOptions) (plugin.Client, error) { + c := &Client{ + logger: logger.With().Str("module", opts.PluginMeta.Name).Logger(), + } + if opts.NoConnection { + return c, nil + } + + if err := json.Unmarshal(s, &c.spec); err != nil { + return nil, fmt.Errorf("failed to unmarshal file spec: %w", err) + } + c.spec.SetDefaults() + if err := c.spec.Validate(); err != nil { + return nil, err + } + + for _, transformationSpec := range c.spec.TransformationSpecs { + tf, err := transformers.NewFromSpec(transformationSpec) + if err != nil { + return nil, err + } + c.tfs = append(c.tfs, tf) + } + + return c, nil +} + +func (c *Client) Transform(ctx context.Context, recvRecords <-chan arrow.Record, sendRecords chan<- arrow.Record) error { + for { + select { + case record, ok := <-recvRecords: + if !ok { + return nil + } + + // Run all transformers sequentially on the record + for _, tf := range c.tfs { + var err error + record, err = tf.Transform(record) + if err != nil { + return err + } + } + + sendRecords <- record + case <-ctx.Done(): + return nil + } + } +} + +func (c *Client) TransformSchema(ctx context.Context, schema *arrow.Schema) (*arrow.Schema, error) { + for _, tf := range c.tfs { + var err error + schema, err = tf.TransformSchema(schema) + if err != nil { + return nil, err + } + } + return schema, nil +} + +func (*Client) Close(ctx context.Context) error { + return nil +} diff --git a/plugins/transformer/basic/client/recordupdater/record_updater.go b/plugins/transformer/basic/client/recordupdater/record_updater.go new file mode 100644 index 00000000000000..e02e3af75a04e1 --- /dev/null +++ b/plugins/transformer/basic/client/recordupdater/record_updater.go @@ -0,0 +1,142 @@ +package recordupdater + +import ( + "crypto/sha256" + "fmt" + "strings" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/memory" + "github.com/cloudquery/cloudquery/plugins/transformer/basic/client/schemaupdater" +) + +// RecordUpdater takes an `arrow.Record` and knows how to make simple subsequent changes to it. +// It doesn't know which table it belongs to or if the changes make sense. +type RecordUpdater struct { + record arrow.Record + schemaUpdater *schemaupdater.SchemaUpdater +} + +func New(record arrow.Record) *RecordUpdater { + return &RecordUpdater{ + record: record, + schemaUpdater: schemaupdater.New(record.Schema()), + } +} + +func (r *RecordUpdater) RemoveColumns(columnNames []string) (arrow.Record, error) { + colIndices, err := r.colIndicesByNames(columnNames) + if err != nil { + return nil, err + } + if len(colIndices) == int(r.record.NumCols()) { + return nil, fmt.Errorf("cannot remove all columns") + } + + oldRecord := r.record.Columns() + newColumns := make([]arrow.Array, 0, len(oldRecord)-len(colIndices)) + for i, column := range oldRecord { + if _, ok := colIndices[i]; ok { + continue + } + newColumns = append(newColumns, column) + } + + r.record = array.NewRecord(r.schemaUpdater.RemoveColumnIndices(colIndices), newColumns, r.record.NumRows()) + return r.record, nil +} + +func (r *RecordUpdater) AddLiteralStringColumn(columnName, columnValue string, position int) (arrow.Record, error) { + if position == -1 { + position = int(r.record.NumCols()) + } + if position < 0 || position > int(r.record.NumCols()) { + return nil, fmt.Errorf("invalid position %v", position) + } + + newColumns := make([]arrow.Array, 0, int(r.record.NumCols())+1) + for i := 0; i < int(r.record.NumCols()); i++ { + if i == position { + newColumns = append(newColumns, r.buildStringColumn(columnValue, int(r.record.NumRows()))) + } + newColumns = append(newColumns, r.record.Column(i)) + } + if position == int(r.record.NumCols()) { + newColumns = append(newColumns, r.buildStringColumn(columnValue, int(r.record.NumRows()))) + } + newSchema, err := r.schemaUpdater.AddStringColumnAtPos(columnName, position, true) + if err != nil { + return nil, err + } + r.record = array.NewRecord(newSchema, newColumns, r.record.NumRows()) + return r.record, nil +} + +func (r *RecordUpdater) ObfuscateColumns(columnNames []string) (arrow.Record, error) { + colIndex, err := r.colIndicesByNames(columnNames) + if err != nil { + return nil, err + } + + oldRecord := r.record.Columns() + newColumns := make([]arrow.Array, 0, len(oldRecord)) + for i, column := range oldRecord { + if _, ok := colIndex[i]; ok { + if column.DataType().ID() != arrow.STRING { + return nil, fmt.Errorf("column %v is not a string column", r.record.ColumnName(i)) + } + newColumns = append(newColumns, r.obfuscateColumn(column)) + } else { + newColumns = append(newColumns, column) + } + } + + r.record = array.NewRecord(r.record.Schema(), newColumns, r.record.NumRows()) + return r.record, nil +} + +func (r *RecordUpdater) colIndicesByNames(columnNames []string) (map[int]struct{}, error) { + colNameMap := make(map[string]struct{}) + for _, columnName := range columnNames { + colNameMap[columnName] = struct{}{} + } + + colIndexes := make(map[int]struct{}) + for i := 0; i < int(r.record.NumCols()); i++ { + colName := r.record.ColumnName(i) + if _, ok := colNameMap[colName]; ok { + colIndexes[i] = struct{}{} + delete(colNameMap, colName) + } + } + if len(colNameMap) > 0 { + missingColumns := make([]string, 0, len(colNameMap)) + for colName := range colNameMap { + missingColumns = append(missingColumns, colName) + } + return nil, fmt.Errorf("columns %v not found", strings.Join(missingColumns, ", ")) + } + + return colIndexes, nil +} + +func (*RecordUpdater) buildStringColumn(literalValue string, numRows int) arrow.Array { + bld := array.NewStringBuilder(memory.DefaultAllocator) + for i := 0; i < numRows; i++ { + bld.AppendString(literalValue) + } + return bld.NewStringArray() +} + +func (*RecordUpdater) obfuscateColumn(column arrow.Array) arrow.Array { + bld := array.NewStringBuilder(memory.DefaultAllocator) + for i := 0; i < column.Len(); i++ { + if !column.IsValid(i) { + bld.AppendNull() + continue + } + bld.AppendString(fmt.Sprintf("%x", sha256.Sum256([]byte(column.ValueStr(i))))) + } + return bld.NewStringArray() +} diff --git a/plugins/transformer/basic/client/recordupdater/record_updater_test.go b/plugins/transformer/basic/client/recordupdater/record_updater_test.go new file mode 100644 index 00000000000000..e907a9399b69b1 --- /dev/null +++ b/plugins/transformer/basic/client/recordupdater/record_updater_test.go @@ -0,0 +1,76 @@ +package recordupdater + +import ( + "testing" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/memory" + "github.com/stretchr/testify/require" +) + +func TestRemoveColumns(t *testing.T) { + record := createTestRecord() + updater := New(record) + + updatedRecord, err := updater.RemoveColumns([]string{"col1"}) + require.NoError(t, err) + + require.Equal(t, int64(1), updatedRecord.NumCols()) + require.Equal(t, int64(2), updatedRecord.NumRows()) + requireAllColsLenMatchRecordsLen(t, updatedRecord) + require.Equal(t, "col2", updatedRecord.ColumnName(0)) +} + +func TestAddLiteralStringColumn(t *testing.T) { + record := createTestRecord() + updater := New(record) + + updatedRecord, err := updater.AddLiteralStringColumn("col3", "literal", -1) + require.NoError(t, err) + + require.Equal(t, int64(3), updatedRecord.NumCols()) + require.Equal(t, int64(2), updatedRecord.NumRows()) + requireAllColsLenMatchRecordsLen(t, updatedRecord) + require.Equal(t, "col3", updatedRecord.ColumnName(2)) + require.Equal(t, "literal", updatedRecord.Column(2).(*array.String).Value(0)) + require.Equal(t, "literal", updatedRecord.Column(2).(*array.String).Value(1)) +} + +func TestObfuscateColumns(t *testing.T) { + record := createTestRecord() + updater := New(record) + + updatedRecord, err := updater.ObfuscateColumns([]string{"col1"}) + require.NoError(t, err) + + require.Equal(t, int64(2), updatedRecord.NumCols()) + require.Equal(t, int64(2), updatedRecord.NumRows()) + requireAllColsLenMatchRecordsLen(t, updatedRecord) + require.Equal(t, "col1", updatedRecord.ColumnName(0)) + require.Equal(t, "col2", updatedRecord.ColumnName(1)) + require.Equal(t, "cc1d9c865e8380c2d566dc724c66369051acfaa3e9e8f36ad6c67d7d9b8461a5", updatedRecord.Column(0).(*array.String).Value(0)) + require.Equal(t, "528e5290f8ff0eb0325f0472b9c1a9ef4fac0b02ff6094b64d9382af4a10444b", updatedRecord.Column(0).(*array.String).Value(1)) +} + +func createTestRecord() arrow.Record { + bld := array.NewRecordBuilder(memory.DefaultAllocator, arrow.NewSchema( + []arrow.Field{ + {Name: "col1", Type: arrow.BinaryTypes.String}, + {Name: "col2", Type: arrow.BinaryTypes.String}, + }, + nil, + )) + defer bld.Release() + + bld.Field(0).(*array.StringBuilder).AppendValues([]string{"val1", "val2"}, nil) + bld.Field(1).(*array.StringBuilder).AppendValues([]string{"val3", "val4"}, nil) + + return bld.NewRecord() +} + +func requireAllColsLenMatchRecordsLen(t *testing.T, record arrow.Record) { + for i := 0; i < int(record.NumCols()); i++ { + require.Equal(t, int(record.NumRows()), record.Column(i).Len(), "Expected length of %d for column %d", record.NumRows(), i) + } +} diff --git a/plugins/transformer/basic/client/schemaupdater/schema_updater.go b/plugins/transformer/basic/client/schemaupdater/schema_updater.go new file mode 100644 index 00000000000000..0b89093835a928 --- /dev/null +++ b/plugins/transformer/basic/client/schemaupdater/schema_updater.go @@ -0,0 +1,37 @@ +package schemaupdater + +import "github.com/apache/arrow/go/v17/arrow" + +// SchemaUpdater takes an `arrow.Schema` and knows how to make simple subsequent changes to it. +// It doesn't know which table it belongs to or if the changes make sense. +type SchemaUpdater struct { + schema *arrow.Schema +} + +func New(schema *arrow.Schema) *SchemaUpdater { + return &SchemaUpdater{schema: schema} +} + +func (s *SchemaUpdater) RemoveColumnIndices(colIndices map[int]struct{}) *arrow.Schema { + oldMetadata := s.schema.Metadata() + oldFields := s.schema.Fields() + newFields := make([]arrow.Field, 0, len(oldFields)-len(colIndices)) + for i := range oldFields { + if _, ok := colIndices[i]; ok { + continue + } + newFields = append(newFields, oldFields[i]) + } + s.schema = arrow.NewSchema(newFields, &oldMetadata) + return s.schema +} + +func (s *SchemaUpdater) AddStringColumnAtPos(columnName string, zeroIndexedPosition int, isNullable bool) (*arrow.Schema, error) { + if zeroIndexedPosition == -1 { + zeroIndexedPosition = s.schema.NumFields() + } + return s.schema.AddField( + zeroIndexedPosition, + arrow.Field{Name: columnName, Type: arrow.BinaryTypes.String, Nullable: isNullable}, + ) +} diff --git a/plugins/transformer/basic/client/schemaupdater/schemaupdater_test.go b/plugins/transformer/basic/client/schemaupdater/schemaupdater_test.go new file mode 100644 index 00000000000000..107983606b200d --- /dev/null +++ b/plugins/transformer/basic/client/schemaupdater/schemaupdater_test.go @@ -0,0 +1,73 @@ +package schemaupdater + +import ( + "testing" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/stretchr/testify/require" +) + +func TestRemoveColumnIndices(t *testing.T) { + schema := createTestSchema() + updater := New(schema) + + colIndices := map[int]struct{}{0: {}} + updatedSchema := updater.RemoveColumnIndices(colIndices) + + require.Equal(t, 1, updatedSchema.NumFields(), "Expected 1 field") + require.Equal(t, "col2", updatedSchema.Field(0).Name, "Expected field name 'col2'") +} + +func TestAddStringColumnAtPos(t *testing.T) { + schema := createTestSchema() + updater := New(schema) + + updatedSchema, err := updater.AddStringColumnAtPos("col3", 1, false) + require.NoError(t, err) + + require.Equal(t, 3, updatedSchema.NumFields(), "Expected 3 fields") + require.Equal(t, "col3", updatedSchema.Field(1).Name, "Expected field name 'col3'") + require.False(t, updatedSchema.Field(1).Nullable, "Expected field 'col3' to be non-nullable") +} + +func TestAddStringColumnAtEnd(t *testing.T) { + schema := createTestSchema() + updater := New(schema) + + updatedSchema, err := updater.AddStringColumnAtPos("col3", -1, true) + require.NoError(t, err) + + require.Equal(t, 3, updatedSchema.NumFields(), "Expected 3 fields") + require.Equal(t, "col3", updatedSchema.Field(2).Name, "Expected field name 'col3'") + require.True(t, updatedSchema.Field(2).Nullable, "Expected field 'col3' to be nullable") +} + +func TestTransformMaintainsMetadata(t *testing.T) { + md1 := arrow.NewMetadata([]string{"key1", "key2"}, []string{"value1", "value2"}) + md2 := arrow.NewMetadata([]string{"key3", "key4"}, []string{"value3", "value4"}) + md3 := arrow.NewMetadata([]string{"key5", "key6"}, []string{"value5", "value6"}) + schema := arrow.NewSchema( + []arrow.Field{ + {Name: "col1", Type: arrow.BinaryTypes.String, Nullable: true, Metadata: md1}, + {Name: "col2", Type: arrow.BinaryTypes.String, Nullable: true, Metadata: md2}, + }, + &md3, + ) + updater := New(schema) + updatedSchema, err := updater.AddStringColumnAtPos("col3", -1, true) + require.NoError(t, err) + + require.Equal(t, schema.Metadata(), updatedSchema.Metadata(), "Expected metadata to be retained") + require.Equal(t, schema.Field(0).Metadata, updatedSchema.Field(0).Metadata, "Expected metadata to be retained") + require.Equal(t, schema.Field(1).Metadata, updatedSchema.Field(1).Metadata, "Expected metadata to be retained") +} + +func createTestSchema() *arrow.Schema { + return arrow.NewSchema( + []arrow.Field{ + {Name: "col1", Type: arrow.BinaryTypes.String, Nullable: true}, + {Name: "col2", Type: arrow.BinaryTypes.String, Nullable: true}, + }, + nil, + ) +} diff --git a/plugins/transformer/basic/client/spec/spec.go b/plugins/transformer/basic/client/spec/spec.go new file mode 100644 index 00000000000000..8a40934abb37f9 --- /dev/null +++ b/plugins/transformer/basic/client/spec/spec.go @@ -0,0 +1,73 @@ +package spec + +import ( + "errors" + "fmt" +) + +const ( + KindRemoveColumns = "remove_columns" + KindAddColumn = "add_column" + KindObfuscateColumns = "obfuscate_columns" +) + +type TransformationSpec struct { + Kind string `json:"kind"` + Tables []string `json:"tables"` // per-transformation table glob patterns + Columns []string `json:"columns"` + Name string `json:"name"` + Value string `json:"value"` +} + +type Spec struct { + TransformationSpecs []TransformationSpec `json:"transformations"` +} + +func (s *Spec) SetDefaults() { + for i := range s.TransformationSpecs { + if len(s.TransformationSpecs[i].Tables) == 0 { + s.TransformationSpecs[i].Tables = append(s.TransformationSpecs[i].Tables, "*") + } + } +} + +func (s *Spec) Validate() error { + var err error + for _, t := range s.TransformationSpecs { + switch t.Kind { + case KindRemoveColumns: + if len(t.Columns) == 0 { + err = errors.Join(err, fmt.Errorf("'columns' field must be specified for remove_columns transformation")) + } + if t.Name != "" { + err = errors.Join(err, fmt.Errorf("'name' field must not be specified for remove_columns transformation")) + } + if t.Value != "" { + err = errors.Join(err, fmt.Errorf("'value' field must not be specified for remove_columns transformation")) + } + case KindAddColumn: + if t.Name == "" { + err = errors.Join(err, fmt.Errorf("'name' field must be specified for add_column transformation")) + } + if t.Value == "" { + err = errors.Join(err, fmt.Errorf("'value' field must be specified for add_column transformation")) + } + if len(t.Columns) > 0 { + err = errors.Join(err, fmt.Errorf("'columns' field must not be specified for add_column transformation")) + } + case KindObfuscateColumns: + if len(t.Columns) == 0 { + err = errors.Join(err, fmt.Errorf("'columns' field must be specified for obfuscate_columns transformation")) + } + if t.Name != "" { + err = errors.Join(err, fmt.Errorf("'name' field must not be specified for obfuscate_columns transformation")) + } + if t.Value != "" { + err = errors.Join(err, fmt.Errorf("'value' field must not be specified for obfuscate_columns transformation")) + } + default: + err = errors.Join(err, fmt.Errorf("invalid transformation kind: %s; must be one of: remove_columns, add_column, obfuscate_columns", t.Kind)) + } + } + return err +} diff --git a/plugins/transformer/basic/client/spec/spec_test.go b/plugins/transformer/basic/client/spec/spec_test.go new file mode 100644 index 00000000000000..f781e2c7015117 --- /dev/null +++ b/plugins/transformer/basic/client/spec/spec_test.go @@ -0,0 +1,135 @@ +package spec + +import ( + "testing" +) + +func TestSetDefaults(t *testing.T) { + tests := []struct { + name string + input Spec + expected Spec + }{ + { + name: "Adds * as default pattern", + input: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: KindRemoveColumns, Columns: []string{"col1"}}, + }, + }, + expected: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: KindRemoveColumns, Tables: []string{"*"}, Columns: []string{"col1"}}, + }, + }, + }, + { + name: "Leaves as is if pattern is already set", + input: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: KindRemoveColumns, Tables: []string{"table1"}, Columns: []string{"col1"}}, + }, + }, + expected: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: KindRemoveColumns, Tables: []string{"table1"}, Columns: []string{"col1"}}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.input.SetDefaults() + if len(tt.input.TransformationSpecs) != len(tt.expected.TransformationSpecs) { + t.Fatalf("Expected %d transformation specs, got %d", len(tt.expected.TransformationSpecs), len(tt.input.TransformationSpecs)) + } + for i, spec := range tt.input.TransformationSpecs { + if len(spec.Tables) != len(tt.expected.TransformationSpecs[i].Tables) { + t.Errorf("Expected tables %v, got %v", tt.expected.TransformationSpecs[i].Tables, spec.Tables) + } + } + }) + } +} + +func TestValidate(t *testing.T) { + tests := []struct { + name string + input Spec + wantErr bool + }{ + { + name: "ValidRemoveColumns", + input: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: KindRemoveColumns, Columns: []string{"col1"}}, + }, + }, + wantErr: false, + }, + { + name: "InvalidRemoveColumnsNoColumns", + input: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: KindRemoveColumns}, + }, + }, + wantErr: true, + }, + { + name: "ValidAddColumn", + input: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: KindAddColumn, Name: "new_col", Value: "default"}, + }, + }, + wantErr: false, + }, + { + name: "InvalidAddColumnNoName", + input: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: KindAddColumn, Value: "default"}, + }, + }, + wantErr: true, + }, + { + name: "ValidObfuscateColumns", + input: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: KindObfuscateColumns, Columns: []string{"col1"}}, + }, + }, + wantErr: false, + }, + { + name: "InvalidObfuscateColumnsNoColumns", + input: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: KindObfuscateColumns}, + }, + }, + wantErr: true, + }, + { + name: "InvalidTransformationKind", + input: Spec{ + TransformationSpecs: []TransformationSpec{ + {Kind: "invalid_kind"}, + }, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.input.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/plugins/transformer/basic/client/tablematcher/table_matcher.go b/plugins/transformer/basic/client/tablematcher/table_matcher.go new file mode 100644 index 00000000000000..2713543983945a --- /dev/null +++ b/plugins/transformer/basic/client/tablematcher/table_matcher.go @@ -0,0 +1,51 @@ +package tablematcher + +import ( + "errors" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/cloudquery/plugin-sdk/v4/glob" + "github.com/cloudquery/plugin-sdk/v4/schema" +) + +// TableMatcher takes a slice of glob patterns and answers whether an `arrow.Record` +// belongs to a table matching any of the patterns (via Schema().Metadata()). +// +// It is used to allowlist records for transformations, based on spec. +// +// Unlike sources, which know which tables exist, the transformer discovers +// existing tables by observing records. Thus, a cache is used to keep constant lookup time. +type TableMatcher struct { + patterns []string + matcherCache map[string]bool +} + +func New(patterns []string) *TableMatcher { + return &TableMatcher{ + patterns: patterns, + matcherCache: make(map[string]bool), + } +} + +func (t *TableMatcher) isTableMatch(tableName string) bool { + if result, ok := t.matcherCache[tableName]; ok { + return result + } + isMatch := false + for _, p := range t.patterns { + if glob.Glob(p, tableName) { + isMatch = true + break + } + } + t.matcherCache[tableName] = isMatch + return isMatch +} + +func (t *TableMatcher) IsSchemasTableMatch(sc *arrow.Schema) (bool, error) { + table, ok := sc.Metadata().GetValue(schema.MetadataTableName) + if !ok { + return false, errors.New("table name not found in record's metadata") + } + return t.isTableMatch(table), nil +} diff --git a/plugins/transformer/basic/client/tablematcher/tablematcher_test.go b/plugins/transformer/basic/client/tablematcher/tablematcher_test.go new file mode 100644 index 00000000000000..00282bf20e1956 --- /dev/null +++ b/plugins/transformer/basic/client/tablematcher/tablematcher_test.go @@ -0,0 +1,89 @@ +package tablematcher + +import ( + "testing" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/memory" + "github.com/cloudquery/plugin-sdk/v4/schema" + "github.com/stretchr/testify/require" +) + +func TestIsTableMatch(t *testing.T) { + patterns := []string{"table_*", "data_*"} + matcher := New(patterns) + + tests := []struct { + tableName string + expected bool + }{ + {"table_1", true}, + {"data_1", true}, + {"other_table", false}, + } + + for _, test := range tests { + result := matcher.isTableMatch(test.tableName) + require.Equal(t, test.expected, result, "Expected %v for table name %s, got %v", test.expected, test.tableName, result) + } +} + +func TestIsSchemasTableMatch(t *testing.T) { + tests := []struct { + patterns []string + tableName string + expected bool + }{ + {[]string{"table_*", "data_*"}, "table_1", true}, + {[]string{"data_*"}, "table_1", false}, + {[]string{"table_*", "data_*"}, "data_1", true}, + {[]string{"table_*"}, "data_1", false}, + {[]string{"table_*", "data_*"}, "other_table", false}, + {[]string{"*"}, "other_table", true}, + } + for _, tt := range tests { + matcher := New(tt.patterns) + + record := createTestRecordWithMetadata(tt.tableName) + result, err := matcher.IsSchemasTableMatch(record.Schema()) + require.NoError(t, err, "Unexpected error") + require.Equal(t, tt.expected, result, "Expected %v for table name %s, got %v", tt.expected, tt.tableName, result) + } +} + +func TestIsSchemasTableMatch_NoMetadata(t *testing.T) { + patterns := []string{"table_*", "data_*"} + matcher := New(patterns) + + bld := array.NewRecordBuilder(memory.DefaultAllocator, arrow.NewSchema( + []arrow.Field{ + {Name: "col1", Type: arrow.BinaryTypes.String}, + }, + nil, + )) + defer bld.Release() + + bld.Field(0).(*array.StringBuilder).AppendValues([]string{"val1"}, nil) + record := bld.NewRecord() + + _, err := matcher.IsSchemasTableMatch(record.Schema()) + require.Error(t, err, "Expected error") + require.EqualError(t, err, "table name not found in record's metadata", "Expected error message 'table name not found in record's metadata'") +} + +func createTestRecordWithMetadata(tableName string) arrow.Record { + md := arrow.NewMetadata([]string{schema.MetadataTableName}, []string{tableName}) + + bld := array.NewRecordBuilder(memory.DefaultAllocator, arrow.NewSchema( + []arrow.Field{ + {Name: "col1", Type: arrow.BinaryTypes.String}, + }, + &md, + )) + defer bld.Release() + + bld.Field(0).(*array.StringBuilder).AppendValues([]string{"val1"}, nil) + + return bld.NewRecord() +} diff --git a/plugins/transformer/basic/client/transformers/transformers.go b/plugins/transformer/basic/client/transformers/transformers.go new file mode 100644 index 00000000000000..a1f36b9f0fbf1b --- /dev/null +++ b/plugins/transformer/basic/client/transformers/transformers.go @@ -0,0 +1,108 @@ +package transformers + +import ( + "fmt" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/memory" + "github.com/cloudquery/cloudquery/plugins/transformer/basic/client/recordupdater" + "github.com/cloudquery/cloudquery/plugins/transformer/basic/client/spec" + "github.com/cloudquery/cloudquery/plugins/transformer/basic/client/tablematcher" +) + +type TransformationFn = func(arrow.Record) (arrow.Record, error) +type SchemaTransformationFn = func(*arrow.Schema) (*arrow.Schema, error) + +type Transformer struct { + matcher *tablematcher.TableMatcher + fn TransformationFn + schemaFn SchemaTransformationFn +} + +func NewFromSpec(sp spec.TransformationSpec) (*Transformer, error) { + tr := &Transformer{matcher: tablematcher.New(sp.Tables)} + + switch sp.Kind { + case spec.KindAddColumn: + tr.fn = AddLiteralStringColumnAsLastColumn(sp.Name, sp.Value) + case spec.KindRemoveColumns: + tr.fn = RemoveColumns(sp.Columns) + case spec.KindObfuscateColumns: + tr.fn = ObfuscateColumns(sp.Columns) + default: + return nil, fmt.Errorf("unknown transformation kind: %s", sp.Kind) + } + + tr.schemaFn = transformSchema(tr.fn) + + return tr, nil +} + +func (tr *Transformer) Transform(record arrow.Record) (arrow.Record, error) { + // Passthrough if the record's table is not a match to any of the spec's + // tablepatterns, but error if the record doesn't have table metadata. + isMatch, err := tr.matcher.IsSchemasTableMatch(record.Schema()) + if err != nil { + return nil, err + } + if !isMatch { + return record, nil + } + // Apply the specific transformation kind + return tr.fn(record) +} + +func (tr *Transformer) TransformSchema(schema *arrow.Schema) (*arrow.Schema, error) { + // Passthrough if the record's table is not a match to any of the spec's + // tablepatterns, but error if the record doesn't have table metadata. + isMatch, err := tr.matcher.IsSchemasTableMatch(schema) + if err != nil { + return nil, err + } + if !isMatch { + return schema, nil + } + + if tr.schemaFn == nil { + return schema, nil + } + return tr.schemaFn(schema) +} + +func AddLiteralStringColumnAsLastColumn(name, value string) TransformationFn { + return func(record arrow.Record) (arrow.Record, error) { + return recordupdater.New(record).AddLiteralStringColumn(name, value, -1) + } +} + +func RemoveColumns(columnNames []string) TransformationFn { + return func(record arrow.Record) (arrow.Record, error) { + return recordupdater.New(record).RemoveColumns(columnNames) + } +} + +func ObfuscateColumns(columnNames []string) TransformationFn { + return func(record arrow.Record) (arrow.Record, error) { + return recordupdater.New(record).ObfuscateColumns(columnNames) + } +} + +func transformSchema(tf TransformationFn) SchemaTransformationFn { + return func(schema *arrow.Schema) (*arrow.Schema, error) { + newRecord, err := tf(makeEmptyRecord(schema)) + if err != nil { + return nil, err + } + return newRecord.Schema(), nil + } +} + +func makeEmptyRecord(s *arrow.Schema) arrow.Record { + cols := []arrow.Array{} + for _, field := range s.Fields() { + cols = append(cols, array.NewBuilder(memory.DefaultAllocator, field.Type).NewArray()) + } + md := s.Metadata() + return array.NewRecord(arrow.NewSchema(s.Fields(), &md), cols, 0) +} diff --git a/plugins/transformer/basic/client/transformers/transformers_test.go b/plugins/transformer/basic/client/transformers/transformers_test.go new file mode 100644 index 00000000000000..8a50e096a26652 --- /dev/null +++ b/plugins/transformer/basic/client/transformers/transformers_test.go @@ -0,0 +1,152 @@ +package transformers + +import ( + "testing" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/memory" + "github.com/cloudquery/cloudquery/plugins/transformer/basic/client/spec" + "github.com/cloudquery/plugin-sdk/v2/schema" + "github.com/stretchr/testify/require" +) + +func TestNewFromSpec(t *testing.T) { + tests := []struct { + name string + spec spec.TransformationSpec + wantErr bool + }{ + { + name: "AddLiteralStringColumn", + spec: spec.TransformationSpec{ + Kind: spec.KindAddColumn, + Name: "new_col", + Value: "default", + }, + wantErr: false, + }, + { + name: "RemoveColumns", + spec: spec.TransformationSpec{ + Kind: spec.KindRemoveColumns, + Columns: []string{"col1"}, + }, + wantErr: false, + }, + { + name: "ObfuscateColumns", + spec: spec.TransformationSpec{ + Kind: spec.KindObfuscateColumns, + Columns: []string{"col2"}, + }, + wantErr: false, + }, + { + name: "InvalidKind", + spec: spec.TransformationSpec{ + Kind: "invalid_kind", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := NewFromSpec(tt.spec) + if (err != nil) != tt.wantErr { + t.Errorf("NewFromSpec() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestTransform(t *testing.T) { + tests := []struct { + name string + spec spec.TransformationSpec + record arrow.Record + validate func(t *testing.T, record arrow.Record) + }{ + { + name: "AddLiteralStringColumn", + spec: spec.TransformationSpec{ + Kind: spec.KindAddColumn, + Name: "new_col", + Value: "default", + Tables: []string{"*"}, + }, + record: createTestRecord(), + validate: func(t *testing.T, record arrow.Record) { + require.Equal(t, int64(3), record.NumCols(), "Expected 3 columns") + require.Equal(t, int64(2), record.NumRows(), "Expected 2 rows") + require.Equal(t, "default", record.Column(2).(*array.String).Value(0), "Expected 'default' value in new_col column") + require.Equal(t, "default", record.Column(2).(*array.String).Value(1), "Expected 'default' value in new_col column") + }, + }, + { + name: "RemoveColumns", + spec: spec.TransformationSpec{ + Kind: spec.KindRemoveColumns, + Columns: []string{"col1"}, + Tables: []string{"*"}, + }, + record: createTestRecord(), + validate: func(t *testing.T, record arrow.Record) { + require.Equal(t, int64(1), record.NumCols(), "Expected 1 column") + require.Equal(t, int64(2), record.NumRows(), "Expected 2 rows") + }, + }, + { + name: "ObfuscateColumns", + spec: spec.TransformationSpec{ + Kind: spec.KindObfuscateColumns, + Columns: []string{"col2"}, + Tables: []string{"*"}, + }, + record: createTestRecord(), + validate: func(t *testing.T, record arrow.Record) { + require.Equal(t, "bac8d4414984861d5199b7a97699c728bee36c4084299b2ca905434cf65d8944", record.Column(1).(*array.String).Value(0), "Expected sha256 value in new_col column") + require.Equal(t, "dd0fff6ac351dd46cd26e2d5c61e479ce7c68ef12489e04284c0fd66648723cb", record.Column(1).(*array.String).Value(1), "Expected sha256 value in new_col column") + require.Equal(t, int64(2), record.NumCols(), "Expected 2 columns") + require.Equal(t, int64(2), record.NumRows(), "Expected 2 rows") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + transformer, err := NewFromSpec(tt.spec) + require.NoError(t, err, "NewFromSpec() should not return an error") + + transformedRecord, err := transformer.Transform(tt.record) + require.NoError(t, err, "Transform() should not return an error") + + requireAllColsLenMatchRecordsLen(t, transformedRecord) + tt.validate(t, transformedRecord) + }) + } +} + +func createTestRecord() arrow.Record { + md := arrow.NewMetadata([]string{schema.MetadataTableName}, []string{"table1"}) + bld := array.NewRecordBuilder(memory.DefaultAllocator, arrow.NewSchema( + []arrow.Field{ + {Name: "col1", Type: arrow.BinaryTypes.String}, + {Name: "col2", Type: arrow.BinaryTypes.String}, + }, + &md, + )) + defer bld.Release() + + bld.Field(0).(*array.StringBuilder).AppendValues([]string{"val1", "val2"}, nil) + bld.Field(1).(*array.StringBuilder).AppendValues([]string{"val3", "val4"}, nil) + + return bld.NewRecord() +} + +func requireAllColsLenMatchRecordsLen(t *testing.T, record arrow.Record) { + for i := 0; i < int(record.NumCols()); i++ { + require.Equal(t, int(record.NumRows()), record.Column(i).Len(), "Expected length of %d for column %d", record.NumRows(), i) + } +} diff --git a/plugins/transformer/basic/docs/_configuration.md b/plugins/transformer/basic/docs/_configuration.md new file mode 100644 index 00000000000000..d210e04429ac8d --- /dev/null +++ b/plugins/transformer/basic/docs/_configuration.md @@ -0,0 +1,19 @@ +```yaml copy +kind: transformer +spec: + name: "basic" + path: "cloudquery/basic" + version: VERSION_TRANSFORMER_BASIC + spec: + transformations: + - kind: obfuscate_columns + tables: ["xkcd_comics"] + columns: ["safe_title", "title"] + - kind: remove_columns + tables: ["xkcd_comics"] + columns: ["transcript", "news"] + - kind: add_column + tables: ["xkcd_comics"] + name: "source" + value: "xkcd" +``` \ No newline at end of file diff --git a/plugins/transformer/basic/docs/overview.md b/plugins/transformer/basic/docs/overview.md new file mode 100644 index 00000000000000..26da219d52a7b6 --- /dev/null +++ b/plugins/transformer/basic/docs/overview.md @@ -0,0 +1,31 @@ +This CloudQuery transformer plugin provides basic transformation capabilities: + +- Removing columns +- Adding literal string columns +- Obfuscating string columns + +## Configuration + +First, add the transformer to your destination. For example, this will add a basic transformer to a PostgreSQL destination: + +```yaml copy +kind: destination +spec: + name: "postgresql" + path: "cloudquery/postgresql" + registry: "cloudquery" + version: "v8.0.7" + write_mode: "overwrite-delete-stale" + migrate_mode: forced # optional + transformers: + - "basic" + + spec: + connection_string: "postgresql://your.user:your.password@localhost:5432/db_name" +``` + +The `migrate_mode: forced` setting might make sense if you plan on modifying the schema from a previous sync. + +Then, add your transformer spec. Here's an example that transforms the XKCD source table: + +:configuration \ No newline at end of file diff --git a/plugins/transformer/basic/go.mod b/plugins/transformer/basic/go.mod new file mode 100644 index 00000000000000..629ddbcc090962 --- /dev/null +++ b/plugins/transformer/basic/go.mod @@ -0,0 +1,73 @@ +module github.com/cloudquery/cloudquery/plugins/transformer/basic + +go 1.22.4 + +require ( + github.com/apache/arrow/go/v17 v17.0.0 + github.com/cloudquery/plugin-sdk/v2 v2.7.0 + github.com/cloudquery/plugin-sdk/v4 v4.56.0 + github.com/rs/zerolog v1.33.0 + github.com/stretchr/testify v1.9.0 +) + +require ( + github.com/adrg/xdg v0.4.0 // indirect + github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cloudquery/cloudquery-api-go v1.12.4 // indirect + github.com/cloudquery/plugin-pb-go v1.21.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/ghodss/yaml v1.0.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/thoas/go-funk v0.9.3 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.4.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect + go.opentelemetry.io/otel/log v0.4.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.4.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect + golang.org/x/mod v0.18.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.22.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +// github.com/cloudquery/jsonschema @ cqmain +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66 diff --git a/plugins/transformer/basic/go.sum b/plugins/transformer/basic/go.sum new file mode 100644 index 00000000000000..6a5325e3d9857e --- /dev/null +++ b/plugins/transformer/basic/go.sum @@ -0,0 +1,170 @@ +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 h1:mFDZW1FQk9yndPvxScp7RpcOpdSHaqcgBWO7sDlx4S8= +github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= +github.com/apache/arrow/go/v17 v17.0.0 h1:RRR2bdqKcdbss9Gxy2NS/hK8i4LDMh23L6BbkN5+F54= +github.com/apache/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cloudquery/cloudquery-api-go v1.12.4 h1:48zJRUONRb0AJD/l4u5QZtGsVBq1QUU3M9+/+sCU3xo= +github.com/cloudquery/cloudquery-api-go v1.12.4/go.mod h1:5oo8HHnv2Y7NgcVvZn59xFlYKJUyeP0tcN8JH3IP2Aw= +github.com/cloudquery/plugin-pb-go v1.21.3 h1:IlhLN6LbZeAzOjIm0VMELmj7PpFkDroJ41QCrAEcWwg= +github.com/cloudquery/plugin-pb-go v1.21.3/go.mod h1:Gv336j2QCqGlUABUA6gBcrqWNjLpzv9NOmAJAHC3xg0= +github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= +github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug= +github.com/cloudquery/plugin-sdk/v4 v4.56.0 h1:x06ypwoCOBhxJTtX5Mtmzyr6j0QxhOYV/y60tB9VkGU= +github.com/cloudquery/plugin-sdk/v4 v4.56.0/go.mod h1:AQeTVmFxAklgKEAXGyglSkNRFinIDt7OLoy2lLLyufY= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= +github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.4.0 h1:zBPZAISA9NOc5cE8zydqDiS0itvg/P/0Hn9m72a5gvM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.4.0/go.mod h1:gcj2fFjEsqpV3fXuzAA+0Ze1p2/4MJ4T7d77AmkvueQ= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 h1:CIHWikMsN3wO+wq1Tp5VGdVRTcON+DmOJSfDjXypKOc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0/go.mod h1:TNupZ6cxqyFEpLXAZW7On+mLFL0/g0TE3unIYL91xWc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= +go.opentelemetry.io/otel/log v0.4.0 h1:/vZ+3Utqh18e8TPjuc3ecg284078KWrR8BRz+PQAj3o= +go.opentelemetry.io/otel/log v0.4.0/go.mod h1:DhGnQvky7pHy82MIRV43iXh3FlKN8UUKftn0KbLOq6I= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/sdk/log v0.4.0 h1:1mMI22L82zLqf6KtkjrRy5BbagOTWdJsqMY/HSqILAA= +go.opentelemetry.io/otel/sdk/log v0.4.0/go.mod h1:AYJ9FVF0hNOgAVzUG/ybg/QttnXhUePWAupmCqtdESo= +go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= +go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/plugins/transformer/basic/main.go b/plugins/transformer/basic/main.go new file mode 100644 index 00000000000000..8eb9f652f02503 --- /dev/null +++ b/plugins/transformer/basic/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "context" + "log" + + "github.com/cloudquery/cloudquery/plugins/transformer/basic/client" + internalPlugin "github.com/cloudquery/cloudquery/plugins/transformer/basic/resources/plugin" + "github.com/cloudquery/plugin-sdk/v4/plugin" + "github.com/cloudquery/plugin-sdk/v4/serve" +) + +func main() { + p := plugin.NewPlugin( + internalPlugin.Name, + internalPlugin.Version, + client.New, + plugin.WithKind(internalPlugin.Kind), + plugin.WithTeam(internalPlugin.Team), + ) + if err := serve.Plugin(p).Serve(context.Background()); err != nil { + log.Fatalf("failed to serve plugin: %v", err) + } +} diff --git a/plugins/transformer/basic/resources/plugin/plugin.go b/plugins/transformer/basic/resources/plugin/plugin.go new file mode 100644 index 00000000000000..7af26635201240 --- /dev/null +++ b/plugins/transformer/basic/resources/plugin/plugin.go @@ -0,0 +1,9 @@ +package plugin + +// Don't move this file to a different package, it's used by Go releaser to embed the version in the binary. +var ( + Name = "basic" + Kind = "transformer" + Team = "cloudquery" + Version = "development" +) diff --git a/release-please-config.json b/release-please-config.json index 60ca5f497c6635..73059815badea0 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -91,6 +91,9 @@ "plugins/source/xkcd": { "component": "plugins-source-xkcd" }, + "plugins/transformer/basic": { + "component": "plugins-transformer-basic" + }, "scaffold": { "component": "scaffold" } diff --git a/scripts/workflows/wait_for_required_workflows.js b/scripts/workflows/wait_for_required_workflows.js index b227ab0cd86fe2..937664ee7341b3 100644 --- a/scripts/workflows/wait_for_required_workflows.js +++ b/scripts/workflows/wait_for_required_workflows.js @@ -34,7 +34,8 @@ module.exports = async ({github, context}) => { const sources = fs.readdirSync("plugins/source", {withFileTypes: true}).filter(dirent => dirent.isDirectory()).map(dirent => `plugins/source/${dirent.name}`) const destinations = fs.readdirSync("plugins/destination", {withFileTypes: true}).filter(dirent => dirent.isDirectory()).map(dirent => `plugins/destination/${dirent.name}`) - const allComponents = ["cli", "scaffold", ...sources, ...destinations] + const transformers = fs.readdirSync("plugins/transformer", {withFileTypes: true}).filter(dirent => dirent.isDirectory()).map(dirent => `plugins/transformer/${dirent.name}`) + const allComponents = ["cli", "scaffold", ...sources, ...destinations, ...transformers] console.log(`All components: ${allComponents.join(", ")}`) let actions = allComponents.filter(action => matchesFile(action)) if (actions.length === 0) {