From ad7cf321b6c52bee02c70d6a6215ac838cd1907a Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 30 Jan 2024 20:22:12 +0000 Subject: [PATCH 1/2] feat: Add JSON schema to Neo4J destination plugin --- plugins/destination/neo4j/Makefile | 8 +++ plugins/destination/neo4j/client/schema.json | 41 +++++++++++++ plugins/destination/neo4j/client/spec.go | 31 +++++++--- .../destination/neo4j/client/spec/gen/main.go | 30 +++++++++ plugins/destination/neo4j/client/spec_test.go | 61 +++++++++++++++++++ plugins/destination/neo4j/go.mod | 8 +++ plugins/destination/neo4j/go.sum | 10 +++ plugins/destination/neo4j/main.go | 1 + 8 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 plugins/destination/neo4j/client/schema.json create mode 100644 plugins/destination/neo4j/client/spec/gen/main.go create mode 100644 plugins/destination/neo4j/client/spec_test.go diff --git a/plugins/destination/neo4j/Makefile b/plugins/destination/neo4j/Makefile index c957b06653ed16..a803c132c3b248 100644 --- a/plugins/destination/neo4j/Makefile +++ b/plugins/destination/neo4j/Makefile @@ -7,3 +7,11 @@ test: .PHONY: lint lint: golangci-lint run --config ../../.golangci.yml + +.PHONY: gen-spec-schema +gen-spec-schema: + go run client/spec/gen/main.go + +# All gen targets +.PHONY: gen +gen: gen-spec-schema diff --git a/plugins/destination/neo4j/client/schema.json b/plugins/destination/neo4j/client/schema.json new file mode 100644 index 00000000000000..966c46f8c211c8 --- /dev/null +++ b/plugins/destination/neo4j/client/schema.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/cloudquery/plugins/destination/neo4j/client/spec", + "$ref": "#/$defs/Spec", + "$defs": { + "Spec": { + "properties": { + "connection_string": { + "type": "string", + "minLength": 1, + "description": "Connection string to connect to the database. This can be a URL or a DSN, as per official [neo4j docs](https://neo4j.com/docs/browser-manual/current/operations/dbms-connection/#uri-scheme)." + }, + "username": { + "type": "string", + "description": "Username to connect to the database." + }, + "password": { + "type": "string", + "description": "Password to connect to the database." + }, + "batch_size": { + "type": "integer", + "minimum": 1, + "description": "Number of records to batch together before sending to the database.", + "default": 1000 + }, + "batch_size_bytes": { + "type": "integer", + "minimum": 1, + "description": "Number of bytes (as Arrow buffer size) to batch together before sending to the database.", + "default": 4194304 + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "connection_string" + ] + } + } +} diff --git a/plugins/destination/neo4j/client/spec.go b/plugins/destination/neo4j/client/spec.go index 53b84d8d7dc053..16c0c2cfc250cf 100644 --- a/plugins/destination/neo4j/client/spec.go +++ b/plugins/destination/neo4j/client/spec.go @@ -1,24 +1,41 @@ package client import ( + _ "embed" "fmt" ) +const ( + defaultBatchSize = 1000 + defaultBatchSizeBytes = 1024 * 1024 * 4 // 4MB +) + type Spec struct { - ConnectionString string `json:"connection_string"` - Username string `json:"username"` - Password string `json:"password"` + // Connection string to connect to the database. This can be a URL or a DSN, as per official [neo4j docs](https://neo4j.com/docs/browser-manual/current/operations/dbms-connection/#uri-scheme). + ConnectionString string `json:"connection_string" jsonschema:"required,minLength=1"` + + // Username to connect to the database. + Username string `json:"username"` - BatchSize int `json:"batch_size"` - BatchSizeBytes int `json:"batch_size_bytes"` + // Password to connect to the database. + Password string `json:"password"` + + // Number of records to batch together before sending to the database. + BatchSize int `json:"batch_size" jsonschema:"minimum=1,default=1000"` + + // Number of bytes (as Arrow buffer size) to batch together before sending to the database. + BatchSizeBytes int `json:"batch_size_bytes" jsonschema:"minimum=1,default=4194304"` } +//go:embed schema.json +var JSONSchema string + func (s *Spec) SetDefaults() { if s.BatchSize == 0 { - s.BatchSize = 1000 + s.BatchSize = defaultBatchSize } if s.BatchSizeBytes == 0 { - s.BatchSizeBytes = 1024 * 1024 * 4 + s.BatchSizeBytes = defaultBatchSizeBytes } } diff --git a/plugins/destination/neo4j/client/spec/gen/main.go b/plugins/destination/neo4j/client/spec/gen/main.go new file mode 100644 index 00000000000000..17b0ed6018cfac --- /dev/null +++ b/plugins/destination/neo4j/client/spec/gen/main.go @@ -0,0 +1,30 @@ +package main + +import ( + "fmt" + "log" + "path" + "runtime" + + "github.com/cloudquery/cloudquery/plugins/destination/neo4j/client" + cqjsonschema "github.com/cloudquery/codegen/jsonschema" + "github.com/invopop/jsonschema" +) + +func main() { + fmt.Println("Generating JSON schema for plugin spec") + cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"), + cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/neo4j/client", path.Join(currDir(), "../..")), + func(reflector *jsonschema.Reflector) { + reflector.NullableFromType = false + }, + ) +} + +func currDir() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Failed to get caller information") + } + return path.Dir(filename) +} diff --git a/plugins/destination/neo4j/client/spec_test.go b/plugins/destination/neo4j/client/spec_test.go new file mode 100644 index 00000000000000..07a4df9909bd33 --- /dev/null +++ b/plugins/destination/neo4j/client/spec_test.go @@ -0,0 +1,61 @@ +package client + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" +) + +func TestJSONSchema(t *testing.T) { + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty spec", + Spec: `{}`, + Err: true, + }, + { + Name: "spec with connection_string", + Spec: `{"connection_string": "file"}`, + }, + { + Name: "spec with bool connection_string", + Spec: `{"connection_string": true}`, + Err: true, + }, + { + Name: "spec with null connection_string", + Spec: `{"connection_string": null}`, + Err: true, + }, + { + Name: "spec with int connection_string", + Spec: `{"connection_string": 123}`, + Err: true, + }, + { + Name: "spec with bool batch_size", + Spec: `{"connection_string": "abc", "batch_size":false}`, + Err: true, + }, + { + Name: "spec with null batch_size", + Spec: `{"connection_string": "abc", "batch_size":null}`, + Err: true, + }, + { + Name: "spec with string batch_size", + Spec: `{"connection_string": "abc", "batch_size":"str"}`, + Err: true, + }, + { + Name: "spec with array batch_size", + Spec: `{"connection_string": "abc", "batch_size":["abc"]}`, + Err: true, + }, + { + Name: "spec with unknown field", + Spec: `{"connection_string": "abc", "unknown": "test"}`, + Err: true, + }, + }) +} diff --git a/plugins/destination/neo4j/go.mod b/plugins/destination/neo4j/go.mod index 0a846e8ac60073..f9d4d318f255de 100644 --- a/plugins/destination/neo4j/go.mod +++ b/plugins/destination/neo4j/go.mod @@ -4,7 +4,9 @@ go 1.21.4 require ( github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1 + github.com/cloudquery/codegen v0.3.12 github.com/cloudquery/plugin-sdk/v4 v4.28.0 + github.com/invopop/jsonschema v0.12.0 github.com/neo4j/neo4j-go-driver/v5 v5.6.0 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 @@ -21,6 +23,8 @@ require ( github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.10.2 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect @@ -91,6 +95,7 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/yosssi/ace v0.0.5 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/otel v1.20.0 // indirect @@ -119,3 +124,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// github.com/cloudquery/jsonschema @ cqmain +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f diff --git a/plugins/destination/neo4j/go.sum b/plugins/destination/neo4j/go.sum index dc4894b3eaf7ba..1eca9d7dff4132 100644 --- a/plugins/destination/neo4j/go.sum +++ b/plugins/destination/neo4j/go.sum @@ -25,9 +25,13 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= @@ -43,6 +47,10 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/cloudquery/cloudquery-api-go v1.7.0 h1:9da/fBNcKnJGTKF3LFoKIMUwfnzhMCsp5RjIOSxCU7s= github.com/cloudquery/cloudquery-api-go v1.7.0/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA= +github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE= +github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8= +github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8= +github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= github.com/cloudquery/plugin-pb-go v1.16.6 h1:UcN7UK89EWxh9SRGCIPQ/Ao2YB5zVugvBtF8ii536ig= github.com/cloudquery/plugin-pb-go v1.16.6/go.mod h1:/dnO/uBQGZlTvbYDPEvSt5J30ciN6DEDrQ8Jy4MKcIM= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= @@ -251,6 +259,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= diff --git a/plugins/destination/neo4j/main.go b/plugins/destination/neo4j/main.go index e55e7114960f31..28bddebfa0105b 100644 --- a/plugins/destination/neo4j/main.go +++ b/plugins/destination/neo4j/main.go @@ -18,6 +18,7 @@ func main() { p := plugin.NewPlugin(internalPlugin.Name, internalPlugin.Version, client.New, plugin.WithKind(internalPlugin.Kind), plugin.WithTeam(internalPlugin.Team), + plugin.WithJSONSchema(client.JSONSchema), ) if err := serve.Plugin(p, serve.WithPluginSentryDSN(sentryDSN), serve.WithDestinationV0V1Server()).Serve(context.Background()); err != nil { log.Fatalf("failed to serve plugin: %v", err) From a6d30cf9db7380954a745eace3d0adf49148a682 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 31 Jan 2024 08:51:29 +0000 Subject: [PATCH 2/2] CR, workflow changes --- .github/workflows/dest_neo4j.yml | 6 ++++++ plugins/destination/neo4j/client/spec/gen/main.go | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dest_neo4j.yml b/.github/workflows/dest_neo4j.yml index afb37dc1ed635f..2f427717b9bf50 100644 --- a/.github/workflows/dest_neo4j.yml +++ b/.github/workflows/dest_neo4j.yml @@ -62,6 +62,12 @@ jobs: args: "--config ../../.golangci.yml" skip-pkg-cache: true skip-build-cache: true + - name: gen + if: github.event_name == 'pull_request' + run: make gen + - name: Fail if generation updated files + if: github.event_name == 'pull_request' + run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1) - name: Build run: go build . - name: Test Neo4j diff --git a/plugins/destination/neo4j/client/spec/gen/main.go b/plugins/destination/neo4j/client/spec/gen/main.go index 17b0ed6018cfac..1d36db8d3ba1f1 100644 --- a/plugins/destination/neo4j/client/spec/gen/main.go +++ b/plugins/destination/neo4j/client/spec/gen/main.go @@ -8,16 +8,12 @@ import ( "github.com/cloudquery/cloudquery/plugins/destination/neo4j/client" cqjsonschema "github.com/cloudquery/codegen/jsonschema" - "github.com/invopop/jsonschema" ) func main() { fmt.Println("Generating JSON schema for plugin spec") cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"), cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/neo4j/client", path.Join(currDir(), "../..")), - func(reflector *jsonschema.Reflector) { - reflector.NullableFromType = false - }, ) }