Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions plugins/destination/neo4j/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,11 @@ test:
.PHONY: lint
lint:
golangci-lint run --config ../../.golangci.yml

.PHONY: gen-spec-schema
gen-spec-schema:
go run client/spec/gen/main.go

# All gen targets
.PHONY: gen
Comment thread
disq marked this conversation as resolved.
gen: gen-spec-schema
41 changes: 41 additions & 0 deletions plugins/destination/neo4j/client/schema.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 24 additions & 7 deletions plugins/destination/neo4j/client/spec.go
Original file line number Diff line number Diff line change
@@ -1,24 +1,41 @@
package client

import (
_ "embed"
"fmt"
)

const (
defaultBatchSize = 1000
defaultBatchSizeBytes = 1024 * 1024 * 4 // 4MB
)

type Spec struct {
ConnectionString string `json:"connection_string"`
Username string `json:"username"`
Password string `json:"password"`
// Connection string to connect to the database. This can be a URL or a DSN, as per official [neo4j docs](https://neo4j.com/docs/browser-manual/current/operations/dbms-connection/#uri-scheme).
ConnectionString string `json:"connection_string" jsonschema:"required,minLength=1"`

// Username to connect to the database.
Username string `json:"username"`

BatchSize int `json:"batch_size"`
BatchSizeBytes int `json:"batch_size_bytes"`
// Password to connect to the database.
Password string `json:"password"`

// Number of records to batch together before sending to the database.
BatchSize int `json:"batch_size" jsonschema:"minimum=1,default=1000"`

// Number of bytes (as Arrow buffer size) to batch together before sending to the database.
BatchSizeBytes int `json:"batch_size_bytes" jsonschema:"minimum=1,default=4194304"`
}

//go:embed schema.json
var JSONSchema string

func (s *Spec) SetDefaults() {
if s.BatchSize == 0 {
s.BatchSize = 1000
s.BatchSize = defaultBatchSize
}
if s.BatchSizeBytes == 0 {
s.BatchSizeBytes = 1024 * 1024 * 4
s.BatchSizeBytes = defaultBatchSizeBytes
}
}

Expand Down
30 changes: 30 additions & 0 deletions plugins/destination/neo4j/client/spec/gen/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package main

import (
"fmt"
"log"
"path"
"runtime"

"github.com/cloudquery/cloudquery/plugins/destination/neo4j/client"
cqjsonschema "github.com/cloudquery/codegen/jsonschema"
"github.com/invopop/jsonschema"
)

func main() {
fmt.Println("Generating JSON schema for plugin spec")
cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"),
cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/neo4j/client", path.Join(currDir(), "../..")),
func(reflector *jsonschema.Reflector) {
reflector.NullableFromType = false
Comment thread
disq marked this conversation as resolved.
Outdated
},
)
}

func currDir() string {
_, filename, _, ok := runtime.Caller(0)
if !ok {
log.Fatal("Failed to get caller information")
}
return path.Dir(filename)
}
61 changes: 61 additions & 0 deletions plugins/destination/neo4j/client/spec_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package client

import (
"testing"

"github.com/cloudquery/codegen/jsonschema"
)

func TestJSONSchema(t *testing.T) {
jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{
{
Name: "empty spec",
Spec: `{}`,
Err: true,
},
{
Name: "spec with connection_string",
Spec: `{"connection_string": "file"}`,
},
{
Name: "spec with bool connection_string",
Spec: `{"connection_string": true}`,
Err: true,
},
{
Name: "spec with null connection_string",
Spec: `{"connection_string": null}`,
Err: true,
},
{
Name: "spec with int connection_string",
Spec: `{"connection_string": 123}`,
Err: true,
},
{
Name: "spec with bool batch_size",
Spec: `{"connection_string": "abc", "batch_size":false}`,
Err: true,
},
{
Name: "spec with null batch_size",
Spec: `{"connection_string": "abc", "batch_size":null}`,
Err: true,
},
{
Name: "spec with string batch_size",
Spec: `{"connection_string": "abc", "batch_size":"str"}`,
Err: true,
},
{
Name: "spec with array batch_size",
Spec: `{"connection_string": "abc", "batch_size":["abc"]}`,
Err: true,
},
{
Name: "spec with unknown field",
Spec: `{"connection_string": "abc", "unknown": "test"}`,
Err: true,
},
})
}
8 changes: 8 additions & 0 deletions plugins/destination/neo4j/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ go 1.21.4

require (
github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1
github.com/cloudquery/codegen v0.3.12
github.com/cloudquery/plugin-sdk/v4 v4.28.0
github.com/invopop/jsonschema v0.12.0
github.com/neo4j/neo4j-go-driver/v5 v5.6.0
github.com/rs/zerolog v1.31.0
github.com/stretchr/testify v1.8.4
Expand All @@ -21,6 +23,8 @@ require (
github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/bytedance/sonic v1.10.2 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
Expand Down Expand Up @@ -91,6 +95,7 @@ require (
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/yosssi/ace v0.0.5 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/otel v1.20.0 // indirect
Expand Down Expand Up @@ -119,3 +124,6 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

// github.com/cloudquery/jsonschema @ cqmain
replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f
10 changes: 10 additions & 0 deletions plugins/destination/neo4j/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,13 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
Expand All @@ -43,6 +47,10 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0
github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
github.com/cloudquery/cloudquery-api-go v1.7.0 h1:9da/fBNcKnJGTKF3LFoKIMUwfnzhMCsp5RjIOSxCU7s=
github.com/cloudquery/cloudquery-api-go v1.7.0/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE=
github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8=
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8=
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
github.com/cloudquery/plugin-pb-go v1.16.6 h1:UcN7UK89EWxh9SRGCIPQ/Ao2YB5zVugvBtF8ii536ig=
github.com/cloudquery/plugin-pb-go v1.16.6/go.mod h1:/dnO/uBQGZlTvbYDPEvSt5J30ciN6DEDrQ8Jy4MKcIM=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
Expand Down Expand Up @@ -251,6 +259,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
Expand Down
1 change: 1 addition & 0 deletions plugins/destination/neo4j/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ func main() {
p := plugin.NewPlugin(internalPlugin.Name, internalPlugin.Version, client.New,
plugin.WithKind(internalPlugin.Kind),
plugin.WithTeam(internalPlugin.Team),
plugin.WithJSONSchema(client.JSONSchema),
)
if err := serve.Plugin(p, serve.WithPluginSentryDSN(sentryDSN), serve.WithDestinationV0V1Server()).Serve(context.Background()); err != nil {
log.Fatalf("failed to serve plugin: %v", err)
Expand Down