Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/dest_neo4j.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ jobs:
args: "--config ../../.golangci.yml"
skip-pkg-cache: true
skip-build-cache: true
- name: gen
if: github.event_name == 'pull_request'
run: make gen
- name: Fail if generation updated files
if: github.event_name == 'pull_request'
run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
- name: Build
run: go build .
- name: Test Neo4j
Expand Down
8 changes: 8 additions & 0 deletions plugins/destination/neo4j/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,11 @@ test:
.PHONY: lint
lint:
golangci-lint run --config ../../.golangci.yml

.PHONY: gen-spec-schema
gen-spec-schema:
go run client/spec/gen/main.go

# All gen targets
.PHONY: gen
Comment thread
disq marked this conversation as resolved.
gen: gen-spec-schema
41 changes: 41 additions & 0 deletions plugins/destination/neo4j/client/schema.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 24 additions & 7 deletions plugins/destination/neo4j/client/spec.go
Original file line number Diff line number Diff line change
@@ -1,24 +1,41 @@
package client

import (
_ "embed"
"fmt"
)

const (
defaultBatchSize = 1000
defaultBatchSizeBytes = 1024 * 1024 * 4 // 4MB
)

type Spec struct {
ConnectionString string `json:"connection_string"`
Username string `json:"username"`
Password string `json:"password"`
// Connection string to connect to the database. This can be a URL or a DSN, as per official [neo4j docs](https://neo4j.com/docs/browser-manual/current/operations/dbms-connection/#uri-scheme).
ConnectionString string `json:"connection_string" jsonschema:"required,minLength=1"`

// Username to connect to the database.
Username string `json:"username"`

BatchSize int `json:"batch_size"`
BatchSizeBytes int `json:"batch_size_bytes"`
// Password to connect to the database.
Password string `json:"password"`

// Number of records to batch together before sending to the database.
BatchSize int `json:"batch_size" jsonschema:"minimum=1,default=1000"`

// Number of bytes (as Arrow buffer size) to batch together before sending to the database.
BatchSizeBytes int `json:"batch_size_bytes" jsonschema:"minimum=1,default=4194304"`
}

//go:embed schema.json
var JSONSchema string

func (s *Spec) SetDefaults() {
if s.BatchSize == 0 {
s.BatchSize = 1000
s.BatchSize = defaultBatchSize
}
if s.BatchSizeBytes == 0 {
s.BatchSizeBytes = 1024 * 1024 * 4
s.BatchSizeBytes = defaultBatchSizeBytes
}
}

Expand Down
26 changes: 26 additions & 0 deletions plugins/destination/neo4j/client/spec/gen/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package main

import (
"fmt"
"log"
"path"
"runtime"

"github.com/cloudquery/cloudquery/plugins/destination/neo4j/client"
cqjsonschema "github.com/cloudquery/codegen/jsonschema"
)

func main() {
fmt.Println("Generating JSON schema for plugin spec")
cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"),
cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/neo4j/client", path.Join(currDir(), "../..")),
)
}

func currDir() string {
_, filename, _, ok := runtime.Caller(0)
if !ok {
log.Fatal("Failed to get caller information")
}
return path.Dir(filename)
}
61 changes: 61 additions & 0 deletions plugins/destination/neo4j/client/spec_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package client

import (
"testing"

"github.com/cloudquery/codegen/jsonschema"
)

func TestJSONSchema(t *testing.T) {
jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{
{
Name: "empty spec",
Spec: `{}`,
Err: true,
},
{
Name: "spec with connection_string",
Spec: `{"connection_string": "file"}`,
},
{
Name: "spec with bool connection_string",
Spec: `{"connection_string": true}`,
Err: true,
},
{
Name: "spec with null connection_string",
Spec: `{"connection_string": null}`,
Err: true,
},
{
Name: "spec with int connection_string",
Spec: `{"connection_string": 123}`,
Err: true,
},
{
Name: "spec with bool batch_size",
Spec: `{"connection_string": "abc", "batch_size":false}`,
Err: true,
},
{
Name: "spec with null batch_size",
Spec: `{"connection_string": "abc", "batch_size":null}`,
Err: true,
},
{
Name: "spec with string batch_size",
Spec: `{"connection_string": "abc", "batch_size":"str"}`,
Err: true,
},
{
Name: "spec with array batch_size",
Spec: `{"connection_string": "abc", "batch_size":["abc"]}`,
Err: true,
},
{
Name: "spec with unknown field",
Spec: `{"connection_string": "abc", "unknown": "test"}`,
Err: true,
},
})
}
8 changes: 8 additions & 0 deletions plugins/destination/neo4j/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ go 1.21.4

require (
github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1
github.com/cloudquery/codegen v0.3.12
github.com/cloudquery/plugin-sdk/v4 v4.28.0
github.com/invopop/jsonschema v0.12.0
github.com/neo4j/neo4j-go-driver/v5 v5.6.0
github.com/rs/zerolog v1.31.0
github.com/stretchr/testify v1.8.4
Expand All @@ -21,6 +23,8 @@ require (
github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/bytedance/sonic v1.10.2 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
Expand Down Expand Up @@ -91,6 +95,7 @@ require (
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/yosssi/ace v0.0.5 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/otel v1.20.0 // indirect
Expand Down Expand Up @@ -119,3 +124,6 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

// github.com/cloudquery/jsonschema @ cqmain
replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f
10 changes: 10 additions & 0 deletions plugins/destination/neo4j/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,13 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
Expand All @@ -43,6 +47,10 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0
github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
github.com/cloudquery/cloudquery-api-go v1.7.0 h1:9da/fBNcKnJGTKF3LFoKIMUwfnzhMCsp5RjIOSxCU7s=
github.com/cloudquery/cloudquery-api-go v1.7.0/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE=
github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8=
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8=
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
github.com/cloudquery/plugin-pb-go v1.16.6 h1:UcN7UK89EWxh9SRGCIPQ/Ao2YB5zVugvBtF8ii536ig=
github.com/cloudquery/plugin-pb-go v1.16.6/go.mod h1:/dnO/uBQGZlTvbYDPEvSt5J30ciN6DEDrQ8Jy4MKcIM=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
Expand Down Expand Up @@ -251,6 +259,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
Expand Down
1 change: 1 addition & 0 deletions plugins/destination/neo4j/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ func main() {
p := plugin.NewPlugin(internalPlugin.Name, internalPlugin.Version, client.New,
plugin.WithKind(internalPlugin.Kind),
plugin.WithTeam(internalPlugin.Team),
plugin.WithJSONSchema(client.JSONSchema),
)
if err := serve.Plugin(p, serve.WithPluginSentryDSN(sentryDSN), serve.WithDestinationV0V1Server()).Serve(context.Background()); err != nil {
log.Fatalf("failed to serve plugin: %v", err)
Expand Down