diff --git a/.github/workflows/dest_sqlite.yml b/.github/workflows/dest_sqlite.yml index 73f0f79512348f..d745d2631a4075 100644 --- a/.github/workflows/dest_sqlite.yml +++ b/.github/workflows/dest_sqlite.yml @@ -42,6 +42,12 @@ jobs: args: "--config ../../.golangci.yml" skip-pkg-cache: true skip-build-cache: true + - name: gen + if: github.event_name == 'pull_request' + run: make gen + - name: Fail if generation updated files + if: github.event_name == 'pull_request' + run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1) - name: Build run: go build . - name: Test diff --git a/plugins/destination/sqlite/Makefile b/plugins/destination/sqlite/Makefile index 757f2ae5ef17b2..b0f4926681c08c 100644 --- a/plugins/destination/sqlite/Makefile +++ b/plugins/destination/sqlite/Makefile @@ -22,4 +22,12 @@ release-dry-run: -v `pwd`:/go/src/$(PACKAGE_NAME) \ -w /go/src/$(PACKAGE_NAME) \ ghcr.io/cloudquery/golang-cross:v10.0.0 \ - --clean --skip-validate --snapshot --skip-publish \ No newline at end of file + --clean --skip-validate --snapshot --skip-publish + +.PHONY: gen-spec-schema +gen-spec-schema: + go run client/spec/gen/main.go + +# All gen targets +.PHONY: gen +gen: gen-spec-schema diff --git a/plugins/destination/sqlite/client/schema.json b/plugins/destination/sqlite/client/schema.json new file mode 100644 index 00000000000000..3a3db21714f73b --- /dev/null +++ b/plugins/destination/sqlite/client/schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/cloudquery/plugins/destination/sqlite/client/spec", + "$ref": "#/$defs/Spec", + "$defs": { + "Spec": { + "properties": { + "connection_string": { + "type": "string", + "minLength": 1, + "description": "Path to a file, such as `./mydb.sql`" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "connection_string" + ] + } + } +} diff --git a/plugins/destination/sqlite/client/spec.go b/plugins/destination/sqlite/client/spec.go index 9323a53f2fe636..ae85a651810d8f 100644 --- a/plugins/destination/sqlite/client/spec.go +++ b/plugins/destination/sqlite/client/spec.go @@ -1,9 +1,15 @@ package client +import _ "embed" + type Spec struct { - ConnectionString string `json:"connection_string,omitempty"` + // Path to a file, such as `./mydb.sql` + ConnectionString string `json:"connection_string,omitempty" jsonschema:"required,minLength=1"` } +//go:embed schema.json +var JSONSchema string + func (*Spec) SetDefaults() { // stub for any future defaults } diff --git a/plugins/destination/sqlite/client/spec/gen/main.go b/plugins/destination/sqlite/client/spec/gen/main.go new file mode 100644 index 00000000000000..403e4a681c8f62 --- /dev/null +++ b/plugins/destination/sqlite/client/spec/gen/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "log" + "path" + "runtime" + + "github.com/cloudquery/cloudquery/plugins/destination/sqlite/client" + cqjsonschema "github.com/cloudquery/codegen/jsonschema" +) + +func main() { + fmt.Println("Generating JSON schema for plugin spec") + cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"), + cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/sqlite/client", path.Join(currDir(), "../..")), + ) +} + +func currDir() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Failed to get caller information") + } + return path.Dir(filename) +} diff --git a/plugins/destination/sqlite/client/spec_test.go b/plugins/destination/sqlite/client/spec_test.go new file mode 100644 index 00000000000000..f98cb12ec5743b --- /dev/null +++ b/plugins/destination/sqlite/client/spec_test.go @@ -0,0 +1,41 @@ +package client + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" +) + +func TestJSONSchema(t *testing.T) { + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty spec", + Spec: `{}`, + Err: true, + }, + { + Name: "spec with connection_string", + Spec: `{"connection_string": "file"}`, + }, + { + Name: "spec with bool connection_string", + Spec: `{"connection_string": true}`, + Err: true, + }, + { + Name: "spec with null connection_string", + Spec: `{"connection_string": null}`, + Err: true, + }, + { + Name: "spec with int connection_string", + Spec: `{"connection_string": 123}`, + Err: true, + }, + { + Name: "spec with unknown field", + Spec: `{"connection_string": "abc", "unknown": "test"}`, + Err: true, + }, + }) +} diff --git a/plugins/destination/sqlite/go.mod b/plugins/destination/sqlite/go.mod index 6cc72de01ef073..8d95741997b95b 100644 --- a/plugins/destination/sqlite/go.mod +++ b/plugins/destination/sqlite/go.mod @@ -4,6 +4,7 @@ go 1.21.4 require ( github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1 + github.com/cloudquery/codegen v0.3.12 github.com/cloudquery/plugin-sdk/v4 v4.29.0 github.com/mattn/go-sqlite3 v1.14.16 github.com/rs/zerolog v1.31.0 @@ -20,6 +21,8 @@ require ( github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.10.2 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect @@ -51,6 +54,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.11.0 // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -91,6 +95,7 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/yosssi/ace v0.0.5 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/otel v1.20.0 // indirect @@ -119,3 +124,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// github.com/cloudquery/jsonschema @ cqmain +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f diff --git a/plugins/destination/sqlite/go.sum b/plugins/destination/sqlite/go.sum index 949a9ea8f9af16..1e8c2a4d57b1ae 100644 --- a/plugins/destination/sqlite/go.sum +++ b/plugins/destination/sqlite/go.sum @@ -25,9 +25,13 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= @@ -43,6 +47,10 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/cloudquery/cloudquery-api-go v1.7.1 h1:q4pxS7EXKqAPh8DSDBU3bE/vnCQXs4jUM3zcoAlGdL8= github.com/cloudquery/cloudquery-api-go v1.7.1/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA= +github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE= +github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8= +github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8= +github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= github.com/cloudquery/plugin-pb-go v1.16.7 h1:wLx5TFvS6gAvD1dcBZdv5YSskcNCnNpF1JNituka5jM= github.com/cloudquery/plugin-pb-go v1.16.7/go.mod h1:Sd08P8HIjwi3gmfoE0X21qo6HL1NiVdNl/00JrK+DkM= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= @@ -251,6 +259,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= diff --git a/plugins/destination/sqlite/main.go b/plugins/destination/sqlite/main.go index 56d8ac39daab57..5618f6ee855259 100644 --- a/plugins/destination/sqlite/main.go +++ b/plugins/destination/sqlite/main.go @@ -28,6 +28,7 @@ func main() { }), plugin.WithKind(internalPlugin.Kind), plugin.WithTeam(internalPlugin.Team), + plugin.WithJSONSchema(client.JSONSchema), ) if err := serve.Plugin(p, serve.WithPluginSentryDSN(sentryDSN), serve.WithDestinationV0V1Server()).Serve(context.Background()); err != nil { fmt.Println(err)