From f47fecf6fa779e8dc0e6ee2be2a058b9cc47853b Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 2 Feb 2024 14:33:36 +0000 Subject: [PATCH 1/4] feat: Add JSON schema to elasticsearch destination --- .github/workflows/dest_elasticsearch.yml | 6 ++ plugins/destination/elasticsearch/Makefile | 8 +++ .../elasticsearch/client/schema.json | 72 +++++++++++++++++++ .../destination/elasticsearch/client/spec.go | 45 +++++++++--- .../elasticsearch/client/spec/gen/main.go | 26 +++++++ .../elasticsearch/client/spec_test.go | 50 +++++++++++++ plugins/destination/elasticsearch/go.mod | 8 +++ plugins/destination/elasticsearch/go.sum | 10 +++ plugins/destination/elasticsearch/main.go | 1 + 9 files changed, 215 insertions(+), 11 deletions(-) create mode 100644 plugins/destination/elasticsearch/client/schema.json create mode 100644 plugins/destination/elasticsearch/client/spec/gen/main.go create mode 100644 plugins/destination/elasticsearch/client/spec_test.go diff --git a/.github/workflows/dest_elasticsearch.yml b/.github/workflows/dest_elasticsearch.yml index 32a2e0c9a1c81b..7869241577e6c6 100644 --- a/.github/workflows/dest_elasticsearch.yml +++ b/.github/workflows/dest_elasticsearch.yml @@ -56,6 +56,12 @@ jobs: args: "--config ../../.golangci.yml" skip-pkg-cache: true skip-build-cache: true + - name: gen + if: github.event_name == 'pull_request' + run: make gen + - name: Fail if generation updated files + if: github.event_name == 'pull_request' + run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1) - name: Build run: go build . - name: Test diff --git a/plugins/destination/elasticsearch/Makefile b/plugins/destination/elasticsearch/Makefile index 8f6efd386d94ce..10ee9b92bd8133 100644 --- a/plugins/destination/elasticsearch/Makefile +++ b/plugins/destination/elasticsearch/Makefile @@ -10,3 +10,11 @@ test: .PHONY: lint lint: golangci-lint run --config ../../.golangci.yml + +.PHONY: gen-spec-schema +gen-spec-schema: + go run client/spec/gen/main.go + +# All gen targets +.PHONY: gen +gen: gen-spec-schema diff --git a/plugins/destination/elasticsearch/client/schema.json b/plugins/destination/elasticsearch/client/schema.json new file mode 100644 index 00000000000000..7c87e276a860f9 --- /dev/null +++ b/plugins/destination/elasticsearch/client/schema.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/cloudquery/plugins/destination/elasticsearch/client/spec", + "$ref": "#/$defs/Spec", + "$defs": { + "Spec": { + "properties": { + "addresses": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array", + "description": "A list of Elasticsearch nodes to use. (Default: [\"http://localhost:9200\"])" + }, + { + "type": "null" + } + ] + }, + "username": { + "type": "string", + "description": "Username for HTTP Basic Authentication." + }, + "password": { + "type": "string", + "description": "Password for HTTP Basic Authentication." + }, + "cloud_id": { + "type": "string", + "description": "Endpoint for the Elastic Service (https://elastic.co/cloud)." + }, + "api_key": { + "type": "string", + "description": "Base64-encoded token for authorization; if set, overrides username/password and service token." + }, + "service_token": { + "type": "string", + "description": "Service token for authorization; if set, overrides username/password." + }, + "certificate_fingerprint": { + "type": "string", + "description": "SHA256 hex fingerprint given by Elasticsearch on first launch." + }, + "ca_cert": { + "type": "string", + "description": "PEM-encoded certificate authorities.\nWhen set, an empty certificate pool will be created, and the certificates will be appended to it." + }, + "concurrency": { + "type": "integer", + "minimum": 1, + "description": "Number of concurrent worker goroutines to use for indexing. (Default: number of CPUs)" + }, + "batch_size": { + "type": "integer", + "minimum": 1, + "description": "Number of documents to batch together per request.", + "default": 1000 + }, + "batch_size_bytes": { + "type": "integer", + "minimum": 1, + "description": "Number of bytes to batch together per request.", + "default": 5242880 + } + }, + "additionalProperties": false, + "type": "object" + } + } +} diff --git a/plugins/destination/elasticsearch/client/spec.go b/plugins/destination/elasticsearch/client/spec.go index 34f718179cf33d..af4d538138a98c 100644 --- a/plugins/destination/elasticsearch/client/spec.go +++ b/plugins/destination/elasticsearch/client/spec.go @@ -1,6 +1,9 @@ package client -import "runtime" +import ( + _ "embed" + "runtime" +) const ( defaultBatchSize = 1000 @@ -8,24 +11,44 @@ const ( ) type Spec struct { - Addresses []string `json:"addresses"` // A list of Elasticsearch nodes to use. - Username string `json:"username"` // Username for HTTP Basic Authentication. - Password string `json:"password"` // Password for HTTP Basic Authentication. + // A list of Elasticsearch nodes to use. (Default: ["http://localhost:9200"]) + Addresses []string `json:"addresses"` + + // Username for HTTP Basic Authentication. + Username string `json:"username"` + + // Password for HTTP Basic Authentication. + Password string `json:"password"` + + // Endpoint for the Elastic Service (https://elastic.co/cloud). + CloudID string `json:"cloud_id"` + + // Base64-encoded token for authorization; if set, overrides username/password and service token. + APIKey string `json:"api_key"` - CloudID string `json:"cloud_id"` // Endpoint for the Elastic Service (https://elastic.co/cloud). - APIKey string `json:"api_key"` // Base64-encoded token for authorization; if set, overrides username/password and service token. - ServiceToken string `json:"service_token"` // Service token for authorization; if set, overrides username/password. - CertificateFingerprint string `json:"certificate_fingerprint"` // SHA256 hex fingerprint given by Elasticsearch on first launch. + // Service token for authorization; if set, overrides username/password. + ServiceToken string `json:"service_token"` + + // SHA256 hex fingerprint given by Elasticsearch on first launch. + CertificateFingerprint string `json:"certificate_fingerprint"` // PEM-encoded certificate authorities. // When set, an empty certificate pool will be created, and the certificates will be appended to it. CACert string `json:"ca_cert"` - Concurrency int `json:"concurrency"` // Number of concurrent worker goroutines to use for indexing. (Default: number of CPUs) - BatchSize int `json:"batch_size"` // Number of documents to batch together per request. (Default: 1000) - BatchSizeBytes int `json:"batch_size_bytes"` // Number of bytes to batch together per request. (Default: 5 MiB) + // Number of concurrent worker goroutines to use for indexing. (Default: number of CPUs) + Concurrency int `json:"concurrency" jsonschema:"minimum=1"` + + // Number of documents to batch together per request. + BatchSize int `json:"batch_size" jsonschema:"minimum=1,default=1000"` + + // Number of bytes to batch together per request. + BatchSizeBytes int `json:"batch_size_bytes" jsonschema:"minimum=1,default=5242880"` } +//go:embed schema.json +var JSONSchema string + func (s *Spec) SetDefaults() { if len(s.Addresses) == 0 { s.Addresses = []string{"http://localhost:9200"} diff --git a/plugins/destination/elasticsearch/client/spec/gen/main.go b/plugins/destination/elasticsearch/client/spec/gen/main.go new file mode 100644 index 00000000000000..e05f1e856ea4b3 --- /dev/null +++ b/plugins/destination/elasticsearch/client/spec/gen/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "log" + "path" + "runtime" + + "github.com/cloudquery/cloudquery/plugins/destination/elasticsearch/client" + cqjsonschema "github.com/cloudquery/codegen/jsonschema" +) + +func main() { + fmt.Println("Generating JSON schema for plugin spec") + cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"), + cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/elasticsearch/client", path.Join(currDir(), "../..")), + ) +} + +func currDir() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Failed to get caller information") + } + return path.Dir(filename) +} diff --git a/plugins/destination/elasticsearch/client/spec_test.go b/plugins/destination/elasticsearch/client/spec_test.go new file mode 100644 index 00000000000000..a8b82a6f054b36 --- /dev/null +++ b/plugins/destination/elasticsearch/client/spec_test.go @@ -0,0 +1,50 @@ +package client + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" +) + +func TestJSONSchema(t *testing.T) { + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty spec", + Spec: `{}`, + }, + { + Name: "spec with str addresses", + Spec: `{"addresses": "address"}`, + Err: true, + }, + { + Name: "spec with valid addresses", + Spec: `{"addresses": ["address"]}`, + }, + { + Name: "spec with bool batch_size", + Spec: `{"batch_size":false}`, + Err: true, + }, + { + Name: "spec with null batch_size", + Spec: `{"batch_size":null}`, + Err: true, + }, + { + Name: "spec with string batch_size", + Spec: `{"batch_size":"str"}`, + Err: true, + }, + { + Name: "spec with array batch_size", + Spec: `{"batch_size":["abc"]}`, + Err: true, + }, + { + Name: "spec with unknown field", + Spec: `{"unknown": "test"}`, + Err: true, + }, + }) +} diff --git a/plugins/destination/elasticsearch/go.mod b/plugins/destination/elasticsearch/go.mod index e051377d1d125b..51384809bc1aff 100644 --- a/plugins/destination/elasticsearch/go.mod +++ b/plugins/destination/elasticsearch/go.mod @@ -5,6 +5,7 @@ go 1.21.4 require ( github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1 github.com/cenkalti/backoff/v4 v4.2.1 + github.com/cloudquery/codegen v0.3.12 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/elastic/go-elasticsearch/v8 v8.6.0 github.com/goccy/go-json v0.10.2 @@ -25,6 +26,8 @@ require ( github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.10.2 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect @@ -55,6 +58,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.11.0 // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -95,6 +99,7 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/yosssi/ace v0.0.5 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/otel v1.20.0 // indirect @@ -121,3 +126,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// github.com/cloudquery/jsonschema @ cqmain +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb diff --git a/plugins/destination/elasticsearch/go.sum b/plugins/destination/elasticsearch/go.sum index 094e18171eedc2..3eb3a53213ea6a 100644 --- a/plugins/destination/elasticsearch/go.sum +++ b/plugins/destination/elasticsearch/go.sum @@ -25,9 +25,13 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= @@ -43,6 +47,10 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/cloudquery/cloudquery-api-go v1.7.2 h1:jpQfeZUxekbV7ASN5ONpGIkrtKIZvC/Y8fOj+tQxLm4= github.com/cloudquery/cloudquery-api-go v1.7.2/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA= +github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE= +github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8= +github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb h1:/l8fbvLOCNlgkHp8VUKTTL+Tk9gs5y/K3Yx/bRfReNk= +github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= github.com/cloudquery/plugin-pb-go v1.16.7 h1:wLx5TFvS6gAvD1dcBZdv5YSskcNCnNpF1JNituka5jM= github.com/cloudquery/plugin-pb-go v1.16.7/go.mod h1:Sd08P8HIjwi3gmfoE0X21qo6HL1NiVdNl/00JrK+DkM= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= @@ -256,6 +264,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= diff --git a/plugins/destination/elasticsearch/main.go b/plugins/destination/elasticsearch/main.go index 82248ba5bd6bc0..a6532bef433e1d 100644 --- a/plugins/destination/elasticsearch/main.go +++ b/plugins/destination/elasticsearch/main.go @@ -18,6 +18,7 @@ func main() { p := plugin.NewPlugin(internalPlugin.Name, internalPlugin.Version, client.New, plugin.WithKind(internalPlugin.Kind), plugin.WithTeam(internalPlugin.Team), + plugin.WithJSONSchema(client.JSONSchema), ) if err := serve.Plugin(p, serve.WithPluginSentryDSN(sentryDSN), serve.WithDestinationV0V1Server()).Serve(context.Background()); err != nil { log.Fatal(err) From 68adaad7d4c7c1c9054bfab9d312f5d26d5a55ed Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 2 Feb 2024 20:40:32 +0000 Subject: [PATCH 2/4] Add default --- plugins/destination/elasticsearch/client/schema.json | 5 ++++- plugins/destination/elasticsearch/client/spec.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/destination/elasticsearch/client/schema.json b/plugins/destination/elasticsearch/client/schema.json index 7c87e276a860f9..a7f3d6b73c64a9 100644 --- a/plugins/destination/elasticsearch/client/schema.json +++ b/plugins/destination/elasticsearch/client/schema.json @@ -12,7 +12,10 @@ "type": "string" }, "type": "array", - "description": "A list of Elasticsearch nodes to use. (Default: [\"http://localhost:9200\"])" + "description": "A list of Elasticsearch nodes to use. (Default: [\"http://localhost:9200\"])", + "default": [ + "http://localhost:9200" + ] }, { "type": "null" diff --git a/plugins/destination/elasticsearch/client/spec.go b/plugins/destination/elasticsearch/client/spec.go index af4d538138a98c..37e71b2d2b873f 100644 --- a/plugins/destination/elasticsearch/client/spec.go +++ b/plugins/destination/elasticsearch/client/spec.go @@ -12,7 +12,7 @@ const ( type Spec struct { // A list of Elasticsearch nodes to use. (Default: ["http://localhost:9200"]) - Addresses []string `json:"addresses"` + Addresses []string `json:"addresses" jsonschema:"default=http://localhost:9200"` // Username for HTTP Basic Authentication. Username string `json:"username"` From 1652ead4280c5752c9166cb6f865b52895111390 Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Sat, 3 Feb 2024 10:11:59 +0200 Subject: [PATCH 3/4] Update plugins/destination/elasticsearch/client/spec.go --- plugins/destination/elasticsearch/client/spec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/destination/elasticsearch/client/spec.go b/plugins/destination/elasticsearch/client/spec.go index 37e71b2d2b873f..116a8d9027a308 100644 --- a/plugins/destination/elasticsearch/client/spec.go +++ b/plugins/destination/elasticsearch/client/spec.go @@ -11,7 +11,7 @@ const ( ) type Spec struct { - // A list of Elasticsearch nodes to use. (Default: ["http://localhost:9200"]) + // A list of Elasticsearch nodes to use. Addresses []string `json:"addresses" jsonschema:"default=http://localhost:9200"` // Username for HTTP Basic Authentication. From 9595f225c3bc96e925edc0e7a8bcf98112e26fdf Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Sat, 3 Feb 2024 10:13:42 +0200 Subject: [PATCH 4/4] gen --- plugins/destination/elasticsearch/client/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/destination/elasticsearch/client/schema.json b/plugins/destination/elasticsearch/client/schema.json index a7f3d6b73c64a9..ff65e11922260c 100644 --- a/plugins/destination/elasticsearch/client/schema.json +++ b/plugins/destination/elasticsearch/client/schema.json @@ -12,7 +12,7 @@ "type": "string" }, "type": "array", - "description": "A list of Elasticsearch nodes to use. (Default: [\"http://localhost:9200\"])", + "description": "A list of Elasticsearch nodes to use.", "default": [ "http://localhost:9200" ]