Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/dest_elasticsearch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@ jobs:
args: "--config ../../.golangci.yml"
skip-pkg-cache: true
skip-build-cache: true
- name: gen
if: github.event_name == 'pull_request'
run: make gen
- name: Fail if generation updated files
if: github.event_name == 'pull_request'
run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
- name: Build
run: go build .
- name: Test
Expand Down
8 changes: 8 additions & 0 deletions plugins/destination/elasticsearch/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,11 @@ test:
.PHONY: lint
lint:
golangci-lint run --config ../../.golangci.yml

.PHONY: gen-spec-schema
gen-spec-schema:
go run client/spec/gen/main.go

# All gen targets
.PHONY: gen
gen: gen-spec-schema
75 changes: 75 additions & 0 deletions plugins/destination/elasticsearch/client/schema.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

45 changes: 34 additions & 11 deletions plugins/destination/elasticsearch/client/spec.go
Original file line number Diff line number Diff line change
@@ -1,31 +1,54 @@
package client

import "runtime"
import (
_ "embed"
"runtime"
)

const (
defaultBatchSize = 1000
defaultBatchSizeBytes = 5 * 1024 * 1024
)

type Spec struct {
Addresses []string `json:"addresses"` // A list of Elasticsearch nodes to use.
Username string `json:"username"` // Username for HTTP Basic Authentication.
Password string `json:"password"` // Password for HTTP Basic Authentication.
// A list of Elasticsearch nodes to use.
Addresses []string `json:"addresses" jsonschema:"default=http://localhost:9200"`

// Username for HTTP Basic Authentication.
Username string `json:"username"`

// Password for HTTP Basic Authentication.
Password string `json:"password"`

// Endpoint for the Elastic Service (https://elastic.co/cloud).
Comment thread
candiduslynx marked this conversation as resolved.
CloudID string `json:"cloud_id"`

// Base64-encoded token for authorization; if set, overrides username/password and service token.
APIKey string `json:"api_key"`

CloudID string `json:"cloud_id"` // Endpoint for the Elastic Service (https://elastic.co/cloud).
APIKey string `json:"api_key"` // Base64-encoded token for authorization; if set, overrides username/password and service token.
ServiceToken string `json:"service_token"` // Service token for authorization; if set, overrides username/password.
CertificateFingerprint string `json:"certificate_fingerprint"` // SHA256 hex fingerprint given by Elasticsearch on first launch.
// Service token for authorization; if set, overrides username/password.
ServiceToken string `json:"service_token"`

// SHA256 hex fingerprint given by Elasticsearch on first launch.
CertificateFingerprint string `json:"certificate_fingerprint"`

// PEM-encoded certificate authorities.
// When set, an empty certificate pool will be created, and the certificates will be appended to it.
CACert string `json:"ca_cert"`

Concurrency int `json:"concurrency"` // Number of concurrent worker goroutines to use for indexing. (Default: number of CPUs)
BatchSize int `json:"batch_size"` // Number of documents to batch together per request. (Default: 1000)
BatchSizeBytes int `json:"batch_size_bytes"` // Number of bytes to batch together per request. (Default: 5 MiB)
// Number of concurrent worker goroutines to use for indexing. (Default: number of CPUs)
Concurrency int `json:"concurrency" jsonschema:"minimum=1"`

// Number of documents to batch together per request.
BatchSize int `json:"batch_size" jsonschema:"minimum=1,default=1000"`

// Number of bytes to batch together per request.
BatchSizeBytes int `json:"batch_size_bytes" jsonschema:"minimum=1,default=5242880"`
}

//go:embed schema.json
var JSONSchema string

func (s *Spec) SetDefaults() {
if len(s.Addresses) == 0 {
s.Addresses = []string{"http://localhost:9200"}
Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is very basic and Validate() is empty... doesn't make a good json schema :)

Expand Down
26 changes: 26 additions & 0 deletions plugins/destination/elasticsearch/client/spec/gen/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package main

import (
"fmt"
"log"
"path"
"runtime"

"github.com/cloudquery/cloudquery/plugins/destination/elasticsearch/client"
cqjsonschema "github.com/cloudquery/codegen/jsonschema"
)

func main() {
fmt.Println("Generating JSON schema for plugin spec")
cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"),
cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/elasticsearch/client", path.Join(currDir(), "../..")),
)
}

func currDir() string {
_, filename, _, ok := runtime.Caller(0)
if !ok {
log.Fatal("Failed to get caller information")
}
return path.Dir(filename)
}
50 changes: 50 additions & 0 deletions plugins/destination/elasticsearch/client/spec_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package client

import (
"testing"

"github.com/cloudquery/codegen/jsonschema"
)

func TestJSONSchema(t *testing.T) {
jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{
{
Name: "empty spec",
Spec: `{}`,
},
{
Name: "spec with str addresses",
Spec: `{"addresses": "address"}`,
Err: true,
},
{
Name: "spec with valid addresses",
Spec: `{"addresses": ["address"]}`,
},
{
Name: "spec with bool batch_size",
Spec: `{"batch_size":false}`,
Err: true,
},
{
Name: "spec with null batch_size",
Spec: `{"batch_size":null}`,
Err: true,
},
{
Name: "spec with string batch_size",
Spec: `{"batch_size":"str"}`,
Err: true,
},
{
Name: "spec with array batch_size",
Spec: `{"batch_size":["abc"]}`,
Err: true,
},
{
Name: "spec with unknown field",
Spec: `{"unknown": "test"}`,
Err: true,
},
})
}
8 changes: 8 additions & 0 deletions plugins/destination/elasticsearch/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.21.4
require (
github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1
github.com/cenkalti/backoff/v4 v4.2.1
github.com/cloudquery/codegen v0.3.12
github.com/cloudquery/plugin-sdk/v4 v4.29.1
github.com/elastic/go-elasticsearch/v8 v8.6.0
github.com/goccy/go-json v0.10.2
Expand All @@ -25,6 +26,8 @@ require (
github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/bytedance/sonic v1.10.2 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
github.com/chenzhuoyu/iasm v0.9.1 // indirect
Expand Down Expand Up @@ -55,6 +58,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/invopop/jsonschema v0.11.0 // indirect
github.com/iris-contrib/schema v0.0.6 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand Down Expand Up @@ -95,6 +99,7 @@ require (
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/yosssi/ace v0.0.5 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/otel v1.20.0 // indirect
Expand All @@ -121,3 +126,6 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

// github.com/cloudquery/jsonschema @ cqmain
replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb
10 changes: 10 additions & 0 deletions plugins/destination/elasticsearch/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,13 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
Expand All @@ -43,6 +47,10 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0
github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
github.com/cloudquery/cloudquery-api-go v1.7.2 h1:jpQfeZUxekbV7ASN5ONpGIkrtKIZvC/Y8fOj+tQxLm4=
github.com/cloudquery/cloudquery-api-go v1.7.2/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE=
github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8=
github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb h1:/l8fbvLOCNlgkHp8VUKTTL+Tk9gs5y/K3Yx/bRfReNk=
github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
github.com/cloudquery/plugin-pb-go v1.16.7 h1:wLx5TFvS6gAvD1dcBZdv5YSskcNCnNpF1JNituka5jM=
github.com/cloudquery/plugin-pb-go v1.16.7/go.mod h1:Sd08P8HIjwi3gmfoE0X21qo6HL1NiVdNl/00JrK+DkM=
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
Expand Down Expand Up @@ -256,6 +264,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
Expand Down
1 change: 1 addition & 0 deletions plugins/destination/elasticsearch/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ func main() {
p := plugin.NewPlugin(internalPlugin.Name, internalPlugin.Version, client.New,
plugin.WithKind(internalPlugin.Kind),
plugin.WithTeam(internalPlugin.Team),
plugin.WithJSONSchema(client.JSONSchema),
)
if err := serve.Plugin(p, serve.WithPluginSentryDSN(sentryDSN), serve.WithDestinationV0V1Server()).Serve(context.Background()); err != nil {
log.Fatal(err)
Expand Down