diff --git a/.github/workflows/source_firestore.yml b/.github/workflows/source_firestore.yml index 10e1dadad47a16..18bc24c329e26f 100644 --- a/.github/workflows/source_firestore.yml +++ b/.github/workflows/source_firestore.yml @@ -50,6 +50,12 @@ jobs: args: "--config ../../.golangci.yml" skip-pkg-cache: true skip-build-cache: true + - name: gen + if: github.event_name == 'pull_request' + run: make gen + - name: Fail if generation updated files + if: github.event_name == 'pull_request' + run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1) - name: Build run: go build . - name: Test firestore diff --git a/plugins/source/firestore/Makefile b/plugins/source/firestore/Makefile index 9c5dbffd3e2af1..c12c057ace25b7 100644 --- a/plugins/source/firestore/Makefile +++ b/plugins/source/firestore/Makefile @@ -11,6 +11,10 @@ lint: gen-docs: echo "skipping docs generation for firestore source plugin" +.PHONY: gen-spec-schema +gen-spec-schema: + go run client/spec/gen/main.go + # All gen targets .PHONY: gen -gen: gen-docs +gen: gen-docs gen-spec-schema diff --git a/plugins/source/firestore/client/schema.json b/plugins/source/firestore/client/schema.json new file mode 100644 index 00000000000000..a27fd024111f0e --- /dev/null +++ b/plugins/source/firestore/client/schema.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/cloudquery/plugins/source/firestore/client/spec", + "$ref": "#/$defs/Spec", + "$defs": { + "Spec": { + "properties": { + "project_id": { + "type": "string", + "description": "The ID of the project to use for this client. If not specified, the project id will be auto-detected from the credentials.", + "default": "*detect-project-id*" + }, + "use_base64": { + "type": "boolean", + "description": "If `true` the `service_account_json` content will be treated as base64-encoded.", + "default": false + }, + "service_account_json": { + "type": "string", + "description": "Service account JSON content." + }, + "max_batch_size": { + "type": "integer", + "minimum": 1, + "description": "Maximum batch size for each request when reading Firestore data.", + "default": 50000 + }, + "order_by": { + "type": "string", + "description": "List of fields to order the results by." + }, + "order_direction": { + "type": "string", + "enum": [ + "asc", + "desc" + ], + "description": "The order direction used when `order_by` is `true`.", + "default": "asc" + } + }, + "additionalProperties": false, + "type": "object", + "description": "Spec is the (nested) spec used by Firestore Source Plugin" + } + } +} diff --git a/plugins/source/firestore/client/spec.go b/plugins/source/firestore/client/spec.go index d8e61d4ba6655b..01bb8ee94c717f 100644 --- a/plugins/source/firestore/client/spec.go +++ b/plugins/source/firestore/client/spec.go @@ -1,23 +1,34 @@ package client import ( + _ "embed" "encoding/base64" "fmt" + "github.com/invopop/jsonschema" "strings" "cloud.google.com/go/firestore" ) +// Spec is the (nested) spec used by Firestore Source Plugin type Spec struct { - ProjectID string `json:"project_id"` - UseBase64 bool `json:"use_base64"` + // The ID of the project to use for this client. If not specified, the project id will be auto-detected from the credentials. + ProjectID string `json:"project_id"` + // If `true` the `service_account_json` content will be treated as base64-encoded. + UseBase64 bool `json:"use_base64" jsonschema:"default=false"` + // Service account JSON content. ServiceAccountJSON string `json:"service_account_json"` - MaxBatchSize int `json:"max_batch_size"` - OrderBy string `json:"order_by"` - OrderDirection string `json:"order_direction"` + // Maximum batch size for each request when reading Firestore data. + MaxBatchSize int `json:"max_batch_size" jsonschema:"minimum=1"` + // List of fields to order the results by. + OrderBy string `json:"order_by"` + // The order direction used when `order_by` is `true`. + OrderDirection string `json:"order_direction" jsonschema:"enum=asc,enum=desc,default=asc"` } func (s *Spec) Validate() error { + // decode base64 if needed - note if the Validate function is removed from the spec, this will need to be done + // elsewhere in the application if s.UseBase64 { data, err := base64.StdEncoding.DecodeString(s.ServiceAccountJSON) if err != nil { @@ -35,6 +46,12 @@ func (s *Spec) Validate() error { return nil } +func (Spec) JSONSchemaExtend(sc *jsonschema.Schema) { + // Configure defaults + sc.Properties.Value("project_id").Default = firestore.DetectProjectID + sc.Properties.Value("max_batch_size").Default = 50_000 +} + func (s *Spec) SetDefaults() { if s.MaxBatchSize == 0 { s.MaxBatchSize = 50_000 @@ -43,3 +60,6 @@ func (s *Spec) SetDefaults() { s.ProjectID = firestore.DetectProjectID } } + +//go:embed schema.json +var JSONSchema string diff --git a/plugins/source/firestore/client/spec/gen/main.go b/plugins/source/firestore/client/spec/gen/main.go new file mode 100644 index 00000000000000..b63ba4e7b8aa27 --- /dev/null +++ b/plugins/source/firestore/client/spec/gen/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "log" + "path" + "runtime" + + "github.com/cloudquery/cloudquery/plugins/source/firestore/client" + "github.com/cloudquery/codegen/jsonschema" +) + +func main() { + fmt.Println("Generating JSON schema for plugin spec") + jsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"), + jsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/source/firestore/client", path.Join(currDir(), "../..")), + ) +} + +func currDir() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Failed to get caller information") + } + return path.Dir(filename) +} diff --git a/plugins/source/firestore/client/spec_test.go b/plugins/source/firestore/client/spec_test.go new file mode 100644 index 00000000000000..fd846e40a40ca5 --- /dev/null +++ b/plugins/source/firestore/client/spec_test.go @@ -0,0 +1,99 @@ +package client + +import ( + "encoding/base64" + "testing" + + "github.com/cloudquery/codegen/jsonschema" +) + +func TestSpecValidate(t *testing.T) { + testCases := []struct { + name string + spec Spec + err bool + }{ + { + name: "empty", + spec: Spec{}, + }, + { + name: "valid", + spec: Spec{ + ProjectID: "project_id", + ServiceAccountJSON: "{}", + OrderDirection: "asc", + MaxBatchSize: 1, + }, + }, + { + name: "invalid order direction", + spec: Spec{ + ProjectID: "project_id", + ServiceAccountJSON: "{}", + OrderDirection: "invalid", + MaxBatchSize: 1, + }, + err: true, + }, + { + name: "invalid max batch size", + spec: Spec{ + ProjectID: "project_id", + ServiceAccountJSON: "{}", + OrderDirection: "asc", + MaxBatchSize: -1, + }, + err: true, + }, + { + name: "base64 service account", + spec: Spec{ + ProjectID: "project_id", + ServiceAccountJSON: base64.StdEncoding.EncodeToString([]byte("{}")), + OrderDirection: "asc", + MaxBatchSize: 1, + UseBase64: true, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.spec.Validate() + if tc.err && err == nil { + t.Errorf("expected error but got none") + } + if !tc.err && err != nil { + t.Errorf("expected no error but got %v", err) + } + }) + } +} + +func TestJSONSchema(t *testing.T) { + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty spec", + Spec: `{}`, + }, + { + Name: "valid spec", + Spec: `{"project_id": "project_id","service_account_json": "{}","order_direction": "asc","max_batch_size": 1}`, + }, + { + Name: "invalid order direction", + Spec: `{"project_id": "project_id","service_account_json": "{}","order_direction": "invalid","max_batch_size": 1}`, + Err: true, + }, + { + Name: "invalid max batch size", + Spec: `{"project_id": "project_id","service_account_json": "{}","order_direction": "asc","max_batch_size": -1}`, + Err: true, + }, + { + Name: "base64 service account", + Spec: `{"project_id": "project_id","service_account_json": "` + base64.StdEncoding.EncodeToString([]byte("{}")) + `","order_direction": "asc","max_batch_size": 1,"use_base64": true}`, + }, + }) +} diff --git a/plugins/source/firestore/go.mod b/plugins/source/firestore/go.mod index 0ea9b14ae59002..94df03b1651af7 100644 --- a/plugins/source/firestore/go.mod +++ b/plugins/source/firestore/go.mod @@ -5,7 +5,9 @@ go 1.21.4 require ( cloud.google.com/go/firestore v1.14.0 github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1 + github.com/cloudquery/codegen v0.3.12 github.com/cloudquery/plugin-sdk/v4 v4.28.0 + github.com/invopop/jsonschema v0.11.0 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 golang.org/x/sync v0.5.0 @@ -27,6 +29,8 @@ require ( github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.10.2 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect @@ -101,6 +105,7 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/yosssi/ace v0.0.5 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect @@ -132,3 +137,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// github.com/cloudquery/jsonschema @ cqmain +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f diff --git a/plugins/source/firestore/go.sum b/plugins/source/firestore/go.sum index 6b3d9bf9ac4c5b..8e8af210cae315 100644 --- a/plugins/source/firestore/go.sum +++ b/plugins/source/firestore/go.sum @@ -37,9 +37,13 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= @@ -57,6 +61,10 @@ github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLI github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudquery/cloudquery-api-go v1.7.0 h1:9da/fBNcKnJGTKF3LFoKIMUwfnzhMCsp5RjIOSxCU7s= github.com/cloudquery/cloudquery-api-go v1.7.0/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA= +github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE= +github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8= +github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8= +github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= github.com/cloudquery/plugin-pb-go v1.16.6 h1:UcN7UK89EWxh9SRGCIPQ/Ao2YB5zVugvBtF8ii536ig= github.com/cloudquery/plugin-pb-go v1.16.6/go.mod h1:/dnO/uBQGZlTvbYDPEvSt5J30ciN6DEDrQ8Jy4MKcIM= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= @@ -297,6 +305,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= diff --git a/plugins/source/firestore/resources/plugin/plugin.go b/plugins/source/firestore/resources/plugin/plugin.go index 2f92558cd93610..8ab633e4635ebe 100644 --- a/plugins/source/firestore/resources/plugin/plugin.go +++ b/plugins/source/firestore/resources/plugin/plugin.go @@ -17,6 +17,7 @@ func Plugin() *plugin.Plugin { Name, Version, client.Configure, + plugin.WithJSONSchema(client.JSONSchema), plugin.WithKind(Kind), plugin.WithTeam(Team), )