cloudquery · kodiakhq · Jan 31, 2024 · Jan 31, 2024 · Jan 31, 2024 · Jan 31, 2024
@@ -50,6 +50,12 @@ jobs:
           args: "--config ../../.golangci.yml"
           skip-pkg-cache: true
           skip-build-cache: true
+      - name: gen
+        if: github.event_name == 'pull_request'
+        run: make gen
+      - name: Fail if generation updated files
+        if: github.event_name == 'pull_request'
+        run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
       - name: Build
         run: go build .
       - name: Test firestore

@@ -11,6 +11,10 @@ lint:
 gen-docs:
 	echo "skipping docs generation for firestore source plugin"
 
+.PHONY: gen-spec-schema
+gen-spec-schema:
+	go run client/spec/gen/main.go
+
 # All gen targets
 .PHONY: gen
-gen: gen-docs
+gen: gen-docs gen-spec-schema
@@ -1,23 +1,34 @@
 package client
 
 import (
+	_ "embed"
 	"encoding/base64"
 	"fmt"
+	"github.com/invopop/jsonschema"
 	"strings"
 
 	"cloud.google.com/go/firestore"
 )
 
+// Spec is the (nested) spec used by Firestore Source Plugin
 type Spec struct {
-	ProjectID          string `json:"project_id"`
-	UseBase64          bool   `json:"use_base64"`
+	// The ID of the project to use for this client. If not specified, the project id will be auto-detected from the credentials.
+	ProjectID string `json:"project_id"`
+	// If `true` the `service_account_json` content will be treated as base64-encoded.
+	UseBase64 bool `json:"use_base64" jsonschema:"default=false"`
+	//  Service account JSON content.
 	ServiceAccountJSON string `json:"service_account_json"`
-	MaxBatchSize       int    `json:"max_batch_size"`
-	OrderBy            string `json:"order_by"`
-	OrderDirection     string `json:"order_direction"`
+	// Maximum batch size for each request when reading Firestore data.
+	MaxBatchSize int `json:"max_batch_size" jsonschema:"minimum=1"`
+	// List of fields to order the results by.
+	OrderBy string `json:"order_by"`
+	// The order direction used when `order_by` is `true`.
+	OrderDirection string `json:"order_direction" jsonschema:"enum=asc,enum=desc,default=asc"`
 }
 
 func (s *Spec) Validate() error {
+	// decode base64 if needed - note if the Validate function is removed from the spec, this will need to be done
+	// elsewhere in the application
 	if s.UseBase64 {
 		data, err := base64.StdEncoding.DecodeString(s.ServiceAccountJSON)
 		if err != nil {
@@ -35,6 +46,12 @@ func (s *Spec) Validate() error {
 	return nil
 }
 
+func (Spec) JSONSchemaExtend(sc *jsonschema.Schema) {
+	// Configure defaults
+	sc.Properties.Value("project_id").Default = firestore.DetectProjectID
+	sc.Properties.Value("max_batch_size").Default = 50_000
+}
+
 func (s *Spec) SetDefaults() {
 	if s.MaxBatchSize == 0 {
 		s.MaxBatchSize = 50_000
@@ -43,3 +60,6 @@ func (s *Spec) SetDefaults() {
 		s.ProjectID = firestore.DetectProjectID
 	}
 }
+
+//go:embed schema.json
+var JSONSchema string
@@ -0,0 +1,26 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"path"
+	"runtime"
+
+	"github.com/cloudquery/cloudquery/plugins/source/firestore/client"
+	"github.com/cloudquery/codegen/jsonschema"
+)
+
+func main() {
+	fmt.Println("Generating JSON schema for plugin spec")
+	jsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"),
+		jsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/source/firestore/client", path.Join(currDir(), "../..")),
+	)
+}
+
+func currDir() string {
+	_, filename, _, ok := runtime.Caller(0)
+	if !ok {
+		log.Fatal("Failed to get caller information")
+	}
+	return path.Dir(filename)
+}
@@ -0,0 +1,99 @@
+package client
+
+import (
+	"encoding/base64"
+	"testing"
+
+	"github.com/cloudquery/codegen/jsonschema"
+)
+
+func TestSpecValidate(t *testing.T) {
+	testCases := []struct {
+		name string
+		spec Spec
+		err  bool
+	}{
+		{
+			name: "empty",
+			spec: Spec{},
+		},
+		{
+			name: "valid",
+			spec: Spec{
+				ProjectID:          "project_id",
+				ServiceAccountJSON: "{}",
+				OrderDirection:     "asc",
+				MaxBatchSize:       1,
+			},
+		},
+		{
+			name: "invalid order direction",
+			spec: Spec{
+				ProjectID:          "project_id",
+				ServiceAccountJSON: "{}",
+				OrderDirection:     "invalid",
+				MaxBatchSize:       1,
+			},
+			err: true,
+		},
+		{
+			name: "invalid max batch size",
+			spec: Spec{
+				ProjectID:          "project_id",
+				ServiceAccountJSON: "{}",
+				OrderDirection:     "asc",
+				MaxBatchSize:       -1,
+			},
+			err: true,
+		},
+		{
+			name: "base64 service account",
+			spec: Spec{
+				ProjectID:          "project_id",
+				ServiceAccountJSON: base64.StdEncoding.EncodeToString([]byte("{}")),
+				OrderDirection:     "asc",
+				MaxBatchSize:       1,
+				UseBase64:          true,
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.spec.Validate()
+			if tc.err && err == nil {
+				t.Errorf("expected error but got none")
+			}
+			if !tc.err && err != nil {
+				t.Errorf("expected no error but got %v", err)
+			}
+		})
+	}
+}
+
+func TestJSONSchema(t *testing.T) {
+	jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{
+		{
+			Name: "empty spec",
+			Spec: `{}`,
+		},
+		{
+			Name: "valid spec",
+			Spec: `{"project_id": "project_id","service_account_json": "{}","order_direction": "asc","max_batch_size": 1}`,
+		},
+		{
+			Name: "invalid order direction",
+			Spec: `{"project_id": "project_id","service_account_json": "{}","order_direction": "invalid","max_batch_size": 1}`,
+			Err:  true,
+		},
+		{
+			Name: "invalid max batch size",
+			Spec: `{"project_id": "project_id","service_account_json": "{}","order_direction": "asc","max_batch_size": -1}`,
+			Err:  true,
+		},
+		{
+			Name: "base64 service account",
+			Spec: `{"project_id": "project_id","service_account_json": "` + base64.StdEncoding.EncodeToString([]byte("{}")) + `","order_direction": "asc","max_batch_size": 1,"use_base64": true}`,
+		},
+	})
+}
diff --git a/plugins/source/firestore/go.mod b/plugins/source/firestore/go.mod
@@ -5,7 +5,9 @@ go 1.21.4
 require (
 	cloud.google.com/go/firestore v1.14.0
 	github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1
+	github.com/cloudquery/codegen v0.3.12
 	github.com/cloudquery/plugin-sdk/v4 v4.28.0
+	github.com/invopop/jsonschema v0.11.0
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
 	golang.org/x/sync v0.5.0
@@ -27,6 +29,8 @@ require (
 	github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/bytedance/sonic v1.10.2 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
@@ -101,6 +105,7 @@ require (
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/yosssi/ace v0.0.5 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
@@ -132,3 +137,6 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+// github.com/cloudquery/jsonschema @ cqmain
+replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f
diff --git a/plugins/source/firestore/go.sum b/plugins/source/firestore/go.sum
@@ -37,9 +37,13 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D
 github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
+github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
 github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
 github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
 github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
@@ -57,6 +61,10 @@ github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLI
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudquery/cloudquery-api-go v1.7.0 h1:9da/fBNcKnJGTKF3LFoKIMUwfnzhMCsp5RjIOSxCU7s=
 github.com/cloudquery/cloudquery-api-go v1.7.0/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
+github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE=
+github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8=
+github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8=
+github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
 github.com/cloudquery/plugin-pb-go v1.16.6 h1:UcN7UK89EWxh9SRGCIPQ/Ao2YB5zVugvBtF8ii536ig=
 github.com/cloudquery/plugin-pb-go v1.16.6/go.mod h1:/dnO/uBQGZlTvbYDPEvSt5J30ciN6DEDrQ8Jy4MKcIM=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
@@ -297,6 +305,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
+github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
+github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=

@@ -17,6 +17,7 @@ func Plugin() *plugin.Plugin {
 		Name,
 		Version,
 		client.Configure,
+		plugin.WithJSONSchema(client.JSONSchema),
 		plugin.WithKind(Kind),
 		plugin.WithTeam(Team),
 	)