Skip to content

Commit 16dab3a

Browse files
authored
feat: Adding JSON schema support to firestore plugin (#16380)
fixes: #16359
1 parent 1205b36 commit 16dab3a

9 files changed

Lines changed: 227 additions & 6 deletions

File tree

.github/workflows/source_firestore.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,12 @@ jobs:
5050
args: "--config ../../.golangci.yml"
5151
skip-pkg-cache: true
5252
skip-build-cache: true
53+
- name: gen
54+
if: github.event_name == 'pull_request'
55+
run: make gen
56+
- name: Fail if generation updated files
57+
if: github.event_name == 'pull_request'
58+
run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
5359
- name: Build
5460
run: go build .
5561
- name: Test firestore

plugins/source/firestore/Makefile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ lint:
1111
gen-docs:
1212
echo "skipping docs generation for firestore source plugin"
1313

14+
.PHONY: gen-spec-schema
15+
gen-spec-schema:
16+
go run client/spec/gen/main.go
17+
1418
# All gen targets
1519
.PHONY: gen
16-
gen: gen-docs
20+
gen: gen-docs gen-spec-schema

plugins/source/firestore/client/schema.json

Lines changed: 47 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

plugins/source/firestore/client/spec.go

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,34 @@
11
package client
22

33
import (
4+
_ "embed"
45
"encoding/base64"
56
"fmt"
7+
"github.com/invopop/jsonschema"
68
"strings"
79

810
"cloud.google.com/go/firestore"
911
)
1012

13+
// Spec is the (nested) spec used by Firestore Source Plugin
1114
type Spec struct {
12-
ProjectID string `json:"project_id"`
13-
UseBase64 bool `json:"use_base64"`
15+
// The ID of the project to use for this client. If not specified, the project id will be auto-detected from the credentials.
16+
ProjectID string `json:"project_id"`
17+
// If `true` the `service_account_json` content will be treated as base64-encoded.
18+
UseBase64 bool `json:"use_base64" jsonschema:"default=false"`
19+
// Service account JSON content.
1420
ServiceAccountJSON string `json:"service_account_json"`
15-
MaxBatchSize int `json:"max_batch_size"`
16-
OrderBy string `json:"order_by"`
17-
OrderDirection string `json:"order_direction"`
21+
// Maximum batch size for each request when reading Firestore data.
22+
MaxBatchSize int `json:"max_batch_size" jsonschema:"minimum=1"`
23+
// List of fields to order the results by.
24+
OrderBy string `json:"order_by"`
25+
// The order direction used when `order_by` is `true`.
26+
OrderDirection string `json:"order_direction" jsonschema:"enum=asc,enum=desc,default=asc"`
1827
}
1928

2029
func (s *Spec) Validate() error {
30+
// decode base64 if needed - note if the Validate function is removed from the spec, this will need to be done
31+
// elsewhere in the application
2132
if s.UseBase64 {
2233
data, err := base64.StdEncoding.DecodeString(s.ServiceAccountJSON)
2334
if err != nil {
@@ -35,6 +46,12 @@ func (s *Spec) Validate() error {
3546
return nil
3647
}
3748

49+
func (Spec) JSONSchemaExtend(sc *jsonschema.Schema) {
50+
// Configure defaults
51+
sc.Properties.Value("project_id").Default = firestore.DetectProjectID
52+
sc.Properties.Value("max_batch_size").Default = 50_000
53+
}
54+
3855
func (s *Spec) SetDefaults() {
3956
if s.MaxBatchSize == 0 {
4057
s.MaxBatchSize = 50_000
@@ -43,3 +60,6 @@ func (s *Spec) SetDefaults() {
4360
s.ProjectID = firestore.DetectProjectID
4461
}
4562
}
63+
64+
//go:embed schema.json
65+
var JSONSchema string
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"path"
7+
"runtime"
8+
9+
"github.com/cloudquery/cloudquery/plugins/source/firestore/client"
10+
"github.com/cloudquery/codegen/jsonschema"
11+
)
12+
13+
func main() {
14+
fmt.Println("Generating JSON schema for plugin spec")
15+
jsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"),
16+
jsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/source/firestore/client", path.Join(currDir(), "../..")),
17+
)
18+
}
19+
20+
func currDir() string {
21+
_, filename, _, ok := runtime.Caller(0)
22+
if !ok {
23+
log.Fatal("Failed to get caller information")
24+
}
25+
return path.Dir(filename)
26+
}
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package client
2+
3+
import (
4+
"encoding/base64"
5+
"testing"
6+
7+
"github.com/cloudquery/codegen/jsonschema"
8+
)
9+
10+
func TestSpecValidate(t *testing.T) {
11+
testCases := []struct {
12+
name string
13+
spec Spec
14+
err bool
15+
}{
16+
{
17+
name: "empty",
18+
spec: Spec{},
19+
},
20+
{
21+
name: "valid",
22+
spec: Spec{
23+
ProjectID: "project_id",
24+
ServiceAccountJSON: "{}",
25+
OrderDirection: "asc",
26+
MaxBatchSize: 1,
27+
},
28+
},
29+
{
30+
name: "invalid order direction",
31+
spec: Spec{
32+
ProjectID: "project_id",
33+
ServiceAccountJSON: "{}",
34+
OrderDirection: "invalid",
35+
MaxBatchSize: 1,
36+
},
37+
err: true,
38+
},
39+
{
40+
name: "invalid max batch size",
41+
spec: Spec{
42+
ProjectID: "project_id",
43+
ServiceAccountJSON: "{}",
44+
OrderDirection: "asc",
45+
MaxBatchSize: -1,
46+
},
47+
err: true,
48+
},
49+
{
50+
name: "base64 service account",
51+
spec: Spec{
52+
ProjectID: "project_id",
53+
ServiceAccountJSON: base64.StdEncoding.EncodeToString([]byte("{}")),
54+
OrderDirection: "asc",
55+
MaxBatchSize: 1,
56+
UseBase64: true,
57+
},
58+
},
59+
}
60+
61+
for _, tc := range testCases {
62+
t.Run(tc.name, func(t *testing.T) {
63+
err := tc.spec.Validate()
64+
if tc.err && err == nil {
65+
t.Errorf("expected error but got none")
66+
}
67+
if !tc.err && err != nil {
68+
t.Errorf("expected no error but got %v", err)
69+
}
70+
})
71+
}
72+
}
73+
74+
func TestJSONSchema(t *testing.T) {
75+
jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{
76+
{
77+
Name: "empty spec",
78+
Spec: `{}`,
79+
},
80+
{
81+
Name: "valid spec",
82+
Spec: `{"project_id": "project_id","service_account_json": "{}","order_direction": "asc","max_batch_size": 1}`,
83+
},
84+
{
85+
Name: "invalid order direction",
86+
Spec: `{"project_id": "project_id","service_account_json": "{}","order_direction": "invalid","max_batch_size": 1}`,
87+
Err: true,
88+
},
89+
{
90+
Name: "invalid max batch size",
91+
Spec: `{"project_id": "project_id","service_account_json": "{}","order_direction": "asc","max_batch_size": -1}`,
92+
Err: true,
93+
},
94+
{
95+
Name: "base64 service account",
96+
Spec: `{"project_id": "project_id","service_account_json": "` + base64.StdEncoding.EncodeToString([]byte("{}")) + `","order_direction": "asc","max_batch_size": 1,"use_base64": true}`,
97+
},
98+
})
99+
}

plugins/source/firestore/go.mod

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ go 1.21.4
55
require (
66
cloud.google.com/go/firestore v1.14.0
77
github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1
8+
github.com/cloudquery/codegen v0.3.12
89
github.com/cloudquery/plugin-sdk/v4 v4.28.0
10+
github.com/invopop/jsonschema v0.11.0
911
github.com/rs/zerolog v1.31.0
1012
github.com/stretchr/testify v1.8.4
1113
golang.org/x/sync v0.5.0
@@ -27,6 +29,8 @@ require (
2729
github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect
2830
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
2931
github.com/aymerick/douceur v0.2.0 // indirect
32+
github.com/bahlo/generic-list-go v0.2.0 // indirect
33+
github.com/buger/jsonparser v1.1.1 // indirect
3034
github.com/bytedance/sonic v1.10.2 // indirect
3135
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
3236
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
@@ -101,6 +105,7 @@ require (
101105
github.com/valyala/fasttemplate v1.2.2 // indirect
102106
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
103107
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
108+
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
104109
github.com/yosssi/ace v0.0.5 // indirect
105110
github.com/zeebo/xxh3 v1.0.2 // indirect
106111
go.opencensus.io v0.24.0 // indirect
@@ -132,3 +137,6 @@ require (
132137
gopkg.in/yaml.v2 v2.4.0 // indirect
133138
gopkg.in/yaml.v3 v3.0.1 // indirect
134139
)
140+
141+
// github.com/cloudquery/jsonschema @ cqmain
142+
replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f

plugins/source/firestore/go.sum

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,13 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D
3737
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
3838
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
3939
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
40+
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
41+
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
4042
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
4143
github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
4244
github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
45+
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
46+
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
4347
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
4448
github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
4549
github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
@@ -57,6 +61,10 @@ github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLI
5761
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
5862
github.com/cloudquery/cloudquery-api-go v1.7.0 h1:9da/fBNcKnJGTKF3LFoKIMUwfnzhMCsp5RjIOSxCU7s=
5963
github.com/cloudquery/cloudquery-api-go v1.7.0/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
64+
github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE=
65+
github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8=
66+
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8=
67+
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
6068
github.com/cloudquery/plugin-pb-go v1.16.6 h1:UcN7UK89EWxh9SRGCIPQ/Ao2YB5zVugvBtF8ii536ig=
6169
github.com/cloudquery/plugin-pb-go v1.16.6/go.mod h1:/dnO/uBQGZlTvbYDPEvSt5J30ciN6DEDrQ8Jy4MKcIM=
6270
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
@@ -297,6 +305,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
297305
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
298306
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
299307
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
308+
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
309+
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
300310
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
301311
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
302312
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=

plugins/source/firestore/resources/plugin/plugin.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ func Plugin() *plugin.Plugin {
1717
Name,
1818
Version,
1919
client.Configure,
20+
plugin.WithJSONSchema(client.JSONSchema),
2021
plugin.WithKind(Kind),
2122
plugin.WithTeam(Team),
2223
)

0 commit comments

Comments
 (0)