Skip to content

Commit e3c566c

Browse files
candiduslynxcandiduslynx
authored
feat: Add JSON schema to azblob destination plugin (#16429)
Closes #16405 Basically, an adapted version of #16404
1 parent dc8df9d commit e3c566c

14 files changed

Lines changed: 741 additions & 160 deletions

File tree

‎.github/workflows/dest_azblob.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ jobs:
4242
args: "--config ../../.golangci.yml"
4343
skip-pkg-cache: true
4444
skip-build-cache: true
45+
- name: gen
46+
if: github.event_name == 'pull_request'
47+
run: make gen
48+
- name: Fail if generation updated files
49+
if: github.event_name == 'pull_request'
50+
run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
4551
- name: Build
4652
run: go build .
4753
- name: Test azblob plugin

‎plugins/destination/azblob/Makefile‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,13 @@ test:
55
.PHONY: lint
66
lint:
77
golangci-lint run --config ../../.golangci.yml
8+
9+
.PHONY: gen-spec-schema
10+
gen-spec-schema:
11+
# required for loading comments from filetypes
12+
go mod vendor
13+
go run client/spec/gen/main.go
14+
15+
# All gen targets
16+
.PHONY: gen
17+
gen: gen-spec-schema

‎plugins/destination/azblob/client/client.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
1010
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
11+
"github.com/cloudquery/cloudquery/plugins/destination/azblob/client/spec"
1112
"github.com/cloudquery/filetypes/v4"
1213
"github.com/cloudquery/plugin-sdk/v4/plugin"
1314
"github.com/cloudquery/plugin-sdk/v4/writers/streamingbatchwriter"
@@ -21,30 +22,30 @@ type Client struct {
2122
streamingbatchwriter.UnimplementedDeleteRecords
2223

2324
logger zerolog.Logger
24-
spec *Spec
25+
spec *spec.Spec
2526
*filetypes.Client
2627
writer *streamingbatchwriter.StreamingBatchWriter
2728

2829
storageClient *azblob.Client
2930
}
3031

31-
func New(ctx context.Context, logger zerolog.Logger, spec []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
32+
func New(ctx context.Context, logger zerolog.Logger, s []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
3233
c := &Client{
3334
logger: logger.With().Str("module", "azb").Logger(),
3435
}
3536
if opts.NoConnection {
3637
return c, nil
3738
}
3839

39-
if err := json.Unmarshal(spec, &c.spec); err != nil {
40+
if err := json.Unmarshal(s, &c.spec); err != nil {
4041
return nil, fmt.Errorf("failed to unmarshal azblob spec: %w", err)
4142
}
4243
if err := c.spec.Validate(); err != nil {
4344
return nil, err
4445
}
4546
c.spec.SetDefaults()
4647

47-
filetypesClient, err := filetypes.NewClient(c.spec.FileSpec)
48+
filetypesClient, err := filetypes.NewClient(&c.spec.FileSpec)
4849
if err != nil {
4950
return nil, fmt.Errorf("failed to create filetypes client: %w", err)
5051
}

‎plugins/destination/azblob/client/client_test.go‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/apache/arrow/go/v15/arrow"
1111
"github.com/apache/arrow/go/v15/arrow/array"
1212
"github.com/apache/arrow/go/v15/arrow/memory"
13+
"github.com/cloudquery/cloudquery/plugins/destination/azblob/client/spec"
1314
"github.com/cloudquery/filetypes/v4"
1415
"github.com/cloudquery/plugin-sdk/v4/message"
1516
"github.com/cloudquery/plugin-sdk/v4/plugin"
@@ -19,8 +20,8 @@ import (
1920
)
2021

2122
const (
22-
storage_account = "cqdestinationazblob"
23-
container = "test"
23+
storageAccount = "cqdestinationazblob"
24+
container = "test"
2425
)
2526

2627
func TestPlugin(t *testing.T) {
@@ -29,30 +30,28 @@ func TestPlugin(t *testing.T) {
2930
filetypes.FormatTypeJSON,
3031
filetypes.FormatTypeParquet,
3132
} {
32-
spec := Spec{
33-
StorageAccount: storage_account,
33+
s := spec.Spec{
34+
StorageAccount: storageAccount,
3435
Container: container,
3536
Path: t.TempDir(),
3637
NoRotate: true,
37-
FileSpec: &filetypes.FileSpec{
38-
Format: ft,
39-
},
38+
FileSpec: filetypes.FileSpec{Format: ft},
4039
}
4140

4241
t.Run("generic/"+string(ft), func(t *testing.T) {
43-
testPlugin(t, &spec)
42+
testPlugin(t, &s)
4443
})
4544

4645
t.Run("write/"+string(ft), func(t *testing.T) {
47-
testPluginCustom(t, &spec)
46+
testPluginCustom(t, &s)
4847
})
4948
}
5049
}
5150

52-
func testPlugin(t *testing.T, spec *Spec) {
51+
func testPlugin(t *testing.T, s *spec.Spec) {
5352
ctx := context.Background()
5453
p := plugin.NewPlugin("azblob", "development", New)
55-
b, err := json.Marshal(spec)
54+
b, err := json.Marshal(s)
5655
if err != nil {
5756
t.Fatal(err)
5857
}
@@ -71,7 +70,7 @@ func testPlugin(t *testing.T, spec *Spec) {
7170
)
7271
}
7372

74-
func testPluginCustom(t *testing.T, spec *Spec) {
73+
func testPluginCustom(t *testing.T, s *spec.Spec) {
7574
ctx := context.Background()
7675

7776
var client plugin.Client
@@ -81,7 +80,7 @@ func testPluginCustom(t *testing.T, spec *Spec) {
8180
client, err = New(ctx, logger, spec, opts)
8281
return client, err
8382
})
84-
b, err := json.Marshal(spec)
83+
b, err := json.Marshal(s)
8584
if err != nil {
8685
t.Fatal(err)
8786
}

‎plugins/destination/azblob/client/spec.go‎

Lines changed: 0 additions & 71 deletions
This file was deleted.

‎plugins/destination/azblob/client/spec/gen/main.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"path"
7+
"runtime"
8+
9+
"github.com/cloudquery/cloudquery/plugins/destination/azblob/client/spec"
10+
"github.com/cloudquery/codegen/jsonschema"
11+
"github.com/cloudquery/filetypes/v4"
12+
)
13+
14+
func main() {
15+
fmt.Println("Generating JSON schema for plugin spec")
16+
jsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json"),
17+
append(filetypes.FileSpec{}.JSONSchemaOptions(),
18+
jsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/azblob/client/spec", path.Join(currDir(), "..")),
19+
jsonschema.WithAddGoComments("github.com/cloudquery/filetypes/v4", path.Join(currDir(), "..", "..", "..", "vendor", "github.com/cloudquery/filetypes/v4")),
20+
)...,
21+
)
22+
}
23+
24+
func currDir() string {
25+
_, filename, _, ok := runtime.Caller(0)
26+
if !ok {
27+
log.Fatal("Failed to get caller information")
28+
}
29+
return path.Dir(filename)
30+
}

‎plugins/destination/azblob/client/spec/schema.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package spec
2+
3+
import (
4+
_ "embed"
5+
6+
"github.com/invopop/jsonschema"
7+
orderedmap "github.com/wk8/go-ordered-map/v2"
8+
)
9+
10+
func (s Spec) JSONSchemaExtend(sc *jsonschema.Schema) {
11+
s.FileSpec.JSONSchemaExtend(sc) // need to call manually
12+
13+
batchTimeout := sc.Properties.Value("batch_timeout").OneOf[0] // 0 - val, 1 - null
14+
batchTimeout.Default = "30s"
15+
16+
// no_rotate:true -> only nulls for batch options
17+
noRotateNoBatch := &jsonschema.Schema{
18+
Title: "Disallow batching when using no_rotate",
19+
If: &jsonschema.Schema{
20+
Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] {
21+
noRotate := *sc.Properties.Value("no_rotate")
22+
noRotate.Default = nil
23+
noRotate.Const = true
24+
noRotate.Description = ""
25+
properties := orderedmap.New[string, *jsonschema.Schema]()
26+
properties.Set("no_rotate", &noRotate)
27+
return properties
28+
}(),
29+
Required: []string{"no_rotate"},
30+
},
31+
Then: &jsonschema.Schema{
32+
Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] {
33+
// we make the non-zero requirement, so we want to allow only null here
34+
null := &jsonschema.Schema{Type: "null"}
35+
properties := orderedmap.New[string, *jsonschema.Schema]()
36+
properties.Set("batch_size", null)
37+
properties.Set("batch_size_bytes", null)
38+
properties.Set("batch_timeout", null)
39+
return properties
40+
}(),
41+
},
42+
}
43+
44+
sc.AllOf = append(sc.AllOf, noRotateNoBatch)
45+
}
46+
47+
//go:embed schema.json
48+
var JSONSchema string

0 commit comments

Comments
 (0)