cloudquery · kodiakhq · Feb 1, 2024 · Feb 1, 2024 · Feb 1, 2024 · Feb 1, 2024
@@ -42,6 +42,12 @@ jobs:
           args: "--config ../../.golangci.yml"
           skip-pkg-cache: true
           skip-build-cache: true
+      - name: gen
+        if: github.event_name == 'pull_request'
+        run: make gen
+      - name: Fail if generation updated files
+        if: github.event_name == 'pull_request'
+        run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
       - name: Build
         run: go build .
       - name: Test azblob plugin

@@ -5,3 +5,13 @@ test:
 .PHONY: lint
 lint:
 	golangci-lint run --config ../../.golangci.yml
+
+.PHONY: gen-spec-schema
+gen-spec-schema:
+	# required for loading comments from filetypes
+	go mod vendor
+	go run client/spec/gen/main.go
+
+# All gen targets
+.PHONY: gen
+gen: gen-spec-schema
@@ -8,6 +8,7 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/cloudquery/cloudquery/plugins/destination/azblob/client/spec"
 	"github.com/cloudquery/filetypes/v4"
 	"github.com/cloudquery/plugin-sdk/v4/plugin"
 	"github.com/cloudquery/plugin-sdk/v4/writers/streamingbatchwriter"
@@ -21,30 +22,30 @@ type Client struct {
 	streamingbatchwriter.UnimplementedDeleteRecords
 
 	logger zerolog.Logger
-	spec   *Spec
+	spec   *spec.Spec
 	*filetypes.Client
 	writer *streamingbatchwriter.StreamingBatchWriter
 
 	storageClient *azblob.Client
 }
 
-func New(ctx context.Context, logger zerolog.Logger, spec []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
+func New(ctx context.Context, logger zerolog.Logger, s []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
 	c := &Client{
 		logger: logger.With().Str("module", "azb").Logger(),
 	}
 	if opts.NoConnection {
 		return c, nil
 	}
 
-	if err := json.Unmarshal(spec, &c.spec); err != nil {
+	if err := json.Unmarshal(s, &c.spec); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal azblob spec: %w", err)
 	}
 	if err := c.spec.Validate(); err != nil {
 		return nil, err
 	}
 	c.spec.SetDefaults()
 
-	filetypesClient, err := filetypes.NewClient(c.spec.FileSpec)
+	filetypesClient, err := filetypes.NewClient(&c.spec.FileSpec)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create filetypes client: %w", err)
 	}

@@ -10,6 +10,7 @@ import (
 	"github.com/apache/arrow/go/v15/arrow"
 	"github.com/apache/arrow/go/v15/arrow/array"
 	"github.com/apache/arrow/go/v15/arrow/memory"
+	"github.com/cloudquery/cloudquery/plugins/destination/azblob/client/spec"
 	"github.com/cloudquery/filetypes/v4"
 	"github.com/cloudquery/plugin-sdk/v4/message"
 	"github.com/cloudquery/plugin-sdk/v4/plugin"
@@ -19,8 +20,8 @@ import (
 )
 
 const (
-	storage_account = "cqdestinationazblob"
-	container       = "test"
+	storageAccount = "cqdestinationazblob"
+	container      = "test"
 )
 
 func TestPlugin(t *testing.T) {
@@ -29,30 +30,28 @@ func TestPlugin(t *testing.T) {
 		filetypes.FormatTypeJSON,
 		filetypes.FormatTypeParquet,
 	} {
-		spec := Spec{
-			StorageAccount: storage_account,
+		s := spec.Spec{
+			StorageAccount: storageAccount,
 			Container:      container,
 			Path:           t.TempDir(),
 			NoRotate:       true,
-			FileSpec: &filetypes.FileSpec{
-				Format: ft,
-			},
+			FileSpec:       filetypes.FileSpec{Format: ft},
 		}
 
 		t.Run("generic/"+string(ft), func(t *testing.T) {
-			testPlugin(t, &spec)
+			testPlugin(t, &s)
 		})
 
 		t.Run("write/"+string(ft), func(t *testing.T) {
-			testPluginCustom(t, &spec)
+			testPluginCustom(t, &s)
 		})
 	}
 }
 
-func testPlugin(t *testing.T, spec *Spec) {
+func testPlugin(t *testing.T, s *spec.Spec) {
 	ctx := context.Background()
 	p := plugin.NewPlugin("azblob", "development", New)
-	b, err := json.Marshal(spec)
+	b, err := json.Marshal(s)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -71,7 +70,7 @@ func testPlugin(t *testing.T, spec *Spec) {
 	)
 }
 
-func testPluginCustom(t *testing.T, spec *Spec) {
+func testPluginCustom(t *testing.T, s *spec.Spec) {
 	ctx := context.Background()
 
 	var client plugin.Client
@@ -81,7 +80,7 @@ func testPluginCustom(t *testing.T, spec *Spec) {
 		client, err = New(ctx, logger, spec, opts)
 		return client, err
 	})
-	b, err := json.Marshal(spec)
+	b, err := json.Marshal(s)
 	if err != nil {
 		t.Fatal(err)
 	}

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"path"
+	"runtime"
+
+	"github.com/cloudquery/cloudquery/plugins/destination/azblob/client/spec"
+	"github.com/cloudquery/codegen/jsonschema"
+	"github.com/cloudquery/filetypes/v4"
+)
+
+func main() {
+	fmt.Println("Generating JSON schema for plugin spec")
+	jsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json"),
+		append(filetypes.FileSpec{}.JSONSchemaOptions(),
+			jsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/azblob/client/spec", path.Join(currDir(), "..")),
+			jsonschema.WithAddGoComments("github.com/cloudquery/filetypes/v4", path.Join(currDir(), "..", "..", "..", "vendor", "github.com/cloudquery/filetypes/v4")),
+		)...,
+	)
+}
+
+func currDir() string {
+	_, filename, _, ok := runtime.Caller(0)
+	if !ok {
+		log.Fatal("Failed to get caller information")
+	}
+	return path.Dir(filename)
+}
@@ -0,0 +1,48 @@
+package spec
+
+import (
+	_ "embed"
+
+	"github.com/invopop/jsonschema"
+	orderedmap "github.com/wk8/go-ordered-map/v2"
+)
+
+func (s Spec) JSONSchemaExtend(sc *jsonschema.Schema) {
+	s.FileSpec.JSONSchemaExtend(sc) // need to call manually
+
+	batchTimeout := sc.Properties.Value("batch_timeout").OneOf[0] // 0 - val, 1 - null
+	batchTimeout.Default = "30s"
+
+	// no_rotate:true -> only nulls for batch options
+	noRotateNoBatch := &jsonschema.Schema{
+		Title: "Disallow batching when using no_rotate",
+		If: &jsonschema.Schema{
+			Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] {
+				noRotate := *sc.Properties.Value("no_rotate")
+				noRotate.Default = nil
+				noRotate.Const = true
+				noRotate.Description = ""
+				properties := orderedmap.New[string, *jsonschema.Schema]()
+				properties.Set("no_rotate", &noRotate)
+				return properties
+			}(),
+			Required: []string{"no_rotate"},
+		},
+		Then: &jsonschema.Schema{
+			Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] {
+				// we make the non-zero requirement, so we want to allow only null here
+				null := &jsonschema.Schema{Type: "null"}
+				properties := orderedmap.New[string, *jsonschema.Schema]()
+				properties.Set("batch_size", null)
+				properties.Set("batch_size_bytes", null)
+				properties.Set("batch_timeout", null)
+				return properties
+			}(),
+		},
+	}
+
+	sc.AllOf = append(sc.AllOf, noRotateNoBatch)
+}
+
+//go:embed schema.json
+var JSONSchema string