cloudquery · kodiakhq · Feb 1, 2024 · Feb 1, 2024 · Feb 1, 2024 · Feb 1, 2024
@@ -67,6 +67,12 @@ jobs:
           args: "--config ../../.golangci.yml"
           skip-pkg-cache: true
           skip-build-cache: true
+      - name: gen
+        if: github.event_name == 'pull_request'
+        run: make gen
+      - name: Fail if generation updated files
+        if: github.event_name == 'pull_request'
+        run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
       - name: Build
         run: go build .
       - name: Test

@@ -7,3 +7,13 @@ test:
 .PHONY: lint
 lint:
 	golangci-lint run --config ../../.golangci.yml
+
+.PHONY: gen-spec-schema
+gen-spec-schema:
+	# required for loading comments from filetypes
+	go mod vendor
+	go run client/spec/gen/main.go
+
+# All gen targets
+.PHONY: gen
+gen: gen-spec-schema
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/Shopify/sarama"
+	"github.com/cloudquery/cloudquery/plugins/destination/kafka/client/spec"
 	"github.com/cloudquery/filetypes/v4"
 	"github.com/cloudquery/plugin-sdk/v4/plugin"
 	"github.com/rs/zerolog"
@@ -20,20 +21,20 @@ type Client struct {
 	producer sarama.SyncProducer
 
 	logger zerolog.Logger
-	spec   *Spec
+	spec   *spec.Spec
 
 	*filetypes.Client
 }
 
-func New(_ context.Context, logger zerolog.Logger, spec []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
+func New(_ context.Context, logger zerolog.Logger, s []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
 	c := &Client{
 		logger: logger.With().Str("module", "dest-kafka").Logger(),
 	}
 	if opts.NoConnection {
 		return c, nil
 	}
 
-	if err := json.Unmarshal(spec, &c.spec); err != nil {
+	if err := json.Unmarshal(s, &c.spec); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal spec: %w", err)
 	}
 	if err := c.spec.Validate(); err != nil {
@@ -55,10 +56,10 @@ func New(_ context.Context, logger zerolog.Logger, spec []byte, opts plugin.NewC
 	c.conf.Metadata.Full = true
 	c.conf.ClientID = c.spec.ClientID
 
-	if c.spec.SaslUsername != "" {
+	if c.spec.SASLUsername != "" {
 		c.conf.Net.SASL.Enable = true
-		c.conf.Net.SASL.User = c.spec.SaslUsername
-		c.conf.Net.SASL.Password = c.spec.SaslPassword
+		c.conf.Net.SASL.User = c.spec.SASLUsername
+		c.conf.Net.SASL.Password = c.spec.SASLPassword
 		c.conf.Net.TLS.Enable = true
 		c.conf.Net.TLS.Config = &tls.Config{InsecureSkipVerify: true}
 		c.conf.Net.SASL.Handshake = true
@@ -70,7 +71,7 @@ func New(_ context.Context, logger zerolog.Logger, spec []byte, opts plugin.NewC
 		return nil, err
 	}
 
-	filetypesClient, err := filetypes.NewClient(c.spec.FileSpec)
+	filetypesClient, err := filetypes.NewClient(&c.spec.FileSpec)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create filetypes client: %w", err)
 	}

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/cloudquery/cloudquery/plugins/destination/kafka/client/spec"
 	"github.com/cloudquery/filetypes/v4"
 	"github.com/cloudquery/plugin-sdk/v4/plugin"
 )
@@ -26,12 +27,12 @@ func getenv(key, fallback string) string {
 func TestPlugin(t *testing.T) {
 	ctx := context.Background()
 	p := plugin.NewPlugin("kafka", "development", New)
-	b, err := json.Marshal(&Spec{
+	b, err := json.Marshal(&spec.Spec{
 		Brokers:      strings.Split(getenv("CQ_DEST_KAFKA_CONNECTION_STRING", defaultConnectionString), ","),
-		SaslUsername: getenv("CQ_DEST_KAFKA_SASL_USERNAME", ""),
-		SaslPassword: getenv("CQ_DEST_KAFKA_SASL_PASSWORD", ""),
+		SASLUsername: getenv("CQ_DEST_KAFKA_SASL_USERNAME", ""),
+		SASLPassword: getenv("CQ_DEST_KAFKA_SASL_PASSWORD", ""),
 		Verbose:      true,
-		FileSpec:     &filetypes.FileSpec{Format: filetypes.FormatTypeJSON},
+		FileSpec:     filetypes.FileSpec{Format: filetypes.FormatTypeJSON},
 	})
 	if err != nil {
 		t.Fatal(err)

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"path"
+	"runtime"
+
+	"github.com/cloudquery/cloudquery/plugins/destination/kafka/client/spec"
+	cqjsonschema "github.com/cloudquery/codegen/jsonschema"
+	"github.com/cloudquery/filetypes/v4"
+	"github.com/invopop/jsonschema"
+)
+
+func main() {
+	fmt.Println("Generating JSON schema for plugin spec")
+	cqjsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json"),
+		append(filetypes.FileSpec{}.JSONSchemaOptions(),
+			cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/kafka/client/spec", path.Join(currDir(), "..")),
+			cqjsonschema.WithAddGoComments("github.com/cloudquery/filetypes/v4", path.Join(currDir(), "..", "..", "..", "vendor", "github.com/cloudquery/filetypes/v4")),
+			func(r *jsonschema.Reflector) {
+				// not required for this plugin
+				r.NullableFromType = false
+			},
+		)...,
+	)
+}
+
+func currDir() string {
+	_, filename, _, ok := runtime.Caller(0)
+	if !ok {
+		log.Fatal("Failed to get caller information")
+	}
+	return path.Dir(filename)
+}
@@ -0,0 +1,46 @@
+package spec
+
+import (
+	_ "embed"
+
+	"github.com/invopop/jsonschema"
+	orderedmap "github.com/wk8/go-ordered-map/v2"
+)
+
+func (s Spec) JSONSchemaExtend(sc *jsonschema.Schema) {
+	s.FileSpec.JSONSchemaExtend(sc) // need to call manually
+
+	strValueIsSet := func(property string) *jsonschema.Schema {
+		return &jsonschema.Schema{
+			Title: "`" + property + "` value is set",
+			Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] {
+				p := *sc.Properties.Value(property)
+				p.Default = nil
+				p.Description = ""
+				p.MinLength = &([]uint64{1}[0])
+				properties := orderedmap.New[string, *jsonschema.Schema]()
+				properties.Set(property, &p)
+				return properties
+			}(),
+			Required: []string{property},
+		}
+	}
+	usernamePresent := strValueIsSet("sasl_username")
+	passwordPresent := strValueIsSet("sasl_password")
+
+	sc.AllOf = append(sc.AllOf,
+		&jsonschema.Schema{
+			Title: "Require `sasl_password` when `sasl_username` is set",
+			If:    usernamePresent,
+			Then:  passwordPresent,
+		},
+		&jsonschema.Schema{
+			Title: "Require `sasl_username` when `sasl_password` is set",
+			If:    passwordPresent,
+			Then:  usernamePresent,
+		},
+	)
+}
+
+//go:embed schema.json
+var JSONSchema string