diff --git a/plugins/source/okta/Makefile b/plugins/source/okta/Makefile index e305c36ab8133b..f0b35e06335727 100644 --- a/plugins/source/okta/Makefile +++ b/plugins/source/okta/Makefile @@ -22,6 +22,10 @@ gen-docs: build cloudquery tables --format markdown --output-dir docs test/config.yml mv docs/$(shell basename $(CURDIR)) docs/tables +.PHONY: gen-spec-schema +gen-spec-schema: + go run client/spec/gen/main.go + # All gen targets .PHONY: gen -gen: gen-docs +gen: gen-spec-schema gen-docs diff --git a/plugins/source/okta/client/schema.json b/plugins/source/okta/client/schema.json new file mode 100644 index 00000000000000..52b89638a137a7 --- /dev/null +++ b/plugins/source/okta/client/schema.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/cloudquery/plugins/source/okta/client/spec", + "$ref": "#/$defs/Spec", + "$defs": { + "Duration": { + "type": "string", + "pattern": "^[-+]?([0-9]*(\\.[0-9]*)?[a-z]+)+$", + "title": "CloudQuery configtype.Duration" + }, + "RateLimit": { + "properties": { + "max_backoff": { + "$ref": "#/$defs/Duration", + "description": "Max backoff interval to be used.\nIf the value specified is less than the default one, the default one is used.", + "default": "30s" + }, + "max_retries": { + "type": "integer", + "minimum": 2, + "description": "Max retries to be performed.", + "default": 2 + } + }, + "additionalProperties": false, + "type": "object" + }, + "Spec": { + "properties": { + "token": { + "type": "string", + "minLength": 1, + "description": "Token for Okta API access." + }, + "domain": { + "type": "string", + "pattern": "^https?://[^\n\u003c\u003e]+\\.okta\\.com$", + "description": "Specify the Okta domain you are fetching from.\n[Visit this link](https://developer.okta.com/docs/guides/find-your-domain/findorg/) to find your Okta domain." + }, + "rate_limit": { + "oneOf": [ + { + "$ref": "#/$defs/RateLimit" + }, + { + "type": "null" + } + ] + }, + "debug": { + "type": "boolean", + "description": "Enables debug logs within the Okta SDK.", + "default": false + }, + "concurrency": { + "type": "integer", + "minimum": 1, + "description": "Number of concurrent requests to be made to Okta API.", + "default": 10000 + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "token", + "domain" + ] + } + } +} diff --git a/plugins/source/okta/client/spec.go b/plugins/source/okta/client/spec.go index 3d97a48bcf76aa..91e13ebf35d674 100644 --- a/plugins/source/okta/client/spec.go +++ b/plugins/source/okta/client/spec.go @@ -1,26 +1,44 @@ package client import ( + _ "embed" "errors" "time" + "github.com/cloudquery/plugin-sdk/v4/configtype" + "github.com/invopop/jsonschema" "github.com/rs/zerolog" ) type ( Spec struct { - Token string `json:"token,omitempty"` - Domain string `json:"domain,omitempty"` - RateLimit *RateLimit `json:"rate_limit,omitempty"` - Debug bool `json:"debug,omitempty"` - Concurrency int `json:"concurrency,omitempty"` + // Token for Okta API access. + Token string `json:"token" jsonschema:"required,minLength=1"` + + // Specify the Okta domain you are fetching from. + // [Visit this link](https://developer.okta.com/docs/guides/find-your-domain/findorg/) to find your Okta domain. + Domain string `json:"domain" jsonschema:"required,pattern=^https?://[^\n<>]+\\.okta\\.com$"` + RateLimit *RateLimit `json:"rate_limit"` + + // Enables debug logs within the Okta SDK. + Debug bool `json:"debug,omitempty" jsonschema:"default=false"` + + // Number of concurrent requests to be made to Okta API. + Concurrency int `json:"concurrency" jsonschema:"minimum=1,default=10000"` } RateLimit struct { - MaxBackoff time.Duration `json:"max_backoff,omitempty"` - MaxRetries int32 `json:"max_retries,omitempty"` + // Max backoff interval to be used. + // If the value specified is less than the default one, the default one is used. + MaxBackoff configtype.Duration `json:"max_backoff,omitempty"` + + // Max retries to be performed. + MaxRetries int32 `json:"max_retries,omitempty" jsonschema:"minimum=2,default=2"` } ) +//go:embed schema.json +var JSONSchema string + func (s *Spec) SetDefaults(logger *zerolog.Logger) { const ( minRetries = int32(2) @@ -35,8 +53,8 @@ func (s *Spec) SetDefaults(logger *zerolog.Logger) { s.RateLimit.MaxRetries = minRetries } - if s.RateLimit.MaxBackoff < minBackOff { - s.RateLimit.MaxBackoff = minBackOff + if s.RateLimit.MaxBackoff.Duration() < minBackOff { + s.RateLimit.MaxBackoff = configtype.NewDuration(minBackOff) } if s.Concurrency < 1 { @@ -57,3 +75,7 @@ func (s Spec) Validate() error { return nil } + +func (RateLimit) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Properties.Value("max_backoff").Default = "30s" +} diff --git a/plugins/source/okta/client/spec/gen/main.go b/plugins/source/okta/client/spec/gen/main.go new file mode 100644 index 00000000000000..dda208b24a06c9 --- /dev/null +++ b/plugins/source/okta/client/spec/gen/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "log" + "path" + "runtime" + + "github.com/cloudquery/cloudquery/plugins/source/okta/client" + cqjsonschema "github.com/cloudquery/codegen/jsonschema" +) + +func main() { + fmt.Println("Generating JSON schema for plugin spec") + cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"), + cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/source/okta/client", path.Join(currDir(), "../..")), + ) +} + +func currDir() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Failed to get caller information") + } + return path.Dir(filename) +} diff --git a/plugins/source/okta/client/spec_test.go b/plugins/source/okta/client/spec_test.go new file mode 100644 index 00000000000000..295c4116052549 --- /dev/null +++ b/plugins/source/okta/client/spec_test.go @@ -0,0 +1,109 @@ +package client + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/stretchr/testify/require" +) + +func TestJSONSchema(t *testing.T) { + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty spec", + Spec: `{}`, + Err: true, + }, + { + Name: "spec with token", + Spec: `{"token": "tok"}`, + Err: true, + }, + { + Name: "spec with domain", + Spec: `{"domain": "https://domain.okta.com"}`, + Err: true, + }, + { + Name: "spec with token and domain", + Spec: `{"token": "tok", "domain": "https://domain.okta.com"}`, + }, + { + Name: "spec with token and invalid domain", + Spec: `{"token": "tok", "domain": "https://.okta.com"}`, + Err: true, + }, + { + Name: "spec with token and domain and empty rate limit", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {}}`, + }, + { + Name: "spec with token and domain and null rate limit", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": null}`, + }, + { + Name: "spec with token and domain and valid rate limit", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {"max_backoff": "60s"}}`, + }, + { + Name: "spec with bool concurrency", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":false}`, + Err: true, + }, + { + Name: "spec with null concurrency", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":null}`, + Err: true, + }, + { + Name: "spec with string concurrency", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":"str"}`, + Err: true, + }, + { + Name: "spec with proper concurrency", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency": 7}`, + }, + { + Name: "spec with array concurrency", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":["abc"]}`, + Err: true, + }, + { + Name: "spec with unknown field", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "unknown": "test"}`, + Err: true, + }, + }) +} + +func TestRateLimitJSONSchema(t *testing.T) { + data, err := jsonschema.Generate(RateLimit{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(data), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "valid max_backoff", + Spec: `{"max_backoff": "60s"}`, + }, + { + Name: "invalid max_backoff", + Spec: `{"max_backoff": true}`, + Err: true, + }, + { + Name: "zero max_backoff", + Spec: `{"max_backoff": 0}`, + Err: true, + }, + { + Name: "unknown field", + Spec: `{"unknown": "test"}`, + Err: true, + }, + }) +} diff --git a/plugins/source/okta/docs/overview.md b/plugins/source/okta/docs/overview.md index 011751d3dddfa4..6e619d3fc12546 100644 --- a/plugins/source/okta/docs/overview.md +++ b/plugins/source/okta/docs/overview.md @@ -50,12 +50,12 @@ Make sure you use [environment variable expansion](/docs/advanced-topics/environ ### Rate limit spec -- `max_backoff` (`duration`) (optional) (default: `5s`) +- `max_backoff` (`duration`) (optional) (default: `30s`) Max backoff interval to be used. If the value specified is less than the default one, the default one is used. -- `max_retries` (`integer`) (optional) (default: `3`) +- `max_retries` (`integer`) (optional) (default: `2`) Max retries to be performed. If the value specified is less than the default one, the default one is used. diff --git a/plugins/source/okta/go.mod b/plugins/source/okta/go.mod index 064cba10f7e4bf..788f7374e8c139 100644 --- a/plugins/source/okta/go.mod +++ b/plugins/source/okta/go.mod @@ -4,8 +4,10 @@ go 1.21.4 require ( github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1 + github.com/cloudquery/codegen v0.3.12 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/gorilla/mux v1.8.0 + github.com/invopop/jsonschema v0.12.0 github.com/okta/okta-sdk-golang/v3 v3.0.2 github.com/rs/zerolog v1.31.0 github.com/thoas/go-funk v0.9.3 @@ -55,7 +57,6 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/jsonschema v0.12.0 // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -129,3 +130,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// github.com/cloudquery/jsonschema @ cqmain +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb diff --git a/plugins/source/okta/go.sum b/plugins/source/okta/go.sum index edd8daf2c68a82..07446f1736c207 100644 --- a/plugins/source/okta/go.sum +++ b/plugins/source/okta/go.sum @@ -88,6 +88,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudquery/cloudquery-api-go v1.7.2 h1:jpQfeZUxekbV7ASN5ONpGIkrtKIZvC/Y8fOj+tQxLm4= github.com/cloudquery/cloudquery-api-go v1.7.2/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA= +github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE= +github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8= +github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb h1:/l8fbvLOCNlgkHp8VUKTTL+Tk9gs5y/K3Yx/bRfReNk= +github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= github.com/cloudquery/plugin-pb-go v1.16.7 h1:wLx5TFvS6gAvD1dcBZdv5YSskcNCnNpF1JNituka5jM= github.com/cloudquery/plugin-pb-go v1.16.7/go.mod h1:Sd08P8HIjwi3gmfoE0X21qo6HL1NiVdNl/00JrK+DkM= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= @@ -228,8 +232,6 @@ github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/C github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= -github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go= github.com/iris-contrib/httpexpect/v2 v2.15.2/go.mod h1:JLDgIqnFy5loDSUv1OA2j0mb6p/rDhiCqigP22Uq9xE= github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= diff --git a/plugins/source/okta/resources/plugin/client.go b/plugins/source/okta/resources/plugin/client.go index 8df4ca61eb7033..f6d291f9f9c516 100644 --- a/plugins/source/okta/resources/plugin/client.go +++ b/plugins/source/okta/resources/plugin/client.go @@ -81,7 +81,7 @@ func Configure(_ context.Context, logger zerolog.Logger, specBytes []byte, opts okta.WithOrgUrl(config.Domain), okta.WithToken(config.Token), okta.WithCache(true), - okta.WithRateLimitMaxBackOff(int64(config.RateLimit.MaxBackoff/time.Second)), // this param takes int64 of seconds + okta.WithRateLimitMaxBackOff(int64(config.RateLimit.MaxBackoff.Duration()/time.Second)), // this param takes int64 of seconds okta.WithRateLimitMaxRetries(config.RateLimit.MaxRetries), ) cf.Debug = config.Debug diff --git a/plugins/source/okta/resources/plugin/plugin.go b/plugins/source/okta/resources/plugin/plugin.go index 82af0be4829b24..d011788c1547b9 100644 --- a/plugins/source/okta/resources/plugin/plugin.go +++ b/plugins/source/okta/resources/plugin/plugin.go @@ -1,6 +1,7 @@ package plugin import ( + "github.com/cloudquery/cloudquery/plugins/source/okta/client" "github.com/cloudquery/plugin-sdk/v4/plugin" ) @@ -18,5 +19,6 @@ func Plugin() *plugin.Plugin { Configure, plugin.WithKind(Kind), plugin.WithTeam(Team), + plugin.WithJSONSchema(client.JSONSchema), ) }