From 653be8ceb250e5a661e6ea103d1d04d789d73b1d Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 2 Feb 2024 16:10:20 +0000 Subject: [PATCH 1/6] feat: Add JSON Schema to Okta source plugin --- plugins/source/okta/Makefile | 6 +- plugins/source/okta/client/schema.json | 59 ++++++++++++++++++ plugins/source/okta/client/spec.go | 34 +++++++---- plugins/source/okta/client/spec/gen/main.go | 26 ++++++++ plugins/source/okta/client/spec_test.go | 60 +++++++++++++++++++ plugins/source/okta/docs/overview.md | 4 +- plugins/source/okta/go.mod | 4 ++ plugins/source/okta/go.sum | 6 +- .../source/okta/resources/plugin/plugin.go | 2 + 9 files changed, 185 insertions(+), 16 deletions(-) create mode 100644 plugins/source/okta/client/schema.json create mode 100644 plugins/source/okta/client/spec/gen/main.go create mode 100644 plugins/source/okta/client/spec_test.go diff --git a/plugins/source/okta/Makefile b/plugins/source/okta/Makefile index e305c36ab8133b..f0b35e06335727 100644 --- a/plugins/source/okta/Makefile +++ b/plugins/source/okta/Makefile @@ -22,6 +22,10 @@ gen-docs: build cloudquery tables --format markdown --output-dir docs test/config.yml mv docs/$(shell basename $(CURDIR)) docs/tables +.PHONY: gen-spec-schema +gen-spec-schema: + go run client/spec/gen/main.go + # All gen targets .PHONY: gen -gen: gen-docs +gen: gen-spec-schema gen-docs diff --git a/plugins/source/okta/client/schema.json b/plugins/source/okta/client/schema.json new file mode 100644 index 00000000000000..6469e9eb678582 --- /dev/null +++ b/plugins/source/okta/client/schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/cloudquery/plugins/source/okta/client/spec", + "$ref": "#/$defs/Spec", + "$defs": { + "RateLimit": { + "properties": { + "max_backoff": { + "type": "integer", + "minimum": 30, + "description": "Max backoff interval to be used.", + "default": 30 + }, + "max_retries": { + "type": "integer", + "minimum": 2, + "description": "Max retries to be performed.", + "default": 2 + } + }, + "additionalProperties": false, + "type": "object" + }, + "Spec": { + "properties": { + "token": { + "type": "string", + "minLength": 1, + "description": "Token for Okta API access.\nYou can set this with an `OKTA_API_TOKEN` environment variable." + }, + "domain": { + "type": "string", + "pattern": "^https?://[^\n\u003c\u003e]+\\.okta\\.com$", + "description": "Specify the Okta domain you are fetching from.\n[Visit this link](https://developer.okta.com/docs/guides/find-your-domain/findorg/) to find your Okta domain." + }, + "rate_limit": { + "$ref": "#/$defs/RateLimit" + }, + "debug": { + "type": "boolean", + "description": "Enables debug logs within the Okta SDK.", + "default": false + }, + "concurrency": { + "type": "integer", + "minimum": 1, + "description": "Number of concurrent requests to be made to Okta API.", + "default": 10000 + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "token", + "domain" + ] + } + } +} diff --git a/plugins/source/okta/client/spec.go b/plugins/source/okta/client/spec.go index 1c31391da692cc..d9855ef085de6c 100644 --- a/plugins/source/okta/client/spec.go +++ b/plugins/source/okta/client/spec.go @@ -1,6 +1,7 @@ package client import ( + _ "embed" "fmt" "os" "time" @@ -10,15 +11,27 @@ import ( type ( Spec struct { - Token string `json:"token,omitempty"` - Domain string `json:"domain,omitempty"` - RateLimit *RateLimit `json:"rate_limit,omitempty"` - Debug bool `json:"debug,omitempty"` - Concurrency int `json:"concurrency,omitempty"` + // Token for Okta API access. + // You can set this with an `OKTA_API_TOKEN` environment variable. + Token string `json:"token" jsonschema:"required,minLength=1"` + + // Specify the Okta domain you are fetching from. + // [Visit this link](https://developer.okta.com/docs/guides/find-your-domain/findorg/) to find your Okta domain. + Domain string `json:"domain" jsonschema:"required,pattern=^https?://[^\n<>]+\\.okta\\.com$"` + RateLimit RateLimit `json:"rate_limit"` + + // Enables debug logs within the Okta SDK. + Debug bool `json:"debug,omitempty" jsonschema:"default=false"` + + // Number of concurrent requests to be made to Okta API. + Concurrency int `json:"concurrency" jsonschema:"minimum=1,default=10000"` } RateLimit struct { - MaxBackoff time.Duration `json:"max_backoff,omitempty"` - MaxRetries int32 `json:"max_retries,omitempty"` + // Max backoff interval to be used. + MaxBackoff time.Duration `json:"max_backoff,omitempty" jsonschema:"minimum=30,default=30"` + + // Max retries to be performed. + MaxRetries int32 `json:"max_retries,omitempty" jsonschema:"minimum=2,default=2"` } ) @@ -26,16 +39,15 @@ const ( OktaAPIToken = "OKTA_API_TOKEN" ) +//go:embed schema.json +var JSONSchema string + func (s *Spec) SetDefaults(logger *zerolog.Logger) { const ( minRetries = int32(2) minBackOff = 30 * time.Second ) - if s.RateLimit == nil { - s.RateLimit = new(RateLimit) - } - if s.RateLimit.MaxRetries < minRetries { s.RateLimit.MaxRetries = minRetries } diff --git a/plugins/source/okta/client/spec/gen/main.go b/plugins/source/okta/client/spec/gen/main.go new file mode 100644 index 00000000000000..dda208b24a06c9 --- /dev/null +++ b/plugins/source/okta/client/spec/gen/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "log" + "path" + "runtime" + + "github.com/cloudquery/cloudquery/plugins/source/okta/client" + cqjsonschema "github.com/cloudquery/codegen/jsonschema" +) + +func main() { + fmt.Println("Generating JSON schema for plugin spec") + cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"), + cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/source/okta/client", path.Join(currDir(), "../..")), + ) +} + +func currDir() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Failed to get caller information") + } + return path.Dir(filename) +} diff --git a/plugins/source/okta/client/spec_test.go b/plugins/source/okta/client/spec_test.go new file mode 100644 index 00000000000000..2e4ababf29cf9e --- /dev/null +++ b/plugins/source/okta/client/spec_test.go @@ -0,0 +1,60 @@ +package client + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" +) + +func TestJSONSchema(t *testing.T) { + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty spec", + Spec: `{}`, + Err: true, + }, + { + Name: "spec with token", + Spec: `{"token": "tok"}`, + Err: true, + }, + { + Name: "spec with token and domain", + Spec: `{"token": "tok", "domain": "https://domain.okta.com"}`, + }, + { + Name: "spec with token and invalid domain", + Spec: `{"token": "tok", "domain": "https://.okta.com"}`, + Err: true, + }, + { + Name: "spec with bool concurrency", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":false}`, + Err: true, + }, + { + Name: "spec with null concurrency", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":null}`, + Err: true, + }, + { + Name: "spec with string concurrency", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":"str"}`, + Err: true, + }, + { + Name: "spec with proper concurrency", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency": 7}`, + }, + { + Name: "spec with array concurrency", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":["abc"]}`, + Err: true, + }, + { + Name: "spec with unknown field", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "unknown": "test"}`, + Err: true, + }, + }) +} diff --git a/plugins/source/okta/docs/overview.md b/plugins/source/okta/docs/overview.md index e6d7bb4e718eaf..bf0c9f8c2ff37f 100644 --- a/plugins/source/okta/docs/overview.md +++ b/plugins/source/okta/docs/overview.md @@ -55,12 +55,12 @@ Make sure you use [environment variable expansion](/docs/advanced-topics/environ ### Rate limit spec -- `max_backoff` (`duration`, optional. Default: `5s`) +- `max_backoff` (`duration`, optional. Default: `30s`) Max backoff interval to be used. If the value specified is less than the default one, the default one is used. -- `max_retries` (`int32`, optional. Default: `3`) +- `max_retries` (`int32`, optional. Default: `2`) Max retries to be performed. If the value specified is less than the default one, the default one is used. diff --git a/plugins/source/okta/go.mod b/plugins/source/okta/go.mod index 064cba10f7e4bf..14e337caa9d747 100644 --- a/plugins/source/okta/go.mod +++ b/plugins/source/okta/go.mod @@ -4,6 +4,7 @@ go 1.21.4 require ( github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1 + github.com/cloudquery/codegen v0.3.12 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/gorilla/mux v1.8.0 github.com/okta/okta-sdk-golang/v3 v3.0.2 @@ -129,3 +130,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// github.com/cloudquery/jsonschema @ cqmain +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb diff --git a/plugins/source/okta/go.sum b/plugins/source/okta/go.sum index edd8daf2c68a82..07446f1736c207 100644 --- a/plugins/source/okta/go.sum +++ b/plugins/source/okta/go.sum @@ -88,6 +88,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudquery/cloudquery-api-go v1.7.2 h1:jpQfeZUxekbV7ASN5ONpGIkrtKIZvC/Y8fOj+tQxLm4= github.com/cloudquery/cloudquery-api-go v1.7.2/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA= +github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE= +github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8= +github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb h1:/l8fbvLOCNlgkHp8VUKTTL+Tk9gs5y/K3Yx/bRfReNk= +github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= github.com/cloudquery/plugin-pb-go v1.16.7 h1:wLx5TFvS6gAvD1dcBZdv5YSskcNCnNpF1JNituka5jM= github.com/cloudquery/plugin-pb-go v1.16.7/go.mod h1:Sd08P8HIjwi3gmfoE0X21qo6HL1NiVdNl/00JrK+DkM= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= @@ -228,8 +232,6 @@ github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/C github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= -github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go= github.com/iris-contrib/httpexpect/v2 v2.15.2/go.mod h1:JLDgIqnFy5loDSUv1OA2j0mb6p/rDhiCqigP22Uq9xE= github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= diff --git a/plugins/source/okta/resources/plugin/plugin.go b/plugins/source/okta/resources/plugin/plugin.go index 82af0be4829b24..d011788c1547b9 100644 --- a/plugins/source/okta/resources/plugin/plugin.go +++ b/plugins/source/okta/resources/plugin/plugin.go @@ -1,6 +1,7 @@ package plugin import ( + "github.com/cloudquery/cloudquery/plugins/source/okta/client" "github.com/cloudquery/plugin-sdk/v4/plugin" ) @@ -18,5 +19,6 @@ func Plugin() *plugin.Plugin { Configure, plugin.WithKind(Kind), plugin.WithTeam(Team), + plugin.WithJSONSchema(client.JSONSchema), ) } From 60018cda9d5e45c78b61418ae88eaf8ac8a2ae89 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 2 Feb 2024 16:14:44 +0000 Subject: [PATCH 2/6] Test RateLimit --- plugins/source/okta/client/schema.json | 9 ++++++++- plugins/source/okta/client/spec.go | 8 ++++++-- plugins/source/okta/client/spec_test.go | 22 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/plugins/source/okta/client/schema.json b/plugins/source/okta/client/schema.json index 6469e9eb678582..21dc0c0f41b000 100644 --- a/plugins/source/okta/client/schema.json +++ b/plugins/source/okta/client/schema.json @@ -34,7 +34,14 @@ "description": "Specify the Okta domain you are fetching from.\n[Visit this link](https://developer.okta.com/docs/guides/find-your-domain/findorg/) to find your Okta domain." }, "rate_limit": { - "$ref": "#/$defs/RateLimit" + "oneOf": [ + { + "$ref": "#/$defs/RateLimit" + }, + { + "type": "null" + } + ] }, "debug": { "type": "boolean", diff --git a/plugins/source/okta/client/spec.go b/plugins/source/okta/client/spec.go index d9855ef085de6c..1c5138d458d69f 100644 --- a/plugins/source/okta/client/spec.go +++ b/plugins/source/okta/client/spec.go @@ -17,8 +17,8 @@ type ( // Specify the Okta domain you are fetching from. // [Visit this link](https://developer.okta.com/docs/guides/find-your-domain/findorg/) to find your Okta domain. - Domain string `json:"domain" jsonschema:"required,pattern=^https?://[^\n<>]+\\.okta\\.com$"` - RateLimit RateLimit `json:"rate_limit"` + Domain string `json:"domain" jsonschema:"required,pattern=^https?://[^\n<>]+\\.okta\\.com$"` + RateLimit *RateLimit `json:"rate_limit"` // Enables debug logs within the Okta SDK. Debug bool `json:"debug,omitempty" jsonschema:"default=false"` @@ -48,6 +48,10 @@ func (s *Spec) SetDefaults(logger *zerolog.Logger) { minBackOff = 30 * time.Second ) + if s.RateLimit == nil { + s.RateLimit = new(RateLimit) + } + if s.RateLimit.MaxRetries < minRetries { s.RateLimit.MaxRetries = minRetries } diff --git a/plugins/source/okta/client/spec_test.go b/plugins/source/okta/client/spec_test.go index 2e4ababf29cf9e..dc61344acfffd1 100644 --- a/plugins/source/okta/client/spec_test.go +++ b/plugins/source/okta/client/spec_test.go @@ -27,6 +27,28 @@ func TestJSONSchema(t *testing.T) { Spec: `{"token": "tok", "domain": "https://.okta.com"}`, Err: true, }, + { + Name: "spec with token and domain and empty rate limit", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {}}`, + }, + { + Name: "spec with token and domain and null rate limit", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": null}`, + }, + { + Name: "spec with token and domain and rate limit", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {"max_backoff": 60}}`, + }, + { + Name: "spec with token and domain and invalid rate limit", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {"max_backoff": true}}`, + Err: true, + }, + { + Name: "spec with token and domain and zero rate limit", + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {"max_backoff": 0}}`, + Err: true, + }, { Name: "spec with bool concurrency", Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":false}`, From c4e1543a9257d609e3d7f00d9f21519bc4d2585f Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 2 Feb 2024 20:52:48 +0000 Subject: [PATCH 3/6] CR --- plugins/source/okta/client/schema.json | 14 ++++++++------ plugins/source/okta/client/spec.go | 17 ++++++++++++----- plugins/source/okta/client/spec_test.go | 6 +++++- plugins/source/okta/docs/overview.md | 2 +- plugins/source/okta/go.mod | 2 +- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/plugins/source/okta/client/schema.json b/plugins/source/okta/client/schema.json index 21dc0c0f41b000..bb44f0e765a61b 100644 --- a/plugins/source/okta/client/schema.json +++ b/plugins/source/okta/client/schema.json @@ -3,13 +3,17 @@ "$id": "https://github.com/cloudquery/cloudquery/plugins/source/okta/client/spec", "$ref": "#/$defs/Spec", "$defs": { + "Duration": { + "type": "string", + "pattern": "^[-+]?([0-9]*(\\.[0-9]*)?[a-z]+)+$", + "title": "CloudQuery configtype.Duration" + }, "RateLimit": { "properties": { "max_backoff": { - "type": "integer", - "minimum": 30, - "description": "Max backoff interval to be used.", - "default": 30 + "$ref": "#/$defs/Duration", + "description": "Max backoff interval to be used.\nIf the value specified is less than the default one, the default one is used.", + "default": "30s" }, "max_retries": { "type": "integer", @@ -25,7 +29,6 @@ "properties": { "token": { "type": "string", - "minLength": 1, "description": "Token for Okta API access.\nYou can set this with an `OKTA_API_TOKEN` environment variable." }, "domain": { @@ -58,7 +61,6 @@ "additionalProperties": false, "type": "object", "required": [ - "token", "domain" ] } diff --git a/plugins/source/okta/client/spec.go b/plugins/source/okta/client/spec.go index 1c5138d458d69f..8746ff8cd67ad6 100644 --- a/plugins/source/okta/client/spec.go +++ b/plugins/source/okta/client/spec.go @@ -6,6 +6,8 @@ import ( "os" "time" + "github.com/cloudquery/plugin-sdk/v4/configtype" + "github.com/invopop/jsonschema" "github.com/rs/zerolog" ) @@ -13,7 +15,7 @@ type ( Spec struct { // Token for Okta API access. // You can set this with an `OKTA_API_TOKEN` environment variable. - Token string `json:"token" jsonschema:"required,minLength=1"` + Token string `json:"token"` // Specify the Okta domain you are fetching from. // [Visit this link](https://developer.okta.com/docs/guides/find-your-domain/findorg/) to find your Okta domain. @@ -27,8 +29,9 @@ type ( Concurrency int `json:"concurrency" jsonschema:"minimum=1,default=10000"` } RateLimit struct { - // Max backoff interval to be used. - MaxBackoff time.Duration `json:"max_backoff,omitempty" jsonschema:"minimum=30,default=30"` + // Max backoff interval to be used. + // If the value specified is less than the default one, the default one is used. + MaxBackoff configtype.Duration `json:"max_backoff,omitempty"` // Max retries to be performed. MaxRetries int32 `json:"max_retries,omitempty" jsonschema:"minimum=2,default=2"` @@ -56,8 +59,8 @@ func (s *Spec) SetDefaults(logger *zerolog.Logger) { s.RateLimit.MaxRetries = minRetries } - if s.RateLimit.MaxBackoff < minBackOff { - s.RateLimit.MaxBackoff = minBackOff + if s.RateLimit.MaxBackoff.Duration() < minBackOff { + s.RateLimit.MaxBackoff = configtype.NewDuration(minBackOff) } if len(s.Token) == 0 { @@ -83,3 +86,7 @@ func (s Spec) Validate() error { return nil } + +func (RateLimit) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Properties.Value("max_backoff").Default = "30s" +} diff --git a/plugins/source/okta/client/spec_test.go b/plugins/source/okta/client/spec_test.go index dc61344acfffd1..b8a54315c20634 100644 --- a/plugins/source/okta/client/spec_test.go +++ b/plugins/source/okta/client/spec_test.go @@ -18,6 +18,10 @@ func TestJSONSchema(t *testing.T) { Spec: `{"token": "tok"}`, Err: true, }, + { + Name: "spec with domain", + Spec: `{"domain": "https://domain.okta.com"}`, + }, { Name: "spec with token and domain", Spec: `{"token": "tok", "domain": "https://domain.okta.com"}`, @@ -37,7 +41,7 @@ func TestJSONSchema(t *testing.T) { }, { Name: "spec with token and domain and rate limit", - Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {"max_backoff": 60}}`, + Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {"max_backoff": "60s"}}`, }, { Name: "spec with token and domain and invalid rate limit", diff --git a/plugins/source/okta/docs/overview.md b/plugins/source/okta/docs/overview.md index bf0c9f8c2ff37f..366c94d21b5d7a 100644 --- a/plugins/source/okta/docs/overview.md +++ b/plugins/source/okta/docs/overview.md @@ -60,7 +60,7 @@ Make sure you use [environment variable expansion](/docs/advanced-topics/environ Max backoff interval to be used. If the value specified is less than the default one, the default one is used. -- `max_retries` (`int32`, optional. Default: `2`) +- `max_retries` (`integer`, optional. Default: `2`) Max retries to be performed. If the value specified is less than the default one, the default one is used. diff --git a/plugins/source/okta/go.mod b/plugins/source/okta/go.mod index 14e337caa9d747..788f7374e8c139 100644 --- a/plugins/source/okta/go.mod +++ b/plugins/source/okta/go.mod @@ -7,6 +7,7 @@ require ( github.com/cloudquery/codegen v0.3.12 github.com/cloudquery/plugin-sdk/v4 v4.29.1 github.com/gorilla/mux v1.8.0 + github.com/invopop/jsonschema v0.12.0 github.com/okta/okta-sdk-golang/v3 v3.0.2 github.com/rs/zerolog v1.31.0 github.com/thoas/go-funk v0.9.3 @@ -56,7 +57,6 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/jsonschema v0.12.0 // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect From f4340d434d056a6bede871668174ab81c96dcf67 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 2 Feb 2024 20:54:39 +0000 Subject: [PATCH 4/6] Fix for configtype.Duration usage --- plugins/source/okta/resources/plugin/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source/okta/resources/plugin/client.go b/plugins/source/okta/resources/plugin/client.go index 8df4ca61eb7033..f6d291f9f9c516 100644 --- a/plugins/source/okta/resources/plugin/client.go +++ b/plugins/source/okta/resources/plugin/client.go @@ -81,7 +81,7 @@ func Configure(_ context.Context, logger zerolog.Logger, specBytes []byte, opts okta.WithOrgUrl(config.Domain), okta.WithToken(config.Token), okta.WithCache(true), - okta.WithRateLimitMaxBackOff(int64(config.RateLimit.MaxBackoff/time.Second)), // this param takes int64 of seconds + okta.WithRateLimitMaxBackOff(int64(config.RateLimit.MaxBackoff.Duration()/time.Second)), // this param takes int64 of seconds okta.WithRateLimitMaxRetries(config.RateLimit.MaxRetries), ) cf.Debug = config.Debug From 0a354306f7ca0d127c674e944437b3d4f95273a8 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 5 Feb 2024 09:41:16 +0000 Subject: [PATCH 5/6] token requirement changes --- plugins/source/okta/client/schema.json | 4 +++- plugins/source/okta/client/spec.go | 5 +---- plugins/source/okta/client/spec_test.go | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/source/okta/client/schema.json b/plugins/source/okta/client/schema.json index bb44f0e765a61b..52b89638a137a7 100644 --- a/plugins/source/okta/client/schema.json +++ b/plugins/source/okta/client/schema.json @@ -29,7 +29,8 @@ "properties": { "token": { "type": "string", - "description": "Token for Okta API access.\nYou can set this with an `OKTA_API_TOKEN` environment variable." + "minLength": 1, + "description": "Token for Okta API access." }, "domain": { "type": "string", @@ -61,6 +62,7 @@ "additionalProperties": false, "type": "object", "required": [ + "token", "domain" ] } diff --git a/plugins/source/okta/client/spec.go b/plugins/source/okta/client/spec.go index b0b935ea6258e9..91e13ebf35d674 100644 --- a/plugins/source/okta/client/spec.go +++ b/plugins/source/okta/client/spec.go @@ -2,8 +2,6 @@ package client import ( _ "embed" - "fmt" - "os" "errors" "time" @@ -15,8 +13,7 @@ import ( type ( Spec struct { // Token for Okta API access. - // You can set this with an `OKTA_API_TOKEN` environment variable. - Token string `json:"token"` + Token string `json:"token" jsonschema:"required,minLength=1"` // Specify the Okta domain you are fetching from. // [Visit this link](https://developer.okta.com/docs/guides/find-your-domain/findorg/) to find your Okta domain. diff --git a/plugins/source/okta/client/spec_test.go b/plugins/source/okta/client/spec_test.go index b8a54315c20634..0ff0bde2460fff 100644 --- a/plugins/source/okta/client/spec_test.go +++ b/plugins/source/okta/client/spec_test.go @@ -21,6 +21,7 @@ func TestJSONSchema(t *testing.T) { { Name: "spec with domain", Spec: `{"domain": "https://domain.okta.com"}`, + Err: true, }, { Name: "spec with token and domain", From e8180cc2682a690d964c3790ea1b4d9bcb25b7d8 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 5 Feb 2024 09:44:45 +0000 Subject: [PATCH 6/6] separate ratelimit schema test --- plugins/source/okta/client/spec_test.go | 44 ++++++++++++++++++------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/plugins/source/okta/client/spec_test.go b/plugins/source/okta/client/spec_test.go index 0ff0bde2460fff..295c4116052549 100644 --- a/plugins/source/okta/client/spec_test.go +++ b/plugins/source/okta/client/spec_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/cloudquery/codegen/jsonschema" + "github.com/stretchr/testify/require" ) func TestJSONSchema(t *testing.T) { @@ -41,19 +42,9 @@ func TestJSONSchema(t *testing.T) { Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": null}`, }, { - Name: "spec with token and domain and rate limit", + Name: "spec with token and domain and valid rate limit", Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {"max_backoff": "60s"}}`, }, - { - Name: "spec with token and domain and invalid rate limit", - Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {"max_backoff": true}}`, - Err: true, - }, - { - Name: "spec with token and domain and zero rate limit", - Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {"max_backoff": 0}}`, - Err: true, - }, { Name: "spec with bool concurrency", Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":false}`, @@ -85,3 +76,34 @@ func TestJSONSchema(t *testing.T) { }, }) } + +func TestRateLimitJSONSchema(t *testing.T) { + data, err := jsonschema.Generate(RateLimit{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(data), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "valid max_backoff", + Spec: `{"max_backoff": "60s"}`, + }, + { + Name: "invalid max_backoff", + Spec: `{"max_backoff": true}`, + Err: true, + }, + { + Name: "zero max_backoff", + Spec: `{"max_backoff": 0}`, + Err: true, + }, + { + Name: "unknown field", + Spec: `{"unknown": "test"}`, + Err: true, + }, + }) +}