Skip to content

Commit 75ce0ab

Browse files
disqdisq
authored
feat: Add JSON Schema to Okta source plugin (#16493)
Closes #16453
1 parent d374ba8 commit 75ce0ab

10 files changed

Lines changed: 255 additions & 16 deletions

File tree

‎plugins/source/okta/Makefile‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ gen-docs: build
2222
cloudquery tables --format markdown --output-dir docs test/config.yml
2323
mv docs/$(shell basename $(CURDIR)) docs/tables
2424

25+
.PHONY: gen-spec-schema
26+
gen-spec-schema:
27+
go run client/spec/gen/main.go
28+
2529
# All gen targets
2630
.PHONY: gen
27-
gen: gen-docs
31+
gen: gen-spec-schema gen-docs

‎plugins/source/okta/client/schema.json‎

Lines changed: 70 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/okta/client/spec.go‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,44 @@
11
package client
22

33
import (
4+
_ "embed"
45
"errors"
56
"time"
67

8+
"github.com/cloudquery/plugin-sdk/v4/configtype"
9+
"github.com/invopop/jsonschema"
710
"github.com/rs/zerolog"
811
)
912

1013
type (
1114
Spec struct {
12-
Token string `json:"token,omitempty"`
13-
Domain string `json:"domain,omitempty"`
14-
RateLimit *RateLimit `json:"rate_limit,omitempty"`
15-
Debug bool `json:"debug,omitempty"`
16-
Concurrency int `json:"concurrency,omitempty"`
15+
// Token for Okta API access.
16+
Token string `json:"token" jsonschema:"required,minLength=1"`
17+
18+
// Specify the Okta domain you are fetching from.
19+
// [Visit this link](https://developer.okta.com/docs/guides/find-your-domain/findorg/) to find your Okta domain.
20+
Domain string `json:"domain" jsonschema:"required,pattern=^https?://[^\n<>]+\\.okta\\.com$"`
21+
RateLimit *RateLimit `json:"rate_limit"`
22+
23+
// Enables debug logs within the Okta SDK.
24+
Debug bool `json:"debug,omitempty" jsonschema:"default=false"`
25+
26+
// Number of concurrent requests to be made to Okta API.
27+
Concurrency int `json:"concurrency" jsonschema:"minimum=1,default=10000"`
1728
}
1829
RateLimit struct {
19-
MaxBackoff time.Duration `json:"max_backoff,omitempty"`
20-
MaxRetries int32 `json:"max_retries,omitempty"`
30+
// Max backoff interval to be used.
31+
// If the value specified is less than the default one, the default one is used.
32+
MaxBackoff configtype.Duration `json:"max_backoff,omitempty"`
33+
34+
// Max retries to be performed.
35+
MaxRetries int32 `json:"max_retries,omitempty" jsonschema:"minimum=2,default=2"`
2136
}
2237
)
2338

39+
//go:embed schema.json
40+
var JSONSchema string
41+
2442
func (s *Spec) SetDefaults(logger *zerolog.Logger) {
2543
const (
2644
minRetries = int32(2)
@@ -35,8 +53,8 @@ func (s *Spec) SetDefaults(logger *zerolog.Logger) {
3553
s.RateLimit.MaxRetries = minRetries
3654
}
3755

38-
if s.RateLimit.MaxBackoff < minBackOff {
39-
s.RateLimit.MaxBackoff = minBackOff
56+
if s.RateLimit.MaxBackoff.Duration() < minBackOff {
57+
s.RateLimit.MaxBackoff = configtype.NewDuration(minBackOff)
4058
}
4159

4260
if s.Concurrency < 1 {
@@ -57,3 +75,7 @@ func (s Spec) Validate() error {
5775

5876
return nil
5977
}
78+
79+
func (RateLimit) JSONSchemaExtend(sc *jsonschema.Schema) {
80+
sc.Properties.Value("max_backoff").Default = "30s"
81+
}

‎plugins/source/okta/client/spec/gen/main.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"path"
7+
"runtime"
8+
9+
"github.com/cloudquery/cloudquery/plugins/source/okta/client"
10+
cqjsonschema "github.com/cloudquery/codegen/jsonschema"
11+
)
12+
13+
func main() {
14+
fmt.Println("Generating JSON schema for plugin spec")
15+
cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"),
16+
cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/source/okta/client", path.Join(currDir(), "../..")),
17+
)
18+
}
19+
20+
func currDir() string {
21+
_, filename, _, ok := runtime.Caller(0)
22+
if !ok {
23+
log.Fatal("Failed to get caller information")
24+
}
25+
return path.Dir(filename)
26+
}

‎plugins/source/okta/client/spec_test.go‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package client
2+
3+
import (
4+
"testing"
5+
6+
"github.com/cloudquery/codegen/jsonschema"
7+
"github.com/stretchr/testify/require"
8+
)
9+
10+
func TestJSONSchema(t *testing.T) {
11+
jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{
12+
{
13+
Name: "empty spec",
14+
Spec: `{}`,
15+
Err: true,
16+
},
17+
{
18+
Name: "spec with token",
19+
Spec: `{"token": "tok"}`,
20+
Err: true,
21+
},
22+
{
23+
Name: "spec with domain",
24+
Spec: `{"domain": "https://domain.okta.com"}`,
25+
Err: true,
26+
},
27+
{
28+
Name: "spec with token and domain",
29+
Spec: `{"token": "tok", "domain": "https://domain.okta.com"}`,
30+
},
31+
{
32+
Name: "spec with token and invalid domain",
33+
Spec: `{"token": "tok", "domain": "https://<CHANGE_THIS_TO_YOUR_OKTA_DOMAIN>.okta.com"}`,
34+
Err: true,
35+
},
36+
{
37+
Name: "spec with token and domain and empty rate limit",
38+
Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {}}`,
39+
},
40+
{
41+
Name: "spec with token and domain and null rate limit",
42+
Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": null}`,
43+
},
44+
{
45+
Name: "spec with token and domain and valid rate limit",
46+
Spec: `{"token": "tok", "domain": "https://domain.okta.com", "rate_limit": {"max_backoff": "60s"}}`,
47+
},
48+
{
49+
Name: "spec with bool concurrency",
50+
Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":false}`,
51+
Err: true,
52+
},
53+
{
54+
Name: "spec with null concurrency",
55+
Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":null}`,
56+
Err: true,
57+
},
58+
{
59+
Name: "spec with string concurrency",
60+
Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":"str"}`,
61+
Err: true,
62+
},
63+
{
64+
Name: "spec with proper concurrency",
65+
Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency": 7}`,
66+
},
67+
{
68+
Name: "spec with array concurrency",
69+
Spec: `{"token": "tok", "domain": "https://domain.okta.com", "concurrency":["abc"]}`,
70+
Err: true,
71+
},
72+
{
73+
Name: "spec with unknown field",
74+
Spec: `{"token": "tok", "domain": "https://domain.okta.com", "unknown": "test"}`,
75+
Err: true,
76+
},
77+
})
78+
}
79+
80+
func TestRateLimitJSONSchema(t *testing.T) {
81+
data, err := jsonschema.Generate(RateLimit{})
82+
require.NoError(t, err)
83+
84+
jsonschema.TestJSONSchema(t, string(data), []jsonschema.TestCase{
85+
{
86+
Name: "empty",
87+
Spec: `{}`,
88+
},
89+
{
90+
Name: "valid max_backoff",
91+
Spec: `{"max_backoff": "60s"}`,
92+
},
93+
{
94+
Name: "invalid max_backoff",
95+
Spec: `{"max_backoff": true}`,
96+
Err: true,
97+
},
98+
{
99+
Name: "zero max_backoff",
100+
Spec: `{"max_backoff": 0}`,
101+
Err: true,
102+
},
103+
{
104+
Name: "unknown field",
105+
Spec: `{"unknown": "test"}`,
106+
Err: true,
107+
},
108+
})
109+
}

‎plugins/source/okta/docs/overview.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,12 @@ Make sure you use [environment variable expansion](/docs/advanced-topics/environ
5050

5151
### Rate limit spec
5252

53-
- `max_backoff` (`duration`) (optional) (default: `5s`)
53+
- `max_backoff` (`duration`) (optional) (default: `30s`)
5454

5555
Max backoff interval to be used.
5656
If the value specified is less than the default one, the default one is used.
5757

58-
- `max_retries` (`integer`) (optional) (default: `3`)
58+
- `max_retries` (`integer`) (optional) (default: `2`)
5959

6060
Max retries to be performed.
6161
If the value specified is less than the default one, the default one is used.

‎plugins/source/okta/go.mod‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@ go 1.21.4
44

55
require (
66
github.com/apache/arrow/go/v15 v15.0.0-20240114144300-7e703aae55c1
7+
github.com/cloudquery/codegen v0.3.12
78
github.com/cloudquery/plugin-sdk/v4 v4.29.1
89
github.com/gorilla/mux v1.8.0
10+
github.com/invopop/jsonschema v0.12.0
911
github.com/okta/okta-sdk-golang/v3 v3.0.2
1012
github.com/rs/zerolog v1.31.0
1113
github.com/thoas/go-funk v0.9.3
@@ -55,7 +57,6 @@ require (
5557
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect
5658
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
5759
github.com/inconshreveable/mousetrap v1.1.0 // indirect
58-
github.com/invopop/jsonschema v0.12.0 // indirect
5960
github.com/iris-contrib/schema v0.0.6 // indirect
6061
github.com/josharian/intern v1.0.0 // indirect
6162
github.com/json-iterator/go v1.1.12 // indirect
@@ -129,3 +130,6 @@ require (
129130
gopkg.in/yaml.v2 v2.4.0 // indirect
130131
gopkg.in/yaml.v3 v3.0.1 // indirect
131132
)
133+
134+
// github.com/cloudquery/jsonschema @ cqmain
135+
replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb

‎plugins/source/okta/go.sum‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
8888
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
8989
github.com/cloudquery/cloudquery-api-go v1.7.2 h1:jpQfeZUxekbV7ASN5ONpGIkrtKIZvC/Y8fOj+tQxLm4=
9090
github.com/cloudquery/cloudquery-api-go v1.7.2/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
91+
github.com/cloudquery/codegen v0.3.12 h1:9BaYdwbMJU1HVT/BHI+ykhOhBGeXt8AjpvBiXN1KhKE=
92+
github.com/cloudquery/codegen v0.3.12/go.mod h1:utqjurr58U8uqcPJe0rZjh06i0Eq9uAPGOmyIjq/1w8=
93+
github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb h1:/l8fbvLOCNlgkHp8VUKTTL+Tk9gs5y/K3Yx/bRfReNk=
94+
github.com/cloudquery/jsonschema v0.0.0-20240202134451-d771afde32fb/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
9195
github.com/cloudquery/plugin-pb-go v1.16.7 h1:wLx5TFvS6gAvD1dcBZdv5YSskcNCnNpF1JNituka5jM=
9296
github.com/cloudquery/plugin-pb-go v1.16.7/go.mod h1:Sd08P8HIjwi3gmfoE0X21qo6HL1NiVdNl/00JrK+DkM=
9397
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
@@ -228,8 +232,6 @@ github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/C
228232
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
229233
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
230234
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
231-
github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI=
232-
github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
233235
github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go=
234236
github.com/iris-contrib/httpexpect/v2 v2.15.2/go.mod h1:JLDgIqnFy5loDSUv1OA2j0mb6p/rDhiCqigP22Uq9xE=
235237
github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw=

‎plugins/source/okta/resources/plugin/client.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ func Configure(_ context.Context, logger zerolog.Logger, specBytes []byte, opts
8181
okta.WithOrgUrl(config.Domain),
8282
okta.WithToken(config.Token),
8383
okta.WithCache(true),
84-
okta.WithRateLimitMaxBackOff(int64(config.RateLimit.MaxBackoff/time.Second)), // this param takes int64 of seconds
84+
okta.WithRateLimitMaxBackOff(int64(config.RateLimit.MaxBackoff.Duration()/time.Second)), // this param takes int64 of seconds
8585
okta.WithRateLimitMaxRetries(config.RateLimit.MaxRetries),
8686
)
8787
cf.Debug = config.Debug

‎plugins/source/okta/resources/plugin/plugin.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package plugin
22

33
import (
4+
"github.com/cloudquery/cloudquery/plugins/source/okta/client"
45
"github.com/cloudquery/plugin-sdk/v4/plugin"
56
)
67

@@ -18,5 +19,6 @@ func Plugin() *plugin.Plugin {
1819
Configure,
1920
plugin.WithKind(Kind),
2021
plugin.WithTeam(Team),
22+
plugin.WithJSONSchema(client.JSONSchema),
2123
)
2224
}

0 commit comments

Comments
 (0)