Skip to content

Commit d81cb01

Browse files
mnorburymnorbury
authored
feat: Adding JSONSchema support to GitHub plugin (#16491)
closes: #16360
1 parent a49554b commit d81cb01

8 files changed

Lines changed: 468 additions & 18 deletions

File tree

‎plugins/source/github/Makefile‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ gen-docs: build
3535
lint:
3636
golangci-lint run --config ../../.golangci.yml
3737

38+
.PHONY: gen-spec-schema
39+
gen-spec-schema:
40+
go run client/spec/gen/main.go
41+
3842
# All gen targets
3943
.PHONY: gen
40-
gen: gen-mocks gen-docs
44+
gen: gen-mocks gen-docs gen-spec-schema

‎plugins/source/github/client/schema.json‎

Lines changed: 159 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/github/client/spec.go‎

Lines changed: 63 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,51 @@
11
package client
22

3-
import "fmt"
3+
import (
4+
_ "embed"
5+
"fmt"
46

7+
"github.com/invopop/jsonschema"
8+
)
9+
10+
// Spec is the (nested) spec used by GitHub Source Plugin
511
type Spec struct {
6-
AccessToken string `json:"access_token"`
7-
Orgs []string `json:"orgs"`
8-
Repos []string `json:"repos"`
9-
AppAuth []AppAuthSpec `json:"app_auth"`
12+
// Personal Access Token, required if not using App Authentication.
13+
AccessToken string `json:"access_token" jsonschema:"minLength=1"`
14+
// List of organizations to sync from. You must specify either orgs or repos in the configuration.
15+
Orgs []string `json:"orgs" jsonschema:"minItems=1"`
16+
// List of repositories to sync from. The format is owner/repo (e.g. cloudquery/cloudquery).
17+
// You must specify either orgs or repos in the configuration.
18+
Repos []string `json:"repos" jsonschema:"minItems=1,minLength=1,pattern=^[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+$"`
19+
AppAuth []AppAuthSpec `json:"app_auth" jsonschema:"minItems=1"`
1020
EnterpriseSettings *EnterpriseSettings `json:"enterprise"`
1121

12-
Concurrency int `json:"concurrency,omitempty"`
13-
DiscoveryConcurrency int `json:"discovery_concurrency,omitempty"`
14-
SkipArchivedRepos bool `json:"skip_archived_repos,omitempty"`
22+
// The best effort maximum number of Go routines to use.
23+
// Lower this number to reduce memory usage.
24+
Concurrency int `json:"concurrency,omitempty" jsonschema:"default=10000"`
25+
// Controls the number of parallel requests to GitHub when discovering repositories, a negative value means unlimited.
26+
DiscoveryConcurrency int `json:"discovery_concurrency,omitempty" jsonschema:"default=1"`
27+
// Skip archived repositories when discovering repositories.
28+
SkipArchivedRepos bool `json:"skip_archived_repos,omitempty"`
1529
}
1630

1731
type EnterpriseSettings struct {
18-
BaseURL string `json:"base_url"`
19-
UploadURL string `json:"upload_url"`
32+
// The base URL of the GitHub Enterprise instance.
33+
BaseURL string `json:"base_url" jsonschema:"required,minLength=1"`
34+
// The upload URL of the GitHub Enterprise instance.
35+
UploadURL string `json:"upload_url" jsonschema:"required,minLength=1"`
2036
}
2137

2238
type AppAuthSpec struct {
23-
Org string `json:"org"`
24-
AppID string `json:"app_id"`
25-
PrivateKeyPath string `json:"private_key_path"`
26-
PrivateKey string `json:"private_key"`
27-
InstallationID string `json:"installation_id"`
39+
// The GitHub organization to sync from.
40+
Org string `json:"org" jsonschema:"required,minLength=1"`
41+
// The GitHub App ID.
42+
AppID string `json:"app_id" jsonschema:"required,minLength=1"`
43+
// The path to the private key file used to authenticate the GitHub App.
44+
PrivateKeyPath string `json:"private_key_path" jsonschema:"minLength=1"`
45+
// The private key used to authenticate the GitHub App.
46+
PrivateKey string `json:"private_key" jsonschema:"minLength=1"`
47+
// The GitHub App installation ID.
48+
InstallationID string `json:"installation_id" jsonschema:"required,minLength=1"`
2849
}
2950

3051
func (s *Spec) SetDefaults() {
@@ -91,3 +112,30 @@ func validateRepo(repo string) error {
91112
}
92113
return nil
93114
}
115+
116+
func (Spec) JSONSchemaExtend(sc *jsonschema.Schema) {
117+
sc.AllOf = []*jsonschema.Schema{
118+
{
119+
OneOf: []*jsonschema.Schema{
120+
{Required: []string{"access_token"}},
121+
{Required: []string{"app_auth"}},
122+
},
123+
},
124+
{
125+
OneOf: []*jsonschema.Schema{
126+
{Required: []string{"orgs"}},
127+
{Required: []string{"repos"}},
128+
},
129+
},
130+
}
131+
}
132+
133+
func (AppAuthSpec) JSONSchemaExtend(sc *jsonschema.Schema) {
134+
sc.OneOf = []*jsonschema.Schema{
135+
{Required: []string{"private_key_path"}},
136+
{Required: []string{"private_key"}},
137+
}
138+
}
139+
140+
//go:embed schema.json
141+
var JSONSchema string

‎plugins/source/github/client/spec/gen/main.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"path"
7+
"runtime"
8+
9+
"github.com/cloudquery/cloudquery/plugins/source/github/client"
10+
cqjsonschema "github.com/cloudquery/codegen/jsonschema"
11+
"github.com/invopop/jsonschema"
12+
)
13+
14+
func main() {
15+
fmt.Println("Generating JSON schema for plugin spec")
16+
cqjsonschema.GenerateIntoFile(new(client.Spec), path.Join(currDir(), "../..", "schema.json"),
17+
cqjsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/source/github/client", path.Join(currDir(), "../..")),
18+
func(r *jsonschema.Reflector) {
19+
r.NullableFromType = false
20+
},
21+
)
22+
}
23+
24+
func currDir() string {
25+
_, filename, _, ok := runtime.Caller(0)
26+
if !ok {
27+
log.Fatal("Failed to get caller information")
28+
}
29+
return path.Dir(filename)
30+
}

0 commit comments

Comments
 (0)