Skip to content

Commit c35f473

Browse files
candiduslynxcandiduslynx
authored
feat: Introduce spec JSON schema (#14296)
Closes #14027 Blocked by: * cloudquery/codegen#39 * invopop/jsonschema#109 – merged to `cloudquery/jsonschema@cqmain` * invopop/jsonschema#110 – merged to `cloudquery/jsonschema@cqmain` I propose reviewing the annotations along with tests, as the JSON schemas generated are just too long to grasp visually.
1 parent 05249ad commit c35f473

58 files changed

Lines changed: 5730 additions & 560 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
website/pages/docs/plugins/sources/**/tables.md linguist-generated
2+
**/schema.json linguist-generated

‎plugins/source/aws/Makefile‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ gen-docs: build
4646
sed -i.bak -e 's_(\(.*\).md)_(\1)_' ../../../website/tables/aws/*.md
4747
rm -rf ../../../website/tables/aws/*.bak
4848

49+
.PHONY: gen-spec-schema
50+
gen-spec-schema:
51+
go run client/spec/gen/main.go
52+
4953
# All gen targets
5054
.PHONY: gen
51-
gen: gen-mocks gen-docs
55+
gen: gen-spec-schema gen-mocks gen-docs

‎plugins/source/aws/client/account.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/aws/aws-sdk-go-v2/service/sts"
1313
"github.com/aws/smithy-go"
1414
"github.com/cloudquery/cloudquery/plugins/source/aws/client/services"
15+
"github.com/cloudquery/cloudquery/plugins/source/aws/client/spec"
1516
"github.com/rs/zerolog"
1617
)
1718

@@ -21,7 +22,7 @@ type svcsDetail struct {
2122
svcs Services
2223
}
2324

24-
func (c *Client) setupAWSAccount(ctx context.Context, logger zerolog.Logger, awsPluginSpec *Spec, adminAccountSts AssumeRoleAPIClient, account Account) (*svcsDetail, error) {
25+
func (c *Client) setupAWSAccount(ctx context.Context, logger zerolog.Logger, awsPluginSpec *spec.Spec, adminAccountSts AssumeRoleAPIClient, account spec.Account) (*svcsDetail, error) {
2526
if account.AccountName == "" {
2627
account.AccountName = account.ID
2728
}
@@ -46,7 +47,7 @@ func (c *Client) setupAWSAccount(ctx context.Context, logger zerolog.Logger, aws
4647
awsCfg, err := ConfigureAwsSDK(ctx, logger, awsPluginSpec, account, adminAccountSts)
4748
if err != nil {
4849
warningMsg := logger.Warn().Str("account", account.AccountName).Err(err)
49-
if account.source == "org" {
50+
if account.Source == spec.AccountSourceOrg {
5051
warningMsg.Msg("Unable to assume role in account")
5152
return nil, nil
5253
}

‎plugins/source/aws/client/aws_sdk.go‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,29 +9,24 @@ import (
99
"github.com/aws/aws-sdk-go-v2/config"
1010
"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
1111
"github.com/aws/aws-sdk-go-v2/service/sts"
12+
"github.com/cloudquery/cloudquery/plugins/source/aws/client/spec"
1213
"github.com/rs/zerolog"
1314
)
1415

15-
func ConfigureAwsSDK(ctx context.Context, logger zerolog.Logger, awsPluginSpec *Spec, account Account, stsClient AssumeRoleAPIClient) (aws.Config, error) {
16+
func ConfigureAwsSDK(ctx context.Context, logger zerolog.Logger, awsPluginSpec *spec.Spec, account spec.Account, stsClient AssumeRoleAPIClient) (aws.Config, error) {
1617
var err error
1718
var awsCfg aws.Config
1819

19-
maxAttempts := 10
20-
if awsPluginSpec.MaxRetries != nil {
21-
maxAttempts = *awsPluginSpec.MaxRetries
22-
}
23-
maxBackoff := 30
24-
if awsPluginSpec.MaxBackoff != nil {
25-
maxBackoff = *awsPluginSpec.MaxBackoff
26-
}
20+
// This sets MaxRetries & MaxBackoff, too
21+
awsPluginSpec.SetDefaults()
2722

2823
configFns := []func(*config.LoadOptions) error{
2924
config.WithDefaultRegion(defaultRegion),
3025
// https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/retries-timeouts/
3126
config.WithRetryer(func() aws.Retryer {
3227
return retry.NewStandard(func(so *retry.StandardOptions) {
33-
so.MaxAttempts = maxAttempts
34-
so.MaxBackoff = time.Duration(maxBackoff) * time.Second
28+
so.MaxAttempts = *awsPluginSpec.MaxRetries
29+
so.MaxBackoff = time.Duration(*awsPluginSpec.MaxBackoff) * time.Second
3530
so.RateLimiter = &NoRateLimiter{}
3631
})
3732
}),

‎plugins/source/aws/client/client.go‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/aws/aws-sdk-go-v2/service/sts"
1212
wafv2types "github.com/aws/aws-sdk-go-v2/service/wafv2/types"
1313
"github.com/aws/smithy-go/logging"
14+
"github.com/cloudquery/cloudquery/plugins/source/aws/client/spec"
1415
"github.com/cloudquery/plugin-sdk/v4/schema"
1516
"github.com/cloudquery/plugin-sdk/v4/state"
1617
"github.com/rs/zerolog"
@@ -32,7 +33,7 @@ type Client struct {
3233
LanguageCode string
3334
Backend state.Client
3435
specificRegions bool
35-
Spec *Spec
36+
Spec *spec.Spec
3637
accountMutex map[string]*sync.Mutex
3738
}
3839

@@ -88,13 +89,13 @@ func (s *ServicesManager) InitServicesForPartitionAccount(partition, accountId s
8889
s.services[partition][accountId].Regions = funk.UniqString(append(s.services[partition][accountId].Regions, svcs.Regions...))
8990
}
9091

91-
func NewAwsClient(logger zerolog.Logger, spec *Spec) Client {
92+
func NewAwsClient(logger zerolog.Logger, s *spec.Spec) Client {
9293
return Client{
9394
ServicesManager: &ServicesManager{
9495
services: ServicesPartitionAccountMap{},
9596
},
9697
logger: logger,
97-
Spec: spec,
98+
Spec: s,
9899
accountMutex: map[string]*sync.Mutex{},
99100
}
100101
}
@@ -200,14 +201,14 @@ func (c *Client) withLanguageCode(code string) *Client {
200201
}
201202

202203
// Configure is the entrypoint into configuring the AWS plugin. It is called by the plugin initialization in resources/plugin/aws.go
203-
func Configure(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.ClientMeta, error) {
204-
if err := spec.Validate(); err != nil {
204+
func Configure(ctx context.Context, logger zerolog.Logger, s spec.Spec) (schema.ClientMeta, error) {
205+
if err := s.Validate(); err != nil {
205206
return nil, fmt.Errorf("spec validation failed: %w", err)
206207
}
207-
spec.SetDefaults()
208+
s.SetDefaults()
208209

209-
if spec.TableOptions != nil {
210-
structVal := reflect.ValueOf(*spec.TableOptions)
210+
if s.TableOptions != nil {
211+
structVal := reflect.ValueOf(*s.TableOptions)
211212
fieldNum := structVal.NumField()
212213
for i := 0; i < fieldNum; i++ {
213214
field := structVal.Field(i)
@@ -218,7 +219,7 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.Cl
218219
}
219220
}
220221

221-
client := NewAwsClient(logger, &spec)
222+
client := NewAwsClient(logger, &s)
222223

223224
var adminAccountSts AssumeRoleAPIClient
224225

@@ -231,11 +232,7 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.Cl
231232
}
232233
}
233234
if len(client.Spec.Accounts) == 0 {
234-
client.Spec.Accounts = []Account{
235-
{
236-
ID: defaultVar,
237-
},
238-
}
235+
client.Spec.Accounts = []spec.Account{{ID: defaultVar}}
239236
}
240237

241238
initLock := sync.Mutex{}

‎plugins/source/aws/client/organizations.go‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
"github.com/cloudquery/cloudquery/plugins/source/aws/client/services"
7+
"github.com/cloudquery/cloudquery/plugins/source/aws/client/spec"
78
"github.com/thoas/go-funk"
89

910
"github.com/aws/aws-sdk-go-v2/aws"
@@ -15,10 +16,10 @@ import (
1516
)
1617

1718
// Parses org configuration and grabs the appropriate accounts
18-
func loadOrgAccounts(ctx context.Context, logger zerolog.Logger, awsPluginSpec *Spec) ([]Account, AssumeRoleAPIClient, error) {
19+
func loadOrgAccounts(ctx context.Context, logger zerolog.Logger, awsPluginSpec *spec.Spec) ([]spec.Account, AssumeRoleAPIClient, error) {
1920
// If user doesn't specify any configs for admin account instantiate default values
2021
if awsPluginSpec.Organization.AdminAccount == nil {
21-
awsPluginSpec.Organization.AdminAccount = &Account{
22+
awsPluginSpec.Organization.AdminAccount = &spec.Account{
2223
AccountName: "Default-Admin-Account",
2324
LocalProfile: "",
2425
}
@@ -46,7 +47,7 @@ func loadOrgAccounts(ctx context.Context, logger zerolog.Logger, awsPluginSpec *
4647
}
4748

4849
// Load accounts from the appropriate endpoint as well as normalizing response
49-
func loadAccounts(ctx context.Context, awsPluginSpec *Spec, accountsApi services.OrganizationsClient, region string) ([]Account, error) {
50+
func loadAccounts(ctx context.Context, awsPluginSpec *spec.Spec, accountsApi services.OrganizationsClient, region string) ([]spec.Account, error) {
5051
var rawAccounts []orgTypes.Account
5152
var err error
5253
if len(awsPluginSpec.Organization.OrganizationUnits) > 0 {
@@ -56,10 +57,10 @@ func loadAccounts(ctx context.Context, awsPluginSpec *Spec, accountsApi services
5657
}
5758

5859
if err != nil {
59-
return []Account{}, err
60+
return []spec.Account{}, err
6061
}
6162
seen := map[string]struct{}{}
62-
accounts := make([]Account, 0)
63+
accounts := make([]spec.Account, 0)
6364
for _, account := range rawAccounts {
6465
// Only load Active accounts
6566
if account.Status != orgTypes.AccountStatusActive || account.Id == nil {
@@ -83,21 +84,21 @@ func loadAccounts(ctx context.Context, awsPluginSpec *Spec, accountsApi services
8384
roleArn.Partition = parsed.Partition
8485
}
8586

86-
accounts = append(accounts, Account{
87+
accounts = append(accounts, spec.Account{
8788
ID: *account.Id,
8889
RoleARN: roleArn.String(),
8990
RoleSessionName: awsPluginSpec.Organization.ChildAccountRoleSessionName,
9091
ExternalID: awsPluginSpec.Organization.ChildAccountExternalID,
9192
LocalProfile: awsPluginSpec.Organization.AdminAccount.LocalProfile,
9293
Regions: awsPluginSpec.Organization.ChildAccountRegions,
93-
source: "org",
94+
Source: spec.AccountSourceOrg,
9495
})
9596
}
9697
return accounts, err
9798
}
9899

99100
// Get Accounts for specific Organizational Units
100-
func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient, awsOrg *AwsOrg, region string) ([]orgTypes.Account, error) {
101+
func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient, awsOrg *spec.Org, region string) ([]orgTypes.Account, error) {
101102
q := awsOrg.OrganizationUnits
102103
var ou string
103104
var rawAccounts []orgTypes.Account
@@ -158,7 +159,7 @@ func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient
158159
}
159160

160161
// Get All accounts in a specific organization
161-
func getAllAccounts(ctx context.Context, accountsApi services.OrganizationsClient, org *AwsOrg, region string) ([]orgTypes.Account, error) {
162+
func getAllAccounts(ctx context.Context, accountsApi services.OrganizationsClient, org *spec.Org, region string) ([]orgTypes.Account, error) {
162163
var rawAccounts []orgTypes.Account
163164
accountsPaginator := organizations.NewListAccountsPaginator(accountsApi, &organizations.ListAccountsInput{})
164165
for accountsPaginator.HasMorePages() {

‎plugins/source/aws/client/organizations_test.go‎

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"testing"
88

99
"github.com/cloudquery/cloudquery/plugins/source/aws/client/mocks"
10+
"github.com/cloudquery/cloudquery/plugins/source/aws/client/spec"
1011
"github.com/golang/mock/gomock"
1112

1213
"github.com/aws/aws-sdk-go-v2/aws"
@@ -111,99 +112,99 @@ func Test_loadAccounts(t *testing.T) {
111112
ctx := context.Background()
112113
tests := []struct {
113114
name string
114-
spec *Spec
115+
spec *spec.Spec
115116
want []string
116117
wantErr error
117118
}{
118119
{
119120
name: "all_accounts",
120-
spec: &Spec{
121-
Organization: &AwsOrg{
122-
AdminAccount: &Account{},
121+
spec: &spec.Spec{
122+
Organization: &spec.Org{
123+
AdminAccount: &spec.Account{},
123124
},
124125
},
125126
want: []string{"id-child1-account", "id-child2-account", "id-parent1-account", "id-parent2-account", "id-top-level-account"},
126127
},
127128
{
128129
name: "all_accounts_with_skip_member_accounts",
129-
spec: &Spec{
130-
Organization: &AwsOrg{
131-
AdminAccount: &Account{},
130+
spec: &spec.Spec{
131+
Organization: &spec.Org{
132+
AdminAccount: &spec.Account{},
132133
SkipMemberAccounts: []string{"id-child2-account", "id-parent1-account", "id-parent2-account", "id-top-level-account"},
133134
},
134135
},
135136
want: []string{"id-child1-account"},
136137
},
137138
{
138139
name: "org_root",
139-
spec: &Spec{
140-
Organization: &AwsOrg{
140+
spec: &spec.Spec{
141+
Organization: &spec.Org{
141142
OrganizationUnits: []string{"root"},
142-
AdminAccount: &Account{},
143+
AdminAccount: &spec.Account{},
143144
},
144145
},
145146
want: []string{"id-top-level-account", "id-child1-account", "id-parent1-account", "id-child2-account", "id-parent2-account"},
146147
},
147148
{
148149
name: "ou_parent1",
149-
spec: &Spec{
150-
Organization: &AwsOrg{
150+
spec: &spec.Spec{
151+
Organization: &spec.Org{
151152
OrganizationUnits: []string{"ou-parent1"},
152-
AdminAccount: &Account{},
153+
AdminAccount: &spec.Account{},
153154
},
154155
},
155156
want: []string{"id-parent1-account", "id-child1-account"},
156157
},
157158
{
158159
name: "ou_parent1_and_parent2",
159-
spec: &Spec{
160-
Organization: &AwsOrg{
160+
spec: &spec.Spec{
161+
Organization: &spec.Org{
161162
OrganizationUnits: []string{"ou-parent1", "ou-parent2"},
162-
AdminAccount: &Account{},
163+
AdminAccount: &spec.Account{},
163164
},
164165
},
165166
want: []string{"id-parent1-account", "id-child1-account", "id-parent2-account", "id-child2-account"},
166167
},
167168
{
168169
name: "ou_parent1_skip_child1",
169-
spec: &Spec{
170-
Organization: &AwsOrg{
170+
spec: &spec.Spec{
171+
Organization: &spec.Org{
171172
OrganizationUnits: []string{"ou-parent1"},
172173
SkipMemberAccounts: []string{"id-child1-account"},
173-
AdminAccount: &Account{},
174+
AdminAccount: &spec.Account{},
174175
},
175176
},
176177
want: []string{"id-parent1-account"},
177178
},
178179
{
179180
name: "ou_root_skip_parent1",
180-
spec: &Spec{
181-
Organization: &AwsOrg{
181+
spec: &spec.Spec{
182+
Organization: &spec.Org{
182183
OrganizationUnits: []string{"root"},
183184
SkipOrganizationalUnits: []string{"ou-parent1"},
184-
AdminAccount: &Account{},
185+
AdminAccount: &spec.Account{},
185186
},
186187
},
187188
want: []string{"id-top-level-account", "id-parent2-account", "id-child2-account"},
188189
},
189190
{
190191
name: "ou_root_skip_parent1",
191-
spec: &Spec{
192-
Organization: &AwsOrg{
192+
spec: &spec.Spec{
193+
Organization: &spec.Org{
193194
OrganizationUnits: []string{"root"},
194195
SkipOrganizationalUnits: []string{"ou-parent1"},
195-
AdminAccount: &Account{},
196+
AdminAccount: &spec.Account{},
196197
},
197198
},
198199
want: []string{"id-top-level-account", "id-parent2-account", "id-child2-account"},
199200
},
200201
{
201202
name: "ou_root_and_parent1",
202-
spec: &Spec{
203-
Organization: &AwsOrg{
203+
spec: &spec.Spec{
204+
Organization: &spec.Org{
204205
OrganizationUnits: []string{"root", "ou-parent1"},
205206
SkipOrganizationalUnits: []string{},
206-
AdminAccount: &Account{},
207+
AdminAccount: &spec.Account{},
207208
},
208209
},
209210
want: []string{"id-top-level-account", "id-parent1-account", "id-child1-account", "id-parent2-account", "id-child2-account"},

0 commit comments

Comments
 (0)