From 212c862f82e75caf45e5ad3011a92e9526799e58 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 8 Feb 2024 14:59:14 +0000 Subject: [PATCH 1/5] fix: Extract `HUBSPOT_APP_TOKEN` to spec --- plugins/source/hubspot/client/client.go | 2 +- .../source/hubspot/client/spec/schema.json | 20 ++++---- plugins/source/hubspot/client/spec/spec.go | 36 ++++++++++---- .../source/hubspot/client/spec/spec_test.go | 48 ++++++++++++------- .../hubspot/client/spec/table_options.go | 4 +- .../source/hubspot/docs/_authentication.md | 5 +- plugins/source/hubspot/docs/overview.md | 4 ++ .../source/hubspot/resources/plugin/plugin.go | 3 ++ 8 files changed, 80 insertions(+), 42 deletions(-) diff --git a/plugins/source/hubspot/client/client.go b/plugins/source/hubspot/client/client.go index bb25346457ecb4..c422d4022f679c 100644 --- a/plugins/source/hubspot/client/client.go +++ b/plugins/source/hubspot/client/client.go @@ -54,7 +54,7 @@ func New(_ context.Context, logger zerolog.Logger, s spec.Spec) (schema.ClientMe Authorizer: hubspot.NewTokenAuthorizer(authToken), Spec: s, RateLimiter: rate.NewLimiter( - /* r= */ rate.Limit(*s.MaxRequestsPerSecond), + /* r= */ rate.Limit(s.MaxRequestsPerSecond), /* b= */ 1, ), }, nil diff --git a/plugins/source/hubspot/client/spec/schema.json b/plugins/source/hubspot/client/spec/schema.json index 850d99408b6194..e7827df87427b7 100644 --- a/plugins/source/hubspot/client/spec/schema.json +++ b/plugins/source/hubspot/client/spec/schema.json @@ -5,18 +5,16 @@ "$defs": { "Spec": { "properties": { + "app_token": { + "type": "string", + "minLength": 1, + "description": "In order for CloudQuery to sync resources from your HubSpot setup, you will need to authenticate with your HubSpot account. You will need to create a [HubSpot Private App](https://developers.hubspot.com/docs/api/private-apps), and copy the App Token here.\nIf not specified `HUBSPOT_APP_TOKEN` environment variable will be used instead." + }, "max_requests_per_second": { - "oneOf": [ - { - "type": "integer", - "minimum": 1, - "description": "Max number of requests per second to perform against the Hubspot API.", - "default": 5 - }, - { - "type": "null" - } - ] + "type": "integer", + "minimum": 1, + "description": "Max number of requests per second to perform against the Hubspot API.", + "default": 5 }, "table_options": { "oneOf": [ diff --git a/plugins/source/hubspot/client/spec/spec.go b/plugins/source/hubspot/client/spec/spec.go index ac935c1e2df808..3d723b3ee13db9 100644 --- a/plugins/source/hubspot/client/spec/spec.go +++ b/plugins/source/hubspot/client/spec/spec.go @@ -1,21 +1,32 @@ package spec -import _ "embed" +import ( + _ "embed" + "fmt" + "os" +) const ( defaultConcurrency = 1000 ) type Spec struct { + // In order for CloudQuery to sync resources from your HubSpot setup, you will need to authenticate with your HubSpot account. You will need to create a [HubSpot Private App](https://developers.hubspot.com/docs/api/private-apps), and copy the App Token here. + // If not specified `HUBSPOT_APP_TOKEN` environment variable will be used instead. + AppToken string `json:"app_token,omitempty" jsonschema:"minLength=1"` // Max number of requests per second to perform against the Hubspot API. - MaxRequestsPerSecond *int `yaml:"max_requests_per_second,omitempty" json:"max_requests_per_second,omitempty" jsonschema:"minimum=1,default=5"` + MaxRequestsPerSecond int `json:"max_requests_per_second,omitempty" jsonschema:"minimum=1,default=5"` // Key-value map of options for each table. The key is the name of the table. The value is an options object. - TableOptions TableOptions `yaml:"table_options,omitempty" json:"table_options,omitempty"` + TableOptions TableOptions `json:"table_options,omitempty"` // Concurrency setting for the CloudQuery scheduler. - Concurrency int `yaml:"concurrency,omitempty" json:"concurrency,omitempty" jsonschema:"minimum=1,default=1000"` + Concurrency int `json:"concurrency,omitempty" jsonschema:"minimum=1,default=1000"` } -func (spec *Spec) SetDefaults() { +func (s *Spec) SetDefaults() { + if s.AppToken == "" { + s.AppToken = os.Getenv("HUBSPOT_APP_TOKEN") + } + // https://developers.hubspot.com/docs/api/usage-details#rate-limits // Hubspot, for Pro and Enterprise, accounts, has rate limits of: // - 15 requests / second / private-app @@ -24,13 +35,20 @@ func (spec *Spec) SetDefaults() { // subscriptions in case cloudquery is run 24/7). var defaultRateLimitPerSecond = 5 - if spec.MaxRequestsPerSecond == nil || *spec.MaxRequestsPerSecond <= 0 { - spec.MaxRequestsPerSecond = &defaultRateLimitPerSecond + if s.MaxRequestsPerSecond <= 0 { + s.MaxRequestsPerSecond = defaultRateLimitPerSecond } - if spec.Concurrency == 0 { - spec.Concurrency = defaultConcurrency + if s.Concurrency == 0 { + s.Concurrency = defaultConcurrency + } +} + +func (s Spec) Validate() error { + if s.AppToken == "" { + return fmt.Errorf("app_token is required") } + return nil } //go:embed schema.json diff --git a/plugins/source/hubspot/client/spec/spec_test.go b/plugins/source/hubspot/client/spec/spec_test.go index 64b2b0bd74d579..3365ea2dfb0020 100644 --- a/plugins/source/hubspot/client/spec/spec_test.go +++ b/plugins/source/hubspot/client/spec/spec_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/cloudquery/codegen/jsonschema" + "github.com/stretchr/testify/require" ) func TestJSONSchema(t *testing.T) { @@ -13,57 +14,70 @@ func TestJSONSchema(t *testing.T) { Spec: `{}`, }, { - Name: "null max_requests_per_second", - Spec: `{"max_requests_per_second": null}`, + Name: "app_token", + Spec: `{"app_token": "token"}`, + }, + { + Name: "empty app_token", + Spec: `{"app_token": ""}`, + Err: true, }, { Name: "max_requests_per_second == -1 is invalid", - Spec: `{"max_requests_per_second": -1}`, + Spec: `{"app_token": "token", "max_requests_per_second": -1}`, Err: true, }, { Name: "max_requests_per_second == 0 is invalid", - Spec: `{"max_requests_per_second": 0}`, + Spec: `{"app_token": "token", "max_requests_per_second": 0}`, Err: true, }, { Name: "max_requests_per_second == 1 is valid", - Spec: `{"max_requests_per_second": 1}`, + Spec: `{"app_token": "token", "max_requests_per_second": 1}`, }, { Name: "concurrency == -1 is invalid", - Spec: `{"concurrency": -1}`, + Spec: `{"app_token": "token", "concurrency": -1}`, Err: true, }, { Name: "concurrency == 0 is invalid", - Spec: `{"concurrency": 0}`, + Spec: `{"app_token": "token", "concurrency": 0}`, Err: true, }, { Name: "concurrency == 1 is valid", - Spec: `{"concurrency": 1}`, + Spec: `{"app_token": "token", "concurrency": 1}`, }, { Name: "table_options == null is valid", - Spec: `{"table_options": null}`, + Spec: `{"app_token": "token", "table_options": null}`, }, + }) +} + +func TestTableOptionsJSONSchema(t *testing.T) { + schema, err := jsonschema.Generate(TableOptions{}) + + require.NoError(t, err) + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ { - Name: "table_options == {} is valid", - Spec: `{"table_options": {}}`, + Name: "empty table options", + Spec: `{}`, }, { - Name: "spec with table options = null is invalid", - Spec: `{"table_options": {"hubspot_crm_companies": null}}`, + Name: "hubspot_crm_companies = null is invalid", + Spec: `{"hubspot_crm_companies": null}`, }, { - Name: "spec with table options = [] is invalid", - Spec: `{"table_options": {"hubspot_crm_companies": []}}`, + Name: "hubspot_crm_companiess = [] is invalid", + Spec: `{"hubspot_crm_companies": []}`, Err: true, }, { - Name: "spec with table options = {} is valid", - Spec: `{"table_options": {"hubspot_crm_companies": {}}}`, + Name: "hubspot_crm_companies = {} is valid", + Spec: `{"hubspot_crm_companies": {}}`, }, }) } diff --git a/plugins/source/hubspot/client/spec/table_options.go b/plugins/source/hubspot/client/spec/table_options.go index 0cc2643cba5b41..37ca69ce6d6cc7 100644 --- a/plugins/source/hubspot/client/spec/table_options.go +++ b/plugins/source/hubspot/client/spec/table_options.go @@ -5,9 +5,9 @@ type TableOptions map[string]*TableOptionsSpec // Table options spec. type TableOptionsSpec struct { // List of properties to sync. If empty, everything is synced. - Properties []string `yaml:"properties,omitempty" json:"properties,omitempty" jsonschema:"minLength=1"` + Properties []string `json:"properties,omitempty" jsonschema:"minLength=1"` // List of associations to sync. If empty, everything is synced. - Associations []string `yaml:"associations,omitempty" json:"associations,omitempty" jsonschema:"minLength=1"` + Associations []string `json:"associations,omitempty" jsonschema:"minLength=1"` } func (ts TableOptions) ForTable(name string) *TableOptionsSpec { diff --git a/plugins/source/hubspot/docs/_authentication.md b/plugins/source/hubspot/docs/_authentication.md index 2b859b31826632..af3b8f5e43b10c 100644 --- a/plugins/source/hubspot/docs/_authentication.md +++ b/plugins/source/hubspot/docs/_authentication.md @@ -1,5 +1,6 @@ -In Order for CloudQuery to sync resources from your HubSpot setup, you will need to authenticate with your HubSpot account. You will need to create a [HubSpot Private App](https://developers.hubspot.com/docs/api/private-apps), and export the App Token in `HUBSPOT_APP_TOKEN` environment variable. +In Order for CloudQuery to sync resources from your HubSpot setup, you will need to authenticate with your HubSpot account. You will need to create a [HubSpot Private App](https://developers.hubspot.com/docs/api/private-apps), and copy the App Token to the spec. +If not specified `HUBSPOT_APP_TOKEN` environment variable will be used instead. ```bash copy -export HUBSPOT_APP_TOKEN= +export HUBSPOT_APP_TOKEN= # optional, if not using spec configuration ``` diff --git a/plugins/source/hubspot/docs/overview.md b/plugins/source/hubspot/docs/overview.md index 4f154d9fbdcaaa..51eb43a35955b0 100644 --- a/plugins/source/hubspot/docs/overview.md +++ b/plugins/source/hubspot/docs/overview.md @@ -31,7 +31,11 @@ The following example sets up the HubSpot plugin, and connects it to a postgresq This is the specs that can be used by the HubSpot source Plugin. +- `app_token` (`string`, optional, default: `HUBSPOT_APP_TOKEN` environment variable) + The HubSpot App Token to use for authentication. This can also be set with the `HUBSPOT_APP_TOKEN` environment variable. + - `concurrency` (int, optional, default: `1000`): + A best effort maximum number of Go routines to use. Lower this number to reduce memory usage. - `max_requests_per_second` (`int`, optional. Default: `5`) diff --git a/plugins/source/hubspot/resources/plugin/plugin.go b/plugins/source/hubspot/resources/plugin/plugin.go index a5c50bc3fded69..d04a60ced72427 100644 --- a/plugins/source/hubspot/resources/plugin/plugin.go +++ b/plugins/source/hubspot/resources/plugin/plugin.go @@ -65,6 +65,9 @@ func newClient(ctx context.Context, logger zerolog.Logger, specBytes []byte, opt return nil, err } s.SetDefaults() + if err := s.Validate(); err != nil { + return nil, err + } syncClient, err := client.New(ctx, logger, *s) if err != nil { return nil, err From baf9ce372847f13745d8c3f7cb041a13a22eecf4 Mon Sep 17 00:00:00 2001 From: Kemal <223029+disq@users.noreply.github.com> Date: Thu, 8 Feb 2024 07:32:05 -0800 Subject: [PATCH 2/5] Update plugins/source/hubspot/client/spec/spec.go Co-authored-by: Alex Shcherbakov --- plugins/source/hubspot/client/spec/spec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source/hubspot/client/spec/spec.go b/plugins/source/hubspot/client/spec/spec.go index 3d723b3ee13db9..219517ab32d8f5 100644 --- a/plugins/source/hubspot/client/spec/spec.go +++ b/plugins/source/hubspot/client/spec/spec.go @@ -23,7 +23,7 @@ type Spec struct { } func (s *Spec) SetDefaults() { - if s.AppToken == "" { + if len(s.AppToken) == 0 { s.AppToken = os.Getenv("HUBSPOT_APP_TOKEN") } From a2f19a564fd88fea88a73b77879ca12f9fe4dff9 Mon Sep 17 00:00:00 2001 From: Kemal <223029+disq@users.noreply.github.com> Date: Thu, 8 Feb 2024 07:32:26 -0800 Subject: [PATCH 3/5] Update plugins/source/hubspot/docs/overview.md Co-authored-by: Alex Shcherbakov --- plugins/source/hubspot/docs/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source/hubspot/docs/overview.md b/plugins/source/hubspot/docs/overview.md index 51eb43a35955b0..2882698cd3614d 100644 --- a/plugins/source/hubspot/docs/overview.md +++ b/plugins/source/hubspot/docs/overview.md @@ -31,7 +31,7 @@ The following example sets up the HubSpot plugin, and connects it to a postgresq This is the specs that can be used by the HubSpot source Plugin. -- `app_token` (`string`, optional, default: `HUBSPOT_APP_TOKEN` environment variable) +- `app_token` (`string`) (optional) (default: `HUBSPOT_APP_TOKEN` environment variable value) The HubSpot App Token to use for authentication. This can also be set with the `HUBSPOT_APP_TOKEN` environment variable. - `concurrency` (int, optional, default: `1000`): From 241656d23647b609f57bc7a15d25658ac2413296 Mon Sep 17 00:00:00 2001 From: Kemal <223029+disq@users.noreply.github.com> Date: Thu, 8 Feb 2024 07:32:35 -0800 Subject: [PATCH 4/5] Update plugins/source/hubspot/client/spec/spec.go Co-authored-by: Alex Shcherbakov --- plugins/source/hubspot/client/spec/spec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source/hubspot/client/spec/spec.go b/plugins/source/hubspot/client/spec/spec.go index 219517ab32d8f5..070b84c286a703 100644 --- a/plugins/source/hubspot/client/spec/spec.go +++ b/plugins/source/hubspot/client/spec/spec.go @@ -39,7 +39,7 @@ func (s *Spec) SetDefaults() { s.MaxRequestsPerSecond = defaultRateLimitPerSecond } - if s.Concurrency == 0 { + if s.Concurrency <= 0 { s.Concurrency = defaultConcurrency } } From 67847d8348f6cbe7912954787808d3efdf3a5379 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 8 Feb 2024 16:18:09 +0000 Subject: [PATCH 5/5] more markdown mangling --- plugins/source/hubspot/docs/overview.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/source/hubspot/docs/overview.md b/plugins/source/hubspot/docs/overview.md index 2882698cd3614d..585a8939929af7 100644 --- a/plugins/source/hubspot/docs/overview.md +++ b/plugins/source/hubspot/docs/overview.md @@ -34,24 +34,24 @@ This is the specs that can be used by the HubSpot source Plugin. - `app_token` (`string`) (optional) (default: `HUBSPOT_APP_TOKEN` environment variable value) The HubSpot App Token to use for authentication. This can also be set with the `HUBSPOT_APP_TOKEN` environment variable. -- `concurrency` (int, optional, default: `1000`): +- `concurrency` (`integer`) (optional) (default: `1000`) A best effort maximum number of Go routines to use. Lower this number to reduce memory usage. -- `max_requests_per_second` (`int`, optional. Default: `5`) +- `max_requests_per_second` (`integer`) (optional) (default: `5`) Rate limit per second for requests done HubSpot API, this will depend on your HubSpot plan (https://developers.hubspot.com/docs/api/usage-details#rate-limits) -- `table_options` (map[string][TableOptions](#table-options) spec, optional. Default: `empty`) +- `table_options` (map[string][TableOptions](#table-options) spec) (optional) (default: `empty`) Table Options for HubSpot entities that will be synced. ### Table Options -- `associations` (`[]string`, optional. Default: empty) +- `associations` (`[]string`) (optional) (default: empty) Additional associations to be retrieved from HubSpot when syncing the table entity -- `properties` (`[]string`, optional. Default: empty) +- `properties` (`[]string`) (optional) (default: empty) Additional properties to be retrieved from HubSpot when syncing the table entity