cloudquery · kodiakhq · Feb 9, 2024 · Feb 8, 2024 · Feb 8, 2024 · Feb 8, 2024
@@ -54,7 +54,7 @@ func New(_ context.Context, logger zerolog.Logger, s spec.Spec) (schema.ClientMe
 		Authorizer: hubspot.NewTokenAuthorizer(authToken),
 		Spec:       s,
 		RateLimiter: rate.NewLimiter(
-			/* r= */ rate.Limit(*s.MaxRequestsPerSecond),
+			/* r= */ rate.Limit(s.MaxRequestsPerSecond),
 			/* b= */ 1,
 		),
 	}, nil

diff --git a/plugins/source/hubspot/client/spec/schema.json b/plugins/source/hubspot/client/spec/schema.json
@@ -1,21 +1,32 @@
 package spec
 
-import _ "embed"
+import (
+	_ "embed"
+	"fmt"
+	"os"
+)
 
 const (
 	defaultConcurrency = 1000
 )
 
 type Spec struct {
+	// In order for CloudQuery to sync resources from your HubSpot setup, you will need to authenticate with your HubSpot account. You will need to create a [HubSpot Private App](https://developers.hubspot.com/docs/api/private-apps), and copy the App Token here.
+	// If not specified `HUBSPOT_APP_TOKEN` environment variable will be used instead.
+	AppToken string `json:"app_token,omitempty" jsonschema:"minLength=1"`
 	// Max number of requests per second to perform against the Hubspot API.
-	MaxRequestsPerSecond *int `yaml:"max_requests_per_second,omitempty" json:"max_requests_per_second,omitempty" jsonschema:"minimum=1,default=5"`
+	MaxRequestsPerSecond int `json:"max_requests_per_second,omitempty" jsonschema:"minimum=1,default=5"`
 	// Key-value map of options for each table. The key is the name of the table. The value is an options object.
-	TableOptions TableOptions `yaml:"table_options,omitempty" json:"table_options,omitempty"`
+	TableOptions TableOptions `json:"table_options,omitempty"`
 	// Concurrency setting for the CloudQuery scheduler.
-	Concurrency int `yaml:"concurrency,omitempty" json:"concurrency,omitempty" jsonschema:"minimum=1,default=1000"`
+	Concurrency int `json:"concurrency,omitempty" jsonschema:"minimum=1,default=1000"`
 }
 
-func (spec *Spec) SetDefaults() {
+func (s *Spec) SetDefaults() {
+	if len(s.AppToken) == 0 {
+		s.AppToken = os.Getenv("HUBSPOT_APP_TOKEN")
+	}
+
 	// https://developers.hubspot.com/docs/api/usage-details#rate-limits
 	// Hubspot, for Pro and Enterprise, accounts, has rate limits of:
 	// - 15 requests / second / private-app
@@ -24,13 +35,20 @@ func (spec *Spec) SetDefaults() {
 	// subscriptions in case cloudquery is run 24/7).
 	var defaultRateLimitPerSecond = 5
 
-	if spec.MaxRequestsPerSecond == nil || *spec.MaxRequestsPerSecond <= 0 {
-		spec.MaxRequestsPerSecond = &defaultRateLimitPerSecond
+	if s.MaxRequestsPerSecond <= 0 {
+		s.MaxRequestsPerSecond = defaultRateLimitPerSecond
 	}
 
-	if spec.Concurrency == 0 {
-		spec.Concurrency = defaultConcurrency
+	if s.Concurrency <= 0 {
+		s.Concurrency = defaultConcurrency
+	}
+}
+
+func (s Spec) Validate() error {
+	if s.AppToken == "" {
+		return fmt.Errorf("app_token is required")
 	}
+	return nil
 }
 
 //go:embed schema.json

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/cloudquery/codegen/jsonschema"
+	"github.com/stretchr/testify/require"
 )
 
 func TestJSONSchema(t *testing.T) {
@@ -13,57 +14,70 @@ func TestJSONSchema(t *testing.T) {
 			Spec: `{}`,
 		},
 		{
-			Name: "null max_requests_per_second",
-			Spec: `{"max_requests_per_second": null}`,
+			Name: "app_token",
+			Spec: `{"app_token": "token"}`,
+		},
+		{
+			Name: "empty app_token",
+			Spec: `{"app_token": ""}`,
+			Err:  true,
 		},
 		{
 			Name: "max_requests_per_second == -1 is invalid",
-			Spec: `{"max_requests_per_second": -1}`,
+			Spec: `{"app_token": "token", "max_requests_per_second": -1}`,
 			Err:  true,
 		},
 		{
 			Name: "max_requests_per_second == 0 is invalid",
-			Spec: `{"max_requests_per_second": 0}`,
+			Spec: `{"app_token": "token", "max_requests_per_second": 0}`,
 			Err:  true,
 		},
 		{
 			Name: "max_requests_per_second == 1 is valid",
-			Spec: `{"max_requests_per_second": 1}`,
+			Spec: `{"app_token": "token", "max_requests_per_second": 1}`,
 		},
 		{
 			Name: "concurrency == -1 is invalid",
-			Spec: `{"concurrency": -1}`,
+			Spec: `{"app_token": "token", "concurrency": -1}`,
 			Err:  true,
 		},
 		{
 			Name: "concurrency == 0 is invalid",
-			Spec: `{"concurrency": 0}`,
+			Spec: `{"app_token": "token", "concurrency": 0}`,
 			Err:  true,
 		},
 		{
 			Name: "concurrency == 1 is valid",
-			Spec: `{"concurrency": 1}`,
+			Spec: `{"app_token": "token", "concurrency": 1}`,
 		},
 		{
 			Name: "table_options == null is valid",
-			Spec: `{"table_options": null}`,
+			Spec: `{"app_token": "token", "table_options": null}`,
 		},
+	})
+}
+
+func TestTableOptionsJSONSchema(t *testing.T) {
+	schema, err := jsonschema.Generate(TableOptions{})
+
+	require.NoError(t, err)
+	jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
 		{
-			Name: "table_options == {} is valid",
-			Spec: `{"table_options": {}}`,
+			Name: "empty table options",
+			Spec: `{}`,
 		},
 		{
-			Name: "spec with table options = null is invalid",
-			Spec: `{"table_options": {"hubspot_crm_companies": null}}`,
+			Name: "hubspot_crm_companies = null is invalid",
+			Spec: `{"hubspot_crm_companies": null}`,
 		},
 		{
-			Name: "spec with table options = [] is invalid",
-			Spec: `{"table_options": {"hubspot_crm_companies": []}}`,
+			Name: "hubspot_crm_companiess = [] is invalid",
+			Spec: `{"hubspot_crm_companies": []}`,
 			Err:  true,
 		},
 		{
-			Name: "spec with table options = {} is valid",
-			Spec: `{"table_options": {"hubspot_crm_companies": {}}}`,
+			Name: "hubspot_crm_companies = {} is valid",
+			Spec: `{"hubspot_crm_companies": {}}`,
 		},
 	})
 }
@@ -5,9 +5,9 @@ type TableOptions map[string]*TableOptionsSpec
 // Table options spec.
 type TableOptionsSpec struct {
 	// List of properties to sync. If empty, everything is synced.
-	Properties []string `yaml:"properties,omitempty" json:"properties,omitempty" jsonschema:"minLength=1"`
+	Properties []string `json:"properties,omitempty" jsonschema:"minLength=1"`
 	// List of associations to sync. If empty, everything is synced.
-	Associations []string `yaml:"associations,omitempty" json:"associations,omitempty" jsonschema:"minLength=1"`
+	Associations []string `json:"associations,omitempty" jsonschema:"minLength=1"`
 }
 
 func (ts TableOptions) ForTable(name string) *TableOptionsSpec {

@@ -1,5 +1,6 @@
-In Order for CloudQuery to sync resources from your HubSpot setup, you will need to authenticate with your HubSpot account. You will need to create a [HubSpot Private App](https://developers.hubspot.com/docs/api/private-apps), and export the App Token in `HUBSPOT_APP_TOKEN` environment variable.
+In Order for CloudQuery to sync resources from your HubSpot setup, you will need to authenticate with your HubSpot account. You will need to create a [HubSpot Private App](https://developers.hubspot.com/docs/api/private-apps), and copy the App Token to the spec.
+If not specified `HUBSPOT_APP_TOKEN` environment variable will be used instead.
 
 ```bash copy
-export HUBSPOT_APP_TOKEN=<your_app_token>
+export HUBSPOT_APP_TOKEN=<your_app_token> # optional, if not using spec configuration
 ```
@@ -31,23 +31,27 @@ The following example sets up the HubSpot plugin, and connects it to a postgresq
 
 This is the specs that can be used by the HubSpot source Plugin.
 
-- `concurrency` (int, optional, default: `1000`):
+- `app_token` (`string`)  (optional) (default: `HUBSPOT_APP_TOKEN` environment variable value)
+  The HubSpot App Token to use for authentication. This can also be set with the `HUBSPOT_APP_TOKEN` environment variable. 
+
+- `concurrency` (`integer`) (optional) (default: `1000`)
+
   A best effort maximum number of Go routines to use. Lower this number to reduce memory usage.
 
-- `max_requests_per_second` (`int`, optional. Default: `5`)
+- `max_requests_per_second` (`integer`) (optional) (default: `5`)
 
   Rate limit per second for requests done HubSpot API, this will depend on your HubSpot plan (https://developers.hubspot.com/docs/api/usage-details#rate-limits)
 
-- `table_options` (map[string][TableOptions](#table-options) spec, optional. Default: `empty`)
+- `table_options` (map[string][TableOptions](#table-options) spec) (optional) (default: `empty`)
 
   Table Options for HubSpot entities that will be synced.
 
 ### Table Options
 
-- `associations` (`[]string`, optional. Default: empty)
+- `associations` (`[]string`) (optional) (default: empty)
 
   Additional associations to be retrieved from HubSpot when syncing the table entity
 
-- `properties` (`[]string`, optional. Default: empty)
+- `properties` (`[]string`) (optional) (default: empty)
 
   Additional properties to be retrieved from HubSpot when syncing the table entity
@@ -65,6 +65,9 @@ func newClient(ctx context.Context, logger zerolog.Logger, specBytes []byte, opt
 		return nil, err
 	}
 	s.SetDefaults()
+	if err := s.Validate(); err != nil {
+		return nil, err
+	}
 	syncClient, err := client.New(ctx, logger, *s)
 	if err != nil {
 		return nil, err