From fef3a908303b9d15253f090b19e2459e056a2286 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 10 Jan 2022 17:42:56 +0000 Subject: [PATCH 01/45] Detect provider version, don't run table creator on V4 --- go.mod | 2 ++ pkg/client/client.go | 49 +++++++++++++++++++++++++++++++++++++------- pkg/plugin/plugin.go | 8 +++++++- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 73ec0596f43ad0..b9b55cd073042d 100644 --- a/go.mod +++ b/go.mod @@ -142,3 +142,5 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) + +replace github.com/cloudquery/cq-provider-sdk v0.6.1 => ../cq-provider-sdk diff --git a/pkg/client/client.go b/pkg/client/client.go index 7113e63fef8766..7657c4fde4fe6f 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -201,7 +201,7 @@ type FetchDoneResult struct { // TableCreator creates tables based on schema received from providers type TableCreator interface { - CreateTable(ctx context.Context, conn *pgxpool.Conn, t *schema.Table, p *schema.Table) error + CreateTable(ctx context.Context, conn *pgxpool.Conn, t, p *schema.Table) error } type TableRemover interface { @@ -244,7 +244,7 @@ type Client struct { ModuleManager module.Manager // ModuleManager manages all modules lifecycle PolicyManager policy.Manager - // TableCreator defines how table are created in the database + // TableCreator defines how table are created in the database, only for plugin protocol < 4 TableCreator TableCreator // HistoryConfig defines configuration for CloudQuery history mode HistoryCfg *history.Config @@ -581,7 +581,13 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes return response, nil } -func (c *Client) GetProviderSchema(ctx context.Context, providerName string) (*cqproto.GetProviderSchemaResponse, error) { +type ProviderSchema struct { + *cqproto.GetProviderSchemaResponse + + ProtocolVersion int +} + +func (c *Client) GetProviderSchema(ctx context.Context, providerName string) (*ProviderSchema, error) { providerPlugin, err := c.Manager.CreatePlugin(providerName, "", nil) if err != nil { c.Logger.Error("failed to create provider plugin", "provider", providerName, "error", err) @@ -596,7 +602,16 @@ func (c *Client) GetProviderSchema(ctx context.Context, providerName string) (*c c.Logger.Warn("failed to kill provider", "provider", providerName) } }() - return providerPlugin.Provider().GetProviderSchema(ctx, &cqproto.GetProviderSchemaRequest{}) + + schema, err := providerPlugin.Provider().GetProviderSchema(ctx, &cqproto.GetProviderSchemaRequest{}) + if err != nil { + return nil, err + } + + return &ProviderSchema{ + GetProviderSchemaResponse: schema, + ProtocolVersion: providerPlugin.ProtocolVersion(), + }, nil } func (c *Client) GetProviderConfiguration(ctx context.Context, providerName string) (*cqproto.GetProviderConfigResponse, error) { @@ -632,17 +647,37 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( return err } defer conn.Release() - for name, t := range s.ResourceTables { - c.Logger.Debug("creating tables for resource for provider", "resource_name", name, "provider", s.Name, "version", s.Version) - if err := c.TableCreator.CreateTable(ctx, conn, t, nil); err != nil { + + runTableCreator := func() error { + for name, t := range s.ResourceTables { + c.Logger.Debug("creating tables for resource for provider", "resource_name", name, "provider", s.Name, "version", s.Version) + if err := c.TableCreator.CreateTable(ctx, conn, t, nil); err != nil { + return err + } + } + return nil + } + + switch s.ProtocolVersion { + case cqproto.V3: + if err := runTableCreator(); err != nil { return err } + + default: // Protocol 4 onwards + if s.Migrations == nil { // Keep the table creator if we don't have any migrations defined for this provider + if err := runTableCreator(); err != nil { + return err + } + } + } if s.Migrations == nil { c.Logger.Debug("provider doesn't support migrations", "provider", providerName) return nil } + // create migration table and set it to version based on latest create table m, cfg, err := c.buildProviderMigrator(s.Migrations, providerName) if err != nil { diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index db3666ed0c8195..98494ac3602b18 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -27,6 +27,7 @@ var pluginMap = map[string]plugin.Plugin{ type Plugin interface { Name() string Version() string + ProtocolVersion() int Provider() cqproto.CQProvider Close() } @@ -46,7 +47,8 @@ func newRemotePlugin(details *registry.ProviderDetails, alias string, env []stri client := plugin.NewClient(&plugin.ClientConfig{ HandshakeConfig: serve.Handshake, VersionedPlugins: map[int]plugin.PluginSet{ - 3: pluginMap, + cqproto.V4: pluginMap, + cqproto.V3: pluginMap, }, Managed: true, Cmd: cmd, @@ -89,6 +91,8 @@ func (m managedPlugin) Name() string { return m.name } func (m managedPlugin) Version() string { return m.version } +func (m managedPlugin) ProtocolVersion() int { return m.client.NegotiatedVersion() } + func (m managedPlugin) Provider() cqproto.CQProvider { return m.provider } func (m managedPlugin) Close() { @@ -141,6 +145,8 @@ func (m unmanagedPlugin) Name() string { return m.name } func (m unmanagedPlugin) Version() string { return Unmanaged } +func (m unmanagedPlugin) ProtocolVersion() int { return m.client.NegotiatedVersion() } + func (m unmanagedPlugin) Provider() cqproto.CQProvider { return m.provider } func (m unmanagedPlugin) Close() {} From 448fa6adce40dade12befecae660505eeb0da343 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 10 Jan 2022 17:53:04 +0000 Subject: [PATCH 02/45] Clean up --- pkg/client/client.go | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 7657c4fde4fe6f..19025695a28756 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -25,6 +25,7 @@ import ( "github.com/cloudquery/cq-provider-sdk/cqproto" "github.com/cloudquery/cq-provider-sdk/helpers" "github.com/cloudquery/cq-provider-sdk/provider" + "github.com/cloudquery/cq-provider-sdk/provider/migrations" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/cloudquery/cq-provider-sdk/provider/schema/diag" "github.com/getsentry/sentry-go" @@ -648,29 +649,14 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( } defer conn.Release() - runTableCreator := func() error { + if s.ProtocolVersion <= cqproto.V3 || s.Migrations == nil { + // Keep the table creator if we don't have any migrations defined for this provider, or if we're running an older protocol for name, t := range s.ResourceTables { c.Logger.Debug("creating tables for resource for provider", "resource_name", name, "provider", s.Name, "version", s.Version) if err := c.TableCreator.CreateTable(ctx, conn, t, nil); err != nil { return err } } - return nil - } - - switch s.ProtocolVersion { - case cqproto.V3: - if err := runTableCreator(); err != nil { - return err - } - - default: // Protocol 4 onwards - if s.Migrations == nil { // Keep the table creator if we don't have any migrations defined for this provider - if err := runTableCreator(); err != nil { - return err - } - } - } if s.Migrations == nil { @@ -1008,7 +994,7 @@ func (c *Client) setupTableCreator(ctx context.Context) error { } if c.HistoryCfg == nil { c.Logger.Debug("using default table creator without history mode enabled.") - c.TableCreator = provider.NewTableCreator(c.Logger) + c.TableCreator = migrations.NewTableCreator(c.Logger) return nil } creator, err := history.NewHistoryTableCreator(c.HistoryCfg, c.Logger) From 8682de7835e5691f2d396da42024d62517f49e2f Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 10 Jan 2022 18:45:55 +0000 Subject: [PATCH 03/45] Use Vunmanaged for unmanaged plugins --- pkg/client/client.go | 2 +- pkg/plugin/plugin.go | 2 +- pkg/ui/console/client.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 19025695a28756..3a1bb6c20ec8ab 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -649,7 +649,7 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( } defer conn.Release() - if s.ProtocolVersion <= cqproto.V3 || s.Migrations == nil { + if s.ProtocolVersion != cqproto.Vunmanaged && (s.ProtocolVersion <= cqproto.V3 || s.Migrations == nil) { // Keep the table creator if we don't have any migrations defined for this provider, or if we're running an older protocol for name, t := range s.ResourceTables { c.Logger.Debug("creating tables for resource for provider", "resource_name", name, "provider", s.Name, "version", s.Version) diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 98494ac3602b18..e7f980046d2c33 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -145,7 +145,7 @@ func (m unmanagedPlugin) Name() string { return m.name } func (m unmanagedPlugin) Version() string { return Unmanaged } -func (m unmanagedPlugin) ProtocolVersion() int { return m.client.NegotiatedVersion() } +func (m unmanagedPlugin) ProtocolVersion() int { return cqproto.Vunmanaged } func (m unmanagedPlugin) Provider() cqproto.CQProvider { return m.provider } diff --git a/pkg/ui/console/client.go b/pkg/ui/console/client.go index da12f271f884e3..3325a3eabb8cb5 100644 --- a/pkg/ui/console/client.go +++ b/pkg/ui/console/client.go @@ -480,7 +480,7 @@ func (c Client) getModuleProviders(ctx context.Context) ([]*cqproto.GetProviderS s.Version = deets.Version } } - list[i] = s + list[i] = s.GetProviderSchemaResponse } return list, nil From 54859b58068b4de45b400825aa3bcb3b91b2f394 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 10 Jan 2022 23:48:45 +0000 Subject: [PATCH 04/45] Migrations hook thing --- pkg/client/client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/client/client.go b/pkg/client/client.go index 3a1bb6c20ec8ab..10299e3f838d2f 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -247,6 +247,8 @@ type Client struct { PolicyManager policy.Manager // TableCreator defines how table are created in the database, only for plugin protocol < 4 TableCreator TableCreator + // MigrationHook is called after migrations are finished in the database when they are being set up, or before they are teared down + MigrationHook provider.MigrationHook // HistoryConfig defines configuration for CloudQuery history mode HistoryCfg *history.Config // pool is a list of connection that are used for policy/query execution @@ -943,6 +945,7 @@ func (c *Client) buildProviderMigrator(migrations map[string][]byte, providerNam if err != nil { return nil, nil, err } + m.SetHook(c.MigrationHook) return m, providerConfig, err } From ed1f0079de9311f9456cbe1cde85ecca42fd4fd1 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 13 Jan 2022 19:46:48 +0000 Subject: [PATCH 05/45] Improvements --- pkg/client/client.go | 90 ++++----- pkg/client/client_test.go | 2 +- pkg/client/database/database.go | 48 +++++ .../postgres/executor.go} | 50 ++--- .../postgres/executor_test.go} | 2 +- .../{history => database/timescale}/table.go | 91 +++------ .../timescale}/table_test.go | 10 +- pkg/client/database/timescale/timescale.go | 182 ++++++++++++++++++ .../database/timescale/timescale_test.go | 43 +++++ pkg/client/history/history.go | 104 ---------- 10 files changed, 372 insertions(+), 250 deletions(-) create mode 100644 pkg/client/database/database.go rename pkg/client/{database.go => database/postgres/executor.go} (70%) rename pkg/client/{database_test.go => database/postgres/executor_test.go} (99%) rename pkg/client/{history => database/timescale}/table.go (57%) rename pkg/client/{history => database/timescale}/table_test.go (91%) create mode 100644 pkg/client/database/timescale/timescale.go create mode 100644 pkg/client/database/timescale/timescale_test.go diff --git a/pkg/client/client.go b/pkg/client/client.go index 10299e3f838d2f..6dd104001522d1 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -14,6 +14,8 @@ import ( "github.com/cloudquery/cloudquery/internal/logging" "github.com/cloudquery/cloudquery/internal/telemetry" + + "github.com/cloudquery/cloudquery/pkg/client/database" "github.com/cloudquery/cloudquery/pkg/client/history" "github.com/cloudquery/cloudquery/pkg/config" "github.com/cloudquery/cloudquery/pkg/module" @@ -23,9 +25,11 @@ import ( "github.com/cloudquery/cloudquery/pkg/policy" "github.com/cloudquery/cloudquery/pkg/ui" "github.com/cloudquery/cq-provider-sdk/cqproto" + sdkdb "github.com/cloudquery/cq-provider-sdk/database" "github.com/cloudquery/cq-provider-sdk/helpers" + "github.com/cloudquery/cq-provider-sdk/migration" + "github.com/cloudquery/cq-provider-sdk/migration/migrator" "github.com/cloudquery/cq-provider-sdk/provider" - "github.com/cloudquery/cq-provider-sdk/provider/migrations" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/cloudquery/cq-provider-sdk/provider/schema/diag" "github.com/getsentry/sentry-go" @@ -59,7 +63,7 @@ type FetchRequest struct { UpdateCallback FetchUpdateCallback // Providers list of providers to call for fetching Providers []*config.Provider - // Optional: Adds extra fields to the provider, this is used for testing purposes. + // Optional: Adds extra fields to the provider, this is used for history mode and testing purposes. ExtraFields map[string]interface{} } @@ -245,14 +249,15 @@ type Client struct { ModuleManager module.Manager // ModuleManager manages all modules lifecycle PolicyManager policy.Manager - // TableCreator defines how table are created in the database, only for plugin protocol < 4 + // TableCreator defines how tables are created in the database, only for plugin protocol < 4 TableCreator TableCreator - // MigrationHook is called after migrations are finished in the database when they are being set up, or before they are teared down - MigrationHook provider.MigrationHook // HistoryConfig defines configuration for CloudQuery history mode HistoryCfg *history.Config // pool is a list of connection that are used for policy/query execution pool *pgxpool.Pool + + db *sdkdb.DB + dialectExecutor database.DialectExecutor } func New(ctx context.Context, options ...Option) (*Client, error) { @@ -280,23 +285,27 @@ func New(ctx context.Context, options ...Option) (*Client, error) { if c.DSN == "" { c.Logger.Warn("missing DSN, some commands won't work") } else { - c.pool, err = CreateDatabase(ctx, c.DSN) + c.db, err = sdkdb.New(ctx, c.Logger, c.DSN) if err != nil { return nil, err } - if err := ValidatePostgresVersion(ctx, c.pool, MinPostgresVersion); err != nil { - c.Logger.Warn("postgres validation warning", "err", err) + _, c.dialectExecutor, err = database.GetExecutor(c.Logger, c.DSN) + if err != nil { + return nil, fmt.Errorf("getExecutor: %w", err) } - } - // migrate cloudquery core tables to latest version - if c.DSN != "" { + + // migrate cloudquery core tables to latest version if err := c.MigrateCore(ctx); err != nil { return nil, fmt.Errorf("failed to migrate cloudquery_core tables: %w", err) } - } - if err := c.setupTableCreator(ctx); err != nil { - return nil, err + if ok, err := c.dialectExecutor.Validate(ctx); err != nil { + return nil, fmt.Errorf("validate: %w", err) + } else if !ok { + c.Logger.Warn("postgres validation warning") + } + + c.TableCreator = migration.NewTableCreator(c.Logger, schema.GetDialect(c.db.DialectType())) } c.initModules() @@ -651,12 +660,13 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( } defer conn.Release() - if s.ProtocolVersion != cqproto.Vunmanaged && (s.ProtocolVersion <= cqproto.V3 || s.Migrations == nil) { + if (s.ProtocolVersion != cqproto.Vunmanaged && s.ProtocolVersion <= cqproto.V3) || s.Migrations == nil { + // TODO Throw error about upgrading // Keep the table creator if we don't have any migrations defined for this provider, or if we're running an older protocol for name, t := range s.ResourceTables { c.Logger.Debug("creating tables for resource for provider", "resource_name", name, "provider", s.Name, "version", s.Version) if err := c.TableCreator.CreateTable(ctx, conn, t, nil); err != nil { - return err + return fmt.Errorf("CreateTable(%s) failed: %w", t.Name, err) } } } @@ -667,7 +677,7 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( } // create migration table and set it to version based on latest create table - m, cfg, err := c.buildProviderMigrator(s.Migrations, providerName) + m, cfg, err := c.buildProviderMigrator(ctx, s.Migrations, providerName) if err != nil { return err } @@ -707,7 +717,7 @@ func (c *Client) UpgradeProvider(ctx context.Context, providerName string) (retE if s.Migrations == nil { return ErrMigrationsNotSupported } - m, cfg, err := c.buildProviderMigrator(s.Migrations, providerName) + m, cfg, err := c.buildProviderMigrator(ctx, s.Migrations, providerName) if err != nil { return err } @@ -748,7 +758,7 @@ func (c *Client) DowngradeProvider(ctx context.Context, providerName string) (re if s.Migrations == nil { return fmt.Errorf("provider doesn't support migrations") } - m, cfg, err := c.buildProviderMigrator(s.Migrations, providerName) + m, cfg, err := c.buildProviderMigrator(ctx, s.Migrations, providerName) if err != nil { return err } @@ -771,7 +781,7 @@ func (c *Client) DropProvider(ctx context.Context, providerName string) (retErr if err != nil { return err } - m, cfg, err := c.buildProviderMigrator(s.Migrations, providerName) + m, cfg, err := c.buildProviderMigrator(ctx, s.Migrations, providerName) if err != nil { return err } @@ -910,7 +920,7 @@ func (c *Client) SetProviderVersion(ctx context.Context, providerName, version s if s.Migrations == nil { return fmt.Errorf("provider doesn't support migrations") } - m, cfg, err := c.buildProviderMigrator(s.Migrations, providerName) + m, cfg, err := c.buildProviderMigrator(ctx, s.Migrations, providerName) if err != nil { return err } @@ -923,7 +933,7 @@ func (c *Client) initModules() { c.ModuleManager.RegisterModule(drift.New(c.Logger)) } -func (c *Client) buildProviderMigrator(migrations map[string][]byte, providerName string) (*provider.Migrator, *config.RequiredProvider, error) { +func (c *Client) buildProviderMigrator(ctx context.Context, migrations map[string][]byte, providerName string) (*migrator.Migrator, *config.RequiredProvider, error) { providerConfig, err := c.getProviderConfig(providerName) if err != nil { return nil, nil, err @@ -933,19 +943,15 @@ func (c *Client) buildProviderMigrator(migrations map[string][]byte, providerNam return nil, nil, err } - dsn := c.DSN - if c.HistoryCfg != nil { - dsn, err = parseDSN(c.DSN, "history") - if err != nil { - return nil, nil, err - } + dsn, err := c.dialectExecutor.Setup(ctx) + if err != nil { + return nil, nil, fmt.Errorf("dialectExecutor.Setup: %w", err) } - m, err := provider.NewMigrator(c.Logger, migrations, dsn, fmt.Sprintf("%s_%s", org, name)) + m, err := migrator.New(c.Logger, migrations, dsn, fmt.Sprintf("%s_%s", org, name), c.dialectExecutor.Finalize) if err != nil { return nil, nil, err } - m.SetHook(c.MigrationHook) return m, providerConfig, err } @@ -990,30 +996,6 @@ func (c *Client) getProviderConfig(providerName string) (*config.RequiredProvide return providerConfig, nil } -func (c *Client) setupTableCreator(ctx context.Context) error { - if c.TableCreator != nil { - c.Logger.Debug("table creator already set") - return nil - } - if c.HistoryCfg == nil { - c.Logger.Debug("using default table creator without history mode enabled.") - c.TableCreator = migrations.NewTableCreator(c.Logger) - return nil - } - creator, err := history.NewHistoryTableCreator(c.HistoryCfg, c.Logger) - if err != nil { - return err - } - // set history table creator - c.TableCreator = creator - conn, err := c.pool.Acquire(ctx) - if err != nil { - return fmt.Errorf("failed to acquire connection for history setup: %w", err) - } - defer conn.Release() - return history.SetupHistory(ctx, conn) -} - func parseDSN(dsn, searchPath string) (string, error) { url, err := helpers.ParseConnectionString(dsn) if err != nil { diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 968d07351b76e5..513cfda84b70fe 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -398,7 +398,7 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { assert.Nil(t, err) // insert dummy migration files like test provider just for version number return - m, _, err := c.buildProviderMigrator(map[string][]byte{ + m, _, err := c.buildProviderMigrator(ctx, map[string][]byte{ "1_v0.0.1.up.sql": []byte(""), "1_v0.0.1.down.sql": []byte(""), "2_v0.0.2.up.sql": []byte(""), diff --git a/pkg/client/database/database.go b/pkg/client/database/database.go new file mode 100644 index 00000000000000..0b63c8a060649f --- /dev/null +++ b/pkg/client/database/database.go @@ -0,0 +1,48 @@ +package database + +import ( + "context" + "fmt" + + "github.com/cloudquery/cloudquery/pkg/client/database/postgres" + "github.com/cloudquery/cloudquery/pkg/client/database/timescale" + sdkdb "github.com/cloudquery/cq-provider-sdk/database" + "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/hashicorp/go-hclog" +) + +type DialectExecutor interface { + // Setup is called on the dialect on initialization, returns the DSN (modified if necessary) to use for migrations + Setup(context.Context) (string, error) + + // Validate is called before startup to check that the dialect can execute properly + Validate(context.Context) (bool, error) + + // Finalize is called after migrations and upgrades are run + Finalize(context.Context) error +} + +var ( + _ DialectExecutor = (*postgres.Executor)(nil) + _ DialectExecutor = (*timescale.Executor)(nil) +) + +func GetExecutor(logger hclog.Logger, dsn string) (schema.DialectType, DialectExecutor, error) { + if dsn == "" { + return schema.Postgres, nil, fmt.Errorf("missing DSN") + } + + dType, dsn, err := sdkdb.DSNtoDialect(dsn) + if err != nil { + return dType, nil, err + } + + switch dType { + case schema.Postgres: + return dType, postgres.New(logger, dsn), nil + case schema.TSDB: + return dType, timescale.New(logger, dsn), nil + default: + return dType, nil, fmt.Errorf("unhandled dialect type") + } +} diff --git a/pkg/client/database.go b/pkg/client/database/postgres/executor.go similarity index 70% rename from pkg/client/database.go rename to pkg/client/database/postgres/executor.go index 4f23a235cb81c3..d8a9827c3c8ee0 100644 --- a/pkg/client/database.go +++ b/pkg/client/database/postgres/executor.go @@ -1,44 +1,50 @@ -package client +package postgres import ( "context" "fmt" "strings" + sdkpg "github.com/cloudquery/cq-provider-sdk/database/postgres" + "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-version" - "github.com/jackc/pgtype" "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/pgxpool" ) +type Executor struct { + logger hclog.Logger + dsn string +} + var MinPostgresVersion = version.Must(version.NewVersion("11.0")) -func CreateDatabase(ctx context.Context, dsn string) (*pgxpool.Pool, error) { - if dsn == "" { - return nil, fmt.Errorf("missing DSN") - } - poolCfg, err := pgxpool.ParseConfig(dsn) - if err != nil { - return nil, err +func New(logger hclog.Logger, dsn string) Executor { + return Executor{ + logger: logger, + dsn: dsn, } +} - poolCfg.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error { - UUIDType := pgtype.DataType{ - Value: &UUID{}, - Name: "uuid", - OID: pgtype.UUIDOID, - } +func (e Executor) Setup(ctx context.Context) (string, error) { + return e.dsn, nil +} - conn.ConnInfo().RegisterDataType(UUIDType) - return nil +func (e Executor) Validate(ctx context.Context) (bool, error) { + pool, err := sdkpg.Connect(ctx, e.dsn) + if err != nil { + return false, err } - poolCfg.LazyConnect = true - pool, err := pgxpool.ConnectConfig(ctx, poolCfg) - if err != nil { - return nil, err + if err := ValidatePostgresVersion(ctx, pool, MinPostgresVersion); err != nil { + return false, err } - return pool, err + + return true, nil +} + +func (e Executor) Finalize(ctx context.Context) error { + return nil } // queryRower helps with unit tests diff --git a/pkg/client/database_test.go b/pkg/client/database/postgres/executor_test.go similarity index 99% rename from pkg/client/database_test.go rename to pkg/client/database/postgres/executor_test.go index e9cb46df87100f..abf39249ac5121 100644 --- a/pkg/client/database_test.go +++ b/pkg/client/database/postgres/executor_test.go @@ -1,4 +1,4 @@ -package client +package postgres import ( "context" diff --git a/pkg/client/history/table.go b/pkg/client/database/timescale/table.go similarity index 57% rename from pkg/client/history/table.go rename to pkg/client/database/timescale/table.go index 06cc629a7ecbe6..629113de289edb 100644 --- a/pkg/client/history/table.go +++ b/pkg/client/database/timescale/table.go @@ -1,49 +1,47 @@ -package history +package timescale import ( "context" "fmt" - "strconv" "strings" + "github.com/cloudquery/cloudquery/pkg/client/history" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/georgysavva/scany/pgxscan" "github.com/hashicorp/go-hclog" - "github.com/huandu/go-sqlbuilder" "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/pgxpool" ) -type CreateHyperTableResult struct { - HypertableId int `db:"hypertable_id"` - SchemaName string `db:"schema_name"` - TableName string `db:"table_name"` - Created bool `db:"created"` -} +/* +setup -> (creates history schema, history functions) +create (migrate) -> creates all tables in history schema (we set schema like we do now) +finalizes -> fix constraints / pks and run create_hypertable + retention policy and all tables -type TableCreator struct { - log hclog.Logger - cfg *Config -} ++ recreate views in public schema -func NewHistoryTableCreator(cfg *Config, l hclog.Logger) (*TableCreator, error) { - return &TableCreator{ - l, - cfg, - }, nil +need to fix PKs and constraints (remove FKs, add cq_fetch_date to UNIQUE) BEFORE we can do create_hypertable +*/ + +type DDLManager struct { + log hclog.Logger + cfg *history.Config + dialect schema.Dialect } -func (h TableCreator) CreateTable(ctx context.Context, conn *pgxpool.Conn, t, p *schema.Table) error { - sql, err := h.buildTableSQL(t, p) - if err != nil { - return err +func NewDDLManager(cfg *history.Config, l hclog.Logger, dt schema.DialectType) (*DDLManager, error) { + if dt != schema.TSDB { + return nil, fmt.Errorf("history is only supported on timescaledb") } - h.log.Debug("creating table if not exists", "table", t.Name) - if _, err := conn.Exec(ctx, sql); err != nil { - return fmt.Errorf("failed to create table: %w", err) - } + return &DDLManager{ + log: l, + cfg: cfg, + dialect: schema.GetDialect(dt), + }, nil +} +func (h DDLManager) CreateTable(ctx context.Context, conn *pgxpool.Conn, t, p *schema.Table) error { if err := h.createHyperTable(ctx, t, p, conn); err != nil { return fmt.Errorf("failed to create hypertable for table: %s: %w", t.Name, err) } @@ -84,8 +82,8 @@ func (h TableCreator) CreateTable(ctx context.Context, conn *pgxpool.Conn, t, p return nil } -func (h TableCreator) createHyperTable(ctx context.Context, t, p *schema.Table, conn *pgxpool.Conn) error { - var hyperTable CreateHyperTableResult +func (h DDLManager) createHyperTable(ctx context.Context, t, p *schema.Table, conn *pgxpool.Conn) error { + var hyperTable createHyperTableResult tName := fmt.Sprintf(`"history"."%s"`, t.Name) if err := pgxscan.Get(ctx, conn, &hyperTable, fmt.Sprintf(createHyperTable, h.cfg.TimeInterval), tName); err != nil { return fmt.Errorf("failed to create hypertable: %w", err) @@ -101,7 +99,7 @@ func (h TableCreator) createHyperTable(ctx context.Context, t, p *schema.Table, return nil } -func (h TableCreator) buildCascadeTrigger(ctx context.Context, conn *pgxpool.Conn, t, p *schema.Table) error { +func (h DDLManager) buildCascadeTrigger(ctx context.Context, conn *pgxpool.Conn, t, p *schema.Table) error { c := h.findParentIdColumn(t) if c == nil { return fmt.Errorf("failed to find parent cq id column for %s", t.Name) @@ -115,7 +113,7 @@ func (h TableCreator) buildCascadeTrigger(ctx context.Context, conn *pgxpool.Con return nil } -func (h TableCreator) findParentIdColumn(t *schema.Table) *schema.Column { +func (h DDLManager) findParentIdColumn(t *schema.Table) *schema.Column { for _, c := range t.Columns { if c.Meta().Resolver != nil && c.Meta().Resolver.Name == "ParentIdResolver" { return &c @@ -130,36 +128,3 @@ func (h TableCreator) findParentIdColumn(t *schema.Table) *schema.Column { return nil } - -func (h TableCreator) buildTableSQL(table, _ *schema.Table) (string, error) { - // Build SQL to create a table. - ctb := sqlbuilder.CreateTable(fmt.Sprintf("history.%s", table.Name)).IfNotExists() - var uniques []string - for _, c := range schema.GetDefaultSDKColumns() { - ctb.Define(c.Name, schema.GetPgTypeFromType(c.Type)) - if c.CreationOptions.Unique { - uniques = append(uniques, c.Name) - } - } - ctb.Define("cq_fetch_date", schema.GetPgTypeFromType(schema.TypeTimestamp)) - h.buildColumns(ctb, table.Columns) - for _, s := range uniques { - ctb.Define(fmt.Sprintf("UNIQUE (cq_fetch_date, %s)", s)) - } - allKeys := append([]string{"cq_fetch_date"}, table.PrimaryKeys()...) - ctb.Define(fmt.Sprintf("constraint %s_pk primary key(%s)", schema.TruncateTableConstraint(table.Name), strings.Join(allKeys, ","))) - - sql, _ := ctb.BuildWithFlavor(sqlbuilder.PostgreSQL) - h.log.Trace("creating table if not exists", "table", table.Name) - return sql, nil -} - -func (h TableCreator) buildColumns(ctb *sqlbuilder.CreateTableBuilder, cc []schema.Column) { - for _, c := range cc { - defs := []string{strconv.Quote(c.Name), schema.GetPgTypeFromType(c.Type)} - if c.CreationOptions.Unique { - defs = []string{strconv.Quote(c.Name), schema.GetPgTypeFromType(c.Type), "unique"} - } - ctb.Define(defs...) - } -} diff --git a/pkg/client/history/table_test.go b/pkg/client/database/timescale/table_test.go similarity index 91% rename from pkg/client/history/table_test.go rename to pkg/client/database/timescale/table_test.go index 87adc2ac204c77..70d6f4629dd44c 100644 --- a/pkg/client/history/table_test.go +++ b/pkg/client/database/timescale/table_test.go @@ -1,7 +1,7 @@ //go:build history // +build history -package history_test +package timescale_test import ( "context" @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/cloudquery/cloudquery/pkg/client" "github.com/cloudquery/cloudquery/pkg/client/history" + "github.com/cloudquery/cq-provider-sdk/database/postgres" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/assert" @@ -55,7 +55,7 @@ var testTable = &schema.Table{ } func TestHistory_SetupHistory(t *testing.T) { - pool, err := client.CreateDatabase(context.Background(), testDBConnection) + pool, err := postgres.Connect(context.Background(), testDBConnection) assert.NoError(t, err) defer pool.Close() conn, err := pool.Acquire(context.Background()) @@ -65,14 +65,14 @@ func TestHistory_SetupHistory(t *testing.T) { } func TestHistoryTableCreator_CreateTables(t *testing.T) { - m, err := history.NewHistoryTableCreator(&history.Config{Retention: 1, + m, err := NewDDLManager(&history.Config{Retention: 1, TimeInterval: 1, TimeTruncation: 24, }, hclog.L()) assert.NoError(t, err) assert.NotNil(t, m) - pool, err := client.CreateDatabase(context.Background(), testDBConnection) + pool, err := postgres.Connect(context.Background(), testDBConnection) assert.NoError(t, err) defer pool.Close() conn, err := pool.Acquire(context.Background()) diff --git a/pkg/client/database/timescale/timescale.go b/pkg/client/database/timescale/timescale.go new file mode 100644 index 00000000000000..172a93e42fb23b --- /dev/null +++ b/pkg/client/database/timescale/timescale.go @@ -0,0 +1,182 @@ +package timescale + +import ( + "context" + "fmt" + + "github.com/cloudquery/cloudquery/pkg/client/database/postgres" + pgsdk "github.com/cloudquery/cq-provider-sdk/database/postgres" + "github.com/cloudquery/cq-provider-sdk/helpers" + "github.com/georgysavva/scany/pgxscan" + "github.com/hashicorp/go-hclog" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" +) + +const ( + createHyperTable = `SELECT * FROM create_hypertable($1, 'cq_fetch_date', chunk_time_interval => INTERVAL '%d day', if_not_exists => true);` + dataRetentionPolicy = `SELECT add_retention_policy($1, INTERVAL '%d day', if_not_exists => true);` + + dropTableView = `DROP VIEW IF EXISTS "%[1]s"` + createTableView = `CREATE VIEW "%[1]s" AS SELECT * FROM history."%[1]s" WHERE cq_fetch_date = find_latest('history', '%[1]s')` + + historySchema = "history" + viewSchema = "public" +) + +type Executor struct { + logger hclog.Logger + dsn string +} + +func New(logger hclog.Logger, dsn string) Executor { + return Executor{ + logger: logger, + dsn: dsn, + } +} + +// Setup sets all required history functions and validation checks that it can run cleanly. +func (e Executor) Setup(ctx context.Context) (string, error) { + pool, err := pgsdk.Connect(ctx, e.dsn) + if err != nil { + return e.dsn, err + } + defer pool.Close() + conn, err := pool.Acquire(ctx) + if err != nil { + return e.dsn, err + } + defer conn.Release() + + if err := addHistoryFunctions(ctx, conn); err != nil { + return e.dsn, fmt.Errorf("failed to create history functions: %w", err) + } + + return setDsnElement(e.dsn, map[string]string{"search_path": historySchema}), nil +} + +func (e Executor) Validate(ctx context.Context) (bool, error) { + const ( + validateTimescaleInstalled = `SELECT EXISTS(SELECT 1 FROM pg_extension where extname = 'timescaledb')` + ) + + pool, err := pgsdk.Connect(ctx, e.dsn) + if err != nil { + return false, err + } + defer pool.Close() + + if err := postgres.ValidatePostgresVersion(ctx, pool, postgres.MinPostgresVersion); err != nil { + return false, err + } + + var installed bool + if err := pgxscan.Get(ctx, pool, &installed, validateTimescaleInstalled); err != nil { + return false, err + } + if !installed { + return false, fmt.Errorf("timescaledb extension not installed, `CREATE EXTENSION IF NOT EXISTS timescaledb;`") + } + + return true, nil +} + +func (e Executor) Finalize(ctx context.Context) error { + // TODO do view stuff + return nil +} + +func addHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { + const ( + createHistorySchema = `CREATE SCHEMA IF NOT EXISTS history;` + cascadeDeleteFunction = ` + CREATE OR REPLACE FUNCTION history.cascade_delete() + RETURNS trigger + LANGUAGE 'plpgsql' + COST 100 + VOLATILE NOT LEAKPROOF + AS $BODY$ + BEGIN + BEGIN + IF (TG_OP = 'DELETE') THEN + EXECUTE format('DELETE FROM history.%I where %I = %L AND cq_fetch_date = %L', TG_ARGV[0], TG_ARGV[1], OLD.cq_id, OLD.cq_fetch_date); + RETURN OLD; + END IF; + RETURN NULL; -- result is ignored since this is an AFTER trigger + END; + END; + $BODY$;` + buildTriggerFunction = ` + CREATE OR REPLACE FUNCTION history.build_trigger(_table_name text, _child_table_name text, _parent_id text) + RETURNS integer + LANGUAGE 'plpgsql' + COST 100 + VOLATILE PARALLEL UNSAFE + AS $BODY$ + BEGIN + IF NOT EXISTS ( SELECT 1 FROM pg_trigger WHERE tgname = _child_table_name ) then + EXECUTE format( + 'CREATE TRIGGER %I BEFORE DELETE ON history.%I FOR EACH ROW EXECUTE PROCEDURE history.cascade_delete(%s, %s)'::text, + _child_table_name, _table_name, _child_table_name, _parent_id); + return 0; + ELSE + return 1; + END IF; + END; + $BODY$;` + + findLatestFetchDate = ` + CREATE OR REPLACE FUNCTION find_latest(schema TEXT, _table_name TEXT) + RETURNS timestamp without time zone AS $body$ + DECLARE + fetchDate timestamp without time zone; + BEGIN + EXECUTE format('SELECT cq_fetch_date FROM %I.%I order by cq_fetch_date desc limit 1', schema, _table_name) into fetchDate; + return fetchDate; + END; + $body$ LANGUAGE plpgsql IMMUTABLE` + ) + + return conn.BeginFunc(ctx, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, createHistorySchema); err != nil { + return err + } + if _, err := tx.Exec(ctx, buildTriggerFunction); err != nil { + return err + } + if _, err := tx.Exec(ctx, cascadeDeleteFunction); err != nil { + return err + } + if _, err := tx.Exec(ctx, findLatestFetchDate); err != nil { + return err + } + return nil + }) +} + +type createHyperTableResult struct { + HypertableId int `db:"hypertable_id"` + SchemaName string `db:"schema_name"` + TableName string `db:"table_name"` + Created bool `db:"created"` +} + +func setPath(ctx context.Context, conn *pgx.Conn, schemaName string) error { + _, err := conn.Exec(ctx, "SET search_path TO $1", schemaName) + return err +} + +func setDsnElement(dsn string, elems map[string]string) string { + u, err := helpers.ParseConnectionString(dsn) + if err != nil { + panic(err.Error()) + } + + vals := u.Query() + for k, v := range elems { + vals.Set(k, v) + } + u.RawQuery = vals.Encode() + return u.String() +} diff --git a/pkg/client/database/timescale/timescale_test.go b/pkg/client/database/timescale/timescale_test.go new file mode 100644 index 00000000000000..3063f5a368979a --- /dev/null +++ b/pkg/client/database/timescale/timescale_test.go @@ -0,0 +1,43 @@ +package timescale + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDSNElement(t *testing.T) { + tbl := []struct { + input string + mod map[string]string + expected string + }{ + { + input: "postgres://a:b@c.d?x=y&z=f", + mod: map[string]string{"ADD": "THIS"}, + expected: "postgres://a:b@c.d?x=y&z=f&ADD=THIS", + }, + { + input: "host=localhost user=postgres password=pass database=postgres port=5432 sslmode=disable", + mod: map[string]string{"ADD": "THIS"}, + expected: "postgres://postgres:pass@localhost:5432/postgres?ADD=THIS&sslmode=disable", + }, + { + input: "tsdb://a:b@c.d?x=y&z=f", + mod: map[string]string{"ADD": "THIS"}, + expected: "tsdb://a:b@c.d?x=y&z=f&ADD=THIS", + }, + } + for _, tc := range tbl { + out := setDsnElement(tc.input, tc.mod) + u1, err := url.Parse(tc.expected) + assert.NoError(t, err) + u2, err := url.Parse(out) + assert.NoError(t, err) + assert.EqualValues(t, u1.Scheme, u2.Scheme) + assert.EqualValues(t, u1.Host, u2.Host) + assert.EqualValues(t, u1.Path, u2.Path) + assert.EqualValues(t, u1.Query(), u2.Query()) + } +} diff --git a/pkg/client/history/history.go b/pkg/client/history/history.go index 5eb80d9110da8e..1ded616867027e 100644 --- a/pkg/client/history/history.go +++ b/pkg/client/history/history.go @@ -1,70 +1,7 @@ package history import ( - "context" - "errors" - "fmt" "time" - - "github.com/georgysavva/scany/pgxscan" - "github.com/jackc/pgx/v4" - "github.com/jackc/pgx/v4/pgxpool" -) - -const ( - createHistorySchema = `CREATE SCHEMA IF NOT EXISTS history;` - validateTimescaleInstalled = `SELECT EXISTS(SELECT 1 FROM pg_extension where extname = 'timescaledb')` - cascadeDeleteFunction = ` - CREATE OR REPLACE FUNCTION history.cascade_delete() - RETURNS trigger - LANGUAGE 'plpgsql' - COST 100 - VOLATILE NOT LEAKPROOF - AS $BODY$ - BEGIN - BEGIN - IF (TG_OP = 'DELETE') THEN - EXECUTE format('DELETE FROM history.%I where %I = %L AND cq_fetch_date = %L', TG_ARGV[0], TG_ARGV[1], OLD.cq_id, OLD.cq_fetch_date); - RETURN OLD; - END IF; - RETURN NULL; -- result is ignored since this is an AFTER trigger - END; - END; - $BODY$;` - buildTriggerFunction = ` - CREATE OR REPLACE FUNCTION history.build_trigger(_table_name text, _child_table_name text, _parent_id text) - RETURNS integer - LANGUAGE 'plpgsql' - COST 100 - VOLATILE PARALLEL UNSAFE - AS $BODY$ - BEGIN - IF NOT EXISTS ( SELECT 1 FROM pg_trigger WHERE tgname = _child_table_name ) then - EXECUTE format( - 'CREATE TRIGGER %I BEFORE DELETE ON history.%I FOR EACH ROW EXECUTE PROCEDURE history.cascade_delete(%s, %s)'::text, - _child_table_name, _table_name, _child_table_name, _parent_id); - return 0; - ELSE - return 1; - END IF; - END; - $BODY$;` - - createHyperTable = `SELECT * FROM create_hypertable($1, 'cq_fetch_date', chunk_time_interval => INTERVAL '%d day', if_not_exists => true);` - dataRetentionPolicy = `SELECT add_retention_policy($1, INTERVAL '%d day', if_not_exists => true);` - findLatestFetchDate = ` - CREATE OR REPLACE FUNCTION find_latest(schema TEXT, _table_name TEXT) - RETURNS timestamp without time zone AS $body$ - DECLARE - fetchDate timestamp without time zone; - BEGIN - EXECUTE format('SELECT cq_fetch_date FROM %I.%I order by cq_fetch_date desc limit 1', schema, _table_name) into fetchDate; - return fetchDate; - END; - $body$ LANGUAGE plpgsql IMMUTABLE` - - dropTableView = `DROP VIEW IF EXISTS "%[1]s"` - createTableView = `CREATE VIEW "%[1]s" AS SELECT * FROM history."%[1]s" WHERE cq_fetch_date = find_latest('history', '%[1]s')` ) type Config struct { @@ -80,44 +17,3 @@ type Config struct { func (c Config) FetchDate() time.Time { return time.Now().UTC().Truncate(time.Duration(c.TimeTruncation) * time.Hour) } - -// SetupHistory sets all required history functions and validation checks that it can run cleanly. -func SetupHistory(ctx context.Context, conn *pgxpool.Conn) error { - installed, err := validateInstalled(ctx, conn) - if err != nil { - return fmt.Errorf("failed to validate timescale installed: %w", err) - } - if !installed { - return errors.New("timescaledb extension not installed, `CREATE EXTENSION IF NOT EXISTS timescaledb;`") - } - if err := addHistoryFunctions(ctx, conn); err != nil { - return fmt.Errorf("failed to create history functions: %w", err) - } - return nil -} - -func addHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { - return conn.BeginFunc(ctx, func(tx pgx.Tx) error { - if _, err := tx.Exec(ctx, createHistorySchema); err != nil { - return err - } - if _, err := tx.Exec(ctx, buildTriggerFunction); err != nil { - return err - } - if _, err := tx.Exec(ctx, cascadeDeleteFunction); err != nil { - return err - } - if _, err := tx.Exec(ctx, findLatestFetchDate); err != nil { - return err - } - return nil - }) -} - -func validateInstalled(ctx context.Context, conn *pgxpool.Conn) (bool, error) { - var installed bool - if err := pgxscan.Get(ctx, conn, &installed, validateTimescaleInstalled); err != nil { - return false, err - } - return installed, nil -} From a05a50d17cc93f413c10ea2a8f6fbf7b1392cd52 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 14 Jan 2022 11:20:47 +0000 Subject: [PATCH 06/45] History things --- pkg/client/client.go | 2 +- pkg/client/database/database.go | 5 +- pkg/client/database/timescale/table.go | 130 ---------- pkg/client/database/timescale/timescale.go | 128 ++-------- .../database/timescale/timescale_test.go | 43 ---- pkg/client/history/ddlmanager.go | 239 ++++++++++++++++++ .../ddlmanager_test.go} | 45 +++- 7 files changed, 302 insertions(+), 290 deletions(-) delete mode 100644 pkg/client/database/timescale/table.go delete mode 100644 pkg/client/database/timescale/timescale_test.go create mode 100644 pkg/client/history/ddlmanager.go rename pkg/client/{database/timescale/table_test.go => history/ddlmanager_test.go} (72%) diff --git a/pkg/client/client.go b/pkg/client/client.go index 6dd104001522d1..043a5871c28282 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -289,7 +289,7 @@ func New(ctx context.Context, options ...Option) (*Client, error) { if err != nil { return nil, err } - _, c.dialectExecutor, err = database.GetExecutor(c.Logger, c.DSN) + _, c.dialectExecutor, err = database.GetExecutor(c.Logger, c.DSN, c.HistoryCfg) if err != nil { return nil, fmt.Errorf("getExecutor: %w", err) } diff --git a/pkg/client/database/database.go b/pkg/client/database/database.go index 0b63c8a060649f..8cc7e34d79b380 100644 --- a/pkg/client/database/database.go +++ b/pkg/client/database/database.go @@ -6,6 +6,7 @@ import ( "github.com/cloudquery/cloudquery/pkg/client/database/postgres" "github.com/cloudquery/cloudquery/pkg/client/database/timescale" + "github.com/cloudquery/cloudquery/pkg/client/history" sdkdb "github.com/cloudquery/cq-provider-sdk/database" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/hashicorp/go-hclog" @@ -27,7 +28,7 @@ var ( _ DialectExecutor = (*timescale.Executor)(nil) ) -func GetExecutor(logger hclog.Logger, dsn string) (schema.DialectType, DialectExecutor, error) { +func GetExecutor(logger hclog.Logger, dsn string, c *history.Config) (schema.DialectType, DialectExecutor, error) { if dsn == "" { return schema.Postgres, nil, fmt.Errorf("missing DSN") } @@ -41,7 +42,7 @@ func GetExecutor(logger hclog.Logger, dsn string) (schema.DialectType, DialectEx case schema.Postgres: return dType, postgres.New(logger, dsn), nil case schema.TSDB: - return dType, timescale.New(logger, dsn), nil + return dType, timescale.New(logger, dsn, c), nil default: return dType, nil, fmt.Errorf("unhandled dialect type") } diff --git a/pkg/client/database/timescale/table.go b/pkg/client/database/timescale/table.go deleted file mode 100644 index 629113de289edb..00000000000000 --- a/pkg/client/database/timescale/table.go +++ /dev/null @@ -1,130 +0,0 @@ -package timescale - -import ( - "context" - "fmt" - "strings" - - "github.com/cloudquery/cloudquery/pkg/client/history" - "github.com/cloudquery/cq-provider-sdk/provider/schema" - "github.com/georgysavva/scany/pgxscan" - "github.com/hashicorp/go-hclog" - "github.com/jackc/pgx/v4" - "github.com/jackc/pgx/v4/pgxpool" -) - -/* -setup -> (creates history schema, history functions) -create (migrate) -> creates all tables in history schema (we set schema like we do now) -finalizes -> fix constraints / pks and run create_hypertable + retention policy and all tables - -+ recreate views in public schema - -need to fix PKs and constraints (remove FKs, add cq_fetch_date to UNIQUE) BEFORE we can do create_hypertable -*/ - -type DDLManager struct { - log hclog.Logger - cfg *history.Config - dialect schema.Dialect -} - -func NewDDLManager(cfg *history.Config, l hclog.Logger, dt schema.DialectType) (*DDLManager, error) { - if dt != schema.TSDB { - return nil, fmt.Errorf("history is only supported on timescaledb") - } - - return &DDLManager{ - log: l, - cfg: cfg, - dialect: schema.GetDialect(dt), - }, nil -} - -func (h DDLManager) CreateTable(ctx context.Context, conn *pgxpool.Conn, t, p *schema.Table) error { - if err := h.createHyperTable(ctx, t, p, conn); err != nil { - return fmt.Errorf("failed to create hypertable for table: %s: %w", t.Name, err) - } - - if err := conn.BeginTxFunc(ctx, pgx.TxOptions{}, func(tx pgx.Tx) error { - // Must drop the view first -- CREATE OR REPLACE view won't cut it if columns are changed. PostgreSQL doc states: - // > The new query must generate the same columns that were generated by the existing view query (that is, the same column names in the same order and with - // > the same data types), but it may add additional columns to the end of the list. - // ref: https://www.postgresql.org/docs/14/sql-createview.html - - if _, err := tx.Exec(ctx, fmt.Sprintf(dropTableView, t.Name)); err != nil { - return fmt.Errorf("failed to drop view for table: %w", err) - } - - if _, err := tx.Exec(ctx, fmt.Sprintf(createTableView, t.Name)); err != nil { - return fmt.Errorf("failed to create view for table: %w", err) - } - - return nil - }); err != nil { - return fmt.Errorf("tx failed for %s: %w", t.Name, err) - } - - if p != nil { - if err := h.buildCascadeTrigger(ctx, conn, t, p); err != nil { - return fmt.Errorf("table build %s failed: %w", t.Name, err) - } - } - - // Create relation tables - for _, r := range t.Relations { - h.log.Debug("creating table relation", "table", r.Name) - if err := h.CreateTable(ctx, conn, r, t); err != nil { - return err - } - } - - return nil -} - -func (h DDLManager) createHyperTable(ctx context.Context, t, p *schema.Table, conn *pgxpool.Conn) error { - var hyperTable createHyperTableResult - tName := fmt.Sprintf(`"history"."%s"`, t.Name) - if err := pgxscan.Get(ctx, conn, &hyperTable, fmt.Sprintf(createHyperTable, h.cfg.TimeInterval), tName); err != nil { - return fmt.Errorf("failed to create hypertable: %w", err) - } - h.log.Debug("created hyper table for table", "table", hyperTable.TableName, "id", hyperTable.HypertableId, "created", hyperTable.Created) - if p != nil { - return nil - } - if _, err := conn.Exec(ctx, fmt.Sprintf(dataRetentionPolicy, h.cfg.Retention), tName); err != nil { - return err - } - h.log.Debug("created data retention policy", "table", hyperTable.TableName, "days", h.cfg.Retention) - return nil -} - -func (h DDLManager) buildCascadeTrigger(ctx context.Context, conn *pgxpool.Conn, t, p *schema.Table) error { - c := h.findParentIdColumn(t) - if c == nil { - return fmt.Errorf("failed to find parent cq id column for %s", t.Name) - } - if _, err := conn.Exec(ctx, "SELECT history.build_trigger($1, $2, $3);", p.Name, t.Name, c.Name); err != nil { - return fmt.Errorf("failed to create trigger: %w", err) - } - if _, err := conn.Exec(ctx, fmt.Sprintf("CREATE INDEX ON \"history\".\"%s\" (cq_fetch_date, %s)", t.Name, c.Name)); err != nil { - return fmt.Errorf("failed to create index on %s (cq_fetch_date, %s): %w", t.Name, c.Name, err) - } - return nil -} - -func (h DDLManager) findParentIdColumn(t *schema.Table) *schema.Column { - for _, c := range t.Columns { - if c.Meta().Resolver != nil && c.Meta().Resolver.Name == "ParentIdResolver" { - return &c - } - } - // Support old school columns instead of meta, this is backwards compatibility for providers using SDK prior v0.5.0 - for _, c := range t.Columns { - if strings.HasSuffix(c.Name, "cq_id") && c.Name != "cq_id" { - return &c - } - } - - return nil -} diff --git a/pkg/client/database/timescale/timescale.go b/pkg/client/database/timescale/timescale.go index 172a93e42fb23b..666c5e4d509892 100644 --- a/pkg/client/database/timescale/timescale.go +++ b/pkg/client/database/timescale/timescale.go @@ -5,34 +5,24 @@ import ( "fmt" "github.com/cloudquery/cloudquery/pkg/client/database/postgres" + "github.com/cloudquery/cloudquery/pkg/client/history" pgsdk "github.com/cloudquery/cq-provider-sdk/database/postgres" - "github.com/cloudquery/cq-provider-sdk/helpers" + "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/georgysavva/scany/pgxscan" "github.com/hashicorp/go-hclog" - "github.com/jackc/pgx/v4" - "github.com/jackc/pgx/v4/pgxpool" -) - -const ( - createHyperTable = `SELECT * FROM create_hypertable($1, 'cq_fetch_date', chunk_time_interval => INTERVAL '%d day', if_not_exists => true);` - dataRetentionPolicy = `SELECT add_retention_policy($1, INTERVAL '%d day', if_not_exists => true);` - - dropTableView = `DROP VIEW IF EXISTS "%[1]s"` - createTableView = `CREATE VIEW "%[1]s" AS SELECT * FROM history."%[1]s" WHERE cq_fetch_date = find_latest('history', '%[1]s')` - - historySchema = "history" - viewSchema = "public" ) type Executor struct { logger hclog.Logger dsn string + cfg *history.Config } -func New(logger hclog.Logger, dsn string) Executor { +func New(logger hclog.Logger, dsn string, cfg *history.Config) Executor { return Executor{ logger: logger, dsn: dsn, + cfg: cfg, } } @@ -49,11 +39,11 @@ func (e Executor) Setup(ctx context.Context) (string, error) { } defer conn.Release() - if err := addHistoryFunctions(ctx, conn); err != nil { + if err := history.AddHistoryFunctions(ctx, conn); err != nil { return e.dsn, fmt.Errorf("failed to create history functions: %w", err) } - return setDsnElement(e.dsn, map[string]string{"search_path": historySchema}), nil + return history.TransformDSN(e.dsn) } func (e Executor) Validate(ctx context.Context) (bool, error) { @@ -83,100 +73,20 @@ func (e Executor) Validate(ctx context.Context) (bool, error) { } func (e Executor) Finalize(ctx context.Context) error { - // TODO do view stuff - return nil -} - -func addHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { - const ( - createHistorySchema = `CREATE SCHEMA IF NOT EXISTS history;` - cascadeDeleteFunction = ` - CREATE OR REPLACE FUNCTION history.cascade_delete() - RETURNS trigger - LANGUAGE 'plpgsql' - COST 100 - VOLATILE NOT LEAKPROOF - AS $BODY$ - BEGIN - BEGIN - IF (TG_OP = 'DELETE') THEN - EXECUTE format('DELETE FROM history.%I where %I = %L AND cq_fetch_date = %L', TG_ARGV[0], TG_ARGV[1], OLD.cq_id, OLD.cq_fetch_date); - RETURN OLD; - END IF; - RETURN NULL; -- result is ignored since this is an AFTER trigger - END; - END; - $BODY$;` - buildTriggerFunction = ` - CREATE OR REPLACE FUNCTION history.build_trigger(_table_name text, _child_table_name text, _parent_id text) - RETURNS integer - LANGUAGE 'plpgsql' - COST 100 - VOLATILE PARALLEL UNSAFE - AS $BODY$ - BEGIN - IF NOT EXISTS ( SELECT 1 FROM pg_trigger WHERE tgname = _child_table_name ) then - EXECUTE format( - 'CREATE TRIGGER %I BEFORE DELETE ON history.%I FOR EACH ROW EXECUTE PROCEDURE history.cascade_delete(%s, %s)'::text, - _child_table_name, _table_name, _child_table_name, _parent_id); - return 0; - ELSE - return 1; - END IF; - END; - $BODY$;` - - findLatestFetchDate = ` - CREATE OR REPLACE FUNCTION find_latest(schema TEXT, _table_name TEXT) - RETURNS timestamp without time zone AS $body$ - DECLARE - fetchDate timestamp without time zone; - BEGIN - EXECUTE format('SELECT cq_fetch_date FROM %I.%I order by cq_fetch_date desc limit 1', schema, _table_name) into fetchDate; - return fetchDate; - END; - $body$ LANGUAGE plpgsql IMMUTABLE` - ) - - return conn.BeginFunc(ctx, func(tx pgx.Tx) error { - if _, err := tx.Exec(ctx, createHistorySchema); err != nil { - return err - } - if _, err := tx.Exec(ctx, buildTriggerFunction); err != nil { - return err - } - if _, err := tx.Exec(ctx, cascadeDeleteFunction); err != nil { - return err - } - if _, err := tx.Exec(ctx, findLatestFetchDate); err != nil { - return err - } - return nil - }) -} - -type createHyperTableResult struct { - HypertableId int `db:"hypertable_id"` - SchemaName string `db:"schema_name"` - TableName string `db:"table_name"` - Created bool `db:"created"` -} - -func setPath(ctx context.Context, conn *pgx.Conn, schemaName string) error { - _, err := conn.Exec(ctx, "SET search_path TO $1", schemaName) - return err -} - -func setDsnElement(dsn string, elems map[string]string) string { - u, err := helpers.ParseConnectionString(dsn) + pool, err := pgsdk.Connect(ctx, e.dsn) if err != nil { - panic(err.Error()) + return err } + defer pool.Close() + conn, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() - vals := u.Query() - for k, v := range elems { - vals.Set(k, v) + ddl, err := history.NewDDLManager(e.logger, conn, e.cfg, schema.TSDB) + if err != nil { + return err } - u.RawQuery = vals.Encode() - return u.String() + return ddl.SetupHistory(ctx, conn) } diff --git a/pkg/client/database/timescale/timescale_test.go b/pkg/client/database/timescale/timescale_test.go deleted file mode 100644 index 3063f5a368979a..00000000000000 --- a/pkg/client/database/timescale/timescale_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package timescale - -import ( - "net/url" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestDSNElement(t *testing.T) { - tbl := []struct { - input string - mod map[string]string - expected string - }{ - { - input: "postgres://a:b@c.d?x=y&z=f", - mod: map[string]string{"ADD": "THIS"}, - expected: "postgres://a:b@c.d?x=y&z=f&ADD=THIS", - }, - { - input: "host=localhost user=postgres password=pass database=postgres port=5432 sslmode=disable", - mod: map[string]string{"ADD": "THIS"}, - expected: "postgres://postgres:pass@localhost:5432/postgres?ADD=THIS&sslmode=disable", - }, - { - input: "tsdb://a:b@c.d?x=y&z=f", - mod: map[string]string{"ADD": "THIS"}, - expected: "tsdb://a:b@c.d?x=y&z=f&ADD=THIS", - }, - } - for _, tc := range tbl { - out := setDsnElement(tc.input, tc.mod) - u1, err := url.Parse(tc.expected) - assert.NoError(t, err) - u2, err := url.Parse(out) - assert.NoError(t, err) - assert.EqualValues(t, u1.Scheme, u2.Scheme) - assert.EqualValues(t, u1.Host, u2.Host) - assert.EqualValues(t, u1.Path, u2.Path) - assert.EqualValues(t, u1.Query(), u2.Query()) - } -} diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/history/ddlmanager.go new file mode 100644 index 00000000000000..2277477b31689e --- /dev/null +++ b/pkg/client/history/ddlmanager.go @@ -0,0 +1,239 @@ +package history + +import ( + "context" + "fmt" + + "github.com/cloudquery/cq-provider-sdk/helpers" + "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/georgysavva/scany/pgxscan" + "github.com/hashicorp/go-hclog" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" +) + +const ( + listTables = `SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' ORDER BY 1` + getColumnComments = `WITH x AS (SELECT column_name, pg_catalog.col_description(format('%s.%s',table_schema,table_name)::regclass::oid,ordinal_position) AS comment FROM information_schema.columns w +here table_schema=$1 AND table_name=$2) SELECT * FROM x WHERE comment IS NOT NULL;` + + createHyperTable = `SELECT * FROM create_hypertable($1, 'cq_fetch_date', chunk_time_interval => INTERVAL '%d day', if_not_exists => true);` + dataRetentionPolicy = `SELECT add_retention_policy($1, INTERVAL '%d day', if_not_exists => true);` + + dropTableView = `DROP VIEW IF EXISTS "%[1]s"` + createTableView = `CREATE VIEW "%[1]s" AS SELECT * FROM history."%[1]s" WHERE cq_fetch_date = find_latest('history', '%[1]s')` + + SchemaName = "history" +) + +type DDLManager struct { + log hclog.Logger + conn *pgxpool.Conn + cfg *Config + dialect schema.Dialect +} + +func NewDDLManager(l hclog.Logger, conn *pgxpool.Conn, cfg *Config, dt schema.DialectType) (*DDLManager, error) { + if dt != schema.TSDB { + return nil, fmt.Errorf("history is only supported on timescaledb") + } + + return &DDLManager{ + log: l, + conn: conn, + cfg: cfg, + dialect: schema.GetDialect(dt), + }, nil +} + +func (h DDLManager) SetupHistory(ctx context.Context, conn *pgxpool.Conn) error { + var tables []string + if err := pgxscan.Get(ctx, conn, &tables, listTables, SchemaName); err != nil { + return fmt.Errorf("failed to list tables: %w", err) + } + + for _, table := range tables { + parentIdCol, parentTable, err := h.getTableParent(ctx, conn, table) + if err != nil { + return fmt.Errorf("getTableParent failed for %s: %w", table, err) + } + + if err := h.createHyperTable(ctx, conn, table, parentTable != ""); err != nil { + return fmt.Errorf("failed to create hypertable for table: %s: %w", table, err) + } + if err := h.recreateView(ctx, conn, table); err != nil { + return fmt.Errorf("recreateView: %w", err) + } + + if parentTable != "" { + if err := h.buildCascadeTrigger(ctx, conn, table, parentIdCol, parentTable); err != nil { + return fmt.Errorf("table build %s failed: %w", table, err) + } + } + } + + return nil +} + +func (h DDLManager) createHyperTable(ctx context.Context, conn *pgxpool.Conn, tableName string, hasParent bool) error { + var hyperTable struct { + HypertableId int `db:"hypertable_id"` + SchemaName string `db:"schema_name"` + TableName string `db:"table_name"` + Created bool `db:"created"` + } + + tName := fmt.Sprintf(`"%s"."%s"`, SchemaName, tableName) + if err := pgxscan.Get(ctx, conn, &hyperTable, fmt.Sprintf(createHyperTable, h.cfg.TimeInterval), tName); err != nil { + return fmt.Errorf("failed to create hypertable: %w", err) + } + h.log.Debug("created hyper table for table", "table", hyperTable.TableName, "id", hyperTable.HypertableId, "created", hyperTable.Created) + if hasParent { // TODO + return nil + } + if _, err := conn.Exec(ctx, fmt.Sprintf(dataRetentionPolicy, h.cfg.Retention), tName); err != nil { + return err + } + h.log.Debug("created data retention policy", "table", hyperTable.TableName, "days", h.cfg.Retention) + return nil +} + +func (h DDLManager) recreateView(ctx context.Context, conn *pgxpool.Conn, table string) error { + if err := conn.BeginTxFunc(ctx, pgx.TxOptions{}, func(tx pgx.Tx) error { + // Must drop the view first -- CREATE OR REPLACE view won't cut it if columns are changed. PostgreSQL doc states: + // > The new query must generate the same columns that were generated by the existing view query (that is, the same column names in the same order and with + // > the same data types), but it may add additional columns to the end of the list. + // ref: https://www.postgresql.org/docs/14/sql-createview.html + + if _, err := tx.Exec(ctx, fmt.Sprintf(dropTableView, table)); err != nil { + return fmt.Errorf("failed to drop view for table: %w", err) + } + + if _, err := tx.Exec(ctx, fmt.Sprintf(createTableView, table)); err != nil { + return fmt.Errorf("failed to create view for table: %w", err) + } + + return nil + }); err != nil { + return fmt.Errorf("tx failed for %s: %w", table, err) + } + return nil +} + +func (h DDLManager) buildCascadeTrigger(ctx context.Context, conn *pgxpool.Conn, table, parentIdColumn, parentTable string) error { + if _, err := conn.Exec(ctx, "SELECT history.build_trigger($1, $2, $3);", parentTable, table, parentIdColumn); err != nil { + return fmt.Errorf("failed to create trigger: %w", err) + } + return nil +} + +func (h DDLManager) getTableParent(ctx context.Context, conn *pgxpool.Conn, tableName string) (parentIdColumn, parentTable string, err error) { + var comments []struct { + Col string `db:"column_name"` + Comment string `db:"comment"` + } + if err := pgxscan.Select(ctx, conn, &comments, getColumnComments, SchemaName, tableName); err != nil { + return "", "", fmt.Errorf("failed to get column comments: %w", err) + } + + found := 0 + for _, c := range comments { + parentTable, _ = schema.GetFKFromComment(c.Comment) + if parentTable != "" { + found++ + parentIdColumn = c.Col + } + } + + if found > 1 { + return "", "", fmt.Errorf("multiple FK comments found in table") + } + + return +} + +func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { + const ( + createHistorySchema = `CREATE SCHEMA IF NOT EXISTS history;` + cascadeDeleteFunction = ` + CREATE OR REPLACE FUNCTION history.cascade_delete() + RETURNS trigger + LANGUAGE 'plpgsql' + COST 100 + VOLATILE NOT LEAKPROOF + AS $BODY$ + BEGIN + BEGIN + IF (TG_OP = 'DELETE') THEN + EXECUTE format('DELETE FROM history.%I where %I = %L AND cq_fetch_date = %L', TG_ARGV[0], TG_ARGV[1], OLD.cq_id, OLD.cq_fetch_date); + RETURN OLD; + END IF; + RETURN NULL; -- result is ignored since this is an AFTER trigger + END; + END; + $BODY$;` + buildTriggerFunction = ` + CREATE OR REPLACE FUNCTION history.build_trigger(_table_name text, _child_table_name text, _parent_id text) + RETURNS integer + LANGUAGE 'plpgsql' + COST 100 + VOLATILE PARALLEL UNSAFE + AS $BODY$ + BEGIN + IF NOT EXISTS ( SELECT 1 FROM pg_trigger WHERE tgname = _child_table_name ) then + EXECUTE format( + 'CREATE TRIGGER %I BEFORE DELETE ON history.%I FOR EACH ROW EXECUTE PROCEDURE history.cascade_delete(%s, %s)'::text, + _child_table_name, _table_name, _child_table_name, _parent_id); + return 0; + ELSE + return 1; + END IF; + END; + $BODY$;` + + findLatestFetchDate = ` + CREATE OR REPLACE FUNCTION find_latest(schema TEXT, _table_name TEXT) + RETURNS timestamp without time zone AS $body$ + DECLARE + fetchDate timestamp without time zone; + BEGIN + EXECUTE format('SELECT cq_fetch_date FROM %I.%I order by cq_fetch_date desc limit 1', schema, _table_name) into fetchDate; + return fetchDate; + END; + $body$ LANGUAGE plpgsql IMMUTABLE` + ) + + return conn.BeginFunc(ctx, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, createHistorySchema); err != nil { + return err + } + if _, err := tx.Exec(ctx, buildTriggerFunction); err != nil { + return err + } + if _, err := tx.Exec(ctx, cascadeDeleteFunction); err != nil { + return err + } + if _, err := tx.Exec(ctx, findLatestFetchDate); err != nil { + return err + } + return nil + }) +} + +func TransformDSN(dsn string) (string, error) { + return setDsnElement(dsn, map[string]string{"search_path": SchemaName}) +} + +func setDsnElement(dsn string, elems map[string]string) (string, error) { + u, err := helpers.ParseConnectionString(dsn) + if err != nil { + return "", err + } + + vals := u.Query() + for k, v := range elems { + vals.Set(k, v) + } + u.RawQuery = vals.Encode() + return u.String(), nil +} diff --git a/pkg/client/database/timescale/table_test.go b/pkg/client/history/ddlmanager_test.go similarity index 72% rename from pkg/client/database/timescale/table_test.go rename to pkg/client/history/ddlmanager_test.go index 70d6f4629dd44c..f2610d3b60519b 100644 --- a/pkg/client/database/timescale/table_test.go +++ b/pkg/client/history/ddlmanager_test.go @@ -1,15 +1,15 @@ //go:build history // +build history -package timescale_test +package history_test import ( "context" "fmt" + "net/url" "testing" "time" - "github.com/cloudquery/cloudquery/pkg/client/history" "github.com/cloudquery/cq-provider-sdk/database/postgres" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/hashicorp/go-hclog" @@ -61,11 +61,11 @@ func TestHistory_SetupHistory(t *testing.T) { conn, err := pool.Acquire(context.Background()) assert.NoError(t, err) defer conn.Release() - assert.NoError(t, history.SetupHistory(context.Background(), conn)) + assert.NoError(t, SetupHistory(context.Background(), conn)) } func TestHistoryTableCreator_CreateTables(t *testing.T) { - m, err := NewDDLManager(&history.Config{Retention: 1, + m, err := NewDDLManager(&Config{Retention: 1, TimeInterval: 1, TimeTruncation: 24, }, hclog.L()) @@ -79,7 +79,7 @@ func TestHistoryTableCreator_CreateTables(t *testing.T) { assert.NoError(t, err) defer conn.Release() // Call setup history as previous test can execute before - assert.NoError(t, history.SetupHistory(context.Background(), conn)) + assert.NoError(t, SetupHistory(context.Background(), conn)) assert.NoError(t, m.CreateTable(context.Background(), conn, testTable, nil)) // creating tables again shouldn't create any errors @@ -106,3 +106,38 @@ func TestHistoryTableCreator_CreateTables(t *testing.T) { assert.Nil(t, err) assert.Equal(t, res.RowsAffected(), int64(0)) } + +func TestDSNElement(t *testing.T) { + tbl := []struct { + input string + mod map[string]string + expected string + }{ + { + input: "postgres://a:b@c.d?x=y&z=f", + mod: map[string]string{"ADD": "THIS"}, + expected: "postgres://a:b@c.d?x=y&z=f&ADD=THIS", + }, + { + input: "host=localhost user=postgres password=pass database=postgres port=5432 sslmode=disable", + mod: map[string]string{"ADD": "THIS"}, + expected: "postgres://postgres:pass@localhost:5432/postgres?ADD=THIS&sslmode=disable", + }, + { + input: "tsdb://a:b@c.d?x=y&z=f", + mod: map[string]string{"ADD": "THIS"}, + expected: "tsdb://a:b@c.d?x=y&z=f&ADD=THIS", + }, + } + for _, tc := range tbl { + out := setDsnElement(tc.input, tc.mod) + u1, err := url.Parse(tc.expected) + assert.NoError(t, err) + u2, err := url.Parse(out) + assert.NoError(t, err) + assert.EqualValues(t, u1.Scheme, u2.Scheme) + assert.EqualValues(t, u1.Host, u2.Host) + assert.EqualValues(t, u1.Path, u2.Path) + assert.EqualValues(t, u1.Query(), u2.Query()) + } +} From b633cc1d7a23439d05ec425fb3a79ea3f4ee0161 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 14 Jan 2022 13:31:44 +0000 Subject: [PATCH 07/45] Fix first migration handling --- pkg/client/client.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 043a5871c28282..52b91aa67dbd8d 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -654,15 +654,17 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( if err != nil { return err } - conn, err := c.pool.Acquire(ctx) - if err != nil { - return err - } - defer conn.Release() if (s.ProtocolVersion != cqproto.Vunmanaged && s.ProtocolVersion <= cqproto.V3) || s.Migrations == nil { // TODO Throw error about upgrading // Keep the table creator if we don't have any migrations defined for this provider, or if we're running an older protocol + + conn, err := c.pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + for name, t := range s.ResourceTables { c.Logger.Debug("creating tables for resource for provider", "resource_name", name, "provider", s.Name, "version", s.Version) if err := c.TableCreator.CreateTable(ctx, conn, t, nil); err != nil { @@ -686,13 +688,10 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( c.Logger.Error("failed to close migrator connection", "error", err) } }() - if _, _, err := m.Version(); err == migrate.ErrNilVersion { - mv, err := m.FindLatestMigration(cfg.Version) - if err != nil { - return err - } - c.Logger.Debug("setting provider schema migration version", "version", cfg.Version, "migration_version", mv) - return m.SetVersion(cfg.Version) + + c.Logger.Debug("setting provider schema migration version", "version", cfg.Version) + if err := m.UpgradeProvider(cfg.Version); err != nil && err != migrate.ErrNoChange { + return err } return nil } @@ -948,7 +947,7 @@ func (c *Client) buildProviderMigrator(ctx context.Context, migrations map[strin return nil, nil, fmt.Errorf("dialectExecutor.Setup: %w", err) } - m, err := migrator.New(c.Logger, migrations, dsn, fmt.Sprintf("%s_%s", org, name), c.dialectExecutor.Finalize) + m, err := migrator.New(c.Logger, c.db.DialectType(), migrations, dsn, fmt.Sprintf("%s_%s", org, name), c.dialectExecutor.Finalize) if err != nil { return nil, nil, err } From 08560feb9b91b75da519d87cdc145ae216132d18 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 14 Jan 2022 14:41:10 +0000 Subject: [PATCH 08/45] Get rid of pool, use QueryExecer --- pkg/client/client.go | 25 +++++++++---------------- pkg/module/drift/drift.go | 4 ++-- pkg/module/drift/terraform.go | 3 +-- pkg/module/manager.go | 17 +++++------------ pkg/module/types.go | 7 +++---- pkg/policy/execute.go | 11 +++++------ pkg/policy/execute_test.go | 34 ++++++++++++---------------------- pkg/policy/manager.go | 17 ++++------------- 8 files changed, 41 insertions(+), 77 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 52b91aa67dbd8d..12ea34f93ea8d8 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -38,7 +38,6 @@ import ( "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" - "github.com/jackc/pgx/v4/pgxpool" zerolog "github.com/rs/zerolog/log" "go.opentelemetry.io/otel/attribute" otrace "go.opentelemetry.io/otel/trace" @@ -206,11 +205,11 @@ type FetchDoneResult struct { // TableCreator creates tables based on schema received from providers type TableCreator interface { - CreateTable(ctx context.Context, conn *pgxpool.Conn, t, p *schema.Table) error + CreateTable(context.Context, schema.QueryExecer, *schema.Table, *schema.Table) error } type TableRemover interface { - DropTable(ctx context.Context, conn *pgxpool.Conn, t *schema.Table) error + DropTable(context.Context, schema.QueryExecer, *schema.Table) error } type FetchUpdateCallback func(update FetchUpdate) @@ -253,8 +252,6 @@ type Client struct { TableCreator TableCreator // HistoryConfig defines configuration for CloudQuery history mode HistoryCfg *history.Config - // pool is a list of connection that are used for policy/query execution - pool *pgxpool.Pool db *sdkdb.DB dialectExecutor database.DialectExecutor @@ -310,7 +307,7 @@ func New(ctx context.Context, options ...Option) (*Client, error) { c.initModules() - c.PolicyManager = policy.NewManager(c.PolicyDirectory, c.pool, c.Logger) + c.PolicyManager = policy.NewManager(c.PolicyDirectory, c.db, c.Logger) return c, nil } @@ -659,18 +656,14 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( // TODO Throw error about upgrading // Keep the table creator if we don't have any migrations defined for this provider, or if we're running an older protocol - conn, err := c.pool.Acquire(ctx) - if err != nil { - return err - } - defer conn.Release() - for name, t := range s.ResourceTables { c.Logger.Debug("creating tables for resource for provider", "resource_name", name, "provider", s.Name, "version", s.Version) - if err := c.TableCreator.CreateTable(ctx, conn, t, nil); err != nil { + if err := c.TableCreator.CreateTable(ctx, c.db, t, nil); err != nil { return fmt.Errorf("CreateTable(%s) failed: %w", t.Name, err) } } + + return nil } if s.Migrations == nil { @@ -906,8 +899,8 @@ func (c *Client) ExecuteModule(ctx context.Context, req ModuleRunRequest) (res * func (c *Client) Close() { c.Manager.Shutdown() - if c.pool != nil { - c.pool.Close() + if c.db != nil { + c.db.Close() } } @@ -928,7 +921,7 @@ func (c *Client) SetProviderVersion(ctx context.Context, providerName, version s } func (c *Client) initModules() { - c.ModuleManager = module.NewManager(c.pool, c.Logger) + c.ModuleManager = module.NewManager(c.db, c.Logger) c.ModuleManager.RegisterModule(drift.New(c.Logger)) } diff --git a/pkg/module/drift/drift.go b/pkg/module/drift/drift.go index 11753bef8af3f4..370688a8af4049 100644 --- a/pkg/module/drift/drift.go +++ b/pkg/module/drift/drift.go @@ -7,6 +7,7 @@ import ( "regexp" "strings" + "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" "github.com/georgysavva/scany/pgxscan" @@ -15,7 +16,6 @@ import ( "github.com/hashicorp/hcl/v2/hclparse" "github.com/jackc/pgconn" "github.com/jackc/pgerrcode" - "github.com/jackc/pgx/v4/pgxpool" "github.com/spf13/afero" "github.com/cloudquery/cloudquery/pkg/module" @@ -260,7 +260,7 @@ func (d *Drift) run(ctx context.Context, req *module.ExecuteRequest) (*Results, return resList, nil } -func queryIntoResourceList(ctx context.Context, logger hclog.Logger, conn *pgxpool.Conn, sel *goqu.SelectDataset) (ResourceList, error) { +func queryIntoResourceList(ctx context.Context, logger hclog.Logger, conn schema.QueryExecer, sel *goqu.SelectDataset) (ResourceList, error) { query, args, err := sel.ToSQL() if err != nil { return nil, fmt.Errorf("goqu build failed: %w", err) diff --git a/pkg/module/drift/terraform.go b/pkg/module/drift/terraform.go index 23600ff0d69d8a..9b12b5c110f58f 100644 --- a/pkg/module/drift/terraform.go +++ b/pkg/module/drift/terraform.go @@ -15,7 +15,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/hashicorp/go-hclog" - "github.com/jackc/pgx/v4/pgxpool" "github.com/olekukonko/tablewriter" "github.com/tidwall/gjson" ) @@ -161,7 +160,7 @@ func parseTerraformAttribute(val interface{}, t schema.ValueType) interface{} { } } -func driftTerraform(ctx context.Context, logger hclog.Logger, conn *pgxpool.Conn, cloudName string, cloudTable *traversedTable, resName string, resources map[string]*ResourceConfig, iacData *IACConfig, states TFStates, runParams RunParams, accountIDs []string) (*Result, error) { +func driftTerraform(ctx context.Context, logger hclog.Logger, conn schema.QueryExecer, cloudName string, cloudTable *traversedTable, resName string, resources map[string]*ResourceConfig, iacData *IACConfig, states TFStates, runParams RunParams, accountIDs []string) (*Result, error) { res := &Result{ Different: nil, Equal: nil, diff --git a/pkg/module/manager.go b/pkg/module/manager.go index 124ff9377a9208..878673e044aa83 100644 --- a/pkg/module/manager.go +++ b/pkg/module/manager.go @@ -4,9 +4,9 @@ import ( "context" "fmt" + "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/hashicorp/go-hclog" "github.com/hashicorp/hcl/v2" - "github.com/jackc/pgx/v4/pgxpool" ) // ManagerImpl is the manager implementation struct. @@ -14,8 +14,8 @@ type ManagerImpl struct { modules map[string]Module modOrder []string - // Instance of a database connection pool - pool *pgxpool.Pool + // Instance of database + pool schema.QueryExecer // Logger instance logger hclog.Logger @@ -35,7 +35,7 @@ type Manager interface { } // NewManager returns a new manager instance. -func NewManager(pool *pgxpool.Pool, logger hclog.Logger) *ManagerImpl { +func NewManager(pool schema.QueryExecer, logger hclog.Logger) *ManagerImpl { return &ManagerImpl{ modules: make(map[string]Module), pool: pool, @@ -64,14 +64,7 @@ func (m *ManagerImpl) ExecuteModule(ctx context.Context, modName string, cfg hcl return nil, fmt.Errorf("module configuration failed: %w", err) } - var err error - - // Acquire connection from the connection pool - execReq.Conn, err = m.pool.Acquire(ctx) - if err != nil { - return nil, fmt.Errorf("failed to acquire connection from the connection pool: %w", err) - } - defer execReq.Conn.Release() + execReq.Conn = m.pool return mod.Execute(ctx, execReq), nil } diff --git a/pkg/module/types.go b/pkg/module/types.go index 11bf96db46d737..b8f1fb0a84d2a0 100644 --- a/pkg/module/types.go +++ b/pkg/module/types.go @@ -3,10 +3,9 @@ package module import ( "context" - "github.com/hashicorp/hcl/v2" - "github.com/jackc/pgx/v4/pgxpool" - "github.com/cloudquery/cq-provider-sdk/cqproto" + "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/hashicorp/hcl/v2" ) type Module interface { @@ -29,7 +28,7 @@ type ExecuteRequest struct { // Providers is the list of providers to process Providers []*cqproto.GetProviderSchemaResponse // Conn is the db connection to use - Conn *pgxpool.Conn + Conn schema.QueryExecer } type ExecutionResult struct { diff --git a/pkg/policy/execute.go b/pkg/policy/execute.go index 121d1add63f897..b55353af320abe 100644 --- a/pkg/policy/execute.go +++ b/pkg/policy/execute.go @@ -6,13 +6,12 @@ import ( "errors" "fmt" "path" - "strings" - "path/filepath" + "strings" + "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-version" - "github.com/jackc/pgx/v4/pgxpool" "github.com/spf13/afero" ) @@ -46,7 +45,7 @@ func (f Update) DoneCount() int { // Executor implements the execution framework. type Executor struct { // Connection to the database - conn *pgxpool.Conn + conn schema.QueryExecer log hclog.Logger PolicyPath []string @@ -99,7 +98,7 @@ type ExecuteRequest struct { } // NewExecutor creates a new executor. -func NewExecutor(conn *pgxpool.Conn, log hclog.Logger, progressUpdate UpdateCallback) *Executor { +func NewExecutor(conn schema.QueryExecer, log hclog.Logger, progressUpdate UpdateCallback) *Executor { return &Executor{ conn: conn, log: log, @@ -239,7 +238,7 @@ func (e *Executor) executeQuery(ctx context.Context, q *Check) (*QueryResult, er func (e *Executor) createViews(ctx context.Context, policy *Policy) error { for _, v := range policy.Views { e.log.Info("creating policy view", "view", v.Name) - if _, err := e.conn.Exec(ctx, fmt.Sprintf("CREATE OR REPLACE TEMPORARY VIEW %s AS %s", v.Name, v.Query)); err != nil { + if err := e.conn.Exec(ctx, fmt.Sprintf("CREATE OR REPLACE TEMPORARY VIEW %s AS %s", v.Name, v.Query)); err != nil { return fmt.Errorf("failed to create view %s/%s: %w", policy.Name, v.Name, err) } } diff --git a/pkg/policy/execute_test.go b/pkg/policy/execute_test.go index f6d24c0a2c00b9..adfdf2de130df4 100644 --- a/pkg/policy/execute_test.go +++ b/pkg/policy/execute_test.go @@ -5,31 +5,27 @@ import ( "fmt" "testing" + sdkdb "github.com/cloudquery/cq-provider-sdk/database" + "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/hashicorp/go-hclog" - "github.com/jackc/pgx/v4/pgxpool" "github.com/stretchr/testify/assert" ) -func setupPolicyDatabase(t *testing.T, tableName string) (*pgxpool.Pool, func(t *testing.T)) { - poolCfg, err := pgxpool.ParseConfig("postgres://postgres:pass@localhost:5432/postgres") - assert.NoError(t, err) - poolCfg.LazyConnect = true - pool, err := pgxpool.ConnectConfig(context.Background(), poolCfg) - assert.NoError(t, err) - conn, err := pool.Acquire(context.Background()) +func setupPolicyDatabase(t *testing.T, tableName string) (schema.QueryExecer, func(t *testing.T)) { + conn, err := sdkdb.New(context.Background(), hclog.NewNullLogger(), "postgres://postgres:pass@localhost:5432/postgres") assert.NoError(t, err) // Setup test data - _, err = conn.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) + err = conn.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) assert.NoError(t, err) - _, err = conn.Exec(context.Background(), fmt.Sprintf("CREATE TABLE %s (id serial PRIMARY KEY, name VARCHAR(50) NOT NULL)", tableName)) + err = conn.Exec(context.Background(), fmt.Sprintf("CREATE TABLE %s (id serial PRIMARY KEY, name VARCHAR(50) NOT NULL)", tableName)) assert.NoError(t, err) - _, err = conn.Exec(context.Background(), fmt.Sprintf("INSERT INTO %s VALUES (1, 'john')", tableName)) + err = conn.Exec(context.Background(), fmt.Sprintf("INSERT INTO %s VALUES (1, 'john')", tableName)) assert.NoError(t, err) // Return conn and tear down func - return pool, func(t *testing.T) { - _, err = conn.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", tableName)) + return conn, func(t *testing.T) { + err = conn.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", tableName)) assert.NoError(t, err) } } @@ -55,10 +51,8 @@ func TestExecutor_executeQuery(t *testing.T) { }, } - pool, tearDownFunc := setupPolicyDatabase(t, t.Name()) + conn, tearDownFunc := setupPolicyDatabase(t, t.Name()) defer tearDownFunc(t) - conn, err := pool.Acquire(context.Background()) - assert.NoError(t, err) executor := NewExecutor(conn, hclog.Default(), nil) for _, tc := range cases { @@ -147,10 +141,8 @@ func TestExecutor_executePolicy(t *testing.T) { }, } - pool, tearDownFunc := setupPolicyDatabase(t, t.Name()) + conn, tearDownFunc := setupPolicyDatabase(t, t.Name()) defer tearDownFunc(t) - conn, err := pool.Acquire(context.Background()) - assert.NoError(t, err) executor := NewExecutor(conn, hclog.Default(), nil) for _, tc := range cases { @@ -310,10 +302,8 @@ func TestExecutor_Execute(t *testing.T) { }, } - pool, tearDownFunc := setupPolicyDatabase(t, t.Name()) + conn, tearDownFunc := setupPolicyDatabase(t, t.Name()) defer tearDownFunc(t) - conn, err := pool.Acquire(context.Background()) - assert.NoError(t, err) executor := NewExecutor(conn, hclog.Default(), nil) for _, tc := range cases { diff --git a/pkg/policy/manager.go b/pkg/policy/manager.go index e5c14b1ccda3f8..cd7b2ad2aec441 100644 --- a/pkg/policy/manager.go +++ b/pkg/policy/manager.go @@ -2,14 +2,13 @@ package policy import ( "context" - "fmt" "strings" + "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/go-hclog" - "github.com/jackc/pgx/v4/pgxpool" ) const ( @@ -22,7 +21,7 @@ type ManagerImpl struct { policyDirectory string // Instance of a database connection pool - pool *pgxpool.Pool + pool schema.QueryExecer // Logger instance logger hclog.Logger @@ -39,7 +38,7 @@ type Manager interface { } // NewManager returns a new manager instance. -func NewManager(policyDir string, pool *pgxpool.Pool, logger hclog.Logger) *ManagerImpl { +func NewManager(policyDir string, pool schema.QueryExecer, logger hclog.Logger) *ManagerImpl { return &ManagerImpl{ policyDirectory: policyDir, pool: pool, @@ -70,14 +69,6 @@ func (m *ManagerImpl) Load(ctx context.Context, policy *Policy) (*Policy, error) } func (m *ManagerImpl) Run(ctx context.Context, request *ExecuteRequest) (*ExecutionResult, error) { - // Acquire connection from the connection pool - conn, err := m.pool.Acquire(ctx) - m.logger.Trace("acquired connection from the connection pool", "err", err) - if err != nil { - return nil, fmt.Errorf("failed to acquire connection from the connection pool: %w", err) - } - defer conn.Release() - var ( totalQueriesToRun = request.Policy.TotalQueries() finishedQueries = 0 @@ -117,7 +108,7 @@ func (m *ManagerImpl) Run(ctx context.Context, request *ExecuteRequest) (*Execut } // execute the queries - return NewExecutor(conn, m.logger, progressUpdate).Execute(ctx, request, request.Policy, selector) + return NewExecutor(m.pool, m.logger, progressUpdate).Execute(ctx, request, request.Policy, selector) } func (m *ManagerImpl) loadPolicyFromSource(ctx context.Context, name, subPolicy, sourceURL string) (*Policy, error) { From 6d7c49dad05d0d2af786fc52717a4cc999c6c7b4 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 14 Jan 2022 14:51:21 +0000 Subject: [PATCH 09/45] Remove older version compatibility which didn't work anyway --- pkg/client/client.go | 11 ++--------- pkg/plugin/plugin.go | 1 - 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 12ea34f93ea8d8..5957ccad209a10 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -652,10 +652,8 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( return err } - if (s.ProtocolVersion != cqproto.Vunmanaged && s.ProtocolVersion <= cqproto.V3) || s.Migrations == nil { - // TODO Throw error about upgrading - // Keep the table creator if we don't have any migrations defined for this provider, or if we're running an older protocol - + if s.Migrations == nil { + // Keep the table creator if we don't have any migrations defined for this provider and hope that it works for name, t := range s.ResourceTables { c.Logger.Debug("creating tables for resource for provider", "resource_name", name, "provider", s.Name, "version", s.Version) if err := c.TableCreator.CreateTable(ctx, c.db, t, nil); err != nil { @@ -666,11 +664,6 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( return nil } - if s.Migrations == nil { - c.Logger.Debug("provider doesn't support migrations", "provider", providerName) - return nil - } - // create migration table and set it to version based on latest create table m, cfg, err := c.buildProviderMigrator(ctx, s.Migrations, providerName) if err != nil { diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index e7f980046d2c33..b4fc8fc51c369d 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -48,7 +48,6 @@ func newRemotePlugin(details *registry.ProviderDetails, alias string, env []stri HandshakeConfig: serve.Handshake, VersionedPlugins: map[int]plugin.PluginSet{ cqproto.V4: pluginMap, - cqproto.V3: pluginMap, }, Managed: true, Cmd: cmd, From 685289b70a68abbca70da708591d777c9abed98f Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 14 Jan 2022 15:06:16 +0000 Subject: [PATCH 10/45] Fix drift working with prerelease --- pkg/module/drift/config.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/module/drift/config.go b/pkg/module/drift/config.go index 83322eaa742d73..465e4e6677a6a1 100644 --- a/pkg/module/drift/config.go +++ b/pkg/module/drift/config.go @@ -422,6 +422,13 @@ func (d *Drift) applyProvider(cfg *ProviderConfig, p *cqproto.GetProviderSchemaR if len(cfg.versionConstraints) > 0 { pver, err := version.NewSemver(p.Version) + if err == nil { + if pr := pver.Prerelease(); pr != "" && strings.HasPrefix(pr, "SNAPSHOT") { + // re-parse without prerelease info + v := strings.SplitN(p.Version, "-", 2) + pver, err = version.NewVersion(v[0]) + } + } if err != nil { return false, []*hcl.Diagnostic{ { From a0bdfb7527e40973ed43084cb3e280ae59662881 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 14 Jan 2022 18:16:06 +0000 Subject: [PATCH 11/45] history fixes --- pkg/client/client.go | 52 ++++++++++++++++++-------------- pkg/client/history/ddlmanager.go | 7 ++--- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 5957ccad209a10..b63b5fd5cb1823 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -14,8 +14,10 @@ import ( "github.com/cloudquery/cloudquery/internal/logging" "github.com/cloudquery/cloudquery/internal/telemetry" + "github.com/jackc/pgx/v4/pgxpool" "github.com/cloudquery/cloudquery/pkg/client/database" + "github.com/cloudquery/cloudquery/pkg/client/database/timescale" "github.com/cloudquery/cloudquery/pkg/client/history" "github.com/cloudquery/cloudquery/pkg/config" "github.com/cloudquery/cloudquery/pkg/module" @@ -286,7 +288,9 @@ func New(ctx context.Context, options ...Option) (*Client, error) { if err != nil { return nil, err } - _, c.dialectExecutor, err = database.GetExecutor(c.Logger, c.DSN, c.HistoryCfg) + + var dt schema.DialectType + dt, c.dialectExecutor, err = database.GetExecutor(c.Logger, c.DSN, c.HistoryCfg) if err != nil { return nil, fmt.Errorf("getExecutor: %w", err) } @@ -296,6 +300,15 @@ func New(ctx context.Context, options ...Option) (*Client, error) { return nil, fmt.Errorf("failed to migrate cloudquery_core tables: %w", err) } + if c.HistoryCfg != nil && dt != schema.TSDB { + // check if we're already on TSDB but the dsn is wrong + if ok, err := timescale.New(c.Logger, c.DSN, c.HistoryCfg).Validate(ctx); ok && err == nil { + return nil, fmt.Errorf("you must update the dsn to use tsdb:// prefix") + } + + return nil, fmt.Errorf("history is only supported on timescaledb") + } + if ok, err := c.dialectExecutor.Validate(ctx); err != nil { return nil, fmt.Errorf("validate: %w", err) } else if !ok { @@ -412,13 +425,19 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes c.Logger.Info("received fetch request", "extra_fields", request.ExtraFields, "history_enabled", c.HistoryCfg != nil) - searchPath := "" + var dsn string if c.HistoryCfg != nil { - searchPath = "history" - } - dsn, err := parseDSN(c.DSN, searchPath) - if err != nil { - return nil, err + var err error + dsn, err = history.TransformDSN(c.DSN) + if err != nil { + return nil, err + } + } else { + parsed, err := helpers.ParseConnectionString(c.DSN) + if err != nil { + return nil, err + } + dsn = parsed.String() } fetchSummaries := make(chan ProviderFetchSummary, len(request.Providers)) @@ -457,11 +476,12 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes pLog := c.Logger.With("provider", providerConfig.Name, "alias", providerConfig.Alias, "version", providerPlugin.Version()) pLog.Info("requesting provider to configure") if c.HistoryCfg != nil { - pLog.Info("history enabled adding fetch date", "fetch_date", c.HistoryCfg.FetchDate().Format(time.RFC3339)) + fd := c.HistoryCfg.FetchDate() + pLog.Info("history enabled adding fetch date", "fetch_date", fd.Format(time.RFC3339)) if request.ExtraFields == nil { request.ExtraFields = make(map[string]interface{}) } - request.ExtraFields["cq_fetch_date"] = c.HistoryCfg.FetchDate() + request.ExtraFields["cq_fetch_date"] = fd } _, err = providerPlugin.Provider().ConfigureProvider(ctx, &cqproto.ConfigureProviderRequest{ CloudQueryVersion: Version, @@ -981,20 +1001,6 @@ func (c *Client) getProviderConfig(providerName string) (*config.RequiredProvide return providerConfig, nil } -func parseDSN(dsn, searchPath string) (string, error) { - url, err := helpers.ParseConnectionString(dsn) - if err != nil { - return "", err - } - if searchPath == "" { - return url.String(), nil - } - if url.RawQuery != "" { - return url.String() + "&search_path=history", nil - } - return url.String() + "search_path=history", nil -} - func parsePartialFetchKV(r *cqproto.FailedResourceFetch) []interface{} { kv := []interface{}{"table", r.TableName, "err", r.Error} if r.RootTableName != "" { diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/history/ddlmanager.go index 2277477b31689e..ee4a9484b45494 100644 --- a/pkg/client/history/ddlmanager.go +++ b/pkg/client/history/ddlmanager.go @@ -13,9 +13,8 @@ import ( ) const ( - listTables = `SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' ORDER BY 1` - getColumnComments = `WITH x AS (SELECT column_name, pg_catalog.col_description(format('%s.%s',table_schema,table_name)::regclass::oid,ordinal_position) AS comment FROM information_schema.columns w -here table_schema=$1 AND table_name=$2) SELECT * FROM x WHERE comment IS NOT NULL;` + listTables = `SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' AND table_name NOT LIKE '%_schema_migrations' ORDER BY 1` + getColumnComments = `WITH x AS (SELECT column_name, pg_catalog.col_description(format('%s.%s',table_schema,table_name)::regclass::oid,ordinal_position) AS comment FROM information_schema.columns WHERE table_schema=$1 AND table_name=$2) SELECT * FROM x WHERE comment IS NOT NULL;` createHyperTable = `SELECT * FROM create_hypertable($1, 'cq_fetch_date', chunk_time_interval => INTERVAL '%d day', if_not_exists => true);` dataRetentionPolicy = `SELECT add_retention_policy($1, INTERVAL '%d day', if_not_exists => true);` @@ -48,7 +47,7 @@ func NewDDLManager(l hclog.Logger, conn *pgxpool.Conn, cfg *Config, dt schema.Di func (h DDLManager) SetupHistory(ctx context.Context, conn *pgxpool.Conn) error { var tables []string - if err := pgxscan.Get(ctx, conn, &tables, listTables, SchemaName); err != nil { + if err := pgxscan.Select(ctx, conn, &tables, listTables, SchemaName); err != nil { return fmt.Errorf("failed to list tables: %w", err) } From 629d85a30eba19a18c764785cdef613570ea3f7f Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 14 Jan 2022 18:46:02 +0000 Subject: [PATCH 12/45] Point to sdk commit --- go.mod | 6 ++---- go.sum | 10 ++-------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index b9b55cd073042d..31cf594dba0004 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.1 + github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220114183641-587ed0610c0f github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -42,7 +42,6 @@ require ( github.com/google/go-cmp v0.5.6 github.com/google/uuid v1.3.0 github.com/hashicorp/go-getter v1.5.10 - github.com/huandu/go-sqlbuilder v1.13.0 github.com/jackc/pgconn v1.10.0 github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 github.com/jackc/pgtype v1.8.1 @@ -82,7 +81,6 @@ require ( github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 // indirect - github.com/huandu/xstrings v1.3.2 // indirect github.com/iancoleman/strcase v0.2.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect @@ -143,4 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -replace github.com/cloudquery/cq-provider-sdk v0.6.1 => ../cq-provider-sdk +//replace github.com/cloudquery/cq-provider-sdk v0.6.1 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index c7955b295d0222..40843600155fcd 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.6.1 h1:pyHabGR81AdsnwtZF0oaJhF9VPK5tHiC8DukA5JEW/A= -github.com/cloudquery/cq-provider-sdk v0.6.1/go.mod h1:lLjzStk8uqMiunTDnAp26QXyQ3XAMexOqzuo8T2riMc= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220114183641-587ed0610c0f h1:7qjyiPseLtq1/2ed8HHVanckKz4Ug899v7aaqbECmZ0= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220114183641-587ed0610c0f/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -666,12 +666,6 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 h1:brI5vBRUlAlM34VFmnLPwjnCL/FxAJp9XvOdX6Zt+XE= github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c= -github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= -github.com/huandu/go-sqlbuilder v1.13.0 h1:IN1VRzcyQ+Kx74L0g5ZAY5qDaRJjwMWVmb6GrFAF8Jc= -github.com/huandu/go-sqlbuilder v1.13.0/go.mod h1:LILlbQo0MOYjlIiGgOSR3UcWQpd5Y/oZ7HLNGyAUz0E= -github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= -github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= From e26df8109396ec8b92c264282fe7314971bbb594 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 14 Jan 2022 19:20:40 +0000 Subject: [PATCH 13/45] Update test provider --- .../provider/migrations/1_v0.0.1.down.sql | 2 -- .../test/provider/migrations/1_v0.0.1.up.sql | 2 -- .../provider/migrations/2_v0.0.2.down.sql | 2 -- .../test/provider/migrations/2_v0.0.2.up.sql | 2 -- .../migrations/postgres/1_v0.0.1.down.sql | 10 ++++++ .../migrations/postgres/1_v0.0.1.up.sql | 28 +++++++++++++++++ .../migrations/postgres/2_v0.0.2.down.sql | 2 ++ .../migrations/postgres/2_v0.0.2.up.sql | 2 ++ .../migrations/postgres/3_v0.0.3.down.sql | 2 ++ .../migrations/postgres/3_v0.0.3.up.sql | 2 ++ .../migrations/timescale/1_v0.0.1.down.sql | 10 ++++++ .../migrations/timescale/1_v0.0.1.up.sql | 31 +++++++++++++++++++ .../migrations/timescale/2_v0.0.2.down.sql | 2 ++ .../migrations/timescale/2_v0.0.2.up.sql | 2 ++ .../migrations/timescale/3_v0.0.3.down.sql | 2 ++ .../migrations/timescale/3_v0.0.3.up.sql | 2 ++ internal/test/test_history_config.hcl | 2 +- internal/test/tools/migrations.go | 17 ++++++++++ 18 files changed, 113 insertions(+), 9 deletions(-) delete mode 100644 internal/test/provider/migrations/1_v0.0.1.down.sql delete mode 100644 internal/test/provider/migrations/1_v0.0.1.up.sql delete mode 100644 internal/test/provider/migrations/2_v0.0.2.down.sql delete mode 100644 internal/test/provider/migrations/2_v0.0.2.up.sql create mode 100644 internal/test/provider/migrations/postgres/1_v0.0.1.down.sql create mode 100644 internal/test/provider/migrations/postgres/1_v0.0.1.up.sql create mode 100644 internal/test/provider/migrations/postgres/2_v0.0.2.down.sql create mode 100644 internal/test/provider/migrations/postgres/2_v0.0.2.up.sql create mode 100644 internal/test/provider/migrations/postgres/3_v0.0.3.down.sql create mode 100644 internal/test/provider/migrations/postgres/3_v0.0.3.up.sql create mode 100644 internal/test/provider/migrations/timescale/1_v0.0.1.down.sql create mode 100644 internal/test/provider/migrations/timescale/1_v0.0.1.up.sql create mode 100644 internal/test/provider/migrations/timescale/2_v0.0.2.down.sql create mode 100644 internal/test/provider/migrations/timescale/2_v0.0.2.up.sql create mode 100644 internal/test/provider/migrations/timescale/3_v0.0.3.down.sql create mode 100644 internal/test/provider/migrations/timescale/3_v0.0.3.up.sql create mode 100644 internal/test/tools/migrations.go diff --git a/internal/test/provider/migrations/1_v0.0.1.down.sql b/internal/test/provider/migrations/1_v0.0.1.down.sql deleted file mode 100644 index 0ea1d04617f1b7..00000000000000 --- a/internal/test/provider/migrations/1_v0.0.1.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE "slow_resource" - DROP COLUMN IF EXISTS upgrade_column \ No newline at end of file diff --git a/internal/test/provider/migrations/1_v0.0.1.up.sql b/internal/test/provider/migrations/1_v0.0.1.up.sql deleted file mode 100644 index 5b37beac44692a..00000000000000 --- a/internal/test/provider/migrations/1_v0.0.1.up.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE "slow_resource" - ADD COLUMN IF NOT EXISTS upgrade_column integer \ No newline at end of file diff --git a/internal/test/provider/migrations/2_v0.0.2.down.sql b/internal/test/provider/migrations/2_v0.0.2.down.sql deleted file mode 100644 index 5db8e2799a8f66..00000000000000 --- a/internal/test/provider/migrations/2_v0.0.2.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE "slow_resource" - DROP COLUMN IF EXISTS upgrade_column_2 \ No newline at end of file diff --git a/internal/test/provider/migrations/2_v0.0.2.up.sql b/internal/test/provider/migrations/2_v0.0.2.up.sql deleted file mode 100644 index 39954932fa4cc8..00000000000000 --- a/internal/test/provider/migrations/2_v0.0.2.up.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE "slow_resource" - ADD COLUMN IF NOT EXISTS upgrade_column_2 integer \ No newline at end of file diff --git a/internal/test/provider/migrations/postgres/1_v0.0.1.down.sql b/internal/test/provider/migrations/postgres/1_v0.0.1.down.sql new file mode 100644 index 00000000000000..198ac7836003cc --- /dev/null +++ b/internal/test/provider/migrations/postgres/1_v0.0.1.down.sql @@ -0,0 +1,10 @@ +-- Autogenerated by migration tool on 2022-01-14 19:05:53 + +-- Resource: error_resource +DROP TABLE IF EXISTS error_resource; + +-- Resource: slow_resource +DROP TABLE IF EXISTS slow_resource; + +-- Resource: very_slow_resource +DROP TABLE IF EXISTS very_slow_resource; diff --git a/internal/test/provider/migrations/postgres/1_v0.0.1.up.sql b/internal/test/provider/migrations/postgres/1_v0.0.1.up.sql new file mode 100644 index 00000000000000..1e568db0ac21dd --- /dev/null +++ b/internal/test/provider/migrations/postgres/1_v0.0.1.up.sql @@ -0,0 +1,28 @@ +-- Autogenerated by migration tool on 2022-01-14 19:05:53 + +-- Resource: error_resource +CREATE TABLE IF NOT EXISTS "error_resource" ( + "cq_id" uuid NOT NULL, + "cq_meta" jsonb, + CONSTRAINT error_resource_pk PRIMARY KEY(cq_id), + UNIQUE(cq_id) +); + +-- Resource: slow_resource +CREATE TABLE IF NOT EXISTS "slow_resource" ( + "cq_id" uuid NOT NULL, + "cq_meta" jsonb, + "some_bool" boolean, + "upgrade_column" integer, + "upgrade_column_2" integer, + CONSTRAINT slow_resource_pk PRIMARY KEY(cq_id), + UNIQUE(cq_id) +); + +-- Resource: very_slow_resource +CREATE TABLE IF NOT EXISTS "very_slow_resource" ( + "cq_id" uuid NOT NULL, + "cq_meta" jsonb, + CONSTRAINT very_slow_resource_pk PRIMARY KEY(cq_id), + UNIQUE(cq_id) +); diff --git a/internal/test/provider/migrations/postgres/2_v0.0.2.down.sql b/internal/test/provider/migrations/postgres/2_v0.0.2.down.sql new file mode 100644 index 00000000000000..683e0a60a9eaef --- /dev/null +++ b/internal/test/provider/migrations/postgres/2_v0.0.2.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE "slow_resource" + DROP COLUMN IF EXISTS upgrade_column; \ No newline at end of file diff --git a/internal/test/provider/migrations/postgres/2_v0.0.2.up.sql b/internal/test/provider/migrations/postgres/2_v0.0.2.up.sql new file mode 100644 index 00000000000000..a513aac13c44b5 --- /dev/null +++ b/internal/test/provider/migrations/postgres/2_v0.0.2.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE "slow_resource" + ADD COLUMN IF NOT EXISTS upgrade_column integer; \ No newline at end of file diff --git a/internal/test/provider/migrations/postgres/3_v0.0.3.down.sql b/internal/test/provider/migrations/postgres/3_v0.0.3.down.sql new file mode 100644 index 00000000000000..60217bb38fe84a --- /dev/null +++ b/internal/test/provider/migrations/postgres/3_v0.0.3.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE "slow_resource" + DROP COLUMN IF EXISTS upgrade_column_2; \ No newline at end of file diff --git a/internal/test/provider/migrations/postgres/3_v0.0.3.up.sql b/internal/test/provider/migrations/postgres/3_v0.0.3.up.sql new file mode 100644 index 00000000000000..157c03fd8aebef --- /dev/null +++ b/internal/test/provider/migrations/postgres/3_v0.0.3.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE "slow_resource" + ADD COLUMN IF NOT EXISTS upgrade_column_2 integer; \ No newline at end of file diff --git a/internal/test/provider/migrations/timescale/1_v0.0.1.down.sql b/internal/test/provider/migrations/timescale/1_v0.0.1.down.sql new file mode 100644 index 00000000000000..198ac7836003cc --- /dev/null +++ b/internal/test/provider/migrations/timescale/1_v0.0.1.down.sql @@ -0,0 +1,10 @@ +-- Autogenerated by migration tool on 2022-01-14 19:05:53 + +-- Resource: error_resource +DROP TABLE IF EXISTS error_resource; + +-- Resource: slow_resource +DROP TABLE IF EXISTS slow_resource; + +-- Resource: very_slow_resource +DROP TABLE IF EXISTS very_slow_resource; diff --git a/internal/test/provider/migrations/timescale/1_v0.0.1.up.sql b/internal/test/provider/migrations/timescale/1_v0.0.1.up.sql new file mode 100644 index 00000000000000..814af9787dc4d4 --- /dev/null +++ b/internal/test/provider/migrations/timescale/1_v0.0.1.up.sql @@ -0,0 +1,31 @@ +-- Autogenerated by migration tool on 2022-01-14 19:05:53 + +-- Resource: error_resource +CREATE TABLE IF NOT EXISTS "error_resource" ( + "cq_id" uuid NOT NULL, + "cq_meta" jsonb, + "cq_fetch_date" timestamp without time zone NOT NULL, + CONSTRAINT error_resource_pk PRIMARY KEY(cq_fetch_date,cq_id), + UNIQUE(cq_fetch_date,cq_id) +); + +-- Resource: slow_resource +CREATE TABLE IF NOT EXISTS "slow_resource" ( + "cq_id" uuid NOT NULL, + "cq_meta" jsonb, + "cq_fetch_date" timestamp without time zone NOT NULL, + "some_bool" boolean, + "upgrade_column" integer, + "upgrade_column_2" integer, + CONSTRAINT slow_resource_pk PRIMARY KEY(cq_fetch_date,cq_id), + UNIQUE(cq_fetch_date,cq_id) +); + +-- Resource: very_slow_resource +CREATE TABLE IF NOT EXISTS "very_slow_resource" ( + "cq_id" uuid NOT NULL, + "cq_meta" jsonb, + "cq_fetch_date" timestamp without time zone NOT NULL, + CONSTRAINT very_slow_resource_pk PRIMARY KEY(cq_fetch_date,cq_id), + UNIQUE(cq_fetch_date,cq_id) +); diff --git a/internal/test/provider/migrations/timescale/2_v0.0.2.down.sql b/internal/test/provider/migrations/timescale/2_v0.0.2.down.sql new file mode 100644 index 00000000000000..683e0a60a9eaef --- /dev/null +++ b/internal/test/provider/migrations/timescale/2_v0.0.2.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE "slow_resource" + DROP COLUMN IF EXISTS upgrade_column; \ No newline at end of file diff --git a/internal/test/provider/migrations/timescale/2_v0.0.2.up.sql b/internal/test/provider/migrations/timescale/2_v0.0.2.up.sql new file mode 100644 index 00000000000000..a513aac13c44b5 --- /dev/null +++ b/internal/test/provider/migrations/timescale/2_v0.0.2.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE "slow_resource" + ADD COLUMN IF NOT EXISTS upgrade_column integer; \ No newline at end of file diff --git a/internal/test/provider/migrations/timescale/3_v0.0.3.down.sql b/internal/test/provider/migrations/timescale/3_v0.0.3.down.sql new file mode 100644 index 00000000000000..60217bb38fe84a --- /dev/null +++ b/internal/test/provider/migrations/timescale/3_v0.0.3.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE "slow_resource" + DROP COLUMN IF EXISTS upgrade_column_2; \ No newline at end of file diff --git a/internal/test/provider/migrations/timescale/3_v0.0.3.up.sql b/internal/test/provider/migrations/timescale/3_v0.0.3.up.sql new file mode 100644 index 00000000000000..157c03fd8aebef --- /dev/null +++ b/internal/test/provider/migrations/timescale/3_v0.0.3.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE "slow_resource" + ADD COLUMN IF NOT EXISTS upgrade_column_2 integer; \ No newline at end of file diff --git a/internal/test/test_history_config.hcl b/internal/test/test_history_config.hcl index 2df1292da7679e..ff1b67e91ca36d 100644 --- a/internal/test/test_history_config.hcl +++ b/internal/test/test_history_config.hcl @@ -1,7 +1,7 @@ cloudquery { connection { - dsn = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" + dsn = "tsdb://postgres:pass@localhost:5432/postgres?sslmode=disable" } provider "test" { source = "cloudquery" diff --git a/internal/test/tools/migrations.go b/internal/test/tools/migrations.go new file mode 100644 index 00000000000000..9d06f1744fc80a --- /dev/null +++ b/internal/test/tools/migrations.go @@ -0,0 +1,17 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/cloudquery/cloudquery/internal/test/provider" + "github.com/cloudquery/cq-provider-sdk/migration" +) + +func main() { + if err := migration.Run(context.Background(), provider.Provider(), "internal/test/provider/migrations"); err != nil { + fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error()) + os.Exit(1) + } +} From 25871dfff722ac5d23f2ed12f3388d1fb02997ac Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Sat, 15 Jan 2022 22:17:43 +0000 Subject: [PATCH 14/45] Adapt SDK changes --- go.mod | 3 ++- pkg/client/client.go | 6 +++++- pkg/client/database/database.go | 2 +- pkg/client/history/ddlmanager.go | 7 ++++++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 31cf594dba0004..734db9aa9fcb0e 100644 --- a/go.mod +++ b/go.mod @@ -141,4 +141,5 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -//replace github.com/cloudquery/cq-provider-sdk v0.6.1 => ../cq-provider-sdk +replace github.com/cloudquery/cq-provider-sdk v0.6.1 => ../cq-provider-sdk +replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220114183641-587ed0610c0f => ../cq-provider-sdk diff --git a/pkg/client/client.go b/pkg/client/client.go index b63b5fd5cb1823..e1c5ea186a710a 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -315,7 +315,11 @@ func New(ctx context.Context, options ...Option) (*Client, error) { c.Logger.Warn("postgres validation warning") } - c.TableCreator = migration.NewTableCreator(c.Logger, schema.GetDialect(c.db.DialectType())) + dialect, err := schema.GetDialect(c.db.DialectType()) + if err != nil { + return nil, err + } + c.TableCreator = migration.NewTableCreator(c.Logger, dialect) } c.initModules() diff --git a/pkg/client/database/database.go b/pkg/client/database/database.go index 8cc7e34d79b380..66dbb333cf8b04 100644 --- a/pkg/client/database/database.go +++ b/pkg/client/database/database.go @@ -33,7 +33,7 @@ func GetExecutor(logger hclog.Logger, dsn string, c *history.Config) (schema.Dia return schema.Postgres, nil, fmt.Errorf("missing DSN") } - dType, dsn, err := sdkdb.DSNtoDialect(dsn) + dType, dsn, err := sdkdb.ParseDialectDSN(dsn) if err != nil { return dType, nil, err } diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/history/ddlmanager.go index ee4a9484b45494..6103871c6d8802 100644 --- a/pkg/client/history/ddlmanager.go +++ b/pkg/client/history/ddlmanager.go @@ -37,11 +37,16 @@ func NewDDLManager(l hclog.Logger, conn *pgxpool.Conn, cfg *Config, dt schema.Di return nil, fmt.Errorf("history is only supported on timescaledb") } + dialect, err := schema.GetDialect(dt) + if err != nil { + return nil, err + } + return &DDLManager{ log: l, conn: conn, cfg: cfg, - dialect: schema.GetDialect(dt), + dialect: dialect, }, nil } From 550ad3ee90e7ecfdac7299982f4b42f2de9eb7e7 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Sat, 15 Jan 2022 22:27:01 +0000 Subject: [PATCH 15/45] Migration proto changes --- go.mod | 5 ++--- go.sum | 4 ++-- pkg/client/client.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 734db9aa9fcb0e..08929435211170 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220114183641-587ed0610c0f + github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220115222550-75ef915d812a github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -141,5 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -replace github.com/cloudquery/cq-provider-sdk v0.6.1 => ../cq-provider-sdk -replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220114183641-587ed0610c0f => ../cq-provider-sdk +//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220115222550-75ef915d812a => ../cq-provider-sdk diff --git a/go.sum b/go.sum index 40843600155fcd..133fb68b597005 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220114183641-587ed0610c0f h1:7qjyiPseLtq1/2ed8HHVanckKz4Ug899v7aaqbECmZ0= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220114183641-587ed0610c0f/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220115222550-75ef915d812a h1:2Q4K7cguK+heD1LqkgPxH7x25DE1WGeLW4U7HLhGR2U= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220115222550-75ef915d812a/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/pkg/client/client.go b/pkg/client/client.go index e1c5ea186a710a..08f685875ae305 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -942,7 +942,7 @@ func (c *Client) initModules() { c.ModuleManager.RegisterModule(drift.New(c.Logger)) } -func (c *Client) buildProviderMigrator(ctx context.Context, migrations map[string][]byte, providerName string) (*migrator.Migrator, *config.RequiredProvider, error) { +func (c *Client) buildProviderMigrator(ctx context.Context, migrations map[string]map[string][]byte, providerName string) (*migrator.Migrator, *config.RequiredProvider, error) { providerConfig, err := c.getProviderConfig(providerName) if err != nil { return nil, nil, err From 403a42482c8f155d818b320da12c25756bedf9a4 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Sun, 16 Jan 2022 17:32:34 +0000 Subject: [PATCH 16/45] Remove disabledelete references --- go.mod | 4 ++-- go.sum | 2 -- pkg/client/client.go | 5 ++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 08929435211170..8fdf48a6e5b2ab 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220115222550-75ef915d812a + github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220116110540-b7a42f90e4b8 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -141,4 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220115222550-75ef915d812a => ../cq-provider-sdk +replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220116110540-b7a42f90e4b8 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index 133fb68b597005..879e8d7ab3b710 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,6 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220115222550-75ef915d812a h1:2Q4K7cguK+heD1LqkgPxH7x25DE1WGeLW4U7HLhGR2U= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220115222550-75ef915d812a/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/pkg/client/client.go b/pkg/client/client.go index 08f685875ae305..64cc98bf31db01 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -492,9 +492,8 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes Connection: cqproto.ConnectionDetails{ DSN: dsn, }, - Config: providerConfig.Configuration, - DisableDelete: true, - ExtraFields: request.ExtraFields, + Config: providerConfig.Configuration, + ExtraFields: request.ExtraFields, }) if err != nil { pLog.Error("failed to configure provider", "error", err) From 7317975490596b2649c4a965f8b5dbf3f828fd84 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Jan 2022 11:38:31 +0000 Subject: [PATCH 17/45] history: Handle define_fk calls --- pkg/client/history/ddlmanager.go | 80 +++++++++++--------------------- 1 file changed, 26 insertions(+), 54 deletions(-) diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/history/ddlmanager.go index 6103871c6d8802..8c44a4a63bfc57 100644 --- a/pkg/client/history/ddlmanager.go +++ b/pkg/client/history/ddlmanager.go @@ -13,8 +13,7 @@ import ( ) const ( - listTables = `SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' AND table_name NOT LIKE '%_schema_migrations' ORDER BY 1` - getColumnComments = `WITH x AS (SELECT column_name, pg_catalog.col_description(format('%s.%s',table_schema,table_name)::regclass::oid,ordinal_position) AS comment FROM information_schema.columns WHERE table_schema=$1 AND table_name=$2) SELECT * FROM x WHERE comment IS NOT NULL;` + listTables = `SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' AND table_name NOT LIKE '%_schema_migrations' ORDER BY 1` createHyperTable = `SELECT * FROM create_hypertable($1, 'cq_fetch_date', chunk_time_interval => INTERVAL '%d day', if_not_exists => true);` dataRetentionPolicy = `SELECT add_retention_policy($1, INTERVAL '%d day', if_not_exists => true);` @@ -22,7 +21,7 @@ const ( dropTableView = `DROP VIEW IF EXISTS "%[1]s"` createTableView = `CREATE VIEW "%[1]s" AS SELECT * FROM history."%[1]s" WHERE cq_fetch_date = find_latest('history', '%[1]s')` - SchemaName = "history" + schemaName = "history" ) type DDLManager struct { @@ -52,34 +51,23 @@ func NewDDLManager(l hclog.Logger, conn *pgxpool.Conn, cfg *Config, dt schema.Di func (h DDLManager) SetupHistory(ctx context.Context, conn *pgxpool.Conn) error { var tables []string - if err := pgxscan.Select(ctx, conn, &tables, listTables, SchemaName); err != nil { + if err := pgxscan.Select(ctx, conn, &tables, listTables, schemaName); err != nil { return fmt.Errorf("failed to list tables: %w", err) } for _, table := range tables { - parentIdCol, parentTable, err := h.getTableParent(ctx, conn, table) - if err != nil { - return fmt.Errorf("getTableParent failed for %s: %w", table, err) - } - - if err := h.createHyperTable(ctx, conn, table, parentTable != ""); err != nil { + if err := h.createHyperTable(ctx, conn, table); err != nil { return fmt.Errorf("failed to create hypertable for table: %s: %w", table, err) } if err := h.recreateView(ctx, conn, table); err != nil { return fmt.Errorf("recreateView: %w", err) } - - if parentTable != "" { - if err := h.buildCascadeTrigger(ctx, conn, table, parentIdCol, parentTable); err != nil { - return fmt.Errorf("table build %s failed: %w", table, err) - } - } } return nil } -func (h DDLManager) createHyperTable(ctx context.Context, conn *pgxpool.Conn, tableName string, hasParent bool) error { +func (h DDLManager) createHyperTable(ctx context.Context, conn *pgxpool.Conn, tableName string) error { var hyperTable struct { HypertableId int `db:"hypertable_id"` SchemaName string `db:"schema_name"` @@ -87,14 +75,13 @@ func (h DDLManager) createHyperTable(ctx context.Context, conn *pgxpool.Conn, ta Created bool `db:"created"` } - tName := fmt.Sprintf(`"%s"."%s"`, SchemaName, tableName) + tName := fmt.Sprintf(`"%s"."%s"`, schemaName, tableName) if err := pgxscan.Get(ctx, conn, &hyperTable, fmt.Sprintf(createHyperTable, h.cfg.TimeInterval), tName); err != nil { return fmt.Errorf("failed to create hypertable: %w", err) } h.log.Debug("created hyper table for table", "table", hyperTable.TableName, "id", hyperTable.HypertableId, "created", hyperTable.Created) - if hasParent { // TODO - return nil - } + + // FIXME: below call is only needed for "parent" tables if _, err := conn.Exec(ctx, fmt.Sprintf(dataRetentionPolicy, h.cfg.Retention), tName); err != nil { return err } @@ -124,38 +111,6 @@ func (h DDLManager) recreateView(ctx context.Context, conn *pgxpool.Conn, table return nil } -func (h DDLManager) buildCascadeTrigger(ctx context.Context, conn *pgxpool.Conn, table, parentIdColumn, parentTable string) error { - if _, err := conn.Exec(ctx, "SELECT history.build_trigger($1, $2, $3);", parentTable, table, parentIdColumn); err != nil { - return fmt.Errorf("failed to create trigger: %w", err) - } - return nil -} - -func (h DDLManager) getTableParent(ctx context.Context, conn *pgxpool.Conn, tableName string) (parentIdColumn, parentTable string, err error) { - var comments []struct { - Col string `db:"column_name"` - Comment string `db:"comment"` - } - if err := pgxscan.Select(ctx, conn, &comments, getColumnComments, SchemaName, tableName); err != nil { - return "", "", fmt.Errorf("failed to get column comments: %w", err) - } - - found := 0 - for _, c := range comments { - parentTable, _ = schema.GetFKFromComment(c.Comment) - if parentTable != "" { - found++ - parentIdColumn = c.Col - } - } - - if found > 1 { - return "", "", fmt.Errorf("multiple FK comments found in table") - } - - return -} - func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { const ( createHistorySchema = `CREATE SCHEMA IF NOT EXISTS history;` @@ -194,6 +149,20 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { END IF; END; $BODY$;` + defineFKFunction = ` + CREATE OR REPLACE FUNCTION history.define_fk(_table_name text, _column_name text, _parent_table_name text, _parent_column_name text) + RETURNS integer + LANGUAGE 'plpgsql' + COST 100 + VOLATILE PARALLEL UNSAFE + AS $BODY$ + DECLARE + result integer; + BEGIN + SELECT history.build_trigger(_parent_table_name, _table_name, _column_name) into result; + RETURN result; + END; + $BODY$;` findLatestFetchDate = ` CREATE OR REPLACE FUNCTION find_latest(schema TEXT, _table_name TEXT) @@ -214,6 +183,9 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { if _, err := tx.Exec(ctx, buildTriggerFunction); err != nil { return err } + if _, err := tx.Exec(ctx, defineFKFunction); err != nil { + return err + } if _, err := tx.Exec(ctx, cascadeDeleteFunction); err != nil { return err } @@ -225,7 +197,7 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { } func TransformDSN(dsn string) (string, error) { - return setDsnElement(dsn, map[string]string{"search_path": SchemaName}) + return setDsnElement(dsn, map[string]string{"search_path": schemaName}) } func setDsnElement(dsn string, elems map[string]string) (string, error) { From 2d73d93d33d4aba75fe445f6661b1afba23737fb Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Jan 2022 13:03:07 +0000 Subject: [PATCH 18/45] Retention fns (doesn't work) --- go.mod | 4 ++-- go.sum | 2 ++ pkg/client/history/ddlmanager.go | 41 +++++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8fdf48a6e5b2ab..7b54187253383c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220116110540-b7a42f90e4b8 + github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117103040-fcf06f268435 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -141,4 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220116110540-b7a42f90e4b8 => ../cq-provider-sdk +replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117103040-fcf06f268435 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index 879e8d7ab3b710..12771cda909386 100644 --- a/go.sum +++ b/go.sum @@ -206,6 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117103040-fcf06f268435 h1:Ql0hoIC/HEQ0K/q59zzMYAsNhnQ332D6V2WsOnsINOs= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117103040-fcf06f268435/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/history/ddlmanager.go index 8c44a4a63bfc57..1b6dfa53f85def 100644 --- a/pkg/client/history/ddlmanager.go +++ b/pkg/client/history/ddlmanager.go @@ -16,7 +16,7 @@ const ( listTables = `SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' AND table_name NOT LIKE '%_schema_migrations' ORDER BY 1` createHyperTable = `SELECT * FROM create_hypertable($1, 'cq_fetch_date', chunk_time_interval => INTERVAL '%d day', if_not_exists => true);` - dataRetentionPolicy = `SELECT add_retention_policy($1, INTERVAL '%d day', if_not_exists => true);` + dataRetentionPolicy = `SELECT history.update_retention($1, INTERVAL '%d day');` dropTableView = `DROP VIEW IF EXISTS "%[1]s"` createTableView = `CREATE VIEW "%[1]s" AS SELECT * FROM history."%[1]s" WHERE cq_fetch_date = find_latest('history', '%[1]s')` @@ -163,6 +163,39 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { RETURN result; END; $BODY$;` + defineMainFunction = ` + CREATE OR REPLACE FUNCTION history.define_main(_table_name text) + RETURNS integer + LANGUAGE 'plpgsql' + COST 100 + VOLATILE PARALLEL UNSAFE + AS $BODY$ + DECLARE + result integer; + BEGIN + SELECT public.add_retention_policy(_table_name, INTERVAL '14 day', if_not_exists => true) into result; + RETURN result; + END; + $BODY$;` + defineRetentionFunction = ` + CREATE OR REPLACE FUNCTION history.update_retention(_table_name text, _retention interval) + RETURNS integer + LANGUAGE 'plpgsql' + COST 100 + VOLATILE PARALLEL UNSAFE + AS $BODY$ + DECLARE + result integer; + BEGIN + IF EXISTS ( SELECT 1 FROM timescaledb_information.jobs WHERE proc_name = 'policy_retention' AND hypertable_name = _table_name) THEN + PERFORM remove_retention_policy(_table_name, if_exists => true); + SELECT add_retention_policy(_table_name, _retention, if_not_exists => true) INTO result; + RETURN result; + ELSE + RETURN -2; + END IF; + END; + $BODY$;` findLatestFetchDate = ` CREATE OR REPLACE FUNCTION find_latest(schema TEXT, _table_name TEXT) @@ -186,6 +219,12 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { if _, err := tx.Exec(ctx, defineFKFunction); err != nil { return err } + if _, err := tx.Exec(ctx, defineMainFunction); err != nil { + return err + } + if _, err := tx.Exec(ctx, defineRetentionFunction); err != nil { + return err + } if _, err := tx.Exec(ctx, cascadeDeleteFunction); err != nil { return err } From 7ee13384f8fd615544dc54bc224dff86644ea1ed Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Jan 2022 14:09:54 +0000 Subject: [PATCH 19/45] Hypertable setup fixes --- go.mod | 4 +-- go.sum | 4 +-- pkg/client/history/ddlmanager.go | 56 +++++++++++++++----------------- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 7b54187253383c..d15a95638993e9 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117103040-fcf06f268435 + github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117140847-a9e40bb7fd21 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -141,4 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117103040-fcf06f268435 => ../cq-provider-sdk +//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117140847-a9e40bb7fd21 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index 12771cda909386..6d5957cdfe63a7 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117103040-fcf06f268435 h1:Ql0hoIC/HEQ0K/q59zzMYAsNhnQ332D6V2WsOnsINOs= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117103040-fcf06f268435/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117140847-a9e40bb7fd21 h1:k7m2jabjUvIBGS4Ty230BkqGbxC2OsSa3V56CRmJxkI= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117140847-a9e40bb7fd21/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/history/ddlmanager.go index 1b6dfa53f85def..06634df546bfb4 100644 --- a/pkg/client/history/ddlmanager.go +++ b/pkg/client/history/ddlmanager.go @@ -15,8 +15,9 @@ import ( const ( listTables = `SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' AND table_name NOT LIKE '%_schema_migrations' ORDER BY 1` - createHyperTable = `SELECT * FROM create_hypertable($1, 'cq_fetch_date', chunk_time_interval => INTERVAL '%d day', if_not_exists => true);` - dataRetentionPolicy = `SELECT history.update_retention($1, INTERVAL '%d day');` + createHyperTable = `SELECT * FROM create_hypertable($1, 'cq_fetch_date', chunk_time_interval => INTERVAL '%d hour', if_not_exists => true);` + setChunkTimeInterval = `SELECT * FROM set_chunk_time_interval($1, INTERVAL '%d hour');` + dataRetentionPolicy = `SELECT history.update_retention($1, INTERVAL '%d day');` dropTableView = `DROP VIEW IF EXISTS "%[1]s"` createTableView = `CREATE VIEW "%[1]s" AS SELECT * FROM history."%[1]s" WHERE cq_fetch_date = find_latest('history', '%[1]s')` @@ -79,9 +80,16 @@ func (h DDLManager) createHyperTable(ctx context.Context, conn *pgxpool.Conn, ta if err := pgxscan.Get(ctx, conn, &hyperTable, fmt.Sprintf(createHyperTable, h.cfg.TimeInterval), tName); err != nil { return fmt.Errorf("failed to create hypertable: %w", err) } - h.log.Debug("created hyper table for table", "table", hyperTable.TableName, "id", hyperTable.HypertableId, "created", hyperTable.Created) + if hyperTable.Created { + h.log.Debug("created hyper table for table", "table", hyperTable.TableName, "id", hyperTable.HypertableId, "interval", h.cfg.TimeInterval) + } else { + if _, err := conn.Exec(ctx, fmt.Sprintf(setChunkTimeInterval, h.cfg.TimeInterval), tName); err != nil { + return err + } + h.log.Debug("updated chunk_time_interval for table", "table", hyperTable.TableName, "id", hyperTable.HypertableId, "interval", h.cfg.TimeInterval) + } - // FIXME: below call is only needed for "parent" tables + // Below call is only needed for "parent" tables. dataRetentionPolicy function takes care of that by updating retention ONLY IF a previous retention policy is set. if _, err := conn.Exec(ctx, fmt.Sprintf(dataRetentionPolicy, h.cfg.Retention), tName); err != nil { return err } @@ -131,40 +139,30 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { END; END; $BODY$;` - buildTriggerFunction = ` - CREATE OR REPLACE FUNCTION history.build_trigger(_table_name text, _child_table_name text, _parent_id text) + + // Creates trigger on a referenced table, so each time a row from the parent table is deleted, referencing (child) rows are also cleared from database. + setupTriggerFunction = ` + CREATE OR REPLACE FUNCTION history.setup_tsdb_trigger(_table_name text, _column_name text, _parent_table_name text, _parent_column_name text) RETURNS integer LANGUAGE 'plpgsql' COST 100 VOLATILE PARALLEL UNSAFE AS $BODY$ BEGIN - IF NOT EXISTS ( SELECT 1 FROM pg_trigger WHERE tgname = _child_table_name ) then + IF NOT EXISTS ( SELECT 1 FROM pg_trigger WHERE tgname = _table_name ) then EXECUTE format( 'CREATE TRIGGER %I BEFORE DELETE ON history.%I FOR EACH ROW EXECUTE PROCEDURE history.cascade_delete(%s, %s)'::text, - _child_table_name, _table_name, _child_table_name, _parent_id); + _table_name, _parent_table_name, _table_name, _column_name); return 0; ELSE return 1; END IF; END; $BODY$;` - defineFKFunction = ` - CREATE OR REPLACE FUNCTION history.define_fk(_table_name text, _column_name text, _parent_table_name text, _parent_column_name text) - RETURNS integer - LANGUAGE 'plpgsql' - COST 100 - VOLATILE PARALLEL UNSAFE - AS $BODY$ - DECLARE - result integer; - BEGIN - SELECT history.build_trigger(_parent_table_name, _table_name, _column_name) into result; - RETURN result; - END; - $BODY$;` - defineMainFunction = ` - CREATE OR REPLACE FUNCTION history.define_main(_table_name text) + + // Creates hypertable on the given table with a default chunk_time_interval, and adds a default retention policy + setupParentFunction = ` + CREATE OR REPLACE FUNCTION history.setup_tsdb_parent(_table_name text) RETURNS integer LANGUAGE 'plpgsql' COST 100 @@ -173,10 +171,13 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { DECLARE result integer; BEGIN + PERFORM public.create_hypertable(_table_name, 'cq_fetch_date', chunk_time_interval => INTERVAL '1 day', if_not_exists => true); SELECT public.add_retention_policy(_table_name, INTERVAL '14 day', if_not_exists => true) into result; RETURN result; END; $BODY$;` + + // Updates the retention policy on the given table, only if a policy already exists. defineRetentionFunction = ` CREATE OR REPLACE FUNCTION history.update_retention(_table_name text, _retention interval) RETURNS integer @@ -213,13 +214,10 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { if _, err := tx.Exec(ctx, createHistorySchema); err != nil { return err } - if _, err := tx.Exec(ctx, buildTriggerFunction); err != nil { - return err - } - if _, err := tx.Exec(ctx, defineFKFunction); err != nil { + if _, err := tx.Exec(ctx, setupTriggerFunction); err != nil { return err } - if _, err := tx.Exec(ctx, defineMainFunction); err != nil { + if _, err := tx.Exec(ctx, setupParentFunction); err != nil { return err } if _, err := tx.Exec(ctx, defineRetentionFunction); err != nil { From d21ba7f480f1a303a52aa97b540da4e3c7d61bff Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Jan 2022 16:12:06 +0000 Subject: [PATCH 20/45] always call create_hypertable in migration, configure hypertable in finalizer instead --- go.mod | 4 ++-- go.sum | 4 ++-- pkg/client/history/ddlmanager.go | 35 +++++++++++--------------------- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index d15a95638993e9..7e8b1c44cd5ab2 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117140847-a9e40bb7fd21 + github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117160908-f7316a3fbdf9 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -141,4 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117140847-a9e40bb7fd21 => ../cq-provider-sdk +//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117160908-f7316a3fbdf9 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index 6d5957cdfe63a7..bfe0ced7cc00d6 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117140847-a9e40bb7fd21 h1:k7m2jabjUvIBGS4Ty230BkqGbxC2OsSa3V56CRmJxkI= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117140847-a9e40bb7fd21/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117160908-f7316a3fbdf9 h1:YTjGmELvA8TwsqpjxzIsJMiej7qi2FMiQXUIAnDdlEk= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117160908-f7316a3fbdf9/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/history/ddlmanager.go index 06634df546bfb4..29ae7d88417546 100644 --- a/pkg/client/history/ddlmanager.go +++ b/pkg/client/history/ddlmanager.go @@ -15,7 +15,6 @@ import ( const ( listTables = `SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' AND table_name NOT LIKE '%_schema_migrations' ORDER BY 1` - createHyperTable = `SELECT * FROM create_hypertable($1, 'cq_fetch_date', chunk_time_interval => INTERVAL '%d hour', if_not_exists => true);` setChunkTimeInterval = `SELECT * FROM set_chunk_time_interval($1, INTERVAL '%d hour');` dataRetentionPolicy = `SELECT history.update_retention($1, INTERVAL '%d day');` @@ -57,8 +56,8 @@ func (h DDLManager) SetupHistory(ctx context.Context, conn *pgxpool.Conn) error } for _, table := range tables { - if err := h.createHyperTable(ctx, conn, table); err != nil { - return fmt.Errorf("failed to create hypertable for table: %s: %w", table, err) + if err := h.configureHyperTable(ctx, conn, table); err != nil { + return fmt.Errorf("failed to configure hypertable for table: %s: %w", table, err) } if err := h.recreateView(ctx, conn, table); err != nil { return fmt.Errorf("recreateView: %w", err) @@ -68,32 +67,20 @@ func (h DDLManager) SetupHistory(ctx context.Context, conn *pgxpool.Conn) error return nil } -func (h DDLManager) createHyperTable(ctx context.Context, conn *pgxpool.Conn, tableName string) error { - var hyperTable struct { - HypertableId int `db:"hypertable_id"` - SchemaName string `db:"schema_name"` - TableName string `db:"table_name"` - Created bool `db:"created"` - } - +func (h DDLManager) configureHyperTable(ctx context.Context, conn *pgxpool.Conn, tableName string) error { tName := fmt.Sprintf(`"%s"."%s"`, schemaName, tableName) - if err := pgxscan.Get(ctx, conn, &hyperTable, fmt.Sprintf(createHyperTable, h.cfg.TimeInterval), tName); err != nil { - return fmt.Errorf("failed to create hypertable: %w", err) - } - if hyperTable.Created { - h.log.Debug("created hyper table for table", "table", hyperTable.TableName, "id", hyperTable.HypertableId, "interval", h.cfg.TimeInterval) - } else { - if _, err := conn.Exec(ctx, fmt.Sprintf(setChunkTimeInterval, h.cfg.TimeInterval), tName); err != nil { - return err - } - h.log.Debug("updated chunk_time_interval for table", "table", hyperTable.TableName, "id", hyperTable.HypertableId, "interval", h.cfg.TimeInterval) + + if _, err := conn.Exec(ctx, fmt.Sprintf(setChunkTimeInterval, h.cfg.TimeInterval), tName); err != nil { + return err } + h.log.Debug("updated chunk_time_interval for table", "table", tableName, "interval", h.cfg.TimeInterval) // Below call is only needed for "parent" tables. dataRetentionPolicy function takes care of that by updating retention ONLY IF a previous retention policy is set. if _, err := conn.Exec(ctx, fmt.Sprintf(dataRetentionPolicy, h.cfg.Retention), tName); err != nil { return err } - h.log.Debug("created data retention policy", "table", hyperTable.TableName, "days", h.cfg.Retention) + + h.log.Debug("created data retention policy", "table", tableName, "days", h.cfg.Retention) return nil } @@ -142,13 +129,15 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { // Creates trigger on a referenced table, so each time a row from the parent table is deleted, referencing (child) rows are also cleared from database. setupTriggerFunction = ` - CREATE OR REPLACE FUNCTION history.setup_tsdb_trigger(_table_name text, _column_name text, _parent_table_name text, _parent_column_name text) + CREATE OR REPLACE FUNCTION history.setup_tsdb_child(_table_name text, _column_name text, _parent_table_name text, _parent_column_name text) RETURNS integer LANGUAGE 'plpgsql' COST 100 VOLATILE PARALLEL UNSAFE AS $BODY$ BEGIN + PERFORM public.create_hypertable(_table_name, 'cq_fetch_date', chunk_time_interval => INTERVAL '1 day', if_not_exists => true); + IF NOT EXISTS ( SELECT 1 FROM pg_trigger WHERE tgname = _table_name ) then EXECUTE format( 'CREATE TRIGGER %I BEFORE DELETE ON history.%I FOR EACH ROW EXECUTE PROCEDURE history.cascade_delete(%s, %s)'::text, From 532b033c6a7f410674d060055fddd5d4ef1c2346 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Jan 2022 17:03:25 +0000 Subject: [PATCH 21/45] Fix history tests --- .../database/timescale/timescale_test.go | 157 ++++++++++++++++++ pkg/client/history/ddlmanager_test.go | 105 +----------- 2 files changed, 160 insertions(+), 102 deletions(-) create mode 100644 pkg/client/database/timescale/timescale_test.go diff --git a/pkg/client/database/timescale/timescale_test.go b/pkg/client/database/timescale/timescale_test.go new file mode 100644 index 00000000000000..ef0f9a37146bf4 --- /dev/null +++ b/pkg/client/database/timescale/timescale_test.go @@ -0,0 +1,157 @@ +//go:build history +// +build history + +package timescale + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/cloudquery/cloudquery/pkg/client/history" + pgsdk "github.com/cloudquery/cq-provider-sdk/database/postgres" + "github.com/cloudquery/cq-provider-sdk/migration" + "github.com/cloudquery/cq-provider-sdk/provider/schema" + "github.com/hashicorp/go-hclog" + "github.com/stretchr/testify/assert" +) + +const ( + testDBConnection = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" // timescale +) + +var testTable = &schema.Table{ + Name: "test_table", + Columns: []schema.Column{ + { + Name: "id", + Type: schema.TypeString, + }, + }, + Relations: []*schema.Table{ + { + Name: "test_rel_table", + Columns: []schema.Column{ + { + Name: "parent_cq_id", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "test", + Type: schema.TypeString, + }, + }, + }, + }, + Options: schema.TableCreationOptions{PrimaryKeys: []string{"id"}}, +} + +func TestSetupHistory(t *testing.T) { + ctx := context.TODO() + ts := New(hclog.L(), testDBConnection, &history.Config{ + Retention: 1, + TimeInterval: 1, + TimeTruncation: 24, + }) + + ok, err := ts.Validate(ctx) + assert.NoError(t, err) + assert.True(t, ok) + + migrationDSN, err := ts.Setup(ctx) + assert.NoError(t, err) + + { + pool, err := pgsdk.Connect(ctx, migrationDSN) + assert.NoError(t, err) + defer pool.Close() + + conn, err := pool.Acquire(ctx) + assert.NoError(t, err) + defer conn.Release() + + tc := migration.NewTableCreator(hclog.L(), schema.TSDBDialect{}) + ups, downs, err := tc.CreateTableDefinitions(ctx, testTable, nil) + assert.NoError(t, err) + + newDowns := make([]string, len(downs)) + for i, sql := range downs { + if strings.HasPrefix(sql, "DROP TABLE ") { + sql += " CASCADE" + } + newDowns[i] = sql + } + defer func() { + for _, sql := range newDowns { + _, err = conn.Exec(ctx, sql) + assert.NoError(t, err) + } + }() + + for _, sql := range append(newDowns, ups...) { // DROP old tables first, if they exist + _, err = conn.Exec(ctx, sql) + assert.NoError(t, err) + } + } + + err = ts.Finalize(ctx) + assert.NoError(t, err) + + t.Run("FinalizeSecondTime", func(t *testing.T) { + // Finalize() again shouldn't create any errors + err := ts.Finalize(ctx) + assert.NoError(t, err) + }) + + pool, err := pgsdk.Connect(ctx, testDBConnection) + assert.NoError(t, err) + defer pool.Close() + + conn, err := pool.Acquire(ctx) + assert.NoError(t, err) + defer conn.Release() + + t.Run("QueryView", func(t *testing.T) { + _, err = conn.Exec(ctx, "select cq_fetch_date from test_table") + assert.Nil(t, err) + }) + + t.Run("QueryHistoryTable", func(t *testing.T) { + _, err = conn.Exec(ctx, "select cq_fetch_date from history.test_table") + assert.Nil(t, err) + }) + + partitionDate := time.Now().Format("2006/01/02") + + t.Run("Insert", func(t *testing.T) { + const ( + sqlInsertMainTable = `INSERT INTO public.test_table(cq_id, cq_meta, cq_fetch_date, id) + VALUES ('0d0bf7c6-c87d-4b3c-a270-60246dcb6ab1', NULL, TO_DATE('%s', 'YYYY/MM/DD'), 'test_id')` + sqlInsertRelTable = `INSERT INTO public.test_rel_table(cq_id, cq_meta, cq_fetch_date, parent_cq_id, test) + VALUES (gen_random_uuid(), null, TO_DATE('%s', 'YYYY/MM/DD'), '0d0bf7c6-c87d-4b3c-a270-60246dcb6ab1', 'test2')` + ) + + _, err = conn.Exec(ctx, fmt.Sprintf(sqlInsertMainTable, partitionDate)) + assert.NoError(t, err) + _, err = conn.Exec(ctx, fmt.Sprintf(sqlInsertRelTable, partitionDate)) + assert.NoError(t, err) + }) + + t.Run("Select", func(t *testing.T) { + res, err := conn.Exec(ctx, "select * from test_rel_table") + assert.NoError(t, err) + assert.Equal(t, res.RowsAffected(), int64(1)) + }) + + t.Run("DeleteCascadeTrigger", func(t *testing.T) { + res, err := conn.Exec(ctx, fmt.Sprintf(`DELETE FROM test_table WHERE cq_fetch_date = TO_DATE('%s', 'YYYY/MM/DD')`, partitionDate)) + assert.NoError(t, err) + assert.Equal(t, res.RowsAffected(), int64(1)) + res, err = conn.Exec(ctx, "select * from test_rel_table") + assert.NoError(t, err) + assert.Equal(t, res.RowsAffected(), int64(0)) + }) +} diff --git a/pkg/client/history/ddlmanager_test.go b/pkg/client/history/ddlmanager_test.go index f2610d3b60519b..dc02f1430ec218 100644 --- a/pkg/client/history/ddlmanager_test.go +++ b/pkg/client/history/ddlmanager_test.go @@ -1,112 +1,12 @@ -//go:build history -// +build history - -package history_test +package history import ( - "context" - "fmt" "net/url" "testing" - "time" - "github.com/cloudquery/cq-provider-sdk/database/postgres" - "github.com/cloudquery/cq-provider-sdk/provider/schema" - "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/assert" ) -const ( - testDBConnection = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - sqlInsertMainTable = `INSERT INTO public.test_table( - cq_id, meta, cq_fetch_date, test) - VALUES ('0d0bf7c6-c87d-4b3c-a270-60246dcb6ab1', NULL, TO_DATE('%s', 'YYYY/MM/DD'), 'test'); - ` - sqlInsertRelTable = `INSERT INTO public.test_rel_table( - cq_id, meta, cq_fetch_date, parent_cq_id, test) - VALUES (gen_random_uuid(), null, TO_DATE('%s', 'YYYY/MM/DD'), '0d0bf7c6-c87d-4b3c-a270-60246dcb6ab1', 'test2'); - ` -) - -var testTable = &schema.Table{ - Name: "test_table", - Columns: []schema.Column{ - { - Name: "test", - Type: schema.TypeString, - }, - }, - Relations: []*schema.Table{ - { - Name: "test_rel_table", - Columns: []schema.Column{ - { - Name: "parent_cq_id", - Type: schema.TypeUUID, - }, - { - Name: "test", - Type: schema.TypeString, - }, - }, - }, - }, - Options: schema.TableCreationOptions{PrimaryKeys: []string{"test"}}, -} - -func TestHistory_SetupHistory(t *testing.T) { - pool, err := postgres.Connect(context.Background(), testDBConnection) - assert.NoError(t, err) - defer pool.Close() - conn, err := pool.Acquire(context.Background()) - assert.NoError(t, err) - defer conn.Release() - assert.NoError(t, SetupHistory(context.Background(), conn)) -} - -func TestHistoryTableCreator_CreateTables(t *testing.T) { - m, err := NewDDLManager(&Config{Retention: 1, - TimeInterval: 1, - TimeTruncation: 24, - }, hclog.L()) - assert.NoError(t, err) - assert.NotNil(t, m) - - pool, err := postgres.Connect(context.Background(), testDBConnection) - assert.NoError(t, err) - defer pool.Close() - conn, err := pool.Acquire(context.Background()) - assert.NoError(t, err) - defer conn.Release() - // Call setup history as previous test can execute before - assert.NoError(t, SetupHistory(context.Background(), conn)) - - assert.NoError(t, m.CreateTable(context.Background(), conn, testTable, nil)) - // creating tables again shouldn't create any errors - assert.NoError(t, m.CreateTable(context.Background(), conn, testTable, nil)) - // query the view - _, err = conn.Exec(context.Background(), "select cq_fetch_date from test_table") - assert.Nil(t, err) - // query the history table itself - _, err = conn.Exec(context.Background(), "select cq_fetch_date from history.test_table") - assert.Nil(t, err) - partitionDate := time.Now().Format("2006/01/02") - _, err = conn.Exec(context.Background(), fmt.Sprintf(sqlInsertMainTable, partitionDate)) - assert.Nil(t, err) - _, err = conn.Exec(context.Background(), fmt.Sprintf(sqlInsertRelTable, partitionDate)) - // Check data was inserted - res, err := conn.Exec(context.Background(), "select * from test_rel_table") - assert.Nil(t, err) - assert.Equal(t, res.RowsAffected(), int64(1)) - // Test that delete cascade trigger works - res, err = conn.Exec(context.Background(), fmt.Sprintf(`DELETE FROM test_table WHERE cq_fetch_date = TO_DATE('%s', 'YYYY/MM/DD')`, partitionDate)) - assert.Nil(t, err) - assert.Equal(t, res.RowsAffected(), int64(1)) - res, err = conn.Exec(context.Background(), "select * from test_rel_table") - assert.Nil(t, err) - assert.Equal(t, res.RowsAffected(), int64(0)) -} - func TestDSNElement(t *testing.T) { tbl := []struct { input string @@ -130,7 +30,8 @@ func TestDSNElement(t *testing.T) { }, } for _, tc := range tbl { - out := setDsnElement(tc.input, tc.mod) + out, err := setDsnElement(tc.input, tc.mod) + assert.NoError(t, err) u1, err := url.Parse(tc.expected) assert.NoError(t, err) u2, err := url.Parse(out) From a96d3ec1751d5d2e6a8d3812709400c0a4d37d66 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Jan 2022 19:06:55 +0000 Subject: [PATCH 22/45] Update SDK --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 7e8b1c44cd5ab2..bb584855bf8846 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117160908-f7316a3fbdf9 + github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117181622-5b2bfd72cf18 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -141,4 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117160908-f7316a3fbdf9 => ../cq-provider-sdk +//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117181622-5b2bfd72cf18 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index bfe0ced7cc00d6..cac39136271916 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117160908-f7316a3fbdf9 h1:YTjGmELvA8TwsqpjxzIsJMiej7qi2FMiQXUIAnDdlEk= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117160908-f7316a3fbdf9/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117181622-5b2bfd72cf18 h1:zVlfnzwMekQN3cVHvicD22aAj3TJrZ4m34SDHDnU1iw= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117181622-5b2bfd72cf18/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= From 395c4140c4fbcadbb35377fecced507340248621 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Jan 2022 19:44:27 +0000 Subject: [PATCH 23/45] Rebase for fetch summary PR --- pkg/client/client.go | 43 ++++++------------- pkg/client/client_test.go | 12 +++--- pkg/client/fetch.go | 12 +----- pkg/client/fetch_test.go | 28 +++--------- .../{ => postgres}/1_v0.19.2.down.sql | 0 .../{ => postgres}/1_v0.19.2.up.sql | 0 6 files changed, 27 insertions(+), 68 deletions(-) rename pkg/client/migrations/{ => postgres}/1_v0.19.2.down.sql (100%) rename pkg/client/migrations/{ => postgres}/1_v0.19.2.up.sql (100%) diff --git a/pkg/client/client.go b/pkg/client/client.go index 64cc98bf31db01..488fad580c744c 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -14,8 +14,6 @@ import ( "github.com/cloudquery/cloudquery/internal/logging" "github.com/cloudquery/cloudquery/internal/telemetry" - "github.com/jackc/pgx/v4/pgxpool" - "github.com/cloudquery/cloudquery/pkg/client/database" "github.com/cloudquery/cloudquery/pkg/client/database/timescale" "github.com/cloudquery/cloudquery/pkg/client/history" @@ -31,7 +29,6 @@ import ( "github.com/cloudquery/cq-provider-sdk/helpers" "github.com/cloudquery/cq-provider-sdk/migration" "github.com/cloudquery/cq-provider-sdk/migration/migrator" - "github.com/cloudquery/cq-provider-sdk/provider" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/cloudquery/cq-provider-sdk/provider/schema/diag" "github.com/getsentry/sentry-go" @@ -50,14 +47,10 @@ import ( var ( ErrMigrationsNotSupported = errors.New("provider doesn't support migrations") - //go:embed migrations/*.sql + //go:embed migrations/*/*.sql coreMigrations embed.FS ) -const ( - latestVersion = "latest" -) - // FetchRequest is provided to the Client to execute a fetch on one or more providers type FetchRequest struct { // UpdateCallback allows gets called when the client receives updates on fetch. @@ -295,11 +288,6 @@ func New(ctx context.Context, options ...Option) (*Client, error) { return nil, fmt.Errorf("getExecutor: %w", err) } - // migrate cloudquery core tables to latest version - if err := c.MigrateCore(ctx); err != nil { - return nil, fmt.Errorf("failed to migrate cloudquery_core tables: %w", err) - } - if c.HistoryCfg != nil && dt != schema.TSDB { // check if we're already on TSDB but the dsn is wrong if ok, err := timescale.New(c.Logger, c.DSN, c.HistoryCfg).Validate(ctx); ok && err == nil { @@ -315,6 +303,11 @@ func New(ctx context.Context, options ...Option) (*Client, error) { c.Logger.Warn("postgres validation warning") } + // migrate cloudquery core tables to latest version + if err := c.MigrateCore(ctx); err != nil { + return nil, fmt.Errorf("failed to migrate cloudquery_core tables: %w", err) + } + dialect, err := schema.GetDialect(c.db.DialectType()) if err != nil { return nil, err @@ -472,7 +465,7 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes } defer func() { - if err := SaveFetchSummary(ctx, c.pool, &fs); err != nil { + if err := c.SaveFetchSummary(ctx, &fs); err != nil { c.Logger.Error("failed to save fetch summary", "err", err) } }() @@ -964,16 +957,16 @@ func (c *Client) buildProviderMigrator(ctx context.Context, migrations map[strin } func (c *Client) MigrateCore(ctx context.Context) error { - err := createCoreSchema(ctx, c.pool) + err := createCoreSchema(ctx, c.db) if err != nil { return err } - migrations, err := provider.ReadMigrationFiles(c.Logger, coreMigrations) + migrations, err := migrator.ReadMigrationFiles(c.Logger, coreMigrations) if err != nil { return err } dsn := c.DSN + "&search_path=cloudquery" - m, err := provider.NewMigrator(c.Logger, migrations, dsn, "cloudquery_core") + m, err := migrator.New(c.Logger, schema.Postgres, migrations, dsn, "cloudquery_core", nil) if err != nil { return err } @@ -984,7 +977,7 @@ func (c *Client) MigrateCore(ctx context.Context) error { } }() - if err := m.UpgradeProvider(latestVersion); err != nil && err != migrate.ErrNoChange { + if err := m.UpgradeProvider("latest"); err != nil && err != migrate.ErrNoChange { return fmt.Errorf("failed to migrate cloudquery core schema: %w", err) } return nil @@ -1106,16 +1099,6 @@ func reportFetchSummaryErrors(span otrace.Span, fetchSummaries map[string]Provid ) } -func createCoreSchema(ctx context.Context, pool *pgxpool.Pool) error { - conn, err := pool.Acquire(ctx) - if err != nil { - return err - } - defer conn.Release() - - _, err = conn.Exec(ctx, "CREATE SCHEMA IF NOT EXISTS cloudquery") - if err != nil { - return err - } - return nil +func createCoreSchema(ctx context.Context, db schema.QueryExecer) error { + return db.Exec(ctx, "CREATE SCHEMA IF NOT EXISTS cloudquery") } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 513cfda84b70fe..a94a8a4eced087 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -398,11 +398,13 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { assert.Nil(t, err) // insert dummy migration files like test provider just for version number return - m, _, err := c.buildProviderMigrator(ctx, map[string][]byte{ - "1_v0.0.1.up.sql": []byte(""), - "1_v0.0.1.down.sql": []byte(""), - "2_v0.0.2.up.sql": []byte(""), - "2_v0.0.2.down.sql": []byte(""), + m, _, err := c.buildProviderMigrator(ctx, map[string]map[string][]byte{ + "postgres": { + "1_v0.0.1.up.sql": []byte(""), + "1_v0.0.1.down.sql": []byte(""), + "2_v0.0.2.up.sql": []byte(""), + "2_v0.0.2.down.sql": []byte(""), + }, }, "test") if err != nil { t.Fatal(err) diff --git a/pkg/client/fetch.go b/pkg/client/fetch.go index 6fe89c3b9ed806..5261684571b4b4 100644 --- a/pkg/client/fetch.go +++ b/pkg/client/fetch.go @@ -10,7 +10,6 @@ import ( "github.com/cloudquery/cq-provider-sdk/provider/schema/diag" "github.com/doug-martin/goqu/v9" "github.com/google/uuid" - "github.com/jackc/pgx/v4/pgxpool" ) // FetchSummary includes a summarized report of fetch, such as fetch id, fetch start and finish, @@ -59,13 +58,7 @@ type ResourceFetchSummary struct { } // SaveFetchSummary saves fetch summary into fetches database -func SaveFetchSummary(ctx context.Context, pool *pgxpool.Pool, fs *FetchSummary) error { - conn, err := pool.Acquire(ctx) - if err != nil { - return err - } - defer conn.Release() - +func (c *Client) SaveFetchSummary(ctx context.Context, fs *FetchSummary) error { id, err := uuid.NewUUID() if err != nil { return err @@ -77,6 +70,5 @@ func SaveFetchSummary(ctx context.Context, pool *pgxpool.Pool, fs *FetchSummary) return err } - _, err = conn.Exec(ctx, sql, args...) - return err + return c.db.Exec(ctx, sql, args...) } diff --git a/pkg/client/fetch_test.go b/pkg/client/fetch_test.go index 244d346deb52f3..254289bba0fbf6 100644 --- a/pkg/client/fetch_test.go +++ b/pkg/client/fetch_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/google/uuid" - "github.com/jackc/pgx/v4/pgxpool" "github.com/stretchr/testify/assert" ) @@ -75,38 +74,21 @@ var fetchSummaryTests = []fetchSummaryTest{ }, } -func setupDatabase(dsn string) (*pgxpool.Pool, error) { - poolCfg, err := pgxpool.ParseConfig(dsn) - if err != nil { - return nil, err - } - poolCfg.LazyConnect = true - pool, err := pgxpool.ConnectConfig(context.Background(), poolCfg) - if err != nil { - return nil, err - } - return pool, nil -} - func TestFetchSummary(t *testing.T) { - option := func(c *Client) { + c, err := New(context.Background(), func(c *Client) { c.DSN = testDBConnection - } - _, err := New(context.Background(), option) - assert.NoError(t, err) - pool, err := setupDatabase(testDBConnection) - assert.NoError(t, err) - defer pool.Close() + }) assert.NoError(t, err) + fetchId := uuid.New() for _, f := range fetchSummaryTests { if !f.skipFetchId { f.summary.FetchId = fetchId } f.summary.Start = time.Now() - err := SaveFetchSummary(context.Background(), pool, &f.summary) + err := c.SaveFetchSummary(context.Background(), &f.summary) if f.err != nil { - assert.Equal(t, f.err.Error(), err.Error()) + assert.EqualError(t, err, f.err.Error()) } else { assert.NoError(t, err) } diff --git a/pkg/client/migrations/1_v0.19.2.down.sql b/pkg/client/migrations/postgres/1_v0.19.2.down.sql similarity index 100% rename from pkg/client/migrations/1_v0.19.2.down.sql rename to pkg/client/migrations/postgres/1_v0.19.2.down.sql diff --git a/pkg/client/migrations/1_v0.19.2.up.sql b/pkg/client/migrations/postgres/1_v0.19.2.up.sql similarity index 100% rename from pkg/client/migrations/1_v0.19.2.up.sql rename to pkg/client/migrations/postgres/1_v0.19.2.up.sql From 9f7c04b31d16d06fd4c4d07f7c46e32e52d2dbba Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Jan 2022 20:06:15 +0000 Subject: [PATCH 24/45] List hypertables, not just tables. Pass DialectExecutor to CoreMigrator to get a proper DSN in history mode --- go.mod | 4 +-- go.sum | 4 +-- pkg/client/client.go | 15 +++++++-- pkg/client/history/ddlmanager.go | 22 +++----------- pkg/client/history/ddlmanager_test.go | 44 --------------------------- 5 files changed, 20 insertions(+), 69 deletions(-) delete mode 100644 pkg/client/history/ddlmanager_test.go diff --git a/go.mod b/go.mod index bb584855bf8846..764a9ed323fc54 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117181622-5b2bfd72cf18 + github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117195354-9102a1578352 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -141,4 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117181622-5b2bfd72cf18 => ../cq-provider-sdk +//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117195354-9102a1578352 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index cac39136271916..f550a1112ca1c1 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117181622-5b2bfd72cf18 h1:zVlfnzwMekQN3cVHvicD22aAj3TJrZ4m34SDHDnU1iw= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117181622-5b2bfd72cf18/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117195354-9102a1578352 h1:U9jbzKbDQc/rSF4jroH5uL65PcLphRLtZPWQjQoI1gA= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117195354-9102a1578352/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/pkg/client/client.go b/pkg/client/client.go index 488fad580c744c..8ebb7c7da8f506 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -304,7 +304,7 @@ func New(ctx context.Context, options ...Option) (*Client, error) { } // migrate cloudquery core tables to latest version - if err := c.MigrateCore(ctx); err != nil { + if err := c.MigrateCore(ctx, c.dialectExecutor); err != nil { return nil, fmt.Errorf("failed to migrate cloudquery_core tables: %w", err) } @@ -956,16 +956,25 @@ func (c *Client) buildProviderMigrator(ctx context.Context, migrations map[strin return m, providerConfig, err } -func (c *Client) MigrateCore(ctx context.Context) error { +func (c *Client) MigrateCore(ctx context.Context, de database.DialectExecutor) error { err := createCoreSchema(ctx, c.db) if err != nil { return err } + + newDSN, err := de.Setup(ctx) + if err != nil { + return err + } + migrations, err := migrator.ReadMigrationFiles(c.Logger, coreMigrations) if err != nil { return err } - dsn := c.DSN + "&search_path=cloudquery" + dsn, err := helpers.SetDSNElement(newDSN, map[string]string{"search_path": "cloudquery"}) + if err != nil { + return err + } m, err := migrator.New(c.Logger, schema.Postgres, migrations, dsn, "cloudquery_core", nil) if err != nil { return err diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/history/ddlmanager.go index 29ae7d88417546..450fd09511c0c4 100644 --- a/pkg/client/history/ddlmanager.go +++ b/pkg/client/history/ddlmanager.go @@ -13,7 +13,7 @@ import ( ) const ( - listTables = `SELECT table_name FROM information_schema.tables WHERE table_schema=$1 AND table_type='BASE TABLE' AND table_name NOT LIKE '%_schema_migrations' ORDER BY 1` + listHyperTables = `SELECT hypertable_name FROM timescaledb_information.hypertables WHERE hypertable_schema=$1 ORDER BY 1` setChunkTimeInterval = `SELECT * FROM set_chunk_time_interval($1, INTERVAL '%d hour');` dataRetentionPolicy = `SELECT history.update_retention($1, INTERVAL '%d day');` @@ -51,8 +51,8 @@ func NewDDLManager(l hclog.Logger, conn *pgxpool.Conn, cfg *Config, dt schema.Di func (h DDLManager) SetupHistory(ctx context.Context, conn *pgxpool.Conn) error { var tables []string - if err := pgxscan.Select(ctx, conn, &tables, listTables, schemaName); err != nil { - return fmt.Errorf("failed to list tables: %w", err) + if err := pgxscan.Select(ctx, conn, &tables, listHyperTables, schemaName); err != nil { + return fmt.Errorf("failed to list hypertables: %w", err) } for _, table := range tables { @@ -223,19 +223,5 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { } func TransformDSN(dsn string) (string, error) { - return setDsnElement(dsn, map[string]string{"search_path": schemaName}) -} - -func setDsnElement(dsn string, elems map[string]string) (string, error) { - u, err := helpers.ParseConnectionString(dsn) - if err != nil { - return "", err - } - - vals := u.Query() - for k, v := range elems { - vals.Set(k, v) - } - u.RawQuery = vals.Encode() - return u.String(), nil + return helpers.SetDSNElement(dsn, map[string]string{"search_path": schemaName}) } diff --git a/pkg/client/history/ddlmanager_test.go b/pkg/client/history/ddlmanager_test.go deleted file mode 100644 index dc02f1430ec218..00000000000000 --- a/pkg/client/history/ddlmanager_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package history - -import ( - "net/url" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestDSNElement(t *testing.T) { - tbl := []struct { - input string - mod map[string]string - expected string - }{ - { - input: "postgres://a:b@c.d?x=y&z=f", - mod: map[string]string{"ADD": "THIS"}, - expected: "postgres://a:b@c.d?x=y&z=f&ADD=THIS", - }, - { - input: "host=localhost user=postgres password=pass database=postgres port=5432 sslmode=disable", - mod: map[string]string{"ADD": "THIS"}, - expected: "postgres://postgres:pass@localhost:5432/postgres?ADD=THIS&sslmode=disable", - }, - { - input: "tsdb://a:b@c.d?x=y&z=f", - mod: map[string]string{"ADD": "THIS"}, - expected: "tsdb://a:b@c.d?x=y&z=f&ADD=THIS", - }, - } - for _, tc := range tbl { - out, err := setDsnElement(tc.input, tc.mod) - assert.NoError(t, err) - u1, err := url.Parse(tc.expected) - assert.NoError(t, err) - u2, err := url.Parse(out) - assert.NoError(t, err) - assert.EqualValues(t, u1.Scheme, u2.Scheme) - assert.EqualValues(t, u1.Host, u2.Host) - assert.EqualValues(t, u1.Path, u2.Path) - assert.EqualValues(t, u1.Query(), u2.Query()) - } -} From f9e8da139a47d7d5e1ae261dff7cb97bf753b6b7 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Jan 2022 21:27:06 +0000 Subject: [PATCH 25/45] Update SDK, dsn changes --- go.mod | 4 ++-- go.sum | 4 ++-- pkg/client/client.go | 16 ++++++++-------- pkg/client/history/ddlmanager.go | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 764a9ed323fc54..53d02802e71b22 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117195354-9102a1578352 + github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117212543-7670f98adb8d github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -141,4 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117195354-9102a1578352 => ../cq-provider-sdk +//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117212543-7670f98adb8d => ../cq-provider-sdk diff --git a/go.sum b/go.sum index f550a1112ca1c1..99ed18e2083b89 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117195354-9102a1578352 h1:U9jbzKbDQc/rSF4jroH5uL65PcLphRLtZPWQjQoI1gA= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117195354-9102a1578352/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117212543-7670f98adb8d h1:R0f8SLYyBWXpu5HMNhsx1l9sHn56NMxS6tmbMXL8yWk= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117212543-7670f98adb8d/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/pkg/client/client.go b/pkg/client/client.go index 8ebb7c7da8f506..3bd5dc61189669 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -26,7 +26,7 @@ import ( "github.com/cloudquery/cloudquery/pkg/ui" "github.com/cloudquery/cq-provider-sdk/cqproto" sdkdb "github.com/cloudquery/cq-provider-sdk/database" - "github.com/cloudquery/cq-provider-sdk/helpers" + "github.com/cloudquery/cq-provider-sdk/database/dsn" "github.com/cloudquery/cq-provider-sdk/migration" "github.com/cloudquery/cq-provider-sdk/migration/migrator" "github.com/cloudquery/cq-provider-sdk/provider/schema" @@ -422,19 +422,19 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes c.Logger.Info("received fetch request", "extra_fields", request.ExtraFields, "history_enabled", c.HistoryCfg != nil) - var dsn string + var dsnURI string if c.HistoryCfg != nil { var err error - dsn, err = history.TransformDSN(c.DSN) + dsnURI, err = history.TransformDSN(c.DSN) if err != nil { return nil, err } } else { - parsed, err := helpers.ParseConnectionString(c.DSN) + parsed, err := dsn.ParseConnectionString(c.DSN) if err != nil { return nil, err } - dsn = parsed.String() + dsnURI = parsed.String() } fetchSummaries := make(chan ProviderFetchSummary, len(request.Providers)) @@ -483,7 +483,7 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes _, err = providerPlugin.Provider().ConfigureProvider(ctx, &cqproto.ConfigureProviderRequest{ CloudQueryVersion: Version, Connection: cqproto.ConnectionDetails{ - DSN: dsn, + DSN: dsnURI, }, Config: providerConfig.Configuration, ExtraFields: request.ExtraFields, @@ -971,11 +971,11 @@ func (c *Client) MigrateCore(ctx context.Context, de database.DialectExecutor) e if err != nil { return err } - dsn, err := helpers.SetDSNElement(newDSN, map[string]string{"search_path": "cloudquery"}) + newDSN, err = dsn.SetDSNElement(newDSN, map[string]string{"search_path": "cloudquery"}) if err != nil { return err } - m, err := migrator.New(c.Logger, schema.Postgres, migrations, dsn, "cloudquery_core", nil) + m, err := migrator.New(c.Logger, schema.Postgres, migrations, newDSN, "cloudquery_core", nil) if err != nil { return err } diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/history/ddlmanager.go index 450fd09511c0c4..fb33c0122d3ff5 100644 --- a/pkg/client/history/ddlmanager.go +++ b/pkg/client/history/ddlmanager.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/cloudquery/cq-provider-sdk/helpers" + "github.com/cloudquery/cq-provider-sdk/database/dsn" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/georgysavva/scany/pgxscan" "github.com/hashicorp/go-hclog" @@ -222,6 +222,6 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { }) } -func TransformDSN(dsn string) (string, error) { - return helpers.SetDSNElement(dsn, map[string]string{"search_path": schemaName}) +func TransformDSN(inputDSN string) (string, error) { + return dsn.SetDSNElement(inputDSN, map[string]string{"search_path": schemaName}) } From e60774dc24fce7af5bc963f4479bcde06c23e5a2 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Jan 2022 21:36:04 +0000 Subject: [PATCH 26/45] Address CR feedback --- pkg/client/client.go | 81 +++++++++++++++++--------------- pkg/client/history/ddlmanager.go | 76 +++++++++++++++--------------- 2 files changed, 81 insertions(+), 76 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 3bd5dc61189669..965491284a24c5 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -203,10 +203,6 @@ type TableCreator interface { CreateTable(context.Context, schema.QueryExecer, *schema.Table, *schema.Table) error } -type TableRemover interface { - DropTable(context.Context, schema.QueryExecer, *schema.Table) error -} - type FetchUpdateCallback func(update FetchUpdate) type Option func(options *Client) @@ -277,42 +273,9 @@ func New(ctx context.Context, options ...Option) (*Client, error) { if c.DSN == "" { c.Logger.Warn("missing DSN, some commands won't work") } else { - c.db, err = sdkdb.New(ctx, c.Logger, c.DSN) - if err != nil { + if err := c.initDatabase(ctx); err != nil { return nil, err } - - var dt schema.DialectType - dt, c.dialectExecutor, err = database.GetExecutor(c.Logger, c.DSN, c.HistoryCfg) - if err != nil { - return nil, fmt.Errorf("getExecutor: %w", err) - } - - if c.HistoryCfg != nil && dt != schema.TSDB { - // check if we're already on TSDB but the dsn is wrong - if ok, err := timescale.New(c.Logger, c.DSN, c.HistoryCfg).Validate(ctx); ok && err == nil { - return nil, fmt.Errorf("you must update the dsn to use tsdb:// prefix") - } - - return nil, fmt.Errorf("history is only supported on timescaledb") - } - - if ok, err := c.dialectExecutor.Validate(ctx); err != nil { - return nil, fmt.Errorf("validate: %w", err) - } else if !ok { - c.Logger.Warn("postgres validation warning") - } - - // migrate cloudquery core tables to latest version - if err := c.MigrateCore(ctx, c.dialectExecutor); err != nil { - return nil, fmt.Errorf("failed to migrate cloudquery_core tables: %w", err) - } - - dialect, err := schema.GetDialect(c.db.DialectType()) - if err != nil { - return nil, err - } - c.TableCreator = migration.NewTableCreator(c.Logger, dialect) } c.initModules() @@ -1111,3 +1074,45 @@ func reportFetchSummaryErrors(span otrace.Span, fetchSummaries map[string]Provid func createCoreSchema(ctx context.Context, db schema.QueryExecer) error { return db.Exec(ctx, "CREATE SCHEMA IF NOT EXISTS cloudquery") } + +func (c *Client) initDatabase(ctx context.Context) error { + var err error + c.db, err = sdkdb.New(ctx, c.Logger, c.DSN) + if err != nil { + return err + } + + var dt schema.DialectType + dt, c.dialectExecutor, err = database.GetExecutor(c.Logger, c.DSN, c.HistoryCfg) + if err != nil { + return fmt.Errorf("getExecutor: %w", err) + } + + if c.HistoryCfg != nil && dt != schema.TSDB { + // check if we're already on TSDB but the dsn is wrong + if ok, err := timescale.New(c.Logger, c.DSN, c.HistoryCfg).Validate(ctx); ok && err == nil { + return fmt.Errorf("you must update the dsn to use tsdb:// prefix") + } + + return fmt.Errorf("history is only supported on timescaledb") + } + + if ok, err := c.dialectExecutor.Validate(ctx); err != nil { + return fmt.Errorf("validate: %w", err) + } else if !ok { + c.Logger.Warn("postgres validation warning") + } + + // migrate cloudquery core tables to latest version + if err := c.MigrateCore(ctx, c.dialectExecutor); err != nil { + return fmt.Errorf("failed to migrate cloudquery_core tables: %w", err) + } + + dialect, err := schema.GetDialect(c.db.DialectType()) + if err != nil { + return err + } + c.TableCreator = migration.NewTableCreator(c.Logger, dialect) + + return nil +} diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/history/ddlmanager.go index fb33c0122d3ff5..2dd52e33f51b71 100644 --- a/pkg/client/history/ddlmanager.go +++ b/pkg/client/history/ddlmanager.go @@ -107,9 +107,36 @@ func (h DDLManager) recreateView(ctx context.Context, conn *pgxpool.Conn, table } func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { - const ( - createHistorySchema = `CREATE SCHEMA IF NOT EXISTS history;` - cascadeDeleteFunction = ` + return conn.BeginFunc(ctx, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, createHistorySchema); err != nil { + return err + } + if _, err := tx.Exec(ctx, setupTriggerFunction); err != nil { + return err + } + if _, err := tx.Exec(ctx, setupParentFunction); err != nil { + return err + } + if _, err := tx.Exec(ctx, defineRetentionFunction); err != nil { + return err + } + if _, err := tx.Exec(ctx, cascadeDeleteFunction); err != nil { + return err + } + if _, err := tx.Exec(ctx, findLatestFetchDate); err != nil { + return err + } + return nil + }) +} + +func TransformDSN(inputDSN string) (string, error) { + return dsn.SetDSNElement(inputDSN, map[string]string{"search_path": schemaName}) +} + +const ( + createHistorySchema = `CREATE SCHEMA IF NOT EXISTS history;` + cascadeDeleteFunction = ` CREATE OR REPLACE FUNCTION history.cascade_delete() RETURNS trigger LANGUAGE 'plpgsql' @@ -127,8 +154,8 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { END; $BODY$;` - // Creates trigger on a referenced table, so each time a row from the parent table is deleted, referencing (child) rows are also cleared from database. - setupTriggerFunction = ` + // Creates trigger on a referenced table, so each time a row from the parent table is deleted, referencing (child) rows are also cleared from database. + setupTriggerFunction = ` CREATE OR REPLACE FUNCTION history.setup_tsdb_child(_table_name text, _column_name text, _parent_table_name text, _parent_column_name text) RETURNS integer LANGUAGE 'plpgsql' @@ -149,8 +176,8 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { END; $BODY$;` - // Creates hypertable on the given table with a default chunk_time_interval, and adds a default retention policy - setupParentFunction = ` + // Creates hypertable on the given table with a default chunk_time_interval, and adds a default retention policy + setupParentFunction = ` CREATE OR REPLACE FUNCTION history.setup_tsdb_parent(_table_name text) RETURNS integer LANGUAGE 'plpgsql' @@ -166,8 +193,8 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { END; $BODY$;` - // Updates the retention policy on the given table, only if a policy already exists. - defineRetentionFunction = ` + // Updates the retention policy on the given table, only if a policy already exists. + defineRetentionFunction = ` CREATE OR REPLACE FUNCTION history.update_retention(_table_name text, _retention interval) RETURNS integer LANGUAGE 'plpgsql' @@ -187,7 +214,7 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { END; $BODY$;` - findLatestFetchDate = ` + findLatestFetchDate = ` CREATE OR REPLACE FUNCTION find_latest(schema TEXT, _table_name TEXT) RETURNS timestamp without time zone AS $body$ DECLARE @@ -197,31 +224,4 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { return fetchDate; END; $body$ LANGUAGE plpgsql IMMUTABLE` - ) - - return conn.BeginFunc(ctx, func(tx pgx.Tx) error { - if _, err := tx.Exec(ctx, createHistorySchema); err != nil { - return err - } - if _, err := tx.Exec(ctx, setupTriggerFunction); err != nil { - return err - } - if _, err := tx.Exec(ctx, setupParentFunction); err != nil { - return err - } - if _, err := tx.Exec(ctx, defineRetentionFunction); err != nil { - return err - } - if _, err := tx.Exec(ctx, cascadeDeleteFunction); err != nil { - return err - } - if _, err := tx.Exec(ctx, findLatestFetchDate); err != nil { - return err - } - return nil - }) -} - -func TransformDSN(inputDSN string) (string, error) { - return dsn.SetDSNElement(inputDSN, map[string]string{"search_path": schemaName}) -} +) From 57052e41b6ad2b1d94866e7808af002d560bce7d Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 18 Jan 2022 10:29:09 +0000 Subject: [PATCH 27/45] Use migrator version const from SDK --- go.mod | 4 ++-- go.sum | 4 ++-- pkg/client/client.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 53d02802e71b22..fab93010d9fc76 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117212543-7670f98adb8d + github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220118102630-832b00cf6e25 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -141,4 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117212543-7670f98adb8d => ../cq-provider-sdk +//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220118102630-832b00cf6e25 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index 99ed18e2083b89..4e1bd6eeb8cc5c 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117212543-7670f98adb8d h1:R0f8SLYyBWXpu5HMNhsx1l9sHn56NMxS6tmbMXL8yWk= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220117212543-7670f98adb8d/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220118102630-832b00cf6e25 h1:ieZ8LRZQbsSqtKGHlrcLy37OOuPDYQcuwisuSB6Y+Mo= +github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220118102630-832b00cf6e25/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/pkg/client/client.go b/pkg/client/client.go index 965491284a24c5..207f8dc0a0f1ec 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -949,7 +949,7 @@ func (c *Client) MigrateCore(ctx context.Context, de database.DialectExecutor) e } }() - if err := m.UpgradeProvider("latest"); err != nil && err != migrate.ErrNoChange { + if err := m.UpgradeProvider(migrator.Latest); err != nil && err != migrate.ErrNoChange { return fmt.Errorf("failed to migrate cloudquery core schema: %w", err) } return nil From ce4a26df138e08c29292b0386130a905c4c7b14f Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 18 Jan 2022 10:35:20 +0000 Subject: [PATCH 28/45] gocritic --- pkg/client/client.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 207f8dc0a0f1ec..8212413594fe04 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -272,10 +272,8 @@ func New(ctx context.Context, options ...Option) (*Client, error) { if c.DSN == "" { c.Logger.Warn("missing DSN, some commands won't work") - } else { - if err := c.initDatabase(ctx); err != nil { - return nil, err - } + } else if err := c.initDatabase(ctx); err != nil { + return nil, err } c.initModules() From 2d18fc83dc12fa1dec84666aef21c087d36a4a84 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 18 Jan 2022 10:47:12 +0000 Subject: [PATCH 29/45] Move history/ddlmanager/timescale stuff around --- .../timescale}/ddlmanager.go | 18 ++++++------------ pkg/client/database/timescale/timescale.go | 4 ++-- pkg/client/history/history.go | 8 ++++++++ 3 files changed, 16 insertions(+), 14 deletions(-) rename pkg/client/{history => database/timescale}/ddlmanager.go (93%) diff --git a/pkg/client/history/ddlmanager.go b/pkg/client/database/timescale/ddlmanager.go similarity index 93% rename from pkg/client/history/ddlmanager.go rename to pkg/client/database/timescale/ddlmanager.go index 2dd52e33f51b71..d85d10275f32fb 100644 --- a/pkg/client/history/ddlmanager.go +++ b/pkg/client/database/timescale/ddlmanager.go @@ -1,10 +1,10 @@ -package history +package timescale import ( "context" "fmt" - "github.com/cloudquery/cq-provider-sdk/database/dsn" + "github.com/cloudquery/cloudquery/pkg/client/history" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/georgysavva/scany/pgxscan" "github.com/hashicorp/go-hclog" @@ -20,18 +20,16 @@ const ( dropTableView = `DROP VIEW IF EXISTS "%[1]s"` createTableView = `CREATE VIEW "%[1]s" AS SELECT * FROM history."%[1]s" WHERE cq_fetch_date = find_latest('history', '%[1]s')` - - schemaName = "history" ) type DDLManager struct { log hclog.Logger conn *pgxpool.Conn - cfg *Config + cfg *history.Config dialect schema.Dialect } -func NewDDLManager(l hclog.Logger, conn *pgxpool.Conn, cfg *Config, dt schema.DialectType) (*DDLManager, error) { +func NewDDLManager(l hclog.Logger, conn *pgxpool.Conn, cfg *history.Config, dt schema.DialectType) (*DDLManager, error) { if dt != schema.TSDB { return nil, fmt.Errorf("history is only supported on timescaledb") } @@ -51,7 +49,7 @@ func NewDDLManager(l hclog.Logger, conn *pgxpool.Conn, cfg *Config, dt schema.Di func (h DDLManager) SetupHistory(ctx context.Context, conn *pgxpool.Conn) error { var tables []string - if err := pgxscan.Select(ctx, conn, &tables, listHyperTables, schemaName); err != nil { + if err := pgxscan.Select(ctx, conn, &tables, listHyperTables, history.SchemaName); err != nil { return fmt.Errorf("failed to list hypertables: %w", err) } @@ -68,7 +66,7 @@ func (h DDLManager) SetupHistory(ctx context.Context, conn *pgxpool.Conn) error } func (h DDLManager) configureHyperTable(ctx context.Context, conn *pgxpool.Conn, tableName string) error { - tName := fmt.Sprintf(`"%s"."%s"`, schemaName, tableName) + tName := fmt.Sprintf(`"%s"."%s"`, history.SchemaName, tableName) if _, err := conn.Exec(ctx, fmt.Sprintf(setChunkTimeInterval, h.cfg.TimeInterval), tName); err != nil { return err @@ -130,10 +128,6 @@ func AddHistoryFunctions(ctx context.Context, conn *pgxpool.Conn) error { }) } -func TransformDSN(inputDSN string) (string, error) { - return dsn.SetDSNElement(inputDSN, map[string]string{"search_path": schemaName}) -} - const ( createHistorySchema = `CREATE SCHEMA IF NOT EXISTS history;` cascadeDeleteFunction = ` diff --git a/pkg/client/database/timescale/timescale.go b/pkg/client/database/timescale/timescale.go index 666c5e4d509892..9bf04b3f91bfe5 100644 --- a/pkg/client/database/timescale/timescale.go +++ b/pkg/client/database/timescale/timescale.go @@ -39,7 +39,7 @@ func (e Executor) Setup(ctx context.Context) (string, error) { } defer conn.Release() - if err := history.AddHistoryFunctions(ctx, conn); err != nil { + if err := AddHistoryFunctions(ctx, conn); err != nil { return e.dsn, fmt.Errorf("failed to create history functions: %w", err) } @@ -84,7 +84,7 @@ func (e Executor) Finalize(ctx context.Context) error { } defer conn.Release() - ddl, err := history.NewDDLManager(e.logger, conn, e.cfg, schema.TSDB) + ddl, err := NewDDLManager(e.logger, conn, e.cfg, schema.TSDB) if err != nil { return err } diff --git a/pkg/client/history/history.go b/pkg/client/history/history.go index 1ded616867027e..f7dfaef1253b69 100644 --- a/pkg/client/history/history.go +++ b/pkg/client/history/history.go @@ -2,8 +2,12 @@ package history import ( "time" + + "github.com/cloudquery/cq-provider-sdk/database/dsn" ) +const SchemaName = "history" + type Config struct { // Retention of data in days, defaults to 7 Retention int `default:"7" hcl:"retention,optional"` @@ -17,3 +21,7 @@ type Config struct { func (c Config) FetchDate() time.Time { return time.Now().UTC().Truncate(time.Duration(c.TimeTruncation) * time.Hour) } + +func TransformDSN(inputDSN string) (string, error) { + return dsn.SetDSNElement(inputDSN, map[string]string{"search_path": SchemaName}) +} From dd0212dcc266f87a2616f58b9d98d1583a87ed3a Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 18 Jan 2022 12:07:07 +0000 Subject: [PATCH 30/45] document history.TransformDSN --- pkg/client/history/history.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/client/history/history.go b/pkg/client/history/history.go index f7dfaef1253b69..82b126c36dbd65 100644 --- a/pkg/client/history/history.go +++ b/pkg/client/history/history.go @@ -22,6 +22,7 @@ func (c Config) FetchDate() time.Time { return time.Now().UTC().Truncate(time.Duration(c.TimeTruncation) * time.Hour) } +// TransformDSN sets the search_path of the given DSN to the history schema func TransformDSN(inputDSN string) (string, error) { return dsn.SetDSNElement(inputDSN, map[string]string{"search_path": SchemaName}) } From a9fa69fee60d77222be583e895bc1ec223122c43 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 10:36:54 +0000 Subject: [PATCH 31/45] Update SDK --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index fab93010d9fc76..a97ccc79bba650 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220118102630-832b00cf6e25 + github.com/cloudquery/cq-provider-sdk v0.7.0-alpha github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -140,5 +140,3 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) - -//replace github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220118102630-832b00cf6e25 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index 4e1bd6eeb8cc5c..d58606c9804989 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220118102630-832b00cf6e25 h1:ieZ8LRZQbsSqtKGHlrcLy37OOuPDYQcuwisuSB6Y+Mo= -github.com/cloudquery/cq-provider-sdk v0.6.2-0.20220118102630-832b00cf6e25/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= +github.com/cloudquery/cq-provider-sdk v0.7.0-alpha h1:3lV8S448JGnHaHZRQAeMkszeujUh3ESwn6aCNdEc+so= +github.com/cloudquery/cq-provider-sdk v0.7.0-alpha/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= From 62b8cf708e27d795bb97cedbadd1e1a0f0e39884 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 12:39:23 +0000 Subject: [PATCH 32/45] Disable TestClient_ProviderSkipVersionMigrations --- pkg/client/client_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index a94a8a4eced087..b1645b11f9d995 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -354,6 +354,8 @@ func TestClient_ProviderMigrations(t *testing.T) { } +/* +// TODO re-enable after more migrations are introduced func TestClient_ProviderSkipVersionMigrations(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() @@ -412,8 +414,8 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { // migrations should be in 2 i.e v0.0.2 v, dirty, err := m.Version() assert.Equal(t, []interface{}{"v0.0.2", false, nil}, []interface{}{v, dirty, err}) - } +*/ const testConfig = `cloudquery { connection { From 05770d2368561eeeee4aefd63beeb588e0a66a21 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 12:47:33 +0000 Subject: [PATCH 33/45] Disable TestClient_ProviderMigrations --- pkg/client/client_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index b1645b11f9d995..1233eee3da31f3 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -16,10 +16,8 @@ import ( "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/cloudquery/cq-provider-sdk/serve" "github.com/fsnotify/fsnotify" - "github.com/golang-migrate/migrate/v4" "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2/hclparse" - "github.com/jackc/pgx/v4" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -310,6 +308,8 @@ func TestClient_ProviderUpgradeNoBuild(t *testing.T) { assert.NoError(t, err) } +/* +// TODO re-enable after more migrations are introduced func TestClient_ProviderMigrations(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() @@ -354,8 +354,6 @@ func TestClient_ProviderMigrations(t *testing.T) { } -/* -// TODO re-enable after more migrations are introduced func TestClient_ProviderSkipVersionMigrations(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() From 878eedaf4a760035bc1d7575e907b38d38102ac5 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 13:31:29 +0000 Subject: [PATCH 34/45] Check if history config is set --- go.mod | 2 ++ pkg/client/client.go | 6 +++++- pkg/client/database/database.go | 6 +++++- pkg/client/database/timescale/timescale.go | 9 ++++++--- pkg/client/database/timescale/timescale_test.go | 3 ++- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a97ccc79bba650..7984579acf42a0 100644 --- a/go.mod +++ b/go.mod @@ -140,3 +140,5 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) + +//replace github.com/cloudquery/cq-provider-sdk v0.7.0-alpha => ../cq-provider-sdk diff --git a/pkg/client/client.go b/pkg/client/client.go index 8212413594fe04..d1ab9587310265 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -1088,7 +1088,11 @@ func (c *Client) initDatabase(ctx context.Context) error { if c.HistoryCfg != nil && dt != schema.TSDB { // check if we're already on TSDB but the dsn is wrong - if ok, err := timescale.New(c.Logger, c.DSN, c.HistoryCfg).Validate(ctx); ok && err == nil { + ts, err := timescale.New(c.Logger, c.DSN, c.HistoryCfg) + if err != nil { + return err + } + if ok, err := ts.Validate(ctx); ok && err == nil { return fmt.Errorf("you must update the dsn to use tsdb:// prefix") } diff --git a/pkg/client/database/database.go b/pkg/client/database/database.go index 66dbb333cf8b04..29bf37e3993617 100644 --- a/pkg/client/database/database.go +++ b/pkg/client/database/database.go @@ -42,7 +42,11 @@ func GetExecutor(logger hclog.Logger, dsn string, c *history.Config) (schema.Dia case schema.Postgres: return dType, postgres.New(logger, dsn), nil case schema.TSDB: - return dType, timescale.New(logger, dsn, c), nil + ts, err := timescale.New(logger, dsn, c) + if err != nil { + return dType, nil, err + } + return dType, ts, nil default: return dType, nil, fmt.Errorf("unhandled dialect type") } diff --git a/pkg/client/database/timescale/timescale.go b/pkg/client/database/timescale/timescale.go index 9bf04b3f91bfe5..c3d89ae4a8c2f3 100644 --- a/pkg/client/database/timescale/timescale.go +++ b/pkg/client/database/timescale/timescale.go @@ -18,12 +18,15 @@ type Executor struct { cfg *history.Config } -func New(logger hclog.Logger, dsn string, cfg *history.Config) Executor { - return Executor{ +func New(logger hclog.Logger, dsn string, cfg *history.Config) (*Executor, error) { + if cfg == nil { + return nil, fmt.Errorf("missing history config") + } + return &Executor{ logger: logger, dsn: dsn, cfg: cfg, - } + }, nil } // Setup sets all required history functions and validation checks that it can run cleanly. diff --git a/pkg/client/database/timescale/timescale_test.go b/pkg/client/database/timescale/timescale_test.go index ef0f9a37146bf4..444f53bc733195 100644 --- a/pkg/client/database/timescale/timescale_test.go +++ b/pkg/client/database/timescale/timescale_test.go @@ -51,11 +51,12 @@ var testTable = &schema.Table{ func TestSetupHistory(t *testing.T) { ctx := context.TODO() - ts := New(hclog.L(), testDBConnection, &history.Config{ + ts, err := New(hclog.L(), testDBConnection, &history.Config{ Retention: 1, TimeInterval: 1, TimeTruncation: 24, }) + assert.NoError(t, err) ok, err := ts.Validate(ctx) assert.NoError(t, err) From d6fbb31bb8d1a9e2c922ca95703e791365786749 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 16:44:23 +0000 Subject: [PATCH 35/45] Add error message if running on old provider schema --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/client/client.go | 10 ++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 7984579acf42a0..459e166403346a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.7.0-alpha + github.com/cloudquery/cq-provider-sdk v0.7.0-alpha.0.20220119154925-739f3acc9e08 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -141,4 +141,4 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -//replace github.com/cloudquery/cq-provider-sdk v0.7.0-alpha => ../cq-provider-sdk +//replace github.com/cloudquery/cq-provider-sdk v0.7.0-alpha.0.20220119154925-739f3acc9e08 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index d58606c9804989..3a6976e2a2c663 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.7.0-alpha h1:3lV8S448JGnHaHZRQAeMkszeujUh3ESwn6aCNdEc+so= -github.com/cloudquery/cq-provider-sdk v0.7.0-alpha/go.mod h1:UG9fpw8UPqIMkKOoPAQ3wi4PU7/aGvhFAbE/7pRE37g= +github.com/cloudquery/cq-provider-sdk v0.7.0-alpha.0.20220119154925-739f3acc9e08 h1:rnmAkH3+ffP2nykRsHuJeephJu6EHltD5Pg0lNFGdAk= +github.com/cloudquery/cq-provider-sdk v0.7.0-alpha.0.20220119154925-739f3acc9e08/go.mod h1:T+ngRXzcjJ6otKDGkWnPrHTsZuHUe3KZKtyhSLcvHCs= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -257,8 +257,8 @@ github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7 github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= -github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw= -github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= +github.com/containerd/containerd v1.5.9 h1:rs6Xg1gtIxaeyG+Smsb/0xaSDu1VgFhOCKBXxMxbsF4= +github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= diff --git a/pkg/client/client.go b/pkg/client/client.go index d1ab9587310265..dc6dcad06a71f4 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "io/fs" "path/filepath" "sort" "strconv" @@ -641,6 +642,15 @@ func (c *Client) BuildProviderTables(ctx context.Context, providerName string) ( return nil } + defer func() { + if retErr == nil || !errors.Is(retErr, fs.ErrNotExist) { + return + } + + c.Logger.Error("BuildProviderTables failed", "error", retErr) + retErr = fmt.Errorf("Incompatible provider schema: Please drop provider tables and recreate, alternatively execute `cq provider drop %s`", providerName) + }() + // create migration table and set it to version based on latest create table m, cfg, err := c.buildProviderMigrator(ctx, s.Migrations, providerName) if err != nil { From a4ef45a7315f2b21761c20e8ae00d6011ec78d99 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 17:19:40 +0000 Subject: [PATCH 36/45] Revert "Disable TestClient_ProviderMigrations" This reverts commit 05770d2368561eeeee4aefd63beeb588e0a66a21. --- pkg/client/client_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 1233eee3da31f3..b1645b11f9d995 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -16,8 +16,10 @@ import ( "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/cloudquery/cq-provider-sdk/serve" "github.com/fsnotify/fsnotify" + "github.com/golang-migrate/migrate/v4" "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2/hclparse" + "github.com/jackc/pgx/v4" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -308,8 +310,6 @@ func TestClient_ProviderUpgradeNoBuild(t *testing.T) { assert.NoError(t, err) } -/* -// TODO re-enable after more migrations are introduced func TestClient_ProviderMigrations(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() @@ -354,6 +354,8 @@ func TestClient_ProviderMigrations(t *testing.T) { } +/* +// TODO re-enable after more migrations are introduced func TestClient_ProviderSkipVersionMigrations(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() From a7e94e0c8efe7653e652c4038e17090f2dbeeed6 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 17:19:41 +0000 Subject: [PATCH 37/45] Revert "Disable TestClient_ProviderSkipVersionMigrations" This reverts commit 62b8cf708e27d795bb97cedbadd1e1a0f0e39884. --- pkg/client/client_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index b1645b11f9d995..a94a8a4eced087 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -354,8 +354,6 @@ func TestClient_ProviderMigrations(t *testing.T) { } -/* -// TODO re-enable after more migrations are introduced func TestClient_ProviderSkipVersionMigrations(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() @@ -414,8 +412,8 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { // migrations should be in 2 i.e v0.0.2 v, dirty, err := m.Version() assert.Equal(t, []interface{}{"v0.0.2", false, nil}, []interface{}{v, dirty, err}) + } -*/ const testConfig = `cloudquery { connection { From 08fdc9866e2806ea4e001dbd2bb2135429a14eb0 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 17:32:11 +0000 Subject: [PATCH 38/45] Fix client_test --- .../migrations/postgres/1_v0.0.1.down.sql | 2 -- .../migrations/postgres/1_v0.0.1.up.sql | 4 ---- .../migrations/timescale/1_v0.0.1.down.sql | 2 -- .../migrations/timescale/1_v0.0.1.up.sql | 4 ---- pkg/client/client_test.go | 23 ++++++++++--------- 5 files changed, 12 insertions(+), 23 deletions(-) diff --git a/internal/test/provider/migrations/postgres/1_v0.0.1.down.sql b/internal/test/provider/migrations/postgres/1_v0.0.1.down.sql index 198ac7836003cc..19db8ca6e8d45d 100644 --- a/internal/test/provider/migrations/postgres/1_v0.0.1.down.sql +++ b/internal/test/provider/migrations/postgres/1_v0.0.1.down.sql @@ -1,5 +1,3 @@ --- Autogenerated by migration tool on 2022-01-14 19:05:53 - -- Resource: error_resource DROP TABLE IF EXISTS error_resource; diff --git a/internal/test/provider/migrations/postgres/1_v0.0.1.up.sql b/internal/test/provider/migrations/postgres/1_v0.0.1.up.sql index 1e568db0ac21dd..c8b26fb0669960 100644 --- a/internal/test/provider/migrations/postgres/1_v0.0.1.up.sql +++ b/internal/test/provider/migrations/postgres/1_v0.0.1.up.sql @@ -1,5 +1,3 @@ --- Autogenerated by migration tool on 2022-01-14 19:05:53 - -- Resource: error_resource CREATE TABLE IF NOT EXISTS "error_resource" ( "cq_id" uuid NOT NULL, @@ -13,8 +11,6 @@ CREATE TABLE IF NOT EXISTS "slow_resource" ( "cq_id" uuid NOT NULL, "cq_meta" jsonb, "some_bool" boolean, - "upgrade_column" integer, - "upgrade_column_2" integer, CONSTRAINT slow_resource_pk PRIMARY KEY(cq_id), UNIQUE(cq_id) ); diff --git a/internal/test/provider/migrations/timescale/1_v0.0.1.down.sql b/internal/test/provider/migrations/timescale/1_v0.0.1.down.sql index 198ac7836003cc..19db8ca6e8d45d 100644 --- a/internal/test/provider/migrations/timescale/1_v0.0.1.down.sql +++ b/internal/test/provider/migrations/timescale/1_v0.0.1.down.sql @@ -1,5 +1,3 @@ --- Autogenerated by migration tool on 2022-01-14 19:05:53 - -- Resource: error_resource DROP TABLE IF EXISTS error_resource; diff --git a/internal/test/provider/migrations/timescale/1_v0.0.1.up.sql b/internal/test/provider/migrations/timescale/1_v0.0.1.up.sql index 814af9787dc4d4..951a2e57f2e657 100644 --- a/internal/test/provider/migrations/timescale/1_v0.0.1.up.sql +++ b/internal/test/provider/migrations/timescale/1_v0.0.1.up.sql @@ -1,5 +1,3 @@ --- Autogenerated by migration tool on 2022-01-14 19:05:53 - -- Resource: error_resource CREATE TABLE IF NOT EXISTS "error_resource" ( "cq_id" uuid NOT NULL, @@ -15,8 +13,6 @@ CREATE TABLE IF NOT EXISTS "slow_resource" ( "cq_meta" jsonb, "cq_fetch_date" timestamp without time zone NOT NULL, "some_bool" boolean, - "upgrade_column" integer, - "upgrade_column_2" integer, CONSTRAINT slow_resource_pk PRIMARY KEY(cq_fetch_date,cq_id), UNIQUE(cq_fetch_date,cq_id) ); diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index a94a8a4eced087..61606c07bacf20 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -341,7 +341,7 @@ func TestClient_ProviderMigrations(t *testing.T) { c.Providers[0].Version = "v0.0.1" err = c.DowngradeProvider(ctx, "test") assert.NoError(t, err) - _, err = conn.Exec(ctx, "select some_bool, upgrade_column from slow_resource") + _, err = conn.Exec(ctx, "select some_bool from slow_resource") assert.NoError(t, err) _, err = conn.Exec(ctx, "select some_bool, upgrade_column, upgrade_column_2 from slow_resource") assert.Error(t, err) @@ -349,7 +349,7 @@ func TestClient_ProviderMigrations(t *testing.T) { c.Providers[0].Version = "v0.0.2" err = c.UpgradeProvider(ctx, "test") assert.NoError(t, err) - _, err = conn.Exec(ctx, "select some_bool, upgrade_column, upgrade_column_2 from slow_resource") + _, err = conn.Exec(ctx, "select some_bool, upgrade_column from slow_resource") assert.NoError(t, err) } @@ -359,7 +359,7 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { defer cancelServe() c, err := New(context.Background(), func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" + options.DSN = "postgres://postgres:pass@localhost:15432/postgres?sslmode=disable" options.Providers = requiredTestProviders }) assert.Nil(t, err) @@ -374,7 +374,7 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { err = c.UpgradeProvider(ctx, "test") assert.ErrorIs(t, err, migrate.ErrNoChange) - conn, err := pgx.Connect(ctx, "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable") + conn, err := pgx.Connect(ctx, "postgres://postgres:pass@localhost:15432/postgres?sslmode=disable") if err != nil { assert.FailNow(t, "failed to create connection") return @@ -385,13 +385,13 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { c.Providers[0].Version = "v0.0.1" err = c.DowngradeProvider(ctx, "test") assert.Nil(t, err) - _, err = conn.Exec(ctx, "select some_bool, upgrade_column from slow_resource") + _, err = conn.Exec(ctx, "select some_bool from slow_resource") assert.Nil(t, err) _, err = conn.Exec(ctx, "select some_bool, upgrade_column, upgrade_column_2 from slow_resource") assert.Error(t, err) c.Providers[0].Version = "v0.0.5" - // latest migration should be to v0.0.2 + // latest migration should be to v0.0.3 err = c.UpgradeProvider(ctx, "test") assert.Nil(t, err) _, err = conn.Exec(ctx, "select some_bool, upgrade_column, upgrade_column_2 from slow_resource") @@ -404,14 +404,15 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { "1_v0.0.1.down.sql": []byte(""), "2_v0.0.2.up.sql": []byte(""), "2_v0.0.2.down.sql": []byte(""), + "3_v0.0.3.up.sql": []byte(""), + "3_v0.0.3.down.sql": []byte(""), }, }, "test") - if err != nil { - t.Fatal(err) - } - // migrations should be in 2 i.e v0.0.2 + assert.NoError(t, err) + + // migrations should be in 3 i.e v0.0.3 v, dirty, err := m.Version() - assert.Equal(t, []interface{}{"v0.0.2", false, nil}, []interface{}{v, dirty, err}) + assert.Equal(t, []interface{}{"v0.0.3", false, nil}, []interface{}{v, dirty, err}) } From fa9bd16d69922657059d3398d9f71d383cc4c098 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 17:36:06 +0000 Subject: [PATCH 39/45] Wrong port --- pkg/client/client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 61606c07bacf20..eb81d4a4127a75 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -359,7 +359,7 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { defer cancelServe() c, err := New(context.Background(), func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:15432/postgres?sslmode=disable" + options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" options.Providers = requiredTestProviders }) assert.Nil(t, err) @@ -374,7 +374,7 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { err = c.UpgradeProvider(ctx, "test") assert.ErrorIs(t, err, migrate.ErrNoChange) - conn, err := pgx.Connect(ctx, "postgres://postgres:pass@localhost:15432/postgres?sslmode=disable") + conn, err := pgx.Connect(ctx, "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable") if err != nil { assert.FailNow(t, "failed to create connection") return From a25e0c0db49a9ad4026584696adb16f6ee0fa600 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 18:45:01 +0000 Subject: [PATCH 40/45] Fix tests --- pkg/client/client_test.go | 164 +++++++++++++++++++++++++++++--------- 1 file changed, 126 insertions(+), 38 deletions(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index eb81d4a4127a75..93ef0d0a62b491 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -3,10 +3,12 @@ package client import ( "context" "errors" + "math/rand" "net" "os" "path/filepath" "reflect" + "strconv" "strings" "testing" "time" @@ -25,24 +27,66 @@ import ( "github.com/stretchr/testify/require" ) -var ( - providerSrc = "cloudquery" - requiredTestProviders = []*config.RequiredProvider{ +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func requiredTestProviders() []*config.RequiredProvider { + providerSrc := "cloudquery" + return []*config.RequiredProvider{ { Name: "test", Source: &providerSrc, Version: "latest", }, } -) +} + +func setupDB(t *testing.T) (dsn string) { + baseDSN := os.Getenv("CQ_CLIENT_TEST_DSN") + if baseDSN == "" { + baseDSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" + } + + conn, err := pgx.Connect(context.Background(), baseDSN) + if err != nil { + assert.FailNow(t, "failed to create connection") + return + } + + newDB := "test_" + strconv.Itoa(rand.Int()) + + _, err = conn.Exec(context.Background(), "CREATE DATABASE "+newDB) + assert.NoError(t, err) + + t.Cleanup(func() { + defer conn.Close(context.Background()) + + //if t.Failed() { + // t.Log("Not dropping database", newDB) + // return + //} + + if _, err := conn.Exec(context.Background(), "DROP DATABASE "+newDB+" WITH(FORCE)"); err != nil { + t.Logf("teardown: drop database failed: %v", err) + } + }) + + return strings.Replace(baseDSN, "/postgres?", "/"+newDB+"?", 1) +} func TestClient_FailOnFetchWithPartialFetch(t *testing.T) { ctx := context.Background() + + dbDSN := setupDB(t) + c, err := New(ctx, func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.Nil(t, err) + t.Cleanup(c.Close) + // download test provider if it doesn't already exist err = c.DownloadProviders(ctx) assert.Nil(t, err) @@ -69,11 +113,15 @@ func TestClient_FailOnFetchWithPartialFetch(t *testing.T) { func TestClient_FailOnFetch(t *testing.T) { ctx := context.Background() + dbDSN := setupDB(t) + c, err := New(ctx, func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.Nil(t, err) + t.Cleanup(c.Close) + // download test provider if it doesn't already exist err = c.DownloadProviders(ctx) assert.Nil(t, err) @@ -99,11 +147,15 @@ func TestClient_FailOnFetch(t *testing.T) { func TestClient_PartialFetch(t *testing.T) { ctx := context.Background() + dbDSN := setupDB(t) + c, err := New(ctx, func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.Nil(t, err) + t.Cleanup(c.Close) + // download test provider if it doesn't already exist err = c.DownloadProviders(ctx) assert.Nil(t, err) @@ -128,12 +180,15 @@ func TestClient_PartialFetch(t *testing.T) { func TestClient_TestNoDownload(t *testing.T) { _ = os.RemoveAll(".cq/downloadTest") + c, err := New(context.Background(), func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = setupDB(t) + options.Providers = requiredTestProviders() options.PluginDirectory = ".cq/downloadTest" }) assert.Nil(t, err) + t.Cleanup(c.Close) + _, err = c.Manager.GetPluginDetails("test") assert.Error(t, err) @@ -147,12 +202,14 @@ func TestClient_TestNoDownload(t *testing.T) { pd, err := c.Manager.GetPluginDetails("test") assert.Nil(t, err) assert.Equal(t, "test", pd.Name) + c, err = New(context.Background(), func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = setupDB(t) + options.Providers = requiredTestProviders() options.PluginDirectory = ".cq/downloadTest" }) assert.Nil(t, err) + t.Cleanup(c.Close) pd2, err := c.Manager.GetPluginDetails("test") assert.Nil(t, err) assert.Equal(t, pd2.FilePath, pd.FilePath) @@ -164,16 +221,20 @@ func TestClient_TestNoDownload(t *testing.T) { func TestClient_FetchTimeout(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() + + dbDSN := setupDB(t) + c, err := New(context.Background(), func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.Nil(t, err) if c == nil { assert.FailNow(t, "failed to create client") } + t.Cleanup(c.Close) assert.Nil(t, err) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() _, err = c.Fetch(ctx, FetchRequest{ Providers: []*config.Provider{ @@ -192,18 +253,24 @@ func TestClient_FetchTimeout(t *testing.T) { func TestClient_FetchNilConfig(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() - cfg, diags := config.NewParser().LoadConfigFromSource("config.hcl", []byte(testConfig)) + + dbDSN := setupDB(t) + + testCfg := []byte(strings.Replace(testConfig, "DSN_PLACEHOLDER", `"`+dbDSN+`"`, 1)) + + cfg, diags := config.NewParser().LoadConfigFromSource("config.hcl", testCfg) assert.Nil(t, diags) // Set configuration to nil cfg.Providers[0].Configuration = nil c, err := New(context.Background(), func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.Nil(t, err) if c == nil { assert.FailNow(t, "failed to create client") } + t.Cleanup(c.Close) ctx := context.Background() _, err = c.Fetch(ctx, FetchRequest{ Providers: []*config.Provider{ @@ -220,15 +287,18 @@ func TestClient_FetchNilConfig(t *testing.T) { func TestClient_Fetch(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() + + dbDSN := setupDB(t) + c, err := New(context.Background(), func(options *Client) { - options.DSN = "host=localhost user=postgres password=pass database=postgres port=5432 sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.Nil(t, err) if c == nil { assert.FailNow(t, "failed to create client") } - assert.Nil(t, err) + t.Cleanup(c.Close) ctx := context.Background() _, err = c.Fetch(ctx, FetchRequest{ @@ -245,15 +315,19 @@ func TestClient_Fetch(t *testing.T) { func TestClient_GetProviderSchema(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() + + dbDSN := setupDB(t) + c, err := New(context.Background(), func(options *Client) { - options.DSN = "host=localhost user=postgres password=pass database=postgres port=5432 sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.Nil(t, err) if c == nil { assert.FailNow(t, "failed to create client") return } + t.Cleanup(c.Close) ctx := context.Background() s, err := c.GetProviderSchema(ctx, "test") if s == nil { @@ -269,15 +343,18 @@ func TestClient_GetProviderConfig(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() + dbDSN := setupDB(t) + c, err := New(context.Background(), func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.Nil(t, err) if c == nil { assert.FailNow(t, "failed to create client") return } + t.Cleanup(c.Close) ctx := context.Background() pConfig, err := c.GetProviderConfiguration(ctx, "test") @@ -295,14 +372,17 @@ func TestClient_ProviderUpgradeNoBuild(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() + dbDSN := setupDB(t) + c, err := New(context.Background(), func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.NoError(t, err) if c == nil { assert.FailNow(t, "failed to create client") } + t.Cleanup(c.Close) ctx := context.Background() err = c.DropProvider(ctx, "test") assert.NoError(t, err) @@ -314,14 +394,17 @@ func TestClient_ProviderMigrations(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() + dbDSN := setupDB(t) + c, err := New(context.Background(), func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.NoError(t, err) if c == nil { assert.FailNow(t, "failed to create client") } + t.Cleanup(c.Close) ctx := context.Background() err = c.DropProvider(ctx, "test") assert.NoError(t, err) @@ -330,7 +413,7 @@ func TestClient_ProviderMigrations(t *testing.T) { err = c.UpgradeProvider(ctx, "test") assert.ErrorIs(t, err, migrate.ErrNoChange) - conn, err := pgx.Connect(ctx, "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable") + conn, err := pgx.Connect(ctx, dbDSN) if err != nil { assert.FailNow(t, "failed to create connection") return @@ -358,14 +441,17 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { cancelServe := setupTestPlugin(t) defer cancelServe() + dbDSN := setupDB(t) + c, err := New(context.Background(), func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" - options.Providers = requiredTestProviders + options.DSN = dbDSN + options.Providers = requiredTestProviders() }) assert.Nil(t, err) if c == nil { assert.FailNow(t, "failed to create client") } + t.Cleanup(c.Close) ctx := context.Background() err = c.DropProvider(ctx, "test") assert.Nil(t, err) @@ -374,7 +460,7 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { err = c.UpgradeProvider(ctx, "test") assert.ErrorIs(t, err, migrate.ErrNoChange) - conn, err := pgx.Connect(ctx, "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable") + conn, err := pgx.Connect(ctx, dbDSN) if err != nil { assert.FailNow(t, "failed to create connection") return @@ -418,7 +504,7 @@ func TestClient_ProviderSkipVersionMigrations(t *testing.T) { const testConfig = `cloudquery { connection { - dsn = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" + dsn = DSN_PLACEHOLDER } provider "test" { source = "cloudquery" @@ -658,7 +744,7 @@ func Test_CheckForProviderUpdates(t *testing.T) { { Name: "test", Source: &source, - Version: "v0.0.9", + Version: "v0.0.11", }, }, 0, @@ -678,11 +764,13 @@ func Test_CheckForProviderUpdates(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() + c, err := New(ctx, func(options *Client) { - options.DSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" + options.DSN = setupDB(t) options.Providers = tt.providers }) assert.Nil(t, err) + t.Cleanup(c.Close) providers, err := c.CheckForProviderUpdates(ctx) assert.Nil(t, err) assert.Len(t, providers, tt.updates) From 9340bcb029d37a7275c0d34481736cb65225638f Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 18:47:32 +0000 Subject: [PATCH 41/45] lint --- pkg/client/client_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 93ef0d0a62b491..20d4e4c0c98923 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -62,10 +62,10 @@ func setupDB(t *testing.T) (dsn string) { t.Cleanup(func() { defer conn.Close(context.Background()) - //if t.Failed() { - // t.Log("Not dropping database", newDB) - // return - //} + if os.Getenv("CQ_TEST_DEBUG") != "" && t.Failed() { + t.Log("Not dropping database", newDB) + return + } if _, err := conn.Exec(context.Background(), "DROP DATABASE "+newDB+" WITH(FORCE)"); err != nil { t.Logf("teardown: drop database failed: %v", err) From 70ae7e3ee3923e75e51556c48028abdbf97f1d6d Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 18:56:46 +0000 Subject: [PATCH 42/45] Update timescale test --- .../database/timescale/timescale_test.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/client/database/timescale/timescale_test.go b/pkg/client/database/timescale/timescale_test.go index 444f53bc733195..5cf96f4b7d9d73 100644 --- a/pkg/client/database/timescale/timescale_test.go +++ b/pkg/client/database/timescale/timescale_test.go @@ -6,6 +6,7 @@ package timescale import ( "context" "fmt" + "os" "strings" "testing" "time" @@ -18,10 +19,6 @@ import ( "github.com/stretchr/testify/assert" ) -const ( - testDBConnection = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" // timescale -) - var testTable = &schema.Table{ Name: "test_table", Columns: []schema.Column{ @@ -49,9 +46,17 @@ var testTable = &schema.Table{ Options: schema.TableCreationOptions{PrimaryKeys: []string{"id"}}, } +func getDSN() { + dbDSN := os.Getenv("CQ_TIMESCALE_TEST_DSN") + if dbDSN == "" { + dbDSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" // timescale + } + return dbDSN +} + func TestSetupHistory(t *testing.T) { ctx := context.TODO() - ts, err := New(hclog.L(), testDBConnection, &history.Config{ + ts, err := New(hclog.L(), getDSN(), &history.Config{ Retention: 1, TimeInterval: 1, TimeTruncation: 24, @@ -81,7 +86,7 @@ func TestSetupHistory(t *testing.T) { newDowns := make([]string, len(downs)) for i, sql := range downs { if strings.HasPrefix(sql, "DROP TABLE ") { - sql += " CASCADE" + sql = strings.TrimSuffix(sql, ";") + " CASCADE" } newDowns[i] = sql } @@ -107,7 +112,7 @@ func TestSetupHistory(t *testing.T) { assert.NoError(t, err) }) - pool, err := pgsdk.Connect(ctx, testDBConnection) + pool, err := pgsdk.Connect(ctx, getDSN()) assert.NoError(t, err) defer pool.Close() From 3a0d33627862df0a77316097e7200e186989ceb4 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 19 Jan 2022 19:01:38 +0000 Subject: [PATCH 43/45] oops --- pkg/client/database/timescale/timescale_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/database/timescale/timescale_test.go b/pkg/client/database/timescale/timescale_test.go index 5cf96f4b7d9d73..fba4f5dee873fb 100644 --- a/pkg/client/database/timescale/timescale_test.go +++ b/pkg/client/database/timescale/timescale_test.go @@ -46,7 +46,7 @@ var testTable = &schema.Table{ Options: schema.TableCreationOptions{PrimaryKeys: []string{"id"}}, } -func getDSN() { +func getDSN() string { dbDSN := os.Getenv("CQ_TIMESCALE_TEST_DSN") if dbDSN == "" { dbDSN = "postgres://postgres:pass@localhost:5432/postgres?sslmode=disable" // timescale From edb8d0d07964b3c98916c66905c34ae810726df3 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 20 Jan 2022 10:16:13 +0000 Subject: [PATCH 44/45] CR nit --- pkg/client/database/timescale/timescale.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/client/database/timescale/timescale.go b/pkg/client/database/timescale/timescale.go index c3d89ae4a8c2f3..8c37c61ccdaa71 100644 --- a/pkg/client/database/timescale/timescale.go +++ b/pkg/client/database/timescale/timescale.go @@ -12,6 +12,10 @@ import ( "github.com/hashicorp/go-hclog" ) +const ( + validateTimescaleInstalled = `SELECT EXISTS(SELECT 1 FROM pg_extension where extname = 'timescaledb')` +) + type Executor struct { logger hclog.Logger dsn string @@ -48,12 +52,7 @@ func (e Executor) Setup(ctx context.Context) (string, error) { return history.TransformDSN(e.dsn) } - func (e Executor) Validate(ctx context.Context) (bool, error) { - const ( - validateTimescaleInstalled = `SELECT EXISTS(SELECT 1 FROM pg_extension where extname = 'timescaledb')` - ) - pool, err := pgsdk.Connect(ctx, e.dsn) if err != nil { return false, err From 02e7a10decc65b7be6db4b4d2af5f0779e379f66 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 20 Jan 2022 12:35:59 +0000 Subject: [PATCH 45/45] Update SDK --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 459e166403346a..8287caa97a7c79 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.7.0-alpha.0.20220119154925-739f3acc9e08 + github.com/cloudquery/cq-provider-sdk v0.7.0-alpha2 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 @@ -140,5 +140,3 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) - -//replace github.com/cloudquery/cq-provider-sdk v0.7.0-alpha.0.20220119154925-739f3acc9e08 => ../cq-provider-sdk diff --git a/go.sum b/go.sum index 3a6976e2a2c663..bacc88c831fda6 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.7.0-alpha.0.20220119154925-739f3acc9e08 h1:rnmAkH3+ffP2nykRsHuJeephJu6EHltD5Pg0lNFGdAk= -github.com/cloudquery/cq-provider-sdk v0.7.0-alpha.0.20220119154925-739f3acc9e08/go.mod h1:T+ngRXzcjJ6otKDGkWnPrHTsZuHUe3KZKtyhSLcvHCs= +github.com/cloudquery/cq-provider-sdk v0.7.0-alpha2 h1:GY0NJLEYf5JSHluVJsdAfFN00ygX5A+HZHw6/LDif5Q= +github.com/cloudquery/cq-provider-sdk v0.7.0-alpha2/go.mod h1:T+ngRXzcjJ6otKDGkWnPrHTsZuHUe3KZKtyhSLcvHCs= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=