diff --git a/.github/workflows/history_test.yml b/.github/workflows/history_test.yml index 11a5e6283f32c1..4250b31abff628 100644 --- a/.github/workflows/history_test.yml +++ b/.github/workflows/history_test.yml @@ -47,6 +47,11 @@ jobs: - name: Build run: go build -v . + - name: Sanity Fetch with old version + run: go run ./main.go fetch --config=internal/test/test_history_config_oldversion.hcl --enable-console-log + env: + CQ_NO_TELEMETRY: 1 + - name: Sanity Fetch run: go run ./main.go fetch --config=internal/test/test_history_config.hcl --enable-console-log env: diff --git a/internal/test/test_history_config_oldversion.hcl b/internal/test/test_history_config_oldversion.hcl new file mode 100644 index 00000000000000..97bff2a785f3a3 --- /dev/null +++ b/internal/test/test_history_config_oldversion.hcl @@ -0,0 +1,25 @@ +cloudquery { + + connection { + dsn = "tsdb://postgres:pass@localhost:5432/postgres?sslmode=disable" + } + provider "test" { + version = "v0.0.10" + } + history { + // Save data retention for 7 days + retention = 7 + // Truncate our fetch by 6 hours per fetch + truncation = 6 + } + +} + +// All Provider Configurations +provider "test" { + configuration {} + + resources = [ + "slow_resource" + ] +} \ No newline at end of file diff --git a/pkg/client/database/timescale/ddlmanager.go b/pkg/client/database/timescale/ddlmanager.go index d85d10275f32fb..b059f4fd6a2499 100644 --- a/pkg/client/database/timescale/ddlmanager.go +++ b/pkg/client/database/timescale/ddlmanager.go @@ -47,6 +47,17 @@ func NewDDLManager(l hclog.Logger, conn *pgxpool.Conn, cfg *history.Config, dt s }, nil } +// PrepareHistory is run before any migrations +func (h DDLManager) PrepareHistory(ctx context.Context, conn *pgxpool.Conn) error { + if err := AddHistoryFunctions(ctx, conn); err != nil { + return fmt.Errorf("AddHistoryFunctions failed: %w", err) + } + + // we need to drop the views before underlying tables can be modified + return h.dropViews(ctx, conn) +} + +// SetupHistory is run after any migrations, finalizing history setup func (h DDLManager) SetupHistory(ctx context.Context, conn *pgxpool.Conn) error { var tables []string if err := pgxscan.Select(ctx, conn, &tables, listHyperTables, history.SchemaName); err != nil { @@ -82,6 +93,25 @@ func (h DDLManager) configureHyperTable(ctx context.Context, conn *pgxpool.Conn, return nil } +func (h DDLManager) dropViews(ctx context.Context, conn *pgxpool.Conn) error { + var tables []string + if err := pgxscan.Select(ctx, conn, &tables, listHyperTables, history.SchemaName); err != nil { + return fmt.Errorf("failed to list hypertables: %w", err) + } + + if err := conn.BeginTxFunc(ctx, pgx.TxOptions{}, func(tx pgx.Tx) error { + for _, table := range tables { + if _, err := tx.Exec(ctx, fmt.Sprintf(dropTableView, table)); err != nil { + return fmt.Errorf("failed to drop view for table: %w", err) + } + } + return nil + }); err != nil { + return err + } + return nil +} + func (h DDLManager) recreateView(ctx context.Context, conn *pgxpool.Conn, table string) error { if err := conn.BeginTxFunc(ctx, pgx.TxOptions{}, func(tx pgx.Tx) error { // Must drop the view first -- CREATE OR REPLACE view won't cut it if columns are changed. PostgreSQL doc states: diff --git a/pkg/client/database/timescale/timescale.go b/pkg/client/database/timescale/timescale.go index 8c37c61ccdaa71..3da1c5d9a5663f 100644 --- a/pkg/client/database/timescale/timescale.go +++ b/pkg/client/database/timescale/timescale.go @@ -46,8 +46,12 @@ func (e Executor) Setup(ctx context.Context) (string, error) { } defer conn.Release() - if err := AddHistoryFunctions(ctx, conn); err != nil { - return e.dsn, fmt.Errorf("failed to create history functions: %w", err) + ddl, err := NewDDLManager(e.logger, conn, e.cfg, schema.TSDB) + if err != nil { + return e.dsn, err + } + if err := ddl.PrepareHistory(ctx, conn); err != nil { + return e.dsn, fmt.Errorf("failed to prepare history: %w", err) } return history.TransformDSN(e.dsn) diff --git a/pkg/module/manager.go b/pkg/module/manager.go index c3d27f66ed6796..219eaf6ac4aed9 100644 --- a/pkg/module/manager.go +++ b/pkg/module/manager.go @@ -3,6 +3,7 @@ package module import ( "context" "fmt" + "sort" "strings" "github.com/cloudquery/cq-provider-sdk/cqproto" @@ -168,6 +169,10 @@ func versionError(modName string, modVersions []uint32, provVersions map[string] } } + sort.Strings(unsupportingProviders) + sort.Strings(olderProviders) + sort.Strings(newerProviders) + if l := len(unsupportingProviders); l == 1 { return fmt.Errorf("provider %s doesn't support %s yet", unsupportingProviders[0], modName) } else if l > 1 {