From 034b324b29aef59456474dc0b1888eb42601cfcf Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Tue, 28 Apr 2026 15:02:51 +0200 Subject: [PATCH 1/3] Add schema migration support Add schema migration generation, application, upload, download, and check support for SQLite and PostgreSQL. Introduce the cloudsync_alter_* API, pending migration tracking, raw SQL migration operations, createTable payload support, and block-level LWW/augment migration commands. Update libcurl and Apple native network layers, add SQLite/PostgreSQL/cross-engine tests, refresh examples and documentation, and remove the old begin/commit alter API. --- API.md | 465 +++- Makefile | 16 +- docker/Makefile.postgresql | 1 + docker/postgresql/docker-compose.debug.yml | 1 + docs/internal/schema-migrations.md | 309 +++ examples/README.md | 8 +- examples/schema-migrations/README.md | 70 + .../client-to-server-v1.json | 31 + .../schema-migrations/client-to-server.sql | 21 + .../schema-migrations/dialect-overrides.sql | 11 + examples/schema-migrations/raw-sql.sql | 15 + .../server-to-client-v2.json | 47 + src/cloudsync.c | 13 +- src/cloudsync.h | 32 +- src/migration.c | 2043 +++++++++++++++++ src/network/network.c | 596 ++++- src/network/network.m | 288 +-- src/network/network_private.h | 4 + src/postgresql/cloudsync.sql.in | 137 +- src/postgresql/cloudsync_postgresql.c | 538 ++++- src/postgresql/database_postgresql.c | 8 +- .../migrations/cloudsync--1.0--1.1.sql | 140 ++ src/sqlite/cloudsync_sqlite.c | 384 +++- src/sqlite/database_sqlite.c | 3 +- src/utils.c | 2 +- test/integration.c | 531 ++++- test/postgresql/31_alter_table_sync.sql | 42 +- test/postgresql/52_schema_migrations.sql | 1509 ++++++++++++ test/postgresql/full_test.sql | 1 + test/schema_migration_cross_dialect.sh | 166 ++ test/unit.c | 1185 +++++++++- 31 files changed, 8135 insertions(+), 482 deletions(-) create mode 100644 docs/internal/schema-migrations.md create mode 100644 examples/schema-migrations/README.md create mode 100644 examples/schema-migrations/client-to-server-v1.json create mode 100644 examples/schema-migrations/client-to-server.sql create mode 100644 examples/schema-migrations/dialect-overrides.sql create mode 100644 examples/schema-migrations/raw-sql.sql create mode 100644 examples/schema-migrations/server-to-client-v2.json create mode 100644 src/migration.c create mode 100644 src/postgresql/migrations/cloudsync--1.0--1.1.sql create mode 100644 test/postgresql/52_schema_migrations.sql create mode 100755 test/schema_migration_cross_dialect.sh diff --git a/API.md b/API.md index 031732df..def94f11 100644 --- a/API.md +++ b/API.md @@ -22,8 +22,24 @@ This document provides a reference for the SQLite functions provided by the `sql - [`cloudsync_db_version()`](#cloudsync_db_version) - [`cloudsync_uuid()`](#cloudsync_uuid) - [Schema Alteration Functions](#schema-alteration-functions) - - [`cloudsync_begin_alter()`](#cloudsync_begin_altertable_name) - - [`cloudsync_commit_alter()`](#cloudsync_commit_altertable_name) + - [`cloudsync_alter_create_table()`](#cloudsync_alter_create_tabletable_name) + - [`cloudsync_alter_add_column()`](#cloudsync_alter_add_columntable_name-column_name-logical_type-nullable-default_value) + - [`cloudsync_alter_add_column_sqlite()`](#cloudsync_alter_add_column_sqlitetable_name-column_name-type_sql-nullable-default_sql) + - [`cloudsync_alter_add_column_postgresql()`](#cloudsync_alter_add_column_postgresqltable_name-column_name-type_sql-nullable-default_sql) + - [`cloudsync_alter_sql()`](#cloudsync_alter_sqlsql) + - [`cloudsync_alter_sqlite()`](#cloudsync_alter_sqlitesql) + - [`cloudsync_alter_postgresql()`](#cloudsync_alter_postgresqlsql) + - [`cloudsync_alter_add_primary_key()`](#cloudsync_alter_add_primary_keytable_name-column_name) + - [`cloudsync_alter_augment_table()`](#cloudsync_alter_augment_tabletable_name-algo-init_flags) + - [`cloudsync_alter_set_block_lww()`](#cloudsync_alter_set_block_lwwtable_name-column_name-delimiter) + - [`cloudsync_alter_set_column()`](#cloudsync_alter_set_columntable_name-column_name-key-value) + - [`cloudsync_alter_set_filter()`](#cloudsync_alter_set_filtertable_name-filter_expr) + - [`cloudsync_alter_drop_column()`](#cloudsync_alter_drop_columntable_name-column_name) + - [`cloudsync_alter_rename_column()`](#cloudsync_alter_rename_columntable_name-from_name-to_name) + - [`cloudsync_alter_preview()`](#cloudsync_alter_preview) + - [`cloudsync_alter_apply()`](#cloudsync_alter_apply) + - [`cloudsync_alter_clear()`](#cloudsync_alter_cleartable_name) + - [`cloudsync_migration_apply()`](#cloudsync_migration_applypayload) - [Network Functions](#network-functions) - [`cloudsync_network_init()`](#cloudsync_network_initmanageddatabaseid) - [`cloudsync_network_cleanup()`](#cloudsync_network_cleanup) @@ -32,6 +48,9 @@ This document provides a reference for the SQLite functions provided by the `sql - [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) - [`cloudsync_network_check_changes()`](#cloudsync_network_check_changes) - [`cloudsync_network_sync()`](#cloudsync_network_syncwait_ms-max_retries) + - [`cloudsync_network_migration_check()`](#cloudsync_network_migration_check) + - [`cloudsync_network_migration_download()`](#cloudsync_network_migration_download) + - [`cloudsync_network_migration_upload()`](#cloudsync_network_migration_uploadpayload) - [`cloudsync_network_reset_sync_version()`](#cloudsync_network_reset_sync_version) - [`cloudsync_network_has_unsent_changes()`](#cloudsync_network_has_unsent_changes) - [`cloudsync_network_logout()`](#cloudsync_network_logout) @@ -365,46 +384,390 @@ INSERT INTO products (id, name) VALUES (cloudsync_uuid(), 'New Product'); ## Schema Alteration Functions -### `cloudsync_begin_alter(table_name)` +Schema migrations should be created with the declarative `cloudsync_alter_*` +functions. Operations are queued in memory for the current connection until +`cloudsync_alter_apply()` applies them locally and stores a generated +payload in `cloudsync_pending_migration` for upload. -**Description:** Prepares a synchronized table for schema changes. This function must be called before altering the table. Failure to use `cloudsync_begin_alter` and `cloudsync_commit_alter` can lead to synchronization errors and data divergence. +The older public SQL functions `cloudsync_begin_alter()` and +`cloudsync_commit_alter()` are no longer exposed. The extension still uses +internal C primitives with those names while replaying migrations on augmented +tables. + +### `cloudsync_alter_create_table(table_name)` + +**Description:** Queues creation of a table inside a schema migration. Add the +table columns with `cloudsync_alter_add_column()` and mark at least one primary +key column with `cloudsync_alter_add_primary_key()` before applying. **Parameters:** -- `table_name` (TEXT): The name of the table that will be altered. +- `table_name` (TEXT): The table to create. **Returns:** None. **Example:** ```sql -SELECT cloudsync_init('my_table'); --- ... later -SELECT cloudsync_begin_alter('my_table'); -ALTER TABLE my_table ADD COLUMN new_column TEXT; -SELECT cloudsync_commit_alter('my_table'); +SELECT cloudsync_alter_create_table('notes'); +SELECT cloudsync_alter_add_column('notes', 'id', 'text', false); +SELECT cloudsync_alter_add_primary_key('notes', 'id'); ``` --- -### `cloudsync_commit_alter(table_name)` +### `cloudsync_alter_add_column(table_name, column_name, logical_type, nullable, [default_value])` -**Description:** Finalizes schema changes for a synchronized table. This function must be called after altering the table's schema, completing the process initiated by `cloudsync_begin_alter` and ensuring CRDT data consistency. +**Description:** Queues a portable column definition. `default_value` is +optional. A `NOT NULL` non-primary-key column must have a default value. **Parameters:** -- `table_name` (TEXT): The name of the table that was altered. +- `table_name` (TEXT): The table to alter. +- `column_name` (TEXT): The column to add. +- `logical_type` (TEXT): Portable type such as `text`, `integer`, `real`, `numeric`, `blob`, `boolean`, `json`, `timestamp`, or `uuid`. +- `nullable` (BOOLEAN/INTEGER): Whether the column may contain NULL values. +- `default_value` (optional): Plain default value. Serialization is inferred from `logical_type`. **Returns:** None. **Example:** ```sql -SELECT cloudsync_init('my_table'); --- ... later -SELECT cloudsync_begin_alter('my_type'); -ALTER TABLE my_table ADD COLUMN new_column TEXT; -SELECT cloudsync_commit_alter('my_table'); +SELECT cloudsync_alter_add_column('notes', 'title', 'text', false, ''); +SELECT cloudsync_alter_add_column('notes', 'metadata', 'json', true); +``` + +--- + +### `cloudsync_alter_add_column_sqlite(table_name, column_name, type_sql, nullable, [default_sql])` + +**Description:** Adds or replaces the SQLite-specific SQL rendering for a +previously queued `cloudsync_alter_add_column()` operation. + +**Parameters:** + +- `table_name` (TEXT): The table being altered. +- `column_name` (TEXT): The column with the override. +- `type_sql` (TEXT): SQLite type SQL fragment. +- `nullable` (BOOLEAN/INTEGER): Whether the column may contain NULL values. +- `default_sql` (TEXT, optional): SQLite default SQL expression, not a plain value. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_add_column('notes', 'metadata', 'json', false, '{}'); +SELECT cloudsync_alter_add_column_sqlite('notes', 'metadata', 'TEXT', false, '''{}'''); +``` + +--- + +### `cloudsync_alter_add_column_postgresql(table_name, column_name, type_sql, nullable, [default_sql])` + +**Description:** Adds or replaces the PostgreSQL-specific SQL rendering for a +previously queued `cloudsync_alter_add_column()` operation. + +**Parameters:** + +- `table_name` (TEXT): The table being altered. +- `column_name` (TEXT): The column with the override. +- `type_sql` (TEXT): PostgreSQL type SQL fragment. +- `nullable` (BOOLEAN/INTEGER): Whether the column may contain NULL values. +- `default_sql` (TEXT, optional): PostgreSQL default SQL expression, not a plain value. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_add_column('notes', 'metadata', 'json', false, '{}'); +SELECT cloudsync_alter_add_column_postgresql('notes', 'metadata', 'JSONB', false, '''{}''::jsonb'); +``` + +--- + +### `cloudsync_alter_sql(sql)` + +**Description:** Queues raw SQL inside the current schema migration. This is an +advanced escape hatch for schema work that cannot be represented by the +portable alter APIs. The SQL runs in order with the other queued operations and +inside the same migration savepoint. Transaction-control statements such as +`BEGIN`, `COMMIT`, `ROLLBACK`, `SAVEPOINT`, and `RELEASE` are rejected. Because +the extension cannot infer the impact of arbitrary SQL, generated raw SQL +migrations use `formatVersion: 2` and require destructive-schema authorization. + +**Parameters:** + +- `sql` (TEXT): SQL statement to run on every database engine. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_sql('CREATE INDEX notes_updated_at_idx ON notes(updated_at)'); +``` + +--- + +### `cloudsync_alter_sqlite(sql)` + +**Description:** Queues raw SQL that should run only when the migration is +applied to SQLite. PostgreSQL skips this operation. + +**Parameters:** + +- `sql` (TEXT): SQLite SQL statement. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_sqlite('CREATE INDEX notes_body_sqlite_idx ON notes(body)'); +``` + +--- + +### `cloudsync_alter_postgresql(sql)` + +**Description:** Queues raw SQL that should run only when the migration is +applied to PostgreSQL. SQLite skips this operation. + +**Parameters:** + +- `sql` (TEXT): PostgreSQL SQL statement. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_postgresql('CREATE INDEX notes_body_pg_idx ON notes(body)'); +``` + +--- + +### `cloudsync_alter_add_primary_key(table_name, column_name)` + +**Description:** Marks a queued column as part of the primary key for a queued +`cloudsync_alter_create_table()` operation. + +**Parameters:** + +- `table_name` (TEXT): The table being created. +- `column_name` (TEXT): The primary-key column. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_add_primary_key('notes', 'id'); +``` + +--- + +### `cloudsync_alter_augment_table(table_name, [algo], [init_flags])` + +**Description:** Queues CloudSync augmentation for the table. This is the +migration equivalent of `cloudsync_init()`. + +**Parameters:** + +- `table_name` (TEXT): The table to augment. +- `algo` (TEXT, optional): CRDT algorithm. Defaults to `cls`. +- `init_flags` (INTEGER, optional): Same bitmask used by `cloudsync_init()`. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_augment_table('notes', 'CLS', 1); +``` + +--- + +### `cloudsync_alter_set_block_lww(table_name, column_name, [delimiter])` + +**Description:** Queues block-level LWW for a text column. + +**Parameters:** + +- `table_name` (TEXT): The synchronized table. +- `column_name` (TEXT): The text column to configure. +- `delimiter` (TEXT, optional): Block delimiter. For line-level blocks use `char(10)`. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_set_block_lww('notes', 'body', char(10)); +``` + +--- + +### `cloudsync_alter_set_column(table_name, column_name, key, value)` + +**Description:** Queues a CloudSync column setting. + +**Parameters:** + +- `table_name` (TEXT): The synchronized table. +- `column_name` (TEXT): The target column. +- `key` (TEXT): Setting key. +- `value` (TEXT): Setting value, or NULL. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_set_column('notes', 'body', 'delimiter', char(10)); +``` + +--- + +### `cloudsync_alter_set_filter(table_name, filter_expr)` + +**Description:** Queues a portable row filter. Use +`cloudsync_alter_set_filter_sqlite()` or +`cloudsync_alter_set_filter_postgresql()` when the expression must differ by +database. + +**Parameters:** + +- `table_name` (TEXT): The synchronized table. +- `filter_expr` (TEXT): SQL predicate for rows that should be tracked. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_set_filter('notes', 'archived = 0'); +SELECT cloudsync_alter_set_filter_postgresql('notes', 'archived IS FALSE'); +``` + +--- + +### `cloudsync_alter_drop_column(table_name, column_name)` + +**Description:** Queues a V2 destructive column drop. Backend uploads must +require `schema:destructive` authorization. + +**Parameters:** + +- `table_name` (TEXT): The table to alter. +- `column_name` (TEXT): The column to drop. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_drop_column('notes', 'legacy_body'); +``` + +--- + +### `cloudsync_alter_rename_column(table_name, from_name, to_name)` + +**Description:** Queues a V2 column rename and updates CloudSync metadata for +the renamed column. + +**Parameters:** + +- `table_name` (TEXT): The table to alter. +- `from_name` (TEXT): Existing column name. +- `to_name` (TEXT): New column name. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_rename_column('notes', 'text', 'body'); +``` + +--- + +### `cloudsync_alter_preview()` + +**Description:** Returns the generated schema migration JSON without applying it. + +**Parameters:** None. + +**Returns:** A JSON payload as TEXT. + +**Example:** + +```sql +SELECT cloudsync_alter_preview(); +``` + +--- + +### `cloudsync_alter_apply()` + +**Description:** Builds the queued migration, applies it locally, stores the +generated payload in `cloudsync_pending_migration`, and clears the queued +operations for the current connection. + +**Parameters:** None. + +**Returns:** A JSON status object with the generated `migrationId`. + +**Example:** + +```sql +SELECT cloudsync_alter_apply(); +-- '{"status":"applied","migrationId":"...","pendingUpload":true}' +``` + +--- + +### `cloudsync_alter_clear([table_name])` + +**Description:** Discards queued in-memory schema alteration operations. With no +argument, clears all queued operations for the current connection, including +global raw SQL operations. + +**Parameters:** + +- `table_name` (TEXT, optional): Table-specific queue to clear. + +**Returns:** None. + +**Example:** + +```sql +SELECT cloudsync_alter_clear('notes'); +SELECT cloudsync_alter_clear(); +``` + +--- + +### `cloudsync_migration_apply(payload)` + +**Description:** Applies an explicit schema migration JSON payload. This is +mainly for custom backends, server-downloaded migrations, and tests; application +code should normally use the declarative `cloudsync_alter_*` functions. + +**Parameters:** + +- `payload` (TEXT): Migration JSON. + +**Returns:** A JSON status object. + +**Example:** + +```sql +SELECT cloudsync_migration_apply(CAST(readfile('client-to-server-v1.json') AS TEXT)); ``` --- @@ -604,6 +967,72 @@ SELECT cloudsync_network_sync(500, 3); --- +### `cloudsync_network_migration_check()` + +**Description:** Checks the schema migration endpoint and applies any returned +migration before normal row payloads are processed. Responses may contain an +inline `migration`/`payload` object, a migration payload directly, a `url` to +download, or no migration. + +**Parameters:** None. + +**Returns:** A JSON status object such as `{"status":"applied"}` or +`{"status":"none"}`. + +**Example:** + +```sql +SELECT cloudsync_network_migration_check(); +``` + +--- + +### `cloudsync_network_migration_download()` + +**Description:** Downloads and applies the next migration from the schema +download endpoint. + +**Parameters:** None. + +**Returns:** A JSON status object such as `{"status":"applied"}` or +`{"status":"none"}`. + +**Example:** + +```sql +SELECT cloudsync_network_migration_download(); +``` + +--- + +### `cloudsync_network_migration_upload([payload])` + +**Description:** Uploads a schema migration to the backend. With no arguments it +uploads the next row from `cloudsync_pending_migration`, which is the normal +flow after `cloudsync_alter_apply()`. With one argument it uploads the +explicit JSON payload. + +Uploading requires a schema-capable API key on the backend. V2/destructive +migrations should require a stronger destructive-schema permission. + +**Parameters:** + +- `payload` (TEXT, optional): Explicit migration JSON. Omit it to upload the next pending generated migration. + +**Returns:** The backend JSON response. + +**Example:** + +```sql +SELECT cloudsync_alter_apply(); +SELECT cloudsync_network_migration_upload(); + +-- Custom backend/test flow: +SELECT cloudsync_network_migration_upload(CAST(readfile('client-to-server-v1.json') AS TEXT)); +``` + +--- + ### `cloudsync_network_reset_sync_version()` **Description:** Resets local synchronization version numbers, forcing the next sync to fetch all changes from the server. diff --git a/Makefile b/Makefile index 376a4e25..4bbd378a 100644 --- a/Makefile +++ b/Makefile @@ -210,10 +210,14 @@ $(TEST_TARGET): $(TEST_OBJ) $(CC) $(filter-out $(patsubst $(DIST_DIR)/%$(EXE),$(BUILD_TEST)/%.o, $(filter-out $@,$(TEST_TARGET))), $(TEST_OBJ)) -o $@ $(T_LDFLAGS) # Object files +$(BUILD_RELEASE)/fractional_indexing.o: $(FI_DIR)/fractional_indexing.c + $(CC) $(CFLAGS) -Wno-sign-compare -O3 -fPIC -c $< -o $@ $(BUILD_RELEASE)/%.o: %.c $(CC) $(CFLAGS) -O3 -fPIC -c $< -o $@ $(BUILD_TEST)/sqlite3.o: $(SQLITE_DIR)/sqlite3.c $(CC) $(CFLAGS) -DSQLITE_DQS=0 -DSQLITE_CORE -c $< -o $@ +$(BUILD_TEST)/fractional_indexing.o: $(FI_DIR)/fractional_indexing.c + $(CC) $(T_CFLAGS) -Wno-sign-compare -c $< -o $@ $(BUILD_TEST)/%.o: %.c $(CC) $(T_CFLAGS) -c $< -o $@ @@ -237,6 +241,15 @@ e2e: $(TARGET) $(DIST_DIR)/integration$(EXE) fi; \ ./$(DIST_DIR)/integration$(EXE) +cross-dialect-migration-test: $(TARGET) + PG_DOCKER_DB_HOST="$(PG_DOCKER_DB_HOST)" \ + PG_DOCKER_DB_PORT="$(PG_DOCKER_DB_PORT)" \ + PG_DOCKER_DB_NAME="$(PG_DOCKER_DB_NAME)" \ + PG_DOCKER_DB_USER="$(PG_DOCKER_DB_USER)" \ + PG_DOCKER_DB_PASSWORD="$(PG_DOCKER_DB_PASSWORD)" \ + SQLITE3="$(SQLITE3)" \ + ./test/schema_migration_cross_dialect.sh + OPENSSL_TARBALL = $(OPENSSL_DIR)/$(OPENSSL_VERSION).tar.gz $(OPENSSL_TARBALL): @@ -456,6 +469,7 @@ help: @echo " clean - Remove built files" @echo " test [COVERAGE=true] - Test the extension with optional coverage output" @echo " unittest - Run only unit tests (test/unit.c)" + @echo " cross-dialect-migration-test - Test schema migrations between SQLite and PostgreSQL" @echo " help - Display this help message" @echo " xcframework - Build the Apple XCFramework" @echo " aar - Build the Android AAR package" @@ -466,4 +480,4 @@ help: # Include PostgreSQL extension targets include docker/Makefile.postgresql -.PHONY: all clean test unittest e2e extension help version xcframework aar +.PHONY: all clean test unittest e2e cross-dialect-migration-test extension help version xcframework aar diff --git a/docker/Makefile.postgresql b/docker/Makefile.postgresql index f5303bdf..a91cebb7 100644 --- a/docker/Makefile.postgresql +++ b/docker/Makefile.postgresql @@ -52,6 +52,7 @@ PG_CORE_SRC = \ src/pk.c \ src/utils.c \ src/lz4.c \ + src/migration.c \ src/block.c \ modules/fractional-indexing/fractional_indexing.c diff --git a/docker/postgresql/docker-compose.debug.yml b/docker/postgresql/docker-compose.debug.yml index d4456704..c42227d2 100644 --- a/docker/postgresql/docker-compose.debug.yml +++ b/docker/postgresql/docker-compose.debug.yml @@ -10,6 +10,7 @@ services: POSTGRES_DB: cloudsync_test ports: - "5432:5432" + command: ["postgres", "-c", "listen_addresses=*"] ulimits: core: -1 cap_add: diff --git a/docs/internal/schema-migrations.md b/docs/internal/schema-migrations.md new file mode 100644 index 00000000..0c5ddfac --- /dev/null +++ b/docs/internal/schema-migrations.md @@ -0,0 +1,309 @@ +# CloudSync Schema Migrations + +This document describes the implemented schema migration flow for SQLite Sync. +Schema migrations can originate from an authorized SQLite client or from the +cloud database, and the same payload can be applied to SQLite and PostgreSQL. + +## Goals + +- Allow schema changes to originate either from a SQLite client or from the cloud database. +- Support an empty SQLite client database that creates its synchronized tables during first sync. +- Keep row CRDT payloads binary and focused on data, while using a separate schema-migration protocol. +- Support SQLite/SQLiteCloud and PostgreSQL backends. +- Keep every database coherent when a schema migration or data sync fails. +- Preserve a raw SQL escape hatch for migrations that cannot be expressed portably. + +## Core Model + +Schema is not a CRDT. Schema migrations are serialized in one ordered log per +`database_id`, and the backend decides whether a migration may be proposed or +applied based on the API key used for the request. + +The extension stores applied migrations locally in `cloudsync_migrations`, and +client-originated migrations waiting for upload in `cloudsync_pending_migration`. +Pending alter operations are not stored in a table: they live in the current +CloudSync context until `cloudsync_alter_apply()` or `cloudsync_alter_clear()`. + +The local migration applier is atomic. `cloudsync_migration_apply()` opens a +savepoint, validates the JSON payload, applies every operation, updates the +schema hash, records the migration id, and rolls everything back on failure. + +## Public SQL Workflow + +Applications should build migrations with declarative SQL functions. They do +not need to write JSON. + +```sql +SELECT cloudsync_alter_create_table('notes'); +SELECT cloudsync_alter_add_column('notes', 'id', 'text', false); +SELECT cloudsync_alter_add_primary_key('notes', 'id'); +SELECT cloudsync_alter_add_column('notes', 'title', 'text', false, ''); +SELECT cloudsync_alter_add_column('notes', 'body', 'text', false, ''); +SELECT cloudsync_alter_add_column('notes', 'updated_at', 'timestamp', false, '1970-01-01T00:00:00Z'); +SELECT cloudsync_alter_augment_table('notes', 'CLS', 1); +SELECT cloudsync_alter_set_block_lww('notes', 'body', char(10)); +SELECT cloudsync_alter_apply(); +``` + +`cloudsync_alter_apply()` applies the queued migration locally and stores the +generated payload in `cloudsync_pending_migration`. After that, an authorized +client uploads it with: + +```sql +SELECT cloudsync_network_migration_upload(); +SELECT cloudsync_network_sync(); +``` + +The zero-argument upload form uploads the next pending local migration and marks +it uploaded only after the backend returns valid JSON. The one-argument form is +still available for custom backends or tests: + +```sql +SELECT cloudsync_network_migration_upload(:json_payload); +``` + +While a local migration is pending upload, `cloudsync_network_send_changes()` +returns an error instead of sending row changes. This prevents data produced +with a new local schema from reaching a server that has not accepted that schema. + +## Declarative API + +The same SQL API is exposed by the SQLite and PostgreSQL extensions: + +- `cloudsync_alter_create_table(table)` +- `cloudsync_alter_add_column(table, column, logical_type, nullable)` +- `cloudsync_alter_add_column(table, column, logical_type, nullable, default_value)` +- `cloudsync_alter_add_column_sqlite(table, column, type_sql, nullable)` +- `cloudsync_alter_add_column_sqlite(table, column, type_sql, nullable, default_sql)` +- `cloudsync_alter_add_column_postgresql(table, column, type_sql, nullable)` +- `cloudsync_alter_add_column_postgresql(table, column, type_sql, nullable, default_sql)` +- `cloudsync_alter_add_primary_key(table, column)` +- `cloudsync_alter_augment_table(table)` +- `cloudsync_alter_augment_table(table, algorithm)` +- `cloudsync_alter_augment_table(table, algorithm, init_flags)` +- `cloudsync_alter_set_block_lww(table, column)` +- `cloudsync_alter_set_block_lww(table, column, delimiter)` +- `cloudsync_alter_set_column(table, column, key, value)` +- `cloudsync_alter_set_filter(table, filter_expr)` +- `cloudsync_alter_set_filter_sqlite(table, filter_expr)` +- `cloudsync_alter_set_filter_postgresql(table, filter_expr)` +- `cloudsync_alter_drop_column(table, column)` +- `cloudsync_alter_rename_column(table, from_name, to_name)` +- `cloudsync_alter_sql(sql)` +- `cloudsync_alter_sqlite(sql)` +- `cloudsync_alter_postgresql(sql)` +- `cloudsync_alter_preview()` +- `cloudsync_alter_apply()` +- `cloudsync_alter_clear()` +- `cloudsync_alter_clear(table)` + +`cloudsync_alter_preview()` returns the generated JSON without applying it. +`cloudsync_alter_clear()` discards queued in-memory operations. + +The dialect override functions are optional. Use them when the portable logical +type or default is not precise enough: + +```sql +SELECT cloudsync_alter_add_column('notes', 'metadata', 'json', false, '{}'); +SELECT cloudsync_alter_add_column_sqlite('notes', 'metadata', 'TEXT', false, '''{}'''); +SELECT cloudsync_alter_add_column_postgresql('notes', 'metadata', 'JSONB', false, '''{}''::jsonb'); +``` + +The override default is a SQL fragment for that dialect, not a plain value. The +portable `default_value` argument is optional, and its serialization is inferred +from the logical column type. + +Raw SQL functions are an escape hatch for migration steps that do not have a +portable command yet. `cloudsync_alter_sql()` runs on every engine, while +`cloudsync_alter_sqlite()` and `cloudsync_alter_postgresql()` are emitted as +dialect-specific raw SQL and skipped by the other engine. They run in queue +order with the structured operations and cannot contain transaction-control +statements. + +`cloudsync_begin_alter()` and `cloudsync_commit_alter()` still exist as internal +C primitives used while replaying migrations on already-augmented tables. They +are not public SQL APIs. + +## Payload Format + +The network payload is JSON. This is intentional even though row sync uses a +binary encoder: schema payloads must be audited, authorized, inspected by a +backend service, and sometimes hand-produced by server tooling. User-facing APIs +generate the JSON automatically, so application code does not need to construct +it directly. + +Generated client payloads omit `baseSchemaHash` and `targetSchemaHash` because +raw SQLite and PostgreSQL schema hashes are not necessarily portable across +dialects. Manual payloads may include those fields; when present, +`cloudsync_migration_apply()` enforces them and rolls back on mismatch. + +Example generated V1 payload: + +```json +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "0197097c-8b35-7c11-8ed4-4e59ddfdb928", + "requiredCapabilities": ["schema:write"], + "ops": [ + { + "op": "createTable", + "table": "notes", + "columns": [ + {"name": "id", "type": "text", "nullable": false, "primaryKey": true}, + {"name": "body", "type": "text", "nullable": false, "default": {"type": "text", "value": ""}} + ] + }, + {"op": "augmentTable", "table": "notes", "algorithm": "CLS", "initFlags": 1}, + {"op": "setBlockLww", "table": "notes", "column": "body", "delimiter": "\n"} + ] +} +``` + +## Version 1 + +Version 1 contains additive and bootstrap operations: + +- `createTable`: create a table from logical column definitions. +- `addColumn`: add a nullable column or a `NOT NULL` column with a default value. +- `augmentTable`: call the same internal path as `cloudsync_init()`. +- `setBlockLww`: configure block-level LWW and materialize block metadata. +- `setColumn`: set a CloudSync column setting. +- `setFilter`: set a row filter, with optional dialect-specific filters. + +Creating a synchronized table requires both `createTable` and `augmentTable`. +`setBlockLww` must run after the table is augmented and after the target column +exists. + +## Version 2 + +Version 2 is implemented for authorized non-additive changes: + +- `dropColumn` +- `renameColumn` +- `rebuildTableSync` +- `rawSql` in V2/destructive payloads + +Generated payloads containing `dropColumn` or `renameColumn` use +`formatVersion: 2` and include `schema:destructive` in `requiredCapabilities`. +The backend must enforce this capability from the API key; the payload field is +for audit and policy clarity, not authentication. + +The declarative raw SQL functions also emit V2/destructive payloads, even when +the SQL is intended to be additive, because the extension cannot safely infer +the behavioral impact of arbitrary SQL. + +`rebuildTableSync` uses `cloudsync_cleanup(..., is_migration = true)` so the +table sync metadata is rebuilt without resetting the database-wide CloudSync +site identity or schema history. The `ddl` and `blockLww` fields are validated +before cleanup/reinit so malformed payloads fail without partially changing the +table. + +Version 3 orchestration is deliberately not implemented. Rolling expand/contract +migrations, payload translation across schema epochs, and long-running backfills +belong to a future protocol layer. + +## Logical Type Mapping + +Portable payloads use logical types and let `migration.c` render backend SQL: + +- `text` -> SQLite `TEXT`, PostgreSQL `TEXT` +- `uuid` -> SQLite `TEXT`, PostgreSQL `UUID` +- `integer` -> SQLite `INTEGER`, PostgreSQL `BIGINT` +- `real` -> SQLite `REAL`, PostgreSQL `DOUBLE PRECISION` +- `numeric` -> SQLite `NUMERIC`, PostgreSQL `NUMERIC` +- `blob` -> SQLite `BLOB`, PostgreSQL `BYTEA` +- `boolean` -> SQLite `INTEGER`, PostgreSQL `BOOLEAN` +- `json` -> SQLite `TEXT`, PostgreSQL `JSONB` +- `timestamp` -> SQLite `TEXT`, PostgreSQL `TIMESTAMPTZ` + +Use dialect override functions when a migration needs exact SQL types or +database-specific default expressions. + +## Backend Protocol + +Schema endpoints live beside the existing data endpoints: + +- `POST /v2/cloudsync/databases/{databaseId}/{siteId}/schema/check` +- `POST /v2/cloudsync/databases/{databaseId}/{siteId}/schema/upload` +- `GET /v2/cloudsync/databases/{databaseId}/{siteId}/schema/download` + +Recommended backend log fields: + +- `database_id` +- `schema_version` or `schema_epoch` +- `migration_id` +- `source`: `client` or `server` +- `author_site_id` +- `payload` +- `payload_hash` +- `required_capabilities` +- `authorized_by_key_id` +- `status`: `pending`, `applied`, `rejected`, `failed` +- `created_at`, `applied_at` +- `error` + +API key classes: + +- `sync`: send/receive data and download already-approved migrations. +- `schema:write`: propose/upload/apply V1 migrations. +- `schema:destructive`: propose/upload/apply V2 destructive migrations. + +Normal application clients should use `sync` keys. A schema-capable key is +required for every schema change, including V1 additive changes. + +## Sync Flow + +Client-originated migration: + +1. Application queues operations with `cloudsync_alter_*`. +2. Application calls `cloudsync_alter_apply()`. +3. Extension applies the migration locally and writes `cloudsync_pending_migration`. +4. Application or `cloudsync_network_sync()` uploads the pending migration. +5. Backend authorizes the API key, applies the payload to the cloud database, + records the migration, and returns success. +6. Client sends row changes after the pending migration is uploaded. + +Server-originated migration: + +1. Backend applies and records a migration. +2. Client calls `cloudsync_network_sync()` or `cloudsync_network_migration_check()`. +3. The network layer downloads the migration when the local schema is missing or stale. +4. `cloudsync_migration_apply()` applies it locally. +5. Data download/retry continues on the new schema. + +Empty client first sync: + +1. Empty SQLite client calls `cloudsync_network_init(database_id)`. +2. Sync checks schema before returning from the empty local send phase. +3. Backend returns a schema snapshot or migration chain. +4. The client creates tables, augments them, applies block LWW, then downloads data. + +## Failure Semantics + +- Migrations are atomic per database connection. +- A malformed JSON payload is rejected before DDL is applied. +- A migration id is idempotent through `cloudsync_migrations`. +- Explicit hash guards are enforced when present. +- Raw SQL runs inside the same savepoint as portable operations. +- Row changes are not uploaded while `cloudsync_pending_migration` contains an unuploaded migration. +- V2 migrations should be blocked by the backend when stale/offline clients may still upload incompatible old-epoch payloads, unless the backend has an explicit rejection or translation policy. + +## Tests + +SQLite coverage is in `test/unit.c` and the mock network tests in +`test/integration.c`. + +PostgreSQL coverage is in `test/postgresql/31_alter_table_sync.sql` and +`test/postgresql/52_schema_migrations.sql`. + +Cross-dialect coverage is in `test/schema_migration_cross_dialect.sh` and can +be run with: + +```sh +make cross-dialect-migration-test +``` + +The cross-dialect test covers SQLite-generated migrations applied to PostgreSQL, +PostgreSQL-generated migrations with dialect overrides applied to SQLite, and +generic plus dialect-specific raw SQL in both directions. diff --git a/examples/README.md b/examples/README.md index cdff4dbc..e27a7630 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,6 +12,12 @@ This directory contains comprehensive examples demonstrating SQLite Sync in vari - Offline scenarios and network synchronization - Perfect for understanding core sync mechanics +### [schema-migrations/](./schema-migrations/) +**Schema Migrations** +- Client-to-server and server-to-client migration examples +- Demonstrates the `cloudsync_alter_*` API, generated pending migrations, table creation, `cloudsync_init`, block-level LWW, dialect overrides, raw SQL escape hatches, and V2 rebuild payloads +- Shows how schema-capable API keys fit into migration upload/download + ### [sport-tracker-app/](./sport-tracker-app/) **Advanced Web App - Production Patterns** - React/TypeScript web application with Vite @@ -46,4 +52,4 @@ Each example includes detailed setup instructions, code explanations, and securi --- -**Note**: For generic extension loading guides please refer to the [SQLite Extension Guide](https://github.com/sqliteai/sqlite-extensions-guide) repository \ No newline at end of file +**Note**: For generic extension loading guides please refer to the [SQLite Extension Guide](https://github.com/sqliteai/sqlite-extensions-guide) repository diff --git a/examples/schema-migrations/README.md b/examples/schema-migrations/README.md new file mode 100644 index 00000000..5c679dfd --- /dev/null +++ b/examples/schema-migrations/README.md @@ -0,0 +1,70 @@ +# Schema Migrations Example + +This example shows the two supported directions for schema migrations: + +- **Client to server**: an authorized SQLite client creates or changes a synchronized table and uploads the generated migration. +- **Server to client**: the backend publishes a migration, and clients download/apply it before receiving data payloads for the new schema. + +Normal sync API keys should only download approved migrations. Uploading any migration requires a schema-capable API key, and V2/destructive migrations require a key with destructive schema permission on the backend. + +## Client-Originated V1 Migration + +The client queues declarative operations, applies them locally, and then uploads the generated pending migration. The user does not need to write JSON: + +```sql +SELECT cloudsync_alter_create_table('notes'); +SELECT cloudsync_alter_add_column('notes', 'id', 'text', false); +SELECT cloudsync_alter_add_primary_key('notes', 'id'); +SELECT cloudsync_alter_add_column('notes', 'title', 'text', false, ''); +SELECT cloudsync_alter_add_column('notes', 'body', 'text', false, ''); +SELECT cloudsync_alter_add_column('notes', 'updated_at', 'timestamp', false, '1970-01-01T00:00:00Z'); +SELECT cloudsync_alter_augment_table('notes', 'CLS', 1); +SELECT cloudsync_alter_set_block_lww('notes', 'body', char(10)); +SELECT cloudsync_alter_apply(); +SELECT cloudsync_network_migration_upload(); +SELECT cloudsync_network_sync(); +``` + +`cloudsync_alter_preview()` can be used before `cloudsync_alter_apply()` to inspect the generated payload. After apply, the payload is saved in `cloudsync_pending_migration`; `cloudsync_network_migration_upload()` uploads the next pending migration and marks it uploaded on success. + +The backend should authorize the API key, apply the payload to the cloud database, append it to the schema migration log for the `database_id`, and distribute it to other clients through `schema/check` or `schema/download`. + +`client-to-server.sql` contains the same flow as an executable SQLite example. `client-to-server-v1.json` is the manual JSON equivalent for backend tests or custom tooling; application code should normally let `cloudsync_alter_apply()` generate that payload. + +## Server-Originated V2 Migration + +The backend applies and records a payload such as `server-to-client-v2.json`, then a client can update itself before receiving data: + +```sql +SELECT cloudsync_network_migration_check(); +SELECT cloudsync_network_sync(); +``` + +If a client receives a data payload first and `cloudsync_payload_apply()` reports a missing schema or unknown schema hash, the network layer runs one schema check and retries the same data payload after the migration is applied. + +`server-to-client-v2.json` shows a server-originated V2 migration: it adds `summary`, renames `title` to `heading`, and creates both generic and dialect-specific indexes. It contains both `sqlite` and `postgresql` raw SQL branches because the backend may need to apply the same migration to a PostgreSQL cloud database, while every client applies the SQLite branch locally. + +## Dialect Overrides + +Portable logical types work for most migrations. When a column needs different SQL on SQLite and PostgreSQL, add overrides after the portable operation: + +```sql +SELECT cloudsync_alter_add_column('notes', 'metadata', 'json', false, '{}'); +SELECT cloudsync_alter_add_column_sqlite('notes', 'metadata', 'TEXT', false, '''{}'''); +SELECT cloudsync_alter_add_column_postgresql('notes', 'metadata', 'JSONB', false, '''{}''::jsonb'); +SELECT cloudsync_alter_apply(); +``` + +The last argument of the dialect-specific functions is a SQL default expression for that database, not a plain value. + +For migration steps that are not covered by portable commands, use raw SQL in +the same queue: + +```sql +SELECT cloudsync_alter_sql('CREATE INDEX notes_updated_at_idx ON notes(updated_at)'); +SELECT cloudsync_alter_sqlite('CREATE INDEX notes_body_sqlite_idx ON notes(body)'); +SELECT cloudsync_alter_postgresql('CREATE INDEX notes_body_pg_idx ON notes(body)'); +SELECT cloudsync_alter_apply(); +``` + +`raw-sql.sql` contains the same raw SQL flow as an executable example. diff --git a/examples/schema-migrations/client-to-server-v1.json b/examples/schema-migrations/client-to-server-v1.json new file mode 100644 index 00000000..31f7ac4c --- /dev/null +++ b/examples/schema-migrations/client-to-server-v1.json @@ -0,0 +1,31 @@ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "0197097c-8b35-7c11-8ed4-4e59ddfdb928", + "schemaEpoch": 1, + "requiredCapabilities": ["schema:write"], + "ops": [ + { + "op": "createTable", + "table": "notes", + "columns": [ + {"name": "id", "type": "text", "primaryKey": true, "nullable": false}, + {"name": "title", "type": "text", "nullable": false, "default": {"type": "text", "value": ""}}, + {"name": "body", "type": "text", "nullable": false, "default": {"type": "text", "value": ""}}, + {"name": "updated_at", "type": "timestamp", "nullable": false, "default": {"type": "timestamp", "value": "1970-01-01T00:00:00Z"}} + ] + }, + { + "op": "augmentTable", + "table": "notes", + "algorithm": "CLS", + "initFlags": 1 + }, + { + "op": "setBlockLww", + "table": "notes", + "column": "body", + "delimiter": "\n" + } + ] +} diff --git a/examples/schema-migrations/client-to-server.sql b/examples/schema-migrations/client-to-server.sql new file mode 100644 index 00000000..5a56f596 --- /dev/null +++ b/examples/schema-migrations/client-to-server.sql @@ -0,0 +1,21 @@ +-- Client-originated schema migration. +-- +-- Run this on an authorized SQLite client. The extension generates the JSON +-- migration payload, applies it locally, stores it in cloudsync_pending_migration, +-- and uploads it with cloudsync_network_migration_upload(). + +SELECT cloudsync_alter_create_table('notes'); +SELECT cloudsync_alter_add_column('notes', 'id', 'text', false); +SELECT cloudsync_alter_add_primary_key('notes', 'id'); +SELECT cloudsync_alter_add_column('notes', 'title', 'text', false, ''); +SELECT cloudsync_alter_add_column('notes', 'body', 'text', false, ''); +SELECT cloudsync_alter_add_column('notes', 'updated_at', 'timestamp', false, '1970-01-01T00:00:00Z'); +SELECT cloudsync_alter_augment_table('notes', 'CLS', 1); +SELECT cloudsync_alter_set_block_lww('notes', 'body', char(10)); + +-- Optional: inspect the generated payload before applying it. +SELECT cloudsync_alter_preview(); + +SELECT cloudsync_alter_apply(); +SELECT cloudsync_network_migration_upload(); +SELECT cloudsync_network_sync(); diff --git a/examples/schema-migrations/dialect-overrides.sql b/examples/schema-migrations/dialect-overrides.sql new file mode 100644 index 00000000..44a19d4f --- /dev/null +++ b/examples/schema-migrations/dialect-overrides.sql @@ -0,0 +1,11 @@ +-- Add a column with database-specific type/default SQL. +-- +-- The portable operation defines the logical schema. The override calls refine +-- the generated SQL for SQLite and PostgreSQL while keeping one migration +-- payload usable across both databases. + +SELECT cloudsync_alter_add_column('notes', 'metadata', 'json', false, '{}'); +SELECT cloudsync_alter_add_column_sqlite('notes', 'metadata', 'TEXT', false, '''{}'''); +SELECT cloudsync_alter_add_column_postgresql('notes', 'metadata', 'JSONB', false, '''{}''::jsonb'); +SELECT cloudsync_alter_apply(); +SELECT cloudsync_network_migration_upload(); diff --git a/examples/schema-migrations/raw-sql.sql b/examples/schema-migrations/raw-sql.sql new file mode 100644 index 00000000..de4f8f15 --- /dev/null +++ b/examples/schema-migrations/raw-sql.sql @@ -0,0 +1,15 @@ +-- Raw SQL schema migration escape hatch. +-- +-- Use structured cloudsync_alter_* commands whenever possible. Raw SQL is useful +-- for indexes, constraints, or migration steps that are not portable yet. +-- Generic SQL runs on every engine; dialect-specific SQL runs only on that +-- engine and is skipped by the other one. +-- Generated raw SQL migrations require destructive-schema authorization because +-- the extension cannot infer the impact of arbitrary SQL. + +SELECT cloudsync_alter_sql('CREATE INDEX notes_updated_at_idx ON notes(updated_at)'); +SELECT cloudsync_alter_sqlite('CREATE INDEX notes_body_sqlite_idx ON notes(body)'); +SELECT cloudsync_alter_postgresql('CREATE INDEX notes_body_pg_idx ON notes(body)'); + +SELECT cloudsync_alter_apply(); +SELECT cloudsync_network_migration_upload(); diff --git a/examples/schema-migrations/server-to-client-v2.json b/examples/schema-migrations/server-to-client-v2.json new file mode 100644 index 00000000..35698ab2 --- /dev/null +++ b/examples/schema-migrations/server-to-client-v2.json @@ -0,0 +1,47 @@ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 2, + "migrationId": "0197097d-3f7e-7f00-8a3f-91dfaa8f0f3a", + "schemaEpoch": 2, + "requiredCapabilities": ["schema:write", "schema:destructive"], + "ops": [ + { + "op": "addColumn", + "table": "notes", + "column": { + "name": "summary", + "type": "text", + "nullable": false, + "default": {"type": "text", "value": ""} + } + }, + { + "op": "renameColumn", + "table": "notes", + "from": "title", + "to": "heading" + }, + { + "op": "rawSql", + "sql": "CREATE INDEX notes_heading_idx ON notes(heading)" + }, + { + "op": "rawSql", + "sql": { + "sqlite": [ + "CREATE INDEX notes_body_sqlite_idx ON notes(body)" + ] + }, + "skipMissingDialect": true + }, + { + "op": "rawSql", + "sql": { + "postgresql": [ + "CREATE INDEX notes_body_pg_idx ON \"notes\" (\"body\")" + ] + }, + "skipMissingDialect": true + } + ] +} diff --git a/src/cloudsync.c b/src/cloudsync.c index 908e9c17..cfdedb1b 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -323,7 +323,7 @@ DBVM_VALUE dbvm_execute (dbvm_t *stmt, cloudsync_context *data) { } else { result = DBVM_VALUE_UNCHANGED; } - + } else if (stmt == data->db_version_stmt) { data->db_version = (rc == DBRES_DONE) ? CLOUDSYNC_MIN_DB_VERSION : database_column_int(stmt, 0); } @@ -688,7 +688,7 @@ char *table_build_value_sql (cloudsync_table_context *table, const char *colname return sql; } #endif - + // SELECT age FROM customers WHERE first_name=? AND last_name=?; return sql_build_select_cols_by_pk(table->context, table->name, colname, table->schema); } @@ -2387,6 +2387,7 @@ void cloudsync_context_free (void *ctx) { // free all table contexts and prepared statements cloudsync_terminate(data); + cloudsync_alter_clear_context(data); cloudsync_memory_free(data->tables); cloudsync_memory_free(data); @@ -3624,17 +3625,17 @@ int cloudsync_cleanup_internal (cloudsync_context *data, cloudsync_table_context return DBRES_OK; } -int cloudsync_cleanup (cloudsync_context *data, const char *table_name) { +int cloudsync_cleanup (cloudsync_context *data, const char *table_name, bool preserve_global_state) { cloudsync_table_context *table = table_lookup(data, table_name); if (!table) return DBRES_OK; - - // TODO: check what happen if cloudsync_cleanup_internal failes (not eveything dropped) and the table is still in memory? - + int rc = cloudsync_cleanup_internal(data, table); if (rc != DBRES_OK) return rc; int counter = table_remove(data, table); table_free(table); + + if (preserve_global_state) return DBRES_OK; if (counter == 0) { // cleanup database on last table diff --git a/src/cloudsync.h b/src/cloudsync.h index d1641ed6..04cc2545 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -18,7 +18,7 @@ extern "C" { #endif -#define CLOUDSYNC_VERSION "1.0.17" +#define CLOUDSYNC_VERSION "1.1.0" #define CLOUDSYNC_MAX_TABLENAME_LEN 512 #define CLOUDSYNC_VALUE_NOTSET -1 @@ -51,7 +51,7 @@ void cloudsync_context_free (void *ctx); // CloudSync global int cloudsync_init_table (cloudsync_context *data, const char *table_name, const char *algo_name, CLOUDSYNC_INIT_FLAG init_flags); -int cloudsync_cleanup (cloudsync_context *data, const char *table_name); +int cloudsync_cleanup (cloudsync_context *data, const char *table_name, bool preserve_global_state); int cloudsync_cleanup_all (cloudsync_context *data); int cloudsync_terminate (cloudsync_context *data); int cloudsync_insync (cloudsync_context *data); @@ -67,10 +67,33 @@ bool cloudsync_config_exists (cloudsync_context *data); bool cloudsync_context_is_initialized (cloudsync_context *data); dbvm_t *cloudsync_colvalue_stmt (cloudsync_context *data, const char *tbl_name, bool *persistent); -// CloudSync alter table +// CloudSync alter table internals used by migration replay int cloudsync_begin_alter (cloudsync_context *data, const char *table_name); int cloudsync_commit_alter (cloudsync_context *data, const char *table_name); +// CloudSync declarative schema alter API +int cloudsync_alter_create_table(cloudsync_context *data, const char *table); +int cloudsync_alter_add_column(cloudsync_context *data, const char *table, const char *column, const char *type, bool nullable, bool has_default, const char *default_value); +int cloudsync_alter_add_column_dialect(cloudsync_context *data, const char *table, const char *column, const char *dialect, const char *type_sql, bool nullable, bool has_default_sql, const char *default_sql); +int cloudsync_alter_add_primary_key(cloudsync_context *data, const char *table, const char *column); +int cloudsync_alter_augment_table(cloudsync_context *data, const char *table, const char *algorithm, int64_t init_flags); +int cloudsync_alter_set_block_lww(cloudsync_context *data, const char *table, const char *column, const char *delimiter); +int cloudsync_alter_set_column(cloudsync_context *data, const char *table, const char *column, const char *key, const char *value); +int cloudsync_alter_set_filter(cloudsync_context *data, const char *table, const char *filter); +int cloudsync_alter_set_filter_dialect(cloudsync_context *data, const char *table, const char *dialect, const char *filter); +int cloudsync_alter_drop_column(cloudsync_context *data, const char *table, const char *column); +int cloudsync_alter_rename_column(cloudsync_context *data, const char *table, const char *from, const char *to); +int cloudsync_alter_sql(cloudsync_context *data, const char *sql); +int cloudsync_alter_sql_dialect(cloudsync_context *data, const char *dialect, const char *sql); +int cloudsync_alter_clear(cloudsync_context *data, const char *table); +void cloudsync_alter_clear_context(cloudsync_context *data); +char *cloudsync_alter_preview(cloudsync_context *data); +int cloudsync_alter_apply(cloudsync_context *data, char **result_json); +int cloudsync_pending_migration_count(cloudsync_context *data); +char *cloudsync_pending_migration_next_id(cloudsync_context *data); +char *cloudsync_pending_migration_payload(cloudsync_context *data, const char *migration_id); +int cloudsync_pending_migration_mark_uploaded(cloudsync_context *data, const char *migration_id); + // CloudSync getter/setter void *cloudsync_db (cloudsync_context *data); void *cloudsync_auxdata (cloudsync_context *data); @@ -95,6 +118,9 @@ size_t cloudsync_payload_context_size (size_t *header_size); int cloudsync_payload_get (cloudsync_context *data, char **blob, int *blob_size, int *db_version, int64_t *new_db_version); int cloudsync_payload_save (cloudsync_context *data, const char *payload_path, int *blob_size); // available only on Desktop OS (no WASM, no mobile) +// Schema migrations +int cloudsync_migration_apply (cloudsync_context *data, const char *payload, int payload_len, char **result_json); + // CloudSync table context int cloudsync_refill_metatable (cloudsync_context *data, const char *table_name); int cloudsync_reset_metatable (cloudsync_context *data, const char *table_name); diff --git a/src/migration.c b/src/migration.c new file mode 100644 index 00000000..cd88e6c1 --- /dev/null +++ b/src/migration.c @@ -0,0 +1,2043 @@ +// +// migration.c +// cloudsync +// +// Schema migration payload application. +// + +#include +#include +#include +#include +#include +#include +#include + +#include "cloudsync.h" +#include "database.h" +#include "dbutils.h" +#include "utils.h" + +#define JSMN_STATIC +#include "jsmn.h" + +#define CLOUDSYNC_MIGRATION_INITIAL_TOKENS 512 +#define CLOUDSYNC_MIGRATION_SAVEPOINT "cloudsync_migration" +#define CLOUDSYNC_ALTER_APPLY_SAVEPOINT "cloudsync_alter_apply" + +typedef struct { + char *ptr; + size_t len; + size_t cap; +} migration_buffer; + +typedef struct { + const char *json; + jsmntok_t *tokens; + int ntokens; +} migration_json; + +typedef enum { + PENDING_ALTER_CREATE_TABLE, + PENDING_ALTER_ADD_COLUMN, + PENDING_ALTER_ADD_PRIMARY_KEY, + PENDING_ALTER_AUGMENT_TABLE, + PENDING_ALTER_SET_BLOCK_LWW, + PENDING_ALTER_SET_COLUMN, + PENDING_ALTER_SET_FILTER, + PENDING_ALTER_DROP_COLUMN, + PENDING_ALTER_RENAME_COLUMN, + PENDING_ALTER_RAW_SQL +} pending_alter_kind; + +typedef struct pending_alter_op { + pending_alter_kind kind; + char *table; + char *a; + char *b; + char *c; + char *d; + bool flag; + int64_t number; + bool has_default; + char *default_kind; + char *default_value; + char *sqlite_type_sql; + char *sqlite_default_sql; + char *postgresql_type_sql; + char *postgresql_default_sql; + char *sqlite_filter_sql; + char *postgresql_filter_sql; + struct pending_alter_op *next; +} pending_alter_op; + +typedef struct pending_alter_context { + cloudsync_context *data; + pending_alter_op *ops; + struct pending_alter_context *next; +} pending_alter_context; + +static pending_alter_context *g_pending_alters = NULL; + +static bool migration_sql_has_statement(const char *sql); +static bool migration_sql_has_transaction_control(const char *sql); + +static bool migration_json_is_whitespace (char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; +} + +static bool migration_json_has_single_root (const char *json, size_t json_len, jsmntok_t *tokens, int ntokens) { + if (!json || !tokens || ntokens < 1 || tokens[0].start < 0 || tokens[0].end < tokens[0].start) return false; + + size_t cursor = 0; + while (cursor < json_len && migration_json_is_whitespace(json[cursor])) cursor++; + if (cursor != (size_t)tokens[0].start) return false; + + cursor = (size_t)tokens[0].end; + while (cursor < json_len && migration_json_is_whitespace(json[cursor])) cursor++; + return cursor == json_len; +} + +static int migration_json_parse_root_object (const char *json, size_t json_len, jsmntok_t *tokens, unsigned int max_tokens) { + if (!json || json_len == 0 || !tokens || max_tokens == 0) return JSMN_ERROR_INVAL; + + jsmn_parser parser; + jsmn_init(&parser); + int ntokens = jsmn_parse(&parser, json, json_len, tokens, max_tokens); + if (ntokens < 0) return ntokens; + if (ntokens < 1 || tokens[0].type != JSMN_OBJECT || !migration_json_has_single_root(json, json_len, tokens, ntokens)) { + return JSMN_ERROR_INVAL; + } + return ntokens; +} + +static int migration_json_parse_root_object_alloc (const char *json, size_t json_len, jsmntok_t **tokens_out, int *ntokens_out) { + if (tokens_out) *tokens_out = NULL; + if (ntokens_out) *ntokens_out = 0; + if (!json || json_len == 0 || !tokens_out || !ntokens_out) return DBRES_MISUSE; + + size_t max_tokens = json_len + 1; + if (max_tokens < CLOUDSYNC_MIGRATION_INITIAL_TOKENS) max_tokens = CLOUDSYNC_MIGRATION_INITIAL_TOKENS; + if (max_tokens > (size_t)INT_MAX) max_tokens = (size_t)INT_MAX; + + size_t cap = CLOUDSYNC_MIGRATION_INITIAL_TOKENS; + if (cap > max_tokens) cap = max_tokens; + while (cap > 0 && cap <= max_tokens) { + if (cap > SIZE_MAX / sizeof(jsmntok_t)) return DBRES_NOMEM; + jsmntok_t *tokens = cloudsync_memory_alloc(cap * sizeof(jsmntok_t)); + if (!tokens) return DBRES_NOMEM; + + int ntokens = migration_json_parse_root_object(json, json_len, tokens, (unsigned int)cap); + if (ntokens == JSMN_ERROR_NOMEM) { + cloudsync_memory_free(tokens); + if (cap == max_tokens) break; + size_t next = cap * 2; + if (next <= cap || next > max_tokens) next = max_tokens; + cap = next; + continue; + } + if (ntokens < 1) { + cloudsync_memory_free(tokens); + return DBRES_MISUSE; + } + + *tokens_out = tokens; + *ntokens_out = ntokens; + return DBRES_OK; + } + + return DBRES_MISUSE; +} + +static int migration_token_skip (migration_json *doc, int index) { + if (!doc || index < 0 || index >= doc->ntokens) return index + 1; + jsmntok_t *tok = &doc->tokens[index]; + int next = index + 1; + if (tok->type == JSMN_OBJECT) { + for (int i = 0; i < tok->size; ++i) { + next = migration_token_skip(doc, next); // key + next = migration_token_skip(doc, next); // value + } + } else if (tok->type == JSMN_ARRAY) { + for (int i = 0; i < tok->size; ++i) { + next = migration_token_skip(doc, next); + } + } + return next; +} + +static bool migration_token_eq (migration_json *doc, int index, const char *value) { + if (!doc || index < 0 || index >= doc->ntokens || !value) return false; + jsmntok_t *tok = &doc->tokens[index]; + int len = tok->end - tok->start; + return tok->type == JSMN_STRING && (int)strlen(value) == len && strncmp(doc->json + tok->start, value, (size_t)len) == 0; +} + +static int migration_object_get (migration_json *doc, int object_index, const char *key) { + if (!doc || object_index < 0 || object_index >= doc->ntokens || !key) return -1; + jsmntok_t *obj = &doc->tokens[object_index]; + if (obj->type != JSMN_OBJECT) return -1; + + int cursor = object_index + 1; + for (int i = 0; i < obj->size; ++i) { + int key_index = cursor; + int value_index = migration_token_skip(doc, key_index); + if (migration_token_eq(doc, key_index, key)) return value_index; + cursor = migration_token_skip(doc, value_index); + } + return -1; +} + +static int migration_array_item (migration_json *doc, int array_index, int item) { + if (!doc || array_index < 0 || array_index >= doc->ntokens || item < 0) return -1; + jsmntok_t *array = &doc->tokens[array_index]; + if (array->type != JSMN_ARRAY || item >= array->size) return -1; + + int cursor = array_index + 1; + for (int i = 0; i < item; ++i) cursor = migration_token_skip(doc, cursor); + return cursor; +} + +static char *migration_token_dup (migration_json *doc, int index) { + if (!doc || index < 0 || index >= doc->ntokens) return NULL; + jsmntok_t *tok = &doc->tokens[index]; + if (tok->start < 0 || tok->end < tok->start) return NULL; + int len = tok->end - tok->start; + if (tok->type != JSMN_STRING) return cloudsync_string_ndup(doc->json + tok->start, (size_t)len); + + char *out = cloudsync_memory_zeroalloc((uint64_t)len + 1); + if (!out) return NULL; + int j = 0; + const char *src = doc->json + tok->start; + for (int i = 0; i < len; ) { + if (src[i] == '\\' && i + 1 < len) { + char c = src[i + 1]; + if (c == '"' || c == '\\' || c == '/') { out[j++] = c; i += 2; } + else if (c == 'n') { out[j++] = '\n'; i += 2; } + else if (c == 'r') { out[j++] = '\r'; i += 2; } + else if (c == 't') { out[j++] = '\t'; i += 2; } + else if (c == 'b') { out[j++] = '\b'; i += 2; } + else if (c == 'f') { out[j++] = '\f'; i += 2; } + else { + out[j++] = src[i++]; + } + } else { + out[j++] = src[i++]; + } + } + out[j] = '\0'; + return out; +} + +static char *migration_object_string (migration_json *doc, int object_index, const char *key) { + int index = migration_object_get(doc, object_index, key); + if (index < 0 || doc->tokens[index].type != JSMN_STRING) return NULL; + return migration_token_dup(doc, index); +} + +static int64_t migration_object_int (migration_json *doc, int object_index, const char *key, int64_t default_value) { + int index = migration_object_get(doc, object_index, key); + if (index < 0 || doc->tokens[index].type != JSMN_PRIMITIVE) return default_value; + char *value = migration_token_dup(doc, index); + if (!value) return default_value; + int64_t result = strtoll(value, NULL, 10); + cloudsync_memory_free(value); + return result; +} + +static bool migration_object_bool (migration_json *doc, int object_index, const char *key, bool default_value) { + int index = migration_object_get(doc, object_index, key); + if (index < 0 || doc->tokens[index].type != JSMN_PRIMITIVE) return default_value; + jsmntok_t *tok = &doc->tokens[index]; + int len = tok->end - tok->start; + if (len == 4 && strncmp(doc->json + tok->start, "true", 4) == 0) return true; + if (len == 5 && strncmp(doc->json + tok->start, "false", 5) == 0) return false; + return default_value; +} + +static bool migration_object_bool_strict (migration_json *doc, int object_index, const char *key, bool default_value, bool *ok) { + int index = migration_object_get(doc, object_index, key); + if (index < 0) return default_value; + if (index >= doc->ntokens || doc->tokens[index].type != JSMN_PRIMITIVE) { + if (ok) *ok = false; + return default_value; + } + jsmntok_t *tok = &doc->tokens[index]; + int len = tok->end - tok->start; + if (len == 4 && strncmp(doc->json + tok->start, "true", 4) == 0) return true; + if (len == 5 && strncmp(doc->json + tok->start, "false", 5) == 0) return false; + if (ok) *ok = false; + return default_value; +} + +static bool migration_buffer_reserve (migration_buffer *buffer, size_t extra) { + if (buffer->len + extra + 1 <= buffer->cap) return true; + size_t cap = buffer->cap ? buffer->cap * 2 : 256; + while (cap < buffer->len + extra + 1) cap *= 2; + char *ptr = cloudsync_memory_realloc(buffer->ptr, cap); + if (!ptr) return false; + buffer->ptr = ptr; + buffer->cap = cap; + return true; +} + +static bool migration_buffer_append_len (migration_buffer *buffer, const char *value, size_t len) { + if (!migration_buffer_reserve(buffer, len)) return false; + memcpy(buffer->ptr + buffer->len, value, len); + buffer->len += len; + buffer->ptr[buffer->len] = '\0'; + return true; +} + +static bool migration_buffer_append (migration_buffer *buffer, const char *value) { + return migration_buffer_append_len(buffer, value, strlen(value)); +} + +static bool migration_buffer_appendf (migration_buffer *buffer, const char *format, ...) { + va_list args; + va_start(args, format); + va_list copy; + va_copy(copy, args); + int needed = vsnprintf(NULL, 0, format, copy); + va_end(copy); + if (needed < 0) { + va_end(args); + return false; + } + if (!migration_buffer_reserve(buffer, (size_t)needed)) { + va_end(args); + return false; + } + vsnprintf(buffer->ptr + buffer->len, buffer->cap - buffer->len, format, args); + va_end(args); + buffer->len += (size_t)needed; + return true; +} + +static char *migration_sql_quote_identifier (const char *value) { + if (!value) return NULL; + size_t len = strlen(value); + char *out = cloudsync_memory_alloc(len * 2 + 3); + if (!out) return NULL; + size_t j = 0; + out[j++] = '"'; + for (size_t i = 0; i < len; ++i) { + if (value[i] == '"') out[j++] = '"'; + out[j++] = value[i]; + } + out[j++] = '"'; + out[j] = '\0'; + return out; +} + +static char *migration_sql_quote_literal (const char *value) { + if (!value) return NULL; + size_t len = strlen(value); + char *out = cloudsync_memory_alloc(len * 2 + 3); + if (!out) return NULL; + size_t j = 0; + out[j++] = '\''; + for (size_t i = 0; i < len; ++i) { + if (value[i] == '\'') out[j++] = '\''; + out[j++] = value[i]; + } + out[j++] = '\''; + out[j] = '\0'; + return out; +} + +static bool migration_json_append_string (migration_buffer *buffer, const char *value) { + if (!migration_buffer_append(buffer, "\"")) return false; + if (value) { + for (const unsigned char *p = (const unsigned char *)value; *p; ++p) { + char tmp[8]; + switch (*p) { + case '"': if (!migration_buffer_append(buffer, "\\\"")) return false; break; + case '\\': if (!migration_buffer_append(buffer, "\\\\")) return false; break; + case '\b': if (!migration_buffer_append(buffer, "\\b")) return false; break; + case '\f': if (!migration_buffer_append(buffer, "\\f")) return false; break; + case '\n': if (!migration_buffer_append(buffer, "\\n")) return false; break; + case '\r': if (!migration_buffer_append(buffer, "\\r")) return false; break; + case '\t': if (!migration_buffer_append(buffer, "\\t")) return false; break; + default: + if (*p < 0x20) { + snprintf(tmp, sizeof(tmp), "\\u%04x", *p); + if (!migration_buffer_append(buffer, tmp)) return false; + } else { + if (!migration_buffer_append_len(buffer, (const char *)p, 1)) return false; + } + break; + } + } + } + return migration_buffer_append(buffer, "\""); +} + +static char *migration_table_ref (cloudsync_context *data, const char *table_name) { +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + const char *schema = cloudsync_schema(data); + return database_build_base_ref(schema, table_name); +#else + (void)data; + return migration_sql_quote_identifier(table_name); +#endif +} + +static bool migration_blocks_table_exists (cloudsync_context *data, const char *table_name) { + if (!table_name) return false; + char *blocks_name = cloudsync_memory_mprintf("%s_cloudsync_blocks", table_name); + if (!blocks_name) return false; + bool exists = database_table_exists(data, blocks_name, cloudsync_schema(data)); + cloudsync_memory_free(blocks_name); + return exists; +} + +static int migration_delete_dropped_column_sync_metadata (cloudsync_context *data, const char *table, const char *column) { + char *table_lit = migration_sql_quote_literal(table); + char *column_lit = migration_sql_quote_literal(column); + if (!table_lit || !column_lit) { + if (table_lit) cloudsync_memory_free(table_lit); + if (column_lit) cloudsync_memory_free(column_lit); + return DBRES_NOMEM; + } + + int rc = DBRES_OK; + if (database_internal_table_exists(data, CLOUDSYNC_TABLE_SETTINGS_NAME)) { + char *sql = cloudsync_memory_mprintf( + "DELETE FROM cloudsync_table_settings WHERE tbl_name = %s AND col_name = %s;", + table_lit, column_lit); + if (!sql) { rc = DBRES_NOMEM; goto cleanup; } + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) goto cleanup; + } + + if (migration_blocks_table_exists(data, table)) { + char *blocks_ref = database_build_blocks_ref(cloudsync_schema(data), table); + char *like_pattern = block_build_colname(column, "%"); + char *like_lit = like_pattern ? migration_sql_quote_literal(like_pattern) : NULL; + if (!blocks_ref || !like_pattern || !like_lit) { + if (blocks_ref) cloudsync_memory_free(blocks_ref); + if (like_pattern) cloudsync_memory_free(like_pattern); + if (like_lit) cloudsync_memory_free(like_lit); + rc = DBRES_NOMEM; + goto cleanup; + } + + char *sql = cloudsync_memory_mprintf("DELETE FROM %s WHERE col_name LIKE %s;", blocks_ref, like_lit); + if (sql) { + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + } else { + rc = DBRES_NOMEM; + } + + cloudsync_memory_free(blocks_ref); + cloudsync_memory_free(like_pattern); + cloudsync_memory_free(like_lit); + if (rc != DBRES_OK) goto cleanup; + } + +cleanup: + cloudsync_memory_free(table_lit); + cloudsync_memory_free(column_lit); + return rc; +} + +static char *migration_sqlite_or_pg_type (const char *logical_type) { + if (!logical_type) logical_type = "text"; +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + if (strcasecmp(logical_type, "uuid") == 0) return cloudsync_string_dup("UUID"); + if (strcasecmp(logical_type, "integer") == 0) return cloudsync_string_dup("BIGINT"); + if (strcasecmp(logical_type, "real") == 0) return cloudsync_string_dup("DOUBLE PRECISION"); + if (strcasecmp(logical_type, "numeric") == 0) return cloudsync_string_dup("NUMERIC"); + if (strcasecmp(logical_type, "blob") == 0) return cloudsync_string_dup("BYTEA"); + if (strcasecmp(logical_type, "boolean") == 0) return cloudsync_string_dup("BOOLEAN"); + if (strcasecmp(logical_type, "json") == 0) return cloudsync_string_dup("JSONB"); + if (strcasecmp(logical_type, "timestamp") == 0) return cloudsync_string_dup("TIMESTAMPTZ"); + return cloudsync_string_dup("TEXT"); +#else + if (strcasecmp(logical_type, "integer") == 0) return cloudsync_string_dup("INTEGER"); + if (strcasecmp(logical_type, "real") == 0) return cloudsync_string_dup("REAL"); + if (strcasecmp(logical_type, "numeric") == 0) return cloudsync_string_dup("NUMERIC"); + if (strcasecmp(logical_type, "blob") == 0) return cloudsync_string_dup("BLOB"); + if (strcasecmp(logical_type, "boolean") == 0) return cloudsync_string_dup("INTEGER"); + return cloudsync_string_dup("TEXT"); +#endif +} + +static char *migration_default_sql (migration_json *doc, int default_index) { + if (default_index < 0) return NULL; + jsmntok_t *tok = &doc->tokens[default_index]; + if (tok->type == JSMN_OBJECT) { + char *type = migration_object_string(doc, default_index, "type"); + int value_index = migration_object_get(doc, default_index, "value"); + char *value = value_index >= 0 ? migration_token_dup(doc, value_index) : NULL; + char *result = NULL; + if (type && strcasecmp(type, "null") == 0) { + result = cloudsync_string_dup("NULL"); + } else if (type && strcasecmp(type, "integer") == 0) { + result = value ? cloudsync_string_dup(value) : cloudsync_string_dup("0"); + } else if (type && strcasecmp(type, "real") == 0) { + result = value ? cloudsync_string_dup(value) : cloudsync_string_dup("0"); + } else if (type && strcasecmp(type, "boolean") == 0) { +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + result = value && (strcmp(value, "true") == 0 || strcmp(value, "1") == 0) ? cloudsync_string_dup("TRUE") : cloudsync_string_dup("FALSE"); +#else + result = value && (strcmp(value, "true") == 0 || strcmp(value, "1") == 0) ? cloudsync_string_dup("1") : cloudsync_string_dup("0"); +#endif + } else if (type && strcasecmp(type, "blob") == 0) { +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + result = cloudsync_memory_mprintf("decode('%s', 'hex')", value ? value : ""); +#else + result = cloudsync_memory_mprintf("X'%s'", value ? value : ""); +#endif + } else { + result = migration_sql_quote_literal(value ? value : ""); + } + if (type) cloudsync_memory_free(type); + if (value) cloudsync_memory_free(value); + return result; + } + if (tok->type == JSMN_STRING) { + char *value = migration_token_dup(doc, default_index); + char *result = migration_sql_quote_literal(value ? value : ""); + if (value) cloudsync_memory_free(value); + return result; + } + if (tok->type == JSMN_PRIMITIVE) { + char *value = migration_token_dup(doc, default_index); + if (!value) return NULL; + if (strcmp(value, "null") == 0) { + cloudsync_memory_free(value); + return cloudsync_string_dup("NULL"); + } + return value; + } + return NULL; +} + +static int migration_current_dialect_object (migration_json *doc, int object_index) { + int dialects = migration_object_get(doc, object_index, "dialects"); + if (dialects < 0 || doc->tokens[dialects].type != JSMN_OBJECT) return -1; +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + return migration_object_get(doc, dialects, "postgresql"); +#else + return migration_object_get(doc, dialects, "sqlite"); +#endif +} + +static char *migration_column_definition (migration_json *doc, int col_index, bool allow_primary_key, bool *primary_key, bool *valid) { + if (primary_key) *primary_key = false; + if (valid) *valid = true; + char *name = migration_object_string(doc, col_index, "name"); + char *logical_type = migration_object_string(doc, col_index, "type"); + char *qname = migration_sql_quote_identifier(name); + bool bool_ok = true; + bool nullable = migration_object_bool_strict(doc, col_index, "nullable", true, &bool_ok); + bool pk_value = migration_object_bool_strict(doc, col_index, "primaryKey", false, &bool_ok); + bool pk = allow_primary_key && pk_value; + int default_index = migration_object_get(doc, col_index, "default"); + int dialect_index = migration_current_dialect_object(doc, col_index); + char *sql_type = NULL; + char *default_sql = NULL; + if (!bool_ok) { + if (valid) *valid = false; + if (name) cloudsync_memory_free(name); + if (logical_type) cloudsync_memory_free(logical_type); + if (qname) cloudsync_memory_free(qname); + return NULL; + } + if (dialect_index >= 0 && doc->tokens[dialect_index].type == JSMN_OBJECT) { + sql_type = migration_object_string(doc, dialect_index, "typeSql"); + default_sql = migration_object_string(doc, dialect_index, "defaultSql"); + } + if (!sql_type) sql_type = migration_sqlite_or_pg_type(logical_type); + if (!default_sql) default_sql = migration_default_sql(doc, default_index); + + migration_buffer buffer = {0}; + if (qname && sql_type) { + migration_buffer_appendf(&buffer, "%s %s", qname, sql_type); + if (!nullable || pk) migration_buffer_append(&buffer, " NOT NULL"); + if (default_sql) migration_buffer_appendf(&buffer, " DEFAULT %s", default_sql); + } + + if (primary_key) *primary_key = pk; + if (name) cloudsync_memory_free(name); + if (logical_type) cloudsync_memory_free(logical_type); + if (qname) cloudsync_memory_free(qname); + if (sql_type) cloudsync_memory_free(sql_type); + if (default_sql) cloudsync_memory_free(default_sql); + return buffer.ptr; +} + +static int migration_create_migrations_table (cloudsync_context *data) { + return database_exec(data, + "CREATE TABLE IF NOT EXISTS cloudsync_migrations (" + "migration_id TEXT PRIMARY KEY NOT NULL," + "schema_epoch INTEGER NOT NULL DEFAULT 0," + "target_schema_hash TEXT," + "applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP" + ");"); +} + +static bool migration_already_applied (cloudsync_context *data, const char *migration_id) { + if (!migration_id) return false; + char *literal = migration_sql_quote_literal(migration_id); + if (!literal) return false; + char *sql = cloudsync_memory_mprintf("SELECT COUNT(*) FROM cloudsync_migrations WHERE migration_id = %s;", literal); + cloudsync_memory_free(literal); + if (!sql) return false; + int64_t count = 0; + int rc = database_select_int(data, sql, &count); + cloudsync_memory_free(sql); + return rc == DBRES_OK && count > 0; +} + +static int migration_record_applied (cloudsync_context *data, const char *migration_id, int64_t schema_epoch, const char *target_hash) { + if (!migration_id) return DBRES_OK; + char *id_lit = migration_sql_quote_literal(migration_id); + char *hash_lit = target_hash ? migration_sql_quote_literal(target_hash) : cloudsync_string_dup("NULL"); + if (!id_lit || !hash_lit) { + if (id_lit) cloudsync_memory_free(id_lit); + if (hash_lit) cloudsync_memory_free(hash_lit); + return DBRES_NOMEM; + } + char *sql = cloudsync_memory_mprintf( + "INSERT INTO cloudsync_migrations (migration_id, schema_epoch, target_schema_hash) " + "VALUES (%s, %lld, %s);", + id_lit, (long long)schema_epoch, hash_lit); + cloudsync_memory_free(id_lit); + cloudsync_memory_free(hash_lit); + if (!sql) return DBRES_NOMEM; + int rc = database_exec(data, sql); + cloudsync_memory_free(sql); + return rc; +} + +static int migration_apply_raw_sql_text (cloudsync_context *data, const char *sql) { + if (!sql || !migration_sql_has_statement(sql)) { + return cloudsync_set_error(data, "rawSql requires SQL", DBRES_MISUSE); + } + if (migration_sql_has_transaction_control(sql)) { + return cloudsync_set_error(data, "rawSql cannot contain transaction control statements", DBRES_MISUSE); + } + return database_exec(data, sql); +} + +static int migration_apply_raw_sql (cloudsync_context *data, migration_json *doc, int op_index) { + int sql_index = migration_object_get(doc, op_index, "sql"); + if (sql_index < 0) return cloudsync_set_error(data, "rawSql requires sql", DBRES_MISUSE); + + int dialect_index = sql_index; + if (doc->tokens[sql_index].type == JSMN_OBJECT) { +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + dialect_index = migration_object_get(doc, sql_index, "postgresql"); +#else + dialect_index = migration_object_get(doc, sql_index, "sqlite"); +#endif + if (dialect_index < 0) { + if (migration_object_bool(doc, op_index, "skipMissingDialect", false)) return DBRES_OK; + return cloudsync_set_error(data, "rawSql does not include SQL for this database dialect", DBRES_MISUSE); + } + } + + if (doc->tokens[dialect_index].type == JSMN_STRING) { + char *sql = migration_token_dup(doc, dialect_index); + if (!sql) return DBRES_NOMEM; + int rc = migration_apply_raw_sql_text(data, sql); + cloudsync_memory_free(sql); + return rc; + } + + if (doc->tokens[dialect_index].type == JSMN_ARRAY) { + int count = doc->tokens[dialect_index].size; + for (int i = 0; i < count; ++i) { + int item = migration_array_item(doc, dialect_index, i); + if (item < 0 || doc->tokens[item].type != JSMN_STRING) return cloudsync_set_error(data, "rawSql array items must be strings", DBRES_MISUSE); + char *sql = migration_token_dup(doc, item); + if (!sql) return DBRES_NOMEM; + int rc = migration_apply_raw_sql_text(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + } + return DBRES_OK; + } + + return cloudsync_set_error(data, "rawSql sql must be a string, array, or dialect object", DBRES_MISUSE); +} + +static int migration_apply_create_table (cloudsync_context *data, migration_json *doc, int op_index) { + char *table = migration_object_string(doc, op_index, "table"); + int cols_index = migration_object_get(doc, op_index, "columns"); + if (!table || cols_index < 0 || doc->tokens[cols_index].type != JSMN_ARRAY) { + if (table) cloudsync_memory_free(table); + return cloudsync_set_error(data, "createTable requires table and columns", DBRES_MISUSE); + } + + if (database_table_exists(data, table, cloudsync_schema(data))) { + cloudsync_memory_free(table); + return cloudsync_set_error(data, "createTable target table already exists", DBRES_MISUSE); + } + + char *table_ref = migration_table_ref(data, table); + if (!table_ref) { + cloudsync_memory_free(table); + return DBRES_NOMEM; + } + migration_buffer sql = {0}; + migration_buffer pk = {0}; + int ncols = doc->tokens[cols_index].size; + if (!migration_buffer_appendf(&sql, "CREATE TABLE %s (", table_ref)) { + cloudsync_memory_free(table); + cloudsync_memory_free(table_ref); + return DBRES_NOMEM; + } + + for (int i = 0; i < ncols; ++i) { + int col_index = migration_array_item(doc, cols_index, i); + bool is_pk = false; + bool valid_column = true; + char *def = migration_column_definition(doc, col_index, true, &is_pk, &valid_column); + char *name = migration_object_string(doc, col_index, "name"); + char *qname = migration_sql_quote_identifier(name); + if (!valid_column) { + if (name) cloudsync_memory_free(name); + if (qname) cloudsync_memory_free(qname); + if (table) cloudsync_memory_free(table); + if (table_ref) cloudsync_memory_free(table_ref); + if (sql.ptr) cloudsync_memory_free(sql.ptr); + if (pk.ptr) cloudsync_memory_free(pk.ptr); + return cloudsync_set_error(data, "Column nullable and primaryKey fields must be boolean values", DBRES_MISUSE); + } + if (!def || !qname) { + if (def) cloudsync_memory_free(def); + if (name) cloudsync_memory_free(name); + if (qname) cloudsync_memory_free(qname); + if (table) cloudsync_memory_free(table); + if (table_ref) cloudsync_memory_free(table_ref); + if (sql.ptr) cloudsync_memory_free(sql.ptr); + if (pk.ptr) cloudsync_memory_free(pk.ptr); + return DBRES_NOMEM; + } + if (i > 0) migration_buffer_append(&sql, ", "); + migration_buffer_append(&sql, def); + if (is_pk) { + if (pk.len > 0) migration_buffer_append(&pk, ", "); + migration_buffer_append(&pk, qname); + } + cloudsync_memory_free(def); + cloudsync_memory_free(name); + cloudsync_memory_free(qname); + } + + if (pk.len > 0) migration_buffer_appendf(&sql, ", PRIMARY KEY (%s)", pk.ptr); + migration_buffer_append(&sql, ");"); + + int rc = database_exec(data, sql.ptr); + if (table) cloudsync_memory_free(table); + if (table_ref) cloudsync_memory_free(table_ref); + if (sql.ptr) cloudsync_memory_free(sql.ptr); + if (pk.ptr) cloudsync_memory_free(pk.ptr); + return rc; +} + +static int migration_apply_add_column (cloudsync_context *data, migration_json *doc, int op_index) { + char *table = migration_object_string(doc, op_index, "table"); + int col_index = migration_object_get(doc, op_index, "column"); + if (!table || col_index < 0 || doc->tokens[col_index].type != JSMN_OBJECT) { + if (table) cloudsync_memory_free(table); + return cloudsync_set_error(data, "addColumn requires table and column", DBRES_MISUSE); + } + + char *table_ref = migration_table_ref(data, table); + bool valid_column = true; + char *def = migration_column_definition(doc, col_index, false, NULL, &valid_column); + if (!valid_column) { + if (table) cloudsync_memory_free(table); + if (table_ref) cloudsync_memory_free(table_ref); + return cloudsync_set_error(data, "Column nullable and primaryKey fields must be boolean values", DBRES_MISUSE); + } + if (!table_ref || !def) { + if (table) cloudsync_memory_free(table); + if (table_ref) cloudsync_memory_free(table_ref); + if (def) cloudsync_memory_free(def); + return DBRES_NOMEM; + } + + bool augmented = table_lookup(data, table) != NULL; + int rc = DBRES_OK; + if (augmented) { + rc = cloudsync_begin_alter(data, table); + if (rc != DBRES_OK) goto cleanup; + } + + char *sql = cloudsync_memory_mprintf("ALTER TABLE %s ADD COLUMN %s;", table_ref, def); + if (!sql) { rc = DBRES_NOMEM; goto cleanup; } + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) goto cleanup; + + if (augmented) rc = cloudsync_commit_alter(data, table); + +cleanup: + cloudsync_memory_free(table); + cloudsync_memory_free(table_ref); + cloudsync_memory_free(def); + return rc; +} + +static int migration_apply_augment_table (cloudsync_context *data, migration_json *doc, int op_index) { + char *table = migration_object_string(doc, op_index, "table"); + char *algo = migration_object_string(doc, op_index, "algorithm"); + int64_t init_flags = migration_object_int(doc, op_index, "initFlags", 0); + if (!table) return cloudsync_set_error(data, "augmentTable requires table", DBRES_MISUSE); + int rc = cloudsync_init_table(data, table, algo ? algo : CLOUDSYNC_DEFAULT_ALGO, (CLOUDSYNC_INIT_FLAG)init_flags); + cloudsync_memory_free(table); + if (algo) cloudsync_memory_free(algo); + return rc; +} + +static int migration_apply_set_block_lww (cloudsync_context *data, migration_json *doc, int op_index) { + char *table = migration_object_string(doc, op_index, "table"); + char *column = migration_object_string(doc, op_index, "column"); + char *delimiter = migration_object_string(doc, op_index, "delimiter"); + if (!table || !column) { + if (table) cloudsync_memory_free(table); + if (column) cloudsync_memory_free(column); + if (delimiter) cloudsync_memory_free(delimiter); + return cloudsync_set_error(data, "setBlockLww requires table and column", DBRES_MISUSE); + } + int rc = cloudsync_setup_block_column(data, table, column, delimiter, true); + cloudsync_memory_free(table); + cloudsync_memory_free(column); + if (delimiter) cloudsync_memory_free(delimiter); + return rc; +} + +static int migration_apply_set_column (cloudsync_context *data, migration_json *doc, int op_index) { + char *table = migration_object_string(doc, op_index, "table"); + char *column = migration_object_string(doc, op_index, "column"); + char *key = migration_object_string(doc, op_index, "key"); + char *value = migration_object_string(doc, op_index, "value"); + if (!table || !column || !key) { + if (table) cloudsync_memory_free(table); + if (column) cloudsync_memory_free(column); + if (key) cloudsync_memory_free(key); + if (value) cloudsync_memory_free(value); + return cloudsync_set_error(data, "setColumn requires table, column, and key", DBRES_MISUSE); + } + int rc; + if (value && strcmp(key, "algo") == 0 && strcmp(value, "block") == 0) { + rc = cloudsync_setup_block_column(data, table, column, NULL, true); + } else { + rc = dbutils_table_settings_set_key_value(data, table, column, key, value); + } + cloudsync_memory_free(table); + cloudsync_memory_free(column); + cloudsync_memory_free(key); + if (value) cloudsync_memory_free(value); + return rc; +} + +static int migration_apply_set_filter (cloudsync_context *data, migration_json *doc, int op_index) { + char *table = migration_object_string(doc, op_index, "table"); + char *filter = migration_object_string(doc, op_index, "filter"); + if (!filter) { + int filters = migration_object_get(doc, op_index, "filters"); + if (filters >= 0 && doc->tokens[filters].type == JSMN_OBJECT) { +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + int dialect_filter = migration_object_get(doc, filters, "postgresql"); +#else + int dialect_filter = migration_object_get(doc, filters, "sqlite"); +#endif + if (dialect_filter >= 0 && doc->tokens[dialect_filter].type == JSMN_STRING) { + filter = migration_token_dup(doc, dialect_filter); + } + } + } + if (!table || !filter) { + if (table) cloudsync_memory_free(table); + if (filter) cloudsync_memory_free(filter); + return cloudsync_set_error(data, "setFilter requires table and filter", DBRES_MISUSE); + } + if (!table_lookup(data, table)) { + cloudsync_memory_free(table); + cloudsync_memory_free(filter); + return cloudsync_set_error(data, "setFilter table is not configured for sync", DBRES_MISUSE); + } + + int rc = dbutils_table_settings_set_key_value(data, table, "*", "filter", filter); + if (rc == DBRES_OK) { + table_algo algo = dbutils_table_settings_get_algo(data, table); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + rc = database_delete_triggers(data, table); + if (rc == DBRES_OK) rc = database_create_triggers(data, table, algo, filter); + if (rc == DBRES_OK) rc = cloudsync_reset_metatable(data, table); + } + + cloudsync_memory_free(table); + cloudsync_memory_free(filter); + return rc; +} + +static int migration_apply_drop_column (cloudsync_context *data, migration_json *doc, int op_index) { + char *table = migration_object_string(doc, op_index, "table"); + char *column = migration_object_string(doc, op_index, "column"); + if (!table || !column) { + if (table) cloudsync_memory_free(table); + if (column) cloudsync_memory_free(column); + return cloudsync_set_error(data, "dropColumn requires table and column", DBRES_MISUSE); + } + char *table_ref = migration_table_ref(data, table); + char *qcol = migration_sql_quote_identifier(column); + if (!table_ref || !qcol) { + if (table) cloudsync_memory_free(table); + if (column) cloudsync_memory_free(column); + if (table_ref) cloudsync_memory_free(table_ref); + if (qcol) cloudsync_memory_free(qcol); + return DBRES_NOMEM; + } + + bool augmented = table_lookup(data, table) != NULL; + int rc = DBRES_OK; + if (augmented) { + rc = cloudsync_begin_alter(data, table); + if (rc != DBRES_OK) goto cleanup; + } + char *sql = cloudsync_memory_mprintf("ALTER TABLE %s DROP COLUMN %s;", table_ref, qcol); + if (!sql) { rc = DBRES_NOMEM; goto cleanup; } + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) goto cleanup; + if (augmented) { + rc = migration_delete_dropped_column_sync_metadata(data, table, column); + if (rc == DBRES_OK) rc = cloudsync_commit_alter(data, table); + } + +cleanup: + cloudsync_memory_free(table); + cloudsync_memory_free(column); + cloudsync_memory_free(table_ref); + cloudsync_memory_free(qcol); + return rc; +} + +static int migration_update_renamed_column_metadata (cloudsync_context *data, const char *table, const char *from, const char *to) { + char *meta_ref = database_build_meta_ref(cloudsync_schema(data), table); + char *blocks_ref = database_build_blocks_ref(cloudsync_schema(data), table); + char *from_lit = migration_sql_quote_literal(from); + char *to_lit = migration_sql_quote_literal(to); + if (!meta_ref || !blocks_ref || !from_lit || !to_lit) { + if (meta_ref) cloudsync_memory_free(meta_ref); + if (blocks_ref) cloudsync_memory_free(blocks_ref); + if (from_lit) cloudsync_memory_free(from_lit); + if (to_lit) cloudsync_memory_free(to_lit); + return DBRES_NOMEM; + } + +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + char *expr = cloudsync_memory_mprintf( + "CASE WHEN col_name = %s THEN %s " + "WHEN col_name LIKE (%s || chr(31) || '%%') THEN %s || substring(col_name from char_length(%s) + 1) " + "ELSE col_name END", + from_lit, to_lit, from_lit, to_lit, from_lit); + char *where = cloudsync_memory_mprintf("col_name = %s OR col_name LIKE (%s || chr(31) || '%%')", from_lit, from_lit); +#else + char *expr = cloudsync_memory_mprintf( + "CASE WHEN col_name = %s THEN %s " + "WHEN col_name LIKE (%s || char(31) || '%%') THEN %s || substr(col_name, length(%s) + 1) " + "ELSE col_name END", + from_lit, to_lit, from_lit, to_lit, from_lit); + char *where = cloudsync_memory_mprintf("col_name = %s OR col_name LIKE (%s || char(31) || '%%')", from_lit, from_lit); +#endif + int rc = DBRES_NOMEM; + if (expr && where) { + char *sql = cloudsync_memory_mprintf("UPDATE %s SET col_name = %s WHERE %s;", meta_ref, expr, where); + if (sql) { + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + } + if (rc == DBRES_OK && migration_blocks_table_exists(data, table)) { + sql = cloudsync_memory_mprintf("UPDATE %s SET col_name = %s WHERE %s;", blocks_ref, expr, where); + if (sql) { + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + } + } + } + if (expr) cloudsync_memory_free(expr); + if (where) cloudsync_memory_free(where); + cloudsync_memory_free(meta_ref); + cloudsync_memory_free(blocks_ref); + cloudsync_memory_free(from_lit); + cloudsync_memory_free(to_lit); + return rc; +} + +static int migration_apply_rename_column (cloudsync_context *data, migration_json *doc, int op_index) { + char *table = migration_object_string(doc, op_index, "table"); + char *from = migration_object_string(doc, op_index, "from"); + char *to = migration_object_string(doc, op_index, "to"); + if (!table || !from || !to) { + if (table) cloudsync_memory_free(table); + if (from) cloudsync_memory_free(from); + if (to) cloudsync_memory_free(to); + return cloudsync_set_error(data, "renameColumn requires table, from, and to", DBRES_MISUSE); + } + char *table_ref = migration_table_ref(data, table); + char *qfrom = migration_sql_quote_identifier(from); + char *qto = migration_sql_quote_identifier(to); + if (!table_ref || !qfrom || !qto) { + if (table) cloudsync_memory_free(table); + if (from) cloudsync_memory_free(from); + if (to) cloudsync_memory_free(to); + if (table_ref) cloudsync_memory_free(table_ref); + if (qfrom) cloudsync_memory_free(qfrom); + if (qto) cloudsync_memory_free(qto); + return DBRES_NOMEM; + } + + bool augmented = table_lookup(data, table) != NULL; + int rc = DBRES_OK; + if (augmented) { + rc = cloudsync_begin_alter(data, table); + if (rc != DBRES_OK) goto cleanup; + } + + char *sql = cloudsync_memory_mprintf("ALTER TABLE %s RENAME COLUMN %s TO %s;", table_ref, qfrom, qto); + if (!sql) { rc = DBRES_NOMEM; goto cleanup; } + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) goto cleanup; + + if (augmented) { + rc = migration_update_renamed_column_metadata(data, table, from, to); + if (rc == DBRES_OK) { + char *from_lit = migration_sql_quote_literal(from); + char *to_lit = migration_sql_quote_literal(to); + char *table_lit = migration_sql_quote_literal(table); + if (!from_lit || !to_lit || !table_lit) rc = DBRES_NOMEM; + else { + sql = cloudsync_memory_mprintf("UPDATE cloudsync_table_settings SET col_name = %s WHERE tbl_name = %s AND col_name = %s;", to_lit, table_lit, from_lit); + if (sql) { + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + } else rc = DBRES_NOMEM; + } + if (from_lit) cloudsync_memory_free(from_lit); + if (to_lit) cloudsync_memory_free(to_lit); + if (table_lit) cloudsync_memory_free(table_lit); + } + if (rc == DBRES_OK) rc = cloudsync_commit_alter(data, table); + } + +cleanup: + cloudsync_memory_free(table); + cloudsync_memory_free(from); + cloudsync_memory_free(to); + cloudsync_memory_free(table_ref); + cloudsync_memory_free(qfrom); + cloudsync_memory_free(qto); + return rc; +} + +static int migration_apply_rebuild_table_sync (cloudsync_context *data, migration_json *doc, int op_index) { + char *table = migration_object_string(doc, op_index, "table"); + char *algo = migration_object_string(doc, op_index, "algorithm"); + int64_t init_flags = migration_object_int(doc, op_index, "initFlags", 0); + if (!table) return cloudsync_set_error(data, "rebuildTableSync requires table", DBRES_MISUSE); + + int ddl_index = migration_object_get(doc, op_index, "ddl"); + if (ddl_index >= 0 && doc->tokens[ddl_index].type != JSMN_ARRAY) { + cloudsync_memory_free(table); + if (algo) cloudsync_memory_free(algo); + return cloudsync_set_error(data, "rebuildTableSync ddl must be an array", DBRES_MISUSE); + } + + int block_index = migration_object_get(doc, op_index, "blockLww"); + if (block_index >= 0 && doc->tokens[block_index].type != JSMN_ARRAY) { + cloudsync_memory_free(table); + if (algo) cloudsync_memory_free(algo); + return cloudsync_set_error(data, "rebuildTableSync blockLww must be an array", DBRES_MISUSE); + } + + if (table_lookup(data, table)) { + int rc = cloudsync_cleanup(data, table, true); + if (rc != DBRES_OK) { + cloudsync_memory_free(table); + if (algo) cloudsync_memory_free(algo); + return rc; + } + } + + if (ddl_index >= 0) { + int count = doc->tokens[ddl_index].size; + for (int i = 0; i < count; ++i) { + int item = migration_array_item(doc, ddl_index, i); + char *op = migration_object_string(doc, item, "op"); + int rc = DBRES_OK; + if (op && strcmp(op, "rawSql") == 0) rc = migration_apply_raw_sql(data, doc, item); + else rc = cloudsync_set_error(data, "rebuildTableSync ddl currently supports rawSql operations only", DBRES_MISUSE); + if (op) cloudsync_memory_free(op); + if (rc != DBRES_OK) { + cloudsync_memory_free(table); + if (algo) cloudsync_memory_free(algo); + return rc; + } + } + } + + int rc = cloudsync_init_table(data, table, algo ? algo : CLOUDSYNC_DEFAULT_ALGO, (CLOUDSYNC_INIT_FLAG)init_flags); + if (rc != DBRES_OK) { + cloudsync_memory_free(table); + if (algo) cloudsync_memory_free(algo); + return rc; + } + + if (block_index >= 0 && doc->tokens[block_index].type == JSMN_ARRAY) { + int count = doc->tokens[block_index].size; + for (int i = 0; i < count; ++i) { + int item = migration_array_item(doc, block_index, i); + char *column = migration_object_string(doc, item, "column"); + char *delimiter = migration_object_string(doc, item, "delimiter"); + if (!column) rc = cloudsync_set_error(data, "rebuildTableSync blockLww requires column", DBRES_MISUSE); + else rc = cloudsync_setup_block_column(data, table, column, delimiter, true); + if (column) cloudsync_memory_free(column); + if (delimiter) cloudsync_memory_free(delimiter); + if (rc != DBRES_OK) break; + } + } + + cloudsync_memory_free(table); + if (algo) cloudsync_memory_free(algo); + return rc; +} + +static int migration_apply_op (cloudsync_context *data, migration_json *doc, int op_index) { + char *op = migration_object_string(doc, op_index, "op"); + if (!op) return cloudsync_set_error(data, "Migration operation requires op", DBRES_MISUSE); + int rc = DBRES_MISUSE; + if (strcmp(op, "createTable") == 0) rc = migration_apply_create_table(data, doc, op_index); + else if (strcmp(op, "addColumn") == 0) rc = migration_apply_add_column(data, doc, op_index); + else if (strcmp(op, "augmentTable") == 0) rc = migration_apply_augment_table(data, doc, op_index); + else if (strcmp(op, "setBlockLww") == 0) rc = migration_apply_set_block_lww(data, doc, op_index); + else if (strcmp(op, "setColumn") == 0) rc = migration_apply_set_column(data, doc, op_index); + else if (strcmp(op, "setFilter") == 0) rc = migration_apply_set_filter(data, doc, op_index); + else if (strcmp(op, "rawSql") == 0) rc = migration_apply_raw_sql(data, doc, op_index); + else if (strcmp(op, "dropColumn") == 0) rc = migration_apply_drop_column(data, doc, op_index); + else if (strcmp(op, "renameColumn") == 0) rc = migration_apply_rename_column(data, doc, op_index); + else if (strcmp(op, "rebuildTableSync") == 0) rc = migration_apply_rebuild_table_sync(data, doc, op_index); + else { + char buffer[256]; + snprintf(buffer, sizeof(buffer), "Unsupported migration operation %s", op); + rc = cloudsync_set_error(data, buffer, DBRES_MISUSE); + } + cloudsync_memory_free(op); + return rc; +} + +int cloudsync_migration_apply (cloudsync_context *data, const char *payload, int payload_len, char **result_json) { + if (result_json) *result_json = NULL; + if (!data || !payload || payload_len <= 0) return cloudsync_set_error(data, "cloudsync_migration_apply requires a JSON payload", DBRES_MISUSE); + + jsmntok_t *tokens = NULL; + int ntokens = 0; + int parse_rc = migration_json_parse_root_object_alloc(payload, (size_t)payload_len, &tokens, &ntokens); + if (parse_rc == DBRES_NOMEM) return DBRES_NOMEM; + if (parse_rc != DBRES_OK) { + return cloudsync_set_error(data, "Invalid migration JSON payload", DBRES_MISUSE); + } + + migration_json doc = {payload, tokens, ntokens}; + char *migration_id = migration_object_string(&doc, 0, "migrationId"); + char *target_hash = migration_object_string(&doc, 0, "targetSchemaHash"); + char *computed_target_hash = NULL; + char *base_hash = migration_object_string(&doc, 0, "baseSchemaHash"); + int64_t schema_epoch = migration_object_int(&doc, 0, "schemaEpoch", migration_object_int(&doc, 0, "targetSchemaEpoch", 0)); + int ops_index = migration_object_get(&doc, 0, "ops"); + if (ops_index < 0 || tokens[ops_index].type != JSMN_ARRAY) { + if (migration_id) cloudsync_memory_free(migration_id); + if (target_hash) cloudsync_memory_free(target_hash); + if (base_hash) cloudsync_memory_free(base_hash); + cloudsync_memory_free(tokens); + return cloudsync_set_error(data, "Migration payload requires ops array", DBRES_MISUSE); + } + + int rc = migration_create_migrations_table(data); + if (rc != DBRES_OK) goto cleanup; + + if (migration_id && migration_already_applied(data, migration_id)) { + if (result_json) *result_json = cloudsync_memory_mprintf("{\"status\":\"already_applied\",\"migrationId\":\"%s\"}", migration_id); + rc = DBRES_OK; + goto cleanup; + } + + if (base_hash) { + uint64_t current = database_schema_hash(data); + uint64_t expected = (uint64_t)strtoull(base_hash, NULL, 10); + if (current != expected) { + rc = cloudsync_set_error(data, "Migration baseSchemaHash does not match the current schema hash", DBRES_MISUSE); + goto cleanup; + } + } + + rc = database_begin_savepoint(data, CLOUDSYNC_MIGRATION_SAVEPOINT); + if (rc != DBRES_OK) goto cleanup; + + int op_count = tokens[ops_index].size; + for (int i = 0; i < op_count; ++i) { + int op_index = migration_array_item(&doc, ops_index, i); + rc = migration_apply_op(data, &doc, op_index); + if (rc != DBRES_OK) break; + } + + if (rc == DBRES_OK && cloudsync_config_exists(data) && dbutils_table_settings_count_tables(data) > 0) { + cloudsync_update_schema_hash(data); + uint64_t current = database_schema_hash(data); + computed_target_hash = cloudsync_memory_mprintf("%" PRIu64, current); + if (target_hash) { + uint64_t expected = (uint64_t)strtoull(target_hash, NULL, 10); + if (current != expected) { + rc = cloudsync_set_error(data, "Migration targetSchemaHash does not match the computed schema hash", DBRES_MISUSE); + } + } + } + + if (rc == DBRES_OK) rc = migration_record_applied(data, migration_id, schema_epoch, target_hash ? target_hash : computed_target_hash); + + if (rc == DBRES_OK) { + rc = database_commit_savepoint(data, CLOUDSYNC_MIGRATION_SAVEPOINT); + if (result_json && rc == DBRES_OK) { + *result_json = cloudsync_memory_mprintf("{\"status\":\"applied\",\"migrationId\":%s%s%s}", + migration_id ? "\"" : "null", + migration_id ? migration_id : "", + migration_id ? "\"" : ""); + } + } else { + database_rollback_savepoint(data, CLOUDSYNC_MIGRATION_SAVEPOINT); + cloudsync_terminate(data); + } + +cleanup: + if (migration_id) cloudsync_memory_free(migration_id); + if (target_hash) cloudsync_memory_free(target_hash); + if (computed_target_hash) cloudsync_memory_free(computed_target_hash); + if (base_hash) cloudsync_memory_free(base_hash); + cloudsync_memory_free(tokens); + return rc; +} + +// MARK: - Declarative alter builder - + +static void pending_alter_op_free(pending_alter_op *op) { + if (!op) return; + if (op->table) cloudsync_memory_free(op->table); + if (op->a) cloudsync_memory_free(op->a); + if (op->b) cloudsync_memory_free(op->b); + if (op->c) cloudsync_memory_free(op->c); + if (op->d) cloudsync_memory_free(op->d); + if (op->default_kind) cloudsync_memory_free(op->default_kind); + if (op->default_value) cloudsync_memory_free(op->default_value); + if (op->sqlite_type_sql) cloudsync_memory_free(op->sqlite_type_sql); + if (op->sqlite_default_sql) cloudsync_memory_free(op->sqlite_default_sql); + if (op->postgresql_type_sql) cloudsync_memory_free(op->postgresql_type_sql); + if (op->postgresql_default_sql) cloudsync_memory_free(op->postgresql_default_sql); + if (op->sqlite_filter_sql) cloudsync_memory_free(op->sqlite_filter_sql); + if (op->postgresql_filter_sql) cloudsync_memory_free(op->postgresql_filter_sql); + cloudsync_memory_free(op); +} + +static pending_alter_context *pending_alter_context_for(cloudsync_context *data, bool create) { + pending_alter_context *ctx = g_pending_alters; + while (ctx) { + if (ctx->data == data) return ctx; + ctx = ctx->next; + } + if (!create) return NULL; + ctx = cloudsync_memory_zeroalloc(sizeof(*ctx)); + if (!ctx) return NULL; + ctx->data = data; + ctx->next = g_pending_alters; + g_pending_alters = ctx; + return ctx; +} + +static int pending_alter_append(cloudsync_context *data, pending_alter_op *op) { + pending_alter_context *ctx = pending_alter_context_for(data, true); + if (!ctx) { + pending_alter_op_free(op); + return DBRES_NOMEM; + } + if (!ctx->ops) ctx->ops = op; + else { + pending_alter_op *tail = ctx->ops; + while (tail->next) tail = tail->next; + tail->next = op; + } + return DBRES_OK; +} + +static pending_alter_op *pending_alter_op_new(pending_alter_kind kind, const char *table) { + pending_alter_op *op = cloudsync_memory_zeroalloc(sizeof(*op)); + if (!op) return NULL; + op->kind = kind; + if (table) { + op->table = cloudsync_string_dup(table); + if (!op->table) { + pending_alter_op_free(op); + return NULL; + } + } + return op; +} + +static bool pending_table_eq(pending_alter_op *op, const char *table) { + return op && op->table && table && strcmp(op->table, table) == 0; +} + +static pending_alter_op *pending_find_add_column(cloudsync_context *data, const char *table, const char *column) { + pending_alter_context *ctx = pending_alter_context_for(data, false); + for (pending_alter_op *op = ctx ? ctx->ops : NULL; op; op = op->next) { + if (op->kind == PENDING_ALTER_ADD_COLUMN && pending_table_eq(op, table) && op->a && column && strcmp(op->a, column) == 0) return op; + } + return NULL; +} + +static bool pending_has_create_table(cloudsync_context *data, const char *table) { + pending_alter_context *ctx = pending_alter_context_for(data, false); + for (pending_alter_op *op = ctx ? ctx->ops : NULL; op; op = op->next) { + if (op->kind == PENDING_ALTER_CREATE_TABLE && pending_table_eq(op, table)) return true; + } + return false; +} + +static bool pending_column_is_pk(cloudsync_context *data, const char *table, const char *column) { + pending_alter_context *ctx = pending_alter_context_for(data, false); + for (pending_alter_op *op = ctx ? ctx->ops : NULL; op; op = op->next) { + if (op->kind == PENDING_ALTER_ADD_PRIMARY_KEY && pending_table_eq(op, table) && op->a && column && strcmp(op->a, column) == 0) return true; + } + return false; +} + +static bool pending_table_has_pk(cloudsync_context *data, const char *table) { + pending_alter_context *ctx = pending_alter_context_for(data, false); + for (pending_alter_op *op = ctx ? ctx->ops : NULL; op; op = op->next) { + if (op->kind == PENDING_ALTER_ADD_PRIMARY_KEY && pending_table_eq(op, table)) return true; + } + return false; +} + +static bool pending_sql_default_is_non_null (const char *sql) { + if (!sql) return false; + while (*sql == ' ' || *sql == '\t' || *sql == '\n' || *sql == '\r') sql++; + if (strncasecmp(sql, "NULL", 4) == 0) { + char next = sql[4]; + if (!((next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') || (next >= '0' && next <= '9') || next == '_')) return false; + } + return true; +} + +static bool pending_column_has_current_default(pending_alter_op *op) { + if (!op) return false; +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + const char *sql = op->postgresql_default_sql; +#else + const char *sql = op->sqlite_default_sql; +#endif + if (sql) return pending_sql_default_is_non_null(sql); + return op->has_default && op->default_kind && strcasecmp(op->default_kind, "null") != 0; +} + +static bool pending_append_column_json(migration_buffer *buffer, cloudsync_context *data, pending_alter_op *op, bool include_pk) { + bool pk = include_pk && pending_column_is_pk(data, op->table, op->a); + if (!migration_buffer_append(buffer, "{")) return false; + if (!migration_buffer_append(buffer, "\"name\":")) return false; + if (!migration_json_append_string(buffer, op->a)) return false; + if (!migration_buffer_append(buffer, ",\"type\":")) return false; + if (!migration_json_append_string(buffer, op->b ? op->b : "text")) return false; + if (!migration_buffer_appendf(buffer, ",\"nullable\":%s", op->flag ? "true" : "false")) return false; + if (pk && !migration_buffer_append(buffer, ",\"primaryKey\":true")) return false; + if (op->has_default) { + if (!migration_buffer_append(buffer, ",\"default\":{\"type\":")) return false; + if (!migration_json_append_string(buffer, op->default_kind ? op->default_kind : "text")) return false; + if (op->default_value || (op->default_kind && strcasecmp(op->default_kind, "null") != 0)) { + if (!migration_buffer_append(buffer, ",\"value\":")) return false; + if (!migration_json_append_string(buffer, op->default_value ? op->default_value : "")) return false; + } + if (!migration_buffer_append(buffer, "}")) return false; + } + bool has_sqlite = op->sqlite_type_sql || op->sqlite_default_sql; + bool has_pg = op->postgresql_type_sql || op->postgresql_default_sql; + if (has_sqlite || has_pg) { + bool first_dialect = true; + if (!migration_buffer_append(buffer, ",\"dialects\":{")) return false; + if (has_sqlite) { + if (!migration_buffer_append(buffer, "\"sqlite\":{")) return false; + bool first = true; + if (op->sqlite_type_sql) { + if (!migration_buffer_append(buffer, "\"typeSql\":")) return false; + if (!migration_json_append_string(buffer, op->sqlite_type_sql)) return false; + first = false; + } + if (op->sqlite_default_sql) { + if (!first && !migration_buffer_append(buffer, ",")) return false; + if (!migration_buffer_append(buffer, "\"defaultSql\":")) return false; + if (!migration_json_append_string(buffer, op->sqlite_default_sql)) return false; + } + if (!migration_buffer_append(buffer, "}")) return false; + first_dialect = false; + } + if (has_pg) { + if (!first_dialect && !migration_buffer_append(buffer, ",")) return false; + if (!migration_buffer_append(buffer, "\"postgresql\":{")) return false; + bool first = true; + if (op->postgresql_type_sql) { + if (!migration_buffer_append(buffer, "\"typeSql\":")) return false; + if (!migration_json_append_string(buffer, op->postgresql_type_sql)) return false; + first = false; + } + if (op->postgresql_default_sql) { + if (!first && !migration_buffer_append(buffer, ",")) return false; + if (!migration_buffer_append(buffer, "\"defaultSql\":")) return false; + if (!migration_json_append_string(buffer, op->postgresql_default_sql)) return false; + } + if (!migration_buffer_append(buffer, "}")) return false; + } + if (!migration_buffer_append(buffer, "}")) return false; + } + return migration_buffer_append(buffer, "}"); +} + +static int pending_validate_ops(cloudsync_context *data, bool *destructive) { + pending_alter_context *ctx = pending_alter_context_for(data, false); + if (!ctx || !ctx->ops) return cloudsync_set_error(data, "No pending alter operations", DBRES_MISUSE); + for (pending_alter_op *op = ctx->ops; op; op = op->next) { + if (op->kind == PENDING_ALTER_RAW_SQL) { + if (destructive) *destructive = true; + continue; + } + if (!op->table || !op->table[0]) return cloudsync_set_error(data, "cloudsync alter operation requires table", DBRES_MISUSE); + if (op->kind == PENDING_ALTER_DROP_COLUMN || op->kind == PENDING_ALTER_RENAME_COLUMN) { + if (destructive) *destructive = true; + } + if (op->kind == PENDING_ALTER_ADD_COLUMN && !op->flag && !pending_column_has_current_default(op) && !pending_column_is_pk(data, op->table, op->a)) { + return cloudsync_set_error(data, "NOT NULL columns require a default value unless they are primary keys", DBRES_MISUSE); + } + if (op->kind == PENDING_ALTER_ADD_PRIMARY_KEY && !pending_has_create_table(data, op->table)) { + return cloudsync_set_error(data, "cloudsync_alter_add_primary_key is supported only while creating a table", DBRES_MISUSE); + } + } + for (pending_alter_op *op = ctx->ops; op; op = op->next) { + if (op->kind == PENDING_ALTER_CREATE_TABLE && !pending_table_has_pk(data, op->table)) { + return cloudsync_set_error(data, "createTable requires at least one primary key", DBRES_MISUSE); + } + } + return DBRES_OK; +} + +static bool pending_append_set_filter_json(migration_buffer *buffer, pending_alter_op *op) { + if (!migration_buffer_append(buffer, "{\"op\":\"setFilter\",\"table\":")) return false; + if (!migration_json_append_string(buffer, op->table)) return false; + if (op->a) { + if (!migration_buffer_append(buffer, ",\"filter\":")) return false; + if (!migration_json_append_string(buffer, op->a)) return false; + } + if (op->sqlite_filter_sql || op->postgresql_filter_sql) { + bool first = true; + if (!migration_buffer_append(buffer, ",\"filters\":{")) return false; + if (op->sqlite_filter_sql) { + if (!migration_buffer_append(buffer, "\"sqlite\":")) return false; + if (!migration_json_append_string(buffer, op->sqlite_filter_sql)) return false; + first = false; + } + if (op->postgresql_filter_sql) { + if (!first && !migration_buffer_append(buffer, ",")) return false; + if (!migration_buffer_append(buffer, "\"postgresql\":")) return false; + if (!migration_json_append_string(buffer, op->postgresql_filter_sql)) return false; + } + if (!migration_buffer_append(buffer, "}")) return false; + } + return migration_buffer_append(buffer, "}"); +} + +static bool pending_append_raw_sql_json(migration_buffer *buffer, pending_alter_op *op) { + if (!migration_buffer_append(buffer, "{\"op\":\"rawSql\",\"sql\":")) return false; + if (op->b) { + if (!migration_buffer_append(buffer, "{")) return false; + if (!migration_json_append_string(buffer, op->b)) return false; + if (!migration_buffer_append(buffer, ":[")) return false; + if (!migration_json_append_string(buffer, op->a)) return false; + if (!migration_buffer_append(buffer, "]},\"skipMissingDialect\":true")) return false; + } else { + if (!migration_json_append_string(buffer, op->a)) return false; + } + return migration_buffer_append(buffer, "}"); +} + +static char *pending_build_payload(cloudsync_context *data, const char *migration_id, const char *base_hash, const char *target_hash, bool *destructive_out) { + bool destructive = false; + int rc = pending_validate_ops(data, &destructive); + if (rc != DBRES_OK) return NULL; + if (destructive_out) *destructive_out = destructive; + + pending_alter_context *ctx = pending_alter_context_for(data, false); + migration_buffer payload = {0}; + if (!migration_buffer_append(&payload, "{")) goto oom; + if (!migration_buffer_append(&payload, "\"type\":\"cloudsync.schema.migration\"")) goto oom; + if (!migration_buffer_appendf(&payload, ",\"formatVersion\":%d", destructive ? 2 : 1)) goto oom; + if (migration_id) { + if (!migration_buffer_append(&payload, ",\"migrationId\":")) goto oom; + if (!migration_json_append_string(&payload, migration_id)) goto oom; + } + if (base_hash) { + if (!migration_buffer_append(&payload, ",\"baseSchemaHash\":")) goto oom; + if (!migration_json_append_string(&payload, base_hash)) goto oom; + } + if (target_hash) { + if (!migration_buffer_append(&payload, ",\"targetSchemaHash\":")) goto oom; + if (!migration_json_append_string(&payload, target_hash)) goto oom; + } + if (!migration_buffer_append(&payload, destructive ? ",\"requiredCapabilities\":[\"schema:write\",\"schema:destructive\"]" : ",\"requiredCapabilities\":[\"schema:write\"]")) goto oom; + if (!migration_buffer_append(&payload, ",\"ops\":[")) goto oom; + bool first_op = true; + + for (pending_alter_op *op = ctx->ops; op; op = op->next) { + if (op->kind == PENDING_ALTER_ADD_PRIMARY_KEY) continue; + if (op->kind == PENDING_ALTER_ADD_COLUMN && pending_has_create_table(data, op->table)) continue; + if (!first_op && !migration_buffer_append(&payload, ",")) goto oom; + switch (op->kind) { + case PENDING_ALTER_CREATE_TABLE: { + if (!migration_buffer_append(&payload, "{\"op\":\"createTable\",\"table\":")) goto oom; + if (!migration_json_append_string(&payload, op->table)) goto oom; + if (!migration_buffer_append(&payload, ",\"columns\":[")) goto oom; + bool first_col = true; + for (pending_alter_op *col = ctx->ops; col; col = col->next) { + if (col->kind != PENDING_ALTER_ADD_COLUMN || !pending_table_eq(col, op->table)) continue; + if (!first_col && !migration_buffer_append(&payload, ",")) goto oom; + if (!pending_append_column_json(&payload, data, col, true)) goto oom; + first_col = false; + } + if (!migration_buffer_append(&payload, "]}")) goto oom; + break; + } + case PENDING_ALTER_ADD_COLUMN: + if (!migration_buffer_append(&payload, "{\"op\":\"addColumn\",\"table\":")) goto oom; + if (!migration_json_append_string(&payload, op->table)) goto oom; + if (!migration_buffer_append(&payload, ",\"column\":")) goto oom; + if (!pending_append_column_json(&payload, data, op, false)) goto oom; + if (!migration_buffer_append(&payload, "}")) goto oom; + break; + case PENDING_ALTER_AUGMENT_TABLE: + if (!migration_buffer_append(&payload, "{\"op\":\"augmentTable\",\"table\":")) goto oom; + if (!migration_json_append_string(&payload, op->table)) goto oom; + if (!migration_buffer_append(&payload, ",\"algorithm\":")) goto oom; + if (!migration_json_append_string(&payload, op->a ? op->a : CLOUDSYNC_DEFAULT_ALGO)) goto oom; + if (!migration_buffer_appendf(&payload, ",\"initFlags\":%lld}", (long long)op->number)) goto oom; + break; + case PENDING_ALTER_SET_BLOCK_LWW: + if (!migration_buffer_append(&payload, "{\"op\":\"setBlockLww\",\"table\":")) goto oom; + if (!migration_json_append_string(&payload, op->table)) goto oom; + if (!migration_buffer_append(&payload, ",\"column\":")) goto oom; + if (!migration_json_append_string(&payload, op->a)) goto oom; + if (op->b) { + if (!migration_buffer_append(&payload, ",\"delimiter\":")) goto oom; + if (!migration_json_append_string(&payload, op->b)) goto oom; + } + if (!migration_buffer_append(&payload, "}")) goto oom; + break; + case PENDING_ALTER_SET_COLUMN: + if (!migration_buffer_append(&payload, "{\"op\":\"setColumn\",\"table\":")) goto oom; + if (!migration_json_append_string(&payload, op->table)) goto oom; + if (!migration_buffer_append(&payload, ",\"column\":")) goto oom; + if (!migration_json_append_string(&payload, op->a)) goto oom; + if (!migration_buffer_append(&payload, ",\"key\":")) goto oom; + if (!migration_json_append_string(&payload, op->b)) goto oom; + if (op->c) { + if (!migration_buffer_append(&payload, ",\"value\":")) goto oom; + if (!migration_json_append_string(&payload, op->c)) goto oom; + } + if (!migration_buffer_append(&payload, "}")) goto oom; + break; + case PENDING_ALTER_SET_FILTER: + if (!pending_append_set_filter_json(&payload, op)) goto oom; + break; + case PENDING_ALTER_DROP_COLUMN: + if (!migration_buffer_append(&payload, "{\"op\":\"dropColumn\",\"table\":")) goto oom; + if (!migration_json_append_string(&payload, op->table)) goto oom; + if (!migration_buffer_append(&payload, ",\"column\":")) goto oom; + if (!migration_json_append_string(&payload, op->a)) goto oom; + if (!migration_buffer_append(&payload, "}")) goto oom; + break; + case PENDING_ALTER_RENAME_COLUMN: + if (!migration_buffer_append(&payload, "{\"op\":\"renameColumn\",\"table\":")) goto oom; + if (!migration_json_append_string(&payload, op->table)) goto oom; + if (!migration_buffer_append(&payload, ",\"from\":")) goto oom; + if (!migration_json_append_string(&payload, op->a)) goto oom; + if (!migration_buffer_append(&payload, ",\"to\":")) goto oom; + if (!migration_json_append_string(&payload, op->b)) goto oom; + if (!migration_buffer_append(&payload, "}")) goto oom; + break; + case PENDING_ALTER_RAW_SQL: + if (!pending_append_raw_sql_json(&payload, op)) goto oom; + break; + default: + break; + } + first_op = false; + } + if (!migration_buffer_append(&payload, "]}")) goto oom; + return payload.ptr; +oom: + if (payload.ptr) cloudsync_memory_free(payload.ptr); + cloudsync_set_error(data, "Unable to build pending alter payload", DBRES_NOMEM); + return NULL; +} + +static int pending_migration_table_create(cloudsync_context *data) { + return database_exec(data, + "CREATE TABLE IF NOT EXISTS cloudsync_pending_migration (" + "migration_id TEXT PRIMARY KEY NOT NULL," + "table_name TEXT," + "payload TEXT NOT NULL," + "created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP," + "uploaded_at TEXT," + "last_error TEXT" + ");"); +} + +static int pending_migration_save(cloudsync_context *data, const char *migration_id, const char *table, const char *payload) { + int rc = pending_migration_table_create(data); + if (rc != DBRES_OK) return rc; + char *id_lit = migration_sql_quote_literal(migration_id); + char *table_lit = table ? migration_sql_quote_literal(table) : cloudsync_string_dup("NULL"); + char *payload_lit = migration_sql_quote_literal(payload); + if (!id_lit || !table_lit || !payload_lit) { + if (id_lit) cloudsync_memory_free(id_lit); + if (table_lit) cloudsync_memory_free(table_lit); + if (payload_lit) cloudsync_memory_free(payload_lit); + return DBRES_NOMEM; + } + char *sql = NULL; +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + sql = cloudsync_memory_mprintf( + "INSERT INTO cloudsync_pending_migration (migration_id, table_name, payload, uploaded_at, last_error) " + "VALUES (%s, %s, %s, NULL, NULL) " + "ON CONFLICT (migration_id) DO UPDATE SET " + "table_name = excluded.table_name, payload = excluded.payload, uploaded_at = NULL, last_error = NULL;", + id_lit, table_lit, payload_lit); +#else + sql = cloudsync_memory_mprintf( + "INSERT OR REPLACE INTO cloudsync_pending_migration (migration_id, table_name, payload, uploaded_at, last_error) " + "VALUES (%s, %s, %s, NULL, NULL);", + id_lit, table_lit, payload_lit); +#endif + cloudsync_memory_free(id_lit); + cloudsync_memory_free(table_lit); + cloudsync_memory_free(payload_lit); + if (!sql) return DBRES_NOMEM; + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + return rc; +} + +int cloudsync_pending_migration_count(cloudsync_context *data) { + if (!database_internal_table_exists(data, "cloudsync_pending_migration")) return 0; + int64_t count = 0; + int rc = database_select_int(data, "SELECT COUNT(*) FROM cloudsync_pending_migration WHERE uploaded_at IS NULL;", &count); + return rc == DBRES_OK ? (int)count : 0; +} + +char *cloudsync_pending_migration_next_id(cloudsync_context *data) { + if (!database_internal_table_exists(data, "cloudsync_pending_migration")) return NULL; + char *migration_id = NULL; + int rc = database_select_text(data, "SELECT migration_id FROM cloudsync_pending_migration WHERE uploaded_at IS NULL ORDER BY created_at, migration_id LIMIT 1;", &migration_id); + return rc == DBRES_OK ? migration_id : NULL; +} + +char *cloudsync_pending_migration_payload(cloudsync_context *data, const char *migration_id) { + if (!migration_id || !database_internal_table_exists(data, "cloudsync_pending_migration")) return NULL; + char *id_lit = migration_sql_quote_literal(migration_id); + if (!id_lit) return NULL; + char *sql = cloudsync_memory_mprintf("SELECT payload FROM cloudsync_pending_migration WHERE migration_id = %s AND uploaded_at IS NULL;", id_lit); + cloudsync_memory_free(id_lit); + if (!sql) return NULL; + char *payload = NULL; + int rc = database_select_text(data, sql, &payload); + cloudsync_memory_free(sql); + return rc == DBRES_OK ? payload : NULL; +} + +int cloudsync_pending_migration_mark_uploaded(cloudsync_context *data, const char *migration_id) { + if (!migration_id) return DBRES_MISUSE; + int rc = pending_migration_table_create(data); + if (rc != DBRES_OK) return rc; + char *id_lit = migration_sql_quote_literal(migration_id); + if (!id_lit) return DBRES_NOMEM; + char *sql = cloudsync_memory_mprintf("UPDATE cloudsync_pending_migration SET uploaded_at = CURRENT_TIMESTAMP, last_error = NULL WHERE migration_id = %s;", id_lit); + cloudsync_memory_free(id_lit); + if (!sql) return DBRES_NOMEM; + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + return rc; +} + +static bool migration_sql_is_space(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; +} + +static bool migration_sql_ident_start(char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; +} + +static bool migration_sql_ident_char(char c) { + return migration_sql_ident_start(c) || (c >= '0' && c <= '9') || c == '$'; +} + +static const char *migration_sql_skip_space_comments(const char *sql) { + const char *p = sql; + while (p && *p) { + while (migration_sql_is_space(*p)) p++; + if (p[0] == '-' && p[1] == '-') { + p += 2; + while (*p && *p != '\n') p++; + continue; + } + if (p[0] == '/' && p[1] == '*') { + p += 2; + while (*p && !(p[0] == '*' && p[1] == '/')) p++; + if (*p) p += 2; + continue; + } + break; + } + return p; +} + +static bool migration_sql_keyword_eq(const char *start, size_t len, const char *keyword) { + return strlen(keyword) == len && strncasecmp(start, keyword, len) == 0; +} + +static bool migration_sql_forbidden_transaction_keyword(const char *start, size_t len) { + return migration_sql_keyword_eq(start, len, "begin") || + migration_sql_keyword_eq(start, len, "commit") || + migration_sql_keyword_eq(start, len, "end") || + migration_sql_keyword_eq(start, len, "abort") || + migration_sql_keyword_eq(start, len, "rollback") || + migration_sql_keyword_eq(start, len, "savepoint") || + migration_sql_keyword_eq(start, len, "release"); +} + +static const char *migration_sql_next_statement(const char *sql) { + bool single_quote = false; + bool double_quote = false; + for (const char *p = sql; p && *p; ++p) { + if (single_quote) { + if (*p == '\'' && p[1] == '\'') p++; + else if (*p == '\'') single_quote = false; + continue; + } + if (double_quote) { + if (*p == '"' && p[1] == '"') p++; + else if (*p == '"') double_quote = false; + continue; + } + if (p[0] == '-' && p[1] == '-') { + p += 2; + while (*p && *p != '\n') p++; + if (!*p) return p; + continue; + } + if (p[0] == '/' && p[1] == '*') { + p += 2; + while (*p && !(p[0] == '*' && p[1] == '/')) p++; + if (!*p) return p; + p++; + continue; + } + if (*p == '$') { + const char *tag_end = p + 1; + while (migration_sql_ident_char(*tag_end)) tag_end++; + if (*tag_end == '$') { + size_t tag_len = (size_t)(tag_end - p) + 1; + const char *q = tag_end + 1; + while (*q && strncmp(q, p, tag_len) != 0) q++; + if (!*q) return q; + p = q + tag_len - 1; + continue; + } + } + if (*p == '\'') { + single_quote = true; + continue; + } + if (*p == '"') { + double_quote = true; + continue; + } + if (*p == ';') return p + 1; + } + return sql + strlen(sql); +} + +static bool migration_sql_has_statement(const char *sql) { + const char *p = migration_sql_skip_space_comments(sql); + while (p && *p == ';') p = migration_sql_skip_space_comments(p + 1); + return p && *p; +} + +static bool migration_sql_has_transaction_control(const char *sql) { + const char *p = sql; + while (p && *p) { + p = migration_sql_skip_space_comments(p); + while (*p == ';') p = migration_sql_skip_space_comments(p + 1); + if (!*p) return false; + if (migration_sql_ident_start(*p)) { + const char *start = p; + while (migration_sql_ident_char(*p)) p++; + if (migration_sql_forbidden_transaction_keyword(start, (size_t)(p - start))) return true; + } + p = migration_sql_next_statement(p); + } + return false; +} + +static int cloudsync_alter_append_simple(cloudsync_context *data, pending_alter_kind kind, const char *table, const char *a, const char *b, const char *c) { + if (!table || !table[0]) return cloudsync_set_error(data, "cloudsync alter requires table", DBRES_MISUSE); + pending_alter_op *op = pending_alter_op_new(kind, table); + if (!op) return DBRES_NOMEM; + op->a = a ? cloudsync_string_dup(a) : NULL; + op->b = b ? cloudsync_string_dup(b) : NULL; + op->c = c ? cloudsync_string_dup(c) : NULL; + return pending_alter_append(data, op); +} + +int cloudsync_alter_create_table(cloudsync_context *data, const char *table) { + return cloudsync_alter_append_simple(data, PENDING_ALTER_CREATE_TABLE, table, NULL, NULL, NULL); +} + +int cloudsync_alter_add_column(cloudsync_context *data, const char *table, const char *column, const char *type, bool nullable, bool has_default, const char *default_value) { + if (!table || !column || !type) return cloudsync_set_error(data, "cloudsync_alter_add_column requires table, column, and type", DBRES_MISUSE); + pending_alter_op *op = pending_alter_op_new(PENDING_ALTER_ADD_COLUMN, table); + if (!op) return DBRES_NOMEM; + op->a = cloudsync_string_dup(column); + op->b = cloudsync_string_dup(type); + op->flag = nullable; + op->has_default = has_default; + op->default_kind = has_default ? cloudsync_string_dup(default_value ? type : "null") : NULL; + op->default_value = (has_default && default_value) ? cloudsync_string_dup(default_value) : NULL; + if (!op->a || !op->b || (has_default && !op->default_kind)) { + pending_alter_op_free(op); + return DBRES_NOMEM; + } + return pending_alter_append(data, op); +} + +int cloudsync_alter_add_column_dialect(cloudsync_context *data, const char *table, const char *column, const char *dialect, const char *type_sql, bool nullable, bool has_default_sql, const char *default_sql) { + if (!table || !column || !dialect || !type_sql) return cloudsync_set_error(data, "cloudsync_alter_add_column_dialect requires table, column, dialect, and type SQL", DBRES_MISUSE); + pending_alter_op *op = pending_find_add_column(data, table, column); + if (!op) return cloudsync_set_error(data, "Dialect override requires a pending addColumn operation", DBRES_MISUSE); + op->flag = nullable; + char **type_slot = NULL; + char **default_slot = NULL; + if (strcasecmp(dialect, "sqlite") == 0) { + type_slot = &op->sqlite_type_sql; + default_slot = &op->sqlite_default_sql; + } else if (strcasecmp(dialect, "postgresql") == 0) { + type_slot = &op->postgresql_type_sql; + default_slot = &op->postgresql_default_sql; + } else { + return cloudsync_set_error(data, "Unsupported dialect override", DBRES_MISUSE); + } + if (*type_slot) cloudsync_memory_free(*type_slot); + *type_slot = cloudsync_string_dup(type_sql); + if (has_default_sql) { + if (*default_slot) cloudsync_memory_free(*default_slot); + *default_slot = default_sql ? cloudsync_string_dup(default_sql) : cloudsync_string_dup("NULL"); + } + return *type_slot ? DBRES_OK : DBRES_NOMEM; +} + +int cloudsync_alter_add_primary_key(cloudsync_context *data, const char *table, const char *column) { + return cloudsync_alter_append_simple(data, PENDING_ALTER_ADD_PRIMARY_KEY, table, column, NULL, NULL); +} + +int cloudsync_alter_augment_table(cloudsync_context *data, const char *table, const char *algorithm, int64_t init_flags) { + if (!table || !table[0]) return cloudsync_set_error(data, "cloudsync_alter_augment_table requires table", DBRES_MISUSE); + pending_alter_op *op = pending_alter_op_new(PENDING_ALTER_AUGMENT_TABLE, table); + if (!op) return DBRES_NOMEM; + op->a = cloudsync_string_dup(algorithm ? algorithm : CLOUDSYNC_DEFAULT_ALGO); + op->number = init_flags; + return pending_alter_append(data, op); +} + +int cloudsync_alter_set_block_lww(cloudsync_context *data, const char *table, const char *column, const char *delimiter) { + return cloudsync_alter_append_simple(data, PENDING_ALTER_SET_BLOCK_LWW, table, column, delimiter, NULL); +} + +int cloudsync_alter_set_column(cloudsync_context *data, const char *table, const char *column, const char *key, const char *value) { + return cloudsync_alter_append_simple(data, PENDING_ALTER_SET_COLUMN, table, column, key, value); +} + +int cloudsync_alter_set_filter(cloudsync_context *data, const char *table, const char *filter) { + return cloudsync_alter_append_simple(data, PENDING_ALTER_SET_FILTER, table, filter, NULL, NULL); +} + +int cloudsync_alter_set_filter_dialect(cloudsync_context *data, const char *table, const char *dialect, const char *filter) { + if (!table || !dialect || !filter) return cloudsync_set_error(data, "cloudsync_alter_set_filter_dialect requires table, dialect, and filter", DBRES_MISUSE); + pending_alter_context *ctx = pending_alter_context_for(data, true); + if (!ctx) return DBRES_NOMEM; + pending_alter_op *op = NULL; + for (pending_alter_op *cur = ctx->ops; cur; cur = cur->next) { + if (cur->kind == PENDING_ALTER_SET_FILTER && pending_table_eq(cur, table)) op = cur; + } + if (!op) { + op = pending_alter_op_new(PENDING_ALTER_SET_FILTER, table); + if (!op) return DBRES_NOMEM; + int rc = pending_alter_append(data, op); + if (rc != DBRES_OK) return rc; + } + char **slot = NULL; + if (strcasecmp(dialect, "sqlite") == 0) slot = &op->sqlite_filter_sql; + else if (strcasecmp(dialect, "postgresql") == 0) slot = &op->postgresql_filter_sql; + else return cloudsync_set_error(data, "Unsupported dialect override", DBRES_MISUSE); + if (*slot) cloudsync_memory_free(*slot); + *slot = cloudsync_string_dup(filter); + return *slot ? DBRES_OK : DBRES_NOMEM; +} + +int cloudsync_alter_drop_column(cloudsync_context *data, const char *table, const char *column) { + return cloudsync_alter_append_simple(data, PENDING_ALTER_DROP_COLUMN, table, column, NULL, NULL); +} + +int cloudsync_alter_rename_column(cloudsync_context *data, const char *table, const char *from, const char *to) { + return cloudsync_alter_append_simple(data, PENDING_ALTER_RENAME_COLUMN, table, from, to, NULL); +} + +static int cloudsync_alter_sql_internal(cloudsync_context *data, const char *sql, const char *dialect) { + if (!sql || !migration_sql_has_statement(sql)) return cloudsync_set_error(data, "cloudsync_alter_sql requires SQL", DBRES_MISUSE); + if (migration_sql_has_transaction_control(sql)) return cloudsync_set_error(data, "cloudsync_alter_sql cannot contain transaction control statements", DBRES_MISUSE); + + const char *canonical_dialect = NULL; + if (dialect) { + if (strcasecmp(dialect, "sqlite") == 0) canonical_dialect = "sqlite"; + else if (strcasecmp(dialect, "postgresql") == 0) canonical_dialect = "postgresql"; + else return cloudsync_set_error(data, "Unsupported SQL dialect", DBRES_MISUSE); + } + + pending_alter_op *op = pending_alter_op_new(PENDING_ALTER_RAW_SQL, NULL); + if (!op) return DBRES_NOMEM; + op->a = cloudsync_string_dup(sql); + op->b = canonical_dialect ? cloudsync_string_dup(canonical_dialect) : NULL; + op->flag = canonical_dialect != NULL; + if (!op->a || (canonical_dialect && !op->b)) { + pending_alter_op_free(op); + return DBRES_NOMEM; + } + return pending_alter_append(data, op); +} + +int cloudsync_alter_sql(cloudsync_context *data, const char *sql) { + return cloudsync_alter_sql_internal(data, sql, NULL); +} + +int cloudsync_alter_sql_dialect(cloudsync_context *data, const char *dialect, const char *sql) { + return cloudsync_alter_sql_internal(data, sql, dialect); +} + +int cloudsync_alter_clear(cloudsync_context *data, const char *table) { + pending_alter_context *ctx = pending_alter_context_for(data, false); + if (!ctx) return DBRES_OK; + pending_alter_op **link = &ctx->ops; + while (*link) { + pending_alter_op *op = *link; + if (!table || pending_table_eq(op, table)) { + *link = op->next; + op->next = NULL; + pending_alter_op_free(op); + } else { + link = &op->next; + } + } + return DBRES_OK; +} + +void cloudsync_alter_clear_context(cloudsync_context *data) { + pending_alter_context **link = &g_pending_alters; + while (*link) { + pending_alter_context *ctx = *link; + if (ctx->data == data) { + *link = ctx->next; + pending_alter_op *op = ctx->ops; + while (op) { + pending_alter_op *next = op->next; + pending_alter_op_free(op); + op = next; + } + cloudsync_memory_free(ctx); + return; + } + link = &ctx->next; + } +} + +char *cloudsync_alter_preview(cloudsync_context *data) { + char uuid[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_string(uuid, true); + bool destructive = false; + return pending_build_payload(data, uuid, NULL, NULL, &destructive); +} + +int cloudsync_alter_apply(cloudsync_context *data, char **result_json) { + if (result_json) *result_json = NULL; + char uuid[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_string(uuid, true); + bool destructive = false; + char *payload = pending_build_payload(data, uuid, NULL, NULL, &destructive); + if (!payload) return cloudsync_errcode(data); + + char *apply_result = NULL; + char *final_payload = NULL; + bool savepoint_open = false; + int rc = database_begin_savepoint(data, CLOUDSYNC_ALTER_APPLY_SAVEPOINT); + if (rc != DBRES_OK) goto cleanup; + savepoint_open = true; + + rc = cloudsync_migration_apply(data, payload, (int)strlen(payload), &apply_result); + if (apply_result) { + cloudsync_memory_free(apply_result); + apply_result = NULL; + } + if (rc != DBRES_OK) goto rollback; + + final_payload = pending_build_payload(data, uuid, NULL, NULL, &destructive); + if (!final_payload) { + rc = cloudsync_errcode(data); + if (rc == DBRES_OK) rc = DBRES_NOMEM; + goto rollback; + } + + rc = pending_migration_save(data, uuid, NULL, final_payload); + if (rc != DBRES_OK) goto rollback; + + rc = database_commit_savepoint(data, CLOUDSYNC_ALTER_APPLY_SAVEPOINT); + if (rc != DBRES_OK) goto rollback; + savepoint_open = false; + + rc = cloudsync_alter_clear(data, NULL); + if (result_json && rc == DBRES_OK) { + *result_json = cloudsync_memory_mprintf("{\"status\":\"applied\",\"migrationId\":\"%s\",\"pendingUpload\":true}", uuid); + } + +cleanup: + cloudsync_memory_free(payload); + if (final_payload) cloudsync_memory_free(final_payload); + return rc; + +rollback: + if (savepoint_open) { + database_rollback_savepoint(data, CLOUDSYNC_ALTER_APPLY_SAVEPOINT); + cloudsync_terminate(data); + } + goto cleanup; +} diff --git a/src/network/network.c b/src/network/network.c index bd6591b8..6ec80866 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -9,6 +9,7 @@ #include #include +#include #include #include "network.h" @@ -56,6 +57,9 @@ struct network_data { char *upload_endpoint; char *apply_endpoint; char *status_endpoint; + char *schema_check_endpoint; + char *schema_upload_endpoint; + char *schema_download_endpoint; }; typedef struct { @@ -72,6 +76,11 @@ typedef struct { size_t read_pos; } network_read_data; +static int cloudsync_network_migration_check_internal(sqlite3_context *context, char **result_json, char **err_out); +static bool network_migration_apply_error(const char *message); +static void network_result_to_sqlite_error(sqlite3_context *context, NETWORK_RESULT res, const char *default_error_message); +network_data *cloudsync_network_data(sqlite3_context *context); + // MARK: - void network_result_cleanup (NETWORK_RESULT *res) { @@ -100,6 +109,9 @@ bool network_data_set_endpoints (network_data *data, char *auth, char *check, ch if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); if (data->apply_endpoint) cloudsync_memory_free(data->apply_endpoint); if (data->status_endpoint) cloudsync_memory_free(data->status_endpoint); + if (data->schema_check_endpoint) cloudsync_memory_free(data->schema_check_endpoint); + if (data->schema_upload_endpoint) cloudsync_memory_free(data->schema_upload_endpoint); + if (data->schema_download_endpoint) cloudsync_memory_free(data->schema_download_endpoint); // clear pointers data->authentication = NULL; @@ -107,6 +119,9 @@ bool network_data_set_endpoints (network_data *data, char *auth, char *check, ch data->upload_endpoint = NULL; data->apply_endpoint = NULL; data->status_endpoint = NULL; + data->schema_check_endpoint = NULL; + data->schema_upload_endpoint = NULL; + data->schema_download_endpoint = NULL; // make a copy of the new endpoints char *auth_copy = NULL; @@ -157,6 +172,9 @@ void network_data_free (network_data *data) { if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); if (data->apply_endpoint) cloudsync_memory_free(data->apply_endpoint); if (data->status_endpoint) cloudsync_memory_free(data->status_endpoint); + if (data->schema_check_endpoint) cloudsync_memory_free(data->schema_check_endpoint); + if (data->schema_upload_endpoint) cloudsync_memory_free(data->schema_upload_endpoint); + if (data->schema_download_endpoint) cloudsync_memory_free(data->schema_download_endpoint); cloudsync_memory_free(data); } @@ -438,9 +456,34 @@ int network_download_changes (sqlite3_context *context, const char *download_url if (rc != DBRES_OK) { const char *msg = cloudsync_errmsg(data); if (!msg || !msg[0]) msg = "cloudsync_payload_apply failed"; - if (err_out) *err_out = cloudsync_string_dup(msg); - else sqlite3_result_error(context, msg, -1); - if (pnrows) *pnrows = 0; + if (network_migration_apply_error(msg)) { + char *migration_err = NULL; + char *migration_result = NULL; + int mrc = cloudsync_network_migration_check_internal(context, &migration_result, &migration_err); + if (migration_result) cloudsync_memory_free(migration_result); + if (mrc == SQLITE_OK && !migration_err) { + if (pnrows) *pnrows = 0; + rc = cloudsync_payload_apply(data, result.buffer, (int)result.blen, pnrows); + msg = rc == DBRES_OK ? NULL : cloudsync_errmsg(data); + } else { + if (migration_err) { + if (err_out) *err_out = migration_err; + else { + sqlite3_result_error(context, migration_err, -1); + cloudsync_memory_free(migration_err); + } + } + rc = SQLITE_ERROR; + if (pnrows) *pnrows = 0; + goto cleanup_download; + } + } + if (rc != DBRES_OK) { + if (!msg || !msg[0]) msg = "cloudsync_payload_apply failed"; + if (err_out) *err_out = cloudsync_string_dup(msg); + else sqlite3_result_error(context, msg, -1); + if (pnrows) *pnrows = 0; + } } } else if (result.code == CLOUDSYNC_NETWORK_ERROR) { network_set_sqlite_result(context, &result); @@ -450,6 +493,7 @@ int network_download_changes (sqlite3_context *context, const char *download_url // CLOUDSYNC_NETWORK_OK — no data, not an error if (pnrows) *pnrows = 0; } +cleanup_download: network_result_cleanup(&result); return rc; @@ -468,7 +512,95 @@ char *network_authentication_token (const char *key, const char *value) { // MARK: - JSON helpers (jsmn) - -#define JSMN_MAX_TOKENS 64 +#define JSMN_INITIAL_TOKENS 512 + +static bool json_is_whitespace(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; +} + +static bool json_has_single_root(const char *json, size_t json_len, const jsmntok_t *tokens, int ntokens) { + if (!json || !tokens || ntokens < 1 || tokens[0].start < 0 || tokens[0].end < tokens[0].start) return false; + + size_t cursor = 0; + while (cursor < json_len && json_is_whitespace(json[cursor])) cursor++; + if (cursor != (size_t)tokens[0].start) return false; + + cursor = (size_t)tokens[0].end; + while (cursor < json_len && json_is_whitespace(json[cursor])) cursor++; + return cursor == json_len; +} + +static int json_parse_root_object(const char *json, size_t json_len, jsmntok_t *tokens, unsigned int max_tokens) { + if (!json || json_len == 0 || !tokens || max_tokens == 0) return JSMN_ERROR_INVAL; + + jsmn_parser parser; + jsmn_init(&parser); + int ntokens = jsmn_parse(&parser, json, json_len, tokens, max_tokens); + if (ntokens < 0) return ntokens; + if (ntokens < 1 || tokens[0].type != JSMN_OBJECT || !json_has_single_root(json, json_len, tokens, ntokens)) { + return JSMN_ERROR_INVAL; + } + return ntokens; +} + +static jsmntok_t *json_parse_root_object_alloc(const char *json, size_t json_len, int *ntokens_out) { + if (ntokens_out) *ntokens_out = 0; + if (!json || json_len == 0 || !ntokens_out) return NULL; + + size_t max_tokens = json_len + 1; + if (max_tokens < JSMN_INITIAL_TOKENS) max_tokens = JSMN_INITIAL_TOKENS; + if (max_tokens > (size_t)INT_MAX) max_tokens = (size_t)INT_MAX; + + size_t cap = JSMN_INITIAL_TOKENS; + if (cap > max_tokens) cap = max_tokens; + while (cap > 0 && cap <= max_tokens) { + if (cap > SIZE_MAX / sizeof(jsmntok_t)) return NULL; + jsmntok_t *tokens = cloudsync_memory_alloc(cap * sizeof(jsmntok_t)); + if (!tokens) return NULL; + + int ntokens = json_parse_root_object(json, json_len, tokens, (unsigned int)cap); + if (ntokens == JSMN_ERROR_NOMEM) { + cloudsync_memory_free(tokens); + if (cap == max_tokens) break; + size_t next = cap * 2; + if (next <= cap || next > max_tokens) next = max_tokens; + cap = next; + continue; + } + if (ntokens < 1) { + cloudsync_memory_free(tokens); + return NULL; + } + + *ntokens_out = ntokens; + return tokens; + } + + return NULL; +} + +static bool json_is_valid_root_object(const char *json, size_t json_len) { + int ntokens = 0; + jsmntok_t *tokens = json_parse_root_object_alloc(json, json_len, &ntokens); + bool valid = tokens != NULL && ntokens >= 1; + if (tokens) cloudsync_memory_free(tokens); + return valid; +} + +static int network_json_error(sqlite3_context *context, const char *source, char **err_out) { + char buffer[192]; + snprintf(buffer, sizeof(buffer), "%s returned invalid JSON.", source ? source : "CloudSync network endpoint"); + if (err_out) *err_out = cloudsync_string_dup(buffer); + else sqlite3_result_error(context, buffer, -1); + return SQLITE_ERROR; +} + +static bool network_validate_json_response(sqlite3_context *context, NETWORK_RESULT *res, const char *source, char **err_out) { + if (!res || res->code != CLOUDSYNC_NETWORK_BUFFER || !res->buffer || res->blen == 0) return true; + if (json_is_valid_root_object(res->buffer, res->blen)) return true; + network_json_error(context, source, err_out); + return false; +} static bool jsmn_token_eq(const char *json, const jsmntok_t *tok, const char *s) { return (tok->type == JSMN_STRING && @@ -522,55 +654,98 @@ static char *json_unescape_string(const char *src, int len) { static char *json_extract_string(const char *json, size_t json_len, const char *key) { if (!json || json_len == 0 || !key) return NULL; - jsmn_parser parser; - jsmntok_t tokens[JSMN_MAX_TOKENS]; - jsmn_init(&parser); - int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS); + int ntokens = 0; + jsmntok_t *tokens = json_parse_root_object_alloc(json, json_len, &ntokens); if (ntokens < 1) return NULL; int i = jsmn_find_key(json, tokens, ntokens, key); - if (i < 0 || i + 1 >= ntokens) return NULL; + if (i < 0 || i + 1 >= ntokens) { + cloudsync_memory_free(tokens); + return NULL; + } jsmntok_t *val = &tokens[i + 1]; - if (val->type != JSMN_STRING) return NULL; + if (val->type != JSMN_STRING) { + cloudsync_memory_free(tokens); + return NULL; + } - return json_unescape_string(json + val->start, val->end - val->start); + char *result = json_unescape_string(json + val->start, val->end - val->start); + cloudsync_memory_free(tokens); + return result; +} + +static char *network_migration_upload_error_message(NETWORK_RESULT *res) { + if (!res || res->code != CLOUDSYNC_NETWORK_BUFFER || !res->buffer || res->blen == 0) return NULL; + + char *status = json_extract_string(res->buffer, res->blen, "status"); + char *error = json_extract_string(res->buffer, res->blen, "error"); + bool rejected = false; + bool accepted = status && (strcmp(status, "uploaded") == 0 || strcmp(status, "accepted") == 0 || + strcmp(status, "applied") == 0 || strcmp(status, "ok") == 0 || + strcmp(status, "success") == 0); + if (error && error[0]) rejected = true; + if (!status || !status[0]) rejected = true; + else if (!accepted) rejected = true; + + char *message = NULL; + if (rejected) { + if (error && error[0]) message = cloudsync_string_dup(error); + else if (status && status[0]) message = cloudsync_memory_mprintf("CloudSync schema migration upload failed with status '%s'.", status); + else message = cloudsync_string_dup("CloudSync schema migration upload response did not include an accepted status."); + } + + if (status) cloudsync_memory_free(status); + if (error) cloudsync_memory_free(error); + return message; } static int64_t json_extract_int(const char *json, size_t json_len, const char *key, int64_t default_value) { if (!json || json_len == 0 || !key) return default_value; - jsmn_parser parser; - jsmntok_t tokens[JSMN_MAX_TOKENS]; - jsmn_init(&parser); - int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS); - if (ntokens < 1 || tokens[0].type != JSMN_OBJECT) return default_value; + int ntokens = 0; + jsmntok_t *tokens = json_parse_root_object_alloc(json, json_len, &ntokens); + if (ntokens < 1) return default_value; int i = jsmn_find_key(json, tokens, ntokens, key); - if (i < 0 || i + 1 >= ntokens) return default_value; + if (i < 0 || i + 1 >= ntokens) { + cloudsync_memory_free(tokens); + return default_value; + } jsmntok_t *val = &tokens[i + 1]; - if (val->type != JSMN_PRIMITIVE) return default_value; + if (val->type != JSMN_PRIMITIVE) { + cloudsync_memory_free(tokens); + return default_value; + } - return strtoll(json + val->start, NULL, 10); + int64_t result = strtoll(json + val->start, NULL, 10); + cloudsync_memory_free(tokens); + return result; } static int json_extract_array_size(const char *json, size_t json_len, const char *key) { if (!json || json_len == 0 || !key) return -1; - jsmn_parser parser; - jsmntok_t tokens[JSMN_MAX_TOKENS]; - jsmn_init(&parser); - int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS); - if (ntokens < 1 || tokens[0].type != JSMN_OBJECT) return -1; + int ntokens = 0; + jsmntok_t *tokens = json_parse_root_object_alloc(json, json_len, &ntokens); + if (ntokens < 1) return -1; int i = jsmn_find_key(json, tokens, ntokens, key); - if (i < 0 || i + 1 >= ntokens) return -1; + if (i < 0 || i + 1 >= ntokens) { + cloudsync_memory_free(tokens); + return -1; + } jsmntok_t *val = &tokens[i + 1]; - if (val->type != JSMN_ARRAY) return -1; + if (val->type != JSMN_ARRAY) { + cloudsync_memory_free(tokens); + return -1; + } - return val->size; + int result = val->size; + cloudsync_memory_free(tokens); + return result; } // Escape a string for safe embedding as a JSON string value (without surrounding quotes). @@ -613,28 +788,264 @@ static char *json_escape_string(const char *src) { static char *json_extract_object_raw(const char *json, size_t json_len, const char *key) { if (!json || json_len == 0 || !key) return NULL; - jsmn_parser parser; - jsmntok_t tokens[JSMN_MAX_TOKENS]; - jsmn_init(&parser); - int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS); + int ntokens = 0; + jsmntok_t *tokens = json_parse_root_object_alloc(json, json_len, &ntokens); if (ntokens < 1) return NULL; int i = jsmn_find_key(json, tokens, ntokens, key); - if (i < 0 || i + 1 >= ntokens) return NULL; + if (i < 0 || i + 1 >= ntokens) { + cloudsync_memory_free(tokens); + return NULL; + } jsmntok_t *val = &tokens[i + 1]; - if (val->type != JSMN_OBJECT) return NULL; + if (val->type != JSMN_OBJECT) { + cloudsync_memory_free(tokens); + return NULL; + } int len = val->end - val->start; - if (len <= 0) return NULL; + if (len <= 0) { + cloudsync_memory_free(tokens); + return NULL; + } char *out = cloudsync_memory_zeroalloc(len + 1); - if (!out) return NULL; + if (!out) { + cloudsync_memory_free(tokens); + return NULL; + } memcpy(out, json + val->start, len); out[len] = '\0'; + cloudsync_memory_free(tokens); return out; } +static bool network_migration_apply_error(const char *message) { + if (!message) return false; + return strstr(message, "schema hash") != NULL || + strstr(message, "cloudsync is not initialized") != NULL; +} + +static int64_t network_migration_current_epoch(cloudsync_context *data) { + if (!database_internal_table_exists(data, "cloudsync_migrations")) return 0; + int64_t epoch = 0; + int rc = database_select_int(data, "SELECT COALESCE(MAX(schema_epoch), 0) FROM cloudsync_migrations;", &epoch); + return rc == DBRES_OK ? epoch : 0; +} + +static char *network_migration_state_json(cloudsync_context *data) { + uint64_t schema_hash = database_schema_hash(data); + int64_t schema_epoch = network_migration_current_epoch(data); + bool has_tables = cloudsync_config_exists(data) && dbutils_table_settings_count_tables(data) > 0; + return cloudsync_memory_mprintf( + "{\"schemaHash\":\"%" PRIu64 "\",\"schemaEpoch\":%lld,\"hasTables\":%s}", + schema_hash, (long long)schema_epoch, has_tables ? "true" : "false"); +} + +static int network_apply_migration_payload(sqlite3_context *context, const char *payload, size_t payload_len, char **result_json, char **err_out) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + char *apply_result = NULL; + int rc = cloudsync_migration_apply(data, payload, (int)payload_len, &apply_result); + if (rc != DBRES_OK) { + const char *msg = cloudsync_errmsg(data); + if (!msg || !msg[0]) msg = "cloudsync_migration_apply failed"; + if (err_out) *err_out = cloudsync_string_dup(msg); + else sqlite3_result_error(context, msg, -1); + if (apply_result) cloudsync_memory_free(apply_result); + return SQLITE_ERROR; + } + + if (result_json) *result_json = apply_result; + else if (apply_result) cloudsync_memory_free(apply_result); + return SQLITE_OK; +} + +static int network_apply_migration_response(sqlite3_context *context, NETWORK_RESULT *res, bool allow_url, char **result_json, char **err_out) { + if (!res) { + const char *msg = "CloudSync schema migration endpoint failed."; + if (err_out) *err_out = cloudsync_string_dup(msg); + else sqlite3_result_error(context, msg, -1); + return SQLITE_ERROR; + } + + if (res->code == CLOUDSYNC_NETWORK_OK || (res->code == CLOUDSYNC_NETWORK_BUFFER && (!res->buffer || res->blen == 0))) { + if (result_json) *result_json = cloudsync_string_dup("{\"status\":\"none\"}"); + return SQLITE_OK; + } + + if (res->code != CLOUDSYNC_NETWORK_BUFFER) { + if (err_out) { + *err_out = cloudsync_string_dup(res->buffer ? res->buffer : "CloudSync schema migration endpoint failed."); + } else { + network_result_to_sqlite_error(context, *res, "CloudSync schema migration endpoint failed."); + } + return SQLITE_ERROR; + } + + if (!network_validate_json_response(context, res, "CloudSync schema migration endpoint", err_out)) { + return SQLITE_ERROR; + } + + network_data *netdata = NULL; + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + if (data) netdata = (network_data *)cloudsync_auxdata(data); + + char *migration = json_extract_object_raw(res->buffer, res->blen, "migration"); + if (!migration) migration = json_extract_object_raw(res->buffer, res->blen, "payload"); + if (migration) { + int rc = network_apply_migration_payload(context, migration, strlen(migration), result_json, err_out); + cloudsync_memory_free(migration); + return rc; + } + + if (allow_url && netdata) { + char *url = json_extract_string(res->buffer, res->blen, "url"); + if (url) { + NETWORK_RESULT download = network_receive_buffer(netdata, url, NULL, true, false, NULL, NULL); + cloudsync_memory_free(url); + int rc = network_apply_migration_response(context, &download, false, result_json, err_out); + network_result_cleanup(&download); + return rc; + } + } + + if (json_extract_array_size(res->buffer, res->blen, "ops") >= 0) { + return network_apply_migration_payload(context, res->buffer, res->blen, result_json, err_out); + } + + if (result_json) *result_json = cloudsync_string_dup("{\"status\":\"none\"}"); + return SQLITE_OK; +} + +static int cloudsync_network_migration_check_internal(sqlite3_context *context, char **result_json, char **err_out) { + if (result_json) *result_json = NULL; + if (err_out) *err_out = NULL; + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata || !netdata->schema_check_endpoint) { + sqlite3_result_error(context, "Unable to retrieve CloudSync schema migration endpoint.", -1); + return SQLITE_ERROR; + } + + char *state = network_migration_state_json(data); + if (!state) { + sqlite3_result_error_code(context, SQLITE_NOMEM); + return SQLITE_NOMEM; + } + + NETWORK_RESULT res = network_receive_buffer(netdata, netdata->schema_check_endpoint, netdata->authentication, true, true, state, CLOUDSYNC_HEADER_SQLITECLOUD); + cloudsync_memory_free(state); + + int rc = network_apply_migration_response(context, &res, true, result_json, err_out); + network_result_cleanup(&res); + return rc; +} + +static int cloudsync_network_migration_download_internal(sqlite3_context *context, char **result_json, char **err_out) { + if (result_json) *result_json = NULL; + if (err_out) *err_out = NULL; + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata || !netdata->schema_download_endpoint) { + sqlite3_result_error(context, "Unable to retrieve CloudSync schema migration download endpoint.", -1); + return SQLITE_ERROR; + } + + NETWORK_RESULT res = network_receive_buffer(netdata, netdata->schema_download_endpoint, netdata->authentication, true, false, NULL, CLOUDSYNC_HEADER_SQLITECLOUD); + int rc = network_apply_migration_response(context, &res, true, result_json, err_out); + network_result_cleanup(&res); + return rc; +} + +void cloudsync_network_migration_check(sqlite3_context *context, int argc, sqlite3_value **argv) { + char *result = NULL; + int rc = cloudsync_network_migration_check_internal(context, &result, NULL); + if (rc == SQLITE_OK) { + if (result) sqlite3_result_text(context, result, -1, cloudsync_memory_free); + else sqlite3_result_text(context, "{\"status\":\"none\"}", -1, SQLITE_TRANSIENT); + } +} + +void cloudsync_network_migration_download(sqlite3_context *context, int argc, sqlite3_value **argv) { + char *result = NULL; + int rc = cloudsync_network_migration_download_internal(context, &result, NULL); + if (rc == SQLITE_OK) { + if (result) sqlite3_result_text(context, result, -1, cloudsync_memory_free); + else sqlite3_result_text(context, "{\"status\":\"none\"}", -1, SQLITE_TRANSIENT); + } +} + +void cloudsync_network_migration_upload(sqlite3_context *context, int argc, sqlite3_value **argv) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = cloudsync_network_data(context); + if (!netdata || !netdata->schema_upload_endpoint) { + sqlite3_result_error(context, "Unable to retrieve CloudSync schema migration upload endpoint.", -1); + return; + } + + bool from_pending = false; + char *pending_id = NULL; + char *pending_payload = NULL; + const char *payload = NULL; + + if (argc == 0) { + pending_id = cloudsync_pending_migration_next_id(data); + if (!pending_id) { + sqlite3_result_error(context, "No pending schema migration to upload.", -1); + return; + } + pending_payload = cloudsync_pending_migration_payload(data, pending_id); + if (!pending_payload) { + sqlite3_result_error(context, "Unable to load pending schema migration payload.", -1); + cloudsync_memory_free(pending_id); + return; + } + payload = pending_payload; + from_pending = true; + } else { + payload = (const char *)sqlite3_value_text(argv[0]); + } + + if (!payload || payload[0] == '\0') { + sqlite3_result_error(context, "cloudsync_network_migration_upload expects a JSON text payload.", -1); + if (pending_id) cloudsync_memory_free(pending_id); + if (pending_payload) cloudsync_memory_free(pending_payload); + return; + } + if (!json_is_valid_root_object(payload, strlen(payload))) { + sqlite3_result_error(context, "cloudsync_network_migration_upload expects a valid JSON object payload.", -1); + if (pending_id) cloudsync_memory_free(pending_id); + if (pending_payload) cloudsync_memory_free(pending_payload); + return; + } + + NETWORK_RESULT res = network_receive_buffer(netdata, netdata->schema_upload_endpoint, netdata->authentication, true, true, (char *)payload, CLOUDSYNC_HEADER_SQLITECLOUD); + if (network_validate_json_response(context, &res, "CloudSync schema migration upload endpoint", NULL)) { + int rc = DBRES_OK; + char *upload_error = network_migration_upload_error_message(&res); + if (res.code == CLOUDSYNC_NETWORK_ERROR) { + network_set_sqlite_result(context, &res); + } else if (upload_error) { + sqlite3_result_error(context, upload_error, -1); + } else { + if (from_pending) rc = cloudsync_pending_migration_mark_uploaded(data, pending_id); + if (rc == DBRES_OK) { + network_set_sqlite_result(context, &res); + } else { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + } + } + if (upload_error) cloudsync_memory_free(upload_error); + } + network_result_cleanup(&res); + if (pending_id) cloudsync_memory_free(pending_id); + if (pending_payload) cloudsync_memory_free(pending_payload); +} + int network_extract_query_param (const char *query, const char *key, char *output, size_t output_size) { if (!query || !key || !output || output_size == 0) { return -1; // Invalid input @@ -719,6 +1130,24 @@ static bool network_compute_endpoints_with_address (sqlite3_context *context, ne snprintf(status_endpoint, requested, "%s/%s/%s/%s/%s", address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_STATUS); + char *schema_check_endpoint = cloudsync_memory_mprintf("%s/%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_SCHEMA, CLOUDSYNC_ENDPOINT_SCHEMA_CHECK); + char *schema_upload_endpoint = cloudsync_memory_mprintf("%s/%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_SCHEMA, CLOUDSYNC_ENDPOINT_SCHEMA_UPLOAD); + char *schema_download_endpoint = cloudsync_memory_mprintf("%s/%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_SCHEMA, CLOUDSYNC_ENDPOINT_SCHEMA_DOWNLOAD); + if (!schema_check_endpoint || !schema_upload_endpoint || !schema_download_endpoint) { + if (schema_check_endpoint) cloudsync_memory_free(schema_check_endpoint); + if (schema_upload_endpoint) cloudsync_memory_free(schema_upload_endpoint); + if (schema_download_endpoint) cloudsync_memory_free(schema_download_endpoint); + cloudsync_memory_free(check_endpoint); + cloudsync_memory_free(upload_endpoint); + cloudsync_memory_free(apply_endpoint); + cloudsync_memory_free(status_endpoint); + sqlite3_result_error_code(context, SQLITE_NOMEM); + return false; + } + if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); data->check_endpoint = check_endpoint; @@ -731,10 +1160,19 @@ static bool network_compute_endpoints_with_address (sqlite3_context *context, ne if (data->status_endpoint) cloudsync_memory_free(data->status_endpoint); data->status_endpoint = status_endpoint; + if (data->schema_check_endpoint) cloudsync_memory_free(data->schema_check_endpoint); + data->schema_check_endpoint = schema_check_endpoint; + + if (data->schema_upload_endpoint) cloudsync_memory_free(data->schema_upload_endpoint); + data->schema_upload_endpoint = schema_upload_endpoint; + + if (data->schema_download_endpoint) cloudsync_memory_free(data->schema_download_endpoint); + data->schema_download_endpoint = schema_download_endpoint; + return true; } -void network_result_to_sqlite_error (sqlite3_context *context, NETWORK_RESULT res, const char *default_error_message) { +static void network_result_to_sqlite_error (sqlite3_context *context, NETWORK_RESULT res, const char *default_error_message) { sqlite3_result_error(context, ((res.code == CLOUDSYNC_NETWORK_ERROR) && (res.buffer)) ? res.buffer : default_error_message, -1); sqlite3_result_error_code(context, SQLITE_ERROR); } @@ -931,6 +1369,10 @@ void cloudsync_network_has_unsent_changes (sqlite3_context *context, int argc, s int64_t last_optimistic_version = -1; if (res.code == CLOUDSYNC_NETWORK_BUFFER && res.buffer) { + if (!network_validate_json_response(context, &res, "CloudSync status endpoint", NULL)) { + network_result_cleanup(&res); + return; + } last_optimistic_version = json_extract_int(res.buffer, res.blen, "lastOptimisticVersion", -1); } else if (res.code != CLOUDSYNC_NETWORK_OK) { network_result_to_sqlite_error(context, res, "unable to retrieve current status from remote host."); @@ -950,6 +1392,35 @@ int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc, network_data *netdata = (network_data *)cloudsync_auxdata(data); if (!netdata) {sqlite3_result_error(context, "Unable to retrieve CloudSync network context.", -1); return SQLITE_ERROR;} + + bool has_sync_tables = cloudsync_config_exists(data) && dbutils_table_settings_count_tables(data) > 0; + if (!has_sync_tables) { + char *migration_result = NULL; + char *migration_err = NULL; + int mrc = cloudsync_network_migration_check_internal(context, &migration_result, &migration_err); + if (migration_result) cloudsync_memory_free(migration_result); + if (migration_err) { + sqlite3_result_error(context, migration_err, -1); + cloudsync_memory_free(migration_err); + return SQLITE_ERROR; + } + if (mrc != SQLITE_OK) return mrc; + + has_sync_tables = cloudsync_config_exists(data) && dbutils_table_settings_count_tables(data) > 0; + if (!has_sync_tables) { + if (out) { + out->server_version = 0; + out->local_version = 0; + out->status = network_compute_status(0, 0, 0, 0); + } + return SQLITE_OK; + } + } + + if (cloudsync_pending_migration_count(data) > 0) { + sqlite3_result_error(context, "A pending schema migration must be uploaded before sending row changes.", -1); + return SQLITE_ERROR; + } // retrieve payload char *blob = NULL; @@ -964,6 +1435,19 @@ int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc, // Case 1: empty local db — no payload and no server state, skip network entirely if ((blob == NULL || blob_size == 0) && db_version == 0) { + bool has_sync_tables = cloudsync_config_exists(data) && dbutils_table_settings_count_tables(data) > 0; + if (!has_sync_tables) { + char *migration_result = NULL; + char *migration_err = NULL; + int mrc = cloudsync_network_migration_check_internal(context, &migration_result, &migration_err); + if (migration_result) cloudsync_memory_free(migration_result); + if (migration_err) { + sqlite3_result_error(context, migration_err, -1); + cloudsync_memory_free(migration_err); + return SQLITE_ERROR; + } + if (mrc != SQLITE_OK) return mrc; + } if (out) { out->server_version = 0; out->local_version = 0; @@ -982,6 +1466,11 @@ int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc, network_result_cleanup(&res); return SQLITE_ERROR; } + if (!network_validate_json_response(context, &res, "CloudSync upload endpoint", NULL)) { + cloudsync_memory_free(blob); + network_result_cleanup(&res); + return SQLITE_ERROR; + } char *s3_url = json_extract_string(res.buffer, res.blen, "url"); if (!s3_url) { @@ -1023,6 +1512,10 @@ int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc, char *last_failure_json = NULL; if (res.code == CLOUDSYNC_NETWORK_BUFFER && res.buffer) { + if (!network_validate_json_response(context, &res, "CloudSync apply/status endpoint", NULL)) { + network_result_cleanup(&res); + return SQLITE_ERROR; + } last_optimistic_version = json_extract_int(res.buffer, res.blen, "lastOptimisticVersion", -1); last_confirmed_version = json_extract_int(res.buffer, res.blen, "lastConfirmedVersion", -1); gaps_size = json_extract_array_size(res.buffer, res.blen, "gaps"); @@ -1104,6 +1597,10 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync NETWORK_RESULT result = network_receive_buffer(netdata, netdata->check_endpoint, netdata->authentication, true, true, json_payload, CLOUDSYNC_HEADER_SQLITECLOUD); int rc = SQLITE_OK; if (result.code == CLOUDSYNC_NETWORK_BUFFER) { + if (!network_validate_json_response(context, &result, "CloudSync check endpoint", err_out)) { + network_result_cleanup(&result); + return SQLITE_ERROR; + } char *download_url = json_extract_string(result.buffer, result.blen, "url"); if (!download_url) { sqlite3_result_error(context, "cloudsync_network_check_changes: missing 'url' in check response.", -1); @@ -1137,6 +1634,18 @@ void cloudsync_network_sync (sqlite3_context *context, int wait_ms, int max_retr int rc = cloudsync_network_send_changes_internal(context, 0, NULL, &sr); if (rc != SQLITE_OK) { if (sr.last_failure_json) cloudsync_memory_free(sr.last_failure_json); return; } + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + if (!cloudsync_config_exists(data) || dbutils_table_settings_count_tables(data) == 0) { + char *buf = cloudsync_memory_mprintf( + "{\"send\":{\"status\":\"%s\",\"localVersion\":%lld,\"serverVersion\":%lld}," + "\"receive\":{\"rows\":0,\"tables\":[]}}", + sr.status ? sr.status : "error", + (long long)sr.local_version, (long long)sr.server_version); + sqlite3_result_text(context, buf, -1, cloudsync_memory_free); + if (sr.last_failure_json) cloudsync_memory_free(sr.last_failure_json); + return; + } + int ntries = 0; int nrows = 0; char *receive_err = NULL; @@ -1356,7 +1865,9 @@ void cloudsync_network_status (sqlite3_context *context, int argc, sqlite3_value } NETWORK_RESULT res = network_receive_buffer(netdata, netdata->status_endpoint, netdata->authentication, true, false, NULL, CLOUDSYNC_HEADER_SQLITECLOUD); - network_set_sqlite_result(context, &res); + if (network_validate_json_response(context, &res, "CloudSync status endpoint", NULL)) { + network_set_sqlite_result(context, &res); + } network_result_cleanup(&res); } @@ -1405,6 +1916,17 @@ int cloudsync_network_register (sqlite3 *db, char **pzErrMsg, void *ctx) { rc = sqlite3_create_function(db, "cloudsync_network_status", 0, DEFAULT_FLAGS, ctx, cloudsync_network_status, NULL, NULL); if (rc != SQLITE_OK) return rc; + rc = sqlite3_create_function(db, "cloudsync_network_migration_check", 0, DEFAULT_FLAGS, ctx, cloudsync_network_migration_check, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_migration_upload", 1, DEFAULT_FLAGS, ctx, cloudsync_network_migration_upload, NULL, NULL); + if (rc != SQLITE_OK) return rc; + rc = sqlite3_create_function(db, "cloudsync_network_migration_upload", 0, DEFAULT_FLAGS, ctx, cloudsync_network_migration_upload, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_migration_download", 0, DEFAULT_FLAGS, ctx, cloudsync_network_migration_download, NULL, NULL); + if (rc != SQLITE_OK) return rc; + cleanup: if ((rc != SQLITE_OK) && (pzErrMsg)) { *pzErrMsg = sqlite3_mprintf("Error creating function in cloudsync_network_register: %s", sqlite3_errmsg(db)); diff --git a/src/network/network.m b/src/network/network.m index da2338c9..dbe5ecfe 100644 --- a/src/network/network.m +++ b/src/network/network.m @@ -13,168 +13,182 @@ void network_buffer_cleanup (void *xdata) { if (xdata) CFRelease(xdata); } -bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { - NSString *urlString = [NSString stringWithUTF8String:endpoint]; - NSURL *url = [NSURL URLWithString:urlString]; - if (!url) return false; - - NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url]; - [request setHTTPMethod:@"PUT"]; - [request setValue:@"application/octet-stream" forHTTPHeaderField:@"Content-Type"]; - [request setValue:@"text/plain" forHTTPHeaderField:@"Accept"]; - - if (authentication && authentication[0] != '\0') { - NSString *authString = [NSString stringWithFormat:@"Bearer %s", authentication]; - [request setValue:authString forHTTPHeaderField:@"Authorization"]; - } +static NSString *network_string_from_cstr (const char *value) { + return value ? [NSString stringWithUTF8String:value] : nil; +} - char *org_id = network_data_get_orgid(data); - if (org_id) { - [request setValue:[NSString stringWithUTF8String:org_id] forHTTPHeaderField:@CLOUDSYNC_HEADER_ORG]; - } +static NETWORK_RESULT network_error_result (NSString *message, size_t blen) { + NETWORK_RESULT result = {}; + result.code = CLOUDSYNC_NETWORK_ERROR; + result.buffer = message ? (char *)message.UTF8String : NULL; + result.xdata = message ? (void *)CFBridgingRetain(message) : NULL; + result.xfree = network_buffer_cleanup; + result.blen = blen; + return result; +} - NSData *bodyData = [NSData dataWithBytes:blob length:blob_size]; - [request setHTTPBody:bodyData]; +static void network_set_header (NSMutableURLRequest *request, const char *header) { + NSString *headerString = network_string_from_cstr(header); + if (!headerString) return; - __block bool success = false; - dispatch_semaphore_t sema = dispatch_semaphore_create(0); + NSRange separator = [headerString rangeOfString:@":"]; + if (separator.location == NSNotFound) return; - NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; - NSURLSession *session = [NSURLSession sessionWithConfiguration:config]; + NSCharacterSet *ws = [NSCharacterSet whitespaceCharacterSet]; + NSString *field = [[headerString substringToIndex:separator.location] stringByTrimmingCharactersInSet:ws]; + NSString *value = [[headerString substringFromIndex:separator.location + 1] stringByTrimmingCharactersInSet:ws]; + if (field.length == 0) return; + [request setValue:value forHTTPHeaderField:field]; +} - NSURLSessionDataTask *task = [session dataTaskWithRequest:request - completionHandler:^(NSData * _Nullable responseBody, - NSURLResponse * _Nullable response, - NSError * _Nullable error) { - if (!error && [response isKindOfClass:[NSHTTPURLResponse class]]) { - NSInteger statusCode = [(NSHTTPURLResponse *)response statusCode]; - success = (statusCode >= 200 && statusCode < 300); +bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { + @autoreleasepool { + if (!endpoint || blob_size < 0 || (!blob && blob_size > 0)) return false; + + NSString *urlString = network_string_from_cstr(endpoint); + NSURL *url = urlString ? [NSURL URLWithString:urlString] : nil; + if (!url) return false; + + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url]; + [request setHTTPMethod:@"PUT"]; + [request setValue:@"application/octet-stream" forHTTPHeaderField:@"Content-Type"]; + [request setValue:@"text/plain" forHTTPHeaderField:@"Accept"]; + + if (authentication && authentication[0] != '\0') { + NSString *authValue = network_string_from_cstr(authentication); + if (authValue) { + [request setValue:[NSString stringWithFormat:@"Bearer %@", authValue] forHTTPHeaderField:@"Authorization"]; + } } - dispatch_semaphore_signal(sema); - }]; - [task resume]; - dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER); - [session finishTasksAndInvalidate]; + char *org_id = data ? network_data_get_orgid(data) : NULL; + NSString *orgValue = network_string_from_cstr(org_id); + if (orgValue) { + [request setValue:orgValue forHTTPHeaderField:@CLOUDSYNC_HEADER_ORG]; + } - return success; -} + NSData *bodyData = blob_size > 0 ? [NSData dataWithBytes:blob length:(NSUInteger)blob_size] : [NSData data]; + [request setHTTPBody:bodyData]; + __block bool success = false; + dispatch_semaphore_t sema = dispatch_semaphore_create(0); -NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header) { - - NSString *urlString = [NSString stringWithUTF8String:endpoint]; - NSURL *url = [NSURL URLWithString:urlString]; - if (!url) { - NETWORK_RESULT result = {}; - NSString *msg = [NSString stringWithCString:"Malformed URL" encoding:NSUTF8StringEncoding]; - result.code = CLOUDSYNC_NETWORK_ERROR; - result.buffer = (char *)msg.UTF8String; - result.xdata = (void *)CFBridgingRetain(msg); - result.xfree = network_buffer_cleanup; - return result; - } + NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; + NSURLSession *session = [NSURLSession sessionWithConfiguration:config]; - NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url]; - request.HTTPMethod = (json_payload || is_post_request) ? @"POST" : @"GET"; + NSURLSessionDataTask *task = [session dataTaskWithRequest:request + completionHandler:^(NSData * _Nullable responseBody, + NSURLResponse * _Nullable response, + NSError * _Nullable error) { + if (!error && [response isKindOfClass:[NSHTTPURLResponse class]]) { + NSInteger statusCode = [(NSHTTPURLResponse *)response statusCode]; + success = (statusCode >= 200 && statusCode < 300); + } + dispatch_semaphore_signal(sema); + }]; - if (custom_header) { - NSString *header = [NSString stringWithUTF8String:custom_header]; - NSArray *parts = [header componentsSeparatedByString:@": "]; - if (parts.count == 2) { - [request setValue:parts[1] forHTTPHeaderField:parts[0]]; - } - } + [task resume]; + dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER); + [session finishTasksAndInvalidate]; - char *org_id = network_data_get_orgid(data); - if (org_id) { - [request setValue:[NSString stringWithUTF8String:org_id] forHTTPHeaderField:@CLOUDSYNC_HEADER_ORG]; + return success; } +} - if (authentication) { - NSString *authString = [NSString stringWithFormat:@"Bearer %s", authentication]; - [request setValue:authString forHTTPHeaderField:@"Authorization"]; - } - if (json_payload) { - [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"]; - NSData *jsonData = [NSData dataWithBytes:json_payload length:strlen(json_payload)]; - request.HTTPBody = jsonData; - } else if (is_post_request) { - request.HTTPBody = [NSData data]; // empty POST - } +NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header) { + @autoreleasepool { + NSString *urlString = network_string_from_cstr(endpoint); + NSURL *url = urlString ? [NSURL URLWithString:urlString] : nil; + if (!url) return network_error_result(@"Malformed URL", 0); - __block NSData *responseData = nil; - __block NSString *responseError = nil; - __block NSInteger statusCode = 0; - __block NSInteger errorCode = 0; + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url]; + request.HTTPMethod = (json_payload || is_post_request) ? @"POST" : @"GET"; - dispatch_semaphore_t sema = dispatch_semaphore_create(0); + network_set_header(request, custom_header); - NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; - NSURLSession *session = [NSURLSession sessionWithConfiguration:config]; - NSURLSessionDataTask *task = [session dataTaskWithRequest:request completionHandler:^(NSData *responseBody, NSURLResponse *response, NSError *error) { - responseData = responseBody; - if (error) { - responseError = [error localizedDescription]; - errorCode = [error code]; - } - if ([response isKindOfClass:[NSHTTPURLResponse class]]) { - statusCode = [(NSHTTPURLResponse *)response statusCode]; + char *org_id = data ? network_data_get_orgid(data) : NULL; + NSString *orgValue = network_string_from_cstr(org_id); + if (orgValue) { + [request setValue:orgValue forHTTPHeaderField:@CLOUDSYNC_HEADER_ORG]; } - dispatch_semaphore_signal(sema); - }]; - [task resume]; - dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER); - [session finishTasksAndInvalidate]; + if (authentication && authentication[0] != '\0') { + NSString *authValue = network_string_from_cstr(authentication); + if (authValue) { + [request setValue:[NSString stringWithFormat:@"Bearer %@", authValue] forHTTPHeaderField:@"Authorization"]; + } + } - if (!responseError && (statusCode >= 200 && statusCode < 300)) { - // check if OK should be returned - if (responseData == nil || [responseData length] == 0) { - return (NETWORK_RESULT){CLOUDSYNC_NETWORK_OK, NULL, 0, NULL, NULL}; + if (json_payload) { + [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"]; + NSData *jsonData = [NSData dataWithBytes:json_payload length:strlen(json_payload)]; + request.HTTPBody = jsonData; + } else if (is_post_request) { + request.HTTPBody = [NSData data]; // empty POST } - - // otherwise return a buffer - NETWORK_RESULT result = {}; - result.code = CLOUDSYNC_NETWORK_BUFFER; - if (zero_terminated) { - NSString *utf8String = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding]; - if (!utf8String) { - NSString *msg = @"Response is not valid UTF-8"; - return (NETWORK_RESULT){CLOUDSYNC_NETWORK_ERROR, (char *)msg.UTF8String, 0, (void *)CFBridgingRetain(msg), network_buffer_cleanup}; + + __block NSData *responseData = nil; + __block NSString *responseError = nil; + __block NSInteger statusCode = 0; + __block NSInteger errorCode = 0; + + dispatch_semaphore_t sema = dispatch_semaphore_create(0); + + NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; + NSURLSession *session = [NSURLSession sessionWithConfiguration:config]; + NSURLSessionDataTask *task = [session dataTaskWithRequest:request completionHandler:^(NSData *responseBody, NSURLResponse *response, NSError *error) { + responseData = responseBody; + if (error) { + responseError = [error localizedDescription]; + errorCode = [error code]; + } + if ([response isKindOfClass:[NSHTTPURLResponse class]]) { + statusCode = [(NSHTTPURLResponse *)response statusCode]; + } + dispatch_semaphore_signal(sema); + }]; + + [task resume]; + dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER); + [session finishTasksAndInvalidate]; + + if (!responseError && (statusCode >= 200 && statusCode < 300)) { + // check if OK should be returned + if (responseData == nil || [responseData length] == 0) { + return (NETWORK_RESULT){CLOUDSYNC_NETWORK_OK, NULL, 0, NULL, NULL}; } - result.buffer = (char *)utf8String.UTF8String; - result.xdata = (void *)CFBridgingRetain(utf8String); - } else { - result.buffer = (char *)responseData.bytes; - result.xdata = (void *)CFBridgingRetain(responseData); + + // otherwise return a buffer + NETWORK_RESULT result = {}; + result.code = CLOUDSYNC_NETWORK_BUFFER; + if (zero_terminated) { + NSString *utf8String = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding]; + if (!utf8String) return network_error_result(@"Response is not valid UTF-8", 0); + result.buffer = (char *)utf8String.UTF8String; + result.xdata = (void *)CFBridgingRetain(utf8String); + } else { + result.buffer = (char *)responseData.bytes; + result.xdata = (void *)CFBridgingRetain(responseData); + } + result.blen = [responseData length]; + result.xfree = network_buffer_cleanup; + + return result; } - result.blen = [responseData length]; - result.xfree = network_buffer_cleanup; - - return result; - } - - // return error - NETWORK_RESULT result = {}; - NSString *msg = nil; - if (responseError) { - msg = responseError; - } else if (responseData && [responseData length] > 0) { - // Use the actual response body as the error message - msg = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding]; - if (!msg) { - msg = [NSString stringWithFormat:@"HTTP %ld error", (long)statusCode]; + + // return error + NSString *msg = nil; + size_t blen = responseError ? (size_t)errorCode : (size_t)statusCode; + if (responseError) { + msg = responseError; + } else if (responseData && [responseData length] > 0) { + // Use the actual response body as the error message + msg = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding]; + if (msg) blen = [responseData length]; + else msg = [NSString stringWithFormat:@"HTTP %ld error", (long)statusCode]; } - } else { - msg = [NSString stringWithFormat:@"HTTP %ld error", (long)statusCode]; + + return network_error_result(msg, blen); } - result.code = CLOUDSYNC_NETWORK_ERROR; - result.buffer = (char *)msg.UTF8String; - result.xdata = (void *)CFBridgingRetain(msg); - result.xfree = network_buffer_cleanup; - result.blen = responseError ? (size_t)errorCode : (size_t)statusCode; - - return result; } diff --git a/src/network/network_private.h b/src/network/network_private.h index b0429593..7d230885 100644 --- a/src/network/network_private.h +++ b/src/network/network_private.h @@ -14,6 +14,10 @@ #define CLOUDSYNC_ENDPOINT_CHECK "check" #define CLOUDSYNC_ENDPOINT_APPLY "apply" #define CLOUDSYNC_ENDPOINT_STATUS "status" +#define CLOUDSYNC_ENDPOINT_SCHEMA "schema" +#define CLOUDSYNC_ENDPOINT_SCHEMA_CHECK "check" +#define CLOUDSYNC_ENDPOINT_SCHEMA_UPLOAD "upload" +#define CLOUDSYNC_ENDPOINT_SCHEMA_DOWNLOAD "download" #define CLOUDSYNC_HEADER_SQLITECLOUD "Accept: sqlc/plain" #define CLOUDSYNC_HEADER_ORG "X-CloudSync-Org" diff --git a/src/postgresql/cloudsync.sql.in b/src/postgresql/cloudsync.sql.in index edfa4d39..a13c78a0 100644 --- a/src/postgresql/cloudsync.sql.in +++ b/src/postgresql/cloudsync.sql.in @@ -120,16 +120,135 @@ RETURNS boolean AS 'MODULE_PATHNAME', 'cloudsync_set_column' LANGUAGE C VOLATILE; --- Begin schema alteration -CREATE OR REPLACE FUNCTION cloudsync_begin_alter(table_name text) +-- Declarative schema alterations +CREATE OR REPLACE FUNCTION cloudsync_alter_create_table(table_name text) RETURNS boolean -AS 'MODULE_PATHNAME', 'pg_cloudsync_begin_alter' +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_create_table' LANGUAGE C VOLATILE; --- Commit schema alteration -CREATE OR REPLACE FUNCTION cloudsync_commit_alter(table_name text) +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column(table_name text, column_name text, logical_type text, nullable boolean) RETURNS boolean -AS 'MODULE_PATHNAME', 'pg_cloudsync_commit_alter' +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column(table_name text, column_name text, logical_type text, nullable boolean, default_value text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column_sqlite(table_name text, column_name text, type_sql text, nullable boolean) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column_sqlite' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column_sqlite(table_name text, column_name text, type_sql text, nullable boolean, default_sql text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column_sqlite' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column_postgresql(table_name text, column_name text, type_sql text, nullable boolean) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column_postgresql' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column_postgresql(table_name text, column_name text, type_sql text, nullable boolean, default_sql text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column_postgresql' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_sql(sql text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_sql' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_sqlite(sql text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_sqlite' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_postgresql(sql text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_postgresql' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_primary_key(table_name text, column_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_primary_key' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_augment_table(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_augment_table' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_augment_table(table_name text, algo text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_augment_table' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_augment_table(table_name text, algo text, init_flags integer) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_augment_table' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_block_lww(table_name text, column_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_block_lww' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_block_lww(table_name text, column_name text, delimiter text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_block_lww' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_column(table_name text, column_name text, key text, value text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_column' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_filter(table_name text, filter_expr text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_filter' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_filter_sqlite(table_name text, filter_expr text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_filter_sqlite' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_filter_postgresql(table_name text, filter_expr text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_filter_postgresql' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_drop_column(table_name text, column_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_drop_column' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_rename_column(table_name text, from_name text, to_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_rename_column' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_clear() +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_clear' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_clear(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_clear' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_preview() +RETURNS text +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_preview' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_apply() +RETURNS text +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_apply' LANGUAGE C VOLATILE; -- Payload encoding (aggregate function) @@ -161,6 +280,12 @@ RETURNS integer AS 'MODULE_PATHNAME', 'pg_cloudsync_payload_apply' LANGUAGE C VOLATILE; +-- Schema migration application +CREATE OR REPLACE FUNCTION cloudsync_migration_apply(payload text) +RETURNS text +AS 'MODULE_PATHNAME', 'pg_cloudsync_migration_apply' +LANGUAGE C VOLATILE; + -- ============================================================================ -- Private/Internal Functions -- ============================================================================ diff --git a/src/postgresql/cloudsync_postgresql.c b/src/postgresql/cloudsync_postgresql.c index 4d0ed6af..a1d35cee 100644 --- a/src/postgresql/cloudsync_postgresql.c +++ b/src/postgresql/cloudsync_postgresql.c @@ -464,7 +464,7 @@ Datum pg_cloudsync_cleanup (PG_FUNCTION_ARGS) { PG_TRY(); { - rc = cloudsync_cleanup(data, table); + rc = cloudsync_cleanup(data, table, false); } PG_CATCH(); { @@ -818,117 +818,6 @@ Datum cloudsync_clear_filter (PG_FUNCTION_ARGS) { PG_RETURN_BOOL(true); } -// MARK: - Schema Alteration - - -// cloudsync_begin_alter - Begin schema alteration -PG_FUNCTION_INFO_V1(pg_cloudsync_begin_alter); -Datum pg_cloudsync_begin_alter (PG_FUNCTION_ARGS) { - if (PG_ARGISNULL(0)) { - ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); - } - - const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); - cloudsync_context *data = get_cloudsync_context(); - int rc = DBRES_OK; - - if (SPI_connect() != SPI_OK_CONNECT) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed"))); - } - - PG_TRY(); - { - database_begin_savepoint(data, "cloudsync_alter"); - rc = cloudsync_begin_alter(data, table_name); - if (rc != DBRES_OK) { - database_rollback_savepoint(data, "cloudsync_alter"); - } - } - PG_CATCH(); - { - SPI_finish(); - PG_RE_THROW(); - } - PG_END_TRY(); - - SPI_finish(); - if (rc != DBRES_OK) { - ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("%s", cloudsync_errmsg(data)))); - } - PG_RETURN_BOOL(true); -} - -// cloudsync_commit_alter - Commit schema alteration -// -// This wrapper manages SPI in two phases to avoid the PostgreSQL warning -// "subtransaction left non-empty SPI stack". The subtransaction was opened -// by a prior cloudsync_begin_alter call, so SPI_connect() here creates a -// connection at the subtransaction level. We must disconnect SPI before -// cloudsync_commit_alter releases that subtransaction, then reconnect -// for post-commit work (cloudsync_update_schema_hash). -// Prepared statements survive SPI_finish via SPI_keepplan/TopMemoryContext. -PG_FUNCTION_INFO_V1(pg_cloudsync_commit_alter); -Datum pg_cloudsync_commit_alter (PG_FUNCTION_ARGS) { - if (PG_ARGISNULL(0)) { - ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); - } - - const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); - cloudsync_context *data = get_cloudsync_context(); - int rc = DBRES_OK; - - // Phase 1: SPI work before savepoint release - if (SPI_connect() != SPI_OK_CONNECT) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed"))); - } - - PG_TRY(); - { - rc = cloudsync_commit_alter(data, table_name); - } - PG_CATCH(); - { - SPI_finish(); - PG_RE_THROW(); - } - PG_END_TRY(); - - // Disconnect SPI before savepoint boundary - SPI_finish(); - - if (rc != DBRES_OK) { - // Rollback savepoint (SPI disconnected, no warning) - database_rollback_savepoint(data, "cloudsync_alter"); - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); - } - - // Release savepoint (SPI disconnected, no warning) - rc = database_commit_savepoint(data, "cloudsync_alter"); - if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to release cloudsync_alter savepoint: %s", database_errmsg(data)))); - } - - // Phase 2: reconnect SPI for post-commit work - if (SPI_connect() != SPI_OK_CONNECT) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed after savepoint release"))); - } - - PG_TRY(); - { - cloudsync_update_schema_hash(data); - } - PG_CATCH(); - { - SPI_finish(); - PG_RE_THROW(); - } - PG_END_TRY(); - - SPI_finish(); - PG_RETURN_BOOL(true); -} - // MARK: - Payload Functions - // Aggregate function: cloudsync_payload_encode transition function @@ -1055,6 +944,431 @@ Datum pg_cloudsync_payload_apply (PG_FUNCTION_ARGS) { return cloudsync_payload_decode(fcinfo); } +// Schema migration apply +PG_FUNCTION_INFO_V1(pg_cloudsync_migration_apply); +Datum pg_cloudsync_migration_apply (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("migration payload cannot be NULL"))); + } + + char *payload = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + char *result_json = NULL; + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + rc = cloudsync_migration_apply(data, payload, (int)strlen(payload), &result_json); + } + PG_CATCH(); + { + if (result_json) cloudsync_memory_free(result_json); + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + + text *result = cstring_to_text(result_json ? result_json : "{\"status\":\"applied\"}"); + if (result_json) cloudsync_memory_free(result_json); + PG_RETURN_TEXT_P(result); +} + +static void pg_cloudsync_raise_error(cloudsync_context *data) { + const char *msg = cloudsync_errmsg(data); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", msg && msg[0] ? msg : "CloudSync operation failed"))); +} + +static void pg_cloudsync_spi_connect_or_error(void) { + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_create_table); +Datum pg_cloudsync_alter_create_table(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_create_table(data, text_to_cstring(PG_GETARG_TEXT_PP(0))); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_add_column); +Datum pg_cloudsync_alter_add_column(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2) || PG_ARGISNULL(3)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cloudsync_alter_add_column requires table, column, type, and nullable"))); + } + const char *default_value = NULL; + bool has_default = PG_NARGS() >= 5; + if (has_default) { + if (!PG_ARGISNULL(4)) default_value = text_to_cstring(PG_GETARG_TEXT_PP(4)); + } + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_add_column(data, + text_to_cstring(PG_GETARG_TEXT_PP(0)), + text_to_cstring(PG_GETARG_TEXT_PP(1)), + text_to_cstring(PG_GETARG_TEXT_PP(2)), + PG_GETARG_BOOL(3), + has_default, + default_value); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +static Datum pg_cloudsync_alter_add_column_dialect(PG_FUNCTION_ARGS, const char *dialect) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2) || PG_ARGISNULL(3)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("dialect add_column requires table, column, type SQL, and nullable"))); + } + bool has_default = PG_NARGS() >= 5; + const char *default_sql = (has_default && !PG_ARGISNULL(4)) ? text_to_cstring(PG_GETARG_TEXT_PP(4)) : NULL; + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_add_column_dialect(data, + text_to_cstring(PG_GETARG_TEXT_PP(0)), + text_to_cstring(PG_GETARG_TEXT_PP(1)), + dialect, + text_to_cstring(PG_GETARG_TEXT_PP(2)), + PG_GETARG_BOOL(3), + has_default, + default_sql); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_add_column_sqlite); +Datum pg_cloudsync_alter_add_column_sqlite(PG_FUNCTION_ARGS) { + return pg_cloudsync_alter_add_column_dialect(fcinfo, "sqlite"); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_add_column_postgresql); +Datum pg_cloudsync_alter_add_column_postgresql(PG_FUNCTION_ARGS) { + return pg_cloudsync_alter_add_column_dialect(fcinfo, "postgresql"); +} + +static Datum pg_cloudsync_alter_sql_dialect(PG_FUNCTION_ARGS, const char *dialect) { + if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("sql cannot be NULL"))); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + const char *sql = text_to_cstring(PG_GETARG_TEXT_PP(0)); + rc = dialect ? cloudsync_alter_sql_dialect(data, dialect, sql) : cloudsync_alter_sql(data, sql); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_sql); +Datum pg_cloudsync_alter_sql(PG_FUNCTION_ARGS) { + return pg_cloudsync_alter_sql_dialect(fcinfo, NULL); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_sqlite); +Datum pg_cloudsync_alter_sqlite(PG_FUNCTION_ARGS) { + return pg_cloudsync_alter_sql_dialect(fcinfo, "sqlite"); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_postgresql); +Datum pg_cloudsync_alter_postgresql(PG_FUNCTION_ARGS) { + return pg_cloudsync_alter_sql_dialect(fcinfo, "postgresql"); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_add_primary_key); +Datum pg_cloudsync_alter_add_primary_key(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name and column_name cannot be NULL"))); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_add_primary_key(data, text_to_cstring(PG_GETARG_TEXT_PP(0)), text_to_cstring(PG_GETARG_TEXT_PP(1))); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_augment_table); +Datum pg_cloudsync_alter_augment_table(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + const char *algo = (PG_NARGS() >= 2 && !PG_ARGISNULL(1)) ? text_to_cstring(PG_GETARG_TEXT_PP(1)) : CLOUDSYNC_DEFAULT_ALGO; + int32 init_flags = (PG_NARGS() >= 3 && !PG_ARGISNULL(2)) ? PG_GETARG_INT32(2) : CLOUDSYNC_INIT_FLAG_NONE; + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_augment_table(data, text_to_cstring(PG_GETARG_TEXT_PP(0)), algo, init_flags); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_set_block_lww); +Datum pg_cloudsync_alter_set_block_lww(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name and column_name cannot be NULL"))); + const char *delimiter = (PG_NARGS() >= 3 && !PG_ARGISNULL(2)) ? text_to_cstring(PG_GETARG_TEXT_PP(2)) : NULL; + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_set_block_lww(data, text_to_cstring(PG_GETARG_TEXT_PP(0)), text_to_cstring(PG_GETARG_TEXT_PP(1)), delimiter); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_set_column); +Datum pg_cloudsync_alter_set_column(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name, column_name, and key cannot be NULL"))); + const char *value = PG_ARGISNULL(3) ? NULL : text_to_cstring(PG_GETARG_TEXT_PP(3)); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_set_column(data, text_to_cstring(PG_GETARG_TEXT_PP(0)), text_to_cstring(PG_GETARG_TEXT_PP(1)), text_to_cstring(PG_GETARG_TEXT_PP(2)), value); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_set_filter); +Datum pg_cloudsync_alter_set_filter(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name and filter cannot be NULL"))); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_set_filter(data, text_to_cstring(PG_GETARG_TEXT_PP(0)), text_to_cstring(PG_GETARG_TEXT_PP(1))); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +static Datum pg_cloudsync_alter_set_filter_dialect(PG_FUNCTION_ARGS, const char *dialect) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name and filter cannot be NULL"))); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_set_filter_dialect(data, text_to_cstring(PG_GETARG_TEXT_PP(0)), dialect, text_to_cstring(PG_GETARG_TEXT_PP(1))); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_set_filter_sqlite); +Datum pg_cloudsync_alter_set_filter_sqlite(PG_FUNCTION_ARGS) { + return pg_cloudsync_alter_set_filter_dialect(fcinfo, "sqlite"); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_set_filter_postgresql); +Datum pg_cloudsync_alter_set_filter_postgresql(PG_FUNCTION_ARGS) { + return pg_cloudsync_alter_set_filter_dialect(fcinfo, "postgresql"); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_drop_column); +Datum pg_cloudsync_alter_drop_column(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name and column_name cannot be NULL"))); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_drop_column(data, text_to_cstring(PG_GETARG_TEXT_PP(0)), text_to_cstring(PG_GETARG_TEXT_PP(1))); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_rename_column); +Datum pg_cloudsync_alter_rename_column(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name, from, and to cannot be NULL"))); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_rename_column(data, text_to_cstring(PG_GETARG_TEXT_PP(0)), text_to_cstring(PG_GETARG_TEXT_PP(1)), text_to_cstring(PG_GETARG_TEXT_PP(2))); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_clear); +Datum pg_cloudsync_alter_clear(PG_FUNCTION_ARGS) { + const char *table = (PG_NARGS() == 0 || PG_ARGISNULL(0)) ? NULL : text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + int rc = cloudsync_alter_clear(data, table); + if (rc != DBRES_OK) pg_cloudsync_raise_error(data); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_preview); +Datum pg_cloudsync_alter_preview(PG_FUNCTION_ARGS) { + cloudsync_context *data = get_cloudsync_context(); + char *payload = NULL; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + payload = cloudsync_alter_preview(data); + } + PG_CATCH(); + { + if (payload) cloudsync_memory_free(payload); + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (!payload) pg_cloudsync_raise_error(data); + text *result = cstring_to_text(payload); + cloudsync_memory_free(payload); + PG_RETURN_TEXT_P(result); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_alter_apply); +Datum pg_cloudsync_alter_apply(PG_FUNCTION_ARGS) { + cloudsync_context *data = get_cloudsync_context(); + char *result_json = NULL; + int rc = DBRES_OK; + pg_cloudsync_spi_connect_or_error(); + PG_TRY(); + { + rc = cloudsync_alter_apply(data, &result_json); + } + PG_CATCH(); + { + if (result_json) cloudsync_memory_free(result_json); + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + SPI_finish(); + if (rc != DBRES_OK) { + if (result_json) cloudsync_memory_free(result_json); + pg_cloudsync_raise_error(data); + } + text *result = cstring_to_text(result_json ? result_json : "{\"status\":\"applied\"}"); + if (result_json) cloudsync_memory_free(result_json); + PG_RETURN_TEXT_P(result); +} + // MARK: - Private/Internal Functions - typedef struct cloudsync_pg_cleanup_state { diff --git a/src/postgresql/database_postgresql.c b/src/postgresql/database_postgresql.c index 3fc6310f..3a503a1b 100644 --- a/src/postgresql/database_postgresql.c +++ b/src/postgresql/database_postgresql.c @@ -2973,7 +2973,12 @@ int database_rollback_savepoint (cloudsync_context *data, const char *savepoint_ PG_TRY(); { RollbackAndReleaseCurrentSubTransaction(); - database_refresh_snapshot(); + /* + * Do not refresh the active snapshot after a rollback. Several callers + * raise a PostgreSQL ERROR immediately after rolling back; pushing a new + * snapshot at that point can leave portal->portalSnapshot non-NULL when + * PL/pgSQL exception handling resumes. + */ } PG_CATCH(); { @@ -3073,4 +3078,3 @@ uint64_t dbmem_size (void *ptr) { return 0; } - diff --git a/src/postgresql/migrations/cloudsync--1.0--1.1.sql b/src/postgresql/migrations/cloudsync--1.0--1.1.sql new file mode 100644 index 00000000..5449eee4 --- /dev/null +++ b/src/postgresql/migrations/cloudsync--1.0--1.1.sql @@ -0,0 +1,140 @@ +-- CloudSync 1.0 -> 1.1 +-- Adds declarative schema migration SQL functions. + +DROP FUNCTION IF EXISTS cloudsync_begin_alter(text); +DROP FUNCTION IF EXISTS cloudsync_commit_alter(text); + +CREATE OR REPLACE FUNCTION cloudsync_alter_create_table(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_create_table' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column(table_name text, column_name text, logical_type text, nullable boolean) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column(table_name text, column_name text, logical_type text, nullable boolean, default_value text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column_sqlite(table_name text, column_name text, type_sql text, nullable boolean) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column_sqlite' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column_sqlite(table_name text, column_name text, type_sql text, nullable boolean, default_sql text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column_sqlite' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column_postgresql(table_name text, column_name text, type_sql text, nullable boolean) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column_postgresql' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column_postgresql(table_name text, column_name text, type_sql text, nullable boolean, default_sql text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column_postgresql' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_sql(sql text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_sql' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_sqlite(sql text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_sqlite' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_postgresql(sql text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_postgresql' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_add_primary_key(table_name text, column_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_primary_key' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_augment_table(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_augment_table' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_augment_table(table_name text, algo text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_augment_table' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_augment_table(table_name text, algo text, init_flags integer) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_augment_table' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_block_lww(table_name text, column_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_block_lww' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_block_lww(table_name text, column_name text, delimiter text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_block_lww' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_column(table_name text, column_name text, key text, value text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_column' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_filter(table_name text, filter_expr text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_filter' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_filter_sqlite(table_name text, filter_expr text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_filter_sqlite' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_set_filter_postgresql(table_name text, filter_expr text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_set_filter_postgresql' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_drop_column(table_name text, column_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_drop_column' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_rename_column(table_name text, from_name text, to_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_rename_column' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_clear() +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_clear' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_clear(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_clear' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_preview() +RETURNS text +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_preview' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_alter_apply() +RETURNS text +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_apply' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_migration_apply(payload text) +RETURNS text +AS 'MODULE_PATHNAME', 'pg_cloudsync_migration_apply' +LANGUAGE C VOLATILE; diff --git a/src/sqlite/cloudsync_sqlite.c b/src/sqlite/cloudsync_sqlite.c index bdff56b5..f0698d85 100644 --- a/src/sqlite/cloudsync_sqlite.c +++ b/src/sqlite/cloudsync_sqlite.c @@ -12,6 +12,7 @@ #include "../block.h" #include "../database.h" #include "../dbutils.h" +#include "../utils.h" #ifndef CLOUDSYNC_OMIT_NETWORK #include "../network/network.h" @@ -825,7 +826,7 @@ void dbsync_cleanup (sqlite3_context *context, int argc, sqlite3_value **argv) { const char *table = (const char *)database_value_text(argv[0]); cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - int rc = cloudsync_cleanup(data, table); + int rc = cloudsync_cleanup(data, table, false); if (rc != DBRES_OK) { sqlite3_result_error(context, cloudsync_errmsg(data), -1); sqlite3_result_error_code(context, rc); @@ -934,56 +935,6 @@ void dbsync_init1 (sqlite3_context *context, int argc, sqlite3_value **argv) { dbsync_init(context, table, NULL, CLOUDSYNC_INIT_FLAG_NONE); } -// MARK: - - -void dbsync_begin_alter (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("dbsync_begin_alter"); - - const char *table_name = (const char *)database_value_text(argv[0]); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - int rc = database_begin_savepoint(data, "cloudsync_alter"); - if (rc != DBRES_OK) { - sqlite3_result_error(context, "Unable to create cloudsync_alter savepoint", -1); - sqlite3_result_error_code(context, rc); - return; - } - - rc = cloudsync_begin_alter(data, table_name); - if (rc != DBRES_OK) { - database_rollback_savepoint(data, "cloudsync_alter"); - sqlite3_result_error(context, cloudsync_errmsg(data), -1); - sqlite3_result_error_code(context, rc); - } -} - -void dbsync_commit_alter (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_commit_alter"); - - //retrieve table argument - const char *table_name = (const char *)database_value_text(argv[0]); - - // retrieve context - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - int rc = cloudsync_commit_alter(data, table_name); - if (rc != DBRES_OK) { - database_rollback_savepoint(data, "cloudsync_alter"); - sqlite3_result_error(context, cloudsync_errmsg(data), -1); - sqlite3_result_error_code(context, rc); - return; - } - - rc = database_commit_savepoint(data, "cloudsync_alter"); - if (rc != DBRES_OK) { - sqlite3_result_error(context, database_errmsg(data), -1); - sqlite3_result_error_code(context, rc); - return; - } - - cloudsync_update_schema_hash(data); -} - // MARK: - Payload - void dbsync_payload_encode_step (sqlite3_context *context, int argc, sqlite3_value **argv) { @@ -1145,6 +1096,266 @@ void dbsync_payload_load (sqlite3_context *context, int argc, sqlite3_value **ar } #endif +// MARK: - Schema Migrations - + +void dbsync_migration_apply (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_migration_apply"); + + if (database_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(context, "cloudsync_migration_apply expects a JSON text payload.", -1); + sqlite3_result_error_code(context, SQLITE_MISUSE); + return; + } + + const char *payload = (const char *)database_value_text(argv[0]); + int payload_len = database_value_bytes(argv[0]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + char *result = NULL; + int rc = cloudsync_migration_apply(data, payload, payload_len, &result); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + if (result) cloudsync_memory_free(result); + return; + } + + if (result) sqlite3_result_text(context, result, -1, cloudsync_memory_free); + else sqlite3_result_int(context, 1); +} + +static void dbsync_result_from_rc(sqlite3_context *context, cloudsync_context *data, int rc) { + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + sqlite3_result_int(context, 1); +} + +static char *dbsync_default_value_text(sqlite3_value *value, bool *oom) { + int type = sqlite3_value_type(value); + switch (type) { + case SQLITE_NULL: + return NULL; + case SQLITE_INTEGER: + return cloudsync_memory_mprintf("%lld", (long long)sqlite3_value_int64(value)); + case SQLITE_FLOAT: { + char buffer[64]; + snprintf(buffer, sizeof(buffer), "%.17g", sqlite3_value_double(value)); + return cloudsync_string_dup(buffer); + } + case SQLITE_BLOB: { + const unsigned char *blob = sqlite3_value_blob(value); + int len = sqlite3_value_bytes(value); + char *hex = cloudsync_memory_alloc((uint64_t)len * 2 + 1); + if (!hex) { + if (oom) *oom = true; + return NULL; + } + static const char digits[] = "0123456789abcdef"; + for (int i = 0; i < len; ++i) { + hex[i * 2] = digits[(blob[i] >> 4) & 0x0f]; + hex[i * 2 + 1] = digits[blob[i] & 0x0f]; + } + hex[len * 2] = '\0'; + return hex; + } + default: + return cloudsync_string_dup((const char *)sqlite3_value_text(value)); + } +} + +void dbsync_alter_create_table(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_alter_create_table(data, (const char *)database_value_text(argv[0])); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_add_column(sqlite3_context *context, int argc, sqlite3_value **argv) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + char *default_value = NULL; + bool oom = false; + bool has_default = argc >= 5; + if (has_default) { + default_value = dbsync_default_value_text(argv[4], &oom); + if (oom || (!default_value && sqlite3_value_type(argv[4]) != SQLITE_NULL)) { + sqlite3_result_error_nomem(context); + return; + } + } + int rc = cloudsync_alter_add_column(data, + (const char *)database_value_text(argv[0]), + (const char *)database_value_text(argv[1]), + (const char *)database_value_text(argv[2]), + database_value_int(argv[3]) != 0, + has_default, + default_value); + if (default_value) cloudsync_memory_free(default_value); + dbsync_result_from_rc(context, data, rc); +} + +static void dbsync_alter_add_column_dialect(sqlite3_context *context, int argc, sqlite3_value **argv, const char *dialect) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_alter_add_column_dialect(data, + (const char *)database_value_text(argv[0]), + (const char *)database_value_text(argv[1]), + dialect, + (const char *)database_value_text(argv[2]), + database_value_int(argv[3]) != 0, + argc >= 5, + argc >= 5 ? (const char *)database_value_text(argv[4]) : NULL); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_add_column_sqlite(sqlite3_context *context, int argc, sqlite3_value **argv) { + dbsync_alter_add_column_dialect(context, argc, argv, "sqlite"); +} + +void dbsync_alter_add_column_postgresql(sqlite3_context *context, int argc, sqlite3_value **argv) { + dbsync_alter_add_column_dialect(context, argc, argv, "postgresql"); +} + +static void dbsync_alter_sql_dialect(sqlite3_context *context, sqlite3_value **argv, const char *dialect) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = dialect + ? cloudsync_alter_sql_dialect(data, dialect, (const char *)database_value_text(argv[0])) + : cloudsync_alter_sql(data, (const char *)database_value_text(argv[0])); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_sql(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + dbsync_alter_sql_dialect(context, argv, NULL); +} + +void dbsync_alter_sqlite(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + dbsync_alter_sql_dialect(context, argv, "sqlite"); +} + +void dbsync_alter_postgresql(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + dbsync_alter_sql_dialect(context, argv, "postgresql"); +} + +void dbsync_alter_add_primary_key(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_alter_add_primary_key(data, (const char *)database_value_text(argv[0]), (const char *)database_value_text(argv[1])); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_augment_table(sqlite3_context *context, int argc, sqlite3_value **argv) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + const char *algorithm = (argc >= 2 && sqlite3_value_type(argv[1]) != SQLITE_NULL) ? (const char *)database_value_text(argv[1]) : CLOUDSYNC_DEFAULT_ALGO; + int init_flags = argc >= 3 ? database_value_int(argv[2]) : CLOUDSYNC_INIT_FLAG_NONE; + int rc = cloudsync_alter_augment_table(data, + (const char *)database_value_text(argv[0]), + algorithm, + init_flags); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_set_block_lww(sqlite3_context *context, int argc, sqlite3_value **argv) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + const char *delimiter = (argc >= 3 && sqlite3_value_type(argv[2]) != SQLITE_NULL) ? (const char *)database_value_text(argv[2]) : NULL; + int rc = cloudsync_alter_set_block_lww(data, (const char *)database_value_text(argv[0]), (const char *)database_value_text(argv[1]), delimiter); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_set_column(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + const char *value = sqlite3_value_type(argv[3]) == SQLITE_NULL ? NULL : (const char *)database_value_text(argv[3]); + int rc = cloudsync_alter_set_column(data, + (const char *)database_value_text(argv[0]), + (const char *)database_value_text(argv[1]), + (const char *)database_value_text(argv[2]), + value); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_set_filter(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_alter_set_filter(data, (const char *)database_value_text(argv[0]), (const char *)database_value_text(argv[1])); + dbsync_result_from_rc(context, data, rc); +} + +static void dbsync_alter_set_filter_dialect(sqlite3_context *context, sqlite3_value **argv, const char *dialect) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_alter_set_filter_dialect(data, + (const char *)database_value_text(argv[0]), + dialect, + (const char *)database_value_text(argv[1])); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_set_filter_sqlite(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + dbsync_alter_set_filter_dialect(context, argv, "sqlite"); +} + +void dbsync_alter_set_filter_postgresql(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + dbsync_alter_set_filter_dialect(context, argv, "postgresql"); +} + +void dbsync_alter_drop_column(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_alter_drop_column(data, (const char *)database_value_text(argv[0]), (const char *)database_value_text(argv[1])); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_rename_column(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_alter_rename_column(data, + (const char *)database_value_text(argv[0]), + (const char *)database_value_text(argv[1]), + (const char *)database_value_text(argv[2])); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_clear(sqlite3_context *context, int argc, sqlite3_value **argv) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + const char *table = argc == 0 || sqlite3_value_type(argv[0]) == SQLITE_NULL ? NULL : (const char *)database_value_text(argv[0]); + int rc = cloudsync_alter_clear(data, table); + dbsync_result_from_rc(context, data, rc); +} + +void dbsync_alter_preview(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + char *payload = cloudsync_alter_preview(data); + if (!payload) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, cloudsync_errcode(data)); + return; + } + sqlite3_result_text(context, payload, -1, cloudsync_memory_free); +} + +void dbsync_alter_apply(sqlite3_context *context, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + char *result = NULL; + int rc = cloudsync_alter_apply(data, &result); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + if (result) cloudsync_memory_free(result); + return; + } + if (result) sqlite3_result_text(context, result, -1, cloudsync_memory_free); + else sqlite3_result_int(context, 1); +} + // MARK: - Register - int dbsync_register_with_flags (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, int flags, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { @@ -1434,12 +1645,6 @@ int dbsync_register_functions (sqlite3 *db, char **pzErrMsg) { rc = dbsync_register_function(db, "cloudsync_db_version_next", dbsync_db_version_next, 1, pzErrMsg, ctx, NULL); if (rc != SQLITE_OK) return rc; - rc = dbsync_register_function(db, "cloudsync_begin_alter", dbsync_begin_alter, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbsync_register_function(db, "cloudsync_commit_alter", dbsync_commit_alter, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - rc = dbsync_register_function(db, "cloudsync_uuid", dbsync_uuid, 0, pzErrMsg, ctx, NULL); if (rc != SQLITE_OK) return rc; @@ -1452,6 +1657,71 @@ int dbsync_register_functions (sqlite3 *db, char **pzErrMsg) { if (rc != SQLITE_OK) return rc; rc = dbsync_register_function(db, "cloudsync_payload_apply", dbsync_payload_decode, -1, pzErrMsg, ctx, NULL); if (rc != SQLITE_OK) return rc; + + // SCHEMA MIGRATIONS + rc = dbsync_register_function(db, "cloudsync_migration_apply", dbsync_migration_apply, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_alter_create_table", dbsync_alter_create_table, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_alter_add_column", dbsync_alter_add_column, 4, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_add_column", dbsync_alter_add_column, 5, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_alter_add_column_sqlite", dbsync_alter_add_column_sqlite, 4, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_add_column_sqlite", dbsync_alter_add_column_sqlite, 5, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_add_column_postgresql", dbsync_alter_add_column_postgresql, 4, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_add_column_postgresql", dbsync_alter_add_column_postgresql, 5, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_alter_sql", dbsync_alter_sql, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_sqlite", dbsync_alter_sqlite, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_postgresql", dbsync_alter_postgresql, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_alter_add_primary_key", dbsync_alter_add_primary_key, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_alter_augment_table", dbsync_alter_augment_table, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_augment_table", dbsync_alter_augment_table, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_augment_table", dbsync_alter_augment_table, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_alter_set_block_lww", dbsync_alter_set_block_lww, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_set_block_lww", dbsync_alter_set_block_lww, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_alter_set_column", dbsync_alter_set_column, 4, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_set_filter", dbsync_alter_set_filter, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_set_filter_sqlite", dbsync_alter_set_filter_sqlite, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_set_filter_postgresql", dbsync_alter_set_filter_postgresql, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_alter_drop_column", dbsync_alter_drop_column, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_rename_column", dbsync_alter_rename_column, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_clear", dbsync_alter_clear, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_clear", dbsync_alter_clear, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_preview", dbsync_alter_preview, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_alter_apply", dbsync_alter_apply, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; #ifdef CLOUDSYNC_DESKTOP_OS rc = dbsync_register_function(db, "cloudsync_payload_save", dbsync_payload_save, 1, pzErrMsg, ctx, NULL); diff --git a/src/sqlite/database_sqlite.c b/src/sqlite/database_sqlite.c index b7864bbd..f5e8f6b0 100644 --- a/src/sqlite/database_sqlite.c +++ b/src/sqlite/database_sqlite.c @@ -672,7 +672,7 @@ int database_cleanup (cloudsync_context *data) { } for (int i = ncols; i < nrows+ncols; i+=ncols) { - int rc2 = cloudsync_cleanup(data, result[i]); + int rc2 = cloudsync_cleanup(data, result[i], false); if (rc2 != SQLITE_OK) {rc = rc2; goto exit_cleanup;} } @@ -1384,4 +1384,3 @@ uint64_t dbmem_size (void *ptr) { return (uint64_t)sqlite3_msize(ptr); } - diff --git a/src/utils.c b/src/utils.c index fff6cdd2..898afca2 100644 --- a/src/utils.c +++ b/src/utils.c @@ -72,7 +72,7 @@ int cloudsync_uuid_v7 (uint8_t value[UUID_LEN]) { // get current timestamp in ms struct timespec ts; - #ifdef __ANDROID__ + #if defined(__ANDROID__) || defined(__APPLE__) if (clock_gettime(CLOCK_REALTIME, &ts) != 0) return -1; #else if (timespec_get(&ts, TIME_UTC) == 0) return -1; diff --git a/test/integration.c b/test/integration.c index 62bb1de2..52c2152d 100644 --- a/test/integration.c +++ b/test/integration.c @@ -14,6 +14,14 @@ #include "utils.h" #include "sqlite3.h" +#ifndef _WIN32 +#include +#include +#include +#include +#include +#endif + // Define the number of simulated peers, when it's 0 it skips the peer test. #if defined(__linux__) && !defined(__ANDROID__) #define PEERS 0 @@ -498,6 +506,511 @@ int version(void){ ABORT_TEST } +#ifndef _WIN32 +typedef enum { + MOCK_INVALID_STATUS, + MOCK_INVALID_CHECK, + MOCK_INVALID_UPLOAD, + MOCK_INVALID_APPLY, + MOCK_INVALID_SCHEMA_CHECK, + MOCK_INVALID_SCHEMA_DOWNLOAD, + MOCK_INVALID_SCHEMA_UPLOAD, + MOCK_SCHEMA_UPLOAD_AUTH_ERROR, + MOCK_SCHEMA_CHECK_HTTP_EMPTY_ERROR, + MOCK_SCHEMA_UPLOAD_MISSING_STATUS, + MOCK_FIRST_SCHEMA_SYNC +} mock_network_scenario; + +typedef struct { + int listen_fd; + int port; + volatile int stop; + pthread_t thread; + mock_network_scenario scenario; + const unsigned char *payload; + int payload_len; + const char *migration_json; +} mock_network_server; + +static int mock_send_all(int fd, const void *buffer, size_t len) { + const char *ptr = (const char *)buffer; + while (len > 0) { + ssize_t sent = send(fd, ptr, len, 0); + if (sent <= 0) return SQLITE_ERROR; + ptr += sent; + len -= (size_t)sent; + } + return SQLITE_OK; +} + +static void mock_send_response(int fd, const char *content_type, const void *body, size_t body_len) { + char header[512]; + snprintf(header, sizeof(header), + "HTTP/1.1 200 OK\r\nContent-Length: %zu\r\nContent-Type: %s\r\nConnection: close\r\n\r\n", + body_len, content_type ? content_type : "application/json"); + mock_send_all(fd, header, strlen(header)); + if (body && body_len > 0) mock_send_all(fd, body, body_len); +} + +static void mock_send_status_response(int fd, const char *status, const char *content_type, const void *body, size_t body_len) { + char header[512]; + snprintf(header, sizeof(header), + "HTTP/1.1 %s\r\nContent-Length: %zu\r\nContent-Type: %s\r\nConnection: close\r\n\r\n", + status, body_len, content_type ? content_type : "application/json"); + mock_send_all(fd, header, strlen(header)); + if (body && body_len > 0) mock_send_all(fd, body, body_len); +} + +static void mock_send_text(int fd, const char *body) { + mock_send_response(fd, "application/json", body, strlen(body)); +} + +static void mock_absorb_request_body(int fd, const char *request, int received) { + const char *content_length = strstr(request, "Content-Length:"); + if (!content_length) content_length = strstr(request, "content-length:"); + const char *body = strstr(request, "\r\n\r\n"); + if (!content_length || !body) return; + long expected = strtol(content_length + strlen("Content-Length:"), NULL, 10); + long have = received - (long)((body + 4) - request); + char scratch[1024]; + while (have < expected) { + ssize_t n = recv(fd, scratch, sizeof(scratch), 0); + if (n <= 0) break; + have += (long)n; + } +} + +static void mock_handle_client(mock_network_server *server, int fd) { + char request[8192]; + int received = 0; + while (received < (int)sizeof(request) - 1) { + ssize_t n = recv(fd, request + received, sizeof(request) - 1 - (size_t)received, 0); + if (n <= 0) return; + received += (int)n; + request[received] = '\0'; + if (strstr(request, "\r\n\r\n")) break; + } + + char method[16] = {0}; + char path[1024] = {0}; + sscanf(request, "%15s %1023s", method, path); + mock_absorb_request_body(fd, request, received); + + if (server->stop) { + mock_send_text(fd, "{}"); + return; + } + + if (strstr(path, "/schema/check")) { + if (server->scenario == MOCK_INVALID_SCHEMA_CHECK) { + mock_send_text(fd, "not-json"); + } else if (server->scenario == MOCK_SCHEMA_CHECK_HTTP_EMPTY_ERROR) { + mock_send_status_response(fd, "503 Service Unavailable", "application/json", NULL, 0); + } else if (server->scenario == MOCK_FIRST_SCHEMA_SYNC) { + char *body = sqlite3_mprintf("{\"migration\":%s}", server->migration_json); + mock_send_text(fd, body ? body : "{}"); + sqlite3_free(body); + } else { + mock_send_text(fd, "{\"status\":\"none\"}"); + } + } else if (strstr(path, "/schema/download")) { + if (server->scenario == MOCK_INVALID_SCHEMA_DOWNLOAD) mock_send_text(fd, "not-json"); + else mock_send_text(fd, "{\"status\":\"none\"}"); + } else if (strstr(path, "/schema/upload")) { + if (server->scenario == MOCK_INVALID_SCHEMA_UPLOAD) mock_send_text(fd, "not-json"); + else if (server->scenario == MOCK_SCHEMA_UPLOAD_AUTH_ERROR) { + const char *body = "{\"error\":\"missing schema api key\"}"; + mock_send_status_response(fd, "403 Forbidden", "application/json", body, strlen(body)); + } else if (server->scenario == MOCK_SCHEMA_UPLOAD_MISSING_STATUS) { + mock_send_text(fd, "{}"); + } + else mock_send_text(fd, "{\"status\":\"uploaded\"}"); + } else if (strstr(path, "/blob-upload")) { + mock_send_text(fd, ""); + } else if (strstr(path, "/download")) { + mock_send_response(fd, "application/octet-stream", server->payload, server->payload_len); + } else if (strstr(path, "/status")) { + if (server->scenario == MOCK_INVALID_STATUS) mock_send_text(fd, "not-json"); + else mock_send_text(fd, "{\"lastOptimisticVersion\":0,\"lastConfirmedVersion\":0,\"gaps\":[]}"); + } else if (strstr(path, "/upload")) { + if (server->scenario == MOCK_INVALID_UPLOAD) { + mock_send_text(fd, "not-json"); + } else { + char body[256]; + snprintf(body, sizeof(body), "{\"url\":\"http://127.0.0.1:%d/blob-upload\"}", server->port); + mock_send_text(fd, body); + } + } else if (strstr(path, "/apply")) { + if (server->scenario == MOCK_INVALID_APPLY) mock_send_text(fd, "not-json"); + else mock_send_text(fd, "{\"lastOptimisticVersion\":1,\"lastConfirmedVersion\":1,\"gaps\":[]}"); + } else if (strstr(path, "/check")) { + if (server->scenario == MOCK_INVALID_CHECK) { + mock_send_text(fd, "not-json"); + } else if (server->scenario == MOCK_FIRST_SCHEMA_SYNC) { + char body[256]; + snprintf(body, sizeof(body), "{\"url\":\"http://127.0.0.1:%d/download\"}", server->port); + mock_send_text(fd, body); + } else { + mock_send_text(fd, "{\"lastOptimisticVersion\":0,\"lastConfirmedVersion\":0,\"gaps\":[]}"); + } + } else { + mock_send_text(fd, "{}"); + } +} + +static void *mock_network_worker(void *arg) { + mock_network_server *server = (mock_network_server *)arg; + while (!server->stop) { + int fd = accept(server->listen_fd, NULL, NULL); + if (fd < 0) continue; + mock_handle_client(server, fd); + close(fd); + } + return NULL; +} + +static int mock_network_start(mock_network_server *server, mock_network_scenario scenario) { + memset(server, 0, sizeof(*server)); + server->scenario = scenario; + server->listen_fd = socket(AF_INET, SOCK_STREAM, 0); + if (server->listen_fd < 0) return SQLITE_ERROR; + + int reuse = 1; + setsockopt(server->listen_fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + if (bind(server->listen_fd, (struct sockaddr *)&addr, sizeof(addr)) != 0) return SQLITE_ERROR; + if (listen(server->listen_fd, 16) != 0) return SQLITE_ERROR; + + socklen_t len = sizeof(addr); + if (getsockname(server->listen_fd, (struct sockaddr *)&addr, &len) != 0) return SQLITE_ERROR; + server->port = ntohs(addr.sin_port); + if (pthread_create(&server->thread, NULL, mock_network_worker, server) != 0) return SQLITE_ERROR; + return SQLITE_OK; +} + +static void mock_network_stop(mock_network_server *server) { + if (!server || server->listen_fd <= 0) return; + server->stop = 1; + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd >= 0) { + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = htons((uint16_t)server->port); + connect(fd, (struct sockaddr *)&addr, sizeof(addr)); + close(fd); + } + pthread_join(server->thread, NULL); + close(server->listen_fd); + server->listen_fd = -1; +} + +static int mock_network_init_db(sqlite3 *db, mock_network_server *server) { + char sql[256]; + snprintf(sql, sizeof(sql), "SELECT cloudsync_network_init_custom('http://127.0.0.1:%d', 'mockdb');", server->port); + return db_exec(db, sql); +} + +static int expect_sql_error_contains(sqlite3 *db, const char *sql, const char *expected) { + char *errmsg = NULL; + int rc = sqlite3_exec(db, sql, NULL, NULL, &errmsg); + if (rc == SQLITE_OK) { + printf("Error: expected SQL failure while executing %s\n", sql); + return SQLITE_ERROR; + } + if (expected && (!errmsg || !strstr(errmsg, expected))) { + printf("Error: expected message containing \"%s\", got \"%s\"\n", expected, errmsg ? errmsg : "NULL"); + sqlite3_free(errmsg); + return SQLITE_ERROR; + } + sqlite3_free(errmsg); + return SQLITE_OK; +} + +static int mock_prepare_synced_row(sqlite3 *db) { + int rc = db_exec(db, + "CREATE TABLE IF NOT EXISTS notes (id TEXT PRIMARY KEY NOT NULL, body TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('notes');" + "INSERT INTO notes (id, body) VALUES ('n1', 'hello');"); + return rc; +} + +static int mock_prepare_large_synced_row(sqlite3 *db) { + sqlite3_str *str = sqlite3_str_new(NULL); + if (!str) return SQLITE_NOMEM; + + sqlite3_str_appendall(str, "CREATE TABLE IF NOT EXISTS notes (id TEXT PRIMARY KEY NOT NULL, body TEXT NOT NULL DEFAULT ''"); + for (int i = 0; i < 90; ++i) { + sqlite3_str_appendf(str, ", extra_%02d TEXT", i); + } + sqlite3_str_appendall(str, + ");" + "SELECT cloudsync_init('notes');" + "INSERT INTO notes (id, body) VALUES ('n1', 'hello');"); + + char *sql = sqlite3_str_finish(str); + if (!sql) return SQLITE_NOMEM; + int rc = db_exec(db, sql); + sqlite3_free(sql); + return rc; +} + +static int select_payload_blob(sqlite3 *db, unsigned char **payload, int *payload_len) { + sqlite3_stmt *stmt = NULL; + const char *sql = "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes;"; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) return rc; + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { sqlite3_finalize(stmt); return SQLITE_ERROR; } + int len = sqlite3_column_bytes(stmt, 0); + const void *blob = sqlite3_column_blob(stmt, 0); + if (!blob || len <= 0) { sqlite3_finalize(stmt); return SQLITE_ERROR; } + unsigned char *copy = sqlite3_malloc(len); + if (!copy) { sqlite3_finalize(stmt); return SQLITE_NOMEM; } + memcpy(copy, blob, (size_t)len); + *payload = copy; + *payload_len = len; + sqlite3_finalize(stmt); + return SQLITE_OK; +} + +static char *mock_build_large_first_schema_migration(void) { + sqlite3_str *str = sqlite3_str_new(NULL); + if (!str) return NULL; + + sqlite3_str_appendall(str, + "{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mock-first-schema-sync-large\"," + "\"ops\":[" + "{\"op\":\"createTable\",\"table\":\"notes\",\"columns\":[" + "{\"name\":\"id\",\"type\":\"text\",\"primaryKey\":true,\"nullable\":false}," + "{\"name\":\"body\",\"type\":\"text\",\"nullable\":false,\"default\":{\"type\":\"text\",\"value\":\"\"}}"); + for (int i = 0; i < 90; ++i) { + sqlite3_str_appendf(str, ",{\"name\":\"extra_%02d\",\"type\":\"text\",\"nullable\":true}", i); + } + sqlite3_str_appendall(str, + "]}," + "{\"op\":\"augmentTable\",\"table\":\"notes\",\"algorithm\":\"CLS\",\"initFlags\":0}" + "]" + "}"); + + return sqlite3_str_finish(str); +} + +static int test_mock_network_json_validation_one(mock_network_scenario scenario, const char *sql, bool expect_error_json) { + mock_network_server server; + int rc = mock_network_start(&server, scenario); + if (rc != SQLITE_OK) return rc; + + sqlite3 *db = NULL; + rc = open_load_ext(":memory:", &db); + if (rc != SQLITE_OK) goto cleanup; + rc = mock_network_init_db(db, &server); + if (rc != SQLITE_OK) goto cleanup; + + if (scenario == MOCK_INVALID_UPLOAD || scenario == MOCK_INVALID_APPLY || scenario == MOCK_INVALID_CHECK) { + rc = mock_prepare_synced_row(db); + if (rc != SQLITE_OK) goto cleanup; + } + + if (expect_error_json) rc = db_expect_int(db, sql, 1); + else rc = expect_sql_error_contains(db, sql, "invalid JSON"); + +cleanup: + if (db) { + db_exec(db, "SELECT cloudsync_terminate();"); + sqlite3_close(db); + } + mock_network_stop(&server); + return rc; +} + +static int test_mock_network_json_validation(void) { + int rc = SQLITE_OK; + rc += test_mock_network_json_validation_one(MOCK_INVALID_STATUS, "SELECT cloudsync_network_status();", false); + rc += test_mock_network_json_validation_one(MOCK_INVALID_CHECK, "SELECT cloudsync_network_check_changes() LIKE '%invalid JSON%';", true); + rc += test_mock_network_json_validation_one(MOCK_INVALID_UPLOAD, "SELECT cloudsync_network_send_changes();", false); + rc += test_mock_network_json_validation_one(MOCK_INVALID_APPLY, "SELECT cloudsync_network_send_changes();", false); + rc += test_mock_network_json_validation_one(MOCK_INVALID_SCHEMA_CHECK, "SELECT cloudsync_network_migration_check();", false); + rc += test_mock_network_json_validation_one(MOCK_INVALID_SCHEMA_DOWNLOAD, "SELECT cloudsync_network_migration_download();", false); + rc += test_mock_network_json_validation_one(MOCK_INVALID_SCHEMA_UPLOAD, "SELECT cloudsync_network_migration_upload('{\"ops\":[]}');", false); + return rc == SQLITE_OK ? SQLITE_OK : SQLITE_ERROR; +} + +static int test_mock_schema_check_empty_error(void) { + mock_network_server server; + int rc = mock_network_start(&server, MOCK_SCHEMA_CHECK_HTTP_EMPTY_ERROR); + if (rc != SQLITE_OK) return rc; + + sqlite3 *db = NULL; + rc = open_load_ext(":memory:", &db); + if (rc != SQLITE_OK) goto cleanup; + rc = mock_network_init_db(db, &server); + if (rc != SQLITE_OK) goto cleanup; + + rc = expect_sql_error_contains(db, "SELECT cloudsync_network_migration_check();", "CloudSync schema migration endpoint failed"); + +cleanup: + if (db) { + db_exec(db, "SELECT cloudsync_terminate();"); + sqlite3_close(db); + } + mock_network_stop(&server); + return rc; +} + +static int test_mock_migration_upload_error_keeps_pending(void) { + mock_network_server server; + int rc = mock_network_start(&server, MOCK_SCHEMA_UPLOAD_AUTH_ERROR); + if (rc != SQLITE_OK) return rc; + + sqlite3 *db = NULL; + rc = open_load_ext(":memory:", &db); + if (rc != SQLITE_OK) goto cleanup; + rc = mock_network_init_db(db, &server); + if (rc != SQLITE_OK) goto cleanup; + + rc = db_exec(db, + "SELECT cloudsync_alter_create_table('upload_notes');" + "SELECT cloudsync_alter_add_column('upload_notes', 'id', 'text', 0);" + "SELECT cloudsync_alter_add_primary_key('upload_notes', 'id');" + "SELECT cloudsync_alter_augment_table('upload_notes');" + "SELECT cloudsync_alter_apply();"); + if (rc != SQLITE_OK) goto cleanup; + + rc = db_expect_int(db, "SELECT count(*) FROM cloudsync_pending_migration WHERE uploaded_at IS NULL;", 1); + if (rc != SQLITE_OK) goto cleanup; + rc = expect_sql_error_contains(db, "SELECT cloudsync_network_migration_upload();", NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = db_expect_int(db, "SELECT count(*) FROM cloudsync_pending_migration WHERE uploaded_at IS NULL;", 1); + if (rc != SQLITE_OK) goto cleanup; + rc = db_exec(db, "INSERT INTO upload_notes (id) VALUES ('u1');"); + if (rc != SQLITE_OK) goto cleanup; + rc = expect_sql_error_contains(db, "SELECT cloudsync_network_send_changes();", "pending schema migration"); + +cleanup: + if (db) { + db_exec(db, "SELECT cloudsync_terminate();"); + sqlite3_close(db); + } + mock_network_stop(&server); + return rc; +} + +static int test_mock_migration_upload_missing_status_keeps_pending(void) { + mock_network_server server; + int rc = mock_network_start(&server, MOCK_SCHEMA_UPLOAD_MISSING_STATUS); + if (rc != SQLITE_OK) return rc; + + sqlite3 *db = NULL; + rc = open_load_ext(":memory:", &db); + if (rc != SQLITE_OK) goto cleanup; + rc = mock_network_init_db(db, &server); + if (rc != SQLITE_OK) goto cleanup; + + rc = db_exec(db, + "SELECT cloudsync_alter_create_table('upload_missing_status_notes');" + "SELECT cloudsync_alter_add_column('upload_missing_status_notes', 'id', 'text', 0);" + "SELECT cloudsync_alter_add_primary_key('upload_missing_status_notes', 'id');" + "SELECT cloudsync_alter_augment_table('upload_missing_status_notes');" + "SELECT cloudsync_alter_apply();"); + if (rc != SQLITE_OK) goto cleanup; + + rc = db_expect_int(db, "SELECT count(*) FROM cloudsync_pending_migration WHERE uploaded_at IS NULL;", 1); + if (rc != SQLITE_OK) goto cleanup; + rc = expect_sql_error_contains(db, "SELECT cloudsync_network_migration_upload();", "accepted status"); + if (rc != SQLITE_OK) goto cleanup; + rc = db_expect_int(db, "SELECT count(*) FROM cloudsync_pending_migration WHERE uploaded_at IS NULL;", 1); + if (rc != SQLITE_OK) goto cleanup; + rc = db_exec(db, "INSERT INTO upload_missing_status_notes (id) VALUES ('u1');"); + if (rc != SQLITE_OK) goto cleanup; + rc = expect_sql_error_contains(db, "SELECT cloudsync_network_send_changes();", "pending schema migration"); + +cleanup: + if (db) { + db_exec(db, "SELECT cloudsync_terminate();"); + sqlite3_close(db); + } + mock_network_stop(&server); + return rc; +} + +static int test_mock_first_schema_sync(void) { + sqlite3 *source = NULL; + sqlite3 *target = NULL; + unsigned char *payload = NULL; + int payload_len = 0; + char *migration_json = NULL; + mock_network_server server; + int rc = open_load_ext(":memory:", &source); + if (rc != SQLITE_OK) goto cleanup; + rc = mock_prepare_large_synced_row(source); + if (rc != SQLITE_OK) goto cleanup; + rc = select_payload_blob(source, &payload, &payload_len); + if (rc != SQLITE_OK) goto cleanup; + + rc = mock_network_start(&server, MOCK_FIRST_SCHEMA_SYNC); + if (rc != SQLITE_OK) goto cleanup; + server.payload = payload; + server.payload_len = payload_len; + migration_json = mock_build_large_first_schema_migration(); + if (!migration_json || strlen(migration_json) < 4096) { rc = SQLITE_ERROR; goto cleanup_server; } + server.migration_json = migration_json; + + rc = open_load_ext(":memory:", &target); + if (rc != SQLITE_OK) goto cleanup_server; + rc = mock_network_init_db(target, &server); + if (rc != SQLITE_OK) goto cleanup_server; + rc = db_exec(target, "SELECT cloudsync_network_sync(10, 1);"); + if (rc != SQLITE_OK) goto cleanup_server; + rc = db_expect_int(target, "SELECT count(*) FROM notes WHERE id='n1' AND body='hello';", 1); + if (rc != SQLITE_OK) goto cleanup_server; + rc = db_expect_int(target, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id='mock-first-schema-sync-large';", 1); + +cleanup_server: + mock_network_stop(&server); +cleanup: + if (source) { db_exec(source, "SELECT cloudsync_terminate();"); sqlite3_close(source); } + if (target) { db_exec(target, "SELECT cloudsync_terminate();"); sqlite3_close(target); } + if (payload) sqlite3_free(payload); + if (migration_json) sqlite3_free(migration_json); + return rc; +} +#else +static int test_mock_network_json_validation(void) { + printf("Skipping local mock network JSON test on Windows.\n"); + return SQLITE_OK; +} + +static int test_mock_schema_check_empty_error(void) { + printf("Skipping local mock schema empty error test on Windows.\n"); + return SQLITE_OK; +} + +static int test_mock_migration_upload_error_keeps_pending(void) { + printf("Skipping local mock migration upload auth test on Windows.\n"); + return SQLITE_OK; +} + +static int test_mock_migration_upload_missing_status_keeps_pending(void) { + printf("Skipping local mock migration upload ack test on Windows.\n"); + return SQLITE_OK; +} + +static int test_mock_first_schema_sync(void) { + printf("Skipping local mock first schema sync test on Windows.\n"); + return SQLITE_OK; +} +#endif + // MARK: - int test_report(const char *description, int rc){ @@ -546,6 +1059,21 @@ int main (void) { printf("===========================================\n"); test_report("Version Test:", rc); + rc += test_report("Mock Network JSON Test:", test_mock_network_json_validation()); + rc += test_report("Mock Schema Empty Error:", test_mock_schema_check_empty_error()); + rc += test_report("Mock Migration Upload Auth:", test_mock_migration_upload_error_keeps_pending()); + rc += test_report("Mock Migration Upload Ack:", test_mock_migration_upload_missing_status_keeps_pending()); + rc += test_report("Mock First Schema Sync Test:", test_mock_first_schema_sync()); + rc += test_report("Double Empty Init Test:", test_double_empty_network_init(":memory:")); + + if (!getenv("INTEGRATION_TEST_DATABASE_ID")) { + printf("Skipping remote integration tests: INTEGRATION_TEST_DATABASE_ID not set.\n"); + remove(DB_PATH); + cloudsync_memory_finalize(); + printf("\n"); + return rc; + } + sqlite3 *db = NULL; rc += open_load_ext(DB_PATH, &db); rc += db_init(db); @@ -556,7 +1084,6 @@ int main (void) { rc += test_report("DB Version Test:", test_db_version(DB_PATH)); rc += test_report("Enable Disable Test:", test_enable_disable(DB_PATH)); rc += test_report("Offline Error Test:", test_offline_error(":memory:")); - rc += test_report("Double Empty Init Test:", test_double_empty_network_init(":memory:")); remove(DB_PATH); // remove the database file @@ -652,4 +1179,4 @@ int main (void) { printf("\n"); return rc; -} \ No newline at end of file +} diff --git a/test/postgresql/31_alter_table_sync.sql b/test/postgresql/31_alter_table_sync.sql index 365c3568..5b798388 100644 --- a/test/postgresql/31_alter_table_sync.sql +++ b/test/postgresql/31_alter_table_sync.sql @@ -1,5 +1,5 @@ -- Alter Table Sync Test --- Tests cloudsync_begin_alter and cloudsync_commit_alter functions. +-- Tests declarative cloudsync_alter_* schema migration functions. -- Verifies that schema changes (add column) are handled correctly -- and data syncs after alteration. @@ -106,7 +106,7 @@ SELECT (:fail::int + 1) AS fail \gset \endif -- ============================================================================ --- ALTER TABLE on Database A (begin_alter + ALTER + commit_alter on SAME connection) +-- ALTER TABLE on Database A through declarative migration API -- ============================================================================ \echo [INFO] (:testid) === ALTER TABLE on Database A === @@ -115,21 +115,19 @@ SELECT (:fail::int + 1) AS fail \gset \ir helper_psql_conn_setup.sql SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset -SELECT cloudsync_begin_alter('products') AS begin_alter_a \gset -\if :begin_alter_a -\echo [PASS] (:testid) cloudsync_begin_alter succeeded on Database A +SELECT cloudsync_alter_add_column('products', 'description', 'text', false, '') AS alter_add_a \gset +\if :alter_add_a +\echo [PASS] (:testid) cloudsync_alter_add_column queued on Database A \else -\echo [FAIL] (:testid) cloudsync_begin_alter failed on Database A +\echo [FAIL] (:testid) cloudsync_alter_add_column failed on Database A SELECT (:fail::int + 1) AS fail \gset \endif -ALTER TABLE products ADD COLUMN description TEXT NOT NULL DEFAULT ''; - -SELECT cloudsync_commit_alter('products') AS commit_alter_a \gset -\if :commit_alter_a -\echo [PASS] (:testid) cloudsync_commit_alter succeeded on Database A +SELECT cloudsync_alter_apply() IS NOT NULL AS alter_apply_a \gset +\if :alter_apply_a +\echo [PASS] (:testid) cloudsync_alter_apply succeeded on Database A \else -\echo [FAIL] (:testid) cloudsync_commit_alter failed on Database A +\echo [FAIL] (:testid) cloudsync_alter_apply failed on Database A SELECT (:fail::int + 1) AS fail \gset \endif @@ -157,7 +155,7 @@ SELECT (:fail::int + 1) AS fail \gset \endif -- ============================================================================ --- ALTER TABLE on Database B (begin_alter + ALTER + commit_alter on SAME connection) +-- ALTER TABLE on Database B through declarative migration API -- Apply A's payload, insert/update, encode B's payload -- ============================================================================ @@ -167,21 +165,19 @@ SELECT (:fail::int + 1) AS fail \gset \ir helper_psql_conn_setup.sql SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset -SELECT cloudsync_begin_alter('products') AS begin_alter_b \gset -\if :begin_alter_b -\echo [PASS] (:testid) cloudsync_begin_alter succeeded on Database B +SELECT cloudsync_alter_add_column('products', 'description', 'text', false, '') AS alter_add_b \gset +\if :alter_add_b +\echo [PASS] (:testid) cloudsync_alter_add_column queued on Database B \else -\echo [FAIL] (:testid) cloudsync_begin_alter failed on Database B +\echo [FAIL] (:testid) cloudsync_alter_add_column failed on Database B SELECT (:fail::int + 1) AS fail \gset \endif -ALTER TABLE products ADD COLUMN description TEXT NOT NULL DEFAULT ''; - -SELECT cloudsync_commit_alter('products') AS commit_alter_b \gset -\if :commit_alter_b -\echo [PASS] (:testid) cloudsync_commit_alter succeeded on Database B +SELECT cloudsync_alter_apply() IS NOT NULL AS alter_apply_b \gset +\if :alter_apply_b +\echo [PASS] (:testid) cloudsync_alter_apply succeeded on Database B \else -\echo [FAIL] (:testid) cloudsync_commit_alter failed on Database B +\echo [FAIL] (:testid) cloudsync_alter_apply failed on Database B SELECT (:fail::int + 1) AS fail \gset \endif diff --git a/test/postgresql/52_schema_migrations.sql b/test/postgresql/52_schema_migrations.sql new file mode 100644 index 00000000..edb87183 --- /dev/null +++ b/test/postgresql/52_schema_migrations.sql @@ -0,0 +1,1509 @@ +-- Schema migration payload tests +-- Covers local V1/V2 migration application for PostgreSQL. + +\set testid '52' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_test_52; +CREATE DATABASE cloudsync_test_52; + +\connect cloudsync_test_52 +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- ============================================================================ +-- V1: create table + augment + block-level LWW +-- ============================================================================ + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-v1-create", + "requiredCapabilities": ["schema:write"], + "ops": [ + { + "op": "createTable", + "table": "notes", + "columns": [ + {"name": "id", "type": "uuid", "primaryKey": true, "nullable": false}, + {"name": "body", "type": "text", "nullable": false, "default": {"type": "text", "value": ""}} + ] + }, + {"op": "augmentTable", "table": "notes", "algorithm": "CLS", "initFlags": 0}, + {"op": "setBlockLww", "table": "notes", "column": "body", "delimiter": "\n"} + ] +} +$json$) AS migration_v1_create \gset + +INSERT INTO notes (id, body) +VALUES ('11111111-1111-1111-1111-111111111111', E'a\nb\nc'); + +SELECT count(*) = 3 AS v1_blocks_ok +FROM notes_cloudsync_blocks +WHERE pk = cloudsync_pk_encode('11111111-1111-1111-1111-111111111111'::uuid) \gset + +SELECT is_nullable = 'NO' AS v1_pk_notnull_ok +FROM information_schema.columns +WHERE table_name = 'notes' AND column_name = 'id' \gset + +\if :v1_blocks_ok +\if :v1_pk_notnull_ok +\echo [PASS] (:testid) V1 create/augment/block migration created block metadata +\else +\echo [FAIL] (:testid) V1 create migration did not mark the primary key NOT NULL +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V1 create/augment/block migration did not create expected block metadata +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V1: add column + idempotency +-- ============================================================================ + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-v1-add-column", + "requiredCapabilities": ["schema:write"], + "ops": [ + { + "op": "addColumn", + "table": "notes", + "column": { + "name": "subtitle", + "type": "text", + "nullable": false, + "default": {"type": "text", "value": ""} + } + } + ] +} +$json$) AS migration_v1_add \gset + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-v1-add-column", + "requiredCapabilities": ["schema:write"], + "ops": [ + { + "op": "addColumn", + "table": "notes", + "column": { + "name": "subtitle", + "type": "text", + "nullable": false, + "default": {"type": "text", "value": ""} + } + } + ] +} +$json$) AS migration_v1_add_again \gset + +SELECT count(*) = 1 AS v1_add_column_ok +FROM information_schema.columns +WHERE table_name = 'notes' AND column_name = 'subtitle' \gset + +SELECT count(*) = 1 AS v1_idempotent_ok +FROM cloudsync_migrations +WHERE migration_id = 'mig-pg-v1-add-column' \gset + +\if :v1_add_column_ok +\if :v1_idempotent_ok +\echo [PASS] (:testid) V1 addColumn migration is idempotent +\else +\echo [FAIL] (:testid) V1 addColumn migration idempotency record mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V1 addColumn migration did not add subtitle +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V1: large payload requires dynamically sized JSON token parsing +-- ============================================================================ + +WITH cols AS ( + SELECT 0 AS ord, jsonb_build_object( + 'name', 'id', + 'type', 'uuid', + 'primaryKey', true, + 'nullable', false + ) AS col + UNION ALL + SELECT gs + 1 AS ord, jsonb_build_object( + 'name', format('extra_%s', lpad(gs::text, 3, '0')), + 'type', 'text', + 'nullable', true + ) AS col + FROM generate_series(0, 699) AS gs +), payload AS ( + SELECT jsonb_build_object( + 'type', 'cloudsync.schema.migration', + 'formatVersion', 1, + 'migrationId', '0197097c-8b35-7c11-8ed4-4e59ddfdb929', + 'requiredCapabilities', jsonb_build_array('schema:write'), + 'ops', jsonb_build_array(jsonb_build_object( + 'op', 'createTable', + 'table', 'pg_wide_dynamic_notes', + 'columns', (SELECT jsonb_agg(col ORDER BY ord) FROM cols) + )) + ) AS doc +) +SELECT cloudsync_migration_apply(doc::text) AS migration_large_json +FROM payload \gset + +SELECT count(*) = 701 AS large_json_columns_ok +FROM information_schema.columns +WHERE table_name = 'pg_wide_dynamic_notes' \gset + +SELECT count(*) = 1 AS large_json_record_ok +FROM cloudsync_migrations +WHERE migration_id = '0197097c-8b35-7c11-8ed4-4e59ddfdb929' \gset + +\if :large_json_columns_ok +\if :large_json_record_ok +\echo [PASS] (:testid) Large migration JSON payload parsed dynamically +\else +\echo [FAIL] (:testid) Large migration JSON payload was not recorded +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Large migration JSON payload did not create all columns +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V2: metadata-preserving renameColumn +-- ============================================================================ + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 2, + "migrationId": "mig-pg-v2-rename-column", + "requiredCapabilities": ["schema:destructive"], + "ops": [ + {"op": "renameColumn", "table": "notes", "from": "subtitle", "to": "summary"} + ] +} +$json$) AS migration_v2_rename \gset + +UPDATE notes +SET summary = 'renamed column works' +WHERE id = '11111111-1111-1111-1111-111111111111'; + +SELECT count(*) = 1 AS v2_rename_col_ok +FROM information_schema.columns +WHERE table_name = 'notes' AND column_name = 'summary' \gset + +SELECT count(*) = 0 AS v2_rename_old_meta_ok +FROM notes_cloudsync +WHERE col_name = 'subtitle' \gset + +\if :v2_rename_col_ok +\if :v2_rename_old_meta_ok +\echo [PASS] (:testid) V2 renameColumn migration updated schema and metadata +\else +\echo [FAIL] (:testid) V2 renameColumn migration left old metadata +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 renameColumn migration did not rename column +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V2: rebuildTableSync with destructive DDL +-- ============================================================================ + +SELECT cloudsync_cleanup('notes') AS _cleanup_notes \gset + +CREATE TABLE rebuild_docs ( + id UUID PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + legacy TEXT NOT NULL DEFAULT '' +); +SELECT cloudsync_init('rebuild_docs', 'CLS', 0) AS _init_rebuild \gset +INSERT INTO rebuild_docs VALUES ('22222222-2222-2222-2222-222222222222', 'title', 'legacy'); + +SELECT encode(site_id, 'hex') AS rebuild_site_before +FROM cloudsync_site_id +WHERE id = 0 \gset + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 2, + "migrationId": "mig-pg-v2-rebuild", + "requiredCapabilities": ["schema:destructive"], + "ops": [ + { + "op": "rebuildTableSync", + "table": "rebuild_docs", + "algorithm": "CLS", + "initFlags": 0, + "ddl": [ + {"op": "rawSql", "sql": {"postgresql": ["ALTER TABLE rebuild_docs DROP COLUMN legacy"]}} + ] + } + ] +} +$json$) AS migration_v2_rebuild \gset + +INSERT INTO rebuild_docs (id, title) +VALUES ('33333333-3333-3333-3333-333333333333', 'new'); + +SELECT count(*) = 0 AS v2_rebuild_drop_ok +FROM information_schema.columns +WHERE table_name = 'rebuild_docs' AND column_name = 'legacy' \gset + +SELECT count(*) = 1 AS v2_rebuild_meta_ok +FROM information_schema.tables +WHERE table_name = 'rebuild_docs_cloudsync' \gset + +SELECT encode(site_id, 'hex') = :'rebuild_site_before' AS v2_rebuild_site_ok +FROM cloudsync_site_id +WHERE id = 0 \gset + +\if :v2_rebuild_drop_ok +\if :v2_rebuild_meta_ok +\if :v2_rebuild_site_ok +\echo [PASS] (:testid) V2 rebuildTableSync migration dropped column and recreated metadata +\else +\echo [FAIL] (:testid) V2 rebuildTableSync migration changed site identity +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 rebuildTableSync migration did not recreate metadata +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 rebuildTableSync migration did not drop legacy column +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V2: reject malformed rebuildTableSync DDL payloads +-- ============================================================================ + +CREATE TABLE bad_rebuild_docs ( + id UUID PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + legacy TEXT NOT NULL DEFAULT '' +); +SELECT cloudsync_init('bad_rebuild_docs', 'CLS', 0) AS _init_bad_rebuild \gset +INSERT INTO bad_rebuild_docs VALUES ('44444444-4444-4444-4444-444444444444', 'title', 'legacy'); + +CREATE TEMP TABLE schema_migration_error_flags ( + key TEXT PRIMARY KEY, + ok BOOLEAN NOT NULL +) ON COMMIT PRESERVE ROWS; + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 2, + "migrationId": "mig-pg-v2-rebuild-bad-ddl", + "requiredCapabilities": ["schema:destructive"], + "ops": [ + { + "op": "rebuildTableSync", + "table": "bad_rebuild_docs", + "algorithm": "CLS", + "initFlags": 0, + "ddl": {"op": "rawSql", "sql": {"postgresql": ["ALTER TABLE bad_rebuild_docs DROP COLUMN legacy"]}} + } + ] +} +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + + INSERT INTO schema_migration_error_flags VALUES ('bad_rebuild_ddl_rejected', rejected); +END +$$; + +SELECT ok AS v2_rebuild_bad_ddl_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'bad_rebuild_ddl_rejected' \gset + +SELECT count(*) = 0 AS v2_rebuild_bad_ddl_unrecorded_ok +FROM cloudsync_migrations +WHERE migration_id = 'mig-pg-v2-rebuild-bad-ddl' \gset + +SELECT count(*) = 1 AS v2_rebuild_bad_ddl_legacy_ok +FROM information_schema.columns +WHERE table_name = 'bad_rebuild_docs' AND column_name = 'legacy' \gset + +\if :v2_rebuild_bad_ddl_rejected_ok +\if :v2_rebuild_bad_ddl_unrecorded_ok +\if :v2_rebuild_bad_ddl_legacy_ok +\echo [PASS] (:testid) V2 rebuildTableSync rejected malformed DDL payload +\else +\echo [FAIL] (:testid) V2 malformed rebuildTableSync DDL altered the table +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 malformed rebuildTableSync DDL was recorded +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 malformed rebuildTableSync DDL was accepted +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V2: reject malformed rebuildTableSync blockLww payloads +-- ============================================================================ + +CREATE TABLE bad_block_docs ( + id UUID PRIMARY KEY, + body TEXT NOT NULL DEFAULT '' +); +SELECT cloudsync_init('bad_block_docs', 'CLS', 0) AS _init_bad_block \gset +INSERT INTO bad_block_docs VALUES ('55555555-5555-5555-5555-555555555555', E'line1\nline2'); + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 2, + "migrationId": "mig-pg-v2-rebuild-bad-block", + "requiredCapabilities": ["schema:destructive"], + "ops": [ + { + "op": "rebuildTableSync", + "table": "bad_block_docs", + "algorithm": "CLS", + "initFlags": 0, + "blockLww": {"column": "body", "delimiter": "\n"} + } + ] +} +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + + INSERT INTO schema_migration_error_flags VALUES ('bad_rebuild_block_rejected', rejected); +END +$$; + +SELECT ok AS v2_rebuild_bad_block_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'bad_rebuild_block_rejected' \gset + +SELECT count(*) = 0 AS v2_rebuild_bad_block_unrecorded_ok +FROM cloudsync_migrations +WHERE migration_id = 'mig-pg-v2-rebuild-bad-block' \gset + +SELECT count(*) = 1 AS v2_rebuild_bad_block_meta_ok +FROM information_schema.tables +WHERE table_name = 'bad_block_docs_cloudsync' \gset + +\if :v2_rebuild_bad_block_rejected_ok +\if :v2_rebuild_bad_block_unrecorded_ok +\if :v2_rebuild_bad_block_meta_ok +\echo [PASS] (:testid) V2 rebuildTableSync rejected malformed blockLww payload +\else +\echo [FAIL] (:testid) V2 malformed rebuildTableSync blockLww dropped metadata +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 malformed rebuildTableSync blockLww was recorded +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 malformed rebuildTableSync blockLww was accepted +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Reject migration payloads with trailing JSON data +-- ============================================================================ + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-invalid-json", + "ops": [ + { + "op": "createTable", + "table": "invalid_json_notes", + "columns": [ + {"name": "id", "type": "uuid", "primaryKey": true, "nullable": false} + ] + } + ] +} +true +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + + INSERT INTO schema_migration_error_flags VALUES ('invalid_json_rejected', rejected); +END +$$; + +SELECT ok AS invalid_json_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'invalid_json_rejected' \gset + +SELECT count(*) = 0 AS invalid_json_table_absent_ok +FROM information_schema.tables +WHERE table_name = 'invalid_json_notes' \gset + +SELECT count(*) = 0 AS invalid_json_unrecorded_ok +FROM cloudsync_migrations +WHERE migration_id = 'mig-pg-invalid-json' \gset + +\if :invalid_json_rejected_ok +\if :invalid_json_table_absent_ok +\if :invalid_json_unrecorded_ok +\echo [PASS] (:testid) Invalid migration JSON payload was rejected before applying +\else +\echo [FAIL] (:testid) Invalid migration JSON payload was recorded +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Invalid migration JSON payload altered the schema +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Invalid migration JSON payload was accepted +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Reject malformed column boolean flags +-- ============================================================================ + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-bad-bool-pk", + "ops": [ + { + "op": "createTable", + "table": "bad_bool_pk_notes", + "columns": [ + {"name": "id", "type": "uuid", "primaryKey": "true", "nullable": false} + ] + } + ] +} +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + + INSERT INTO schema_migration_error_flags VALUES ('bad_bool_pk_rejected', rejected); +END +$$; + +CREATE TABLE bad_bool_add_notes ( + id UUID PRIMARY KEY, + title TEXT NOT NULL DEFAULT '' +); +SELECT cloudsync_init('bad_bool_add_notes', 'CLS', 0) AS _bad_bool_init \gset + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-bad-bool-nullable", + "ops": [ + { + "op": "addColumn", + "table": "bad_bool_add_notes", + "column": { + "name": "summary", + "type": "text", + "nullable": "false", + "default": {"type": "text", "value": ""} + } + } + ] +} +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + + INSERT INTO schema_migration_error_flags VALUES ('bad_bool_nullable_rejected', rejected); +END +$$; + +SELECT ok AS bad_bool_pk_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'bad_bool_pk_rejected' \gset + +SELECT ok AS bad_bool_nullable_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'bad_bool_nullable_rejected' \gset + +SELECT count(*) = 0 AS bad_bool_pk_table_absent_ok +FROM information_schema.tables +WHERE table_name = 'bad_bool_pk_notes' \gset + +SELECT count(*) = 0 AS bad_bool_add_column_absent_ok +FROM information_schema.columns +WHERE table_name = 'bad_bool_add_notes' AND column_name = 'summary' \gset + +SELECT count(*) = 0 AS bad_bool_unrecorded_ok +FROM cloudsync_migrations +WHERE migration_id IN ('mig-pg-bad-bool-pk', 'mig-pg-bad-bool-nullable') \gset + +\if :bad_bool_pk_rejected_ok +\if :bad_bool_nullable_rejected_ok +\if :bad_bool_pk_table_absent_ok +\if :bad_bool_add_column_absent_ok +\if :bad_bool_unrecorded_ok +\echo [PASS] (:testid) Malformed column boolean flags were rejected +\else +\echo [FAIL] (:testid) Malformed column boolean migration was recorded +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Malformed nullable flag added a column +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Malformed primaryKey flag created a table +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Malformed nullable flag was accepted +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Malformed primaryKey flag was accepted +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V2: rebuildTableSync with valid blockLww array +-- ============================================================================ + +CREATE TABLE rebuild_block_docs ( + id UUID PRIMARY KEY, + body TEXT NOT NULL DEFAULT '' +); +SELECT cloudsync_init('rebuild_block_docs', 'CLS', 0) AS _init_rebuild_block \gset +INSERT INTO rebuild_block_docs VALUES ('66666666-6666-6666-6666-666666666666', E'one\ntwo'); + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 2, + "migrationId": "mig-pg-v2-rebuild-good-block", + "requiredCapabilities": ["schema:destructive"], + "ops": [ + { + "op": "rebuildTableSync", + "table": "rebuild_block_docs", + "algorithm": "CLS", + "initFlags": 0, + "blockLww": [{"column": "body", "delimiter": "\n"}] + } + ] +} +$json$) AS migration_v2_rebuild_good_block \gset + +UPDATE rebuild_block_docs +SET body = E'one\ntwo\nthree' +WHERE id = '66666666-6666-6666-6666-666666666666'; + +SELECT count(*) = 3 AS v2_rebuild_good_block_blocks_ok +FROM rebuild_block_docs_cloudsync_blocks \gset + +SELECT count(*) = 1 AS v2_rebuild_good_block_setting_ok +FROM cloudsync_table_settings +WHERE tbl_name = 'rebuild_block_docs' AND col_name = 'body' AND key = 'algo' AND value = 'block' \gset + +\if :v2_rebuild_good_block_blocks_ok +\if :v2_rebuild_good_block_setting_ok +\echo [PASS] (:testid) V2 rebuildTableSync applied valid blockLww array +\else +\echo [FAIL] (:testid) V2 rebuildTableSync blockLww did not persist setting +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 rebuildTableSync blockLww did not create expected blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V2: rename block-level LWW column metadata +-- ============================================================================ + +CREATE TABLE rename_block_docs ( + id UUID PRIMARY KEY, + body TEXT NOT NULL DEFAULT '' +); +SELECT cloudsync_init('rename_block_docs', 'CLS', 0) AS _init_rename_block \gset +SELECT cloudsync_set_column('rename_block_docs', 'body', 'algo', 'block') AS _set_rename_block \gset +INSERT INTO rename_block_docs VALUES ('77777777-7777-7777-7777-777777777777', E'a\nb'); + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 2, + "migrationId": "mig-pg-v2-rename-block", + "requiredCapabilities": ["schema:destructive"], + "ops": [ + {"op": "renameColumn", "table": "rename_block_docs", "from": "body", "to": "content"} + ] +} +$json$) AS migration_v2_rename_block \gset + +UPDATE rename_block_docs +SET content = E'a\nb\nc' +WHERE id = '77777777-7777-7777-7777-777777777777'; + +SELECT count(*) = 1 AS v2_rename_block_col_ok +FROM information_schema.columns +WHERE table_name = 'rename_block_docs' AND column_name = 'content' \gset + +SELECT count(*) = 0 AS v2_rename_block_old_meta_ok +FROM rename_block_docs_cloudsync +WHERE col_name = 'body' OR col_name LIKE 'body' || chr(31) || '%' \gset + +SELECT count(*) > 0 AS v2_rename_block_new_meta_ok +FROM rename_block_docs_cloudsync +WHERE col_name = 'content' OR col_name LIKE 'content' || chr(31) || '%' \gset + +SELECT count(*) = 0 AS v2_rename_block_old_blocks_ok +FROM rename_block_docs_cloudsync_blocks +WHERE col_name = 'body' OR col_name LIKE 'body' || chr(31) || '%' \gset + +SELECT count(*) > 0 AS v2_rename_block_new_blocks_ok +FROM rename_block_docs_cloudsync_blocks +WHERE col_name = 'content' OR col_name LIKE 'content' || chr(31) || '%' \gset + +SELECT count(*) = 1 AS v2_rename_block_setting_ok +FROM cloudsync_table_settings +WHERE tbl_name = 'rename_block_docs' AND col_name = 'content' AND key = 'algo' AND value = 'block' \gset + +\if :v2_rename_block_col_ok +\if :v2_rename_block_old_meta_ok +\if :v2_rename_block_new_meta_ok +\if :v2_rename_block_old_blocks_ok +\if :v2_rename_block_new_blocks_ok +\if :v2_rename_block_setting_ok +\echo [PASS] (:testid) V2 renameColumn migration updated block metadata and settings +\else +\echo [FAIL] (:testid) V2 renameColumn block setting was not renamed +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 renameColumn block metadata missing new block entries +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 renameColumn block metadata left old block entries +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 renameColumn block metadata missing new meta entries +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 renameColumn block metadata left old meta entries +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 renameColumn block column was not renamed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V2: direct dropColumn operation +-- ============================================================================ + +CREATE TABLE direct_drop_docs ( + id UUID PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + legacy TEXT NOT NULL DEFAULT '' +); +SELECT cloudsync_init('direct_drop_docs', 'CLS', 0) AS _init_direct_drop \gset +INSERT INTO direct_drop_docs VALUES ('88888888-8888-8888-8888-888888888888', 'title', 'legacy'); + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 2, + "migrationId": "mig-pg-v2-drop-column", + "requiredCapabilities": ["schema:destructive"], + "ops": [ + {"op": "dropColumn", "table": "direct_drop_docs", "column": "legacy"} + ] +} +$json$) AS migration_v2_drop_column \gset + +INSERT INTO direct_drop_docs (id, title) +VALUES ('99999999-9999-9999-9999-999999999999', 'new'); + +SELECT count(*) = 0 AS v2_drop_column_absent_ok +FROM information_schema.columns +WHERE table_name = 'direct_drop_docs' AND column_name = 'legacy' \gset + +SELECT count(*) = 0 AS v2_drop_column_meta_ok +FROM direct_drop_docs_cloudsync +WHERE col_name = 'legacy' \gset + +SELECT count(*) = 2 AS v2_drop_column_rows_ok +FROM direct_drop_docs \gset + +\if :v2_drop_column_absent_ok +\if :v2_drop_column_meta_ok +\if :v2_drop_column_rows_ok +\echo [PASS] (:testid) V2 dropColumn migration dropped schema and metadata +\else +\echo [FAIL] (:testid) V2 dropColumn migration left wrong row count +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 dropColumn migration left old metadata +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 dropColumn migration did not drop column +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V2: drop block-level LWW column cleans table-local metadata +-- ============================================================================ + +CREATE TABLE drop_block_docs ( + id UUID PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '' +); +SELECT cloudsync_init('drop_block_docs', 'CLS', 0) AS _init_drop_block \gset +SELECT cloudsync_set_column('drop_block_docs', 'body', 'algo', 'block') AS _set_drop_block \gset +INSERT INTO drop_block_docs VALUES ('abababab-abab-abab-abab-abababababab', 'title', E'a\nb'); + +SELECT count(*) > 0 AS v2_drop_block_before_blocks_ok +FROM drop_block_docs_cloudsync_blocks +WHERE col_name LIKE 'body' || chr(31) || '%' \gset + +SELECT count(*) > 0 AS v2_drop_block_before_settings_ok +FROM cloudsync_table_settings +WHERE tbl_name = 'drop_block_docs' AND col_name = 'body' \gset + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 2, + "migrationId": "mig-pg-v2-drop-block-column", + "requiredCapabilities": ["schema:destructive"], + "ops": [ + {"op": "dropColumn", "table": "drop_block_docs", "column": "body"} + ] +} +$json$) AS migration_v2_drop_block_column \gset + +UPDATE drop_block_docs +SET title = 'changed' +WHERE id = 'abababab-abab-abab-abab-abababababab'; + +SELECT count(*) = 0 AS v2_drop_block_column_absent_ok +FROM information_schema.columns +WHERE table_name = 'drop_block_docs' AND column_name = 'body' \gset + +SELECT count(*) = 0 AS v2_drop_block_meta_ok +FROM drop_block_docs_cloudsync +WHERE col_name = 'body' OR col_name LIKE 'body' || chr(31) || '%' \gset + +SELECT count(*) = 0 AS v2_drop_block_rows_ok +FROM drop_block_docs_cloudsync_blocks +WHERE col_name LIKE 'body' || chr(31) || '%' \gset + +SELECT count(*) = 0 AS v2_drop_block_settings_ok +FROM cloudsync_table_settings +WHERE tbl_name = 'drop_block_docs' AND col_name = 'body' \gset + +\if :v2_drop_block_before_blocks_ok +\if :v2_drop_block_before_settings_ok +\if :v2_drop_block_column_absent_ok +\if :v2_drop_block_meta_ok +\if :v2_drop_block_rows_ok +\if :v2_drop_block_settings_ok +\echo [PASS] (:testid) V2 dropColumn cleaned block metadata and settings +\else +\echo [FAIL] (:testid) V2 dropColumn left block column settings +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 dropColumn left block table rows +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 dropColumn left block metadata rows +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V2 dropColumn did not drop block column +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Test setup did not persist block column settings +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Test setup did not create block rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- V1: setFilter and setColumn operations +-- ============================================================================ + +CREATE TABLE filtered_tasks ( + id UUID PRIMARY KEY, + owner TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '' +); +SELECT cloudsync_init('filtered_tasks', 'CLS', 0) AS _init_filtered \gset + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-v1-filter-column", + "requiredCapabilities": ["schema:write"], + "ops": [ + {"op": "setColumn", "table": "filtered_tasks", "column": "title", "key": "label", "value": "sync-title"}, + {"op": "setFilter", "table": "filtered_tasks", "filter": "owner = 'alice'"} + ] +} +$json$) AS migration_v1_filter_column \gset + +INSERT INTO filtered_tasks VALUES ('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'alice', 'visible'); +INSERT INTO filtered_tasks VALUES ('bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'bob', 'hidden'); + +SELECT count(*) = 1 AS v1_set_column_ok +FROM cloudsync_table_settings +WHERE tbl_name = 'filtered_tasks' AND col_name = 'title' AND key = 'label' AND value = 'sync-title' \gset + +SELECT count(*) = 1 AS v1_set_filter_ok +FROM cloudsync_table_settings +WHERE tbl_name = 'filtered_tasks' AND col_name = '*' AND key = 'filter' AND value = 'owner = ''alice''' \gset + +SELECT count(*) > 0 AS v1_filter_alice_ok +FROM filtered_tasks_cloudsync +WHERE pk = cloudsync_pk_encode('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'::uuid) \gset + +SELECT count(*) = 0 AS v1_filter_bob_ok +FROM filtered_tasks_cloudsync +WHERE pk = cloudsync_pk_encode('bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb'::uuid) \gset + +\if :v1_set_column_ok +\if :v1_set_filter_ok +\if :v1_filter_alice_ok +\if :v1_filter_bob_ok +\echo [PASS] (:testid) V1 setFilter and setColumn migration applied settings and triggers +\else +\echo [FAIL] (:testid) V1 setFilter tracked non-matching row +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V1 setFilter did not track matching row +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V1 setFilter setting missing +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) V1 setColumn setting missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Declarative alter rejects explicit NULL defaults for NOT NULL columns +-- ============================================================================ + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_alter_add_column('filtered_tasks', 'required_null', 'text', false, NULL); + PERFORM cloudsync_alter_apply(); + EXCEPTION WHEN OTHERS THEN + rejected := true; + PERFORM cloudsync_alter_clear('filtered_tasks'); + END; + + INSERT INTO schema_migration_error_flags VALUES ('alter_required_null_rejected', rejected); +END +$$; + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_alter_add_column('filtered_tasks', 'required_null_override', 'text', false, ''); + PERFORM cloudsync_alter_add_column_postgresql('filtered_tasks', 'required_null_override', 'TEXT', false, NULL); + PERFORM cloudsync_alter_apply(); + EXCEPTION WHEN OTHERS THEN + rejected := true; + PERFORM cloudsync_alter_clear('filtered_tasks'); + END; + + INSERT INTO schema_migration_error_flags VALUES ('alter_required_null_override_rejected', rejected); +END +$$; + +SELECT ok AS alter_required_null_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'alter_required_null_rejected' \gset + +SELECT ok AS alter_required_null_override_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'alter_required_null_override_rejected' \gset + +SELECT count(*) = 0 AS alter_required_null_absent_ok +FROM information_schema.columns +WHERE table_name = 'filtered_tasks' AND column_name IN ('required_null', 'required_null_override') \gset + +\if :alter_required_null_rejected_ok +\if :alter_required_null_override_rejected_ok +\if :alter_required_null_absent_ok +\echo [PASS] (:testid) Declarative alter rejected NOT NULL column with NULL default +\else +\echo [FAIL] (:testid) Declarative alter NULL default changed schema +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative alter accepted dialect override with NULL default +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative alter accepted NOT NULL column with NULL default +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Hash guards reject and roll back migrations +-- ============================================================================ + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-bad-base", + "baseSchemaHash": "1", + "ops": [ + {"op": "addColumn", "table": "filtered_tasks", "column": {"name": "base_fail", "type": "text", "nullable": true}} + ] +} +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + INSERT INTO schema_migration_error_flags VALUES ('bad_base_rejected', rejected); +END +$$; + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-bad-target", + "targetSchemaHash": "1", + "ops": [ + {"op": "addColumn", "table": "filtered_tasks", "column": {"name": "target_fail", "type": "text", "nullable": true}} + ] +} +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + INSERT INTO schema_migration_error_flags VALUES ('bad_target_rejected', rejected); +END +$$; + +SELECT ok AS hash_bad_base_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'bad_base_rejected' \gset + +SELECT ok AS hash_bad_target_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'bad_target_rejected' \gset + +SELECT count(*) = 0 AS hash_guard_columns_absent_ok +FROM information_schema.columns +WHERE table_name = 'filtered_tasks' AND column_name IN ('base_fail', 'target_fail') \gset + +SELECT count(*) = 0 AS hash_guard_unrecorded_ok +FROM cloudsync_migrations +WHERE migration_id IN ('mig-pg-bad-base', 'mig-pg-bad-target') \gset + +\if :hash_bad_base_rejected_ok +\if :hash_bad_target_rejected_ok +\if :hash_guard_columns_absent_ok +\if :hash_guard_unrecorded_ok +\echo [PASS] (:testid) Migration hash guards rejected and rolled back invalid payloads +\else +\echo [FAIL] (:testid) Hash guard failure was recorded +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Hash guard failure altered schema +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) targetSchemaHash mismatch was accepted +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) baseSchemaHash mismatch was accepted +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- rawSql dialect validation +-- ============================================================================ + +SELECT cloudsync_migration_apply($json$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-raw-success", + "requiredCapabilities": ["schema:write"], + "ops": [ + {"op": "rawSql", "sql": {"postgresql": ["CREATE TABLE raw_ok (id UUID PRIMARY KEY)"]}} + ] +} +$json$) AS migration_raw_success \gset + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-raw-missing", + "ops": [ + {"op": "rawSql", "sql": {"sqlite": ["CREATE TABLE raw_missing (id TEXT PRIMARY KEY NOT NULL)"]}} + ] +} +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + INSERT INTO schema_migration_error_flags VALUES ('raw_missing_rejected', rejected); +END +$$; + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-raw-bad-item", + "ops": [ + {"op": "rawSql", "sql": {"postgresql": [123]}} + ] +} +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + INSERT INTO schema_migration_error_flags VALUES ('raw_bad_item_rejected', rejected); +END +$$; + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-raw-tx", + "ops": [ + {"op": "rawSql", "sql": {"postgresql": ["COMMIT"]}} + ] +} +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + INSERT INTO schema_migration_error_flags VALUES ('raw_tx_rejected', rejected); +END +$$; + +SELECT count(*) = 1 AS raw_success_ok +FROM information_schema.tables +WHERE table_name = 'raw_ok' \gset + +SELECT ok AS raw_missing_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'raw_missing_rejected' \gset + +SELECT ok AS raw_bad_item_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'raw_bad_item_rejected' \gset + +SELECT ok AS raw_tx_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'raw_tx_rejected' \gset + +SELECT count(*) = 0 AS raw_missing_absent_ok +FROM information_schema.tables +WHERE table_name = 'raw_missing' \gset + +SELECT count(*) = 0 AS raw_bad_unrecorded_ok +FROM cloudsync_migrations +WHERE migration_id IN ('mig-pg-raw-missing', 'mig-pg-raw-bad-item', 'mig-pg-raw-tx') \gset + +\if :raw_success_ok +\if :raw_missing_rejected_ok +\if :raw_bad_item_rejected_ok +\if :raw_tx_rejected_ok +\if :raw_missing_absent_ok +\if :raw_bad_unrecorded_ok +\echo [PASS] (:testid) rawSql migration selected dialect and rejected malformed variants +\else +\echo [FAIL] (:testid) rawSql malformed migration was recorded +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) rawSql missing dialect altered schema +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) rawSql transaction control was accepted +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) rawSql non-string array item was accepted +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) rawSql missing dialect was accepted +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) rawSql dialect success branch did not run +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Declarative raw SQL builder API and guardrails. +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_alter_sql('SAVEPOINT bad_raw_alter'); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + INSERT INTO schema_migration_error_flags VALUES ('alter_raw_guard_rejected', rejected); +END +$$; + +DO $$ +DECLARE + end_rejected BOOLEAN := false; + abort_rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_alter_sql('END'); + EXCEPTION WHEN OTHERS THEN + end_rejected := true; + END; + + BEGIN + PERFORM cloudsync_alter_sql('ABORT'); + EXCEPTION WHEN OTHERS THEN + abort_rejected := true; + END; + + INSERT INTO schema_migration_error_flags VALUES ('alter_raw_end_guard_rejected', end_rejected); + INSERT INTO schema_migration_error_flags VALUES ('alter_raw_abort_guard_rejected', abort_rejected); +END +$$; + +SELECT cloudsync_alter_create_table('raw_alter_pg') AS alter_raw_create_ok \gset +SELECT cloudsync_alter_add_column('raw_alter_pg', 'id', 'uuid', false) AS alter_raw_id_ok \gset +SELECT cloudsync_alter_add_primary_key('raw_alter_pg', 'id') AS alter_raw_pk_ok \gset +SELECT cloudsync_alter_add_column('raw_alter_pg', 'title', 'text', false, '') AS alter_raw_title_ok \gset +SELECT cloudsync_alter_sql('CREATE INDEX raw_alter_pg_title_common_idx ON raw_alter_pg(title)') AS alter_raw_common_ok \gset +SELECT cloudsync_alter_sqlite('CREATE INDEX raw_alter_pg_title_sqlite_idx ON raw_alter_pg(title)') AS alter_raw_sqlite_ok \gset +SELECT cloudsync_alter_postgresql('CREATE INDEX raw_alter_pg_title_pg_idx ON raw_alter_pg(title)') AS alter_raw_pg_ok \gset +SELECT position('"skipMissingDialect":true' in cloudsync_alter_preview()) > 0 AS alter_raw_preview_skip_ok \gset +SELECT cloudsync_alter_apply() IS NOT NULL AS alter_raw_apply_ok \gset + +SELECT ok AS alter_raw_guard_ok +FROM schema_migration_error_flags +WHERE key = 'alter_raw_guard_rejected' \gset + +SELECT ok AS alter_raw_end_guard_ok +FROM schema_migration_error_flags +WHERE key = 'alter_raw_end_guard_rejected' \gset + +SELECT ok AS alter_raw_abort_guard_ok +FROM schema_migration_error_flags +WHERE key = 'alter_raw_abort_guard_rejected' \gset + +SELECT count(*) = 1 AS alter_raw_table_ok +FROM information_schema.tables +WHERE table_name = 'raw_alter_pg' \gset + +SELECT count(*) = 1 AS alter_raw_common_idx_ok +FROM pg_indexes +WHERE indexname = 'raw_alter_pg_title_common_idx' \gset + +SELECT count(*) = 1 AS alter_raw_pg_idx_ok +FROM pg_indexes +WHERE indexname = 'raw_alter_pg_title_pg_idx' \gset + +SELECT count(*) = 0 AS alter_raw_sqlite_idx_skipped_ok +FROM pg_indexes +WHERE indexname = 'raw_alter_pg_title_sqlite_idx' \gset + +\if :alter_raw_guard_ok +\if :alter_raw_end_guard_ok +\if :alter_raw_abort_guard_ok +\if :alter_raw_preview_skip_ok +\if :alter_raw_apply_ok +\if :alter_raw_table_ok +\if :alter_raw_common_idx_ok +\if :alter_raw_pg_idx_ok +\if :alter_raw_sqlite_idx_skipped_ok +\echo [PASS] (:testid) Declarative raw SQL builder applied PostgreSQL SQL and skipped SQLite SQL +\else +\echo [FAIL] (:testid) SQLite-only raw SQL ran on PostgreSQL +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) PostgreSQL raw SQL index was not created +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Common raw SQL index was not created +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative raw SQL createTable did not create table +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative raw SQL apply failed +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative raw SQL preview did not mark dialect skip +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative raw SQL accepted ABORT transaction control +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative raw SQL accepted END transaction control +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative raw SQL accepted transaction control +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- createTable conflict and declarative alter atomicity +-- ============================================================================ + +CREATE TABLE existing_create_conflict ( + id UUID PRIMARY KEY, + title TEXT NOT NULL DEFAULT '' +); + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_migration_apply($payload$ +{ + "type": "cloudsync.schema.migration", + "formatVersion": 1, + "migrationId": "mig-pg-existing-create", + "ops": [ + { + "op": "createTable", + "table": "existing_create_conflict", + "columns": [ + {"name": "id", "type": "uuid", "primaryKey": true, "nullable": false}, + {"name": "body", "type": "text", "nullable": true} + ] + } + ] +} +$payload$); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + INSERT INTO schema_migration_error_flags VALUES ('existing_create_rejected', rejected); +END +$$; + +SELECT ok AS existing_create_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'existing_create_rejected' \gset + +SELECT count(*) = 0 AS existing_create_body_absent_ok +FROM information_schema.columns +WHERE table_name = 'existing_create_conflict' AND column_name = 'body' \gset + +SELECT count(*) = 0 AS existing_create_unrecorded_ok +FROM cloudsync_migrations +WHERE migration_id = 'mig-pg-existing-create' \gset + +SELECT count(*) AS migrations_before_atomic +FROM cloudsync_migrations \gset + +DROP TABLE IF EXISTS cloudsync_pending_migration; +CREATE TABLE cloudsync_pending_migration (x TEXT); + +DO $$ +DECLARE + rejected BOOLEAN := false; +BEGIN + BEGIN + PERFORM cloudsync_alter_create_table('atomic_pg_notes'); + PERFORM cloudsync_alter_add_column('atomic_pg_notes', 'id', 'uuid', false); + PERFORM cloudsync_alter_add_primary_key('atomic_pg_notes', 'id'); + PERFORM cloudsync_alter_apply(); + EXCEPTION WHEN OTHERS THEN + rejected := true; + END; + INSERT INTO schema_migration_error_flags VALUES ('alter_atomic_rejected', rejected); +END +$$; + +SELECT ok AS alter_atomic_rejected_ok +FROM schema_migration_error_flags +WHERE key = 'alter_atomic_rejected' \gset + +SELECT count(*) = 0 AS alter_atomic_table_absent_ok +FROM information_schema.tables +WHERE table_name = 'atomic_pg_notes' \gset + +SELECT count(*) = 1 AS alter_atomic_pending_shape_ok +FROM information_schema.columns +WHERE table_name = 'cloudsync_pending_migration' AND column_name = 'x' \gset + +SELECT count(*) = :migrations_before_atomic AS alter_atomic_records_ok +FROM cloudsync_migrations \gset + +\if :existing_create_rejected_ok +\if :existing_create_body_absent_ok +\if :existing_create_unrecorded_ok +\echo [PASS] (:testid) createTable rejected an existing conflicting table +\else +\echo [FAIL] (:testid) createTable conflict was recorded +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) createTable conflict changed the existing table +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) createTable accepted an existing conflicting table +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :alter_atomic_rejected_ok +\if :alter_atomic_table_absent_ok +\if :alter_atomic_pending_shape_ok +\if :alter_atomic_records_ok +\echo [PASS] (:testid) declarative alter rollback kept local schema and pending save atomic +\else +\echo [FAIL] (:testid) declarative alter atomic rollback recorded a migration +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) declarative alter atomic rollback recreated pending table +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) declarative alter atomic rollback left a local table +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) declarative alter accepted malformed pending migration table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_52; +\endif diff --git a/test/postgresql/full_test.sql b/test/postgresql/full_test.sql index 9ff000a6..b46cf243 100644 --- a/test/postgresql/full_test.sql +++ b/test/postgresql/full_test.sql @@ -59,6 +59,7 @@ \ir 49_row_filter_prefill.sql \ir 50_block_lww_existing_data.sql \ir 51_stale_table_settings_dropped_meta.sql +\ir 52_schema_migrations.sql -- 'Test summary' \echo '\nTest summary:' diff --git a/test/schema_migration_cross_dialect.sh b/test/schema_migration_cross_dialect.sh new file mode 100755 index 00000000..799025cf --- /dev/null +++ b/test/schema_migration_cross_dialect.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SQLITE3_BIN="${SQLITE3:-sqlite3}" +PGHOST="${PG_DOCKER_DB_HOST:-localhost}" +PGPORT="${PG_DOCKER_DB_PORT:-5432}" +PGDATABASE_ADMIN="${PG_DOCKER_DB_NAME:-postgres}" +PGUSER="${PG_DOCKER_DB_USER:-postgres}" +export PGPASSWORD="${PG_DOCKER_DB_PASSWORD:-postgres}" + +EXTENSION_PATH="" +for candidate in "$ROOT_DIR/dist/cloudsync.dylib" "$ROOT_DIR/dist/cloudsync.so" "$ROOT_DIR/dist/cloudsync.dll"; do + if [ -f "$candidate" ]; then + EXTENSION_PATH="$candidate" + break + fi +done + +if [ -z "$EXTENSION_PATH" ]; then + echo "[FAIL] SQLite extension not found in dist/" + exit 1 +fi + +command -v "$SQLITE3_BIN" >/dev/null 2>&1 || { echo "[FAIL] sqlite3 not found"; exit 1; } +command -v psql >/dev/null 2>&1 || { echo "[FAIL] psql not found"; exit 1; } + +TMPDIR="$(mktemp -d)" +SQLITE_DB="$TMPDIR/cross.sqlite" +SQLITE_TO_PG="$TMPDIR/sqlite-to-postgresql.json" +PG_TO_SQLITE="$TMPDIR/postgresql-to-sqlite.json" +PG_TEST_DB="cloudsync_cross_migration" +PG_ADMIN_URL="postgresql://$PGUSER@$PGHOST:$PGPORT/$PGDATABASE_ADMIN" +PG_TEST_URL="postgresql://$PGUSER@$PGHOST:$PGPORT/$PG_TEST_DB" + +cleanup() { + PGPASSWORD="$PGPASSWORD" psql "$PG_ADMIN_URL" -qAt -v ON_ERROR_STOP=0 -c "DROP DATABASE IF EXISTS $PG_TEST_DB;" >/dev/null 2>&1 || true + rm -rf "$TMPDIR" +} +trap cleanup EXIT + +sqlite_exec() { + "$SQLITE3_BIN" "$SQLITE_DB" </dev/null +} + +expect_eq() { + local label="$1" + local actual="$2" + local expected="$3" + if [ "$actual" != "$expected" ]; then + echo "[FAIL] $label: expected $expected, got $actual" + exit 1 + fi +} + +PGPASSWORD="$PGPASSWORD" psql "$PG_ADMIN_URL" -qAt -v ON_ERROR_STOP=1 </dev/null +DROP DATABASE IF EXISTS $PG_TEST_DB; +CREATE DATABASE $PG_TEST_DB; +SQL +pg_exec "CREATE EXTENSION cloudsync;" >/dev/null + +"$SQLITE3_BIN" "$SQLITE_DB" </dev/null +.bail on +.load $EXTENSION_PATH +SELECT cloudsync_alter_create_table('events'); +SELECT cloudsync_alter_add_column('events', 'id', 'text', 0); +SELECT cloudsync_alter_add_primary_key('events', 'id'); +SELECT cloudsync_alter_add_column('events', 'body', 'text', 0, ''); +SELECT cloudsync_alter_add_column('events', 'created_at', 'timestamp', 0, '2026-04-28T00:00:00Z'); +SELECT cloudsync_alter_augment_table('events'); +SELECT cloudsync_alter_set_block_lww('events', 'body', char(10)); +SELECT cloudsync_alter_sql('CREATE INDEX events_created_common_idx ON events(created_at)'); +SELECT cloudsync_alter_sqlite('CREATE INDEX events_body_sqlite_idx ON events(body)'); +SELECT cloudsync_alter_postgresql('CREATE INDEX events_body_pg_idx ON events(body)'); +.once $SQLITE_TO_PG +SELECT cloudsync_alter_preview(); +.output /dev/null +SELECT cloudsync_terminate(); +SQL + +sqlite_apply_payload "$SQLITE_TO_PG" >/dev/null +pg_apply_payload "$SQLITE_TO_PG" + +sqlite_exec "INSERT INTO events (id, body) VALUES ('s1', 'one' || char(10) || 'two');" >/dev/null +pg_exec "INSERT INTO events (id, body) VALUES ('p1', E'one\\ntwo');" >/dev/null + +expect_eq "SQLite->PostgreSQL table" "$(pg_scalar "SELECT count(*) FROM information_schema.tables WHERE table_name='events';")" "1" +expect_eq "SQLite->PostgreSQL block settings" "$(pg_scalar "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='events' AND col_name='body' AND key='algo' AND value='block';")" "1" +expect_eq "SQLite common raw SQL index" "$(sqlite_scalar "SELECT count(*) FROM sqlite_master WHERE type='index' AND name='events_created_common_idx';")" "1" +expect_eq "SQLite dialect raw SQL index" "$(sqlite_scalar "SELECT count(*) FROM sqlite_master WHERE type='index' AND name='events_body_sqlite_idx';")" "1" +expect_eq "SQLite skipped PostgreSQL raw SQL index" "$(sqlite_scalar "SELECT count(*) FROM sqlite_master WHERE type='index' AND name='events_body_pg_idx';")" "0" +expect_eq "PostgreSQL common raw SQL index" "$(pg_scalar "SELECT count(*) FROM pg_indexes WHERE indexname='events_created_common_idx';")" "1" +expect_eq "PostgreSQL dialect raw SQL index" "$(pg_scalar "SELECT count(*) FROM pg_indexes WHERE indexname='events_body_pg_idx';")" "1" +expect_eq "PostgreSQL skipped SQLite raw SQL index" "$(pg_scalar "SELECT count(*) FROM pg_indexes WHERE indexname='events_body_sqlite_idx';")" "0" +expect_eq "SQLite block rows" "$(sqlite_scalar "SELECT count(*) FROM events_cloudsync_blocks;")" "2" +expect_eq "PostgreSQL block rows" "$(pg_scalar "SELECT count(*) FROM events_cloudsync_blocks;")" "2" +echo "[PASS] SQLite -> PostgreSQL createTable/augment/block migration" + +PGPASSWORD="$PGPASSWORD" psql "$PG_TEST_URL" -qAt -v ON_ERROR_STOP=1 </dev/null +\\o /dev/null +SELECT cloudsync_alter_add_column('events', 'metadata', 'json', false, '{}'); +SELECT cloudsync_alter_add_column_sqlite('events', 'metadata', 'TEXT', false, '''{}'''); +SELECT cloudsync_alter_add_column_postgresql('events', 'metadata', 'JSONB', false, '''{}''::jsonb'); +SELECT cloudsync_alter_sql('CREATE INDEX events_metadata_common_idx ON events(created_at)'); +SELECT cloudsync_alter_sqlite('CREATE INDEX events_metadata_sqlite_idx ON events(metadata)'); +SELECT cloudsync_alter_postgresql('CREATE INDEX events_metadata_pg_idx ON events(metadata)'); +\\o $PG_TO_SQLITE +SELECT cloudsync_alter_preview(); +\\o +SQL + +pg_apply_payload "$PG_TO_SQLITE" +sqlite_apply_payload "$PG_TO_SQLITE" >/dev/null + +expect_eq "PostgreSQL->SQLite column on SQLite" "$(sqlite_scalar "SELECT count(*) FROM pragma_table_info('events') WHERE name='metadata';")" "1" +expect_eq "PostgreSQL JSONB override" "$(pg_scalar "SELECT count(*) FROM information_schema.columns WHERE table_name='events' AND column_name='metadata' AND data_type='jsonb';")" "1" +expect_eq "PostgreSQL->SQLite common raw SQL index on SQLite" "$(sqlite_scalar "SELECT count(*) FROM sqlite_master WHERE type='index' AND name='events_metadata_common_idx';")" "1" +expect_eq "PostgreSQL->SQLite dialect raw SQL index on SQLite" "$(sqlite_scalar "SELECT count(*) FROM sqlite_master WHERE type='index' AND name='events_metadata_sqlite_idx';")" "1" +expect_eq "PostgreSQL skipped SQLite-specific raw SQL index" "$(pg_scalar "SELECT count(*) FROM pg_indexes WHERE indexname='events_metadata_sqlite_idx';")" "0" +expect_eq "PostgreSQL dialect raw SQL index" "$(pg_scalar "SELECT count(*) FROM pg_indexes WHERE indexname='events_metadata_pg_idx';")" "1" + +sqlite_exec "INSERT INTO events (id, body) VALUES ('s2', 'three');" >/dev/null +pg_exec "INSERT INTO events (id, body) VALUES ('p2', 'three');" >/dev/null +expect_eq "SQLite default override" "$(sqlite_scalar "SELECT metadata FROM events WHERE id='s2';")" "{}" +expect_eq "PostgreSQL default override" "$(pg_scalar "SELECT metadata::text FROM events WHERE id='p2';")" "{}" +echo "[PASS] PostgreSQL -> SQLite dialect override migration" + +echo "[PASS] Cross-dialect schema migration tests completed" diff --git a/test/unit.c b/test/unit.c index 05e9c958..9664cd23 100644 --- a/test/unit.c +++ b/test/unit.c @@ -1079,6 +1079,51 @@ bool do_create_tables (int table_mask, sqlite3 *db) { return false; } +static void test_json_append_string(sqlite3_str *json, const char *value) { + sqlite3_str_appendchar(json, 1, '"'); + if (value) { + for (const unsigned char *p = (const unsigned char *)value; *p; ++p) { + switch (*p) { + case '"': sqlite3_str_appendall(json, "\\\""); break; + case '\\': sqlite3_str_appendall(json, "\\\\"); break; + case '\b': sqlite3_str_appendall(json, "\\b"); break; + case '\f': sqlite3_str_appendall(json, "\\f"); break; + case '\n': sqlite3_str_appendall(json, "\\n"); break; + case '\r': sqlite3_str_appendall(json, "\\r"); break; + case '\t': sqlite3_str_appendall(json, "\\t"); break; + default: + if (*p < 0x20) sqlite3_str_appendf(json, "\\u%04x", *p); + else sqlite3_str_appendchar(json, 1, (char)*p); + break; + } + } + } + sqlite3_str_appendchar(json, 1, '"'); +} + +static bool do_rebuild_table_migration(sqlite3 *db, const char *migration_id, const char *table, const char **ddl, int ddl_count) { + sqlite3_str *json = sqlite3_str_new(NULL); + sqlite3_str_appendall(json, "{\"type\":\"cloudsync.schema.migration\",\"formatVersion\":2,\"migrationId\":"); + test_json_append_string(json, migration_id); + sqlite3_str_appendall(json, ",\"requiredCapabilities\":[\"schema:write\",\"schema:destructive\"],\"ops\":[{\"op\":\"rebuildTableSync\",\"table\":"); + test_json_append_string(json, table); + sqlite3_str_appendall(json, ",\"algorithm\":\"cls\",\"initFlags\":1,\"ddl\":[{\"op\":\"rawSql\",\"sql\":["); + for (int i = 0; i < ddl_count; ++i) { + if (i > 0) sqlite3_str_appendchar(json, 1, ','); + test_json_append_string(json, ddl[i]); + } + sqlite3_str_appendall(json, "]}]}]}"); + char *payload = sqlite3_str_finish(json); + if (!payload) return false; + + char *sql = sqlite3_mprintf("SELECT cloudsync_migration_apply('%q');", payload); + sqlite3_free(payload); + if (!sql) return false; + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + sqlite3_free(sql); + return rc == SQLITE_OK; +} + bool do_alter_tables (int table_mask, sqlite3 *db, int alter_version) { // declare tables if (table_mask & TEST_PRIKEYS) { @@ -1086,37 +1131,45 @@ bool do_alter_tables (int table_mask, sqlite3 *db, int alter_version) { char *sql = NULL; switch (alter_version) { case 1: - sql = sqlite3_mprintf("SELECT cloudsync_begin_alter('%q'); " - "ALTER TABLE \"%w\" ADD new_column_1 TEXT; " - "ALTER TABLE \"%w\" ADD new_column_2 TEXT DEFAULT 'default value'; " - "ALTER TABLE \"%w\" DROP note; " - "SELECT cloudsync_commit_alter('%q') ", - CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE); + sql = sqlite3_mprintf("SELECT cloudsync_alter_add_column('%q', 'new_column_1', 'text', 1); " + "SELECT cloudsync_alter_add_column('%q', 'new_column_2', 'text', 1, 'default value'); " + "SELECT cloudsync_alter_drop_column('%q', 'note'); " + "SELECT cloudsync_alter_apply();", + CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE); break; case 2: - sql = sqlite3_mprintf("SELECT cloudsync_begin_alter('%q'); " - "ALTER TABLE \"%w\" RENAME TO do_alter_tables_temp_customers; " - "CREATE TABLE \"%w\" (first_name TEXT NOT NULL, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" TEXT NOT NULL, note TEXT, note3 TEXT DEFAULT 'note', note4 DATETIME DEFAULT(datetime('subsec')), stamp TEXT DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\")); " - "INSERT INTO \"%w\" (\"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",\"note\", \"note3\", \"stamp\") SELECT \"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",\"note\",'a new note',\"stamp\" FROM do_alter_tables_temp_customers; " - "DROP TABLE do_alter_tables_temp_customers; " - "SELECT cloudsync_commit_alter('%q') ", - CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE); + { + char *ddl0 = sqlite3_mprintf("ALTER TABLE \"%w\" RENAME TO do_alter_tables_temp_customers;", CUSTOMERS_TABLE); + char *ddl1 = sqlite3_mprintf("CREATE TABLE \"%w\" (first_name TEXT NOT NULL, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" TEXT NOT NULL, note TEXT, note3 TEXT DEFAULT 'note', note4 DATETIME DEFAULT(datetime('subsec')), stamp TEXT DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\"));", CUSTOMERS_TABLE); + char *ddl2 = sqlite3_mprintf("INSERT INTO \"%w\" (\"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",\"note\", \"note3\", \"stamp\") SELECT \"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",\"note\",'a new note',\"stamp\" FROM do_alter_tables_temp_customers;", CUSTOMERS_TABLE); + const char *ddl[] = {ddl0, ddl1, ddl2, "DROP TABLE do_alter_tables_temp_customers;"}; + bool ok = ddl0 && ddl1 && ddl2 && do_rebuild_table_migration(db, "test-rebuild-customers-v2", CUSTOMERS_TABLE, ddl, 4); + if (ddl0) sqlite3_free(ddl0); + if (ddl1) sqlite3_free(ddl1); + if (ddl2) sqlite3_free(ddl2); + if (!ok) goto abort_alter_tables; + sql = NULL; break; + } case 3: - sql = sqlite3_mprintf("SELECT cloudsync_begin_alter('%q'); " - "ALTER TABLE \"%w\" RENAME TO do_alter_tables_temp_customers; " - "CREATE TABLE \"%w\" (name TEXT NOT NULL, note TEXT, note3 TEXT DEFAULT 'note', note4 DATETIME DEFAULT(datetime('subsec')), stamp TEXT DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(name)); " - "INSERT INTO \"%w\" (\"name\",\"note\", \"note3\", \"stamp\") SELECT \"first_name\" || \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" ,\"note\",'a new note',\"stamp\" FROM do_alter_tables_temp_customers; " - "DROP TABLE do_alter_tables_temp_customers; " - "SELECT cloudsync_commit_alter('%q') ", - CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE); + { + char *ddl0 = sqlite3_mprintf("ALTER TABLE \"%w\" RENAME TO do_alter_tables_temp_customers;", CUSTOMERS_TABLE); + char *ddl1 = sqlite3_mprintf("CREATE TABLE \"%w\" (name TEXT NOT NULL, note TEXT, note3 TEXT DEFAULT 'note', note4 DATETIME DEFAULT(datetime('subsec')), stamp TEXT DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(name));", CUSTOMERS_TABLE); + char *ddl2 = sqlite3_mprintf("INSERT INTO \"%w\" (\"name\",\"note\", \"note3\", \"stamp\") SELECT \"first_name\" || \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" ,\"note\",'a new note',\"stamp\" FROM do_alter_tables_temp_customers;", CUSTOMERS_TABLE); + const char *ddl[] = {ddl0, ddl1, ddl2, "DROP TABLE do_alter_tables_temp_customers;"}; + bool ok = ddl0 && ddl1 && ddl2 && do_rebuild_table_migration(db, "test-rebuild-customers-v3", CUSTOMERS_TABLE, ddl, 4); + if (ddl0) sqlite3_free(ddl0); + if (ddl1) sqlite3_free(ddl1); + if (ddl2) sqlite3_free(ddl2); + if (!ok) goto abort_alter_tables; + sql = NULL; break; + } case 4: // only add columns, not drop - sql = sqlite3_mprintf("SELECT cloudsync_begin_alter('%q'); " - "ALTER TABLE \"%w\" ADD new_column_1 TEXT; " - "ALTER TABLE \"%w\" ADD new_column_2 TEXT DEFAULT 'default value'; " - "SELECT cloudsync_commit_alter('%q') ", - CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE, CUSTOMERS_TABLE); + sql = sqlite3_mprintf("SELECT cloudsync_alter_add_column('%q', 'new_column_1', 'text', 1); " + "SELECT cloudsync_alter_add_column('%q', 'new_column_2', 'text', 1, 'default value'); " + "SELECT cloudsync_alter_apply();", + CUSTOMERS_TABLE, CUSTOMERS_TABLE); break; default: sql = NULL; @@ -1133,37 +1186,46 @@ bool do_alter_tables (int table_mask, sqlite3 *db, int alter_version) { const char *sql; switch (alter_version) { case 1: - sql = "SELECT cloudsync_begin_alter('" CUSTOMERS_NOCOLS_TABLE "'); " - "ALTER TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" ADD new_column_1 TEXT; " - "ALTER TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" ADD new_column_2 TEXT DEFAULT 'default value'; " - "SELECT cloudsync_commit_alter('" CUSTOMERS_NOCOLS_TABLE "'); "; + sql = "SELECT cloudsync_alter_add_column('" CUSTOMERS_NOCOLS_TABLE "', 'new_column_1', 'text', 1); " + "SELECT cloudsync_alter_add_column('" CUSTOMERS_NOCOLS_TABLE "', 'new_column_2', 'text', 1, 'default value'); " + "SELECT cloudsync_alter_apply(); "; break; case 2: - sql = "SELECT cloudsync_begin_alter('" CUSTOMERS_NOCOLS_TABLE "'); " - "ALTER TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" RENAME TO do_alter_tables_temp_customers_nocols; " - "CREATE TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" (first_name TEXT NOT NULL, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME " 2\" TEXT NOT NULL, PRIMARY KEY(first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME " 2\")); " - "INSERT INTO \"" CUSTOMERS_NOCOLS_TABLE "\" (\"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME " 2\") SELECT \"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" FROM do_alter_tables_temp_customers_nocols; " - "DROP TABLE do_alter_tables_temp_customers_nocols; " - "SELECT cloudsync_commit_alter('" CUSTOMERS_NOCOLS_TABLE "'); " - "SELECT cloudsync_begin_alter('" CUSTOMERS_NOCOLS_TABLE "'); " - "ALTER TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" RENAME TO do_alter_tables_temp_customers_nocols; " - "CREATE TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" (first_name TEXT NOT NULL, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" TEXT NOT NULL, PRIMARY KEY(first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\")); " - "INSERT INTO \"" CUSTOMERS_NOCOLS_TABLE "\" (\"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\") SELECT \"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME " 2\" FROM do_alter_tables_temp_customers_nocols; " - "DROP TABLE do_alter_tables_temp_customers_nocols; " - "SELECT cloudsync_commit_alter('" CUSTOMERS_NOCOLS_TABLE "');" ; + { + const char *ddl1[] = { + "ALTER TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" RENAME TO do_alter_tables_temp_customers_nocols;", + "CREATE TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" (first_name TEXT NOT NULL, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME " 2\" TEXT NOT NULL, PRIMARY KEY(first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME " 2\"));", + "INSERT INTO \"" CUSTOMERS_NOCOLS_TABLE "\" (\"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME " 2\") SELECT \"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" FROM do_alter_tables_temp_customers_nocols;", + "DROP TABLE do_alter_tables_temp_customers_nocols;" + }; + const char *ddl2[] = { + "ALTER TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" RENAME TO do_alter_tables_temp_customers_nocols;", + "CREATE TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" (first_name TEXT NOT NULL, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" TEXT NOT NULL, PRIMARY KEY(first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\"));", + "INSERT INTO \"" CUSTOMERS_NOCOLS_TABLE "\" (\"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\") SELECT \"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME " 2\" FROM do_alter_tables_temp_customers_nocols;", + "DROP TABLE do_alter_tables_temp_customers_nocols;" + }; + if (!do_rebuild_table_migration(db, "test-rebuild-customers-nocols-v2a", CUSTOMERS_NOCOLS_TABLE, ddl1, 4)) goto abort_alter_tables; + if (!do_rebuild_table_migration(db, "test-rebuild-customers-nocols-v2b", CUSTOMERS_NOCOLS_TABLE, ddl2, 4)) goto abort_alter_tables; + sql = NULL; break; + } case 3: - sql = "SELECT cloudsync_begin_alter('" CUSTOMERS_NOCOLS_TABLE "'); " - "ALTER TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" RENAME TO do_alter_tables_temp_customers_nocols; " - "CREATE TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" (name TEXT NOT NULL PRIMARY KEY); " - "INSERT INTO \"" CUSTOMERS_NOCOLS_TABLE "\" (\"name\") SELECT \"first_name\" || \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" FROM do_alter_tables_temp_customers_nocols; " - "DROP TABLE do_alter_tables_temp_customers_nocols; " - "SELECT cloudsync_commit_alter('" CUSTOMERS_NOCOLS_TABLE "'); "; + { + const char *ddl[] = { + "ALTER TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" RENAME TO do_alter_tables_temp_customers_nocols;", + "CREATE TABLE \"" CUSTOMERS_NOCOLS_TABLE "\" (name TEXT NOT NULL PRIMARY KEY);", + "INSERT INTO \"" CUSTOMERS_NOCOLS_TABLE "\" (\"name\") SELECT \"first_name\" || \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" FROM do_alter_tables_temp_customers_nocols;", + "DROP TABLE do_alter_tables_temp_customers_nocols;" + }; + if (!do_rebuild_table_migration(db, "test-rebuild-customers-nocols-v3", CUSTOMERS_NOCOLS_TABLE, ddl, 4)) goto abort_alter_tables; + sql = NULL; break; + } default: + sql = NULL; break; } - if (sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) goto abort_alter_tables; + if (sql && sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) goto abort_alter_tables; } if (table_mask & TEST_NOPRIKEYS) { @@ -1171,35 +1233,39 @@ bool do_alter_tables (int table_mask, sqlite3 *db, int alter_version) { const char *sql; switch (alter_version) { case 1: - sql = "SELECT cloudsync_begin_alter('customers_noprikey'); " - "ALTER TABLE customers_noprikey ADD new_column_1 TEXT, new_column_2 TEXT DEFAULT CURRENT_TIMESTAMP; " - "SELECT cloudsync_commit_alter('customers_noprikey') "; + sql = "SELECT cloudsync_alter_add_column('customers_noprikey', 'new_column_1', 'text', 1); " + "SELECT cloudsync_alter_add_column('customers_noprikey', 'new_column_2', 'text', 1, CURRENT_TIMESTAMP); " + "SELECT cloudsync_alter_apply(); "; break; case 2: - sql = "SELECT cloudsync_begin_alter('customers_noprikey'); " - "ALTER TABLE \"customers_noprikey\" RENAME TO do_alter_tables_temp_customers_noprikey; " - "CREATE TABLE \"customers_noprikey\" (first_name TEXT NOT NULL, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" TEXT NOT NULL, note TEXT, note3 TEXT DEFAULT 'note', stamp TEXT DEFAULT CURRENT_TIMESTAMP); " - "INSERT INTO \"customers_noprikey\" (\"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",\"note\", \"note3\", \"stamp\") SELECT \"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",\"note\",'a new note',\"stamp\" FROM do_alter_tables_temp_customers_noprikey; " - "DROP TABLE do_alter_tables_temp_customers_noprikey; " - "SELECT cloudsync_commit_alter('customers_noprikey') " - "SELECT cloudsync_begin_alter('customers_noprikey'); " - "ALTER TABLE \"customers_noprikey\" RENAME TO do_alter_tables_temp_customers_noprikey; " - "CREATE TABLE customers_noprikey (first_name TEXT NOT NULL, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" TEXT NOT NULL, PRIMARY KEY(first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\")); " - "INSERT INTO \"customers_noprikey\" (\"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\") SELECT \"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME " 2\" FROM do_alter_tables_temp_customers_noprikey; " - "DROP TABLE do_alter_tables_temp_customers_noprikey; " - "SELECT cloudsync_commit_alter('customers_noprikey');" ; + { + const char *ddl[] = { + "ALTER TABLE \"customers_noprikey\" RENAME TO do_alter_tables_temp_customers_noprikey;", + "CREATE TABLE \"customers_noprikey\" (first_name TEXT NOT NULL, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\" TEXT NOT NULL, note TEXT, note3 TEXT DEFAULT 'note', stamp TEXT DEFAULT CURRENT_TIMESTAMP);", + "INSERT INTO \"customers_noprikey\" (\"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",\"note\", \"note3\", \"stamp\") SELECT \"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",NULL,'a new note',CURRENT_TIMESTAMP FROM do_alter_tables_temp_customers_noprikey;", + "DROP TABLE do_alter_tables_temp_customers_noprikey;" + }; + if (!do_rebuild_table_migration(db, "test-rebuild-customers-noprikey-v2", "customers_noprikey", ddl, 4)) goto abort_alter_tables; + sql = NULL; break; + } case 3: - sql = "SELECT cloudsync_begin_alter('customers_noprikey'); " - "ALTER TABLE \"customers_noprikey\" RENAME TO do_alter_tables_temp_customers_noprikey; " - "CREATE TABLE \"customers_noprikey\" (name TEXT NOT NULL, note TEXT, note3 TEXT DEFAULT 'note', stamp TEXT DEFAULT CURRENT_TIMESTAMP); " - "INSERT INTO \"customers_noprikey\" (\"first_name\" || \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",\"note\", \"note3\", \"stamp\") SELECT \"first_name\",\"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",\"note\",'a new note',\"stamp\" FROM do_alter_tables_temp_customers_noprikey; " - "DROP TABLE do_alter_tables_temp_customers_noprikey; " - "SELECT cloudsync_commit_alter('customers_noprikey') "; + { + const char *ddl[] = { + "ALTER TABLE \"customers_noprikey\" RENAME TO do_alter_tables_temp_customers_noprikey;", + "CREATE TABLE \"customers_noprikey\" (name TEXT NOT NULL, note TEXT, note3 TEXT DEFAULT 'note', stamp TEXT DEFAULT CURRENT_TIMESTAMP);", + "INSERT INTO \"customers_noprikey\" (\"name\",\"note\", \"note3\", \"stamp\") SELECT \"first_name\" || \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\",NULL,'a new note',CURRENT_TIMESTAMP FROM do_alter_tables_temp_customers_noprikey;", + "DROP TABLE do_alter_tables_temp_customers_noprikey;" + }; + if (!do_rebuild_table_migration(db, "test-rebuild-customers-noprikey-v3", "customers_noprikey", ddl, 4)) goto abort_alter_tables; + sql = NULL; + break; + } default: + sql = NULL; break; } - if (sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) goto abort_alter_tables; + if (sql && sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) goto abort_alter_tables; } return true; @@ -2005,13 +2071,41 @@ bool do_test_error_cases (sqlite3 *db) { int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_ERROR) return false; - // test error - sql = "SELECT cloudsync_begin_alter('foo2');"; + // test empty declarative migration + sql = "SELECT cloudsync_alter_apply();"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_MISUSE) return false; - // test error - sql = "SELECT cloudsync_commit_alter('foo2');"; + // test NOT NULL add-column without default + sql = "SELECT cloudsync_alter_add_column('foo2', 'required_value', 'text', 0);" + "SELECT cloudsync_alter_apply();"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_MISUSE) return false; + sqlite3_exec(db, "SELECT cloudsync_alter_clear('foo2');", NULL, NULL, NULL); + + // test NOT NULL add-column with explicit SQL NULL default + sql = "SELECT cloudsync_alter_add_column('foo2', 'required_null', 'text', 0, NULL);" + "SELECT cloudsync_alter_apply();"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_MISUSE) return false; + sqlite3_exec(db, "SELECT cloudsync_alter_clear('foo2');", NULL, NULL, NULL); + + // test current-dialect override with explicit SQL NULL default + sql = "SELECT cloudsync_alter_add_column('foo2', 'required_null_override', 'text', 0, 'fallback');" + "SELECT cloudsync_alter_add_column_sqlite('foo2', 'required_null_override', 'TEXT', 0, NULL);" + "SELECT cloudsync_alter_apply();"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_MISUSE) return false; + sqlite3_exec(db, "SELECT cloudsync_alter_clear('foo2');", NULL, NULL, NULL); + + // test raw SQL guardrails + sql = "SELECT cloudsync_alter_sql('BEGIN');"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_MISUSE) return false; + sql = "SELECT cloudsync_alter_sql('END');"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_MISUSE) return false; + sql = "SELECT cloudsync_alter_sql('ABORT');"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_MISUSE) return false; @@ -12313,6 +12407,929 @@ bool do_test_schema_hash_mismatch (int nclients, bool print_result, bool cleanup return result; } +bool do_test_migration_v1_create_augment_block (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-v1-create\"," + "\"ops\":[" + "{\"op\":\"createTable\",\"table\":\"notes\",\"columns\":[" + "{\"name\":\"id\",\"type\":\"text\",\"primaryKey\":true,\"nullable\":false}," + "{\"name\":\"body\",\"type\":\"text\",\"nullable\":false,\"default\":{\"type\":\"text\",\"value\":\"\"}}" + "]}," + "{\"op\":\"augmentTable\",\"table\":\"notes\",\"algorithm\":\"CLS\",\"initFlags\":0}," + "{\"op\":\"setBlockLww\",\"table\":\"notes\",\"column\":\"body\",\"delimiter\":\"\\n\"}" + "]" + "}');"; + + int rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v1_create: apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + rc = sqlite3_exec(db, "INSERT INTO notes (id, body) VALUES ('n1', 'a\nb\nc');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v1_create: insert failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t settings = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='notes' AND key='algo';"); + int64_t blocks = do_select_int(db, "SELECT count(*) FROM notes_cloudsync_blocks WHERE pk = cloudsync_pk_encode('n1');"); + int64_t migrations = do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id='mig-sqlite-v1-create';"); + int64_t pk_notnull = do_select_int(db, "SELECT count(*) FROM pragma_table_info('notes') WHERE name='id' AND pk > 0 AND \"notnull\" = 1;"); + result = (settings == 2 && blocks == 3 && migrations == 1 && pk_notnull == 1); + if (!result) { + printf("migration_v1_create: unexpected settings=%lld blocks=%lld migrations=%lld pk_notnull=%lld\n", + (long long)settings, (long long)blocks, (long long)migrations, (long long)pk_notnull); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_v1_add_column_idempotent (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE tasks (id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('tasks');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-v1-add-column\"," + "\"ops\":[{\"op\":\"addColumn\",\"table\":\"tasks\",\"column\":{" + "\"name\":\"description\",\"type\":\"text\",\"nullable\":false," + "\"default\":{\"type\":\"text\",\"value\":\"\"}}}]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v1_add: apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v1_add: idempotent apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t col_exists = do_select_int(db, "SELECT count(*) FROM pragma_table_info('tasks') WHERE name='description';"); + int64_t migrations = do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id='mig-sqlite-v1-add-column';"); + rc = sqlite3_exec(db, "INSERT INTO tasks (id, title, description) VALUES ('t1', 'title', 'desc');", NULL, NULL, NULL); + result = (col_exists == 1 && migrations == 1 && rc == SQLITE_OK); + if (!result) { + printf("migration_v1_add: col=%lld migrations=%lld rc=%d err=%s\n", + (long long)col_exists, (long long)migrations, rc, sqlite3_errmsg(db)); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_v1_rollback_on_failure (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE tasks (id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('tasks');" + "INSERT INTO tasks (id, title) VALUES ('t1', 'title');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-v1-fail\"," + "\"ops\":[{\"op\":\"addColumn\",\"table\":\"tasks\",\"column\":{" + "\"name\":\"must_fail\",\"type\":\"text\",\"nullable\":false}}]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc == SQLITE_OK) { + printf("migration_v1_fail: migration unexpectedly succeeded\n"); + goto cleanup; + } + + int64_t col_exists = do_select_int(db, "SELECT count(*) FROM pragma_table_info('tasks') WHERE name='must_fail';"); + int64_t migrations = do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id='mig-sqlite-v1-fail';"); + result = (col_exists == 0 && migrations == 0); + if (!result) { + printf("migration_v1_fail: rollback failed col=%lld migrations=%lld\n", + (long long)col_exists, (long long)migrations); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_v2_rename_column (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE docs (id TEXT PRIMARY KEY, body TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('docs');" + "INSERT INTO docs (id, body) VALUES ('d1', 'line1\nline2');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":2," + "\"migrationId\":\"mig-sqlite-v2-rename\"," + "\"requiredCapabilities\":[\"schema:destructive\"]," + "\"ops\":[{\"op\":\"renameColumn\",\"table\":\"docs\",\"from\":\"body\",\"to\":\"content\"}]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v2_rename: apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t col_exists = do_select_int(db, "SELECT count(*) FROM pragma_table_info('docs') WHERE name='content';"); + int64_t old_meta = do_select_int(db, "SELECT count(*) FROM docs_cloudsync WHERE col_name='body';"); + int64_t new_meta = do_select_int(db, "SELECT count(*) FROM docs_cloudsync WHERE col_name='content';"); + rc = sqlite3_exec(db, "UPDATE docs SET content='changed' WHERE id='d1';", NULL, NULL, NULL); + result = (col_exists == 1 && old_meta == 0 && new_meta > 0 && rc == SQLITE_OK); + if (!result) { + printf("migration_v2_rename: col=%lld old_meta=%lld new_meta=%lld rc=%d err=%s\n", + (long long)col_exists, (long long)old_meta, (long long)new_meta, rc, sqlite3_errmsg(db)); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_v2_rebuild_drop_column (void) { + sqlite3 *db = do_create_database(); + bool result = false; + char *site_before = NULL; + char *site_after = NULL; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE docs (id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT '', legacy TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('docs');" + "INSERT INTO docs (id, title, legacy) VALUES ('d1', 'title', 'legacy');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + site_before = do_select_text(db, "SELECT hex(site_id) FROM cloudsync_site_id WHERE rowid=0;"); + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":2," + "\"migrationId\":\"mig-sqlite-v2-rebuild\"," + "\"requiredCapabilities\":[\"schema:destructive\"]," + "\"ops\":[{\"op\":\"rebuildTableSync\",\"table\":\"docs\",\"algorithm\":\"CLS\",\"initFlags\":0," + "\"ddl\":[{\"op\":\"rawSql\",\"sql\":{\"sqlite\":[\"ALTER TABLE docs DROP COLUMN legacy\"]}}]}]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v2_rebuild: apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t legacy_exists = do_select_int(db, "SELECT count(*) FROM pragma_table_info('docs') WHERE name='legacy';"); + int64_t meta_exists = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='docs_cloudsync';"); + site_after = do_select_text(db, "SELECT hex(site_id) FROM cloudsync_site_id WHERE rowid=0;"); + rc = sqlite3_exec(db, "INSERT INTO docs (id, title) VALUES ('d2', 'new');", NULL, NULL, NULL); + int64_t row_count = do_select_int(db, "SELECT count(*) FROM docs;"); + bool site_preserved = site_before && site_after && strcmp(site_before, site_after) == 0; + result = (legacy_exists == 0 && meta_exists == 1 && row_count == 2 && rc == SQLITE_OK && site_preserved); + if (!result) { + printf("migration_v2_rebuild: legacy=%lld meta=%lld rows=%lld site_preserved=%d rc=%d err=%s\n", + (long long)legacy_exists, (long long)meta_exists, (long long)row_count, site_preserved, rc, sqlite3_errmsg(db)); + } +cleanup: + if (site_before) sqlite3_free(site_before); + if (site_after) sqlite3_free(site_after); + close_db(db); + return result; +} + +bool do_test_migration_v2_rebuild_rejects_malformed_ddl (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE docs (id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT '', legacy TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('docs');" + "INSERT INTO docs (id, title, legacy) VALUES ('d1', 'title', 'legacy');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":2," + "\"migrationId\":\"mig-sqlite-v2-rebuild-bad-ddl\"," + "\"requiredCapabilities\":[\"schema:destructive\"]," + "\"ops\":[{\"op\":\"rebuildTableSync\",\"table\":\"docs\",\"algorithm\":\"CLS\",\"initFlags\":0," + "\"ddl\":{\"op\":\"rawSql\",\"sql\":{\"sqlite\":[\"ALTER TABLE docs DROP COLUMN legacy\"]}}}]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc == SQLITE_OK) { + printf("migration_v2_rebuild_bad_ddl: migration unexpectedly succeeded\n"); + goto cleanup; + } + + int64_t legacy_exists = do_select_int(db, "SELECT count(*) FROM pragma_table_info('docs') WHERE name='legacy';"); + int64_t meta_exists = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='docs_cloudsync';"); + int64_t migrations = do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id='mig-sqlite-v2-rebuild-bad-ddl';"); + result = (legacy_exists == 1 && meta_exists == 1 && migrations == 0); + if (!result) { + printf("migration_v2_rebuild_bad_ddl: legacy=%lld meta=%lld migrations=%lld\n", + (long long)legacy_exists, (long long)meta_exists, (long long)migrations); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_v2_rebuild_rejects_malformed_block_lww (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE docs (id TEXT PRIMARY KEY, body TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('docs');" + "INSERT INTO docs (id, body) VALUES ('d1', 'line1\nline2');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":2," + "\"migrationId\":\"mig-sqlite-v2-rebuild-bad-block\"," + "\"requiredCapabilities\":[\"schema:destructive\"]," + "\"ops\":[{\"op\":\"rebuildTableSync\",\"table\":\"docs\",\"algorithm\":\"CLS\",\"initFlags\":0," + "\"blockLww\":{\"column\":\"body\",\"delimiter\":\"\\n\"}}]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc == SQLITE_OK) { + printf("migration_v2_rebuild_bad_block: migration unexpectedly succeeded\n"); + goto cleanup; + } + + int64_t meta_exists = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='docs_cloudsync';"); + int64_t migrations = do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id='mig-sqlite-v2-rebuild-bad-block';"); + int64_t row_count = do_select_int(db, "SELECT count(*) FROM docs WHERE id='d1' AND body='line1\nline2';"); + result = (meta_exists == 1 && migrations == 0 && row_count == 1); + if (!result) { + printf("migration_v2_rebuild_bad_block: meta=%lld migrations=%lld rows=%lld\n", + (long long)meta_exists, (long long)migrations, (long long)row_count); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_rejects_trailing_json (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-invalid-json\"," + "\"ops\":[{\"op\":\"createTable\",\"table\":\"invalid_json_notes\",\"columns\":[" + "{\"name\":\"id\",\"type\":\"text\",\"primaryKey\":true,\"nullable\":false}" + "]}]" + "} true');"; + + int rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc == SQLITE_OK) { + printf("migration_invalid_json: migration unexpectedly succeeded\n"); + goto cleanup; + } + + int64_t table_exists = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='invalid_json_notes';"); + int64_t migrations_exists = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='cloudsync_migrations';"); + result = (table_exists == 0 && migrations_exists == 0); + if (!result) { + printf("migration_invalid_json: table_exists=%lld migrations_exists=%lld\n", + (long long)table_exists, (long long)migrations_exists); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_v2_rebuild_valid_block_lww (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE docs (id TEXT PRIMARY KEY, body TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('docs');" + "INSERT INTO docs (id, body) VALUES ('d1', 'one\ntwo');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":2," + "\"migrationId\":\"mig-sqlite-v2-rebuild-good-block\"," + "\"requiredCapabilities\":[\"schema:destructive\"]," + "\"ops\":[{\"op\":\"rebuildTableSync\",\"table\":\"docs\",\"algorithm\":\"CLS\",\"initFlags\":0," + "\"blockLww\":[{\"column\":\"body\",\"delimiter\":\"\\n\"}]}]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v2_rebuild_good_block: apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + rc = sqlite3_exec(db, "UPDATE docs SET body='one\ntwo\nthree' WHERE id='d1';", NULL, NULL, NULL); + int64_t blocks = do_select_int(db, "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('d1');"); + int64_t settings = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='docs' AND col_name='body' AND key='algo' AND value='block';"); + int64_t migrations = do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id='mig-sqlite-v2-rebuild-good-block';"); + result = (rc == SQLITE_OK && blocks == 3 && settings == 1 && migrations == 1); + if (!result) { + printf("migration_v2_rebuild_good_block: rc=%d blocks=%lld settings=%lld migrations=%lld err=%s\n", + rc, (long long)blocks, (long long)settings, (long long)migrations, sqlite3_errmsg(db)); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_v2_rename_block_column (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE docs (id TEXT PRIMARY KEY, body TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('docs');" + "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');" + "INSERT INTO docs (id, body) VALUES ('d1', 'a\nb');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":2," + "\"migrationId\":\"mig-sqlite-v2-rename-block\"," + "\"requiredCapabilities\":[\"schema:destructive\"]," + "\"ops\":[{\"op\":\"renameColumn\",\"table\":\"docs\",\"from\":\"body\",\"to\":\"content\"}]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v2_rename_block: apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t content_col = do_select_int(db, "SELECT count(*) FROM pragma_table_info('docs') WHERE name='content';"); + int64_t old_meta = do_select_int(db, "SELECT count(*) FROM docs_cloudsync WHERE col_name='body' OR col_name LIKE 'body' || x'1f' || '%';"); + int64_t new_meta = do_select_int(db, "SELECT count(*) FROM docs_cloudsync WHERE col_name='content' OR col_name LIKE 'content' || x'1f' || '%';"); + int64_t old_blocks = do_select_int(db, "SELECT count(*) FROM docs_cloudsync_blocks WHERE col_name='body' OR col_name LIKE 'body' || x'1f' || '%';"); + int64_t new_blocks = do_select_int(db, "SELECT count(*) FROM docs_cloudsync_blocks WHERE col_name='content' OR col_name LIKE 'content' || x'1f' || '%';"); + int64_t setting = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='docs' AND col_name='content' AND key='algo' AND value='block';"); + rc = sqlite3_exec(db, "UPDATE docs SET content='a\nb\nc' WHERE id='d1';", NULL, NULL, NULL); + result = (rc == SQLITE_OK && content_col == 1 && old_meta == 0 && new_meta > 0 && old_blocks == 0 && new_blocks > 0 && setting == 1); + if (!result) { + printf("migration_v2_rename_block: rc=%d col=%lld old_meta=%lld new_meta=%lld old_blocks=%lld new_blocks=%lld setting=%lld err=%s\n", + rc, (long long)content_col, (long long)old_meta, (long long)new_meta, + (long long)old_blocks, (long long)new_blocks, (long long)setting, sqlite3_errmsg(db)); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_v2_drop_column_direct (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE docs (id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT '', legacy TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('docs');" + "INSERT INTO docs (id, title, legacy) VALUES ('d1', 'title', 'legacy');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":2," + "\"migrationId\":\"mig-sqlite-v2-drop-column\"," + "\"requiredCapabilities\":[\"schema:destructive\"]," + "\"ops\":[{\"op\":\"dropColumn\",\"table\":\"docs\",\"column\":\"legacy\"}]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v2_drop_column: apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t legacy_exists = do_select_int(db, "SELECT count(*) FROM pragma_table_info('docs') WHERE name='legacy';"); + int64_t legacy_meta = do_select_int(db, "SELECT count(*) FROM docs_cloudsync WHERE col_name='legacy';"); + rc = sqlite3_exec(db, "INSERT INTO docs (id, title) VALUES ('d2', 'new');", NULL, NULL, NULL); + int64_t row_count = do_select_int(db, "SELECT count(*) FROM docs;"); + result = (legacy_exists == 0 && legacy_meta == 0 && rc == SQLITE_OK && row_count == 2); + if (!result) { + printf("migration_v2_drop_column: legacy=%lld meta=%lld rows=%lld rc=%d err=%s\n", + (long long)legacy_exists, (long long)legacy_meta, (long long)row_count, rc, sqlite3_errmsg(db)); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_v2_drop_block_column_cleans_metadata (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE docs (id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT '', body TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('docs');" + "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');" + "INSERT INTO docs (id, title, body) VALUES ('d1', 'title', 'a\nb');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + int64_t blocks_before = do_select_int(db, "SELECT count(*) FROM docs_cloudsync_blocks WHERE col_name LIKE 'body' || char(31) || '%';"); + int64_t settings_before = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='docs' AND col_name='body';"); + if (blocks_before == 0 || settings_before == 0) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":2," + "\"migrationId\":\"mig-sqlite-v2-drop-block-column\"," + "\"requiredCapabilities\":[\"schema:destructive\"]," + "\"ops\":[{\"op\":\"dropColumn\",\"table\":\"docs\",\"column\":\"body\"}]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v2_drop_block_column: apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t body_exists = do_select_int(db, "SELECT count(*) FROM pragma_table_info('docs') WHERE name='body';"); + int64_t body_meta = do_select_int(db, "SELECT count(*) FROM docs_cloudsync WHERE col_name='body' OR col_name LIKE 'body' || char(31) || '%';"); + int64_t body_blocks = do_select_int(db, "SELECT count(*) FROM docs_cloudsync_blocks WHERE col_name LIKE 'body' || char(31) || '%';"); + int64_t body_settings = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='docs' AND col_name='body';"); + rc = sqlite3_exec(db, "UPDATE docs SET title='changed' WHERE id='d1';", NULL, NULL, NULL); + result = (body_exists == 0 && body_meta == 0 && body_blocks == 0 && body_settings == 0 && rc == SQLITE_OK); + if (!result) { + printf("migration_v2_drop_block_column: exists=%lld meta=%lld blocks=%lld settings=%lld rc=%d err=%s\n", + (long long)body_exists, (long long)body_meta, (long long)body_blocks, + (long long)body_settings, rc, sqlite3_errmsg(db)); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_v1_set_filter_and_set_column (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE tasks (id TEXT PRIMARY KEY, owner TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('tasks');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-v1-filter-column\"," + "\"ops\":[" + "{\"op\":\"setColumn\",\"table\":\"tasks\",\"column\":\"title\",\"key\":\"label\",\"value\":\"sync-title\"}," + "{\"op\":\"setFilter\",\"table\":\"tasks\",\"filter\":\"owner = ''alice''\"}" + "]" + "}');"; + + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_v1_filter_column: apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + rc = sqlite3_exec(db, + "INSERT INTO tasks (id, owner, title) VALUES ('t1', 'alice', 'visible');" + "INSERT INTO tasks (id, owner, title) VALUES ('t2', 'bob', 'hidden');", + NULL, NULL, NULL); + int64_t custom_setting = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='tasks' AND col_name='title' AND key='label' AND value='sync-title';"); + int64_t filter_setting = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='tasks' AND col_name='*' AND key='filter' AND value='owner = ''alice''';"); + int64_t alice_meta = do_select_int(db, "SELECT count(*) FROM tasks_cloudsync WHERE pk = cloudsync_pk_encode('t1');"); + int64_t bob_meta = do_select_int(db, "SELECT count(*) FROM tasks_cloudsync WHERE pk = cloudsync_pk_encode('t2');"); + result = (rc == SQLITE_OK && custom_setting == 1 && filter_setting == 1 && alice_meta > 0 && bob_meta == 0); + if (!result) { + printf("migration_v1_filter_column: rc=%d custom=%lld filter=%lld alice=%lld bob=%lld err=%s\n", + rc, (long long)custom_setting, (long long)filter_setting, + (long long)alice_meta, (long long)bob_meta, sqlite3_errmsg(db)); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_hash_guards (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE tasks (id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('tasks');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *bad_base = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-bad-base\"," + "\"baseSchemaHash\":\"1\"," + "\"ops\":[{\"op\":\"addColumn\",\"table\":\"tasks\",\"column\":{\"name\":\"base_fail\",\"type\":\"text\",\"nullable\":true}}]" + "}');"; + rc = sqlite3_exec(db, bad_base, NULL, NULL, NULL); + bool base_rejected = (rc != SQLITE_OK); + + const char *bad_target = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-bad-target\"," + "\"targetSchemaHash\":\"1\"," + "\"ops\":[{\"op\":\"addColumn\",\"table\":\"tasks\",\"column\":{\"name\":\"target_fail\",\"type\":\"text\",\"nullable\":true}}]" + "}');"; + rc = sqlite3_exec(db, bad_target, NULL, NULL, NULL); + bool target_rejected = (rc != SQLITE_OK); + + int64_t base_col = do_select_int(db, "SELECT count(*) FROM pragma_table_info('tasks') WHERE name='base_fail';"); + int64_t target_col = do_select_int(db, "SELECT count(*) FROM pragma_table_info('tasks') WHERE name='target_fail';"); + int64_t records = do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id IN ('mig-sqlite-bad-base', 'mig-sqlite-bad-target');"); + result = (base_rejected && target_rejected && base_col == 0 && target_col == 0 && records == 0); + if (!result) { + printf("migration_hash_guards: base_rejected=%d target_rejected=%d base_col=%lld target_col=%lld records=%lld\n", + base_rejected, target_rejected, (long long)base_col, (long long)target_col, (long long)records); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_raw_sql_dialect_validation (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + const char *success = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-raw-success\"," + "\"ops\":[{\"op\":\"rawSql\",\"sql\":{\"sqlite\":[\"CREATE TABLE raw_ok (id TEXT PRIMARY KEY NOT NULL)\"]}}]" + "}');"; + int rc = sqlite3_exec(db, success, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_raw_sql: success branch failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + const char *missing_dialect = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-raw-missing\"," + "\"ops\":[{\"op\":\"rawSql\",\"sql\":{\"postgresql\":[\"CREATE TABLE raw_missing (id TEXT PRIMARY KEY NOT NULL)\"]}}]" + "}');"; + rc = sqlite3_exec(db, missing_dialect, NULL, NULL, NULL); + bool missing_rejected = (rc != SQLITE_OK); + + const char *bad_item = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-raw-bad-item\"," + "\"ops\":[{\"op\":\"rawSql\",\"sql\":{\"sqlite\":[123]}}]" + "}');"; + rc = sqlite3_exec(db, bad_item, NULL, NULL, NULL); + bool item_rejected = (rc != SQLITE_OK); + + const char *tx_control = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-raw-tx\"," + "\"ops\":[{\"op\":\"rawSql\",\"sql\":{\"sqlite\":[\"COMMIT\"]}}]" + "}');"; + rc = sqlite3_exec(db, tx_control, NULL, NULL, NULL); + bool tx_rejected = (rc != SQLITE_OK); + + int64_t raw_ok = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='raw_ok';"); + int64_t raw_missing = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='raw_missing';"); + int64_t bad_records = do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id IN ('mig-sqlite-raw-missing', 'mig-sqlite-raw-bad-item', 'mig-sqlite-raw-tx');"); + result = (raw_ok == 1 && missing_rejected && item_rejected && tx_rejected && raw_missing == 0 && bad_records == 0); + if (!result) { + printf("migration_raw_sql: raw_ok=%lld missing_rejected=%d item_rejected=%d tx_rejected=%d raw_missing=%lld bad_records=%lld\n", + (long long)raw_ok, missing_rejected, item_rejected, tx_rejected, (long long)raw_missing, (long long)bad_records); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_rejects_non_boolean_column_flags (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + const char *bad_primary_key = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-bad-bool-pk\"," + "\"ops\":[{\"op\":\"createTable\",\"table\":\"bad_bool_pk_notes\",\"columns\":[" + "{\"name\":\"id\",\"type\":\"text\",\"primaryKey\":\"true\",\"nullable\":false}" + "]}]" + "}');"; + int rc = sqlite3_exec(db, bad_primary_key, NULL, NULL, NULL); + bool pk_rejected = (rc != SQLITE_OK); + + rc = sqlite3_exec(db, + "CREATE TABLE bad_bool_add_notes (id TEXT PRIMARY KEY NOT NULL, title TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('bad_bool_add_notes');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *bad_nullable = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-bad-bool-nullable\"," + "\"ops\":[{\"op\":\"addColumn\",\"table\":\"bad_bool_add_notes\",\"column\":{" + "\"name\":\"summary\",\"type\":\"text\",\"nullable\":\"false\",\"default\":{\"type\":\"text\",\"value\":\"\"}}}]" + "}');"; + rc = sqlite3_exec(db, bad_nullable, NULL, NULL, NULL); + bool nullable_rejected = (rc != SQLITE_OK); + + int64_t pk_table_exists = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='bad_bool_pk_notes';"); + int64_t add_col_exists = do_select_int(db, "SELECT count(*) FROM pragma_table_info('bad_bool_add_notes') WHERE name='summary';"); + int64_t bad_records = do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id IN ('mig-sqlite-bad-bool-pk', 'mig-sqlite-bad-bool-nullable');"); + result = (pk_rejected && nullable_rejected && pk_table_exists == 0 && add_col_exists == 0 && bad_records == 0); + if (!result) { + printf("migration_bad_bool: pk_rejected=%d nullable_rejected=%d pk_table=%lld add_col=%lld records=%lld\n", + pk_rejected, nullable_rejected, (long long)pk_table_exists, + (long long)add_col_exists, (long long)bad_records); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_create_table_rejects_existing_table (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, "CREATE TABLE notes (id TEXT PRIMARY KEY NOT NULL, title TEXT NOT NULL DEFAULT '');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + const char *payload = + "SELECT cloudsync_migration_apply('{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"mig-sqlite-existing-create\"," + "\"ops\":[{\"op\":\"createTable\",\"table\":\"notes\",\"columns\":[" + "{\"name\":\"id\",\"type\":\"text\",\"primaryKey\":true,\"nullable\":false}," + "{\"name\":\"body\",\"type\":\"text\",\"nullable\":true}" + "]}]" + "}');"; + rc = sqlite3_exec(db, payload, NULL, NULL, NULL); + bool rejected = (rc != SQLITE_OK); + int64_t body_col = do_select_int(db, "SELECT count(*) FROM pragma_table_info('notes') WHERE name='body';"); + int64_t migrations_table = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='cloudsync_migrations';"); + int64_t migration_record = migrations_table == 1 ? do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id='mig-sqlite-existing-create';") : 0; + + result = (rejected && body_col == 0 && migration_record == 0); + if (!result) { + printf("migration_create_existing: rejected=%d body_col=%lld record=%lld err=%s\n", + rejected, (long long)body_col, (long long)migration_record, sqlite3_errmsg(db)); + } + +cleanup: + close_db(db); + return result; +} + +bool do_test_migration_large_dynamic_json_parse (void) { + sqlite3 *db = do_create_database(); + bool result = false; + char *payload = NULL; + char *sql = NULL; + if (!db) return false; + + sqlite3_str *json = sqlite3_str_new(NULL); + if (!json) goto cleanup; + sqlite3_str_appendall(json, + "{" + "\"type\":\"cloudsync.schema.migration\"," + "\"formatVersion\":1," + "\"migrationId\":\"0197097c-8b35-7c11-8ed4-4e59ddfdb928\"," + "\"ops\":[{\"op\":\"createTable\",\"table\":\"wide_dynamic_notes\",\"columns\":[" + "{\"name\":\"id\",\"type\":\"text\",\"primaryKey\":true,\"nullable\":false}"); + for (int i = 0; i < 700; ++i) { + sqlite3_str_appendf(json, ",{\"name\":\"extra_%03d\",\"type\":\"text\",\"nullable\":true}", i); + } + sqlite3_str_appendall(json, "]}]}"); + payload = sqlite3_str_finish(json); + if (!payload) goto cleanup; + + sql = sqlite3_mprintf("SELECT cloudsync_migration_apply('%q');", payload); + if (!sql) goto cleanup; + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("migration_large_dynamic_json: apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t col_count = do_select_int(db, "SELECT count(*) FROM pragma_table_info('wide_dynamic_notes');"); + int64_t migration_record = do_select_int(db, "SELECT count(*) FROM cloudsync_migrations WHERE migration_id='0197097c-8b35-7c11-8ed4-4e59ddfdb928';"); + result = (col_count == 701 && migration_record == 1); + if (!result) { + printf("migration_large_dynamic_json: col_count=%lld migration_record=%lld\n", + (long long)col_count, (long long)migration_record); + } + +cleanup: + if (sql) sqlite3_free(sql); + if (payload) sqlite3_free(payload); + close_db(db); + return result; +} + +bool do_test_declarative_alter_apply_atomic_pending_save (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "CREATE TABLE cloudsync_pending_migration (x TEXT);" + "SELECT cloudsync_alter_create_table('atomic_notes');" + "SELECT cloudsync_alter_add_column('atomic_notes', 'id', 'text', 0);" + "SELECT cloudsync_alter_add_primary_key('atomic_notes', 'id');" + "SELECT cloudsync_alter_apply();", + NULL, NULL, NULL); + bool rejected = (rc != SQLITE_OK); + int64_t table_exists = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='atomic_notes';"); + int64_t pending_shape = do_select_int(db, "SELECT count(*) FROM pragma_table_info('cloudsync_pending_migration') WHERE name='x';"); + int64_t migrations_table = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='cloudsync_migrations';"); + int64_t migration_records = migrations_table == 1 ? do_select_int(db, "SELECT count(*) FROM cloudsync_migrations;") : 0; + + result = (rejected && table_exists == 0 && pending_shape == 1 && migration_records == 0); + if (!result) { + printf("declarative_alter_atomic: rejected=%d table=%lld pending_shape=%lld records=%lld err=%s\n", + rejected, (long long)table_exists, (long long)pending_shape, + (long long)migration_records, sqlite3_errmsg(db)); + } + + close_db(db); + return result; +} + +bool do_test_declarative_alter_builder (void) { + sqlite3 *db = do_create_database(); + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "SELECT cloudsync_alter_create_table('drafts');" + "SELECT cloudsync_alter_add_column('drafts', 'id', 'text', 0);" + "SELECT cloudsync_alter_add_primary_key('drafts', 'id');" + "SELECT cloudsync_alter_add_column('drafts', 'body', 'text', 0, '');" + "SELECT cloudsync_alter_add_column('drafts', 'rank', 'integer', 0, 0);" + "SELECT cloudsync_alter_augment_table('drafts');" + "SELECT cloudsync_alter_set_block_lww('drafts', 'body', '\n');" + "SELECT cloudsync_alter_sql('CREATE INDEX drafts_rank_idx ON drafts(rank)');" + "SELECT cloudsync_alter_sqlite('CREATE INDEX drafts_body_sqlite_idx ON drafts(body)');" + "SELECT cloudsync_alter_postgresql('CREATE INDEX drafts_body_pg_idx ON drafts(body)');" + "SELECT cloudsync_alter_apply();", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("declarative_alter: create/apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + rc = sqlite3_exec(db, "INSERT INTO drafts (id, body, rank) VALUES ('d1', 'a\nb', 5);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + int64_t table_exists = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='drafts';"); + int64_t pk_notnull = do_select_int(db, "SELECT count(*) FROM pragma_table_info('drafts') WHERE name='id' AND pk > 0 AND \"notnull\" = 1;"); + int64_t block_setting = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='drafts' AND col_name='body' AND key='algo' AND value='block';"); + int64_t block_rows = do_select_int(db, "SELECT count(*) FROM drafts_cloudsync_blocks WHERE pk = cloudsync_pk_encode('d1');"); + int64_t common_index = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='index' AND name='drafts_rank_idx';"); + int64_t sqlite_index = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='index' AND name='drafts_body_sqlite_idx';"); + int64_t pg_index = do_select_int(db, "SELECT count(*) FROM sqlite_master WHERE type='index' AND name='drafts_body_pg_idx';"); + int64_t pending_after_create = do_select_int(db, "SELECT count(*) FROM cloudsync_pending_migration WHERE uploaded_at IS NULL;"); + if (table_exists != 1 || pk_notnull != 1 || block_setting != 1 || block_rows != 2 || common_index != 1 || sqlite_index != 1 || pg_index != 0 || pending_after_create != 1) { + printf("declarative_alter: create checks failed table=%lld pk=%lld block_setting=%lld block_rows=%lld common_index=%lld sqlite_index=%lld pg_index=%lld pending=%lld\n", + (long long)table_exists, (long long)pk_notnull, (long long)block_setting, (long long)block_rows, + (long long)common_index, (long long)sqlite_index, (long long)pg_index, (long long)pending_after_create); + goto cleanup; + } + + rc = sqlite3_exec(db, + "SELECT cloudsync_alter_add_column('drafts', 'tag', 'text', 1);" + "SELECT cloudsync_alter_add_column_sqlite('drafts', 'tag', 'TEXT COLLATE NOCASE', 1, '''misc''');" + "SELECT cloudsync_alter_add_column_postgresql('drafts', 'tag', 'TEXT', 1, '''misc''');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("declarative_alter: dialect queue failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t preview_has_dialects = do_select_int(db, "SELECT instr(cloudsync_alter_preview(), '\"dialects\"') > 0;"); + rc = sqlite3_exec(db, "SELECT cloudsync_alter_apply();", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("declarative_alter: dialect apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t tag_exists = do_select_int(db, "SELECT count(*) FROM pragma_table_info('drafts') WHERE name='tag';"); + rc = sqlite3_exec(db, + "SELECT cloudsync_alter_set_filter_sqlite('drafts', 'rank >= 0');" + "SELECT cloudsync_alter_set_filter_postgresql('drafts', 'rank >= 0');" + "SELECT cloudsync_alter_apply();", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("declarative_alter: filter apply failed: %s\n", sqlite3_errmsg(db)); + goto cleanup; + } + + int64_t filter_saved = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='drafts' AND col_name='*' AND key='filter' AND value='rank >= 0';"); + int64_t pending_total = do_select_int(db, "SELECT count(*) FROM cloudsync_pending_migration WHERE uploaded_at IS NULL;"); + + result = (preview_has_dialects == 1 && tag_exists == 1 && filter_saved == 1 && pending_total == 3); + if (!result) { + printf("declarative_alter: final checks failed preview=%lld tag=%lld filter=%lld pending=%lld\n", + (long long)preview_has_dialects, (long long)tag_exists, (long long)filter_saved, (long long)pending_total); + } + +cleanup: + close_db(db); + return result; +} + int test_report(const char *description, bool result){ printf("%-30s %s\n", description, (result) ? "OK" : "FAILED"); return result ? 0 : 1; @@ -12492,6 +13509,26 @@ int main (int argc, const char * argv[]) { result += test_report("Delete/Resurrect Order:", do_test_delete_resurrect_ordering(3, print_result, cleanup_databases)); result += test_report("Large Composite PK Test:", do_test_large_composite_pk(2, print_result, cleanup_databases)); result += test_report("Schema Hash Mismatch:", do_test_schema_hash_mismatch(2, print_result, cleanup_databases)); + result += test_report("Migration V1 Create:", do_test_migration_v1_create_augment_block()); + result += test_report("Migration V1 Add Column:", do_test_migration_v1_add_column_idempotent()); + result += test_report("Migration V1 Rollback:", do_test_migration_v1_rollback_on_failure()); + result += test_report("Migration V2 Rename Col:", do_test_migration_v2_rename_column()); + result += test_report("Migration V2 Rebuild:", do_test_migration_v2_rebuild_drop_column()); + result += test_report("Migration V2 Bad DDL:", do_test_migration_v2_rebuild_rejects_malformed_ddl()); + result += test_report("Migration V2 Bad Block:", do_test_migration_v2_rebuild_rejects_malformed_block_lww()); + result += test_report("Migration Bad JSON:", do_test_migration_rejects_trailing_json()); + result += test_report("Migration V2 Block Rebuild:", do_test_migration_v2_rebuild_valid_block_lww()); + result += test_report("Migration V2 Rename Block:", do_test_migration_v2_rename_block_column()); + result += test_report("Migration V2 Drop Column:", do_test_migration_v2_drop_column_direct()); + result += test_report("Migration V2 Drop Block Column:", do_test_migration_v2_drop_block_column_cleans_metadata()); + result += test_report("Migration V1 Filter/Column:", do_test_migration_v1_set_filter_and_set_column()); + result += test_report("Migration Hash Guards:", do_test_migration_hash_guards()); + result += test_report("Migration Raw SQL:", do_test_migration_raw_sql_dialect_validation()); + result += test_report("Migration Bad Bool Flags:", do_test_migration_rejects_non_boolean_column_flags()); + result += test_report("Migration Existing Table:", do_test_migration_create_table_rejects_existing_table()); + result += test_report("Migration Large JSON:", do_test_migration_large_dynamic_json_parse()); + result += test_report("Declarative Alter API:", do_test_declarative_alter_builder()); + result += test_report("Declarative Alter Atomic:", do_test_declarative_alter_apply_atomic_pending_save()); result += test_report("Stale Table Settings:", do_test_stale_table_settings(cleanup_databases)); result += test_report("Stale Table Settings Dropped Meta:", do_test_stale_table_settings_dropped_meta(cleanup_databases)); result += test_report("DBVersion Rebuild Error:", do_test_dbversion_rebuild_error()); From 2284480bf2fd1e9c02e53b50bf296e143c2dbe4f Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Tue, 28 Apr 2026 16:16:42 -0600 Subject: [PATCH 2/3] test: cover schema migration replay regressions Add focused migration unit coverage for dialect override behavior, generated migration JSON replay across simulated devices, and row payload retry after applying the missing migration. Fix the dialect override builder so SQLite/PostgreSQL overrides no longer mutate the portable nullable flag and replacing an override clears stale defaultSql state before preview/apply. Allow PostgreSQL declarative alters to accept non-text default arguments through an anyelement overload and stringify defaults before passing them to the shared migration builder. Validate with SQLite migration tests and PostgreSQL schema migration coverage, including non-text defaults for integer, boolean, and json columns. --- src/migration.c | 7 +- src/postgresql/cloudsync.sql.in | 5 + src/postgresql/cloudsync_postgresql.c | 17 +- .../migrations/cloudsync--1.0--1.1.sql | 5 + test/postgresql/52_schema_migrations.sql | 59 +++++ test/unit.c | 203 ++++++++++++++++++ 6 files changed, 293 insertions(+), 3 deletions(-) diff --git a/src/migration.c b/src/migration.c index cd88e6c1..1784761d 100644 --- a/src/migration.c +++ b/src/migration.c @@ -1837,7 +1837,6 @@ int cloudsync_alter_add_column_dialect(cloudsync_context *data, const char *tabl if (!table || !column || !dialect || !type_sql) return cloudsync_set_error(data, "cloudsync_alter_add_column_dialect requires table, column, dialect, and type SQL", DBRES_MISUSE); pending_alter_op *op = pending_find_add_column(data, table, column); if (!op) return cloudsync_set_error(data, "Dialect override requires a pending addColumn operation", DBRES_MISUSE); - op->flag = nullable; char **type_slot = NULL; char **default_slot = NULL; if (strcasecmp(dialect, "sqlite") == 0) { @@ -1851,10 +1850,14 @@ int cloudsync_alter_add_column_dialect(cloudsync_context *data, const char *tabl } if (*type_slot) cloudsync_memory_free(*type_slot); *type_slot = cloudsync_string_dup(type_sql); + if (*default_slot) { + cloudsync_memory_free(*default_slot); + *default_slot = NULL; + } if (has_default_sql) { - if (*default_slot) cloudsync_memory_free(*default_slot); *default_slot = default_sql ? cloudsync_string_dup(default_sql) : cloudsync_string_dup("NULL"); } + (void)nullable; return *type_slot ? DBRES_OK : DBRES_NOMEM; } diff --git a/src/postgresql/cloudsync.sql.in b/src/postgresql/cloudsync.sql.in index a13c78a0..fef8ae11 100644 --- a/src/postgresql/cloudsync.sql.in +++ b/src/postgresql/cloudsync.sql.in @@ -136,6 +136,11 @@ RETURNS boolean AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column' LANGUAGE C VOLATILE; +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column(table_name text, column_name text, logical_type text, nullable boolean, default_value anyelement) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column' +LANGUAGE C VOLATILE; + CREATE OR REPLACE FUNCTION cloudsync_alter_add_column_sqlite(table_name text, column_name text, type_sql text, nullable boolean) RETURNS boolean AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column_sqlite' diff --git a/src/postgresql/cloudsync_postgresql.c b/src/postgresql/cloudsync_postgresql.c index a1d35cee..f47a3ccc 100644 --- a/src/postgresql/cloudsync_postgresql.c +++ b/src/postgresql/cloudsync_postgresql.c @@ -55,6 +55,21 @@ PG_MODULE_MAGIC; // External declaration Datum database_column_datum (dbvm_t *vm, int index); +static char *pg_argument_to_cstring(PG_FUNCTION_ARGS, int argno) { + Oid argtype = get_fn_expr_argtype(fcinfo->flinfo, argno); + Oid typoutput = InvalidOid; + bool typisvarlena = false; + + if (!OidIsValid(argtype)) { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Unable to determine argument type"))); + } + + getTypeOutputInfo(argtype, &typoutput, &typisvarlena); + return OidOutputFunctionCall(typoutput, PG_GETARG_DATUM(argno)); +} + // MARK: - Context Management - // Global context stored per backend @@ -1026,7 +1041,7 @@ Datum pg_cloudsync_alter_add_column(PG_FUNCTION_ARGS) { const char *default_value = NULL; bool has_default = PG_NARGS() >= 5; if (has_default) { - if (!PG_ARGISNULL(4)) default_value = text_to_cstring(PG_GETARG_TEXT_PP(4)); + if (!PG_ARGISNULL(4)) default_value = pg_argument_to_cstring(fcinfo, 4); } cloudsync_context *data = get_cloudsync_context(); int rc = DBRES_OK; diff --git a/src/postgresql/migrations/cloudsync--1.0--1.1.sql b/src/postgresql/migrations/cloudsync--1.0--1.1.sql index 5449eee4..ce44a8b6 100644 --- a/src/postgresql/migrations/cloudsync--1.0--1.1.sql +++ b/src/postgresql/migrations/cloudsync--1.0--1.1.sql @@ -19,6 +19,11 @@ RETURNS boolean AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column' LANGUAGE C VOLATILE; +CREATE OR REPLACE FUNCTION cloudsync_alter_add_column(table_name text, column_name text, logical_type text, nullable boolean, default_value anyelement) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column' +LANGUAGE C VOLATILE; + CREATE OR REPLACE FUNCTION cloudsync_alter_add_column_sqlite(table_name text, column_name text, type_sql text, nullable boolean) RETURNS boolean AS 'MODULE_PATHNAME', 'pg_cloudsync_alter_add_column_sqlite' diff --git a/test/postgresql/52_schema_migrations.sql b/test/postgresql/52_schema_migrations.sql index edb87183..27d93497 100644 --- a/test/postgresql/52_schema_migrations.sql +++ b/test/postgresql/52_schema_migrations.sql @@ -1026,6 +1026,65 @@ SELECT (:fail::int + 1) AS fail \gset SELECT (:fail::int + 1) AS fail \gset \endif +-- ============================================================================ +-- Declarative alter accepts non-text default arguments +-- ============================================================================ + +CREATE TABLE pg_default_tasks ( + id UUID PRIMARY KEY +); +SELECT cloudsync_init('pg_default_tasks', 'CLS', 0) AS _init_pg_default_tasks \gset + +SELECT cloudsync_alter_add_column('pg_default_tasks', 'rank', 'integer', false, 0) AS alter_default_rank_ok \gset +SELECT cloudsync_alter_add_column('pg_default_tasks', 'done', 'boolean', false, false) AS alter_default_done_ok \gset +SELECT cloudsync_alter_add_column('pg_default_tasks', 'metadata', 'json', false, '{}'::jsonb) AS alter_default_metadata_ok \gset +SELECT cloudsync_alter_apply() IS NOT NULL AS alter_non_text_defaults_apply_ok \gset + +INSERT INTO pg_default_tasks (id) +VALUES ('cccccccc-cccc-cccc-cccc-cccccccccccc'); + +SELECT count(*) = 3 AS alter_non_text_defaults_cols_ok +FROM information_schema.columns +WHERE table_name = 'pg_default_tasks' + AND column_name IN ('rank', 'done', 'metadata') + AND is_nullable = 'NO' \gset + +SELECT rank = 0 AND done = false AND metadata = '{}'::jsonb AS alter_non_text_defaults_values_ok +FROM pg_default_tasks +WHERE id = 'cccccccc-cccc-cccc-cccc-cccccccccccc' \gset + +\if :alter_default_rank_ok +\if :alter_default_done_ok +\if :alter_default_metadata_ok +\if :alter_non_text_defaults_apply_ok +\if :alter_non_text_defaults_cols_ok +\if :alter_non_text_defaults_values_ok +\echo [PASS] (:testid) Declarative alter accepted non-text default arguments +\else +\echo [FAIL] (:testid) Declarative alter non-text defaults did not populate inserted row +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative alter non-text defaults did not create NOT NULL columns +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative alter non-text defaults apply failed +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative alter JSON default argument failed +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative alter boolean default argument failed +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) Declarative alter integer default argument failed +SELECT (:fail::int + 1) AS fail \gset +\endif + -- ============================================================================ -- Hash guards reject and roll back migrations -- ============================================================================ diff --git a/test/unit.c b/test/unit.c index 9664cd23..8ccfcd89 100644 --- a/test/unit.c +++ b/test/unit.c @@ -13218,6 +13218,205 @@ bool do_test_migration_large_dynamic_json_parse (void) { return result; } +static bool do_apply_migration_json(sqlite3 *db, const char *payload) { + if (!db || !payload) return false; + char *sql = sqlite3_mprintf("SELECT cloudsync_migration_apply('%q');", payload); + if (!sql) return false; + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("apply_migration_json: %s\n", sqlite3_errmsg(db)); + } + sqlite3_free(sql); + return rc == SQLITE_OK; +} + +static unsigned char *do_select_payload_blob(sqlite3 *db, bool only_local, int *payload_len) { + sqlite3_stmt *stmt = NULL; + unsigned char *copy = NULL; + if (payload_len) *payload_len = 0; + const char *sql = only_local + ? "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes WHERE site_id=cloudsync_siteid();" + : "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes;"; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW || sqlite3_column_type(stmt, 0) == SQLITE_NULL) goto cleanup; + + int len = sqlite3_column_bytes(stmt, 0); + const unsigned char *blob = sqlite3_column_blob(stmt, 0); + if (!blob || len <= 0) goto cleanup; + copy = sqlite3_malloc(len); + if (!copy) goto cleanup; + memcpy(copy, blob, len); + if (payload_len) *payload_len = len; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + return copy; +} + +static bool do_apply_payload_blob(sqlite3 *db, const unsigned char *payload, int payload_len, bool print_error_msg) { + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc = sqlite3_prepare_v2(db, "SELECT cloudsync_payload_decode(?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_blob(stmt, 1, payload, payload_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_step(stmt); + result = (rc == SQLITE_ROW); + +cleanup: + if (!result && print_error_msg) printf("apply_payload_blob: %s\n", sqlite3_errmsg(db)); + if (stmt) sqlite3_finalize(stmt); + return result; +} + +bool do_test_declarative_alter_dialect_nullable_is_portable (void) { + sqlite3 *db = do_create_database(); + char *preview = NULL; + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "SELECT cloudsync_alter_add_column('dialect_notes', 'tag', 'text', 0, '');" + "SELECT cloudsync_alter_add_column_sqlite('dialect_notes', 'tag', 'TEXT', 1);", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + preview = do_select_text(db, "SELECT cloudsync_alter_preview();"); + result = preview && strstr(preview, "\"nullable\":false") != NULL && strstr(preview, "\"nullable\":true") == NULL; + if (!result) { + printf("alter_dialect_nullable: unexpected preview=%s\n", preview ? preview : "NULL"); + } + +cleanup: + if (preview) sqlite3_free(preview); + close_db(db); + return result; +} + +bool do_test_declarative_alter_dialect_replacement_clears_default (void) { + sqlite3 *db = do_create_database(); + char *preview = NULL; + bool result = false; + if (!db) return false; + + int rc = sqlite3_exec(db, + "SELECT cloudsync_alter_add_column('dialect_notes', 'tag', 'text', 1);" + "SELECT cloudsync_alter_add_column_sqlite('dialect_notes', 'tag', 'TEXT', 1, '''stale''');" + "SELECT cloudsync_alter_add_column_sqlite('dialect_notes', 'tag', 'TEXT COLLATE NOCASE', 1);", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + preview = do_select_text(db, "SELECT cloudsync_alter_preview();"); + result = preview && + strstr(preview, "\"typeSql\":\"TEXT COLLATE NOCASE\"") != NULL && + strstr(preview, "defaultSql") == NULL && + strstr(preview, "stale") == NULL; + if (!result) { + printf("alter_dialect_replace: unexpected preview=%s\n", preview ? preview : "NULL"); + } + +cleanup: + if (preview) sqlite3_free(preview); + close_db(db); + return result; +} + +bool do_test_generated_migration_json_roundtrip_two_devices (void) { + sqlite3 *db[2] = {NULL, NULL}; + char *payload = NULL; + bool result = false; + + db[0] = do_create_database(); + db[1] = do_create_database(); + if (!db[0] || !db[1]) goto cleanup; + + int rc = sqlite3_exec(db[0], + "SELECT cloudsync_alter_create_table('roundtrip_notes');" + "SELECT cloudsync_alter_add_column('roundtrip_notes', 'id', 'text', 0);" + "SELECT cloudsync_alter_add_primary_key('roundtrip_notes', 'id');" + "SELECT cloudsync_alter_add_column('roundtrip_notes', 'body', 'text', 0, '');" + "SELECT cloudsync_alter_add_column('roundtrip_notes', 'rank', 'integer', 0, 0);" + "SELECT cloudsync_alter_augment_table('roundtrip_notes');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + payload = do_select_text(db[0], "SELECT cloudsync_alter_preview();"); + if (!payload) goto cleanup; + + rc = sqlite3_exec(db[0], "SELECT cloudsync_alter_apply();", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + if (!do_apply_migration_json(db[1], payload)) goto cleanup; + + rc = sqlite3_exec(db[0], "INSERT INTO roundtrip_notes (id, body, rank) VALUES ('n1', 'hello', 7);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + if (!do_merge_using_payload(db[0], db[1], true, true)) goto cleanup; + + int64_t cols0 = do_select_int(db[0], "SELECT count(*) FROM pragma_table_info('roundtrip_notes') WHERE name IN ('id', 'body', 'rank');"); + int64_t cols1 = do_select_int(db[1], "SELECT count(*) FROM pragma_table_info('roundtrip_notes') WHERE name IN ('id', 'body', 'rank');"); + int64_t row1 = do_select_int(db[1], "SELECT count(*) FROM roundtrip_notes WHERE id='n1' AND body='hello' AND rank=7;"); + int64_t meta1 = do_select_int(db[1], "SELECT count(*) FROM roundtrip_notes_cloudsync WHERE pk=cloudsync_pk_encode('n1');"); + result = (cols0 == 3 && cols1 == 3 && row1 == 1 && meta1 > 0); + if (!result) { + printf("migration_roundtrip_two_devices: cols0=%lld cols1=%lld row1=%lld meta1=%lld\n", + (long long)cols0, (long long)cols1, (long long)row1, (long long)meta1); + } + +cleanup: + if (payload) sqlite3_free(payload); + for (int i = 0; i < 2; ++i) if (db[i]) close_db(db[i]); + return result; +} + +bool do_test_row_payload_retry_after_migration_json (void) { + sqlite3 *db[2] = {NULL, NULL}; + char *migration_payload = NULL; + unsigned char *row_payload = NULL; + int row_payload_len = 0; + bool result = false; + + db[0] = do_create_database(); + db[1] = do_create_database(); + if (!db[0] || !db[1]) goto cleanup; + + for (int i = 0; i < 2; ++i) { + int rc = sqlite3_exec(db[i], + "CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT NOT NULL DEFAULT '');" + "SELECT cloudsync_init('tasks');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + } + + int rc = sqlite3_exec(db[0], + "SELECT cloudsync_alter_add_column('tasks', 'description', 'text', 0, '');" + "SELECT cloudsync_alter_apply();" + "INSERT INTO tasks (id, title, description) VALUES ('t1', 'title', 'desc');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + migration_payload = do_select_text(db[0], "SELECT payload FROM cloudsync_pending_migration WHERE uploaded_at IS NULL ORDER BY created_at, migration_id LIMIT 1;"); + row_payload = do_select_payload_blob(db[0], true, &row_payload_len); + if (!migration_payload || !row_payload) goto cleanup; + + bool rejected_before_migration = !do_apply_payload_blob(db[1], row_payload, row_payload_len, false); + if (!do_apply_migration_json(db[1], migration_payload)) goto cleanup; + bool applied_after_migration = do_apply_payload_blob(db[1], row_payload, row_payload_len, true); + int64_t row_count = do_select_int(db[1], "SELECT count(*) FROM tasks WHERE id='t1' AND title='title' AND description='desc';"); + + result = rejected_before_migration && applied_after_migration && row_count == 1; + if (!result) { + printf("row_payload_retry_after_migration: rejected=%d applied=%d rows=%lld\n", + rejected_before_migration, applied_after_migration, (long long)row_count); + } + +cleanup: + if (row_payload) sqlite3_free(row_payload); + if (migration_payload) sqlite3_free(migration_payload); + for (int i = 0; i < 2; ++i) if (db[i]) close_db(db[i]); + return result; +} + bool do_test_declarative_alter_apply_atomic_pending_save (void) { sqlite3 *db = do_create_database(); bool result = false; @@ -13527,6 +13726,10 @@ int main (int argc, const char * argv[]) { result += test_report("Migration Bad Bool Flags:", do_test_migration_rejects_non_boolean_column_flags()); result += test_report("Migration Existing Table:", do_test_migration_create_table_rejects_existing_table()); result += test_report("Migration Large JSON:", do_test_migration_large_dynamic_json_parse()); + result += test_report("Migration Dialect Nullable:", do_test_declarative_alter_dialect_nullable_is_portable()); + result += test_report("Migration Dialect Replace:", do_test_declarative_alter_dialect_replacement_clears_default()); + result += test_report("Migration JSON Two Devices:", do_test_generated_migration_json_roundtrip_two_devices()); + result += test_report("Migration Payload Retry:", do_test_row_payload_retry_after_migration_json()); result += test_report("Declarative Alter API:", do_test_declarative_alter_builder()); result += test_report("Declarative Alter Atomic:", do_test_declarative_alter_apply_atomic_pending_save()); result += test_report("Stale Table Settings:", do_test_stale_table_settings(cleanup_databases)); From 46339f77c8a50112e1dfcd4182cef72197d14dbf Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Wed, 29 Apr 2026 08:35:32 +0200 Subject: [PATCH 3/3] Added support for first sync --- docs/internal/schema-migrations.md | 53 +++++-- examples/schema-migrations/README.md | 15 +- .../schema-migrations/client-to-server.sql | 10 +- src/network/network.c | 149 +++++++++++++----- test/integration.c | 8 +- test/postgresql/52_schema_migrations.sql | 53 +++++++ test/unit.c | 51 ++++++ 7 files changed, 280 insertions(+), 59 deletions(-) diff --git a/docs/internal/schema-migrations.md b/docs/internal/schema-migrations.md index 0c5ddfac..2993bf50 100644 --- a/docs/internal/schema-migrations.md +++ b/docs/internal/schema-migrations.md @@ -46,25 +46,56 @@ SELECT cloudsync_alter_apply(); ``` `cloudsync_alter_apply()` applies the queued migration locally and stores the -generated payload in `cloudsync_pending_migration`. After that, an authorized -client uploads it with: +generated payload in `cloudsync_pending_migration`. On the next +`cloudsync_network_sync()` or `cloudsync_network_send_changes()`, the network +layer uploads every pending schema migration before it sends row data: ```sql -SELECT cloudsync_network_migration_upload(); SELECT cloudsync_network_sync(); ``` -The zero-argument upload form uploads the next pending local migration and marks -it uploaded only after the backend returns valid JSON. The one-argument form is -still available for custom backends or tests: +The zero-argument upload form is still available when an application wants to +publish schema separately from row data. It uploads the next pending local +migration and marks it uploaded only after the backend returns an accepted +response. The one-argument form is available for custom backends or tests: ```sql SELECT cloudsync_network_migration_upload(:json_payload); ``` -While a local migration is pending upload, `cloudsync_network_send_changes()` -returns an error instead of sending row changes. This prevents data produced -with a new local schema from reaching a server that has not accepted that schema. +If a pending migration upload fails, row changes are not sent. This prevents +data produced with a new local schema from reaching a server that has not +accepted that schema. + +## Initial Schema Sync + +The first version of a database is distributed with the same migration protocol +used for later schema changes. There is no separate bootstrap format. + +Client-to-cloud first sync: + +1. The client must have `database_id` configured and must sync with an API key + that is allowed to initiate schema migrations. +2. The app defines the first schema with `cloudsync_alter_create_table()`, + `cloudsync_alter_add_column()`, `cloudsync_alter_augment_table()`, and any + optional commands such as `cloudsync_alter_set_block_lww()`. +3. `cloudsync_alter_apply()` creates the local tables, records the schema epoch, + and stores a pending upload in `cloudsync_pending_migration`. +4. The app may insert initial data. +5. `cloudsync_network_sync()` uploads the pending schema migration first. The + backend applies it to the cloud database, records it in the per-`database_id` + schema log, and only then can row data be uploaded. + +Cloud-to-client first sync: + +1. The backend applies and records the initial migration for the `database_id`. +2. A new SQLite client only needs `database_id` and a valid API key. +3. `cloudsync_network_sync()` detects that the local database has no augmented + tables, calls the schema check endpoint, applies the first migration locally, + and then continues with normal row download. + +An empty client that has no local schema and no server-side migration simply +returns an empty sync result. ## Declarative API @@ -259,7 +290,7 @@ Client-originated migration: 1. Application queues operations with `cloudsync_alter_*`. 2. Application calls `cloudsync_alter_apply()`. 3. Extension applies the migration locally and writes `cloudsync_pending_migration`. -4. Application or `cloudsync_network_sync()` uploads the pending migration. +4. `cloudsync_network_sync()` uploads the pending migration before row changes; applications may call `cloudsync_network_migration_upload()` explicitly when they want a separate schema publish step. 5. Backend authorizes the API key, applies the payload to the cloud database, records the migration, and returns success. 6. Client sends row changes after the pending migration is uploaded. @@ -286,7 +317,7 @@ Empty client first sync: - A migration id is idempotent through `cloudsync_migrations`. - Explicit hash guards are enforced when present. - Raw SQL runs inside the same savepoint as portable operations. -- Row changes are not uploaded while `cloudsync_pending_migration` contains an unuploaded migration. +- Row changes are uploaded only after all pending local schema migrations have been accepted by the backend. - V2 migrations should be blocked by the backend when stale/offline clients may still upload incompatible old-epoch payloads, unless the backend has an explicit rejection or translation policy. ## Tests diff --git a/examples/schema-migrations/README.md b/examples/schema-migrations/README.md index 5c679dfd..3e635d8b 100644 --- a/examples/schema-migrations/README.md +++ b/examples/schema-migrations/README.md @@ -21,16 +21,27 @@ SELECT cloudsync_alter_add_column('notes', 'updated_at', 'timestamp', false, '19 SELECT cloudsync_alter_augment_table('notes', 'CLS', 1); SELECT cloudsync_alter_set_block_lww('notes', 'body', char(10)); SELECT cloudsync_alter_apply(); -SELECT cloudsync_network_migration_upload(); SELECT cloudsync_network_sync(); ``` -`cloudsync_alter_preview()` can be used before `cloudsync_alter_apply()` to inspect the generated payload. After apply, the payload is saved in `cloudsync_pending_migration`; `cloudsync_network_migration_upload()` uploads the next pending migration and marks it uploaded on success. +`cloudsync_alter_preview()` can be used before `cloudsync_alter_apply()` to inspect the generated payload. After apply, the payload is saved in `cloudsync_pending_migration`; `cloudsync_network_sync()` uploads pending migrations before it sends row changes. `cloudsync_network_migration_upload()` is still available when schema should be published separately from data. The backend should authorize the API key, apply the payload to the cloud database, append it to the schema migration log for the `database_id`, and distribute it to other clients through `schema/check` or `schema/download`. `client-to-server.sql` contains the same flow as an executable SQLite example. `client-to-server-v1.json` is the manual JSON equivalent for backend tests or custom tooling; application code should normally let `cloudsync_alter_apply()` generate that payload. +## Initial Database Sync + +The first version of a database uses the same flow. A schema-capable client +queues the first `createTable` migration, calls `cloudsync_alter_apply()`, +optionally inserts initial rows, and then calls `cloudsync_network_sync()`. +The schema upload is accepted by the backend before any row payload is sent. + +For cloud-to-client bootstrap, the backend records the initial migration first. +A new SQLite client with only `database_id` and an API key calls +`cloudsync_network_sync()`; the client downloads the first schema migration, +creates/augments the tables, and then downloads data normally. + ## Server-Originated V2 Migration The backend applies and records a payload such as `server-to-client-v2.json`, then a client can update itself before receiving data: diff --git a/examples/schema-migrations/client-to-server.sql b/examples/schema-migrations/client-to-server.sql index 5a56f596..e4045cb7 100644 --- a/examples/schema-migrations/client-to-server.sql +++ b/examples/schema-migrations/client-to-server.sql @@ -2,7 +2,7 @@ -- -- Run this on an authorized SQLite client. The extension generates the JSON -- migration payload, applies it locally, stores it in cloudsync_pending_migration, --- and uploads it with cloudsync_network_migration_upload(). +-- and uploads it automatically before row data on the next cloudsync_network_sync(). SELECT cloudsync_alter_create_table('notes'); SELECT cloudsync_alter_add_column('notes', 'id', 'text', false); @@ -17,5 +17,11 @@ SELECT cloudsync_alter_set_block_lww('notes', 'body', char(10)); SELECT cloudsync_alter_preview(); SELECT cloudsync_alter_apply(); -SELECT cloudsync_network_migration_upload(); + +-- Optional initial data can be inserted here. cloudsync_network_sync() uploads +-- the pending schema migration first and sends row data only after the backend +-- accepts that schema. +INSERT INTO notes (id, title, body, updated_at) +VALUES (cloudsync_uuid(), 'First note', 'Created before first sync', '1970-01-01T00:00:00Z'); + SELECT cloudsync_network_sync(); diff --git a/src/network/network.c b/src/network/network.c index 6ec80866..32051fa5 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -77,6 +77,7 @@ typedef struct { } network_read_data; static int cloudsync_network_migration_check_internal(sqlite3_context *context, char **result_json, char **err_out); +static int cloudsync_network_migration_upload_next_pending(sqlite3_context *context, char **result_json, char **err_out); static bool network_migration_apply_error(const char *message); static void network_result_to_sqlite_error(sqlite3_context *context, NETWORK_RESULT res, const char *default_error_message); network_data *cloudsync_network_data(sqlite3_context *context); @@ -979,71 +980,124 @@ void cloudsync_network_migration_download(sqlite3_context *context, int argc, sq } void cloudsync_network_migration_upload(sqlite3_context *context, int argc, sqlite3_value **argv) { - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); network_data *netdata = cloudsync_network_data(context); if (!netdata || !netdata->schema_upload_endpoint) { sqlite3_result_error(context, "Unable to retrieve CloudSync schema migration upload endpoint.", -1); return; } - bool from_pending = false; - char *pending_id = NULL; - char *pending_payload = NULL; const char *payload = NULL; if (argc == 0) { - pending_id = cloudsync_pending_migration_next_id(data); - if (!pending_id) { - sqlite3_result_error(context, "No pending schema migration to upload.", -1); + char *result = NULL; + int rc = cloudsync_network_migration_upload_next_pending(context, &result, NULL); + if (rc == SQLITE_OK && result) sqlite3_result_text(context, result, -1, cloudsync_memory_free); + else if (rc == SQLITE_OK) sqlite3_result_text(context, "{\"status\":\"uploaded\"}", -1, SQLITE_TRANSIENT); + } else { + payload = (const char *)sqlite3_value_text(argv[0]); + + if (!payload || payload[0] == '\0') { + sqlite3_result_error(context, "cloudsync_network_migration_upload expects a JSON text payload.", -1); return; } - pending_payload = cloudsync_pending_migration_payload(data, pending_id); - if (!pending_payload) { - sqlite3_result_error(context, "Unable to load pending schema migration payload.", -1); - cloudsync_memory_free(pending_id); + if (!json_is_valid_root_object(payload, strlen(payload))) { + sqlite3_result_error(context, "cloudsync_network_migration_upload expects a valid JSON object payload.", -1); return; } - payload = pending_payload; - from_pending = true; - } else { - payload = (const char *)sqlite3_value_text(argv[0]); + + NETWORK_RESULT res = network_receive_buffer(netdata, netdata->schema_upload_endpoint, netdata->authentication, true, true, (char *)payload, CLOUDSYNC_HEADER_SQLITECLOUD); + if (network_validate_json_response(context, &res, "CloudSync schema migration upload endpoint", NULL)) { + char *upload_error = network_migration_upload_error_message(&res); + if (res.code == CLOUDSYNC_NETWORK_ERROR) { + network_set_sqlite_result(context, &res); + } else if (upload_error) { + sqlite3_result_error(context, upload_error, -1); + } else { + network_set_sqlite_result(context, &res); + } + if (upload_error) cloudsync_memory_free(upload_error); + } + network_result_cleanup(&res); } +} - if (!payload || payload[0] == '\0') { - sqlite3_result_error(context, "cloudsync_network_migration_upload expects a JSON text payload.", -1); - if (pending_id) cloudsync_memory_free(pending_id); - if (pending_payload) cloudsync_memory_free(pending_payload); - return; +static int cloudsync_network_migration_upload_next_pending(sqlite3_context *context, char **result_json, char **err_out) { + if (result_json) *result_json = NULL; + if (err_out) *err_out = NULL; + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = cloudsync_network_data(context); + if (!netdata || !netdata->schema_upload_endpoint) { + const char *message = "Unable to retrieve CloudSync schema migration upload endpoint."; + if (err_out) *err_out = cloudsync_string_dup(message); + else sqlite3_result_error(context, message, -1); + return SQLITE_ERROR; } - if (!json_is_valid_root_object(payload, strlen(payload))) { - sqlite3_result_error(context, "cloudsync_network_migration_upload expects a valid JSON object payload.", -1); - if (pending_id) cloudsync_memory_free(pending_id); - if (pending_payload) cloudsync_memory_free(pending_payload); - return; + + char *pending_id = cloudsync_pending_migration_next_id(data); + if (!pending_id) { + const char *message = "No pending schema migration to upload."; + if (err_out) *err_out = cloudsync_string_dup(message); + else sqlite3_result_error(context, message, -1); + return SQLITE_ERROR; } - NETWORK_RESULT res = network_receive_buffer(netdata, netdata->schema_upload_endpoint, netdata->authentication, true, true, (char *)payload, CLOUDSYNC_HEADER_SQLITECLOUD); - if (network_validate_json_response(context, &res, "CloudSync schema migration upload endpoint", NULL)) { - int rc = DBRES_OK; - char *upload_error = network_migration_upload_error_message(&res); - if (res.code == CLOUDSYNC_NETWORK_ERROR) { - network_set_sqlite_result(context, &res); - } else if (upload_error) { - sqlite3_result_error(context, upload_error, -1); + char *pending_payload = cloudsync_pending_migration_payload(data, pending_id); + if (!pending_payload) { + const char *message = "Unable to load pending schema migration payload."; + if (err_out) *err_out = cloudsync_string_dup(message); + else sqlite3_result_error(context, message, -1); + cloudsync_memory_free(pending_id); + return SQLITE_ERROR; + } + + int rc = SQLITE_ERROR; + if (!json_is_valid_root_object(pending_payload, strlen(pending_payload))) { + const char *message = "cloudsync_network_migration_upload expects a valid JSON object payload."; + if (err_out) *err_out = cloudsync_string_dup(message); + else sqlite3_result_error(context, message, -1); + goto cleanup; + } + + NETWORK_RESULT res = network_receive_buffer(netdata, netdata->schema_upload_endpoint, netdata->authentication, true, true, pending_payload, CLOUDSYNC_HEADER_SQLITECLOUD); + if (!network_validate_json_response(context, &res, "CloudSync schema migration upload endpoint", err_out)) { + network_result_cleanup(&res); + goto cleanup; + } + + char *upload_error = network_migration_upload_error_message(&res); + if (res.code == CLOUDSYNC_NETWORK_ERROR) { + if (err_out) *err_out = res.buffer ? cloudsync_string_dup(res.buffer) : cloudsync_string_dup("CloudSync schema migration upload failed."); + else network_set_sqlite_result(context, &res); + } else if (upload_error) { + if (err_out) *err_out = cloudsync_string_dup(upload_error); + else sqlite3_result_error(context, upload_error, -1); + } else { + int db_rc = cloudsync_pending_migration_mark_uploaded(data, pending_id); + if (db_rc == DBRES_OK) { + if (result_json && res.code == CLOUDSYNC_NETWORK_BUFFER && res.buffer) { + *result_json = cloudsync_memory_zeroalloc(res.blen + 1); + if (*result_json) memcpy(*result_json, res.buffer, res.blen); + else rc = SQLITE_NOMEM; + } + if (rc != SQLITE_NOMEM) rc = SQLITE_OK; + else if (!err_out) sqlite3_result_error_code(context, SQLITE_NOMEM); + if (!result_json && !err_out) network_set_sqlite_result(context, &res); } else { - if (from_pending) rc = cloudsync_pending_migration_mark_uploaded(data, pending_id); - if (rc == DBRES_OK) { - network_set_sqlite_result(context, &res); - } else { + if (err_out) *err_out = cloudsync_string_dup(cloudsync_errmsg(data)); + else { sqlite3_result_error(context, cloudsync_errmsg(data), -1); - sqlite3_result_error_code(context, rc); + sqlite3_result_error_code(context, db_rc); } } - if (upload_error) cloudsync_memory_free(upload_error); } + if (upload_error) cloudsync_memory_free(upload_error); network_result_cleanup(&res); + +cleanup: if (pending_id) cloudsync_memory_free(pending_id); if (pending_payload) cloudsync_memory_free(pending_payload); + return rc; } int network_extract_query_param (const char *query, const char *key, char *output, size_t output_size) { @@ -1417,9 +1471,20 @@ int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc, } } - if (cloudsync_pending_migration_count(data) > 0) { - sqlite3_result_error(context, "A pending schema migration must be uploaded before sending row changes.", -1); - return SQLITE_ERROR; + while (cloudsync_pending_migration_count(data) > 0) { + char *migration_result = NULL; + char *migration_err = NULL; + int mrc = cloudsync_network_migration_upload_next_pending(context, &migration_result, &migration_err); + if (migration_result) cloudsync_memory_free(migration_result); + if (migration_err) { + sqlite3_result_error(context, migration_err, -1); + cloudsync_memory_free(migration_err); + return SQLITE_ERROR; + } + if (mrc != SQLITE_OK) { + if (mrc == SQLITE_NOMEM) sqlite3_result_error_code(context, SQLITE_NOMEM); + return mrc; + } } // retrieve payload diff --git a/test/integration.c b/test/integration.c index 52c2152d..f6651365 100644 --- a/test/integration.c +++ b/test/integration.c @@ -894,7 +894,9 @@ static int test_mock_migration_upload_error_keeps_pending(void) { if (rc != SQLITE_OK) goto cleanup; rc = db_exec(db, "INSERT INTO upload_notes (id) VALUES ('u1');"); if (rc != SQLITE_OK) goto cleanup; - rc = expect_sql_error_contains(db, "SELECT cloudsync_network_send_changes();", "pending schema migration"); + rc = expect_sql_error_contains(db, "SELECT cloudsync_network_send_changes();", "missing schema api key"); + if (rc != SQLITE_OK) goto cleanup; + rc = db_expect_int(db, "SELECT count(*) FROM cloudsync_pending_migration WHERE uploaded_at IS NULL;", 1); cleanup: if (db) { @@ -932,7 +934,9 @@ static int test_mock_migration_upload_missing_status_keeps_pending(void) { if (rc != SQLITE_OK) goto cleanup; rc = db_exec(db, "INSERT INTO upload_missing_status_notes (id) VALUES ('u1');"); if (rc != SQLITE_OK) goto cleanup; - rc = expect_sql_error_contains(db, "SELECT cloudsync_network_send_changes();", "pending schema migration"); + rc = expect_sql_error_contains(db, "SELECT cloudsync_network_send_changes();", "accepted status"); + if (rc != SQLITE_OK) goto cleanup; + rc = db_expect_int(db, "SELECT count(*) FROM cloudsync_pending_migration WHERE uploaded_at IS NULL;", 1); cleanup: if (db) { diff --git a/test/postgresql/52_schema_migrations.sql b/test/postgresql/52_schema_migrations.sql index 27d93497..6d865623 100644 --- a/test/postgresql/52_schema_migrations.sql +++ b/test/postgresql/52_schema_migrations.sql @@ -1482,6 +1482,37 @@ SELECT count(*) = 0 AS existing_create_unrecorded_ok FROM cloudsync_migrations WHERE migration_id = 'mig-pg-existing-create' \gset +-- ============================================================================ +-- Initial schema sync generated from declarative API +-- ============================================================================ + +SELECT cloudsync_alter_create_table('initial_pg_notes'); +SELECT cloudsync_alter_add_column('initial_pg_notes', 'id', 'uuid', false); +SELECT cloudsync_alter_add_primary_key('initial_pg_notes', 'id'); +SELECT cloudsync_alter_add_column('initial_pg_notes', 'body', 'text', false, ''); +SELECT cloudsync_alter_augment_table('initial_pg_notes', 'CLS', 0); +SELECT cloudsync_alter_set_block_lww('initial_pg_notes', 'body', E'\n'); +SELECT cloudsync_alter_apply() IS NOT NULL AS initial_pg_apply_ok \gset + +SELECT count(*) = 1 AS initial_pg_pending_ok +FROM cloudsync_pending_migration +WHERE uploaded_at IS NULL + AND payload::jsonb->>'type' = 'cloudsync.schema.migration' + AND payload::jsonb->'ops' @> '[{"op":"createTable","table":"initial_pg_notes"}]'::jsonb \gset + +SELECT count(*) = 1 AS initial_pg_recorded_ok +FROM cloudsync_migrations +WHERE migration_id = ( + SELECT migration_id + FROM cloudsync_pending_migration + WHERE uploaded_at IS NULL AND payload::jsonb->'ops' @> '[{"op":"createTable","table":"initial_pg_notes"}]'::jsonb + LIMIT 1 +) \gset + +SELECT count(*) = 1 AS initial_pg_block_ok +FROM cloudsync_table_settings +WHERE tbl_name = 'initial_pg_notes' AND col_name = 'body' AND key = 'algo' AND value = 'block' \gset + SELECT count(*) AS migrations_before_atomic FROM cloudsync_migrations \gset @@ -1536,6 +1567,28 @@ SELECT (:fail::int + 1) AS fail \gset SELECT (:fail::int + 1) AS fail \gset \endif +\if :initial_pg_apply_ok +\if :initial_pg_pending_ok +\if :initial_pg_recorded_ok +\if :initial_pg_block_ok +\echo [PASS] (:testid) declarative initial schema sync generated pending PostgreSQL migration +\else +\echo [FAIL] (:testid) initial PostgreSQL schema sync missed block LWW settings +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) initial PostgreSQL schema sync was not recorded locally +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) initial PostgreSQL schema sync did not create pending upload +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) initial PostgreSQL schema sync apply failed +SELECT (:fail::int + 1) AS fail \gset +\endif + \if :alter_atomic_rejected_ok \if :alter_atomic_table_absent_ok \if :alter_atomic_pending_shape_ok diff --git a/test/unit.c b/test/unit.c index 8ccfcd89..479fdfd7 100644 --- a/test/unit.c +++ b/test/unit.c @@ -13369,6 +13369,56 @@ bool do_test_generated_migration_json_roundtrip_two_devices (void) { return result; } +bool do_test_initial_schema_sync_to_empty_client (void) { + sqlite3 *db[2] = {NULL, NULL}; + char *migration_payload = NULL; + bool result = false; + + db[0] = do_create_database(); + db[1] = do_create_database(); + if (!db[0] || !db[1]) goto cleanup; + + int rc = sqlite3_exec(db[0], + "SELECT cloudsync_alter_create_table('initial_notes');" + "SELECT cloudsync_alter_add_column('initial_notes', 'id', 'text', 0);" + "SELECT cloudsync_alter_add_primary_key('initial_notes', 'id');" + "SELECT cloudsync_alter_add_column('initial_notes', 'title', 'text', 0, '');" + "SELECT cloudsync_alter_add_column('initial_notes', 'body', 'text', 0, '');" + "SELECT cloudsync_alter_augment_table('initial_notes');" + "SELECT cloudsync_alter_set_block_lww('initial_notes', 'body', char(10));" + "SELECT cloudsync_alter_apply();", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + int64_t pending = do_select_int(db[0], "SELECT count(*) FROM cloudsync_pending_migration WHERE uploaded_at IS NULL;"); + migration_payload = do_select_text(db[0], "SELECT payload FROM cloudsync_pending_migration WHERE uploaded_at IS NULL ORDER BY created_at, migration_id LIMIT 1;"); + if (pending != 1 || !migration_payload) goto cleanup; + + if (!do_apply_migration_json(db[1], migration_payload)) goto cleanup; + + rc = sqlite3_exec(db[0], + "INSERT INTO initial_notes (id, title, body) VALUES ('n1', 'hello', 'line 1' || char(10) || 'line 2');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + if (!do_merge_using_payload(db[0], db[1], true, true)) goto cleanup; + + int64_t table_exists = do_select_int(db[1], "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='initial_notes';"); + int64_t block_settings = do_select_int(db[1], "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='initial_notes' AND col_name='body' AND key='algo' AND value='block';"); + int64_t row_count = do_select_int(db[1], "SELECT count(*) FROM initial_notes WHERE id='n1' AND title='hello' AND body='line 1' || char(10) || 'line 2';"); + int64_t blocks_table = do_select_int(db[1], "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='initial_notes_cloudsync_blocks';"); + + result = (table_exists == 1 && block_settings == 1 && row_count == 1 && blocks_table == 1); + if (!result) { + printf("initial_schema_sync: table=%lld block_settings=%lld rows=%lld blocks_table=%lld\n", + (long long)table_exists, (long long)block_settings, (long long)row_count, (long long)blocks_table); + } + +cleanup: + if (migration_payload) sqlite3_free(migration_payload); + for (int i = 0; i < 2; ++i) if (db[i]) close_db(db[i]); + return result; +} + bool do_test_row_payload_retry_after_migration_json (void) { sqlite3 *db[2] = {NULL, NULL}; char *migration_payload = NULL; @@ -13729,6 +13779,7 @@ int main (int argc, const char * argv[]) { result += test_report("Migration Dialect Nullable:", do_test_declarative_alter_dialect_nullable_is_portable()); result += test_report("Migration Dialect Replace:", do_test_declarative_alter_dialect_replacement_clears_default()); result += test_report("Migration JSON Two Devices:", do_test_generated_migration_json_roundtrip_two_devices()); + result += test_report("Initial Schema Sync:", do_test_initial_schema_sync_to_empty_client()); result += test_report("Migration Payload Retry:", do_test_row_payload_retry_after_migration_json()); result += test_report("Declarative Alter API:", do_test_declarative_alter_builder()); result += test_report("Declarative Alter Atomic:", do_test_declarative_alter_apply_atomic_pending_save());