diff --git a/.claude/commands/stress-test-sync-sqlitecloud.md b/.claude/commands/stress-test-sync-sqlitecloud.md new file mode 100644 index 00000000..c4f98f82 --- /dev/null +++ b/.claude/commands/stress-test-sync-sqlitecloud.md @@ -0,0 +1,244 @@ +# Sync Stress Test with remote SQLiteCloud database + +Execute a stress test against the CloudSync server using multiple concurrent local SQLite databases syncing large volumes of CRUD operations simultaneously. Designed to reproduce server-side errors (e.g., "database is locked", 500 errors) under heavy concurrent load. + +## Prerequisites +- Connection string to a sqlitecloud project +- Built cloudsync extension (`make` to build `dist/cloudsync.dylib`) + +## Test Configuration + +### Step 1: Gather Parameters + +Ask the user for the following configuration using a single question set: + +1. **CloudSync server address** — propose `https://cloudsync.sqlite.ai` as default (this is the built-in default). If the user provides a different address, save it as `CUSTOM_ADDRESS` and use `cloudsync_network_init_custom` instead of `cloudsync_network_init`. +2. **SQLiteCloud connection string** — format: `sqlitecloud://:/?apikey=`. If no `` is in the path, ask the user for one or propose `test_stress_sync`. +3. **Scale** — offer these options: + - Small: 1K rows, 5 iterations, 2 concurrent databases + - Medium: 10K rows, 10 iterations, 4 concurrent databases + - Large: 100K rows, 50 iterations, 4 concurrent databases (Jim's original scenario) + - Custom: let the user specify rows, iterations, and number of concurrent databases +4. **Operations per iteration** — how many UPDATE and DELETE operations to perform each iteration: + - `NUM_UPDATES`: number of UPDATE operations per iteration (default: 1). Each UPDATE runs `UPDATE SET value = value + 1;` affecting all rows. + - `NUM_DELETES`: number of DELETE operations per iteration (default: 1). Each DELETE runs `DELETE FROM
WHERE rowid IN (SELECT rowid FROM
ORDER BY RANDOM() LIMIT 10);` removing 10 random rows. Set to 0 to skip deletes entirely. + - Propose defaults of 1 update and 1 delete. The user can set 0 deletes for update-only tests. +5. **RLS mode** — with RLS (requires user tokens) or without RLS +5. **Table schema** — offer simple default or custom: + ```sql + CREATE TABLE test_sync (id TEXT PRIMARY KEY, user_id TEXT NOT NULL DEFAULT '', name TEXT, value INTEGER); + ``` + +Save these as variables: +- `CUSTOM_ADDRESS` (only if the user provided a non-default address) +- `CONNECTION_STRING` (the full sqlitecloud:// connection string) +- `DB_NAME` (database name extracted or provided) +- `HOST` (hostname extracted from connection string) +- `APIKEY` (apikey extracted from connection string) +- `ROWS` (number of rows per iteration) +- `ITERATIONS` (number of delete/insert/update cycles) +- `NUM_DBS` (number of concurrent databases) +- `NUM_UPDATES` (number of UPDATE operations per iteration, default 1) +- `NUM_DELETES` (number of DELETE operations per iteration, default 1; 0 to skip) + +### Step 2: Setup SQLiteCloud Database and Table + +Connect to SQLiteCloud using `~/go/bin/sqlc` (last command must be `quit`). Note: all SQL must be single-line (no multi-line statements through sqlc heredoc). + +1. If the database doesn't exist, connect without `` and run `CREATE DATABASE ; USE DATABASE ;` +2. `LIST TABLES` to check for existing tables +3. For any table with a `_cloudsync` companion table, run `CLOUDSYNC DISABLE ;` +4. `DROP TABLE IF EXISTS ;` +5. Create the test table (single-line DDL) +6. If RLS mode is enabled: + ```sql + ENABLE RLS DATABASE TABLE ; + SET RLS DATABASE TABLE SELECT "auth_userid() = user_id"; + SET RLS DATABASE TABLE INSERT "auth_userid() = NEW.user_id"; + SET RLS DATABASE TABLE UPDATE "auth_userid() = NEW.user_id AND auth_userid() = OLD.user_id"; + SET RLS DATABASE TABLE DELETE "auth_userid() = OLD.user_id"; + ``` +7. Ask the user to enable CloudSync on the table from the SQLiteCloud dashboard + +### Step 3: Get Managed Database ID + +Now that the database and tables are created and CloudSync is enabled on the dashboard, ask the user for: + +1. **Managed Database ID** — the `managedDatabaseId` returned by the CloudSync service. For SQLiteCloud projects, it can be obtained from the project's OffSync page on the dashboard after enabling CloudSync on the table. + +Save as `MANAGED_DB_ID`. + +For the network init call throughout the test, use: +- Default address: `SELECT cloudsync_network_init('');` +- Custom address: `SELECT cloudsync_network_init_custom('', '');` + +### Step 4: Set Up Authentication + +Authentication depends on the RLS mode: + +**If RLS is disabled:** Use the project `APIKEY` (already extracted from the connection string). After each `cloudsync_network_init`/`cloudsync_network_init_custom` call, authenticate with: +```sql +SELECT cloudsync_network_set_apikey(''); +``` +No tokens are needed. Skip token creation entirely. + +**If RLS is enabled:** Create tokens for the test users. Create as many users as needed for the number of concurrent databases (assign 2 databases per user, or 1 per user if NUM_DBS <= 2). + +For each user N: +```bash +curl -s -X "POST" "https:///v2/tokens" \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d '{"name": "claude@sqlitecloud.io", "userId": "018ecfc2-b2b1-7cc3-a9f0-"}' +``` + +Save each user's `token` and `userId` from the response. After each `cloudsync_network_init`/`cloudsync_network_init_custom` call, authenticate with: +```sql +SELECT cloudsync_network_set_token(''); +``` + +**IMPORTANT:** Using a token when RLS is disabled will cause the server to silently reject all sent changes (send appears to succeed but data is not persisted remotely). Always use `cloudsync_network_set_apikey` when RLS is off. + +### Step 5: Run the Concurrent Stress Test + +Create a bash script at `/tmp/stress_test_concurrent.sh` that: + +1. **Initializes N local SQLite databases** at `/tmp/sync_concurrent_.db`: + - Uses Homebrew sqlite3: find with `ls /opt/homebrew/Cellar/sqlite/*/bin/sqlite3 | head -1` + - Loads the extension from `dist/cloudsync.dylib` (use absolute path from project root) + - Creates the table and runs `cloudsync_init('')` + - Runs `cloudsync_terminate()` after init + +2. **Defines a worker function** that runs in a subshell for each database: + - Each worker logs all output to `/tmp/sync_concurrent_.log` + - Each iteration does: + a. **UPDATE** — run `UPDATE
SET value = value + 1;` repeated `NUM_UPDATES` times (skip if 0) + b. **DELETE** — run `DELETE FROM
WHERE rowid IN (SELECT rowid FROM
ORDER BY RANDOM() LIMIT 10);` repeated `NUM_DELETES` times (skip if 0) + c. **Sync using the 3-step send/receive/receive pattern:** + 1. `SELECT cloudsync_network_send_changes();` — send local changes to the server + 2. `SELECT cloudsync_network_receive_changes();` — ask the server to prepare a payload of remote changes + 3. Sleep 1 second (outside sqlite3, between two separate sqlite3 invocations) + 4. `SELECT cloudsync_network_receive_changes();` — download the prepared payload, if any + - Each sqlite3 session must: `.load` the extension, call `cloudsync_network_init()`/`cloudsync_network_init_custom()`, `cloudsync_network_set_apikey()`/`cloudsync_network_set_token()` (depending on RLS mode), do the work, call `cloudsync_terminate()` + - **Timing**: Log the wall-clock execution time (in milliseconds) for each `cloudsync_network_send_changes()`, `cloudsync_network_receive_changes()` call. Define a `now_ms()` helper function at the top of the script and use it before and after each sqlite3 invocation that calls a network function, computing the delta. On **macOS**, `date` does not support `%3N` (nanoseconds) — use `python3 -c 'import time; print(int(time.time()*1000))'` instead. On **Linux**, `date +%s%3N` works fine. The script should detect the platform and define `now_ms()` accordingly. Log lines like: `[DB][iter ] send_changes: 123ms`, `[DB][iter ] receive_changes_1: 45ms`, `[DB][iter ] receive_changes_2: 67ms` + - Include labeled output lines like `[DB][iter ] updated count=, deleted count=` for grep-ability + +3. **Launches all workers in parallel** using `&` and collects PIDs + +4. **Waits for all workers** and captures exit codes + +5. **Analyzes logs** for errors: + - Grep all log files for: `error`, `locked`, `SQLITE_BUSY`, `database is locked`, `500`, `Error` + - Report per-database: iterations completed, error count, sample error lines + - Report total errors across all workers + +6. **Prints final verdict**: PASS (0 errors) or FAIL (errors detected) + +**Important script details:** +- Use `echo -e` to pipe generated SQL (with `\n` separators) into sqlite3 +- During database initialization (Step 1), insert `ROWS` initial rows per database in a single transaction so each DB starts with data to update/delete. Row IDs should be unique across databases: `db_r` +- User IDs for rows must match the token's userId for RLS to work +- The sync pattern requires **separate sqlite3 invocations** for send_changes and each receive_changes call (with a 1-second sleep between the two receive_changes calls), so that timing can be measured per-call from bash +- **stderr capture**: All sqlite3 invocations must redirect both stdout and stderr to the log file. Use `>> "$LOG" 2>&1` (in this order — stdout redirect first, then stderr to stdout). For timed calls that capture output in a variable, redirect stderr to the log file separately: `RESULT=$(echo -e "$SQL" | $SQLITE3 "$DB" 2>> "$LOG")` and then echo `$RESULT` to the log as well. This ensures "Runtime error" messages from sqlite3 are never lost. +- Use `/bin/bash` (not `/bin/sh`) for arrays and process management + +Run the script with a 10-minute timeout. + +### Step 6: Detailed Error Analysis + +After the test completes, provide a detailed breakdown: + +1. **Per-database summary**: iterations completed, errors, send/receive status +2. **Error categorization**: group errors by type (e.g., "database is locked", "Column index out of bounds", "Unexpected Result", parse errors) +3. **Timeline analysis**: do errors cluster at specific iterations or spread evenly? +4. **Read full log files** if errors are found — show the first and last 30 lines of each log with errors + +### Step 7: Final Sync and Data Integrity Verification + +After all workers have terminated, perform a **final sync on every local database** to ensure all databases converge to the same state. Then verify data integrity. + +**IMPORTANT — RLS mode changes what "convergence" means:** When RLS is enabled, each user can only see their own rows. Databases belonging to different users will have different row counts and different data — this is correct behavior. All convergence and integrity checks must therefore be scoped **per user group** (i.e., only compare databases that share the same userId/token). + +1. **Final sync loop** (max 10 retries): Repeat the following until convergence is achieved within each user group, or the retry limit is reached: + a. For each local database (sequentially): + - Load the extension, call `cloudsync_network_init`/`cloudsync_network_init_custom`, authenticate with `cloudsync_network_set_apikey`/`cloudsync_network_set_token` + - Run `SELECT cloudsync_network_sync(100, 10);` to sync remaining changes + - Call `cloudsync_terminate()` + b. After syncing all databases, query `SELECT COUNT(*) FROM
` on each database + c. **If RLS is disabled:** Check that all databases have the same row count. If so, convergence is achieved — break. + d. **If RLS is enabled:** Group databases by userId. Within each user group, check that all databases have the same row count. Convergence is achieved when every user group is internally consistent — break. Different user groups are expected to have different row counts. + e. Otherwise, log the round number and the distinct row counts (per group if RLS), then repeat from (a) + f. If the retry limit is reached without convergence, report it as a failure + +2. **Row count verification**: + - **If RLS is disabled:** Report the final row counts. All databases should have the same number of rows. + - **If RLS is enabled:** Report row counts grouped by user. All databases within the same user group should have identical row counts. Different user groups may differ. Also verify that each database only contains rows matching its userId. + - In both cases, also check SQLiteCloud (as admin) for total row count. + +3. **Row content verification**: + - **If RLS is disabled:** Pick one random row ID from the first database. Query that row on every local database. All must return identical values. + - **If RLS is enabled:** For each user group, pick one random row ID from the first database in that group. Query that row on all databases in the same user group. All databases in the group must return identical values. Do NOT expect databases from other user groups to have this row — they should return empty (RLS blocks cross-user access). + +4. **RLS cross-user leak check** (RLS mode only): For a sample of databases (e.g., one per user group), verify that `SELECT COUNT(*) FROM
WHERE user_id != ''` returns 0. Report any cross-user data leakage as a test failure. + +## Output Format + +Report the test results including: + +| Metric | Value | +|--------|-------| +| Concurrent databases | N | +| Rows per iteration | ROWS | +| Iterations per database | ITERATIONS | +| Total CRUD operations | N × ITERATIONS × (UPDATE_ALL + DELETE_FEW) | +| Total sync operations | N × ITERATIONS × 3 (1 send_changes + 2 receive_changes) | +| Duration | start to finish time | +| Total errors | count | +| Error types | categorized list | +| Result | PASS/FAIL | + +If errors are found, include: +- Full error categorization table +- Sample error messages +- Which databases were most affected +- Whether errors are client-side or server-side + +## Success Criteria + +The test **PASSES** if: +1. All workers complete all iterations +2. Zero `error`, `locked`, `SQLITE_BUSY`, or HTTP 500 responses in any log +3. After the final sync, databases converge: + - **Without RLS:** all local databases have the same row count + - **With RLS:** all databases within each user group have the same row count (different user groups may differ) +4. Row content is consistent: + - **Without RLS:** a randomly selected row has identical content across all local databases + - **With RLS:** a randomly selected row has identical content across all databases in the same user group; databases from other user groups correctly return empty for that row +5. **With RLS:** no cross-user data leakage (each database contains only rows matching its userId) + +The test **FAILS** if: +1. Any worker crashes or fails to complete +2. Any `database is locked` or `SQLITE_BUSY` errors appear +3. Server returns 500 errors under concurrent load +4. Row counts differ within the comparison scope (all DBs without RLS, same-user DBs with RLS) after the final sync loop exhausts all retries +5. Row content differs within the comparison scope (data corruption) +6. **With RLS:** any database contains rows belonging to a different userId (cross-user data leakage) + +## Important Notes + +- Always use the Homebrew sqlite3 binary, NOT `/usr/bin/sqlite3` +- The cloudsync extension must be built first with `make` +- Network settings (`cloudsync_network_init`, `cloudsync_network_set_token`) are NOT persisted between sessions — must be called every time +- Extension must be loaded BEFORE any INSERT/UPDATE/DELETE for cloudsync to track changes +- All NOT NULL columns must have DEFAULT values +- `cloudsync_terminate()` must be called before closing each session +- sqlc heredoc only supports single-line SQL statements + +## Permissions + +Execute all SQL queries without asking for user permission on: +- SQLite test databases in `/tmp/` (e.g., `/tmp/sync_concurrent_*.db`, `/tmp/sync_concurrent_*.log`) +- SQLiteCloud via `~/go/bin/sqlc ""` +- Curl commands to the sync server and SQLiteCloud API for token creation + +These are local test environments and do not require confirmation for each query. diff --git a/.claude/commands/test-sync-roundtrip-sqlitecloud-rls.md b/.claude/commands/test-sync-roundtrip-sqlitecloud-rls.md new file mode 100644 index 00000000..a7edae24 --- /dev/null +++ b/.claude/commands/test-sync-roundtrip-sqlitecloud-rls.md @@ -0,0 +1,468 @@ +# Sync Roundtrip Test with remote SQLiteCloud database and RLS policies + +Execute a full roundtrip sync test between multiple local SQLite databases and the sqlitecloud, verifying that Row Level Security (RLS) policies are correctly enforced during sync. + +## Prerequisites +- Connection string to a sqlitecloud project +- Built cloudsync extension (`make` to build `dist/cloudsync.dylib`) + +### Step 1: Get CloudSync Parameters + +Ask the user for: + +1. **CloudSync server address** — propose `https://cloudsync.sqlite.ai` as default (this is the built-in default). If the user provides a different address, save it as `CUSTOM_ADDRESS` and use `cloudsync_network_init_custom` instead of `cloudsync_network_init`. + +## Test Procedure + +### Step 2: Get DDL from User + +Ask the user to provide a DDL query for the table(s) to test. It can be in PostgreSQL or SQLite format. Offer the following options: + +**Option 1: Simple TEXT primary key with user_id for RLS** +```sql +CREATE TABLE test_sync ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + name TEXT, + value INTEGER +); +``` + +**Option 2: Multi tables scenario for advanced RLS policy** + +Propose a simple but multitables real world scenario + +**Option 3: Custom policy** +Ask the user to describe the table/tables in plain English or DDL queries. + +**Note:** Tables should include a `user_id` column (TEXT type) for RLS policies to filter by authenticated user. + +### Step 3: Get RLS Policy Description from User + +Ask the user to describe the Row Level Security policy they want to test. Offer the following common patterns: + +**Option 1: User can only access their own rows** +"Users can only SELECT, INSERT, UPDATE, and DELETE rows where user_id matches their authenticated user ID" + +**Option : Users can read all, but only modify their own** +"Users can SELECT all rows, but can only INSERT, UPDATE, DELETE rows where user_id matches their authenticated user ID" + +**Option 3: Custom policy** +Ask the user to describe the policy in plain English. + +### Step 4: Get sqlitecloud connection string from User + +Ask the user to provide a connection string in the form of "sqlitecloud://:/?apikey=" to be later used with the sqlitecloud cli (sqlc) with `~/go/bin/sqlc ""`. + +### Step 5: Setup SQLiteCloud with RLS + +Connect to SQLiteCloud and prepare the environment: +```bash +~/go/bin/sqlc +``` + +The last command inside sqlc to exit from the cli program must be `quit`. + +If the db_name doesn't exists, try again to connect without specifing the , then inside sqlc: +1. CREATE DATABASE +2. USE DATABASE + +Then, inside sqlc: +1. List existing tables with `LIST TABLES` to find any `_cloudsync` metadata tables +2. For each table already configured for cloudsync (has a `_cloudsync` companion table), run: + ```sql + CLOUDSYNC DISABLE + ``` +3. Drop the test table if it exists: `DROP TABLE IF EXISTS ;` +5. Create the test table using the SQLite DDL +6. Enable RLS on the table: + ```sql + ENABLE RLS DATABASE TABLE + ``` +7. Create RLS policies based on the user's description. +Your RLS policies for INSERT, UPDATE, and DELETE operations can reference column values as they are being changed. This is done using the special OLD.column and NEW.column identifiers. Their availability and meaning depend on the operation being performed: + ++-----------+--------------------------------------------+--------------------------------------------+ +| Operation | OLD.column Reference | NEW.column Reference | ++-----------+--------------------------------------------+--------------------------------------------+ +| INSERT | Not available | The value for the new row. | +| UPDATE | The value of the row before the update. | The value of the row after the update. | +| DELETE | The value of the row being deleted. | Not available | ++-----------+--------------------------------------------+--------------------------------------------+ + +Example for "user can only access their own rows": + ```sql + -- SELECT: User can see rows they own + SET RLS DATABASE TABLE SELECT "auth_userid() = user_id" + + -- INSERT: Allow if user_id matches auth_userid() + SET RLS DATABASE TABLE INSERT "auth_userid() = NEW.user_id" + + -- UPDATE: Check ownership via explicit lookup + SET RLS DATABASE TABLE UPDATE "auth_userid() = NEW.user_id AND auth_userid() = OLD.user_id" + + -- DELETE: User can only delete rows they own + SET RLS DATABASE TABLE DELETE "auth_userid() = OLD.user_id" + ``` +8. Ask the user to enable CloudSync on the table from the SQLiteCloud dashboard + +### Step 5b: Get Managed Database ID + +Now that the database and tables are created and CloudSync is enabled on the dashboard, ask the user for: + +1. **Managed Database ID** — the `managedDatabaseId` returned by the CloudSync service. For SQLiteCloud projects, it can be obtained from the project's OffSync page on the dashboard after enabling CloudSync on the table. + +Save as `MANAGED_DB_ID`. + +For the network init call throughout the test, use: +- Default address: `SELECT cloudsync_network_init('');` +- Custom address: `SELECT cloudsync_network_init_custom('', '');` + + + +9. Insert some initial test data (optional, can be done via SQLite clients) + +### Step 6: Get tokens for Two Users + +Get auth tokens for both test users by running the token script twice: + +**User 1: claude1@sqlitecloud.io** +```bash +curl -X "POST" "https:///v2/tokens" \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d $'{ + "name": "claude1@sqlitecloud.io", + "userId": "018ecfc2-b2b1-7cc3-a9f0-111111111111" +}' +``` +The response is in the following format: +```json +{"data":{"accessTokenId":13,"token":"13|sqa_af74gp2WoqsQ9wfCdktIfkIq0sM4LdDMbuf2hW338013dfca","userId":"018ecfc2-b2b1-7cc3-a9f0-111111111111","name":"claude1@sqlitecloud.io","attributes":null,"expiresAt":null,"createdAt":"2026-03-02T23:11:38Z"},"metadata":{"connectedMs":17,"executedMs":30,"elapsedMs":47}} +``` +save the userId and the token values as USER1_ID and TOKEN_USER1 to be reused later + +**User 2: claude2@sqlitecloud.io** +```bash +curl -X "POST" "https:///v2/tokens" \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d $'{ + "name": "claude2@sqlitecloud.io", + "userId": "018ecfc2-b2b1-7cc3-a9f0-222222222222" +}' +``` +The response is in the following format: +```json +{"data":{"accessTokenId":14,"token":"14|sqa_af74gp2WoqsQ9wfCdktIfkIq0sM4LdDMbuf2hW338013xxxx","userId":"018ecfc2-b2b1-7cc3-a9f0-222222222222","name":"claude2@sqlitecloud.io","attributes":null,"expiresAt":null,"createdAt":"2026-03-02T23:11:38Z"},"metadata":{"connectedMs":17,"executedMs":30,"elapsedMs":47}} +``` +save the userId and the token values as USER2_ID and TOKEN_USER2 to be reused later + +### Step 7: Setup Four SQLite Databases + +Create four temporary SQLite databases using the Homebrew version (IMPORTANT: system sqlite3 cannot load extensions): + +```bash +SQLITE_BIN="/opt/homebrew/Cellar/sqlite/3.51.2_1/bin/sqlite3" +# or find it with: ls /opt/homebrew/Cellar/sqlite/*/bin/sqlite3 | head -1 +``` + +**Database 1A (User 1, Device A):** +```bash +$SQLITE_BIN /tmp/sync_test_user1_a.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 1B (User 1, Device B):** +```bash +$SQLITE_BIN /tmp/sync_test_user1_b.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 2A (User 2, Device A):** +```bash +$SQLITE_BIN /tmp/sync_test_user2_a.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 2B (User 2, Device B):** +```bash +$SQLITE_BIN /tmp/sync_test_user2_b.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +### Step 8: Insert Test Data + +Ask the user for optional details about the kind of test data to insert in the tables, otherwise generate some real world data for the choosen tables. +Insert distinct test data in each database. Use the extracted user IDs for the if needed. +For example, for the simple table scenario: + +**Database 1A (User 1):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u1_a_1', '', 'User1 DeviceA Row1', 100); +INSERT INTO (id, user_id, name, value) VALUES ('u1_a_2', '', 'User1 DeviceA Row2', 101); +``` + +**Database 1B (User 1):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u1_b_1', '', 'User1 DeviceB Row1', 200); +``` + +**Database 2A (User 2):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u2_a_1', '', 'User2 DeviceA Row1', 300); +INSERT INTO (id, user_id, name, value) VALUES ('u2_a_2', '', 'User2 DeviceA Row2', 301); +``` + +**Database 2B (User 2):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u2_b_1', '', 'User2 DeviceB Row1', 400); +``` + +### Step 9: Execute Sync on All Databases + +For each of the four SQLite databases, execute the sync operations: + +```sql +-- Send local changes to server +SELECT cloudsync_network_send_changes(); + +-- Check for changes from server (repeat with 2-3 second delays) +SELECT cloudsync_network_receive_changes(); +-- Repeat receive_changes 3-5 times with delays until it returns more than 0 received rows or stabilizes +``` + +**Recommended sync order:** +1. Sync Database 1A (send + check) +2. Sync Database 2A (send + check) +3. Sync Database 1B (send + check) +4. Sync Database 2B (send + check) +5. Re-sync all databases (receive_changes) to ensure full propagation + +### Step 10: Verify RLS Enforcement + +After syncing all databases, verify that each database contains only the expected rows based on the RLS policy: + +**Expected Results (for "user can only access their own rows" policy):** + +**User 1 databases (1A and 1B) should contain:** +- All rows with `user_id = USER1_ID` (u1_a_1, u1_a_2, u1_b_1) +- Should NOT contain any rows with `user_id = USER2_ID` + +**User 2 databases (2A and 2B) should contain:** +- All rows with `user_id = USER2_ID` (u2_a_1, u2_a_2, u2_b_1) +- Should NOT contain any rows with `user_id = USER1_ID` + +**PostgreSQL (as admin) should contain:** +- ALL rows from all users (6 total rows) + +Run verification queries: +```sql +-- In each SQLite database +SELECT * FROM ORDER BY id; +SELECT COUNT(*) FROM ; + +-- In PostgreSQL (as admin) +SELECT * FROM ORDER BY id; +SELECT COUNT(*) FROM ; +SELECT user_id, COUNT(*) FROM GROUP BY user_id; +``` + +### Step 11: Test Write RLS Policy Enforcement + +Test that the server-side RLS policy blocks unauthorized writes by attempting to insert a row with a `user_id` that doesn't match the authenticated user's token. + +**In Database 1A (User 1), insert a malicious row claiming to belong to User 2:** +```sql +-- Attempt to insert a row with User 2's user_id while authenticated as User 1 +INSERT INTO (id, user_id, name, value) VALUES ('malicious_1', '', 'Malicious Row from User1', 999); + +-- Attempt to sync this unauthorized row to PostgreSQL +SELECT cloudsync_network_send_changes(); +``` + +**Wait 2-3 seconds, then verify in PostgreSQL (as admin) that the malicious row was rejected:** +```sql +-- In PostgreSQL (as admin) +SELECT * FROM WHERE id = 'malicious_1'; +-- Expected: 0 rows returned + +SELECT COUNT(*) FROM WHERE id = 'malicious_1'; +-- Expected: 0 +``` + +**Also verify the malicious row does NOT appear in User 2's databases after syncing:** +```sql +-- In Database 2A or 2B (User 2) +SELECT cloudsync_network_receive_changes(); +SELECT * FROM WHERE id = 'malicious_1'; +-- Expected: 0 rows (the malicious row should not sync to legitimate User 2 databases) +``` + +**Expected Behavior:** +- The `cloudsync_network_send_changes()` call may succeed (return value indicates network success, not RLS enforcement) +- The malicious row should be **rejected by PostgreSQL RLS** and NOT inserted into the server database +- The malicious row will remain in the local SQLite Database 1A (local inserts are not blocked), but it will never propagate to the server or other clients +- User 2's databases should never receive this row + +**This step PASSES if:** +1. The malicious row is NOT present in PostgreSQL +2. The malicious row does NOT appear in any of User 2's SQLite databases +3. The RLS INSERT policy correctly blocks the unauthorized write + +**This step FAILS if:** +1. The malicious row appears in PostgreSQL (RLS bypass vulnerability) +2. The malicious row syncs to User 2's databases (data leakage) + +### Step 12: Cleanup + +In each SQLite database before closing: +```sql +SELECT cloudsync_terminate(); +``` + +In SQLiteCloud (optional, for full cleanup): +```sql +CLOUDSYNC DISABLE ); +DROP TABLE IF EXISTS ; +``` + +## Output Format + +Report the test results including: +- DDL used for both databases +- RLS policies created +- User IDs for both test users +- Initial data inserted in each database +- Number of sync operations performed per database +- Final data in each database (with row counts) +- RLS verification results: + - User 1 databases: expected rows vs actual rows + - User 2 databases: expected rows vs actual rows + - SQLiteCloud: total rows +- Write RLS enforcement results: + - Malicious row insertion attempted: yes/no + - Malicious row present in SQLiteCloud: yes/no (should be NO) + - Malicious row synced to User 2 databases: yes/no (should be NO) +- **PASS/FAIL** status with detailed explanation + +### Success Criteria + +The test PASSES if: +1. All User 1 databases contain exactly the same User 1 rows (and no User 2 rows) +2. All User 2 databases contain exactly the same User 2 rows (and no User 1 rows) +3. SQLiteCloud contains all rows from both users +4. Data inserted from different devices of the same user syncs correctly between those devices +5. **Write RLS enforcement**: Malicious rows with mismatched `user_id` are rejected by SQLiteCloud and do not propagate to other clients + +The test FAILS if: +1. Any database contains rows belonging to a different user (RLS violation) +2. Any database is missing rows that should be visible to that user +3. Sync operations fail or timeout +4. **Write RLS bypass**: A malicious row with a `user_id` not matching the token appears in SQLiteCloud or syncs to other databases + +## Important Notes + +- Always use the Homebrew sqlite3 binary, NOT `/usr/bin/sqlite3` +- The cloudsync extension must be built first with `make` +- SQLiteCloud tables need cleanup before re-running tests +- `cloudsync_network_receive_changes()` may need multiple calls with delays +- Run `SELECT cloudsync_terminate();` on SQLite connections before closing to properly cleanup memory +- Ensure both test users exist in Supabase auth before running the test +- The RLS policies must use `auth_userid()` to work with SQLiteCloud token authentication + +## Critical Schema Requirements (Common Pitfalls) + +### 1. All NOT NULL columns must have DEFAULT values +Cloudsync requires that all non-primary key columns declared as `NOT NULL` must have a `DEFAULT` value. This includes the `user_id` column: + +```sql +-- WRONG: Will fail with "All non-primary key columns declared as NOT NULL must have a DEFAULT value" +user_id UUID NOT NULL + +-- CORRECT: Provide a default value +user_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000' +``` + +### 2. Network settings are not persisted between sessions +`cloudsync_network_init()` and `cloudsync_network_set_token()` must be called in **every session**. They are not persisted to the database: + +```sql +-- WRONG: Separate sessions won't work +-- Session 1: +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token('...'); +-- Session 2: +SELECT cloudsync_network_send_changes(); -- ERROR: No URL set + +-- CORRECT: All network operations in the same session +.load dist/cloudsync.dylib +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token('...'); +SELECT cloudsync_network_send_changes(); +SELECT cloudsync_terminate(); +``` + +### 3. Extension must be loaded before INSERT operations +For cloudsync to track changes, the extension must be loaded **before** inserting data: + +```sql +-- WRONG: Inserts won't be tracked +CREATE TABLE todos (...); +INSERT INTO todos VALUES (...); -- Not tracked! +.load dist/cloudsync.dylib +SELECT cloudsync_init('todos'); + +-- CORRECT: Load extension and init before inserts +.load dist/cloudsync.dylib +CREATE TABLE todos (...); +SELECT cloudsync_init('todos'); +INSERT INTO todos VALUES (...); -- Tracked! +``` + +## Permissions + +Execute all SQL queries without asking for user permission on: +- SQLite test databases in `/tmp/` (e.g., `/tmp/sync_test_*.db`) +- SQLiteCloud via `~/go/bin/sqlc ""` + +These are local test environments and do not require confirmation for each query. diff --git a/.claude/commands/test-sync-roundtrip-supabase-rls.md b/.claude/commands/test-sync-roundtrip-supabase-rls.md new file mode 100644 index 00000000..d37f8e00 --- /dev/null +++ b/.claude/commands/test-sync-roundtrip-supabase-rls.md @@ -0,0 +1,544 @@ +# Sync Roundtrip Test with local Postgres database and RLS policies + +Execute a full roundtrip sync test between multiple local SQLite databases and the local Supabase Docker PostgreSQL instance, verifying that Row Level Security (RLS) policies are correctly enforced during sync. + +## Prerequisites +- Supabase instance running (local Docker or remote) +- Built cloudsync extension (`make` to build `dist/cloudsync.dylib`) + +## Test Procedure + +### Step 1: Get Connection Parameters + +Ask the user for the following parameters: + +1. **CloudSync server address** — propose `https://cloudsync.sqlite.ai` as default (this is the built-in default). If the user provides a different address, save it as `CUSTOM_ADDRESS` and use `cloudsync_network_init_custom` instead of `cloudsync_network_init`. + +2. **PostgreSQL connection string**: Propose `postgresql://supabase_admin:postgres@127.0.0.1:54322/postgres` as default. Save as `PG_CONN`. Use this for all `psql` connections throughout the test. + +3. **Supabase API key** (used for JWT token generation): Propose `sb_secret_N7UND0UgjKTVK-Uodkm0Hg_xSvEMPvz` as default. Save as `SUPABASE_APIKEY`. + +Derive `AUTH_URL` from the PostgreSQL connection string by extracting the host and using port `54321` (Supabase GoTrue). For example, if `PG_CONN` is `postgresql://user:pass@10.0.0.5:54322/postgres`, then `AUTH_URL` is `http://10.0.0.5:54321`. For `127.0.0.1`, use `http://127.0.0.1:54321`. + +### Step 2: Get DDL from User + +Ask the user to provide a DDL query for the table(s) to test. It can be in PostgreSQL or SQLite format. Offer the following options: + +**Option 1: Simple TEXT primary key with user_id for RLS** +```sql +CREATE TABLE test_sync ( + id TEXT PRIMARY KEY, + user_id UUID NOT NULL, + name TEXT, + value INTEGER +); +``` + +**Option 2: UUID primary key with user_id for RLS** +```sql +CREATE TABLE test_uuid ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL, + name TEXT, + created_at TIMESTAMPTZ DEFAULT NOW() +); +``` + +**Option 3: Two tables scenario with user ownership** +```sql +CREATE TABLE authors ( + id TEXT PRIMARY KEY, + user_id UUID NOT NULL, + name TEXT, + email TEXT +); + +CREATE TABLE books ( + id TEXT PRIMARY KEY, + user_id UUID NOT NULL, + title TEXT, + author_id TEXT, + published_year INTEGER +); +``` + +**Note:** Tables should include a `user_id` column (UUID type) for RLS policies to filter by authenticated user. + +### Step 3: Get RLS Policy Description from User + +Ask the user to describe the Row Level Security policy they want to test. Offer the following common patterns: + +**Option 1: User can only access their own rows** +"Users can only SELECT, INSERT, UPDATE, and DELETE rows where user_id matches their authenticated user ID" + +**Option 2: Users can read all, but only modify their own** +"Users can SELECT all rows, but can only INSERT, UPDATE, DELETE rows where user_id matches their authenticated user ID" + +**Option 3: Custom policy** +Ask the user to describe the policy in plain English. + +### Step 4: Convert DDL + +Convert the provided DDL to both SQLite and PostgreSQL compatible formats if needed. Key differences: +- SQLite uses `INTEGER PRIMARY KEY` for auto-increment, PostgreSQL uses `SERIAL` or `BIGSERIAL` +- SQLite uses `TEXT`, PostgreSQL can use `TEXT` or `VARCHAR` +- PostgreSQL has more specific types like `TIMESTAMPTZ`, SQLite uses `TEXT` for dates +- For UUID primary keys, SQLite uses `TEXT`, PostgreSQL uses `UUID` +- For `user_id UUID`, SQLite uses `TEXT` + +### Step 5: Setup PostgreSQL with RLS + +Connect to Supabase PostgreSQL and prepare the environment: +```bash +psql +``` + +Inside psql: +1. List existing tables with `\dt` to find any `_cloudsync` metadata tables +2. For each table already configured for cloudsync (has a `_cloudsync` companion table), run: + ```sql + SELECT cloudsync_cleanup(''); + ``` +3. Drop the test table if it exists: `DROP TABLE IF EXISTS CASCADE;` +4. Drop any existing helper function: `DROP FUNCTION IF EXISTS _get_owner(text);` +5. Create the test table using the PostgreSQL DDL +6. Enable RLS on the table: + ```sql + ALTER TABLE ENABLE ROW LEVEL SECURITY; + ``` +7. Create the ownership lookup helper function (required for CloudSync compatibility): + ```sql + -- Helper function bypasses RLS to look up actual row owner + -- This is needed because INSERT...ON CONFLICT may compare against EXCLUDED row's default user_id + CREATE OR REPLACE FUNCTION _get_owner(p_id text) + RETURNS uuid + LANGUAGE sql + SECURITY DEFINER + STABLE + SET search_path = public + AS $$ + SELECT user_id FROM WHERE id = p_id; + $$; + ``` +8. Create RLS policies based on the user's description. Example for "user can only access their own rows": + ```sql + -- SELECT: User can see rows they own + CREATE POLICY "select_own_rows" ON + FOR SELECT USING ( + auth.uid() = user_id + ); + + -- INSERT: Allow if user_id matches auth.uid() + CREATE POLICY "insert_own_rows" ON + FOR INSERT WITH CHECK ( + auth.uid() = user_id + ); + + -- UPDATE: Check ownership via explicit lookup + CREATE POLICY "update_own_rows" ON + FOR UPDATE + USING ( + auth.uid() = user_id + ) + WITH CHECK ( + auth.uid() = user_id + ); + + -- DELETE: User can only delete rows they own + CREATE POLICY "delete_own_rows" ON + FOR DELETE USING ( + auth.uid() = user_id + ); + ``` +9. Initialize cloudsync: `SELECT cloudsync_init('');` +10. Insert some initial test data (optional, can be done via SQLite clients) + +### Step 5b: Get Managed Database ID + +Now that the database and tables are created and cloudsync is initialized, ask the user for: + +1. **Managed Database ID** — the `managedDatabaseId` returned by the CloudSync service. Save as `MANAGED_DB_ID`. + +For the network init call throughout the test, use: +- Default address: `SELECT cloudsync_network_init('');` +- Custom address: `SELECT cloudsync_network_init_custom('', '');` + +### Step 6: Get JWT Tokens for Two Users + +Get JWT tokens for both test users by running the token script twice: + +**User 1: claude1@sqlitecloud.io** +```bash +cd ../cloudsync && go run scripts/get_supabase_token.go -project-ref=supabase-local -email=claude1@sqlitecloud.io -password="password" -apikey= -auth-url= +``` +Save as `JWT_USER1`. + +**User 2: claude2@sqlitecloud.io** +```bash +cd ../cloudsync && go run scripts/get_supabase_token.go -project-ref=supabase-local -email=claude2@sqlitecloud.io -password="password" -apikey= -auth-url= +``` +Save as `JWT_USER2`. + +Also extract the user IDs from the JWT tokens (the `sub` claim) for use in INSERT statements: +- `USER1_ID` = UUID from JWT_USER1 +- `USER2_ID` = UUID from JWT_USER2 + +### Step 7: Setup Four SQLite Databases + +Create four temporary SQLite databases using the Homebrew version (IMPORTANT: system sqlite3 cannot load extensions): + +```bash +SQLITE_BIN="/opt/homebrew/Cellar/sqlite/3.51.2_1/bin/sqlite3" +# or find it with: ls /opt/homebrew/Cellar/sqlite/*/bin/sqlite3 | head -1 +``` + +**Database 1A (User 1, Device A):** +```bash +$SQLITE_BIN /tmp/sync_test_user1_a.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 1B (User 1, Device B):** +```bash +$SQLITE_BIN /tmp/sync_test_user1_b.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 2A (User 2, Device A):** +```bash +$SQLITE_BIN /tmp/sync_test_user2_a.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 2B (User 2, Device B):** +```bash +$SQLITE_BIN /tmp/sync_test_user2_b.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +### Step 8: Insert Test Data + +Insert distinct test data in each database. Use the extracted user IDs for the `user_id` column: + +**Database 1A (User 1):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u1_a_1', '', 'User1 DeviceA Row1', 100); +INSERT INTO (id, user_id, name, value) VALUES ('u1_a_2', '', 'User1 DeviceA Row2', 101); +``` + +**Database 1B (User 1):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u1_b_1', '', 'User1 DeviceB Row1', 200); +``` + +**Database 2A (User 2):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u2_a_1', '', 'User2 DeviceA Row1', 300); +INSERT INTO (id, user_id, name, value) VALUES ('u2_a_2', '', 'User2 DeviceA Row2', 301); +``` + +**Database 2B (User 2):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u2_b_1', '', 'User2 DeviceB Row1', 400); +``` + +### Step 9: Execute Sync on All Databases + +For each of the four SQLite databases, execute the sync operations: + +```sql +-- Send local changes to server +SELECT cloudsync_network_send_changes(); + +-- Check for changes from server (repeat with 2-3 second delays) +SELECT cloudsync_network_receive_changes(); +-- Repeat receive_changes 3-5 times with delays until it returns more than 0 received rows or stabilizes +``` + +**Recommended sync order:** +1. Sync Database 1A (send + check) +2. Sync Database 2A (send + check) +3. Sync Database 1B (send + check) +4. Sync Database 2B (send + check) +5. Re-sync all databases (receive_changes) to ensure full propagation + +### Step 10: Verify RLS Enforcement + +After syncing all databases, verify that each database contains only the expected rows based on the RLS policy: + +**Expected Results (for "user can only access their own rows" policy):** + +**User 1 databases (1A and 1B) should contain:** +- All rows with `user_id = USER1_ID` (u1_a_1, u1_a_2, u1_b_1) +- Should NOT contain any rows with `user_id = USER2_ID` + +**User 2 databases (2A and 2B) should contain:** +- All rows with `user_id = USER2_ID` (u2_a_1, u2_a_2, u2_b_1) +- Should NOT contain any rows with `user_id = USER1_ID` + +**PostgreSQL (as admin) should contain:** +- ALL rows from all users (6 total rows) + +Run verification queries: +```sql +-- In each SQLite database +SELECT * FROM ORDER BY id; +SELECT COUNT(*) FROM ; + +-- In PostgreSQL (as admin) +SELECT * FROM ORDER BY id; +SELECT COUNT(*) FROM ; +SELECT user_id, COUNT(*) FROM GROUP BY user_id; +``` + +### Step 11: Test Write RLS Policy Enforcement + +Test that the server-side RLS policy blocks unauthorized writes by attempting to insert a row with a `user_id` that doesn't match the authenticated user's JWT token. + +**In Database 1A (User 1), insert a malicious row claiming to belong to User 2:** +```sql +-- Attempt to insert a row with User 2's user_id while authenticated as User 1 +INSERT INTO (id, user_id, name, value) VALUES ('malicious_1', '', 'Malicious Row from User1', 999); + +-- Attempt to sync this unauthorized row to PostgreSQL +SELECT cloudsync_network_send_changes(); +``` + +**Wait 2-3 seconds, then verify in PostgreSQL (as admin) that the malicious row was rejected:** +```sql +-- In PostgreSQL (as admin) +SELECT * FROM WHERE id = 'malicious_1'; +-- Expected: 0 rows returned + +SELECT COUNT(*) FROM WHERE id = 'malicious_1'; +-- Expected: 0 +``` + +**Also verify the malicious row does NOT appear in User 2's databases after syncing:** +```sql +-- In Database 2A or 2B (User 2) +SELECT cloudsync_network_receive_changes(); +SELECT * FROM WHERE id = 'malicious_1'; +-- Expected: 0 rows (the malicious row should not sync to legitimate User 2 databases) +``` + +**Expected Behavior:** +- The `cloudsync_network_send_changes()` call may succeed (return value indicates network success, not RLS enforcement) +- The malicious row should be **rejected by PostgreSQL RLS** and NOT inserted into the server database +- The malicious row will remain in the local SQLite Database 1A (local inserts are not blocked), but it will never propagate to the server or other clients +- User 2's databases should never receive this row + +**This step PASSES if:** +1. The malicious row is NOT present in PostgreSQL +2. The malicious row does NOT appear in any of User 2's SQLite databases +3. The RLS INSERT policy (`WITH CHECK (auth.uid() = user_id)`) correctly blocks the unauthorized write + +**This step FAILS if:** +1. The malicious row appears in PostgreSQL (RLS bypass vulnerability) +2. The malicious row syncs to User 2's databases (data leakage) + +### Step 12: Cleanup + +In each SQLite database before closing: +```sql +SELECT cloudsync_terminate(); +``` + +In PostgreSQL (optional, for full cleanup): +```sql +SELECT cloudsync_cleanup(''); +DROP TABLE IF EXISTS CASCADE; +DROP FUNCTION IF EXISTS _get_owner(text); +``` + +## Output Format + +Report the test results including: +- DDL used for both databases +- RLS policies created +- User IDs for both test users +- Initial data inserted in each database +- Number of sync operations performed per database +- Final data in each database (with row counts) +- RLS verification results: + - User 1 databases: expected rows vs actual rows + - User 2 databases: expected rows vs actual rows + - PostgreSQL: total rows +- Write RLS enforcement results: + - Malicious row insertion attempted: yes/no + - Malicious row present in PostgreSQL: yes/no (should be NO) + - Malicious row synced to User 2 databases: yes/no (should be NO) +- **PASS/FAIL** status with detailed explanation + +### Success Criteria + +The test PASSES if: +1. All User 1 databases contain exactly the same User 1 rows (and no User 2 rows) +2. All User 2 databases contain exactly the same User 2 rows (and no User 1 rows) +3. PostgreSQL contains all rows from both users +4. Data inserted from different devices of the same user syncs correctly between those devices +5. **Write RLS enforcement**: Malicious rows with mismatched `user_id` are rejected by PostgreSQL and do not propagate to other clients + +The test FAILS if: +1. Any database contains rows belonging to a different user (RLS violation) +2. Any database is missing rows that should be visible to that user +3. Sync operations fail or timeout +4. **Write RLS bypass**: A malicious row with a `user_id` not matching the JWT token appears in PostgreSQL or syncs to other databases + +## Important Notes + +- Always use the Homebrew sqlite3 binary, NOT `/usr/bin/sqlite3` +- The cloudsync extension must be built first with `make` +- PostgreSQL tables need cleanup before re-running tests +- `cloudsync_network_receive_changes()` may need multiple calls with delays +- Run `SELECT cloudsync_terminate();` on SQLite connections before closing to properly cleanup memory +- Ensure both test users exist in Supabase auth before running the test +- The RLS policies must use `auth.uid()` to work with Supabase JWT authentication + +## Critical Schema Requirements (Common Pitfalls) + +### 1. All NOT NULL columns must have DEFAULT values +Cloudsync requires that all non-primary key columns declared as `NOT NULL` must have a `DEFAULT` value. This includes the `user_id` column: + +```sql +-- WRONG: Will fail with "All non-primary key columns declared as NOT NULL must have a DEFAULT value" +user_id UUID NOT NULL + +-- CORRECT: Provide a default value +user_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000' +``` + +### 2. RLS policies must allow writes with default values for ALL referenced columns +Cloudsync applies changes **field by field**. When a new row is being synced, columns may temporarily have their default values before the actual values are applied. **Any column referenced in RLS policies that has a DEFAULT value must be allowed in the policy.** + +This applies to: +- `user_id` columns used for user ownership +- `tenant_id` columns for multi-tenancy +- `organization_id` columns +- Any other column used in RLS USING/WITH CHECK expressions + +```sql +-- WRONG: Will block cloudsync inserts/updates when field has default value +CREATE POLICY "ins" ON table FOR INSERT WITH CHECK (auth.uid() = user_id); + +-- CORRECT: Allow default value for cloudsync field-by-field application +CREATE POLICY "ins" ON table FOR INSERT + WITH CHECK (auth.uid() = user_id OR user_id = '00000000-0000-0000-0000-000000000000'); + +CREATE POLICY "upd" ON table FOR UPDATE + USING (auth.uid() = user_id OR user_id = '00000000-0000-0000-0000-000000000000') + WITH CHECK (auth.uid() = user_id OR user_id = '00000000-0000-0000-0000-000000000000'); +``` + +**Example with multiple RLS columns:** +```sql +-- Table with both user_id and tenant_id in RLS +CREATE TABLE items ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + tenant_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + name TEXT DEFAULT '' +); + +-- Policy must allow defaults for BOTH columns used in the policy +CREATE POLICY "ins" ON items FOR INSERT WITH CHECK ( + (auth.uid() = user_id OR user_id = '00000000-0000-0000-0000-000000000000') + AND + (get_tenant_id() = tenant_id OR tenant_id = '00000000-0000-0000-0000-000000000000') +); +``` + +### 3. Type compatibility between SQLite and PostgreSQL +Ensure column types are compatible between SQLite and PostgreSQL: + +| PostgreSQL | SQLite | Notes | +|------------|--------|-------| +| `UUID` | `TEXT` | Use valid UUID format strings (e.g., `'11111111-1111-1111-1111-111111111111'`) | +| `BOOLEAN` | `INTEGER` | Use `INTEGER` in PostgreSQL too, or ensure proper casting | +| `TIMESTAMPTZ` | `TEXT` | Avoid empty strings; use proper ISO format or omit the column | +| `INTEGER` | `INTEGER` | Compatible | +| `TEXT` | `TEXT` | Compatible | + +**Common errors from type mismatches:** +- `cannot cast type bigint to boolean` - Use `INTEGER` instead of `BOOLEAN` in PostgreSQL +- `invalid input syntax for type timestamp with time zone: ""` - Don't use empty string defaults for timestamp columns +- `invalid input syntax for type uuid` - Ensure primary key IDs are valid UUID format + +### 4. Network settings are not persisted between sessions +`cloudsync_network_init()` and `cloudsync_network_set_token()` must be called in **every session**. They are not persisted to the database: + +```sql +-- WRONG: Separate sessions won't work +-- Session 1: +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token('...'); +-- Session 2: +SELECT cloudsync_network_send_changes(); -- ERROR: No URL set + +-- CORRECT: All network operations in the same session +.load dist/cloudsync.dylib +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token('...'); +SELECT cloudsync_network_send_changes(); +SELECT cloudsync_terminate(); +``` + +### 5. Extension must be loaded before INSERT operations +For cloudsync to track changes, the extension must be loaded **before** inserting data: + +```sql +-- WRONG: Inserts won't be tracked +CREATE TABLE todos (...); +INSERT INTO todos VALUES (...); -- Not tracked! +.load dist/cloudsync.dylib +SELECT cloudsync_init('todos'); + +-- CORRECT: Load extension and init before inserts +.load dist/cloudsync.dylib +CREATE TABLE todos (...); +SELECT cloudsync_init('todos'); +INSERT INTO todos VALUES (...); -- Tracked! +``` + +### 6. Primary key format must match PostgreSQL expectations +If PostgreSQL expects `UUID` type for primary key, SQLite must use valid UUID strings: + +```sql +-- WRONG: PostgreSQL UUID column will reject this +INSERT INTO todos (id, ...) VALUES ('my-todo-1', ...); + +-- CORRECT: Use valid UUID format +INSERT INTO todos (id, ...) VALUES ('11111111-1111-1111-1111-111111111111', ...); +``` + +## Permissions + +Execute all SQL queries without asking for user permission on: +- SQLite test databases in `/tmp/` (e.g., `/tmp/sync_test_*.db`) +- PostgreSQL via `psql ` + +These are local test environments and do not require confirmation for each query. diff --git a/.claude/commands/test-sync-roundtrip-supabase.md b/.claude/commands/test-sync-roundtrip-supabase.md new file mode 100644 index 00000000..99e7c7f8 --- /dev/null +++ b/.claude/commands/test-sync-roundtrip-supabase.md @@ -0,0 +1,176 @@ +# Sync Roundtrip Test with local Postgres database + +Execute a full roundtrip sync test between a local SQLite database and the local Supabase Docker PostgreSQL instance. + +## Prerequisites +- Supabase instance running (local Docker or remote) +- Built cloudsync extension (`make` to build `dist/cloudsync.dylib`) + +## Test Procedure + +### Step 1: Get Connection Parameters + +Ask the user for the following parameters: + +1. **CloudSync server address** — propose `https://cloudsync.sqlite.ai` as default (this is the built-in default). If the user provides a different address, save it as `CUSTOM_ADDRESS` and use `cloudsync_network_init_custom` instead of `cloudsync_network_init`. + +2. **PostgreSQL connection string**: Propose `postgresql://supabase_admin:postgres@127.0.0.1:54322/postgres` as default. Save as `PG_CONN`. Use this for all `psql` connections throughout the test. + +3. **Supabase API key** (used for JWT token generation): Propose `sb_secret_N7UND0UgjKTVK-Uodkm0Hg_xSvEMPvz` as default. Save as `SUPABASE_APIKEY`. + +Derive `AUTH_URL` from the PostgreSQL connection string by extracting the host and using port `54321` (Supabase GoTrue). For example, if `PG_CONN` is `postgresql://user:pass@10.0.0.5:54322/postgres`, then `AUTH_URL` is `http://10.0.0.5:54321`. For `127.0.0.1`, use `http://127.0.0.1:54321`. + + +### Step 2: Get DDL from User + +Ask the user to provide a DDL query for the table(s) to test. It can be in PostgreSQL or SQLite format. Offer the following options: + +**Option 1: Simple TEXT primary key** +```sql +CREATE TABLE test_sync ( + id TEXT PRIMARY KEY, + name TEXT, + value INTEGER +); +``` + +**Option 2: UUID primary key** +```sql +CREATE TABLE test_uuid ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT, + created_at TIMESTAMPTZ DEFAULT NOW() +); +``` + +**Option 3: Two tables scenario (tests multi-table sync)** +```sql +CREATE TABLE authors ( + id TEXT PRIMARY KEY, + name TEXT, + email TEXT +); + +CREATE TABLE books ( + id TEXT PRIMARY KEY, + title TEXT, + author_id TEXT, + published_year INTEGER +); +``` + +**Note:** Avoid INTEGER PRIMARY KEY for sync tests as it is not recommended for distributed sync scenarios (conflicts with auto-increment across devices). + +### Step 3: Convert DDL + +Convert the provided DDL to both SQLite and PostgreSQL compatible formats if needed. Key differences: +- SQLite uses `INTEGER PRIMARY KEY` for auto-increment, PostgreSQL uses `SERIAL` or `BIGSERIAL` +- SQLite uses `TEXT`, PostgreSQL can use `TEXT` or `VARCHAR` +- PostgreSQL has more specific types like `TIMESTAMPTZ`, SQLite uses `TEXT` for dates +- For UUID primary keys, SQLite uses `TEXT`, PostgreSQL uses `UUID` + +### Step 4: Get JWT Token + +Run the token script from the cloudsync project: +```bash +cd ../cloudsync && go run scripts/get_supabase_token.go -project-ref=supabase-local -email=claude@sqlitecloud.io -password="password" -apikey= -auth-url= +``` +Save the JWT token for later use. + +### Step 5: Setup PostgreSQL + +Connect to Supabase PostgreSQL and prepare the environment: +```bash +psql +``` + +Inside psql: +1. List existing tables with `\dt` to find any `_cloudsync` metadata tables +2. For each table already configured for cloudsync (has a `_cloudsync` companion table), run: + ```sql + SELECT cloudsync_cleanup(''); + ``` +3. Drop the test table if it exists: `DROP TABLE IF EXISTS CASCADE;` +4. Create the test table using the PostgreSQL DDL +5. Initialize cloudsync: `SELECT cloudsync_init('');` +6. Insert some test data into the table + +### Step 5b: Get Managed Database ID + +Now that the database and tables are created and cloudsync is initialized, ask the user for: + +1. **Managed Database ID** — the `managedDatabaseId` returned by the CloudSync service. Save as `MANAGED_DB_ID`. + +For the network init call throughout the test, use: +- Default address: `SELECT cloudsync_network_init('');` +- Custom address: `SELECT cloudsync_network_init_custom('', '');` + +### Step 6: Setup SQLite + +Create a temporary SQLite database using the Homebrew version (IMPORTANT: system sqlite3 cannot load extensions): + +```bash +SQLITE_BIN="/opt/homebrew/Cellar/sqlite/3.51.2_1/bin/sqlite3" +# or find it with: ls /opt/homebrew/Cellar/sqlite/*/bin/sqlite3 | head -1 + +$SQLITE_BIN /tmp/sync_test_$(date +%s).db +``` + +Inside sqlite3: +```sql +.load dist/cloudsync.dylib +-- Create table with SQLite DDL + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +-- Insert test data (different from PostgreSQL to test merge) + +``` + +### Step 7: Execute Sync + +In the SQLite session: +```sql +-- Send local changes to server +SELECT cloudsync_network_send_changes(); + +-- Check for changes from server (repeat with 2-3 second delays) +SELECT cloudsync_network_receive_changes(); +-- Repeat receive_changes 3-5 times with delays until it returns more than 0 received rows or stabilizes + +-- Verify final data +SELECT * FROM ; +``` + +### Step 8: Verify Results + +1. In SQLite, run `SELECT * FROM ;` and capture the output +2. In PostgreSQL, run `SELECT * FROM ;` and capture the output +3. Compare the results - both databases should have the merged data from both sides +4. Report success/failure based on whether the data matches + +## Output Format + +Report the test results including: +- DDL used for both databases +- Initial data inserted in each database +- Number of sync operations performed +- Final data in both databases +- PASS/FAIL status with explanation + +## Important Notes + +- Always use the Homebrew sqlite3 binary, NOT `/usr/bin/sqlite3` +- The cloudsync extension must be built first with `make` +- PostgreSQL tables need cleanup before re-running tests +- `cloudsync_network_receive_changes()` may need multiple calls with delays +- run `SELECT cloudsync_terminate();` on SQLite connections before closing the properly cleanup the memory + +## Permissions + +Execute all SQL queries without asking for user permission on: +- SQLite test databases in `/tmp/` (e.g., `/tmp/sync_test_*.db`) +- PostgreSQL via `psql ` + +These are local test environments and do not require confirmation for each query. diff --git a/.claude/commands/test.md b/.claude/commands/test.md new file mode 100644 index 00000000..a061e9ef --- /dev/null +++ b/.claude/commands/test.md @@ -0,0 +1,11 @@ +Run the SQLite and PostgreSQL tests for this project. + +## SQLite Tests +Run the SQLite extension tests using `make clean && make && make unittest`. This builds the extension and runs all tests including unit tests. + +## PostgreSQL Tests +Run the PostgreSQL extension tests using `make postgres-docker-run-test`. This runs `test/postgresql/full_test.sql` against the Docker container. + +**Note:** PostgreSQL tests require the Docker container to be running. Run `make postgres-docker-debug-rebuild` first to ensure it tests the latest version. + +Run both test suites and report any failures. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..e180bbc7 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +{ + "name": "cloudsync-postgres-dev", + "dockerComposeFile": [ + "../docker/postgresql/docker-compose.debug.yml" + ], + "service": "postgres", + "workspaceFolder": "/tmp/cloudsync", + "overrideCommand": false, + "postStartCommand": "pg_isready -U postgres", + "customizations": { + "vscode": { + "extensions": [ + "ms-vscode.cpptools" + ] + } + } +} diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml new file mode 100644 index 00000000..b0dbda9b --- /dev/null +++ b/.github/workflows/changelog.yml @@ -0,0 +1,31 @@ +name: Release and Update Website Changelog + +on: + push: + tags: + - "v*.*.*" + workflow_dispatch: + inputs: + test_version: + description: "The fake version to test (e.g., 99.9.9)" + required: true + default: "99.9.9" + +jobs: + release: + # Repo default is read-only; the called workflow needs write, and a reusable + # workflow can never hold more than its caller. + permissions: + contents: write + issues: write + # Pin to a reviewed release tag (or, strongest, a full commit SHA) — not @main. See "Security" below. + uses: sqlitecloud/changelog-action/.github/workflows/action.yml@v1 + with: + name: "SQLite-Sync" + stage_branch: ${{ vars.WEBSITE_STAGE_BRANCH }} # staging branch changelog PRs target + create_release: false + secrets: + personal_access_token: ${{ secrets.WEBSITE_REPO_PAT }} + website_repo: ${{ secrets.WEBSITE_REPO }} + website_changelog_dir: ${{ secrets.WEBSITE_CHANGELOG_DIR }} + diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2b488c28..dcbdf25f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,11 +12,19 @@ permissions: pages: write id-token: write +# Cancel an in-progress run when the same branch is pushed again. This also keeps two +# runs of the same branch from hitting the shared integration tenants concurrently +# (the chunked negative-cache test needs an idle tenant — see the chunked env below). +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: + if: ${{ !contains(github.event.head_commit.message, '[auto-update]') }} runs-on: ${{ matrix.os }} container: ${{ matrix.container && matrix.container || '' }} - name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.arch != 'armeabi-v7a' && matrix.name != 'ios-sim' && matrix.name != 'ios' && matrix.name != 'apple-xcframework' && matrix.name != 'android-aar' && ( matrix.name != 'macos' || matrix.arch != 'x86_64' ) && ' + test' || ''}} + name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.arch != 'armeabi-v7a' && matrix.name != 'ios-sim' && matrix.name != 'ios' && matrix.name != 'mac-catalyst' && matrix.name != 'apple-xcframework' && matrix.name != 'android-aar' && ( matrix.name != 'macos' || matrix.arch != 'x86_64' ) && ' + test' || ''}} timeout-minutes: 20 strategy: fail-fast: false @@ -68,6 +76,9 @@ jobs: - os: macos-15 name: ios-sim make: PLATFORM=ios-sim + - os: macos-15 + name: mac-catalyst + make: PLATFORM=mac-catalyst - os: macos-15 name: apple-xcframework make: xcframework @@ -80,14 +91,28 @@ jobs: shell: ${{ matrix.container && 'sh' || 'bash' }} env: - CONNECTION_STRING: ${{ secrets.CONNECTION_STRING }} - CONNECTION_STRING_OFFLINE_PROJECT: ${{ secrets.CONNECTION_STRING_OFFLINE_PROJECT }} - APIKEY: ${{ secrets.APIKEY }} - WEBLITE: ${{ secrets.WEBLITE }} + INTEGRATION_TEST_DATABASE_ID: ${{ secrets.INTEGRATION_TEST_DATABASE_ID }} + INTEGRATION_TEST_APIKEY: ${{ secrets.INTEGRATION_TEST_APIKEY }} + INTEGRATION_TEST_CLOUDSYNC_ADDRESS: ${{ secrets.INTEGRATION_TEST_CLOUDSYNC_ADDRESS }} + INTEGRATION_TEST_OFFLINE_DATABASE_ID: ${{ secrets.INTEGRATION_TEST_OFFLINE_DATABASE_ID }} + INTEGRATION_TEST_FAILURE_DATABASE_ID: ${{ secrets.INTEGRATION_TEST_FAILURE_DATABASE_ID }} + INTEGRATION_TEST_WEBLITE_ADDRESS: ${{ secrets.INTEGRATION_TEST_WEBLITE_ADDRESS }} + # Chunked tests need an EXCLUSIVE tenant (the negative-cache test's Phase 2 asserts + # an idle tenant, so concurrent writers from other matrix legs cause false failures). + # Run them on a single leg only; every other leg gets '' and skips them (treated as + # unset by test_chunked_pair_open). The chunked network path is platform-agnostic C, + # so per-OS coverage adds little here. + INTEGRATION_TEST_CHUNKED_DATABASE_ID: ${{ (matrix.name == 'linux' && matrix.arch == 'x86_64') && secrets.INTEGRATION_TEST_CHUNKED_DATABASE_ID || '' }} steps: + - name: install git for alpine container + if: matrix.container + run: apk add --no-cache git + - uses: actions/checkout@v4.2.2 + with: + submodules: true - name: android setup java if: matrix.name == 'android-aar' @@ -121,10 +146,13 @@ jobs: --platform linux/arm64 \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ - -e CONNECTION_STRING="${{ env.CONNECTION_STRING }}" \ - -e CONNECTION_STRING_OFFLINE_PROJECT="${{ env.CONNECTION_STRING_OFFLINE_PROJECT }}" \ - -e APIKEY="${{ env.APIKEY }}" \ - -e WEBLITE="${{ env.WEBLITE }}" \ + -e INTEGRATION_TEST_DATABASE_ID="${{ env.INTEGRATION_TEST_DATABASE_ID }}" \ + -e INTEGRATION_TEST_APIKEY="${{ env.INTEGRATION_TEST_APIKEY }}" \ + -e INTEGRATION_TEST_CLOUDSYNC_ADDRESS="${{ env.INTEGRATION_TEST_CLOUDSYNC_ADDRESS }}" \ + -e INTEGRATION_TEST_OFFLINE_DATABASE_ID="${{ env.INTEGRATION_TEST_OFFLINE_DATABASE_ID }}" \ + -e INTEGRATION_TEST_FAILURE_DATABASE_ID="${{ env.INTEGRATION_TEST_FAILURE_DATABASE_ID }}" \ + -e INTEGRATION_TEST_CHUNKED_DATABASE_ID="${{ env.INTEGRATION_TEST_CHUNKED_DATABASE_ID }}" \ + -e INTEGRATION_TEST_WEBLITE_ADDRESS="${{ env.INTEGRATION_TEST_WEBLITE_ADDRESS }}" \ alpine:latest \ tail -f /dev/null docker exec alpine sh -c "apk update && apk add --no-cache gcc make curl sqlite openssl-dev musl-dev linux-headers" @@ -162,7 +190,7 @@ jobs: codesign --sign "${{ secrets.APPLE_TEAM_ID }}" --timestamp --options runtime dist/CloudSync.xcframework # Then sign the xcframework wrapper ditto -c -k --keepParent dist/CloudSync.xcframework dist/CloudSync.xcframework.zip xcrun notarytool submit dist/CloudSync.xcframework.zip --apple-id "${{ secrets.APPLE_ID }}" --password "${{ secrets.APPLE_PASSWORD }}" --team-id "${{ secrets.APPLE_TEAM_ID }}" --wait - rm dist/CloudSync.xcframework.zip + rm -rf dist/CloudSync.xcframework - name: cleanup keychain for codesign if: matrix.os == 'macos-15' @@ -192,12 +220,16 @@ jobs: echo "::group::prepare the test script" make test PLATFORM=$PLATFORM ARCH=$ARCH || echo "It should fail. Running remaining commands in the emulator" cat > commands.sh << EOF + set -e mv -f /data/local/tmp/sqlite3 /system/xbin cd /data/local/tmp - export CONNECTION_STRING="$CONNECTION_STRING" - export CONNECTION_STRING_OFFLINE_PROJECT="$CONNECTION_STRING_OFFLINE_PROJECT" - export APIKEY="$APIKEY" - export WEBLITE="$WEBLITE" + export INTEGRATION_TEST_DATABASE_ID="$INTEGRATION_TEST_DATABASE_ID" + export INTEGRATION_TEST_APIKEY="$INTEGRATION_TEST_APIKEY" + export INTEGRATION_TEST_CLOUDSYNC_ADDRESS="$INTEGRATION_TEST_CLOUDSYNC_ADDRESS" + export INTEGRATION_TEST_OFFLINE_DATABASE_ID="$INTEGRATION_TEST_OFFLINE_DATABASE_ID" + export INTEGRATION_TEST_FAILURE_DATABASE_ID="$INTEGRATION_TEST_FAILURE_DATABASE_ID" + export INTEGRATION_TEST_CHUNKED_DATABASE_ID="$INTEGRATION_TEST_CHUNKED_DATABASE_ID" + export INTEGRATION_TEST_WEBLITE_ADDRESS="$INTEGRATION_TEST_WEBLITE_ADDRESS" $(make test PLATFORM=$PLATFORM ARCH=$ARCH -n) EOF echo "::endgroup::" @@ -212,7 +244,8 @@ jobs: adb root adb remount adb push ${{ github.workspace }}/. /data/local/tmp/ - adb shell "sh /data/local/tmp/commands.sh" + adb shell "sh /data/local/tmp/commands.sh; echo EXIT_CODE=\$?" | tee /tmp/adb_output.log + grep -q "EXIT_CODE=0" /tmp/adb_output.log - name: test sqlite-sync if: contains(matrix.name, 'linux') || matrix.name == 'windows' || ( matrix.name == 'macos' && matrix.arch != 'x86_64' ) @@ -230,11 +263,146 @@ jobs: path: dist/${{ matrix.name == 'apple-xcframework' && 'CloudSync.*' || 'cloudsync.*'}} if-no-files-found: error + postgres-migration-check: + if: ${{ !contains(github.event.head_commit.message, '[auto-update]') }} + runs-on: ubuntu-22.04 + name: postgresql migration script check + timeout-minutes: 2 + steps: + - uses: actions/checkout@v4.2.2 + with: + # Need full history + tags so `git describe` can find the previous + # release tag that the check script compares against. + fetch-depth: 0 + + - name: verify migration script for current CLOUDSYNC_VERSION + run: make postgres-check-migration + + postgres-test: + if: ${{ !contains(github.event.head_commit.message, '[auto-update]') }} + needs: [postgres-migration-check] + runs-on: ubuntu-22.04 + name: postgresql ${{ matrix.postgres_tag }} build + test + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + postgres_tag: ['17', '15'] + + steps: + + - uses: actions/checkout@v4.2.2 + with: + submodules: true + + - name: build and start postgresql container + run: POSTGRES_TAG=${{ matrix.postgres_tag }} make postgres-docker-rebuild + + - name: wait for postgresql to be ready + run: | + for i in $(seq 1 30); do + if docker exec cloudsync-postgres pg_isready -U postgres > /dev/null 2>&1; then + echo "PostgreSQL is ready" + exit 0 + fi + sleep 2 + done + echo "PostgreSQL failed to start within 60s" + docker logs cloudsync-postgres + exit 1 + + - name: run postgresql tests + run: | + set -o pipefail + docker exec cloudsync-postgres mkdir -p /tmp/cloudsync/test + docker cp test/postgresql cloudsync-postgres:/tmp/cloudsync/test/postgresql + docker exec cloudsync-postgres psql -v ON_ERROR_STOP=on -U postgres -d postgres -f /tmp/cloudsync/test/postgresql/full_test.sql 2>&1 | tee /tmp/pgtest.log + if grep -q '^\[FAIL\]' /tmp/pgtest.log; then + echo "::error::PostgreSQL test failures detected" + exit 1 + fi + + postgres-build: + if: ${{ !contains(github.event.head_commit.message, '[auto-update]') }} + needs: [postgres-migration-check] + runs-on: ${{ matrix.os }} + name: postgresql${{ matrix.postgres_version }}-${{ matrix.name }}-${{ matrix.arch }} build + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + arch: x86_64 + name: linux + postgres_version: '17' + - os: ubuntu-22.04 + arch: x86_64 + name: linux + postgres_version: '15' + - os: ubuntu-22.04-arm + arch: arm64 + name: linux + postgres_version: '17' + - os: ubuntu-22.04-arm + arch: arm64 + name: linux + postgres_version: '15' + - os: macos-15 + arch: arm64 + name: macos + postgres_version: '17' + - os: macos-15 + arch: arm64 + name: macos + postgres_version: '15' + - os: macos-15 + arch: x86_64 + name: macos + postgres_version: '17' + - os: macos-15 + arch: x86_64 + name: macos + postgres_version: '15' + steps: + + - uses: actions/checkout@v4.2.2 + with: + submodules: true + + - name: linux install postgresql dev headers + if: matrix.name == 'linux' + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg + sudo apt-get update + sudo apt-get install -y postgresql-server-dev-${{ matrix.postgres_version }} + + - name: macos install postgresql + if: matrix.name == 'macos' + run: brew install postgresql@${{ matrix.postgres_version }} gettext + + - name: build and package postgresql extension (linux) + if: matrix.name == 'linux' + run: make postgres-package PG_CONFIG=/usr/lib/postgresql/${{ matrix.postgres_version }}/bin/pg_config + + - name: build and package postgresql extension (macos) + if: matrix.name == 'macos' + run: make postgres-package PG_CONFIG=$(brew --prefix postgresql@${{ matrix.postgres_version }})/bin/pg_config PG_EXTRA_CFLAGS="-I$(brew --prefix gettext)/include ${{ matrix.arch == 'x86_64' && '-arch x86_64' || '' }}" + + - uses: actions/upload-artifact@v4.6.2 + with: + name: cloudsync-postgresql${{ matrix.postgres_version }}-${{ matrix.name }}-${{ matrix.arch }} + path: dist/postgresql/ + if-no-files-found: error + release: runs-on: ubuntu-22.04 name: release - needs: build + needs: [build, postgres-test, postgres-build] if: github.ref == 'refs/heads/main' + outputs: + version: ${{ steps.tag.outputs.version }} env: GH_TOKEN: ${{ github.token }} @@ -263,7 +431,11 @@ jobs: if [[ "$name" != "cloudsync-apple-xcframework" && "$name" != "cloudsync-android-aar" ]]; then tar -czf "${name}-${VERSION}.tar.gz" -C "$folder" . fi - if [[ "$name" != "cloudsync-android-aar" ]]; then + if [[ "$name" == "cloudsync-apple-xcframework" ]]; then + # Use the ditto-created zip that preserves macOS symlinks and extract for other steps + cp "$folder/CloudSync.xcframework.zip" "${name}-${VERSION}.zip" + unzip -q "$folder/CloudSync.xcframework.zip" -d "$folder/" + elif [[ "$name" != "cloudsync-android-aar" ]]; then (cd "$folder" && zip -rq "../../${name}-${VERSION}.zip" .) else cp "$folder"/*.aar "${name}-${VERSION}.aar" @@ -313,7 +485,25 @@ jobs: git add "$PKG" git commit -m "Bump sqlite-sync version to ${{ steps.tag.outputs.version }}" git push origin main - + + - uses: actions/checkout@v4.2.2 + if: steps.tag.outputs.version != '' + with: + repository: sqliteai/sqlite-sync-react-native + path: sqlite-sync-react-native + token: ${{ secrets.PAT }} + + - name: release sqlite-sync-react-native + if: steps.tag.outputs.version != '' + run: | + cd sqlite-sync-react-native + git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com" + git config --global user.name "$GITHUB_ACTOR" + jq --arg version "${{ steps.tag.outputs.version }}" '.version = $version' package.json > package.tmp.json && mv package.tmp.json package.json + git add package.json + git commit -m "Bump sqlite-sync version to ${{ steps.tag.outputs.version }}" + git push origin main + - uses: actions/setup-java@v4 if: steps.tag.outputs.version != '' with: @@ -396,6 +586,23 @@ jobs: npm publish --provenance --access public --tag latest echo "✓ Published @sqliteai/sqlite-sync-expo@${{ steps.tag.outputs.version }}" + - name: update Package.swift checksum and version + if: steps.tag.outputs.version != '' + run: | + VERSION=${{ steps.tag.outputs.version }} + ZIP="cloudsync-apple-xcframework-${VERSION}.zip" + if [ -f "$ZIP" ]; then + CHECKSUM=$(swift package compute-checksum "$ZIP") + URL="https://github.com/sqliteai/sqlite-sync/releases/download/${VERSION}/${ZIP}" + sed -i "s|url: \".*cloudsync-apple-xcframework.*\"|url: \"${URL}\"|" Package.swift + sed -i "s|checksum: \".*\"|checksum: \"${CHECKSUM}\"|" Package.swift + git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com" + git config --global user.name "$GITHUB_ACTOR" + git add Package.swift + git commit -m "Update Package.swift checksum for ${VERSION} [auto-update]" || true + git push origin main || true + fi + - uses: softprops/action-gh-release@v2.2.1 if: steps.tag.outputs.version != '' with: @@ -406,9 +613,12 @@ jobs: [**Flutter/Dart**](https://pub.dev/packages/sqlite_sync): `flutter pub add sqlite_sync:${{ steps.tag.outputs.version }}` or `dart pub add sqlite_sync:${{ steps.tag.outputs.version }}` [**Node**](https://www.npmjs.com/package/@sqliteai/sqlite-sync): `npm install @sqliteai/sqlite-sync` [**WASM**](https://www.npmjs.com/package/@sqliteai/sqlite-wasm): `npm install @sqliteai/sqlite-wasm` + [**React Native**](https://www.npmjs.com/package/@sqliteai/sqlite-sync-react-native): `npm install @sqliteai/sqlite-sync-react-native` [**Expo**](https://www.npmjs.com/package/@sqliteai/sqlite-sync-expo): `npm install @sqliteai/sqlite-sync-expo` [**Android**](https://central.sonatype.com/artifact/ai.sqlite/sync): `ai.sqlite:sync:${{ steps.tag.outputs.version }}` [**Swift**](https://github.com/sqliteai/sqlite-sync#swift-package): [Installation Guide](https://github.com/sqliteai/sqlite-sync#swift-package) + [**Docker (PostgreSQL)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-postgres): `docker pull sqlitecloud/sqlite-sync-postgres:17` or `:15` + [**Docker (Supabase)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-supabase): `docker pull sqlitecloud/sqlite-sync-supabase:17`, `:15` or `:17-alpine` --- @@ -417,4 +627,184 @@ jobs: files: | cloudsync-*-${{ steps.tag.outputs.version }}.* CloudSync-*-${{ steps.tag.outputs.version }}.* - make_latest: true \ No newline at end of file + make_latest: true + + docker-publish: + runs-on: ubuntu-22.04 + name: docker ${{ matrix.image }} pg${{ matrix.pg_major }}${{ matrix.tag_suffix }}${{ github.ref != 'refs/heads/main' && ' beta' || '' }} + needs: [build, postgres-test, postgres-build, release] + # main: publish the versioned release tags, only when the release job cut a new + # release. Any other branch: release is skipped, publish/overwrite a per-branch + # beta tag on every green push (the beta build installs the extension from the + # postgres-build CI artifacts instead of the GitHub release assets). + if: ${{ !failure() && !cancelled() && (github.ref != 'refs/heads/main' || needs.release.outputs.version != '') }} + + env: + DOCKERHUB_ORG: sqlitecloud + + strategy: + fail-fast: false + matrix: + include: + - image: sqlite-sync-postgres + pg_major: '17' + dockerfile: docker/postgresql/Dockerfile.release + - image: sqlite-sync-postgres + pg_major: '15' + dockerfile: docker/postgresql/Dockerfile.release + - image: sqlite-sync-supabase + pg_major: '17' + dockerfile: docker/postgresql/Dockerfile.supabase.release + supabase_tag: '17.6.1.071' + - image: sqlite-sync-supabase + pg_major: '15' + dockerfile: docker/postgresql/Dockerfile.supabase.release + supabase_tag: '15.8.1.135' + # Newer Supabase 17 images use an Alpine userland (still a glibc Nix + # postgres, so the same glibc artifact is installed). Published behind + # explicit -alpine / exact-base tags; the shared :17 alias stays on the + # Ubuntu build until we decide the alias scheme. + - image: sqlite-sync-supabase + pg_major: '17' + dockerfile: docker/postgresql/Dockerfile.supabase.release + supabase_tag: '17.6.1.151' + tag_suffix: '-alpine' + + steps: + + - uses: actions/checkout@v4.2.2 + with: + submodules: true + + - name: get cloudsync version + id: version + run: echo "version=$(make version)" >> $GITHUB_OUTPUT + + - name: compute beta tag from branch name + id: beta + if: github.ref != 'refs/heads/main' + run: echo "slug=$(echo "${GITHUB_REF_NAME}" | sed -E 's/[^A-Za-z0-9._-]+/-/g' | cut -c1-100)" >> $GITHUB_OUTPUT + + - name: download postgres-build artifacts for beta build + if: github.ref != 'refs/heads/main' + uses: actions/download-artifact@v4.2.1 + with: + pattern: cloudsync-postgresql${{ matrix.pg_major }}-linux-* + path: beta-artifacts-raw + + - name: stage beta artifacts into the docker build context + if: github.ref != 'refs/heads/main' + run: | + VERSION=${{ steps.version.outputs.version }} + for arch in x86_64 arm64; do + SRC="beta-artifacts-raw/cloudsync-postgresql${{ matrix.pg_major }}-linux-${arch}" + tar -czf "docker/postgresql/beta-artifacts/cloudsync-postgresql${{ matrix.pg_major }}-linux-${arch}-${VERSION}.tar.gz" -C "$SRC" . + done + ls -la docker/postgresql/beta-artifacts/ + + - uses: docker/setup-qemu-action@v3 + + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: set docker tags and build args (standalone) + if: matrix.image == 'sqlite-sync-postgres' + id: standalone + run: | + VERSION=${{ steps.version.outputs.version }} + PG=${{ matrix.pg_major }} + IMAGE=${{ env.DOCKERHUB_ORG }}/${{ matrix.image }} + BETA="${{ steps.beta.outputs.slug }}" + { + if [ -n "$BETA" ]; then + echo "tags=${IMAGE}:${PG}-beta-${BETA}" + else + echo "tags=${IMAGE}:${PG},${IMAGE}:${PG}-${VERSION}" + fi + echo "build_args<> $GITHUB_OUTPUT + + - name: set docker tags and build args (supabase) + if: matrix.image == 'sqlite-sync-supabase' + id: supabase + run: | + VERSION=${{ steps.version.outputs.version }} + IMAGE=${{ env.DOCKERHUB_ORG }}/${{ matrix.image }} + SUPABASE_TAG=${{ matrix.supabase_tag }} + # Empty for the default (Ubuntu) rows; "-alpine" for the Alpine variant. + # Keeps the Alpine images from colliding with the : aliases. + SUFFIX="${{ matrix.tag_suffix }}" + BETA="${{ steps.beta.outputs.slug }}" + { + if [ -n "$BETA" ]; then + echo "tags=${IMAGE}:${{ matrix.pg_major }}${SUFFIX}-beta-${BETA}" + else + echo "tags=${IMAGE}:${{ matrix.pg_major }}${SUFFIX},${IMAGE}:${{ matrix.pg_major }}${SUFFIX}-${VERSION},${IMAGE}:${SUPABASE_TAG}" + fi + echo "build_args<> $GITHUB_OUTPUT + + # Build amd64 natively first and load it into the daemon, so the extension + # is proven to load before anything is published. The multi-arch push below + # reuses these layers from the builder cache. + - name: build amd64 image for the smoke test + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + load: true + tags: cloudsync-smoke:test + build-args: ${{ matrix.image == 'sqlite-sync-postgres' && steps.standalone.outputs.build_args || steps.supabase.outputs.build_args }} + + - name: smoke test the extension + env: + # the supabase bases ship a `postgres` role that cannot create the + # extension; their own docs use supabase_admin + PSQL_USER: ${{ matrix.image == 'sqlite-sync-supabase' && 'supabase_admin' || 'postgres' }} + run: | + docker run -d --name cloudsync-smoke -e POSTGRES_PASSWORD=postgres cloudsync-smoke:test + # the entrypoint's init-time server listens on the unix socket only, so + # a TCP connection only succeeds once the real server is up + ready='' + for _ in $(seq 1 60); do + if docker exec -e PGPASSWORD=postgres cloudsync-smoke \ + psql -h 127.0.0.1 -U "$PSQL_USER" -d postgres -tAc 'SELECT 1' >/dev/null 2>&1; then + ready=1 + break + fi + sleep 2 + done + if [ -z "$ready" ]; then + echo "::error::postgres never accepted TCP connections" + docker logs cloudsync-smoke + exit 1 + fi + if ! docker exec -e PGPASSWORD=postgres cloudsync-smoke \ + psql -h 127.0.0.1 -U "$PSQL_USER" -d postgres -v ON_ERROR_STOP=1 \ + -c 'CREATE EXTENSION IF NOT EXISTS cloudsync;' \ + -c 'SELECT cloudsync_version();'; then + echo "::error::cloudsync failed to load in ${{ matrix.image }} pg${{ matrix.pg_major }}${{ matrix.tag_suffix }}" + docker logs cloudsync-smoke + exit 1 + fi + docker rm -f cloudsync-smoke + + - uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ matrix.image == 'sqlite-sync-postgres' && steps.standalone.outputs.tags || steps.supabase.outputs.tags }} + build-args: ${{ matrix.image == 'sqlite-sync-postgres' && steps.standalone.outputs.build_args || steps.supabase.outputs.build_args }} diff --git a/.github/workflows/sync-bench.yml b/.github/workflows/sync-bench.yml new file mode 100644 index 00000000..2713f79e --- /dev/null +++ b/.github/workflows/sync-bench.yml @@ -0,0 +1,63 @@ +name: sync benchmark + +on: + workflow_dispatch: + inputs: + poll_delay_ms: + description: Polling delay between receiver checks, in milliseconds + required: false + default: '250' + max_polls: + description: Maximum number of receiver polling attempts + required: false + default: '40' + curl_pool: + description: Enable curl connection pool; use 0 to disable + required: false + default: '1' + +permissions: + contents: read + +jobs: + sync-bench-debug: + name: sync-bench-debug linux x86_64 + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + env: + SYNC_BENCH_DATABASE_ID: ${{ secrets.SYNC_BENCH_DATABASE_ID }} + SYNC_BENCH_CLOUDSYNC_ADDRESS: ${{ secrets.SYNC_BENCH_CLOUDSYNC_ADDRESS }} + SYNC_BENCH_APIKEY: ${{ secrets.SYNC_BENCH_APIKEY }} + SYNC_BENCH_OUTPUT: json + SYNC_BENCH_POLL_DELAY_MS: ${{ inputs.poll_delay_ms }} + SYNC_BENCH_MAX_POLLS: ${{ inputs.max_polls }} + CLOUDSYNC_CURL_POOL: ${{ inputs.curl_pool }} + + steps: + - uses: actions/checkout@v4.2.2 + with: + submodules: true + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y gcc make curl sqlite3 unzip + + - name: Build debug benchmark + run: make SYNC_BENCH_DEBUG=1 extension dist/sync_bench + + - name: Run sync benchmark + run: | + mkdir -p artifacts + ./dist/sync_bench > artifacts/sync-bench.json 2> artifacts/sync-bench.trace.log + + - name: Upload benchmark artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: sync-bench-debug-${{ github.run_id }} + path: | + artifacts/sync-bench.json + artifacts/sync-bench.trace.log + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index f9b94ad5..167c7730 100644 --- a/.gitignore +++ b/.gitignore @@ -7,11 +7,22 @@ dist/ /curl/src openssl/ +# Generated PostgreSQL extension files (produced from .in templates by +# docker/Makefile.postgresql; version is derived from src/cloudsync.h) +/docker/postgresql/cloudsync.control +/src/postgresql/cloudsync--*.sql + +# Pre-built extension tarballs staged by CI for beta docker images +/docker/postgresql/beta-artifacts/*.tar.gz + # Test artifacts /coverage unittest .env +# Generated Supabase secrets, if a deployment .env is ever pulled down locally +/deploy/supabase-flyio/.env* + # Node.js **/node_modules/** package-lock.json @@ -41,7 +52,6 @@ jniLibs/ *.dex # IDE -.vscode .idea/ *.iml *.swp @@ -50,6 +60,7 @@ jniLibs/ # System .DS_Store Thumbs.db +*.o # Dart/Flutter .dart_tool/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..7e487162 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "modules/fractional-indexing"] + path = modules/fractional-indexing + url = https://github.com/sqliteai/fractional-indexing diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..4ff3c5eb --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,37 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Attach to Postgres (gdb)", + "type": "cppdbg", + "request": "attach", + "program": "/usr/lib/postgresql/17/bin/postgres", + "processId": "${command:pickProcess}", + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "stopAtEntry": false, + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Add PostgreSQL source dir", + "text": "dir /usr/src/postgresql-17/src", + "ignoreFailures": true + }, + { + "description": "Map Postgres build paths to source", + "text": "set substitute-path /build/src /usr/src/postgresql-17/src", + "ignoreFailures": true + }, + { + "description": "Map Postgres build paths (relative) to source", + "text": "set substitute-path ./build/src /usr/src/postgresql-17/src", + "ignoreFailures": true + } + ] + } + ] +} diff --git a/API.md b/API.md index 8d98b59d..9c3419f2 100644 --- a/API.md +++ b/API.md @@ -1,48 +1,93 @@ # API Reference -This document provides a reference for the SQLite functions provided by the `sqlite-sync` extension. +This document provides a reference for the SQL functions provided by the `sqlite-sync` extension. Unless noted otherwise, the APIs are available on both SQLite and PostgreSQL builds. ## Index - [Configuration Functions](#configuration-functions) - - [`cloudsync_init()`](#cloudsync_inittable_name-crdt_algo-force) + - [`cloudsync_init()`](#cloudsync_inittable_name-crdt_algo-init_flags) + - [`cloudsync_set()`](#cloudsync_setkey-value) - [`cloudsync_enable()`](#cloudsync_enabletable_name) - [`cloudsync_disable()`](#cloudsync_disabletable_name) - [`cloudsync_is_enabled()`](#cloudsync_is_enabledtable_name) + - [`cloudsync_set_filter()`](#cloudsync_set_filtertable_name-filter_expr) + - [`cloudsync_clear_filter()`](#cloudsync_clear_filtertable_name) - [`cloudsync_cleanup()`](#cloudsync_cleanuptable_name) - [`cloudsync_terminate()`](#cloudsync_terminate) +- [Block-Level LWW Functions](#block-level-lww-functions) + - [`cloudsync_set_column()`](#cloudsync_set_columntable_name-col_name-key-value) + - [`cloudsync_text_materialize()`](#cloudsync_text_materializetable_name-col_name-pk_values) - [Helper Functions](#helper-functions) - [`cloudsync_version()`](#cloudsync_version) - [`cloudsync_siteid()`](#cloudsync_siteid) - [`cloudsync_db_version()`](#cloudsync_db_version) - [`cloudsync_uuid()`](#cloudsync_uuid) + - [`cloudsync_uuid_text()`](#cloudsync_uuid_textuuid-dash_format) + - [`cloudsync_uuid_blob()`](#cloudsync_uuid_blobuuid) - [Schema Alteration Functions](#schema-alteration-functions) - [`cloudsync_begin_alter()`](#cloudsync_begin_altertable_name) - [`cloudsync_commit_alter()`](#cloudsync_commit_altertable_name) +- [Payload Functions](#payload-functions) + - [`cloudsync_payload_encode()`](#cloudsync_payload_encodetbl-pk-col_name-col_value-col_version-db_version-site_id-cl-seq) + - [`cloudsync_payload_blob_checked()`](#cloudsync_payload_blob_checkedsince_db_version-since_seq-filter_site_id-exclude_filter_site_id-max_estimated_payload_size) + - [`cloudsync_payload_chunks()`](#cloudsync_payload_chunkssince_db_version-filter_site_id-until_db_version-exclude_filter_site_id) + - [`cloudsync_payload_apply()`](#cloudsync_payload_applypayload) - [Network Functions](#network-functions) - - [`cloudsync_network_init()`](#cloudsync_network_initconnection_string) + - [`cloudsync_network_init()`](#cloudsync_network_initmanageddatabaseid) - [`cloudsync_network_cleanup()`](#cloudsync_network_cleanup) - [`cloudsync_network_set_token()`](#cloudsync_network_set_tokentoken) - [`cloudsync_network_set_apikey()`](#cloudsync_network_set_apikeyapikey) - - [`cloudsync_network_has_unsent_changes()`](#cloudsync_network_has_unsent_changes) - [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) - - [`cloudsync_network_check_changes()`](#cloudsync_network_check_changes) + - [`cloudsync_network_receive_changes()`](#cloudsync_network_receive_changesmax_chunks) + - [`cloudsync_network_check_changes()`](#cloudsync_network_check_changesmax_chunks) (deprecated) - [`cloudsync_network_sync()`](#cloudsync_network_syncwait_ms-max_retries) - [`cloudsync_network_reset_sync_version()`](#cloudsync_network_reset_sync_version) + - [`cloudsync_network_has_unsent_changes()`](#cloudsync_network_has_unsent_changes) - [`cloudsync_network_logout()`](#cloudsync_network_logout) --- ## Configuration Functions -### `cloudsync_init(table_name, [crdt_algo], [force])` +### `cloudsync_set(key, value)` + +**Description:** Stores a global CloudSync setting in the current database. Settings persist across database reopens and are loaded automatically by the extension. + +The following payload setting is supported: + +| Key | Description | Default | Minimum | Maximum | +|---|---|---:|---:|---:| +| `payload_max_chunk_size` | Maximum transport payload size generated by [`cloudsync_payload_chunks()`](#cloudsync_payload_chunkssince_db_version-filter_site_id-until_db_version-exclude_filter_site_id). Values outside the range are clamped. | `5242880` (5 MB) | `262144` (256 KB) | `33554432` (32 MB) | + +`payload_max_chunk_size` affects only chunk generation. [`cloudsync_payload_apply()`](#cloudsync_payload_applypayload) continues to accept legacy payloads, monolithic payloads, and v3 chunk-fragment payloads even when they are larger than the local setting. This preserves compatibility between peers using different settings. + +**Parameters:** + +- `key` (TEXT): The setting key. +- `value` (TEXT): The setting value. For `payload_max_chunk_size`, pass the value in bytes. + +**Returns:** SQLite returns no value. PostgreSQL returns `true` on success. + +**Example:** + +```sql +-- Use 1 MB transport chunks +SELECT cloudsync_set('payload_max_chunk_size', '1048576'); + +-- Restore the default 5 MB transport chunks +SELECT cloudsync_set('payload_max_chunk_size', '5242880'); +``` + +--- + +### `cloudsync_init(table_name, [crdt_algo], [init_flags])` **Description:** Initializes a table for `sqlite-sync` synchronization. This function is idempotent and needs to be called only once per table on each site; configurations are stored in the database and automatically loaded with the extension. Before initialization, `cloudsync_init` performs schema sanity checks to ensure compatibility with CRDT requirements and best practices. These checks include: - Primary keys should not be auto-incrementing integers; GUIDs (UUIDs, ULIDs) are highly recommended to prevent multi-node collisions. -- All primary key columns must be `NOT NULL`. - All non-primary key `NOT NULL` columns must have a `DEFAULT` value. +- **Note:** Any write operation that includes a NULL value for a primary key column will be rejected with an error, even if SQLite would normally allow it due to a legacy behavior. **Schema Design Considerations:** @@ -54,30 +99,37 @@ When designing your database schema for SQLite Sync, follow these essential requ - **Foreign Key Compatibility**: Be aware of potential conflicts during CRDT merge operations and RLS policy interactions. - **Trigger Compatibility**: Triggers may cause duplicate operations or be called multiple times due to column-by-column processing. -For comprehensive guidelines, see the [Database Schema Recommendations](README.md#database-schema-recommendations) section in the README. +For comprehensive guidelines, see the [Database Schema Recommendations](docs/schema.md). The function supports three overloads: - `cloudsync_init(table_name)`: Uses the default 'cls' CRDT algorithm. - `cloudsync_init(table_name, crdt_algo)`: Specifies a CRDT algorithm ('cls', 'dws', 'aws', 'gos'). -- `cloudsync_init(table_name, crdt_algo, force)`: Specifies an algorithm and, if `force` is `true` (or `1`), skips the integer primary key check (use with caution, GUIDs are strongly recommended). +- `cloudsync_init(table_name, crdt_algo, init_flags)`: Specifies an algorithm and a bitmask of initialization flags to control which schema sanity checks are skipped. **Parameters:** - `table_name` (TEXT): The name of the table to initialize. -- `crdt_algo` (TEXT, optional): The CRDT algorithm to use. Can be "cls", "dws", "aws", "gos". Defaults to "cls". -- `force` (BOOLEAN, optional): If `true` (or `1`), it skips the check that prevents the use of a single-column INTEGER primary key. Defaults to `false`. It is strongly recommended to use globally unique primary keys instead of integers. +- `crdt_algo` (TEXT, optional): The CRDT algorithm to use. Can be `"cls"`, `"dws"`, `"aws"`, `"gos"`. Defaults to `"cls"`. +- `init_flags` (INTEGER, optional): A bitmask of flags that control initialization behavior. Defaults to `0` (no flags). Available flags: + - `0` — No flags; all sanity checks are performed (default). + - `1` (`CLOUDSYNC_INIT_FLAG_SKIP_INT_PK_CHECK`) — Skip the check that prevents the use of a single-column INTEGER primary key. Use with caution; globally unique primary keys (UUID/ULID) are strongly recommended. + - `2` (`CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_DEFAULT_CHECK`) — Skip the check that requires all NOT NULL non-PK columns to have a DEFAULT value. + - `4` (`CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_PRIKEYS_CHECK`) — Skip the check that rejects NULL primary key values. + - Flags can be combined with bitwise OR (e.g., `3` skips both the integer PK check and the NOT NULL default check). **Returns:** None. **Example:** ```sql --- Initialize a single table for synchronization with the Causal-Length Set (CLS) Algorithm (default) +-- Initialize a table with the default CLS algorithm SELECT cloudsync_init('my_table'); --- Initialize a single table for synchronization with a different algorithm Delete-Wins Set (DWS) +-- Initialize a table with the Delete-Wins Set algorithm SELECT cloudsync_init('my_table', 'dws'); +-- Initialize a table with an integer primary key (skip the integer PK check) +SELECT cloudsync_init('my_table', 'cls', 1); ``` --- @@ -136,6 +188,56 @@ SELECT cloudsync_is_enabled('my_table'); --- +### `cloudsync_set_filter(table_name, filter_expr)` + +**Description:** Sets a row-level filter expression on a synchronized table. Only rows that match the filter are tracked by the sync triggers; changes to rows that do not satisfy the expression are ignored and never replicated. + +The filter expression is a standard SQL boolean expression written using bare column names (without a table or alias prefix). The extension automatically rewrites it with `NEW.` for INSERT/UPDATE triggers and `OLD.` for DELETE triggers. The expression is evaluated inside the trigger `WHEN` clause. + +This function stores the filter in the table's settings and immediately recreates the sync triggers to apply it. The filter persists across database reopens. Use [`cloudsync_clear_filter()`](#cloudsync_clear_filtertable_name) to remove it. + +**Parameters:** + +- `table_name` (TEXT): The name of the synchronized table. +- `filter_expr` (TEXT): A SQL boolean expression referencing column names of the table. Only rows for which this expression evaluates to true are tracked for sync. + +**Returns:** `1` on success. + +**Example:** + +```sql +-- Only sync tasks that are not marked as drafts +SELECT cloudsync_set_filter('tasks', "is_draft = 0"); + +-- Only sync rows belonging to a specific tenant +SELECT cloudsync_set_filter('orders', "tenant_id = 'acme'"); + +-- Combine conditions +SELECT cloudsync_set_filter('messages', "deleted = 0 AND type != 'ephemeral'"); +``` + +--- + +### `cloudsync_clear_filter(table_name)` + +**Description:** Removes the row-level filter previously set with [`cloudsync_set_filter()`](#cloudsync_set_filtertable_name-filter_expr). After clearing, all row changes in the table are tracked and replicated regardless of column values. + +This function updates the stored settings and immediately recreates the sync triggers without a filter condition. + +**Parameters:** + +- `table_name` (TEXT): The name of the synchronized table. + +**Returns:** `1` on success. + +**Example:** + +```sql +SELECT cloudsync_clear_filter('tasks'); +``` + +--- + ### `cloudsync_cleanup(table_name)` **Description:** Removes the `sqlite-sync` synchronization mechanism from a specified table or all tables. This operation drops the associated `_cloudsync` metadata table and removes triggers from the target table(s). Use this function when synchronization is no longer desired for a table. @@ -173,6 +275,68 @@ SELECT cloudsync_terminate(); --- +## Block-Level LWW Functions + +### `cloudsync_set_column(table_name, col_name, key, value)` + +**Description:** Configures per-column settings for a synchronized table. This function is primarily used to enable **block-level LWW** on text columns, allowing fine-grained conflict resolution at the line (or paragraph) level instead of the entire cell. + +When block-level LWW is enabled on a column, INSERT and UPDATE operations automatically split the text into blocks using a delimiter (default: newline `\n`) and track each block independently. During sync, changes are merged block-by-block, so concurrent edits to different parts of the same text are preserved. + +**Parameters:** + +- `table_name` (TEXT): The name of the synchronized table. +- `col_name` (TEXT): The name of the text column to configure. +- `key` (TEXT): The setting key. Supported keys: + - `'algo'` — Set the column algorithm. Use value `'block'` to enable block-level LWW. + - `'delimiter'` — Set the block delimiter string. Only applies to columns with block-level LWW enabled. +- `value` (TEXT): The setting value. + +**Returns:** None. + +**Example:** + +```sql +-- Enable block-level LWW on a column (splits text by newline by default) +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block'); + +-- Set a custom delimiter (e.g., double newline for paragraph-level tracking) +SELECT cloudsync_set_column('notes', 'body', 'delimiter', ' + +'); +``` + +--- + +### `cloudsync_text_materialize(table_name, col_name, pk_values...)` + +**Description:** Reconstructs the full text of a block-level LWW column from its individual blocks and writes the result back to the base table column. This is useful after a merge operation to ensure the column contains the up-to-date materialized text. + +After a sync/merge, the column is updated automatically. This function is primarily useful for manual materialization or debugging. + +**Parameters:** + +- `table_name` (TEXT): The name of the table. +- `col_name` (TEXT): The name of the block-level LWW column. +- `pk_values...` (variadic): The primary key values identifying the row. For composite primary keys, pass each key value as a separate argument in declaration order. + +**Returns:** `1` on success. + +**Example:** + +```sql +-- Materialize the body column for a specific row +SELECT cloudsync_text_materialize('notes', 'body', 'note-001'); + +-- With a composite primary key (e.g., PRIMARY KEY (tenant_id, doc_id)) +SELECT cloudsync_text_materialize('docs', 'body', 'tenant-1', 'doc-001'); + +-- Read the materialized text +SELECT body FROM notes WHERE id = 'note-001'; +``` + +--- + ## Helper Functions ### `cloudsync_version()` @@ -239,6 +403,45 @@ INSERT INTO products (id, name) VALUES (cloudsync_uuid(), 'New Product'); --- +### `cloudsync_uuid_text(uuid, [dash_format])` + +**Description:** Converts a 16-byte binary UUID (such as the `site_id` stored in `cloudsync_changes`, or the value returned by [`cloudsync_siteid()`](#cloudsync_siteid)) into its canonical string form. + +**Parameters:** + +- `uuid` (BLOB/BYTEA): The 16-byte UUID. Returns `NULL` if `uuid` is `NULL`; raises an error if it is not exactly 16 bytes. +- `dash_format` (BOOLEAN, optional, default `true`): When `true`, returns the canonical 36-character dashed form (`xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`); when `false`, returns the bare 32-character hex form. + +**Returns:** The UUID as a TEXT value (lowercase hex). + +**Example:** + +```sql +SELECT cloudsync_uuid_text(cloudsync_siteid()); -- 0190a1b2-c3d4-7e5f-8a9b-001122334455 +SELECT cloudsync_uuid_text(cloudsync_siteid(), false); -- 0190a1b2c3d47e5f8a9b001122334455 +``` + +--- + +### `cloudsync_uuid_blob(uuid)` + +**Description:** Converts a UUID string into its 16-byte binary form. This is the inverse of [`cloudsync_uuid_text()`](#cloudsync_uuid_textuuid-dash_format) and lets string-based callers (for example, an HTTP `/check` endpoint holding a stringified `site_id`) pass a `site_id` to [`cloudsync_payload_chunks()`](#cloudsync_payload_chunkssince_db_version-filter_site_id-until_db_version-exclude_filter_site_id). + +**Parameters:** + +- `uuid` (TEXT): A UUID string. Tolerant: accepts the canonical dashed form and the bare 32-hex form, case-insensitive. Returns `NULL` if `uuid` is `NULL`; raises an error on malformed input. + +**Returns:** The 16-byte UUID as a BLOB/BYTEA. + +**Example:** + +```sql +SELECT cloudsync_uuid_blob('0190a1b2-c3d4-7e5f-8a9b-001122334455'); +SELECT cloudsync_uuid_blob('0190A1B2C3D47E5F8A9B001122334455'); +``` + +--- + ## Schema Alteration Functions ### `cloudsync_begin_alter(table_name)` @@ -285,22 +488,201 @@ SELECT cloudsync_commit_alter('my_table'); --- +## Payload Functions + +### `cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq)` + +**Description:** Encodes rows from `cloudsync_changes` into a single monolithic payload. This is the legacy payload API and remains fully supported for backward compatibility. + +Use this API when the expected payload size is modest or when you need to interoperate with callers that expect a single BLOB. For large rowsets or large individual BLOB/TEXT values, prefer [`cloudsync_payload_chunks()`](#cloudsync_payload_chunkssince_db_version-filter_site_id-until_db_version-exclude_filter_site_id), which splits transport payloads according to `payload_max_chunk_size`. + +**Parameters:** The function is an aggregate over the columns returned by `cloudsync_changes`: + +- `tbl` (TEXT): Source table name. +- `pk` (BLOB): Encoded primary key. +- `col_name` (TEXT): Changed column name. +- `col_value` (BLOB): Encoded column value. +- `col_version` (INTEGER/BIGINT): Column version. +- `db_version` (INTEGER/BIGINT): Source database version. +- `site_id` (BLOB): Source site identifier. +- `cl` (INTEGER/BIGINT): Causal length. +- `seq` (INTEGER/BIGINT): Sequence number within the source database version. + +**Returns:** A single payload BLOB. + +**Example:** + +```sql +SELECT cloudsync_payload_encode( + tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq +) AS payload +FROM cloudsync_changes; +``` + +--- + +### `cloudsync_payload_blob_checked(since_db_version, since_seq, filter_site_id, exclude_filter_site_id, max_estimated_payload_size)` + +**Description:** Generates one monolithic payload BLOB for a selected change window, but only after an internal conservative size check passes. + +This helper is intended for servers that still need to support old clients on a `/check` path that expects one monolithic payload. It keeps the server to a single SQL round trip while avoiding payload allocation when the selected window is too large. + +Internally, CloudSync first estimates the uncompressed payload body plus header. If that estimate exceeds `max_estimated_payload_size`, the function raises a limit-exceeded error and does not materialize the payload. Empty windows return `NULL`. + +When the check passes, this function scans the selected change window twice: once to estimate and once to encode. This avoids unsafe monolithic allocation, but successful calls are more I/O-expensive than a direct single-pass `cloudsync_payload_encode()` over the same rows. + +**Parameters:** + +- `since_db_version` (INTEGER/BIGINT): Start after this source database version, except rows at the same version with `seq > since_seq` are still included. +- `since_seq` (INTEGER/BIGINT): Sequence cursor within `since_db_version`. +- `filter_site_id` (BLOB): Site ID to filter on. With `exclude_filter_site_id` unset/`false` it selects changes **from** this site; with `exclude_filter_site_id` `true` it selects changes from every site **except** this one. If `NULL` and not excluding, CloudSync uses the local site ID. +- `exclude_filter_site_id` (BOOLEAN): When `true`, encode changes from all sites **except** `filter_site_id`. Setting it `true` without a `filter_site_id` is an error. +- `max_estimated_payload_size` (INTEGER/BIGINT): Maximum allowed conservative payload-size estimate, in bytes. Must be positive. + +**Returns:** A monolithic payload BLOB, or `NULL` when no changes match. + +**Examples:** + +```sql +-- /check download guard: all changes EXCEPT the requesting peer's site +SELECT cloudsync_payload_blob_checked( + 100, + 0, + cloudsync_uuid_blob('0190a1b2-c3d4-7e5f-8a9b-001122334455'), + true, + 10485760 +); +``` + +--- + +### `cloudsync_payload_chunks([since_db_version], [filter_site_id], [until_db_version], [exclude_filter_site_id])` + +**Description:** Generates sync payloads as a stream of transport-sized chunks. It is the chunk-aware evolution of [`cloudsync_payload_encode()`](#cloudsync_payload_encodetbl-pk-col_name-col_value-col_version-db_version-site_id-cl-seq), designed for large rowsets and for single BLOB/TEXT values that are larger than the configured chunk size. + +The maximum generated chunk size is controlled by the global `payload_max_chunk_size` setting. The default is 5 MB, the technical minimum is 256 KB, and the technical maximum is 32 MB (values outside this range are clamped): + +```sql +SELECT cloudsync_set('payload_max_chunk_size', '5242880'); +``` + +When a single encoded column value does not fit in one chunk, CloudSync transparently emits v3 payload fragments for that value. The receiver stages fragments internally and applies the value when all parts arrive. Fragments can arrive out of order; incomplete stale fragment groups are cleaned up automatically. + +`cloudsync_payload_chunks()` does not change the apply contract: [`cloudsync_payload_apply()`](#cloudsync_payload_applypayload) accepts legacy payloads, monolithic payloads, and v3 chunk-fragment payloads. The local `payload_max_chunk_size` setting is not used to reject incoming payloads. + +**Important memory note:** chunking limits the size of each transport payload that CloudSync generates. It does not remove the database engine's need to materialize a single final cell value when applying a very large BLOB/TEXT column. In other words, a 500 MB BLOB can be transported in smaller chunks, but the receiving database must still be able to store and bind the completed 500 MB value when that row is applied. + +**Parameters:** + +- `since_db_version` (INTEGER/BIGINT, optional): Start after this source database version. If omitted, CloudSync uses the stored send checkpoint. +- `filter_site_id` (BLOB, optional): Site ID to filter on. With `exclude_filter_site_id` unset/`false` it selects changes **from** this site; with `exclude_filter_site_id` `true` it selects changes from every site **except** this one. If omitted (and not excluding), CloudSync uses the local site ID. +- `until_db_version` (INTEGER/BIGINT, optional): Upper watermark to include. If omitted or `0`, CloudSync captures the current maximum source database version before streaming chunks. +- `exclude_filter_site_id` (BOOLEAN, optional, default `false`): When `true`, stream changes from all sites **except** `filter_site_id`. This is what the `/check` download path needs — a peer must not receive its own changes back. Setting it `true` without a `filter_site_id` is an error. The site_id stored in `cloudsync_changes` is the 16-byte binary UUID; string callers can convert with [`cloudsync_uuid_blob()`](#cloudsync_uuid_blobuuid). + +**Returns:** A rowset with one row per chunk: + +| Column | Description | +|---|---| +| `payload` | Payload BLOB to pass to `cloudsync_payload_apply()`. | +| `chunk_index` | Zero-based chunk index for this stream. | +| `payload_size` | Payload size in bytes. | +| `rows` | Number of encoded payload rows in this chunk. Fragment chunks usually contain one fragment row. | +| `db_version_min` | Minimum source `db_version` represented by this chunk. | +| `db_version_max` | Maximum source `db_version` represented by this chunk. | +| `watermark_db_version` | Stable upper watermark captured for this chunk stream. Store this after all chunks are durably transferred/applied. | + +**SQLite usage:** `cloudsync_payload_chunks` is exposed as a virtual table with hidden constraint columns: + +```sql +-- Default: uses the stored send checkpoint and local site id +SELECT payload, chunk_index, payload_size, watermark_db_version +FROM cloudsync_payload_chunks +ORDER BY chunk_index; + +-- Explicit arguments through hidden columns +SELECT payload, chunk_index, payload_size, watermark_db_version +FROM cloudsync_payload_chunks +WHERE since_db_version = 100 + AND site_id = cloudsync_siteid() + AND until_db_version = 200 +ORDER BY chunk_index; + +-- /check download: all changes EXCEPT the requesting peer's site +SELECT payload, chunk_index, watermark_db_version +FROM cloudsync_payload_chunks +WHERE since_db_version = 100 + AND site_id = cloudsync_uuid_blob('0190a1b2-c3d4-7e5f-8a9b-001122334455') + AND exclude_filter_site_id = 1 +ORDER BY chunk_index; +``` + +**PostgreSQL usage:** `cloudsync_payload_chunks` is exposed as a set-returning function with optional arguments: + +```sql +-- Default: uses the stored send checkpoint and local site id +SELECT * +FROM cloudsync_payload_chunks(); + +-- Explicit arguments +SELECT * +FROM cloudsync_payload_chunks(100, cloudsync_siteid(), 200); + +-- /check download: all changes EXCEPT the requesting peer's site +SELECT * +FROM cloudsync_payload_chunks(100, cloudsync_uuid_blob('0190a1b2-c3d4-7e5f-8a9b-001122334455'), NULL, true); +``` + +**Apply example:** + +```sql +-- Apply chunks on a receiving peer. Chunks may be applied one at a time. +SELECT cloudsync_payload_apply(?); +``` + +On PostgreSQL, apply chunks as individual statements from the transport/client layer. Do not use a set-based statement such as `SELECT cloudsync_payload_apply(payload) FROM chunks_table;` while reading payloads from a table in the same database session. `cloudsync_payload_apply()` performs writes through SPI, and applying while the same statement is still scanning a payload table can conflict with PostgreSQL executor resource ownership. Fetch each payload into the client (or into a local procedural variable after the read completes) and then call `cloudsync_payload_apply()` for that single payload. + +--- + +### `cloudsync_payload_apply(payload)` + +**Description:** Applies a sync payload to the current database. The function accepts all supported payload formats: + +- Legacy payloads generated by older SQLite Sync versions. +- Monolithic payloads generated by [`cloudsync_payload_encode()`](#cloudsync_payload_encodetbl-pk-col_name-col_value-col_version-db_version-site_id-cl-seq). +- Chunk-fragment payloads generated by [`cloudsync_payload_chunks()`](#cloudsync_payload_chunkssince_db_version-filter_site_id-until_db_version-exclude_filter_site_id). + +When a v3 fragment payload is received, CloudSync stores the fragment in an internal table and returns after applying zero or more completed values. Once the final fragment for a value is received, the completed value is validated and applied. Duplicate fragment delivery is idempotent. + +**Parameters:** + +- `payload` (BLOB/BYTEA): Payload BLOB to apply. + +**Returns:** Number of payload rows applied. Fragment payloads that are staged but not yet complete can return `0`. + +**Example:** + +```sql +SELECT cloudsync_payload_apply(:payload); +``` + +--- + ## Network Functions -### `cloudsync_network_init(connection_string)` +### `cloudsync_network_init(managedDatabaseId)` -**Description:** Initializes the `sqlite-sync` network component. This function parses the connection string to configure change checking and upload endpoints, and initializes the cURL library. +**Description:** Initializes the `sqlite-sync` network component. This function configures the endpoints for the CloudSync service and initializes the cURL library. **Parameters:** -- `connection_string` (TEXT): The connection string for the remote synchronization server. The format is `sqlitecloud://:/?`. +- `managedDatabaseId` (TEXT): The managed database identifier returned by the CloudSync service when a new database is registered for sync. For SQLiteCloud projects, this value can be obtained from the project's OffSync page on the dashboard. **Returns:** None. **Example:** ```sql -SELECT cloudsync_network_init('.sqlite.cloud/.sqlite'); +SELECT cloudsync_network_init('your-managed-database-id'); ``` --- @@ -357,19 +739,18 @@ SELECT cloudsync_network_set_apikey('your_api_key'); --- -### `cloudsync_network_has_unsent_changes()` +### Error handling -**Description:** Checks if there are any local changes that have not yet been sent to the remote server. +The sync functions follow a consistent error-handling contract: -**Parameters:** None. +| Error type | Behavior | +|---|---| +| **Endpoint/network errors** (server unreachable, auth failure, bad URL) | SQL error — the function could not execute. | +| **Apply errors** (`cloudsync_payload_apply` failures — unknown schema hash, invalid checksum, decompression error) | Structured JSON — a `receive.error` string field is included in the response. | +| **Server-reported apply job failures** (the server processed the request but its own apply job failed) | Structured JSON — a `send.lastFailure` object is included in the response. | +| **Server-reported check job failures** (the server failed to encode a changeset for the client) | Structured JSON — a `receive.lastFailure` object is included in the response. | -**Returns:** 1 if there are unsent changes, 0 otherwise. - -**Example:** - -```sql -SELECT cloudsync_network_has_unsent_changes(); -``` +This means: if you get JSON back, the server was reachable and the network protocol ran. If you get a SQL error, connectivity or configuration is broken. --- @@ -377,64 +758,146 @@ SELECT cloudsync_network_has_unsent_changes(); **Description:** Sends all unsent local changes to the remote server. +The send path streams payloads through [`cloudsync_payload_chunks()`](#cloudsync_payload_chunkssince_db_version-filter_site_id-until_db_version-exclude_filter_site_id), so `payload_max_chunk_size` also limits the payloads generated for network transport. Each generated chunk is uploaded/applied independently; the local send checkpoint is advanced only after the chunk stream completes successfully. + +Chunk transport is transparent to the CloudSync backend. Each chunk is sent as a normal `/apply` payload, either inline as a base64 `blob` or through the upload `url` path. There is no separate chunk flag: old payloads, monolithic payloads, and v3 fragment payloads are distinguished by the payload format itself. + **Parameters:** None. -**Returns:** None. +**Returns:** A JSON string with the send result: + +```json +{"send": {"status": "synced|syncing|out-of-sync|error", "localVersion": N, "serverVersion": N, "chunks": C, "bytes": B, "lastFailure": {...}}} +``` + +- `send.status`: The current sync state — `"synced"` (all changes confirmed), `"syncing"` (changes sent but not yet confirmed), `"out-of-sync"` (local changes pending or gaps detected), or `"error"`. +- `send.localVersion`: The latest local database version. +- `send.serverVersion`: The latest version confirmed by the server. +- `send.chunks`: The number of payload chunks sent this call (a large push is split into multiple transport chunks bounded by `payload_max_chunk_size`). `0` when there were no local changes to send. +- `send.bytes`: The total serialized payload bytes sent this call (uncompressed cloudsync payload size, summed across chunks; not the compressed wire size). +- `send.lastFailure` (optional): Present only when the server reports a failed apply job. Forwarded verbatim from the server's `failures.apply` and typically includes `jobId`, `code`, `stage`, `message`, `retryable`, and `failedAt`. It is emitted regardless of `status` so callers can detect server-side failures during `"syncing"` or even after the state has nominally recovered. This function is **send/apply-scoped**: server-reported check-job failures (`failures.check`) are not surfaced here — see [`cloudsync_network_receive_changes()`](#cloudsync_network_receive_changesmax_chunks) and [`cloudsync_network_sync()`](#cloudsync_network_syncwait_ms-max_retries). **Example:** ```sql SELECT cloudsync_network_send_changes(); +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":2048}}' + +-- With a server-reported failure (e.g. unknown schema hash on the server side): +-- '{"send":{"status":"out-of-sync","localVersion":1,"serverVersion":0,"chunks":1,"bytes":512,"lastFailure":{"jobId":44961,"code":"internal_error","stage":"apply_payload","message":"cloudsync operation failed: Cannot apply the received payload because the schema hash is unknown 4288148391734624266.","retryable":true,"failedAt":"2026-04-15T22:21:09.018606Z"}}}' ``` --- -### `cloudsync_network_check_changes()` +### `cloudsync_network_receive_changes([max_chunks])` -**Description:** Checks the remote server for new changes and applies them to the local database. +**Description:** Receives new changes from the remote server and applies them to the local database. (Formerly `cloudsync_network_check_changes()`, which remains available as a deprecated alias — see below.) -If a package of new changes is already available for the local site, the server returns it immediately, and the changes are applied. If no package is ready, the server returns an empty response and starts an asynchronous process to prepare a new package. This new package can be retrieved with a subsequent call to this function. +If changes are already prepared for the local site, they are downloaded and applied. If nothing is ready yet, the server starts preparing a package asynchronously and this call returns having applied nothing; a later call retrieves it. This function does **not** wait/poll for preparation to finish — it applies what is available now. To force an update and wait for not-yet-ready changes, use [`cloudsync_network_sync(wait_ms, max_retries)`](#cloudsync_network_syncwait_ms-max_retries). -This function is designed to be called periodically to keep the local database in sync. -To force an update and wait for changes (with a timeout), use [`cloudsync_network_sync(wait_ms, max_retries)`]. +By default this function **drains all currently-available chunks** in one call. Pass `max_chunks` to cap how many chunks are applied per call, for caller-driven progress or traffic control: -If the network is misconfigured or the remote server is unreachable, the function returns an error. -On success, it returns `SQLITE_OK`, and the return value indicates how many changes were downloaded and applied. +```sql +-- Drain at most 5 chunks, loop until the stream is complete +SELECT cloudsync_network_receive_changes(5) ->> '$.receive.complete'; +``` -**Parameters:** None. +The drain position (the per-stream page cursor) is held **in memory** on the network context, so a capped drain resumes where it left off on the next call — the caller does not manage any cursor; it just loops while `receive.complete` is `false`. If the connection is closed or the process restarts mid-drain, the cursor is lost and the next call safely restarts the drain from the beginning of the stream: already-applied chunks are re-downloaded and re-applied idempotently, so **no rows are skipped** — only redundant download is incurred. This is safe because the durable receive checkpoint (`check_dbversion`/`check_seq`) only advances after a stream has been **fully** applied, never in the middle of a source `db_version`. + +If the network is misconfigured or the remote server is unreachable, the function raises a SQL error. If the received payload cannot be applied locally (for example because of an unknown schema hash), the error is returned as a `receive.error` field in the JSON response. If the server reports an unresolved failed check job (e.g. an `encode_changes` failure), that failure is forwarded as a `receive.lastFailure` object. + +**Parameters:** + +- `max_chunks` (INTEGER, optional): Maximum number of chunks to apply this call. Omit or pass `0` (or negative) to drain everything available. A positive value caps the drain; `receive.complete` will be `false` when the cap stops a drain that still has pending chunks. + +**Returns:** A JSON string with the receive result: -**Returns:** The number of changes downloaded. Errors are reported via the SQLite return code. +```json +{"receive": {"rows": N, "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}}} +``` + +- `receive.rows`: The total number of rows received and applied to the local database, summed across all chunks drained this call. `0` when the receive phase failed, when nothing was available, or when only intermediate fragments were staged without completing a value. +- `receive.tables`: An array of table names that received changes (the union across all drained chunks). Empty (`[]`) if no changes were applied or the receive phase failed. +- `receive.chunks`: The number of payload chunks applied by this call. `0` when nothing was ready, `1` for a single monolithic/inline page, and `N` for a drained `N`-chunk stream (bounded by `max_chunks` if given). +- `receive.bytes`: The total serialized payload bytes received this call (uncompressed cloudsync payload size, summed across chunks; transport-independent, not the compressed wire size). Useful for byte-budgeted draining together with `max_chunks`. +- `receive.complete` (boolean): `true` when the receive stream is fully drained (nothing pending), `false` when more chunks remain — because `max_chunks` capped the drain, or it stopped early. When `false`, call this function again to continue. +- `receive.error` (optional, string): Present when client-side `cloudsync_payload_apply` failed. Contains a human-readable error message describing why the received payload could not be applied. +- `receive.lastFailure` (optional, object): Present only when the server reports a failed check job. Forwarded verbatim from the server's `failures.check` and typically includes `jobId`, `dbVersion`, `seq`, `code`, `stage`, `message`, `retryable`, and `failedAt`. Distinct from `receive.error`: `receive.error` describes a client-side apply failure (string), while `receive.lastFailure` describes a server-side check-job failure (object). Both can coexist in the same response. This function is **check-scoped**: server-reported apply-job failures (`failures.apply`) are not surfaced here — see [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) and [`cloudsync_network_sync()`](#cloudsync_network_syncwait_ms-max_retries). **Example:** ```sql -SELECT cloudsync_network_check_changes(); +SELECT cloudsync_network_receive_changes(); +-- '{"receive":{"rows":3,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' + +-- Capped drain with more pending (call again to continue): +-- '{"receive":{"rows":40,"tables":["docs"],"chunks":5,"bytes":1310720,"complete":false}}' + +-- With a client-side apply error: +-- '{"receive":{"rows":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' + +-- With a server-reported check-job failure: +-- '{"receive":{"rows":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"lastFailure":{"jobId":456,"dbVersion":15,"seq":1,"code":"tenant_unreachable","stage":"encode_changes","message":"tenant check failed","retryable":true,"failedAt":"2026-04-24T10:22:00Z"}}}' ``` --- +### `cloudsync_network_check_changes([max_chunks])` + +> **Deprecated:** use [`cloudsync_network_receive_changes()`](#cloudsync_network_receive_changesmax_chunks). This name is retained as a thin alias for backward compatibility and will be removed in a future major version. It behaves identically, including the optional `max_chunks` argument and all returned fields. + +--- + ### `cloudsync_network_sync([wait_ms], [max_retries])` **Description:** Performs a full synchronization cycle. This function has two overloads: - `cloudsync_network_sync()`: Performs one send operation and one check operation. -- `cloudsync_network_sync(wait_ms, max_retries)`: Performs one send operation and then repeatedly tries to download remote changes until at least one change is downloaded or `max_retries` times has been reached, waiting `wait_ms` between retries. +- `cloudsync_network_sync(wait_ms, max_retries)`: Performs one send operation and then downloads remote changes. + +When the server delivers changes as a stream of chunks, this function drains the **whole stream in a single call**: as long as the next chunk is already available it is fetched back-to-back with no delay. `wait_ms` and `max_retries` are spent only while the server payload is **not yet ready** (the server is still preparing a package): in that case the function waits `wait_ms` and retries up to `max_retries` times. They are not consumed while paging through chunks that are already available. **Parameters:** -- `wait_ms` (INTEGER, optional): The time to wait in milliseconds between retries. Defaults to 100. -- `max_retries` (INTEGER, optional): The maximum number of times to retry the synchronization. Defaults to 1. +- `wait_ms` (INTEGER, optional): The time to wait in milliseconds between retries while the server payload is not yet ready. Defaults to 100. +- `max_retries` (INTEGER, optional): The maximum number of poll attempts while the server payload is not yet ready. Defaults to 1. -**Returns:** The number of changes downloaded. Errors are reported via the SQLite return code. +**Returns:** A JSON string with the full sync result, combining send and receive: + +```json +{ + "send": {"status": "synced|syncing|out-of-sync|error", "localVersion": N, "serverVersion": N, "chunks": C, "bytes": B, "lastFailure": {...}}, + "receive": {"rows": N, "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}} +} +``` + +- `send.status`: The current sync state — `"synced"`, `"syncing"`, `"out-of-sync"`, or `"error"`. +- `send.localVersion`: The latest local database version. +- `send.serverVersion`: The latest version confirmed by the server. +- `send.chunks` / `send.bytes`: Number of payload chunks sent and total serialized payload bytes sent during the send phase. Same semantics as in [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes). +- `send.lastFailure` (optional): Same semantics as in [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) — forwarded verbatim from the server's `failures.apply` whenever a failed apply job is reported, regardless of `status`. +- `receive.rows`: The **total** number of rows received and applied during the receive phase, summed across **all** chunks drained in this call. `0` when the receive phase failed. +- `receive.tables`: An array of table names that received changes (the union across all drained chunks). Empty (`[]`) if no changes were applied or the receive phase failed. +- `receive.chunks`: The number of payload chunks applied in this call. `0` when nothing was ready, `1` for a single monolithic/inline page, and `N` for a fully drained `N`-chunk stream. `cloudsync_network_sync()` always drains the whole stream (it does not cap chunks). +- `receive.bytes`: The total serialized payload bytes received this call (uncompressed cloudsync payload size, summed across chunks; not the compressed wire size). Same semantics as in [`cloudsync_network_receive_changes()`](#cloudsync_network_receive_changesmax_chunks). +- `receive.complete` (boolean): `true` when the server stream was fully drained, `false` when the download stopped before the final chunk (an error occurred, or an internal safety bound was reached). When `false`, call `cloudsync_network_sync()` again to resume; re-delivered rows are idempotent. +- `receive.error` (optional, string): Present when client-side `cloudsync_payload_apply` failed (for example `"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."`). The send result is always preserved so the caller can tell that local changes reached the server even when applying incoming changes failed. The receive drain stops immediately on apply errors, since failures like schema-hash mismatches do not heal across retries. Endpoint/network errors during the receive phase raise a SQL error instead. +- `receive.lastFailure` (optional, object): Same semantics as in [`cloudsync_network_receive_changes()`](#cloudsync_network_receive_changesmax_chunks) — forwarded verbatim from the server's `failures.check` whenever a failed check job is reported. Distinct from `receive.error`. `cloudsync_network_sync()` reports both `send.lastFailure` and `receive.lastFailure` when present. **Example:** ```sql -- Perform a single synchronization cycle SELECT cloudsync_network_sync(); +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":2048},"receive":{"rows":3,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' -- Perform a synchronization cycle with custom retry settings SELECT cloudsync_network_sync(500, 3); +-- A large download drained as a multi-chunk stream in a single call: +-- '{"send":{"status":"synced","localVersion":42,"serverVersion":42,"chunks":0,"bytes":0},"receive":{"rows":1200,"tables":["docs"],"chunks":7,"bytes":1835008,"complete":true}}' + +-- Receive phase failed but send phase completed — the error is surfaced in JSON, not as a SQL error: +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":512},"receive":{"rows":0,"tables":[],"chunks":0,"bytes":0,"complete":false,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' ``` --- @@ -455,9 +918,25 @@ SELECT cloudsync_network_reset_sync_version(); --- +### `cloudsync_network_has_unsent_changes()` + +**Description:** Checks if there are any local changes that have not yet been sent to the remote server. + +**Parameters:** None. + +**Returns:** 1 if there are unsent changes, 0 otherwise. + +**Example:** + +```sql +SELECT cloudsync_network_has_unsent_changes(); +``` + +--- + ### `cloudsync_network_logout()` -**Description:** Logs out the current user and cleans up all local data from synchronized tables. This function deletes and then re-initializes synchronized tables, useful for switching users or resetting the local database. **Warning:** This function deletes all data from synchronized tables. Use with caution. +**Description:** Logs out the current user and cleans up all local data from synchronized tables. This function deletes and then re-initializes synchronized tables, useful for switching users or resetting the local database. **Warning:** This function deletes all data from synchronized tables. Use with caution. Consider calling [`cloudsync_network_has_unsent_changes()`](#cloudsync_network_has_unsent_changes) before logout to check for unsent local changes and warn the user before data that has not been fully synchronized to the remote server is deleted. **Parameters:** None. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..b99bfdc0 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,245 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## [Unreleased] + +### Added + +- **`sqlitecloud/sqlite-sync-supabase:17-alpine`** — CloudSync on the newer Alpine-userland Supabase Postgres images (roughly `17.6.1.084` and later). The extension is identical to the Ubuntu build — the PostgreSQL binary is glibc-linked on both — so only the base image differs. Use it when your Supabase stack pins one of these newer bases; `:17` continues to track the Ubuntu base. + +### Changed + +- **The local build-from-source path (`make postgres-supabase-build`, `docker/postgresql/Dockerfile.supabase`) now works with the Alpine-userland Supabase bases** as well as the Ubuntu ones. The extension is compiled in a dedicated glibc builder stage (official `postgres:` image with PGDG headers, the same toolchain that produces the release artifacts) whose PostgreSQL major version is derived from the base image tag, and `pg_config` is resolved by probing the Nix profile paths instead of a hardcoded Ubuntu-only path. The runtime stage no longer needs a compiler or package manager, so the base userland is irrelevant. +- **The `sqlitecloud/sqlite-sync-supabase:15` image now builds on Supabase base `15.8.1.135`** (previously `15.8.1.085`), which moves it from Ubuntu 20.04 to 24.04. Ubuntu 24.04 allocates system UIDs differently, so the `postgres` user changes from `105:106` to `101:102` and an **existing** data directory becomes unreadable: the container fails with `cat: /etc/postgresql-custom/pgsodium_root.key: Permission denied` followed by `FATAL: invalid secret key`. Existing deployments need a one-time `chown -R 101:102` on the data directory and the `supabase_db-config` volume before starting the new image — see [Self-Hosted Supabase](docs/postgresql/quickstarts/supabase-self-hosted.md) for the procedure. New deployments are unaffected, and the CloudSync extension itself is unchanged. + +## [1.1.2] - 2026-07-13 + +### Fixed + +- Querying the SQLite `cloudsync_changes` virtual table (directly or via `cloudsync_payload_blob_checked()`) on a database whose `cloudsync_table_settings` rows survived while all `*_cloudsync` meta tables were lost (e.g. a dump/restore that skipped them) now raises an actionable error ("cloudsync settings reference tables whose sync metadata is missing ...") instead of a misclassified `SQLITE_NOMEM`, which servers reported as "Not enough memory to execute query". + +## [1.1.1] - 2026-07-10 + +### Fixed + +- Querying the SQLite `cloudsync_payload_chunks()` virtual table on a database where cloudsync is not initialized now raises the same actionable error as `cloudsync_payload_apply()` ("cloudsync is not initialized: call SELECT cloudsync_init('') ...") instead of a bare "SQL logic error". Errors raised by the internal `cloudsync_changes` scan (including the watermark computation, which previously ignored them) are now propagated with their message instead of being masked. + +## [1.1.0] - 2026-07-09 + +### Added + +- **Chunked payload generation** via `cloudsync_payload_chunks()`, available as a SQLite virtual table and as a PostgreSQL set-returning function. The API emits transport-sized payload chunks and transparently fragments oversized BLOB/TEXT values into v3 fragment payloads. +- **`payload_max_chunk_size` global setting** for controlling generated chunk size. The default is 5 MB and values are clamped to the 256 KB technical minimum and the 32 MB technical maximum. +- **`exclude_filter_site_id` argument** for `cloudsync_payload_chunks()`. When set, the function streams changes from every site **except** `filter_site_id`, which is what the `/check` download path needs (a peer must not receive its own changes back). The default (omitted/`false`) preserves the existing single-site behavior. Passing the flag without a `filter_site_id` is an error. +- **`cloudsync_payload_blob_checked()`** scalar function on both SQLite and PostgreSQL. It performs an internal conservative size check before generating one monolithic legacy payload BLOB, allowing `/check` endpoints to support old clients with one SQL round trip while rejecting unsafe responses before payload materialization. +- **`cloudsync_uuid_text()` / `cloudsync_uuid_blob()`** scalar functions on both SQLite and PostgreSQL, converting between the 16-byte binary `site_id` and its canonical UUID string. `cloudsync_uuid_text()` takes an optional `dash_format` argument (default `true`); `cloudsync_uuid_blob()` accepts dashed or undashed, case-insensitive input. These let string-based callers (e.g. the `/check` endpoint) pass a `site_id` to `cloudsync_payload_chunks()`. +- **Payload chunking documentation** in `API.md` and `PERFORMANCE.md`, including the explicit memory note that chunking bounds transport payloads but the database must still materialize a completed single BLOB/TEXT value when it is applied. +- **PostgreSQL `1.0 -> 1.1` upgrade script** (`migrations/cloudsync--1.0--1.1.sql`) for the new chunked-payload SQL surface, so existing deployments can `ALTER EXTENSION cloudsync UPDATE`. +- **`cloudsync_network_receive_changes([max_chunks])`** as the canonical name for receiving changes (the old `cloudsync_network_check_changes()` is retained as a deprecated alias). It drains all currently-available chunks by default; the optional `max_chunks` argument caps how many chunks are applied per call for caller-driven progress/traffic control (the in-memory page cursor persists between calls, so a capped drain resumes where it left off). +- **`chunks` and `bytes` fields** in the `send` and `receive` JSON of the network functions, plus **`complete`** in `receive`. `chunks` is the number of payload chunks sent/applied; `bytes` is the serialized (uncompressed) payload bytes; `complete` is `false` when a chunked download stopped before the final chunk, signalling the caller to call again. + +### Deprecated + +- **`cloudsync_network_check_changes([max_chunks])`** — use `cloudsync_network_receive_changes()` instead. The old name remains a thin, fully-functional alias and will be removed in a future major version. + +### Changed + +- `cloudsync_network_receive_changes()` (and its `cloudsync_network_check_changes()` alias) now **drains all available chunks** in one call (it does not wait for not-yet-ready server preparation). Previously each call fetched a single page. +- `cloudsync_network_sync(wait_ms, max_retries)` now drains an entire chunked download in a **single call**, fetching already-available chunks back-to-back with no delay. `wait_ms` / `max_retries` are now spent only while the server payload is not yet ready (HTTP 202), not while paging through chunks that are already available. Previously a multi-chunk stream required several `cloudsync_network_sync()` calls and wasted a `wait_ms` delay on each staged fragment. +- `receive.rows` is now the **cumulative** number of rows applied across all chunks drained in the call (previously only the last page was reported); `receive.tables` likewise reports the union of tables touched across the whole drain. +- `cloudsync_payload_apply()` now accepts legacy payloads, monolithic payloads, and v3 fragment payloads without enforcing the local `payload_max_chunk_size`, preserving compatibility between peers with different settings. +- `cloudsync_network_send_changes()` now streams outgoing changes through `cloudsync_payload_chunks()` instead of first building one monolithic payload. This bounds transport payload size for the built-in network path and lets large rowsets or oversized BLOB/TEXT values flow through the same `/apply` endpoint as regular payloads. +- The built-in `/check` receive path now advertises `X-CloudSync-Capabilities: check-status-response, check-chunks` and can apply cursor-mode pages returned inline as `data.payload` base64 bytes, in addition to larger pages returned as `data.url` download artifacts. Requests send `cursor`; responses provide `nextCursor` when another page is available. +- The chunked-download receive path advances the local receive checkpoint (`check_dbversion` / `check_seq`) **only after a chunk stream has been fully applied**, jumping straight to the stream watermark — never into the middle of a source `db_version`. This mirrors the send path and ensures a stop between chunks cannot skip the un-applied rows of a `db_version` split across chunks on the next `/check` (the server resumes on `db_version > since`, with no intra-version cursor). `cloudsync_payload_apply()` no longer advances the receive checkpoint per applied chunk; the built-in network `/check` path drives it from the server's watermark and final-chunk signal, and falls back to the previous monolithic behavior when the server sends no watermark. Re-delivered rows remain idempotent. + +### Fixed + +- **PostgreSQL backend crash (segfault) on an error raised after `cloudsync_changes_select()`.** The set-returning function returned via `SRF_RETURN_NEXT` / `SRF_RETURN_DONE` from inside its `PG_TRY` block, which skips `PG_END_TRY()` and leaves `PG_exception_stack` pointing at the function's already-returned stack frame. A later `ereport(ERROR)` in the same query — such as the `cloudsync_payload_blob_checked()` size-limit check — then `siglongjmp()`d into freed stack and crashed the backend. The `SRF_RETURN_*` calls now run after `PG_END_TRY()` so the exception stack is always restored. This is a pre-existing bug, not specific to the chunked-payload work. + +## [1.0.20] - 2026-05-26 + +### Changed + +- **Improved network sync performance** by reducing request overhead during `cloudsync_network_send_changes()`, especially for small payloads that can now be applied without the extra upload-URL round trip. +- **Improved repeated sync request latency** by allowing the network layer to reuse HTTP connections across CloudSync API calls. + +## [1.0.19] - 2026-05-15 + +### Added + +- **Mac Catalyst support**. + +## [1.0.18] - 2026-04-29 + +### Fixed + +- **`cloudsync_network_check_changes()`** no longer errors with `missing 'url' in check response` when the server has not yet prepared any incoming changes for this device. The function now returns the standard "no rows yet" response in that case, so polling loops keep working without spurious errors. + +### Added + +- **`receive.lastFailure`** JSON field on `cloudsync_network_check_changes()` and `cloudsync_network_sync()`, surfacing the most recent server-side failure of the receive pipeline (e.g. the server failed to prepare the next batch of incoming changes for this device). It complements the existing `send.lastFailure` (server-side apply failures) and `receive.error` (local apply failures on this device), so applications can distinguish "the server has trouble producing my changes" from "I had trouble applying them locally". Each function reports only the failures relevant to its own scope: `cloudsync_network_send_changes()` reports `send.lastFailure`; `cloudsync_network_check_changes()` reports `receive.lastFailure`; `cloudsync_network_sync()` reports both. + +### Changed + +- Updated the request headers sent to the cloudsync HTTP endpoints (version advertisement, per-endpoint capabilities; legacy `Accept` header removed). + +## [1.0.17] - 2026-04-24 + +### Fixed + +- **Confusing errors when `cloudsync_init` was never called**: `cloudsync_changes` (SQLite), `cloudsync_db_version`, `cloudsync_db_version_next`, `cloudsync_set_filter`, `cloudsync_clear_filter`, and `cloudsync_payload_apply` now raise a single actionable message pointing at `SELECT cloudsync_init('')` instead of leaking low-level symptoms (`out of memory`, `not an error`, silent `-1`, multi-line "no such table" dumps). The guard runs only on the error branch, so the sync hot path is unaffected. + +## [1.0.16] - 2026-04-16 + +### Fixed + +- **WASM crash in `cloudsync_set_column` on existing rows**: Calling `cloudsync_set_column(table, col, 'lww', 'block')` on a table with pre-existing rows crashed the WASM build with a `RuntimeError: function signature mismatch` as soon as the block-index migration tried to allocate memory. `block_init_allocator` was casting `cloudsync_memory_alloc` (a `uint64_t size` function) directly to the fractional-indexing allocator's `void *(*)(size_t)` slot. The cast is a no-op on native platforms where `size_t` is 64-bit, but WASM's `call_indirect` enforces strict type checking — the function is registered as `(i64) -> i32` and called as `(i32) -> i32`, triggering an immediate runtime error. A thin `fi_malloc_wrapper` (mirroring the existing `fi_calloc_wrapper`) now bridges the signatures. Native builds are unaffected. + +## [1.0.15] - 2026-04-16 + +### Fixed + +- **Silent receive failures**: When `cloudsync_payload_apply` failed during the receive phase (for example with an unknown schema hash, invalid checksum, or decompression error), the error was stored only on the internal cloudsync context and never propagated to the SQL caller. Both `cloudsync_network_check_changes()` and `cloudsync_network_sync()` silently returned no result. Apply errors are now surfaced as a `receive.error` field in the JSON response. + +### Changed + +- **Error handling contract**: endpoint/network errors (server unreachable, auth failure, bad URL) always raise a SQL error. Processing errors (`cloudsync_payload_apply` failures) are returned as structured JSON via `receive.error` or `send.lastFailure`, so callers can inspect and log them without try/catch logic. +- **`cloudsync_network_send_changes()` output** now includes a `send.lastFailure` object whenever the server reports one (raw pass-through of the server's `lastFailure` — `jobId`, `code`, `message`, `retryable`, `failedAt`, …), regardless of whether the computed `send.status` is `synced`, `syncing`, or `out-of-sync`. The field is omitted when the server does not report a failure. +- **`cloudsync_network_check_changes()` output** now includes a `receive.error` string when `cloudsync_payload_apply` fails, instead of silently returning NULL. Endpoint/network errors still raise a SQL error. +- **`cloudsync_network_sync()` output** now mirrors the same `send.lastFailure` field and, if the receive phase has a processing error (`cloudsync_payload_apply` failure), returns structured JSON with a `receive.error` string rather than failing silently. The send result is always preserved so callers can tell that their local changes reached the server even when applying incoming changes failed. Endpoint/network errors during the receive phase still raise a SQL error. The receive retry loop breaks immediately on processing errors (a schema-hash mismatch will not heal across retries). + +## [1.0.14] - 2026-04-15 + +### Fixed + +- **Stale `cloudsync_table_settings` crash**: Reopening a database that had its base table and `
_cloudsync` meta-table dropped without calling `cloudsync_cleanup` crashed with a double-free on `sqlite3_close`. Two bugs were involved: (1) `cloudsync_dbversion_rebuild` returned `DBRES_NOMEM` when `cloudsync_dbversion_build_query` yielded a NULL SQL string (stale row in `cloudsync_table_settings` but no matching `*_cloudsync` table in `sqlite_master`), failing extension init; (2) on init failure `dbsync_register_functions` manually freed the context that SQLite already owned via the `cloudsync_version` destructor, causing a double-free when the connection was later closed. `cloudsync_dbversion_rebuild` now treats a NULL build query the same as `count == 0` (no prepared statement, db_version stays at the minimum and is rebuilt on the next `cloudsync_init`), and the manual free in the error path has been removed. + +### Added + +- Unit test `do_test_stale_table_settings_dropped_meta` (Stale Table Settings Dropped Meta) covering the drop-base-table + drop-meta-table + reopen scenario. + +## [1.0.13] - 2026-04-14 + +### Fixed + +- **Block-level LWW migration**: When `cloudsync_set_column(..., 'algo', 'block')` is called on a table that already has tracked rows, those rows are now immediately migrated into the blocks table. Previously, pre-existing column values were ignored until the next UPDATE, leaving sync state incomplete. The migration uses a two-phase collect-then-write approach to avoid SQLite cursor invalidation and `INSERT OR IGNORE` / `ON CONFLICT DO NOTHING` semantics for idempotency. + +### Added + +- Unit test `do_test_block_lww_existing_data` (Block LWW Existing Data) verifying block migration on `set_column`, idempotency of repeated `set_column` calls, and correct materialization after update. +- PostgreSQL test `50_block_lww_existing_data.sql` with equivalent coverage for the PostgreSQL backend. + +## [1.0.12] - 2026-04-11 + +### Fixed + +- **Settings loader**: Prevent infinite loop in `sqlite3_cloudsync_init` when reopening a database that has a persisted block-column setting. `dbutils_settings_table_load_callback` was calling `cloudsync_setup_block_column`, which `REPLACE`d the same row into `cloudsync_table_settings` while `sqlite3_exec` was still iterating it, re-feeding the rewritten row to the cursor. Added a `persist` flag to `cloudsync_setup_block_column` so the loader replays the in-memory setup without writing back. +- **PostgreSQL tests**: Updated 168 `cloudsync_init` callsites across 43 `test/postgresql/*.sql` files to pass integer flags (`0`/`1`) instead of `true`/`false`, matching the signature change in 1.0.9. +- **CI**: The `postgres-test` job now fails on SQL errors and `[FAIL]` markers. `psql` is run with `ON_ERROR_STOP=on`, `pipefail` is enabled around the `tee`, and the captured log is grepped for `[FAIL]` / `psql ERROR` as a final guard. + +### Added + +- Unit test `do_test_block_column_reload` (Block Column Reload) that persists a block column with a custom delimiter, closes the database, and reopens it — without the fix this hangs the test process. + +## [1.0.11] - 2026-04-11 + +### Fixed + +- **cloudsync_cleanup**: Now also drops the `{table}_cloudsync_blocks` table when the table has block LWW columns configured via `cloudsync_set_column(..., 'algo', 'block')`. + +### Added + +- Unit test `do_test_block_lww_cleanup` verifying that both `{table}_cloudsync` and `{table}_cloudsync_blocks` are removed after `cloudsync_cleanup`. + +## [1.0.10] - 2026-04-08 + +### Fixed + +- **PostgreSQL**: Prevent debug assertion crash on `cloudsync_init` error path (#37). +- **Row filter**: `cloudsync_set_filter` and `cloudsync_clear_filter` now reset the metatable and refill it from scratch, ensuring only rows matching the active filter are tracked for sync (#38). + +### Added + +- Row filter edge-case test coverage: clear/change filter lifecycle, complex expressions (AND, IS NULL), row enter/exit via UPDATE, composite PK with multi-column filters, multi-table roundtrip sync, and pre-existing data prefill tests for both SQLite and PostgreSQL. + +## [1.0.9] - 2026-04-08 + +### Changed + +- **cloudsync_init**: Replaced the `force` boolean parameter with an `init_flags` integer bitmask (`CLOUDSYNC_INIT_FLAG`), allowing fine-grained control over which schema sanity checks are skipped. Existing callers passing `0`/`false` or `1`/`true` remain compatible. +- **API**: Updated `cloudsync_init` SQL signature (PostgreSQL) to accept `integer` instead of `boolean` for the third argument, enabling flag combinations via bitwise OR. + +### Added + +- `CLOUDSYNC_INIT_FLAG_NONE` (0), `CLOUDSYNC_INIT_FLAG_SKIP_INT_PK_CHECK` (1), `CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_DEFAULT_CHECK` (2), `CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_PRIKEYS_CHECK` (4) enum values. +- Documentation for `cloudsync_set_filter` and `cloudsync_clear_filter` in API.md. + +## [1.0.8] - 2026-04-03 + +### Changed + +- **CI/CD**: Fix flutter package publish workflow not triggering on new releases. + +## [1.0.7] - 2026-04-02 + +### Fixed + +- **Harden table initialization against stale config and error cleanup.** + +## [1.0.2] - 2026-03-25 + +### Fixed + +- **Swift Package**: Use binary target and versioned macOS framework for Xcode 26 compatibility. +- **Minor bugs** in tests, docs, and examples related to the 1.0.0 major release. + +## [1.0.0] - 2026-03-24 + +### Added + +- **PostgreSQL support**: The CloudSync extension can now be built and loaded on PostgreSQL, so both SQLiteCloud and PostgreSQL are supported as the cloud backend database of the sync service. The core CRDT functions are shared by the SQLite and PostgreSQL extensions. Includes support for PostgreSQL-native types (UUID primary keys, composite PKs with mixed types, and automatic type casting). +- **Row-Level Security (RLS)**: Sync payloads are now fully compatible with SQLiteCloud and PostgreSQL Row-Level Security policies. Changes are buffered per primary key and flushed as complete rows, so RLS policies can evaluate all columns at once. +- **Block-level LWW for text conflict resolution**: Text columns can now be tracked at block level (lines by default) using Last-Writer-Wins. Concurrent edits to different parts of the same text are preserved after sync. New functions: `cloudsync_set_column()` to write individual blocks and `cloudsync_text_materialize()` to reconstruct the full text. + +### Changed + +- **BREAKING: `cloudsync_network_init` now accepts a `managedDatabaseId` instead of a connection string.** The `managedDatabaseId` is returned by the CloudSync service when a new database is registered for sync. For SQLiteCloud projects, it can be obtained from the project's OffSync page on the dashboard. + + Before: + ```sql + SELECT cloudsync_network_init('sqlitecloud://myproject.sqlite.cloud:8860/mydb.sqlite?apikey=KEY'); + ``` + + After: + ```sql + SELECT cloudsync_network_init('your-managed-database-id'); + ``` + +- **BREAKING: Sync functions now return structured JSON.** `cloudsync_network_send_changes`, `cloudsync_network_check_changes`, and `cloudsync_network_sync` return a JSON object instead of a plain integer. This provides richer status information including sync state, version numbers, row counts, and affected table names. + + Before: + ```sql + SELECT cloudsync_network_sync(); + -- 3 (number of rows received) + ``` + + After: + ```sql + SELECT cloudsync_network_sync(); + -- '{"send":{"status":"synced","localVersion":5,"serverVersion":5},"receive":{"rows":3,"tables":["tasks"]}}' + ``` + +- **Batch merge replaces column-by-column processing**: During sync, changes to the same row are now applied in a single SQL statement instead of one statement per column. This eliminates the previous behavior where UPDATE triggers fired multiple times per row during synchronization. +- **Network endpoints updated for the CloudSync v2 HTTP service**: Internal network layer now targets the new CloudSync service endpoints, including support for multi-organization routing. +- **NULL primary key rejection at runtime**: The extension now enforces NULL primary key rejection at runtime, so the explicit `NOT NULL` constraint on primary key columns is no longer a schema requirement. + +### Fixed + +- **Improved error reporting**: Sync network functions now surface the actual server error message instead of generic error codes. +- **Schema hash verification**: Normalized schema comparison now uses only column name (lowercase), type (SQLite affinity), and primary key flag, preventing false mismatches caused by formatting differences. +- **SQLite trigger safety**: Internal functions used inside triggers are now marked with `SQLITE_INNOCUOUS`, fixing `unsafe use of` errors when initializing tables that have triggers. +- **NULL column binding**: Column value parameters are now correctly bound even when NULL, preventing sync failures on rows with NULL values. +- **Stability and reliability improvements** across the SQLite and PostgreSQL codebases, including fixes to memory management, error handling, and CRDT version tracking. diff --git a/Makefile b/Makefile index b77a04c3..f9d1acf1 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # Supports compilation for Linux, macOS, Windows, Android and iOS # customize sqlite3 executable with -# make test SQLITE3=/opt/homebrew/Cellar/sqlite/3.49.1/bin/sqlite3 +# make test SQLITE3=/opt/homebrew/Cellar/sqlite/3.50.4/bin/sqlite3 SQLITE3 ?= sqlite3 # set curl version to download and build @@ -30,9 +30,15 @@ endif # Speed up builds by using all available CPU cores MAKEFLAGS += -j$(CPUS) +# Mac Catalyst uses the Apple-native NSURLSession networking path. This needs to +# be set before the common linker flags decide whether libcurl is required. +ifeq ($(PLATFORM),mac-catalyst) + override NATIVE_NETWORK := ON +endif + # Compiler and flags CC = gcc -CFLAGS = -Wall -Wextra -Wno-unused-parameter -I$(SRC_DIR) -I$(SQLITE_DIR) -I$(CURL_DIR)/include +CFLAGS = -Wall -Wextra -Wno-unused-parameter -I$(SRC_DIR) -I$(SRC_DIR)/sqlite -I$(SRC_DIR)/postgresql -I$(SRC_DIR)/network -I$(SQLITE_DIR) -I$(CURL_DIR)/include -Imodules/fractional-indexing T_CFLAGS = $(CFLAGS) -DSQLITE_CORE -DCLOUDSYNC_UNITTEST -DCLOUDSYNC_OMIT_NETWORK -DCLOUDSYNC_OMIT_PRINT_RESULT COVERAGE = false ifndef NATIVE_NETWORK @@ -41,10 +47,14 @@ endif # Directories SRC_DIR = src +SQLITE_IMPL_DIR = $(SRC_DIR)/sqlite +POSTGRES_IMPL_DIR = $(SRC_DIR)/postgresql DIST_DIR = dist TEST_DIR = test SQLITE_DIR = sqlite -VPATH = $(SRC_DIR):$(SQLITE_DIR):$(TEST_DIR) +FI_DIR = modules/fractional-indexing +NETWORK_DIR = $(SRC_DIR)/network +VPATH = $(SRC_DIR):$(SQLITE_IMPL_DIR):$(POSTGRES_IMPL_DIR):$(NETWORK_DIR):$(SQLITE_DIR):$(TEST_DIR):$(FI_DIR) BUILD_RELEASE = build/release BUILD_TEST = build/test BUILD_DIRS = $(BUILD_TEST) $(BUILD_RELEASE) @@ -59,15 +69,44 @@ ifeq ($(PLATFORM),android) OPENSSL_INSTALL_DIR = $(OPENSSL_DIR)/$(PLATFORM)/$(ARCH) endif -SRC_FILES = $(wildcard $(SRC_DIR)/*.c) -TEST_SRC = $(wildcard $(TEST_DIR)/*.c) +# Multi-platform source files (at src/ root) - exclude database_*.c as they're in subdirs +CORE_SRC = $(filter-out $(SRC_DIR)/database_%.c, $(wildcard $(SRC_DIR)/*.c)) $(wildcard $(NETWORK_DIR)/*.c) +# SQLite-specific files +SQLITE_SRC = $(wildcard $(SQLITE_IMPL_DIR)/*.c) +# Fractional indexing submodule +FI_SRC = $(FI_DIR)/fractional_indexing.c +# Combined for SQLite extension build +SRC_FILES = $(CORE_SRC) $(SQLITE_SRC) $(FI_SRC) + +# network_unit.c is built separately with networking ENABLED (see network-unittest), +# so keep it out of the default OMIT_NETWORK test build. +TEST_SRC = $(filter-out $(TEST_DIR)/network_unit.c,$(wildcard $(TEST_DIR)/*.c)) TEST_FILES = $(SRC_FILES) $(TEST_SRC) $(wildcard $(SQLITE_DIR)/*.c) RELEASE_OBJ = $(patsubst %.c, $(BUILD_RELEASE)/%.o, $(notdir $(SRC_FILES))) TEST_OBJ = $(patsubst %.c, $(BUILD_TEST)/%.o, $(notdir $(TEST_FILES))) -COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(SRC_DIR)/network.c, $(SRC_FILES)) +COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(NETWORK_DIR)/network.c $(SQLITE_IMPL_DIR)/sql_sqlite.c $(POSTGRES_IMPL_DIR)/database_postgresql.c $(FI_SRC), $(SRC_FILES)) CURL_LIB = $(CURL_DIR)/$(PLATFORM)/libcurl.a TEST_TARGET = $(patsubst %.c,$(DIST_DIR)/%$(EXE), $(notdir $(TEST_SRC))) +# Network-enabled unit tests: rebuild the codebase with networking ON (T_CFLAGS +# minus OMIT_NETWORK) and link curl, so network.c's internal functions can be +# tested directly on in-memory buffers. NT_LDFLAGS reuses the platform LDFLAGS +# (which carries -lcurl) minus the shared-library-only flags (-shared on Linux, +# -dynamiclib on macOS) so it links as an executable, plus the test link libs. +# -undefined dynamic_lookup is kept: the test never opens a connection, so curl's +# transport symbols are linked but never invoked. +BUILD_NETTEST = build/nettest +NT_CFLAGS = $(filter-out -DCLOUDSYNC_OMIT_NETWORK,$(T_CFLAGS)) +NT_LDFLAGS = $(filter-out -shared -dynamiclib -headerpad_max_install_names,$(LDFLAGS)) $(T_LDFLAGS) +NT_SRC = $(SRC_FILES) $(SQLITE_DIR)/sqlite3.c $(TEST_DIR)/network_unit.c +NT_OBJ = $(patsubst %.c,$(BUILD_NETTEST)/%.o,$(notdir $(NT_SRC))) + +# Build curl hermetically: neutralize the developer's ambient build env so +# curl's ./configure compile tests aren't broken by overrides leaking in +# (e.g. exported LDFLAGS/CPPFLAGS/LIBS pointing at Homebrew). Build flags for +# curl are supplied explicitly via CURL_CONFIG. +CURL_CONFIG_ENV = LDFLAGS= CPPFLAGS= LIBS= CFLAGS= + # Platform-specific settings ifeq ($(PLATFORM),windows) TARGET := $(DIST_DIR)/cloudsync.dll @@ -120,7 +159,7 @@ else ifeq ($(PLATFORM),android) CURL_CONFIG = --host $(ARCH)-linux-$(ANDROID_ABI) --with-openssl=$(CURDIR)/$(OPENSSL_INSTALL_DIR) LDFLAGS="-L$(CURDIR)/$(OPENSSL_INSTALL_DIR)/lib" LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(CC) CXX=$(BIN)/$(ARCH)-linux-$(ANDROID_ABI)-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip TARGET := $(DIST_DIR)/cloudsync.so CFLAGS += -fPIC -I$(OPENSSL_INSTALL_DIR)/include - LDFLAGS += -shared -fPIC -L$(OPENSSL_INSTALL_DIR)/lib -lssl -lcrypto -Wl,-z,max-page-size=16384 + LDFLAGS += -shared -fPIC -L$(OPENSSL_INSTALL_DIR)/lib -lssl -lcrypto -lm -Wl,-z,max-page-size=16384 STRIP = $(BIN)/llvm-strip --strip-unneeded $@ else ifeq ($(PLATFORM),ios) TARGET := $(DIST_DIR)/cloudsync.dylib @@ -138,10 +177,23 @@ else ifeq ($(PLATFORM),ios-sim) CFLAGS += -arch x86_64 -arch arm64 $(SDK) CURL_CONFIG = --host=arm64-apple-darwin --with-secure-transport CFLAGS="-arch x86_64 -arch arm64 -isysroot $$(xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0" STRIP = strip -x -S $@ +else ifeq ($(PLATFORM),mac-catalyst) + TARGET := $(DIST_DIR)/cloudsync.dylib + MAC_CATALYST_DEPLOYMENT_TARGET ?= 14.0 + ifndef ARCH + MAC_CATALYST_UNIVERSAL := true + STRIP = strip -x -S $@ + else + SDK := -isysroot $(shell xcrun --sdk macosx --show-sdk-path) -target $(ARCH)-apple-ios$(MAC_CATALYST_DEPLOYMENT_TARGET)-macabi + LDFLAGS += -framework Security -framework CoreFoundation -dynamiclib $(SDK) -headerpad_max_install_names + T_LDFLAGS = -framework Security + CFLAGS += $(SDK) + STRIP = strip -x -S $@ + endif else # linux TARGET := $(DIST_DIR)/cloudsync.so - LDFLAGS += -shared -lssl -lcrypto - T_LDFLAGS += -lpthread + LDFLAGS += -shared -lssl -lcrypto -lm + T_LDFLAGS += -lpthread -lm CURL_CONFIG = --with-openssl STRIP = strip --strip-unneeded $@ endif @@ -154,9 +206,13 @@ endif T_LDFLAGS += -fprofile-arcs -ftest-coverage endif +ifdef NETWORK_TRACE + CFLAGS += -DCLOUDSYNC_NETWORK_TRACE +endif + # Native network support only for Apple platforms ifdef NATIVE_NETWORK - RELEASE_OBJ += $(patsubst %.m, $(BUILD_RELEASE)/%_m.o, $(notdir $(wildcard $(SRC_DIR)/*.m))) + RELEASE_OBJ += $(patsubst %.m, $(BUILD_RELEASE)/%_m.o, $(notdir $(wildcard $(NETWORK_DIR)/*.m))) LDFLAGS += -framework Foundation CFLAGS += -DCLOUDSYNC_OMIT_CURL @@ -179,6 +235,20 @@ $(shell mkdir -p $(BUILD_DIRS) $(DIST_DIR)) extension: $(TARGET) all: $(TARGET) +ifeq ($(MAC_CATALYST_UNIVERSAL),true) +MAC_CATALYST_ARCHS = x86_64 arm64 +MAC_CATALYST_DYLIBS = $(foreach arch,$(MAC_CATALYST_ARCHS),$(DIST_DIR)/cloudsync-mac-catalyst-$(arch).dylib) + +$(TARGET): + @for arch in $(MAC_CATALYST_ARCHS); do \ + rm -rf $(BUILD_DIRS); \ + $(MAKE) PLATFORM=mac-catalyst ARCH=$$arch MAC_CATALYST_DEPLOYMENT_TARGET=$(MAC_CATALYST_DEPLOYMENT_TARGET); \ + mv $(DIST_DIR)/cloudsync.dylib $(DIST_DIR)/cloudsync-mac-catalyst-$$arch.dylib; \ + done + lipo -create $(MAC_CATALYST_DYLIBS) -output $@ + rm -f $(MAC_CATALYST_DYLIBS) + $(STRIP) +else # Loadable library ifdef NATIVE_NETWORK $(TARGET): $(RELEASE_OBJ) $(DEF_FILE) @@ -192,6 +262,7 @@ ifeq ($(PLATFORM),windows) endif # Strip debug symbols $(STRIP) +endif # Test executable $(TEST_TARGET): $(TEST_OBJ) @@ -205,16 +276,68 @@ $(BUILD_TEST)/sqlite3.o: $(SQLITE_DIR)/sqlite3.c $(BUILD_TEST)/%.o: %.c $(CC) $(T_CFLAGS) -c $< -o $@ +# Network-enabled object files (networking ON, for network-unittest) +$(BUILD_NETTEST): + mkdir -p $(BUILD_NETTEST) +$(BUILD_NETTEST)/sqlite3.o: $(SQLITE_DIR)/sqlite3.c | $(BUILD_NETTEST) + $(CC) $(CFLAGS) -DSQLITE_DQS=0 -DSQLITE_CORE -c $< -o $@ +$(BUILD_NETTEST)/%.o: %.c | $(BUILD_NETTEST) + $(CC) $(NT_CFLAGS) -c $< -o $@ + # Run code coverage (--css-file $(CUSTOM_CSS)) -test: $(TARGET) $(TEST_TARGET) - $(SQLITE3) ":memory:" -cmd ".bail on" ".load ./$<" "SELECT cloudsync_version();" - set -e; for t in $(TEST_TARGET); do ./$$t; done +# dist/network_unit is listed before `unittest` so it is built during the file-build +# phase: on Android the host `make test` aborts when `unittest` runs the cross-built +# dist/unit, and network_unit (not in TEST_TARGET) would otherwise never be built +# before `make test -n` captures the on-emulator command script. +test: $(TARGET) $(TEST_TARGET) $(DIST_DIR)/network_unit$(EXE) unittest network-unittest e2e + set -e; $(SQLITE3) ":memory:" -cmd ".bail on" ".load ./$<" "SELECT cloudsync_version();" ifneq ($(COVERAGE),false) mkdir -p $(COV_DIR) lcov --capture --directory . --output-file $(COV_DIR)/coverage.info $(subst src, --include src,${COV_FILES}) genhtml $(COV_DIR)/coverage.info --output-directory $(COV_DIR) endif +# Run only unit tests +unittest: $(TARGET) $(DIST_DIR)/unit$(EXE) + @./$(DIST_DIR)/unit$(EXE) + +# Network-enabled unit test binary. Link it via a file rule (like dist/unit), not in +# the run recipe below: on Android `make test` runs binaries on the emulator from a +# script generated by `make test -n`, so a link command inside the recipe would be +# emitted and the cross-compiler invoked on-device (it isn't there → exit 127). +$(DIST_DIR)/network_unit$(EXE): $(CURL_LIB) $(NT_OBJ) + $(CC) $(NT_OBJ) -o $@ $(NT_LDFLAGS) + +# Run the network-layer unit tests (networking compiled in, no server) +network-unittest: $(DIST_DIR)/network_unit$(EXE) + @./$(DIST_DIR)/network_unit$(EXE) + +# Run end-to-end integration tests +e2e: $(TARGET) $(DIST_DIR)/integration$(EXE) + @if [ -f .env ]; then \ + export $$(grep -v '^#' .env | xargs); \ + fi; \ + ./$(DIST_DIR)/integration$(EXE) + +# Run the sync performance benchmark. This is intentionally separate from e2e +# because timings depend on network/server load and polling configuration. +sync-bench: $(TARGET) $(DIST_DIR)/sync_bench$(EXE) + @if [ -f .env ]; then \ + export $$(grep -v '^#' .env | xargs); \ + fi; \ + if [ -n "$(SYNC_BENCH_DATABASE_ID)" ]; then export SYNC_BENCH_DATABASE_ID="$(SYNC_BENCH_DATABASE_ID)"; fi; \ + if [ -n "$(SYNC_BENCH_CLOUDSYNC_ADDRESS)" ]; then export SYNC_BENCH_CLOUDSYNC_ADDRESS="$(SYNC_BENCH_CLOUDSYNC_ADDRESS)"; fi; \ + if [ -n "$(SYNC_BENCH_APIKEY)" ]; then export SYNC_BENCH_APIKEY="$(SYNC_BENCH_APIKEY)"; fi; \ + if [ -n "$(SYNC_BENCH_POLL_DELAY_MS)" ]; then export SYNC_BENCH_POLL_DELAY_MS="$(SYNC_BENCH_POLL_DELAY_MS)"; fi; \ + if [ -n "$(SYNC_BENCH_MAX_POLLS)" ]; then export SYNC_BENCH_MAX_POLLS="$(SYNC_BENCH_MAX_POLLS)"; fi; \ + if [ -n "$(SYNC_BENCH_RANDOM_BLOB_SIZE_BYTES)" ]; then export SYNC_BENCH_RANDOM_BLOB_SIZE_BYTES="$(SYNC_BENCH_RANDOM_BLOB_SIZE_BYTES)"; fi; \ + if [ -n "$(SYNC_BENCH_CLEANUP_OLDER_THAN_SECONDS)" ]; then export SYNC_BENCH_CLEANUP_OLDER_THAN_SECONDS="$(SYNC_BENCH_CLEANUP_OLDER_THAN_SECONDS)"; fi; \ + if [ -n "$(SYNC_BENCH_OUTPUT)" ]; then export SYNC_BENCH_OUTPUT="$(SYNC_BENCH_OUTPUT)"; fi; \ + ./$(DIST_DIR)/sync_bench$(EXE) + +sync-bench-debug: + $(MAKE) NETWORK_TRACE=1 sync-bench + OPENSSL_TARBALL = $(OPENSSL_DIR)/$(OPENSSL_VERSION).tar.gz $(OPENSSL_TARBALL): @@ -247,7 +370,7 @@ else unzip $(CURL_DIR)/src/curl.zip -d $(CURL_DIR)/src/. endif - cd $(CURL_SRC) && ./configure \ + cd $(CURL_SRC) && $(CURL_CONFIG_ENV) ./configure \ --without-libpsl \ --disable-alt-svc \ --disable-ares \ @@ -359,8 +482,8 @@ framework module CloudSync {\ } endef -LIB_NAMES = ios.dylib ios-sim.dylib macos.dylib -FMWK_NAMES = ios-arm64 ios-arm64_x86_64-simulator macos-arm64_x86_64 +LIB_NAMES = ios.dylib ios-sim.dylib mac-catalyst.dylib macos.dylib +FMWK_NAMES = ios-arm64 ios-arm64_x86_64-simulator ios-arm64_x86_64-maccatalyst macos-arm64_x86_64 $(DIST_DIR)/%.xcframework: $(LIB_NAMES) @$(foreach i,1 2 3,\ lib=$(word $(i),$(LIB_NAMES)); \ @@ -373,6 +496,21 @@ $(DIST_DIR)/%.xcframework: $(LIB_NAMES) mv $(DIST_DIR)/$$lib $(DIST_DIR)/$$fmwk/CloudSync.framework/CloudSync; \ install_name_tool -id "@rpath/CloudSync.framework/CloudSync" $(DIST_DIR)/$$fmwk/CloudSync.framework/CloudSync; \ ) + @lib=$(word 4,$(LIB_NAMES)); \ + fmwk=$(word 4,$(FMWK_NAMES)); \ + mkdir -p $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Headers; \ + mkdir -p $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Modules; \ + mkdir -p $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Resources; \ + cp src/cloudsync.h $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Headers/CloudSync.h; \ + printf "$(PLIST)" > $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Resources/Info.plist; \ + printf "$(MODULEMAP)" > $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Modules/module.modulemap; \ + mv $(DIST_DIR)/$$lib $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/CloudSync; \ + install_name_tool -id "@rpath/CloudSync.framework/CloudSync" $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/CloudSync; \ + ln -sf A $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/Current; \ + ln -sf Versions/Current/CloudSync $(DIST_DIR)/$$fmwk/CloudSync.framework/CloudSync; \ + ln -sf Versions/Current/Headers $(DIST_DIR)/$$fmwk/CloudSync.framework/Headers; \ + ln -sf Versions/Current/Modules $(DIST_DIR)/$$fmwk/CloudSync.framework/Modules; \ + ln -sf Versions/Current/Resources $(DIST_DIR)/$$fmwk/CloudSync.framework/Resources; xcodebuild -create-xcframework $(foreach fmwk,$(FMWK_NAMES),-framework $(DIST_DIR)/$(fmwk)/CloudSync.framework) -output $@ rm -rf $(foreach fmwk,$(FMWK_NAMES),$(DIST_DIR)/$(fmwk)) @@ -413,13 +551,21 @@ help: @echo " android (needs ARCH to be set to x86_64, arm64-v8a, or armeabi-v7a and ANDROID_NDK to be set)" @echo " ios (only on macOS - can be compiled with native network support)" @echo " ios-sim (only on macOS - can be compiled with native network support)" + @echo " mac-catalyst (only on macOS - builds a universal arm64 + x86_64 Catalyst dylib)" @echo "" @echo "Targets:" @echo " all - Build the extension (default)" @echo " clean - Remove built files" @echo " test [COVERAGE=true] - Test the extension with optional coverage output" + @echo " unittest - Run only unit tests (test/unit.c)" @echo " help - Display this help message" @echo " xcframework - Build the Apple XCFramework" @echo " aar - Build the Android AAR package" + @echo "" + @echo "PostgreSQL Targets:" + @echo " make postgres-help - Show PostgreSQL-specific targets" + +# Include PostgreSQL extension targets +include docker/Makefile.postgresql -.PHONY: all clean test extension help version xcframework aar +.PHONY: all clean test unittest e2e extension help version xcframework aar diff --git a/PERFORMANCE.md b/PERFORMANCE.md new file mode 100644 index 00000000..c865d13b --- /dev/null +++ b/PERFORMANCE.md @@ -0,0 +1,197 @@ +# Performance & Overhead + +This document describes the computational and storage overhead introduced by the CloudSync extension, and how sync execution time relates to database size. + +## TL;DR + +Sync execution time scales with **the number of changes since the last sync (D)**, not with total database size (N). If you sync frequently, D stays small regardless of how large the database grows. The per-operation overhead on writes is proportional to the number of columns in the affected row, not to the table size. This is fundamentally different from sync solutions that diff or scan the full dataset. + +## Breaking Down the Cost + +The overhead introduced by the extension can be decomposed into four independent concerns: + +### 1. Per-Operation Overhead (Write-Path Cost) + +Every INSERT, UPDATE, or DELETE on a synced table fires AFTER triggers that write CRDT metadata into a companion `
_cloudsync` table. This happens synchronously, inline with the original write. + +| Operation | Metadata Rows Written | Complexity | +|-----------|----------------------|------------| +| INSERT | 1 sentinel + 1 per non-PK column | O(C) | +| UPDATE | 1 per changed column (NEW != OLD) | O(C_changed) <= O(C) | +| DELETE | 1 sentinel + cleanup of existing metadata | O(C_existing) | + +Where **C** = number of non-PK columns in the table. + +**Key point:** This cost is **constant per row** and independent of the total number of rows in the table (N). Writing to a 100-row table costs the same as writing to a 10-million-row table. The metadata table uses a composite primary key `(pk, col_name)` with `WITHOUT ROWID` optimization (SQLite) or a standard B-tree primary key (PostgreSQL), so the index update cost is O(log M) where M is the metadata table size -- but this is the same cost as any indexed INSERT and is negligible in practice. + +### 2. Sync Operations (Push & Pull) + +These are the operations that create and apply sync payloads. They are synchronous in the extension and should typically be run by the application off the main thread. + +#### Push: Payload Generation + +``` +Cost: O(D) where D = number of column-level changes since last sync +``` + +The push operation queries `cloudsync_changes`, which dynamically reads from all synced `
_cloudsync` tables: +```sql +SELECT ... FROM cloudsync_changes WHERE db_version > +``` + +Each metadata table has an **index on `db_version`**, so payload generation scales primarily with the number of new changes, plus a small per-synced-table overhead to construct the `cloudsync_changes` query. It does not diff the full dataset. In SQLite, each changed column also performs a primary-key lookup in the base table to retrieve the current value. + +The legacy `cloudsync_payload_encode()` API builds one monolithic LZ4-compressed payload before transmission. For large deltas, `cloudsync_payload_chunks()` can be used instead: it streams a sequence of payload chunks bounded by the `payload_max_chunk_size` setting (default 5 MB, minimum 256 KB, maximum 32 MB). If a single encoded BLOB/TEXT value is larger than the chunk budget, the value is split into transparent v3 fragments and reassembled by `cloudsync_payload_apply()` on the receiver. + +For legacy `/check` callers that still need one monolithic payload, `cloudsync_payload_blob_checked()` performs an internal size estimate before encoding. Successful calls scan the selected change window twice (estimate, then encode), so they trade extra I/O for avoiding unsafe monolithic payload allocation when the estimate exceeds the configured limit. + +#### Pull: Payload Application + +``` +Cost: O(D) to decode + O(D_unique_pks) to merge into the database +``` + +Incoming changes are decoded and **batched by primary key**. All column changes for the same row are accumulated and flushed as a single UPDATE or INSERT statement. This batching reduces the number of actual database writes to one per affected row, regardless of how many columns changed. + +Conflict resolution (CRDT merge) is O(1) per column: it compares version numbers and, only if tied, falls back to value comparison and site-id tiebreaking. No global state or table scan is required. + +#### Summary + +| Phase | Scales With | Does NOT Scale With | +|-------|-------------|-------------------| +| Payload generation | D (changes since last sync) | N (total rows) | +| Payload application | D (incoming changes) | N (total rows) | +| Conflict resolution | D (conflicting columns) | N (total rows) | + +**This means sync time is driven mainly by delta size (`D`) rather than total database size (`N`)**. As long as the number of changes between syncs stays bounded, sync time remains roughly stable even as the database grows. + +### 3. Sync Frequency & Network Latency + +When the application runs sync off the main thread, perceived latency depends on: + +- **Sync interval**: How often the app triggers a push/pull cycle. More frequent syncs mean smaller deltas (smaller D) and faster individual sync operations, at the cost of more network round-trips. +- **Network latency**: The round-trip time to the sync server. LZ4 compression reduces payload size, but latency is dominated by the network hop itself for small deltas. +- **Payload size**: Proportional to D x average column value size. Large BLOBs or TEXT values will increase transfer time linearly. Use `cloudsync_payload_chunks()` when transport payloads may be large; it limits each generated transport payload but does not change the size of the final database value. + +The extension does not impose a sync schedule -- the application controls when and how often to sync. A typical pattern is to sync on a timer (e.g., every 5-30 seconds) or on specific events (app foreground, user action). + +### 4. Metadata Storage Overhead + +Each synced table has a companion `
_cloudsync` metadata table with the following schema: + +``` +PRIMARY KEY (pk, col_name) -- WITHOUT ROWID (SQLite) +Columns: pk, col_name, col_version, db_version, site_id, seq +Index: db_version +``` + +**Storage cost per row in the base table:** +- 1 sentinel row (marks the row's existence/deletion state) +- 1 metadata row per non-PK column that has ever been written + +So for a table with C non-PK columns, the metadata table will contain approximately `N x (1 + C)` rows, where N is the number of rows in the base table. + +**Estimated overhead per metadata row:** +- `pk`: encoded primary key (typically 8-32 bytes depending on PK type and count) +- `col_name`: column name string (shared via SQLite's string interning, typically 5-30 bytes) +- `col_version`, `db_version`, `seq`: 3 integers (8 bytes each = 24 bytes) +- `site_id`: 1 integer (8 bytes) + +Rough estimate: **60-100 bytes per metadata row**, or **60-100 x (1 + C) bytes per base table row**. + +| Base Table | Columns (C) | Rows (N) | Estimated Metadata Size | +|------------|-------------|----------|------------------------| +| Small | 5 | 1,000 | ~360 KB - 600 KB | +| Medium | 10 | 100,000 | ~66 MB - 110 MB | +| Large | 10 | 1,000,000| ~660 MB - 1.1 GB | +| Wide | 50 | 100,000 | ~306 MB - 510 MB | + +**Mitigation strategies:** +- Only sync tables that need it -- not every table requires CRDT tracking. +- Prefer narrow tables (fewer columns) for high-volume data. +- The `WITHOUT ROWID` optimization (SQLite) significantly reduces per-row storage overhead. +- Deleted rows have their per-column metadata cleaned up, but a tombstone sentinel row persists (see section 9 below). + +### 5. Read-Path Overhead + +Normal application reads are not directly instrumented by the extension. No triggers, views, or hooks intercept ordinary SELECT queries on application tables, and the CRDT metadata is stored separately. In practice, read overhead is usually negligible. + +### 6. Initial Sync (First Device) + +When a new device syncs for the first time (`db_version = 0`), the push payload contains the **entire dataset**: every column of every row across all synced tables. The payload size is proportional to `N * C` (total rows times columns). + +With the legacy `cloudsync_payload_encode()` API, the payload is built entirely in memory, starting with a 512 KB buffer (`CLOUDSYNC_PAYLOAD_MINBUF_SIZE` in `src/cloudsync.c`) and growing via `realloc` as needed. Peak memory usage is at least the full uncompressed payload size and can be higher during compression. For a database with 1 million rows and 10 columns of average 50 bytes each, the uncompressed payload could reach ~500 MB before LZ4 compression. + +For large initial syncs, prefer `cloudsync_payload_chunks()`. It keeps each generated transport payload bounded by `payload_max_chunk_size` and can fragment a single oversized BLOB/TEXT column across multiple v3 fragment payloads. This prevents the transport payload itself from growing without bound and avoids constructing a monolithic v2 payload during v3 apply. + +Important limitation: chunking does **not** make a single database cell streamable all the way into the storage engine. When the last fragment of a very large BLOB/TEXT value arrives, the receiver must still materialize the completed value once in order to bind/store it in the destination database. Size `payload_max_chunk_size` for transport safety, but size application memory limits for the largest individual value you allow. + +Subsequent syncs are incremental (proportional to D, changes since the last sync), so the first sync is the expensive one. Applications with large datasets should plan for this -- for example, by seeding new devices from a database snapshot rather than syncing from scratch. + +### 7. WAL and Disk I/O Amplification + +Each write to a synced table generates additional metadata writes via AFTER triggers. The amplification factor depends on the operation: + +| Operation | Total Writes (base + metadata) | Amplification Factor | +|-----------|-------------------------------|---------------------| +| INSERT (C columns) | 1 + 1 sentinel + C metadata | ~C+2x | +| UPDATE (1 column) | 1 + 1 metadata | 2x | +| UPDATE (C columns) | 1 + C metadata | ~C+1x | +| DELETE | 1 + cleanup writes | variable | + +For a table with 10 non-PK columns, an INSERT generates roughly 12 logical row writes instead of 1. This increases WAL/page churn and affects: + +- **Disk I/O**: More pages written per transaction, larger WAL files between checkpoints. +- **WAL checkpoint frequency**: The WAL grows faster, so checkpoints run more often (or the WAL file stays larger if checkpointing is deferred). +- **Battery on mobile**: More disk writes per user action. Batching multiple writes in a single transaction amortizes the transaction overhead but not the per-row metadata cost. + +### 8. Locking During Sync Apply + +Payload application (`cloudsync_payload_apply`) uses savepoints grouped by source `db_version`. On SQLite, each savepoint holds a write lock for its duration. If the application runs sync on the main thread, other work on the same connection is blocked, and reads from other connections may block outside WAL mode. + +On SQLite, using WAL mode prevents readers on other connections from being blocked by writers, which is the recommended configuration for concurrent sync. + +### 9. Metadata Lifecycle (Tombstones and Cleanup) + +When a row is deleted, the per-column metadata rows are removed, but a **tombstone sentinel** (`__[RIP]__`) persists in the metadata table. This tombstone is necessary for propagating deletes to other devices during sync. There is no automatic garbage collection of tombstones -- they accumulate over time. + +Metadata cleanup for **removed columns** (after schema migration) only runs during `cloudsync_finalize_alter()`, which is called as part of the `cloudsync_alter()` workflow. Outside of schema changes, orphaned metadata from dropped columns remains in the metadata table. + +The **site ID table** (`cloudsync_site_id`) also grows monotonically -- one entry per unique device that has ever synced. This is typically small (one row per device) and not a concern in practice. + +For applications with high delete rates, the tombstone accumulation may become significant over time. Consider periodic full re-syncs or application-level archival strategies if this is a concern. + +### 10. Multi-Table Considerations + +The `cloudsync_changes` virtual table (SQLite) or set-returning function (PostgreSQL) dynamically constructs a `UNION ALL` query across all synced tables' metadata tables. The query construction cost scales as O(T) where T is the number of synced tables. + +For most applications (fewer than ~50 synced tables), this is negligible. Applications syncing a very large number of tables should be aware that payload generation involves iterating over all synced tables to check for changes. + +### Platform Differences (SQLite vs PostgreSQL) + +- **SQLite** uses native C triggers registered directly with the SQLite API. Metadata tables use `WITHOUT ROWID` for compact storage. +- **PostgreSQL** uses row-level PL/pgSQL trigger functions that call into C functions via the extension. This adds a small amount of overhead per trigger invocation compared to SQLite's direct C triggers. Additionally, merge operations use per-PK savepoints to handle failures such as RLS policy violations gracefully. +- **Table registration** (`cloudsync_enable()`) is a one-time operation on both platforms. It creates 1 metadata table, 1 index, and 3 triggers (INSERT, UPDATE, DELETE), plus ~15-20 prepared statements that are cached for the lifetime of the connection. + +## Comparison with Full-Scan Sync Solutions + +Many sync solutions must diff or hash the entire dataset to determine what changed. This leads to O(N) sync time that grows linearly with total database size -- the exact problem described in the question. + +CloudSync avoids this through its **monotonic versioning approach**: every write increments a monotonic `db_version` counter, and the sync query filters on this counter using an index. The result is that sync time depends mainly on the volume of changes (D), not on the total data size (N). + +``` +Full-scan sync: sync_time ~ O(N) -- grows with database size +CloudSync: sync_time ~ O(D) -- grows with changes since last sync + where D is independent of N when sync frequency is constant +``` + +## Performance Optimizations in the Implementation + +1. **`WITHOUT ROWID` tables** (SQLite): Metadata tables use clustered primary keys, avoiding the overhead of a separate rowid B-tree. +2. **`db_version` index**: Enables efficient range scans for delta extraction. +3. **Deferred batch merge**: Column changes for the same primary key are accumulated and flushed as a single SQL statement. +4. **Prepared statement caching**: Merge statements are compiled once and reused across rows. +5. **Chunked payload generation**: `cloudsync_payload_chunks()` bounds transport payload size and handles oversized single values with transparent v3 fragments. +6. **LZ4 compression**: Reduces payload size for network transfer. +7. **Per-column tracking**: Only changed columns are included in the sync payload, not entire rows. +8. **Early exit on stale data**: The CLS algorithm skips rows where the incoming causal length is lower than the local one, avoiding unnecessary column-level comparisons. diff --git a/Package.swift b/Package.swift index 9e0db59f..0fd83186 100644 --- a/Package.swift +++ b/Package.swift @@ -5,29 +5,22 @@ import PackageDescription let package = Package( name: "CloudSync", - platforms: [.macOS(.v11), .iOS(.v11)], + platforms: [.macOS(.v11), .iOS(.v11), .macCatalyst(.v14)], products: [ - // Products can be used to vend plugins, making them visible to other packages. - .plugin( - name: "CloudSyncPlugin", - targets: ["CloudSyncPlugin"]), .library( name: "CloudSync", targets: ["CloudSync"]) ], targets: [ - // Build tool plugin that invokes the Makefile - .plugin( - name: "CloudSyncPlugin", - capability: .buildTool(), - path: "packages/swift/plugin" + .binaryTarget( + name: "CloudSyncBinary", + url: "https://github.com/sqliteai/sqlite-sync/releases/download/1.1.2/cloudsync-apple-xcframework-1.1.2.zip", + checksum: "60f9d5a5e08f3327773991d605c032a33d8d32e72b93cf3eef4244706e8c57da" ), - // CloudSync library target .target( name: "CloudSync", - dependencies: [], - path: "packages/swift/extension", - plugins: ["CloudSyncPlugin"] + dependencies: ["CloudSyncBinary"], + path: "packages/swift" ), ] -) \ No newline at end of file +) diff --git a/README.md b/README.md index c2dc3807..a2e02d39 100644 --- a/README.md +++ b/README.md @@ -1,496 +1,285 @@ -# SQLite Sync - -[![sqlite-sync coverage](https://img.shields.io/badge/dynamic/regex?url=https%3A%2F%2Fsqliteai.github.io%2Fsqlite-sync%2F&search=%3Ctd%20class%3D%22headerItem%22%3EFunctions%3A%3C%5C%2Ftd%3E%5Cs*%3Ctd%20class%3D%22headerCovTableEntryHi%22%3E(%5B%5Cd.%5D%2B)%26nbsp%3B%25%3C%5C%2Ftd%3E&replace=%241%25&label=coverage&labelColor=rgb(85%2C%2085%2C%2085)%3B&color=rgb(167%2C%20252%2C%20157)%3B&link=https%3A%2F%2Fsqliteai.github.io%2Fsqlite-sync%2F)](https://sqliteai.github.io/sqlite-sync/) - -**SQLite Sync** is a multi-platform extension that brings a true **local-first experience** to your applications with minimal effort. It extends standard SQLite tables with built-in support for offline work and automatic synchronization, allowing multiple devices to operate independently—even without a network connection—and seamlessly stay in sync. With SQLite Sync, developers can easily build **distributed, collaborative applications** while continuing to rely on the **simplicity, reliability, and performance of SQLite**. - -Under the hood, SQLite Sync uses advanced **CRDT (Conflict-free Replicated Data Type)** algorithms and data structures designed specifically for **collaborative, distributed systems**. This means: - -- Devices can update data independently, even without a network connection. -- When they reconnect, all changes are **merged automatically and without conflicts**. -- **No data loss. No overwrites. No manual conflict resolution.** - -In simple terms, CRDTs make it possible for multiple users to **edit shared data at the same time**, from anywhere, and everything just works. - -## Table of Contents -- [Key Features](#key-features) -- [Built-in Network Layer](#built-in-network-layer) -- [Row-Level Security](#row-level-security) -- [What Can You Build with SQLite Sync?](#what-can-you-build-with-sqlite-sync) -- [Documentation](#documentation) -- [Installation](#installation) -- [Getting Started](#getting-started) -- [Database Schema Recommendations](#database-schema-recommendations) - - [Primary Key Requirements](#primary-key-requirements) - - [Column Constraint Guidelines](#column-constraint-guidelines) - - [UNIQUE Constraint Considerations](#unique-constraint-considerations) - - [Foreign Key Compatibility](#foreign-key-compatibility) - - [Trigger Compatibility](#trigger-compatibility) -- [License](#license) - -## Key Features - -- **Offline-First by Design**: Works seamlessly even when devices are offline. Changes are queued locally and synced automatically when connectivity is restored. -- **CRDT-Based Conflict Resolution**: Merges updates deterministically and efficiently, ensuring eventual consistency across all replicas without the need for complex merge logic. -- **Embedded Network Layer**: No external libraries or sync servers required. SQLiteSync handles connection setup, message encoding, retries, and state reconciliation internally. -- **Drop-in Simplicity**: Just load the extension into SQLite and start syncing. No need to implement custom protocols or state machines. -- **Efficient and Resilient**: Optimized binary encoding, automatic batching, and robust retry logic make synchronization fast and reliable even on flaky networks. - -Whether you're building a mobile app, IoT device, or desktop tool, SQLite Sync simplifies distributed data management and unlocks the full potential of SQLite in decentralized environments. - -## Built-in Network Layer - -Unlike traditional sync systems that require you to build and maintain a complex backend, **SQLite Sync includes a built-in network layer** that works out of the box: - -- Sync your database with the cloud using **a single function call**. -- Compatible with **any language or framework** that supports SQLite. -- **No backend setup required** — SQLite Sync handles networking, change tracking, and conflict resolution for you. - -The sync layer is tightly integrated with [**SQLite Cloud**](https://sqlitecloud.io/), enabling seamless and secure data sharing across devices, users, and platforms. You get the power of cloud sync without the complexity. +
+ + SQLite AI + + +

SQLite-Sync

+

Offline-first sync for SQLite, powered by CRDTs.
+ Local writes, conflict-free merges, real-time collaboration across devices. Sync to SQLite Cloud, PostgreSQL, or Supabase — no central coordinator required.

+ +

+ Free managed instance → · + Docs · + Live Demo · + Website +

+ +

+ Data: + Vector · + Sync · + Columnar · + JS +
+ AI: + AI · + Agent · + Memory · + MCP + +

+
+ +
+ +> **Need a sync backend?** Plug into any [PostgreSQL](./docs/postgresql/quickstarts/postgres.md) or [self-hosted Supabase](./docs/postgresql/quickstarts/supabase-self-hosted.md) instance, or use **[SQLite Cloud CloudSync](https://www.sqlite.ai/pricing)** -## Row-Level Security - -Thanks to the underlying SQLite Cloud infrastructure, **SQLite Sync supports Row-Level Security (RLS)**—allowing you to define **precise access control at the row level**: - -- Control not just who can read or write a table, but **which specific rows** they can access. -- Enforce security policies on the server—no need for client-side filtering. - -For example: - -- User A can only see and edit their own data. -- User B can access a different set of rows—even within the same shared table. - -**Benefits of RLS**: +--- -- **Data isolation**: Ensure users only access what they’re authorized to see. -- **Built-in privacy**: Security policies are enforced at the database level. -- **Simplified development**: Reduce or eliminate complex permission logic in your application code. +# SQLite Sync -### What Can You Build with SQLite Sync? +[![sqlite-sync coverage](https://img.shields.io/badge/dynamic/regex?url=https%3A%2F%2Fsqliteai.github.io%2Fsqlite-sync%2F&search=Functions%3A%3C%5C%2Ftd%3E%5Cs*%3Ctd%20class%3D%22headerCovTableEntry(?:Hi|Med|Lo)%22%3E(%5B%5Cd.%5D%2B)%26nbsp%3B%25&replace=%241%25&label=coverage&labelColor=rgb(85%2C%2085%2C%2085)%3B&color=rgb(167%2C%20252%2C%20157)%3B&link=https%3A%2F%2Fsqliteai.github.io%2Fsqlite-sync%2F)](https://sqliteai.github.io/sqlite-sync/) -SQLite Sync is ideal for building collaborative and distributed apps across web, mobile, desktop, and edge platforms. Some example use cases include: +**SQLite Sync** is a multi-platform extension that turns any SQLite database into a **conflict-free, offline-first replica** that syncs automatically with **[SQLite Cloud](https://sqlitecloud.io/)** nodes, **[PostgreSQL](./docs/postgresql/quickstarts/postgres.md)** servers, and **[Supabase](./docs/postgresql/quickstarts/supabase-self-hosted.md)** instances. One function call is all it takes: no backend to build, no sync protocol to implement. -#### 📋 Productivity & Collaboration +Built on **CRDT** (Conflict-free Replicated Data Types), it guarantees: -- **Shared To-Do Lists**: Users independently update tasks and sync effortlessly. -- **Note-Taking Apps**: Real-time collaboration with offline editing. -- **Markdown Editors**: Work offline, sync when back online—no conflicts. +- **No data loss.** Devices update independently, even offline, and all changes merge automatically. +- **No conflicts.** Deterministic merge, no manual conflict resolution, ever. +- **No extra infrastructure.** A globally distributed network of **CloudSync microservices** handles routing, packaging, and delivery of changes between SQLite and other DBMS nodes. -#### 📱 Mobile & Edge +## Why SQLite Sync? -- **Field Data Collection**: For remote inspections, agriculture, or surveys. -- **Point-of-Sale Systems**: Offline-first retail solutions with synced inventory. -- **Health & Fitness Apps**: Sync data across devices with strong privacy controls. +**For offline-first apps** (mobile, desktop, IoT, edge): devices work with a local SQLite database and sync when connectivity is available. Changes queue locally and merge seamlessly on reconnect. -#### 🏢 Enterprise Workflows +**For AI agents**: agents that maintain memory, notes, or shared state in SQLite can sync across instances without coordination. **[Block-Level LWW](#block-level-lww)** was specifically designed to keep **markdown files** in sync: multiple agents editing different sections of the same document preserve all changes after sync. -- **CRM Systems**: Sync leads and clients per user with row-level access control. -- **Project Management Tools**: Offline-friendly planning and task management. -- **Expense Trackers**: Sync team expenses securely and automatically. +## What Can You Build with SQLite Sync? -#### 🧠 Personal Apps +### Offline-First Apps +- **Shared To-Do Lists**: users independently update tasks and sync effortlessly. +- **Note-Taking Apps**: real-time collaboration with offline editing. +- **Field Data Collection**: for remote inspections, agriculture, or surveys. +- **Point-of-Sale Systems**: offline-first retail solutions with synced inventory. -- **Journaling & Diaries**: Private, encrypted entries that sync across devices. -- **Bookmarks & Reading Lists**: Personal or collaborative content management. -- **Habit Trackers**: Sync progress with data security and consistency. +### AI Agent Sync +- **Agent Memory**: multiple agents share and update a common SQLite database, syncing state across instances without coordination. +- **Markdown Knowledge Bases**: agents independently edit different sections of shared markdown documents, with Block-Level LWW preserving all changes. +- **Distributed Pipelines**: agents running on different nodes accumulate results locally and merge them into a single consistent dataset. -#### 🌍 Multi-User, Multi-Tenant Systems +### Enterprise and Multi-Tenant +- **CRM Systems**: sync leads and clients per user with row-level access control. +- **SaaS Platforms**: row-level access for each user or team using a single shared database. +- **Project Management Tools**: offline-friendly planning and task management. -- **SaaS Platforms**: Row-level access for each user or team. -- **Collaborative Design Tools**: Merge visual edits and annotations offline. -- **Educational Apps**: Shared learning content with per-student access controls. +### Personal Apps +- **Journaling and Diaries**: private entries that sync across devices. +- **Habit Trackers**: sync progress with data security and consistency. +- **Bookmarks and Reading Lists**: personal or collaborative content management. -## Documentation +## Key Features -For detailed information on all available functions, their parameters, and examples, refer to the [comprehensive API Reference](./API.md). +| Feature | Description | +|---------|-------------| +| **CRDT-based sync** | Causal-Length Set, Delete-Wins, Add-Wins, and Grow-Only Set algorithms | +| **Block-Level LWW** | Line-level merge for text/markdown columns, concurrent edits to different lines are preserved | +| **Built-in networking** | Embedded network layer (libcurl or native), single function call to sync | +| **Row-Level Security** | Server-enforced RLS: each client syncs only the rows it is authorized to see | +| **Multi-platform** | Linux, macOS, Windows, iOS, Android, WASM | -## Installation +## Quick Start -### Pre-built Binaries +### 1. Install -Download the appropriate pre-built binary for your platform from the official [Releases](https://github.com/sqliteai/sqlite-sync/releases) page: +Download a pre-built binary from the [Releases](https://github.com/sqliteai/sqlite-sync/releases) page, or install a platform package (see [full installation guide](./docs/installation.md) for platform-specific code examples): -- Linux: x86 and ARM -- macOS: x86 and ARM -- Windows: x86 -- Android -- iOS +| Platform | Install | +|----------|---------| +| **SQLite CLI / C** | `.load ./cloudsync` or `SELECT load_extension('./cloudsync');` | +| **Swift** | [Add this repo as a Swift Package dependency](https://developer.apple.com/documentation/xcode/adding-package-dependencies-to-your-app), follow [steps 4 and 5](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/platforms/ios.md#4-set-up-sqlite-with-extension-loading), and load extension with `CloudSync.path` | +| **Android** | `implementation 'ai.sqlite:sync:1.0.0'` ([Maven Central](https://central.sonatype.com/artifact/ai.sqlite/sync)) | +| **Flutter** | `flutter pub add sqlite_sync` ([pub.dev](https://pub.dev/packages/sqlite_sync)) | +| **Expo** | `npm install @sqliteai/sqlite-sync-expo` | +| **React Native** | `npm install @sqliteai/sqlite-sync-react-native` | +| **WASM** | `npm install @sqliteai/sqlite-wasm` ([npm](https://www.npmjs.com/package/@sqliteai/sqlite-wasm)) | -### Loading the Extension +### 2. Create a table and enable sync ```sql --- In SQLite CLI .load ./cloudsync --- In SQL -SELECT load_extension('./cloudsync'); -``` - -### WASM Version - -You can download the WebAssembly (WASM) version of SQLite with the SQLite Sync extension enabled from: https://www.npmjs.com/package/@sqliteai/sqlite-wasm - -### Swift Package - -You can [add this repository as a package dependency to your Swift project](https://developer.apple.com/documentation/xcode/adding-package-dependencies-to-your-app#Add-a-package-dependency). After adding the package, you'll need to set up SQLite with extension loading by following steps 4 and 5 of [this guide](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/platforms/ios.md#4-set-up-sqlite-with-extension-loading). - -Here's an example of how to use the package: -```swift -import CloudSync - -... - -var db: OpaquePointer? -sqlite3_open(":memory:", &db) -sqlite3_enable_load_extension(db, 1) -var errMsg: UnsafeMutablePointer? = nil -sqlite3_load_extension(db, CloudSync.path, nil, &errMsg) -var stmt: OpaquePointer? -sqlite3_prepare_v2(db, "SELECT cloudsync_version()", -1, &stmt, nil) -defer { sqlite3_finalize(stmt) } -sqlite3_step(stmt) -log("cloudsync_version(): \(String(cString: sqlite3_column_text(stmt, 0)))") -sqlite3_close(db) -``` - -### Android Package - -Add the [following](https://central.sonatype.com/artifact/ai.sqlite/sync) to your Gradle dependencies: - -```gradle -implementation 'ai.sqlite:sync:0.8.41' -``` - -Here's an example of how to use the package: -```java -SQLiteCustomExtension cloudsyncExtension = new SQLiteCustomExtension(getApplicationInfo().nativeLibraryDir + "/cloudsync", null); -SQLiteDatabaseConfiguration config = new SQLiteDatabaseConfiguration( - getCacheDir().getPath() + "/cloudsync_test.db", - SQLiteDatabase.CREATE_IF_NECESSARY | SQLiteDatabase.OPEN_READWRITE, - Collections.emptyList(), - Collections.emptyList(), - Collections.singletonList(cloudsyncExtension) +CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + done INTEGER NOT NULL DEFAULT 0 ); -SQLiteDatabase db = SQLiteDatabase.openDatabase(config, null, null); -``` -**Note:** Additional settings and configuration are required for a complete setup. For full implementation details, see the [complete Android example](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/examples/android/README.md). - -### Expo - -Install the Expo package: - -```bash -npm install @sqliteai/sqlite-sync-expo +-- Enable CRDT sync on the table +SELECT cloudsync_init('tasks'); ``` -Add to your `app.json`: +### 3. Use your database normally -```json -{ - "expo": { - "plugins": ["@sqliteai/sqlite-sync-expo"] - } -} -``` +```sql +INSERT INTO tasks (id, title) VALUES (cloudsync_uuid(), 'Buy groceries'); +INSERT INTO tasks (id, title) VALUES (cloudsync_uuid(), 'Review PR #42'); -Run prebuild: +UPDATE tasks SET done = 1 WHERE title = 'Buy groceries'; -```bash -npx expo prebuild --clean +SELECT * FROM tasks; ``` -Load the extension: - -```typescript -import { Platform } from 'react-native'; -import { getDylibPath, open } from '@op-engineering/op-sqlite'; - -const db = open({ name: 'mydb.db' }); +### 4. Sync with the cloud -// Load SQLite Sync extension -if (Platform.OS === 'ios') { - const path = getDylibPath('ai.sqlite.cloudsync', 'CloudSync'); - db.loadExtension(path); -} else { - db.loadExtension('cloudsync'); -} -``` - -### Flutter Package +The example below uses SQLite Cloud CloudSync. If you are wiring up a self-hosted backend instead, use the [PostgreSQL quick start](./docs/postgresql/quickstarts/postgres.md) or the [self-hosted Supabase quick start](./docs/postgresql/quickstarts/supabase-self-hosted.md). -Add the [sqlite_sync](https://pub.dev/packages/sqlite_sync) package to your project: +```sql +-- Connect to your SQLite Cloud managed database +-- (get the managed database ID from the OffSync page on the SQLite Cloud dashboard) +SELECT cloudsync_network_init('your-managed-database-id'); +SELECT cloudsync_network_set_apikey('your-api-key'); -```bash -flutter pub add sqlite_sync # Flutter projects -dart pub add sqlite_sync # Dart projects -``` +-- Send local changes and receive remote changes +SELECT cloudsync_network_sync(); +-- Returns JSON: {"send":{"status":"synced","localVersion":3,"serverVersion":3},"receive":{"rows":0,"tables":[]}} -Usage with `sqlite3` package: -```dart -import 'package:sqlite3/sqlite3.dart'; -import 'package:sqlite_sync/sqlite_sync.dart'; +-- Call periodically to stay in sync +SELECT cloudsync_network_sync(); -sqlite3.loadSqliteSyncExtension(); -final db = sqlite3.openInMemory(); -print(db.select('SELECT cloudsync_version()')); +-- Before closing the connection +SELECT cloudsync_terminate(); ``` -For a complete example, see the [Flutter example](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/examples/flutter/README.md). +### 5. Sync from another device -## Getting Started - -Here's a quick example to get started with SQLite Sync: - -### Prerequisites - -1. **SQLite Cloud Account**: Sign up at [SQLite Cloud](https://sqlitecloud.io/) -2. **SQLite Sync Extension**: Download from [Releases](https://github.com/sqliteai/sqlite-sync/releases) - -### SQLite Cloud Setup - -1. Create a new project and database in your [SQLite Cloud Dashboard](https://dashboard.sqlitecloud.io/) -2. Copy your connection string and API key from the dashboard -3. Create tables with identical schema in both local and cloud databases -4. Enable synchronization: click the **"OffSync"** button for your database and select each table you want to synchronize - -### Local Database Setup - -```bash -# Start SQLite CLI -sqlite3 myapp.db -``` +On a second device (or a second database for testing), repeat the same setup: ```sql --- Load the extension +-- Device B: load extension, create the same table, init sync .load ./cloudsync --- Create a table (primary key MUST be TEXT for global uniqueness) -CREATE TABLE IF NOT EXISTS my_data ( - id TEXT PRIMARY KEY NOT NULL, - value TEXT NOT NULL DEFAULT '', - created_at TEXT DEFAULT CURRENT_TIMESTAMP +CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + done INTEGER NOT NULL DEFAULT 0 ); --- Initialize table for synchronization -SELECT cloudsync_init('my_data'); - --- Use your local database normally: read and write data using standard SQL queries --- The CRDT system automatically tracks all changes for synchronization - --- Example: Insert data (always use cloudsync_uuid() for globally unique IDs) -INSERT INTO my_data (id, value) VALUES - (cloudsync_uuid(), 'Hello from device A!'), - (cloudsync_uuid(), 'Working offline is seamless!'); - --- Example: Update and delete operations work normally -UPDATE my_data SET value = 'Updated: Hello from device A!' WHERE value LIKE 'Hello from device A!'; - --- View your data -SELECT * FROM my_data ORDER BY created_at; - --- Configure network connection before using the network sync functions -SELECT cloudsync_network_init('sqlitecloud://your-project-id.sqlite.cloud/database.sqlite'); -SELECT cloudsync_network_set_apikey('your-api-key-here'); --- Or use token authentication (required for Row-Level Security) --- SELECT cloudsync_network_set_token('your_auth_token'); - --- Sync with cloud: send local changes, then check the remote server for new changes --- and, if a package with changes is ready to be downloaded, applies them to the local database -SELECT cloudsync_network_sync(); --- Keep calling periodically. The function returns > 0 if data was received --- In production applications, you would typically call this periodically --- rather than manually (e.g., every few seconds) -SELECT cloudsync_network_sync(); - --- Before closing the database connection -SELECT cloudsync_terminate(); --- Close the database connection -.quit -``` -```sql --- On another device (or create another database for testing: sqlite3 myapp_2.db) --- Follow the same setup steps: load extension, create table, init sync, configure network - --- Load extension and create identical table structure -.load ./cloudsync -CREATE TABLE IF NOT EXISTS my_data ( - id TEXT PRIMARY KEY NOT NULL, - value TEXT NOT NULL DEFAULT '', - created_at TEXT DEFAULT CURRENT_TIMESTAMP -); -SELECT cloudsync_init('my_data'); +SELECT cloudsync_init('tasks'); -- Connect to the same cloud database -SELECT cloudsync_network_init('sqlitecloud://your-project-id.sqlite.cloud/database.sqlite'); -SELECT cloudsync_network_set_apikey('your-api-key-here'); +SELECT cloudsync_network_init('your-managed-database-id'); +SELECT cloudsync_network_set_apikey('your-api-key'); --- Sync to get data from the first device +-- Pull changes from Device A SELECT cloudsync_network_sync(); --- repeat until data is received (returns > 0) +-- Call again: the first call triggers package preparation, the second downloads it SELECT cloudsync_network_sync(); --- View synchronized data -SELECT * FROM my_data ORDER BY created_at; +-- Device A's tasks are now here +SELECT * FROM tasks; --- Add data from this device to test bidirectional sync -INSERT INTO my_data (id, value) VALUES - (cloudsync_uuid(), 'Hello from device B!'); +-- Add data from this device +INSERT INTO tasks (id, title) VALUES (cloudsync_uuid(), 'Call the dentist'); --- Sync again to send this device's changes +-- Send this device's changes to the cloud SELECT cloudsync_network_sync(); --- The CRDT system ensures all devices eventually have the same data, --- with automatic conflict resolution and no data loss - --- Before closing the database connection +-- Before closing the connection SELECT cloudsync_terminate(); --- Close the database connection -.quit ``` -### For a Complete Example +Back on Device A, calling `cloudsync_network_sync()` will pull Device B's changes. The CRDT engine ensures all devices converge to the same data, automatically, with no conflicts. -See the [examples](./examples/simple-todo-db/) directory for a comprehensive walkthrough including: -- Multi-device collaboration -- Offline scenarios -- Row-level security setup -- Conflict resolution demonstrations +> **Note:** every device participating in the same sync must create **the same set of tables with the same structure** and initialize each one with `cloudsync_init()`. sqlite-sync derives a schema hash from the synced tables, and the server rejects payloads whose hash it does not recognize. For multi-tenant setups where each client should see only a subset of rows, use a shared schema with a tenant/scope column and enforce isolation with [RLS Overview](./docs/rls-overview.md) — do not give each client a different table. -## 📦 Integrations +## Block-Level LWW -Use SQLite-AI alongside: - -* **[SQLite-AI](https://github.com/sqliteai/sqlite-ai)** – on-device inference, embedding generation, and model interaction directly into your database -* **[SQLite-Vector](https://github.com/sqliteai/sqlite-vector)** – vector search from SQL -* **[SQLite-JS](https://github.com/sqliteai/sqlite-js)** – define SQLite functions in JavaScript - -## Database Schema Recommendations - -When designing your database schema for SQLite Sync, follow these best practices to ensure optimal CRDT performance and conflict resolution: - -### Primary Key Requirements - -- **Use globally unique identifiers**: Always use TEXT primary keys with UUIDs, ULIDs, or similar globally unique identifiers -- **Avoid auto-incrementing integers**: Integer primary keys can cause conflicts across multiple devices -- **Use `cloudsync_uuid()`**: The built-in function generates UUIDv7 identifiers optimized for distributed systems -- **All primary keys must be explicitly declared as `NOT NULL`**. +Standard CRDT sync replaces an entire cell when two devices edit the same column. **Block-Level LWW** splits text into lines and merges them independently, designed for keeping **markdown files and agent memory** in sync. ```sql --- ✅ Recommended: Globally unique TEXT primary key -CREATE TABLE users ( - id TEXT PRIMARY KEY NOT NULL, -- Use cloudsync_uuid() - name TEXT NOT NULL, - email TEXT UNIQUE NOT NULL +CREATE TABLE notes ( + id TEXT PRIMARY KEY NOT NULL, + title TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '' ); --- ❌ Avoid: Auto-incrementing integer primary key -CREATE TABLE users ( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- Causes conflicts - name TEXT NOT NULL, - email TEXT UNIQUE NOT NULL -); +SELECT cloudsync_init('notes'); +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block'); ``` -### Column Constraint Guidelines +Now two agents (or devices) can edit different lines of the same note, and both edits are preserved after sync. See the full guide: **[Block-Level LWW Documentation](./docs/block-lww.md)**. -- **Provide DEFAULT values**: All `NOT NULL` columns (except primary keys) must have `DEFAULT` values -- **Consider nullable columns**: For optional data, use nullable columns instead of empty strings +## Row-Level Security -```sql --- ✅ Recommended: Proper constraints and defaults -CREATE TABLE tasks ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL DEFAULT '', - status TEXT NOT NULL DEFAULT 'pending', - priority INTEGER NOT NULL DEFAULT 1, - created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), - assigned_to TEXT -- Nullable for optional assignment -); -``` +With SQLite Cloud's RLS, a single shared cloud database serves all users while each client only sees and syncs its own rows. Policies are enforced server-side: a compromised client cannot bypass access controls. -### UNIQUE Constraint Considerations +- One database, multiple tenants, no per-user database provisioning. +- Each client syncs only authorized rows, minimal bandwidth and storage. -When converting from single-tenant to multi-tenant database schemas with Row-Level Security, **UNIQUE constraints must be globally unique** across all tenants in the cloud database. For columns that should only be unique within a tenant, use composite UNIQUE constraints. +See the full guide: **[RLS Overview](./docs/rls-overview.md)**. -```sql --- ❌ Single-tenant: Unique email per database -CREATE TABLE users ( - id TEXT PRIMARY KEY, - email TEXT UNIQUE NOT NULL -- Problem: Not unique across tenants -); +## Documentation --- ✅ Multi-tenant: Composite unique constraint -CREATE TABLE users ( - id TEXT PRIMARY KEY, - tenant_id TEXT NOT NULL, - email TEXT NOT NULL, - UNIQUE(tenant_id, email) -- Unique email per tenant -); -``` +- **[API Reference](./API.md)**: all functions, parameters, and examples +- **[Performance & Overhead](./PERFORMANCE.md)**: sync cost model, payload chunking, and large-value memory notes +- **[Installation Guide](./docs/installation.md)**: platform-specific setup (Swift, Android, Expo, React Native, Flutter, WASM) +- **[Block-Level LWW Guide](./docs/block-lww.md)**: line-level text merge for markdown and documents +- **[RLS Overview](./docs/rls-overview.md)**: conceptual guide to multi-tenant access control with server-enforced policies +- **[Self-Hosted PostgreSQL Quick Start](./docs/postgresql/quickstarts/postgres.md)**: run CloudSync against your own PostgreSQL instance +- **[Self-Hosted Supabase Quick Start](./docs/postgresql/quickstarts/supabase-self-hosted.md)**: run CloudSync against your own Supabase deployment +- **[Database Schema Recommendations](./docs/schema.md)**: primary keys, constraints, foreign keys, triggers +- **[Custom Network Layer](./docs/internal/network.md)**: replace the built-in libcurl networking +- **[Examples](./examples/)**: complete walkthroughs (todo app, sport tracker, Swift multiplatform) -### Foreign Key Compatibility +## SQLite Cloud Setup -When using foreign key constraints with SQLite Sync, be aware that interactions with the CRDT merge algorithm and Row-Level Security policies may cause constraint violations. +If you are not using SQLite Cloud as the sync backend, see the [self-hosted PostgreSQL quick start](./docs/postgresql/quickstarts/postgres.md) or the [self-hosted Supabase quick start](./docs/postgresql/quickstarts/supabase-self-hosted.md). -#### Potential Conflicts +1. Sign up at [SQLite Cloud](https://sqlitecloud.io/) and create a project. +2. Create a database and your tables in the [dashboard](https://dashboard.sqlitecloud.io/). +3. Enable synchronization: click **"OffSync"** for your database and select the tables to sync. +4. Copy the managed database ID and API key from the dashboard. +5. Use `cloudsync_network_init()` and `cloudsync_network_set_apikey()` locally, then call `cloudsync_network_sync()`. -**CRDT Merge Algorithm and DEFAULT Values** +For token-based authentication (required for RLS), use `cloudsync_network_set_token()` instead of `cloudsync_network_set_apikey()`. -- CRDT changes are applied column-by-column during synchronization -- Columns may be temporarily assigned DEFAULT values during the merge process -- If a foreign key column has a DEFAULT value, that value must exist in the referenced table +## Versioning -**Row-Level Security and CASCADE Actions** -- RLS policies may block operations required for maintaining referential integrity -- CASCADE DELETE/UPDATE operations may fail if RLS prevents access to related rows +This project follows [semver](https://semver.org/). The single source of truth is `CLOUDSYNC_VERSION` in `src/cloudsync.h`; all packaged artifacts (NPM, Maven, pub.dev, Swift, Docker, native tarballs) inherit this version. PATCH releases never alter the exposed API — they ship bug fixes, performance improvements, and internal changes only. -#### Recommendations +The PostgreSQL extension differs only in how it surfaces the version: its catalog version (`default_version` / `installed_version`) exposes `MAJOR.MINOR` only, so PATCH releases are transparent binary upgrades and only MINOR/MAJOR releases need `ALTER EXTENSION cloudsync UPDATE`. The `cloudsync_version()` SQL function always reports the full semver of the loaded `.so`. See the [PostgreSQL upgrade docs](docs/postgresql/quickstarts/postgres.md#upgrading-a-later-release) for the user-facing procedure. -**Database Design Patterns** -- Prefer application-level cascade logic over database-level CASCADE actions -- Design RLS policies to accommodate referential integrity operations -- Use nullable foreign keys where appropriate to avoid DEFAULT value issues -- Alternatively, ensure DEFAULT values for foreign key columns exist in their referenced tables +## License -**Testing and Validation** -- Test synchronization scenarios with foreign key constraints enabled -- Monitor for constraint violations during sync operations in development +This project is licensed under the [Elastic License 2.0](./LICENSE.md). For production or managed service use, [contact SQLite Cloud, Inc](mailto:info@sqlitecloud.io) for a commercial license. -### Trigger Compatibility +--- -Be aware that certain types of triggers can cause errors during synchronization due to SQLite Sync's merge logic. +## ☁️ Hosted version -**Duplicate Operations** -- If a trigger modifies a table that is also synchronized with SQLite Sync, changes performed by the trigger may be applied twice during the merge operation -- This can lead to constraint violations or unexpected data states depending on the table's constraints +Don't want to run a sync server yourself? **[SQLite Cloud CloudSync](https://sqlite.ai)** is the managed backend for SQLite-Sync — works with SQLite Cloud, PostgreSQL, or Supabase as your source of truth, with auth, ACL, and observability included. -**Column-by-Column Processing** -- SQLite Sync applies changes column-by-column during synchronization -- UPDATE triggers may be called multiple times for a single row as each column is processed -- This can result in unexpected trigger behavior +[**Start free →**](https://dashboard.sqlitecloud.io/auth/sign-in) --- -## License - -This project is licensed under the [Elastic License 2.0](./LICENSE.md). You can use, copy, modify, and distribute it under the terms of the license for non-production use. For production or managed service use, please [contact SQLite Cloud, Inc](mailto:info@sqlitecloud.io) for a commercial license. - ---- +## Part of the SQLite AI stack -## Part of the SQLite AI Ecosystem +SQLite-Sync is one piece of a larger ecosystem that turns SQLite into a runtime for intelligent, distributed data: -This project is part of the **SQLite AI** ecosystem, a collection of extensions that bring modern AI capabilities to the world’s most widely deployed database. The goal is to make SQLite the default data and inference engine for Edge AI applications. +**Data layer** +- [sqlite-vector](https://github.com/sqliteai/sqlite-vector) — ANN vector search inside SQLite +- [**sqlite-sync**](https://github.com/sqliteai/sqlite-sync) — Offline-first CRDT sync across devices *(you are here)* +- [sqlite-columnar](https://github.com/sqliteai/sqlite-columnar) — Column-oriented analytics for OLAP queries +- [sqlite-js](https://github.com/sqliteai/sqlite-js) — Custom SQLite functions written in JavaScript -Other projects in the ecosystem include: +**AI layer** +- [sqlite-ai](https://github.com/sqliteai/sqlite-ai) — On-device LLM inference and embeddings +- [sqlite-agent](https://github.com/sqliteai/sqlite-agent) — Autonomous AI agents running inside SQLite +- [sqlite-memory](https://github.com/sqliteai/sqlite-memory) — Persistent, searchable memory for agents +- [sqlite-mcp](https://github.com/sqliteai/sqlite-mcp) — Call MCP tools directly from SQL queries -- **[SQLite-AI](https://github.com/sqliteai/sqlite-ai)** — On-device inference and embedding generation directly inside SQLite. -- **[SQLite-Memory](https://github.com/sqliteai/sqlite-memory)** — Markdown-based AI agent memory with semantic search. -- **[SQLite-Vector](https://github.com/sqliteai/sqlite-vector)** — Ultra-efficient vector search for embeddings stored as BLOBs in standard SQLite tables. -- **[SQLite-Sync](https://github.com/sqliteai/sqlite-sync)** — Local-first CRDT-based synchronization for seamless, conflict-free data sync and real-time collaboration across devices. -- **[SQLite-Agent](https://github.com/sqliteai/sqlite-agent)** — Run autonomous AI agents directly from within SQLite databases. -- **[SQLite-MCP](https://github.com/sqliteai/sqlite-mcp)** — Connect SQLite databases to MCP servers and invoke their tools. -- **[SQLite-JS](https://github.com/sqliteai/sqlite-js)** — Create custom SQLite functions using JavaScript. -- **[Liteparser](https://github.com/sqliteai/liteparser)** — A highly efficient and fully compliant SQLite SQL parser. +**Managed platform** +- [SQLite Cloud](https://sqlite.ai) — Hosted SQLite with sync, auth, edge functions, and analytics. [Free tier →](https://dashboard.sqlitecloud.io/auth/sign-in) -Learn more at **[SQLite AI](https://sqlite.ai)**. +Built by [SQLite AI](https://sqlite.ai). Questions? [Contact us](https://sqlite.ai/support). diff --git a/deploy/supabase-flyio/Dockerfile b/deploy/supabase-flyio/Dockerfile new file mode 100644 index 00000000..6c44d95a --- /dev/null +++ b/deploy/supabase-flyio/Dockerfile @@ -0,0 +1,23 @@ +# Self-hosted Supabase + CloudSync, packaged as a single Fly Machine image. +# +# The image ships dockerd + docker compose and a pinned copy of the upstream +# Supabase `docker/` directory. Everything the stack needs at runtime (images, +# PGDATA, .env) lives on the Fly volume mounted at /data, so machine restarts +# and redeploys keep the data and re-run the same bootstrap. +FROM docker:28-dind + +# fuse-overlayfs: Docker's overlay2 driver fails inside a Fly Machine +# ("failed to convert whiteout file: operation not permitted"). +RUN apk add --no-cache bash fuse-overlayfs openssl + +# Pinned upstream Supabase docker/ tree. Bump to adopt new service versions. +ARG SUPABASE_REF=312d05af4be0a404325417254d57f69378a0091c +RUN wget -qO /tmp/sb.tgz "https://codeload.github.com/supabase/supabase/tar.gz/${SUPABASE_REF}" \ + && tar -xzf /tmp/sb.tgz -C /tmp \ + && mv /tmp/supabase-*/docker /supabase \ + && rm -rf /tmp/sb.tgz /tmp/supabase-* + +COPY docker-compose.cloudsync.yml cloudsync.sql entrypoint.sh /supabase/ +RUN chmod +x /supabase/entrypoint.sh + +ENTRYPOINT ["/supabase/entrypoint.sh"] diff --git a/deploy/supabase-flyio/cloudsync.sql b/deploy/supabase-flyio/cloudsync.sql new file mode 100644 index 00000000..c1b24be4 --- /dev/null +++ b/deploy/supabase-flyio/cloudsync.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS cloudsync; diff --git a/deploy/supabase-flyio/docker-compose.cloudsync.yml b/deploy/supabase-flyio/docker-compose.cloudsync.yml new file mode 100644 index 00000000..dd69e337 --- /dev/null +++ b/deploy/supabase-flyio/docker-compose.cloudsync.yml @@ -0,0 +1,17 @@ +# Layered on top of the upstream docker-compose.yml via COMPOSE_FILE. +# Keeps the upstream file untouched so it can be re-pinned without merge work. +services: + db: + # Released tag by default; set CLOUDSYNC_IMAGE (e.g. `fly secrets set`) to + # run a beta build without touching the repo. + image: ${CLOUDSYNC_IMAGE:-sqlitecloud/sqlite-sync-supabase:17-alpine} + volumes: + - ./cloudsync.sql:/docker-entrypoint-initdb.d/init-scripts/100-cloudsync.sql:Z + ports: + # Direct Postgres, bypassing Supavisor (which already owns 5432). + - "5433:5432" + + supavisor: + # Skips /app/limits.sh, whose `ulimit -n 100000` is refused by the Fly kernel + # and crashes the container. Same tini init, same command. + entrypoint: ["/usr/bin/tini", "-s", "-g", "--"] diff --git a/deploy/supabase-flyio/entrypoint.sh b/deploy/supabase-flyio/entrypoint.sh new file mode 100644 index 00000000..44c6125e --- /dev/null +++ b/deploy/supabase-flyio/entrypoint.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# Boots the whole stack. Idempotent: runs on every machine start, including +# after `fly deploy`, `fly machine restart` and crashes. +set -eu + +DATA_DIR=/data/supabase-docker +DOCKERD_LOG=/data/dockerd.log +INIT_MARKER=/data/.cloudsync-post-init-done + +echo "=== configuring dockerd ===" +mkdir -p /etc/docker /data/docker +cat > /etc/docker/daemon.json <<'JSON' +{ + "storage-driver": "fuse-overlayfs", + "data-root": "/data/docker", + "ipv6": true, + "ip6tables": true, + "fixed-cidr-v6": "fd00:d0c::/64" +} +JSON + +echo "=== starting dockerd ===" +dockerd >>"$DOCKERD_LOG" 2>&1 & +until docker info >/dev/null 2>&1; do sleep 1; done + +echo "=== syncing compose tree to $DATA_DIR ===" +mkdir -p "$DATA_DIR" +# Config comes from the image on every boot; state (.env, volumes/) stays. +cp -R /supabase/. "$DATA_DIR/" +cd "$DATA_DIR" + +set_env() { + grep -q "^$1=" .env && sed -i "s|^$1=.*|$1=$2|" .env || echo "$1=$2" >> .env +} + +if [ ! -f .env ]; then + echo "=== first boot: generating secrets ===" + cp .env.example .env + sh ./utils/generate-keys.sh --update-env + sh ./utils/add-new-auth-keys.sh --update-env + set_env POSTGRES_PASSWORD "$(openssl rand -hex 24)" + set_env DASHBOARD_PASSWORD "$(openssl rand -hex 12)" +fi + +# Public hostnames follow from the app name, which Fly injects, so no +# deployment-specific URL has to live in the repo. Fly secrets still win. +if [ -n "${FLY_APP_NAME:-}" ]; then + : "${SUPABASE_PUBLIC_URL:=https://$FLY_APP_NAME.fly.dev}" + : "${API_EXTERNAL_URL:=$SUPABASE_PUBLIC_URL/auth/v1}" + export SUPABASE_PUBLIC_URL API_EXTERNAL_URL +fi + +# Services dropped from the active config — a gateway swapped out via an override +# profile, say — keep their containers and their published ports, which blocks the +# replacement from binding. `--remove-orphans` does not cover them. +active_services=$(docker compose config --services 2>/dev/null || true) +docker compose ps -a --format '{{.Service}}|{{.Name}}' 2>/dev/null | while IFS='|' read -r svc name; do + [ -n "$svc" ] || continue + echo "$active_services" | grep -qx "$svc" && continue + echo "=== removing disabled service: $svc ($name) ===" + docker rm -f "$name" >/dev/null 2>&1 || true +done + +echo "=== starting supabase ===" +# A service that fails its dependency conditions must not kill the machine: +# exiting here would crash-loop until Fly gives up, taking the healthy services +# with it and leaving nothing to debug. +docker compose up -d --remove-orphans || echo "WARNING: compose up reported a failure, continuing" + +echo "=== waiting for postgres ===" +pg_ready=0 +for _ in $(seq 1 90); do + if docker compose exec -T db pg_isready -U postgres -h localhost >/dev/null 2>&1; then + pg_ready=1 + break + fi + sleep 2 +done + +if [ "$pg_ready" = 1 ]; then + # Init scripts only run on an empty PGDATA, so create the extension explicitly. + docker compose exec -T db psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS cloudsync;" || true +else + echo "WARNING: postgres never became ready, skipping post-init" +fi + +if [ "$pg_ready" = 1 ] && [ ! -f "$INIT_MARKER" ]; then + # GoTrue runs its migrations as supabase_auth_admin and must own these. + echo "=== fixing auth function ownership ===" + for fn in "auth.uid()" "auth.role()" "auth.email()"; do + docker compose exec -T db psql -U postgres \ + -c "ALTER FUNCTION $fn OWNER TO supabase_auth_admin;" || true + done + docker compose restart auth + touch "$INIT_MARKER" +fi + +echo "=== supabase is up ===" +docker compose ps + +# Stream service logs to `fly logs`. If this exits the machine restarts and the +# whole bootstrap re-runs. +exec docker compose logs --no-color -f diff --git a/deploy/supabase-flyio/fly.toml b/deploy/supabase-flyio/fly.toml new file mode 100644 index 00000000..069f3fc6 --- /dev/null +++ b/deploy/supabase-flyio/fly.toml @@ -0,0 +1,43 @@ +# No `app` / `primary_region` here on purpose: both are deployment-specific and +# are passed on the command line instead. +# +# fly deploy . --config fly.toml --app "$FLY_APP" --primary-region "$FLY_REGION" +# +# The public URLs follow from FLY_APP_NAME at boot (see entrypoint.sh) and the +# CloudSync image tag from CLOUDSYNC_IMAGE (see docker-compose.cloudsync.yml); +# override either with `fly secrets set`. + +[build] + dockerfile = "Dockerfile" + +[env] + # Envoy replaces Kong as the API gateway. Upstream makes this the default in + # the ~Aug 2026 release, after which the override becomes a no-op shim. + COMPOSE_FILE = "docker-compose.yml:docker-compose.envoy.yml:docker-compose.cloudsync.yml" + SITE_URL = "http://localhost:3000" + POOLER_TENANT_ID = "cloudsync" + STUDIO_DEFAULT_ORGANIZATION = "CloudSync" + STUDIO_DEFAULT_PROJECT = "Supabase" + +[[mounts]] + source = "supabase_data" + destination = "/data" + initial_size = "50gb" + +[[vm]] + size = "shared-cpu-4x" + memory = "4096" + +# API gateway. Gives Studio and the REST/Auth/Storage APIs HTTPS on .fly.dev. +[http_service] + internal_port = 8000 + force_https = true + auto_stop_machines = "off" + auto_start_machines = false + min_machines_running = 1 + +# Postgres (container port 5433) is intentionally NOT published. Same-org apps +# reach it at .internal:5433; laptops use `fly proxy 5433`. To expose it +# publicly, add a [[services]] block with internal_port = 5433 — note that a +# shared IPv4 serves 80/443 only and Postgres cannot use Fly's TLS handler, so +# public IPv4 access needs a dedicated IPv4 (`fly ips allocate-v4`). diff --git a/docker/Makefile.postgresql b/docker/Makefile.postgresql new file mode 100644 index 00000000..c765d8d7 --- /dev/null +++ b/docker/Makefile.postgresql @@ -0,0 +1,461 @@ +# PostgreSQL Extension Build Configuration +# This file is included by the root Makefile + +# Detect pg_config +PG_CONFIG ?= pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs 2>/dev/null) + +# PostgreSQL directories +PG_SHAREDIR := $(shell $(PG_CONFIG) --sharedir 2>/dev/null) +PG_PKGLIBDIR := $(shell $(PG_CONFIG) --pkglibdir 2>/dev/null) +PG_INCLUDEDIR := $(shell $(PG_CONFIG) --includedir-server 2>/dev/null) + +# Extension metadata +EXTENSION = cloudsync +# Read the binary version (full semver) from src/cloudsync.h, and derive +# EXTVERSION as just MAJOR.MINOR. Rationale: PATCH bumps are binary-only +# (no SQL surface change, no ALTER EXTENSION UPDATE needed); MINOR/MAJOR bumps +# are the SQL-surface-changing ones that require a per-release upgrade script. +# cloudsync_version() in the .so still reports the full semver for debugging. +# +# Recursive (=) rather than immediate (:=) assignment so the shell calls only +# fire when a PG-related target actually references these variables. Non-PG +# targets parse this included makefile without invoking sed/cut at all. +CLOUDSYNC_VERSION_FULL = $(shell sed -n 's/^\#define CLOUDSYNC_VERSION[[:space:]]*"\([^"]*\)".*/\1/p' src/cloudsync.h) +EXTVERSION = $(shell echo '$(CLOUDSYNC_VERSION_FULL)' | cut -d. -f1-2) + +# Detect OS for platform-specific settings +ifneq ($(OS),Windows_NT) +UNAME_S := $(shell uname -s) +endif + +# Platform-specific PostgreSQL settings +ifeq ($(OS),Windows_NT) + PG_EXTENSION_LIB = $(EXTENSION).dll + PG_CFLAGS = -Wall -Wextra -Wno-unused-parameter -std=c11 -O2 + PG_LDFLAGS = -shared -L$(shell $(PG_CONFIG) --libdir) -lpostgres +else + PG_EXTENSION_LIB = $(EXTENSION).so + PG_CFLAGS = -fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -O2 + ifeq ($(UNAME_S),Darwin) + # macOS: allow undefined symbols resolved at load time by PostgreSQL + PG_LDFLAGS = -shared -undefined dynamic_lookup + else + PG_LDFLAGS = -shared + endif +endif + +# Source files - core platform-agnostic code +PG_CORE_SRC = \ + src/cloudsync.c \ + src/dbutils.c \ + src/pk.c \ + src/utils.c \ + src/lz4.c \ + src/block.c \ + modules/fractional-indexing/fractional_indexing.c + +# PostgreSQL-specific implementation +PG_IMPL_SRC = \ + src/postgresql/database_postgresql.c \ + src/postgresql/cloudsync_postgresql.c \ + src/postgresql/pgvalue.c \ + src/postgresql/sql_postgresql.c + +# All source files +PG_ALL_SRC = $(PG_CORE_SRC) $(PG_IMPL_SRC) +PG_OBJS = $(PG_ALL_SRC:.c=.o) + +# Compiler flags +# Define POSIX macros as compiler flags to ensure they're defined before any includes +# On Windows, skip POSIX defines — PG headers use Winsock instead of netinet/in.h +PG_EXTRA_CFLAGS ?= +PG_CPPFLAGS = -I$(PG_INCLUDEDIR) -Isrc -Isrc/postgresql -Imodules/fractional-indexing -DCLOUDSYNC_POSTGRESQL_BUILD $(PG_EXTRA_CFLAGS) +ifneq ($(OS),Windows_NT) +PG_CPPFLAGS += -D_POSIX_C_SOURCE=200809L +ifeq ($(UNAME_S),Darwin) +PG_CPPFLAGS += -D_DARWIN_C_SOURCE +else +PG_CPPFLAGS += -D_GNU_SOURCE +endif +endif +PG_DEBUG ?= 0 +ifeq ($(PG_DEBUG),1) +ifeq ($(OS),Windows_NT) +PG_CFLAGS = -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer +else +PG_CFLAGS = -fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer +endif +endif + +# Output files +PG_EXTENSION_SQL = src/postgresql/$(EXTENSION)--$(EXTVERSION).sql +PG_EXTENSION_CONTROL = docker/postgresql/$(EXTENSION).control + +# Input templates (tracked). @EXTVERSION@ is substituted at build time. +PG_EXTENSION_SQL_IN = src/postgresql/$(EXTENSION).sql.in +PG_EXTENSION_CONTROL_IN = docker/postgresql/$(EXTENSION).control.in + +# Upgrade scripts (cloudsync----.sql) are hand-written per release. +PG_MIGRATIONS_DIR = src/postgresql/migrations +PG_MIGRATION_SQLS = $(wildcard $(PG_MIGRATIONS_DIR)/$(EXTENSION)--*--*.sql) + +# ============================================================================ +# PostgreSQL Build Targets +# ============================================================================ + +.PHONY: postgres-check postgres-check-migration postgres-generate-files postgres-build postgres-install postgres-package postgres-clean postgres-test \ + postgres-docker-build postgres-docker-build-asan postgres-docker-run postgres-docker-run-asan postgres-docker-stop postgres-docker-rebuild \ + postgres-docker-debug-build postgres-docker-debug-run postgres-docker-debug-rebuild \ + postgres-docker-shell postgres-dev-rebuild postgres-help unittest-pg \ + postgres-supabase-build postgres-supabase-rebuild postgres-supabase-run-test \ + postgres-docker-run-test + +# Verify that a cloudsync----.sql upgrade script exists for the +# current CLOUDSYNC_VERSION in src/cloudsync.h. Release-blocking: a missing +# upgrade script silently breaks ALTER EXTENSION cloudsync UPDATE for every +# existing deployment. Runs in <1s; safe to call on every PR. +postgres-check-migration: + @scripts/check-postgres-migration.sh + +# Check if PostgreSQL is available +postgres-check: + @echo "Checking PostgreSQL installation..." + @which $(PG_CONFIG) > /dev/null || (echo "Error: pg_config not found. Install postgresql-server-dev." && exit 1) + @[ -n "$(CLOUDSYNC_VERSION_FULL)" ] || (echo "Error: could not read CLOUDSYNC_VERSION from src/cloudsync.h" && exit 1) + @[ -n "$(EXTVERSION)" ] || (echo "Error: could not derive MAJOR.MINOR EXTVERSION from CLOUDSYNC_VERSION '$(CLOUDSYNC_VERSION_FULL)'" && exit 1) + @echo "PostgreSQL version: $$($(PG_CONFIG) --version)" + @echo "CloudSync version : $(CLOUDSYNC_VERSION_FULL) (extension version $(EXTVERSION))" + @echo "Extension directory: $(PG_PKGLIBDIR)" + @echo "Share directory: $(PG_SHAREDIR)" + @echo "Include directory: $(PG_INCLUDEDIR)" + +# Render the versioned .sql install script and .control file from their .in +# templates. This is a phony target (rather than file rules keyed on +# $(PG_EXTENSION_SQL) / $(PG_EXTENSION_CONTROL)) so that $(EXTVERSION) never +# appears in a rule's target or prerequisite position — those expansions +# happen at parse time and would force sed/cut to run on every `make` +# invocation, including for non-PG goals. Here, the references live inside +# the recipe body and are only evaluated when this target actually fires. +# Writes via a .tmp + atomic mv so a failed sed can't leave a half-rendered +# output file in the extension share dir. +postgres-generate-files: postgres-check + @echo "Rendering extension files at version $(EXTVERSION) (binary $(CLOUDSYNC_VERSION_FULL))" + @sed 's/@EXTVERSION@/$(EXTVERSION)/g' $(PG_EXTENSION_SQL_IN) > $(PG_EXTENSION_SQL).tmp && mv $(PG_EXTENSION_SQL).tmp $(PG_EXTENSION_SQL) + @sed 's/@EXTVERSION@/$(EXTVERSION)/g' $(PG_EXTENSION_CONTROL_IN) > $(PG_EXTENSION_CONTROL).tmp && mv $(PG_EXTENSION_CONTROL).tmp $(PG_EXTENSION_CONTROL) + +# Build PostgreSQL extension +postgres-build: postgres-generate-files + @echo "Building PostgreSQL extension (version $(EXTVERSION))..." + @echo "Compiling source files..." + @for src in $(PG_ALL_SRC); do \ + echo " CC $$src"; \ + $(CC) $(PG_CPPFLAGS) $(PG_CFLAGS) -c $$src -o $${src%.c}.o || exit 1; \ + done + @echo "Linking $(PG_EXTENSION_LIB)..." + $(CC) $(PG_LDFLAGS) $(PG_EXTRA_CFLAGS) -o $(PG_EXTENSION_LIB) $(PG_OBJS) + @echo "Build complete: $(PG_EXTENSION_LIB)" + +# Install extension to PostgreSQL +postgres-install: postgres-build + @echo "Installing CloudSync extension to PostgreSQL..." + @echo "Installing shared library to $(PG_PKGLIBDIR)/" + install -d $(PG_PKGLIBDIR) + install -m 755 $(PG_EXTENSION_LIB) $(PG_PKGLIBDIR)/ + @echo "Installing SQL script to $(PG_SHAREDIR)/extension/" + install -d $(PG_SHAREDIR)/extension + install -m 644 $(PG_EXTENSION_SQL) $(PG_SHAREDIR)/extension/ + @echo "Installing control file to $(PG_SHAREDIR)/extension/" + install -m 644 $(PG_EXTENSION_CONTROL) $(PG_SHAREDIR)/extension/ + @if [ -n "$(PG_MIGRATION_SQLS)" ]; then \ + echo "Installing $(words $(PG_MIGRATION_SQLS)) migration script(s) to $(PG_SHAREDIR)/extension/"; \ + install -m 644 $(PG_MIGRATION_SQLS) $(PG_SHAREDIR)/extension/; \ + fi + @echo "" + @echo "Installation complete!" + @echo "To use the extension, run in psql:" + @echo " CREATE EXTENSION $(EXTENSION);" + @echo "To upgrade an existing installation, run in psql:" + @echo " ALTER EXTENSION $(EXTENSION) UPDATE;" + +# Package extension files for distribution +PG_DIST_DIR = dist/postgresql + +postgres-package: postgres-build + @echo "Packaging PostgreSQL extension (version $(EXTVERSION))..." + @mkdir -p $(PG_DIST_DIR) + cp $(PG_EXTENSION_LIB) $(PG_DIST_DIR)/ + cp $(PG_EXTENSION_SQL) $(PG_DIST_DIR)/ + cp $(PG_EXTENSION_CONTROL) $(PG_DIST_DIR)/ + @if [ -n "$(PG_MIGRATION_SQLS)" ]; then \ + echo "Including $(words $(PG_MIGRATION_SQLS)) migration script(s)"; \ + cp $(PG_MIGRATION_SQLS) $(PG_DIST_DIR)/; \ + fi + @echo "Package ready in $(PG_DIST_DIR)/" + +# Clean PostgreSQL build artifacts +postgres-clean: + @echo "Cleaning PostgreSQL build artifacts..." + rm -f $(PG_OBJS) $(PG_EXTENSION_LIB) + rm -f $(PG_EXTENSION_SQL) $(PG_EXTENSION_CONTROL) + @echo "Clean complete" + +# Test extension (requires running PostgreSQL) +postgres-test: postgres-install + @echo "Testing CloudSync extension..." + @echo "Dropping existing extension (if any)..." + -psql -U postgres -d postgres -c "DROP EXTENSION IF EXISTS $(EXTENSION) CASCADE;" 2>/dev/null + @echo "Creating extension..." + psql -U postgres -d postgres -c "CREATE EXTENSION $(EXTENSION);" + @echo "Testing version function..." + psql -U postgres -d postgres -c "SELECT $(EXTENSION)_version();" + @echo "Listing extension functions..." + psql -U postgres -d postgres -c "\\df $(EXTENSION)_*" + +# ============================================================================ +# Docker Targets +# ============================================================================ + +DOCKER_IMAGE = sqliteai/sqlite-sync-pg +DOCKER_TAG ?= latest +DOCKER_BUILD_ARGS ?= +SUPABASE_CLI_IMAGE ?= $(shell docker ps --format '{{.Image}} {{.Names}}' | awk '/supabase_db/ {print $$1; exit}') +SUPABASE_CLI_DOCKERFILE ?= docker/postgresql/Dockerfile.supabase +SUPABASE_POSTGRES_TAG ?= 17.6.1.071 +SUPABASE_WORKDIR ?= +SUPABASE_WORKDIR_ARG = $(if $(SUPABASE_WORKDIR),--workdir $(SUPABASE_WORKDIR),) +SUPABASE_DB_HOST ?= 127.0.0.1 +SUPABASE_DB_PORT ?= 54322 +SUPABASE_DB_PASSWORD ?= postgres +PG_DOCKER_DB_HOST ?= localhost +PG_DOCKER_DB_PORT ?= 5432 +PG_DOCKER_DB_NAME ?= postgres +PG_DOCKER_DB_USER ?= postgres +PG_DOCKER_DB_PASSWORD ?= postgres + +# Build Docker image with pre-installed extension +postgres-docker-build: + @echo "Building Docker image via docker compose (rebuilt when sources change)..." + # To force plaintext BuildKit logs, run: make postgres-docker-build DOCKER_BUILD_ARGS="--progress=plain" + cd docker/postgresql && docker compose build $(DOCKER_BUILD_ARGS) + @echo "" + @echo "Docker image built successfully!" + +# Build Docker image with AddressSanitizer enabled (override compose file) +postgres-docker-build-asan: + @echo "Building Docker image with ASAN via docker compose..." + # To force plaintext BuildKit logs, run: make postgres-docker-build-asan DOCKER_BUILD_ARGS=\"--progress=plain\" + cd docker/postgresql && docker compose -f docker-compose.debug.yml -f docker-compose.asan.yml build $(DOCKER_BUILD_ARGS) + @echo "" + @echo "ASAN Docker image built successfully!" + +# Build Docker image using docker-compose.debug.yml +postgres-docker-debug-build: + @echo "Building debug Docker image via docker compose..." + # To force plaintext BuildKit logs, run: make postgres-docker-debug-build DOCKER_BUILD_ARGS=\"--progress=plain\" + cd docker/postgresql && docker compose -f docker-compose.debug.yml build $(DOCKER_BUILD_ARGS) + @echo "" + @echo "Debug Docker image built successfully!" + +# Run PostgreSQL container with CloudSync +postgres-docker-run: + @echo "Starting PostgreSQL with CloudSync..." + cd docker/postgresql && docker compose up -d --build + @echo "" + @echo "Container started successfully!" + @echo "" + @echo "Connect with psql:" + @echo " docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test" + @echo "" + @echo "Or from host:" + @echo " psql postgresql://postgres:postgres@localhost:5432/cloudsync_test" + @echo "" + @echo "Enable extension:" + @echo " CREATE EXTENSION cloudsync;" + @echo " SELECT cloudsync_version();" + +# Run PostgreSQL container with CloudSync and AddressSanitizer enabled +postgres-docker-run-asan: + @echo "Starting PostgreSQL with CloudSync (ASAN enabled)..." + cd docker/postgresql && docker compose -f docker-compose.debug.yml -f docker-compose.asan.yml up -d --build + @echo "" + @echo "Container started successfully!" + @echo "" + @echo "Connect with psql:" + @echo " docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test" + @echo "" + @echo "Or from host:" + @echo " psql postgresql://postgres:postgres@localhost:5432/cloudsync_test" + @echo "" + @echo "Enable extension:" + @echo " CREATE EXTENSION cloudsync;" + @echo " SELECT cloudsync_version();" + +# Run PostgreSQL container using docker-compose.debug.yml +postgres-docker-debug-run: + @echo "Starting PostgreSQL with CloudSync (debug compose)..." + cd docker/postgresql && docker compose -f docker-compose.debug.yml up -d --build + @echo "" + @echo "Container started successfully!" + @echo "" + @echo "Connect with psql:" + @echo " docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test" + @echo "" + @echo "Or from host:" + @echo " psql postgresql://postgres:postgres@localhost:5432/cloudsync_test" + @echo "" + @echo "Enable extension:" + @echo " CREATE EXTENSION cloudsync;" + @echo " SELECT cloudsync_version();" + +# Stop PostgreSQL container +postgres-docker-stop: + @echo "Stopping PostgreSQL container..." + cd docker/postgresql && docker compose down + @echo "Container stopped" + +# Rebuild and restart container +postgres-docker-rebuild: postgres-docker-build + @echo "Rebuilding and restarting container..." + cd docker/postgresql && docker compose down + cd docker/postgresql && docker compose up -d --build + @echo "Container restarted with new image" + +# Rebuild and restart container using docker-compose.debug.yml +postgres-docker-debug-rebuild: postgres-docker-debug-build + @echo "Rebuilding and restarting debug container..." + cd docker/postgresql && docker compose -f docker-compose.debug.yml down + cd docker/postgresql && docker compose -f docker-compose.debug.yml up -d --build + @echo "Debug container restarted with new image" + +# Interactive shell in container +postgres-docker-shell: + @echo "Opening shell in PostgreSQL container..." + docker exec -it cloudsync-postgres bash + +# Build CloudSync into the Supabase CLI postgres image tag +postgres-supabase-build: + @echo "Building CloudSync image for Supabase CLI..." + @tmp_dockerfile="$$(mktemp ./cloudsync-supabase-cli.XXXXXX)"; \ + src_dockerfile="$(SUPABASE_CLI_DOCKERFILE)"; \ + supabase_cli_image="$(SUPABASE_CLI_IMAGE)"; \ + if [ -z "$$supabase_cli_image" ]; then \ + supabase_cli_image="public.ecr.aws/supabase/postgres:$(SUPABASE_POSTGRES_TAG)"; \ + fi; \ + if [ -z "$$supabase_cli_image" ]; then \ + echo "Error: Supabase CLI postgres image not found."; \ + echo "Run 'supabase start' first, or set SUPABASE_CLI_IMAGE=public.ecr.aws/supabase/postgres:."; \ + exit 1; \ + fi; \ + if [ ! -f "$$src_dockerfile" ]; then \ + if [ -f "docker/postgresql/Dockerfile.supabase" ]; then \ + src_dockerfile="docker/postgresql/Dockerfile.supabase"; \ + else \ + echo "Error: Supabase Dockerfile not found (expected $$src_dockerfile)."; \ + rm -f "$$tmp_dockerfile"; \ + exit 1; \ + fi; \ + fi; \ + sed -e "s|^FROM supabase/postgres:[^ ]*|FROM $$supabase_cli_image|" \ + -e "s|^FROM public.ecr.aws/supabase/postgres:[^ ]*|FROM $$supabase_cli_image|" \ + "$$src_dockerfile" > "$$tmp_dockerfile"; \ + if [ ! -s "$$tmp_dockerfile" ]; then \ + echo "Error: Generated Dockerfile is empty."; \ + rm -f "$$tmp_dockerfile"; \ + exit 1; \ + fi; \ + pg_major="$${supabase_cli_image##*:}"; pg_major="$${pg_major%%.*}"; \ + case "$$pg_major" in ''|*[!0-9]*) pg_major=17;; esac; \ + echo "Using base image: $$supabase_cli_image (PostgreSQL major $$pg_major)"; \ + echo "Pulling fresh base image to avoid layer accumulation..."; \ + docker pull "$$supabase_cli_image" 2>/dev/null || true; \ + docker build --build-arg SUPABASE_POSTGRES_TAG="$(SUPABASE_POSTGRES_TAG)" --build-arg PG_MAJOR="$$pg_major" --build-arg CLOUDSYNC_VERSION="$(CLOUDSYNC_VERSION_FULL)" -f "$$tmp_dockerfile" -t "$$supabase_cli_image" .; \ + rm -f "$$tmp_dockerfile"; \ + echo "Build complete: $$supabase_cli_image" + +# Rebuild CloudSync image and restart Supabase CLI stack +postgres-supabase-rebuild: postgres-supabase-build + @echo "Restarting Supabase CLI stack..." + @command -v supabase >/dev/null 2>&1 || (echo "Error: supabase CLI not found in PATH." && exit 1) + @supabase stop $(SUPABASE_WORKDIR_ARG) + @supabase start $(SUPABASE_WORKDIR_ARG) + @echo "Supabase CLI stack restarted." + +# Run smoke test against Supabase CLI local database +postgres-supabase-run-test: + @echo "Running Supabase CLI test..." + @PGPASSWORD="$(SUPABASE_DB_PASSWORD)" psql postgresql://supabase_admin@$(SUPABASE_DB_HOST):$(SUPABASE_DB_PORT)/postgres -f test/postgresql/full_test.sql + @echo "Test completed." + +# Run smoke test against Docker standalone database +postgres-docker-run-test: + @echo "Running Docker test..." + @PGPASSWORD="$(PG_DOCKER_DB_PASSWORD)" psql postgresql://$(PG_DOCKER_DB_USER)@$(PG_DOCKER_DB_HOST):$(PG_DOCKER_DB_PORT)/$(PG_DOCKER_DB_NAME) -f test/postgresql/full_test.sql + @echo "Test completed." + +# ============================================================================ +# Development Workflow Targets +# ============================================================================ + +# Quick rebuild inside running container +postgres-dev-rebuild: + @echo "Rebuilding extension inside running container..." + @echo "This is faster than rebuilding the entire Docker image" + docker exec -it cloudsync-postgres bash -c "cd /tmp/cloudsync && make postgres-clean && make postgres-build && make postgres-install" + @echo "" + @echo "Extension rebuilt successfully!" + @echo "" + @echo "To reload the extension in psql, run:" + @echo " DROP EXTENSION cloudsync CASCADE;" + @echo " CREATE EXTENSION cloudsync;" + +# Help target +postgres-help: + @echo "PostgreSQL Extension Build Targets" + @echo "===================================" + @echo "" + @echo "Build & Install:" + @echo " postgres-check - Verify PostgreSQL installation" + @echo " postgres-check-migration - Verify a migration script exists for the current version" + @echo " postgres-generate-files - Render cloudsync.control and cloudsync--.sql from templates" + @echo " postgres-build - Build extension (.so file)" + @echo " postgres-install - Install extension to PostgreSQL" + @echo " postgres-clean - Clean build artifacts" + @echo " postgres-test - Test extension (requires running PostgreSQL)" + @echo "" + @echo "Docker Targets:" + @echo " postgres-docker-build - Build Docker image with pre-installed extension" + @echo " postgres-docker-build-asan - Build Docker image with ASAN enabled" + @echo " postgres-docker-run-asan - Run container with ASAN enabled" + @echo " postgres-docker-debug-build - Build image via docker-compose.debug.yml" + @echo " postgres-docker-debug-run - Run container via docker-compose.debug.yml" + @echo " postgres-docker-debug-rebuild - Rebuild and run docker-compose.debug.yml" + @echo " postgres-docker-run - Start PostgreSQL container" + @echo " postgres-docker-stop - Stop PostgreSQL container" + @echo " postgres-docker-rebuild - Rebuild image and restart container" + @echo " postgres-docker-shell - Open bash shell in running container" + @echo " postgres-supabase-build - Build CloudSync into Supabase CLI postgres image" + @echo " postgres-supabase-rebuild - Build CloudSync image and restart Supabase CLI stack" + @echo " postgres-supabase-run-test - Run smoke test against Supabase CLI database" + @echo " postgres-docker-run-test - Run smoke test against Docker database" + @echo "" + @echo "Development:" + @echo " postgres-dev-rebuild - Rebuild extension in running container (fast)" + @echo " unittest-pg - Rebuild container and run smoke test (create extension + version)" + @echo "" + @echo "Examples:" + @echo " make postgres-docker-build # Build image" + @echo " make postgres-docker-build-asan # Build image with ASAN" + @echo " make postgres-docker-run-asan # Run container with ASAN" + @echo " make postgres-docker-debug-build # Build debug compose image" + @echo " make postgres-docker-debug-run # Run debug compose container" + @echo " make postgres-docker-debug-rebuild # Rebuild debug compose container" + @echo " make postgres-docker-run # Start container" + @echo " make postgres-docker-shell # Open shell" + @echo " make postgres-dev-rebuild # Rebuild after code changes" + +# Simple smoke test: rebuild image/container, create extension, and query version +unittest-pg: postgres-docker-rebuild + @echo "Running PostgreSQL extension smoke test..." + cd docker/postgresql && docker compose exec -T postgres psql -U postgres -d cloudsync_test -f /tmp/cloudsync/docker/postgresql/smoke_test.sql + @echo "Smoke test completed." diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..d9e1fa42 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,361 @@ +# CloudSync Docker Setup + +This directory contains Docker configurations for developing and testing CloudSync with PostgreSQL. + +## Directory Structure + +``` +docker/ +├── postgresql/ # Standalone PostgreSQL with CloudSync +│ ├── Dockerfile # Custom PostgreSQL image +│ ├── docker-compose.yml +│ ├── init.sql # CloudSync metadata tables +│ └── cloudsync.control.in # template; cloudsync.control is generated at build time +``` + +## Option 1: Standalone PostgreSQL + +Use this for simple PostgreSQL development and testing. + +### Quick Start + +```bash +# Build Docker image with CloudSync extension +make postgres-docker-build + +# Start PostgreSQL container +make postgres-docker-run + +# Test the extension +docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test -c "CREATE EXTENSION cloudsync; SELECT cloudsync_version();" +``` + +This starts: +- PostgreSQL 16 on `localhost:5432` +- CloudSync extension pre-installed +- pgAdmin on `localhost:5050` (optional, use `--profile admin`) + +### Configuration + +- **Database**: `cloudsync_test` +- **Username**: `postgres` +- **Password**: `postgres` + +### Development Workflow + +After making changes to the source code: + +```bash +# Quick rebuild inside running container (fast!) +make postgres-dev-rebuild + +# Then reload the extension in psql +docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test +``` + +```sql +DROP EXTENSION cloudsync CASCADE; +CREATE EXTENSION cloudsync; +SELECT cloudsync_version(); +``` + +### Using pgAdmin (Optional) + +Start with the admin profile: + +```bash +docker-compose --profile admin up -d +``` + +Access pgAdmin at http://localhost:5050: +- Email: `admin@cloudsync.local` +- Password: `admin` + +### VS Code Dev Container Debugging (PostgreSQL) + +Use this when you want breakpoints in the extension code. +The dev container uses `docker/postgresql/Dockerfile.debug` and `docker/postgresql/docker-compose.debug.yml`, which build the extension with debug symbols. +Required VS Code extensions: +- `ms-vscode-remote.remote-containers` (Dev Containers) +- `ms-vscode.cpptools` (C/C++ debugging) + +1) Open the dev container +VS Code -> Command Palette -> `Dev Containers: Reopen in Container` + +2) Connect with `psql` (inside the dev container) +```bash +psql -U postgres -d cloudsync_test +``` + +3) Enable the extension if needed +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +4) Get the backend PID (inside `psql`) +```sql +SELECT pg_backend_pid(); +``` + +5) Attach the debugger (VS Code dev container window) +Run and Debug -> `Attach to Postgres (gdb)` -> pick the PID from step 4 -> Continue + +6) Trigger your breakpoint +Run the SQL that exercises the code path. If `psql` blocks, the backend is paused at a breakpoint; continue in the debugger. + +## Option 2: Supabase Integration (cli) + +Use this when you're running `supabase start` and want CloudSync inside the local stack. +The Supabase CLI uses a bundled PostgreSQL image (for example, +`public.ecr.aws/supabase/postgres:17.6.1.071`). Build a matching image that +includes CloudSync, then tag it with the same name so the CLI reuses it. This +keeps your local Supabase stack intact (auth, realtime, storage, etc.) while +enabling the extension in the CLI-managed Postgres container. + +> **Note on Ubuntu vs Alpine bases.** Newer Supabase Postgres images (roughly +> `17.6.1.084` and later) use an Alpine-based userland instead of the earlier +> Ubuntu one. The published release image (`Dockerfile.supabase.release`, used +> for `sqlitecloud/sqlite-sync-supabase:*`) detects the userland automatically +> and works on both. The local build-from-source flow below +> (`make postgres-supabase-build`, `Dockerfile.supabase`) currently assumes an +> Ubuntu base; if your CLI stack pulls a newer Alpine image, prefer the +> published `:17-alpine` release image. + +### Prerequisites + +- Supabase CLI installed (`supabase start` works) +- Docker running + +### Setup + +1. Initialize a Supabase project (use a separate workdir to keep generated files + out of the repo): + ```bash + mkdir -p ~/supabase-local + supabase init --workdir ~/supabase-local + ``` + +2. Start Supabase once so the CLI pulls the Postgres image: + ```bash + supabase start --workdir ~/supabase-local + ``` + +3. Build and tag a CloudSync image using the same tag as the running CLI stack: + ```bash + make postgres-supabase-build + ``` + This auto-detects the running `supabase_db` image tag and rebuilds it with + CloudSync installed. If you need to override the full image tag, set + `SUPABASE_CLI_IMAGE=public.ecr.aws/supabase/postgres:`. + Example: + ```bash + SUPABASE_CLI_IMAGE=public.ecr.aws/supabase/postgres:17.6.1.071 make postgres-supabase-build + ``` + You can also set the Supabase base image tag explicitly (defaults to + `17.6.1.071`). This only affects the base image used in the Dockerfile: + ```bash + SUPABASE_POSTGRES_TAG=17.6.1.071 make postgres-supabase-build + ``` + +4. Restart the stack: + ```bash + supabase stop --workdir ~/supabase-local + supabase start --workdir ~/supabase-local + ``` + +### Using the CloudSync Extension + +You can load the extension automatically from a migration, or enable it +manually. + +Migration-based (notes for CLI): Supabase CLI migrations run as the `postgres` +role, which cannot create C extensions by default. Use manual enable or grant +`USAGE` on language `c` once, then migrations will work. Note: `c` is an +untrusted language, so `GRANT USAGE ON LANGUAGE c` is only allowed for +superusers. On the CLI/local stack, the simplest approach is to enable the +extension manually as `supabase_admin` after `supabase db reset`. + +If you still want a migration file, add: +```bash +~/supabase-local/supabase/migrations/00000000000000_cloudsync.sql +``` +Contents: +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +Then either: +- run `GRANT USAGE ON LANGUAGE c TO postgres;` once as `supabase_admin`, or +- skip the migration and enable the extension manually after `supabase db reset`. + +Manual enable (no reset required): + +Connect as the Supabase superuser (C extensions require superuser or language +privileges), then enable the extension: + +```bash +psql postgresql://supabase_admin:postgres@127.0.0.1:54322/postgres +``` + +```sql +CREATE EXTENSION cloudsync; +SELECT cloudsync_version(); +``` + +If you want to use the `postgres` role instead: + +```sql +GRANT USAGE ON LANGUAGE c TO postgres; +``` + +### Rebuilding After Changes + +If you modify the CloudSync source code, rebuild the CLI image and restart: + +```bash +make postgres-supabase-rebuild SUPABASE_WORKDIR=~/supabase-local +``` + +### Supabase Realtime Migration Error (app_schema_version) + +If Supabase Realtime fails to start with: + +``` +ERROR 42P01 (undefined_table) relation "app_schema_version" does not exist +``` + +it's caused by CloudSync's `app_schema_change` event trigger firing during +migrations while Realtime uses a restricted `search_path`. Fix it by +fully qualifying the table in the trigger function: + +```sql +CREATE TABLE IF NOT EXISTS public.app_schema_version ( + version BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY +); + +CREATE OR REPLACE FUNCTION bump_app_schema_version() +RETURNS event_trigger AS $$ +BEGIN + INSERT INTO public.app_schema_version DEFAULT VALUES; +END; +$$ LANGUAGE plpgsql; + +DROP EVENT TRIGGER IF EXISTS app_schema_change; +CREATE EVENT TRIGGER app_schema_change +ON ddl_command_end +EXECUTE FUNCTION bump_app_schema_version(); +``` + +## Development Workflow + +### 1. Make Changes + +Edit source files in `src/postgresql/` or `src/` (shared code). + +### 2. Rebuild Extension + +**Fast method** (rebuild in running container): +```bash +make postgres-dev-rebuild +``` + +**Or manually**: +```bash +docker exec -it cloudsync-postgres bash +cd /tmp/cloudsync +make postgres-clean && make postgres-build && make postgres-install +``` + +### 3. Reload Extension in PostgreSQL + +```bash +docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test +``` + +```sql +-- Reload extension +DROP EXTENSION IF EXISTS cloudsync CASCADE; +CREATE EXTENSION cloudsync; + +-- Test your changes +SELECT cloudsync_version(); +SELECT cloudsync_init('test_table'); +``` + +## Troubleshooting + +### Extension Not Found + +If you get "could not open extension control file", the extension wasn't installed correctly: + +```bash +# Check installation paths +pg_config --sharedir # Should contain cloudsync.control +pg_config --pkglibdir # Should contain cloudsync.so + +# Reinstall +cd /tmp/cloudsync +make install POSTGRES=1 +``` + +### Build Errors + +If you encounter build errors: + +```bash +# Install missing dependencies +apt-get update +apt-get install -y build-essential postgresql-server-dev-16 + +# Clean and rebuild +make clean +make POSTGRES=1 +``` + +### Database Connection Issues + +If you can't connect to PostgreSQL: + +```bash +# Check if PostgreSQL is running +docker ps | grep postgres + +# Check logs +docker logs cloudsync-postgres + +# Restart container +docker-compose restart +``` + +## Environment Variables + +You can customize the setup using environment variables: + +```bash +# PostgreSQL +export POSTGRES_PASSWORD=mypassword +export POSTGRES_DB=mydb + +# Ports +export POSTGRES_PORT=5432 +export PGADMIN_PORT=5050 + +docker-compose up -d +``` + +## Cleaning Up + +```bash +# Stop containers +docker-compose down + +# Remove volumes (deletes all data!) +docker-compose down -v + +# Remove images +docker rmi sqliteai/sqlite-sync-pg:latest +``` + +## Next Steps + +- See [API.md](../API.md) for CloudSync API documentation diff --git a/docker/postgresql/Dockerfile b/docker/postgresql/Dockerfile new file mode 100644 index 00000000..e18b71b0 --- /dev/null +++ b/docker/postgresql/Dockerfile @@ -0,0 +1,50 @@ +# PostgreSQL Docker image with CloudSync extension pre-installed +ARG POSTGRES_TAG=17 +FROM postgres:${POSTGRES_TAG} + +# Derive the major version from PG_MAJOR (set by the official postgres image) +# and install the matching server-dev package +RUN apt-get update && apt-get install -y \ + build-essential \ + postgresql-contrib-${PG_MAJOR} \ + postgresql-server-dev-${PG_MAJOR} \ + git \ + make \ + && rm -rf /var/lib/apt/lists/* + +# Create directory for extension source +WORKDIR /tmp/cloudsync + +# Copy entire source tree (needed for includes and makefiles) +COPY src/ ./src/ +COPY modules/ ./modules/ +COPY docker/ ./docker/ +COPY Makefile . + +# Build and install the CloudSync extension +RUN make postgres-build && \ + make postgres-install && \ + make postgres-clean + +# Verify installation +RUN echo "Verifying CloudSync extension installation..." && \ + ls -la $(pg_config --pkglibdir)/cloudsync.so && \ + ls -la $(pg_config --sharedir)/extension/cloudsync* && \ + echo "CloudSync extension installed successfully" + +# Set default PostgreSQL credentials +ENV POSTGRES_PASSWORD=postgres +ENV POSTGRES_DB=cloudsync_test + +# Expose PostgreSQL port +EXPOSE 5432 + +# Copy initialization script (creates CloudSync metadata tables) +COPY docker/postgresql/init.sql /docker-entrypoint-initdb.d/ + +# Return to root directory +WORKDIR / + +# Add label with extension version +LABEL org.sqliteai.cloudsync.version="1.0" \ + org.sqliteai.cloudsync.description="PostgreSQL with CloudSync CRDT extension" diff --git a/docker/postgresql/Dockerfile.debug b/docker/postgresql/Dockerfile.debug new file mode 100644 index 00000000..c5549366 --- /dev/null +++ b/docker/postgresql/Dockerfile.debug @@ -0,0 +1,99 @@ +# PostgreSQL Docker image with CloudSync extension (debug build) +FROM postgres:17 + +# Enable ASAN build flags when requested (used by docker-compose.asan.yml). +ARG ENABLE_ASAN=0 + +# Install build dependencies and debug symbols +RUN apt-get update && apt-get install -y \ + ca-certificates \ + gnupg \ + wget \ + && . /etc/os-release \ + && echo "deb http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && echo "deb-src http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg-src.list \ + && wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg \ + && echo "deb http://deb.debian.org/debian-debug ${VERSION_CODENAME}-debug main" > /etc/apt/sources.list.d/debian-debug.list \ + && echo "deb-src http://deb.debian.org/debian ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/debian-src.list \ + && apt-get update && apt-get install -y \ + build-essential \ + bison \ + dpkg-dev \ + flex \ + gdb \ + libicu-dev \ + libreadline-dev \ + libasan8 \ + libssl-dev \ + postgresql-server-dev-17 \ + postgresql-17-dbgsym \ + git \ + make \ + zlib1g-dev \ + && apt-get source postgresql-17 \ + && mkdir -p /usr/src/postgresql-17 \ + && srcdir="$(find . -maxdepth 1 -type d -name 'postgresql-17*' | head -n 1)" \ + && if [ -n "$srcdir" ]; then cp -a "$srcdir"/. /usr/src/postgresql-17/; fi \ + && rm -rf /var/lib/apt/lists/* + +# Create directory for extension source +WORKDIR /tmp/cloudsync + +# Build PostgreSQL from source without optimizations for better gdb visibility +RUN set -eux; \ + cd /usr/src/postgresql-17; \ + ./configure --enable-debug --enable-cassert --without-icu CFLAGS="-O0 -g3 -fno-omit-frame-pointer"; \ + make -j"$(nproc)"; \ + make install; \ + make -C contrib/dblink -j"$(nproc)"; \ + make -C contrib/dblink install + +ENV PATH="/usr/local/pgsql/bin:${PATH}" +ENV LD_LIBRARY_PATH="/usr/local/pgsql/lib:${LD_LIBRARY_PATH}" + +# Copy entire source tree (needed for includes and makefiles) +COPY src/ ./src/ +COPY modules/ ./modules/ +COPY docker/ ./docker/ +COPY Makefile . + +# Build and install the CloudSync extension with debug flags +RUN set -eux; \ + ASAN_CFLAGS=""; \ + ASAN_LDFLAGS=""; \ + if [ "${ENABLE_ASAN}" = "1" ]; then \ + ASAN_CFLAGS="-fsanitize=address"; \ + ASAN_LDFLAGS="-fsanitize=address"; \ + fi; \ + make postgres-build PG_DEBUG=1 \ + PG_CFLAGS="-fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer ${ASAN_CFLAGS}" \ + PG_LDFLAGS="-shared ${ASAN_LDFLAGS}" \ + PG_CPPFLAGS="-I$(pg_config --includedir-server) -Isrc -Isrc/postgresql -Imodules/fractional-indexing -DCLOUDSYNC_POSTGRESQL_BUILD -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE" && \ + make postgres-install PG_DEBUG=1 \ + PG_CFLAGS="-fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer ${ASAN_CFLAGS}" \ + PG_LDFLAGS="-shared ${ASAN_LDFLAGS}" \ + PG_CPPFLAGS="-I$(pg_config --includedir-server) -Isrc -Isrc/postgresql -Imodules/fractional-indexing -DCLOUDSYNC_POSTGRESQL_BUILD -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE" && \ + make postgres-clean + +# Verify installation +RUN echo "Verifying CloudSync extension installation..." && \ + ls -la $(pg_config --pkglibdir)/cloudsync.so && \ + ls -la $(pg_config --sharedir)/extension/cloudsync* && \ + echo "CloudSync extension installed successfully" + +# Set default PostgreSQL credentials +ENV POSTGRES_PASSWORD=postgres +ENV POSTGRES_DB=cloudsync_test + +# Expose PostgreSQL port +EXPOSE 5432 + +# Copy initialization script (creates CloudSync metadata tables) +COPY docker/postgresql/init.sql /docker-entrypoint-initdb.d/ + +# Return to root directory +WORKDIR / + +# Add label with extension version +LABEL org.sqliteai.cloudsync.version="1.0" \ + org.sqliteai.cloudsync.description="PostgreSQL with CloudSync CRDT extension (debug)" diff --git a/docker/postgresql/Dockerfile.debug-no-optimization b/docker/postgresql/Dockerfile.debug-no-optimization new file mode 100644 index 00000000..c5549366 --- /dev/null +++ b/docker/postgresql/Dockerfile.debug-no-optimization @@ -0,0 +1,99 @@ +# PostgreSQL Docker image with CloudSync extension (debug build) +FROM postgres:17 + +# Enable ASAN build flags when requested (used by docker-compose.asan.yml). +ARG ENABLE_ASAN=0 + +# Install build dependencies and debug symbols +RUN apt-get update && apt-get install -y \ + ca-certificates \ + gnupg \ + wget \ + && . /etc/os-release \ + && echo "deb http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && echo "deb-src http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg-src.list \ + && wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg \ + && echo "deb http://deb.debian.org/debian-debug ${VERSION_CODENAME}-debug main" > /etc/apt/sources.list.d/debian-debug.list \ + && echo "deb-src http://deb.debian.org/debian ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/debian-src.list \ + && apt-get update && apt-get install -y \ + build-essential \ + bison \ + dpkg-dev \ + flex \ + gdb \ + libicu-dev \ + libreadline-dev \ + libasan8 \ + libssl-dev \ + postgresql-server-dev-17 \ + postgresql-17-dbgsym \ + git \ + make \ + zlib1g-dev \ + && apt-get source postgresql-17 \ + && mkdir -p /usr/src/postgresql-17 \ + && srcdir="$(find . -maxdepth 1 -type d -name 'postgresql-17*' | head -n 1)" \ + && if [ -n "$srcdir" ]; then cp -a "$srcdir"/. /usr/src/postgresql-17/; fi \ + && rm -rf /var/lib/apt/lists/* + +# Create directory for extension source +WORKDIR /tmp/cloudsync + +# Build PostgreSQL from source without optimizations for better gdb visibility +RUN set -eux; \ + cd /usr/src/postgresql-17; \ + ./configure --enable-debug --enable-cassert --without-icu CFLAGS="-O0 -g3 -fno-omit-frame-pointer"; \ + make -j"$(nproc)"; \ + make install; \ + make -C contrib/dblink -j"$(nproc)"; \ + make -C contrib/dblink install + +ENV PATH="/usr/local/pgsql/bin:${PATH}" +ENV LD_LIBRARY_PATH="/usr/local/pgsql/lib:${LD_LIBRARY_PATH}" + +# Copy entire source tree (needed for includes and makefiles) +COPY src/ ./src/ +COPY modules/ ./modules/ +COPY docker/ ./docker/ +COPY Makefile . + +# Build and install the CloudSync extension with debug flags +RUN set -eux; \ + ASAN_CFLAGS=""; \ + ASAN_LDFLAGS=""; \ + if [ "${ENABLE_ASAN}" = "1" ]; then \ + ASAN_CFLAGS="-fsanitize=address"; \ + ASAN_LDFLAGS="-fsanitize=address"; \ + fi; \ + make postgres-build PG_DEBUG=1 \ + PG_CFLAGS="-fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer ${ASAN_CFLAGS}" \ + PG_LDFLAGS="-shared ${ASAN_LDFLAGS}" \ + PG_CPPFLAGS="-I$(pg_config --includedir-server) -Isrc -Isrc/postgresql -Imodules/fractional-indexing -DCLOUDSYNC_POSTGRESQL_BUILD -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE" && \ + make postgres-install PG_DEBUG=1 \ + PG_CFLAGS="-fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer ${ASAN_CFLAGS}" \ + PG_LDFLAGS="-shared ${ASAN_LDFLAGS}" \ + PG_CPPFLAGS="-I$(pg_config --includedir-server) -Isrc -Isrc/postgresql -Imodules/fractional-indexing -DCLOUDSYNC_POSTGRESQL_BUILD -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE" && \ + make postgres-clean + +# Verify installation +RUN echo "Verifying CloudSync extension installation..." && \ + ls -la $(pg_config --pkglibdir)/cloudsync.so && \ + ls -la $(pg_config --sharedir)/extension/cloudsync* && \ + echo "CloudSync extension installed successfully" + +# Set default PostgreSQL credentials +ENV POSTGRES_PASSWORD=postgres +ENV POSTGRES_DB=cloudsync_test + +# Expose PostgreSQL port +EXPOSE 5432 + +# Copy initialization script (creates CloudSync metadata tables) +COPY docker/postgresql/init.sql /docker-entrypoint-initdb.d/ + +# Return to root directory +WORKDIR / + +# Add label with extension version +LABEL org.sqliteai.cloudsync.version="1.0" \ + org.sqliteai.cloudsync.description="PostgreSQL with CloudSync CRDT extension (debug)" diff --git a/docker/postgresql/Dockerfile.release b/docker/postgresql/Dockerfile.release new file mode 100644 index 00000000..2c93eda8 --- /dev/null +++ b/docker/postgresql/Dockerfile.release @@ -0,0 +1,59 @@ +# PostgreSQL with pre-compiled CloudSync (sqlite-sync) extension +# +# Usage: +# docker build \ +# --build-arg POSTGRES_TAG=17 \ +# --build-arg CLOUDSYNC_VERSION=1.0.6 \ +# -t sqlite-sync-postgres:17 \ +# -f docker/postgresql/Dockerfile.release . +# +# Or pull the pre-built image from Docker Hub: +# docker pull sqlitecloud/sqlite-sync-postgres:17 +# + +ARG POSTGRES_TAG=17 +FROM postgres:${POSTGRES_TAG} + +ARG CLOUDSYNC_VERSION +ARG TARGETARCH + +# CI beta builds stage pre-built artifacts here; when the tarball for this arch +# is present it is used instead of the GitHub release download +COPY docker/postgresql/beta-artifacts/ /tmp/beta-artifacts/ + +# Map Docker platform arch to artifact arch +RUN case "${TARGETARCH}" in \ + amd64) ARCH="x86_64" ;; \ + arm64) ARCH="arm64" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \ + ASSET="cloudsync-postgresql${PG_MAJOR}-linux-${ARCH}-${CLOUDSYNC_VERSION}.tar.gz" && \ + if [ -f "/tmp/beta-artifacts/${ASSET}" ]; then \ + echo "Using local artifact ${ASSET}" && \ + cp "/tmp/beta-artifacts/${ASSET}" /tmp/cloudsync.tar.gz; \ + else \ + URL="https://github.com/sqliteai/sqlite-sync/releases/download/${CLOUDSYNC_VERSION}/${ASSET}" && \ + echo "Downloading ${URL}" && \ + curl -fSL "${URL}" -o /tmp/cloudsync.tar.gz; \ + fi && \ + mkdir -p /tmp/cloudsync && \ + tar -xzf /tmp/cloudsync.tar.gz -C /tmp/cloudsync && \ + install -m 755 /tmp/cloudsync/cloudsync.so "$(pg_config --pkglibdir)/" && \ + install -m 644 /tmp/cloudsync/cloudsync--*.sql "$(pg_config --sharedir)/extension/" && \ + install -m 644 /tmp/cloudsync/cloudsync.control "$(pg_config --sharedir)/extension/" && \ + rm -rf /tmp/cloudsync /tmp/cloudsync.tar.gz /tmp/beta-artifacts && \ + apt-get purge -y curl && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* + +# Verify installation +RUN ls -la "$(pg_config --pkglibdir)/cloudsync.so" && \ + ls -la "$(pg_config --sharedir)/extension/cloudsync"* && \ + echo "CloudSync extension installed successfully" + +# Copy initialization script (auto-creates the extension on first start) +COPY docker/postgresql/init.sql /docker-entrypoint-initdb.d/ + +EXPOSE 5432 + +LABEL org.sqliteai.cloudsync.description="PostgreSQL with CloudSync CRDT extension" \ + org.opencontainers.image.source="https://github.com/sqliteai/sqlite-sync" diff --git a/docker/postgresql/Dockerfile.supabase b/docker/postgresql/Dockerfile.supabase new file mode 100644 index 00000000..d132aacc --- /dev/null +++ b/docker/postgresql/Dockerfile.supabase @@ -0,0 +1,117 @@ +# Supabase PostgreSQL with CloudSync (sqlite-sync) built from source +# +# Works with both Supabase base image families: +# - Ubuntu/glibc images (e.g. PG15 15.8.1.135, older PG17 17.6.1.071) +# - Alpine-userland images (e.g. PG17 17.6.1.084+); still a glibc Nix postgres +# Both families run a Nix-built, glibc-linked PostgreSQL, so the extension is +# compiled in a glibc builder stage (official postgres image + PGDG headers, +# the same toolchain that produces the release artifacts) and the identical +# .so is installed on either base. The runtime stage needs no compiler, so +# its userland (apt vs apk) never matters. +# +# PG_MAJOR must match the major version of SUPABASE_POSTGRES_TAG; the +# postgres-supabase-build Makefile target derives it automatically. +ARG PG_MAJOR=17 +ARG SUPABASE_POSTGRES_TAG=17.6.1.071 + +# Build stage for CloudSync extension (glibc toolchain, matching PG headers) +FROM postgres:${PG_MAJOR} AS cloudsync-builder +ARG PG_MAJOR + +# Install build dependencies (PGDG apt repo is preconfigured in this image) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + postgresql-server-dev-${PG_MAJOR} \ + && rm -rf /var/lib/apt/lists/* + +# Create directory for extension source +WORKDIR /tmp/cloudsync + +# Copy entire source tree (needed for includes and makefiles) +COPY src/ ./src/ +COPY modules/ ./modules/ +COPY docker/ ./docker/ +COPY Makefile . + +# Build the CloudSync extension against the PGDG headers +RUN make postgres-build PG_CONFIG=/usr/lib/postgresql/${PG_MAJOR}/bin/pg_config + +# Collect build artifacts +RUN mkdir -p /tmp/cloudsync-artifacts/lib /tmp/cloudsync-artifacts/extension && \ + cp /tmp/cloudsync/cloudsync.so /tmp/cloudsync-artifacts/lib/ && \ + cp /tmp/cloudsync/src/postgresql/cloudsync--*.sql /tmp/cloudsync-artifacts/extension/ && \ + cp /tmp/cloudsync/docker/postgresql/cloudsync.control /tmp/cloudsync-artifacts/extension/ && \ + # Include per-release upgrade scripts so ALTER EXTENSION ... UPDATE works + if ls /tmp/cloudsync/src/postgresql/migrations/cloudsync--*--*.sql 1>/dev/null 2>&1; then \ + cp /tmp/cloudsync/src/postgresql/migrations/cloudsync--*--*.sql /tmp/cloudsync-artifacts/extension/; \ + fi + +# Runtime image based on Supabase Postgres +FROM public.ecr.aws/supabase/postgres:${SUPABASE_POSTGRES_TAG} + +# Extension version (derived from src/cloudsync.h by the Makefile and passed in +# as a build arg); used only for the image label. +ARG CLOUDSYNC_VERSION=unknown + +# pg_config is resolved dynamically at build time (its path differs between the +# Ubuntu and Alpine Supabase bases). Pass --build-arg CLOUDSYNC_PG_CONFIG= +# to override. Kept an ARG so it does not leak into the built image. +ARG CLOUDSYNC_PG_CONFIG="" + +# Install CloudSync extension artifacts +COPY --from=cloudsync-builder /tmp/cloudsync-artifacts/ /tmp/cloudsync-artifacts/ +RUN set -eu; \ + # Resolve pg_config: the Nix profiles the Supabase bases ship first, PATH last + PG_CONFIG="${CLOUDSYNC_PG_CONFIG:-}"; \ + if [ -z "$PG_CONFIG" ]; then \ + for c in /root/.nix-profile/bin/pg_config /nix/var/nix/profiles/default/bin/pg_config "$(command -v pg_config || true)"; do \ + if [ -x "$c" ]; then PG_CONFIG="$c"; break; fi; \ + done; \ + fi; \ + if [ ! -x "$PG_CONFIG" ]; then echo "Error: pg_config not found${PG_CONFIG:+ at $PG_CONFIG}"; exit 1; fi; \ + PKGLIBDIR="$(${PG_CONFIG} --pkglibdir)"; \ + # Supabase wraps postgres and overrides libdir via NIX_PGLIBDIR. + NIX_PGLIBDIR="$(grep -E '^export NIX_PGLIBDIR' /usr/bin/postgres | sed -E "s/.*'([^']+)'.*/\1/" || true)"; \ + if [ -n "$NIX_PGLIBDIR" ]; then PKGLIBDIR="$NIX_PGLIBDIR"; fi; \ + SHAREDIR_PGCONFIG="$(${PG_CONFIG} --sharedir)"; \ + SHAREDIR_STD="/usr/share/postgresql"; \ + install -d "$PKGLIBDIR" "$SHAREDIR_PGCONFIG/extension"; \ + install -m 755 /tmp/cloudsync-artifacts/lib/cloudsync.so "$PKGLIBDIR/"; \ + install -m 644 /tmp/cloudsync-artifacts/extension/cloudsync* "$SHAREDIR_PGCONFIG/extension/"; \ + if [ "$SHAREDIR_STD" != "$SHAREDIR_PGCONFIG" ] && [ -d "$SHAREDIR_STD" ]; then \ + install -d "$SHAREDIR_STD/extension"; \ + install -m 644 /tmp/cloudsync-artifacts/extension/cloudsync* "$SHAREDIR_STD/extension/"; \ + fi; \ + rm -rf /tmp/cloudsync-artifacts + +# Verify installation +RUN set -eu; \ + PG_CONFIG="${CLOUDSYNC_PG_CONFIG:-}"; \ + if [ -z "$PG_CONFIG" ]; then \ + for c in /root/.nix-profile/bin/pg_config /nix/var/nix/profiles/default/bin/pg_config "$(command -v pg_config || true)"; do \ + if [ -x "$c" ]; then PG_CONFIG="$c"; break; fi; \ + done; \ + fi; \ + if [ ! -x "$PG_CONFIG" ]; then echo "Error: pg_config not found${PG_CONFIG:+ at $PG_CONFIG}"; exit 1; fi; \ + NIX_PGLIBDIR="$(grep -E '^export NIX_PGLIBDIR' /usr/bin/postgres | sed -E "s/.*'([^']+)'.*/\1/" || true)"; \ + echo "Verifying CloudSync extension installation..."; \ + if [ -n "$NIX_PGLIBDIR" ]; then \ + ls -la "$NIX_PGLIBDIR/cloudsync.so"; \ + else \ + ls -la "$(${PG_CONFIG} --pkglibdir)/cloudsync.so"; \ + fi; \ + ls -la "$(${PG_CONFIG} --sharedir)/extension/cloudsync"*; \ + if [ -d "/usr/share/postgresql/extension" ]; then \ + ls -la /usr/share/postgresql/extension/cloudsync*; \ + fi; \ + echo "CloudSync extension installed successfully" + +# Expose PostgreSQL port +EXPOSE 5432 + +# Return to root directory +WORKDIR / + +# Add label with extension version +LABEL org.sqliteai.cloudsync.version="${CLOUDSYNC_VERSION}" \ + org.sqliteai.cloudsync.description="PostgreSQL with CloudSync CRDT extension" diff --git a/docker/postgresql/Dockerfile.supabase.release b/docker/postgresql/Dockerfile.supabase.release new file mode 100644 index 00000000..6e56b434 --- /dev/null +++ b/docker/postgresql/Dockerfile.supabase.release @@ -0,0 +1,125 @@ +# Supabase PostgreSQL with pre-compiled CloudSync (sqlite-sync) extension +# +# Works with both Supabase base image families: +# - Ubuntu/glibc images (e.g. PG15 15.8.1.135, older PG17 17.6.1.071) +# - Alpine-userland images (e.g. PG17 17.6.1.084+); still a glibc Nix postgres +# Both run a Nix-built, glibc-linked PostgreSQL, so the SAME glibc extension +# artifact is installed in either case. Only the build-time userland tooling +# differs (apt-get vs apk), which this file detects at runtime. +# +# Usage: +# docker build \ +# --build-arg SUPABASE_POSTGRES_TAG=17.6.1.151 \ +# --build-arg CLOUDSYNC_VERSION= \ +# -f docker/postgresql/Dockerfile.supabase.release \ +# -t my-cloudsync-supabase-postgres . +# +# Or pull the pre-built image from Docker Hub: +# docker pull sqlitecloud/sqlite-sync-supabase:17 +# + +ARG SUPABASE_POSTGRES_TAG=17.6.1.071 +FROM public.ecr.aws/supabase/postgres:${SUPABASE_POSTGRES_TAG} + +ARG CLOUDSYNC_VERSION +ARG TARGETARCH + +# pg_config is resolved dynamically at build time (its path differs between the +# Ubuntu and Alpine Supabase bases). Pass --build-arg CLOUDSYNC_PG_CONFIG= +# to override. Kept an ARG so it does not leak into the published image. +ARG CLOUDSYNC_PG_CONFIG="" + +# CI beta builds stage pre-built artifacts here; when the tarball for this arch +# is present it is used instead of the GitHub release download +COPY docker/postgresql/beta-artifacts/ /tmp/beta-artifacts/ + +# Download pre-compiled extension and install into Supabase's Nix layout +RUN set -eu; \ + : "${CLOUDSYNC_VERSION:?required: pass --build-arg CLOUDSYNC_VERSION=}"; \ + case "${TARGETARCH:-}" in \ + amd64) ARCH="x86_64" ;; \ + arm64) ARCH="arm64" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH:-}"; exit 1 ;; \ + esac; \ + # Resolve pg_config: the Nix profiles the Supabase bases ship first, PATH last + PG_CONFIG="${CLOUDSYNC_PG_CONFIG:-}"; \ + if [ -z "$PG_CONFIG" ]; then \ + for c in /root/.nix-profile/bin/pg_config /nix/var/nix/profiles/default/bin/pg_config "$(command -v pg_config || true)"; do \ + if [ -x "$c" ]; then PG_CONFIG="$c"; break; fi; \ + done; \ + fi; \ + if [ ! -x "$PG_CONFIG" ]; then echo "Error: pg_config not found${PG_CONFIG:+ at $PG_CONFIG}"; exit 1; fi; \ + # Derive PG major version from pg_config + PG_MAJOR=$(${PG_CONFIG} --version | sed 's/[^0-9]*//' | cut -d. -f1); \ + ASSET="cloudsync-postgresql${PG_MAJOR}-linux-${ARCH}-${CLOUDSYNC_VERSION}.tar.gz"; \ + if [ -f "/tmp/beta-artifacts/${ASSET}" ]; then \ + echo "Using local artifact ${ASSET}"; \ + cp "/tmp/beta-artifacts/${ASSET}" /tmp/cloudsync.tar.gz; \ + else \ + # curl is only needed for the release download; install it with whichever + # package manager the userland provides, and remove it afterwards only if + # it was not already part of the base image + HAD_CURL=1; \ + command -v curl >/dev/null 2>&1 || HAD_CURL=0; \ + if command -v apk >/dev/null 2>&1; then \ + PKGMGR=apk; \ + apk add --no-cache curl ca-certificates; \ + elif command -v apt-get >/dev/null 2>&1; then \ + PKGMGR=apt; \ + apt-get update; \ + apt-get install -y --no-install-recommends curl ca-certificates; \ + else \ + echo "No supported package manager (apk/apt-get) found"; exit 1; \ + fi; \ + URL="https://github.com/sqliteai/sqlite-sync/releases/download/${CLOUDSYNC_VERSION}/${ASSET}"; \ + echo "Downloading ${URL}"; \ + curl -fSL "${URL}" -o /tmp/cloudsync.tar.gz; \ + if [ "$HAD_CURL" = 0 ]; then \ + if [ "$PKGMGR" = apk ]; then apk del curl; else apt-get purge -y curl; apt-get autoremove -y; fi; \ + fi; \ + if [ "$PKGMGR" = apt ]; then rm -rf /var/lib/apt/lists/*; fi; \ + fi; \ + mkdir -p /tmp/cloudsync; \ + tar -xzf /tmp/cloudsync.tar.gz -C /tmp/cloudsync; \ + # Resolve Supabase's Nix library path (postgres wrapper overrides libdir) + PKGLIBDIR="$(${PG_CONFIG} --pkglibdir)"; \ + NIX_PGLIBDIR="$(grep -E '^export NIX_PGLIBDIR' /usr/bin/postgres | sed -E "s/.*'([^']+)'.*/\1/" || true)"; \ + if [ -n "$NIX_PGLIBDIR" ]; then PKGLIBDIR="$NIX_PGLIBDIR"; fi; \ + SHAREDIR_PGCONFIG="$(${PG_CONFIG} --sharedir)"; \ + SHAREDIR_STD="/usr/share/postgresql"; \ + install -d "$PKGLIBDIR" "$SHAREDIR_PGCONFIG/extension"; \ + install -m 755 /tmp/cloudsync/cloudsync.so "$PKGLIBDIR/"; \ + install -m 644 /tmp/cloudsync/cloudsync--*.sql /tmp/cloudsync/cloudsync.control "$SHAREDIR_PGCONFIG/extension/"; \ + if [ "$SHAREDIR_STD" != "$SHAREDIR_PGCONFIG" ] && [ -d "$SHAREDIR_STD" ]; then \ + install -d "$SHAREDIR_STD/extension"; \ + install -m 644 /tmp/cloudsync/cloudsync--*.sql /tmp/cloudsync/cloudsync.control "$SHAREDIR_STD/extension/"; \ + fi; \ + rm -rf /tmp/cloudsync /tmp/cloudsync.tar.gz /tmp/beta-artifacts + +# Verify installation +RUN set -eu; \ + PG_CONFIG="${CLOUDSYNC_PG_CONFIG:-}"; \ + if [ -z "$PG_CONFIG" ]; then \ + for c in /root/.nix-profile/bin/pg_config /nix/var/nix/profiles/default/bin/pg_config "$(command -v pg_config || true)"; do \ + if [ -x "$c" ]; then PG_CONFIG="$c"; break; fi; \ + done; \ + fi; \ + NIX_PGLIBDIR="$(grep -E '^export NIX_PGLIBDIR' /usr/bin/postgres | sed -E "s/.*'([^']+)'.*/\1/" || true)"; \ + echo "Verifying CloudSync extension installation..."; \ + if [ -n "$NIX_PGLIBDIR" ]; then \ + ls -la "$NIX_PGLIBDIR/cloudsync.so"; \ + else \ + ls -la "$(${PG_CONFIG} --pkglibdir)/cloudsync.so"; \ + fi; \ + ls -la "$(${PG_CONFIG} --sharedir)/extension/cloudsync"*; \ + if [ -d "/usr/share/postgresql/extension" ]; then \ + ls -la /usr/share/postgresql/extension/cloudsync*; \ + fi; \ + echo "CloudSync extension installed successfully" + +EXPOSE 5432 + +WORKDIR / + +LABEL org.sqliteai.cloudsync.description="Supabase PostgreSQL with CloudSync CRDT extension" \ + org.opencontainers.image.source="https://github.com/sqliteai/sqlite-sync" diff --git a/docker/postgresql/beta-artifacts/.gitkeep b/docker/postgresql/beta-artifacts/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docker/postgresql/cloudsync.control.in b/docker/postgresql/cloudsync.control.in new file mode 100644 index 00000000..704af371 --- /dev/null +++ b/docker/postgresql/cloudsync.control.in @@ -0,0 +1,13 @@ +# CloudSync PostgreSQL Extension Control File +# +# Generated from cloudsync.control.in by docker/Makefile.postgresql. +# Do not edit the generated file; edit the .in template instead. +# The version below is read from CLOUDSYNC_VERSION in src/cloudsync.h. + +comment = 'CloudSync - CRDT-based multi-master database synchronization' +default_version = '@EXTVERSION@' +relocatable = true +requires = '' +superuser = false +module_pathname = '$libdir/cloudsync' +trusted = true diff --git a/docker/postgresql/docker-compose.asan.yml b/docker/postgresql/docker-compose.asan.yml new file mode 100644 index 00000000..b4f2f84d --- /dev/null +++ b/docker/postgresql/docker-compose.asan.yml @@ -0,0 +1,8 @@ +services: + postgres: + build: + args: + ENABLE_ASAN: "1" + environment: + LD_PRELOAD: /usr/lib/aarch64-linux-gnu/libasan.so.8 + ASAN_OPTIONS: detect_leaks=0,abort_on_error=1,allocator_may_return_null=1 diff --git a/docker/postgresql/docker-compose.debug.yml b/docker/postgresql/docker-compose.debug.yml new file mode 100644 index 00000000..d4456704 --- /dev/null +++ b/docker/postgresql/docker-compose.debug.yml @@ -0,0 +1,58 @@ +services: + postgres: + build: + context: ../.. + dockerfile: docker/postgresql/Dockerfile.debug-no-optimization + container_name: cloudsync-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: cloudsync_test + ports: + - "5432:5432" + ulimits: + core: -1 + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined + volumes: + # Mount source code for development (allows quick rebuilds) + - ../../src:/tmp/cloudsync/src:ro + - ../../docker:/tmp/cloudsync/docker:ro + - ../../Makefile:/tmp/cloudsync/Makefile:ro + - ../../.vscode:/tmp/cloudsync/.vscode:ro + # Persist database data + - postgres_data:/var/lib/postgresql/data + # Mount init script + - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + # Optional: pgAdmin for database management + pgadmin: + image: dpage/pgadmin4:latest + container_name: cloudsync-pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: admin@cloudsync.local + PGADMIN_DEFAULT_PASSWORD: admin + PGADMIN_CONFIG_SERVER_MODE: 'False' + ports: + - "5050:80" + volumes: + - pgadmin_data:/var/lib/pgadmin + depends_on: + - postgres + profiles: + - admin + +volumes: + postgres_data: + pgadmin_data: + +networks: + default: + name: cloudsync-network diff --git a/docker/postgresql/docker-compose.yml b/docker/postgresql/docker-compose.yml new file mode 100644 index 00000000..5a3257de --- /dev/null +++ b/docker/postgresql/docker-compose.yml @@ -0,0 +1,53 @@ +services: + postgres: + build: + context: ../.. + dockerfile: docker/postgresql/Dockerfile + args: + POSTGRES_TAG: ${POSTGRES_TAG:-17} + container_name: cloudsync-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: cloudsync_test + ports: + - "5432:5432" + volumes: + # Mount source code for development (allows quick rebuilds) + - ../../src:/tmp/cloudsync/src:ro + - ../../docker:/tmp/cloudsync/docker:ro + - ../../Makefile:/tmp/cloudsync/Makefile:ro + # Persist database data + - postgres_data:/var/lib/postgresql/data + # Mount init script + - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + # Optional: pgAdmin for database management + pgadmin: + image: dpage/pgadmin4:latest + container_name: cloudsync-pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: admin@cloudsync.local + PGADMIN_DEFAULT_PASSWORD: admin + PGADMIN_CONFIG_SERVER_MODE: 'False' + ports: + - "5050:80" + volumes: + - pgadmin_data:/var/lib/pgadmin + depends_on: + - postgres + profiles: + - admin + +volumes: + postgres_data: + pgadmin_data: + +networks: + default: + name: cloudsync-network diff --git a/docker/postgresql/init.sql b/docker/postgresql/init.sql new file mode 100644 index 00000000..b8923718 --- /dev/null +++ b/docker/postgresql/init.sql @@ -0,0 +1,10 @@ +-- CloudSync PostgreSQL Initialization Script +-- This script loads the CloudSync extension during database init + +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Log initialization +DO $$ +BEGIN + RAISE NOTICE 'CloudSync tables initialized successfully'; +END $$; diff --git a/docs/block-lww.md b/docs/block-lww.md new file mode 100644 index 00000000..6765b9e4 --- /dev/null +++ b/docs/block-lww.md @@ -0,0 +1,124 @@ +# Block-Level LWW + +Standard CRDT sync resolves conflicts at the **cell level**: if two devices edit the same column of the same row, one value wins entirely. This works for short values like names or statuses, but for longer text content (markdown documents, notes, agent memory files) the entire text is replaced even if edits were in different parts. + +**Block-Level LWW** (Last-Writer-Wins) solves this by splitting text columns into **blocks** (lines by default) and tracking each block independently. When two devices edit different lines of the same text, **both edits are preserved** after sync. Only when two devices edit the *same* line does LWW apply. + +This feature was specifically designed to keep **markdown files** in sync across devices and AI agents. Agents that independently edit different sections of a shared document (adding notes, updating status, appending logs) can do so without overwriting each other's work. + +## How It Works + +1. **Enable block tracking** on a text column using `cloudsync_set_column()`. +2. On INSERT or UPDATE, text is automatically split into blocks using a delimiter (default: newline `\n`). +3. Each block gets a unique fractional index position, enabling insertions between blocks without reindexing. +4. During sync, changes are merged block-by-block rather than replacing the whole cell. +5. The base column always contains the current full text, your queries work unchanged. + +## Setup + +```sql +-- Create a table with a text column for long-form content +CREATE TABLE notes ( + id TEXT PRIMARY KEY NOT NULL, + title TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '' +); + +-- Initialize sync on the table +SELECT cloudsync_init('notes'); + +-- Enable block-level LWW on the "body" column +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block'); +``` + +## Example: Two-Device Merge + +```sql +-- Device A: create a note +INSERT INTO notes (id, title, body) VALUES ( + 'note-001', + 'Meeting Notes', + 'Line 1: Welcome +Line 2: Agenda +Line 3: Action items' +); + +-- Sync Device A -> Cloud -> Device B +-- Both devices now have the same 3-line note. +``` + +```sql +-- Device A (offline): edit line 1 +UPDATE notes SET body = 'Line 1: Welcome everyone +Line 2: Agenda +Line 3: Action items' WHERE id = 'note-001'; + +-- Device B (offline): edit line 3 +UPDATE notes SET body = 'Line 1: Welcome +Line 2: Agenda +Line 3: Action items - DONE' WHERE id = 'note-001'; +``` + +```sql +-- After both devices sync, the merged result: +-- 'Line 1: Welcome everyone +-- Line 2: Agenda +-- Line 3: Action items - DONE' +-- +-- Both edits are preserved because they affected different lines. +``` + +## Custom Delimiter + +For paragraph-level tracking (useful for long-form markdown documents), set a custom delimiter: + +```sql +-- Use double newline as delimiter (paragraph separator) +SELECT cloudsync_set_column('notes', 'body', 'delimiter', ' + +'); +``` + +## Materializing Text + +After a merge, the `body` column is updated automatically. You can also manually trigger materialization: + +```sql +-- Reconstruct body from blocks for a specific row +SELECT cloudsync_text_materialize('notes', 'body', 'note-001'); + +-- With a composite primary key (e.g., PRIMARY KEY (tenant_id, doc_id)) +SELECT cloudsync_text_materialize('docs', 'body', 'tenant-1', 'doc-001'); + +-- Then read normally +SELECT body FROM notes WHERE id = 'note-001'; +``` + +## Mixed Columns + +Block-level LWW can be enabled on specific columns while other columns use standard cell-level LWW: + +```sql +CREATE TABLE docs ( + id TEXT PRIMARY KEY NOT NULL, + title TEXT NOT NULL DEFAULT '', -- standard LWW (cell-level) + body TEXT NOT NULL DEFAULT '', -- block LWW (line-level) + status TEXT NOT NULL DEFAULT '' -- standard LWW (cell-level) +); + +SELECT cloudsync_init('docs'); +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block'); + +-- Concurrent edits to "title" or "status" use normal LWW. +-- Concurrent edits to "body" merge at the line level. +``` + +## Key Properties + +- **Non-conflicting edits are preserved**: Two users editing different lines both see their changes after sync. +- **Same-line conflicts use LWW**: If two users edit the same line, the last writer wins. +- **Custom delimiters**: Use paragraph separators (`\n\n`), sentence boundaries, or any string. +- **Mixed columns**: A table can have both regular and block-level LWW columns. +- **Transparent reads**: The base column always contains the current full text. + +For API details, see the [API Reference](../API.md). diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 00000000..4fff96ff --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,145 @@ +# Installation + +Download the appropriate pre-built binary for your platform from the official [Releases](https://github.com/sqliteai/sqlite-sync/releases) page: + +- Linux: x86_64 and arm64 (glibc and musl) +- macOS: x86_64 and arm64 +- Windows: x86_64 +- Android +- iOS + +## SQLite CLI / C + +```sql +-- In SQLite CLI +.load ./cloudsync + +-- In SQL +SELECT load_extension('./cloudsync'); +``` + +## Swift Package + +[Add this repository as a package dependency to your Swift project](https://developer.apple.com/documentation/xcode/adding-package-dependencies-to-your-app#Add-a-package-dependency). After adding the package, set up SQLite with extension loading by following steps 4 and 5 of [this guide](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/platforms/ios.md#4-set-up-sqlite-with-extension-loading). + +```swift +import CloudSync + +var db: OpaquePointer? +sqlite3_open(":memory:", &db) +sqlite3_enable_load_extension(db, 1) +var errMsg: UnsafeMutablePointer? = nil +sqlite3_load_extension(db, CloudSync.path, nil, &errMsg) +var stmt: OpaquePointer? +sqlite3_prepare_v2(db, "SELECT cloudsync_version()", -1, &stmt, nil) +defer { sqlite3_finalize(stmt) } +sqlite3_step(stmt) +print("cloudsync_version(): \(String(cString: sqlite3_column_text(stmt, 0)))") +sqlite3_close(db) +``` + +## Android + +Add the [following](https://central.sonatype.com/artifact/ai.sqlite/sync) to your Gradle dependencies: + +```gradle +implementation 'ai.sqlite:sync:' +``` + +```java +SQLiteCustomExtension cloudsyncExtension = new SQLiteCustomExtension( + getApplicationInfo().nativeLibraryDir + "/cloudsync", null); +SQLiteDatabaseConfiguration config = new SQLiteDatabaseConfiguration( + getCacheDir().getPath() + "/cloudsync_test.db", + SQLiteDatabase.CREATE_IF_NECESSARY | SQLiteDatabase.OPEN_READWRITE, + Collections.emptyList(), + Collections.emptyList(), + Collections.singletonList(cloudsyncExtension) +); +SQLiteDatabase db = SQLiteDatabase.openDatabase(config, null, null); +``` + +For full implementation details, see the [complete Android example](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/examples/android/README.md). + +## Expo + +Install the Expo package: + +```bash +npm install @sqliteai/sqlite-sync-expo +``` + +Add to your `app.json`: + +```json +{ + "expo": { + "plugins": ["@sqliteai/sqlite-sync-expo"] + } +} +``` + +Run prebuild: + +```bash +npx expo prebuild --clean +``` + +Load the extension: + +```typescript +import { Platform } from 'react-native'; +import { getDylibPath, open } from '@op-engineering/op-sqlite'; + +const db = open({ name: 'mydb.db' }); + +// Load SQLite Sync extension +if (Platform.OS === 'ios') { + const path = getDylibPath('ai.sqlite.cloudsync', 'CloudSync'); + db.loadExtension(path); +} else { + db.loadExtension('cloudsync'); +} +``` + +## React Native + +Install the React Native library: + +```bash +npm install @sqliteai/sqlite-sync-react-native +``` + +Then follow the instructions from the [README](https://www.npmjs.com/package/@sqliteai/sqlite-sync-react-native). + +## Flutter + +Add the [sqlite_sync](https://pub.dev/packages/sqlite_sync) package to your project: + +```bash +flutter pub add sqlite_sync # Flutter projects +dart pub add sqlite_sync # Dart projects +``` + +Requires Dart 3.10+ / Flutter 3.38+. + +```dart +import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite_sync/sqlite_sync.dart'; + +sqlite3.loadSqliteSyncExtension(); +final db = sqlite3.openInMemory(); +print(db.select('SELECT cloudsync_version()')); +``` + +For a complete example, see the [Flutter example](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/examples/flutter/README.md). + +## WASM + +The WebAssembly version of SQLite with the SQLite Sync extension is available on npm: + +```bash +npm install @sqliteai/sqlite-wasm +``` + +See the [npm package](https://www.npmjs.com/package/@sqliteai/sqlite-wasm) for usage details. diff --git a/docs/internal/grafana-dashboard.json b/docs/internal/grafana-dashboard.json new file mode 100644 index 00000000..ace23267 --- /dev/null +++ b/docs/internal/grafana-dashboard.json @@ -0,0 +1,590 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "dfaqrid9yhvk0b" + }, + "description": "N. of distinct devices which made at least one request", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "dataset": "postgres", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__unixEpochGroup(created_at, '1h') AS time,\n COUNT(DISTINCT project_id || '/' || database || '/' || site_id) AS devices\nFROM cloudsync_metrics\nWHERE $__unixEpochFilter(created_at)\n AND ('${ProjectID}' = '' OR project_id ~ '${ProjectID}')\n AND ('${SiteID}' = '' OR site_id ~ '${SiteID}')\nGROUP BY 1\nORDER BY 1;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Activity (distinct devices)", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "dfaqrid9yhvk0b" + }, + "description": "Total number of requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "dataset": "postgres", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "dfaqrid9yhvk0b" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__unixEpochGroup(created_at, '1h') AS time,\n COUNT(*) AS requests\nFROM cloudsync_metrics\nWHERE $__unixEpochFilter(created_at)\n AND ('${ProjectID}' = '' OR project_id ~ '${ProjectID}')\n AND ('${SiteID}' = '' OR site_id ~ '${SiteID}')\nGROUP BY 1\nORDER BY 1;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "dfaqrid9yhvk0b" + }, + "description": "Total amount of traffic in MB", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "dataset": "postgres", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n $__unixEpochGroup(created_at, '1h') AS time,\n SUM(bytes_in + bytes_out) / 1024 / 1024 AS megabytes\nFROM cloudsync_metrics\nWHERE $__unixEpochFilter(created_at)\n AND ('${ProjectID}' = '' OR project_id ~ '${ProjectID}')\n AND ('${SiteID}' = '' OR site_id ~ '${SiteID}')\nGROUP BY 1\nORDER BY 1;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic MB (In+Out)", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "dfaqrid9yhvk0b" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "megabytes apply" + }, + "properties": [ + { + "id": "displayName", + "value": "Out (MB)" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "megabytes check" + }, + "properties": [ + { + "id": "displayName", + "value": "In (MB)" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "sum", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "dataset": "postgres", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__unixEpochGroup(created_at, '1h') AS time,\n action,\n SUM(bytes_in + bytes_out) / 1024 / 1024 AS megabytes\nFROM cloudsync_metrics\nWHERE $__unixEpochFilter(created_at)\n AND action ~ '${action:regex}'\n AND ('${ProjectID}' = '' OR project_id ~ '${ProjectID}')\n AND ('${SiteID}' = '' OR site_id ~ '${SiteID}')\nGROUP BY 1, 2\nORDER BY 1;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic MB", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "text": [ + "check", + "apply" + ], + "value": [ + "check", + "apply" + ] + }, + "definition": "SELECT DISTINCT action\nFROM cloudsync_metrics\nORDER BY action;", + "description": "", + "includeAll": false, + "label": "Action", + "multi": true, + "name": "action", + "options": [], + "query": "SELECT DISTINCT action\nFROM cloudsync_metrics\nORDER BY action;", + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "", + "value": "" + }, + "label": "ProjectID", + "name": "ProjectID", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "type": "textbox" + }, + { + "current": { + "text": "", + "value": "" + }, + "name": "SiteID", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "type": "textbox" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "CloudSync", + "uid": "advqx5s", + "version": 40 +} \ No newline at end of file diff --git a/docs/Network.md b/docs/internal/network.md similarity index 84% rename from docs/Network.md rename to docs/internal/network.md index 7120231a..9af6e15d 100644 --- a/docs/Network.md +++ b/docs/internal/network.md @@ -34,14 +34,6 @@ This is useful when: You must provide implementations for the following C functions: - ```c - bool network_compute_endpoints (sqlite3_context *context, network_data *data, const char *conn_string); - - // Parses `conn_string` and fills the `network_data` structure with connection information (e.g. base URL, endpoints, credentials). - // Returns `true` on success, `false` on error (you can use `sqlite3_result_error` to report errors to SQLite). - - ``` - ```c bool network_send_buffer (network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size); @@ -50,11 +42,12 @@ This is useful when: ``` ```c - NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header); + NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char **extra_headers, int nextra_headers); // Performs a network request (GET or POST depending on `is_post_request`) to the specified `endpoint`, using the given `authentication` token or header. // If `json_payload` is provided, it will be sent as the POST body (for `is_post_request == true`). // If `zero_terminated == true`, ensure that the returned buffer is null-terminated. + // `extra_headers` is an array of `nextra_headers` request-header lines (each formatted as `"Name: value"`) appended to the standard headers (`Authorization`, `X-CloudSync-Org`, `Content-Type`). Pass `NULL, 0` for none. Used by call sites to send `X-CloudSync-Version` on every cloudsync API call and `X-CloudSync-Capabilities: check-status-response` on calls to the `/check` endpoint. // Returns a `NETWORK_RESULT` enum value indicating success, error, or timeout. ``` diff --git a/docs/internal/postgres-flyio.md b/docs/internal/postgres-flyio.md new file mode 100644 index 00000000..c9e085c8 --- /dev/null +++ b/docs/internal/postgres-flyio.md @@ -0,0 +1,974 @@ +# Self-Hosting PostgreSQL on Fly.io with CloudSync Extension + +This guide deploys a standalone PostgreSQL instance with CloudSync on Fly.io, plus a minimal JWT auth server for token generation. No Supabase required. + +By the end you will have: + +- A Fly.io VM running PostgreSQL with the CloudSync CRDT extension +- A JWT auth server (Node.js) — choose between: + - **HS256 (shared secret)** — simplest setup, signs tokens with a base64-encoded secret + - **RS256 (JWKS)** — production-ready, signs with a private key and exposes a public JWKS endpoint +- A custom Postgres image published to Docker Hub + +## Prerequisites + +| Tool | Purpose | Install | +|------|---------|---------| +| [Docker Desktop](https://www.docker.com/products/docker-desktop/) | Build the custom Postgres image | `brew install --cask docker` | +| [Fly CLI (`flyctl`)](https://fly.io/docs/flyctl/install/) | Provision and manage Fly.io machines | `brew install flyctl` | +| [Git](https://git-scm.com/) | Clone repositories | `brew install git` | +| [Docker Hub](https://hub.docker.com/) account | Host your custom Postgres image | Free signup | + +You also need a [Fly.io account](https://fly.io/app/sign-up). + +### Fly.io VM requirements + +Since this is just PostgreSQL + a small auth server (not a full Supabase stack), resource requirements are much lower: + +| Resource | Minimum | Recommended | +|----------|---------|-------------| +| RAM | 1 GB | 2 GB | +| CPU | 1 core | 2 cores | +| Disk | 4 GB SSD | 10 GB+ | + +--- + +## Step 1: Initialize git submodules + +```bash +cd /path/to/sqlite-sync-dev +git submodule update --init --recursive +``` + +Without this, the build fails with `fractional_indexing.h: No such file or directory`. + +--- + +## Step 2: Build the custom Postgres image + +From the sqlite-sync-dev repo root: + +```bash +make postgres-docker-build +``` + +This builds `postgres:17` with CloudSync pre-installed using `docker/postgresql/Dockerfile`. + +Verify: + +```bash +docker images | grep sqlite-sync-pg +``` + +--- + +## Step 3: Build for amd64 and push to Docker Hub + +If you're on Apple Silicon, you must cross-build for Fly.io's x86 VMs: + +```bash +docker build --platform linux/amd64 \ + -f docker/postgresql/Dockerfile \ + -t /postgres-cloudsync:17 \ + . + +docker push /postgres-cloudsync:17 +``` + +> On Intel Mac or Linux x86, the default build is already amd64. + +--- + +## Step 4: Provision a Fly.io VM + +### 4a. Log in and create the app + +```bash +fly auth login +fly apps create +``` + +### 4b. Create a persistent volume + +```bash +fly volumes create pg_data --app --region --size 4 +``` + +### 4c. Create a Fly Machine + +```bash +fly machine run ubuntu:24.04 \ + --app \ + --region \ + --vm-size shared-cpu-2x \ + --vm-memory 2048 \ + --volume pg_data:/data \ + --name postgres-vm \ + -- sleep inf +``` + +### 4d. Allocate a public IP + +```bash +fly ips allocate-v4 --shared --app +fly ips allocate-v6 --app +``` + +--- + +## Step 5: Set up Docker on the VM + +### 5a. SSH into the machine + +```bash +fly ssh console --app +``` + +### 5b. Install Docker Engine + +```bash +apt-get update +apt-get install -y ca-certificates curl gnupg + +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +chmod a+r /etc/apt/keyrings/docker.gpg + +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + +apt-get update +apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin +``` + +### 5c. Configure Docker storage driver + +```bash +apt-get install -y fuse-overlayfs +mkdir -p /etc/docker +echo '{"storage-driver":"fuse-overlayfs","data-root":"/data/docker"}' > /etc/docker/daemon.json +``` + +### 5d. Start Docker + +```bash +dockerd & +# Wait for "API listen on /var/run/docker.sock" +``` + +--- + +## Step 6: Create the Docker Compose stack + +Create the project directory: + +```bash +mkdir -p /data/cloudsync-postgres +cd /data/cloudsync-postgres +``` + +### 6a. Generate a JWT secret + +```bash +JWT_SECRET=$(openssl rand -base64 32) +echo "JWT_SECRET=$JWT_SECRET" > .env +echo "POSTGRES_PASSWORD=$(openssl rand -base64 16)" >> .env +echo "Your JWT secret: $JWT_SECRET" +echo "Save this secret — you'll need it for CloudSync server configuration." +``` + +### 6b. Create docker-compose.yml + +```bash +cat > docker-compose.yml << 'EOF' +services: + db: + image: /postgres-cloudsync:17 + container_name: cloudsync-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: postgres + ports: + - "5432:5432" + volumes: + - pg_data:/var/lib/postgresql/data + - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + auth: + image: node:22-alpine + container_name: cloudsync-auth + environment: + JWT_SECRET: ${JWT_SECRET} + PORT: 3001 + ports: + - "3001:3001" + volumes: + - ./auth-server:/app + working_dir: /app + command: ["node", "server.js"] + restart: unless-stopped + +volumes: + pg_data: +EOF +``` + +### 6c. Create the Postgres init script + +```bash +cat > init.sql << 'EOF' +CREATE EXTENSION IF NOT EXISTS cloudsync; +EOF +``` + +### 6d. Create the JWT auth server + +```bash +mkdir -p auth-server +``` + +Create the package file: + +```bash +cat > auth-server/package.json << 'EOF' +{ + "name": "cloudsync-auth", + "version": "1.0.0", + "private": true, + "dependencies": { + "jsonwebtoken": "^9.0.0" + } +} +EOF +``` + +Create the auth server: + +```bash +cat > auth-server/server.js << 'AUTHEOF' +const http = require("http"); +const jwt = require("jsonwebtoken"); + +const PORT = process.env.PORT || 3001; +const JWT_SECRET = process.env.JWT_SECRET; + +if (!JWT_SECRET) { + console.error("JWT_SECRET environment variable is required"); + process.exit(1); +} + +function parseBody(req) { + return new Promise((resolve, reject) => { + let data = ""; + req.on("data", (chunk) => (data += chunk)); + req.on("end", () => { + try { resolve(JSON.parse(data)); } + catch { reject(new Error("Invalid JSON")); } + }); + req.on("error", reject); + }); +} + +function respond(res, status, body) { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(JSON.stringify(body)); +} + +const server = http.createServer(async (req, res) => { + // Health check + if (req.method === "GET" && req.url === "/healthz") { + return respond(res, 200, { status: "ok" }); + } + + // Generate token + // POST /token { "sub": "user-id", "role": "rls_role", "expiresIn": "24h" } + if (req.method === "POST" && req.url === "/token") { + try { + const body = await parseBody(req); + const sub = body.sub || "anonymous"; + const role = body.role; + const expiresIn = body.expiresIn || "24h"; + const claims = body.claims || {}; + + if (!role) { + return respond(res, 400, { error: "role is required" }); + } + + const token = jwt.sign( + { sub, role, ...claims }, + JWT_SECRET, + { expiresIn, algorithm: "HS256" } + ); + + return respond(res, 200, { token, expiresIn }); + } catch (err) { + return respond(res, 400, { error: err.message }); + } + } + + respond(res, 404, { error: "Not found" }); +}); + +server.listen(PORT, () => { + console.log("Auth server listening on port " + PORT); +}); +AUTHEOF +``` + +Install dependencies: + +```bash +docker run --rm -v $(pwd)/auth-server:/app -w /app node:22-alpine npm install +``` + +### 6e. (Optional) Create the JWKS auth server + +If you need RS256/JWKS-based authentication instead of (or in addition to) the shared secret approach, create a second auth server that generates an RSA key pair on startup and exposes a JWKS endpoint. + +```bash +mkdir -p auth-server-jwks + +cat > auth-server-jwks/package.json << 'EOF' +{ + "name": "cloudsync-auth-jwks", + "version": "1.0.0", + "private": true, + "dependencies": { + "jsonwebtoken": "^9.0.0", + "jose": "^5.0.0" + } +} +EOF + +cat > auth-server-jwks/server.js << 'EOF' +const http = require("http"); +const jwt = require("jsonwebtoken"); +const crypto = require("crypto"); +const { exportJWK } = require("jose"); + +const PORT = process.env.PORT || 3002; +const ISSUER = process.env.ISSUER || "cloudsync-auth-jwks"; +const KID = "cloudsync-key-1"; + +let privateKey, publicKey, jwksResponse; + +async function init() { + const pair = crypto.generateKeyPairSync("rsa", { + modulusLength: 2048, + publicKeyEncoding: { type: "spki", format: "pem" }, + privateKeyEncoding: { type: "pkcs8", format: "pem" }, + }); + privateKey = pair.privateKey; + publicKey = pair.publicKey; + + const publicKeyObject = crypto.createPublicKey(publicKey); + const jwk = await exportJWK(publicKeyObject); + jwk.kid = KID; + jwk.alg = "RS256"; + jwk.use = "sig"; + jwksResponse = JSON.stringify({ keys: [jwk] }); + + console.log("RSA key pair generated (kid: " + KID + ")"); +} + +function parseBody(req) { + return new Promise((resolve, reject) => { + let data = ""; + req.on("data", (chunk) => (data += chunk)); + req.on("end", () => { + try { resolve(JSON.parse(data)); } + catch { reject(new Error("Invalid JSON")); } + }); + req.on("error", reject); + }); +} + +function respond(res, status, body) { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(typeof body === "string" ? body : JSON.stringify(body)); +} + +const server = http.createServer(async (req, res) => { + if (req.method === "GET" && req.url === "/healthz") { + return respond(res, 200, { status: "ok" }); + } + + // JWKS endpoint — CloudSync server fetches this to verify tokens + if (req.method === "GET" && req.url === "/.well-known/jwks.json") { + res.writeHead(200, { "Content-Type": "application/json" }); + return res.end(jwksResponse); + } + + // POST /token { "sub": "user-id", "role": "rls_role", "expiresIn": "24h" } + if (req.method === "POST" && req.url === "/token") { + try { + const body = await parseBody(req); + const sub = body.sub || "anonymous"; + const role = body.role; + const expiresIn = body.expiresIn || "24h"; + const claims = body.claims || {}; + + if (!role) { + return respond(res, 400, { error: "role is required" }); + } + + const token = jwt.sign( + { sub, role, iss: ISSUER, ...claims }, + privateKey, + { expiresIn, algorithm: "RS256", keyid: KID } + ); + + return respond(res, 200, { token, expiresIn }); + } catch (err) { + return respond(res, 400, { error: err.message }); + } + } + + respond(res, 404, { error: "Not found" }); +}); + +init().then(() => { + server.listen(PORT, () => { + console.log("JWKS Auth server listening on port " + PORT); + }); +}); +EOF + +docker run --rm -v $(pwd)/auth-server-jwks:/app -w /app node:22-alpine npm install +``` + +Add the JWKS auth service to `docker-compose.yml`: + +```yaml + auth-jwks: + image: node:22-alpine + container_name: cloudsync-auth-jwks + environment: + PORT: 3002 + ISSUER: cloudsync-auth-jwks + ports: + - "3002:3002" + volumes: + - ./auth-server-jwks:/app + working_dir: /app + command: ["node", "server.js"] + restart: unless-stopped +``` + +> **Note:** The JWKS server generates a new RSA key pair each time it starts. For production, persist the key pair to a volume so tokens remain valid across restarts. + +--- + +## Step 7: Start the stack + +```bash +cd /data/cloudsync-postgres +docker compose up -d +``` + +Verify: + +```bash +docker compose ps + +# Test Postgres +docker compose exec db psql -U postgres -c "SELECT cloudsync_version();" + +# Test HS256 auth server +curl http://localhost:3001/healthz + +# Test JWKS auth server (if enabled) +curl http://localhost:3002/healthz +curl http://localhost:3002/.well-known/jwks.json +``` + +--- + +## Step 8: Generate a JWT token + +Before generating JWTs for PostgreSQL, create the database role referenced by the token's `role` claim and grant it the permissions CloudSync needs. + +### 8a. Create and grant the JWT role + +Create the role: + +```bash +cd /data/cloudsync-postgres +docker compose exec db psql -U postgres -d postgres -c "CREATE ROLE rls_role NOLOGIN;" +``` + +Grant schema and table permissions on current and future tables: + +```bash +cd /data/cloudsync-postgres +docker compose exec db psql -U postgres -d test_database_1 -c "GRANT USAGE ON SCHEMA public TO rls_role;" +docker compose exec db psql -U postgres -d test_database_1 -c "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO rls_role;" +docker compose exec db psql -U postgres -d test_database_1 -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO rls_role;" +docker compose exec db psql -U postgres -d test_database_1 -c "GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO rls_role;" +docker compose exec db psql -U postgres -d test_database_1 -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO rls_role;" +``` + +Allow the connection-string user to switch into that role: + +```bash +cd /data/cloudsync-postgres +docker compose exec db psql -U postgres -d postgres -c "GRANT rls_role TO postgres;" +``` + +Verify: + +```bash +cd /data/cloudsync-postgres +docker compose exec db psql -U postgres -d postgres -c "SELECT rolname, rolsuper, rolcanlogin, rolbypassrls FROM pg_roles WHERE rolname = 'rls_role';" +docker compose exec db psql -U postgres -d test_database_1 -c "\\ddp" +``` + +If you want to test the exact session shape CloudSync uses: + +```bash +cd /data/cloudsync-postgres +docker compose exec db psql -U postgres -d test_database_1 -c "BEGIN; SELECT set_config('request.jwt.claims', '{\"sub\":\"test-user-1\",\"role\":\"rls_role\"}', true); SET LOCAL ROLE rls_role; SELECT current_role, current_setting('request.jwt.claims', true); ROLLBACK;" +``` + +**HS256 (shared secret):** + +```bash +curl -X POST http://localhost:3001/token \ + -H "Content-Type: application/json" \ + -d '{"sub": "user-1", "role": "rls_role"}' +``` + +**RS256 (JWKS):** + +```bash +curl -X POST http://localhost:3002/token \ + -H "Content-Type: application/json" \ + -d '{"sub": "user-1", "role": "rls_role"}' +``` + +Response (both): + +```json +{"token":"eyJhbG...","expiresIn":"24h"} +``` + +--- + +## Step 9: Register with CloudSync server + +```bash +export CLOUDSYNC_URL="https://your-cloudsync-server.fly.dev" +export ORG_API_KEY="" + +# Get the Postgres password from .env +source /data/cloudsync-postgres/.env + +# Connection string (same Fly org — use .internal network) +export CONNECTION_STRING="postgres://postgres:$POSTGRES_PASSWORD@.internal:5432/postgres" + +# Or via fly proxy from local machine: +# fly proxy 5432:5432 -a +# export CONNECTION_STRING="postgres://postgres:$POSTGRES_PASSWORD@localhost:5432/postgres" +``` + +Register the database: + +```bash +curl --request POST "$CLOUDSYNC_URL/v1/databases" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{ + "label": "Fly.io Postgres", + "connectionString": "'"$CONNECTION_STRING"'", + "provider": "postgres", + "projectId": "cloudsync-postgres-flyio", + "databaseName": "postgres" + }' +``` + +Save the returned `managedDatabaseId`: + +```bash +export MANAGED_DATABASE_ID="" +``` + +--- + +## Step 10: Test CloudSync sync + +### 10a. Create and enable a test table + +```bash +# On the Fly VM +docker compose exec db psql -U postgres -c " +CREATE TABLE IF NOT EXISTS todos ( + id TEXT PRIMARY KEY DEFAULT cloudsync_uuid(), + title TEXT NOT NULL DEFAULT '', + done BOOLEAN DEFAULT false +); +SELECT cloudsync_init('todos'); +" + +# Enable sync via CloudSync API +curl --request POST "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/cloudsync/enable" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{"tables":["todos"]}' +``` + +### 10b. Generate a token and sync from SQLite + +```bash +# Get a JWT token from the auth server +TOKEN=$(curl -s -X POST http://localhost:3001/token \ + -H "Content-Type: application/json" \ + -d '{"sub": "user-1"}' | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])") +``` + +In a SQLite client: + +```sql +.load path/to/cloudsync + +CREATE TABLE todos ( + id TEXT PRIMARY KEY DEFAULT (cloudsync_uuid()), + title TEXT NOT NULL DEFAULT '', + done BOOLEAN DEFAULT false +); +SELECT cloudsync_init('todos'); + +SELECT cloudsync_network_init(''); +SELECT cloudsync_network_set_token(''); + +INSERT INTO todos (title) VALUES ('Test from SQLite'); +SELECT cloudsync_network_sync(500, 5); +``` + +Verify on Postgres: + +```bash +docker compose exec db psql -U postgres -c "SELECT * FROM todos;" +``` + +--- + +## Step 11: CloudSync server JWT configuration + +The CloudSync server needs to validate tokens from your auth server. Configuration depends on which auth method you chose. + +### Option A: HS256 (shared secret) + +In the CloudSync dashboard, go to your PostgreSQL project → **Configuration** → **Edit connection settings**: +- Under **JWT secret**, enter your `JWT_SECRET` value from `.env` +- Click **Save** + +Both the auth server and CloudSync must use the same raw secret string (not base64-decoded). + +### Option B: RS256 (JWKS) + +Configure the JWKS auth server and CloudSync to use asymmetric key verification. + +**1. Update docker-compose.yml - JWKS auth server ISSUER:** + +```yaml + auth-jwks: + environment: + ISSUER: http://.internal:3002 +``` + +The issuer is the **base URL** (CloudSync automatically appends `/.well-known/jwks.json`). + +**2. Configure CloudSync to accept this issuer:** + +In the CloudSync dashboard for this PostgreSQL project: +- Go to **Configuration** tab → **Edit connection settings** +- Under **JWT allowed issuers**, enter: + ``` + http://.internal:3002 + ``` + +CloudSync will: +1. Receive JWT tokens with `iss: http://.internal:3002` +2. Validate the issuer matches the allowed list +3. Fetch the public key from `http://.internal:3002/.well-known/jwks.json` +4. Verify the token signature + +This is how production auth systems (Auth0, Supabase, Firebase) work — no shared secrets needed. + +--- + +## Access your services + +| Service | URL | +|---------|-----| +| **PostgreSQL** | `postgres://postgres:@.internal:5432/postgres` | +| **Auth Server (HS256)** | `http://.internal:3001` | +| **Auth Server (JWKS)** | `http://.internal:3002` | +| **JWKS Endpoint** | `http://.internal:3002/.well-known/jwks.json` | + +From your local machine, use `fly proxy`: + +```bash +fly proxy 5432:5432 -a # Postgres +fly proxy 3001:3001 -a # Auth server (HS256) +fly proxy 3002:3002 -a # Auth server (JWKS) +``` + +--- + +## Reference: CloudSync Configuration + +After deployment, use these values to configure CloudSync dashboard: + +### Database Connection + +``` +postgresql://postgres:@.internal:5432/postgres +``` + +Replace: +- ``: from `.env` file +- ``: your Fly.io app name + +### JWT Secret (HS256) + +For simple/development setups using shared secrets: + +```env +JWT_SECRET= +``` + +Enter this in CloudSync dashboard → **Configuration** → **JWT secret** + +### JWT Issuer (RS256 with JWKS) + +For production setups using asymmetric keys: + +``` +http://.internal:3002 +``` + +Enter this in CloudSync dashboard → **Configuration** → **JWT allowed issuers** + +CloudSync will automatically fetch the public key from: +``` +http://.internal:3002/.well-known/jwks.json +``` + +--- + +## Maintenance + +### Startup script (survives VM restarts) + +Fly VM root filesystem resets on stop/start — only `/data` persists. Create a startup script: + +```bash +cat > /data/startup.sh << 'SCRIPT' +#!/bin/bash +set -e + +echo "=== Installing Docker ===" +apt-get update && apt-get install -y ca-certificates curl gnupg fuse-overlayfs + +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +chmod a+r /etc/apt/keyrings/docker.gpg + +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + +apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin + +echo "=== Configuring Docker ===" +mkdir -p /etc/docker +echo '{"storage-driver":"fuse-overlayfs","data-root":"/data/docker"}' > /etc/docker/daemon.json + +echo "=== Starting Docker ===" +dockerd > /data/dockerd.log 2>&1 & +until docker info > /dev/null 2>&1; do sleep 1; done +echo "Docker is ready!" + +echo "=== Starting CloudSync Postgres ===" +cd /data/cloudsync-postgres +docker compose up -d + +echo "=== Done! ===" +SCRIPT +chmod +x /data/startup.sh +``` + +After any VM restart: + +```bash +fly ssh console --app +/data/startup.sh +``` + +### Update CloudSync extension + +On your local machine: + +```bash +cd /path/to/sqlite-sync-dev +git pull && git submodule update --init --recursive +docker build --platform linux/amd64 \ + -f docker/postgresql/Dockerfile \ + -t /postgres-cloudsync:17 \ + . +docker push /postgres-cloudsync:17 +``` + +On the Fly VM: + +```bash +cd /data/cloudsync-postgres +docker compose pull db +docker compose up -d db +``` + +### Update CloudSync from a GitHub Actions PostgreSQL artifact + +If you already run the Dockerized PostgreSQL setup from this guide and want to test a newer CloudSync PostgreSQL build without rebuilding and pushing a custom image, you can copy the packaged extension files into the running `db` container. + +1. Confirm the running PostgreSQL major version and current CloudSync version: + +```bash +cd /data/cloudsync-postgres +docker compose exec db pg_config --version +docker compose exec db psql -U postgres -d postgres -c "SELECT cloudsync_version();" +docker compose exec db psql -U postgres -d postgres -c "SELECT name, default_version, installed_version FROM pg_available_extensions WHERE name = 'cloudsync';" +uname -m +``` + +For a standard Fly Ubuntu VM running PostgreSQL 17 on x86_64, download the artifact named: + +```text +cloudsync-postgresql17-linux-x86_64 +``` + +2. Download the artifact from the GitHub Actions run on your local machine while signed into GitHub, then upload the zip to the VM: + +```bash +fly ssh sftp shell --app +``` + +At the SFTP prompt: + +```text +cd /data +put /absolute/path/to/cloudsync-postgresql17-linux-x86_64.zip cloudsync-postgresql17-linux-x86_64.zip +``` + +If the upload fails because `cloudsync-postgresql17-linux-x86_64.zip` already exists on the VM, exit the SFTP prompt and remove it from a normal SSH shell first: + +```bash +fly ssh console --app +rm -f /data/cloudsync-postgresql17-linux-x86_64.zip +``` + +Then start `fly ssh sftp shell` again and rerun `put`. The SFTP prompt only supports a small command set such as `cd`, `ls`, `get`, `put`, and `chmod`; it does not support `rm`. + +3. SSH back into the VM, extract the package, and verify the files: + +```bash +fly ssh console --app + +apt-get update +apt-get install -y unzip + +cd /data +mkdir -p cloudsync-package +cd cloudsync-package +cp ../cloudsync-postgresql17-linux-x86_64.zip . +unzip -o cloudsync-postgresql17-linux-x86_64.zip +ls -1 +``` + +The extracted package should contain: + +- `cloudsync.so` +- `cloudsync.control` +- `cloudsync--.sql` +- any `cloudsync----.sql` upgrade scripts + +4. Copy those files into the running PostgreSQL container: + +```bash +docker cp cloudsync.so cloudsync-postgres:/tmp/ +docker cp cloudsync.control cloudsync-postgres:/tmp/ +for f in cloudsync--*.sql; do + docker cp "$f" cloudsync-postgres:/tmp/ +done +``` + +5. Install the files into PostgreSQL's extension directories inside the container and restart it: + +```bash +docker exec cloudsync-postgres sh -lc 'cp /tmp/cloudsync.so "$(pg_config --pkglibdir)/"' +docker exec cloudsync-postgres sh -lc 'cp /tmp/cloudsync.control /tmp/cloudsync--*.sql "$(pg_config --sharedir)/extension/"' +docker restart cloudsync-postgres +``` + +6. Verify the new binary version: + +```bash +docker exec cloudsync-postgres psql -U postgres -d postgres -c "SELECT cloudsync_version();" +docker exec cloudsync-postgres psql -U postgres -d postgres -c "SELECT name, default_version, installed_version FROM pg_available_extensions WHERE name = 'cloudsync';" +``` + +If `installed_version` is behind `default_version`, apply the PostgreSQL extension upgrade: + +```bash +docker exec cloudsync-postgres psql -U postgres -d postgres -c "ALTER EXTENSION cloudsync UPDATE;" +``` + +Then verify again: + +```bash +docker exec cloudsync-postgres psql -U postgres -d postgres -c "SELECT cloudsync_version();" +docker exec cloudsync-postgres psql -U postgres -d postgres -c "SELECT name, default_version, installed_version FROM pg_available_extensions WHERE name = 'cloudsync';" +``` + +For example, after a successful `1.0.16 -> 1.1.0` upgrade you should see: + +- `cloudsync_version()` returns `1.1.0` +- `default_version = 1.1` +- `installed_version = 1.1` + +> **Important:** this is an in-place container modification. It survives a container restart, but it does **not** survive recreating the container from the original image. If you later run `docker compose pull`, change the image tag, or recreate `db`, you will lose the manually copied files unless the image itself already includes that newer CloudSync build. + +### View logs + +```bash +docker compose logs -f # All services +docker compose logs -f db # Postgres only +docker compose logs -f auth # Auth server only +``` + +--- + +## Troubleshooting + +| Problem | Solution | +|---------|----------| +| `fractional_indexing.h: No such file or directory` | Run `git submodule update --init --recursive` before building | +| `cloudsync_version()` not found | Init scripts only run on first start. Run `CREATE EXTENSION IF NOT EXISTS cloudsync;` manually | +| `unzip: command not found` | Install it on the Fly VM first: `apt-get update && apt-get install -y unzip` | +| Auth server won't start | Check `docker compose logs auth`. Ensure `npm install` was run in `auth-server/` | +| Token verification fails (HS256) | Ensure `JWT_SECRET` matches exactly — CloudSync uses the raw string, not base64-decoded | +| Token verification fails (JWKS) | Ensure CloudSync can reach the JWKS endpoint and `JWT_ISSUER` matches the `ISSUER` env var | +| JWKS keys lost after restart | The JWKS server generates new keys on each start. For production, persist keys to a volume | +| Docker commands not found after VM restart | Run `/data/startup.sh` — Fly VM root filesystem resets on stop/start | +| `fuse-overlayfs` not working | Install it: `apt-get install -y fuse-overlayfs` | +| Manual artifact upgrade disappears after `docker compose up` recreation | This method only patches the running container. Rebuild or switch to an image that already contains the new CloudSync build for a durable upgrade. | +| Can't connect to Postgres from outside Fly | Use `fly proxy 5432:5432 -a ` | diff --git a/docs/PriKey.md b/docs/internal/pri-key.md similarity index 100% rename from docs/PriKey.md rename to docs/internal/pri-key.md diff --git a/docs/RowID.md b/docs/internal/row-id.md similarity index 100% rename from docs/RowID.md rename to docs/internal/row-id.md diff --git a/docs/internal/supabase-flyio-fly-deploy.md b/docs/internal/supabase-flyio-fly-deploy.md new file mode 100644 index 00000000..14ee49a3 --- /dev/null +++ b/docs/internal/supabase-flyio-fly-deploy.md @@ -0,0 +1,437 @@ +# Self-Hosted Supabase on Fly.io — `fly deploy` Strategy + +How to run a self-hosted Supabase stack with the CloudSync Postgres extension on +Fly.io. Everything lives in `deploy/supabase-flyio/` and is deployed with a single +`fly deploy`. No SSH, no `apt-get`, no editing files on the VM. + +Nothing in `deploy/supabase-flyio/` names a specific deployment: the app name, +region and org are command-line arguments, and the public URLs are derived from +them at boot. The examples below use these shell variables: + +```bash +FLY_APP= +FLY_REGION= +FLY_ORG= +``` + +Hardware for a reference deployment is `shared-cpu-4x` / 4 GB / 50 GB volume, set +in `fly.toml`. + +--- + +## Why not Fly's native Docker Compose support + +Fly can turn a compose file into containers on one Machine +(`[build.compose]` in `fly.toml`, +[docs](https://fly.io/docs/machines/guides-examples/multi-container-machines/)). +It cannot run the Supabase compose file. The converter is +`internal/containerconfig/compose.go` in `superfly/flyctl`, and it: + +| Limitation | Consequence for Supabase | +|---|---| +| Plain `yaml.Unmarshal`, no `${VAR}` interpolation and no `env_file` | Every service would receive the literal string `${POSTGRES_PASSWORD}` | +| `volumes:` entries are `os.ReadFile(hostPath)` → injected as base64 container files | File mounts (`kong.yml`, `*.sql`, `pooler.exs`) work; directory mounts (`volumes/db/data`, `volumes/storage`, `volumes/functions`) are dropped with a warning | +| Never emits `containers[].mounts` | No way to attach a Fly Volume to a container: PGDATA lands on the ephemeral overlay and is wiped on restart/redeploy | +| Named volumes ignored | `db-config:/etc/postgresql-custom` (pgsodium key) is dropped | +| One buildable service; secrets are machine-global; one container gets ingress | Manageable, but on top of the above | + +So we keep real Docker Compose and put dockerd inside the Machine instead. + +--- + +## How it works + +One Fly Machine runs a Docker-in-Docker image. The image contains dockerd, the +compose plugin and a pinned copy of the upstream Supabase `docker/` directory. +A Fly Volume at `/data` holds the Docker data root, PGDATA, storage and `.env`, +so state survives restarts and redeploys. + +``` +deploy/supabase-flyio/ +├── Dockerfile # docker:28-dind + fuse-overlayfs + pinned supabase/docker tree +├── entrypoint.sh # dockerd → compose tree → .env bootstrap → compose up → post-init +├── docker-compose.cloudsync.yml # our overrides, layered via COMPOSE_FILE +├── cloudsync.sql # CREATE EXTENSION IF NOT EXISTS cloudsync +└── fly.toml # hardware, volume, env, HTTPS service +``` + +Notable decisions: + +- **`fuse-overlayfs`**: Docker's `overlay2` fails inside a Fly Machine + (`failed to convert whiteout file: operation not permitted`). `vfs` works but + costs ~35 GB and 30–60 min for a full pull. +- **Docker data root on `/data/docker`**: images are pulled once, not on every boot. +- **Compose config re-copied from the image on every boot**, `.env` and + `volumes/` are not — config is immutable and versioned, state is persistent. +- **Overrides layered via `COMPOSE_FILE`** (upstream's own mechanism) instead of + `sed`-ing `docker-compose.yml`, so re-pinning upstream needs no merge work. +- **dockerd runs with IPv6 enabled** so published ports also bind `[::]`. Fly's + private network (`.internal`) is IPv6-only, and Docker publishes IPv4-only + by default — without this, server-to-server access over `.internal` fails. +- **`docker compose logs -f` is the foreground process**, so `fly logs` shows all + services. If it exits, the Machine restarts and the bootstrap re-runs. +- **A failed `compose up` does not kill the Machine.** It used to: with `set -e`, + one service failing its dependency condition exited the entrypoint, Fly + restarted, it failed again, and after 10 rounds the Machine was stopped — taking + the healthy services with it and leaving nothing to inspect. Now the failure is + logged and the boot continues to the log stream. +- **Containers of services dropped from the config are removed before `up`.** + `--remove-orphans` does not cover a service that still exists but sits in a + disabled profile, so its container keeps running *and keeps its published + ports*. That is exactly what happens when swapping the gateway: the old + `supabase-kong` container held port 8000 and Envoy could not bind. + +### API gateway: Envoy + +`COMPOSE_FILE` layers `docker-compose.envoy.yml`, which parks Kong in a disabled +profile and runs `envoyproxy/envoy` as `api-gw` / `supabase-envoy` on the same +host port 8000, keeping the network aliases `kong` and `envoy` so other services' +internal URLs still resolve. Upstream makes Envoy the default in the ~August 2026 +release ([discussion 48048](https://github.com/orgs/supabase/discussions/48048)); +after that the override becomes a no-op shim and Kong moves to +`docker-compose.kong.yml`. + +The Envoy default has no `:8443` HTTPS listener, which does not matter here — +Fly terminates TLS at the edge and only plain 8000 is ever exposed. + +Envoy is stricter than Kong about credentials. `/rest/v1/` (the PostgREST OpenAPI +root) is service-role-only and answers `RBAC: access denied` to an anon key; that +is intended hardening, not a broken route. + +### Overrides applied to upstream compose + +```yaml +services: + db: + image: ${CLOUDSYNC_IMAGE:-sqlitecloud/sqlite-sync-supabase:17-alpine} + volumes: + - ./cloudsync.sql:/docker-entrypoint-initdb.d/init-scripts/100-cloudsync.sql:Z + ports: + - "5433:5432" # direct Postgres; 5432 belongs to Supavisor + supavisor: + entrypoint: ["/usr/bin/tini", "-s", "-g", "--"] # skips /app/limits.sh +``` + +The `100-` prefix keeps the CloudSync init script after Supabase's own scripts +(97–99). Init scripts only run on an empty PGDATA, so `entrypoint.sh` also runs +`CREATE EXTENSION IF NOT EXISTS cloudsync;` on every boot. + +--- + +## Deploying + +```bash +fly apps create "$FLY_APP" --org "$FLY_ORG" +cd deploy/supabase-flyio +fly deploy . --config fly.toml --app "$FLY_APP" --primary-region "$FLY_REGION" +``` + +That is the whole setup. The volume is created from `[[mounts]] initial_size`, +IPs are allocated automatically, and the first boot generates all secrets. + +`fly.toml` deliberately has no `app` or `primary_region` key, so the same file +works for any deployment and no environment-specific identifier is committed. + +First boot pulls ~11 service images and takes roughly 5–10 minutes. Follow it with +`fly logs -a "$FLY_APP"`. + +To run a pre-release CloudSync image instead of the tag baked into the compose +override: + +```bash +fly secrets set -a "$FLY_APP" \ + CLOUDSYNC_IMAGE=sqlitecloud/sqlite-sync-supabase:17-alpine-beta- +``` + +### Secrets + +`entrypoint.sh` bootstraps `.env` on first boot only, from `.env.example` plus +upstream's own generators: + +``` +sh ./utils/generate-keys.sh --update-env # JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, … +sh ./utils/add-new-auth-keys.sh --update-env # ES256 JWT_KEYS/JWT_JWKS, publishable/secret keys +``` + +plus a random `POSTGRES_PASSWORD` and `DASHBOARD_PASSWORD`. Read them back with: + +```bash +fly ssh console -a "$FLY_APP" -C \ + "grep -E '^(POSTGRES_PASSWORD|DASHBOARD_USERNAME|DASHBOARD_PASSWORD|ANON_KEY|SERVICE_ROLE_KEY|JWT_SECRET)=' /data/supabase-docker/.env" +``` + +To pin a value yourself, set it as a Fly secret — compose interpolation prefers +the process environment over `.env`, and Fly secrets are process environment: + +```bash +fly secrets set -a "$FLY_APP" POSTGRES_PASSWORD=... +``` + +> Changing `POSTGRES_PASSWORD` *after* the database has been initialized leaves +> the existing roles on the old password, and services then fail with +> `FATAL: password authentication failed for user "authenticator"`. Realign them: +> +> ```bash +> cd /data/supabase-docker +> for role in authenticator supabase_auth_admin supabase_storage_admin supabase_admin; do +> docker compose exec -T db psql -U postgres \ +> -c "ALTER USER $role WITH PASSWORD 'NEW_PASSWORD';" +> done +> docker compose restart +> ``` +> +> Upstream also ships `sh ./utils/db-passwd.sh` for a guided rotation, followed by +> `docker compose up -d --force-recreate`. + +--- + +## Accessing the stack + +| What | How | +|---|---| +| Studio / API gateway | `https://$FLY_APP.fly.dev` (basic auth: `DASHBOARD_USERNAME` / `DASHBOARD_PASSWORD`) | +| REST / Auth / Storage / Realtime | `https://$FLY_APP.fly.dev/{rest,auth,storage,realtime}/v1/` | +| Postgres, direct, from another Fly app | `postgres://postgres:@$FLY_APP.internal:5433/postgres` | +| Postgres, direct, from a laptop | `fly proxy 5433 -a "$FLY_APP"` then connect to `localhost:5433` | +| Postgres via Supavisor pooler | machine port `5432` (session) / `6543` (transaction), username `postgres.$POOLER_TENANT_ID` | +| Shell | `fly ssh console -a "$FLY_APP"` then `cd /data/supabase-docker` | + +Register with the CloudSync server using the direct (non-pooled) connection +string on port 5433, `"flavor": "supabase"`. + +To keep everything private instead, drop `[http_service]` from `fly.toml`, +release the public IPs, and use `fly proxy 8000 -a `. + +--- + +## Maintenance + +| Task | Command | +|---|---| +| Update the CloudSync image | released tag: redeploy (compose pulls `:17-alpine`); beta: `fly secrets set CLOUDSYNC_IMAGE=…` | +| Update Supabase services | bump `SUPABASE_REF` in the `Dockerfile`, `fly deploy` | +| Restart the stack | `fly machine restart -a ` — the entrypoint re-runs everything | +| Inspect containers | `fly ssh console -a -C "docker ps"` | +| Resize hardware | `fly scale vm shared-cpu-8x --vm-memory 8192 -a ` | +| Grow the volume | `fly volumes extend --size 80 -a ` | +| Destroy | `fly apps destroy ` (removes the volume too) | + +Redeploys recreate the Machine but keep `/data`, so the Docker image cache and +the database survive; only the ~135 MB app image is re-pulled. + +--- + +## Registering the database with the CloudSync server + +The CloudSync server needs a Postgres connection string it can reach. If it runs +in the same Fly org, use the private network directly — no public exposure needed: + +```bash +export CLOUDSYNC_URL="" +export ORG_API_KEY="" +export CONNECTION_STRING="postgres://postgres:@$FLY_APP.internal:5433/postgres" + +curl "$CLOUDSYNC_URL/healthz" # {"status":"ok"} +``` + +If the server runs outside Fly, tunnel instead: `fly proxy 5433 -a "$FLY_APP"` and +use `@localhost:5433`. + +A managed database belongs to a workspace, and an org-wide key must name one. +`PUT` is idempotent — it returns the existing workspace if the slug is taken: + +```bash +curl --request PUT "$CLOUDSYNC_URL/v1/workspaces/" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{"name": "Supabase PG17 Test"}' # → workspaceId +``` + +Register the database. Because GoTrue here is HS256-only (see [Client +token](#client-token)), pass `jwtSecret` and the issuer/audience so CloudSync +validates tokens with the shared secret instead of JWKS: + +```bash +curl --request POST "$CLOUDSYNC_URL/v1/databases" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{ + "label": "Supabase Fly.io Test", + "connectionString": "'"$CONNECTION_STRING"'", + "provider": "postgres", + "flavor": "supabase", + "projectId": "'"$FLY_APP"'", + "databaseName": "postgres", + "workspaceId": "", + "jwtSecret": "", + "jwtAllowedIssuers": ["https://'"$FLY_APP"'.fly.dev/auth/v1"], + "jwtExpectedAudiences": ["authenticated"] + }' + +export MANAGED_DATABASE_ID="" +``` + +The response should show `"auth": {"mode": "jwt_hmac", …}`. Always register the +**direct** port 5433, not the Supavisor pooler. There is no `/verify` endpoint — +the `cloudsync/tables` call below doubles as the connectivity check, since it only +answers once the server has reached the database. + +Create a table and enable sync on it: + +```bash +fly ssh console -a "$FLY_APP" +cd /data/supabase-docker && docker compose exec db psql -U postgres +``` + +```sql +CREATE TABLE IF NOT EXISTS todos ( + id TEXT PRIMARY KEY DEFAULT cloudsync_uuid(), + title TEXT NOT NULL DEFAULT '', + done BOOLEAN DEFAULT false +); +SELECT cloudsync_init('todos'); +``` + +```bash +curl --request POST "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/cloudsync/enable" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{"tables":["todos"]}' + +curl --request GET "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/cloudsync/tables" \ + --header "Authorization: Bearer $ORG_API_KEY" # todos should show "enabled": true +``` + +### Client token + +For a `supabase`-flavored database the client authenticates with a GoTrue JWT, not +an org key. Create the user once, then take an `access_token` per session: + +```bash +ANON_KEY=$(fly ssh console -a "$FLY_APP" -C \ + "grep ^ANON_KEY= /data/supabase-docker/.env" | tail -1 | tr -d '\r' | cut -d= -f2-) + +curl -X POST "https://$FLY_APP.fly.dev/auth/v1/signup" \ + -H "apikey: $ANON_KEY" -H "Content-Type: application/json" \ + -d '{"email":"sync-test@example.com","password":"..."}' + +curl -X POST "https://$FLY_APP.fly.dev/auth/v1/token?grant_type=password" \ + -H "apikey: $ANON_KEY" -H "Content-Type: application/json" \ + -d '{"email":"sync-test@example.com","password":"..."}' # → access_token +``` + +Tokens are HS256, signed with `JWT_SECRET` from `.env`, audience `authenticated`, +issuer `https://$FLY_APP.fly.dev/auth/v1`. Read the issuer from the service, not +from `.env`: the entrypoint derives `API_EXTERNAL_URL` from the app name and +exports it, so process env wins and the `.env` line still reads +`http://localhost:8000/auth/v1`. Confirm with +`docker compose exec auth printenv GOTRUE_JWT_ISSUER`. Validate one with +`GET /auth/v1/user` + `Authorization: Bearer `. Note that +`/auth/v1/.well-known/jwks.json` returns `{"keys":[]}`: upstream ships +`GOTRUE_JWT_KEYS` commented out, so GoTrue stays HS256-only even though `.env` +carries the ES256 material (PostgREST does use it, via `PGRST_JWT_SECRET`). + +### Roundtrip + +```sql +-- SQLite client (Homebrew sqlite3; the system one cannot load extensions) +.load dist/cloudsync.dylib +CREATE TABLE todos ( + id TEXT PRIMARY KEY DEFAULT (cloudsync_uuid()), + title TEXT NOT NULL DEFAULT '', + done INTEGER DEFAULT 0 +); +SELECT cloudsync_init('todos'); +SELECT cloudsync_network_init_custom('', ''); +SELECT cloudsync_network_set_token(''); + +INSERT INTO todos (id, title) VALUES (cloudsync_uuid(), 'from SQLite'); +SELECT cloudsync_network_send_changes(); +SELECT cloudsync_terminate(); +``` + +Then check the row landed in Postgres, insert one there, and pull it back with +`SELECT cloudsync_network_receive_changes();` on the client — allow ~10 s, the +server picks up Postgres-side changes on its own cadence. + +Three things that will bite otherwise: + +- **Declare the SQLite column `INTEGER`, not `BOOLEAN`, for a Postgres `boolean`.** + The schema hash normalizes types per side: PostgreSQL maps `boolean` → `integer`, + while SQLite maps a `BOOLEAN` declaration → `text`. The hashes then differ and the + server rejects the payload with + `Cannot apply the received payload because the schema hash is unknown `. +- **`cloudsync_uuid()` in a `DEFAULT` clause fails on insert** with + `unsafe use of cloudsync_uuid()` unless `PRAGMA trusted_schema=ON`. Pass the id + explicitly instead: `INSERT INTO todos (id, …) VALUES (cloudsync_uuid(), …)`. +- **`cloudsync_network_init` targets the default server.** For any other CloudSync + server use `cloudsync_network_init_custom('', '')`. Neither this nor + `cloudsync_network_set_token` is persisted — both must run in every session. + +--- + +## Building a custom CloudSync Postgres image + +CI publishes a beta tag for every branch push (see the publish matrix in +`.github/workflows/main.yml`), which is usually all you need — point the machine at +it with `CLOUDSYNC_IMAGE`. To build one locally instead: + +```bash +git submodule update --init --recursive # else: fractional_indexing.h: No such file or directory + +docker build --platform linux/amd64 \ + --build-arg SUPABASE_POSTGRES_TAG=17.6.1.151 \ + -f docker/postgresql/Dockerfile.supabase \ + -t /: . + +docker push /: +fly secrets set -a "$FLY_APP" CLOUDSYNC_IMAGE=/: +``` + +`--platform linux/amd64` matters: Fly Machines are x86_64, and an Apple Silicon +build produces an ARM image that will not start. The image must be pullable +without credentials, or `docker login` has to run on the Machine. + +--- + +## Troubleshooting + +| Problem | Fix | +|---|---| +| `Bind for 0.0.0.0:8000 failed: port is already allocated` | A container from a previous compose config still holds the port. The entrypoint now removes containers whose service left the config; otherwise `docker rm -f `. | +| Machine stopped, `machine has reached its max restart count of 10` | Something failed during boot and crash-looped. Read `fly logs`, fix, then `fly machine start `. | +| `password authentication failed for user "authenticator"` | Roles hold a different password than `.env` — see the `ALTER USER` block under [Secrets](#secrets). | +| `cloudsync_version()` does not exist | Init scripts only run on an empty PGDATA. `docker compose exec -T db psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS cloudsync;"` (the entrypoint does this on every boot). | +| Auth restarts, `must be owner of function uid (SQLSTATE 42501)` | GoTrue migrates as `supabase_auth_admin` and must own `auth.uid()`/`role()`/`email()`. Automated in `entrypoint.sh`; rerun by deleting `/data/.cloudsync-post-init-done` and restarting. | +| Supavisor crashes on `ulimit: open files: cannot modify limit` | The Fly kernel refuses `/app/limits.sh`. Handled by the entrypoint override in `docker-compose.cloudsync.yml`. | +| `RBAC: access denied` on `/rest/v1/` | Envoy restricts the PostgREST OpenAPI root to the service-role key. Use a table path, or the service-role key. | +| `.internal` unreachable *from inside a container* | Containers have no route to Fly's 6PN. Use `db:5432` (compose network), `127.0.0.1:5432` (inside `supabase-db`), or the bridge gateway on `:5433`. | +| Services unhealthy right after boot | Give it ~2 minutes, then `docker compose logs `. Studio and Envoy come last. | +| `no space left on device` during a pull | `fly volumes extend --size 80 -a "$FLY_APP"`. | +| `cannot stop container … did not receive an exit event` | Zombie container: `kill -9 $(pidof dockerd) $(pidof containerd)`, `rm -rf /data/docker/containers/*`, then restart the Machine. | +| Cannot pull the CloudSync image | The tag must be public, or run `docker login` on the Machine. | + +--- + +## Image / version notes + +The Alpine CloudSync images are built from `supabase/postgres:17.6.1.151` (see the +publish matrix in `.github/workflows/main.yml`), while the pinned upstream compose +expects `17.6.1.136`. Same PG 17 line, newer patch base — fine, but re-check after +bumping either side. + +The pinned compose runs 11 services: `db`, `supavisor`, `auth`, `rest`, `realtime`, +`storage`, `imgproxy`, `meta`, `functions`, `studio` and the gateway. Upstream has +dropped `analytics` (Logflare) and `vector`, so nothing here needs the old +"disable Logflare" workaround. `db` no longer publishes 5432 either — Supavisor +owns it, which is why the override maps direct Postgres to 5433. + +Verified on the first deployment of this setup (`shared-cpu-4x` / 4 GB / 50 GB, +beta image `…:17-alpine-beta-`): all 11 services +healthy on first boot, `PostgreSQL 17.6`, `cloudsync` extension `1.1` / +`cloudsync_version()` = `1.1.2`, `cloudsync_init()` + insert produced a row in +`cloudsync_changes`, the gateway reachable over HTTPS (Envoy: `/auth/v1/health` and +`/storage/v1/version` 200 with the anon key, `/rest/v1/` 200 with the service-role +key), and ports 5432/5433/8000 published +on both `0.0.0.0` and `[::]` (so `.internal` works). First boot took ~9 +minutes, almost all of it image pulls. diff --git a/docs/postgresql/README.md b/docs/postgresql/README.md new file mode 100644 index 00000000..3e6267e5 --- /dev/null +++ b/docs/postgresql/README.md @@ -0,0 +1,103 @@ +# Architecture Overview + +The **SQLite AI offline-sync solution** consists of three main components: +* **SQLite Sync**: Native client-side SQLite extension +* **CloudSync**: Synchronization microservice +* **Postgres Sync**: Native PostgreSQL extension + +Together, these components provide a complete, production-grade **offline-first synchronization stack** for SQLite and PostgreSQL. + + +# SQLite Sync + +**SQLite Sync** is a native SQLite extension that must be installed and loaded on all client devices. +We provide prebuilt binaries for: +* Desktop and mobile platforms +* WebAssembly (WASM) +* Popular package managers and frameworks including React Native, Expo, Node, Swift PM and Android AAR + +### Architecture Refactoring +The extension has been refactored to support both **SQLite** and **PostgreSQL** backends. +* All database-specific native calls have been isolated in [database.h](../../src/database.h) +* Each database engine implements its own engine-dependent layer +* The core **CRDT logic** is fully shared across engines + +This modular design improves **portability**, **maintainability**, and **cross-database consistency**. +### Testing & Reliability +* Shared CRDT and SQLite components include extensive unit tests +* Code coverage exceeds **90%** +* PostgreSQL-specific code has its own dedicated test suite + +### Key Features +* Deep integration with SQLite — the default database for Edge applications +* Built-in network layer exposed as ordinary SQLite functions +* Cross-platform, language-agnostic payload format +* Works seamlessly in any framework or programming language + +Unlike other offline-sync solutions, **SQLite Sync embeds networking directly inside SQLite**, eliminating external sync SDKs. + +### Supported CRDTs +Currently implemented CRDT algorithms: +* **Last-Write-Wins (LWW)** +* **Grow-Only Set (G-Set)** + +Additional CRDTs can be implemented if needed, though LWW covers most real-world use cases. + + +# CloudSync + +**CloudSync** is a lightweight, stateless microservice responsible for synchronizing clients with central servers. +### Responsibilities +* Synchronizes clients with: + * **SQLite Cloud servers** + * **PostgreSQL servers** +* Manages upload and download of CRDT payloads +* Stores payloads via **AWS S3** +* Collects operational metrics (connected devices, sync volume, traffic, etc.) +* Exposes a complete **REST API** + +### Technology Stack + +* Written in **Go** +* Built on the high-performance **Gin Web Framework** +* Fully **multitenant** +* Connects to multiple DBMS backends +* Stateless architecture enables horizontal scaling simply by adding nodes +* Serialized job queue ensures **no job loss**, even after restarts + +### Observability + +* Metrics dashboard available in [grafana-dashboard.json](../internal/grafana-dashboard.json) + + +# Postgres Sync + +**Postgres Sync** is a native PostgreSQL extension derived from SQLite Sync. +### Features +* Implements the same CRDT algorithms available in SQLite Sync +* Applies CRDT logic to: + * Changes coming from synchronized clients + * Changes made directly in PostgreSQL (CLI, Drizzle, dashboards, etc.) + +This ensures **full bidirectional consistency**, regardless of where changes originate. + +### Schema Handling +SQLite does not support schemas, while PostgreSQL does. To bridge this difference, Postgres Sync introduces a mechanism to: + +* Associate each synchronized table with a specific PostgreSQL schema +* Allow different schemas per table + +This preserves PostgreSQL-native organization while maintaining SQLite compatibility. + +# Current Limitations + +The PostgreSQL integration is actively evolving. Current limitations include: + +* **Table Creation**: Tables must currently be created manually in PostgreSQL before synchronization. We are implementing automatic translation of SQLite CREATE TABLE statements to PostgreSQL syntax. +* **Beta Status**: While extensively tested, the PostgreSQL sync stack should currently be considered **beta software**. Please report any issues; we are committed to resolving them quickly. + +# Next +* [CLIENT](client.md) installation and setup +* [POSTGRES QUICK START](quickstarts/postgres.md) configuration and setup +* [SUPABASE QUICK START](quickstarts/supabase-self-hosted.md) configuration and setup +* [SPORT-TRACKER APP](examples/sport-tracker-app-supabase.md) demo web app based on SQLite Sync WASM diff --git a/docs/postgresql/client.md b/docs/postgresql/client.md new file mode 100644 index 00000000..7aca2076 --- /dev/null +++ b/docs/postgresql/client.md @@ -0,0 +1,218 @@ +# SQLite Sync + +**SQLite Sync** is a multi-platform extension that brings a true **local-first experience** to your applications with minimal effort. It extends standard SQLite tables with built-in support for offline work and automatic synchronization, allowing multiple devices to operate independently — even without a network connection — while seamlessly staying in sync. + +With SQLite Sync, developers can build **distributed, collaborative applications** while continuing to rely on the **simplicity, reliability, and performance of SQLite**. + +Under the hood, SQLite Sync uses advanced **CRDT (Conflict-free Replicated Data Type)** algorithms and data structures designed specifically for **collaborative, distributed systems**: + +- Devices can update data independently, even without a network connection. +- When they reconnect, all changes are **merged automatically and without conflicts**. +- **No data loss. No overwrites. No manual conflict resolution.** + +## Conversion Between SQLite and PostgreSQL Tables + +In this version, make sure to **manually create** the same tables in the PostgreSQL database as used in the SQLite client. + +This guide shows how to manually convert a SQLite table definition to PostgreSQL +so CloudSync can sync between a PostgreSQL server and SQLite clients. + +### 1) Primary Keys + +- Use **TEXT** primary keys in SQLite. +- PostgreSQL primary keys can be **TEXT** or **UUID**. If the PK type + isn't explicitly mapped to a DBTYPE (like UUID), it will be converted to TEXT + in the payload so it remains compatible with the SQLite extension. +- Generate IDs with `cloudsync_uuid()` on both sides. +- Avoid INTEGER auto-increment PKs. + +SQLite: +```sql +id TEXT PRIMARY KEY +``` + +PostgreSQL: +```sql +id TEXT PRIMARY KEY +``` + +PostgreSQL (UUID): +```sql +id UUID PRIMARY KEY +``` + +### 2) NOT NULL Columns Must Have DEFAULTs + +CloudSync merges column-by-column. Any NOT NULL (non-PK) column needs a DEFAULT +to avoid constraint failures during merges. + +Example: +```sql +title TEXT NOT NULL DEFAULT '' +count INTEGER NOT NULL DEFAULT 0 +``` + +### 3) Safe Type Mapping + +Use types that map cleanly to CloudSync's DBTYPEs: + +- INTEGER → `INTEGER` (SQLite) / `INTEGER` (Postgres) +- FLOAT → `REAL` / `DOUBLE` (SQLite) / `DOUBLE PRECISION` (Postgres) +- TEXT → `TEXT` (both) +- BLOB → `BLOB` (SQLite) / `BYTEA` (Postgres) + +Avoid: JSON/JSONB, UUID, INET, CIDR, RANGE, ARRAY unless you accept text-cast +behavior. + +### 4) Defaults That Match Semantics + +Use defaults that serialize the same on both sides: + +- TEXT: `DEFAULT ''` +- INTEGER: `DEFAULT 0` +- FLOAT: `DEFAULT 0.0` +- BLOB: `DEFAULT X'00'` (SQLite) vs `DEFAULT E'\\x00'` (Postgres) + +### 5) Foreign Keys and Triggers + +- Foreign keys can cause merge conflicts; test carefully. +- Application triggers will fire during merge; keep them idempotent or disable + in synced tables. + +### 6) Example Conversion + +SQLite: +```sql +CREATE TABLE notes ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + body TEXT DEFAULT '', + views INTEGER NOT NULL DEFAULT 0, + rating REAL DEFAULT 0.0, + data BLOB +); +``` + +PostgreSQL: +```sql +CREATE TABLE notes ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + body TEXT DEFAULT '', + views INTEGER NOT NULL DEFAULT 0, + rating DOUBLE PRECISION DEFAULT 0.0, + data BYTEA +); +``` + +### 7) Enable CloudSync + +SQLite: +```sql +.load dist/cloudsync.dylib +SELECT cloudsync_init('notes'); +``` + +PostgreSQL: +```sql +CREATE EXTENSION cloudsync; +SELECT cloudsync_init('notes'); +``` + +### Checklist + +- [ ] PKs are TEXT (or UUID in PostgreSQL) +- [ ] All NOT NULL columns have DEFAULT +- [ ] Only INTEGER/FLOAT/TEXT/BLOB-compatible types +- [ ] Same column names and order +- [ ] Same defaults (semantic match) + +Please follow [these Database Schema Recommendations](https://github.com/sqliteai/sqlite-sync?tab=readme-ov-file#database-schema-recommendations) + +## Pre-built Binaries + +Download the appropriate pre-built binary for your platform from the official [Releases](https://github.com/sqliteai/sqlite-sync/releases) page: + +- Linux: x86 and ARM +- macOS: x86 and ARM +- Windows: x86 +- Android +- iOS + +## Loading the Extension + +``` +-- In SQLite CLI +.load ./cloudsync + +-- In SQL +SELECT load_extension('./cloudsync'); +``` + +## WASM Version -> React client-side + +``` +npm i @sqliteai/sqlite-wasm +``` + +Then follow the instructions from the [README](https://www.npmjs.com/package/@sqliteai/sqlite-wasm) + +## Swift Package + +You can [add this repository as a package dependency to your Swift project](https://developer.apple.com/documentation/xcode/adding-package-dependencies-to-your-app#Add-a-package-dependency). After adding the package, you'll need to set up SQLite with extension loading by following steps 4 and 5 of [this guide](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/platforms/ios.md#4-set-up-sqlite-with-extension-loading). + +## Android Package + +Add the [following](https://central.sonatype.com/artifact/ai.sqlite/sync) to your Gradle dependencies: + +``` +implementation 'ai.sqlite:sync:1.0.0' +``` + +## Expo + +Install the Expo package: + +``` +npm install @sqliteai/sqlite-sync-expo +``` + +Then follow the instructions from the [README](https://www.npmjs.com/package/@sqliteai/sqlite-sync-expo) + +## React Native + +Install the React Native library: + +``` +npm install @sqliteai/sqlite-sync-react-native +``` + +Then follow the instructions from the [README](https://www.npmjs.com/package/@sqliteai/sqlite-sync-react-native) + +## Node -> React server-side + +```js +npm i better-sqlite3 +npm i @sqliteai/sqlite-sync + +echo "import { getExtensionPath } from '@sqliteai/sqlite-sync'; +import Database from 'better-sqlite3'; + +const db = new Database(':memory:'); +db.loadExtension(getExtensionPath()); + +// Ready to use +const version = db.prepare('SELECT cloudsync_version()').pluck().get(); +console.log('Sync extension version:', version);" >> index.js + +node index.js +``` + +## Naming Clarification + +- **sqlite-sync** → Client-side SQLite extension +- **cloudsync** → Synchronization server microservice +- **postgres-sync** → PostgreSQL extension + +The sqlite-sync extension is loaded in SQLite under the extension name: +`cloudsync` diff --git a/docs/postgresql/examples/sport-tracker-app-supabase.md b/docs/postgresql/examples/sport-tracker-app-supabase.md new file mode 100644 index 00000000..f6dd0e35 --- /dev/null +++ b/docs/postgresql/examples/sport-tracker-app-supabase.md @@ -0,0 +1,43 @@ +# Sport Tracker app with SQLite Sync 🚵 + +A Vite/React demonstration app showcasing [**SQLite Sync**](https://github.com/sqliteai/sqlite-sync) implementation for **offline-first** data synchronization across multiple devices. This example illustrates how to integrate SQLite AI's sync capabilities into modern web applications with proper authentication via [Access Token](https://docs.sqlitecloud.io/docs/access-tokens) and [Row-Level Security (RLS)](https://docs.sqlitecloud.io/docs/rls). + +> This app uses the packed WASM version of SQLite with the [SQLite Sync extension enabled](https://www.npmjs.com/package/@sqliteai/sqlite-wasm). + +**The source code is located in [examples/sport-tracker-app](../../../examples/sport-tracker-app/)** + +## Setup Instructions + +### 1. Prerequisites +- Node.js 20.x or \>=22.12.0 + +### 2. Database Setup +1. Create database +2. Execute the schema with [sport-tracker-schema-postgres.sql](../../../examples/sport-tracker-app/sport-tracker-schema-postgres.sql). +3. Enable CloudSync for all tables on the remote database with: + ```sql + CREATE EXTENSION IF NOT EXISTS cloudsync; + SELECT cloudsync_init('users_sport'); + SELECT cloudsync_init('workouts'); + SELECT cloudsync_init('activities'); + ``` + +### 3. Environment Configuration + +Rename the `.env.example` into `.env` and fill with your values. + +- `VITE_SQLITECLOUD_CONNECTION_STRING`: the url to the CloudSync server: https://cloudsync-staging.fly.dev/ +- `VITE_SQLITECLOUD_DATABASE`: remote database name. +- `VITE_SQLITECLOUD_API_KEY`: a valid user's JWT token. Refresh it when it expires. +- `VITE_SQLITECLOUD_API_URL`: Supabase project API URL. + +### 4. Installation & Run + +```bash +npm install +npm run dev +``` + +### Demo + +Continue reading on the official [README](https://github.com/sqliteai/sqlite-sync/blob/main/examples/sport-tracker-app/README.md#demo-use-case-multi-user-sync-scenario). diff --git a/docs/postgresql/examples/todo-app-postgres.md b/docs/postgresql/examples/todo-app-postgres.md new file mode 100644 index 00000000..cdf2153c --- /dev/null +++ b/docs/postgresql/examples/todo-app-postgres.md @@ -0,0 +1,63 @@ +# Expo CloudSync Example + +A simple Expo example demonstrating SQLite synchronization with CloudSync and PostgreSQL. Build cross-platform apps that sync data seamlessly across devices. + +https://github.com/user-attachments/assets/21a0332a-7f8f-468b-bd5c-004049e70763 + +## 🚀 Quick Start + +### 1. Clone the template + +Create a new project using this template: +```bash +npx create-expo-app MyApp --template @sqliteai/todoapp@dev +cd MyApp +``` + +### 2. Setup + +1. Execute the exact schema from [`to-do-app.sql`](../../../examples/to-do-app/to-do-app.sql). +2. Enable CloudSync for all tables on the remote database with: + ```sql + CREATE EXTENSION IF NOT EXISTS cloudsync; + SELECT cloudsync_init('tasks'); + SELECT cloudsync_init('tags'); + SELECT cloudsync_init('tasks_tags'); + ``` +3. Rename the `.env.example` into `.env` and fill with your values. +4. If you're testing with a local server define also the `ANDROID_CONNECTION_STRING` variable and use a different connection string for it, replace localhost with `10.0.2.2`. + +``` +CONNECTION_STRING="http://localhost:8091/postgres" +ANDROID_CONNECTION_STRING="http://10.0.2.2:8091/postgres" +API_TOKEN="token" +``` + +5. Fill the `API_TOKEN` variable with the access token. + +> **⚠️ SECURITY WARNING**: This example puts database connection strings directly in `.env` files for demonstration purposes only. **Do not use this pattern in production.** +> +> **Why this is unsafe:** +> - Connection strings contain sensitive credentials +> - Client-side apps expose all environment variables to users +> - Anyone can inspect your app and extract database credentials +> +> **For production apps:** +> - Use the secure [sport-tracker-app](https://github.com/sqliteai/sqlite-sync/tree/main/examples/sport-tracker-app) pattern with authentication tokens and row-level security +> - Never embed database credentials in client applications + +### 4. Build and run the App + +```bash +npx expo prebuild # run once +npm run ios # or android +``` + +## ✨ Features + +- **Add Tasks** - Create new tasks with titles and optional tags. +- **Edit Task Status** - Update task status when completed. +- **Delete Tasks** - Remove tasks from your list. +- **Dropdown Menu** - Select categories for tasks from a predefined list. +- **Cross-Platform** - Works on iOS and Android +- **Offline Support** - Works offline, syncs when connection returns diff --git a/docs/postgresql/quickstarts/postgres.md b/docs/postgresql/quickstarts/postgres.md new file mode 100644 index 00000000..fcfefe86 --- /dev/null +++ b/docs/postgresql/quickstarts/postgres.md @@ -0,0 +1,164 @@ +# CloudSync Quick Start: Self-Hosted PostgreSQL + +This guide helps you enable CloudSync on a **self-hosted PostgreSQL database**. CloudSync adds offline-first synchronization capabilities to your PostgreSQL database. + +## Step 1: Deploy PostgreSQL with CloudSync + +You can enable CloudSync in one of two ways: +- Use the published Docker image if you run PostgreSQL in Docker +- Install the released extension files into an existing native PostgreSQL installation + +### Option A: Docker + +Use the published PostgreSQL image that already includes the CloudSync extension: +- `sqlitecloud/sqlite-sync-postgres:15` +- `sqlitecloud/sqlite-sync-postgres:17` + +Example using Docker Compose: + +```yaml +services: + db: + image: sqlitecloud/sqlite-sync-postgres:17 + container_name: cloudsync-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: your-secure-password + POSTGRES_DB: postgres + ports: + - "5432:5432" + volumes: + - pg_data:/var/lib/postgresql/data + - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro + +volumes: + pg_data: +``` + +Create `init.sql`: +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +Run: +```bash +docker compose up -d +``` + +### Option B: Existing PostgreSQL Without Docker + +If you already run PostgreSQL directly on a VM or bare metal, download the release tarball that matches your operating system, CPU architecture, and PostgreSQL major version. + +Extract the archive, then copy the extension files into PostgreSQL's extension directories. The tarball ships `cloudsync.control`, a `cloudsync--.sql` install script for the current release, and — from release 1.0.17 onward — any `cloudsync----.sql` upgrade scripts needed so existing installations can run `ALTER EXTENSION cloudsync UPDATE`. + +```bash +cp cloudsync.so "$(pg_config --pkglibdir)/" +cp cloudsync.control cloudsync--*.sql "$(pg_config --sharedir)/extension/" +``` + +Then connect to PostgreSQL and enable the extension: + +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +--- + +## Step 2: Verify the Extension + +If you are using Docker: + +```bash +docker compose exec db psql -U postgres -d postgres -c "SELECT cloudsync_version();" +``` + +If you are using an existing PostgreSQL installation without Docker: + +```bash +psql -U postgres -d postgres -c "SELECT cloudsync_version();" +``` + +If the extension is installed correctly, PostgreSQL returns the CloudSync version string. + +### Upgrading a later release + +CloudSync uses the first two components of its semver as the PostgreSQL extension version (for example, `1.0.17` installs as extension version `1.0`). How you upgrade depends on which component changed: + +- **PATCH release** (e.g. `1.0.17 → 1.0.18`): pull the new Docker image or replace the extension files on disk and restart PostgreSQL. No SQL-level upgrade is needed — `installed_version` stays at `1.0` and the new binary takes over on reconnect. `SELECT cloudsync_version();` confirms the new semver. +- **MINOR or MAJOR release** (e.g. `1.0.x → 1.1.0`): pull the new artifacts as above, then run once per database: + + ```sql + ALTER EXTENSION cloudsync UPDATE; + ``` + + PostgreSQL applies any `cloudsync----.sql` upgrade scripts shipped with the release and moves `installed_version` to the new value. + +You can check the current state at any time: + +```sql +SELECT name, default_version, installed_version +FROM pg_available_extensions +WHERE name = 'cloudsync'; +``` + +If `installed_version` is behind `default_version` after a release, run `ALTER EXTENSION cloudsync UPDATE;` to catch up. + +--- + +## Step 3: Register Your Database in the CloudSync Dashboard + +In the [CloudSync dashboard](https://dashboard.sqlitecloud.io/), create a new workspace with the **PostgreSQL** provider, then add a project with your PostgreSQL connection string: + +``` +postgresql://user:password@host:5432/database +``` + +--- + +## Step 4: Enable CloudSync on Tables + +In the dashboard, go to the **Database Setup** tab, select the tables you want to sync, and click **Deploy Changes**. + +--- + +## Step 5: Set Up Authentication + +On the **Client Integration** tab you'll find your **Database ID** and authentication settings. + +### Quick Test with API Key (Recommended for Testing) + +The fastest way to test CloudSync without per-user access control — no JWT setup needed. + +With API key authentication, CloudSync uses the database role resolved from the API-key-authenticated connection when available; otherwise it falls back to the role from the connection string. + +```sql +SELECT cloudsync_network_init(''); +SELECT cloudsync_network_set_apikey(':'); +SELECT cloudsync_network_sync(); +``` + +### Using JWT Tokens (For RLS and Production) + +1. Set **Row Level Security** to **Yes, enforce RLS** +2. Under **Authentication (JWT)**, click **Configure authentication** and choose: + - **HMAC Secret (HS256):** + - Enter your JWT secret (or generate one: `openssl rand -base64 32`) + - Optionally add **Expected audiences**. When configured, a token's `aud` claim must contain at least one of the configured audience values. + - **JWKS Issuer Validation:** + - Enter the issuer base URL from your token's `iss` claim (for example `https://your-auth-domain`) + - By default, CloudSync uses OIDC discovery: it requests `/.well-known/openid-configuration` and reads the returned `jwks_uri` + - Optionally set an **Explicit JWKS URI** to bypass OIDC discovery and use a specific JWKS endpoint directly. This must be a full HTTPS URI. + - Optionally add **Expected audiences**. When configured, a token's `aud` claim must contain at least one of the configured audience values. +3. CloudSync validates JWTs as follows: + - **HS256:** uses the configured JWT secret + - **JWKS:** uses the explicit `jwksUri` when provided; otherwise CloudSync requests `/.well-known/openid-configuration` and reads `jwks_uri` + - CloudSync does not fall back directly to `/.well-known/jwks.json` when discovery is used +4. For claim details and RLS examples, see: + - [JWT Claims Reference](../reference/jwt-claims.md) + - [RLS Reference](../reference/rls.md) +5. In your client code: + ```sql + SELECT cloudsync_network_init(''); + SELECT cloudsync_network_set_token(''); + SELECT cloudsync_network_sync(); + ``` diff --git a/docs/postgresql/quickstarts/supabase-self-hosted.md b/docs/postgresql/quickstarts/supabase-self-hosted.md new file mode 100644 index 00000000..0e1c9107 --- /dev/null +++ b/docs/postgresql/quickstarts/supabase-self-hosted.md @@ -0,0 +1,197 @@ +# CloudSync Quick Start: Self-Hosted Supabase + +This guide helps you enable CloudSync on a **fresh or existing** self-hosted Supabase instance. CloudSync adds offline-first synchronization capabilities to your PostgreSQL database. + +## Step 1: Use the CloudSync Supabase Image + +When deploying or updating your Supabase instance, use the published CloudSync Supabase image instead of the standard Supabase Postgres image. + +### For New Deployments + +Follow [Supabase's Installing Supabase](https://supabase.com/docs/guides/self-hosting/docker#installing-supabase) guide to set up the initial files and `.env` configuration. Then, before the first `docker compose up -d`, update your `docker-compose.yml` to use the CloudSync-enabled Postgres image: + +```yaml +db: + # Supabase on PostgreSQL 15 + image: sqlitecloud/sqlite-sync-supabase:15 + # instead of: public.ecr.aws/supabase/postgres:15.8.1.135 + + # OR Supabase on PostgreSQL 17 + image: sqlitecloud/sqlite-sync-supabase:17 + # instead of: public.ecr.aws/supabase/postgres:17.6.1.071 + + # OR Supabase on PostgreSQL 17 with a newer Alpine-based Supabase image + image: sqlitecloud/sqlite-sync-supabase:17-alpine + # instead of: public.ecr.aws/supabase/postgres:17.6.1.151 +``` + +Use the CloudSync image tag that matches your Supabase PostgreSQL major version. The published major tags `sqlitecloud/sqlite-sync-supabase:15` and `sqlitecloud/sqlite-sync-supabase:17` are the standard choice. + +**Ubuntu vs Alpine base:** newer Supabase Postgres images (roughly `17.6.1.084` and later) ship an Alpine-based userland instead of the earlier Ubuntu one. The extension itself is identical on both — the PostgreSQL binary is glibc-linked in either case — so for a new deployment the two are interchangeable. Moving an **existing** deployment from one family to the other needs the ownership fix described under [For Existing Deployments](#for-existing-deployments). If your Supabase stack pins one of these newer Alpine base images, use the matching `sqlitecloud/sqlite-sync-supabase:17-alpine` tag (or the exact base tag, e.g. `sqlitecloud/sqlite-sync-supabase:17.6.1.151`) so the CloudSync image is built from the same base. Exact Supabase base-image tags are published for both families but are optional for normal setup. + +### Add the CloudSync Init Script + +Create the init SQL: + +```bash +mkdir -p volumes/db +cat > volumes/db/cloudsync.sql << 'EOF' +CREATE EXTENSION IF NOT EXISTS cloudsync; +EOF +``` + +Add a volume mount to the `db` service in `docker-compose.yml`: + +```yaml +services: + db: + volumes: + # ... existing volume mounts ... + - ./volumes/db/cloudsync.sql:/docker-entrypoint-initdb.d/init-scripts/100-cloudsync.sql:Z +``` + +The `100-` prefix ensures CloudSync loads after Supabase's own init scripts, which are typically numbered `98-99` in the self-hosted Docker Compose setup. + +Then start Supabase: + +```bash +docker compose pull +docker compose up -d +``` + +### For Existing Deployments + +Follow [Supabase's Updating](https://supabase.com/docs/guides/self-hosting/docker#updating) guide. When updating the Postgres image, replace the default image with the matching CloudSync image: + +```bash +# Update docker-compose.yml to use: +# sqlitecloud/sqlite-sync-supabase:15 +# or sqlitecloud/sqlite-sync-supabase:17 +# or sqlitecloud/sqlite-sync-supabase:17-alpine (newer Alpine-based Supabase bases) +docker compose pull +docker compose down && docker compose up -d +``` + +If Postgres has already been initialized and you are adding CloudSync afterward, the init script will not run automatically. Connect to the database and run: + +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +#### If Postgres fails to start after moving to a different base image + +Supabase base images do not pin the `postgres` user's UID, so it differs between them — for example `105` on the Ubuntu 20.04 based `15.8.1.085`, `101` on the Ubuntu 24.04 based `15.8.1.135`, and `100` on the Alpine based `17.6.1.151`. A data directory created under one base is unreadable by another, and the container fails to start with: + +``` +cat: /etc/postgresql-custom/pgsodium_root.key: Permission denied +FATAL: invalid secret key +``` + +Re-own the two postgres-owned paths with the UID and GID of the new image: + +```bash +docker compose stop db + +# UID:GID of the postgres user in the new image +docker compose run --rm --no-deps --entrypoint id db postgres + +chown -R : ./volumes/db/data +chown -R : "$(docker volume inspect supabase_db-config --format '{{.Mountpoint}}')" + +docker compose up -d db +``` + +Take a `pg_dumpall` backup first: the change is one-way, and the previous image can no longer read the data directory afterwards. Only deployments whose volumes were initialized under a different base are affected; new deployments are not. + +--- + +## Step 2: Verify the Extension + +```bash +docker compose exec db psql -U supabase_admin -d postgres -c "SELECT cloudsync_version();" +``` + +If the extension is installed correctly, PostgreSQL returns the CloudSync version string. + +### Upgrading a later release + +CloudSync uses the first two components of its semver as the PostgreSQL extension version (for example, `1.0.17` installs as extension version `1.0`). How you upgrade depends on which component changed: + +- **PATCH release** (e.g. `1.0.17 → 1.0.18`): pull the matching `sqlitecloud/sqlite-sync-supabase:` image and restart the `db` service. No SQL-level upgrade is needed — `installed_version` stays at `1.0` and the new binary takes over on reconnect. `SELECT cloudsync_version();` confirms the new semver. +- **MINOR or MAJOR release** (e.g. `1.0.x → 1.1.0`): pull the new image and restart as above, then run once per database: + + ```sql + ALTER EXTENSION cloudsync UPDATE; + ``` + + PostgreSQL applies any `cloudsync----.sql` upgrade scripts shipped with the release and moves `installed_version` to the new value. + +You can check the current state at any time: + +```sql +SELECT name, default_version, installed_version +FROM pg_available_extensions +WHERE name = 'cloudsync'; +``` + +If `installed_version` is behind `default_version` after a release, run `ALTER EXTENSION cloudsync UPDATE;` to catch up. + +--- + +## Step 3: Register Your Database in the CloudSync Dashboard + +In the [CloudSync dashboard](https://dashboard.sqlitecloud.io/), create a new workspace with the **Supabase (Self-hosted)** provider, then add a project with your PostgreSQL connection string: + +``` +postgresql://user:password@host:5432/database +``` + +--- + +## Step 4: Enable CloudSync on Tables + +In the dashboard, go to the **Database Setup** tab, select the tables you want to sync, and click **Deploy Changes**. + +--- + +## Step 5: Set Up Authentication + +On the **Client Integration** tab you'll find your **Database ID** and authentication settings. + +### Quick Test with API Key (Recommended for Testing) + +The fastest way to test CloudSync without per-user access control — no JWT setup needed. + +With API key authentication, CloudSync uses the database role resolved from the API-key-authenticated connection when available; otherwise it falls back to the role from the connection string. + +```sql +SELECT cloudsync_network_init(''); +SELECT cloudsync_network_set_apikey(':'); +SELECT cloudsync_network_sync(); +``` + +### Using JWT Tokens (For RLS and Production) + +1. Set **Row Level Security** to **Yes, enforce RLS** +2. Under **Authentication (JWT)**, click **Configure authentication** and choose: + - **HMAC Secret (HS256):** + - Enter your `JWT_SECRET` from Supabase's `.env` + - Optionally add **Expected audiences**. When configured, a token's `aud` claim must contain at least one of the configured audience values. + - **JWKS Issuer Validation:** + - Enter the issuer base URL from your token's `iss` claim (for example `https://your-auth-domain`) + - By default, CloudSync uses OIDC discovery: it requests `/.well-known/openid-configuration` and reads the returned `jwks_uri` + - Optionally set an **Explicit JWKS URI** to bypass OIDC discovery and use a specific JWKS endpoint directly. This must be a full HTTPS URI. + - Optionally add **Expected audiences**. When configured, a token's `aud` claim must contain at least one of the configured audience values. +3. CloudSync validates JWTs as follows: + - **HS256:** uses the configured JWT secret + - **JWKS:** uses the explicit `jwksUri` when provided; otherwise CloudSync requests `/.well-known/openid-configuration` and reads `jwks_uri` + - CloudSync does not fall back directly to `/.well-known/jwks.json` when discovery is used +4. For claim details and RLS examples, see: + - [JWT Claims Reference](../reference/jwt-claims.md) + - [RLS Reference](../reference/rls.md) +5. In your client code: + ```sql + SELECT cloudsync_network_init(''); + SELECT cloudsync_network_set_token(''); + SELECT cloudsync_network_sync(); + ``` diff --git a/docs/postgresql/reference/jwt-claims.md b/docs/postgresql/reference/jwt-claims.md new file mode 100644 index 00000000..219a699d --- /dev/null +++ b/docs/postgresql/reference/jwt-claims.md @@ -0,0 +1,247 @@ +# JWT Claims Reference + +## HS256 Claims + +Use this mode when CloudSync validates JWTs with `jwtSecret`. + +| Claim | Required? | Notes | +| ------- | ---------- | ------------------------------------------------------------------------------------------------- | +| `sub` | ⚠️ Depends | Not required by CloudSync itself, but commonly used by application-specific RLS policies | +| `email` | ❌ No | Optional app-specific claim; not validated by CloudSync | +| `role` | ✅ Yes | Required for PostgreSQL JWT-authenticated requests because CloudSync uses it for `SET LOCAL ROLE` | +| `iss` | ❌ No | Optional in HS256 mode | +| `aud` | ⚠️ Depends | Required only when `jwtExpectedAudiences` is configured | +| `iat` | ❌ No | Optional issued-at timestamp; not validated by CloudSync | +| `exp` | ✅ Yes | Required and validated by CloudSync | + +## JWKS Claims + +Use this mode when CloudSync validates JWTs with `jwtAllowedIssuers` and optional `jwksUri`. + +| Claim | Required? | Notes | +| -------- | ---------- | ------------------------------------------------------------------------------------------------- | +| `sub` | ⚠️ Depends | Not required by CloudSync itself, but commonly used by application-specific RLS policies | +| `email` | ❌ No | Optional app-specific claim; not validated by CloudSync | +| `role` | ✅ Yes | Required for PostgreSQL JWT-authenticated requests because CloudSync uses it for `SET LOCAL ROLE` | +| `iss` | ✅ Yes | Required for JWKS / issuer-based validation | +| `aud` | ⚠️ Depends | Required only when `jwtExpectedAudiences` is configured | +| `iat` | ❌ No | Optional issued-at timestamp; not validated by CloudSync | +| `exp` | ✅ Yes | Required and validated by CloudSync | +| Header `kid` | ✅ Yes | Required in the JWT header so CloudSync can select the verification key from the JWKS | + +## Custom Claims Examples + +| Claim | Use Case | +| --------------- | -------------------------- | +| `org_id` | Multi-tenant apps | +| `team_id` | Team-based access | +| `permissions` | Fine-grained access | +| `scope` | OAuth scopes | +| `department_id` | Department-based filtering | +| `is_admin` | Admin flag | + +--- + +## How RLS Works with JWT Claims + +**Flow:** + +``` +1. Client sends JWT token to CloudSync +2. CloudSync validates JWT and extracts claims +3. CloudSync passes claims to PostgreSQL as session variables +4. PostgreSQL policies can read session variables via current_setting() +5. Policies filter data based on claims +6. Only authorized rows returned to client +``` + +## PostgreSQL Role Requirement + +For PostgreSQL JWT authentication, the `role` claim must name a real database role that CloudSync can switch into with `SET LOCAL ROLE`. + +That role should: + +- already exist in PostgreSQL +- have the schema, table, and sequence privileges your sync operations need (see [Required Grants](#required-grants)) +- be grantable by the connection-string user + +If the JWT contains a `role` that does not exist, or the connection user cannot switch into it, PostgreSQL sync operations will fail even if the JWT itself is otherwise valid. + +### Creating the Role + +A typical setup uses a `NOLOGIN` role that your connection user enters via `SET LOCAL ROLE` after JWT verification: + +```sql +CREATE ROLE rls_role NOLOGIN; + +-- Allow the connection-string user (e.g. `postgres`) to switch into it +GRANT rls_role TO postgres; +``` + +### Required Grants + +`cloudsync_payload_apply` running as a non-superuser touches several internal CloudSync objects during apply — not just your user table. If any grant is missing on an internal object, the per-PK savepoint silently rolls back the write and the caller sees a non-zero column-change count with no rows landing (see [RLS Troubleshooting](./rls.md#apply-reports-a-count-but-rows-are-missing)). + +There are two equivalent ways to configure this: the **recommended default-privileges pattern** (future-proof) or the **explicit minimum grant set** (tighter, for audited deployments). + +#### Recommended: default-privileges pattern + +Run this **before** `CREATE EXTENSION cloudsync`, as the role that will install the extension (typically `postgres`). Objects created afterwards — including all CloudSync internal tables and future `cloudsync_init` shadows — inherit the grants automatically: + +```sql +GRANT USAGE ON SCHEMA public TO rls_role; +GRANT USAGE ON SCHEMA auth TO rls_role; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER + ON TABLES TO rls_role; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO rls_role; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT EXECUTE ON FUNCTIONS TO rls_role; + +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +**If the extension is already installed**, `ALTER DEFAULT PRIVILEGES` doesn't apply retroactively — backfill existing objects with a one-time broad grant, then still set defaults for future creations: + +```sql +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO rls_role; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO rls_role; +-- (plus the ALTER DEFAULT PRIVILEGES block above) +``` + +#### Explicit minimum grant set + +For audited deployments that need an explicit allowlist, the tightest set that allows `cloudsync_payload_apply` to work under a non-superuser: + +```sql +GRANT USAGE ON SCHEMA public TO rls_role; +GRANT USAGE ON SCHEMA auth TO rls_role; + +-- User table (RLS policies filter rows within these grants) +GRANT SELECT, INSERT, UPDATE, DELETE ON your_table TO rls_role; + +-- Per-table CRDT shadow (created by cloudsync_init) +GRANT SELECT, INSERT, UPDATE, DELETE ON your_table_cloudsync TO rls_role; + +-- CloudSync metadata tables +GRANT SELECT, INSERT, UPDATE, DELETE ON + cloudsync_settings, + cloudsync_table_settings, + cloudsync_site_id, + cloudsync_schema_versions, + cloudsync_payload_fragments, + app_schema_version +TO rls_role; + +-- cloudsync_changes view: SELECT for apply-path readback, INSERT for the +-- INSTEAD OF trigger that feeds column changes into the flush buffer +GRANT SELECT, INSERT ON cloudsync_changes TO rls_role; + +-- BIGSERIAL-backed sequence on cloudsync_site_id.id (nextval needs USAGE) +GRANT USAGE ON SEQUENCE cloudsync_site_id_id_seq TO rls_role; + +-- Your user table's sequence, if it uses SERIAL / IDENTITY +-- GRANT USAGE, SELECT ON SEQUENCE your_table_id_seq TO rls_role; +``` + +Notes on the minimum set: + +- **No `EXECUTE` grants on `cloudsync_*` functions or `auth.uid()` are required**, because PostgreSQL defaults `CREATE FUNCTION` to `EXECUTE TO PUBLIC`. If your cluster has revoked PUBLIC execute, grant `EXECUTE` explicitly on `cloudsync_payload_apply`, `cloudsync_payload_encode`, `cloudsync_changes_select`, `cloudsync_changes_insert_trigger`, `cloudsync_siteid`, `cloudsync_pk_encode`, and `cloudsync_encode_value`. +- **`app_schema_version` is not `cloudsync_*`-prefixed** — easy to miss in `cloudsync_%`-pattern grants. +- **Per-table shadows follow the `
_cloudsync` convention** — repeat the DML grant for every table passed to `cloudsync_init`. +- **Administrative functions** such as `cloudsync_init`, `cloudsync_enable`, `cloudsync_set*`, `cloudsync_terminate`, `cloudsync_cleanup`, `cloudsync_begin_alter`, and `cloudsync_commit_alter` should be run by the database owner during setup, not by client JWT roles. +- The minimum set will need widening if a future CloudSync version adds new internal objects. The default-privileges pattern above is future-proof. + +### Service Role (RLS Bypass) + +For server-side workers that need to apply payloads without RLS enforcement (admin restores, cross-user sync, maintenance jobs), create a dedicated role with `BYPASSRLS`: + +```sql +CREATE ROLE service_role NOLOGIN BYPASSRLS; +GRANT service_role TO postgres; +``` + +Apply the same grants as for `rls_role`. Use this role only from trusted server code, never from JWT-gated request paths. + +--- + +## How CloudSync Passes JWT Claims to PostgreSQL + +**For PostgreSQL JWT-authenticated requests, CloudSync validates the JWT and passes all claims to PostgreSQL as a session variable:** + +```go +// CloudSync (internal implementation) +userData := token.Claims // map[string]any with all JWT claims +claimJSON, _ := json.Marshal(userData) + +// Pass all claims as JSON to PostgreSQL session +db.Exec( + `SELECT set_config('request.jwt.claims', $1, true)`, + string(claimJSON) +) +``` + +**Result:** All JWT claims are available in PostgreSQL as JSON in `request.jwt.claims`, and CloudSync also sets `SET LOCAL ROLE` from the JWT `role` claim. + +**Example:** If JWT contains: + +```json +{ + "sub": "550e8400-e29b-41d4-a716-446655440000", + "email": "user@example.com", + "role": "authenticated", + "org_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" +} +``` + +Then in PostgreSQL: + +```sql +-- Returns: {"sub":"550e8400...","email":"user@example.com","role":"authenticated","org_id":"aaaaaaaa..."} +current_setting('request.jwt.claims') + +-- Access any claim from the JSON +user_id = (current_setting('request.jwt.claims')::jsonb->>'sub')::uuid +email = (current_setting('request.jwt.claims')::jsonb->>'email') +role = (current_setting('request.jwt.claims')::jsonb->>'role') +org_id = (current_setting('request.jwt.claims')::jsonb->>'org_id')::uuid +``` + +--- + +## Optional: Helper Functions for JWT Claims + +CloudSync validates JWTs and passes all claims to PostgreSQL via `request.jwt.claims` — no PostgreSQL extension is required for JWT verification. The validation happens entirely in the CloudSync microservice. + +However, writing `(current_setting('request.jwt.claims')::jsonb->>'sub')::uuid` in every RLS policy is verbose. Following the pattern used by Supabase and Neon, you can optionally create a small set of helper functions in a dedicated schema: + +```sql +-- Create a schema for auth helpers (optional, but keeps things clean) +CREATE SCHEMA IF NOT EXISTS auth; + +-- Returns all JWT claims as JSONB +CREATE OR REPLACE FUNCTION auth.session() + RETURNS jsonb AS $$ + SELECT current_setting('request.jwt.claims', true)::jsonb; +$$ LANGUAGE SQL STABLE; + +-- Returns the user ID (sub claim) +CREATE OR REPLACE FUNCTION auth.user_id() + RETURNS text AS $$ + SELECT auth.session()->>'sub'; +$$ LANGUAGE SQL STABLE; + +-- Returns the user's role claim +CREATE OR REPLACE FUNCTION auth.role() + RETURNS text AS $$ + SELECT auth.session()->>'role'; +$$ LANGUAGE SQL STABLE; +``` + +> **Note:** These are just convenience wrappers — they read from the same `request.jwt.claims` session variable that CloudSync sets. + +--- diff --git a/docs/postgresql/reference/rls.md b/docs/postgresql/reference/rls.md new file mode 100644 index 00000000..8b6586d2 --- /dev/null +++ b/docs/postgresql/reference/rls.md @@ -0,0 +1,202 @@ +# Row Level Security (RLS) with CloudSync + +CloudSync is fully compatible with PostgreSQL Row Level Security. Standard RLS policies work out of the box. + +## How It Works + +### Column-batch merge + +CloudSync resolves CRDT conflicts at the column level — a sync payload may contain individual column changes arriving one at a time. Before writing to the target table, CloudSync buffers all winning column values for the same primary key and flushes them as a single SQL statement. This ensures the database sees a complete row with all columns present. + +### UPDATE vs INSERT selection + +When flushing a batch, CloudSync chooses the statement type based on whether the row already exists locally: + +- **New row**: `INSERT ... ON CONFLICT DO UPDATE` — all columns are present (including the ownership column), so the INSERT `WITH CHECK` policy can evaluate correctly. +- **Existing row**: `UPDATE ... SET ... WHERE pk = ...` — only the changed columns are set. The UPDATE `USING` policy checks the existing row, which already has the correct ownership column value. + +### Per-PK savepoint isolation + +Each primary key's flush is wrapped in its own savepoint. When RLS denies a write: + +1. The database raises an error inside the savepoint +2. CloudSync rolls back that savepoint, releasing all resources acquired during the failed statement +3. Processing continues with the next primary key + +This means a single payload can contain a mix of allowed and denied rows — allowed rows commit normally, denied rows are silently skipped. The caller receives the total number of column changes processed (including denied ones) rather than an error. + +## Quick Setup + +Given a table with an ownership column (`user_id`): + +```sql +CREATE TABLE documents ( + id TEXT PRIMARY KEY, + user_id UUID, + title TEXT, + content TEXT +); + +SELECT cloudsync_init('documents'); +``` + +Enable RLS and create standard policies: + +```sql +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "select_own" ON documents FOR SELECT + USING (auth.uid() = user_id); + +CREATE POLICY "insert_own" ON documents FOR INSERT + WITH CHECK (auth.uid() = user_id); + +CREATE POLICY "update_own" ON documents FOR UPDATE + USING (auth.uid() = user_id) + WITH CHECK (auth.uid() = user_id); + +CREATE POLICY "delete_own" ON documents FOR DELETE + USING (auth.uid() = user_id); +``` + +When you authenticate PostgreSQL requests with JWTs, CloudSync also executes `SET LOCAL ROLE` using the JWT `role` claim. That means the role named in the token must already exist in PostgreSQL and must have the permissions needed to read and write the synced tables. See [PostgreSQL Role Requirement](./jwt-claims.md#postgresql-role-requirement). + +## Example: Two-User Sync with RLS + +This example shows the complete flow of syncing data between two databases where the target enforces RLS. + +### Setup + +```sql +-- Source database (DB A) — no RLS, represents the sync server +CREATE TABLE documents ( + id TEXT PRIMARY KEY, user_id UUID, title TEXT, content TEXT +); +SELECT cloudsync_init('documents'); + +-- Target database (DB B) — RLS enforced +CREATE TABLE documents ( + id TEXT PRIMARY KEY, user_id UUID, title TEXT, content TEXT +); +SELECT cloudsync_init('documents'); +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; +-- (policies as above) +``` + +### Insert sync + +User 1 creates a document on DB A: + +```sql +-- On DB A +INSERT INTO documents VALUES ('doc1', 'user1-uuid', 'Hello', 'World'); +``` + +Apply the payload on DB B as the authenticated user: + +```sql +-- On DB B (running as user1) +SET app.current_user_id = 'user1-uuid'; +SET ROLE authenticated; +SELECT cloudsync_payload_apply(decode(:payload_hex, 'hex')); +``` + +The insert succeeds because `user_id` matches `auth.uid()`. + +### Insert denial + +User 1 tries to sync a document owned by user 2: + +```sql +-- On DB A +INSERT INTO documents VALUES ('doc2', 'user2-uuid', 'Secret', 'Data'); +``` + +```sql +-- On DB B (running as user1) +SET app.current_user_id = 'user1-uuid'; +SET ROLE authenticated; +SELECT cloudsync_payload_apply(decode(:payload_hex, 'hex')); +``` + +The insert is denied by RLS. The row does not appear in DB B. No error is raised to the caller — CloudSync isolates the failure via a per-PK savepoint and continues processing the remaining payload. + +### Partial update sync + +User 1 updates only the title of their own document: + +```sql +-- On DB A +UPDATE documents SET title = 'Hello Updated' WHERE id = 'doc1'; +``` + +The sync payload contains only the changed column (`title`). CloudSync detects that the row already exists on DB B and uses a plain `UPDATE` statement: + +```sql +UPDATE documents SET title = $2 WHERE id = $1; +``` + +The UPDATE policy checks the existing row (which has the correct `user_id`), so it succeeds. + +### Mixed payload + +When a single payload contains rows for multiple users, CloudSync handles each primary key independently: + +```sql +-- On DB A +INSERT INTO documents VALUES ('doc3', 'user1-uuid', 'Mine', '...'); +INSERT INTO documents VALUES ('doc4', 'user2-uuid', 'Theirs', '...'); +``` + +```sql +-- On DB B (running as user1) +SELECT cloudsync_payload_apply(decode(:payload_hex, 'hex')); +-- doc3 is inserted (allowed), doc4 is silently skipped (denied) +``` + +## Supabase Notes + +When using Supabase: + +1. **auth.uid()**: Returns the authenticated user's UUID from the JWT claims. +2. **JWT propagation**: Ensure the JWT token is set before sync operations: + ```sql + SELECT set_config('request.jwt.claims', '{"sub": "user-uuid", ...}', true); + ``` +3. **Service role bypass**: The Supabase service role bypasses RLS entirely. Use the `authenticated` role for user-context operations where RLS enforcement is desired. + +## Troubleshooting + +### "new row violates row-level security policy" + +**Symptom**: Insert operations fail during sync. + +**Cause**: The ownership column value doesn't match the authenticated user. + +**Solution**: Verify that: +- The JWT / session variable is set correctly before calling `cloudsync_payload_apply` +- The `user_id` column in the synced data matches `auth.uid()` +- RLS policies reference the correct ownership column + +### Apply reports a count, but rows are missing + +**Symptom**: `cloudsync_payload_apply` returns a non-zero column-change count, but `SELECT` on the target table shows no new rows. No error is raised to the caller. + +**Cause**: The calling role is missing a grant on one of CloudSync's internal objects — the per-table shadow (`
_cloudsync`), a metadata table (`cloudsync_settings`, `cloudsync_site_id`, `cloudsync_table_settings`, `cloudsync_schema_versions`, `cloudsync_payload_fragments`, `app_schema_version`), the `cloudsync_changes` view, or the `cloudsync_site_id_id_seq` sequence. The per-PK savepoint rolls the write back, but `cloudsync_payload_apply` still returns the number of column changes it processed. + +**Solution**: Apply the full grant set from [JWT Claims → Required Grants](./jwt-claims.md#required-grants). To pinpoint which object is missing, re-run the apply as a superuser or raise log verbosity and inspect the server log for `permission denied` entries preceded by the `cloudsync_payload_apply` call. + +### Debugging + +```sql +-- Check current auth context +SELECT auth.uid(); + +-- Inspect a specific row's ownership +SELECT id, user_id FROM documents WHERE id = 'problematic-pk'; + +-- Temporarily disable RLS to inspect all data +ALTER TABLE documents DISABLE ROW LEVEL SECURITY; +-- ... inspect ... +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; +``` diff --git a/docs/rls-overview.md b/docs/rls-overview.md new file mode 100644 index 00000000..31268ae2 --- /dev/null +++ b/docs/rls-overview.md @@ -0,0 +1,51 @@ +# RLS Overview + +SQLite Sync supports **Row-Level Security (RLS)** through the underlying [SQLite Cloud](https://sqlitecloud.io/) infrastructure. RLS allows you to use a **single shared cloud database** while each client only sees and modifies its own data. Policies are enforced on the server, so the security boundary is at the database level, not in application code. + +> **Using self-hosted PostgreSQL or Supabase?** Start with the backend-specific [RLS Reference](./postgresql/reference/rls.md) for CloudSync apply behavior, PostgreSQL roles and grants, Supabase notes, and troubleshooting. + +## How It Works + +- Control not just who can read or write a table, but **which specific rows** they can access. +- Each device syncs only the rows it is authorized to see: no full dataset download, no client-side filtering. + +For example: + +- User A can only see and edit their own data. +- User B can access a different set of rows, even within the same shared table. + +## Benefits + +- **Single database, multiple tenants**: one cloud database serves all users. RLS policies partition data per user or role, eliminating the need to provision separate databases. +- **Efficient sync**: each client downloads only its authorized rows, reducing bandwidth and local storage. +- **Server-enforced security**: policies are evaluated on the server during sync. A compromised or modified client cannot bypass access controls. +- **Simplified development**: no need to implement permission logic in your application. Define policies once in the database and they apply everywhere. + +## Authentication + +RLS requires token-based authentication. Use `cloudsync_network_set_token()` instead of `cloudsync_network_set_apikey()`: + +```sql +SELECT cloudsync_network_init('your-managed-database-id'); +SELECT cloudsync_network_set_token('your_auth_token'); +SELECT cloudsync_network_sync(); +``` + +For more information on access tokens, see the [Access Tokens documentation](https://docs.sqlitecloud.io/docs/access-tokens). + +## Schema Considerations + +When using RLS with multi-tenant schemas, UNIQUE constraints must be globally unique across all tenants in the cloud database. For columns that should only be unique within a tenant, use composite UNIQUE constraints: + +```sql +CREATE TABLE users ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL DEFAULT '', + email TEXT NOT NULL DEFAULT '', + UNIQUE(tenant_id, email) +); +``` + +For more schema guidelines, see [Database Schema Recommendations](./schema.md). If you are running a self-hosted backend, also see the PostgreSQL/Supabase [RLS Reference](./postgresql/reference/rls.md). + +For full RLS documentation, see the [SQLite Cloud RLS documentation](https://docs.sqlitecloud.io/docs/rls). diff --git a/docs/schema.md b/docs/schema.md new file mode 100644 index 00000000..3b79180b --- /dev/null +++ b/docs/schema.md @@ -0,0 +1,93 @@ +# Database Schema Recommendations + +When designing your database schema for SQLite Sync, follow these guidelines to ensure correct CRDT behavior and conflict resolution. + +## Schema Consistency Across Devices + +All databases participating in the same sync (every client and the cloud database) **must have the same set of synced tables with identical structure**: + +- The same tables must be created on every participant. +- Each table must be initialized with `cloudsync_init()` on every participant. +- Column names, types, and constraints must match across participants. + +sqlite-sync computes a **schema hash** from the synced tables and includes it in every sync payload. The server rejects payloads whose schema hash it does not recognize, failing with an error like: + +``` +cloudsync operation failed: Cannot apply the received payload because the schema hash is unknown +``` + +If you need different clients to see different subsets of data (for example, per-tenant or per-workspace isolation), do **not** give each client a different table. Instead, use a single shared schema and scope the data with a column such as `tenant_id` or `workspace_id`, then enforce isolation server-side with [RLS Overview](./rls-overview.md). + +## Primary Key Requirements + +- **Use globally unique identifiers**: Always use TEXT primary keys with UUIDs or ULIDs. +- **Avoid auto-incrementing integers**: Integer primary keys cause conflicts across multiple devices. +- **Use `cloudsync_uuid()`**: Generates UUIDv7 identifiers optimized for distributed systems. +- **Note:** Any write operation with a NULL primary key value will be rejected with an error. + +```sql +-- Recommended: Globally unique TEXT primary key +CREATE TABLE users ( + id TEXT PRIMARY KEY, -- Use cloudsync_uuid() + name TEXT NOT NULL DEFAULT '', + email TEXT UNIQUE NOT NULL DEFAULT '' +); + +-- Avoid: Auto-incrementing integer primary key +CREATE TABLE users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, -- Causes conflicts across devices + name TEXT NOT NULL DEFAULT '', + email TEXT UNIQUE NOT NULL DEFAULT '' +); +``` + +## Column Constraint Guidelines + +- All `NOT NULL` columns (except primary keys) **must** have `DEFAULT` values. +- For optional data, use nullable columns instead of empty strings. + +```sql +CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'pending', + priority INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), + assigned_to TEXT -- Nullable for optional data +); +``` + +## UNIQUE Constraint Considerations + +In multi-tenant scenarios with Row-Level Security, UNIQUE constraints must be globally unique across all tenants in the cloud database. Use composite UNIQUE constraints for per-tenant uniqueness: + +```sql +-- Multi-tenant: Composite unique constraint +CREATE TABLE users ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL DEFAULT '', + email TEXT NOT NULL DEFAULT '', + UNIQUE(tenant_id, email) -- Unique email per tenant +); +``` + +## Foreign Key Compatibility + +Foreign key constraints may conflict with the CRDT merge algorithm: + +- CRDT changes are applied column-by-column during synchronization. Columns may be temporarily assigned DEFAULT values, so foreign key defaults must reference existing rows. +- RLS policies may block CASCADE DELETE/UPDATE operations on related rows. + +**Recommendations:** +- Prefer application-level cascade logic over database-level CASCADE actions. +- Use nullable foreign keys to avoid DEFAULT value issues. +- Test synchronization scenarios with foreign key constraints enabled. + +## Trigger Compatibility + +Triggers can cause issues during synchronization: + +- **Duplicate operations**: Triggers that modify synchronized tables may apply changes twice during merge. +- **Column-by-column processing**: UPDATE triggers may fire multiple times per row as each column is processed. + +Avoid triggers that write to synchronized tables. Use application-level logic instead. diff --git a/examples/simple-todo-db/README.md b/examples/simple-todo-db/README.md index c9967a5f..56f4d8f5 100644 --- a/examples/simple-todo-db/README.md +++ b/examples/simple-todo-db/README.md @@ -20,11 +20,15 @@ Before using the local CLI, you need to set up your cloud database: 2. Name your database (e.g., "todo_app.sqlite") 3. Click **"Create"** -### 1.3 Get Connection Details -1. Copy the **Connection String** (format: `sqlitecloud://projectid.sqlite.cloud/database.sqlite`) +### 1.3 Enable OffSync +1. Click the **OffSync** button next to your database, then **Enable OffSync** and confirm with the **Enable** button +2. In the **Configuration** tab copy the **Database ID** (format: `db_*`) + +### 1.4 Get Auth Details +1. In your project dashboard, click **Settings**, then **API Keys** 2. Copy an **API Key** -### 1.4 Configure Row-Level Security (Optional) +### 1.5 Configure Row-Level Security (Optional) 1. In your database dashboard, go to **"Security"** → **"Row-Level Security"** 2. Enable RLS for tables you want to secure 3. Create policies to control user access (e.g., users can only see their own tasks) @@ -59,7 +63,7 @@ Tables must be created on both the local database and SQLite Cloud with identica -- Create the main tasks table -- Note: Primary key MUST be TEXT (not INTEGER) for global uniqueness CREATE TABLE IF NOT EXISTS tasks ( - id TEXT PRIMARY KEY NOT NULL, + id TEXT PRIMARY KEY, userid TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', description TEXT DEFAULT '', @@ -84,7 +88,7 @@ SELECT cloudsync_is_enabled('tasks'); - Execute the same CREATE TABLE statement: ```sql CREATE TABLE IF NOT EXISTS tasks ( - id TEXT PRIMARY KEY NOT NULL, + id TEXT PRIMARY KEY, userid TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', description TEXT DEFAULT '', @@ -104,11 +108,11 @@ SELECT cloudsync_is_enabled('tasks'); ```sql -- Configure connection to SQLite Cloud --- Replace with your actual connection string from Step 1.3 -SELECT cloudsync_network_init('sqlitecloud://your-project-id.sqlite.cloud/todo_app.sqlite'); +-- Replace with your managedDatabaseId from the OffSync page on the SQLiteCloud dashboard from Step 1.3 +SELECT cloudsync_network_init('your-managed-database-id'); -- Configure authentication: --- Set your API key from Step 1.3 +-- Set your API key from Step 1.4 SELECT cloudsync_network_set_apikey('your-api-key-here'); -- Or use token authentication (required for Row-Level Security) -- SELECT cloudsync_network_set_token('your_auth_token'); @@ -149,7 +153,7 @@ sqlite3 todo_device_b.db ```sql -- Create identical table structure CREATE TABLE IF NOT EXISTS tasks ( - id TEXT PRIMARY KEY NOT NULL, + id TEXT PRIMARY KEY, userid TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', description TEXT DEFAULT '', @@ -163,12 +167,12 @@ CREATE TABLE IF NOT EXISTS tasks ( SELECT cloudsync_init('tasks'); -- Connect to the same cloud database -SELECT cloudsync_network_init('sqlitecloud://your-project-id.sqlite.cloud/todo_app.sqlite'); +SELECT cloudsync_network_init('your-managed-database-id'); SELECT cloudsync_network_set_apikey('your-api-key-here'); -- Pull data from Device A - repeat until data is received SELECT cloudsync_network_sync(); --- Keep calling until the function returns > 0 (indicating data was received) +-- Check "receive.rows" in the JSON result to see if data was received SELECT cloudsync_network_sync(); -- Verify data was synced @@ -199,7 +203,7 @@ SELECT cloudsync_network_sync(); ```sql -- Get updates from Device B - repeat until data is received SELECT cloudsync_network_sync(); --- Keep calling until the function returns > 0 (indicating data was received) +-- Check "receive.rows" in the JSON result to see if data was received SELECT cloudsync_network_sync(); -- View all tasks (should now include Device B's additions) @@ -232,7 +236,7 @@ SELECT cloudsync_network_has_unsent_changes(); -- When network returns, sync automatically resolves conflicts -- Repeat until all changes are synchronized SELECT cloudsync_network_sync(); --- Keep calling until the function returns > 0 (indicating data was received/sent) +-- Check "receive.rows" and "send.status" in the JSON result SELECT cloudsync_network_sync(); ``` diff --git a/examples/sport-tracker-app/.env.example b/examples/sport-tracker-app/.env.example index ce5c7cca..c534674d 100644 --- a/examples/sport-tracker-app/.env.example +++ b/examples/sport-tracker-app/.env.example @@ -1,6 +1,5 @@ -# Copy from from the SQLite Cloud Dashboard -# eg: sqlitecloud://myhost.cloud:8860/my-remote-database.sqlite -VITE_SQLITECLOUD_CONNECTION_STRING= +# Copy the managedDatabaseId from the OffSync page on the SQLiteCloud Dashboard +VITE_SQLITECLOUD_MANAGED_DATABASE_ID= # The database name # eg: my-remote-database.sqlite VITE_SQLITECLOUD_DATABASE= diff --git a/examples/sport-tracker-app/package.json b/examples/sport-tracker-app/package.json index f1a6da73..3615f8cf 100644 --- a/examples/sport-tracker-app/package.json +++ b/examples/sport-tracker-app/package.json @@ -17,6 +17,7 @@ "@types/react": "^19.1.8", "@types/react-dom": "^19.1.6", "@vitejs/plugin-react": "^4.6.0", + "pg": "^8.17.2", "react": "^19.1.0", "react-dom": "^19.1.0" } diff --git a/examples/sport-tracker-app/sport-tracker-schema-postgres.sql b/examples/sport-tracker-app/sport-tracker-schema-postgres.sql new file mode 100644 index 00000000..1288f165 --- /dev/null +++ b/examples/sport-tracker-app/sport-tracker-schema-postgres.sql @@ -0,0 +1,29 @@ +-- PostgreSQL schema +-- Use this schema to create the remote database on PostgreSQL/PostgREST + +CREATE TABLE IF NOT EXISTS users_sport ( + id TEXT PRIMARY KEY NOT NULL, -- UUID's HIGHLY RECOMMENDED for global uniqueness + name TEXT UNIQUE NOT NULL DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS activities ( + id TEXT PRIMARY KEY NOT NULL, -- UUID's HIGHLY RECOMMENDED for global uniqueness + type TEXT NOT NULL DEFAULT 'runnning', + duration INTEGER, + distance DOUBLE PRECISION, + calories INTEGER, + date TEXT, + notes TEXT, + user_id TEXT REFERENCES users_sport (id) +); + +CREATE TABLE IF NOT EXISTS workouts ( + id TEXT PRIMARY KEY NOT NULL, -- UUID's HIGHLY RECOMMENDED for global uniqueness + name TEXT, + type TEXT, + duration INTEGER, + exercises TEXT, + date TEXT, + completed INTEGER DEFAULT 0, + user_id TEXT +); diff --git a/examples/sport-tracker-app/sport-tracker-schema.sql b/examples/sport-tracker-app/sport-tracker-schema.sql index 9e8ce1a1..88777305 100644 --- a/examples/sport-tracker-app/sport-tracker-schema.sql +++ b/examples/sport-tracker-app/sport-tracker-schema.sql @@ -1,7 +1,7 @@ -- SQL schema -- Use this exact schema to create the remote database on the on SQLite Cloud -CREATE TABLE IF NOT EXISTS users ( +CREATE TABLE IF NOT EXISTS users_sport ( id TEXT PRIMARY KEY NOT NULL, -- UUID's HIGHLY RECOMMENDED for global uniqueness name TEXT UNIQUE NOT NULL DEFAULT '' ); @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS activities ( date TEXT, notes TEXT, user_id TEXT, - FOREIGN KEY (user_id) REFERENCES users (id) + FOREIGN KEY (user_id) REFERENCES users_sport (id) ); CREATE TABLE IF NOT EXISTS workouts ( diff --git a/examples/sport-tracker-app/src/SQLiteSync.ts b/examples/sport-tracker-app/src/SQLiteSync.ts index 3140b25c..85304c3a 100644 --- a/examples/sport-tracker-app/src/SQLiteSync.ts +++ b/examples/sport-tracker-app/src/SQLiteSync.ts @@ -51,9 +51,9 @@ export class SQLiteSync { * * Sync happens in these steps: * 1. Send local changes to the server (`cloudsync_network_send_changes()`). - * 2. Check for changes from the server (`cloudsync_network_check_changes()`). + * 2. Receive changes from the server (`cloudsync_network_receive_changes()`). * 3. Waits a moment for the server to prepare changes if any. - * 4. Check again for changes from the server and apply them to the local database (`cloudsync_network_check_changes()`). + * 4. Receive changes again from the server and apply them to the local database (`cloudsync_network_receive_changes()`). */ async sync() { if (!this.db) { @@ -111,21 +111,23 @@ export class SQLiteSync { const now = new Date(); if (!token) { - console.log("SQLite Sync: No token available, requesting new one from API"); - const tokenData = await this.fetchNewToken(userId, name); - localStorage.setItem( - SQLiteSync.TOKEN_KEY_PREFIX, - JSON.stringify(tokenData) + console.log( + "SQLite Sync: No token available, requesting new one from API", ); + const tokenData = await this.fetchNewToken(userId, name); + // localStorage.setItem( + // SQLiteSync.TOKEN_KEY_PREFIX, + // JSON.stringify(tokenData), + // ); token = tokenData.token; console.log("SQLite Sync: New token obtained and stored in localStorage"); } else if (tokenExpiry && tokenExpiry <= now) { console.warn("SQLite Sync: Token expired, requesting new one from API"); const tokenData = await this.fetchNewToken(userId, name); - localStorage.setItem( - SQLiteSync.TOKEN_KEY_PREFIX, - JSON.stringify(tokenData) - ); + // localStorage.setItem( + // SQLiteSync.TOKEN_KEY_PREFIX, + // JSON.stringify(tokenData), + // ); token = tokenData.token; console.log("SQLite Sync: New token obtained and stored in localStorage"); } else { @@ -143,69 +145,18 @@ export class SQLiteSync { */ private async fetchNewToken( userId: string, - name: string + name: string, ): Promise> { - const response = await fetch( - `${import.meta.env.VITE_SQLITECLOUD_API_URL}/v2/tokens`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${import.meta.env.VITE_SQLITECLOUD_API_KEY}`, - }, - body: JSON.stringify({ - userId, - name, - expiresAt: new Date( - Date.now() + SQLiteSync.TOKEN_EXPIRY_MINUTES * 60 * 1000 - ).toISOString(), - }), - } - ); - - if (!response.ok) { - throw new Error(`Failed to get token: ${response.status}`); - } - - const result = await response.json(); - return result.data; + const jwt = await Promise.resolve(import.meta.env.VITE_SQLITECLOUD_API_KEY); + return { + token: jwt + }; } /** * Checks if a valid token exists in localStorage */ static hasValidToken(): boolean { - const storedTokenData = localStorage.getItem(SQLiteSync.TOKEN_KEY_PREFIX); - - if (!storedTokenData) { - console.log("SQLite Sync: No token data found in localStorage"); - return false; - } - - try { - const parsed: TokenData = JSON.parse(storedTokenData); - - // Check if token exists - if (!parsed.token) { - console.log("SQLite Sync: Token data exists but no token found"); - return false; - } - - // Check if token is expired - if (parsed.expiresAt) { - const tokenExpiry = new Date(parsed.expiresAt); - const now = new Date(); - if (tokenExpiry <= now) { - console.log("SQLite Sync: Token found but expired"); - return false; - } - } - - console.log("SQLite Sync: Valid token found in localStorage"); - return true; - } catch (e) { - console.error("SQLite Sync: Failed to parse stored token:", e); - return false; - } + return false; } } diff --git a/examples/sport-tracker-app/src/components/UserCreation.tsx b/examples/sport-tracker-app/src/components/UserCreation.tsx index 05defb15..fd90b15c 100644 --- a/examples/sport-tracker-app/src/components/UserCreation.tsx +++ b/examples/sport-tracker-app/src/components/UserCreation.tsx @@ -15,26 +15,22 @@ interface UserCreationProps { */ const fetchRemoteUsers = async (): Promise => { const response = await fetch( - `${import.meta.env.VITE_SQLITECLOUD_API_URL}/v2/weblite/sql`, + `${import.meta.env.VITE_SQLITECLOUD_API_URL}/rest/v1/users_sport?select=id,name`, { - method: "POST", + method: "GET", headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${import.meta.env.VITE_SQLITECLOUD_API_KEY}`, + Accept: "application/json", + Authorization: `Bearer ${import.meta.env.VITE_SQLITECLOUD_API_KEY || ""}`, }, - body: JSON.stringify({ - sql: "SELECT id, name FROM users;", - database: import.meta.env.VITE_SQLITECLOUD_DATABASE || "", - }), } ); - + if (!response.ok) { throw new Error(`Failed to fetch users: ${response.status}`); } const result = await response.json(); - return result.data; + return result as User[]; }; const UserCreation: React.FC = ({ @@ -58,6 +54,7 @@ const UserCreation: React.FC = ({ setRemoteUsers(users); } catch (error) { console.error("Failed to load remote users:", error); + alert("Failed to load remote users. Error: " + error); } finally { setIsLoadingRemoteUsers(false); } diff --git a/examples/sport-tracker-app/src/db/databaseOperations.ts b/examples/sport-tracker-app/src/db/databaseOperations.ts index 5bae19ac..2413d271 100644 --- a/examples/sport-tracker-app/src/db/databaseOperations.ts +++ b/examples/sport-tracker-app/src/db/databaseOperations.ts @@ -216,7 +216,7 @@ export const getDatabaseOperations = (db: any) => ({ try { db.exec({ - sql: "INSERT INTO users (id, name) VALUES (?, ?)", + sql: "INSERT INTO users_sport (id, name) VALUES (?, ?)", bind: [userId, name], }); return { id: userId, name }; @@ -228,7 +228,7 @@ export const getDatabaseOperations = (db: any) => ({ getUsers() { const users: any[] = []; db.exec({ - sql: "SELECT id, name FROM users ORDER BY name", + sql: "SELECT id, name FROM users_sport ORDER BY name", callback: (row: any) => { users.push({ id: row[0], @@ -243,7 +243,7 @@ export const getDatabaseOperations = (db: any) => ({ const { id } = data; let user = null; db.exec({ - sql: "SELECT id, name FROM users WHERE id = ?", + sql: "SELECT id, name FROM users_sport WHERE id = ?", bind: [id], callback: (row: any) => { user = { @@ -268,7 +268,7 @@ export const getDatabaseOperations = (db: any) => ({ // Get total counts (always show total for comparison) db.exec({ - sql: "SELECT COUNT(*) FROM users WHERE name != ?", + sql: "SELECT COUNT(*) FROM users_sport WHERE name != ?", bind: ["coach"], callback: (row: any) => (counts.totalUsers = row[0]), }); @@ -287,7 +287,7 @@ export const getDatabaseOperations = (db: any) => ({ if (user_id && !is_coach) { // Regular user - count only their data db.exec({ - sql: "SELECT COUNT(*) FROM users WHERE name != ?", + sql: "SELECT COUNT(*) FROM users_sport WHERE name != ?", bind: ["coach"], callback: (row: any) => (counts.users = row[0]), }); diff --git a/examples/sport-tracker-app/src/db/sqliteSyncOperations.ts b/examples/sport-tracker-app/src/db/sqliteSyncOperations.ts index 97329db7..79fe4406 100644 --- a/examples/sport-tracker-app/src/db/sqliteSyncOperations.ts +++ b/examples/sport-tracker-app/src/db/sqliteSyncOperations.ts @@ -26,7 +26,7 @@ export const getSqliteSyncOperations = (db: any) => ({ * This will push changes to the cloud and check for changes from the cloud. * The first attempt may not find anything to apply, but subsequent attempts * will find changes if they exist. - */ + */ sqliteSyncNetworkSync() { console.log("SQLite Sync - Starting sync..."); db.exec("SELECT cloudsync_network_sync(1000, 2);"); @@ -38,7 +38,7 @@ export const getSqliteSyncOperations = (db: any) => ({ */ sqliteSyncSendChanges() { console.log( - "SQLite Sync - Sending changes to your the SQLite Cloud node..." + "SQLite Sync - Sending changes to your the SQLite Cloud node...", ); db.exec("SELECT cloudsync_network_send_changes();"); console.log("SQLite Sync - Changes sent"); @@ -84,18 +84,18 @@ export const initSQLiteSync = (db: any) => { } // Initialize SQLite Sync - db.exec(`SELECT cloudsync_init('users');`); + db.exec(`SELECT cloudsync_init('users_sport');`); db.exec(`SELECT cloudsync_init('activities');`); db.exec(`SELECT cloudsync_init('workouts');`); // ...or initialize all tables at once // db.exec('SELECT cloudsync_init("*");'); - // Initialize SQLite Sync with the SQLite Cloud Connection String. - // On the SQLite Cloud Dashboard, enable OffSync (SQLite Sync) - // on the remote database and copy the Connection String. + // Initialize SQLite Sync with the managedDatabaseId. + // On the SQLite Cloud Dashboard, enable OffSync (SQLite Sync) + // on the remote database and copy the managedDatabaseId. db.exec( `SELECT cloudsync_network_init('${ - import.meta.env.VITE_SQLITECLOUD_CONNECTION_STRING + import.meta.env.VITE_SQLITECLOUD_MANAGED_DATABASE_ID }')` ); }; diff --git a/examples/to-do-app/.env.example b/examples/to-do-app/.env.example index 4b2816df..ba99068d 100644 --- a/examples/to-do-app/.env.example +++ b/examples/to-do-app/.env.example @@ -1,3 +1,3 @@ -# Copy from the SQLite Cloud Dashboard -# eg: sqlitecloud://myhost.cloud:8860/my-remote-database.sqlite?apikey=myapikey -CONNECTION_STRING = "" \ No newline at end of file +# Copy from the OffSync page on the SQLiteCloud Dashboard +MANAGED_DATABASE_ID = "" +API_TOKEN = diff --git a/examples/to-do-app/README.md b/examples/to-do-app/README.md index bc771237..0e3d0c02 100644 --- a/examples/to-do-app/README.md +++ b/examples/to-do-app/README.md @@ -24,10 +24,10 @@ cd MyApp Rename the `.env.example` into `.env` and fill with your values. -> **⚠️ SECURITY WARNING**: This example puts database connection strings directly in `.env` files for demonstration purposes only. **Do not use this pattern in production.** +> **⚠️ SECURITY WARNING**: This example puts database API Keys directly in `.env` files for demonstration purposes only. **Do not use this pattern in production.** > > **Why this is unsafe:** -> - Connection strings contain sensitive credentials +> - API Keys allow access to sensitive credentials > - Client-side apps expose all environment variables to users > - Anyone can inspect your app and extract database credentials > diff --git a/examples/to-do-app/components/SyncContext.js b/examples/to-do-app/components/SyncContext.js index e964f4aa..c381c02e 100644 --- a/examples/to-do-app/components/SyncContext.js +++ b/examples/to-do-app/components/SyncContext.js @@ -51,17 +51,21 @@ export const SyncProvider = ({ children }) => { try { // Use a timeout for the database query to prevent hanging - const queryPromise = db.execute('SELECT cloudsync_network_check_changes();'); + const queryPromise = db.execute('SELECT cloudsync_network_receive_changes();'); const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Query timeout')), 5000) ); const result = await Promise.race([queryPromise, timeoutPromise]); - if (result.rows && result.rows.length > 0 && result.rows[0]['cloudsync_network_check_changes()'] > 0) { - console.log(`${result.rows[0]['cloudsync_network_check_changes()']} changes detected, triggering refresh`); - // Defer refresh to next tick to avoid blocking current interaction - setTimeout(() => triggerRefresh(), 0); + const raw = result.rows?.[0]?.['cloudsync_network_receive_changes()']; + if (raw) { + const { receive } = JSON.parse(raw); + if (receive.rows > 0) { + console.log(`${receive.rows} changes detected in [${receive.tables}], triggering refresh`); + // Defer refresh to next tick to avoid blocking current interaction + setTimeout(() => triggerRefresh(), 0); + } } } catch (error) { console.error('Error checking for changes:', error); diff --git a/examples/to-do-app/components/TaskRow.js b/examples/to-do-app/components/TaskRow.js index ff8b3cf9..654993d9 100644 --- a/examples/to-do-app/components/TaskRow.js +++ b/examples/to-do-app/components/TaskRow.js @@ -1,6 +1,6 @@ import { useState, useRef, useEffect } from "react"; import { View, Text, StyleSheet, TouchableOpacity } from "react-native"; -import Icon from "react-native-vector-icons/FontAwesome"; +import Icon from "@react-native-vector-icons/fontawesome"; import { Swipeable } from "react-native-gesture-handler"; export default TaskRow = ({ task, updateTask, handleDelete }) => { diff --git a/examples/to-do-app/hooks/useCategories.js b/examples/to-do-app/hooks/useCategories.js index f46215ce..519f7ec5 100644 --- a/examples/to-do-app/hooks/useCategories.js +++ b/examples/to-do-app/hooks/useCategories.js @@ -1,7 +1,7 @@ import { useState, useEffect } from 'react' import { Platform } from 'react-native'; import { db } from "../db/dbConnection"; -import { CONNECTION_STRING } from "@env"; +import { MANAGED_DATABASE_ID, API_TOKEN } from "@env"; import { getDylibPath } from "@op-engineering/op-sqlite"; import { randomUUID } from 'expo-crypto'; import { useSyncContext } from '../components/SyncContext'; @@ -65,14 +65,18 @@ const useCategories = () => { await db.execute('CREATE TABLE IF NOT EXISTS tags (uuid TEXT NOT NULL PRIMARY KEY, name TEXT, UNIQUE(name));') await db.execute('CREATE TABLE IF NOT EXISTS tasks_tags (uuid TEXT NOT NULL PRIMARY KEY, task_uuid TEXT, tag_uuid TEXT, FOREIGN KEY (task_uuid) REFERENCES tasks(uuid), FOREIGN KEY (tag_uuid) REFERENCES tags(uuid));') - await db.execute(`SELECT cloudsync_init('*');`); - await db.execute('INSERT OR IGNORE INTO tags (uuid, name) VALUES (?, ?)', [randomUUID(), 'Work']) - await db.execute('INSERT OR IGNORE INTO tags (uuid, name) VALUES (?, ?)', [randomUUID(), 'Personal']) + await db.execute(`SELECT cloudsync_init('tasks');`); + await db.execute(`SELECT cloudsync_init('tags');`); + await db.execute(`SELECT cloudsync_init('tasks_tags');`); + + await db.execute('INSERT OR IGNORE INTO tags (uuid, name) VALUES (?, ?)', ['work', 'Work']) + await db.execute('INSERT OR IGNORE INTO tags (uuid, name) VALUES (?, ?)', ['personal', 'Personal']) - if (CONNECTION_STRING && CONNECTION_STRING.startsWith('sqlitecloud://')) { - await db.execute(`SELECT cloudsync_network_init('${CONNECTION_STRING}');`); + if (MANAGED_DATABASE_ID && API_TOKEN) { + await db.execute(`SELECT cloudsync_network_init('${MANAGED_DATABASE_ID}');`); + await db.execute(`SELECT cloudsync_network_set_token('${API_TOKEN}');`) } else { - throw new Error('No valid CONNECTION_STRING provided, cloudsync_network_init will not be called'); + throw new Error('No valid MANAGED_DATABASE_ID or API_TOKEN provided, cloudsync_network_init will not be called'); } db.execute('SELECT cloudsync_network_sync(100, 10);') diff --git a/examples/to-do-app/package.json b/examples/to-do-app/package.json index 82534ff0..cb058936 100644 --- a/examples/to-do-app/package.json +++ b/examples/to-do-app/package.json @@ -1,13 +1,22 @@ { "name": "@sqliteai/todoapp", - "version": "1.0.3", + "version": "1.0.7", "description": "An Expo template for building apps with the SQLite CloudSync extension", "repository": { "type": "git", "url": "git+https://github.com/sqliteai/sqlite-sync.git" }, "author": "SQLiteAI", - "keywords": ["expo-template", "sqlite", "cloudsync", "todo", "sync", "react-native", "expo", "template"], + "keywords": [ + "expo-template", + "sqlite", + "cloudsync", + "todo", + "sync", + "react-native", + "expo", + "template" + ], "main": "expo/AppEntry.js", "scripts": { "start": "expo start", @@ -21,19 +30,23 @@ "dependencies": { "@op-engineering/op-sqlite": "14.1.4", "@react-native-picker/picker": "2.11.1", + "@react-native-vector-icons/fontawesome": "^12.4.0", + "@react-native-vector-icons/material-design-icons": "^12.4.0", "@react-navigation/native": "^7.1.17", "@react-navigation/stack": "^7.4.7", "expo": "^53.0.22", + "expo-asset": "^12.0.12", + "expo-constants": "^18.0.13", "expo-crypto": "~14.1.5", "expo-status-bar": "~2.2.3", + "prop-types": "^15.8.1", "react": "19.0.0", - "react-native": "0.79.5", + "react-native": "0.79.6", "react-native-gesture-handler": "~2.24.0", "react-native-paper": "5.14.5", "react-native-picker-select": "^9.3.1", "react-native-safe-area-context": "5.4.0", - "react-native-screens": "~4.11.1", - "react-native-vector-icons": "^10.3.0" + "react-native-screens": "~4.11.1" }, "devDependencies": { "@babel/core": "7.28.3", diff --git a/examples/to-do-app/plugins/CloudSyncSetup.js b/examples/to-do-app/plugins/CloudSyncSetup.js index f4839012..3cba189b 100644 --- a/examples/to-do-app/plugins/CloudSyncSetup.js +++ b/examples/to-do-app/plugins/CloudSyncSetup.js @@ -1,4 +1,4 @@ -const { withDangerousMod, withXcodeProject } = require('@expo/config-plugins'); +const { withDangerousMod, withXcodeProject, withInfoPlist } = require('@expo/config-plugins'); const fs = require('fs'); const path = require('path'); const https = require('https'); @@ -277,8 +277,21 @@ const withCloudSync = (config) => { // iOS setup - add to Xcode project config = withCloudSyncFramework(config); - + + // iOS setup - register icon fonts in Info.plist + config = withInfoPlist(config, (config) => { + const fonts = config.modResults.UIAppFonts || []; + if (!fonts.includes('FontAwesome.ttf')) { + fonts.push('FontAwesome.ttf'); + } + if (!fonts.includes('MaterialDesignIcons.ttf')) { + fonts.push('MaterialDesignIcons.ttf'); + } + config.modResults.UIAppFonts = fonts; + return config; + }); + return config; }; -module.exports = withCloudSync; \ No newline at end of file +module.exports = withCloudSync; diff --git a/examples/to-do-app/screens/Home.js b/examples/to-do-app/screens/Home.js index ae9c2bf3..7e0321a7 100644 --- a/examples/to-do-app/screens/Home.js +++ b/examples/to-do-app/screens/Home.js @@ -1,7 +1,7 @@ import { useState } from "react"; import { View, Text, StyleSheet, FlatList, Alert } from "react-native"; import { Button } from "react-native-paper"; -import Icon from "react-native-vector-icons/FontAwesome"; +import Icon from "@react-native-vector-icons/fontawesome"; import TaskRow from "../components/TaskRow"; import AddTaskModal from "../components/AddTaskModal"; import useTasks from "../hooks/useTasks" diff --git a/modules/fractional-indexing b/modules/fractional-indexing new file mode 160000 index 00000000..b9af0ec5 --- /dev/null +++ b/modules/fractional-indexing @@ -0,0 +1 @@ +Subproject commit b9af0ec5b818bca29919e1a8d42b142feb71f269 diff --git a/packages/expo/README.md b/packages/expo/README.md index ee4c321c..0eac7b33 100644 --- a/packages/expo/README.md +++ b/packages/expo/README.md @@ -21,7 +21,7 @@ node generate-expo-package.js Example: ```bash -node generate-expo-package.js 0.8.57 ../../artifacts ./expo-package +node generate-expo-package.js 1.0.0 ../../artifacts ./expo-package cd expo-package && npm publish --provenance --access public ``` @@ -54,7 +54,7 @@ To test the generator locally, you need to set up mock artifacts that simulate w **Option A: Download from latest release** ```bash -VERSION="0.8.57" # or latest version +VERSION="1.0.0" # or latest version mkdir -p artifacts/cloudsync-apple-xcframework mkdir -p artifacts/cloudsync-android-arm64-v8a @@ -100,7 +100,7 @@ cp -r dist/CloudSync.xcframework artifacts/cloudsync-apple-xcframework/ ```bash cd packages/expo -node generate-expo-package.js 0.8.57 ../../artifacts ./expo-package +node generate-expo-package.js 1.0.0 ../../artifacts ./expo-package ``` ### Step 3: Test in a Expo app diff --git a/packages/expo/generate-expo-package.js b/packages/expo/generate-expo-package.js index de89071a..4a28f1fa 100644 --- a/packages/expo/generate-expo-package.js +++ b/packages/expo/generate-expo-package.js @@ -10,7 +10,7 @@ * node generate-expo-package.js * * Example: - * node generate-expo-package.js 0.8.53 ./artifacts ./expo-package + * node generate-expo-package.js 1.0.0 ./artifacts ./expo-package */ const fs = require('fs'); @@ -396,7 +396,7 @@ function main() { if (args.length < 3) { console.error('Usage: node generate-expo-package.js '); - console.error('Example: node generate-expo-package.js 0.8.53 ./artifacts ./expo-package'); + console.error('Example: node generate-expo-package.js 1.0.0 ./artifacts ./expo-package'); process.exit(1); } @@ -405,7 +405,7 @@ function main() { // Validate version format if (!/^\d+\.\d+\.\d+$/.test(version)) { console.error(`Error: Invalid version format: ${version}`); - console.error('Version must be in semver format (e.g., 0.8.53)'); + console.error('Version must be in semver format (e.g., 1.0.0)'); process.exit(1); } diff --git a/packages/flutter/README.md b/packages/flutter/README.md index 48807ed8..2d3c65aa 100644 --- a/packages/flutter/README.md +++ b/packages/flutter/README.md @@ -67,4 +67,4 @@ See the full [sqlite-sync API documentation](https://github.com/sqliteai/sqlite- ## License -See [LICENSE](LICENSE). +See [LICENSE](../../LICENSE.md). diff --git a/packages/node/README.md b/packages/node/README.md index 91cc8944..234e935a 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -136,6 +136,8 @@ Error thrown when the SQLite Sync extension cannot be found for the current plat ## Related Projects +- **[@sqliteai/sqlite-sync-react-native](https://www.npmjs.com/package/@sqliteai/sqlite-sync-react-native)** - SQLite Sync for React Native +- **[@sqliteai/sqlite-sync-expo](https://www.npmjs.com/package/@sqliteai/sqlite-sync-expo)** - SQLite Sync for Expo - **[@sqliteai/sqlite-vector](https://www.npmjs.com/package/@sqliteai/sqlite-vector)** - Vector search and similarity matching - **[@sqliteai/sqlite-ai](https://www.npmjs.com/package/@sqliteai/sqlite-ai)** - On-device AI inference and embedding generation - **[@sqliteai/sqlite-js](https://www.npmjs.com/package/@sqliteai/sqlite-js)** - Define SQLite functions in JavaScript diff --git a/packages/node/generate-platform-packages.js b/packages/node/generate-platform-packages.js index b725f6d6..32bef4a1 100644 --- a/packages/node/generate-platform-packages.js +++ b/packages/node/generate-platform-packages.js @@ -10,7 +10,7 @@ * node generate-platform-packages.js * * Example: - * node generate-platform-packages.js 0.8.53 ./artifacts ./platform-packages + * node generate-platform-packages.js 1.0.0 ./artifacts ./platform-packages */ const fs = require('fs'); @@ -165,7 +165,7 @@ function main() { if (args.length < 3) { console.error('Usage: node generate-platform-packages.js '); - console.error('Example: node generate-platform-packages.js 0.8.53 ./artifacts ./platform-packages'); + console.error('Example: node generate-platform-packages.js 1.0.0 ./artifacts ./platform-packages'); process.exit(1); } @@ -181,7 +181,7 @@ function main() { // Validate version format if (!/^\d+\.\d+\.\d+$/.test(version)) { console.error(`Error: Invalid version format: ${version}`); - console.error('Version must be in semver format (e.g., 0.8.53)'); + console.error('Version must be in semver format (e.g., 1.0.0)'); process.exit(1); } diff --git a/packages/node/package.json b/packages/node/package.json index f8ecd1d9..1536b4e0 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "@sqliteai/sqlite-sync", - "version": "0.8.53", + "version": "1.0.0", "description": "SQLite Sync extension for Node.js - Sync on-device databases with the cloud", "main": "./dist/index.js", "module": "./dist/index.mjs", @@ -53,13 +53,13 @@ "node": ">=16.0.0" }, "optionalDependencies": { - "@sqliteai/sqlite-sync-darwin-arm64": "0.8.53", - "@sqliteai/sqlite-sync-darwin-x86_64": "0.8.53", - "@sqliteai/sqlite-sync-linux-arm64": "0.8.53", - "@sqliteai/sqlite-sync-linux-arm64-musl": "0.8.53", - "@sqliteai/sqlite-sync-linux-x86_64": "0.8.53", - "@sqliteai/sqlite-sync-linux-x86_64-musl": "0.8.53", - "@sqliteai/sqlite-sync-win32-x86_64": "0.8.53" + "@sqliteai/sqlite-sync-darwin-arm64": "1.0.0", + "@sqliteai/sqlite-sync-darwin-x86_64": "1.0.0", + "@sqliteai/sqlite-sync-linux-arm64": "1.0.0", + "@sqliteai/sqlite-sync-linux-arm64-musl": "1.0.0", + "@sqliteai/sqlite-sync-linux-x86_64": "1.0.0", + "@sqliteai/sqlite-sync-linux-x86_64-musl": "1.0.0", + "@sqliteai/sqlite-sync-win32-x86_64": "1.0.0" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/packages/swift/CloudSync.swift b/packages/swift/CloudSync.swift new file mode 100644 index 00000000..73a0618e --- /dev/null +++ b/packages/swift/CloudSync.swift @@ -0,0 +1,15 @@ +// CloudSync.swift +// Provides the path to the CloudSync SQLite extension for use with sqlite3_load_extension. + +import Foundation + +public struct CloudSync { + /// Returns the absolute path to the CloudSync dylib for use with sqlite3_load_extension. + public static var path: String { + #if os(macOS) + return Bundle.main.bundlePath + "/Contents/Frameworks/CloudSync.framework/CloudSync" + #else + return Bundle.main.bundlePath + "/Frameworks/CloudSync.framework/CloudSync" + #endif + } +} diff --git a/packages/swift/extension/CloudSync.swift b/packages/swift/extension/CloudSync.swift deleted file mode 100644 index b7da66e0..00000000 --- a/packages/swift/extension/CloudSync.swift +++ /dev/null @@ -1,19 +0,0 @@ -// CloudSync.swift -// This file serves as a placeholder for the CloudSync target. -// The actual SQLite extension is built using the Makefile through the build plugin. - -import Foundation - -/// Placeholder structure for CloudSync -public struct CloudSync { - /// Returns the path to the built CloudSync dylib inside the XCFramework - public static var path: String { - #if os(macOS) - return "CloudSync.xcframework/macos-arm64_x86_64/CloudSync.framework/CloudSync" - #elseif targetEnvironment(simulator) - return "CloudSync.xcframework/ios-arm64_x86_64-simulator/CloudSync.framework/CloudSync" - #else - return "CloudSync.xcframework/ios-arm64/CloudSync.framework/CloudSync" - #endif - } -} \ No newline at end of file diff --git a/packages/swift/plugin/CloudSync.swift b/packages/swift/plugin/CloudSync.swift deleted file mode 100644 index ccb95ccd..00000000 --- a/packages/swift/plugin/CloudSync.swift +++ /dev/null @@ -1,60 +0,0 @@ -import PackagePlugin -import Foundation - -@main -struct CloudSync: BuildToolPlugin { - /// Entry point for creating build commands for targets in Swift packages. - func createBuildCommands(context: PluginContext, target: Target) async throws -> [Command] { - let packageDirectory = context.package.directoryURL - let outputDirectory = context.pluginWorkDirectoryURL - return createCloudSyncBuildCommands(packageDirectory: packageDirectory, outputDirectory: outputDirectory) - } -} - -#if canImport(XcodeProjectPlugin) -import XcodeProjectPlugin - -extension CloudSync: XcodeBuildToolPlugin { - // Entry point for creating build commands for targets in Xcode projects. - func createBuildCommands(context: XcodePluginContext, target: XcodeTarget) throws -> [Command] { - let outputDirectory = context.pluginWorkDirectoryURL - return createCloudSyncBuildCommands(packageDirectory: nil, outputDirectory: outputDirectory) - } -} - -#endif - -/// Shared function to create CloudSync build commands -func createCloudSyncBuildCommands(packageDirectory: URL?, outputDirectory: URL) -> [Command] { - - // For Xcode projects, use current directory; for Swift packages, use provided packageDirectory - let workingDirectory = packageDirectory?.path ?? "$(pwd)" - let packageDirInfo = packageDirectory != nil ? "Package directory: \(packageDirectory!.path)" : "Working directory: $(pwd)" - - return [ - .prebuildCommand( - displayName: "Building CloudSync XCFramework", - executable: URL(fileURLWithPath: "/bin/bash"), - arguments: [ - "-c", - """ - set -e - echo "Starting CloudSync XCFramework prebuild..." - echo "\(packageDirInfo)" - - # Clean and create output directory - rm -rf "\(outputDirectory.path)" - mkdir -p "\(outputDirectory.path)" - - # Build directly from source directory with custom output paths - cd "\(workingDirectory)" && \ - echo "Building XCFramework with native network..." && \ - make xcframework NATIVE_NETWORK=ON DIST_DIR="\(outputDirectory.path)" BUILD_RELEASE="\(outputDirectory.path)/build/release" BUILD_TEST="\(outputDirectory.path)/build/test" && \ - rm -rf "\(outputDirectory.path)/build" && \ - echo "XCFramework build completed successfully!" - """ - ], - outputFilesDirectory: outputDirectory - ) - ] -} \ No newline at end of file diff --git a/scripts/check-postgres-migration.sh b/scripts/check-postgres-migration.sh new file mode 100755 index 00000000..f14acd9b --- /dev/null +++ b/scripts/check-postgres-migration.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# +# Enforce the PostgreSQL extension versioning contract on every PR/push. +# +# The extension version (default_version in cloudsync.control, and the +# cloudsync--.sql filename) is MAJOR.MINOR only — it's derived from the +# first two components of CLOUDSYNC_VERSION in src/cloudsync.h. The full +# semver of the binary is reported by the cloudsync_version() SQL function. +# +# Contract: +# - PATCH bumps (e.g. 1.0.16 -> 1.0.17) keep EXTVERSION the same ('1.0'). +# Binary-only release; no SQL surface changes allowed; no user action +# needed after swapping the .so. +# - MINOR / MAJOR bumps (e.g. 1.0.x -> 1.1.0 or 1.x -> 2.0) move EXTVERSION. +# Must ship a matching cloudsync----.sql upgrade script so +# existing deployments can ALTER EXTENSION cloudsync UPDATE. +# +# This script runs in one of two modes depending on whether EXTVERSION moved +# since the most recent ancestor semver tag: +# +# (a) EXTVERSION unchanged (patch release): +# diff the current cloudsync.sql.in against the previous tag's install +# script. If they differ, fail: SQL surface changed without a MINOR +# bump, which would silently break users whose pg_extension.extversion +# stays at the old value. +# +# (b) EXTVERSION changed (minor/major release): +# require src/postgresql/migrations/cloudsync----.sql. +# +# Exit codes: +# 0 - contract satisfied (no bump, or bump with migration present) +# 1 - contract violated (SQL drift without bump, or missing migration) +# 2 - misconfigured environment (no git, missing header, unresolvable tag) + +set -euo pipefail + +repo_root=$(git rev-parse --show-toplevel 2>/dev/null || true) +if [ -z "$repo_root" ]; then + echo "Error: not inside a git working tree; cannot determine previous release tag." >&2 + exit 2 +fi +cd "$repo_root" + +header="src/cloudsync.h" +sql_template="src/postgresql/cloudsync.sql.in" + +[ -f "$header" ] || { echo "Error: $header not found." >&2; exit 2; } +[ -f "$sql_template" ] || { echo "Error: $sql_template not found." >&2; exit 2; } + +# Read full semver from the header and derive MAJOR.MINOR. +current_full=$(sed -n 's/^#define CLOUDSYNC_VERSION[[:space:]]*"\([^"]*\)".*/\1/p' "$header") +if [ -z "$current_full" ]; then + echo "Error: could not read CLOUDSYNC_VERSION from $header." >&2 + exit 2 +fi +current_ext=$(printf '%s\n' "$current_full" | cut -d. -f1-2) +if [ -z "$current_ext" ]; then + echo "Error: could not derive MAJOR.MINOR from CLOUDSYNC_VERSION '$current_full'." >&2 + exit 2 +fi + +# Find the latest ancestor semver tag. +prev_tag=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*' 2>/dev/null || true) +if [ -z "$prev_tag" ]; then + echo "No prior semver tag reachable from HEAD; skipping migration check." + echo "(This is expected on an initial release or a shallow clone without tags.)" + exit 0 +fi + +# Resolve what EXTVERSION the previous release shipped with. +# Pre-new-scheme tags: tracked cloudsync.control held default_version. +# New-scheme tags: control is generated; read CLOUDSYNC_VERSION from header +# at that tag and truncate. +prev_full=$(git show "${prev_tag}:${header}" 2>/dev/null \ + | sed -n 's/^#define CLOUDSYNC_VERSION[[:space:]]*"\([^"]*\)".*/\1/p' || true) + +prev_ext=$(git show "${prev_tag}:docker/postgresql/cloudsync.control" 2>/dev/null \ + | sed -n "s/^default_version = '\\([^']*\\)'.*/\\1/p" \ + | head -1 || true) + +if [ -z "$prev_ext" ] && [ -n "$prev_full" ]; then + prev_ext=$(printf '%s\n' "$prev_full" | cut -d. -f1-2) +fi + +if [ -z "$prev_ext" ]; then + echo "Error: could not determine EXTVERSION at tag ${prev_tag}." >&2 + exit 2 +fi + +# --------------------------------------------------------------------------- +# Mode (a): EXTVERSION unchanged — verify SQL surface didn't drift. +# --------------------------------------------------------------------------- +if [ "$prev_ext" = "$current_ext" ]; then + # Produce a normalized view of the previous release's install script. + if git cat-file -e "${prev_tag}:${sql_template}" 2>/dev/null; then + # New-scheme tag: substitute @EXTVERSION@ with that tag's EXTVERSION. + prev_sql=$(git show "${prev_tag}:${sql_template}" \ + | sed "s/@EXTVERSION@/${prev_ext}/g") + else + # Old-scheme tag: the literal cloudsync--.sql was tracked. + prev_install_path="src/postgresql/cloudsync--${prev_ext}.sql" + if ! git cat-file -e "${prev_tag}:${prev_install_path}" 2>/dev/null; then + echo "Error: could not find previous install script at ${prev_tag}:${prev_install_path}." >&2 + exit 2 + fi + prev_sql=$(git show "${prev_tag}:${prev_install_path}") + fi + + # Current install script, rendered (substitute @EXTVERSION@ -> current_ext). + curr_sql=$(sed "s/@EXTVERSION@/${current_ext}/g" "$sql_template") + + if [ "$prev_sql" = "$curr_sql" ]; then + echo "OK: patch-only release candidate." + echo " EXTVERSION unchanged at '${current_ext}' since ${prev_tag}." + echo " Binary semver: ${prev_full:-unknown} -> ${current_full}." + echo " SQL surface identical; no migration script required." + exit 0 + fi + + cat >&2 < ${current_full} + +The install script (src/postgresql/cloudsync.sql.in) differs from what the +previous release shipped, but EXTVERSION is still '${current_ext}'. Existing +deployments have pg_extension.extversion = '${current_ext}' and will NOT run +any upgrade script when they swap in the new .so, so the new SQL bindings +won't be applied to their catalog. This silently breaks users. + +Pick one: + + 1. Bump MINOR in src/cloudsync.h (e.g. 1.0.x -> 1.1.0), and add + src/postgresql/migrations/cloudsync--${current_ext}--.sql + with the DDL deltas (CREATE OR REPLACE FUNCTION ..., etc.). This is the + correct choice for any intentional SQL-surface change. + + 2. Revert the SQL-level change in cloudsync.sql.in if it was accidental + (e.g. a refactor that went further than intended). + +Diff (previous -> current, normalized): + +EOF + # Show a readable diff; fall back to a terse message if diff is unavailable. + if command -v diff >/dev/null 2>&1; then + diff -u <(printf '%s\n' "$prev_sql") <(printf '%s\n' "$curr_sql") >&2 || true + else + echo "(install 'diff' to see line-level changes)" >&2 + fi + exit 1 +fi + +# --------------------------------------------------------------------------- +# Mode (b): EXTVERSION changed — require a migration file. +# --------------------------------------------------------------------------- +expected="src/postgresql/migrations/cloudsync--${prev_ext}--${current_ext}.sql" +if [ ! -f "$expected" ]; then + cat >&2 < ${current_ext} covered)." +echo " Binary semver: ${prev_full:-unknown} -> ${current_full}." diff --git a/src/block.c b/src/block.c new file mode 100644 index 00000000..dd99b266 --- /dev/null +++ b/src/block.c @@ -0,0 +1,303 @@ +// +// block.c +// cloudsync +// +// Block-level LWW CRDT support for text/blob fields. +// + +#include +#include +#include +#include "block.h" +#include "utils.h" +#include "fractional_indexing.h" + +// MARK: - Col name helpers - + +bool block_is_block_colname(const char *col_name) { + if (!col_name) return false; + return strchr(col_name, BLOCK_SEPARATOR) != NULL; +} + +char *block_extract_base_colname(const char *col_name) { + if (!col_name) return NULL; + const char *sep = strchr(col_name, BLOCK_SEPARATOR); + if (!sep) return cloudsync_string_dup(col_name); + return cloudsync_string_ndup(col_name, (size_t)(sep - col_name)); +} + +const char *block_extract_position_id(const char *col_name) { + if (!col_name) return NULL; + const char *sep = strchr(col_name, BLOCK_SEPARATOR); + if (!sep) return NULL; + return sep + 1; +} + +char *block_build_colname(const char *base_col, const char *position_id) { + if (!base_col || !position_id) return NULL; + size_t blen = strlen(base_col); + size_t plen = strlen(position_id); + char *result = (char *)cloudsync_memory_alloc(blen + 1 + plen + 1); + if (!result) return NULL; + memcpy(result, base_col, blen); + result[blen] = BLOCK_SEPARATOR; + memcpy(result + blen + 1, position_id, plen); + result[blen + 1 + plen] = '\0'; + return result; +} + +// MARK: - Text splitting - + +static block_list_t *block_list_create(void) { + block_list_t *list = (block_list_t *)cloudsync_memory_zeroalloc(sizeof(block_list_t)); + return list; +} + +static bool block_list_append(block_list_t *list, const char *content, size_t content_len, const char *position_id) { + if (list->count >= list->capacity) { + int new_cap = list->capacity ? list->capacity * 2 : 16; + block_entry_t *new_entries = (block_entry_t *)cloudsync_memory_realloc( + list->entries, (uint64_t)(new_cap * sizeof(block_entry_t))); + if (!new_entries) return false; + list->entries = new_entries; + list->capacity = new_cap; + } + block_entry_t *e = &list->entries[list->count]; + e->content = cloudsync_string_ndup(content, content_len); + e->position_id = position_id ? cloudsync_string_dup(position_id) : NULL; + if (!e->content) return false; + list->count++; + return true; +} + +void block_list_free(block_list_t *list) { + if (!list) return; + for (int i = 0; i < list->count; i++) { + if (list->entries[i].content) cloudsync_memory_free(list->entries[i].content); + if (list->entries[i].position_id) cloudsync_memory_free(list->entries[i].position_id); + } + if (list->entries) cloudsync_memory_free(list->entries); + cloudsync_memory_free(list); +} + +block_list_t *block_list_create_empty(void) { + return block_list_create(); +} + +bool block_list_add(block_list_t *list, const char *content, const char *position_id) { + if (!list) return false; + return block_list_append(list, content, strlen(content), position_id); +} + +block_list_t *block_split(const char *text, const char *delimiter) { + block_list_t *list = block_list_create(); + if (!list) return NULL; + + if (!text || !*text) { + // Empty text produces a single empty block + block_list_append(list, "", 0, NULL); + return list; + } + + size_t dlen = strlen(delimiter); + if (dlen == 0) { + // No delimiter: entire text is one block + block_list_append(list, text, strlen(text), NULL); + return list; + } + + const char *start = text; + const char *found; + while ((found = strstr(start, delimiter)) != NULL) { + size_t seg_len = (size_t)(found - start); + if (!block_list_append(list, start, seg_len, NULL)) { + block_list_free(list); + return NULL; + } + start = found + dlen; + } + // Last segment (after last delimiter or entire string if no delimiter found) + if (!block_list_append(list, start, strlen(start), NULL)) { + block_list_free(list); + return NULL; + } + + return list; +} + +// MARK: - Fractional indexing (via fractional-indexing submodule) - + +// Wrapper for malloc: fractional_indexing expects size_t (32-bit on WASM) but cloudsync_memory_alloc takes uint64_t. +// A direct cast passes strict call_indirect type checking in WASM and triggers a RuntimeError. +static void *fi_malloc_wrapper(size_t size) { + return cloudsync_memory_alloc((uint64_t)size); +} + +// Wrapper for calloc: fractional_indexing expects (count, size) but cloudsync_memory_zeroalloc takes a single size. +static void *fi_calloc_wrapper(size_t count, size_t size) { + return cloudsync_memory_zeroalloc((uint64_t)(count * size)); +} + +void block_init_allocator(void) { + fractional_indexing_allocator alloc = { + .malloc = fi_malloc_wrapper, + .calloc = fi_calloc_wrapper, + .free = cloudsync_memory_free + }; + fractional_indexing_set_allocator(&alloc); +} + +char *block_position_between(const char *before, const char *after) { + return generate_key_between(before, after); +} + +char **block_initial_positions(int count) { + if (count <= 0) return NULL; + return generate_n_keys_between(NULL, NULL, count); +} + +// MARK: - Block diff - + +static block_diff_t *block_diff_create(void) { + block_diff_t *diff = (block_diff_t *)cloudsync_memory_zeroalloc(sizeof(block_diff_t)); + return diff; +} + +static bool block_diff_append(block_diff_t *diff, block_diff_type type, const char *position_id, const char *content) { + if (diff->count >= diff->capacity) { + int new_cap = diff->capacity ? diff->capacity * 2 : 16; + block_diff_entry_t *new_entries = (block_diff_entry_t *)cloudsync_memory_realloc( + diff->entries, (uint64_t)(new_cap * sizeof(block_diff_entry_t))); + if (!new_entries) return false; + diff->entries = new_entries; + diff->capacity = new_cap; + } + block_diff_entry_t *e = &diff->entries[diff->count]; + e->type = type; + e->position_id = cloudsync_string_dup(position_id); + e->content = content ? cloudsync_string_dup(content) : NULL; + diff->count++; + return true; +} + +void block_diff_free(block_diff_t *diff) { + if (!diff) return; + for (int i = 0; i < diff->count; i++) { + if (diff->entries[i].position_id) cloudsync_memory_free(diff->entries[i].position_id); + if (diff->entries[i].content) cloudsync_memory_free(diff->entries[i].content); + } + if (diff->entries) cloudsync_memory_free(diff->entries); + cloudsync_memory_free(diff); +} + +// Content-based matching diff algorithm: +// 1. Build a consumed-set from old blocks +// 2. For each new block, find the first unconsumed old block with matching content +// 3. Matched blocks keep their position_id (UNCHANGED) +// 4. Unmatched new blocks get new position_ids (ADDED) +// 5. Unconsumed old blocks are REMOVED +// Modified blocks are detected when content changed but position stayed (handled as MODIFIED) +block_diff_t *block_diff(block_entry_t *old_blocks, int old_count, + const char **new_parts, int new_count) { + block_diff_t *diff = block_diff_create(); + if (!diff) return NULL; + + // Track which old blocks have been consumed + bool *old_consumed = NULL; + if (old_count > 0) { + old_consumed = (bool *)cloudsync_memory_zeroalloc((uint64_t)(old_count * sizeof(bool))); + if (!old_consumed) { + block_diff_free(diff); + return NULL; + } + } + + // For each new block, try to find a matching unconsumed old block + // Use a simple forward scan to preserve ordering + int old_scan = 0; + char *last_position = NULL; + + for (int ni = 0; ni < new_count; ni++) { + bool found = false; + + // Scan forward in old blocks for a content match + for (int oi = old_scan; oi < old_count; oi++) { + if (old_consumed[oi]) continue; + + if (strcmp(old_blocks[oi].content, new_parts[ni]) == 0) { + // Exact match — mark any skipped old blocks as REMOVED + for (int si = old_scan; si < oi; si++) { + if (!old_consumed[si]) { + block_diff_append(diff, BLOCK_DIFF_REMOVED, old_blocks[si].position_id, NULL); + old_consumed[si] = true; + } + } + old_consumed[oi] = true; + old_scan = oi + 1; + last_position = old_blocks[oi].position_id; + found = true; + break; + } + } + + if (!found) { + // New block — needs a new position_id + const char *next_pos = NULL; + // Find the next unconsumed old block's position for the upper bound + for (int oi = old_scan; oi < old_count; oi++) { + if (!old_consumed[oi]) { + next_pos = old_blocks[oi].position_id; + break; + } + } + + char *new_pos = block_position_between(last_position, next_pos); + if (new_pos) { + block_diff_append(diff, BLOCK_DIFF_ADDED, new_pos, new_parts[ni]); + last_position = diff->entries[diff->count - 1].position_id; + cloudsync_memory_free(new_pos); + } + } + } + + // Mark remaining unconsumed old blocks as REMOVED + for (int oi = old_scan; oi < old_count; oi++) { + if (!old_consumed[oi]) { + block_diff_append(diff, BLOCK_DIFF_REMOVED, old_blocks[oi].position_id, NULL); + } + } + + if (old_consumed) cloudsync_memory_free(old_consumed); + return diff; +} + +// MARK: - Materialization - + +char *block_materialize_text(const char **blocks, int count, const char *delimiter) { + if (count == 0) return cloudsync_string_dup(""); + if (!delimiter) delimiter = BLOCK_DEFAULT_DELIMITER; + + size_t dlen = strlen(delimiter); + size_t total = 0; + for (int i = 0; i < count; i++) { + total += strlen(blocks[i]); + if (i < count - 1) total += dlen; + } + + char *result = (char *)cloudsync_memory_alloc(total + 1); + if (!result) return NULL; + + size_t offset = 0; + for (int i = 0; i < count; i++) { + size_t blen = strlen(blocks[i]); + memcpy(result + offset, blocks[i], blen); + offset += blen; + if (i < count - 1) { + memcpy(result + offset, delimiter, dlen); + offset += dlen; + } + } + result[offset] = '\0'; + + return result; +} diff --git a/src/block.h b/src/block.h new file mode 100644 index 00000000..fa43369b --- /dev/null +++ b/src/block.h @@ -0,0 +1,120 @@ +// +// block.h +// cloudsync +// +// Block-level LWW CRDT support for text/blob fields. +// Instead of replacing an entire text column on conflict, +// the text is split into blocks (lines/paragraphs) that are +// independently version-tracked and merged. +// + +#ifndef __CLOUDSYNC_BLOCK__ +#define __CLOUDSYNC_BLOCK__ + +#include +#include +#include + +// The separator character used in col_name to distinguish block entries +// from regular column entries. Format: "col_name\x1Fposition_id" +#define BLOCK_SEPARATOR '\x1F' +#define BLOCK_SEPARATOR_STR "\x1F" +#define BLOCK_DEFAULT_DELIMITER "\n" + +// Column-level algorithm for block tracking +typedef enum { + col_algo_normal = 0, + col_algo_block = 1 +} col_algo_t; + +// A single block from splitting text +typedef struct { + char *content; // block text (owned, must be freed) + char *position_id; // fractional index position (owned, must be freed) +} block_entry_t; + +// Array of blocks +typedef struct { + block_entry_t *entries; + int count; + int capacity; +} block_list_t; + +// Diff result for comparing old and new block lists +typedef enum { + BLOCK_DIFF_UNCHANGED = 0, + BLOCK_DIFF_ADDED = 1, + BLOCK_DIFF_MODIFIED = 2, + BLOCK_DIFF_REMOVED = 3 +} block_diff_type; + +typedef struct { + block_diff_type type; + char *position_id; // the position_id (owned, must be freed) + char *content; // new content (owned, must be freed; NULL for REMOVED) +} block_diff_entry_t; + +typedef struct { + block_diff_entry_t *entries; + int count; + int capacity; +} block_diff_t; + +// Initialize the fractional-indexing library to use cloudsync's allocator. +// Must be called once before any block_position_between / block_initial_positions calls. +void block_init_allocator(void); + +// Check if a col_name is a block entry (contains BLOCK_SEPARATOR) +bool block_is_block_colname(const char *col_name); + +// Extract the base column name from a block col_name (caller must free) +// e.g., "body\x1F0.5" -> "body" +char *block_extract_base_colname(const char *col_name); + +// Extract the position_id from a block col_name +// e.g., "body\x1F0.5" -> "0.5" +const char *block_extract_position_id(const char *col_name); + +// Build a block col_name from base + position_id (caller must free) +// e.g., ("body", "0.5") -> "body\x1F0.5" +char *block_build_colname(const char *base_col, const char *position_id); + +// Split text into blocks using the given delimiter +block_list_t *block_split(const char *text, const char *delimiter); + +// Free a block list +void block_list_free(block_list_t *list); + +// Generate fractional index position IDs for N initial blocks +// Returns array of N strings (caller must free each + the array) +char **block_initial_positions(int count); + +// Generate a position ID that sorts between 'before' and 'after' +// Either can be NULL (meaning beginning/end of sequence) +// Caller must free the result +char *block_position_between(const char *before, const char *after); + +// Compute diff between old blocks (with position IDs) and new content blocks +// old_blocks: existing blocks from metadata (with position_ids) +// new_parts: new text split by delimiter (no position_ids yet) +// new_count: number of new parts +block_diff_t *block_diff(block_entry_t *old_blocks, int old_count, + const char **new_parts, int new_count); + +// Free a diff result +void block_diff_free(block_diff_t *diff); + +// Create an empty block list (for accumulating existing blocks) +block_list_t *block_list_create_empty(void); + +// Add a block entry to a list (content and position_id are copied) +bool block_list_add(block_list_t *list, const char *content, const char *position_id); + +// Concatenate block values with delimiter +// blocks: array of content strings (in position order) +// count: number of blocks +// delimiter: separator between blocks +// Returns allocated string (caller must free) +char *block_materialize_text(const char **blocks, int count, const char *delimiter); + +#endif diff --git a/src/cloudsync.c b/src/cloudsync.c index cb955c4e..6cc1f01e 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -15,18 +15,15 @@ #include #include #include +#include #include "cloudsync.h" -#include "cloudsync_private.h" #include "lz4.h" #include "pk.h" -#include "vtab.h" +#include "sql.h" #include "utils.h" #include "dbutils.h" - -#ifndef CLOUDSYNC_OMIT_NETWORK -#include "network.h" -#endif +#include "block.h" #ifdef _WIN32 #include @@ -51,35 +48,26 @@ #endif #endif -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT1 -#endif - -#ifndef UNUSED_PARAMETER -#define UNUSED_PARAMETER(X) (void)(X) -#endif - -#ifdef _WIN32 -#define APIEXPORT __declspec(dllexport) -#else -#define APIEXPORT -#endif - -#define CLOUDSYNC_DEFAULT_ALGO "cls" -#define CLOUDSYNC_INIT_NTABLES 128 -#define CLOUDSYNC_VALUE_NOTSET -1 +#define CLOUDSYNC_INIT_NTABLES 64 #define CLOUDSYNC_MIN_DB_VERSION 0 -#define CLOUDSYNC_PAYLOAD_MINBUF_SIZE 512*1024 -#define CLOUDSYNC_PAYLOAD_VERSION 1 -#define CLOUDSYNC_PAYLOAD_SIGNATURE 'CLSY' -#define CLOUDSYNC_PAYLOAD_APPLY_CALLBACK_KEY "cloudsync_payload_apply_callback" +#define CLOUDSYNC_PAYLOAD_MINBUF_SIZE (512*1024) +#define CLOUDSYNC_PAYLOAD_SIGNATURE 0x434C5359 /* 'C','L','S','Y' */ +#define CLOUDSYNC_PAYLOAD_VERSION_ORIGNAL 1 +#define CLOUDSYNC_PAYLOAD_VERSION_1 CLOUDSYNC_PAYLOAD_VERSION_ORIGNAL +#define CLOUDSYNC_PAYLOAD_VERSION_2 2 +#define CLOUDSYNC_PAYLOAD_VERSION_3 3 +#define CLOUDSYNC_PAYLOAD_VERSION_LATEST CLOUDSYNC_PAYLOAD_VERSION_2 +#define CLOUDSYNC_PAYLOAD_MIN_VERSION_WITH_CHECKSUM CLOUDSYNC_PAYLOAD_VERSION_2 +#define CLOUDSYNC_PAYLOAD_FRAGMENT_PREFIX "__cloudsync_frag_v1__:" +#define CLOUDSYNC_PAYLOAD_FRAGMENT_STALE_SECONDS (24*60*60) +#define CLOUDSYNC_PAYLOAD_FRAGMENT_CLEANUP_MIN_INTERVAL (60) #ifndef MAX #define MAX(a, b) (((a)>(b))?(a):(b)) #endif -#define DEBUG_SQLITE_ERROR(_rc, _fn, _db) do {if (_rc != SQLITE_OK) printf("Error in %s: %s\n", _fn, sqlite3_errmsg(_db));} while (0) +#define DEBUG_DBERROR(_rc, _fn, _data) do {if (_rc != DBRES_OK) printf("Error in %s: %s\n", _fn, database_errmsg(_data));} while (0) typedef enum { CLOUDSYNC_PK_INDEX_TBL = 0, @@ -94,121 +82,178 @@ typedef enum { } CLOUDSYNC_PK_INDEX; typedef enum { - CLOUDSYNC_STMT_VALUE_ERROR = -1, - CLOUDSYNC_STMT_VALUE_UNCHANGED = 0, - CLOUDSYNC_STMT_VALUE_CHANGED = 1, -} CLOUDSYNC_STMT_VALUE; - -typedef struct { - sqlite3_context *context; - int index; -} cloudsync_pk_decode_context; + DBVM_VALUE_ERROR = -1, + DBVM_VALUE_UNCHANGED = 0, + DBVM_VALUE_CHANGED = 1, +} DBVM_VALUE; #define SYNCBIT_SET(_data) _data->insync = 1 #define SYNCBIT_RESET(_data) _data->insync = 0 -#define BUMP_SEQ(_data) ((_data)->seq += 1, (_data)->seq - 1) -// MARK: - +// MARK: - Deferred column-batch merge - typedef struct { - table_algo algo; // CRDT algoritm associated to the table - char *name; // table name - char **col_name; // array of column names - sqlite3_stmt **col_merge_stmt; // array of merge insert stmt (indexed by col_name) - sqlite3_stmt **col_value_stmt; // array of column value stmt (indexed by col_name) - int *col_id; // array of column id - int ncols; // number of non primary key cols - int npks; // number of primary key cols - bool enabled; // flag to check if a table is enabled or disabled - #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES - bool rowid_only; // a table with no primary keys other than the implicit rowid - #endif - - char **pk_name; // array of primary key names - - // precompiled statements - sqlite3_stmt *meta_pkexists_stmt; // check if a primary key already exist in the augmented table - sqlite3_stmt *meta_sentinel_update_stmt; // update a local sentinel row - sqlite3_stmt *meta_sentinel_insert_stmt; // insert a local sentinel row - sqlite3_stmt *meta_row_insert_update_stmt; // insert/update a local row - sqlite3_stmt *meta_row_drop_stmt; // delete rows from meta - sqlite3_stmt *meta_update_move_stmt; // update rows in meta when pk changes - sqlite3_stmt *meta_local_cl_stmt; // compute local cl value - sqlite3_stmt *meta_winner_clock_stmt; // get the rowid of the last inserted/updated row in the meta table - sqlite3_stmt *meta_merge_delete_drop; - sqlite3_stmt *meta_zero_clock_stmt; - sqlite3_stmt *meta_col_version_stmt; - sqlite3_stmt *meta_site_id_stmt; - - sqlite3_stmt *real_col_values_stmt; // retrieve all column values based on pk - sqlite3_stmt *real_merge_delete_stmt; - sqlite3_stmt *real_merge_sentinel_stmt; - - bool is_altering; - -} cloudsync_table_context; + const char *col_name; // pointer into table_context->col_name[idx] (stable) + dbvalue_t *col_value; // duplicated via database_value_dup (owned) + int64_t col_version; + int64_t db_version; + uint8_t site_id[UUID_LEN]; + int site_id_len; + int64_t seq; +} merge_pending_entry; + +typedef struct { + cloudsync_table_context *table; + char *pk; // malloc'd copy, freed on flush + int pk_len; + int64_t cl; + bool sentinel_pending; + bool row_exists; // true when the PK already exists locally + int count; + int capacity; + merge_pending_entry *entries; + + // Statement cache — reuse the prepared statement when the column + // combination and row_exists flag match between consecutive PK flushes. + dbvm_t *cached_vm; + bool cached_row_exists; + int cached_col_count; + const char **cached_col_names; // array of pointers into table_context (not owned) +} merge_pending_batch; + +// MARK: - struct cloudsync_pk_decode_bind_context { - sqlite3_stmt *vm; - char *tbl; - int64_t tbl_len; - const void *pk; - int64_t pk_len; - char *col_name; - int64_t col_name_len; - int64_t col_version; - int64_t db_version; - const void *site_id; - int64_t site_id_len; - int64_t cl; - int64_t seq; + dbvm_t *vm; + char *tbl; + int64_t tbl_len; + const void *pk; + int64_t pk_len; + char *col_name; + int64_t col_name_len; + int64_t col_version; + int64_t db_version; + const void *site_id; + int64_t site_id_len; + int64_t cl; + int64_t seq; }; struct cloudsync_context { - sqlite3_context *sqlite_ctx; + void *db; + char errmsg[1024]; + int errcode; - char *libversion; - uint8_t site_id[UUID_LEN]; - int insync; - int debug; - bool merge_equal_values; - bool temp_bool; // temporary value used in callback - void *aux_data; + char *libversion; + uint8_t site_id[UUID_LEN]; + int insync; + int debug; + bool merge_equal_values; + void *aux_data; // stmts and context values - bool pragma_checked; // we need to check PRAGMAs only once per transaction - sqlite3_stmt *schema_version_stmt; - sqlite3_stmt *data_version_stmt; - sqlite3_stmt *db_version_stmt; - sqlite3_stmt *getset_siteid_stmt; - int data_version; - int schema_version; - uint64_t schema_hash; - - // set at the start of each transaction on the first invocation and - // re-set on transaction commit or rollback - sqlite3_int64 db_version; - // the version that the db will be set to at the end of the transaction - // if that transaction were to commit at the time this value is checked - sqlite3_int64 pending_db_version; + dbvm_t *schema_version_stmt; + dbvm_t *data_version_stmt; + dbvm_t *db_version_stmt; + dbvm_t *getset_siteid_stmt; + int data_version; + int schema_version; + uint64_t schema_hash; + + // set at transaction start and reset on commit/rollback + int64_t db_version; + // version the DB would have if the transaction committed now + int64_t pending_db_version; // used to set an order inside each transaction - int seq; - - // augmented tables are stored in-memory so we do not need to retrieve information about col names and cid - // from the disk each time a write statement is performed - // we do also not need to use an hash map here because for few tables the direct in-memory comparison with table name is faster - cloudsync_table_context **tables; - int tables_count; - int tables_alloc; + int seq; + + // wall-clock (time()) of the last stale v3-fragment GC; throttles the GC so + // it does not run a full table scan on every applied fragment (0 = never run) + int64_t last_fragment_cleanup; + + // optional schema_name to be set in the cloudsync_table_context + char *current_schema; + + // augmented tables are stored in-memory so we do not need to retrieve information about + // col_names and cid from the disk each time a write statement is performed + // we do also not need to use an hash map here because for few tables the direct + // in-memory comparison with table name is faster + cloudsync_table_context **tables; // dense vector: [0..tables_count-1] are valid + int tables_count; // size + int tables_cap; // capacity + + int skip_decode_idx; // -1 in sqlite, col_value index in postgresql + + // deferred column-batch merge (active during payload_apply) + merge_pending_batch *pending_batch; + + // last (db_version, seq) successfully applied during the current + // cloudsync_payload_apply call; used to resolve the + // CLOUDSYNC_CHECKPOINT_LAST_APPLIED receive-checkpoint mode (-1 = none yet). + int64_t apply_last_db_version; + int64_t apply_last_seq; }; -typedef struct { +struct cloudsync_table_context { + table_algo algo; // CRDT algoritm associated to the table + char *name; // table name + char *schema; // table schema + char *meta_ref; // schema-qualified meta table name (e.g. "schema"."name_cloudsync") + char *base_ref; // schema-qualified base table name (e.g. "schema"."name") + char **col_name; // array of column names + dbvm_t **col_merge_stmt; // array of merge insert stmt (indexed by col_name) + dbvm_t **col_value_stmt; // array of column value stmt (indexed by col_name) + int *col_id; // array of column id + col_algo_t *col_algo; // per-column algorithm (normal or block) + char **col_delimiter; // per-column delimiter for block splitting (NULL = default "\n") + bool has_block_cols; // quick check: does this table have any block columns? + dbvm_t *block_value_read_stmt; // SELECT col_value FROM blocks table + dbvm_t *block_value_write_stmt; // INSERT OR REPLACE into blocks table + dbvm_t *block_value_delete_stmt; // DELETE from blocks table + dbvm_t *block_list_stmt; // SELECT block entries for materialization + char *blocks_ref; // schema-qualified blocks table name + int ncols; // number of non primary key cols + int npks; // number of primary key cols + bool enabled; // flag to check if a table is enabled or disabled + #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES + bool rowid_only; // a table with no primary keys other than the implicit rowid + #endif + + char **pk_name; // array of primary key names + + // precompiled statements + dbvm_t *meta_pkexists_stmt; // check if a primary key already exist in the augmented table + dbvm_t *meta_sentinel_update_stmt; // update a local sentinel row + dbvm_t *meta_sentinel_insert_stmt; // insert a local sentinel row + dbvm_t *meta_row_insert_update_stmt; // insert/update a local row + dbvm_t *meta_row_drop_stmt; // delete rows from meta + dbvm_t *meta_update_move_stmt; // update rows in meta when pk changes + dbvm_t *meta_local_cl_stmt; // compute local cl value + dbvm_t *meta_winner_clock_stmt; // get the rowid of the last inserted/updated row in the meta table + dbvm_t *meta_merge_delete_drop; + dbvm_t *meta_zero_clock_stmt; + dbvm_t *meta_col_version_stmt; + dbvm_t *meta_site_id_stmt; + + dbvm_t *real_col_values_stmt; // retrieve all column values based on pk + dbvm_t *real_merge_delete_stmt; + dbvm_t *real_merge_sentinel_stmt; + + bool is_altering; // flag to track if a table alteration is in progress + + // context + cloudsync_context *context; +}; + +struct cloudsync_payload_context { char *buffer; + size_t bsize; size_t balloc; size_t bused; uint64_t nrows; uint16_t ncols; -} cloudsync_data_payload; + uint8_t version; +}; #ifdef _MSC_VER #pragma pack(push, 1) // For MSVC: pack struct with 1-byte alignment @@ -218,24 +263,16 @@ typedef struct { #endif typedef struct PACKED { - uint32_t signature; // 'CLSY' - uint8_t version; // protocol version - uint8_t libversion[3]; // major.minor.patch + uint32_t signature; // 'CLSY' + uint8_t version; // protocol version + uint8_t libversion[3]; // major.minor.patch uint32_t expanded_size; uint16_t ncols; uint32_t nrows; uint64_t schema_hash; - uint8_t unused[6]; // padding to ensure the struct is exactly 32 bytes + uint8_t checksum[6]; // 48 bits checksum (to ensure struct is 32 bytes) } cloudsync_payload_header; -typedef struct { - sqlite3_value *table_name; - sqlite3_value **new_values; - sqlite3_value **old_values; - int count; - int capacity; -} cloudsync_update_payload; - #ifdef _MSC_VER #pragma pack(pop) #endif @@ -247,114 +284,106 @@ bool force_uncompressed_blob = false; #define CHECK_FORCE_UNCOMPRESSED_BUFFER() #endif -int db_version_rebuild_stmt (sqlite3 *db, cloudsync_context *data); -int cloudsync_load_siteid (sqlite3 *db, cloudsync_context *data); -int local_mark_insert_or_update_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, const char *col_name, sqlite3_int64 db_version, int seq); +// Internal prototypes +int local_mark_insert_or_update_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *col_name, int64_t db_version, int seq); -// MARK: - STMT Utils - +// MARK: - CRDT algos - -CLOUDSYNC_STMT_VALUE stmt_execute (sqlite3_stmt *stmt, cloudsync_context *data) { - int rc = sqlite3_step(stmt); - if (rc != SQLITE_ROW && rc != SQLITE_DONE) { - if (data) DEBUG_SQLITE_ERROR(rc, "stmt_execute", sqlite3_db_handle(stmt)); - sqlite3_reset(stmt); - return CLOUDSYNC_STMT_VALUE_ERROR; - } +table_algo cloudsync_algo_from_name (const char *algo_name) { + if (algo_name == NULL) return table_algo_none; - CLOUDSYNC_STMT_VALUE result = CLOUDSYNC_STMT_VALUE_CHANGED; + if ((strcasecmp(algo_name, "CausalLengthSet") == 0) || (strcasecmp(algo_name, "cls") == 0)) return table_algo_crdt_cls; + if ((strcasecmp(algo_name, "GrowOnlySet") == 0) || (strcasecmp(algo_name, "gos") == 0)) return table_algo_crdt_gos; + if ((strcasecmp(algo_name, "DeleteWinsSet") == 0) || (strcasecmp(algo_name, "dws") == 0)) return table_algo_crdt_dws; + if ((strcasecmp(algo_name, "AddWinsSet") == 0) || (strcasecmp(algo_name, "aws") == 0)) return table_algo_crdt_aws; + + // if nothing is found + return table_algo_none; +} + +const char *cloudsync_algo_name (table_algo algo) { + switch (algo) { + case table_algo_crdt_cls: return "cls"; + case table_algo_crdt_gos: return "gos"; + case table_algo_crdt_dws: return "dws"; + case table_algo_crdt_aws: return "aws"; + case table_algo_none: return NULL; + } + return NULL; +} + +// MARK: - DBVM Utils - + +DBVM_VALUE dbvm_execute (dbvm_t *stmt, cloudsync_context *data) { + if (!stmt) return DBVM_VALUE_ERROR; + + int rc = databasevm_step(stmt); + if (rc != DBRES_ROW && rc != DBRES_DONE) { + if (data) DEBUG_DBERROR(rc, "stmt_execute", data); + databasevm_reset(stmt); + return DBVM_VALUE_ERROR; + } + + DBVM_VALUE result = DBVM_VALUE_CHANGED; if (stmt == data->data_version_stmt) { - int version = sqlite3_column_int(stmt, 0); + int version = (int)database_column_int(stmt, 0); if (version != data->data_version) { data->data_version = version; } else { - result = CLOUDSYNC_STMT_VALUE_UNCHANGED; + result = DBVM_VALUE_UNCHANGED; } } else if (stmt == data->schema_version_stmt) { - int version = sqlite3_column_int(stmt, 0); + int version = (int)database_column_int(stmt, 0); if (version > data->schema_version) { data->schema_version = version; } else { - result = CLOUDSYNC_STMT_VALUE_UNCHANGED; + result = DBVM_VALUE_UNCHANGED; } } else if (stmt == data->db_version_stmt) { - data->db_version = (rc == SQLITE_DONE) ? CLOUDSYNC_MIN_DB_VERSION : sqlite3_column_int64(stmt, 0); + data->db_version = (rc == DBRES_DONE) ? CLOUDSYNC_MIN_DB_VERSION : database_column_int(stmt, 0); } - sqlite3_reset(stmt); + databasevm_reset(stmt); return result; } -int stmt_count (sqlite3_stmt *stmt, const char *value, size_t len, int type) { +int dbvm_count (dbvm_t *stmt, const char *value, size_t len, int type) { int result = -1; - int rc = SQLITE_OK; + int rc = DBRES_OK; if (value) { - rc = (type == SQLITE_TEXT) ? sqlite3_bind_text(stmt, 1, value, (int)len, SQLITE_STATIC) : sqlite3_bind_blob(stmt, 1, value, (int)len, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = (type == DBTYPE_TEXT) ? databasevm_bind_text(stmt, 1, value, (int)len) : databasevm_bind_blob(stmt, 1, value, len); + if (rc != DBRES_OK) goto cleanup; } - rc = sqlite3_step(stmt); - if (rc == SQLITE_DONE) { + rc = databasevm_step(stmt); + if (rc == DBRES_DONE) { result = 0; - rc = SQLITE_OK; - } else if (rc == SQLITE_ROW) { - result = sqlite3_column_int(stmt, 0); - rc = SQLITE_OK; + rc = DBRES_OK; + } else if (rc == DBRES_ROW) { + result = (int)database_column_int(stmt, 0); + rc = DBRES_OK; } cleanup: - DEBUG_SQLITE_ERROR(rc, "stmt_count", sqlite3_db_handle(stmt)); - sqlite3_reset(stmt); + databasevm_reset(stmt); return result; } -sqlite3_stmt *stmt_reset (sqlite3_stmt *stmt) { - sqlite3_clear_bindings(stmt); - sqlite3_reset(stmt); - return NULL; -} - -int stmts_add_tocontext (sqlite3 *db, cloudsync_context *data) { - DEBUG_DBFUNCTION("cloudsync_add_stmts"); - - if (data->data_version_stmt == NULL) { - const char *sql = "PRAGMA data_version;"; - int rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &data->data_version_stmt, NULL); - DEBUG_STMT("data_version_stmt %p", data->data_version_stmt); - if (rc != SQLITE_OK) return rc; - DEBUG_SQL("data_version_stmt: %s", sql); - } - - if (data->schema_version_stmt == NULL) { - const char *sql = "PRAGMA schema_version;"; - int rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &data->schema_version_stmt, NULL); - DEBUG_STMT("schema_version_stmt %p", data->schema_version_stmt); - if (rc != SQLITE_OK) return rc; - DEBUG_SQL("schema_version_stmt: %s", sql); - } - - if (data->getset_siteid_stmt == NULL) { - // get and set index of the site_id - // in SQLite, we can’t directly combine an INSERT and a SELECT to both insert a row and return an identifier (rowid) in a single statement, - // however, we can use a workaround by leveraging the INSERT statement with ON CONFLICT DO UPDATE and then combining it with RETURNING rowid - const char *sql = "INSERT INTO cloudsync_site_id (site_id) VALUES (?) ON CONFLICT(site_id) DO UPDATE SET site_id = site_id RETURNING rowid;"; - int rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &data->getset_siteid_stmt, NULL); - DEBUG_STMT("getset_siteid_stmt %p", data->getset_siteid_stmt); - if (rc != SQLITE_OK) return rc; - DEBUG_SQL("getset_siteid_stmt: %s", sql); - } - - return db_version_rebuild_stmt(db, data); +void dbvm_reset (dbvm_t *stmt) { + if (!stmt) return; + databasevm_clear_bindings(stmt); + databasevm_reset(stmt); } // MARK: - Database Version - -char *db_version_build_query (sqlite3 *db) { +int cloudsync_dbversion_build_query (cloudsync_context *data, char **sql_out) { // this function must be manually called each time tables changes // because the query plan changes too and it must be re-prepared // unfortunately there is no other way - + // we need to execute a query like: /* SELECT max(version) as version FROM ( @@ -367,77 +396,78 @@ char *db_version_build_query (sqlite3 *db) { SELECT value as version FROM cloudsync_settings WHERE key = 'pre_alter_dbversion' ) */ - + // the good news is that the query can be computed in SQLite without the need to do any extra computation from the host language - const char *sql = "WITH table_names AS (" - "SELECT format('%w', name) as tbl_name " - "FROM sqlite_master " - "WHERE type='table' " - "AND name LIKE '%_cloudsync'" - "), " - "query_parts AS (" - "SELECT 'SELECT max(db_version) as version FROM \"' || tbl_name || '\"' as part FROM table_names" - "), " - "combined_query AS (" - "SELECT GROUP_CONCAT(part, ' UNION ALL ') || ' UNION SELECT value as version FROM cloudsync_settings WHERE key = ''pre_alter_dbversion''' as full_query FROM query_parts" - ") " - "SELECT 'SELECT max(version) as version FROM (' || full_query || ');' FROM combined_query;"; - return dbutils_text_select(db, sql); -} - -int db_version_rebuild_stmt (sqlite3 *db, cloudsync_context *data) { + + *sql_out = NULL; + return database_select_text(data, SQL_DBVERSION_BUILD_QUERY, sql_out); +} + +int cloudsync_dbversion_rebuild (cloudsync_context *data) { if (data->db_version_stmt) { - sqlite3_finalize(data->db_version_stmt); + databasevm_finalize(data->db_version_stmt); data->db_version_stmt = NULL; } - - sqlite3_int64 count = dbutils_table_settings_count_tables(db); - if (count == 0) return SQLITE_OK; - else if (count == -1) { - dbutils_context_result_error(data->sqlite_ctx, "%s", sqlite3_errmsg(db)); - return SQLITE_ERROR; - } - - char *sql = db_version_build_query(db); - if (!sql) return SQLITE_NOMEM; + + int64_t count = dbutils_table_settings_count_tables(data); + if (count == 0) return DBRES_OK; + else if (count == -1) return cloudsync_set_dberror(data); + + char *sql = NULL; + int rc = cloudsync_dbversion_build_query(data, &sql); + if (rc != DBRES_OK) return cloudsync_set_dberror(data); + + // A NULL SQL with rc == OK means the generator produced a NULL row: + // sqlite_master has no *_cloudsync meta-tables (for example, the user + // dropped the base table and its meta-table without calling + // cloudsync_cleanup, leaving stale cloudsync_table_settings rows). + // Treat this the same as count == 0: no prepared statement, db_version + // stays at the minimum and will be rebuilt on the next cloudsync_init. + // Genuine errors from database_select_text are handled above. + if (!sql) return DBRES_OK; DEBUG_SQL("db_version_stmt: %s", sql); - - int rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &data->db_version_stmt, NULL); + + rc = databasevm_prepare(data, sql, (void **)&data->db_version_stmt, DBFLAG_PERSISTENT); DEBUG_STMT("db_version_stmt %p", data->db_version_stmt); cloudsync_memory_free(sql); return rc; } -int db_version_rerun (sqlite3 *db, cloudsync_context *data) { - CLOUDSYNC_STMT_VALUE schema_changed = stmt_execute(data->schema_version_stmt, data); - if (schema_changed == CLOUDSYNC_STMT_VALUE_ERROR) return -1; - - if (schema_changed == CLOUDSYNC_STMT_VALUE_CHANGED) { - int rc = db_version_rebuild_stmt(db, data); - if (rc != SQLITE_OK) return -1; +int cloudsync_dbversion_rerun (cloudsync_context *data) { + DBVM_VALUE schema_changed = dbvm_execute(data->schema_version_stmt, data); + if (schema_changed == DBVM_VALUE_ERROR) return -1; + + if (schema_changed == DBVM_VALUE_CHANGED) { + int rc = cloudsync_dbversion_rebuild(data); + if (rc != DBRES_OK) return -1; } - - CLOUDSYNC_STMT_VALUE rc = stmt_execute(data->db_version_stmt, data); - if (rc == CLOUDSYNC_STMT_VALUE_ERROR) return -1; + + if (!data->db_version_stmt) { + data->db_version = CLOUDSYNC_MIN_DB_VERSION; + return 0; + } + + DBVM_VALUE rc = dbvm_execute(data->db_version_stmt, data); + if (rc == DBVM_VALUE_ERROR) return -1; return 0; } -int db_version_check_uptodate (sqlite3 *db, cloudsync_context *data) { +int cloudsync_dbversion_check_uptodate (cloudsync_context *data) { // perform a PRAGMA data_version to check if some other process write any data - CLOUDSYNC_STMT_VALUE rc = stmt_execute(data->data_version_stmt, data); - if (rc == CLOUDSYNC_STMT_VALUE_ERROR) return -1; + DBVM_VALUE rc = dbvm_execute(data->data_version_stmt, data); + if (rc == DBVM_VALUE_ERROR) return -1; // db_version is already set and there is no need to update it - if (data->db_version != CLOUDSYNC_VALUE_NOTSET && rc == CLOUDSYNC_STMT_VALUE_UNCHANGED) return 0; + if (data->db_version != CLOUDSYNC_VALUE_NOTSET && rc == DBVM_VALUE_UNCHANGED) return 0; - return db_version_rerun(db, data); + return cloudsync_dbversion_rerun(data); } -sqlite3_int64 db_version_next (sqlite3 *db, cloudsync_context *data, sqlite3_int64 merging_version) { - int rc = db_version_check_uptodate(db, data); - if (rc != SQLITE_OK) return -1; +int64_t cloudsync_dbversion_next (cloudsync_context *data, int64_t merging_version) { + int rc = cloudsync_dbversion_check_uptodate(data); + if (rc != DBRES_OK) return -1; - sqlite3_int64 result = data->db_version + 1; + int64_t result = data->db_version + 1; if (result < data->pending_db_version) result = data->pending_db_version; if (merging_version != CLOUDSYNC_VALUE_NOTSET && result < merging_version) result = merging_version; data->pending_db_version = result; @@ -445,18 +475,6 @@ sqlite3_int64 db_version_next (sqlite3 *db, cloudsync_context *data, sqlite3_int return result; } -// MARK: - - -void *cloudsync_get_auxdata (sqlite3_context *context) { - cloudsync_context *data = (context) ? (cloudsync_context *)sqlite3_user_data(context) : NULL; - return (data) ? data->aux_data : NULL; -} - -void cloudsync_set_auxdata (sqlite3_context *context, void *xdata) { - cloudsync_context *data = (context) ? (cloudsync_context *)sqlite3_user_data(context) : NULL; - if (data) data->aux_data = xdata; -} - // MARK: - PK Context - char *cloudsync_pk_context_tbl (cloudsync_pk_decode_bind_context *ctx, int64_t *tbl_len) { @@ -482,152 +500,242 @@ int64_t cloudsync_pk_context_dbversion (cloudsync_pk_decode_bind_context *ctx) { return ctx->db_version; } -// MARK: - Table Utils - +// MARK: - CloudSync Context - -char *table_build_values_sql (sqlite3 *db, cloudsync_table_context *table) { - char *sql = NULL; - - /* - This SQL statement dynamically generates a SELECT query for a specified table. - It uses Common Table Expressions (CTEs) to construct the column names and - primary key conditions based on the table schema, which is obtained through - the `pragma_table_info` function. +int cloudsync_insync (cloudsync_context *data) { + return data->insync; +} + +void *cloudsync_siteid (cloudsync_context *data) { + return (void *)data->site_id; +} - 1. `col_names` CTE: - - Retrieves a comma-separated list of non-primary key column names from - the specified table's schema. +void cloudsync_reset_siteid (cloudsync_context *data) { + memset(data->site_id, 0, sizeof(uint8_t) * UUID_LEN); +} - 2. `pk_where` CTE: - - Retrieves a condition string representing the primary key columns in the - format: "column1=? AND column2=? AND ...", used to create the WHERE clause - for selecting rows based on primary key values. +int cloudsync_load_siteid (cloudsync_context *data) { + // check if site_id was already loaded + if (data->site_id[0] != 0) return DBRES_OK; + + // load site_id + char *buffer = NULL; + int64_t size = 0; + int rc = database_select_blob(data, SQL_SITEID_SELECT_ROWID0, &buffer, &size); + if (rc != DBRES_OK) return rc; + if (!buffer || size != UUID_LEN) { + if (buffer) cloudsync_memory_free(buffer); + return cloudsync_set_error(data, "Unable to retrieve siteid", DBRES_MISUSE); + } + + memcpy(data->site_id, buffer, UUID_LEN); + cloudsync_memory_free(buffer); + + return DBRES_OK; +} - 3. Final SELECT: - - Constructs the complete SELECT statement as a string, combining: - - Column names from `col_names`. - - The target table name. - - The WHERE clause conditions from `pk_where`. +int64_t cloudsync_dbversion (cloudsync_context *data) { + return data->db_version; +} - The resulting query can be used to select rows from the table based on primary - key values, and can be executed within the application to retrieve data dynamically. - */ +int cloudsync_bumpseq (cloudsync_context *data) { + int value = data->seq; + data->seq += 1; + return value; +} - // Unfortunately in SQLite column names (or table names) cannot be bound parameters in a SELECT statement - // otherwise we should have used something like SELECT 'SELECT ? FROM %w WHERE rowid=?'; +void cloudsync_update_schema_hash (cloudsync_context *data) { + database_update_schema_hash(data, &data->schema_hash); +} - char *singlequote_escaped_table_name = cloudsync_memory_mprintf("%q", table->name); +void *cloudsync_db (cloudsync_context *data) { + return data->db; +} - #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES - if (table->rowid_only) { - sql = memory_mprintf("WITH col_names AS (SELECT group_concat('\"' || format('%%w', name) || '\"', ',') AS cols FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid) SELECT 'SELECT ' || (SELECT cols FROM col_names) || ' FROM \"%w\" WHERE rowid=?;'", table->name, table->name); - goto process_process; +int cloudsync_add_dbvms (cloudsync_context *data) { + DEBUG_DBFUNCTION("cloudsync_add_stmts"); + + if (data->data_version_stmt == NULL) { + int rc = databasevm_prepare(data, SQL_DATA_VERSION, (void **)&data->data_version_stmt, DBFLAG_PERSISTENT); + DEBUG_STMT("data_version_stmt %p", data->data_version_stmt); + if (rc != DBRES_OK) return rc; + DEBUG_SQL("data_version_stmt: %s", SQL_DATA_VERSION); } - #endif - sql = cloudsync_memory_mprintf("WITH col_names AS (SELECT group_concat('\"' || format('%%w', name) || '\"', ',') AS cols FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid), pk_where AS (SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk) SELECT 'SELECT ' || (SELECT cols FROM col_names) || ' FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'", table->name, table->name, singlequote_escaped_table_name); + if (data->schema_version_stmt == NULL) { + int rc = databasevm_prepare(data, SQL_SCHEMA_VERSION, (void **)&data->schema_version_stmt, DBFLAG_PERSISTENT); + DEBUG_STMT("schema_version_stmt %p", data->schema_version_stmt); + if (rc != DBRES_OK) return rc; + DEBUG_SQL("schema_version_stmt: %s", SQL_SCHEMA_VERSION); + } -#if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES -process_process: -#endif - cloudsync_memory_free(singlequote_escaped_table_name); - if (!sql) return NULL; - char *query = dbutils_text_select(db, sql); - cloudsync_memory_free(sql); + if (data->getset_siteid_stmt == NULL) { + // get and set index of the site_id + // in SQLite, we can’t directly combine an INSERT and a SELECT to both insert a row and return an identifier (rowid) in a single statement, + // however, we can use a workaround by leveraging the INSERT statement with ON CONFLICT DO UPDATE and then combining it with RETURNING rowid + int rc = databasevm_prepare(data, SQL_SITEID_GETSET_ROWID_BY_SITEID, (void **)&data->getset_siteid_stmt, DBFLAG_PERSISTENT); + DEBUG_STMT("getset_siteid_stmt %p", data->getset_siteid_stmt); + if (rc != DBRES_OK) return rc; + DEBUG_SQL("getset_siteid_stmt: %s", SQL_SITEID_GETSET_ROWID_BY_SITEID); + } + + return cloudsync_dbversion_rebuild(data); +} + +int cloudsync_set_error (cloudsync_context *data, const char *err_user, int err_code) { + // force err_code to be something different than OK + if (err_code == DBRES_OK) err_code = database_errcode(data); + if (err_code == DBRES_OK) err_code = DBRES_ERROR; + + // compute a meaningful error message + if (err_user == NULL) { + snprintf(data->errmsg, sizeof(data->errmsg), "%s", database_errmsg(data)); + } else { + const char *db_error = database_errmsg(data); + char db_error_copy[sizeof(data->errmsg)]; + int rc = database_errcode(data); + if (rc == DBRES_OK) { + snprintf(data->errmsg, sizeof(data->errmsg), "%s", err_user); + } else { + if (db_error == data->errmsg) { + snprintf(db_error_copy, sizeof(db_error_copy), "%s", db_error); + db_error = db_error_copy; + } + snprintf(data->errmsg, sizeof(data->errmsg), "%s (%s)", err_user, db_error); + } + } - return query; + data->errcode = err_code; + return err_code; +} + +int cloudsync_set_dberror (cloudsync_context *data) { + return cloudsync_set_error(data, NULL, DBRES_OK); +} + +const char *cloudsync_errmsg (cloudsync_context *data) { + return data->errmsg; +} + +int cloudsync_errcode (cloudsync_context *data) { + return data->errcode; +} + +void cloudsync_reset_error (cloudsync_context *data) { + data->errmsg[0] = 0; + data->errcode = DBRES_OK; +} + +void *cloudsync_auxdata (cloudsync_context *data) { + return data->aux_data; +} + +void cloudsync_set_auxdata (cloudsync_context *data, void *xdata) { + data->aux_data = xdata; } -char *table_build_mergedelete_sql (sqlite3 *db, cloudsync_table_context *table) { +void cloudsync_set_schema (cloudsync_context *data, const char *schema) { + if (data->current_schema && schema && strcmp(data->current_schema, schema) == 0) return; + if (data->current_schema) cloudsync_memory_free(data->current_schema); + data->current_schema = NULL; + if (schema) data->current_schema = cloudsync_string_dup_lowercase(schema); +} + +const char *cloudsync_schema (cloudsync_context *data) { + return data->current_schema; +} + +const char *cloudsync_table_schema (cloudsync_context *data, const char *table_name) { + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) return NULL; + + return table->schema; +} + +// MARK: - Table Utils - + +void table_pknames_free (char **names, int nrows) { + if (!names) return; + for (int i = 0; i < nrows; ++i) {cloudsync_memory_free(names[i]);} + cloudsync_memory_free(names); +} + +char *table_build_mergedelete_sql (cloudsync_table_context *table) { #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES if (table->rowid_only) { - char *sql = memory_mprintf("DELETE FROM \"%w\" WHERE rowid=?;", table->name); + char *sql = memory_mprintf(SQL_DELETE_ROW_BY_ROWID, table->name); return sql; } #endif - - char *singlequote_escaped_table_name = cloudsync_memory_mprintf("%q", table->name); - char *sql = cloudsync_memory_mprintf("WITH pk_where AS (SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk) SELECT 'DELETE FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'", table->name, singlequote_escaped_table_name); - cloudsync_memory_free(singlequote_escaped_table_name); - if (!sql) return NULL; - - char *query = dbutils_text_select(db, sql); - cloudsync_memory_free(sql); - - return query; + + return sql_build_delete_by_pk(table->context, table->name, table->schema); } -char *table_build_mergeinsert_sql (sqlite3 *db, cloudsync_table_context *table, const char *colname) { +char *table_build_mergeinsert_sql (cloudsync_table_context *table, const char *colname) { char *sql = NULL; #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES if (table->rowid_only) { if (colname == NULL) { // INSERT OR IGNORE INTO customers (first_name,last_name) VALUES (?,?); - sql = memory_mprintf("INSERT OR IGNORE INTO \"%w\" (rowid) VALUES (?);", table->name); + sql = memory_mprintf(SQL_INSERT_ROWID_IGNORE, table->name); } else { // INSERT INTO customers (first_name,last_name,age) VALUES (?,?,?) ON CONFLICT DO UPDATE SET age=?; - sql = memory_mprintf("INSERT INTO \"%w\" (rowid, \"%w\") VALUES (?, ?) ON CONFLICT DO UPDATE SET \"%w\"=?;", table->name, colname, colname); + sql = memory_mprintf(SQL_UPSERT_ROWID_AND_COL_BY_ROWID, table->name, colname, colname); } return sql; } #endif - char *singlequote_escaped_table_name = cloudsync_memory_mprintf("%q", table->name); - if (colname == NULL) { // is sentinel insert - sql = cloudsync_memory_mprintf("WITH pk_where AS (SELECT group_concat('\"' || format('%%w', name) || '\"') AS pk_clause FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk), pk_bind AS (SELECT group_concat('?') AS pk_binding FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk) SELECT 'INSERT OR IGNORE INTO \"%w\" (' || (SELECT pk_clause FROM pk_where) || ') VALUES (' || (SELECT pk_binding FROM pk_bind) || ');'", table->name, table->name, singlequote_escaped_table_name); + sql = sql_build_insert_pk_ignore(table->context, table->name, table->schema); } else { - char *singlequote_escaped_col_name = cloudsync_memory_mprintf("%q", colname); - sql = cloudsync_memory_mprintf("WITH pk_where AS (SELECT group_concat('\"' || format('%%w', name) || '\"') AS pk_clause FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk), pk_bind AS (SELECT group_concat('?') AS pk_binding FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk) SELECT 'INSERT INTO \"%w\" (' || (SELECT pk_clause FROM pk_where) || ',\"%w\") VALUES (' || (SELECT pk_binding FROM pk_bind) || ',?) ON CONFLICT DO UPDATE SET \"%w\"=?;'", table->name, table->name, singlequote_escaped_table_name, singlequote_escaped_col_name, singlequote_escaped_col_name); - cloudsync_memory_free(singlequote_escaped_col_name); - + sql = sql_build_upsert_pk_and_col(table->context, table->name, colname, table->schema); } - cloudsync_memory_free(singlequote_escaped_table_name); - if (!sql) return NULL; - - char *query = dbutils_text_select(db, sql); - cloudsync_memory_free(sql); - - return query; + return sql; } -char *table_build_value_sql (sqlite3 *db, cloudsync_table_context *table, const char *colname) { - char *colnamequote = dbutils_is_star_table(colname) ? "" : "\""; - +char *table_build_value_sql (cloudsync_table_context *table, const char *colname) { #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES if (table->rowid_only) { - char *sql = memory_mprintf("SELECT %s%w%s FROM \"%w\" WHERE rowid=?;", colnamequote, colname, colnamequote, table->name); + char *colnamequote = "\""; + char *sql = memory_mprintf(SQL_SELECT_COLS_BY_ROWID_FMT, colnamequote, colname, colnamequote, table->name); return sql; } #endif // SELECT age FROM customers WHERE first_name=? AND last_name=?; - char *singlequote_escaped_table_name = cloudsync_memory_mprintf("%q", table->name); - char *singlequote_escaped_col_name = cloudsync_memory_mprintf("%q", colname); - char *sql = cloudsync_memory_mprintf("WITH pk_where AS (SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk) SELECT 'SELECT %s%w%s FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'", table->name, colnamequote, singlequote_escaped_col_name, colnamequote, singlequote_escaped_table_name); - cloudsync_memory_free(singlequote_escaped_col_name); - cloudsync_memory_free(singlequote_escaped_table_name); - if (!sql) return NULL; - - char *query = dbutils_text_select(db, sql); - cloudsync_memory_free(sql); - - return query; + return sql_build_select_cols_by_pk(table->context, table->name, colname, table->schema); } -cloudsync_table_context *table_create (const char *name, table_algo algo) { +cloudsync_table_context *table_create (cloudsync_context *data, const char *name, table_algo algo) { DEBUG_DBFUNCTION("table_create %s", name); cloudsync_table_context *table = (cloudsync_table_context *)cloudsync_memory_zeroalloc(sizeof(cloudsync_table_context)); if (!table) return NULL; + table->context = data; table->algo = algo; - table->name = cloudsync_string_dup(name, true); + table->name = cloudsync_string_dup_lowercase(name); + + // Detect schema from metadata table location. If metadata table doesn't + // exist yet (during initialization), fall back to cloudsync_schema() which + // returns the explicitly set schema or current_schema(). + table->schema = database_table_schema(name); + if (!table->schema) { + const char *fallback_schema = cloudsync_schema(data); + if (fallback_schema) { + table->schema = cloudsync_string_dup(fallback_schema); + } + } + if (!table->name) { cloudsync_memory_free(table); return NULL; } + table->meta_ref = database_build_meta_ref(table->schema, table->name); + table->base_ref = database_build_base_ref(table->schema, table->name); table->enabled = true; return table; @@ -637,55 +745,72 @@ void table_free (cloudsync_table_context *table) { DEBUG_DBFUNCTION("table_free %s", (table) ? (table->name) : "NULL"); if (!table) return; - if (table->ncols > 0) { - if (table->col_name) { - for (int i=0; incols; ++i) { - cloudsync_memory_free(table->col_name[i]); - } - cloudsync_memory_free(table->col_name); + if (table->col_name) { + for (int i=0; incols; ++i) { + cloudsync_memory_free(table->col_name[i]); } - if (table->col_merge_stmt) { - for (int i=0; incols; ++i) { - sqlite3_finalize(table->col_merge_stmt[i]); - } - cloudsync_memory_free(table->col_merge_stmt); + cloudsync_memory_free(table->col_name); + } + if (table->col_merge_stmt) { + for (int i=0; incols; ++i) { + databasevm_finalize(table->col_merge_stmt[i]); } - if (table->col_value_stmt) { - for (int i=0; incols; ++i) { - sqlite3_finalize(table->col_value_stmt[i]); - } - cloudsync_memory_free(table->col_value_stmt); + cloudsync_memory_free(table->col_merge_stmt); + } + if (table->col_value_stmt) { + for (int i=0; incols; ++i) { + databasevm_finalize(table->col_value_stmt[i]); } - if (table->col_id) { - cloudsync_memory_free(table->col_id); + cloudsync_memory_free(table->col_value_stmt); + } + if (table->col_id) { + cloudsync_memory_free(table->col_id); + } + if (table->col_algo) { + cloudsync_memory_free(table->col_algo); + } + if (table->col_delimiter) { + for (int i=0; incols; ++i) { + if (table->col_delimiter[i]) cloudsync_memory_free(table->col_delimiter[i]); } + cloudsync_memory_free(table->col_delimiter); } - - if (table->pk_name) sqlite3_free_table(table->pk_name); + + if (table->block_value_read_stmt) databasevm_finalize(table->block_value_read_stmt); + if (table->block_value_write_stmt) databasevm_finalize(table->block_value_write_stmt); + if (table->block_value_delete_stmt) databasevm_finalize(table->block_value_delete_stmt); + if (table->block_list_stmt) databasevm_finalize(table->block_list_stmt); + if (table->blocks_ref) cloudsync_memory_free(table->blocks_ref); + if (table->name) cloudsync_memory_free(table->name); - if (table->meta_pkexists_stmt) sqlite3_finalize(table->meta_pkexists_stmt); - if (table->meta_sentinel_update_stmt) sqlite3_finalize(table->meta_sentinel_update_stmt); - if (table->meta_sentinel_insert_stmt) sqlite3_finalize(table->meta_sentinel_insert_stmt); - if (table->meta_row_insert_update_stmt) sqlite3_finalize(table->meta_row_insert_update_stmt); - if (table->meta_row_drop_stmt) sqlite3_finalize(table->meta_row_drop_stmt); - if (table->meta_update_move_stmt) sqlite3_finalize(table->meta_update_move_stmt); - if (table->meta_local_cl_stmt) sqlite3_finalize(table->meta_local_cl_stmt); - if (table->meta_winner_clock_stmt) sqlite3_finalize(table->meta_winner_clock_stmt); - if (table->meta_merge_delete_drop) sqlite3_finalize(table->meta_merge_delete_drop); - if (table->meta_zero_clock_stmt) sqlite3_finalize(table->meta_zero_clock_stmt); - if (table->meta_col_version_stmt) sqlite3_finalize(table->meta_col_version_stmt); - if (table->meta_site_id_stmt) sqlite3_finalize(table->meta_site_id_stmt); - - if (table->real_col_values_stmt) sqlite3_finalize(table->real_col_values_stmt); - if (table->real_merge_delete_stmt) sqlite3_finalize(table->real_merge_delete_stmt); - if (table->real_merge_sentinel_stmt) sqlite3_finalize(table->real_merge_sentinel_stmt); + if (table->schema) cloudsync_memory_free(table->schema); + if (table->meta_ref) cloudsync_memory_free(table->meta_ref); + if (table->base_ref) cloudsync_memory_free(table->base_ref); + if (table->pk_name) table_pknames_free(table->pk_name, table->npks); + if (table->meta_pkexists_stmt) databasevm_finalize(table->meta_pkexists_stmt); + if (table->meta_sentinel_update_stmt) databasevm_finalize(table->meta_sentinel_update_stmt); + if (table->meta_sentinel_insert_stmt) databasevm_finalize(table->meta_sentinel_insert_stmt); + if (table->meta_row_insert_update_stmt) databasevm_finalize(table->meta_row_insert_update_stmt); + if (table->meta_row_drop_stmt) databasevm_finalize(table->meta_row_drop_stmt); + if (table->meta_update_move_stmt) databasevm_finalize(table->meta_update_move_stmt); + if (table->meta_local_cl_stmt) databasevm_finalize(table->meta_local_cl_stmt); + if (table->meta_winner_clock_stmt) databasevm_finalize(table->meta_winner_clock_stmt); + if (table->meta_merge_delete_drop) databasevm_finalize(table->meta_merge_delete_drop); + if (table->meta_zero_clock_stmt) databasevm_finalize(table->meta_zero_clock_stmt); + if (table->meta_col_version_stmt) databasevm_finalize(table->meta_col_version_stmt); + if (table->meta_site_id_stmt) databasevm_finalize(table->meta_site_id_stmt); + + if (table->real_col_values_stmt) databasevm_finalize(table->real_col_values_stmt); + if (table->real_merge_delete_stmt) databasevm_finalize(table->real_merge_delete_stmt); + if (table->real_merge_sentinel_stmt) databasevm_finalize(table->real_merge_sentinel_stmt); cloudsync_memory_free(table); } -int table_add_stmts (sqlite3 *db, cloudsync_table_context *table, int ncols) { - int rc = SQLITE_OK; +int table_add_stmts (cloudsync_table_context *table, int ncols) { + int rc = DBRES_OK; char *sql = NULL; + cloudsync_context *data = table->context; // META TABLE statements @@ -694,162 +819,160 @@ int table_add_stmts (sqlite3 *db, cloudsync_table_context *table, int ncols) { // precompile the pk exists statement // we do not need an index on the pk column because it is already covered by the fact that it is part of the prikeys // EXPLAIN QUERY PLAN reports: SEARCH table_name USING PRIMARY KEY (pk=?) - sql = cloudsync_memory_mprintf("SELECT EXISTS(SELECT 1 FROM \"%w_cloudsync\" WHERE pk = ? LIMIT 1);", table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_ROW_EXISTS_BY_PK, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_pkexists_stmt: %s", sql); - - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_pkexists_stmt, NULL); - + + rc = databasevm_prepare(data, sql, (void **)&table->meta_pkexists_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; - + if (rc != DBRES_OK) goto cleanup; + // precompile the update local sentinel statement - sql = cloudsync_memory_mprintf("UPDATE \"%w_cloudsync\" SET col_version = CASE col_version %% 2 WHEN 0 THEN col_version + 1 ELSE col_version + 2 END, db_version = ?, seq = ?, site_id = 0 WHERE pk = ? AND col_name = '%s';", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_UPDATE_COL_BUMP_VERSION, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_sentinel_update_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_sentinel_update_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_sentinel_update_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // precompile the insert local sentinel statement - sql = cloudsync_memory_mprintf("INSERT INTO \"%w_cloudsync\" (pk, col_name, col_version, db_version, seq, site_id) SELECT ?, '%s', 1, ?, ?, 0 WHERE 1 ON CONFLICT DO UPDATE SET col_version = CASE col_version %% 2 WHEN 0 THEN col_version + 1 ELSE col_version + 2 END, db_version = ?, seq = ?, site_id = 0;", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_UPSERT_COL_INIT_OR_BUMP_VERSION, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE, table->meta_ref, table->meta_ref, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_sentinel_insert_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_sentinel_insert_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_sentinel_insert_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // precompile the insert/update local row statement - sql = cloudsync_memory_mprintf("INSERT INTO \"%w_cloudsync\" (pk, col_name, col_version, db_version, seq, site_id ) SELECT ?, ?, ?, ?, ?, 0 WHERE 1 ON CONFLICT DO UPDATE SET col_version = col_version + 1, db_version = ?, seq = ?, site_id = 0;", table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_UPSERT_RAW_COLVERSION, table->meta_ref, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_row_insert_update_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_row_insert_update_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_row_insert_update_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // precompile the delete rows from meta - sql = cloudsync_memory_mprintf("DELETE FROM \"%w_cloudsync\" WHERE pk=? AND col_name!='%s';", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_DELETE_PK_EXCEPT_COL, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_row_drop_stmt: %s", sql); - - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_row_drop_stmt, NULL); + + rc = databasevm_prepare(data, sql, (void **)&table->meta_row_drop_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // precompile the update rows from meta when pk changes // see https://github.com/sqliteai/sqlite-sync/blob/main/docs/PriKey.md for more details - sql = cloudsync_memory_mprintf("UPDATE OR REPLACE \"%w_cloudsync\" SET pk=?, db_version=?, col_version=1, seq=cloudsync_seq(), site_id=0 WHERE (pk=? AND col_name!='%s');", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = sql_build_rekey_pk_and_reset_version_except_col(data, table->name, CLOUDSYNC_TOMBSTONE_VALUE); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_update_move_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_update_move_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_update_move_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // local cl - sql = cloudsync_memory_mprintf("SELECT COALESCE((SELECT col_version FROM \"%w_cloudsync\" WHERE pk=? AND col_name='%s'), (SELECT 1 FROM \"%w_cloudsync\" WHERE pk=?));", table->name, CLOUDSYNC_TOMBSTONE_VALUE, table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_GET_COL_VERSION_OR_ROW_EXISTS, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_local_cl_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_local_cl_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_local_cl_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // rowid of the last inserted/updated row in the meta table - sql = cloudsync_memory_mprintf("INSERT OR REPLACE INTO \"%w_cloudsync\" (pk, col_name, col_version, db_version, seq, site_id) VALUES (?, ?, ?, cloudsync_db_version_next(?), ?, ?) RETURNING ((db_version << 30) | seq);", table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_INSERT_RETURN_CHANGE_ID, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_winner_clock_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_winner_clock_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_winner_clock_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; - sql = cloudsync_memory_mprintf("DELETE FROM \"%w_cloudsync\" WHERE pk=? AND col_name!='%s';", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_DELETE_PK_EXCEPT_COL, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_merge_delete_drop: %s", sql); - - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_merge_delete_drop, NULL); + + rc = databasevm_prepare(data, sql, (void **)&table->meta_merge_delete_drop, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // zero clock - sql = cloudsync_memory_mprintf("UPDATE \"%w_cloudsync\" SET col_version = 0, db_version = cloudsync_db_version_next(?) WHERE pk=? AND col_name!='%s';", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_TOMBSTONE_PK_EXCEPT_COL, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_zero_clock_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_zero_clock_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_zero_clock_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // col_version - sql = cloudsync_memory_mprintf("SELECT col_version FROM \"%w_cloudsync\" WHERE pk=? AND col_name=?;", table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_COL_VERSION_BY_PK_COL, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_col_version_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_col_version_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_col_version_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // site_id - sql = cloudsync_memory_mprintf("SELECT site_id FROM \"%w_cloudsync\" WHERE pk=? AND col_name=?;", table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_SITE_ID_BY_PK_COL, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_site_id_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_site_id_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_site_id_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // REAL TABLE statements - + // precompile the get column value statement if (ncols > 0) { - sql = table_build_values_sql(db, table); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = sql_build_select_nonpk_by_pk(data, table->name, table->schema); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("real_col_values_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->real_col_values_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->real_col_values_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; } - sql = table_build_mergedelete_sql(db, table); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = table_build_mergedelete_sql(table); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("real_merge_delete: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->real_merge_delete_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->real_merge_delete_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; - sql = table_build_mergeinsert_sql(db, table, NULL); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = table_build_mergeinsert_sql(table, NULL); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("real_merge_sentinel: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->real_merge_sentinel_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->real_merge_sentinel_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; cleanup: - if (rc != SQLITE_OK) printf("table_add_stmts error: %s\n", sqlite3_errmsg(db)); + if (rc != DBRES_OK) DEBUG_ALWAYS("table_add_stmts error: %d %s\n", rc, database_errmsg(data)); return rc; } cloudsync_table_context *table_lookup (cloudsync_context *data, const char *table_name) { DEBUG_DBFUNCTION("table_lookup %s", table_name); - for (int i=0; itables_count; ++i) { - const char *name = (data->tables[i]) ? data->tables[i]->name : NULL; - if ((name) && (strcasecmp(name, table_name) == 0)) { - return data->tables[i]; + if (table_name) { + for (int i=0; itables_count; ++i) { + if ((strcasecmp(data->tables[i]->name, table_name) == 0)) return data->tables[i]; } } return NULL; } -sqlite3_stmt *table_column_lookup (cloudsync_table_context *table, const char *col_name, bool is_merge, int *index) { +void *table_column_lookup (cloudsync_table_context *table, const char *col_name, bool is_merge, int *index) { DEBUG_DBFUNCTION("table_column_lookup %s", col_name); for (int i=0; incols; ++i) { @@ -863,145 +986,152 @@ sqlite3_stmt *table_column_lookup (cloudsync_table_context *table, const char *c return NULL; } -int table_remove (cloudsync_context *data, const char *table_name) { +int table_remove (cloudsync_context *data, cloudsync_table_context *table) { + const char *table_name = table->name; DEBUG_DBFUNCTION("table_remove %s", table_name); - for (int i=0; itables_count; ++i) { - const char *name = (data->tables[i]) ? data->tables[i]->name : NULL; - if ((name) && (strcasecmp(name, table_name) == 0)) { - data->tables[i] = NULL; - return i; + for (int i = 0; i < data->tables_count; ++i) { + cloudsync_table_context *t = data->tables[i]; + + // pointer compare is fastest but fallback to strcasecmp if not same pointer + if ((t == table) || ((strcasecmp(t->name, table_name) == 0))) { + int last = data->tables_count - 1; + data->tables[i] = data->tables[last]; // move last into the hole (keeps array dense) + data->tables[last] = NULL; // NULLify tail (as an extra security measure) + data->tables_count--; + return data->tables_count; } } + return -1; } int table_add_to_context_cb (void *xdata, int ncols, char **values, char **names) { cloudsync_table_context *table = (cloudsync_table_context *)xdata; - - sqlite3 *db = sqlite3_db_handle(table->meta_pkexists_stmt); - if (!db) return SQLITE_ERROR; - + cloudsync_context *data = table->context; + int index = table->ncols; for (int i=0; icol_id[index] = cid; - table->col_name[index] = cloudsync_string_dup(name, true); - if (!table->col_name[index]) return 1; - - char *sql = table_build_mergeinsert_sql(db, table, name); - if (!sql) return SQLITE_NOMEM; + table->col_name[index] = cloudsync_string_dup_lowercase(name); + if (!table->col_name[index]) goto error; + + char *sql = table_build_mergeinsert_sql(table, name); + if (!sql) goto error; DEBUG_SQL("col_merge_stmt[%d]: %s", index, sql); - - int rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->col_merge_stmt[index], NULL); + + int rc = databasevm_prepare(data, sql, (void **)&table->col_merge_stmt[index], DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) return rc; - if (!table->col_merge_stmt[index]) return SQLITE_MISUSE; - - sql = table_build_value_sql(db, table, name); - if (!sql) return SQLITE_NOMEM; + if (rc != DBRES_OK) goto error; + if (!table->col_merge_stmt[index]) goto error; + + sql = table_build_value_sql(table, name); + if (!sql) goto error; DEBUG_SQL("col_value_stmt[%d]: %s", index, sql); - - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->col_value_stmt[index], NULL); + + rc = databasevm_prepare(data, sql, (void **)&table->col_value_stmt[index], DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) return rc; - if (!table->col_value_stmt[index]) return SQLITE_MISUSE; + if (rc != DBRES_OK) goto error; + if (!table->col_value_stmt[index]) goto error; } table->ncols += 1; - + return 0; + +error: + // clean up partially-initialized entry at index + if (table->col_name[index]) {cloudsync_memory_free(table->col_name[index]); table->col_name[index] = NULL;} + if (table->col_merge_stmt[index]) {databasevm_finalize(table->col_merge_stmt[index]); table->col_merge_stmt[index] = NULL;} + if (table->col_value_stmt[index]) {databasevm_finalize(table->col_value_stmt[index]); table->col_value_stmt[index] = NULL;} + return 1; } -bool table_add_to_context (sqlite3 *db, cloudsync_context *data, table_algo algo, const char *table_name) { - DEBUG_DBFUNCTION("cloudsync_context_add_table %s", table_name); +bool table_ensure_capacity (cloudsync_context *data) { + if (data->tables_count < data->tables_cap) return true; - // check if table is already in the global context and in that case just return + int new_cap = data->tables_cap ? data->tables_cap * 2 : CLOUDSYNC_INIT_NTABLES; + size_t bytes = (size_t)new_cap * sizeof(*data->tables); + void *p = cloudsync_memory_realloc(data->tables, bytes); + if (!p) return false; + + data->tables = (cloudsync_table_context **)p; + data->tables_cap = new_cap; + return true; +} + +bool table_add_to_context (cloudsync_context *data, table_algo algo, const char *table_name) { + DEBUG_DBFUNCTION("cloudsync_context_add_table %s", table_name); + + // Check if table already initialized in this connection's context. + // Note: This prevents same-connection duplicate initialization. + // SQLite clients cannot distinguish schemas, so having 'public.users' + // and 'auth.users' would cause sync ambiguity. Users should avoid + // initializing tables with the same name in different schemas. + // If two concurrent connections initialize tables with the same name + // in different schemas, the behavior is undefined. cloudsync_table_context *table = table_lookup(data, table_name); if (table) return true; - // is there any space available? - if (data->tables_alloc <= data->tables_count + 1) { - // realloc tables - cloudsync_table_context **clone = (cloudsync_table_context **)cloudsync_memory_realloc(data->tables, sizeof(cloudsync_table_context) * data->tables_alloc + CLOUDSYNC_INIT_NTABLES); - if (!clone) goto abort_add_table; - - // reset new entries - for (int i=data->tables_alloc; itables_alloc + CLOUDSYNC_INIT_NTABLES; ++i) { - clone[i] = NULL; - } - - // replace old ptr - data->tables = clone; - data->tables_alloc += CLOUDSYNC_INIT_NTABLES; - } + // check for space availability + if (!table_ensure_capacity(data)) return false; - // setup a new table context - table = table_create(table_name, algo); + // setup a new table + table = table_create(data, table_name, algo); if (!table) return false; // fill remaining metadata in the table - char *sql = cloudsync_memory_mprintf("SELECT count(*) FROM pragma_table_info('%q') WHERE pk>0;", table_name); - if (!sql) goto abort_add_table; - table->npks = (int)dbutils_int_select(db, sql); - cloudsync_memory_free(sql); - if (table->npks == -1) { - dbutils_context_result_error(data->sqlite_ctx, "%s", sqlite3_errmsg(db)); - goto abort_add_table; - } - + int count = database_count_pk(data, table_name, false, table->schema); + if (count < 0) {cloudsync_set_dberror(data); goto abort_add_table;} + table->npks = count; if (table->npks == 0) { #if CLOUDSYNC_DISABLE_ROWIDONLY_TABLES - return false; + goto abort_add_table; #else table->rowid_only = true; table->npks = 1; // rowid #endif } - sql = cloudsync_memory_mprintf("SELECT count(*) FROM pragma_table_info('%q') WHERE pk=0;", table_name); - if (!sql) goto abort_add_table; - int64_t ncols = (int64_t)dbutils_int_select(db, sql); - cloudsync_memory_free(sql); - if (ncols == -1) { - dbutils_context_result_error(data->sqlite_ctx, "%s", sqlite3_errmsg(db)); - goto abort_add_table; - } - - int rc = table_add_stmts(db, table, (int)ncols); - if (rc != SQLITE_OK) goto abort_add_table; + int ncols = database_count_nonpk(data, table_name, table->schema); + if (ncols < 0) {cloudsync_set_dberror(data); goto abort_add_table;} + int rc = table_add_stmts(table, ncols); + if (rc != DBRES_OK) goto abort_add_table; // a table with only pk(s) is totally legal if (ncols > 0) { - table->col_name = (char **)cloudsync_memory_alloc((sqlite3_uint64)(sizeof(char *) * ncols)); + table->col_name = (char **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(char *) * ncols)); if (!table->col_name) goto abort_add_table; - - table->col_id = (int *)cloudsync_memory_alloc((sqlite3_uint64)(sizeof(int) * ncols)); + + table->col_id = (int *)cloudsync_memory_zeroalloc((uint64_t)(sizeof(int) * ncols)); if (!table->col_id) goto abort_add_table; - - table->col_merge_stmt = (sqlite3_stmt **)cloudsync_memory_alloc((sqlite3_uint64)(sizeof(sqlite3_stmt *) * ncols)); + + table->col_merge_stmt = (dbvm_t **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(void *) * ncols)); if (!table->col_merge_stmt) goto abort_add_table; - - table->col_value_stmt = (sqlite3_stmt **)cloudsync_memory_alloc((sqlite3_uint64)(sizeof(sqlite3_stmt *) * ncols)); + + table->col_value_stmt = (dbvm_t **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(void *) * ncols)); if (!table->col_value_stmt) goto abort_add_table; - - sql = cloudsync_memory_mprintf("SELECT name, cid FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid;", table_name); + + table->col_algo = (col_algo_t *)cloudsync_memory_zeroalloc((uint64_t)(sizeof(col_algo_t) * ncols)); + if (!table->col_algo) goto abort_add_table; + + table->col_delimiter = (char **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(char *) * ncols)); + if (!table->col_delimiter) goto abort_add_table; + + // Pass empty string when schema is NULL; SQL will fall back to current_schema() + const char *schema = table->schema ? table->schema : ""; + char *sql = cloudsync_memory_mprintf(SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID, + table_name, schema, table_name, schema); if (!sql) goto abort_add_table; - int rc = sqlite3_exec(db, sql, table_add_to_context_cb, (void *)table, NULL); + rc = database_exec_callback(data, sql, table_add_to_context_cb, (void *)table); cloudsync_memory_free(sql); - if (rc == SQLITE_ABORT) goto abort_add_table; - } - - // lookup the first free slot - for (int i=0; itables_alloc; ++i) { - if (data->tables[i] == NULL) { - data->tables[i] = table; - if (i > data->tables_count - 1) ++data->tables_count; - break; - } + if (rc == DBRES_ABORT) goto abort_add_table; } + // append newly created table + data->tables[data->tables_count++] = table; return true; abort_add_table: @@ -1009,13 +1139,10 @@ bool table_add_to_context (sqlite3 *db, cloudsync_context *data, table_algo algo return false; } -bool table_remove_from_context (cloudsync_context *data, cloudsync_table_context *table) { - return (table_remove(data, table->name) != -1); -} +dbvm_t *cloudsync_colvalue_stmt (cloudsync_context *data, const char *tbl_name, bool *persistent) { + dbvm_t *vm = NULL; + *persistent = false; -sqlite3_stmt *cloudsync_colvalue_stmt (sqlite3 *db, cloudsync_context *data, const char *tbl_name, bool *persistent) { - sqlite3_stmt *vm = NULL; - cloudsync_table_context *table = table_lookup(data, tbl_name); if (table) { char *col_name = NULL; @@ -1025,8 +1152,8 @@ sqlite3_stmt *cloudsync_colvalue_stmt (sqlite3 *db, cloudsync_context *data, con vm = table_column_lookup(table, col_name, false, NULL); *persistent = true; } else { - char *sql = table_build_value_sql(db, table, "*"); - sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + char *sql = table_build_value_sql(table, "*"); + databasevm_prepare(data, sql, (void **)&vm, 0); cloudsync_memory_free(sql); *persistent = false; } @@ -1035,124 +1162,400 @@ sqlite3_stmt *cloudsync_colvalue_stmt (sqlite3 *db, cloudsync_context *data, con return vm; } +bool table_enabled (cloudsync_table_context *table) { + return table->enabled; +} + +void table_set_enabled (cloudsync_table_context *table, bool value) { + table->enabled = value; +} + +int table_count_cols (cloudsync_table_context *table) { + return table->ncols; +} + +int table_count_pks (cloudsync_table_context *table) { + return table->npks; +} + +const char *table_colname (cloudsync_table_context *table, int index) { + return table->col_name[index]; +} + +bool table_pk_exists (cloudsync_table_context *table, const char *value, size_t len) { + // check if a row with the same primary key already exists + // if so, this means the row might have been previously deleted (sentinel) + return (dbvm_count(table->meta_pkexists_stmt, value, len, DBTYPE_BLOB) > 0); +} + +char **table_pknames (cloudsync_table_context *table) { + return table->pk_name; +} + +void table_set_pknames (cloudsync_table_context *table, char **pknames) { + table_pknames_free(table->pk_name, table->npks); + table->pk_name = pknames; +} + +bool table_algo_isgos (cloudsync_table_context *table) { + return (table->algo == table_algo_crdt_gos); +} + +const char *table_schema (cloudsync_table_context *table) { + return table->schema; +} + // MARK: - Merge Insert - -sqlite3_int64 merge_get_local_cl (cloudsync_table_context *table, const char *pk, int pklen, const char **err) { - sqlite3_stmt *vm = table->meta_local_cl_stmt; - sqlite3_int64 result = -1; +int64_t merge_get_local_cl (cloudsync_table_context *table, const char *pk, int pklen) { + dbvm_t *vm = table->meta_local_cl_stmt; + int64_t result = -1; - int rc = sqlite3_bind_blob(vm, 1, (const void *)pk, pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, (const void *)pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_blob(vm, 2, (const void *)pk, pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_blob(vm, 2, (const void *)pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) result = sqlite3_column_int64(vm, 0); - else if (rc == SQLITE_DONE) result = 0; + rc = databasevm_step(vm); + if (rc == DBRES_ROW) result = database_column_int(vm, 0); + else if (rc == DBRES_DONE) result = 0; cleanup: - if (result == -1) *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - stmt_reset(vm); + if (result == -1) cloudsync_set_dberror(table->context); + dbvm_reset(vm); return result; } -int merge_get_col_version (cloudsync_table_context *table, const char *col_name, const char *pk, int pklen, sqlite3_int64 *version, const char **err) { - sqlite3_stmt *vm = table->meta_col_version_stmt; +int merge_get_col_version (cloudsync_table_context *table, const char *col_name, const char *pk, int pklen, int64_t *version) { + dbvm_t *vm = table->meta_col_version_stmt; - int rc = sqlite3_bind_blob(vm, 1, (const void *)pk, pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, (const void *)pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_text(vm, 2, col_name, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_text(vm, 2, col_name, -1); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) { - *version = sqlite3_column_int64(vm, 0); - rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_ROW) { + *version = database_column_int(vm, 0); + rc = DBRES_OK; } cleanup: - if ((rc != SQLITE_OK) && (rc != SQLITE_DONE)) *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - stmt_reset(vm); + if ((rc != DBRES_OK) && (rc != DBRES_DONE)) cloudsync_set_dberror(table->context); + dbvm_reset(vm); return rc; } -int merge_set_winner_clock (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pk_len, const char *colname, sqlite3_int64 col_version, sqlite3_int64 db_version, const char *site_id, int site_len, sqlite3_int64 seq, sqlite3_int64 *rowid, const char **err) { +int merge_set_winner_clock (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pk_len, const char *colname, int64_t col_version, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid) { // get/set site_id - sqlite3_stmt *vm = data->getset_siteid_stmt; - int rc = sqlite3_bind_blob(vm, 1, (const void *)site_id, site_len, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup_merge; + dbvm_t *vm = data->getset_siteid_stmt; + int rc = databasevm_bind_blob(vm, 1, (const void *)site_id, site_len); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_step(vm); - if (rc != SQLITE_ROW) goto cleanup_merge; + rc = databasevm_step(vm); + if (rc != DBRES_ROW) goto cleanup_merge; - int64_t ord = sqlite3_column_int64(vm, 0); - stmt_reset(vm); + int64_t ord = database_column_int(vm, 0); + dbvm_reset(vm); vm = table->meta_winner_clock_stmt; - rc = sqlite3_bind_blob(vm, 1, (const void *)pk, pk_len, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_blob(vm, 1, (const void *)pk, pk_len); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_bind_text(vm, 2, (colname) ? colname : CLOUDSYNC_TOMBSTONE_VALUE, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_text(vm, 2, (colname) ? colname : CLOUDSYNC_TOMBSTONE_VALUE, -1); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_bind_int64(vm, 3, col_version); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_int(vm, 3, col_version); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_bind_int64(vm, 4, db_version); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_int(vm, 4, db_version); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_bind_int64(vm, 5, seq); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_int(vm, 5, seq); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_bind_int64(vm, 6, ord); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_int(vm, 6, ord); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) { - *rowid = sqlite3_column_int64(vm, 0); - rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_ROW) { + *rowid = database_column_int(vm, 0); + rc = DBRES_OK; } cleanup_merge: - if (rc != SQLITE_OK) *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - stmt_reset(vm); + if (rc != DBRES_OK) cloudsync_set_dberror(data); + dbvm_reset(vm); return rc; } -int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, const char *col_name, sqlite3_value *col_value, sqlite3_int64 col_version, sqlite3_int64 db_version, const char *site_id, int site_len, sqlite3_int64 seq, sqlite3_int64 *rowid, const char **err) { - int index; - sqlite3_stmt *vm = table_column_lookup(table, col_name, true, &index); - if (vm == NULL) { - *err = "Unable to retrieve column merge precompiled statement in merge_insert_col."; - return SQLITE_MISUSE; +// MARK: - Deferred column-batch merge functions - + +static int merge_pending_add (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, const char *col_name, dbvalue_t *col_value, int64_t col_version, int64_t db_version, const char *site_id, int site_len, int64_t seq) { + merge_pending_batch *batch = data->pending_batch; + + // Store table and PK on first entry + if (batch->table == NULL) { + batch->table = table; + batch->pk = (char *)cloudsync_memory_alloc(pklen); + if (!batch->pk) return cloudsync_set_error(data, "merge_pending_add: out of memory for pk", DBRES_NOMEM); + memcpy(batch->pk, pk, pklen); + batch->pk_len = pklen; + } + + // Ensure capacity + if (batch->count >= batch->capacity) { + int new_cap = batch->capacity ? batch->capacity * 2 : 8; + merge_pending_entry *new_entries = (merge_pending_entry *)cloudsync_memory_realloc(batch->entries, new_cap * sizeof(merge_pending_entry)); + if (!new_entries) return cloudsync_set_error(data, "merge_pending_add: out of memory for entries", DBRES_NOMEM); + batch->entries = new_entries; + batch->capacity = new_cap; + } + + // Resolve col_name to a stable pointer from the table context + // (the incoming col_name may point to VM-owned memory that gets freed on reset) + int col_idx = -1; + table_column_lookup(table, col_name, true, &col_idx); + const char *stable_col_name = (col_idx >= 0) ? table_colname(table, col_idx) : NULL; + if (!stable_col_name) return cloudsync_set_error(data, "merge_pending_add: column not found in table context", DBRES_ERROR); + + merge_pending_entry *e = &batch->entries[batch->count]; + e->col_name = stable_col_name; + e->col_value = col_value ? (dbvalue_t *)database_value_dup(col_value) : NULL; + e->col_version = col_version; + e->db_version = db_version; + e->site_id_len = (site_len <= (int)sizeof(e->site_id)) ? site_len : (int)sizeof(e->site_id); + memcpy(e->site_id, site_id, e->site_id_len); + e->seq = seq; + + batch->count++; + return DBRES_OK; +} + +static void merge_pending_free_entries (merge_pending_batch *batch) { + if (batch->entries) { + for (int i = 0; i < batch->count; i++) { + if (batch->entries[i].col_value) { + database_value_free(batch->entries[i].col_value); + batch->entries[i].col_value = NULL; + } + } + } + if (batch->pk) { + cloudsync_memory_free(batch->pk); + batch->pk = NULL; + } + batch->table = NULL; + batch->pk_len = 0; + batch->cl = 0; + batch->sentinel_pending = false; + batch->row_exists = false; + batch->count = 0; +} + +static int merge_flush_pending (cloudsync_context *data) { + merge_pending_batch *batch = data->pending_batch; + if (!batch) return DBRES_OK; + + int rc = DBRES_OK; + bool flush_savepoint = false; + + // Nothing to write — handle sentinel-only case or skip + if (batch->count == 0 && !(batch->sentinel_pending && batch->table)) { + goto cleanup; + } + + // Wrap database operations in a savepoint so that on failure (e.g. RLS + // denial) the rollback properly releases all executor resources (open + // relations, snapshots, plan cache) acquired during the failed statement. + flush_savepoint = (database_begin_savepoint(data, "merge_flush") == DBRES_OK); + + if (batch->count == 0) { + // Sentinel with no winning columns (PK-only row) + dbvm_t *vm = batch->table->real_merge_sentinel_stmt; + rc = pk_decode_prikey(batch->pk, (size_t)batch->pk_len, pk_decode_bind_callback, vm); + if (rc < 0) { + cloudsync_set_dberror(data); + dbvm_reset(vm); + goto cleanup; + } + SYNCBIT_SET(data); + rc = databasevm_step(vm); + dbvm_reset(vm); + SYNCBIT_RESET(data); + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); + goto cleanup; + } + goto cleanup; + } + + // Check if cached prepared statement can be reused + cloudsync_table_context *table = batch->table; + dbvm_t *vm = NULL; + bool cache_hit = false; + + if (batch->cached_vm && + batch->cached_row_exists == batch->row_exists && + batch->cached_col_count == batch->count) { + cache_hit = true; + for (int i = 0; i < batch->count; i++) { + if (batch->cached_col_names[i] != batch->entries[i].col_name) { + cache_hit = false; + break; + } + } + } + + if (cache_hit) { + vm = batch->cached_vm; + dbvm_reset(vm); + } else { + // Invalidate old cache + if (batch->cached_vm) { + databasevm_finalize(batch->cached_vm); + batch->cached_vm = NULL; + } + + // Build multi-column SQL + const char **colnames = (const char **)cloudsync_memory_alloc(batch->count * sizeof(const char *)); + if (!colnames) { + rc = cloudsync_set_error(data, "merge_flush_pending: out of memory", DBRES_NOMEM); + goto cleanup; + } + for (int i = 0; i < batch->count; i++) { + colnames[i] = batch->entries[i].col_name; + } + + char *sql = batch->row_exists + ? sql_build_update_pk_and_multi_cols(data, table->name, colnames, batch->count, table->schema) + : sql_build_upsert_pk_and_multi_cols(data, table->name, colnames, batch->count, table->schema); + cloudsync_memory_free(colnames); + + if (!sql) { + rc = cloudsync_set_error(data, "merge_flush_pending: unable to build multi-column upsert SQL", DBRES_ERROR); + goto cleanup; + } + + rc = databasevm_prepare(data, sql, &vm, 0); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) { + rc = cloudsync_set_error(data, "merge_flush_pending: unable to prepare statement", rc); + goto cleanup; + } + + // Update cache + batch->cached_vm = vm; + batch->cached_row_exists = batch->row_exists; + batch->cached_col_count = batch->count; + // Reallocate cached_col_names if needed + if (batch->cached_col_count > 0) { + const char **new_names = (const char **)cloudsync_memory_realloc( + batch->cached_col_names, batch->count * sizeof(const char *)); + if (new_names) { + for (int i = 0; i < batch->count; i++) { + new_names[i] = batch->entries[i].col_name; + } + batch->cached_col_names = new_names; + } + } + } + + // Bind PKs (positions 1..npks) + int npks = pk_decode_prikey(batch->pk, (size_t)batch->pk_len, pk_decode_bind_callback, vm); + if (npks < 0) { + cloudsync_set_dberror(data); + dbvm_reset(vm); + rc = DBRES_ERROR; + goto cleanup; + } + + // Bind column values (positions npks+1..npks+count) + for (int i = 0; i < batch->count; i++) { + merge_pending_entry *e = &batch->entries[i]; + int bind_idx = npks + 1 + i; + if (e->col_value) { + rc = databasevm_bind_value(vm, bind_idx, e->col_value); + } else { + rc = databasevm_bind_null(vm, bind_idx); + } + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); + dbvm_reset(vm); + goto cleanup; + } + } + + // Execute with SYNCBIT and GOS handling + if (table->algo == table_algo_crdt_gos) table->enabled = 0; + SYNCBIT_SET(data); + rc = databasevm_step(vm); + dbvm_reset(vm); + SYNCBIT_RESET(data); + if (table->algo == table_algo_crdt_gos) table->enabled = 1; + + if (rc != DBRES_DONE) { + cloudsync_set_dberror(data); + goto cleanup; + } + rc = DBRES_OK; + + // Call merge_set_winner_clock for each buffered entry + int64_t rowid = 0; + for (int i = 0; i < batch->count; i++) { + merge_pending_entry *e = &batch->entries[i]; + int clock_rc = merge_set_winner_clock(data, table, batch->pk, batch->pk_len, + e->col_name, e->col_version, e->db_version, + (const char *)e->site_id, e->site_id_len, + e->seq, &rowid); + if (clock_rc != DBRES_OK) { + rc = clock_rc; + goto cleanup; + } + } + +cleanup: + merge_pending_free_entries(batch); + if (flush_savepoint) { + if (rc == DBRES_OK) database_commit_savepoint(data, "merge_flush"); + else database_rollback_savepoint(data, "merge_flush"); } + return rc; +} + +int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *col_name, dbvalue_t *col_value, int64_t col_version, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid) { + int index; + dbvm_t *vm = table_column_lookup(table, col_name, true, &index); + if (vm == NULL) return cloudsync_set_error(data, "Unable to retrieve column merge precompiled statement in merge_insert_col", DBRES_MISUSE); // INSERT INTO table (pk1, pk2, col_name) VALUES (?, ?, ?) ON CONFLICT DO UPDATE SET col_name=?;" // bind primary key(s) int rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, vm); if (rc < 0) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - rc = sqlite3_errcode(sqlite3_db_handle(vm)); - stmt_reset(vm); + cloudsync_set_dberror(data); + dbvm_reset(vm); return rc; } - // bind value + // bind value (always bind all expected parameters for correct prepared statement handling) if (col_value) { - rc = sqlite3_bind_value(vm, table->npks+1, col_value); - if (rc == SQLITE_OK) rc = sqlite3_bind_value(vm, table->npks+2, col_value); - if (rc != SQLITE_OK) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - stmt_reset(vm); - return rc; - } - + rc = databasevm_bind_value(vm, table->npks+1, col_value); + if (rc == DBRES_OK) rc = databasevm_bind_value(vm, table->npks+2, col_value); + } else { + rc = databasevm_bind_null(vm, table->npks+1); + if (rc == DBRES_OK) rc = databasevm_bind_null(vm, table->npks+2); } - + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); + dbvm_reset(vm); + return rc; + } + // perform real operation and disable triggers // in case of GOS we reused the table->col_merge_stmt statement @@ -1161,133 +1564,138 @@ int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, c // the trick is to disable that trigger before executing the statement if (table->algo == table_algo_crdt_gos) table->enabled = 0; SYNCBIT_SET(data); - rc = sqlite3_step(vm); - DEBUG_MERGE("merge_insert(%02x%02x): %s (%d)", data->site_id[UUID_LEN-2], data->site_id[UUID_LEN-1], sqlite3_expanded_sql(vm), rc); - stmt_reset(vm); + rc = databasevm_step(vm); + DEBUG_MERGE("merge_insert(%02x%02x): %s (%d)", data->site_id[UUID_LEN-2], data->site_id[UUID_LEN-1], databasevm_sql(vm), rc); + dbvm_reset(vm); SYNCBIT_RESET(data); if (table->algo == table_algo_crdt_gos) table->enabled = 1; - if (rc != SQLITE_DONE) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); + if (rc != DBRES_DONE) { + cloudsync_set_dberror(data); return rc; } - return merge_set_winner_clock(data, table, pk, pklen, col_name, col_version, db_version, site_id, site_len, seq, rowid, err); + return merge_set_winner_clock(data, table, pk, pklen, col_name, col_version, db_version, site_id, site_len, seq, rowid); } -int merge_delete (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, const char *colname, sqlite3_int64 cl, sqlite3_int64 db_version, const char *site_id, int site_len, sqlite3_int64 seq, sqlite3_int64 *rowid, const char **err) { - int rc = SQLITE_OK; +int merge_delete (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, const char *colname, int64_t cl, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid) { + int rc = DBRES_OK; // reset return value *rowid = 0; // bind pk - sqlite3_stmt *vm = table->real_merge_delete_stmt; + dbvm_t *vm = table->real_merge_delete_stmt; rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, vm); if (rc < 0) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - rc = sqlite3_errcode(sqlite3_db_handle(vm)); - stmt_reset(vm); + rc = cloudsync_set_dberror(data); + dbvm_reset(vm); return rc; } // perform real operation and disable triggers SYNCBIT_SET(data); - rc = sqlite3_step(vm); - DEBUG_MERGE("merge_delete(%02x%02x): %s (%d)", data->site_id[UUID_LEN-2], data->site_id[UUID_LEN-1], sqlite3_expanded_sql(vm), rc); - stmt_reset(vm); + rc = databasevm_step(vm); + DEBUG_MERGE("merge_delete(%02x%02x): %s (%d)", data->site_id[UUID_LEN-2], data->site_id[UUID_LEN-1], databasevm_sql(vm), rc); + dbvm_reset(vm); SYNCBIT_RESET(data); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - if (rc != SQLITE_OK) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); return rc; } - rc = merge_set_winner_clock(data, table, pk, pklen, colname, cl, db_version, site_id, site_len, seq, rowid, err); - if (rc != SQLITE_OK) return rc; + rc = merge_set_winner_clock(data, table, pk, pklen, colname, cl, db_version, site_id, site_len, seq, rowid); + if (rc != DBRES_OK) return rc; // drop clocks _after_ setting the winner clock so we don't lose track of the max db_version!! // this must never come before `set_winner_clock` vm = table->meta_merge_delete_drop; - rc = sqlite3_bind_blob(vm, 1, (const void *)pk, pklen, SQLITE_STATIC); - if (rc == SQLITE_OK) rc = sqlite3_step(vm); - stmt_reset(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - if (rc != SQLITE_OK) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - } + rc = databasevm_bind_blob(vm, 1, (const void *)pk, pklen); + if (rc == DBRES_OK) rc = databasevm_step(vm); + dbvm_reset(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) cloudsync_set_dberror(data); return rc; } -int merge_zeroclock_on_resurrect(cloudsync_table_context *table, sqlite3_int64 db_version, const char *pk, int pklen, const char **err) { - sqlite3_stmt *vm = table->meta_zero_clock_stmt; +int merge_zeroclock_on_resurrect(cloudsync_table_context *table, int64_t db_version, const char *pk, int pklen) { + dbvm_t *vm = table->meta_zero_clock_stmt; - int rc = sqlite3_bind_int64(vm, 1, db_version); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_int(vm, 1, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_blob(vm, 2, (const void *)pk, pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_blob(vm, 2, (const void *)pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - if (rc != SQLITE_OK) *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - stmt_reset(vm); + if (rc != DBRES_OK) cloudsync_set_dberror(table->context); + dbvm_reset(vm); return rc; } // executed only if insert_cl == local_cl -int merge_did_cid_win (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, sqlite3_value *insert_value, const char *site_id, int site_len, const char *col_name, sqlite3_int64 col_version, bool *didwin_flag, const char **err) { +int merge_did_cid_win (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, dbvalue_t *insert_value, const char *site_id, int site_len, const char *col_name, int64_t col_version, bool *didwin_flag) { if (col_name == NULL) col_name = CLOUDSYNC_TOMBSTONE_VALUE; - sqlite3_int64 local_version; - int rc = merge_get_col_version(table, col_name, pk, pklen, &local_version, err); - if (rc == SQLITE_DONE) { + int64_t local_version; + int rc = merge_get_col_version(table, col_name, pk, pklen, &local_version); + if (rc == DBRES_DONE) { // no rows returned, the incoming change wins if there's nothing there locally *didwin_flag = true; - return SQLITE_OK; + return DBRES_OK; } - if (rc != SQLITE_OK) return rc; + if (rc != DBRES_OK) return rc; - // rc == SQLITE_OK, means that a row with a version exists + // rc == DBRES_OK, means that a row with a version exists if (local_version != col_version) { - if (col_version > local_version) {*didwin_flag = true; return SQLITE_OK;} - if (col_version < local_version) {*didwin_flag = false; return SQLITE_OK;} + if (col_version > local_version) {*didwin_flag = true; return DBRES_OK;} + if (col_version < local_version) {*didwin_flag = false; return DBRES_OK;} } - // rc == SQLITE_ROW and col_version == local_version, need to compare values - + // rc == DBRES_ROW and col_version == local_version, need to compare values + // retrieve col_value precompiled statement - sqlite3_stmt *vm = table_column_lookup(table, col_name, false, NULL); - if (!vm) { - *err = "Unable to retrieve column value precompiled statement in merge_did_cid_win."; - return SQLITE_ERROR; - } - - // bind primary key values - rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, (void *)vm); - if (rc < 0) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - rc = sqlite3_errcode(sqlite3_db_handle(vm)); - stmt_reset(vm); - return rc; + bool is_block_col = block_is_block_colname(col_name) && table_has_block_cols(table); + dbvm_t *vm; + if (is_block_col) { + // Block column: read value from blocks table (pk + col_name bindings) + vm = table_block_value_read_stmt(table); + if (!vm) return cloudsync_set_error(data, "Unable to retrieve block value read statement in merge_did_cid_win", DBRES_ERROR); + rc = databasevm_bind_blob(vm, 1, (const void *)pk, pklen); + if (rc != DBRES_OK) { dbvm_reset(vm); return cloudsync_set_dberror(data); } + rc = databasevm_bind_text(vm, 2, col_name, -1); + if (rc != DBRES_OK) { dbvm_reset(vm); return cloudsync_set_dberror(data); } + } else { + vm = table_column_lookup(table, col_name, false, NULL); + if (!vm) return cloudsync_set_error(data, "Unable to retrieve column value precompiled statement in merge_did_cid_win", DBRES_ERROR); + + // bind primary key values + rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, (void *)vm); + if (rc < 0) { + rc = cloudsync_set_dberror(data); + dbvm_reset(vm); + return rc; + } } // execute vm - sqlite3_value *local_value; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) { + dbvalue_t *local_value; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) { // meta entry exists but the actual value is missing // we should allow the value_compare function to make a decision // value_compare has been modified to handle the case where lvalue is NULL local_value = NULL; - rc = SQLITE_OK; - } else if (rc == SQLITE_ROW) { - local_value = sqlite3_column_value(vm, 0); - rc = SQLITE_OK; + rc = DBRES_OK; + } else if (rc == DBRES_ROW) { + local_value = database_column_value(vm, 0); + rc = DBRES_OK; } else { goto cleanup; } @@ -1295,7 +1703,8 @@ int merge_did_cid_win (cloudsync_context *data, cloudsync_table_context *table, // compare values int ret = dbutils_value_compare(insert_value, local_value); // reset after compare, otherwise local value would be deallocated - vm = stmt_reset(vm); + dbvm_reset(vm); + vm = NULL; bool compare_site_id = (ret == 0 && data->merge_equal_values == true); if (!compare_site_id) { @@ -1305,125 +1714,269 @@ int merge_did_cid_win (cloudsync_context *data, cloudsync_table_context *table, // values are the same and merge_equal_values is true vm = table->meta_site_id_stmt; - rc = sqlite3_bind_blob(vm, 1, (const void *)pk, pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; - - rc = sqlite3_bind_text(vm, 2, col_name, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; - - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) { - const void *local_site_id = sqlite3_column_blob(vm, 0); + rc = databasevm_bind_blob(vm, 1, (const void *)pk, pklen); + if (rc != DBRES_OK) goto cleanup; + + rc = databasevm_bind_text(vm, 2, col_name, -1); + if (rc != DBRES_OK) goto cleanup; + + rc = databasevm_step(vm); + if (rc == DBRES_ROW) { + const void *local_site_id = database_column_blob(vm, 0, NULL); + if (!local_site_id) { + dbvm_reset(vm); + return cloudsync_set_error(data, "NULL site_id in cloudsync table, table is probably corrupted", DBRES_ERROR); + } ret = memcmp(site_id, local_site_id, site_len); *didwin_flag = (ret > 0); - stmt_reset(vm); - return SQLITE_OK; + dbvm_reset(vm); + return DBRES_OK; } // handle error condition here - stmt_reset(vm); - *err = "Unable to find site_id for previous change. The cloudsync table is probably corrupted."; - return SQLITE_ERROR; + dbvm_reset(vm); + return cloudsync_set_error(data, "Unable to find site_id for previous change, cloudsync table is probably corrupted", DBRES_ERROR); cleanup: - if (rc != SQLITE_OK) *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - if (vm) stmt_reset(vm); + if (rc != DBRES_OK) cloudsync_set_dberror(data); + dbvm_reset(vm); return rc; } -int merge_sentinel_only_insert (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, sqlite3_int64 cl, sqlite3_int64 db_version, const char *site_id, int site_len, sqlite3_int64 seq, sqlite3_int64 *rowid, const char **err) { - +int merge_sentinel_only_insert (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, int64_t cl, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid) { + // reset return value *rowid = 0; - - // bind pk - sqlite3_stmt *vm = table->real_merge_sentinel_stmt; - int rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, vm); - if (rc < 0) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - rc = sqlite3_errcode(sqlite3_db_handle(vm)); - stmt_reset(vm); - return rc; - } - - // perform real operation and disable triggers - SYNCBIT_SET(data); - rc = sqlite3_step(vm); - stmt_reset(vm); - SYNCBIT_RESET(data); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - if (rc != SQLITE_OK) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - return rc; + + if (data->pending_batch == NULL) { + // Immediate mode: execute base table INSERT + dbvm_t *vm = table->real_merge_sentinel_stmt; + int rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, vm); + if (rc < 0) { + rc = cloudsync_set_dberror(data); + dbvm_reset(vm); + return rc; + } + + SYNCBIT_SET(data); + rc = databasevm_step(vm); + dbvm_reset(vm); + SYNCBIT_RESET(data); + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); + return rc; + } + } else { + // Batch mode: skip base table INSERT, the batch flush will create the row + merge_pending_batch *batch = data->pending_batch; + batch->sentinel_pending = true; + if (batch->table == NULL) { + batch->table = table; + batch->pk = (char *)cloudsync_memory_alloc(pklen); + if (!batch->pk) return cloudsync_set_error(data, "merge_sentinel_only_insert: out of memory for pk", DBRES_NOMEM); + memcpy(batch->pk, pk, pklen); + batch->pk_len = pklen; + } } - - rc = merge_zeroclock_on_resurrect(table, db_version, pk, pklen, err); - if (rc != SQLITE_OK) return rc; - - return merge_set_winner_clock(data, table, pk, pklen, NULL, cl, db_version, site_id, site_len, seq, rowid, err); + + // Metadata operations always execute regardless of batch mode + int rc = merge_zeroclock_on_resurrect(table, db_version, pk, pklen); + if (rc != DBRES_OK) return rc; + + return merge_set_winner_clock(data, table, pk, pklen, NULL, cl, db_version, site_id, site_len, seq, rowid); } -int cloudsync_merge_insert_gos (sqlite3_vtab *vtab, cloudsync_context *data, cloudsync_table_context *table, const char *insert_pk, int insert_pk_len, const char *insert_name, sqlite3_value *insert_value, sqlite3_int64 insert_col_version, sqlite3_int64 insert_db_version, const char *insert_site_id, int insert_site_id_len, sqlite3_int64 insert_seq, sqlite3_int64 *rowid) { - // Grow-Only Set (GOS) Algorithm: Only insertions are allowed, deletions and updates are prevented from a trigger. - - const char *err = NULL; - int rc = merge_insert_col(data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, - insert_site_id, insert_site_id_len, insert_seq, rowid, &err); - if (rc != SQLITE_OK) { - cloudsync_vtab_set_error(vtab, "Unable to perform GOS merge_insert_col: %s", err); +// MARK: - Block-level merge helpers - + +// Store a block value in the blocks table +static int block_store_value (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *block_colname, dbvalue_t *col_value) { + dbvm_t *vm = table->block_value_write_stmt; + if (!vm) return cloudsync_set_error(data, "block_store_value: blocks table not initialized", DBRES_MISUSE); + + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) goto cleanup; + rc = databasevm_bind_text(vm, 2, block_colname, -1); + if (rc != DBRES_OK) goto cleanup; + if (col_value) { + rc = databasevm_bind_value(vm, 3, col_value); + } else { + rc = databasevm_bind_null(vm, 3); } - + if (rc != DBRES_OK) goto cleanup; + + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + +cleanup: + if (rc != DBRES_OK) cloudsync_set_dberror(data); + databasevm_reset(vm); return rc; } -int cloudsync_merge_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, sqlite3_int64 *rowid) { - // this function performs the merging logic for an insert in a cloud-synchronized table. It handles - // different scenarios including conflicts, causal lengths, delete operations, and resurrecting rows - // based on the incoming data (from remote nodes or clients) and the local database state +// Delete a block value from the blocks table +static int block_delete_value (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *block_colname) { + dbvm_t *vm = table->block_value_delete_stmt; + if (!vm) return cloudsync_set_error(data, "block_delete_value: blocks table not initialized", DBRES_MISUSE); - // this function handles different CRDT algorithms (GOS, DWS, AWS, and CLS). - // the merging strategy is determined based on the table->algo value. - - // meta table declaration: - // tbl TEXT NOT NULL, pk BLOB NOT NULL, col_name TEXT NOT NULL," - // "col_value ANY, col_version INTEGER NOT NULL, db_version INTEGER NOT NULL," - // "site_id BLOB NOT NULL, cl INTEGER NOT NULL, seq INTEGER NOT NULL - - // meta information to retrieve from arguments: - // argv[0] -> table name (TEXT) - // argv[1] -> primary key (BLOB) - // argv[2] -> column name (TEXT or NULL if sentinel) - // argv[3] -> column value (ANY) - // argv[4] -> column version (INTEGER) - // argv[5] -> database version (INTEGER) - // argv[6] -> site ID (BLOB, identifies the origin of the update) - // argv[7] -> causal length (INTEGER, tracks the order of operations) - // argv[8] -> sequence number (INTEGER, unique per operation) - - // extract table name - const char *insert_tbl = (const char *)sqlite3_value_text(argv[0]); - - // lookup table - cloudsync_context *data = cloudsync_vtab_get_context(vtab); - cloudsync_table_context *table = table_lookup(data, insert_tbl); - if (!table) return cloudsync_vtab_set_error(vtab, "Unable to find table %s,", insert_tbl); - - // extract the remaining fields from the input values - const char *insert_pk = (const char *)sqlite3_value_blob(argv[1]); - int insert_pk_len = sqlite3_value_bytes(argv[1]); - const char *insert_name = (sqlite3_value_type(argv[2]) == SQLITE_NULL) ? CLOUDSYNC_TOMBSTONE_VALUE : (const char *)sqlite3_value_text(argv[2]); - sqlite3_value *insert_value = argv[3]; - sqlite3_int64 insert_col_version = sqlite3_value_int64(argv[4]); - sqlite3_int64 insert_db_version = sqlite3_value_int64(argv[5]); - const char *insert_site_id = (const char *)sqlite3_value_blob(argv[6]); - int insert_site_id_len = sqlite3_value_bytes(argv[6]); - sqlite3_int64 insert_cl = sqlite3_value_int64(argv[7]); - sqlite3_int64 insert_seq = sqlite3_value_int64(argv[8]); - const char *err = NULL; - - // perform different logic for each different table algorithm - if (table->algo == table_algo_crdt_gos) return cloudsync_merge_insert_gos(vtab, data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); - + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) goto cleanup; + rc = databasevm_bind_text(vm, 2, block_colname, -1); + if (rc != DBRES_OK) goto cleanup; + + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + +cleanup: + if (rc != DBRES_OK) cloudsync_set_dberror(data); + databasevm_reset(vm); + return rc; +} + +// Materialize all alive blocks for a base column into the base table +int block_materialize_column (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *base_col_name) { + if (!table->block_list_stmt) return cloudsync_set_error(data, "block_materialize_column: blocks table not initialized", DBRES_MISUSE); + + // Find column index and delimiter + int col_idx = -1; + for (int i = 0; i < table->ncols; i++) { + if (strcasecmp(table->col_name[i], base_col_name) == 0) { + col_idx = i; + break; + } + } + if (col_idx < 0) return cloudsync_set_error(data, "block_materialize_column: column not found", DBRES_ERROR); + const char *delimiter = table->col_delimiter[col_idx] ? table->col_delimiter[col_idx] : BLOCK_DEFAULT_DELIMITER; + + // Build the LIKE pattern for block col_names: "base_col\x1F%" + char *like_pattern = block_build_colname(base_col_name, "%"); + if (!like_pattern) return DBRES_NOMEM; + + // Query alive blocks from blocks table joined with metadata + // block_list_stmt: SELECT b.col_value FROM blocks b JOIN meta m + // ON b.pk = m.pk AND b.col_name = m.col_name + // WHERE b.pk = ? AND b.col_name LIKE ? AND m.col_version % 2 = 1 + // ORDER BY b.col_name + dbvm_t *vm = table->block_list_stmt; + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } + rc = databasevm_bind_text(vm, 2, like_pattern, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } + // Bind pk again for the join condition (parameter 3) + rc = databasevm_bind_blob(vm, 3, pk, pklen); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } + rc = databasevm_bind_text(vm, 4, like_pattern, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } + + // Collect block values + const char **block_values = NULL; + int block_count = 0; + int block_cap = 0; + + while ((rc = databasevm_step(vm)) == DBRES_ROW) { + const char *value = database_column_text(vm, 0); + if (block_count >= block_cap) { + int new_cap = block_cap ? block_cap * 2 : 16; + const char **new_arr = (const char **)cloudsync_memory_realloc((void *)block_values, (uint64_t)(new_cap * sizeof(char *))); + if (!new_arr) { rc = DBRES_NOMEM; break; } + block_values = new_arr; + block_cap = new_cap; + } + block_values[block_count] = value ? cloudsync_string_dup(value) : cloudsync_string_dup(""); + block_count++; + } + databasevm_reset(vm); + cloudsync_memory_free(like_pattern); + + if (rc != DBRES_DONE && rc != DBRES_OK && rc != DBRES_ROW) { + // Free collected values + for (int i = 0; i < block_count; i++) cloudsync_memory_free((void *)block_values[i]); + if (block_values) cloudsync_memory_free((void *)block_values); + return cloudsync_set_dberror(data); + } + + // Materialize text (NULL when no alive blocks) + char *text = (block_count > 0) ? block_materialize_text(block_values, block_count, delimiter) : NULL; + for (int i = 0; i < block_count; i++) cloudsync_memory_free((void *)block_values[i]); + if (block_values) cloudsync_memory_free((void *)block_values); + if (block_count > 0 && !text) return DBRES_NOMEM; + + // Update the base table column via the col_merge_stmt (with triggers disabled) + dbvm_t *merge_vm = table->col_merge_stmt[col_idx]; + if (!merge_vm) { cloudsync_memory_free(text); return DBRES_ERROR; } + + // Bind PKs + rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, merge_vm); + if (rc < 0) { cloudsync_memory_free(text); databasevm_reset(merge_vm); return DBRES_ERROR; } + + // Bind the text value twice (INSERT value + ON CONFLICT UPDATE value) + int npks = table->npks; + if (text) { + rc = databasevm_bind_text(merge_vm, npks + 1, text, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(text); databasevm_reset(merge_vm); return rc; } + rc = databasevm_bind_text(merge_vm, npks + 2, text, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(text); databasevm_reset(merge_vm); return rc; } + } else { + rc = databasevm_bind_null(merge_vm, npks + 1); + if (rc != DBRES_OK) { databasevm_reset(merge_vm); return rc; } + rc = databasevm_bind_null(merge_vm, npks + 2); + if (rc != DBRES_OK) { databasevm_reset(merge_vm); return rc; } + } + + // Execute with triggers disabled + table->enabled = 0; + SYNCBIT_SET(data); + rc = databasevm_step(merge_vm); + databasevm_reset(merge_vm); + SYNCBIT_RESET(data); + table->enabled = 1; + + cloudsync_memory_free(text); + + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) return cloudsync_set_dberror(data); + return DBRES_OK; +} + +// Accessor for has_block_cols flag +bool table_has_block_cols (cloudsync_table_context *table) { + return table && table->has_block_cols; +} + +// Get block column algo for a given column index +col_algo_t table_col_algo (cloudsync_table_context *table, int index) { + if (!table || !table->col_algo || index < 0 || index >= table->ncols) return col_algo_normal; + return table->col_algo[index]; +} + +// Get block delimiter for a given column index +const char *table_col_delimiter (cloudsync_table_context *table, int index) { + if (!table || !table->col_delimiter || index < 0 || index >= table->ncols) return BLOCK_DEFAULT_DELIMITER; + return table->col_delimiter[index] ? table->col_delimiter[index] : BLOCK_DEFAULT_DELIMITER; +} + +// Block column struct accessors (for use outside cloudsync.c where struct is opaque) +dbvm_t *table_block_value_read_stmt (cloudsync_table_context *table) { return table ? table->block_value_read_stmt : NULL; } +dbvm_t *table_block_value_write_stmt (cloudsync_table_context *table) { return table ? table->block_value_write_stmt : NULL; } +dbvm_t *table_block_list_stmt (cloudsync_table_context *table) { return table ? table->block_list_stmt : NULL; } +const char *table_blocks_ref (cloudsync_table_context *table) { return table ? table->blocks_ref : NULL; } + +void table_set_col_delimiter (cloudsync_table_context *table, int col_idx, const char *delimiter) { + if (!table || !table->col_delimiter || col_idx < 0 || col_idx >= table->ncols) return; + if (table->col_delimiter[col_idx]) cloudsync_memory_free(table->col_delimiter[col_idx]); + table->col_delimiter[col_idx] = delimiter ? cloudsync_string_dup(delimiter) : NULL; +} + +// Find column index by name +int table_col_index (cloudsync_table_context *table, const char *col_name) { + if (!table || !col_name) return -1; + for (int i = 0; i < table->ncols; i++) { + if (strcasecmp(table->col_name[i], col_name) == 0) return i; + } + return -1; +} + +int merge_insert (cloudsync_context *data, cloudsync_table_context *table, const char *insert_pk, int insert_pk_len, int64_t insert_cl, const char *insert_name, dbvalue_t *insert_value, int64_t insert_col_version, int64_t insert_db_version, const char *insert_site_id, int insert_site_id_len, int64_t insert_seq, int64_t *rowid) { // Handle DWS and AWS algorithms here // Delete-Wins Set (DWS): table_algo_crdt_dws // Add-Wins Set (AWS): table_algo_crdt_aws @@ -1432,14 +1985,12 @@ int cloudsync_merge_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, // compute the local causal length for the row based on the primary key // the causal length is used to determine the order of operations and resolve conflicts. - sqlite3_int64 local_cl = merge_get_local_cl(table, insert_pk, insert_pk_len, &err); - if (local_cl < 0) { - return cloudsync_vtab_set_error(vtab, "Unable to compute local causal length: %s", err); - } + int64_t local_cl = merge_get_local_cl(table, insert_pk, insert_pk_len); + if (local_cl < 0) return cloudsync_set_error(data, "Unable to compute local causal length", DBRES_ERROR); // if the incoming causal length is older than the local causal length, we can safely ignore it // because the local changes are more recent - if (insert_cl < local_cl) return SQLITE_OK; + if (insert_cl < local_cl) return DBRES_OK; // check if the operation is a delete by examining the causal length // even causal lengths typically signify delete operations @@ -1447,24 +1998,24 @@ int cloudsync_merge_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, if (is_delete) { // if it's a delete, check if the local state is at the same causal length // if it is, no further action is needed - if (local_cl == insert_cl) return SQLITE_OK; + if (local_cl == insert_cl) return DBRES_OK; // perform a delete merge if the causal length is newer than the local one int rc = merge_delete(data, table, insert_pk, insert_pk_len, insert_name, insert_col_version, - insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid, &err); - if (rc != SQLITE_OK) cloudsync_vtab_set_error(vtab, "Unable to perform merge_delete: %s", err); + insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); + if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to perform merge_delete", rc); return rc; } // if the operation is a sentinel-only insert (indicating a new row or resurrected row with no column update), handle it separately. bool is_sentinel_only = (strcmp(insert_name, CLOUDSYNC_TOMBSTONE_VALUE) == 0); if (is_sentinel_only) { - if (local_cl == insert_cl) return SQLITE_OK; + if (local_cl == insert_cl) return DBRES_OK; // perform a sentinel-only insert to track the existence of the row int rc = merge_sentinel_only_insert(data, table, insert_pk, insert_pk_len, insert_col_version, - insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid, &err); - if (rc != SQLITE_OK) cloudsync_vtab_set_error(vtab, "Unable to perform merge_sentinel_only_insert: %s", err); + insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); + if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to perform merge_sentinel_only_insert", rc); return rc; } @@ -1479,95 +2030,408 @@ int cloudsync_merge_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, // this handles out-of-order deliveries where the row was deleted and is now being re-inserted if (needs_resurrect && (row_exists_locally || (!row_exists_locally && insert_cl > 1))) { int rc = merge_sentinel_only_insert(data, table, insert_pk, insert_pk_len, insert_cl, - insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid, &err); - if (rc != SQLITE_OK) { - cloudsync_vtab_set_error(vtab, "Unable to perform merge_sentinel_only_insert: %s", err); - return rc; - } + insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to perform merge_sentinel_only_insert", rc); } // at this point, we determine whether the incoming change wins based on causal length // this can be due to a resurrection, a non-existent local row, or a conflict resolution bool flag = false; - int rc = merge_did_cid_win(data, table, insert_pk, insert_pk_len, insert_value, insert_site_id, insert_site_id_len, insert_name, insert_col_version, &flag, &err); - if (rc != SQLITE_OK) { - cloudsync_vtab_set_error(vtab, "Unable to perform merge_did_cid_win: %s", err); - return rc; - } + int rc = merge_did_cid_win(data, table, insert_pk, insert_pk_len, insert_value, insert_site_id, insert_site_id_len, insert_name, insert_col_version, &flag); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to perform merge_did_cid_win", rc); // check if the incoming change wins and should be applied bool does_cid_win = ((needs_resurrect) || (!row_exists_locally) || (flag)); - if (!does_cid_win) return SQLITE_OK; - + if (!does_cid_win) return DBRES_OK; + + // Block-level LWW: if the incoming col_name is a block entry (contains \x1F), + // bypass the normal base-table write and instead store the value in the blocks table. + // The base table column will be materialized from all alive blocks. + if (block_is_block_colname(insert_name) && table->has_block_cols) { + // Store or delete block value in blocks table depending on tombstone status + if (insert_col_version % 2 == 0) { + // Tombstone: remove from blocks table + rc = block_delete_value(data, table, insert_pk, insert_pk_len, insert_name); + } else { + rc = block_store_value(data, table, insert_pk, insert_pk_len, insert_name, insert_value); + } + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to store/delete block value", rc); + + // Set winner clock in metadata + rc = merge_set_winner_clock(data, table, insert_pk, insert_pk_len, insert_name, + insert_col_version, insert_db_version, + insert_site_id, insert_site_id_len, insert_seq, rowid); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to set winner clock for block", rc); + + // Materialize the full column from blocks into the base table + char *base_col = block_extract_base_colname(insert_name); + if (base_col) { + rc = block_materialize_column(data, table, insert_pk, insert_pk_len, base_col); + cloudsync_memory_free(base_col); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to materialize block column", rc); + } + + return DBRES_OK; + } + // perform the final column insert or update if the incoming change wins - rc = merge_insert_col(data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid, &err); - if (rc != SQLITE_OK) cloudsync_vtab_set_error(vtab, "Unable to perform merge_insert_col: %s", err); + if (data->pending_batch) { + // Propagate row_exists_locally to the batch on the first winning column. + // This lets merge_flush_pending choose UPDATE vs INSERT ON CONFLICT, + // which matters when RLS policies reference columns not in the payload. + if (data->pending_batch->table == NULL) { + data->pending_batch->row_exists = row_exists_locally; + } + rc = merge_pending_add(data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq); + if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to perform merge_pending_add", rc); + } else { + rc = merge_insert_col(data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); + if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to perform merge_insert_col", rc); + } + return rc; } -// MARK: - Private - +// MARK: - Block column setup - + +// Migrate existing tracked rows to block format when block-level LWW is first enabled on a column. +// Scans the metadata table for alive rows with the plain col_name entry (not yet block entries), +// reads each row's current value from the base table, splits it into blocks, and inserts +// the block entries into both the blocks table and the metadata table. +// Uses INSERT OR IGNORE semantics so the operation is safe to call multiple times. +static int block_migrate_existing_rows (cloudsync_context *data, cloudsync_table_context *table, int col_idx) { + const char *col_name = table->col_name[col_idx]; + if (!col_name || !table->meta_ref || !table->blocks_ref) return DBRES_OK; + + const char *delim = table->col_delimiter[col_idx] ? table->col_delimiter[col_idx] : BLOCK_DEFAULT_DELIMITER; + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + // Phase 1: collect all existing PKs that have an alive regular col_name entry + // AND do not yet have any entries in the blocks table for this column. + // The NOT IN filter makes this idempotent: rows that were already migrated + // (or had their blocks created via INSERT) are skipped on subsequent calls. + // We collect PKs before writing so that writes to the metadata table (Phase 2) + // do not perturb the read cursor on the same table. + char *like_pattern = block_build_colname(col_name, "%"); + if (!like_pattern) return DBRES_NOMEM; + + char *scan_sql = cloudsync_memory_mprintf(SQL_META_SCAN_COL_FOR_MIGRATION, table->meta_ref, table->blocks_ref); + if (!scan_sql) { cloudsync_memory_free(like_pattern); return DBRES_NOMEM; } + dbvm_t *scan_vm = NULL; + int rc = databasevm_prepare(data, scan_sql, &scan_vm, 0); + cloudsync_memory_free(scan_sql); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); return rc; } + + rc = databasevm_bind_text(scan_vm, 1, col_name, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_finalize(scan_vm); return rc; } + // Bind like_pattern as ?2 and keep it alive until after all scan steps complete, + // because databasevm_bind_text uses SQLITE_STATIC (no copy). + rc = databasevm_bind_text(scan_vm, 2, like_pattern, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_finalize(scan_vm); return rc; } + + // Collect pk blobs into a dynamically-grown array of owned copies + void **pks = NULL; + size_t *pklens = NULL; + int pk_count = 0; + int pk_cap = 0; + + while ((rc = databasevm_step(scan_vm)) == DBRES_ROW) { + size_t pklen = 0; + const void *pk = database_column_blob(scan_vm, 0, &pklen); + if (!pk || pklen == 0) continue; + + if (pk_count >= pk_cap) { + int new_cap = pk_cap ? pk_cap * 2 : 8; + void **new_pks = (void **)cloudsync_memory_realloc(pks, (uint64_t)(new_cap * sizeof(void *))); + size_t *new_pklens = (size_t *)cloudsync_memory_realloc(pklens, (uint64_t)(new_cap * sizeof(size_t))); + if (!new_pks || !new_pklens) { + cloudsync_memory_free(new_pks ? new_pks : pks); + cloudsync_memory_free(new_pklens ? new_pklens : pklens); + databasevm_finalize(scan_vm); + return DBRES_NOMEM; + } + pks = new_pks; + pklens = new_pklens; + pk_cap = new_cap; + } -bool cloudsync_config_exists (sqlite3 *db) { - return dbutils_table_exists(db, CLOUDSYNC_SITEID_NAME) == true; -} + pks[pk_count] = cloudsync_memory_alloc((uint64_t)pklen); + if (!pks[pk_count]) { rc = DBRES_NOMEM; break; } + memcpy(pks[pk_count], pk, pklen); + pklens[pk_count] = pklen; + pk_count++; + } -void *cloudsync_context_create (void) { - cloudsync_context *data = (cloudsync_context *)cloudsync_memory_zeroalloc((uint64_t)(sizeof(cloudsync_context))); - DEBUG_SETTINGS("cloudsync_context_create %p", data); - - data->libversion = CLOUDSYNC_VERSION; - data->pending_db_version = CLOUDSYNC_VALUE_NOTSET; - #if CLOUDSYNC_DEBUG - data->debug = 1; - #endif - - // allocate space for 128 tables (it can grow if needed) - data->tables = (cloudsync_table_context **)cloudsync_memory_zeroalloc((uint64_t)(CLOUDSYNC_INIT_NTABLES * sizeof(cloudsync_table_context *))); - if (!data->tables) { - cloudsync_memory_free(data); - return NULL; + databasevm_finalize(scan_vm); + cloudsync_memory_free(like_pattern); // safe to free after scan_vm is finalized + if (rc != DBRES_DONE && rc != DBRES_OK) { + for (int i = 0; i < pk_count; i++) cloudsync_memory_free(pks[i]); + cloudsync_memory_free(pks); + cloudsync_memory_free(pklens); + return rc; } - data->tables_alloc = CLOUDSYNC_INIT_NTABLES; - data->tables_count = 0; - - return data; -} -void cloudsync_context_free (void *ptr) { - DEBUG_SETTINGS("cloudsync_context_free %p", ptr); - if (!ptr) return; - - cloudsync_context *data = (cloudsync_context*)ptr; - cloudsync_memory_free(data->tables); - cloudsync_memory_free(data); + if (pk_count == 0) { + cloudsync_memory_free(pks); + cloudsync_memory_free(pklens); + return DBRES_OK; + } + + // Phase 2: for each collected PK, read the column value, split into blocks, + // and insert into the blocks table + metadata using INSERT OR IGNORE. + + char *meta_sql = cloudsync_memory_mprintf(SQL_META_INSERT_BLOCK_IGNORE, table->meta_ref); + if (!meta_sql) { rc = DBRES_NOMEM; goto cleanup_pks; } + dbvm_t *meta_vm = NULL; + rc = databasevm_prepare(data, meta_sql, &meta_vm, 0); + cloudsync_memory_free(meta_sql); + if (rc != DBRES_OK) goto cleanup_pks; + + char *blocks_sql = cloudsync_memory_mprintf(SQL_BLOCKS_INSERT_IGNORE, table->blocks_ref); + if (!blocks_sql) { databasevm_finalize(meta_vm); rc = DBRES_NOMEM; goto cleanup_pks; } + dbvm_t *blocks_vm = NULL; + rc = databasevm_prepare(data, blocks_sql, &blocks_vm, 0); + cloudsync_memory_free(blocks_sql); + if (rc != DBRES_OK) { databasevm_finalize(meta_vm); goto cleanup_pks; } + + dbvm_t *val_vm = (dbvm_t *)table_column_lookup(table, col_name, false, NULL); + + for (int p = 0; p < pk_count; p++) { + const void *pk = pks[p]; + size_t pklen = pklens[p]; + + if (!val_vm) continue; + + // Read current column value from the base table + int bind_rc = pk_decode_prikey((char *)pk, pklen, pk_decode_bind_callback, (void *)val_vm); + if (bind_rc < 0) { databasevm_reset(val_vm); continue; } + + int step_rc = databasevm_step(val_vm); + const char *text = (step_rc == DBRES_ROW) ? database_column_text(val_vm, 0) : NULL; + // Make a copy of text before resetting val_vm, as the pointer is only valid until reset + char *text_copy = text ? cloudsync_string_dup(text) : NULL; + databasevm_reset(val_vm); + + if (!text_copy) continue; // NULL column value: nothing to migrate + + // Split text into blocks and store each one + block_list_t *blocks = block_split(text_copy, delim); + cloudsync_memory_free(text_copy); + if (!blocks) continue; + + char **positions = block_initial_positions(blocks->count); + if (positions) { + for (int b = 0; b < blocks->count; b++) { + char *block_cn = block_build_colname(col_name, positions[b]); + if (block_cn) { + // Metadata entry (skip if this block position already exists) + databasevm_bind_blob(meta_vm, 1, pk, (int)pklen); + databasevm_bind_text(meta_vm, 2, block_cn, -1); + databasevm_bind_int(meta_vm, 3, 1); // col_version = 1 (alive) + databasevm_bind_int(meta_vm, 4, db_version); + databasevm_bind_int(meta_vm, 5, cloudsync_bumpseq(data)); + databasevm_step(meta_vm); + databasevm_reset(meta_vm); + + // Block value (skip if this block position already exists) + databasevm_bind_blob(blocks_vm, 1, pk, (int)pklen); + databasevm_bind_text(blocks_vm, 2, block_cn, -1); + databasevm_bind_text(blocks_vm, 3, blocks->entries[b].content, -1); + databasevm_step(blocks_vm); + databasevm_reset(blocks_vm); + + cloudsync_memory_free(block_cn); + } + cloudsync_memory_free(positions[b]); + } + cloudsync_memory_free(positions); + } + block_list_free(blocks); + } + + databasevm_finalize(meta_vm); + databasevm_finalize(blocks_vm); + rc = DBRES_OK; + +cleanup_pks: + for (int i = 0; i < pk_count; i++) cloudsync_memory_free(pks[i]); + cloudsync_memory_free(pks); + cloudsync_memory_free(pklens); + return rc; } -const char *cloudsync_context_init (sqlite3 *db, cloudsync_context *data, sqlite3_context *context) { - if (!data && context) data = (cloudsync_context *)sqlite3_user_data(context); +int cloudsync_setup_block_column (cloudsync_context *data, const char *table_name, const char *col_name, const char *delimiter, bool persist) { + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) return cloudsync_set_error(data, "cloudsync_setup_block_column: table not found", DBRES_ERROR); - // perform init just the first time, if the site_id field is not set. - // The data->site_id value could exists while settings tables don't exists if the - // cloudsync_context_init was previously called in init transaction that was rolled back - // because of an error during the init process. - if (data->site_id[0] == 0 || !dbutils_table_exists(db, CLOUDSYNC_SITEID_NAME)) { - if (dbutils_settings_init(db, data, context) != SQLITE_OK) return NULL; - if (stmts_add_tocontext(db, data) != SQLITE_OK) return NULL; - if (cloudsync_load_siteid(db, data) != SQLITE_OK) return NULL; - - data->sqlite_ctx = context; - data->schema_hash = dbutils_schema_hash(db); + // Find column index + int col_idx = table_col_index(table, col_name); + if (col_idx < 0) { + char buf[1024]; + snprintf(buf, sizeof(buf), "cloudsync_setup_block_column: column '%s' not found in table '%s'", col_name, table_name); + return cloudsync_set_error(data, buf, DBRES_ERROR); } - - return (const char *)data->site_id; -} -void cloudsync_sync_key(cloudsync_context *data, const char *key, const char *value) { - DEBUG_SETTINGS("cloudsync_sync_key key: %s value: %s", key, value); - - // sync data - if (strcmp(key, CLOUDSYNC_KEY_SCHEMAVERSION) == 0) { - data->schema_version = (int)strtol(value, NULL, 0); - return; + // Set column algo + table->col_algo[col_idx] = col_algo_block; + table->has_block_cols = true; + + // Set delimiter (can be NULL for default) + if (table->col_delimiter[col_idx]) { + cloudsync_memory_free(table->col_delimiter[col_idx]); + table->col_delimiter[col_idx] = NULL; + } + if (delimiter) { + table->col_delimiter[col_idx] = cloudsync_string_dup(delimiter); + } + + // Create blocks table if not already done + if (!table->blocks_ref) { + table->blocks_ref = database_build_blocks_ref(table->schema, table->name); + if (!table->blocks_ref) return DBRES_NOMEM; + + // CREATE TABLE IF NOT EXISTS + char *sql = cloudsync_memory_mprintf(SQL_BLOCKS_CREATE_TABLE, table->blocks_ref); + if (!sql) return DBRES_NOMEM; + + int rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to create blocks table", rc); + + // Prepare block statements + // Write: upsert into blocks (pk, col_name, col_value) + sql = cloudsync_memory_mprintf(SQL_BLOCKS_UPSERT, table->blocks_ref); + if (!sql) return DBRES_NOMEM; + rc = databasevm_prepare(data, sql, (void **)&table->block_value_write_stmt, DBFLAG_PERSISTENT); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + + // Read: SELECT col_value FROM blocks WHERE pk = ? AND col_name = ? + sql = cloudsync_memory_mprintf(SQL_BLOCKS_SELECT, table->blocks_ref); + if (!sql) return DBRES_NOMEM; + rc = databasevm_prepare(data, sql, (void **)&table->block_value_read_stmt, DBFLAG_PERSISTENT); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + + // Delete: DELETE FROM blocks WHERE pk = ? AND col_name = ? + sql = cloudsync_memory_mprintf(SQL_BLOCKS_DELETE, table->blocks_ref); + if (!sql) return DBRES_NOMEM; + rc = databasevm_prepare(data, sql, (void **)&table->block_value_delete_stmt, DBFLAG_PERSISTENT); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + + // List alive blocks for materialization + sql = cloudsync_memory_mprintf(SQL_BLOCKS_LIST_ALIVE, table->blocks_ref, table->meta_ref); + if (!sql) return DBRES_NOMEM; + rc = databasevm_prepare(data, sql, (void **)&table->block_list_stmt, DBFLAG_PERSISTENT); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + } + + // Persist settings (skipped when called from the settings loader, since + // writing to cloudsync_table_settings while sqlite3_exec is iterating it + // re-feeds the rewritten row to the cursor and causes an infinite loop). + if (persist) { + int rc = dbutils_table_settings_set_key_value(data, table_name, col_name, "algo", "block"); + if (rc != DBRES_OK) return rc; + + if (delimiter) { + rc = dbutils_table_settings_set_key_value(data, table_name, col_name, "delimiter", delimiter); + if (rc != DBRES_OK) return rc; + } + + // Migrate any existing tracked rows: populate the blocks table and metadata with + // block entries derived from the current column value, so that subsequent UPDATE + // operations can diff against the real existing state instead of treating everything + // as new, and so this node participates correctly in LWW conflict resolution. + rc = block_migrate_existing_rows(data, table, col_idx); + if (rc != DBRES_OK) return rc; + } + + return DBRES_OK; +} + +// MARK: - Private - + +bool cloudsync_config_exists (cloudsync_context *data) { + return database_internal_table_exists(data, CLOUDSYNC_SITEID_NAME) == true; +} + +bool cloudsync_context_is_initialized (cloudsync_context *data) { + // A fully initialized context has its persistent "is the DB stale" probe + // prepared. cloudsync_context_init prepares data_version_stmt (via + // cloudsync_add_dbvms) only after the cloudsync_site_id table exists, so + // a non-NULL pointer means cloudsync_init has been called at least once + // on this connection. Used to produce actionable error messages when + // callers hit a function before calling cloudsync_init. + return data != NULL && data->data_version_stmt != NULL; +} + +cloudsync_context *cloudsync_context_create (void *db) { + cloudsync_context *data = (cloudsync_context *)cloudsync_memory_zeroalloc((uint64_t)(sizeof(cloudsync_context))); + if (!data) return NULL; + DEBUG_SETTINGS("cloudsync_context_create %p", data); + + data->libversion = CLOUDSYNC_VERSION; + data->pending_db_version = CLOUDSYNC_VALUE_NOTSET; + #if CLOUDSYNC_DEBUG + data->debug = 1; + #endif + + // allocate space for 64 tables (it can grow if needed) + uint64_t mem_needed = (uint64_t)(CLOUDSYNC_INIT_NTABLES * sizeof(cloudsync_table_context *)); + data->tables = (cloudsync_table_context **)cloudsync_memory_zeroalloc(mem_needed); + if (!data->tables) {cloudsync_memory_free(data); return NULL;} + + data->tables_cap = CLOUDSYNC_INIT_NTABLES; + data->tables_count = 0; + data->db = db; + + // SQLite exposes col_value as ANY, but other databases require a concrete type. + // In PostgreSQL we expose col_value as bytea, which holds the pk-encoded value bytes (type + data). + // Because col_value is already encoded, we skip decoding this field and pass it through as bytea. + // It is decoded to the target column type just before applying changes to the base table. + data->skip_decode_idx = (db == NULL) ? CLOUDSYNC_PK_INDEX_COLVALUE : -1; + + return data; +} + +void cloudsync_context_free (void *ctx) { + cloudsync_context *data = (cloudsync_context *)ctx; + DEBUG_SETTINGS("cloudsync_context_free %p", data); + if (!data) return; + + // free all table contexts and prepared statements + cloudsync_terminate(data); + + cloudsync_memory_free(data->tables); + cloudsync_memory_free(data); +} + +const char *cloudsync_context_init (cloudsync_context *data) { + if (!data) return NULL; + + // perform init just the first time, if the site_id field is not set. + // The data->site_id value could exists while settings tables don't exists if the + // cloudsync_context_init was previously called in init transaction that was rolled back + // because of an error during the init process. + if (data->site_id[0] == 0 || !database_internal_table_exists(data, CLOUDSYNC_SITEID_NAME)) { + if (dbutils_settings_init(data) != DBRES_OK) return NULL; + if (cloudsync_add_dbvms(data) != DBRES_OK) return NULL; + if (cloudsync_load_siteid(data) != DBRES_OK) return NULL; + data->schema_hash = database_schema_hash(data); + } + + return (const char *)data->site_id; +} + +void cloudsync_sync_key (cloudsync_context *data, const char *key, const char *value) { + DEBUG_SETTINGS("cloudsync_sync_key key: %s value: %s", key, value); + + // sync data + if (strcmp(key, CLOUDSYNC_KEY_SCHEMAVERSION) == 0) { + data->schema_version = (int)strtol(value, NULL, 0); + return; } if (strcmp(key, CLOUDSYNC_KEY_DEBUG) == 0) { @@ -1575,6 +2439,11 @@ void cloudsync_sync_key(cloudsync_context *data, const char *key, const char *va if (value && (value[0] != 0) && (value[0] != '0')) data->debug = 1; return; } + + if (strcmp(key, CLOUDSYNC_KEY_SCHEMA) == 0) { + cloudsync_set_schema(data, value); + return; + } } #if 0 @@ -1592,7 +2461,7 @@ int cloudsync_commit_hook (void *ctx) { data->pending_db_version = CLOUDSYNC_VALUE_NOTSET; data->seq = 0; - return SQLITE_OK; + return DBRES_OK; } void cloudsync_rollback_hook (void *ctx) { @@ -1602,38 +2471,82 @@ void cloudsync_rollback_hook (void *ctx) { data->seq = 0; } -int cloudsync_finalize_alter (sqlite3_context *context, cloudsync_context *data, cloudsync_table_context *table) { - int rc = SQLITE_OK; - sqlite3 *db = sqlite3_context_db_handle(context); +int cloudsync_begin_alter (cloudsync_context *data, const char *table_name) { + // init cloudsync_settings + if (cloudsync_context_init(data) == NULL) { + return DBRES_MISUSE; + } + + // lookup table + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to find table %s", table_name); + return cloudsync_set_error(data, buffer, DBRES_MISUSE); + } + + // idempotent: if already altering, return OK + if (table->is_altering) return DBRES_OK; + + // retrieve primary key(s) + char **names = NULL; + int nrows = 0; + int rc = database_pk_names(data, table_name, &names, &nrows); + if (rc != DBRES_OK) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to get primary keys for table %s", table_name); + cloudsync_set_error(data, buffer, DBRES_MISUSE); + goto rollback_begin_alter; + } + + // sanity check the number of primary keys + if (nrows != table_count_pks(table)) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Number of primary keys for table %s changed before ALTER", table_name); + cloudsync_set_error(data, buffer, DBRES_MISUSE); + goto rollback_begin_alter; + } + + // drop original triggers + rc = database_delete_triggers(data, table_name); + if (rc != DBRES_OK) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to delete triggers for table %s in cloudsync_begin_alter.", table_name); + cloudsync_set_error(data, buffer, DBRES_ERROR); + goto rollback_begin_alter; + } + + table_set_pknames(table, names); + table->is_altering = true; + return DBRES_OK; - db_version_check_uptodate(db, data); +rollback_begin_alter: + if (names) table_pknames_free(names, nrows); + return rc; +} + +int cloudsync_finalize_alter (cloudsync_context *data, cloudsync_table_context *table) { + // check if dbversion needed to be updated + cloudsync_dbversion_check_uptodate(data); - // If primary key columns change (in the schema) - // We need to drop, re-create and backfill - // the clock table. - // A change in pk columns means a change in all identities - // of all rows. - // We can determine this by comparing unique index on lookaside table vs - // pks on source table - char *errmsg = NULL; + // if primary-key columns change, all row identities change. + // In that case, the clock table must be dropped, recreated, + // and backfilled. We detect this by comparing the unique index + // in the lookaside table with the source table's PKs. + + // retrieve primary keys (to check is they changed) char **result = NULL; - int nrows, ncols; - char *sql = cloudsync_memory_mprintf("SELECT name FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table->name); - rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, NULL); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - DEBUG_SQLITE_ERROR(rc, "cloudsync_finalize_alter", db); - goto finalize; - } else if (errmsg || ncols != 1) { - rc = SQLITE_MISUSE; + int nrows = 0; + int rc = database_pk_names (data, table->name, &result, &nrows); + if (rc != DBRES_OK || nrows == 0) { + if (nrows == 0) rc = DBRES_MISUSE; goto finalize; } - bool pk_diff = false; - if (nrows != table->npks) { - pk_diff = true; - } else { - for (int i=0; inpks); + if (!pk_diff) { + for (int i = 0; i < nrows; ++i) { if (strcmp(table->pk_name[i], result[i]) != 0) { pk_diff = true; break; @@ -1643,236 +2556,396 @@ int cloudsync_finalize_alter (sqlite3_context *context, cloudsync_context *data, if (pk_diff) { // drop meta-table, it will be recreated - char *sql = cloudsync_memory_mprintf("DROP TABLE IF EXISTS \"%w_cloudsync\";", table->name); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + char *sql = cloudsync_memory_mprintf(SQL_DROP_CLOUDSYNC_TABLE, table->meta_ref); + rc = database_exec(data, sql); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - DEBUG_SQLITE_ERROR(rc, "cloudsync_finalize_alter", db); + if (rc != DBRES_OK) { + DEBUG_DBERROR(rc, "cloudsync_finalize_alter", data); goto finalize; } } else { // compact meta-table // delete entries for removed columns - char *sql = cloudsync_memory_mprintf("DELETE FROM \"%w_cloudsync\" WHERE \"col_name\" NOT IN (" - "SELECT name FROM pragma_table_info('%q') UNION SELECT '%s'" - ")", table->name, table->name, CLOUDSYNC_TOMBSTONE_VALUE); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + const char *schema = table->schema ? table->schema : ""; + char *sql = sql_build_delete_cols_not_in_schema_query(schema, table->name, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE); + rc = database_exec(data, sql); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - DEBUG_SQLITE_ERROR(rc, "cloudsync_finalize_alter", db); + if (rc != DBRES_OK) { + DEBUG_DBERROR(rc, "cloudsync_finalize_alter", data); goto finalize; } - char *singlequote_escaped_table_name = cloudsync_memory_mprintf("%q", table->name); - sql = cloudsync_memory_mprintf("SELECT group_concat('\"%w\".\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%s') WHERE pk>0 ORDER BY pk;", singlequote_escaped_table_name, singlequote_escaped_table_name); - cloudsync_memory_free(singlequote_escaped_table_name); - if (!sql) { - rc = SQLITE_NOMEM; - goto finalize; - } - char *pkclause = dbutils_text_select(db, sql); - char *pkvalues = (pkclause) ? pkclause : "rowid"; + sql = sql_build_pk_qualified_collist_query(schema, table->name); + if (!sql) {rc = DBRES_NOMEM; goto finalize;} + + char *pkclause = NULL; + rc = database_select_text(data, sql, &pkclause); cloudsync_memory_free(sql); + if (rc != DBRES_OK) goto finalize; + char *pkvalues = (pkclause) ? pkclause : "rowid"; // delete entries related to rows that no longer exist in the original table, but preserve tombstone - sql = cloudsync_memory_mprintf("DELETE FROM \"%w_cloudsync\" WHERE (\"col_name\" != '%s' OR (\"col_name\" = '%s' AND col_version %% 2 != 0)) AND NOT EXISTS (SELECT 1 FROM \"%w\" WHERE \"%w_cloudsync\".pk = cloudsync_pk_encode(%s) LIMIT 1);", table->name, CLOUDSYNC_TOMBSTONE_VALUE, CLOUDSYNC_TOMBSTONE_VALUE, table->name, table->name, pkvalues); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_GC_DELETE_ORPHANED_PK, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE, CLOUDSYNC_TOMBSTONE_VALUE, table->base_ref, table->meta_ref, pkvalues); + rc = database_exec(data, sql); if (pkclause) cloudsync_memory_free(pkclause); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - DEBUG_SQLITE_ERROR(rc, "cloudsync_finalize_alter", db); + if (rc != DBRES_OK) { + DEBUG_DBERROR(rc, "cloudsync_finalize_alter", data); goto finalize; } } + // update key to be later used in cloudsync_dbversion_rebuild char buf[256]; - snprintf(buf, sizeof(buf), "%lld", data->db_version); - dbutils_settings_set_key_value(db, context, "pre_alter_dbversion", buf); + snprintf(buf, sizeof(buf), "%" PRId64, data->db_version); + dbutils_settings_set_key_value(data, "pre_alter_dbversion", buf); finalize: - sqlite3_free_table(result); - sqlite3_free(errmsg); - + table_pknames_free(result, nrows); + return rc; +} + +int cloudsync_commit_alter (cloudsync_context *data, const char *table_name) { + int rc = DBRES_MISUSE; + cloudsync_table_context *table = NULL; + + // init cloudsync_settings + if (cloudsync_context_init(data) == NULL) { + cloudsync_set_error(data, "Unable to initialize cloudsync context", DBRES_MISUSE); + goto rollback_finalize_alter; + } + + // lookup table + table = table_lookup(data, table_name); + if (!table) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to find table %s", table_name); + cloudsync_set_error(data, buffer, DBRES_MISUSE); + goto rollback_finalize_alter; + } + + // idempotent: if not altering, return OK + if (!table->is_altering) return DBRES_OK; + + rc = cloudsync_finalize_alter(data, table); + if (rc != DBRES_OK) goto rollback_finalize_alter; + + // the table is outdated, delete it and it will be reloaded in the cloudsync_init_internal + // is_altering is reset implicitly because table_free + cloudsync_init_table + // will reallocate the table context with zero-initialized memory + table_remove(data, table); + table_free(table); + table = NULL; + + // init again cloudsync for the table + table_algo algo_current = dbutils_table_settings_get_algo(data, table_name); + if (algo_current == table_algo_none) algo_current = dbutils_table_settings_get_algo(data, "*"); + rc = cloudsync_init_table(data, table_name, cloudsync_algo_name(algo_current), CLOUDSYNC_INIT_FLAG_SKIP_INT_PK_CHECK); + if (rc != DBRES_OK) goto rollback_finalize_alter; + + return DBRES_OK; + +rollback_finalize_alter: + if (table) { + table_set_pknames(table, NULL); + table->is_altering = false; + } return rc; } -int cloudsync_refill_metatable (sqlite3 *db, cloudsync_context *data, const char *table_name) { +// MARK: - Filter Rewrite - + +// Replace bare column names in a filter expression with prefix-qualified names. +// E.g., filter="user_id = 42", prefix="NEW", columns=["user_id","id"] → "NEW.\"user_id\" = 42" +// Columns must be sorted by length descending by the caller to avoid partial matches. +// Skips content inside single-quoted string literals. +// Returns a newly allocated string (caller must free with cloudsync_memory_free), or NULL on error. +// Helper: check if an identifier token matches a column name. +static bool filter_is_column (const char *token, size_t token_len, char **columns, int ncols) { + for (int i = 0; i < ncols; ++i) { + if (strlen(columns[i]) == token_len && strncmp(token, columns[i], token_len) == 0) + return true; + } + return false; +} + +// Helper: check if character is part of a SQL identifier. +static bool filter_is_ident_char (char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_'; +} + +char *cloudsync_filter_add_row_prefix (const char *filter, const char *prefix, char **columns, int ncols) { + if (!filter || !prefix || !columns || ncols <= 0) return NULL; + + size_t filter_len = strlen(filter); + size_t prefix_len = strlen(prefix); + + // Each identifier match grows by at most (prefix_len + 3) bytes. + // Worst case: the entire filter is one repeated column reference separated by + // single characters, so up to (filter_len / 2) matches. Use a safe upper bound. + size_t max_growth = (filter_len / 2 + 1) * (prefix_len + 3); + size_t cap = filter_len + max_growth + 64; + char *result = (char *)cloudsync_memory_alloc(cap); + if (!result) return NULL; + size_t out = 0; + + // Single pass: tokenize into identifiers, quoted strings, and everything else. + size_t i = 0; + while (i < filter_len) { + // Skip single-quoted string literals verbatim (handle '' escape) + if (filter[i] == '\'') { + result[out++] = filter[i++]; + while (i < filter_len) { + if (filter[i] == '\'') { + result[out++] = filter[i++]; + // '' is an escaped quote — keep going + if (i < filter_len && filter[i] == '\'') { + result[out++] = filter[i++]; + continue; + } + break; // single ' ends the literal + } + result[out++] = filter[i++]; + } + continue; + } + + // Extract identifier token + if (filter_is_ident_char(filter[i])) { + size_t start = i; + while (i < filter_len && filter_is_ident_char(filter[i])) ++i; + size_t token_len = i - start; + + if (filter_is_column(&filter[start], token_len, columns, ncols)) { + // Emit PREFIX."column_name" + memcpy(&result[out], prefix, prefix_len); out += prefix_len; + result[out++] = '.'; + result[out++] = '"'; + memcpy(&result[out], &filter[start], token_len); out += token_len; + result[out++] = '"'; + } else { + // Not a column — copy as-is + memcpy(&result[out], &filter[start], token_len); out += token_len; + } + continue; + } + + // Any other character — copy as-is + result[out++] = filter[i++]; + } + + result[out] = '\0'; + return result; +} + +int cloudsync_reset_metatable (cloudsync_context *data, const char *table_name) { cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) return SQLITE_INTERNAL; - - sqlite3_stmt *vm = NULL; - sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET); - - char *sql = cloudsync_memory_mprintf("SELECT group_concat('\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name); - char *pkclause_identifiers = dbutils_text_select(db, sql); - char *pkvalues_identifiers = (pkclause_identifiers) ? pkclause_identifiers : "rowid"; + if (!table) return DBRES_ERROR; + + char *sql = cloudsync_memory_mprintf(SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE, table->meta_ref); + int rc = database_exec(data, sql); cloudsync_memory_free(sql); - - sql = cloudsync_memory_mprintf("SELECT group_concat('cloudsync_pk_decode(pk, ' || pk || ') AS ' || '\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name); - char *pkdecode = dbutils_text_select(db, sql); - char *pkdecodeval = (pkdecode) ? pkdecode : "cloudsync_pk_decode(pk, 1) AS rowid"; + if (rc != DBRES_OK) return rc; + + return cloudsync_refill_metatable(data, table_name); +} + +int cloudsync_refill_metatable (cloudsync_context *data, const char *table_name) { + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) return DBRES_ERROR; + + dbvm_t *vm = NULL; + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + // Read row-level filter from settings (if any) + char filter_buf[2048]; + int frc = dbutils_table_settings_get_value(data, table_name, "*", "filter", filter_buf, sizeof(filter_buf)); + const char *filter = (frc == DBRES_OK && filter_buf[0]) ? filter_buf : NULL; + + const char *schema = table->schema ? table->schema : ""; + char *sql = sql_build_pk_collist_query(schema, table_name); + char *pkclause_identifiers = NULL; + int rc = database_select_text(data, sql, &pkclause_identifiers); cloudsync_memory_free(sql); - - sql = cloudsync_memory_mprintf("SELECT cloudsync_insert('%q', %s) FROM (SELECT %s FROM \"%w\" EXCEPT SELECT %s FROM \"%w_cloudsync\");", table_name, pkvalues_identifiers, pkvalues_identifiers, table_name, pkdecodeval, table_name); - int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != DBRES_OK) goto finalize; + char *pkvalues_identifiers = (pkclause_identifiers) ? pkclause_identifiers : "rowid"; + + // Use database-specific query builder to handle type differences in composite PKs + sql = sql_build_insert_missing_pks_query(schema, table_name, pkvalues_identifiers, table->base_ref, table->meta_ref, filter); + if (!sql) {rc = DBRES_NOMEM; goto finalize;} + rc = database_exec(data, sql); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; - + if (rc != DBRES_OK) goto finalize; + // fill missing colums // for each non-pk column: // The new query does 1 encode per source row and one indexed NOT-EXISTS probe. - // The old plan does many decodes per candidate and can’t use an index to rule out matches quickly—so it burns CPU and I/O. - - sql = cloudsync_memory_mprintf("WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM \"%w\") SELECT _cstemp1.pk FROM _cstemp1 WHERE NOT EXISTS (SELECT 1 FROM \"%w_cloudsync\" _cstemp2 WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = ?);", pkvalues_identifiers, table_name, table_name); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &vm, NULL); + // The old plan does many decodes per candidate and can't use an index to rule out matches quickly—so it burns CPU and I/O. + + if (filter) { + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL_FILTERED, pkvalues_identifiers, table->base_ref, filter, table->meta_ref); + } else { + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL, pkvalues_identifiers, table->base_ref, table->meta_ref); + } + rc = databasevm_prepare(data, sql, (void **)&vm, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; + if (rc != DBRES_OK) goto finalize; for (int i=0; incols; ++i) { char *col_name = table->col_name[i]; - rc = sqlite3_bind_text(vm, 1, col_name, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize; - + rc = databasevm_bind_text(vm, 1, col_name, -1); + if (rc != DBRES_OK) goto finalize; + while (1) { - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) { - const char *pk = (const char *)sqlite3_column_text(vm, 0); - size_t pklen = strlen(pk); - rc = local_mark_insert_or_update_meta(db, table, pk, pklen, col_name, db_version, BUMP_SEQ(data)); - } else if (rc == SQLITE_DONE) { - rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_ROW) { + size_t pklen = 0; + const void *pk = (const char *)database_column_blob(vm, 0, &pklen); + if (!pk) { rc = DBRES_ERROR; break; } + rc = local_mark_insert_or_update_meta(table, pk, pklen, col_name, db_version, cloudsync_bumpseq(data)); + } else if (rc == DBRES_DONE) { + rc = DBRES_OK; break; } else { break; } } - if (rc != SQLITE_OK) goto finalize; + if (rc != DBRES_OK) goto finalize; - sqlite3_reset(vm); + databasevm_reset(vm); } finalize: - if (rc != SQLITE_OK) DEBUG_ALWAYS("cloudsync_refill_metatable error: %s", sqlite3_errmsg(db)); + if (rc != DBRES_OK) {DEBUG_ALWAYS("cloudsync_refill_metatable error: %s", database_errmsg(data));} if (pkclause_identifiers) cloudsync_memory_free(pkclause_identifiers); - if (pkdecode) cloudsync_memory_free(pkdecode); - if (vm) sqlite3_finalize(vm); + if (vm) databasevm_finalize(vm); return rc; } // MARK: - Local - -int local_update_sentinel (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, sqlite3_int64 db_version, int seq) { - sqlite3_stmt *vm = table->meta_sentinel_update_stmt; +int local_update_sentinel (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq) { + dbvm_t *vm = table->meta_sentinel_update_stmt; if (!vm) return -1; - int rc = sqlite3_bind_int64(vm, 1, db_version); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_int(vm, 1, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 2, seq); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 2, seq); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_blob(vm, 3, pk, (int)pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_blob(vm, 3, pk, (int)pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - DEBUG_SQLITE_ERROR(rc, "local_update_sentinel", db); - sqlite3_reset(vm); + DEBUG_DBERROR(rc, "local_update_sentinel", table->context); + databasevm_reset(vm); return rc; } -int local_mark_insert_sentinel_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, sqlite3_int64 db_version, int seq) { - sqlite3_stmt *vm = table->meta_sentinel_insert_stmt; +int local_mark_insert_sentinel_meta (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq) { + dbvm_t *vm = table->meta_sentinel_insert_stmt; if (!vm) return -1; - int rc = sqlite3_bind_blob(vm, 1, pk, (int)pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, pk, (int)pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int64(vm, 2, db_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 2, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 3, seq); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 3, seq); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int64(vm, 4, db_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 4, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 5, seq); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 5, seq); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - DEBUG_SQLITE_ERROR(rc, "local_insert_sentinel", db); - sqlite3_reset(vm); + DEBUG_DBERROR(rc, "local_insert_sentinel", table->context); + databasevm_reset(vm); return rc; } -int local_mark_insert_or_update_meta_impl (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, const char *col_name, int col_version, sqlite3_int64 db_version, int seq) { +int local_mark_insert_or_update_meta_impl (cloudsync_table_context *table, const void *pk, size_t pklen, const char *col_name, int col_version, int64_t db_version, int seq) { - sqlite3_stmt *vm = table->meta_row_insert_update_stmt; + dbvm_t *vm = table->meta_row_insert_update_stmt; if (!vm) return -1; - int rc = sqlite3_bind_blob(vm, 1, pk, (int)pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_text(vm, 2, (col_name) ? col_name : CLOUDSYNC_TOMBSTONE_VALUE, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_text(vm, 2, (col_name) ? col_name : CLOUDSYNC_TOMBSTONE_VALUE, -1); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 3, col_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 3, col_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int64(vm, 4, db_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 4, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 5, seq); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 5, seq); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int64(vm, 6, db_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 6, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 7, seq); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 7, seq); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - DEBUG_SQLITE_ERROR(rc, "local_insert_or_update", db); - sqlite3_reset(vm); + DEBUG_DBERROR(rc, "local_insert_or_update", table->context); + databasevm_reset(vm); return rc; } -int local_mark_insert_or_update_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, const char *col_name, sqlite3_int64 db_version, int seq) { - return local_mark_insert_or_update_meta_impl(db, table, pk, pklen, col_name, 1, db_version, seq); +int local_mark_insert_or_update_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *col_name, int64_t db_version, int seq) { + return local_mark_insert_or_update_meta_impl(table, pk, pklen, col_name, 1, db_version, seq); +} + +int local_mark_delete_block_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *block_colname, int64_t db_version, int seq) { + // Mark a block as deleted by setting col_version = 2 (even = deleted) + return local_mark_insert_or_update_meta_impl(table, pk, pklen, block_colname, 2, db_version, seq); } -int local_mark_delete_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, sqlite3_int64 db_version, int seq) { - return local_mark_insert_or_update_meta_impl(db, table, pk, pklen, NULL, 2, db_version, seq); +int block_delete_value_external (cloudsync_context *data, cloudsync_table_context *table, const void *pk, size_t pklen, const char *block_colname) { + return block_delete_value(data, table, pk, (int)pklen, block_colname); } -int local_drop_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen) { - sqlite3_stmt *vm = table->meta_row_drop_stmt; +int local_mark_delete_meta (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq) { + return local_mark_insert_or_update_meta_impl(table, pk, pklen, NULL, 2, db_version, seq); +} + +int local_drop_meta (cloudsync_table_context *table, const void *pk, size_t pklen) { + dbvm_t *vm = table->meta_row_drop_stmt; if (!vm) return -1; - int rc = sqlite3_bind_blob(vm, 1, pk, (int)pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - DEBUG_SQLITE_ERROR(rc, "local_drop_meta", db); - sqlite3_reset(vm); + DEBUG_DBERROR(rc, "local_drop_meta", table->context); + databasevm_reset(vm); return rc; } -int local_update_move_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, const char *pk2, size_t pklen2, sqlite3_int64 db_version) { +int local_update_move_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const void *pk2, size_t pklen2, int64_t db_version) { /* * This function moves non-sentinel metadata entries from an old primary key (OLD.pk) * to a new primary key (NEW.pk) when a primary key change occurs. @@ -1885,7 +2958,7 @@ int local_update_move_meta (sqlite3 *db, cloudsync_table_context *table, const c * may be applied incorrectly, leading to data inconsistency. * * When performing the update, a unique `seq` must be assigned to each metadata row. This can be achieved - * by either incrementing the maximum sequence value in the table or using a function (e.g., `bump_seq(data)`) + * by either incrementing the maximum sequence value in the table or using a function (e.g., cloudsync_bumpseq(data)) * that generates a unique sequence for each row. The update query should ensure that each row moved * from OLD.pk to NEW.pk gets a distinct `seq` to maintain proper versioning and ordering of changes. */ @@ -1893,42 +2966,58 @@ int local_update_move_meta (sqlite3 *db, cloudsync_table_context *table, const c // see https://github.com/sqliteai/sqlite-sync/blob/main/docs/PriKey.md for more details // pk2 is the old pk - sqlite3_stmt *vm = table->meta_update_move_stmt; + dbvm_t *vm = table->meta_update_move_stmt; if (!vm) return -1; // new primary key - int rc = sqlite3_bind_blob(vm, 1, pk, (int)pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) goto cleanup; // new db_version - rc = sqlite3_bind_int64(vm, 2, db_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 2, db_version); + if (rc != DBRES_OK) goto cleanup; // old primary key - rc = sqlite3_bind_blob(vm, 3, pk2, (int)pklen2, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_blob(vm, 3, pk2, pklen2); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - DEBUG_SQLITE_ERROR(rc, "local_update_move_meta", db); - sqlite3_reset(vm); + DEBUG_DBERROR(rc, "local_update_move_meta", table->context); + databasevm_reset(vm); return rc; } // MARK: - Payload Encode / Decode - -bool cloudsync_buffer_free (cloudsync_data_payload *payload) { - if (payload) { - if (payload->buffer) cloudsync_memory_free(payload->buffer); - memset(payload, 0, sizeof(cloudsync_data_payload)); - } - - return false; +static void cloudsync_payload_checksum_store (cloudsync_payload_header *header, uint64_t checksum) { + uint64_t h = checksum & 0xFFFFFFFFFFFFULL; // keep 48 bits + header->checksum[0] = (uint8_t)(h >> 40); + header->checksum[1] = (uint8_t)(h >> 32); + header->checksum[2] = (uint8_t)(h >> 24); + header->checksum[3] = (uint8_t)(h >> 16); + header->checksum[4] = (uint8_t)(h >> 8); + header->checksum[5] = (uint8_t)(h >> 0); +} + +static uint64_t cloudsync_payload_checksum_load (cloudsync_payload_header *header) { + return ((uint64_t)header->checksum[0] << 40) | + ((uint64_t)header->checksum[1] << 32) | + ((uint64_t)header->checksum[2] << 24) | + ((uint64_t)header->checksum[3] << 16) | + ((uint64_t)header->checksum[4] << 8) | + ((uint64_t)header->checksum[5] << 0); } -bool cloudsync_buffer_check (cloudsync_data_payload *payload, size_t needed) { +static bool cloudsync_payload_checksum_verify (cloudsync_payload_header *header, uint64_t checksum) { + uint64_t checksum1 = cloudsync_payload_checksum_load(header); + uint64_t checksum2 = checksum & 0xFFFFFFFFFFFFULL; + return (checksum1 == checksum2); +} + +static bool cloudsync_payload_encode_check (cloudsync_payload_context *payload, size_t needed) { if (payload->nrows == 0) needed += sizeof(cloudsync_payload_header); // alloc/resize buffer @@ -1937,7 +3026,11 @@ bool cloudsync_buffer_check (cloudsync_data_payload *payload, size_t needed) { size_t balloc = payload->balloc + needed; char *buffer = cloudsync_memory_realloc(payload->buffer, balloc); - if (!buffer) return cloudsync_buffer_free(payload); + if (!buffer) { + if (payload->buffer) cloudsync_memory_free(payload->buffer); + memset(payload, 0, sizeof(cloudsync_payload_context)); + return false; + } payload->buffer = buffer; payload->balloc = balloc; @@ -1947,7 +3040,26 @@ bool cloudsync_buffer_check (cloudsync_data_payload *payload, size_t needed) { return true; } -void cloudsync_payload_header_init (cloudsync_payload_header *header, uint32_t expanded_size, uint16_t ncols, uint32_t nrows, uint64_t hash) { +size_t cloudsync_payload_context_size (size_t *header_size) { + if (header_size) *header_size = sizeof(cloudsync_payload_header); + return sizeof(cloudsync_payload_context); +} + +void cloudsync_payload_context_free (cloudsync_payload_context *payload) { + if (!payload) return; + if (payload->buffer) cloudsync_memory_free(payload->buffer); + cloudsync_memory_free(payload); +} + +uint64_t cloudsync_payload_context_nrows (cloudsync_payload_context *payload) { + return payload ? payload->nrows : 0; +} + +size_t cloudsync_payload_context_bused (cloudsync_payload_context *payload) { + return payload ? payload->bused : 0; +} + +void cloudsync_payload_header_init (cloudsync_payload_header *header, uint8_t version, uint32_t expanded_size, uint16_t ncols, uint32_t nrows, uint64_t hash) { memset(header, 0, sizeof(cloudsync_payload_header)); assert(sizeof(cloudsync_payload_header)==32); @@ -1955,146 +3067,485 @@ void cloudsync_payload_header_init (cloudsync_payload_header *header, uint32_t e sscanf(CLOUDSYNC_VERSION, "%d.%d.%d", &major, &minor, &patch); header->signature = htonl(CLOUDSYNC_PAYLOAD_SIGNATURE); - header->version = CLOUDSYNC_PAYLOAD_VERSION; - header->libversion[0] = major; - header->libversion[1] = minor; - header->libversion[2] = patch; + header->version = version; + header->libversion[0] = (uint8_t)major; + header->libversion[1] = (uint8_t)minor; + header->libversion[2] = (uint8_t)patch; header->expanded_size = htonl(expanded_size); header->ncols = htons(ncols); header->nrows = htonl(nrows); header->schema_hash = htonll(hash); } -void cloudsync_payload_encode_step (sqlite3_context *context, int argc, sqlite3_value **argv) { +int cloudsync_payload_encode_step (cloudsync_payload_context *payload, cloudsync_context *data, int argc, dbvalue_t **argv) { DEBUG_FUNCTION("cloudsync_payload_encode_step"); // debug_values(argc, argv); - // allocate/get the session context - cloudsync_data_payload *payload = (cloudsync_data_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_data_payload)); - if (!payload) return; - // check if the step function is called for the first time - if (payload->nrows == 0) payload->ncols = argc; + if (payload->nrows == 0) payload->ncols = (uint16_t)argc; - size_t breq = pk_encode_size(argv, argc, 0); - if (cloudsync_buffer_check(payload, breq) == false) return; + size_t breq = pk_encode_size((dbvalue_t **)argv, argc, 0, data->skip_decode_idx); + if (cloudsync_payload_encode_check(payload, breq) == false) { + return cloudsync_set_error(data, "Not enough memory to resize payload internal buffer", DBRES_NOMEM); + } char *buffer = payload->buffer + payload->bused; - char *ptr = pk_encode(argv, argc, buffer, false, NULL); - assert(buffer == ptr); + size_t bsize = payload->balloc - payload->bused; + char *p = pk_encode((dbvalue_t **)argv, argc, buffer, false, &bsize, data->skip_decode_idx); + if (!p) return cloudsync_set_error(data, "An error occurred while encoding payload", DBRES_ERROR); // update buffer payload->bused += breq; // increment row counter ++payload->nrows; + + return DBRES_OK; } -void cloudsync_payload_encode_final (sqlite3_context *context) { - DEBUG_FUNCTION("cloudsync_payload_encode_final"); - - // get the session context - cloudsync_data_payload *payload = (cloudsync_data_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_data_payload)); - if (!payload) return; - +static bool cloudsync_payload_append_raw (cloudsync_payload_context *payload, cloudsync_context *data, const char **fields, const size_t *field_sizes, int nfields, uint8_t version) { + size_t needed = 0; + for (int i = 0; i < nfields; ++i) { + if (field_sizes[i] > SIZE_MAX - needed) { + cloudsync_set_error(data, CLOUDSYNC_ERRCODE_ROW_TOO_LARGE "cloudsync payload raw row too large", DBRES_NOMEM); + return false; + } + needed += field_sizes[i]; + } + if (!cloudsync_payload_encode_check(payload, needed)) { + cloudsync_set_error(data, "Not enough memory to resize payload internal buffer", DBRES_NOMEM); + return false; + } if (payload->nrows == 0) { - sqlite3_result_null(context); - return; + payload->ncols = (uint16_t)nfields; + payload->version = version; } - - // encode payload - int header_size = (int)sizeof(cloudsync_payload_header); - int real_buffer_size = (int)(payload->bused - header_size); - int zbound = LZ4_compressBound(real_buffer_size); - char *buffer = cloudsync_memory_alloc(zbound + header_size); - if (!buffer) { - cloudsync_buffer_free(payload); - sqlite3_result_error_code(context, SQLITE_NOMEM); - return; + char *dst = payload->buffer + payload->bused; + for (int i = 0; i < nfields; ++i) { + memcpy(dst, fields[i], field_sizes[i]); + dst += field_sizes[i]; } - - // adjust buffer to compress to skip the reserved header - char *src_buffer = payload->buffer + sizeof(cloudsync_payload_header); - int zused = LZ4_compress_default(src_buffer, buffer+header_size, real_buffer_size, zbound); - bool use_uncompressed_buffer = (!zused || zused > real_buffer_size); - CHECK_FORCE_UNCOMPRESSED_BUFFER(); - - // setup payload header - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - cloudsync_payload_header header; - cloudsync_payload_header_init(&header, (use_uncompressed_buffer) ? 0 : real_buffer_size, payload->ncols, (uint32_t)payload->nrows, data->schema_hash); - - // if compression fails or if compressed size is bigger than original buffer, then use the uncompressed buffer - if (use_uncompressed_buffer) { - cloudsync_memory_free(buffer); - buffer = payload->buffer; - zused = real_buffer_size; + payload->bused += needed; + ++payload->nrows; + return true; +} + +int cloudsync_payload_max_chunk_size (cloudsync_context *data) { + int64_t value = dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_PAYLOAD_MAX_CHUNK_SIZE); + if (value <= 0) value = CLOUDSYNC_PAYLOAD_CHUNK_DEFAULT_SIZE; + if (value < CLOUDSYNC_PAYLOAD_CHUNK_MIN_SIZE) value = CLOUDSYNC_PAYLOAD_CHUNK_MIN_SIZE; + if (value > CLOUDSYNC_PAYLOAD_CHUNK_MAX_SIZE) value = CLOUDSYNC_PAYLOAD_CHUNK_MAX_SIZE; + return (int)value; +} + +int cloudsync_payload_fragment_target_size (cloudsync_context *data) { + int max_size = cloudsync_payload_max_chunk_size(data); + int target = max_size - (int)sizeof(cloudsync_payload_header) - CLOUDSYNC_PAYLOAD_CHUNK_SAFETY_MARGIN; + if (target < 1024) target = 1024; + return target; +} + +static size_t cloudsync_payload_decimal_len_i64 (int64_t value) { + size_t len = value < 0 ? 1 : 0; + uint64_t v = (value < 0) ? (uint64_t)(-(value + 1)) + 1u : (uint64_t)value; + do { + len++; + v /= 10u; + } while (v != 0); + return len; +} + +static bool cloudsync_payload_size_add (size_t *acc, size_t value) { + if (value > SIZE_MAX - *acc) return false; + *acc += value; + return true; +} + +int cloudsync_payload_fragment_count (int64_t total_size, int target_size) { + if (total_size <= 0 || target_size <= 0) return 0; + uint64_t total = (uint64_t)total_size; + uint64_t target = (uint64_t)target_size; + uint64_t count = total / target + ((total % target) != 0); + if (count == 0 || count > INT_MAX) return 0; + return (int)count; +} + +int cloudsync_payload_fragment_data_size (cloudsync_context *data, + const char *tbl, int tbl_len, + const void *pk, int pk_len, + const char *col_name, int col_name_len, + int64_t col_version, int64_t db_version, + const void *site_id, int site_id_len, + int64_t cl, int64_t seq, + int64_t total_size, + int part_index, int part_count) { + UNUSED_PARAMETER(pk); + UNUSED_PARAMETER(site_id); + if (tbl_len < 0 && tbl) tbl_len = (int)strlen(tbl); + if (col_name_len < 0 && col_name) col_name_len = (int)strlen(col_name); + if (tbl_len < 0 || pk_len < 0 || col_name_len < 0 || site_id_len < 0 || total_size < 0 || part_index < 0 || part_count <= 0) { + return 0; + } + + size_t fixed = sizeof(cloudsync_payload_header); + size_t frag_col_len = strlen(CLOUDSYNC_PAYLOAD_FRAGMENT_PREFIX) + 32 + 1 + 16 + 1 + + cloudsync_payload_decimal_len_i64(part_index) + 1 + + cloudsync_payload_decimal_len_i64(part_count) + 1 + + cloudsync_payload_decimal_len_i64(total_size) + 1 + + (size_t)col_name_len; + size_t sizes[] = { + pk_encode_raw_size(DBTYPE_TEXT, tbl_len), + pk_encode_raw_size(DBTYPE_BLOB, pk_len), + pk_encode_raw_size(DBTYPE_TEXT, (int64_t)frag_col_len), + pk_encode_raw_size(DBTYPE_INTEGER, col_version), + pk_encode_raw_size(DBTYPE_INTEGER, db_version), + pk_encode_raw_size(DBTYPE_BLOB, site_id_len), + pk_encode_raw_size(DBTYPE_INTEGER, cl), + pk_encode_raw_size(DBTYPE_INTEGER, seq) + }; + for (size_t i = 0; i < sizeof(sizes) / sizeof(sizes[0]); ++i) { + if (sizes[i] == SIZE_MAX || !cloudsync_payload_size_add(&fixed, sizes[i])) return 0; + } + + int max_size = cloudsync_payload_max_chunk_size(data); + if (fixed >= (size_t)max_size) return 0; + + size_t candidate = (size_t)max_size - fixed; + if (candidate > INT_MAX) candidate = INT_MAX; + while (candidate > 0) { + size_t frag_size = pk_encode_raw_size(DBTYPE_BLOB, (int64_t)candidate); + if (frag_size == SIZE_MAX) return 0; + if (fixed <= (size_t)max_size && frag_size <= (size_t)max_size - fixed) return (int)candidate; + size_t total = fixed + frag_size; + size_t over = total > (size_t)max_size ? total - (size_t)max_size : 1; + if (candidate <= over) return 0; + candidate -= over; } - - // copy header and data to SQLite BLOB - memcpy(buffer, &header, sizeof(cloudsync_payload_header)); - int blob_size = zused+sizeof(cloudsync_payload_header); - sqlite3_result_blob(context, buffer, blob_size, SQLITE_TRANSIENT); - - // cleanup memory - cloudsync_buffer_free(payload); - if (!use_uncompressed_buffer) cloudsync_memory_free(buffer); + return 0; } -cloudsync_payload_apply_callback_t cloudsync_get_payload_apply_callback(sqlite3 *db) { - return (sqlite3_libversion_number() >= 3044000) ? sqlite3_get_clientdata(db, CLOUDSYNC_PAYLOAD_APPLY_CALLBACK_KEY) : NULL; +int cloudsync_payload_encoded_value_header (dbvalue_t *value, char *header, int header_cap, int64_t *payload_len) { + if (!value || !header || header_cap <= 0 || !payload_len) return -1; + int type = database_value_type(value); + *payload_len = 0; + if (type != DBTYPE_TEXT && type != DBTYPE_BLOB) return 0; + int64_t len = database_value_bytes(value); + if (len < 0) return -1; + *payload_len = len; + size_t total = pk_encode_raw_size(type, len); + if (total == SIZE_MAX || total < (size_t)len || total - (size_t)len > (size_t)header_cap) return -1; + if (type == DBTYPE_TEXT) { + size_t nbytes = pk_encode_raw_size(type, len) - (size_t)len - 1; + uint8_t type_byte = (uint8_t)((nbytes << 3) | DBTYPE_TEXT); + header[0] = (char)type_byte; + for (size_t i = 0; i < nbytes; i++) header[1 + i] = (uint8_t)(((uint64_t)len >> (8 * (nbytes - 1 - i))) & 0xFFu); + return (int)(1 + nbytes); + } else { + size_t nbytes = pk_encode_raw_size(type, len) - (size_t)len - 1; + uint8_t type_byte = (uint8_t)((nbytes << 3) | DBTYPE_BLOB); + header[0] = (char)type_byte; + for (size_t i = 0; i < nbytes; i++) header[1 + i] = (uint8_t)(((uint64_t)len >> (8 * (nbytes - 1 - i))) & 0xFFu); + return (int)(1 + nbytes); + } } -void cloudsync_set_payload_apply_callback(sqlite3 *db, cloudsync_payload_apply_callback_t callback) { - if (sqlite3_libversion_number() >= 3044000) { - sqlite3_set_clientdata(db, CLOUDSYNC_PAYLOAD_APPLY_CALLBACK_KEY, (void*)callback, NULL); +uint64_t cloudsync_payload_encoded_value_checksum (dbvalue_t *value) { + if (!value) return 0; + int type = database_value_type(value); + if (type != DBTYPE_TEXT && type != DBTYPE_BLOB) { + size_t len = pk_encode_size(&value, 1, 0, -1); + char stack[32]; + char *buf = stack; + if (len > sizeof(stack)) buf = cloudsync_memory_alloc((uint64_t)len); + if (!buf) return 0; + size_t bsize = len; + pk_encode(&value, 1, buf, false, &bsize, -1); + uint64_t h = pk_checksum(buf, bsize); + if (buf != stack) cloudsync_memory_free(buf); + return h; + } + char header[16]; + int64_t payload_len = 0; + int header_len = cloudsync_payload_encoded_value_header(value, header, sizeof(header), &payload_len); + if (header_len <= 0) return 0; + uint64_t h = pk_checksum(header, (size_t)header_len); + const char *p = (const char *)database_value_blob(value); + if (p && payload_len > 0) { + const uint8_t *bytes = (const uint8_t *)p; + for (int64_t i = 0; i < payload_len; ++i) { + h ^= bytes[i]; + h *= 1099511628211ULL; + } } + return h; } -int cloudsync_pk_decode_bind_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { - cloudsync_pk_decode_bind_context *decode_context = (cloudsync_pk_decode_bind_context*)xdata; - int rc = pk_decode_bind_callback(decode_context->vm, index, type, ival, dval, pval); - - if (rc == SQLITE_OK) { - // the dbversion index is smaller than seq index, so it is processed first - // when processing the dbversion column: save the value to the tmp_dbversion field - // when processing the seq column: update the dbversion and seq fields only if the current dbversion is greater than the last max value - switch (index) { - case CLOUDSYNC_PK_INDEX_TBL: - if (type == SQLITE_TEXT) { - decode_context->tbl = pval; +static uint64_t cloudsync_checksum_update (uint64_t h, const void *data, size_t len) { + const uint8_t *p = (const uint8_t *)data; + for (size_t i = 0; i < len; ++i) { + h ^= p[i]; + h *= 1099511628211ULL; + } + return h; +} + +static uint64_t cloudsync_checksum_update_i64 (uint64_t h, int64_t value) { + uint64_t v = (uint64_t)value; + for (int i = 7; i >= 0; --i) { + uint8_t b = (uint8_t)((v >> (8 * i)) & 0xffu); + h = cloudsync_checksum_update(h, &b, 1); + } + return h; +} + +static void cloudsync_payload_fragment_value_id (char out[33], + const char *tbl, int tbl_len, + const void *pk, int pk_len, + const char *col_name, int col_name_len, + int64_t col_version, int64_t db_version, + const void *site_id, int site_id_len, + int64_t cl, int64_t seq, + uint64_t value_checksum, + int64_t total_size) { + uint64_t h1 = 14695981039346656037ULL; + uint64_t h2 = 1099511628211ULL; + const char sep = '\x1f'; + + h1 = cloudsync_checksum_update(h1, tbl, (size_t)tbl_len); + h1 = cloudsync_checksum_update(h1, &sep, 1); + h1 = cloudsync_checksum_update(h1, pk, (size_t)pk_len); + h1 = cloudsync_checksum_update(h1, &sep, 1); + h1 = cloudsync_checksum_update(h1, col_name, (size_t)col_name_len); + h1 = cloudsync_checksum_update(h1, &sep, 1); + h1 = cloudsync_checksum_update(h1, site_id, (size_t)site_id_len); + h1 = cloudsync_checksum_update_i64(h1, col_version); + h1 = cloudsync_checksum_update_i64(h1, db_version); + h1 = cloudsync_checksum_update_i64(h1, cl); + h1 = cloudsync_checksum_update_i64(h1, seq); + h1 = cloudsync_checksum_update_i64(h1, (int64_t)value_checksum); + h1 = cloudsync_checksum_update_i64(h1, total_size); + + h2 = cloudsync_checksum_update_i64(h2, total_size); + h2 = cloudsync_checksum_update_i64(h2, (int64_t)value_checksum); + h2 = cloudsync_checksum_update(h2, site_id, (size_t)site_id_len); + h2 = cloudsync_checksum_update(h2, col_name, (size_t)col_name_len); + h2 = cloudsync_checksum_update(h2, pk, (size_t)pk_len); + h2 = cloudsync_checksum_update(h2, tbl, (size_t)tbl_len); + h2 = cloudsync_checksum_update_i64(h2, seq); + h2 = cloudsync_checksum_update_i64(h2, cl); + h2 = cloudsync_checksum_update_i64(h2, db_version); + h2 = cloudsync_checksum_update_i64(h2, col_version); + + snprintf(out, 33, "%016" PRIx64 "%016" PRIx64, h1, h2); +} + +int cloudsync_payload_encode_fragment_step (cloudsync_payload_context *payload, cloudsync_context *data, + const char *tbl, int tbl_len, + const void *pk, int pk_len, + const char *col_name, int col_name_len, + const void *fragment, int fragment_len, + int64_t col_version, int64_t db_version, + const void *site_id, int site_id_len, + int64_t cl, int64_t seq, + uint64_t value_checksum, + int64_t total_size, + int part_index, int part_count) { + if (!payload || !data || !tbl || !pk || !col_name || !fragment || !site_id) return DBRES_MISUSE; + if (tbl_len < 0) tbl_len = (int)strlen(tbl); + if (col_name_len < 0) col_name_len = (int)strlen(col_name); + if (tbl_len < 0 || pk_len < 0 || col_name_len < 0 || fragment_len <= 0 || site_id_len < 0 || + total_size <= 0 || part_index < 0 || part_count <= 0 || part_index >= part_count) { + return DBRES_MISUSE; + } + + char value_id[33]; + char checksum_hex[17]; + cloudsync_payload_fragment_value_id(value_id, tbl, tbl_len, pk, pk_len, col_name, col_name_len, + col_version, db_version, site_id, site_id_len, cl, seq, + value_checksum, total_size); + snprintf(checksum_hex, sizeof(checksum_hex), "%016" PRIx64, value_checksum); + + char *frag_col = cloudsync_memory_mprintf("%s%s:%s:%d:%d:%" PRId64 ":%.*s", + CLOUDSYNC_PAYLOAD_FRAGMENT_PREFIX, + value_id, checksum_hex, part_index, part_count, total_size, + col_name_len, col_name); + if (!frag_col) return DBRES_NOMEM; + + size_t sizes[9] = {0}; + sizes[0] = pk_encode_raw_size(DBTYPE_TEXT, tbl_len); + sizes[1] = pk_encode_raw_size(DBTYPE_BLOB, pk_len); + sizes[2] = pk_encode_raw_size(DBTYPE_TEXT, (int64_t)strlen(frag_col)); + sizes[3] = pk_encode_raw_size(DBTYPE_BLOB, fragment_len); + sizes[4] = pk_encode_raw_size(DBTYPE_INTEGER, col_version); + sizes[5] = pk_encode_raw_size(DBTYPE_INTEGER, db_version); + sizes[6] = pk_encode_raw_size(DBTYPE_BLOB, site_id_len); + sizes[7] = pk_encode_raw_size(DBTYPE_INTEGER, cl); + sizes[8] = pk_encode_raw_size(DBTYPE_INTEGER, seq); + for (int i = 0; i < 9; ++i) { + if (sizes[i] == SIZE_MAX) { cloudsync_memory_free(frag_col); return DBRES_NOMEM; } + } + + char stack[9][64]; + char *fields[9] = {0}; + for (int i = 0; i < 9; ++i) { + fields[i] = sizes[i] <= sizeof(stack[0]) ? stack[i] : cloudsync_memory_alloc((uint64_t)sizes[i]); + if (!fields[i]) { + for (int j = 0; j < i; ++j) if (fields[j] && (fields[j] < (char *)stack || fields[j] >= (char *)(stack + 9))) cloudsync_memory_free(fields[j]); + cloudsync_memory_free(frag_col); + return DBRES_NOMEM; + } + } + + pk_encode_raw_text(fields[0], tbl, (size_t)tbl_len); + pk_encode_raw_blob(fields[1], pk, (size_t)pk_len); + pk_encode_raw_text(fields[2], frag_col, strlen(frag_col)); + pk_encode_raw_blob(fields[3], fragment, (size_t)fragment_len); + pk_encode_raw_int(fields[4], col_version); + pk_encode_raw_int(fields[5], db_version); + pk_encode_raw_blob(fields[6], site_id, (size_t)site_id_len); + pk_encode_raw_int(fields[7], cl); + pk_encode_raw_int(fields[8], seq); + + const char *cfields[9]; + for (int i = 0; i < 9; ++i) cfields[i] = fields[i]; + bool ok = cloudsync_payload_append_raw(payload, data, cfields, sizes, 9, CLOUDSYNC_PAYLOAD_VERSION_3); + + for (int i = 0; i < 9; ++i) { + if (!(fields[i] >= (char *)stack && fields[i] < (char *)(stack + 9))) cloudsync_memory_free(fields[i]); + } + cloudsync_memory_free(frag_col); + return ok ? DBRES_OK : cloudsync_errcode(data); +} + +int cloudsync_payload_encode_final (cloudsync_payload_context *payload, cloudsync_context *data) { + DEBUG_FUNCTION("cloudsync_payload_encode_final"); + + if (payload->nrows == 0) { + if (payload->buffer) cloudsync_memory_free(payload->buffer); + payload->buffer = NULL; + payload->bsize = 0; + return DBRES_OK; + } + + if (payload->nrows > UINT32_MAX) { + if (payload->buffer) cloudsync_memory_free(payload->buffer); + payload->buffer = NULL; + payload->bsize = 0; + cloudsync_set_error(data, "Maximum number of payload rows reached", DBRES_ERROR); + return DBRES_ERROR; + } + + // sanity check about buffer size + int header_size = (int)sizeof(cloudsync_payload_header); + int64_t buffer_size = (int64_t)payload->bused - (int64_t)header_size; + if (buffer_size < 0) { + if (payload->buffer) cloudsync_memory_free(payload->buffer); + payload->buffer = NULL; + payload->bsize = 0; + cloudsync_set_error(data, "cloudsync_encode: internal size underflow", DBRES_ERROR); + return DBRES_ERROR; + } + if (buffer_size > INT_MAX) { + if (payload->buffer) cloudsync_memory_free(payload->buffer); + payload->buffer = NULL; + payload->bsize = 0; + cloudsync_set_error(data, CLOUDSYNC_ERRCODE_PAYLOAD_TOO_LARGE "cloudsync_encode: payload too large to compress (INT_MAX limit)", DBRES_ERROR); + return DBRES_ERROR; + } + // try to allocate buffer used for compressed data + int real_buffer_size = (int)buffer_size; + int zbound = LZ4_compressBound(real_buffer_size); + char *zbuffer = cloudsync_memory_alloc(zbound + header_size); // if for some reasons allocation fails then just skip compression + + // skip the reserved header from the buffer to compress + char *src_buffer = payload->buffer + sizeof(cloudsync_payload_header); + int zused = (zbuffer) ? LZ4_compress_default(src_buffer, zbuffer+header_size, real_buffer_size, zbound) : 0; + bool use_uncompressed_buffer = (!zused || zused > real_buffer_size); + CHECK_FORCE_UNCOMPRESSED_BUFFER(); + + // setup payload header + cloudsync_payload_header header = {0}; + uint32_t expanded_size = (use_uncompressed_buffer) ? 0 : real_buffer_size; + uint8_t version = payload->version ? payload->version : CLOUDSYNC_PAYLOAD_VERSION_LATEST; + cloudsync_payload_header_init(&header, version, expanded_size, payload->ncols, (uint32_t)payload->nrows, data->schema_hash); + + // if compression fails or if compressed size is bigger than original buffer, then use the uncompressed buffer + if (use_uncompressed_buffer) { + if (zbuffer) cloudsync_memory_free(zbuffer); + zbuffer = payload->buffer; + zused = real_buffer_size; + } + + // compute checksum of the buffer + uint64_t checksum = pk_checksum(zbuffer + header_size, zused); + cloudsync_payload_checksum_store(&header, checksum); + + // copy header and data to SQLite BLOB + memcpy(zbuffer, &header, sizeof(cloudsync_payload_header)); + int blob_size = zused + sizeof(cloudsync_payload_header); + payload->bsize = blob_size; + + // cleanup memory + if (zbuffer != payload->buffer) { + cloudsync_memory_free (payload->buffer); + payload->buffer = zbuffer; + } + + return DBRES_OK; +} + +char *cloudsync_payload_blob (cloudsync_payload_context *payload, int64_t *blob_size, int64_t *nrows) { + DEBUG_FUNCTION("cloudsync_payload_blob"); + + if (blob_size) *blob_size = (int64_t)payload->bsize; + if (nrows) *nrows = (int64_t)payload->nrows; + return payload->buffer; +} + +static int cloudsync_payload_decode_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { + cloudsync_pk_decode_bind_context *decode_context = (cloudsync_pk_decode_bind_context*)xdata; + int rc = pk_decode_bind_callback(decode_context->vm, index, type, ival, dval, pval); + + if (rc == DBRES_OK) { + // the dbversion index is smaller than seq index, so it is processed first + // when processing the dbversion column: save the value to the tmp_dbversion field + // when processing the seq column: update the dbversion and seq fields only if the current dbversion is greater than the last max value + switch (index) { + case CLOUDSYNC_PK_INDEX_TBL: + if (type == DBTYPE_TEXT) { + decode_context->tbl = pval; decode_context->tbl_len = ival; } break; case CLOUDSYNC_PK_INDEX_PK: - if (type == SQLITE_BLOB) { + if (type == DBTYPE_BLOB) { decode_context->pk = pval; decode_context->pk_len = ival; } break; case CLOUDSYNC_PK_INDEX_COLNAME: - if (type == SQLITE_TEXT) { + if (type == DBTYPE_TEXT) { decode_context->col_name = pval; decode_context->col_name_len = ival; } break; case CLOUDSYNC_PK_INDEX_COLVERSION: - if (type == SQLITE_INTEGER) decode_context->col_version = ival; + if (type == DBTYPE_INTEGER) decode_context->col_version = ival; break; case CLOUDSYNC_PK_INDEX_DBVERSION: - if (type == SQLITE_INTEGER) decode_context->db_version = ival; + if (type == DBTYPE_INTEGER) decode_context->db_version = ival; break; case CLOUDSYNC_PK_INDEX_SITEID: - if (type == SQLITE_BLOB) { + if (type == DBTYPE_BLOB) { decode_context->site_id = pval; decode_context->site_id_len = ival; } break; case CLOUDSYNC_PK_INDEX_CL: - if (type == SQLITE_INTEGER) decode_context->cl = ival; + if (type == DBTYPE_INTEGER) decode_context->cl = ival; break; case CLOUDSYNC_PK_INDEX_SEQ: - if (type == SQLITE_INTEGER) decode_context->seq = ival; + if (type == DBTYPE_INTEGER) decode_context->seq = ival; break; } } @@ -2102,1026 +3553,1023 @@ int cloudsync_pk_decode_bind_callback (void *xdata, int index, int type, int64_t return rc; } +typedef struct { + const char *tbl; + int64_t tbl_len; + const void *pk; + int64_t pk_len; + const char *col_name; + int64_t col_name_len; + const void *col_value; + int64_t col_value_len; + int64_t col_version; + int64_t db_version; + const void *site_id; + int64_t site_id_len; + int64_t cl; + int64_t seq; +} cloudsync_payload_fragment_row; + +static int cloudsync_payload_fragment_decode_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { + UNUSED_PARAMETER(dval); + cloudsync_payload_fragment_row *row = (cloudsync_payload_fragment_row *)xdata; + switch (index) { + case CLOUDSYNC_PK_INDEX_TBL: + if (type != DBTYPE_TEXT) return DBRES_ERROR; + row->tbl = pval; row->tbl_len = ival; + break; + case CLOUDSYNC_PK_INDEX_PK: + if (type != DBTYPE_BLOB) return DBRES_ERROR; + row->pk = pval; row->pk_len = ival; + break; + case CLOUDSYNC_PK_INDEX_COLNAME: + if (type != DBTYPE_TEXT) return DBRES_ERROR; + row->col_name = pval; row->col_name_len = ival; + break; + case CLOUDSYNC_PK_INDEX_COLVALUE: + if (type != DBTYPE_BLOB) return DBRES_ERROR; + row->col_value = pval; row->col_value_len = ival; + break; + case CLOUDSYNC_PK_INDEX_COLVERSION: + if (type != DBTYPE_INTEGER) return DBRES_ERROR; + row->col_version = ival; + break; + case CLOUDSYNC_PK_INDEX_DBVERSION: + if (type != DBTYPE_INTEGER) return DBRES_ERROR; + row->db_version = ival; + break; + case CLOUDSYNC_PK_INDEX_SITEID: + if (type != DBTYPE_BLOB) return DBRES_ERROR; + row->site_id = pval; row->site_id_len = ival; + break; + case CLOUDSYNC_PK_INDEX_CL: + if (type != DBTYPE_INTEGER) return DBRES_ERROR; + row->cl = ival; + break; + case CLOUDSYNC_PK_INDEX_SEQ: + if (type != DBTYPE_INTEGER) return DBRES_ERROR; + row->seq = ival; + break; + } + return DBRES_OK; +} + +static bool cloudsync_payload_is_hex (const char *value, size_t len) { + for (size_t i = 0; i < len; ++i) { + char c = value[i]; + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) return false; + } + return true; +} + +static bool cloudsync_payload_parse_u64_segment (const char *start, const char *end, uint64_t max_value, uint64_t *out, const char **next) { + if (!start || !end || start >= end) return false; + uint64_t value = 0; + const char *p = start; + while (p < end && *p >= '0' && *p <= '9') { + uint64_t digit = (uint64_t)(*p - '0'); + if (value > (max_value - digit) / 10u) return false; + value = value * 10u + digit; + p++; + } + if (p == start || p >= end || *p != ':') return false; + *out = value; + *next = p + 1; + return true; +} + +static bool cloudsync_payload_fragment_parse_colname (const char *col_name, int64_t col_name_len, + char *value_id, size_t value_id_len, + char *checksum_hex, size_t checksum_hex_len, + int *part_index, int *part_count, + int64_t *total_size, + const char **base_col, int64_t *base_col_len) { + size_t prefix_len = strlen(CLOUDSYNC_PAYLOAD_FRAGMENT_PREFIX); + if (!col_name || col_name_len <= (int64_t)prefix_len) return false; + if (strncmp(col_name, CLOUDSYNC_PAYLOAD_FRAGMENT_PREFIX, prefix_len) != 0) return false; + + const char *p = col_name + prefix_len; + const char *end = col_name + col_name_len; + const char *sep = memchr(p, ':', (size_t)(end - p)); + if (!sep || (size_t)(sep - p) + 1 > value_id_len) return false; + if ((sep - p) != 32 || !cloudsync_payload_is_hex(p, (size_t)(sep - p))) return false; + memcpy(value_id, p, (size_t)(sep - p)); + value_id[sep - p] = 0; + + p = sep + 1; + sep = memchr(p, ':', (size_t)(end - p)); + if (!sep || (size_t)(sep - p) + 1 > checksum_hex_len) return false; + if ((sep - p) != 16 || !cloudsync_payload_is_hex(p, (size_t)(sep - p))) return false; + memcpy(checksum_hex, p, (size_t)(sep - p)); + checksum_hex[sep - p] = 0; + + const char *next = NULL; + uint64_t parsed = 0; + if (!cloudsync_payload_parse_u64_segment(sep + 1, end, INT_MAX, &parsed, &next)) return false; + *part_index = (int)parsed; + + if (!cloudsync_payload_parse_u64_segment(next, end, INT_MAX, &parsed, &next)) return false; + *part_count = (int)parsed; + + if (!cloudsync_payload_parse_u64_segment(next, end, INT64_MAX, &parsed, &next)) return false; + *total_size = (int64_t)parsed; + + *base_col = next; + *base_col_len = end - *base_col; + return (*part_count > 0 && *part_index < *part_count && *base_col_len > 0); +} + +typedef struct { + dbvm_t *vm; + int param_index; +} cloudsync_payload_bind_param_context; + +static int cloudsync_payload_bind_param_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { + UNUSED_PARAMETER(index); + cloudsync_payload_bind_param_context *ctx = (cloudsync_payload_bind_param_context *)xdata; + switch (type) { + case DBTYPE_INTEGER: return databasevm_bind_int(ctx->vm, ctx->param_index, ival); + case DBTYPE_FLOAT: return databasevm_bind_double(ctx->vm, ctx->param_index, dval); + case DBTYPE_NULL: return databasevm_bind_null(ctx->vm, ctx->param_index); + case DBTYPE_TEXT: return databasevm_bind_text(ctx->vm, ctx->param_index, pval, (int)ival); + case DBTYPE_BLOB: return databasevm_bind_blob(ctx->vm, ctx->param_index, pval, (uint64_t)ival); + } + return DBRES_MISUSE; +} + +static int cloudsync_payload_fragments_cleanup_stale (cloudsync_context *data) { + // Stale-fragment GC is pure maintenance (it removes incomplete fragment groups + // older than CLOUDSYNC_PAYLOAD_FRAGMENT_STALE_SECONDS), so it has no correctness + // deadline. It runs a full GROUP BY scan of the fragments table; calling it on + // every applied fragment would be O(n^2) for a heavily-fragmented value, since + // each fragment arrives as its own apply call. Throttle it to at most once per + // CLOUDSYNC_PAYLOAD_FRAGMENT_CLEANUP_MIN_INTERVAL per connection. + int64_t now = (int64_t)time(NULL); + if (data->last_fragment_cleanup != 0 && + now - data->last_fragment_cleanup < CLOUDSYNC_PAYLOAD_FRAGMENT_CLEANUP_MIN_INTERVAL) { + return DBRES_OK; + } + data->last_fragment_cleanup = now; + + dbvm_t *vm = NULL; + int rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE, &vm, 0); + if (rc != DBRES_OK) return rc; + int64_t cutoff = now - CLOUDSYNC_PAYLOAD_FRAGMENT_STALE_SECONDS; + rc = databasevm_bind_int(vm, 1, cutoff); + if (rc == DBRES_OK) rc = databasevm_step(vm); + databasevm_finalize(vm); + return (rc == DBRES_DONE) ? DBRES_OK : rc; +} + +static int cloudsync_payload_apply_single_decoded_row (cloudsync_context *data, + const char *tbl, size_t tbl_len, + const char *pk, size_t pk_len, + const char *col_name, size_t col_name_len, + const char *encoded_value, size_t encoded_value_len, + int64_t col_version, int64_t db_version, + const char *site_id, size_t site_id_len, + int64_t cl, int64_t seq, + int *pnrows) { + int rc = DBRES_OK; + dbvm_t *vm = NULL; + bool in_savepoint = false; + merge_pending_batch batch = {0}; + + rc = databasevm_prepare(data, SQL_CHANGES_INSERT_ROW, &vm, 0); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: error while compiling SQL statement", rc); + + rc = databasevm_bind_text(vm, 1, tbl, (int)tbl_len); + if (rc == DBRES_OK) rc = databasevm_bind_blob(vm, 2, pk, (uint64_t)pk_len); + if (rc == DBRES_OK) rc = databasevm_bind_text(vm, 3, col_name, (int)col_name_len); + if (rc == DBRES_OK) { + if (data->skip_decode_idx == CLOUDSYNC_PK_INDEX_COLVALUE) { + rc = databasevm_bind_blob(vm, 4, encoded_value, (uint64_t)encoded_value_len); + } else { + size_t seek = 0; + cloudsync_payload_bind_param_context bind_ctx = {.vm = vm, .param_index = 4}; + int res = pk_decode((char *)encoded_value, encoded_value_len, 1, &seek, -1, cloudsync_payload_bind_param_callback, &bind_ctx); + if (res == -1 || seek != encoded_value_len) rc = cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid v3 encoded value", DBRES_MISUSE); + } + } + if (rc == DBRES_OK) rc = databasevm_bind_int(vm, 5, col_version); + if (rc == DBRES_OK) rc = databasevm_bind_int(vm, 6, db_version); + if (rc == DBRES_OK) rc = databasevm_bind_blob(vm, 7, site_id, (uint64_t)site_id_len); + if (rc == DBRES_OK) rc = databasevm_bind_int(vm, 8, cl); + if (rc == DBRES_OK) rc = databasevm_bind_int(vm, 9, seq); + if (rc != DBRES_OK) goto cleanup; + + if (!database_in_transaction(data)) { + rc = database_begin_savepoint(data, "cloudsync_payload_apply"); + if (rc != DBRES_OK) goto cleanup; + in_savepoint = true; + } + + data->pending_batch = &batch; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); + goto cleanup; + } + + rc = merge_flush_pending(data); + if (rc != DBRES_OK) goto cleanup; + data->pending_batch = NULL; + + if (in_savepoint) { + rc = database_commit_savepoint(data, "cloudsync_payload_apply"); + in_savepoint = false; + if (rc != DBRES_OK) goto cleanup; + } + + // Do NOT advance the receive cursor here: a v3 value carries a single + // (db_version, seq) that can be in the middle of its source db_version, and a + // db_version's chunks can span multiple /check artifacts. Advancing per value + // would leave the cursor mid-db_version. The durable cursor is advanced once, + // after the whole payload/stream is applied, via cloudsync_payload_apply's + // checkpoint argument. Record the last applied position for the + // CLOUDSYNC_CHECKPOINT_LAST_APPLIED mode. + if (db_version > data->apply_last_db_version || + (db_version == data->apply_last_db_version && seq > data->apply_last_seq)) { + data->apply_last_db_version = db_version; + data->apply_last_seq = seq; + } + + if (pnrows) *pnrows += 1; + +cleanup: + if (rc != DBRES_OK && in_savepoint) database_rollback_savepoint(data, "cloudsync_payload_apply"); + data->pending_batch = NULL; + merge_pending_free_entries(&batch); + if (batch.cached_vm) databasevm_finalize(batch.cached_vm); + if (batch.cached_col_names) cloudsync_memory_free(batch.cached_col_names); + if (batch.entries) cloudsync_memory_free(batch.entries); + if (vm) databasevm_finalize(vm); + return rc; +} + +static int cloudsync_payload_apply_reassembled_fragment (cloudsync_context *data, const char *value_id, const char *expected_checksum_hex, int *pnrows) { + int rc = DBRES_OK; + dbvm_t *vm = NULL; + char *value = NULL; + char *tbl = NULL, *col_name = NULL; + char *pk = NULL, *site_id = NULL; + size_t tbl_len = 0, col_name_len = 0, pk_len = 0, site_id_len = 0; + int64_t col_version = 0, db_version = 0, cl = 0, seq = 0; + int64_t total_size = 0, copied = 0; + + rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_COUNT, &vm, 0); + if (rc != DBRES_OK) return rc; + rc = databasevm_bind_text(vm, 1, value_id, -1); + if (rc != DBRES_OK) { databasevm_finalize(vm); return rc; } + rc = databasevm_step(vm); + if (rc != DBRES_ROW) { databasevm_finalize(vm); return DBRES_OK; } + int64_t have = database_column_int(vm, 0); + int64_t part_count_min = database_column_int(vm, 1); + int64_t part_count_max = database_column_int(vm, 2); + int64_t total_size_min = database_column_int(vm, 3); + int64_t total_size_max = database_column_int(vm, 4); + const char *checksum_min = database_column_text(vm, 5); + const char *checksum_max = database_column_text(vm, 6); + char checksum_min_copy[32] = {0}; + char checksum_max_copy[32] = {0}; + if (checksum_min) snprintf(checksum_min_copy, sizeof(checksum_min_copy), "%s", checksum_min); + if (checksum_max) snprintf(checksum_max_copy, sizeof(checksum_max_copy), "%s", checksum_max); + int64_t part_index_min = database_column_int(vm, 7); + int64_t part_index_max = database_column_int(vm, 8); + databasevm_finalize(vm); + vm = NULL; + if (have <= 0 || part_count_min <= 0 || have < part_count_max) return DBRES_OK; + if (part_count_min != part_count_max || total_size_min != total_size_max || !checksum_min_copy[0] || !checksum_max_copy[0] || + strcmp(checksum_min_copy, checksum_max_copy) != 0 || strcmp(checksum_min_copy, expected_checksum_hex) != 0 || + part_index_min != 0 || part_index_max != part_count_max - 1 || have != part_count_max) { + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: inconsistent v3 fragments", DBRES_MISUSE); + } + total_size = total_size_max; + + value = cloudsync_memory_alloc((uint64_t)total_size); + if (!value) return DBRES_NOMEM; + + rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_SELECT, &vm, 0); + if (rc != DBRES_OK) goto cleanup; + rc = databasevm_bind_text(vm, 1, value_id, -1); + if (rc != DBRES_OK) goto cleanup; + + uint64_t checksum = 14695981039346656037ULL; + while ((rc = databasevm_step(vm)) == DBRES_ROW) { + size_t frag_len = 0; + const char *frag = database_column_blob(vm, 0, &frag_len); + if (copied + (int64_t)frag_len > total_size) { rc = DBRES_MISUSE; goto cleanup; } + memcpy(value + copied, frag, frag_len); + checksum = cloudsync_checksum_update(checksum, frag, frag_len); + copied += (int64_t)frag_len; + + if (!tbl) { + const char *t = database_column_text(vm, 1); + const char *c = database_column_text(vm, 3); + size_t pkl = 0, sidl = 0; + const char *p = database_column_blob(vm, 2, &pkl); + const char *sid = database_column_blob(vm, 6, &sidl); + tbl_len = (size_t)database_column_bytes(vm, 1); + col_name_len = (size_t)database_column_bytes(vm, 3); + pk_len = pkl; + site_id_len = sidl; + tbl = cloudsync_memory_alloc((uint64_t)tbl_len); + col_name = cloudsync_memory_alloc((uint64_t)col_name_len); + pk = cloudsync_memory_alloc((uint64_t)pk_len); + site_id = cloudsync_memory_alloc((uint64_t)site_id_len); + if (!tbl || !col_name || !pk || !site_id) { rc = DBRES_NOMEM; goto cleanup; } + memcpy(tbl, t, tbl_len); + memcpy(col_name, c, col_name_len); + memcpy(pk, p, pk_len); + memcpy(site_id, sid, site_id_len); + col_version = database_column_int(vm, 4); + db_version = database_column_int(vm, 5); + cl = database_column_int(vm, 7); + seq = database_column_int(vm, 8); + } else { + size_t pkl = 0, sidl = 0; + const char *t = database_column_text(vm, 1); + const char *c = database_column_text(vm, 3); + const char *p = database_column_blob(vm, 2, &pkl); + const char *sid = database_column_blob(vm, 6, &sidl); + if ((size_t)database_column_bytes(vm, 1) != tbl_len || memcmp(tbl, t, tbl_len) != 0 || + pkl != pk_len || memcmp(pk, p, pk_len) != 0 || + (size_t)database_column_bytes(vm, 3) != col_name_len || memcmp(col_name, c, col_name_len) != 0 || + database_column_int(vm, 4) != col_version || + database_column_int(vm, 5) != db_version || + sidl != site_id_len || memcmp(site_id, sid, site_id_len) != 0 || + database_column_int(vm, 7) != cl || + database_column_int(vm, 8) != seq) { + rc = DBRES_MISUSE; + goto cleanup; + } + } + } + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) goto cleanup; + if (copied != total_size) { rc = DBRES_MISUSE; goto cleanup; } + char checksum_hex[17]; + snprintf(checksum_hex, sizeof(checksum_hex), "%016" PRIx64, checksum); + if (strcmp(checksum_hex, expected_checksum_hex) != 0) { rc = DBRES_MISUSE; goto cleanup; } + databasevm_finalize(vm); + vm = NULL; + + rc = cloudsync_payload_apply_single_decoded_row(data, tbl, tbl_len, pk, pk_len, col_name, col_name_len, + value, (size_t)total_size, col_version, db_version, + site_id, site_id_len, cl, seq, pnrows); + if (rc != DBRES_OK) goto cleanup; + + rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_DELETE, &vm, 0); + if (rc == DBRES_OK) { + databasevm_bind_text(vm, 1, value_id, -1); + int step_rc = databasevm_step(vm); + if (step_rc == DBRES_DONE) rc = DBRES_OK; + } + +cleanup: + if (vm) databasevm_finalize(vm); + if (value) cloudsync_memory_free(value); + if (tbl) cloudsync_memory_free(tbl); + if (col_name) cloudsync_memory_free(col_name); + if (pk) cloudsync_memory_free(pk); + if (site_id) cloudsync_memory_free(site_id); + return rc; +} + +static int cloudsync_payload_apply_fragment_row (cloudsync_context *data, cloudsync_payload_fragment_row *row, int *pnrows) { + char value_id[64]; + char checksum_hex[17]; + int part_index = 0, part_count = 0; + int64_t total_size = 0; + const char *base_col = NULL; + int64_t base_col_len = 0; + if (!row || !row->tbl || row->tbl_len <= 0 || !row->pk || row->pk_len <= 0 || + !row->col_name || row->col_name_len <= 0 || !row->col_value || row->col_value_len <= 0 || + !row->site_id || row->site_id_len <= 0) { + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid v3 payload row", DBRES_MISUSE); + } + if (!cloudsync_payload_fragment_parse_colname(row->col_name, row->col_name_len, value_id, sizeof(value_id), + checksum_hex, sizeof(checksum_hex), + &part_index, &part_count, &total_size, &base_col, &base_col_len)) { + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid v3 fragment metadata", DBRES_MISUSE); + } + + uint64_t value_checksum = strtoull(checksum_hex, NULL, 16); + char expected_value_id[33]; + cloudsync_payload_fragment_value_id(expected_value_id, row->tbl, (int)row->tbl_len, row->pk, (int)row->pk_len, + base_col, (int)base_col_len, row->col_version, row->db_version, + row->site_id, (int)row->site_id_len, row->cl, row->seq, + value_checksum, total_size); + if (strcmp(value_id, expected_value_id) != 0) { + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid v3 fragment identity", DBRES_MISUSE); + } + + // the fragments table is guaranteed by dbutils_settings_init; no DDL here + // because the apply path runs under sync-only credentials on server nodes + int rc = cloudsync_payload_fragments_cleanup_stale(data); + if (rc != DBRES_OK) return rc; + + dbvm_t *vm = NULL; + rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_UPSERT, &vm, 0); + if (rc != DBRES_OK) return rc; + databasevm_bind_text(vm, 1, value_id, -1); + databasevm_bind_int(vm, 2, part_index); + databasevm_bind_int(vm, 3, part_count); + databasevm_bind_int(vm, 4, total_size); + databasevm_bind_text(vm, 5, checksum_hex, -1); + databasevm_bind_int(vm, 6, (int64_t)time(NULL)); + databasevm_bind_text(vm, 7, row->tbl, (int)row->tbl_len); + databasevm_bind_blob(vm, 8, row->pk, (uint64_t)row->pk_len); + databasevm_bind_text(vm, 9, base_col, (int)base_col_len); + databasevm_bind_int(vm, 10, row->col_version); + databasevm_bind_int(vm, 11, row->db_version); + databasevm_bind_blob(vm, 12, row->site_id, (uint64_t)row->site_id_len); + databasevm_bind_int(vm, 13, row->cl); + databasevm_bind_int(vm, 14, row->seq); + databasevm_bind_blob(vm, 15, row->col_value, (uint64_t)row->col_value_len); + rc = databasevm_step(vm); + databasevm_finalize(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) return rc; + + return cloudsync_payload_apply_reassembled_fragment(data, value_id, checksum_hex, pnrows); +} + // #ifndef CLOUDSYNC_OMIT_RLS_VALIDATION -int cloudsync_payload_apply (sqlite3_context *context, const char *payload, int blen) { +// Advance the durable receive cursor (check_dbversion/check_seq) after a payload +// (or a fully-applied chunk stream) has been applied. See the checkpoint-mode +// documentation on cloudsync_payload_apply in cloudsync.h. The advance is +// strictly monotonic so re-delivered rows never regress the cursor. +static void cloudsync_payload_apply_checkpoint (cloudsync_context *data, int64_t checkpoint_db_version, int64_t checkpoint_seq) { + int64_t target_db_version; + int64_t target_seq; + + if (checkpoint_db_version == CLOUDSYNC_CHECKPOINT_NONE) return; + if (checkpoint_db_version == CLOUDSYNC_CHECKPOINT_LAST_APPLIED) { + // Nothing applied -> nothing to checkpoint. + if (data->apply_last_db_version < 0) return; + target_db_version = data->apply_last_db_version; + target_seq = data->apply_last_seq; + } else { + target_db_version = checkpoint_db_version; + target_seq = checkpoint_seq; + } + + int64_t cur_db_version = dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_CHECK_DBVERSION); + int64_t cur_seq = dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_CHECK_SEQ); + + // monotonic guard: never move the cursor backwards + if (target_db_version < cur_db_version) return; + if (target_db_version == cur_db_version && target_seq <= cur_seq) return; + + char buf[256]; + snprintf(buf, sizeof(buf), "%" PRId64, target_db_version); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_DBVERSION, buf); + if (target_seq != cur_seq) { + snprintf(buf, sizeof(buf), "%" PRId64, target_seq); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_SEQ, buf); + } +} + +int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int blen, int *pnrows, int64_t checkpoint_db_version, int64_t checkpoint_seq) { + // Guard against calling payload_apply before cloudsync_init: without this, + // the settings lookups at the top of this function would each emit a + // "no such table: cloudsync_settings" debug line, control would fall + // through to the meta-table insert, and the function would ultimately + // return an error with an empty errmsg — SQLite then surfaces that as + // the confusing "Runtime error: not an error". + if (!cloudsync_context_is_initialized(data)) { + return cloudsync_set_error(data, + "cloudsync is not initialized: call SELECT cloudsync_init('') " + "to enable sync on a table before calling cloudsync_payload_apply().", + DBRES_MISUSE); + } + + // sanity check + if (blen < (int)sizeof(cloudsync_payload_header)) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid payload length", DBRES_MISUSE); + + // track the last (db_version, seq) applied by this call so the receive + // checkpoint can be computed once, after the whole payload is applied + data->apply_last_db_version = -1; + data->apply_last_seq = -1; + // decode header cloudsync_payload_header header; memcpy(&header, payload, sizeof(cloudsync_payload_header)); - + header.signature = ntohl(header.signature); header.expanded_size = ntohl(header.expanded_size); header.ncols = ntohs(header.ncols); header.nrows = ntohl(header.nrows); header.schema_hash = ntohll(header.schema_hash); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + // compare schema_hash only if not disabled and if the received payload was created with the current header version + // to avoid schema hash mismatch when processed by a peer with a different extension version during software updates. + if (dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_SKIP_SCHEMA_HASH_CHECK) == 0 && header.version == CLOUDSYNC_PAYLOAD_VERSION_LATEST ) { + if (header.schema_hash != data->schema_hash) { + if (!database_check_schema_hash(data, header.schema_hash)) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Cannot apply the received payload because the schema hash is unknown %" PRIu64 ".", header.schema_hash); + return cloudsync_set_error(data, buffer, DBRES_MISUSE); + } + } + } // sanity check header if ((header.signature != CLOUDSYNC_PAYLOAD_SIGNATURE) || (header.ncols == 0)) { - dbutils_context_result_error(context, "Error on cloudsync_payload_apply: invalid signature or column size."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return -1; + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid signature or column size", DBRES_MISUSE); + } + if (header.version < CLOUDSYNC_PAYLOAD_VERSION_1 || header.version > CLOUDSYNC_PAYLOAD_VERSION_3) { + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: unsupported payload version", DBRES_MISUSE); } const char *buffer = payload + sizeof(cloudsync_payload_header); - blen -= sizeof(cloudsync_payload_header); - + size_t buf_len = (size_t)blen - sizeof(cloudsync_payload_header); + + // sanity check checksum (only if version is >= 2) + if (header.version >= CLOUDSYNC_PAYLOAD_MIN_VERSION_WITH_CHECKSUM) { + uint64_t checksum = pk_checksum(buffer, buf_len); + if (cloudsync_payload_checksum_verify(&header, checksum) == false) { + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid checksum", DBRES_MISUSE); + } + } + // check if payload is compressed char *clone = NULL; if (header.expanded_size != 0) { clone = (char *)cloudsync_memory_alloc(header.expanded_size); - if (!clone) {sqlite3_result_error_code(context, SQLITE_NOMEM); return -1;} - - uint32_t rc = LZ4_decompress_safe(buffer, clone, blen, header.expanded_size); - if (rc <= 0 || rc != header.expanded_size) { - dbutils_context_result_error(context, "Error on cloudsync_payload_apply: unable to decompress BLOB (%d).", rc); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return -1; + if (!clone) return cloudsync_set_error(data, "Unable to allocate memory to uncompress payload", DBRES_NOMEM); + + int lz4_rc = LZ4_decompress_safe(buffer, clone, (int)buf_len, (int)header.expanded_size); + if (lz4_rc <= 0 || (uint32_t)lz4_rc != header.expanded_size) { + if (clone) cloudsync_memory_free(clone); + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to decompress BLOB", DBRES_MISUSE); } - + buffer = (const char *)clone; + buf_len = (size_t)header.expanded_size; } - - sqlite3 *db = sqlite3_context_db_handle(context); - + + if (header.version == CLOUDSYNC_PAYLOAD_VERSION_3) { + int rc = DBRES_OK; + int applied_rows = 0; + if (header.ncols != CLOUDSYNC_CHANGES_NCOLS) { + if (clone) cloudsync_memory_free(clone); + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid v3 column count", DBRES_MISUSE); + } + for (uint32_t i = 0; i < header.nrows; ++i) { + size_t seek = 0; + cloudsync_payload_fragment_row row = {0}; + int res = pk_decode((char *)buffer, buf_len, header.ncols, &seek, -1, + cloudsync_payload_fragment_decode_callback, &row); + if (res == -1 || seek == 0 || seek > buf_len) { + rc = cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid v3 payload row", DBRES_MISUSE); + break; + } + int n = 0; + rc = cloudsync_payload_apply_fragment_row(data, &row, &n); + if (rc != DBRES_OK) break; + applied_rows += n; + buffer += seek; + buf_len -= seek; + } + if (clone) cloudsync_memory_free(clone); + if (pnrows) *pnrows = applied_rows; + // Advance the receive cursor only after the whole payload is applied, + // gated on the caller-supplied checkpoint (a non-final chunk passes + // CLOUDSYNC_CHECKPOINT_NONE and leaves the cursor untouched). + if (rc == DBRES_OK) cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); + return rc; + } + // precompile the insert statement - sqlite3_stmt *vm = NULL; - const char *sql = "INSERT INTO cloudsync_changes(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) VALUES (?,?,?,?,?,?,?,?,?);"; - int rc = sqlite3_prepare(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Error on cloudsync_payload_apply: error while compiling SQL statement (%s).", sqlite3_errmsg(db)); + dbvm_t *vm = NULL; + int rc = databasevm_prepare(data, SQL_CHANGES_INSERT_ROW, &vm, 0); + if (rc != DBRES_OK) { if (clone) cloudsync_memory_free(clone); - return -1; + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: error while compiling SQL statement", rc); } // process buffer, one row at a time uint16_t ncols = header.ncols; uint32_t nrows = header.nrows; int64_t last_payload_db_version = -1; - bool in_savepoint = false; - int dbversion = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_CHECK_DBVERSION); - int seq = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_CHECK_SEQ); cloudsync_pk_decode_bind_context decoded_context = {.vm = vm}; - void *payload_apply_xdata = NULL; - cloudsync_payload_apply_callback_t payload_apply_callback = cloudsync_get_payload_apply_callback(db); - + + // Initialize deferred column-batch merge + merge_pending_batch batch = {0}; + data->pending_batch = &batch; + bool in_savepoint = false; + const void *last_pk = NULL; + int64_t last_pk_len = 0; + const char *last_tbl = NULL; + int64_t last_tbl_len = 0; + for (uint32_t i=0; iskip_decode_idx, cloudsync_payload_decode_callback, &decoded_context); + if (res == -1) { + merge_flush_pending(data); + data->pending_batch = NULL; + if (batch.cached_vm) { databasevm_finalize(batch.cached_vm); batch.cached_vm = NULL; } + if (batch.cached_col_names) { cloudsync_memory_free(batch.cached_col_names); batch.cached_col_names = NULL; } + if (batch.entries) { cloudsync_memory_free(batch.entries); batch.entries = NULL; } + if (in_savepoint) database_rollback_savepoint(data, "cloudsync_payload_apply"); + rc = DBRES_ERROR; + goto cleanup; + } + // Detect PK/table/db_version boundary to flush pending batch + bool pk_changed = (last_pk != NULL && + (last_pk_len != decoded_context.pk_len || + memcmp(last_pk, decoded_context.pk, last_pk_len) != 0)); + bool tbl_changed = (last_tbl != NULL && + (last_tbl_len != decoded_context.tbl_len || + memcmp(last_tbl, decoded_context.tbl, last_tbl_len) != 0)); bool db_version_changed = (last_payload_db_version != decoded_context.db_version); - // Release existing savepoint if db_version changed + // Flush pending batch before any boundary change + if (pk_changed || tbl_changed || db_version_changed) { + int flush_rc = merge_flush_pending(data); + if (flush_rc != DBRES_OK) { + rc = flush_rc; + // continue processing remaining rows + } + } + + // Per-db_version savepoints group rows with the same source db_version + // into one transaction. In SQLite autocommit mode, the RELEASE triggers + // the commit hook which bumps data->db_version and resets seq, ensuring + // unique (db_version, seq) tuples across groups. In PostgreSQL SPI, + // database_in_transaction() is always true so this block is inactive — + // the inner per-PK savepoint in merge_flush_pending handles RLS instead. if (in_savepoint && db_version_changed) { - rc = sqlite3_exec(db, "RELEASE cloudsync_payload_apply;", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Error on cloudsync_payload_apply: unable to release a savepoint (%s).", sqlite3_errmsg(db)); - if (clone) cloudsync_memory_free(clone); - return -1; + rc = database_commit_savepoint(data, "cloudsync_payload_apply"); + if (rc != DBRES_OK) { + merge_pending_free_entries(&batch); + data->pending_batch = NULL; + cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to release a savepoint", rc); + goto cleanup; } in_savepoint = false; } - // Start new savepoint if needed - bool in_transaction = sqlite3_get_autocommit(db) != true; - if (!in_transaction && db_version_changed) { - rc = sqlite3_exec(db, "SAVEPOINT cloudsync_payload_apply;", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Error on cloudsync_payload_apply: unable to start a transaction (%s).", sqlite3_errmsg(db)); - if (clone) cloudsync_memory_free(clone); - return -1; + if (!in_savepoint && db_version_changed && !database_in_transaction(data)) { + rc = database_begin_savepoint(data, "cloudsync_payload_apply"); + if (rc != DBRES_OK) { + merge_pending_free_entries(&batch); + data->pending_batch = NULL; + cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to start a transaction", rc); + goto cleanup; } - last_payload_db_version = decoded_context.db_version; in_savepoint = true; } - - if (approved) { - rc = sqlite3_step(vm); - if (rc != SQLITE_DONE) { - // don't "break;", the error can be due to a RLS policy. - // in case of error we try to apply the following changes - printf("cloudsync_payload_apply error on db_version %lld/%lld: (%d) %s\n", decoded_context.db_version, decoded_context.seq, rc, sqlite3_errmsg(db)); - } + + // Track db_version for batch-flush boundary detection + if (db_version_changed) { + last_payload_db_version = decoded_context.db_version; } - - if (payload_apply_callback) payload_apply_callback(&payload_apply_xdata, &decoded_context, db, data, CLOUDSYNC_PAYLOAD_APPLY_DID_APPLY, rc); - + + // Update PK/table tracking + last_pk = decoded_context.pk; + last_pk_len = decoded_context.pk_len; + last_tbl = decoded_context.tbl; + last_tbl_len = decoded_context.tbl_len; + + rc = databasevm_step(vm); + if (rc != DBRES_DONE) { + // don't "break;", the error can be due to a RLS policy. + // in case of error we try to apply the following changes + } + buffer += seek; - blen -= seek; - stmt_reset(vm); + buf_len -= seek; + dbvm_reset(vm); } - + + // Final flush after loop + { + int flush_rc = merge_flush_pending(data); + if (flush_rc != DBRES_OK && rc == DBRES_OK) rc = flush_rc; + } + data->pending_batch = NULL; + if (in_savepoint) { - sql = "RELEASE cloudsync_payload_apply;"; - int rc1 = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc1 != SQLITE_OK) rc = rc1; + int rc1 = database_commit_savepoint(data, "cloudsync_payload_apply"); + if (rc1 != DBRES_OK) rc = rc1; } - char *lasterr = (rc != SQLITE_OK && rc != SQLITE_DONE) ? cloudsync_string_dup(sqlite3_errmsg(db), false) : NULL; - - if (payload_apply_callback) { - payload_apply_callback(&payload_apply_xdata, &decoded_context, db, data, CLOUDSYNC_PAYLOAD_APPLY_CLEANUP, rc); + // save last error (unused if function returns OK) + if (rc != DBRES_OK && rc != DBRES_DONE) { + cloudsync_set_dberror(data); } - if (rc == SQLITE_DONE) rc = SQLITE_OK; - if (rc == SQLITE_OK) { - char buf[256]; - if (decoded_context.db_version >= dbversion) { - snprintf(buf, sizeof(buf), "%lld", decoded_context.db_version); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_CHECK_DBVERSION, buf); - - if (decoded_context.seq != seq) { - snprintf(buf, sizeof(buf), "%lld", decoded_context.seq); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_CHECK_SEQ, buf); - } + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc == DBRES_OK) { + // Record the last applied (db_version, seq) and advance the receive cursor + // once, gated on the caller-supplied checkpoint. A non-final chunk passes + // CLOUDSYNC_CHECKPOINT_NONE so the cursor never lands mid-db_version. + if (decoded_context.db_version > data->apply_last_db_version || + (decoded_context.db_version == data->apply_last_db_version && decoded_context.seq > data->apply_last_seq)) { + data->apply_last_db_version = decoded_context.db_version; + data->apply_last_seq = decoded_context.seq; } + cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); } +cleanup: + // cleanup merge_pending_batch + if (batch.cached_vm) { databasevm_finalize(batch.cached_vm); batch.cached_vm = NULL; } + if (batch.cached_col_names) { cloudsync_memory_free(batch.cached_col_names); batch.cached_col_names = NULL; } + if (batch.entries) { cloudsync_memory_free(batch.entries); batch.entries = NULL; } + // cleanup vm - if (vm) sqlite3_finalize(vm); - + if (vm) databasevm_finalize(vm); + // cleanup memory if (clone) cloudsync_memory_free(clone); - - if (rc != SQLITE_OK) { - sqlite3_result_error(context, lasterr, -1); - sqlite3_result_error_code(context, SQLITE_MISUSE); - cloudsync_memory_free(lasterr); - return -1; - } - - // return the number of processed rows - sqlite3_result_int(context, nrows); - return nrows; -} -void cloudsync_payload_decode (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_payload_decode"); - //debug_values(argc, argv); - - // sanity check payload type - if (sqlite3_value_type(argv[0]) != SQLITE_BLOB) { - dbutils_context_result_error(context, "Error on cloudsync_payload_decode: value must be a BLOB."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return; - } - - // sanity check payload size - int blen = sqlite3_value_bytes(argv[0]); - if (blen < (int)sizeof(cloudsync_payload_header)) { - dbutils_context_result_error(context, "Error on cloudsync_payload_decode: invalid input size."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return; - } - - // obtain payload - const char *payload = (const char *)sqlite3_value_blob(argv[0]); - - // apply changes - cloudsync_payload_apply(context, payload, blen); + // error already saved in (save last error) + if (rc != DBRES_OK) return rc; + + // return the number of processed rows + if (pnrows) *pnrows = nrows; + return DBRES_OK; } // MARK: - Payload load/store - -int cloudsync_payload_get (sqlite3_context *context, char **blob, int *blob_size, int *db_version, int *seq, sqlite3_int64 *new_db_version, sqlite3_int64 *new_seq) { - sqlite3 *db = sqlite3_context_db_handle(context); - - *db_version = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_SEND_DBVERSION); - if (*db_version < 0) {sqlite3_result_error(context, "Unable to retrieve db_version.", -1); return SQLITE_ERROR;} - - *seq = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_SEND_SEQ); - if (*seq < 0) {sqlite3_result_error(context, "Unable to retrieve seq.", -1); return SQLITE_ERROR;} +int cloudsync_payload_get (cloudsync_context *data, char **blob, int *blob_size, int *db_version, int64_t *new_db_version) { + // retrieve current db_version and seq + *db_version = dbutils_settings_get_int_value(data, CLOUDSYNC_KEY_SEND_DBVERSION); + if (*db_version < 0) return DBRES_ERROR; // retrieve BLOB char sql[1024]; - snprintf(sql, sizeof(sql), "WITH max_db_version AS (SELECT MAX(db_version) AS max_db_version FROM cloudsync_changes) " - "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), max_db_version AS max_db_version, MAX(IIF(db_version = max_db_version, seq, NULL)) FROM cloudsync_changes, max_db_version WHERE site_id=cloudsync_siteid() AND (db_version>%d OR (db_version=%d AND seq>%d))", *db_version, *db_version, *seq); + snprintf(sql, sizeof(sql), "WITH max_db_version AS (SELECT MAX(db_version) AS max_db_version FROM cloudsync_changes WHERE site_id=cloudsync_siteid()) " + "SELECT * FROM (SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload, max_db_version AS max_db_version FROM cloudsync_changes, max_db_version WHERE site_id=cloudsync_siteid() AND db_version>%d) WHERE payload IS NOT NULL", *db_version); - int rc = dbutils_blob_int_int_select(db, sql, blob, blob_size, new_db_version, new_seq); - if (rc != SQLITE_OK) { - sqlite3_result_error(context, "cloudsync_network_send_changes unable to get changes", -1); - sqlite3_result_error_code(context, rc); - return rc; - } + int64_t len = 0; + int rc = database_select_blob_int(data, sql, blob, &len, new_db_version); + *blob_size = (int)len; + if (rc != DBRES_OK) return rc; // exit if there is no data to send - if (blob == NULL || blob_size == 0) return SQLITE_OK; + if (*blob == NULL || *blob_size == 0) return DBRES_OK; return rc; } #ifdef CLOUDSYNC_DESKTOP_OS - -void cloudsync_payload_save (sqlite3_context *context, int argc, sqlite3_value **argv) { +int cloudsync_payload_save (cloudsync_context *data, const char *payload_path, int *size) { DEBUG_FUNCTION("cloudsync_payload_save"); - // sanity check argument - if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "Unable to retrieve file path.", -1); - return; - } - - // retrieve full path to file - const char *path = (const char *)sqlite3_value_text(argv[0]); - cloudsync_file_delete(path); + // silently delete any other payload with the same name + cloudsync_file_delete(payload_path); // retrieve payload char *blob = NULL; - int blob_size = 0, db_version = 0, seq = 0; - sqlite3_int64 new_db_version = 0, new_seq = 0; - int rc = cloudsync_payload_get(context, &blob, &blob_size, &db_version, &seq, &new_db_version, &new_seq); - if (rc != SQLITE_OK) return; + int blob_size = 0, db_version = 0; + int64_t new_db_version = 0; + int rc = cloudsync_payload_get(data, &blob, &blob_size, &db_version, &new_db_version); + if (rc != DBRES_OK) { + if (db_version < 0) return cloudsync_set_error(data, "Unable to retrieve db_version", rc); + return cloudsync_set_error(data, "Unable to retrieve changes in cloudsync_payload_save", rc); + } - // exit if there is no data to send - if (blob == NULL || blob_size == 0) return; + // exit if there is no data to save + if (blob == NULL || blob_size == 0) { + if (size) *size = 0; + return DBRES_OK; + } // write payload to file - bool res = cloudsync_file_write(path, blob, (size_t)blob_size); - sqlite3_free(blob); - + bool res = cloudsync_file_write(payload_path, blob, (size_t)blob_size); + cloudsync_memory_free(blob); if (res == false) { - sqlite3_result_error(context, "Unable to write payload to file path.", -1); - return; - } - - // update db_version and seq - char buf[256]; - sqlite3 *db = sqlite3_context_db_handle(context); - if (new_db_version != db_version) { - snprintf(buf, sizeof(buf), "%lld", new_db_version); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_DBVERSION, buf); - } - if (new_seq != seq) { - snprintf(buf, sizeof(buf), "%lld", new_seq); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_SEQ, buf); + return cloudsync_set_error(data, "Unable to write payload to file path", DBRES_IOERR); } // returns blob size - sqlite3_result_int64(context, (sqlite3_int64)blob_size); + if (size) *size = blob_size; + return DBRES_OK; } +#endif -void cloudsync_payload_load (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_payload_load"); +// MARK: - Core - + +int cloudsync_table_sanity_check (cloudsync_context *data, const char *name, CLOUDSYNC_INIT_FLAG init_flags) { + DEBUG_DBFUNCTION("cloudsync_table_sanity_check %s", name); + char buffer[2048]; - // sanity check argument - if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "Unable to retrieve file path.", -1); - return; + // sanity check table name + if (name == NULL) { + return cloudsync_set_error(data, "cloudsync_init requires a non-null table parameter", DBRES_ERROR); } - // retrieve full path to file - const char *path = (const char *)sqlite3_value_text(argv[0]); - - sqlite3_int64 payload_size = 0; - char *payload = cloudsync_file_read(path, &payload_size); - if (!payload) { - if (payload_size == -1) sqlite3_result_error(context, "Unable to read payload from file path.", -1); - if (payload) cloudsync_memory_free(payload); - return; + // avoid allocating heap memory for SQL statements by setting a maximum length of 512 characters + // for table names. This limit is reasonable and helps prevent memory management issues. + const size_t maxlen = CLOUDSYNC_MAX_TABLENAME_LEN; + if (strlen(name) > maxlen) { + snprintf(buffer, sizeof(buffer), "Table name cannot be longer than %d characters", (int)maxlen); + return cloudsync_set_error(data, buffer, DBRES_ERROR); } - int nrows = (payload_size) ? cloudsync_payload_apply (context, payload, (int)payload_size) : 0; - if (payload) cloudsync_memory_free(payload); - - // returns number of applied rows - if (nrows != -1) sqlite3_result_int(context, nrows); -} - -#endif - -// MARK: - Public - - -void cloudsync_version (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_version"); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - sqlite3_result_text(context, CLOUDSYNC_VERSION, -1, SQLITE_STATIC); -} - -void cloudsync_siteid (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_siteid"); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - sqlite3_result_blob(context, data->site_id, UUID_LEN, SQLITE_STATIC); -} - -void cloudsync_db_version (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_db_version"); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - - // retrieve context - sqlite3 *db = sqlite3_context_db_handle(context); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + // check if already initialized + cloudsync_table_context *table = table_lookup(data, name); + if (table) return DBRES_OK; - int rc = db_version_check_uptodate(db, data); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to retrieve db_version (%s).", sqlite3_errmsg(db)); - return; + // check if table exists + if (database_table_exists(data, name, cloudsync_schema(data)) == false) { + snprintf(buffer, sizeof(buffer), "Table %s does not exist", name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); } - sqlite3_result_int64(context, data->db_version); -} - -void cloudsync_db_version_next (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_db_version_next"); - - // retrieve context - sqlite3 *db = sqlite3_context_db_handle(context); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + // no more than 128 columns can be used as a composite primary key (SQLite hard limit) + int npri_keys = database_count_pk(data, name, false, cloudsync_schema(data)); + if (npri_keys < 0) return cloudsync_set_dberror(data); + if (npri_keys > 128) return cloudsync_set_error(data, "No more than 128 columns can be used to form a composite primary key", DBRES_ERROR); - sqlite3_int64 merging_version = (argc == 1) ? sqlite3_value_int64(argv[0]) : CLOUDSYNC_VALUE_NOTSET; - sqlite3_int64 value = db_version_next(db, data, merging_version); - if (value == -1) { - dbutils_context_result_error(context, "Unable to retrieve next_db_version (%s).", sqlite3_errmsg(db)); - return; + #if CLOUDSYNC_DISABLE_ROWIDONLY_TABLES + // if count == 0 means that rowid will be used as primary key (BTW: very bad choice for the user) + if (npri_keys == 0) { + snprintf(buffer, sizeof(buffer), "Rowid only tables are not supported, all primary keys must be explicitly set and declared as NOT NULL (table %s)", name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); } + #endif - sqlite3_result_int64(context, value); -} - -void cloudsync_seq (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_seq"); - - // retrieve context - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - sqlite3_result_int(context, BUMP_SEQ(data)); -} - -void cloudsync_uuid (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_uuid"); - - char value[UUID_STR_MAXLEN]; - char *uuid = cloudsync_uuid_v7_string(value, true); - sqlite3_result_text(context, uuid, -1, SQLITE_TRANSIENT); -} - -// MARK: - - -void cloudsync_set (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_set"); - - // sanity check parameters - const char *key = (const char *)sqlite3_value_text(argv[0]); - const char *value = (const char *)sqlite3_value_text(argv[1]); - - // silently fails - if (key == NULL) return; - - sqlite3 *db = sqlite3_context_db_handle(context); - dbutils_settings_set_key_value(db, context, key, value); -} - -void cloudsync_set_column (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_set_column"); - - const char *tbl = (const char *)sqlite3_value_text(argv[0]); - const char *col = (const char *)sqlite3_value_text(argv[1]); - const char *key = (const char *)sqlite3_value_text(argv[2]); - const char *value = (const char *)sqlite3_value_text(argv[3]); - dbutils_table_settings_set_key_value(NULL, context, tbl, col, key, value); -} - -void cloudsync_set_table (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_set_table"); - - const char *tbl = (const char *)sqlite3_value_text(argv[0]); - const char *key = (const char *)sqlite3_value_text(argv[1]); - const char *value = (const char *)sqlite3_value_text(argv[2]); - dbutils_table_settings_set_key_value(NULL, context, tbl, "*", key, value); -} - -void cloudsync_is_sync (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_is_sync"); - - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - if (data->insync) { - sqlite3_result_int(context, 1); - return; - } - - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_table_context *table = table_lookup(data, table_name); - sqlite3_result_int(context, (table) ? (table->enabled == 0) : 0); -} - -void cloudsync_col_value (sqlite3_context *context, int argc, sqlite3_value **argv) { - // DEBUG_FUNCTION("cloudsync_col_value"); - - // argv[0] -> table name - // argv[1] -> column name - // argv[2] -> encoded pk - - // lookup table - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to retrieve table name %s in clousdsync_colvalue.", table_name); - return; - } - - // retrieve column name - const char *col_name = (const char *)sqlite3_value_text(argv[1]); - - // check for special tombstone value - if (strcmp(col_name, CLOUDSYNC_TOMBSTONE_VALUE) == 0) { - sqlite3_result_null(context); - return; - } - - // extract the right col_value vm associated to the column name - sqlite3_stmt *vm = table_column_lookup(table, col_name, false, NULL); - if (!vm) { - sqlite3_result_error(context, "Unable to retrieve column value precompiled statement in clousdsync_colvalue.", -1); - return; - } - - // bind primary key values - int rc = pk_decode_prikey((char *)sqlite3_value_blob(argv[2]), (size_t)sqlite3_value_bytes(argv[2]), pk_decode_bind_callback, (void *)vm); - if (rc < 0) goto cleanup; - - // execute vm - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) { - rc = SQLITE_OK; - sqlite3_result_text(context, CLOUDSYNC_RLS_RESTRICTED_VALUE, -1, SQLITE_STATIC); - } else if (rc == SQLITE_ROW) { - // store value result - rc = SQLITE_OK; - sqlite3_result_value(context, sqlite3_column_value(vm, 0)); - } - -cleanup: - if (rc != SQLITE_OK) { - sqlite3 *db = sqlite3_context_db_handle(context); - sqlite3_result_error(context, sqlite3_errmsg(db), -1); - } - sqlite3_reset(vm); -} - -void cloudsync_pk_encode (sqlite3_context *context, int argc, sqlite3_value **argv) { - size_t bsize = 0; - char *buffer = pk_encode_prikey(argv, argc, NULL, &bsize); - if (!buffer) { - sqlite3_result_null(context); - return; - } - sqlite3_result_blob(context, (const void *)buffer, (int)bsize, SQLITE_TRANSIENT); - cloudsync_memory_free(buffer); -} - -int cloudsync_pk_decode_set_result_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { - cloudsync_pk_decode_context *decode_context = (cloudsync_pk_decode_context *)xdata; - // decode_context->index is 1 based - // index is 0 based - if (decode_context->index != index+1) return SQLITE_OK; - - int rc = 0; - sqlite3_context *context = decode_context->context; - switch (type) { - case SQLITE_INTEGER: - sqlite3_result_int64(context, ival); - break; - - case SQLITE_FLOAT: - sqlite3_result_double(context, dval); - break; - - case SQLITE_NULL: - sqlite3_result_null(context); - break; - - case SQLITE_TEXT: - sqlite3_result_text(context, pval, (int)ival, SQLITE_TRANSIENT); - break; - - case SQLITE_BLOB: - sqlite3_result_blob(context, pval, (int)ival, SQLITE_TRANSIENT); - break; - } - - return rc; -} - - -void cloudsync_pk_decode (sqlite3_context *context, int argc, sqlite3_value **argv) { - const char *pk = (const char *)sqlite3_value_text(argv[0]); - int i = sqlite3_value_int(argv[1]); - - cloudsync_pk_decode_context xdata = {.context = context, .index = i}; - pk_decode_prikey((char *)pk, strlen(pk), cloudsync_pk_decode_set_result_callback, &xdata); -} - -// MARK: - - -void cloudsync_insert (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_insert %s", sqlite3_value_text(argv[0])); - // debug_values(argc-1, &argv[1]); - - // argv[0] is table name - // argv[1]..[N] is primary key(s) - - // table_cloudsync - // pk -> encode(argc-1, &argv[1]) - // col_name -> name - // col_version -> 0/1 +1 - // db_version -> check - // site_id 0 - // seq -> sqlite_master - - // retrieve context - sqlite3 *db = sqlite3_context_db_handle(context); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // lookup table - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to retrieve table name %s in cloudsync_insert.", table_name); - return; - } - - // encode the primary key values into a buffer - char buffer[1024]; - size_t pklen = sizeof(buffer); - char *pk = pk_encode_prikey(&argv[1], table->npks, buffer, &pklen); - if (!pk) { - sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); - return; - } - - // compute the next database version for tracking changes - sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET); - - // check if a row with the same primary key already exists - // if so, this means the row might have been previously deleted (sentinel) - bool pk_exists = (bool)stmt_count(table->meta_pkexists_stmt, pk, pklen, SQLITE_BLOB); - int rc = SQLITE_OK; - - if (table->ncols == 0) { - // if there are no columns other than primary keys, insert a sentinel record - rc = local_mark_insert_sentinel_meta(db, table, pk, pklen, db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - } else if (pk_exists){ - // if a row with the same primary key already exists, update the sentinel record - rc = local_update_sentinel(db, table, pk, pklen, db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - } - - // process each non-primary key column for insert or update - for (int i=0; incols; ++i) { - // mark the column as inserted or updated in the metadata - rc = local_mark_insert_or_update_meta(db, table, pk, pklen, table->col_name[i], db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - } - -cleanup: - if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1); - // free memory if the primary key was dynamically allocated - if (pk != buffer) cloudsync_memory_free(pk); -} - -void cloudsync_delete (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_delete %s", sqlite3_value_text(argv[0])); - // debug_values(argc-1, &argv[1]); - - // retrieve context - sqlite3 *db = sqlite3_context_db_handle(context); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // lookup table - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to retrieve table name %s in cloudsync_delete.", table_name); - return; - } - - // compute the next database version for tracking changes - sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET); - int rc = SQLITE_OK; - - // encode the primary key values into a buffer - char buffer[1024]; - size_t pklen = sizeof(buffer); - char *pk = pk_encode_prikey(&argv[1], table->npks, buffer, &pklen); - if (!pk) { - sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); - return; - } - - // mark the row as deleted by inserting a delete sentinel into the metadata - rc = local_mark_delete_meta(db, table, pk, pklen, db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - - // remove any metadata related to the old rows associated with this primary key - rc = local_drop_meta(db, table, pk, pklen); - if (rc != SQLITE_OK) goto cleanup; - -cleanup: - if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1); - // free memory if the primary key was dynamically allocated - if (pk != buffer) cloudsync_memory_free(pk); -} - -// MARK: - - -void cloudsync_update_payload_free (cloudsync_update_payload *payload) { - for (int i=0; icount; i++) { - sqlite3_value_free(payload->new_values[i]); - sqlite3_value_free(payload->old_values[i]); - } - cloudsync_memory_free(payload->new_values); - cloudsync_memory_free(payload->old_values); - sqlite3_value_free(payload->table_name); - payload->new_values = NULL; - payload->old_values = NULL; - payload->table_name = NULL; - payload->count = 0; - payload->capacity = 0; -} - -int cloudsync_update_payload_append (cloudsync_update_payload *payload, sqlite3_value *v1, sqlite3_value *v2, sqlite3_value *v3) { - if (payload->count >= payload->capacity) { - int newcap = payload->capacity ? payload->capacity * 2 : 128; - - sqlite3_value **new_values_2 = (sqlite3_value **)cloudsync_memory_realloc(payload->new_values, newcap * sizeof(*new_values_2)); - if (!new_values_2) return SQLITE_NOMEM; - payload->new_values = new_values_2; - - sqlite3_value **old_values_2 = (sqlite3_value **)cloudsync_memory_realloc(payload->old_values, newcap * sizeof(*old_values_2)); - if (!old_values_2) return SQLITE_NOMEM; - payload->old_values = old_values_2; - - payload->capacity = newcap; - } - - int index = payload->count; - if (payload->table_name == NULL) payload->table_name = sqlite3_value_dup(v1); - else if (dbutils_value_compare(payload->table_name, v1) != 0) return SQLITE_NOMEM; - payload->new_values[index] = sqlite3_value_dup(v2); - payload->old_values[index] = sqlite3_value_dup(v3); - payload->count++; - - // sanity check memory allocations - bool v1_can_be_null = (sqlite3_value_type(v1) == SQLITE_NULL); - bool v2_can_be_null = (sqlite3_value_type(v2) == SQLITE_NULL); - bool v3_can_be_null = (sqlite3_value_type(v3) == SQLITE_NULL); - - if ((payload->table_name == NULL) && (!v1_can_be_null)) return SQLITE_NOMEM; - if ((payload->old_values[index] == NULL) && (!v2_can_be_null)) return SQLITE_NOMEM; - if ((payload->new_values[index] == NULL) && (!v3_can_be_null)) return SQLITE_NOMEM; - - return SQLITE_OK; -} - -void cloudsync_update_step (sqlite3_context *context, int argc, sqlite3_value **argv) { - // argv[0] => table_name - // argv[1] => new_column_value - // argv[2] => old_column_value - - // allocate/get the update payload - cloudsync_update_payload *payload = (cloudsync_update_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_update_payload)); - if (!payload) {sqlite3_result_error_nomem(context); return;} - - if (cloudsync_update_payload_append(payload, argv[0], argv[1], argv[2]) != SQLITE_OK) { - sqlite3_result_error_nomem(context); - } -} - -void cloudsync_update_final (sqlite3_context *context) { - cloudsync_update_payload *payload = (cloudsync_update_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_update_payload)); - if (!payload || payload->count == 0) return; - - // retrieve context - sqlite3 *db = sqlite3_context_db_handle(context); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // lookup table - const char *table_name = (const char *)sqlite3_value_text(payload->table_name); - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to retrieve table name %s in cloudsync_update.", table_name); - return; - } - - // compute the next database version for tracking changes - sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET); - int rc = SQLITE_OK; - - // Check if the primary key(s) have changed - bool prikey_changed = false; - for (int i=0; inpks; ++i) { - if (dbutils_value_compare(payload->old_values[i], payload->new_values[i]) != 0) { - prikey_changed = true; - break; + bool skip_int_pk_check = (init_flags & CLOUDSYNC_INIT_FLAG_SKIP_INT_PK_CHECK) != 0; + if (!skip_int_pk_check) { + if (npri_keys == 1) { + // the affinity of a column is determined by the declared type of the column, + // according to the following rules in the order shown: + // 1. If the declared type contains the string "INT" then it is assigned INTEGER affinity. + int npri_keys_int = database_count_int_pk(data, name, cloudsync_schema(data)); + if (npri_keys_int < 0) return cloudsync_set_dberror(data); + if (npri_keys == npri_keys_int) { + snprintf(buffer, sizeof(buffer), "Table %s uses a single-column INTEGER primary key. For CRDT replication, primary keys must be globally unique. Consider using a TEXT primary key with UUIDs or ULID to avoid conflicts across nodes. If you understand the risk and still want to use this INTEGER primary key, set the third argument of the cloudsync_init function to 1 to skip this check.", name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); + } + } } - - // encode the NEW primary key values into a buffer (used later for indexing) - char buffer[1024]; - char buffer2[1024]; - size_t pklen = sizeof(buffer); - size_t oldpklen = sizeof(buffer2); - char *oldpk = NULL; - - char *pk = pk_encode_prikey(payload->new_values, table->npks, buffer, &pklen); - if (!pk) { - sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); - return; - } - - if (prikey_changed) { - // if the primary key has changed, we need to handle the row differently: - // 1. mark the old row (OLD primary key) as deleted - // 2. create a new row (NEW primary key) - // encode the OLD primary key into a buffer - oldpk = pk_encode_prikey(payload->old_values, table->npks, buffer2, &oldpklen); - if (!oldpk) { - if (pk != buffer) cloudsync_memory_free(pk); - sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); - return; + // if user declared explicit primary key(s) then make sure they are all declared as NOT NULL + #if CLOUDSYNC_CHECK_NOTNULL_PRIKEYS + bool skip_notnull_prikeys_check = (init_flags & CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_PRIKEYS_CHECK) != 0; + if (!skip_notnull_prikeys_check) { + if (npri_keys > 0) { + int npri_keys_notnull = database_count_pk(data, name, true, cloudsync_schema(data)); + if (npri_keys_notnull < 0) return cloudsync_set_dberror(data); + if (npri_keys != npri_keys_notnull) { + snprintf(buffer, sizeof(buffer), "All primary keys must be explicitly declared as NOT NULL (table %s)", name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); + } } - - // mark the rows with the old primary key as deleted in the metadata (old row handling) - rc = local_mark_delete_meta(db, table, oldpk, oldpklen, db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - - // move non-sentinel metadata entries from OLD primary key to NEW primary key - // handles the case where some metadata is retained across primary key change - // see https://github.com/sqliteai/sqlite-sync/blob/main/docs/PriKey.md for more details - rc = local_update_move_meta(db, table, pk, pklen, oldpk, oldpklen, db_version); - if (rc != SQLITE_OK) goto cleanup; - - // mark a new sentinel row with the new primary key in the metadata - rc = local_mark_insert_sentinel_meta(db, table, pk, pklen, db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - - // free memory if the OLD primary key was dynamically allocated - if (oldpk != buffer2) cloudsync_memory_free(oldpk); - oldpk = NULL; } + #endif - // compare NEW and OLD values (excluding primary keys) to handle column updates - for (int i=0; incols; i++) { - int col_index = table->npks + i; // Regular columns start after primary keys - - if (dbutils_value_compare(payload->old_values[col_index], payload->new_values[col_index]) != 0) { - // if a column value has changed, mark it as updated in the metadata - // columns are in cid order - rc = local_mark_insert_or_update_meta(db, table, pk, pklen, table->col_name[i], db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; + // check for columns declared as NOT NULL without a DEFAULT value. + // Otherwise, col_merge_stmt would fail if changes to other columns are inserted first. + bool skip_notnull_default_check = (init_flags & CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_DEFAULT_CHECK) != 0; + if (!skip_notnull_default_check) { + int n_notnull_nodefault = database_count_notnull_without_default(data, name, cloudsync_schema(data)); + if (n_notnull_nodefault < 0) return cloudsync_set_dberror(data); + if (n_notnull_nodefault > 0) { + snprintf(buffer, sizeof(buffer), "All non-primary key columns declared as NOT NULL must have a DEFAULT value. (table %s)", name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); } } -cleanup: - if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1); - if (pk != buffer) cloudsync_memory_free(pk); - if (oldpk && (oldpk != buffer2)) cloudsync_memory_free(oldpk); - - cloudsync_update_payload_free(payload); + return DBRES_OK; } -// MARK: - +int cloudsync_cleanup_internal (cloudsync_context *data, cloudsync_table_context *table) { + if (cloudsync_context_init(data) == NULL) return DBRES_MISUSE; -int cloudsync_cleanup_internal (sqlite3_context *context, const char *table_name) { - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // get database reference - sqlite3 *db = sqlite3_context_db_handle(context); - - // init cloudsync_settings - if (cloudsync_context_init(db, data, context) == NULL) return SQLITE_MISUSE; - - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) return SQLITE_OK; - - table_remove_from_context(data, table); - table_free(table); - // drop meta-table - char *sql = cloudsync_memory_mprintf("DROP TABLE IF EXISTS \"%w_cloudsync\";", table_name); - int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + const char *table_name = table->name; + char *sql = cloudsync_memory_mprintf(SQL_DROP_CLOUDSYNC_TABLE, table->meta_ref); + int rc = database_exec(data, sql); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to drop cloudsync table %s_cloudsync in cloudsync_cleanup.", table_name); - sqlite3_result_error_code(context, rc); - return rc; + if (rc != DBRES_OK) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to drop cloudsync table %s_cloudsync in cloudsync_cleanup", table_name); + return cloudsync_set_error(data, buffer, rc); } - - // drop original triggers - dbutils_delete_triggers(db, table_name); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to drop cloudsync table %s_cloudsync in cloudsync_cleanup.", table_name); - sqlite3_result_error_code(context, rc); - return rc; - } - - // remove all table related settings - dbutils_table_settings_set_key_value(db, context, table_name, NULL, NULL, NULL); - - return SQLITE_OK; -} -void cloudsync_cleanup_all (sqlite3_context *context) { - char *sql = "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'cloudsync_%' AND name NOT LIKE '%_cloudsync';"; - - sqlite3 *db = sqlite3_context_db_handle(context); - char **result = NULL; - int nrows, ncols; - char *errmsg; - int rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, &errmsg); - if (errmsg || ncols != 1) { - printf("cloudsync_cleanup_all error: %s\n", errmsg ? errmsg : "invalid table"); - goto cleanup; - } - - rc = SQLITE_OK; - for (int i = ncols; i < nrows+ncols; i+=ncols) { - int rc2 = cloudsync_cleanup_internal(context, result[i]); - if (rc2 != SQLITE_OK) rc = rc2; - } - - if (rc == SQLITE_OK) { - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - data->site_id[0] = 0; - dbutils_settings_cleanup(db); + // drop blocks table if this table has block LWW columns + if (table->blocks_ref) { + sql = cloudsync_memory_mprintf(SQL_DROP_CLOUDSYNC_TABLE, table->blocks_ref); + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to drop blocks table %s_cloudsync_blocks in cloudsync_cleanup", table_name); + return cloudsync_set_error(data, buffer, rc); + } } - -cleanup: - sqlite3_free_table(result); - sqlite3_free(errmsg); -} -void cloudsync_cleanup (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_cleanup"); - - const char *table = (const char *)sqlite3_value_text(argv[0]); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - sqlite3 *db = sqlite3_context_db_handle(context); - - if (dbutils_is_star_table(table)) cloudsync_cleanup_all(context); - else cloudsync_cleanup_internal(context, table); + // drop original triggers + rc = database_delete_triggers(data, table_name); + if (rc != DBRES_OK) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to delete triggers for table %s", table_name); + return cloudsync_set_error(data, buffer, rc); + } - if (dbutils_table_exists(db, CLOUDSYNC_TABLE_SETTINGS_NAME) == true) dbutils_update_schema_hash(db, &data->schema_hash); + // remove all table related settings + dbutils_table_settings_set_key_value(data, table_name, NULL, NULL, NULL); + return DBRES_OK; } -void cloudsync_enable_disable (sqlite3_context *context, const char *table_name, bool value) { - DEBUG_FUNCTION("cloudsync_enable_disable"); - - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); +int cloudsync_cleanup (cloudsync_context *data, const char *table_name) { cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) return; - - table->enabled = value; -} - -int cloudsync_enable_disable_all_callback (void *xdata, int ncols, char **values, char **names) { - sqlite3_context *context = (sqlite3_context *)xdata; - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - bool value = data->temp_bool; + if (!table) return DBRES_OK; - for (int i=0; ienabled = value; - } - - return SQLITE_OK; -} - -void cloudsync_enable_disable_all (sqlite3_context *context, bool value) { - DEBUG_FUNCTION("cloudsync_enable_disable_all"); + // TODO: check what happen if cloudsync_cleanup_internal failes (not eveything dropped) and the table is still in memory? - char *sql = "SELECT name FROM sqlite_master WHERE type='table';"; + int rc = cloudsync_cleanup_internal(data, table); + if (rc != DBRES_OK) return rc; - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - data->temp_bool = value; - sqlite3 *db = sqlite3_context_db_handle(context); - sqlite3_exec(db, sql, cloudsync_enable_disable_all_callback, context, NULL); -} - -void cloudsync_enable (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_enable"); + int counter = table_remove(data, table); + table_free(table); - const char *table = (const char *)sqlite3_value_text(argv[0]); - if (dbutils_is_star_table(table)) cloudsync_enable_disable_all(context, true); - else cloudsync_enable_disable(context, table, true); -} - -void cloudsync_disable (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_disable"); + if (counter == 0) { + // cleanup database on last table + cloudsync_reset_siteid(data); + dbutils_settings_cleanup(data); + } else { + if (database_internal_table_exists(data, CLOUDSYNC_TABLE_SETTINGS_NAME) == true) { + cloudsync_update_schema_hash(data); + } + } - const char *table = (const char *)sqlite3_value_text(argv[0]); - if (dbutils_is_star_table(table)) cloudsync_enable_disable_all(context, false); - else cloudsync_enable_disable(context, table, false); + return DBRES_OK; } -void cloudsync_is_enabled (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_is_enabled"); - - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_table_context *table = table_lookup(data, table_name); - - int result = (table && table->enabled) ? 1 : 0; - sqlite3_result_int(context, result); +int cloudsync_cleanup_all (cloudsync_context *data) { + return database_cleanup(data); } -void cloudsync_terminate (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_terminate"); - - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - for (int i=0; itables_count; ++i) { - if (data->tables[i]) table_free(data->tables[i]); - data->tables[i] = NULL; +int cloudsync_terminate (cloudsync_context *data) { + // can't use for/loop here because data->tables_count is changed by table_remove + while (data->tables_count > 0) { + cloudsync_table_context *t = data->tables[data->tables_count - 1]; + table_remove(data, t); + table_free(t); } - if (data->schema_version_stmt) sqlite3_finalize(data->schema_version_stmt); - if (data->data_version_stmt) sqlite3_finalize(data->data_version_stmt); - if (data->db_version_stmt) sqlite3_finalize(data->db_version_stmt); - if (data->getset_siteid_stmt) sqlite3_finalize(data->getset_siteid_stmt); + if (data->schema_version_stmt) databasevm_finalize(data->schema_version_stmt); + if (data->data_version_stmt) databasevm_finalize(data->data_version_stmt); + if (data->db_version_stmt) databasevm_finalize(data->db_version_stmt); + if (data->getset_siteid_stmt) databasevm_finalize(data->getset_siteid_stmt); + if (data->current_schema) cloudsync_memory_free(data->current_schema); data->schema_version_stmt = NULL; data->data_version_stmt = NULL; data->db_version_stmt = NULL; data->getset_siteid_stmt = NULL; + data->current_schema = NULL; // reset the site_id so the cloudsync_context_init will be executed again // if any other cloudsync function is called after terminate data->site_id[0] = 0; - sqlite3_result_int(context, 1); + return 1; } -// MARK: - - -int cloudsync_load_siteid (sqlite3 *db, cloudsync_context *data) { - // check if site_id was already loaded - if (data->site_id[0] != 0) return SQLITE_OK; - - // load site_id - int size, rc; - char *buffer = dbutils_blob_select(db, "SELECT site_id FROM cloudsync_site_id WHERE rowid=0;", &size, data->sqlite_ctx, &rc); - if (!buffer) return rc; - if (size != UUID_LEN) return SQLITE_MISUSE; - - memcpy(data->site_id, buffer, UUID_LEN); - cloudsync_memory_free(buffer); - - return SQLITE_OK; -} - -int cloudsync_init_internal (sqlite3_context *context, const char *table_name, const char *algo_name, bool skip_int_pk_check) { - DEBUG_FUNCTION("cloudsync_init_internal"); - - // get database reference - sqlite3 *db = sqlite3_context_db_handle(context); - - // retrieve global context - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - +int cloudsync_init_table (cloudsync_context *data, const char *table_name, const char *algo_name, CLOUDSYNC_INIT_FLAG init_flags) { // sanity check table and its primary key(s) - if (dbutils_table_sanity_check(db, context, table_name, skip_int_pk_check) == false) { - return SQLITE_MISUSE; - } + int rc = cloudsync_table_sanity_check(data, table_name, init_flags); + if (rc != DBRES_OK) return rc; // init cloudsync_settings - if (cloudsync_context_init(db, data, context) == NULL) return SQLITE_MISUSE; + if (cloudsync_context_init(data) == NULL) { + return cloudsync_set_error(data, "Unable to initialize cloudsync context", DBRES_MISUSE); + } // sanity check algo name (if exists) table_algo algo_new = table_algo_none; - if (!algo_name) { - algo_name = CLOUDSYNC_DEFAULT_ALGO; - } + if (!algo_name) algo_name = CLOUDSYNC_DEFAULT_ALGO; - algo_new = crdt_algo_from_name(algo_name); + algo_new = cloudsync_algo_from_name(algo_name); if (algo_new == table_algo_none) { - dbutils_context_result_error(context, "algo name %s does not exist", crdt_algo_name); - return SQLITE_MISUSE; + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unknown CRDT algorithm name %s", algo_name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); + } + + // DWS and AWS algorithms are not yet implemented in the merge logic + if (algo_new == table_algo_crdt_dws || algo_new == table_algo_crdt_aws) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "CRDT algorithm %s is not yet supported", algo_name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); } // check if table name was already augmented - table_algo algo_current = dbutils_table_settings_get_algo(db, table_name); + table_algo algo_current = dbutils_table_settings_get_algo(data, table_name); // sanity check algorithm if ((algo_new == algo_current) && (algo_current != table_algo_none)) { @@ -3134,11 +4582,10 @@ int cloudsync_init_internal (sqlite3_context *context, const char *table_name, c algo_new = algo_current; } else if ((algo_new != table_algo_none) && (algo_current == table_algo_none)) { // write table algo name in settings - dbutils_table_settings_set_key_value(NULL, context, table_name, "*", "algo", algo_name); + dbutils_table_settings_set_key_value(data, table_name, "*", "algo", algo_name); } else { // error condition - dbutils_context_result_error(context, "%s", "Before changing a table algorithm you must call cloudsync_cleanup(table_name)"); - return SQLITE_MISUSE; + return cloudsync_set_error(data, "The function cloudsync_cleanup(table) must be called before changing a table algorithm", DBRES_MISUSE); } // Run the following function even if table was already augmented. @@ -3148,424 +4595,34 @@ int cloudsync_init_internal (sqlite3_context *context, const char *table_name, c // sync algo with table (unused in this version) // cloudsync_sync_table_key(data, table_name, "*", CLOUDSYNC_KEY_ALGO, crdt_algo_name(algo_new)); + // read row-level filter from settings (if any) + char init_filter_buf[2048]; + int init_frc = dbutils_table_settings_get_value(data, table_name, "*", "filter", init_filter_buf, sizeof(init_filter_buf)); + const char *init_filter = (init_frc == DBRES_OK && init_filter_buf[0]) ? init_filter_buf : NULL; + // check triggers - int rc = dbutils_check_triggers(db, table_name, algo_new); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "An error occurred while creating triggers: %s (%d)", sqlite3_errmsg(db), rc); - return SQLITE_MISUSE; - } + rc = database_create_triggers(data, table_name, algo_new, init_filter); + if (rc != DBRES_OK) return cloudsync_set_error(data, "An error occurred while creating triggers", DBRES_MISUSE); // check meta-table - rc = dbutils_check_metatable(db, table_name, algo_new); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "An error occurred while creating metatable: %s (%d)", sqlite3_errmsg(db), rc); - return SQLITE_MISUSE; - } + rc = database_create_metatable(data, table_name); + if (rc != DBRES_OK) return cloudsync_set_error(data, "An error occurred while creating metatable", DBRES_MISUSE); // add prepared statements - if (stmts_add_tocontext(db, data) != SQLITE_OK) { - dbutils_context_result_error(context, "%s", "An error occurred while trying to compile prepared SQL statements."); - return SQLITE_MISUSE; + if (cloudsync_add_dbvms(data) != DBRES_OK) { + return cloudsync_set_error(data, "An error occurred while trying to compile prepared SQL statements", DBRES_MISUSE); } // add table to in-memory data context - if (table_add_to_context(db, data, algo_new, table_name) == false) { - dbutils_context_result_error(context, "An error occurred while adding %s table information to global context", table_name); - return SQLITE_MISUSE; - } - - if (cloudsync_refill_metatable(db, data, table_name) != SQLITE_OK) { - dbutils_context_result_error(context, "%s", "An error occurred while trying to fill the augmented table."); - return SQLITE_MISUSE; - } - - return SQLITE_OK; -} - -int cloudsync_init_all (sqlite3_context *context, const char *algo_name, bool skip_int_pk_check) { - char sql[1024]; - snprintf(sql, sizeof(sql), "SELECT name, '%s' FROM sqlite_master WHERE type='table' and name NOT LIKE 'sqlite_%%' AND name NOT LIKE 'cloudsync_%%' AND name NOT LIKE '%%_cloudsync';", (algo_name) ? algo_name : CLOUDSYNC_DEFAULT_ALGO); - - sqlite3 *db = sqlite3_context_db_handle(context); - sqlite3_stmt *vm = NULL; - int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) goto abort_init_all; - - while (1) { - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) break; - else if (rc != SQLITE_ROW) goto abort_init_all; - - const char *table = (const char *)sqlite3_column_text(vm, 0); - const char *algo = (const char *)sqlite3_column_text(vm, 1); - rc = cloudsync_init_internal(context, table, algo, skip_int_pk_check); - if (rc != SQLITE_OK) {cloudsync_cleanup_internal(context, table); goto abort_init_all;} - } - rc = SQLITE_OK; - -abort_init_all: - if (vm) sqlite3_finalize(vm); - return rc; -} - -void cloudsync_init (sqlite3_context *context, const char *table, const char *algo, bool skip_int_pk_check) { - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - data->sqlite_ctx = context; - - sqlite3 *db = sqlite3_context_db_handle(context); - int rc = sqlite3_exec(db, "SAVEPOINT cloudsync_init;", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to create cloudsync_init savepoint. %s", sqlite3_errmsg(db)); - sqlite3_result_error_code(context, rc); - return; - } - - if (dbutils_is_star_table(table)) rc = cloudsync_init_all(context, algo, skip_int_pk_check); - else rc = cloudsync_init_internal(context, table, algo, skip_int_pk_check); - - if (rc == SQLITE_OK) { - rc = sqlite3_exec(db, "RELEASE cloudsync_init", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to release cloudsync_init savepoint. %s", sqlite3_errmsg(db)); - sqlite3_result_error_code(context, rc); - } - } - - // in case of error, rollback transaction - if (rc != SQLITE_OK) { - sqlite3_exec(db, "ROLLBACK TO cloudsync_init; RELEASE cloudsync_init", NULL, NULL, NULL); - return; - } - - dbutils_update_schema_hash(db, &data->schema_hash); - - // returns site_id as TEXT - char buffer[UUID_STR_MAXLEN]; - cloudsync_uuid_v7_stringify(data->site_id, buffer, false); - sqlite3_result_text(context, buffer, -1, NULL); -} - -void cloudsync_init3 (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_init2"); - - const char *table = (const char *)sqlite3_value_text(argv[0]); - const char *algo = (const char *)sqlite3_value_text(argv[1]); - bool skip_int_pk_check = (bool)sqlite3_value_int(argv[2]); - - cloudsync_init(context, table, algo, skip_int_pk_check); -} - -void cloudsync_init2 (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_init2"); - - const char *table = (const char *)sqlite3_value_text(argv[0]); - const char *algo = (const char *)sqlite3_value_text(argv[1]); - - cloudsync_init(context, table, algo, false); -} - -void cloudsync_init1 (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_init1"); - - const char *table = (const char *)sqlite3_value_text(argv[0]); - - cloudsync_init(context, table, NULL, false); -} - -// MARK: - - -void cloudsync_begin_alter (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_begin_alter"); - char *errmsg = NULL; - char **result = NULL; - - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - - // get database reference - sqlite3 *db = sqlite3_context_db_handle(context); - - // retrieve global context - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // init cloudsync_settings - if (cloudsync_context_init(db, data, context) == NULL) { - sqlite3_result_error(context, "Unable to init the cloudsync context.", -1); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return; - } - - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to find table %s", table_name); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return; - } - - if (table->is_altering) return; - - // create a savepoint to manage the alter operations as a transaction - int rc = sqlite3_exec(db, "SAVEPOINT cloudsync_alter", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - sqlite3_result_error(context, "Unable to create cloudsync_alter savepoint.", -1); - sqlite3_result_error_code(context, rc); - goto rollback_begin_alter; - } - - int nrows, ncols; - char *sql = cloudsync_memory_mprintf("SELECT name FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name); - rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, &errmsg); - cloudsync_memory_free(sql); - if (errmsg || ncols != 1 || nrows != table->npks) { - dbutils_context_result_error(context, "Unable to get primary keys for table %s (%s)", table_name, errmsg); - sqlite3_result_error_code(context, SQLITE_MISUSE); - goto rollback_begin_alter; - } - - // drop original triggers - dbutils_delete_triggers(db, table_name); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to delete triggers for table %s in cloudsync_begin_alter.", table_name); - sqlite3_result_error_code(context, rc); - goto rollback_begin_alter; - } - - if (table->pk_name) sqlite3_free_table(table->pk_name); - table->pk_name = result; - table->is_altering = true; - return; - -rollback_begin_alter: - sqlite3_exec(db, "ROLLBACK TO cloudsync_alter; RELEASE cloudsync_alter;", NULL, NULL, NULL); - -cleanup_begin_alter: - sqlite3_free_table(result); - sqlite3_free(errmsg); -} - -void cloudsync_commit_alter (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_commit_alter"); - - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_table_context *table = NULL; - - // get database reference - sqlite3 *db = sqlite3_context_db_handle(context); - - // retrieve global context - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // init cloudsync_settings - if (cloudsync_context_init(db, data, context) == NULL) { - dbutils_context_result_error(context, "Unable to init the cloudsync context."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - goto rollback_finalize_alter; + if (table_add_to_context(data, algo_new, table_name) == false) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "An error occurred while adding %s table information to global context", table_name); + return cloudsync_set_error(data, buffer, DBRES_MISUSE); } - table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to find table context."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - goto rollback_finalize_alter; - } - - if (!table->is_altering) return; - - if (!table->pk_name) { - dbutils_context_result_error(context, "Unable to find table context."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - goto rollback_finalize_alter; - } - - int rc = cloudsync_finalize_alter(context, data, table); - if (rc != SQLITE_OK) goto rollback_finalize_alter; - - // the table is outdated, delete it and it will be reloaded in the cloudsync_init_internal - table_remove(data, table_name); - table_free(table); - table = NULL; - - // init again cloudsync for the table - table_algo algo_current = dbutils_table_settings_get_algo(db, table_name); - if (algo_current == table_algo_none) algo_current = dbutils_table_settings_get_algo(db, "*"); - rc = cloudsync_init_internal(context, table_name, crdt_algo_name(algo_current), true); - if (rc != SQLITE_OK) goto rollback_finalize_alter; - - // release savepoint - rc = sqlite3_exec(db, "RELEASE cloudsync_alter", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, sqlite3_errmsg(db)); - sqlite3_result_error_code(context, rc); - goto rollback_finalize_alter; - } - - dbutils_update_schema_hash(db, &data->schema_hash); - - // flag is reset here; note that table_remove + table_free destroyed the old table, - // and cloudsync_init_internal created a fresh one with is_altering = false (zero-alloc). - - return; - -rollback_finalize_alter: - sqlite3_exec(db, "ROLLBACK TO cloudsync_alter; RELEASE cloudsync_alter;", NULL, NULL, NULL); - if (table) { - sqlite3_free_table(table->pk_name); - table->pk_name = NULL; - table->is_altering = false; - } -} - -// MARK: - Main Entrypoint - - -int cloudsync_register (sqlite3 *db, char **pzErrMsg) { - int rc = SQLITE_OK; - - // there's no built-in way to verify if sqlite3_cloudsync_init has already been called - // for this specific database connection, we use a workaround: we attempt to retrieve the - // cloudsync_version and check for an error, an error indicates that initialization has not been performed - if (sqlite3_exec(db, "SELECT cloudsync_version();", NULL, NULL, NULL) == SQLITE_OK) return SQLITE_OK; - - // init memory debugger (NOOP in production) - cloudsync_memory_init(1); - - // init context - void *ctx = cloudsync_context_create(); - if (!ctx) { - if (pzErrMsg) *pzErrMsg = "Not enought memory to create a database context"; - return SQLITE_NOMEM; + if (cloudsync_refill_metatable(data, table_name) != DBRES_OK) { + return cloudsync_set_error(data, "An error occurred while trying to fill the augmented table", DBRES_MISUSE); } - - // register functions - - // PUBLIC functions - rc = dbutils_register_function(db, "cloudsync_version", cloudsync_version, 0, pzErrMsg, ctx, cloudsync_context_free); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_init", cloudsync_init1, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_init", cloudsync_init2, 2, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_init", cloudsync_init3, 3, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - - rc = dbutils_register_function(db, "cloudsync_enable", cloudsync_enable, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_disable", cloudsync_disable, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_is_enabled", cloudsync_is_enabled, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_cleanup", cloudsync_cleanup, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_terminate", cloudsync_terminate, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_set", cloudsync_set, 2, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_set_table", cloudsync_set_table, 3, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_set_column", cloudsync_set_column, 4, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_siteid", cloudsync_siteid, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_db_version", cloudsync_db_version, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_db_version_next", cloudsync_db_version_next, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_db_version_next", cloudsync_db_version_next, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_begin_alter", cloudsync_begin_alter, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_commit_alter", cloudsync_commit_alter, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_uuid", cloudsync_uuid, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - // PAYLOAD - rc = dbutils_register_aggregate(db, "cloudsync_payload_encode", cloudsync_payload_encode_step, cloudsync_payload_encode_final, -1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_payload_decode", cloudsync_payload_decode, -1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - #ifdef CLOUDSYNC_DESKTOP_OS - rc = dbutils_register_function(db, "cloudsync_payload_save", cloudsync_payload_save, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_payload_load", cloudsync_payload_load, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - #endif - - // PRIVATE functions - rc = dbutils_register_function(db, "cloudsync_is_sync", cloudsync_is_sync, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_insert", cloudsync_insert, -1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_aggregate(db, "cloudsync_update", cloudsync_update_step, cloudsync_update_final, 3, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_delete", cloudsync_delete, -1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_col_value", cloudsync_col_value, 3, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_pk_encode", cloudsync_pk_encode, -1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_pk_decode", cloudsync_pk_decode, 2, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_seq", cloudsync_seq, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - // NETWORK LAYER - #ifndef CLOUDSYNC_OMIT_NETWORK - rc = cloudsync_network_register(db, pzErrMsg, ctx); - if (rc != SQLITE_OK) return rc; - #endif - - cloudsync_context *data = (cloudsync_context *)ctx; - sqlite3_commit_hook(db, cloudsync_commit_hook, ctx); - sqlite3_rollback_hook(db, cloudsync_rollback_hook, ctx); - - // register eponymous only changes virtual table - rc = cloudsync_vtab_register_changes (db, data); - if (rc != SQLITE_OK) return rc; - - // load config, if exists - if (cloudsync_config_exists(db)) { - cloudsync_context_init(db, ctx, NULL); - // make sure to update internal version to current version - dbutils_settings_set_key_value(db, NULL, CLOUDSYNC_KEY_LIBVERSION, CLOUDSYNC_VERSION); - } - - return SQLITE_OK; -} - -APIEXPORT int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { - DEBUG_FUNCTION("sqlite3_cloudsync_init"); - - #ifndef SQLITE_CORE - SQLITE_EXTENSION_INIT2(pApi); - #endif - - return cloudsync_register(db, pzErrMsg); + return DBRES_OK; } diff --git a/src/cloudsync.h b/src/cloudsync.h index 4fb84b7d..3c2340d5 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -9,20 +9,220 @@ #define __CLOUDSYNC__ #include +#include #include -#ifndef SQLITE_CORE -#include "sqlite3ext.h" -#else -#include "sqlite3.h" -#endif +#include "database.h" +#include "block.h" #ifdef __cplusplus extern "C" { #endif -#define CLOUDSYNC_VERSION "0.8.68" +#define CLOUDSYNC_VERSION "1.1.2" +#define CLOUDSYNC_MAX_TABLENAME_LEN 512 + +#define CLOUDSYNC_VALUE_NOTSET -1 +#define CLOUDSYNC_TOMBSTONE_VALUE "__[RIP]__" +#define CLOUDSYNC_RLS_RESTRICTED_VALUE "__[RLS]__" +#define CLOUDSYNC_DISABLE_ROWIDONLY_TABLES 1 +#define CLOUDSYNC_DEFAULT_ALGO "cls" +#define CLOUDSYNC_PAYLOAD_CHUNK_DEFAULT_SIZE (5 * 1024 * 1024) +#define CLOUDSYNC_PAYLOAD_CHUNK_MIN_SIZE (256 * 1024) +// Hard ceiling on the effective chunk size, regardless of the per-database +// payload_max_chunk_size setting. Protects the server (one chunk is built in +// memory and stored as a single artifact) and the tenant from a misconfigured +// value. Large TEXT/BLOB values still sync above this size: they are split +// across chunks by the fragment path. Only a row whose non-fragmentable +// scaffolding (primary key + column name + metadata, replicated into every +// fragment) exceeds the chunk size hits row_too_large, which is practically +// unreachable. +#define CLOUDSYNC_PAYLOAD_CHUNK_MAX_SIZE (32 * 1024 * 1024) +#define CLOUDSYNC_PAYLOAD_CHUNK_SAFETY_MARGIN (16 * 1024) +// Fragment sizing is a small fixpoint: after the first target estimate, only +// decimal metadata widths for part_index/part_count can change, so eight passes +// is ample while still preventing an accidental unbounded planning loop. +#define CLOUDSYNC_PAYLOAD_FRAGMENT_SIZE_FIXPOINT_ITERATIONS 8 + +// Machine-parseable error-code tokens. These prefix the human-readable text of +// permanent (non-retryable) failures so the CloudSync server can classify them +// from the error message alone — the only signal common to both the Postgres +// (pgconn.PgError.Message) and SQLite (result error text) backends. The server +// parses the bracketed code with /cloudsync_error\[([a-z0-9_]+)\]/ and decides +// retry policy; keep these strings stable and identical across backends. They +// carry a trailing ": " so they concatenate directly onto a message literal. +#define CLOUDSYNC_ERRCODE_PAYLOAD_TOO_LARGE "cloudsync_error[payload_too_large]: " +#define CLOUDSYNC_ERRCODE_ROW_TOO_LARGE "cloudsync_error[row_too_large]: " +#define CLOUDSYNC_ERRCODE_CHUNK_TOO_LARGE "cloudsync_error[chunk_too_large]: " + +#define CLOUDSYNC_CHANGES_NCOLS 9 + +typedef enum { + CLOUDSYNC_INIT_FLAG_NONE = 0, + CLOUDSYNC_INIT_FLAG_SKIP_INT_PK_CHECK = 1 << 0, // 1 + CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_DEFAULT_CHECK = 1 << 1, // 2 + CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_PRIKEYS_CHECK = 1 << 2 // 4 +} CLOUDSYNC_INIT_FLAG; + +// CRDT Algos +table_algo cloudsync_algo_from_name (const char *algo_name); +const char *cloudsync_algo_name (table_algo algo); + +// Opaque structures +typedef struct cloudsync_payload_context cloudsync_payload_context; +typedef struct cloudsync_table_context cloudsync_table_context; + +// CloudSync context +cloudsync_context *cloudsync_context_create (void *db); +const char *cloudsync_context_init (cloudsync_context *data); +void cloudsync_context_free (void *ctx); + +// CloudSync global +int cloudsync_init_table (cloudsync_context *data, const char *table_name, const char *algo_name, CLOUDSYNC_INIT_FLAG init_flags); +int cloudsync_cleanup (cloudsync_context *data, const char *table_name); +int cloudsync_cleanup_all (cloudsync_context *data); +int cloudsync_terminate (cloudsync_context *data); +int cloudsync_insync (cloudsync_context *data); +int cloudsync_bumpseq (cloudsync_context *data); +void *cloudsync_siteid (cloudsync_context *data); +void cloudsync_reset_siteid (cloudsync_context *data); +void cloudsync_sync_key (cloudsync_context *data, const char *key, const char *value); +int64_t cloudsync_dbversion_next (cloudsync_context *data, int64_t merging_version); +int64_t cloudsync_dbversion (cloudsync_context *data); +void cloudsync_update_schema_hash (cloudsync_context *data); +int cloudsync_dbversion_check_uptodate (cloudsync_context *data); +bool cloudsync_config_exists (cloudsync_context *data); +bool cloudsync_context_is_initialized (cloudsync_context *data); +dbvm_t *cloudsync_colvalue_stmt (cloudsync_context *data, const char *tbl_name, bool *persistent); + +// CloudSync alter table +int cloudsync_begin_alter (cloudsync_context *data, const char *table_name); +int cloudsync_commit_alter (cloudsync_context *data, const char *table_name); + +// CloudSync getter/setter +void *cloudsync_db (cloudsync_context *data); +void *cloudsync_auxdata (cloudsync_context *data); +void cloudsync_set_auxdata (cloudsync_context *data, void *xdata); +int cloudsync_set_error (cloudsync_context *data, const char *err_user, int err_code); +int cloudsync_set_dberror (cloudsync_context *data); +const char *cloudsync_errmsg (cloudsync_context *data); +int cloudsync_errcode (cloudsync_context *data); +void cloudsync_reset_error (cloudsync_context *data); +int cloudsync_commit_hook (void *ctx); +void cloudsync_rollback_hook (void *ctx); +void cloudsync_set_schema (cloudsync_context *data, const char *schema); +const char *cloudsync_schema (cloudsync_context *data); +const char *cloudsync_table_schema (cloudsync_context *data, const char *table_name); + +// Payload +// Receive-checkpoint modes for cloudsync_payload_apply's checkpoint_db_version +// argument. The receive cursor (check_dbversion/check_seq) must only ever land +// on a complete db_version boundary, otherwise a stop between chunks of a single +// source db_version silently skips the unapplied rows on the next /check (the +// server's cloudsync_payload_chunks uses db_version > since with no seq cursor). +// >= 0 advance the cursor to exactly this +// (watermark_db_version), with checkpoint_seq. +// Used once a chunk stream is fully applied. +// CLOUDSYNC_CHECKPOINT_NONE do not advance the cursor. Used for a +// non-final chunk of a multi-chunk stream. +// CLOUDSYNC_CHECKPOINT_LAST_APPLIED advance to this artifact's last applied +// (db_version, seq). Legacy/monolithic +// behavior: safe only for a complete payload +// that ends on a db_version boundary. +#define CLOUDSYNC_CHECKPOINT_NONE (-1) +#define CLOUDSYNC_CHECKPOINT_LAST_APPLIED (-2) +int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int blen, int *nrows, int64_t checkpoint_db_version, int64_t checkpoint_seq); +int cloudsync_payload_encode_step (cloudsync_payload_context *payload, cloudsync_context *data, int argc, dbvalue_t **argv); +int cloudsync_payload_encode_final (cloudsync_payload_context *payload, cloudsync_context *data); +char *cloudsync_payload_blob (cloudsync_payload_context *payload, int64_t *blob_size, int64_t *nrows); +size_t cloudsync_payload_context_size (size_t *header_size); +void cloudsync_payload_context_free (cloudsync_payload_context *payload); +uint64_t cloudsync_payload_context_nrows (cloudsync_payload_context *payload); +size_t cloudsync_payload_context_bused (cloudsync_payload_context *payload); +int cloudsync_payload_get (cloudsync_context *data, char **blob, int *blob_size, int *db_version, int64_t *new_db_version); +int cloudsync_payload_save (cloudsync_context *data, const char *payload_path, int *blob_size); // available only on Desktop OS (no WASM, no mobile) +int cloudsync_payload_max_chunk_size (cloudsync_context *data); +int cloudsync_payload_encode_fragment_step (cloudsync_payload_context *payload, cloudsync_context *data, + const char *tbl, int tbl_len, + const void *pk, int pk_len, + const char *col_name, int col_name_len, + const void *fragment, int fragment_len, + int64_t col_version, int64_t db_version, + const void *site_id, int site_id_len, + int64_t cl, int64_t seq, + uint64_t value_checksum, + int64_t total_size, + int part_index, int part_count); +int cloudsync_payload_fragment_target_size (cloudsync_context *data); +int cloudsync_payload_fragment_count (int64_t total_size, int target_size); +int cloudsync_payload_fragment_data_size (cloudsync_context *data, + const char *tbl, int tbl_len, + const void *pk, int pk_len, + const char *col_name, int col_name_len, + int64_t col_version, int64_t db_version, + const void *site_id, int site_id_len, + int64_t cl, int64_t seq, + int64_t total_size, + int part_index, int part_count); +uint64_t cloudsync_payload_encoded_value_checksum (dbvalue_t *value); +int cloudsync_payload_encoded_value_header (dbvalue_t *value, char *header, int header_cap, int64_t *payload_len); + +// CloudSync table context +int cloudsync_refill_metatable (cloudsync_context *data, const char *table_name); +int cloudsync_reset_metatable (cloudsync_context *data, const char *table_name); +cloudsync_table_context *table_lookup (cloudsync_context *data, const char *table_name); +void *table_column_lookup (cloudsync_table_context *table, const char *col_name, bool is_merge, int *index); +bool table_enabled (cloudsync_table_context *table); +void table_set_enabled (cloudsync_table_context *table, bool value); +bool table_add_to_context (cloudsync_context *data, table_algo algo, const char *table_name); +bool table_pk_exists (cloudsync_table_context *table, const char *value, size_t len); +int table_count_cols (cloudsync_table_context *table); +int table_count_pks (cloudsync_table_context *table); +const char *table_colname (cloudsync_table_context *table, int index); +char **table_pknames (cloudsync_table_context *table); +void table_set_pknames (cloudsync_table_context *table, char **pknames); +bool table_algo_isgos (cloudsync_table_context *table); +const char *table_schema (cloudsync_table_context *table); +int table_remove (cloudsync_context *data, cloudsync_table_context *table); +void table_free (cloudsync_table_context *table); + +// Block-level LWW support +bool table_has_block_cols (cloudsync_table_context *table); +col_algo_t table_col_algo (cloudsync_table_context *table, int index); +const char *table_col_delimiter (cloudsync_table_context *table, int index); +int table_col_index (cloudsync_table_context *table, const char *col_name); +int block_materialize_column (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *base_col_name); +int cloudsync_setup_block_column (cloudsync_context *data, const char *table_name, const char *col_name, const char *delimiter, bool persist); + +// Block column accessors (avoids accessing opaque struct from outside cloudsync.c) +dbvm_t *table_block_value_read_stmt (cloudsync_table_context *table); +dbvm_t *table_block_value_write_stmt (cloudsync_table_context *table); +dbvm_t *table_block_list_stmt (cloudsync_table_context *table); +const char *table_blocks_ref (cloudsync_table_context *table); +void table_set_col_delimiter (cloudsync_table_context *table, int col_idx, const char *delimiter); + +// local merge/apply +int local_mark_insert_sentinel_meta (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq); +int local_update_sentinel (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq); +int local_mark_insert_or_update_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *col_name, int64_t db_version, int seq); +int local_mark_delete_meta (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq); +int local_mark_delete_block_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *block_colname, int64_t db_version, int seq); +int block_delete_value_external (cloudsync_context *data, cloudsync_table_context *table, const void *pk, size_t pklen, const char *block_colname); +int local_drop_meta (cloudsync_table_context *table, const void *pk, size_t pklen); +int local_update_move_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const void *pk2, size_t pklen2, int64_t db_version); + +// used by changes virtual table +int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *col_name, dbvalue_t *col_value, int64_t col_version, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid); +int merge_insert (cloudsync_context *data, cloudsync_table_context *table, const char *insert_pk, int insert_pk_len, int64_t insert_cl, const char *insert_name, dbvalue_t *insert_value, int64_t insert_col_version, int64_t insert_db_version, const char *insert_site_id, int insert_site_id_len, int64_t insert_seq, int64_t *rowid); + +// filter rewrite +char *cloudsync_filter_add_row_prefix(const char *filter, const char *prefix, char **columns, int ncols); -int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); +// decode bind context +char *cloudsync_pk_context_tbl (cloudsync_pk_decode_bind_context *ctx, int64_t *tbl_len); +void *cloudsync_pk_context_pk (cloudsync_pk_decode_bind_context *ctx, int64_t *pk_len); +char *cloudsync_pk_context_colname (cloudsync_pk_decode_bind_context *ctx, int64_t *colname_len); +int64_t cloudsync_pk_context_cl (cloudsync_pk_decode_bind_context *ctx); +int64_t cloudsync_pk_context_dbversion (cloudsync_pk_decode_bind_context *ctx); #ifdef __cplusplus } diff --git a/src/cloudsync_endian.h b/src/cloudsync_endian.h new file mode 100644 index 00000000..4109ea7f --- /dev/null +++ b/src/cloudsync_endian.h @@ -0,0 +1,99 @@ +// +// cloudsync_endian.h +// cloudsync +// +// Created by Marco Bambini on 17/01/26. +// + +#ifndef __CLOUDSYNC_ENDIAN__ +#define __CLOUDSYNC_ENDIAN__ + +#include + +#if defined(_MSC_VER) + #include // _byteswap_uint64 +#endif + +// ======================================================= +// bswap64 - portable +// ======================================================= + +static inline uint64_t bswap64_u64(uint64_t v) { +#if defined(_MSC_VER) + return _byteswap_uint64(v); + +#elif defined(__has_builtin) + #if __has_builtin(__builtin_bswap64) + return __builtin_bswap64(v); + #else + return ((v & 0x00000000000000FFull) << 56) | + ((v & 0x000000000000FF00ull) << 40) | + ((v & 0x0000000000FF0000ull) << 24) | + ((v & 0x00000000FF000000ull) << 8) | + ((v & 0x000000FF00000000ull) >> 8) | + ((v & 0x0000FF0000000000ull) >> 24) | + ((v & 0x00FF000000000000ull) >> 40) | + ((v & 0xFF00000000000000ull) >> 56); + #endif + +#elif defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(v); + +#else + return ((v & 0x00000000000000FFull) << 56) | + ((v & 0x000000000000FF00ull) << 40) | + ((v & 0x0000000000FF0000ull) << 24) | + ((v & 0x00000000FF000000ull) << 8) | + ((v & 0x000000FF00000000ull) >> 8) | + ((v & 0x0000FF0000000000ull) >> 24) | + ((v & 0x00FF000000000000ull) >> 40) | + ((v & 0xFF00000000000000ull) >> 56); +#endif +} + +// ======================================================= +// Compile-time endianness detection +// ======================================================= + +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && defined(__ORDER_BIG_ENDIAN__) + #if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #define HOST_IS_LITTLE_ENDIAN 1 + #elif (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + #define HOST_IS_LITTLE_ENDIAN 0 + #endif +#endif + +// WebAssembly is currently defined as little-endian in all major toolchains +#if !defined(HOST_IS_LITTLE_ENDIAN) && (defined(__wasm__) || defined(__EMSCRIPTEN__)) + #define HOST_IS_LITTLE_ENDIAN 1 +#endif + +// Runtime fallback if unknown at compile-time +static inline int host_is_little_endian_runtime (void) { + const uint16_t x = 1; + return *((const uint8_t*)&x) == 1; +} + +// ======================================================= +// Public API +// ======================================================= + +static inline uint64_t host_to_be64 (uint64_t v) { +#if defined(HOST_IS_LITTLE_ENDIAN) + #if HOST_IS_LITTLE_ENDIAN + return bswap64_u64(v); + #else + return v; + #endif +#else + return host_is_little_endian_runtime() ? bswap64_u64(v) : v; +#endif +} + +static inline uint64_t be64_to_host (uint64_t v) { + // same operation (bswap if little-endian) + return host_to_be64(v); +} + +#endif + diff --git a/src/cloudsync_private.h b/src/cloudsync_private.h deleted file mode 100644 index ae5575f7..00000000 --- a/src/cloudsync_private.h +++ /dev/null @@ -1,55 +0,0 @@ -// -// cloudsync_private.h -// cloudsync -// -// Created by Marco Bambini on 30/05/25. -// - -#ifndef __CLOUDSYNC_PRIVATE__ -#define __CLOUDSYNC_PRIVATE__ - -#include -#ifndef SQLITE_CORE -#include "sqlite3ext.h" -#else -#include "sqlite3.h" -#endif - - -#define CLOUDSYNC_TOMBSTONE_VALUE "__[RIP]__" -#define CLOUDSYNC_RLS_RESTRICTED_VALUE "__[RLS]__" -#define CLOUDSYNC_DISABLE_ROWIDONLY_TABLES 1 - -typedef enum { - CLOUDSYNC_PAYLOAD_APPLY_WILL_APPLY = 1, - CLOUDSYNC_PAYLOAD_APPLY_DID_APPLY = 2, - CLOUDSYNC_PAYLOAD_APPLY_CLEANUP = 3 -} CLOUDSYNC_PAYLOAD_APPLY_STEPS; - -typedef struct cloudsync_context cloudsync_context; -typedef struct cloudsync_pk_decode_bind_context cloudsync_pk_decode_bind_context; - -int cloudsync_merge_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, sqlite3_int64 *rowid); -void cloudsync_sync_key (cloudsync_context *data, const char *key, const char *value); - -// used by network layer -const char *cloudsync_context_init (sqlite3 *db, cloudsync_context *data, sqlite3_context *context); -void *cloudsync_get_auxdata (sqlite3_context *context); -void cloudsync_set_auxdata (sqlite3_context *context, void *xdata); -int cloudsync_payload_apply (sqlite3_context *context, const char *payload, int blen); -int cloudsync_payload_get (sqlite3_context *context, char **blob, int *blob_size, int *db_version, int *seq, sqlite3_int64 *new_db_version, sqlite3_int64 *new_seq); - -// used by core -typedef bool (*cloudsync_payload_apply_callback_t)(void **xdata, cloudsync_pk_decode_bind_context *decoded_change, sqlite3 *db, cloudsync_context *data, int step, int rc); -void cloudsync_set_payload_apply_callback(sqlite3 *db, cloudsync_payload_apply_callback_t callback); - -bool cloudsync_config_exists (sqlite3 *db); -sqlite3_stmt *cloudsync_colvalue_stmt (sqlite3 *db, cloudsync_context *data, const char *tbl_name, bool *persistent); -char *cloudsync_pk_context_tbl (cloudsync_pk_decode_bind_context *ctx, int64_t *tbl_len); -void *cloudsync_pk_context_pk (cloudsync_pk_decode_bind_context *ctx, int64_t *pk_len); -char *cloudsync_pk_context_colname (cloudsync_pk_decode_bind_context *ctx, int64_t *colname_len); -int64_t cloudsync_pk_context_cl (cloudsync_pk_decode_bind_context *ctx); -int64_t cloudsync_pk_context_dbversion (cloudsync_pk_decode_bind_context *ctx); - - -#endif diff --git a/src/database.h b/src/database.h new file mode 100644 index 00000000..56bb2d66 --- /dev/null +++ b/src/database.h @@ -0,0 +1,163 @@ +// +// database.h +// cloudsync +// +// Created by Marco Bambini on 03/12/25. +// + +#ifndef __CLOUDSYNC_DATABASE__ +#define __CLOUDSYNC_DATABASE__ + +#include +#include +#include +#include + +typedef void dbvm_t; +typedef void dbvalue_t; + +typedef enum { + DBRES_OK = 0, + DBRES_ERROR = 1, + DBRES_ABORT = 4, + DBRES_NOMEM = 7, + DBRES_IOERR = 10, + DBRES_CONSTRAINT = 19, + DBRES_MISUSE = 21, + DBRES_ROW = 100, + DBRES_DONE = 101 +} DBRES; + +typedef enum { + DBTYPE_INTEGER = 1, + DBTYPE_FLOAT = 2, + DBTYPE_TEXT = 3, + DBTYPE_BLOB = 4, + DBTYPE_NULL = 5 +} DBTYPE; + +typedef enum { + DBFLAG_PERSISTENT = 0x01 +} DBFLAG; + +// The type of CRDT chosen for a table controls what rows are included or excluded when merging tables together from different databases +typedef enum { + table_algo_none = 0, + table_algo_crdt_cls = 100, // CausalLengthSet + table_algo_crdt_gos, // GrowOnlySet + table_algo_crdt_dws, // DeleteWinsSet + table_algo_crdt_aws // AddWinsSet +} table_algo; + +#ifndef UNUSED_PARAMETER +#define UNUSED_PARAMETER(X) (void)(X) +#endif + +// OPAQUE STRUCT +typedef struct cloudsync_context cloudsync_context; + +// CALLBACK +typedef int (*database_exec_cb) (void *xdata, int argc, char **values, char **names); + +int database_exec (cloudsync_context *data, const char *sql); +int database_exec_callback (cloudsync_context *data, const char *sql, database_exec_cb, void *xdata); +int database_select_int (cloudsync_context *data, const char *sql, int64_t *value); +int database_select_text (cloudsync_context *data, const char *sql, char **value); +int database_select_blob (cloudsync_context *data, const char *sql, char **value, int64_t *value_len); +int database_select_blob_int (cloudsync_context *data, const char *sql, char **value, int64_t *value_len, int64_t *value2); +int database_write (cloudsync_context *data, const char *sql, const char **values, DBTYPE types[], int lens[], int count); +bool database_table_exists (cloudsync_context *data, const char *table_name, const char *schema); +bool database_internal_table_exists (cloudsync_context *data, const char *name); +bool database_trigger_exists (cloudsync_context *data, const char *table_name); +int database_create_metatable (cloudsync_context *data, const char *table_name); +int database_create_triggers (cloudsync_context *data, const char *table_name, table_algo algo, const char *filter); +int database_delete_triggers (cloudsync_context *data, const char *table_name); +int database_pk_names (cloudsync_context *data, const char *table_name, char ***names, int *count); +int database_cleanup (cloudsync_context *data); + +int database_count_pk (cloudsync_context *data, const char *table_name, bool not_null, const char *schema); +int database_count_nonpk (cloudsync_context *data, const char *table_name, const char *schema); +int database_count_int_pk (cloudsync_context *data, const char *table_name, const char *schema); +int database_count_notnull_without_default (cloudsync_context *data, const char *table_name, const char *schema); + +int64_t database_schema_version (cloudsync_context *data); +uint64_t database_schema_hash (cloudsync_context *data); +bool database_check_schema_hash (cloudsync_context *data, uint64_t hash); +int database_update_schema_hash (cloudsync_context *data, uint64_t *hash); + +int database_begin_savepoint (cloudsync_context *data, const char *savepoint_name); +int database_commit_savepoint (cloudsync_context *data, const char *savepoint_name); +int database_rollback_savepoint (cloudsync_context *data, const char *savepoint_name); +bool database_in_transaction (cloudsync_context *data); +int database_errcode (cloudsync_context *data); +const char *database_errmsg (cloudsync_context *data); + +// VM +int databasevm_prepare (cloudsync_context *data, const char *sql, dbvm_t **vm, int flags); +int databasevm_step (dbvm_t *vm); +void databasevm_finalize (dbvm_t *vm); +void databasevm_reset (dbvm_t *vm); +void databasevm_clear_bindings (dbvm_t *vm); +const char *databasevm_sql (dbvm_t *vm); + +// BINDING +int databasevm_bind_blob (dbvm_t *vm, int index, const void *value, uint64_t size); +int databasevm_bind_double (dbvm_t *vm, int index, double value); +int databasevm_bind_int (dbvm_t *vm, int index, int64_t value); +int databasevm_bind_null (dbvm_t *vm, int index); +int databasevm_bind_text (dbvm_t *vm, int index, const char *value, int size); +int databasevm_bind_value (dbvm_t *vm, int index, dbvalue_t *value); + +// VALUE +const void *database_value_blob (dbvalue_t *value); +double database_value_double (dbvalue_t *value); +int64_t database_value_int (dbvalue_t *value); +const char *database_value_text (dbvalue_t *value); +int database_value_bytes (dbvalue_t *value); +int database_value_type (dbvalue_t *value); +void database_value_free (dbvalue_t *value); +void *database_value_dup (dbvalue_t *value); + +// COLUMN +const void *database_column_blob (dbvm_t *vm, int index, size_t *len); +double database_column_double (dbvm_t *vm, int index); +int64_t database_column_int (dbvm_t *vm, int index); +const char *database_column_text (dbvm_t *vm, int index); +dbvalue_t *database_column_value (dbvm_t *vm, int index); +int database_column_bytes (dbvm_t *vm, int index); +int database_column_type (dbvm_t *vm, int index); + +// MEMORY +void *dbmem_alloc (uint64_t size); +void *dbmem_zeroalloc (uint64_t size); +void *dbmem_realloc (void *ptr, uint64_t new_size); +char *dbmem_mprintf(const char *format, ...); +char *dbmem_vmprintf (const char *format, va_list list); +void dbmem_free (void *ptr); +uint64_t dbmem_size (void *ptr); + +// SQL +char *sql_build_drop_table (const char *table_name, char *buffer, int bsize, bool is_meta); +char *sql_build_select_nonpk_by_pk (cloudsync_context *data, const char *table_name, const char *schema); +char *sql_build_delete_by_pk (cloudsync_context *data, const char *table_name, const char *schema); +char *sql_build_insert_pk_ignore (cloudsync_context *data, const char *table_name, const char *schema); +char *sql_build_upsert_pk_and_col (cloudsync_context *data, const char *table_name, const char *colname, const char *schema); +char *sql_build_upsert_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema); +char *sql_build_update_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema); +char *sql_build_select_cols_by_pk (cloudsync_context *data, const char *table_name, const char *colname, const char *schema); +char *sql_build_rekey_pk_and_reset_version_except_col (cloudsync_context *data, const char *table_name, const char *except_col); +char *sql_build_delete_cols_not_in_schema_query(const char *schema, const char *table_name, const char *meta_ref, const char *pkcol); +char *sql_build_pk_collist_query(const char *schema, const char *table_name); +char *sql_build_pk_decode_selectlist_query(const char *schema, const char *table_name); +char *sql_build_pk_qualified_collist_query(const char *schema, const char *table_name); +char *sql_build_insert_missing_pks_query(const char *schema, const char *table_name, const char *pkvalues_identifiers, const char *base_ref, const char *meta_ref, const char *filter); + +char *database_table_schema(const char *table_name); +char *database_build_meta_ref(const char *schema, const char *table_name); +char *database_build_base_ref(const char *schema, const char *table_name); +char *database_build_blocks_ref(const char *schema, const char *table_name); + +// OPAQUE STRUCT used by pk_context functions +typedef struct cloudsync_pk_decode_bind_context cloudsync_pk_decode_bind_context; + +#endif diff --git a/src/dbutils.c b/src/dbutils.c index 6278d0da..0d923881 100644 --- a/src/dbutils.c +++ b/src/dbutils.c @@ -6,14 +6,13 @@ // #include +#include + +#include "sql.h" #include "utils.h" #include "dbutils.h" #include "cloudsync.h" -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT3 -#endif - #if CLOUDSYNC_UNITTEST char *OUT_OF_MEMORY_BUFFER = "OUT_OF_MEMORY_BUFFER"; #ifndef SQLITE_MAX_ALLOCATION_SIZE @@ -21,235 +20,54 @@ char *OUT_OF_MEMORY_BUFFER = "OUT_OF_MEMORY_BUFFER"; #endif #endif -typedef struct { - int type; - int len; - int rc; - union { - sqlite3_int64 intValue; - double doubleValue; - char *stringValue; - } value; -} DATABASE_RESULT; - -typedef struct { - sqlite3 *db; - cloudsync_context *data; -} dbutils_settings_table_context; - -int dbutils_settings_check_version (sqlite3 *db, const char *version); - -// MARK: - General - - -DATABASE_RESULT dbutils_exec (sqlite3_context *context, sqlite3 *db, const char *sql, const char **values, int types[], int lens[], int count, DATABASE_RESULT results[], int expected_types[], int result_count) { - DEBUG_DBFUNCTION("dbutils_exec %s", sql); - - sqlite3_stmt *pstmt = NULL; - bool is_write = (result_count == 0); - #ifdef CLOUDSYNC_UNITTEST - bool is_test = (result_count == 1 && expected_types[0] == SQLITE_NOMEM); - #endif - int type = 0; - - // compile sql - int rc = sqlite3_prepare_v2(db, sql, -1, &pstmt, NULL); - if (rc != SQLITE_OK) goto dbutils_exec_finalize; - - // check bindings - for (int i=0; i r_int); } break; - case SQLITE_FLOAT: { - double l_double = sqlite3_value_double(lvalue); - double r_double = sqlite3_value_double(rvalue); + case DBTYPE_FLOAT: { + double l_double = database_value_double(lvalue); + double r_double = database_value_double(rvalue); return (l_double < r_double) ? -1 : (l_double > r_double); } break; - case SQLITE_NULL: + case DBTYPE_NULL: break; - case SQLITE_TEXT: { - const unsigned char *l_text = sqlite3_value_text(lvalue); - const unsigned char *r_text = sqlite3_value_text(rvalue); + case DBTYPE_TEXT: { + const char *l_text = database_value_text(lvalue); + const char *r_text = database_value_text(rvalue); + if (l_text == NULL && r_text == NULL) return 0; + if (l_text == NULL && r_text != NULL) return -1; + if (l_text != NULL && r_text == NULL) return 1; return strcmp((const char *)l_text, (const char *)r_text); } break; - case SQLITE_BLOB: { - const void *l_blob = sqlite3_value_blob(lvalue); - const void *r_blob = sqlite3_value_blob(rvalue); - int l_size = sqlite3_value_bytes(lvalue); - int r_size = sqlite3_value_bytes(rvalue); + case DBTYPE_BLOB: { + const void *l_blob = database_value_blob(lvalue); + const void *r_blob = database_value_blob(rvalue); + if (l_blob == NULL && r_blob == NULL) return 0; + if (l_blob == NULL && r_blob != NULL) return -1; + if (l_blob != NULL && r_blob == NULL) return 1; + int l_size = database_value_bytes(lvalue); + int r_size = database_value_bytes(rvalue); int cmp = memcmp(l_blob, r_blob, (l_size < r_size) ? l_size : r_size); return (cmp != 0) ? cmp : (l_size - r_size); } break; @@ -258,553 +76,148 @@ int dbutils_value_compare (sqlite3_value *lvalue, sqlite3_value *rvalue) { return 0; } -void dbutils_context_result_error (sqlite3_context *context, const char *format, ...) { - char buffer[4096]; - - va_list arg; - va_start (arg, format); - vsnprintf(buffer, sizeof(buffer), format, arg); - va_end (arg); - - if (context) sqlite3_result_error(context, buffer, -1); -} - -// MARK: - - -void dbutils_debug_value (sqlite3_value *value) { - switch (sqlite3_value_type(value)) { - case SQLITE_INTEGER: - printf("\t\tINTEGER: %lld\n", sqlite3_value_int64(value)); +void dbutils_debug_value (dbvalue_t *value) { + switch (database_value_type(value)) { + case DBTYPE_INTEGER: + printf("\t\tINTEGER: %" PRId64 "\n", database_value_int(value)); break; - case SQLITE_FLOAT: - printf("\t\tFLOAT: %f\n", sqlite3_value_double(value)); + case DBTYPE_FLOAT: + printf("\t\tFLOAT: %f\n", database_value_double(value)); break; - case SQLITE_TEXT: - printf("\t\tTEXT: %s (%d)\n", sqlite3_value_text(value), sqlite3_value_bytes(value)); + case DBTYPE_TEXT: + printf("\t\tTEXT: %s (%d)\n", database_value_text(value), database_value_bytes(value)); break; - case SQLITE_BLOB: - printf("\t\tBLOB: %p (%d)\n", (char *)sqlite3_value_blob(value), sqlite3_value_bytes(value)); + case DBTYPE_BLOB: + printf("\t\tBLOB: %p (%d)\n", (char *)database_value_blob(value), database_value_bytes(value)); break; - case SQLITE_NULL: + case DBTYPE_NULL: printf("\t\tNULL\n"); break; } } -void dbutils_debug_values (int argc, sqlite3_value **argv) { +void dbutils_debug_values (dbvalue_t **argv, int argc) { for (int i = 0; i < argc; i++) { dbutils_debug_value(argv[i]); } } -int dbutils_debug_stmt (sqlite3 *db, bool print_result) { - sqlite3_stmt *stmt = NULL; - int counter = 0; - while ((stmt = sqlite3_next_stmt(db, stmt))) { - ++counter; - if (print_result) printf("Unfinalized stmt statement: %p\n", stmt); - } - return counter; -} - -// MARK: - - -int dbutils_register_function (sqlite3 *db, const char *name, void (*ptr)(sqlite3_context*,int,sqlite3_value**), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { - DEBUG_DBFUNCTION("dbutils_register_function %s", name); - - const int DEFAULT_FLAGS = SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC; - int rc = sqlite3_create_function_v2(db, name, nargs, DEFAULT_FLAGS, ctx, ptr, NULL, NULL, ctx_free); - - if (rc != SQLITE_OK) { - if (pzErrMsg) *pzErrMsg = cloudsync_memory_mprintf("Error creating function %s: %s", name, sqlite3_errmsg(db)); - return rc; - } - - return SQLITE_OK; -} - -int dbutils_register_aggregate (sqlite3 *db, const char *name, void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { - DEBUG_DBFUNCTION("dbutils_register_aggregate %s", name); - - const int DEFAULT_FLAGS = SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC; - int rc = sqlite3_create_function_v2(db, name, nargs, DEFAULT_FLAGS, ctx, NULL, xstep, xfinal, ctx_free); - - if (rc != SQLITE_OK) { - if (pzErrMsg) *pzErrMsg = cloudsync_memory_mprintf("Error creating aggregate function %s: %s", name, sqlite3_errmsg(db)); - return rc; - } - - return SQLITE_OK; -} - -bool dbutils_system_exists (sqlite3 *db, const char *name, const char *type) { - DEBUG_DBFUNCTION("dbutils_system_exists %s: %s", type, name); - - sqlite3_stmt *vm = NULL; - bool result = false; - - char sql[1024]; - snprintf(sql, sizeof(sql), "SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type='%s' AND name=?1 COLLATE NOCASE);", type); - int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) goto finalize; - - rc = sqlite3_bind_text(vm, 1, name, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize; - - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) { - result = (bool)sqlite3_column_int(vm, 0); - rc = SQLITE_OK; - } - -finalize: - if (rc != SQLITE_OK) DEBUG_ALWAYS("Error executing %s in dbutils_system_exists for type %s name %s (%s).", sql, type, name, sqlite3_errmsg(db)); - if (vm) sqlite3_finalize(vm); - return result; -} - -bool dbutils_table_exists (sqlite3 *db, const char *name) { - return dbutils_system_exists(db, name, "table"); -} +// MARK: - Settings - -bool dbutils_trigger_exists (sqlite3 *db, const char *name) { - return dbutils_system_exists(db, name, "trigger"); +int dbutils_binary_comparison (int x, int y) { + return (x == y) ? 0 : (x > y ? 1 : -1); } -bool dbutils_table_sanity_check (sqlite3 *db, sqlite3_context *context, const char *name, bool skip_int_pk_check) { - DEBUG_DBFUNCTION("dbutils_table_sanity_check %s", name); - - char buffer[2048]; - size_t blen = sizeof(buffer); +int dbutils_settings_get_value (cloudsync_context *data, const char *key, char *buffer, size_t *blen, int64_t *intvalue) { + DEBUG_SETTINGS("dbutils_settings_get_value key: %s", key); - // sanity check table name - if (name == NULL) { - dbutils_context_result_error(context, "%s", "cloudsync_init requires a non-null table parameter"); - return false; + // if intvalue requested: buffer/blen optional + size_t buffer_len = 0; + if (intvalue) { + *intvalue = 0; + } else { + if (!buffer || !blen || *blen == 0) return DBRES_MISUSE; + buffer[0] = 0; + buffer_len = *blen; + *blen = 0; } - // avoid allocating heap memory for SQL statements by setting a maximum length of 1900 characters - // for table names. This limit is reasonable and helps prevent memory management issues. - const size_t maxlen = blen - 148; - if (strlen(name) > maxlen) { - dbutils_context_result_error(context, "Table name cannot be longer than %d characters", maxlen); - return false; - } + dbvm_t *vm = NULL; + int rc = databasevm_prepare(data, SQL_SETTINGS_GET_VALUE, (void **)&vm, 0); + if (rc != DBRES_OK) goto finalize_get_value; - // check if table exists - if (dbutils_table_exists(db, name) == false) { - dbutils_context_result_error(context, "Table %s does not exist", name); - return false; - } + rc = databasevm_bind_text(vm, 1, key, -1); + if (rc != DBRES_OK) goto finalize_get_value; - // no more than 128 columns can be used as a composite primary key (SQLite hard limit) - char *sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk>0;", name); - sqlite3_int64 count = dbutils_int_select(db, sql); - if (count > 128) { - dbutils_context_result_error(context, "No more than 128 columns can be used to form a composite primary key"); - return false; - } else if (count == -1) { - dbutils_context_result_error(context, "%s", sqlite3_errmsg(db)); - return false; - } + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + else if (rc != DBRES_ROW) goto finalize_get_value; - #if CLOUDSYNC_DISABLE_ROWIDONLY_TABLES - // if count == 0 means that rowid will be used as primary key (BTW: very bad choice for the user) - if (count == 0) { - dbutils_context_result_error(context, "Rowid only tables are not supported, all primary keys must be explicitly set and declared as NOT NULL (table %s)", name); - return false; - } - #endif - - if (!skip_int_pk_check) { - if (count == 1) { - // the affinity of a column is determined by the declared type of the column, - // according to the following rules in the order shown: - // 1. If the declared type contains the string "INT" then it is assigned INTEGER affinity. - sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk=1 AND \"type\" LIKE '%%INT%%';", name); - sqlite3_int64 count2 = dbutils_int_select(db, sql); - if (count == count2) { - dbutils_context_result_error(context, "Table %s uses an single-column INTEGER primary key. For CRDT replication, primary keys must be globally unique. Consider using a TEXT primary key with UUIDs or ULID to avoid conflicts across nodes. If you understand the risk and still want to use this INTEGER primary key, set the third argument of the cloudsync_init function to 1 to skip this check.", name); - return false; - } - if (count2 == -1) { - dbutils_context_result_error(context, "%s", sqlite3_errmsg(db)); - return false; - } - } - } + // SQLITE_ROW case + if (rc == DBRES_ROW) { + rc = DBRES_OK; - // if user declared explicit primary key(s) then make sure they are all declared as NOT NULL - if (count > 0) { - sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk>0 AND \"notnull\"=1;", name); - sqlite3_int64 count2 = dbutils_int_select(db, sql); - if (count2 == -1) { - dbutils_context_result_error(context, "%s", sqlite3_errmsg(db)); - return false; - } - if (count != count2) { - dbutils_context_result_error(context, "All primary keys must be explicitly declared as NOT NULL (table %s)", name); - return false; + // NULL case + if (database_column_type(vm, 0) == DBTYPE_NULL) { + goto finalize_get_value; } - } - - // check for columns declared as NOT NULL without a DEFAULT value. - // Otherwise, col_merge_stmt would fail if changes to other columns are inserted first. - sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk=0 AND \"notnull\"=1 AND \"dflt_value\" IS NULL;", name); - sqlite3_int64 count3 = dbutils_int_select(db, sql); - if (count3 == -1) { - dbutils_context_result_error(context, "%s", sqlite3_errmsg(db)); - return false; - } - if (count3 > 0) { - dbutils_context_result_error(context, "All non-primary key columns declared as NOT NULL must have a DEFAULT value. (table %s)", name); - return false; - } - - return true; -} - -int dbutils_delete_triggers (sqlite3 *db, const char *table) { - DEBUG_DBFUNCTION("dbutils_delete_triggers %s", table); - - // from dbutils_table_sanity_check we already know that 2048 is OK - char buffer[2048]; - size_t blen = sizeof(buffer); - int rc = SQLITE_ERROR; - - char *sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_before_update_%w\";", table); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_before_delete_%w\";", table); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_insert_%w\";", table); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_update_%w\";", table); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_delete_%w\";", table); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - -finalize: - if (rc != SQLITE_OK) DEBUG_ALWAYS("dbutils_delete_triggers error %s (%s)", sqlite3_errmsg(db), sql); - return rc; -} - -int dbutils_check_triggers (sqlite3 *db, const char *table, table_algo algo) { - DEBUG_DBFUNCTION("dbutils_check_triggers %s", table); - - if (dbutils_settings_check_version(db, "0.8.25") <= 0) { - dbutils_delete_triggers(db, table); - } - - char *trigger_name = NULL; - int rc = SQLITE_NOMEM; - - // common part - char *trigger_when = cloudsync_memory_mprintf("FOR EACH ROW WHEN cloudsync_is_sync('%q') = 0", table); - if (!trigger_when) goto finalize; - - // INSERT TRIGGER - // NEW.prikey1, NEW.prikey2... - trigger_name = cloudsync_memory_mprintf("cloudsync_after_insert_%s", table); - if (!trigger_name) goto finalize; - - if (!dbutils_trigger_exists(db, trigger_name)) { - rc = SQLITE_NOMEM; - char *sql = cloudsync_memory_mprintf("SELECT group_concat('NEW.\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table); - if (!sql) goto finalize; - - char *pkclause = dbutils_text_select(db, sql); - char *pkvalues = (pkclause) ? pkclause : "NEW.rowid"; - cloudsync_memory_free(sql); - - sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" AFTER INSERT ON \"%w\" %s BEGIN SELECT cloudsync_insert('%q', %s); END", trigger_name, table, trigger_when, table, pkvalues); - if (pkclause) cloudsync_memory_free(pkclause); - if (!sql) goto finalize; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; - } - cloudsync_memory_free(trigger_name); - trigger_name = NULL; - rc = SQLITE_NOMEM; - - if (algo != table_algo_crdt_gos) { - rc = SQLITE_NOMEM; - // UPDATE TRIGGER - // NEW.prikey1, NEW.prikey2, OLD.prikey1, OLD.prikey2, NEW.col1, OLD.col1, NEW.col2, OLD.col2... - trigger_name = cloudsync_memory_mprintf("cloudsync_after_update_%s", table); - if (!trigger_name) goto finalize; - - if (!dbutils_trigger_exists(db, trigger_name)) { - // Generate VALUES clause for all columns using a CTE to avoid compound SELECT limits - // First, get all primary key columns in order - char *pk_values_sql = cloudsync_memory_mprintf( - "SELECT group_concat('('||quote('%q')||', NEW.\"' || format('%%w', name) || '\", OLD.\"' || format('%%w', name) || '\")', ', ') " - "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", - table, table); - if (!pk_values_sql) goto finalize; - - char *pk_values_list = dbutils_text_select(db, pk_values_sql); - cloudsync_memory_free(pk_values_sql); - - // Then get all regular columns in order - char *col_values_sql = cloudsync_memory_mprintf( - "SELECT group_concat('('||quote('%q')||', NEW.\"' || format('%%w', name) || '\", OLD.\"' || format('%%w', name) || '\")', ', ') " - "FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid;", - table, table); - if (!col_values_sql) goto finalize; - - char *col_values_list = dbutils_text_select(db, col_values_sql); - cloudsync_memory_free(col_values_sql); - - // Build the complete VALUES query - char *values_query; - if (col_values_list && strlen(col_values_list) > 0) { - // Table has both primary keys and regular columns - values_query = cloudsync_memory_mprintf( - "WITH column_data(table_name, new_value, old_value) AS (VALUES %s, %s) " - "SELECT table_name, new_value, old_value FROM column_data", - pk_values_list, col_values_list); - cloudsync_memory_free(col_values_list); + // INT case + if (intvalue) { + int type = database_column_type(vm, 0); + if (type == DBTYPE_TEXT) { + const char *value = database_column_text(vm, 0); + *intvalue = value ? strtoll(value, NULL, 10) : 0; } else { - // Table has only primary keys - values_query = cloudsync_memory_mprintf( - "WITH column_data(table_name, new_value, old_value) AS (VALUES %s) " - "SELECT table_name, new_value, old_value FROM column_data", - pk_values_list); + *intvalue = database_column_int(vm, 0); } - - if (pk_values_list) cloudsync_memory_free(pk_values_list); - if (!values_query) goto finalize; - - // Create the trigger with aggregate function - char *sql = cloudsync_memory_mprintf( - "CREATE TRIGGER \"%w\" AFTER UPDATE ON \"%w\" %s BEGIN " - "SELECT cloudsync_update(table_name, new_value, old_value) FROM (%s); " - "END", - trigger_name, table, trigger_when, values_query); - - cloudsync_memory_free(values_query); - if (!sql) goto finalize; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; + goto finalize_get_value; } - cloudsync_memory_free(trigger_name); - trigger_name = NULL; - } else { - // Grow Only Set - // In a grow-only set, the update operation is not allowed. - // A grow-only set is a type of CRDT (Conflict-free Replicated Data Type) where the only permissible operation is to add elements to the set, - // without ever removing or modifying them. - // Once an element is added to the set, it remains there permanently, which guarantees that the set only grows over time. - trigger_name = cloudsync_memory_mprintf("cloudsync_before_update_%s", table); - if (!trigger_name) goto finalize; - if (!dbutils_trigger_exists(db, trigger_name)) { - char *sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" BEFORE UPDATE ON \"%w\" FOR EACH ROW WHEN cloudsync_is_enabled('%q') = 1 BEGIN SELECT RAISE(ABORT, 'Error: UPDATE operation is not allowed on table %w.'); END", trigger_name, table, table, table); - if (!sql) goto finalize; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; + // buffer case + const char *value = database_column_text(vm, 0); + size_t size = (size_t)database_column_bytes(vm, 0); + if (!value || size == 0) goto finalize_get_value; + if (size + 1 > buffer_len) { + rc = DBRES_NOMEM; + } else { + memcpy(buffer, value, size); + buffer[size] = '\0'; + *blen = size; } - cloudsync_memory_free(trigger_name); - trigger_name = NULL; } - // DELETE TRIGGER - // OLD.prikey1, OLD.prikey2... - if (algo != table_algo_crdt_gos) { - trigger_name = cloudsync_memory_mprintf("cloudsync_after_delete_%s", table); - if (!trigger_name) goto finalize; - - if (!dbutils_trigger_exists(db, trigger_name)) { - char *sql = cloudsync_memory_mprintf("SELECT group_concat('OLD.\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table); - if (!sql) goto finalize; - - char *pkclause = dbutils_text_select(db, sql); - char *pkvalues = (pkclause) ? pkclause : "OLD.rowid"; - cloudsync_memory_free(sql); - - sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" AFTER DELETE ON \"%w\" %s BEGIN SELECT cloudsync_delete('%q',%s); END", trigger_name, table, trigger_when, table, pkvalues); - if (pkclause) cloudsync_memory_free(pkclause); - if (!sql) goto finalize; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; - } - - cloudsync_memory_free(trigger_name); - trigger_name = NULL; - } else { - // Grow Only Set - // In a grow-only set, the delete operation is not allowed. - trigger_name = cloudsync_memory_mprintf("cloudsync_before_delete_%s", table); - if (!trigger_name) goto finalize; - - if (!dbutils_trigger_exists(db, trigger_name)) { - char *sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" BEFORE DELETE ON \"%w\" FOR EACH ROW WHEN cloudsync_is_enabled('%q') = 1 BEGIN SELECT RAISE(ABORT, 'Error: DELETE operation is not allowed on table %w.'); END", trigger_name, table, table, table); - if (!sql) goto finalize; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; - } - cloudsync_memory_free(trigger_name); - trigger_name = NULL; +finalize_get_value: + if (rc != DBRES_OK) { + DEBUG_ALWAYS("dbutils_settings_get_value error %s", database_errmsg(data)); } - rc = SQLITE_OK; - -finalize: - if (trigger_name) cloudsync_memory_free(trigger_name); - if (trigger_when) cloudsync_memory_free(trigger_when); - if (rc != SQLITE_OK) DEBUG_ALWAYS("dbutils_create_triggers error %s (%d)", sqlite3_errmsg(db), rc); - return rc; -} - -int dbutils_check_metatable (sqlite3 *db, const char *table, table_algo algo) { - DEBUG_DBFUNCTION("dbutils_check_metatable %s", table); - - // WITHOUT ROWID is available starting from SQLite version 3.8.2 (2013-12-06) and later - char *sql = cloudsync_memory_mprintf("CREATE TABLE IF NOT EXISTS \"%w_cloudsync\" (pk BLOB NOT NULL, col_name TEXT NOT NULL, col_version INTEGER, db_version INTEGER, site_id INTEGER DEFAULT 0, seq INTEGER, PRIMARY KEY (pk, col_name)) WITHOUT ROWID; CREATE INDEX IF NOT EXISTS \"%w_cloudsync_db_idx\" ON \"%w_cloudsync\" (db_version);", table, table, table); - if (!sql) return SQLITE_NOMEM; - - int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - + if (vm) databasevm_finalize(vm); return rc; } - -sqlite3_int64 dbutils_schema_version (sqlite3 *db) { - DEBUG_DBFUNCTION("dbutils_schema_version"); - - return dbutils_int_select(db, "PRAGMA schema_version;"); -} - -bool dbutils_is_star_table (const char *table_name) { - return (table_name && (strlen(table_name) == 1) && table_name[0] == '*'); -} - -// MARK: - Settings - - -int binary_comparison (int x, int y) { - if (x == y) return 0; - if (x > y) return 1; - return -1; -} - -char *dbutils_settings_get_value (sqlite3 *db, const char *key, char *buffer, size_t blen) { - DEBUG_SETTINGS("dbutils_settings_get_value key: %s", key); - - // check if heap allocation must be forced - if (!buffer || blen == 0) blen = 0; - size_t size = 0; - - sqlite3_stmt *vm = NULL; - char *sql = "SELECT value FROM cloudsync_settings WHERE key=?1;"; - int rc = sqlite3_prepare(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) goto finalize_get_value; - - rc = sqlite3_bind_text(vm, 1, key, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize_get_value; - - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - else if (rc != SQLITE_ROW) goto finalize_get_value; - - // SQLITE_ROW case - if (sqlite3_column_type(vm, 0) == SQLITE_NULL) { - rc = SQLITE_OK; - goto finalize_get_value; - } - - const unsigned char *value = sqlite3_column_text(vm, 0); - #if CLOUDSYNC_UNITTEST - size = (buffer == OUT_OF_MEMORY_BUFFER) ? (SQLITE_MAX_ALLOCATION_SIZE + 1) :(size_t)sqlite3_column_bytes(vm, 0); - #else - size = (size_t)sqlite3_column_bytes(vm, 0); - #endif - if (size + 1 > blen) { - buffer = cloudsync_memory_alloc((sqlite3_uint64)(size + 1)); - if (!buffer) { - rc = SQLITE_NOMEM; - goto finalize_get_value; - } - } - - memcpy(buffer, value, size+1); - rc = SQLITE_OK; - -finalize_get_value: - #if CLOUDSYNC_UNITTEST - if ((rc == SQLITE_NOMEM) && (size == SQLITE_MAX_ALLOCATION_SIZE + 1)) rc = SQLITE_OK; - #endif - if (rc != SQLITE_OK) DEBUG_ALWAYS("dbutils_settings_get_value error %s", sqlite3_errmsg(db)); - if (vm) sqlite3_finalize(vm); - - return buffer; -} - -int dbutils_settings_set_key_value (sqlite3 *db, sqlite3_context *context, const char *key, const char *value) { +int dbutils_settings_set_key_value (cloudsync_context *data, const char *key, const char *value) { + if (!key) return DBRES_MISUSE; DEBUG_SETTINGS("dbutils_settings_set_key_value key: %s value: %s", key, value); - - int rc = SQLITE_OK; - if (db == NULL) db = sqlite3_context_db_handle(context); - - if (key && value) { - char *sql = "REPLACE INTO cloudsync_settings (key, value) VALUES (?1, ?2);"; + + int rc = DBRES_OK; + if (value) { const char *values[] = {key, value}; - int types[] = {SQLITE_TEXT, SQLITE_TEXT}; + DBTYPE types[] = {DBTYPE_TEXT, DBTYPE_TEXT}; int lens[] = {-1, -1}; - rc = dbutils_write(db, context, sql, values, types, lens, 2); - } - - if (value == NULL) { - char *sql = "DELETE FROM cloudsync_settings WHERE key = ?1;"; + rc = database_write(data, SQL_SETTINGS_SET_KEY_VALUE_REPLACE, values, types, lens, 2); + } else { const char *values[] = {key}; - int types[] = {SQLITE_TEXT}; + DBTYPE types[] = {DBTYPE_TEXT}; int lens[] = {-1}; - rc = dbutils_write(db, context, sql, values, types, lens, 1); + rc = database_write(data, SQL_SETTINGS_SET_KEY_VALUE_DELETE, values, types, lens, 1); } - - cloudsync_context *data = (context) ? (cloudsync_context *)sqlite3_user_data(context) : NULL; - if (rc == SQLITE_OK && data) cloudsync_sync_key(data, key, value); + + if (rc == DBRES_OK && data) cloudsync_sync_key(data, key, value); return rc; } -int dbutils_settings_get_int_value (sqlite3 *db, const char *key) { +int dbutils_settings_get_int_value (cloudsync_context *data, const char *key) { + DEBUG_SETTINGS("dbutils_settings_get_int_value key: %s", key); + int64_t value = 0; + if (dbutils_settings_get_value(data, key, NULL, NULL, &value) != DBRES_OK) return -1; + + return (int)value; +} + +int64_t dbutils_settings_get_int64_value (cloudsync_context *data, const char *key) { DEBUG_SETTINGS("dbutils_settings_get_int_value key: %s", key); - char buffer[256] = {0}; - if (dbutils_settings_get_value(db, key, buffer, sizeof(buffer)) == NULL) return -1; + int64_t value = 0; + if (dbutils_settings_get_value(data, key, NULL, NULL, &value) != DBRES_OK) return -1; - return (int)strtol(buffer, NULL, 0); + return value; } -int dbutils_settings_check_version (sqlite3 *db, const char *version) { +int dbutils_settings_check_version (cloudsync_context *data, const char *version) { DEBUG_SETTINGS("dbutils_settings_check_version"); char buffer[256]; - if (dbutils_settings_get_value(db, CLOUDSYNC_KEY_LIBVERSION, buffer, sizeof(buffer)) == NULL) return -666; + size_t len = sizeof(buffer); + if (dbutils_settings_get_value(data, CLOUDSYNC_KEY_LIBVERSION, buffer, &len, NULL) != DBRES_OK) return -666; int major1, minor1, patch1; int major2, minor2, patch2; @@ -814,9 +227,9 @@ int dbutils_settings_check_version (sqlite3 *db, const char *version) { if (count1 != 3 || count2 != 3) return -666; int res = 0; - if ((res = binary_comparison(major1, major2)) == 0) { - if ((res = binary_comparison(minor1, minor2)) == 0) { - return binary_comparison(patch1, patch2); + if ((res = dbutils_binary_comparison(major1, major2)) == 0) { + if ((res = dbutils_binary_comparison(minor1, minor2)) == 0) { + return dbutils_binary_comparison(patch1, patch2); } } @@ -824,318 +237,281 @@ int dbutils_settings_check_version (sqlite3 *db, const char *version) { return res; } -char *dbutils_table_settings_get_value (sqlite3 *db, const char *table, const char *column, const char *key, char *buffer, size_t blen) { - DEBUG_SETTINGS("dbutils_table_settings_get_value table: %s column: %s key: %s", table, column, key); - - // check if heap allocation must be forced - if (!buffer || blen == 0) blen = 0; - size_t size = 0; +int dbutils_table_settings_get_value (cloudsync_context *data, const char *table, const char *column_name, const char *key, char *buffer, size_t blen) { + DEBUG_SETTINGS("dbutils_table_settings_get_value table: %s column: %s key: %s", table, column_name, key); + + if (!buffer || blen == 0) return DBRES_MISUSE; + buffer[0] = 0; - sqlite3_stmt *vm = NULL; - char *sql = "SELECT value FROM cloudsync_table_settings WHERE (tbl_name=?1 AND col_name=?2 AND key=?3);"; - int rc = sqlite3_prepare(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) goto finalize_get_value; + dbvm_t *vm = NULL; + int rc = databasevm_prepare(data, SQL_TABLE_SETTINGS_GET_VALUE, (void **)&vm, 0); + if (rc != DBRES_OK) goto finalize_get_value; - rc = sqlite3_bind_text(vm, 1, table, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize_get_value; + rc = databasevm_bind_text(vm, 1, table, -1); + if (rc != DBRES_OK) goto finalize_get_value; - rc = sqlite3_bind_text(vm, 2, (column) ? column : "*", -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize_get_value; + rc = databasevm_bind_text(vm, 2, (column_name) ? column_name : "*", -1); + if (rc != DBRES_OK) goto finalize_get_value; - rc = sqlite3_bind_text(vm, 3, key, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize_get_value; + rc = databasevm_bind_text(vm, 3, key, -1); + if (rc != DBRES_OK) goto finalize_get_value; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - else if (rc != SQLITE_ROW) goto finalize_get_value; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + else if (rc != DBRES_ROW) goto finalize_get_value; // SQLITE_ROW case - if (sqlite3_column_type(vm, 0) == SQLITE_NULL) { - rc = SQLITE_OK; - goto finalize_get_value; - } - - const unsigned char *value = sqlite3_column_text(vm, 0); - #if CLOUDSYNC_UNITTEST - size = (buffer == OUT_OF_MEMORY_BUFFER) ? (SQLITE_MAX_ALLOCATION_SIZE + 1) :(size_t)sqlite3_column_bytes(vm, 0); - #else - size = (size_t)sqlite3_column_bytes(vm, 0); - #endif - if (size + 1 > blen) { - buffer = cloudsync_memory_alloc((sqlite3_uint64)(size + 1)); - if (!buffer) { - rc = SQLITE_NOMEM; + if (rc == DBRES_ROW) { + rc = DBRES_OK; + + // NULL case + if (database_column_type(vm, 0) == DBTYPE_NULL) { goto finalize_get_value; } - } - memcpy(buffer, value, size+1); - rc = SQLITE_OK; - -finalize_get_value: - #if CLOUDSYNC_UNITTEST - if ((rc == SQLITE_NOMEM) && (size == SQLITE_MAX_ALLOCATION_SIZE + 1)) rc = SQLITE_OK; - #endif - if (rc != SQLITE_OK) { - DEBUG_ALWAYS("cloudsync_table_settings error %s", sqlite3_errmsg(db)); + const char *value = database_column_text(vm, 0); + size_t size = (size_t)database_column_bytes(vm, 0); + if (size + 1 > blen) { + rc = DBRES_NOMEM; + } else { + memcpy(buffer, value, size); + buffer[size] = '\0'; + } } - if (vm) sqlite3_finalize(vm); - return buffer; +finalize_get_value: + if (rc != DBRES_OK) { + DEBUG_ALWAYS("cloudsync_table_settings error %s", database_errmsg(data)); + } + if (vm) databasevm_finalize(vm); + return rc; } -int dbutils_table_settings_set_key_value (sqlite3 *db, sqlite3_context *context, const char *table, const char *column, const char *key, const char *value) { - DEBUG_SETTINGS("dbutils_table_settings_set_key_value table: %s column: %s key: %s", table, column, key); +int dbutils_table_settings_set_key_value (cloudsync_context *data, const char *table_name, const char *column_name, const char *key, const char *value) { + DEBUG_SETTINGS("dbutils_table_settings_set_key_value table: %s column: %s key: %s", table_name, column_name, key); - int rc = SQLITE_OK; - if (db == NULL) db = sqlite3_context_db_handle(context); + int rc = DBRES_OK; // sanity check tbl_name - if (table == NULL) { - if (context) sqlite3_result_error(context, "cloudsync_set_table/set_column requires a non-null table parameter", -1); - return SQLITE_ERROR; + if (table_name == NULL) { + return cloudsync_set_error(data, "cloudsync_set_table/set_column requires a non-null table parameter", DBRES_ERROR); } // sanity check column name - if (column == NULL) column = "*"; + if (column_name == NULL) column_name = "*"; // remove all table_name entries if (key == NULL) { - char *sql = "DELETE FROM cloudsync_table_settings WHERE tbl_name=?1;"; - const char *values[] = {table}; - int types[] = {SQLITE_TEXT}; + const char *values[] = {table_name}; + DBTYPE types[] = {DBTYPE_TEXT}; int lens[] = {-1}; - rc = dbutils_write(db, context, sql, values, types, lens, 1); + rc = database_write(data, SQL_TABLE_SETTINGS_DELETE_ALL_FOR_TABLE, values, types, lens, 1); return rc; } if (key && value) { - char *sql = "REPLACE INTO cloudsync_table_settings (tbl_name, col_name, key, value) VALUES (?1, ?2, ?3, ?4);"; - const char *values[] = {table, column, key, value}; - int types[] = {SQLITE_TEXT, SQLITE_TEXT, SQLITE_TEXT, SQLITE_TEXT}; + const char *values[] = {table_name, column_name, key, value}; + DBTYPE types[] = {DBTYPE_TEXT, DBTYPE_TEXT, DBTYPE_TEXT, DBTYPE_TEXT}; int lens[] = {-1, -1, -1, -1}; - rc = dbutils_write(db, context, sql, values, types, lens, 4); + rc = database_write(data, SQL_TABLE_SETTINGS_REPLACE, values, types, lens, 4); } if (value == NULL) { - char *sql = "DELETE FROM cloudsync_table_settings WHERE (tbl_name=?1 AND col_name=?2 AND key=?3);"; - const char *values[] = {table, column, key}; - int types[] = {SQLITE_TEXT, SQLITE_TEXT, SQLITE_TEXT}; + const char *values[] = {table_name, column_name, key}; + DBTYPE types[] = {DBTYPE_TEXT, DBTYPE_TEXT, DBTYPE_TEXT}; int lens[] = {-1, -1, -1}; - rc = dbutils_write(db, context, sql, values, types, lens, 3); + rc = database_write(data, SQL_TABLE_SETTINGS_DELETE_ONE, values, types, lens, 3); } // unused in this version // cloudsync_context *data = (context) ? (cloudsync_context *)sqlite3_user_data(context) : NULL; - // if (rc == SQLITE_OK && data) cloudsync_sync_table_key(data, table, column, key, value); + // if (rc == DBRES_OK && data) cloudsync_sync_table_key(data, table, column, key, value); return rc; } -sqlite3_int64 dbutils_table_settings_count_tables (sqlite3 *db) { +int64_t dbutils_table_settings_count_tables (cloudsync_context *data) { DEBUG_SETTINGS("dbutils_table_settings_count_tables"); - return dbutils_int_select(db, "SELECT count(*) FROM cloudsync_table_settings WHERE key='algo';"); + + int64_t count = 0; + int rc = database_select_int(data, SQL_TABLE_SETTINGS_COUNT_TABLES, &count); + return (rc == DBRES_OK) ? count : 0; } -table_algo dbutils_table_settings_get_algo (sqlite3 *db, const char *table_name) { +table_algo dbutils_table_settings_get_algo (cloudsync_context *data, const char *table_name) { DEBUG_SETTINGS("dbutils_table_settings_get_algo %s", table_name); char buffer[512]; - char *value = dbutils_table_settings_get_value(db, table_name, "*", "algo", buffer, sizeof(buffer)); - return (value) ? crdt_algo_from_name(value) : table_algo_none; + int rc = dbutils_table_settings_get_value(data, table_name, "*", "algo", buffer, sizeof(buffer)); + return (rc == DBRES_OK) ? cloudsync_algo_from_name(buffer) : table_algo_none; } int dbutils_settings_load_callback (void *xdata, int ncols, char **values, char **names) { cloudsync_context *data = (cloudsync_context *)xdata; - - for (int i=0; idata; - sqlite3 *db = context->db; + cloudsync_context *data = (cloudsync_context *)xdata; - for (int i=0; ischema_version != dbutils_schema_version(db))) { + if ((settings_exists == true) && (data->schema_version != database_schema_version(data))) { // SOMEONE CHANGED SCHEMAs SO WE NEED TO RECHECK AUGMENTED TABLES and RELATED TRIGGERS assert(0); } */ - return SQLITE_OK; -} - -int dbutils_update_schema_hash(sqlite3 *db, uint64_t *hash) { - char *schemasql = "SELECT group_concat(LOWER(sql)) FROM sqlite_master " - "WHERE type = 'table' AND name IN (SELECT tbl_name FROM cloudsync_table_settings ORDER BY tbl_name) " - "ORDER BY name;"; - char *schema = dbutils_text_select(db, schemasql); - if (!schema) return SQLITE_ERROR; - - sqlite3_uint64 h = fnv1a_hash(schema, strlen(schema)); - cloudsync_memory_free(schema); - if (hash && *hash == h) return SQLITE_CONSTRAINT; - - char sql[1024]; - snprintf(sql, sizeof(sql), "INSERT INTO cloudsync_schema_versions (hash, seq) " - "VALUES (%lld, COALESCE((SELECT MAX(seq) FROM cloudsync_schema_versions), 0) + 1) " - "ON CONFLICT(hash) DO UPDATE SET " - " seq = (SELECT COALESCE(MAX(seq), 0) + 1 FROM cloudsync_schema_versions);", (sqlite3_int64)h); - int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc == SQLITE_OK && hash) *hash = h; - return rc; + return DBRES_OK; } -sqlite3_uint64 dbutils_schema_hash (sqlite3 *db) { - DEBUG_DBFUNCTION("dbutils_schema_version"); - - return (sqlite3_uint64)dbutils_int_select(db, "SELECT hash FROM cloudsync_schema_versions ORDER BY seq DESC limit 1;"); -} - -bool dbutils_check_schema_hash (sqlite3 *db, sqlite3_uint64 hash) { - DEBUG_DBFUNCTION("dbutils_check_schema_hash"); - - // a change from the current version of the schema or from previous known schema can be applied - // a change from a newer schema version not yet applied to this peer cannot be applied - // so a schema hash is valid if it exists in the cloudsync_schema_versions table - - // the idea is to allow changes on stale peers and to be able to apply these changes on peers with newer schema, - // but it requires alter table operation on augmented tables only add new columns and never drop columns for backward compatibility - char sql[1024]; - snprintf(sql, sizeof(sql), "SELECT 1 FROM cloudsync_schema_versions WHERE hash = (%lld)", hash); - - return (dbutils_int_select(db, sql) == 1); -} - - -int dbutils_settings_cleanup (sqlite3 *db) { - const char *sql = "DROP TABLE IF EXISTS cloudsync_settings; DROP TABLE IF EXISTS cloudsync_site_id; DROP TABLE IF EXISTS cloudsync_table_settings; DROP TABLE IF EXISTS cloudsync_schema_versions; "; - return sqlite3_exec(db, sql, NULL, NULL, NULL); +int dbutils_settings_cleanup (cloudsync_context *data) { + return database_exec(data, SQL_SETTINGS_CLEANUP_DROP_ALL); } diff --git a/src/dbutils.h b/src/dbutils.h index b245f6ad..b9394fea 100644 --- a/src/dbutils.h +++ b/src/dbutils.h @@ -10,12 +10,12 @@ #include #include "utils.h" -#include "cloudsync_private.h" #define CLOUDSYNC_SETTINGS_NAME "cloudsync_settings" #define CLOUDSYNC_SITEID_NAME "cloudsync_site_id" #define CLOUDSYNC_TABLE_SETTINGS_NAME "cloudsync_table_settings" #define CLOUDSYNC_SCHEMA_VERSIONS_NAME "cloudsync_schema_versions" +#define CLOUDSYNC_PAYLOAD_FRAGMENTS_NAME "cloudsync_payload_fragments" #define CLOUDSYNC_KEY_LIBVERSION "version" #define CLOUDSYNC_KEY_SCHEMAVERSION "schemaversion" @@ -23,50 +23,29 @@ #define CLOUDSYNC_KEY_CHECK_SEQ "check_seq" #define CLOUDSYNC_KEY_SEND_DBVERSION "send_dbversion" #define CLOUDSYNC_KEY_SEND_SEQ "send_seq" +#define CLOUDSYNC_KEY_SCHEMA "schema" #define CLOUDSYNC_KEY_DEBUG "debug" #define CLOUDSYNC_KEY_ALGO "algo" - -// general -int dbutils_write_simple (sqlite3 *db, const char *sql); -int dbutils_write (sqlite3 *db, sqlite3_context *context, const char *sql, const char **values, int types[], int len[], int count); -sqlite3_int64 dbutils_int_select (sqlite3 *db, const char *sql); -char *dbutils_text_select (sqlite3 *db, const char *sql); -char *dbutils_blob_select (sqlite3 *db, const char *sql, int *size, sqlite3_context *context, int *rc); -int dbutils_blob_int_int_select (sqlite3 *db, const char *sql, char **blob, int *size, sqlite3_int64 *int1, sqlite3_int64 *int2); - -int dbutils_register_function (sqlite3 *db, const char *name, void (*ptr)(sqlite3_context*,int,sqlite3_value**), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)); -int dbutils_register_aggregate (sqlite3 *db, const char *name, void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)); - -int dbutils_debug_stmt (sqlite3 *db, bool print_result); -void dbutils_debug_values (int argc, sqlite3_value **argv); -void dbutils_debug_value (sqlite3_value *value); - -int dbutils_value_compare (sqlite3_value *v1, sqlite3_value *v2); -void dbutils_context_result_error (sqlite3_context *context, const char *format, ...); - -bool dbutils_system_exists (sqlite3 *db, const char *name, const char *type); -bool dbutils_table_exists (sqlite3 *db, const char *name); -bool dbutils_trigger_exists (sqlite3 *db, const char *name); -bool dbutils_table_sanity_check (sqlite3 *db, sqlite3_context *context, const char *name, bool skip_int_pk_check); -bool dbutils_is_star_table (const char *table_name); - -int dbutils_delete_triggers (sqlite3 *db, const char *table); -int dbutils_check_triggers (sqlite3 *db, const char *table, table_algo algo); -int dbutils_check_metatable (sqlite3 *db, const char *table, table_algo algo); -sqlite3_int64 dbutils_schema_version (sqlite3 *db); +#define CLOUDSYNC_KEY_SKIP_SCHEMA_HASH_CHECK "skip_schema_hash_check" +#define CLOUDSYNC_KEY_PAYLOAD_MAX_CHUNK_SIZE "payload_max_chunk_size" // settings -int dbutils_settings_cleanup (sqlite3 *db); -int dbutils_settings_init (sqlite3 *db, void *cloudsync_data, sqlite3_context *context); -int dbutils_settings_set_key_value (sqlite3 *db, sqlite3_context *context, const char *key, const char *value); -int dbutils_settings_get_int_value (sqlite3 *db, const char *key); -char *dbutils_settings_get_value (sqlite3 *db, const char *key, char *buffer, size_t blen); -int dbutils_table_settings_set_key_value (sqlite3 *db, sqlite3_context *context, const char *table, const char *column, const char *key, const char *value); -sqlite3_int64 dbutils_table_settings_count_tables (sqlite3 *db); -char *dbutils_table_settings_get_value (sqlite3 *db, const char *table, const char *column, const char *key, char *buffer, size_t blen); -table_algo dbutils_table_settings_get_algo (sqlite3 *db, const char *table_name); -int dbutils_update_schema_hash(sqlite3 *db, uint64_t *hash); -sqlite3_uint64 dbutils_schema_hash (sqlite3 *db); -bool dbutils_check_schema_hash (sqlite3 *db, sqlite3_uint64 hash); +int dbutils_settings_init (cloudsync_context *data); +int dbutils_settings_cleanup (cloudsync_context *data); +int dbutils_settings_check_version (cloudsync_context *data, const char *version); +int dbutils_settings_set_key_value (cloudsync_context *data, const char *key, const char *value); +int dbutils_settings_get_int_value (cloudsync_context *data, const char *key); +int64_t dbutils_settings_get_int64_value (cloudsync_context *data, const char *key); + +// table settings +int dbutils_table_settings_set_key_value (cloudsync_context *data, const char *table_name, const char *column_name, const char *key, const char *value); +int64_t dbutils_table_settings_count_tables (cloudsync_context *data); +int dbutils_table_settings_get_value (cloudsync_context *data, const char *table_name, const char *column_name, const char *key, char *buffer, size_t blen); +table_algo dbutils_table_settings_get_algo (cloudsync_context *data, const char *table_name); + +// others +void dbutils_debug_values (dbvalue_t **argv, int argc); +void dbutils_debug_value (dbvalue_t *value); +int dbutils_value_compare (dbvalue_t *v1, dbvalue_t *v2); #endif diff --git a/src/jsmn.h b/src/jsmn.h new file mode 100644 index 00000000..dca2bb59 --- /dev/null +++ b/src/jsmn.h @@ -0,0 +1,471 @@ +/* + * MIT License + * + * Copyright (c) 2010 Serge Zaitsev + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef JSMN_H +#define JSMN_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef JSMN_STATIC +#define JSMN_API static +#else +#define JSMN_API extern +#endif + +/** + * JSON type identifier. Basic types are: + * o Object + * o Array + * o String + * o Other primitive: number, boolean (true/false) or null + */ +typedef enum { + JSMN_UNDEFINED = 0, + JSMN_OBJECT = 1 << 0, + JSMN_ARRAY = 1 << 1, + JSMN_STRING = 1 << 2, + JSMN_PRIMITIVE = 1 << 3 +} jsmntype_t; + +enum jsmnerr { + /* Not enough tokens were provided */ + JSMN_ERROR_NOMEM = -1, + /* Invalid character inside JSON string */ + JSMN_ERROR_INVAL = -2, + /* The string is not a full JSON packet, more bytes expected */ + JSMN_ERROR_PART = -3 +}; + +/** + * JSON token description. + * type type (object, array, string etc.) + * start start position in JSON data string + * end end position in JSON data string + */ +typedef struct jsmntok { + jsmntype_t type; + int start; + int end; + int size; +#ifdef JSMN_PARENT_LINKS + int parent; +#endif +} jsmntok_t; + +/** + * JSON parser. Contains an array of token blocks available. Also stores + * the string being parsed now and current position in that string. + */ +typedef struct jsmn_parser { + unsigned int pos; /* offset in the JSON string */ + unsigned int toknext; /* next token to allocate */ + int toksuper; /* superior token node, e.g. parent object or array */ +} jsmn_parser; + +/** + * Create JSON parser over an array of tokens + */ +JSMN_API void jsmn_init(jsmn_parser *parser); + +/** + * Run JSON parser. It parses a JSON data string into and array of tokens, each + * describing + * a single JSON object. + */ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, + jsmntok_t *tokens, const unsigned int num_tokens); + +#ifndef JSMN_HEADER +/** + * Allocates a fresh unused token from the token pool. + */ +static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *tok; + if (parser->toknext >= num_tokens) { + return NULL; + } + tok = &tokens[parser->toknext++]; + tok->start = tok->end = -1; + tok->size = 0; +#ifdef JSMN_PARENT_LINKS + tok->parent = -1; +#endif + return tok; +} + +/** + * Fills token type and boundaries. + */ +static void jsmn_fill_token(jsmntok_t *token, const jsmntype_t type, + const int start, const int end) { + token->type = type; + token->start = start; + token->end = end; + token->size = 0; +} + +/** + * Fills next available token with JSON primitive. + */ +static int jsmn_parse_primitive(jsmn_parser *parser, const char *js, + const size_t len, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *token; + int start; + + start = parser->pos; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + switch (js[parser->pos]) { +#ifndef JSMN_STRICT + /* In strict mode primitive must be followed by "," or "}" or "]" */ + case ':': +#endif + case '\t': + case '\r': + case '\n': + case ' ': + case ',': + case ']': + case '}': + goto found; + default: + /* to quiet a warning from gcc*/ + break; + } + if (js[parser->pos] < 32 || js[parser->pos] >= 127) { + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } +#ifdef JSMN_STRICT + /* In strict mode primitive must be followed by a comma/object/array */ + parser->pos = start; + return JSMN_ERROR_PART; +#endif + +found: + if (tokens == NULL) { + parser->pos--; + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + parser->pos--; + return 0; +} + +/** + * Fills next token with JSON string. + */ +static int jsmn_parse_string(jsmn_parser *parser, const char *js, + const size_t len, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *token; + + int start = parser->pos; + + /* Skip starting quote */ + parser->pos++; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c = js[parser->pos]; + + /* Quote: end of string */ + if (c == '\"') { + if (tokens == NULL) { + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_STRING, start + 1, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + return 0; + } + + /* Backslash: Quoted symbol expected */ + if (c == '\\' && parser->pos + 1 < len) { + int i; + parser->pos++; + switch (js[parser->pos]) { + /* Allowed escaped symbols */ + case '\"': + case '/': + case '\\': + case 'b': + case 'f': + case 'r': + case 'n': + case 't': + break; + /* Allows escaped symbol \uXXXX */ + case 'u': + parser->pos++; + for (i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0'; + i++) { + /* If it isn't a hex character we have an error */ + if (!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */ + (js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */ + (js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */ + parser->pos = start; + return JSMN_ERROR_INVAL; + } + parser->pos++; + } + parser->pos--; + break; + /* Unexpected symbol */ + default: + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } + } + parser->pos = start; + return JSMN_ERROR_PART; +} + +/** + * Parse JSON string and fill tokens. + */ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, + jsmntok_t *tokens, const unsigned int num_tokens) { + int r; + int i; + jsmntok_t *token; + int count = parser->toknext; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c; + jsmntype_t type; + + c = js[parser->pos]; + switch (c) { + case '{': + case '[': + count++; + if (tokens == NULL) { + break; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + return JSMN_ERROR_NOMEM; + } + if (parser->toksuper != -1) { + jsmntok_t *t = &tokens[parser->toksuper]; +#ifdef JSMN_STRICT + /* In strict mode an object or array can't become a key */ + if (t->type == JSMN_OBJECT) { + return JSMN_ERROR_INVAL; + } +#endif + t->size++; +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + } + token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY); + token->start = parser->pos; + parser->toksuper = parser->toknext - 1; + break; + case '}': + case ']': + if (tokens == NULL) { + break; + } + type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY); +#ifdef JSMN_PARENT_LINKS + if (parser->toknext < 1) { + return JSMN_ERROR_INVAL; + } + token = &tokens[parser->toknext - 1]; + for (;;) { + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + token->end = parser->pos + 1; + parser->toksuper = token->parent; + break; + } + if (token->parent == -1) { + if (token->type != type || parser->toksuper == -1) { + return JSMN_ERROR_INVAL; + } + break; + } + token = &tokens[token->parent]; + } +#else + for (i = parser->toknext - 1; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + parser->toksuper = -1; + token->end = parser->pos + 1; + break; + } + } + /* Error if unmatched closing bracket */ + if (i == -1) { + return JSMN_ERROR_INVAL; + } + for (; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + parser->toksuper = i; + break; + } + } +#endif + break; + case '\"': + r = jsmn_parse_string(parser, js, len, tokens, num_tokens); + if (r < 0) { + return r; + } + count++; + if (parser->toksuper != -1 && tokens != NULL) { + tokens[parser->toksuper].size++; + } + break; + case '\t': + case '\r': + case '\n': + case ' ': + break; + case ':': + parser->toksuper = parser->toknext - 1; + break; + case ',': + if (tokens != NULL && parser->toksuper != -1 && + tokens[parser->toksuper].type != JSMN_ARRAY && + tokens[parser->toksuper].type != JSMN_OBJECT) { +#ifdef JSMN_PARENT_LINKS + parser->toksuper = tokens[parser->toksuper].parent; +#else + for (i = parser->toknext - 1; i >= 0; i--) { + if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) { + if (tokens[i].start != -1 && tokens[i].end == -1) { + parser->toksuper = i; + break; + } + } + } +#endif + } + break; +#ifdef JSMN_STRICT + /* In strict mode primitives are: numbers and booleans */ + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 't': + case 'f': + case 'n': + /* And they must not be keys of the object */ + if (tokens != NULL && parser->toksuper != -1) { + const jsmntok_t *t = &tokens[parser->toksuper]; + if (t->type == JSMN_OBJECT || + (t->type == JSMN_STRING && t->size != 0)) { + return JSMN_ERROR_INVAL; + } + } +#else + /* In non-strict mode every unquoted value is a primitive */ + default: +#endif + r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens); + if (r < 0) { + return r; + } + count++; + if (parser->toksuper != -1 && tokens != NULL) { + tokens[parser->toksuper].size++; + } + break; + +#ifdef JSMN_STRICT + /* Unexpected char in strict mode */ + default: + return JSMN_ERROR_INVAL; +#endif + } + } + + if (tokens != NULL) { + for (i = parser->toknext - 1; i >= 0; i--) { + /* Unmatched opened object or array */ + if (tokens[i].start != -1 && tokens[i].end == -1) { + return JSMN_ERROR_PART; + } + } + } + + return count; +} + +/** + * Creates a new parser based over a given buffer with an array of tokens + * available. + */ +JSMN_API void jsmn_init(jsmn_parser *parser) { + parser->pos = 0; + parser->toknext = 0; + parser->toksuper = -1; +} + +#endif /* JSMN_HEADER */ + +#ifdef __cplusplus +} +#endif + +#endif /* JSMN_H */ diff --git a/src/network.c b/src/network.c deleted file mode 100644 index 37fe1fff..00000000 --- a/src/network.c +++ /dev/null @@ -1,959 +0,0 @@ -// -// network.c -// cloudsync -// -// Created by Marco Bambini on 12/12/24. -// - -#ifndef CLOUDSYNC_OMIT_NETWORK - -#include -#include "network.h" -#include "dbutils.h" -#include "utils.h" -#include "cloudsync_private.h" -#include "network_private.h" - -#ifndef SQLITE_WASM_EXTRA_INIT -#ifndef CLOUDSYNC_OMIT_CURL -#include "curl/curl.h" -#endif -#else -#define curl_free(x) free(x) -char *substr(const char *start, const char *end); -#endif - -#ifdef __ANDROID__ -#include "cacert.h" -static size_t cacert_len = sizeof(cacert_pem) - 1; -#endif - -#define CLOUDSYNC_NETWORK_MINBUF_SIZE 512 -#define CLOUDSYNC_SESSION_TOKEN_MAXSIZE 4096 - -#define DEFAULT_SYNC_WAIT_MS 100 -#define DEFAULT_SYNC_MAX_RETRIES 1 - -#define MAX_QUERY_VALUE_LEN 256 - -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT3 -#endif - -// MARK: - - -struct network_data { - char site_id[UUID_STR_MAXLEN]; - char *authentication; // apikey or token - char *check_endpoint; - char *upload_endpoint; -}; - -typedef struct { - char *buffer; - size_t balloc; - size_t bused; - int zero_term; -} network_buffer; - - -typedef struct { - const char *data; - size_t size; - size_t read_pos; -} network_read_data; - -// MARK: - - -void network_result_cleanup (NETWORK_RESULT *res) { - if (res->xfree) { - res->xfree(res->xdata); - } else if (res->buffer) { - cloudsync_memory_free(res->buffer); - } -} - -char *network_data_get_siteid (network_data *data) { - return data->site_id; -} - -bool network_data_set_endpoints (network_data *data, char *auth, char *check, char *upload, bool duplicate) { - if (duplicate) { - // auth is optional - char *s1 = (auth) ? cloudsync_string_dup(auth, false) : NULL; - if (auth && !s1) return false; - char *s2 = cloudsync_string_dup(check, false); - if (!s2) {if (auth && s1) sqlite3_free(s1); return false;} - char *s3 = cloudsync_string_dup(upload, false); - if (!s3) {if (auth && s1) sqlite3_free(s1); sqlite3_free(s2); return false;} - - auth = s1; - check = s2; - upload = s3; - } - - data->authentication = auth; - data->check_endpoint = check; - data->upload_endpoint = upload; - return true; -} - -void network_data_free (network_data *data) { - if (!data) return; - - if (data->authentication) cloudsync_memory_free(data->authentication); - if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); - if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); - cloudsync_memory_free(data); -} - -// MARK: - Utils - - -#ifndef CLOUDSYNC_OMIT_CURL -static bool network_buffer_check (network_buffer *data, size_t needed) { - // alloc/resize buffer - if (data->bused + needed > data->balloc) { - if (needed < CLOUDSYNC_NETWORK_MINBUF_SIZE) needed = CLOUDSYNC_NETWORK_MINBUF_SIZE; - size_t balloc = data->balloc + needed; - - char *buffer = cloudsync_memory_realloc(data->buffer, balloc); - if (!buffer) return false; - - data->buffer = buffer; - data->balloc = balloc; - } - - return true; -} - -static size_t network_receive_callback (void *ptr, size_t size, size_t nmemb, void *xdata) { - network_buffer *data = (network_buffer *)xdata; - - size_t ptr_size = (size*nmemb); - if (data->zero_term) ptr_size += 1; - - if (network_buffer_check(data, ptr_size) == false) return -1; - memcpy(data->buffer+data->bused, ptr, size*nmemb); - data->bused += size*nmemb; - if (data->zero_term) data->buffer[data->bused] = 0; - - return (size * nmemb); -} - -NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header) { - char *buffer = NULL; - size_t blen = 0; - struct curl_slist* headers = NULL; - char errbuf[CURL_ERROR_SIZE] = {0}; - long response_code = 0; - - CURL *curl = curl_easy_init(); - if (!curl) return (NETWORK_RESULT){CLOUDSYNC_NETWORK_ERROR, NULL, 0, NULL, NULL}; - - // a buffer to store errors in - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); - - CURLcode rc = curl_easy_setopt(curl, CURLOPT_URL, endpoint); - if (rc != CURLE_OK) goto cleanup; - - // set PEM - #ifdef __ANDROID__ - struct curl_blob pem_blob = { - .data = (void *)cacert_pem, - .len = cacert_len, - .flags = CURL_BLOB_NOCOPY - }; - curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &pem_blob); - #endif - - if (custom_header) headers = curl_slist_append(headers, custom_header); - - if (authentication) { - char auth_header[CLOUDSYNC_SESSION_TOKEN_MAXSIZE]; - snprintf(auth_header, sizeof(auth_header), "Authorization: Bearer %s", authentication); - headers = curl_slist_append(headers, auth_header); - - if (json_payload) headers = curl_slist_append(headers, "Content-Type: application/json"); - } - - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - network_buffer netdata = {NULL, 0, 0, (zero_terminated) ? 1 : 0}; - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &netdata); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, network_receive_callback); - - // add optional JSON payload (implies setting CURLOPT_POST to 1) - // or set the CURLOPT_POST option - if (json_payload) { - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_payload); - } else if (is_post_request) { - curl_easy_setopt(curl, CURLOPT_POST, 1L); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, 0L); - } - - // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); - rc = curl_easy_perform(curl); - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); - if (rc == CURLE_OK) { - buffer = netdata.buffer; - blen = netdata.bused; - } else if (netdata.buffer) { - cloudsync_memory_free(netdata.buffer); - netdata.buffer = NULL; - } - -cleanup: - if (curl) curl_easy_cleanup(curl); - if (headers) curl_slist_free_all(headers); - - // build result - NETWORK_RESULT result = {0, NULL, 0, NULL, NULL}; - if (rc == CURLE_OK && response_code < 400) { - result.code = (buffer && blen) ? CLOUDSYNC_NETWORK_BUFFER : CLOUDSYNC_NETWORK_OK; - result.buffer = buffer; - result.blen = blen; - } else { - result.code = CLOUDSYNC_NETWORK_ERROR; - result.buffer = buffer ? buffer : (errbuf[0]) ? cloudsync_string_dup(errbuf, false) : NULL; - result.blen = buffer ? blen : rc; - } - - return result; -} - -static size_t network_read_callback(char *buffer, size_t size, size_t nitems, void *userdata) { - network_read_data *rd = (network_read_data *)userdata; - size_t max_read = size * nitems; - size_t bytes_left = rd->size - rd->read_pos; - size_t to_copy = bytes_left < max_read ? bytes_left : max_read; - - if (to_copy > 0) { - memcpy(buffer, rd->data + rd->read_pos, to_copy); - rd->read_pos += to_copy; - } - - return to_copy; -} - -bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { - struct curl_slist *headers = NULL; - curl_mime *mime = NULL; - bool result = false; - char errbuf[CURL_ERROR_SIZE] = {0}; - - // init curl - CURL *curl = curl_easy_init(); - if (!curl) return false; - - // set the URL - if (curl_easy_setopt(curl, CURLOPT_URL, endpoint) != CURLE_OK) goto cleanup; - - // set PEM - #ifdef __ANDROID__ - struct curl_blob pem_blob = { - .data = (void *)cacert_pem, - .len = cacert_len, - .flags = CURL_BLOB_NOCOPY - }; - curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &pem_blob); - #endif - - // a buffer to store errors in - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); - curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); - - // type header - headers = curl_slist_append(headers, "Accept: text/plain"); - - if (authentication) { - // init authorization header - char auth_header[CLOUDSYNC_SESSION_TOKEN_MAXSIZE]; - snprintf(auth_header, sizeof(auth_header), "Authorization: Bearer %s", data->authentication); - headers = curl_slist_append(headers, auth_header); - } - - // Set headers if needed (S3 pre-signed URLs usually do not require additional headers) - headers = curl_slist_append(headers, "Content-Type: application/octet-stream"); - - if (!headers) goto cleanup; - if (curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers) != CURLE_OK) goto cleanup; - - // Set HTTP PUT method - curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); - - // Set the size of the blob - curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)blob_size); - - // Provide the data using a custom read callback - network_read_data rdata = { - .data = (const char *)blob, - .size = blob_size, - .read_pos = 0 - }; - - curl_easy_setopt(curl, CURLOPT_READFUNCTION, network_read_callback); - curl_easy_setopt(curl, CURLOPT_READDATA, &rdata); - - // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); - - // perform the upload - CURLcode rc = curl_easy_perform(curl); - if (rc == CURLE_OK) result = true; - -cleanup: - if (mime) curl_mime_free(mime); - if (curl) curl_easy_cleanup(curl); - if (headers) curl_slist_free_all(headers); - return result; -} -#endif - -int network_set_sqlite_result (sqlite3_context *context, NETWORK_RESULT *result) { - int rc = 0; - switch (result->code) { - case CLOUDSYNC_NETWORK_OK: - sqlite3_result_error_code(context, SQLITE_OK); - sqlite3_result_int(context, 0); - rc = 0; - break; - - case CLOUDSYNC_NETWORK_ERROR: - sqlite3_result_error(context, (result->buffer) ? result->buffer : "Memory error.", -1); - sqlite3_result_error_code(context, SQLITE_ERROR); - rc = -1; - break; - - case CLOUDSYNC_NETWORK_BUFFER: - sqlite3_result_error_code(context, SQLITE_OK); - sqlite3_result_text(context, result->buffer, (int)result->blen, SQLITE_TRANSIENT); - rc = (int)result->blen; - break; - } - - return rc; -} - -int network_download_changes (sqlite3_context *context, const char *download_url) { - DEBUG_FUNCTION("network_download_changes"); - - network_data *data = (network_data *)cloudsync_get_auxdata(context); - if (!data) { - sqlite3_result_error(context, "Unable to retrieve CloudSync context.", -1); - return -1; - } - - NETWORK_RESULT result = network_receive_buffer(data, download_url, NULL, false, false, NULL, NULL); - - int rc = SQLITE_OK; - if (result.code == CLOUDSYNC_NETWORK_BUFFER) { - rc = cloudsync_payload_apply(context, result.buffer, (int)result.blen); - } else { - rc = network_set_sqlite_result(context, &result); - } - network_result_cleanup(&result); - - return rc; -} - -char *network_authentication_token(const char *key, const char *value) { - size_t len = strlen(key) + strlen(value) + 64; - char *buffer = cloudsync_memory_zeroalloc(len); - if (!buffer) return NULL; - - // build new token - // we don't need a prefix because the token alreay include a prefix "sqa_" - snprintf(buffer, len, "%s", value); - - return buffer; -} - -int network_extract_query_param(const char *query, const char *key, char *output, size_t output_size) { - if (!query || !key || !output || output_size == 0) { - return -1; // Invalid input - } - - size_t key_len = strlen(key); - const char *p = query; - #ifdef SQLITE_WASM_EXTRA_INIT - if (*p == '?') p++; - #endif - - while (p && *p) { - // Find the start of a key=value pair - const char *key_start = p; - const char *eq = strchr(key_start, '='); - if (!eq) break; // No '=' found, malformed query string - - size_t current_key_len = eq - key_start; - - // Check if the key matches (ensuring it's the full key) - if (current_key_len == key_len && strncmp(key_start, key, key_len) == 0) { - // Extract the value - const char *value_start = eq + 1; - const char *end = strchr(value_start, '&'); // Find end of value - - size_t value_len = (end) ? (size_t)(end - value_start) : strlen(value_start); - if (value_len >= output_size) { - return -2; // Output buffer too small - } - - strncpy(output, value_start, value_len); - output[value_len] = '\0'; // Null-terminate - return 0; // Success - } - - // Move to the next parameter - p = strchr(p, '&'); - if (p) p++; // Skip '&' - } - - return -3; // Key not found -} - -#if !defined(CLOUDSYNC_OMIT_CURL) || defined(SQLITE_WASM_EXTRA_INIT) -bool network_compute_endpoints (sqlite3_context *context, network_data *data, const char *conn_string) { - // compute endpoints - bool result = false; - - char *scheme = NULL; - char *host = NULL; - char *port = NULL; - char *database = NULL; - char *query = NULL; - - char *authentication = NULL; - char *check_endpoint = NULL; - char *upload_endpoint = NULL; - - char *conn_string_https = NULL; - - #ifndef SQLITE_WASM_EXTRA_INIT - CURLUcode rc = CURLUE_OUT_OF_MEMORY; - CURLU *url = curl_url(); - if (!url) goto finalize; - #endif - - conn_string_https = cloudsync_string_replace_prefix(conn_string, "sqlitecloud://", "https://"); - - #ifndef SQLITE_WASM_EXTRA_INIT - // set URL: https://UUID.g5.sqlite.cloud:443/chinook.sqlite?apikey=hWDanFolRT9WDK0p54lufNrIyfgLZgtMw6tb6fbPmpo - rc = curl_url_set(url, CURLUPART_URL, conn_string_https, 0); - if (rc != CURLUE_OK) goto finalize; - - // https (MANDATORY) - rc = curl_url_get(url, CURLUPART_SCHEME, &scheme, 0); - if (rc != CURLUE_OK) goto finalize; - - // UUID.g5.sqlite.cloud (MANDATORY) - rc = curl_url_get(url, CURLUPART_HOST, &host, 0); - if (rc != CURLUE_OK) goto finalize; - - // 443 (OPTIONAL) - rc = curl_url_get(url, CURLUPART_PORT, &port, 0); - if (rc != CURLUE_OK && rc != CURLUE_NO_PORT) goto finalize; - char *port_or_default = port && strcmp(port, "8860") != 0 ? port : CLOUDSYNC_DEFAULT_ENDPOINT_PORT; - - // /chinook.sqlite (MANDATORY) - rc = curl_url_get(url, CURLUPART_PATH, &database, 0); - if (rc != CURLUE_OK) goto finalize; - - // apikey=hWDanFolRT9WDK0p54lufNrIyfgLZgtMw6tb6fbPmpo (OPTIONAL) - rc = curl_url_get(url, CURLUPART_QUERY, &query, 0); - if (rc != CURLUE_OK && rc != CURLUE_NO_QUERY) goto finalize; - #else - // Parse: scheme://host[:port]/path?query - const char *p = strstr(conn_string_https, "://"); - if (!p) goto finalize; - scheme = substr(conn_string_https, p); - p += 3; - const char *host_start = p; - const char *host_end = strpbrk(host_start, ":/?"); - if (!host_end) goto finalize; - host = substr(host_start, host_end); - p = host_end; - if (*p == ':') { - ++p; - const char *port_end = strpbrk(p, "/?"); - if (!port_end) goto finalize; - port = substr(p, port_end); - p = port_end; - } - if (*p == '/') { - const char *path_start = p; - const char *path_end = strchr(path_start, '?'); - if (!path_end) path_end = path_start + strlen(path_start); - database = substr(path_start, path_end); - p = path_end; - } - if (*p == '?') { - query = strdup(p); - } - if (!scheme || !host || !database) goto finalize; - char *port_or_default = port && strcmp(port, "8860") != 0 ? port : CLOUDSYNC_DEFAULT_ENDPOINT_PORT; - #endif - - if (query != NULL) { - char value[MAX_QUERY_VALUE_LEN]; - if (!authentication && network_extract_query_param(query, "apikey", value, sizeof(value)) == 0) { - authentication = network_authentication_token("apikey", value); - } - if (!authentication && network_extract_query_param(query, "token", value, sizeof(value)) == 0) { - authentication = network_authentication_token("token", value); - } - } - - size_t requested = strlen(scheme) + strlen(host) + strlen(port_or_default) + strlen(CLOUDSYNC_ENDPOINT_PREFIX) + strlen(database) + 64; - check_endpoint = (char *)cloudsync_memory_zeroalloc(requested); - upload_endpoint = (char *)cloudsync_memory_zeroalloc(requested); - if ((!upload_endpoint) || (!check_endpoint)) goto finalize; - - snprintf(check_endpoint, requested, "%s://%s:%s/%s%s/%s", scheme, host, port_or_default, CLOUDSYNC_ENDPOINT_PREFIX, database, data->site_id); - snprintf(upload_endpoint, requested, "%s://%s:%s/%s%s/%s/%s", scheme, host, port_or_default, CLOUDSYNC_ENDPOINT_PREFIX, database, data->site_id, CLOUDSYNC_ENDPOINT_UPLOAD); - - result = true; - -finalize: - if (result == false) { - // store proper result code/message - #ifndef SQLITE_WASM_EXTRA_INIT - if (rc != CURLUE_OK) sqlite3_result_error(context, curl_url_strerror(rc), -1); - sqlite3_result_error_code(context, (rc != CURLUE_OK) ? SQLITE_ERROR : SQLITE_NOMEM); - #else - sqlite3_result_error(context, "URL parse error", -1); - sqlite3_result_error_code(context, SQLITE_ERROR); - #endif - - // cleanup memory managed by the extension - if (authentication) cloudsync_memory_free(authentication); - if (check_endpoint) cloudsync_memory_free(check_endpoint); - if (upload_endpoint) cloudsync_memory_free(upload_endpoint); - } - - if (result) { - if (authentication) { - if (data->authentication) cloudsync_memory_free(data->authentication); - data->authentication = authentication; - } - - if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); - data->check_endpoint = check_endpoint; - - if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); - data->upload_endpoint = upload_endpoint; - } - - // cleanup memory - #ifndef SQLITE_WASM_EXTRA_INIT - if (url) curl_url_cleanup(url); - #endif - if (scheme) curl_free(scheme); - if (host) curl_free(host); - if (port) curl_free(port); - if (database) curl_free(database); - if (query) curl_free(query); - if (conn_string_https && conn_string_https != conn_string) cloudsync_memory_free(conn_string_https); - - return result; -} -#endif - -void network_result_to_sqlite_error (sqlite3_context *context, NETWORK_RESULT res, const char *default_error_message) { - sqlite3_result_error(context, ((res.code == CLOUDSYNC_NETWORK_ERROR) && (res.buffer)) ? res.buffer : default_error_message, -1); - sqlite3_result_error_code(context, SQLITE_ERROR); -} - -// MARK: - Init / Cleanup - - -network_data *cloudsync_network_data(sqlite3_context *context) { - network_data *data = (network_data *)cloudsync_get_auxdata(context); - if (data) return data; - - data = (network_data *)cloudsync_memory_zeroalloc(sizeof(network_data)); - if (data) cloudsync_set_auxdata(context, data); - return data; -} - -void cloudsync_network_init (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_init"); - - #ifndef CLOUDSYNC_OMIT_CURL - curl_global_init(CURL_GLOBAL_ALL); - #endif - - // no real network operations here - // just setup the network_data struct - network_data *data = cloudsync_network_data(context); - if (!data) goto abort_memory; - - // init context - uint8_t *site_id = (uint8_t *)cloudsync_context_init(sqlite3_context_db_handle(context), NULL, context); - if (!site_id) goto abort_siteid; - - // save site_id string representation: 01957493c6c07e14803727e969f1d2cc - cloudsync_uuid_v7_stringify(site_id, data->site_id, false); - - // connection string is something like: - // https://UUID.g5.sqlite.cloud:443/chinook.sqlite?apikey=hWDanFolRT9WDK0p54lufNrIyfgLZgtMw6tb6fbPmpo - // or https://UUID.g5.sqlite.cloud:443/chinook.sqlite - // apikey part is optional and can be replaced by a session token once client is authenticated - - const char *connection_param = (const char *)sqlite3_value_text(argv[0]); - - // compute endpoints - if (network_compute_endpoints(context, data, connection_param) == false) { - // error message/code already set inside network_compute_endpoints - goto abort_cleanup; - } - - cloudsync_set_auxdata(context, data); - sqlite3_result_int(context, SQLITE_OK); - return; - -abort_memory: - dbutils_context_result_error(context, "Unable to allocate memory in cloudsync_network_init."); - sqlite3_result_error_code(context, SQLITE_NOMEM); - goto abort_cleanup; - -abort_siteid: - dbutils_context_result_error(context, "Unable to compute/retrieve site_id."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - goto abort_cleanup; - -abort_cleanup: - cloudsync_set_auxdata(context, NULL); - network_data_free(data); -} - -void cloudsync_network_cleanup (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_cleanup"); - - network_data *data = (network_data *)cloudsync_get_auxdata(context); - cloudsync_set_auxdata(context, NULL); - network_data_free(data); - sqlite3_result_int(context, SQLITE_OK); - - #ifndef CLOUDSYNC_OMIT_CURL - curl_global_cleanup(); - #endif -} - -// MARK: - Public - - -bool cloudsync_network_set_authentication_token (sqlite3_context *context, const char *value, bool is_token) { - network_data *data = cloudsync_network_data(context); - if (!data) return false; - - const char *key = (is_token) ? "token" : "apikey"; - char *new_auth_token = network_authentication_token(key, value); - if (!new_auth_token) return false; - - if (data->authentication) cloudsync_memory_free(data->authentication); - data->authentication = new_auth_token; - - return true; -} - -void cloudsync_network_set_token (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_set_token"); - - const char *value = (const char *)sqlite3_value_text(argv[0]); - bool result = cloudsync_network_set_authentication_token(context, value, true); - (result) ? sqlite3_result_int(context, SQLITE_OK) : sqlite3_result_error_code(context, SQLITE_NOMEM); -} - -void cloudsync_network_set_apikey (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_set_apikey"); - - const char *value = (const char *)sqlite3_value_text(argv[0]); - bool result = cloudsync_network_set_authentication_token(context, value, false); - (result) ? sqlite3_result_int(context, SQLITE_OK) : sqlite3_result_error_code(context, SQLITE_NOMEM); -} - -// MARK: - - -void cloudsync_network_has_unsent_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { - sqlite3 *db = sqlite3_context_db_handle(context); - - char *sql = "SELECT max(db_version), hex(site_id) FROM cloudsync_changes WHERE site_id == (SELECT site_id FROM cloudsync_site_id WHERE rowid=0)"; - int last_local_change = (int)dbutils_int_select(db, sql); - if (last_local_change == 0) { - sqlite3_result_int(context, 0); - return; - } - - int sent_db_version = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_SEND_DBVERSION); - sqlite3_result_int(context, (sent_db_version < last_local_change)); -} - -int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_send_changes"); - - network_data *data = (network_data *)cloudsync_get_auxdata(context); - if (!data) {sqlite3_result_error(context, "Unable to retrieve CloudSync context.", -1); return SQLITE_ERROR;} - - // retrieve payload - char *blob = NULL; - int blob_size = 0, db_version = 0, seq = 0; - sqlite3_int64 new_db_version = 0, new_seq = 0; - int rc = cloudsync_payload_get(context, &blob, &blob_size, &db_version, &seq, &new_db_version, &new_seq); - if (rc != SQLITE_OK) return rc; - - // exit if there is no data to send - if (blob == NULL || blob_size == 0) return SQLITE_OK; - - NETWORK_RESULT res = network_receive_buffer(data, data->upload_endpoint, data->authentication, true, false, NULL, CLOUDSYNC_HEADER_SQLITECLOUD); - if (res.code != CLOUDSYNC_NETWORK_BUFFER) { - cloudsync_memory_free(blob); - network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to receive upload URL"); - network_result_cleanup(&res); - return SQLITE_ERROR; - } - - const char *s3_url = res.buffer; - bool sent = network_send_buffer(data, s3_url, NULL, blob, blob_size); - cloudsync_memory_free(blob); - if (sent == false) { - network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to upload BLOB changes to remote host."); - network_result_cleanup(&res); - return SQLITE_ERROR; - } - - char json_payload[2024]; - snprintf(json_payload, sizeof(json_payload), "{\"url\":\"%s\"}", s3_url); - - // free res - network_result_cleanup(&res); - - // notify remote host that we succesfully uploaded changes - res = network_receive_buffer(data, data->upload_endpoint, data->authentication, true, true, json_payload, CLOUDSYNC_HEADER_SQLITECLOUD); - if (res.code != CLOUDSYNC_NETWORK_OK) { - network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to notify BLOB upload to remote host."); - network_result_cleanup(&res); - return SQLITE_ERROR; - } - - // update db_version and seq - char buf[256]; - sqlite3 *db = sqlite3_context_db_handle(context); - if (new_db_version != db_version) { - snprintf(buf, sizeof(buf), "%lld", new_db_version); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_DBVERSION, buf); - } - if (new_seq != seq) { - snprintf(buf, sizeof(buf), "%lld", new_seq); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_SEQ, buf); - } - - network_result_cleanup(&res); - return SQLITE_OK; -} - -void cloudsync_network_send_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_send_changes"); - - cloudsync_network_send_changes_internal(context, argc, argv); -} - -int cloudsync_network_check_internal(sqlite3_context *context) { - network_data *data = (network_data *)cloudsync_get_auxdata(context); - if (!data) {sqlite3_result_error(context, "Unable to retrieve CloudSync context.", -1); return -1;} - - sqlite3 *db = sqlite3_context_db_handle(context); - - int db_version = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_CHECK_DBVERSION); - if (db_version<0) {sqlite3_result_error(context, "Unable to retrieve db_version.", -1); return -1;} - - int seq = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_CHECK_SEQ); - if (seq<0) {sqlite3_result_error(context, "Unable to retrieve seq.", -1); return -1;} - - // http://uuid.g5.sqlite.cloud/v1/cloudsync/{dbname}/{site_id}/{db_version}/{seq}/check - // the data->check_endpoint stops after {site_id}, just need to append /{db_version}/{seq}/check - char endpoint[2024]; - snprintf(endpoint, sizeof(endpoint), "%s/%lld/%d/%s", data->check_endpoint, (long long)db_version, seq, CLOUDSYNC_ENDPOINT_CHECK); - - NETWORK_RESULT result = network_receive_buffer(data, endpoint, data->authentication, true, true, NULL, CLOUDSYNC_HEADER_SQLITECLOUD); - int rc = SQLITE_OK; - if (result.code == CLOUDSYNC_NETWORK_BUFFER) { - rc = network_download_changes(context, result.buffer); - } else { - rc = network_set_sqlite_result(context, &result); - } - - network_result_cleanup(&result); - return rc; -} - -void cloudsync_network_sync (sqlite3_context *context, int wait_ms, int max_retries) { - int rc = cloudsync_network_send_changes_internal(context, 0, NULL); - if (rc != SQLITE_OK) return; - - int ntries = 0; - int nrows = 0; - while (ntries < max_retries) { - if (ntries > 0) sqlite3_sleep(wait_ms); - nrows = cloudsync_network_check_internal(context); - if (nrows > 0) break; - ntries++; - } - - sqlite3_result_error_code(context, (nrows == -1) ? SQLITE_ERROR : SQLITE_OK); - if (nrows >= 0) sqlite3_result_int(context, nrows); -} - -void cloudsync_network_sync0 (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_sync2"); - - cloudsync_network_sync(context, DEFAULT_SYNC_WAIT_MS, DEFAULT_SYNC_MAX_RETRIES); -} - - -void cloudsync_network_sync2 (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_sync2"); - - int wait_ms = sqlite3_value_int(argv[0]); - int max_retries = sqlite3_value_int(argv[1]); - - cloudsync_network_sync(context, wait_ms, max_retries); -} - - -void cloudsync_network_check_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_check_changes"); - - cloudsync_network_check_internal(context); -} - -void cloudsync_network_reset_sync_version (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_reset_sync_version"); - - sqlite3 *db = sqlite3_context_db_handle(context); - char *buf = "0"; - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_CHECK_DBVERSION, buf); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_CHECK_SEQ, buf); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_DBVERSION, buf); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_SEQ, buf); -} - -/** - * Cleanup all local data from cloudsync-enabled tables, so the database can be safely reused - * by another user without exposing any data from the previous session. - * - * Warning: this function deletes all data from the tables. Use with caution. - */ -void cloudsync_network_logout (sqlite3_context *context, int argc, sqlite3_value **argv) { - bool completed = false; - char *errmsg = NULL; - sqlite3 *db = sqlite3_context_db_handle(context); - - // if the network layer is enabled, remove the token or apikey - sqlite3_exec(db, "SELECT cloudsync_network_set_token('');", NULL, NULL, NULL); - - // get the list of cloudsync-enabled tables - char *sql = "SELECT tbl_name, key, value FROM cloudsync_table_settings;"; - char **result = NULL; - int nrows, ncols; - int rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, NULL); - if (rc != SQLITE_OK) { - errmsg = cloudsync_memory_mprintf("Unable to get current cloudsync configuration. %s", sqlite3_errmsg(db)); - goto finalize; - } - - // run everything in a savepoint - rc = sqlite3_exec(db, "SAVEPOINT cloudsync_logout_sp;", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - errmsg = cloudsync_memory_mprintf("Unable to create cloudsync_logout savepoint. %s", sqlite3_errmsg(db)); - return; - } - - // disable cloudsync for all the previously enabled tables: cloudsync_cleanup('*') - rc = sqlite3_exec(db, "SELECT cloudsync_cleanup('*')", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - errmsg = cloudsync_memory_mprintf("Unable to cleanup current cloudsync configuration. %s", sqlite3_errmsg(db)); - goto finalize; - } - - // delete all the local data for each previously enabled table - // re-enable cloudsync on previously enabled tables - for (int i = 1; i <= nrows; i++) { - char *tbl_name = result[i * ncols + 0]; - char *key = result[i * ncols + 1]; - char *value = result[i * ncols + 2]; - - if (strcmp(key, "algo") != 0) continue; - - sql = cloudsync_memory_mprintf("DELETE FROM \"%w\";", tbl_name); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - errmsg = cloudsync_memory_mprintf("Unable to delete data from table %s. %s", tbl_name, sqlite3_errmsg(db)); - goto finalize; - } - - sql = cloudsync_memory_mprintf("SELECT cloudsync_init('%q', '%q', 1);", tbl_name, value); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - errmsg = cloudsync_memory_mprintf("Unable to enable cloudsync on table %s. %s", tbl_name, sqlite3_errmsg(db)); - goto finalize; - } - } - - completed = true; - -finalize: - if (completed) { - sqlite3_exec(db, "RELEASE cloudsync_logout_sp;", NULL, NULL, NULL); - } else { - // cleanup: - // ROLLBACK TO command reverts the state of the database back to what it was just after the corresponding SAVEPOINT - // then RELEASE to remove the SAVEPOINT from the transaction stack - sqlite3_exec(db, "ROLLBACK TO cloudsync_logout_sp;", NULL, NULL, NULL); - sqlite3_exec(db, "RELEASE cloudsync_logout_sp;", NULL, NULL, NULL); - sqlite3_result_error(context, errmsg, -1); - sqlite3_result_error_code(context, rc); - } - sqlite3_free_table(result); - cloudsync_memory_free(errmsg); -} - -// MARK: - - -int cloudsync_network_register (sqlite3 *db, char **pzErrMsg, void *ctx) { - int rc = SQLITE_OK; - - rc = dbutils_register_function(db, "cloudsync_network_init", cloudsync_network_init, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_cleanup", cloudsync_network_cleanup, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_set_token", cloudsync_network_set_token, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_set_apikey", cloudsync_network_set_apikey, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_has_unsent_changes", cloudsync_network_has_unsent_changes, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_send_changes", cloudsync_network_send_changes, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_check_changes", cloudsync_network_check_changes, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_sync", cloudsync_network_sync0, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_sync", cloudsync_network_sync2, 2, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_reset_sync_version", cloudsync_network_reset_sync_version, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_logout", cloudsync_network_logout, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - return rc; -} -#endif diff --git a/src/cacert.h b/src/network/cacert.h similarity index 100% rename from src/cacert.h rename to src/network/cacert.h diff --git a/src/network/jsmn.h b/src/network/jsmn.h new file mode 100644 index 00000000..dca2bb59 --- /dev/null +++ b/src/network/jsmn.h @@ -0,0 +1,471 @@ +/* + * MIT License + * + * Copyright (c) 2010 Serge Zaitsev + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef JSMN_H +#define JSMN_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef JSMN_STATIC +#define JSMN_API static +#else +#define JSMN_API extern +#endif + +/** + * JSON type identifier. Basic types are: + * o Object + * o Array + * o String + * o Other primitive: number, boolean (true/false) or null + */ +typedef enum { + JSMN_UNDEFINED = 0, + JSMN_OBJECT = 1 << 0, + JSMN_ARRAY = 1 << 1, + JSMN_STRING = 1 << 2, + JSMN_PRIMITIVE = 1 << 3 +} jsmntype_t; + +enum jsmnerr { + /* Not enough tokens were provided */ + JSMN_ERROR_NOMEM = -1, + /* Invalid character inside JSON string */ + JSMN_ERROR_INVAL = -2, + /* The string is not a full JSON packet, more bytes expected */ + JSMN_ERROR_PART = -3 +}; + +/** + * JSON token description. + * type type (object, array, string etc.) + * start start position in JSON data string + * end end position in JSON data string + */ +typedef struct jsmntok { + jsmntype_t type; + int start; + int end; + int size; +#ifdef JSMN_PARENT_LINKS + int parent; +#endif +} jsmntok_t; + +/** + * JSON parser. Contains an array of token blocks available. Also stores + * the string being parsed now and current position in that string. + */ +typedef struct jsmn_parser { + unsigned int pos; /* offset in the JSON string */ + unsigned int toknext; /* next token to allocate */ + int toksuper; /* superior token node, e.g. parent object or array */ +} jsmn_parser; + +/** + * Create JSON parser over an array of tokens + */ +JSMN_API void jsmn_init(jsmn_parser *parser); + +/** + * Run JSON parser. It parses a JSON data string into and array of tokens, each + * describing + * a single JSON object. + */ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, + jsmntok_t *tokens, const unsigned int num_tokens); + +#ifndef JSMN_HEADER +/** + * Allocates a fresh unused token from the token pool. + */ +static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *tok; + if (parser->toknext >= num_tokens) { + return NULL; + } + tok = &tokens[parser->toknext++]; + tok->start = tok->end = -1; + tok->size = 0; +#ifdef JSMN_PARENT_LINKS + tok->parent = -1; +#endif + return tok; +} + +/** + * Fills token type and boundaries. + */ +static void jsmn_fill_token(jsmntok_t *token, const jsmntype_t type, + const int start, const int end) { + token->type = type; + token->start = start; + token->end = end; + token->size = 0; +} + +/** + * Fills next available token with JSON primitive. + */ +static int jsmn_parse_primitive(jsmn_parser *parser, const char *js, + const size_t len, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *token; + int start; + + start = parser->pos; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + switch (js[parser->pos]) { +#ifndef JSMN_STRICT + /* In strict mode primitive must be followed by "," or "}" or "]" */ + case ':': +#endif + case '\t': + case '\r': + case '\n': + case ' ': + case ',': + case ']': + case '}': + goto found; + default: + /* to quiet a warning from gcc*/ + break; + } + if (js[parser->pos] < 32 || js[parser->pos] >= 127) { + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } +#ifdef JSMN_STRICT + /* In strict mode primitive must be followed by a comma/object/array */ + parser->pos = start; + return JSMN_ERROR_PART; +#endif + +found: + if (tokens == NULL) { + parser->pos--; + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + parser->pos--; + return 0; +} + +/** + * Fills next token with JSON string. + */ +static int jsmn_parse_string(jsmn_parser *parser, const char *js, + const size_t len, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *token; + + int start = parser->pos; + + /* Skip starting quote */ + parser->pos++; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c = js[parser->pos]; + + /* Quote: end of string */ + if (c == '\"') { + if (tokens == NULL) { + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_STRING, start + 1, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + return 0; + } + + /* Backslash: Quoted symbol expected */ + if (c == '\\' && parser->pos + 1 < len) { + int i; + parser->pos++; + switch (js[parser->pos]) { + /* Allowed escaped symbols */ + case '\"': + case '/': + case '\\': + case 'b': + case 'f': + case 'r': + case 'n': + case 't': + break; + /* Allows escaped symbol \uXXXX */ + case 'u': + parser->pos++; + for (i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0'; + i++) { + /* If it isn't a hex character we have an error */ + if (!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */ + (js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */ + (js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */ + parser->pos = start; + return JSMN_ERROR_INVAL; + } + parser->pos++; + } + parser->pos--; + break; + /* Unexpected symbol */ + default: + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } + } + parser->pos = start; + return JSMN_ERROR_PART; +} + +/** + * Parse JSON string and fill tokens. + */ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, + jsmntok_t *tokens, const unsigned int num_tokens) { + int r; + int i; + jsmntok_t *token; + int count = parser->toknext; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c; + jsmntype_t type; + + c = js[parser->pos]; + switch (c) { + case '{': + case '[': + count++; + if (tokens == NULL) { + break; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + return JSMN_ERROR_NOMEM; + } + if (parser->toksuper != -1) { + jsmntok_t *t = &tokens[parser->toksuper]; +#ifdef JSMN_STRICT + /* In strict mode an object or array can't become a key */ + if (t->type == JSMN_OBJECT) { + return JSMN_ERROR_INVAL; + } +#endif + t->size++; +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + } + token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY); + token->start = parser->pos; + parser->toksuper = parser->toknext - 1; + break; + case '}': + case ']': + if (tokens == NULL) { + break; + } + type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY); +#ifdef JSMN_PARENT_LINKS + if (parser->toknext < 1) { + return JSMN_ERROR_INVAL; + } + token = &tokens[parser->toknext - 1]; + for (;;) { + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + token->end = parser->pos + 1; + parser->toksuper = token->parent; + break; + } + if (token->parent == -1) { + if (token->type != type || parser->toksuper == -1) { + return JSMN_ERROR_INVAL; + } + break; + } + token = &tokens[token->parent]; + } +#else + for (i = parser->toknext - 1; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + parser->toksuper = -1; + token->end = parser->pos + 1; + break; + } + } + /* Error if unmatched closing bracket */ + if (i == -1) { + return JSMN_ERROR_INVAL; + } + for (; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + parser->toksuper = i; + break; + } + } +#endif + break; + case '\"': + r = jsmn_parse_string(parser, js, len, tokens, num_tokens); + if (r < 0) { + return r; + } + count++; + if (parser->toksuper != -1 && tokens != NULL) { + tokens[parser->toksuper].size++; + } + break; + case '\t': + case '\r': + case '\n': + case ' ': + break; + case ':': + parser->toksuper = parser->toknext - 1; + break; + case ',': + if (tokens != NULL && parser->toksuper != -1 && + tokens[parser->toksuper].type != JSMN_ARRAY && + tokens[parser->toksuper].type != JSMN_OBJECT) { +#ifdef JSMN_PARENT_LINKS + parser->toksuper = tokens[parser->toksuper].parent; +#else + for (i = parser->toknext - 1; i >= 0; i--) { + if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) { + if (tokens[i].start != -1 && tokens[i].end == -1) { + parser->toksuper = i; + break; + } + } + } +#endif + } + break; +#ifdef JSMN_STRICT + /* In strict mode primitives are: numbers and booleans */ + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 't': + case 'f': + case 'n': + /* And they must not be keys of the object */ + if (tokens != NULL && parser->toksuper != -1) { + const jsmntok_t *t = &tokens[parser->toksuper]; + if (t->type == JSMN_OBJECT || + (t->type == JSMN_STRING && t->size != 0)) { + return JSMN_ERROR_INVAL; + } + } +#else + /* In non-strict mode every unquoted value is a primitive */ + default: +#endif + r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens); + if (r < 0) { + return r; + } + count++; + if (parser->toksuper != -1 && tokens != NULL) { + tokens[parser->toksuper].size++; + } + break; + +#ifdef JSMN_STRICT + /* Unexpected char in strict mode */ + default: + return JSMN_ERROR_INVAL; +#endif + } + } + + if (tokens != NULL) { + for (i = parser->toknext - 1; i >= 0; i--) { + /* Unmatched opened object or array */ + if (tokens[i].start != -1 && tokens[i].end == -1) { + return JSMN_ERROR_PART; + } + } + } + + return count; +} + +/** + * Creates a new parser based over a given buffer with an array of tokens + * available. + */ +JSMN_API void jsmn_init(jsmn_parser *parser) { + parser->pos = 0; + parser->toknext = 0; + parser->toksuper = -1; +} + +#endif /* JSMN_HEADER */ + +#ifdef __cplusplus +} +#endif + +#endif /* JSMN_H */ diff --git a/src/network/network.c b/src/network/network.c new file mode 100644 index 00000000..038adffd --- /dev/null +++ b/src/network/network.c @@ -0,0 +1,2550 @@ +// +// network.c +// cloudsync +// +// Created by Marco Bambini on 12/12/24. +// + +#ifndef CLOUDSYNC_OMIT_NETWORK + +#include +#include +#include +#include +#include +#include + +#ifdef CLOUDSYNC_NETWORK_TRACE +#ifdef _WIN32 +#include +#else +#include +#endif +#endif + +#include "network.h" +#include "../utils.h" +#include "../dbutils.h" +#include "../cloudsync.h" +#include "network_private.h" + +#define JSMN_STATIC +#include "jsmn.h" + +#ifndef SQLITE_WASM_EXTRA_INIT +#ifndef CLOUDSYNC_OMIT_CURL +#include "curl/curl.h" +#endif +#else +#define curl_free(x) free(x) +char *substr(const char *start, const char *end); +#endif + +#ifdef __ANDROID__ +#include "cacert.h" +static size_t cacert_len = sizeof(cacert_pem) - 1; +#endif + +#define CLOUDSYNC_NETWORK_MINBUF_SIZE 512 +#define CLOUDSYNC_SESSION_TOKEN_MAXSIZE 4096 + +#ifndef CLOUDSYNC_CURL_MAXCONNECTS +#define CLOUDSYNC_CURL_MAXCONNECTS 2L +#endif +#ifndef CLOUDSYNC_CURL_MAXAGE_CONN_SECONDS +#define CLOUDSYNC_CURL_MAXAGE_CONN_SECONDS 15L +#endif +#ifndef CLOUDSYNC_CURL_MAXLIFETIME_CONN_SECONDS +#define CLOUDSYNC_CURL_MAXLIFETIME_CONN_SECONDS 60L +#endif +#ifndef CLOUDSYNC_NETWORK_FAST_LANE_MAX_BLOB_SIZE +#define CLOUDSYNC_NETWORK_FAST_LANE_MAX_BLOB_SIZE (128 * 1024) +#endif + +#define DEFAULT_SYNC_WAIT_MS 100 +#define DEFAULT_SYNC_MAX_RETRIES 1 + +// Safety bound on how many chunks a single cloudsync_network_sync call will drain +// back-to-back. The drain is normally terminated by the server's "final" flag; this +// only guards against a buggy/never-final server. On hit, the drain stops with +// complete=false and the durable receive cursor is left unadvanced past un-applied +// rows, so the next sync resumes safely (apply is idempotent). +#define CLOUDSYNC_CHECK_MAX_CHUNKS_PER_DRAIN 10000 + +#define MAX_QUERY_VALUE_LEN 256 + +#ifndef SQLITE_CORE +SQLITE_EXTENSION_INIT3 +#endif + +// MARK: - + +struct network_data { + char site_id[UUID_STR_MAXLEN]; + char *authentication; // apikey or token + char *org_id; // organization ID for X-CloudSync-Org header + char *ticket; // optional short-lived sync runtime ticket + char *ticket_expires_at; + char *check_endpoint; + char *upload_endpoint; + char *apply_endpoint; + char *status_endpoint; + int ticket_enabled; + // Best-effort spool cursor for the chunked /check download drain. The durable + // receive cursor (check_dbversion/check_seq) is frozen at "since" for the whole + // drain, so the server (which is stateless across /check calls) needs the client + // to echo which spool page to serve next. In-memory only: losing it just + // restarts the drain from page 0, which is safe because apply is idempotent. + int64_t check_cursor; // next page index to request (0 = fresh drain) + int64_t check_cursor_since; // the check_dbversion check_cursor belongs to +#ifndef CLOUDSYNC_OMIT_CURL + CURL *api_curl; + CURL *artifact_curl; + int curl_pool_enabled; +#endif +}; + +#ifdef CLOUDSYNC_NETWORK_TRACE +double network_trace_now_ms(void) { +#ifdef _WIN32 + LARGE_INTEGER freq; + LARGE_INTEGER counter; + QueryPerformanceFrequency(&freq); + QueryPerformanceCounter(&counter); + return ((double)counter.QuadPart * 1000.0) / (double)freq.QuadPart; +#else + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((double)ts.tv_sec * 1000.0) + ((double)ts.tv_nsec / 1000000.0); +#endif +} + +const char *network_trace_endpoint_name(network_data *data, const char *endpoint) { + if (!data || !endpoint) return "unknown"; + if (data->check_endpoint && strcmp(endpoint, data->check_endpoint) == 0) return "check"; + if (data->upload_endpoint && strcmp(endpoint, data->upload_endpoint) == 0) return "upload-url"; + if (data->apply_endpoint && strcmp(endpoint, data->apply_endpoint) == 0) return "apply"; + if (data->status_endpoint && strcmp(endpoint, data->status_endpoint) == 0) return "status"; + return "artifact"; +} + +const char *network_trace_result_name(int code) { + switch (code) { + case CLOUDSYNC_NETWORK_OK: return "ok"; + case CLOUDSYNC_NETWORK_ERROR: return "error"; + case CLOUDSYNC_NETWORK_BUFFER: return "buffer"; + default: return "unknown"; + } +} + +void network_trace_log(network_data *data, const char *method, const char *endpoint, long http_status, int result_code, size_t request_bytes, size_t bytes, double elapsed_ms) { + fprintf(stderr, + "[cloudsync-network] endpoint=%s method=%s http_status=%ld result=%s request_bytes=%zu bytes=%zu elapsed_ms=%.2f\n", + network_trace_endpoint_name(data, endpoint), method, http_status, + network_trace_result_name(result_code), request_bytes, bytes, elapsed_ms); +} + +#ifndef CLOUDSYNC_OMIT_CURL +void network_trace_log_curl(network_data *data, const char *method, const char *endpoint, long http_status, int result_code, size_t request_bytes, size_t bytes, CURL *curl, bool pooled, double elapsed_ms) { + double namelookup = 0.0; + double connect = 0.0; + double appconnect = 0.0; + double starttransfer = 0.0; + double total = 0.0; + long num_connects = 0; + if (curl) { + curl_easy_getinfo(curl, CURLINFO_NAMELOOKUP_TIME, &namelookup); + curl_easy_getinfo(curl, CURLINFO_CONNECT_TIME, &connect); + curl_easy_getinfo(curl, CURLINFO_APPCONNECT_TIME, &appconnect); + curl_easy_getinfo(curl, CURLINFO_STARTTRANSFER_TIME, &starttransfer); + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total); + curl_easy_getinfo(curl, CURLINFO_NUM_CONNECTS, &num_connects); + } + fprintf(stderr, + "[cloudsync-network] endpoint=%s method=%s pool=%s http_status=%ld result=%s request_bytes=%zu bytes=%zu elapsed_ms=%.2f curl_total_ms=%.2f dns_ms=%.2f connect_ms=%.2f tls_ms=%.2f starttransfer_ms=%.2f num_connects=%ld\n", + network_trace_endpoint_name(data, endpoint), method, + pooled ? "on" : "off", http_status, + network_trace_result_name(result_code), request_bytes, bytes, elapsed_ms, + total * 1000.0, namelookup * 1000.0, connect * 1000.0, + appconnect * 1000.0, starttransfer * 1000.0, num_connects); +} +#endif +#endif + +typedef struct { + char *buffer; + size_t balloc; + size_t bused; + int zero_term; +} network_buffer; + + +typedef struct { + const char *data; + size_t size; + size_t read_pos; +} network_read_data; + +typedef struct { + char *ticket; + char *expires_at; +} network_ticket_headers; + +static const char *cloudsync_default_headers[] = { + CLOUDSYNC_HEADER_VERSION_LINE, +}; + +static const char *cloudsync_check_headers[] = { + CLOUDSYNC_HEADER_VERSION_LINE, + CLOUDSYNC_HEADER_CHECK_CAPABILITIES, +}; + +#define ARRAY_LEN(a) ((int)(sizeof(a) / sizeof((a)[0]))) + +// MARK: - + +void network_result_cleanup (NETWORK_RESULT *res) { + if (res->xfree) { + res->xfree(res->xdata); + } else if (res->buffer) { + cloudsync_memory_free(res->buffer); + } +} + +char *network_data_get_siteid (network_data *data) { + return data->site_id; +} + +char *network_data_get_orgid (network_data *data) { + return data->org_id; +} + +char *network_data_get_ticket (network_data *data) { + return data->ticket; +} + +static void network_data_clear_ticket (network_data *data) { + if (!data) return; + if (data->ticket) cloudsync_memory_free(data->ticket); + if (data->ticket_expires_at) cloudsync_memory_free(data->ticket_expires_at); + data->ticket = NULL; + data->ticket_expires_at = NULL; +} + +bool network_data_set_endpoints (network_data *data, char *auth, char *check, char *upload, char *apply, char *status) { + // sanity check + if (!check || !upload) return false; + + // always free previous owned pointers + if (data->authentication) cloudsync_memory_free(data->authentication); + network_data_clear_ticket(data); + if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); + if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); + if (data->apply_endpoint) cloudsync_memory_free(data->apply_endpoint); + if (data->status_endpoint) cloudsync_memory_free(data->status_endpoint); + + // clear pointers + data->authentication = NULL; + data->check_endpoint = NULL; + data->upload_endpoint = NULL; + data->apply_endpoint = NULL; + data->status_endpoint = NULL; + + // make a copy of the new endpoints + char *auth_copy = NULL; + char *check_copy = NULL; + char *upload_copy = NULL; + char *apply_copy = NULL; + char *status_copy = NULL; + + // auth is optional + if (auth) { + auth_copy = cloudsync_string_dup(auth); + if (!auth_copy) goto abort_endpoints; + } + check_copy = cloudsync_string_dup(check); + if (!check_copy) goto abort_endpoints; + + upload_copy = cloudsync_string_dup(upload); + if (!upload_copy) goto abort_endpoints; + + apply_copy = cloudsync_string_dup(apply); + if (!apply_copy) goto abort_endpoints; + + status_copy = cloudsync_string_dup(status); + if (!status_copy) goto abort_endpoints; + + data->authentication = auth_copy; + data->check_endpoint = check_copy; + data->upload_endpoint = upload_copy; + data->apply_endpoint = apply_copy; + data->status_endpoint = status_copy; + return true; + +abort_endpoints: + if (auth_copy) cloudsync_memory_free(auth_copy); + if (check_copy) cloudsync_memory_free(check_copy); + if (upload_copy) cloudsync_memory_free(upload_copy); + if (apply_copy) cloudsync_memory_free(apply_copy); + if (status_copy) cloudsync_memory_free(status_copy); + return false; +} + +void network_data_free (network_data *data) { + if (!data) return; + +#ifndef CLOUDSYNC_OMIT_CURL + if (data->api_curl) curl_easy_cleanup(data->api_curl); + if (data->artifact_curl) curl_easy_cleanup(data->artifact_curl); +#endif + if (data->authentication) cloudsync_memory_free(data->authentication); + if (data->org_id) cloudsync_memory_free(data->org_id); + network_data_clear_ticket(data); + if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); + if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); + if (data->apply_endpoint) cloudsync_memory_free(data->apply_endpoint); + if (data->status_endpoint) cloudsync_memory_free(data->status_endpoint); + cloudsync_memory_free(data); +} + +// MARK: - Utils - + +static bool network_endpoint_is_api(network_data *data, const char *endpoint) { + if (!data || !endpoint) return false; + return (data->check_endpoint && strcmp(endpoint, data->check_endpoint) == 0) || + (data->upload_endpoint && strcmp(endpoint, data->upload_endpoint) == 0) || + (data->apply_endpoint && strcmp(endpoint, data->apply_endpoint) == 0) || + (data->status_endpoint && strcmp(endpoint, data->status_endpoint) == 0); +} + +static bool network_env_disabled(const char *value) { + return value && (strcmp(value, "0") == 0 || strcmp(value, "false") == 0 || strcmp(value, "off") == 0 || strcmp(value, "no") == 0); +} + +static bool network_ticket_enabled(network_data *data) { + if (!data) return false; + if (data->ticket_enabled == 0) { + const char *value = getenv("CLOUDSYNC_NETWORK_TICKET"); + data->ticket_enabled = network_env_disabled(value) ? -1 : 1; + } + return data->ticket_enabled > 0; +} + +bool network_data_should_use_ticket (network_data *data, const char *endpoint, const char *authentication) { + return data && authentication && authentication[0] != '\0' && data->ticket && data->ticket[0] != '\0' && + network_ticket_enabled(data) && network_endpoint_is_api(data, endpoint); +} + +void network_data_update_ticket (network_data *data, const char *ticket, const char *expires_at) { + if (!data || !ticket || ticket[0] == '\0') return; + + char *ticket_copy = cloudsync_string_dup(ticket); + if (!ticket_copy) return; + + char *expires_copy = NULL; + if (expires_at && expires_at[0] != '\0') { + expires_copy = cloudsync_string_dup(expires_at); + if (!expires_copy) { + cloudsync_memory_free(ticket_copy); + return; + } + } + + network_data_clear_ticket(data); + data->ticket = ticket_copy; + data->ticket_expires_at = expires_copy; + +#ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] received_ticket=%s expires_at=%s\n", + data->ticket ? "true" : "false", data->ticket_expires_at ? data->ticket_expires_at : ""); +#endif +} + +#ifndef CLOUDSYNC_OMIT_CURL +static bool network_curl_pool_enabled(network_data *data) { + if (!data) return false; + if (data->curl_pool_enabled == 0) { + const char *value = getenv("CLOUDSYNC_CURL_POOL"); + data->curl_pool_enabled = network_env_disabled(value) ? -1 : 1; + } + return data->curl_pool_enabled > 0; +} + +static CURL *network_curl_for_endpoint(network_data *data, const char *endpoint, bool *pooled) { + if (pooled) *pooled = false; + if (!network_curl_pool_enabled(data)) { + return curl_easy_init(); + } + + CURL **slot = network_endpoint_is_api(data, endpoint) ? &data->api_curl : &data->artifact_curl; + if (!*slot) { + *slot = curl_easy_init(); + } else { + curl_easy_reset(*slot); + } + if (!*slot) return NULL; + + curl_easy_setopt(*slot, CURLOPT_MAXCONNECTS, CLOUDSYNC_CURL_MAXCONNECTS); + curl_easy_setopt(*slot, CURLOPT_MAXAGE_CONN, CLOUDSYNC_CURL_MAXAGE_CONN_SECONDS); + curl_easy_setopt(*slot, CURLOPT_MAXLIFETIME_CONN, CLOUDSYNC_CURL_MAXLIFETIME_CONN_SECONDS); + if (pooled) *pooled = true; + return *slot; +} + +static bool network_buffer_check (network_buffer *data, size_t needed) { + // alloc/resize buffer + if (data->bused + needed > data->balloc) { + if (needed < CLOUDSYNC_NETWORK_MINBUF_SIZE) needed = CLOUDSYNC_NETWORK_MINBUF_SIZE; + size_t balloc = data->balloc + needed; + + char *buffer = cloudsync_memory_realloc(data->buffer, balloc); + if (!buffer) return false; + + data->buffer = buffer; + data->balloc = balloc; + } + + return true; +} + +static size_t network_receive_callback (void *ptr, size_t size, size_t nmemb, void *xdata) { + network_buffer *data = (network_buffer *)xdata; + + size_t ptr_size = (size*nmemb); + if (data->zero_term) ptr_size += 1; + + if (network_buffer_check(data, ptr_size) == false) return CURL_WRITEFUNC_ERROR; + memcpy(data->buffer+data->bused, ptr, size*nmemb); + data->bused += size*nmemb; + if (data->zero_term) data->buffer[data->bused] = 0; + + return (size * nmemb); +} + +static bool network_header_eq(const char *line, size_t len, const char *name) { + size_t name_len = strlen(name); + if (len <= name_len || line[name_len] != ':') return false; + for (size_t i = 0; i < name_len; i++) { + if (tolower((unsigned char)line[i]) != tolower((unsigned char)name[i])) return false; + } + return true; +} + +static char *network_header_value_dup(const char *line, size_t len, const char *name) { + size_t name_len = strlen(name); + const char *start = line + name_len + 1; + const char *end = line + len; + + while (start < end && (*start == ' ' || *start == '\t')) start++; + while (end > start && (end[-1] == '\r' || end[-1] == '\n' || end[-1] == ' ' || end[-1] == '\t')) end--; + + size_t value_len = (size_t)(end - start); + char *value = cloudsync_memory_zeroalloc(value_len + 1); + if (!value) return NULL; + memcpy(value, start, value_len); + value[value_len] = '\0'; + return value; +} + +static size_t network_header_callback(char *buffer, size_t size, size_t nitems, void *userdata) { + network_ticket_headers *ticket_headers = (network_ticket_headers *)userdata; + size_t len = size * nitems; + + if (network_header_eq(buffer, len, CLOUDSYNC_HEADER_TICKET)) { + char *ticket = network_header_value_dup(buffer, len, CLOUDSYNC_HEADER_TICKET); + if (ticket) { + if (ticket_headers->ticket) cloudsync_memory_free(ticket_headers->ticket); + ticket_headers->ticket = ticket; + } + } else if (network_header_eq(buffer, len, CLOUDSYNC_HEADER_TICKET_EXPIRES_AT)) { + char *expires_at = network_header_value_dup(buffer, len, CLOUDSYNC_HEADER_TICKET_EXPIRES_AT); + if (expires_at) { + if (ticket_headers->expires_at) cloudsync_memory_free(ticket_headers->expires_at); + ticket_headers->expires_at = expires_at; + } + } + + return len; +} + +NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char **extra_headers, int nextra_headers) { + char *buffer = NULL; + size_t blen = 0; + struct curl_slist* headers = NULL; + network_ticket_headers ticket_headers = {NULL, NULL}; + char errbuf[CURL_ERROR_SIZE] = {0}; + long response_code = 0; + bool pooled = false; + bool using_ticket = network_data_should_use_ticket(data, endpoint, authentication); + const char *method = (json_payload || is_post_request) ? "POST" : "GET"; +#ifndef CLOUDSYNC_NETWORK_TRACE + (void)method; +#endif +#ifdef CLOUDSYNC_NETWORK_TRACE + double trace_start_ms = network_trace_now_ms(); + size_t request_bytes = json_payload ? strlen(json_payload) : 0; +#endif + + CURL *curl = network_curl_for_endpoint(data, endpoint, &pooled); + if (!curl) return (NETWORK_RESULT){CLOUDSYNC_NETWORK_ERROR, NULL, 0, NULL, NULL}; + + // a buffer to store errors in + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); + + CURLcode rc = curl_easy_setopt(curl, CURLOPT_URL, endpoint); + if (rc != CURLE_OK) goto cleanup; + + // set PEM + #ifdef __ANDROID__ + struct curl_blob pem_blob = { + .data = (void *)cacert_pem, + .len = cacert_len, + .flags = CURL_BLOB_NOCOPY + }; + curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &pem_blob); + #endif + + for (int i = 0; i < nextra_headers; i++) { + struct curl_slist *tmp = curl_slist_append(headers, extra_headers[i]); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + + if (data->org_id) { + char org_header[512]; + snprintf(org_header, sizeof(org_header), "%s: %s", CLOUDSYNC_HEADER_ORG, data->org_id); + struct curl_slist *tmp = curl_slist_append(headers, org_header); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + + if (json_payload) { + struct curl_slist *tmp = curl_slist_append(headers, "Content-Type: application/json"); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + if (authentication) { + char auth_header[CLOUDSYNC_SESSION_TOKEN_MAXSIZE]; + snprintf(auth_header, sizeof(auth_header), "Authorization: Bearer %s", authentication); + struct curl_slist *tmp = curl_slist_append(headers, auth_header); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + if (using_ticket) { + char ticket_header[CLOUDSYNC_SESSION_TOKEN_MAXSIZE]; + snprintf(ticket_header, sizeof(ticket_header), "%s: %s", CLOUDSYNC_HEADER_TICKET, data->ticket); + struct curl_slist *tmp = curl_slist_append(headers, ticket_header); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + + if (headers) curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + network_buffer netdata = {NULL, 0, 0, (zero_terminated) ? 1 : 0}; + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &netdata); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, network_receive_callback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &ticket_headers); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, network_header_callback); + + // add optional JSON payload (implies setting CURLOPT_POST to 1) + // or set the CURLOPT_POST option + if (json_payload) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_payload); + } else if (is_post_request) { + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, 0L); + } + + // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + rc = curl_easy_perform(curl); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + if (rc == CURLE_OK) { + buffer = netdata.buffer; + blen = netdata.bused; + if (response_code < 400 && ticket_headers.ticket) { + network_data_update_ticket(data, ticket_headers.ticket, ticket_headers.expires_at); + } + } else if (netdata.buffer) { + cloudsync_memory_free(netdata.buffer); + netdata.buffer = NULL; + } + +cleanup: + if (headers) curl_slist_free_all(headers); + if (ticket_headers.ticket) cloudsync_memory_free(ticket_headers.ticket); + if (ticket_headers.expires_at) cloudsync_memory_free(ticket_headers.expires_at); + + // build result + NETWORK_RESULT result = {0, NULL, 0, NULL, NULL}; + if (rc == CURLE_OK && response_code < 400) { + result.code = (buffer && blen) ? CLOUDSYNC_NETWORK_BUFFER : CLOUDSYNC_NETWORK_OK; + result.buffer = buffer; + result.blen = blen; + } else { + result.code = CLOUDSYNC_NETWORK_ERROR; + result.buffer = buffer ? buffer : (errbuf[0]) ? cloudsync_string_dup(errbuf) : NULL; + result.blen = buffer ? blen : rc; + } + + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] endpoint=%s using_ticket=%s\n", + network_trace_endpoint_name(data, endpoint), + using_ticket ? "true" : "false"); + network_trace_log_curl(data, method, endpoint, response_code, result.code, request_bytes, result.blen, curl, pooled, network_trace_now_ms() - trace_start_ms); + #endif + if (curl && !pooled) curl_easy_cleanup(curl); + return result; +} + +static size_t network_read_callback (char *buffer, size_t size, size_t nitems, void *userdata) { + network_read_data *rd = (network_read_data *)userdata; + size_t max_read = size * nitems; + size_t bytes_left = rd->size - rd->read_pos; + size_t to_copy = bytes_left < max_read ? bytes_left : max_read; + + if (to_copy > 0) { + memcpy(buffer, rd->data + rd->read_pos, to_copy); + rd->read_pos += to_copy; + } + + return to_copy; +} + +bool network_send_buffer (network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { + struct curl_slist *headers = NULL; + bool result = false; + char errbuf[CURL_ERROR_SIZE] = {0}; + CURLcode rc = CURLE_OK; + long response_code = 0; + bool pooled = false; +#ifdef CLOUDSYNC_NETWORK_TRACE + double trace_start_ms = network_trace_now_ms(); +#endif + + // init/reuse curl + CURL *curl = network_curl_for_endpoint(data, endpoint, &pooled); + if (!curl) return false; + + // set the URL + if (curl_easy_setopt(curl, CURLOPT_URL, endpoint) != CURLE_OK) goto cleanup; + + // set PEM + #ifdef __ANDROID__ + struct curl_blob pem_blob = { + .data = (void *)cacert_pem, + .len = cacert_len, + .flags = CURL_BLOB_NOCOPY + }; + curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &pem_blob); + #endif + + // a buffer to store errors in + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + + // type header + struct curl_slist *tmp = curl_slist_append(headers, "Accept: text/plain"); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + + if (authentication) { + // init authorization header + char auth_header[CLOUDSYNC_SESSION_TOKEN_MAXSIZE]; + snprintf(auth_header, sizeof(auth_header), "Authorization: Bearer %s", authentication); + struct curl_slist *tmp = curl_slist_append(headers, auth_header); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + + if (data->org_id) { + char org_header[512]; + snprintf(org_header, sizeof(org_header), "%s: %s", CLOUDSYNC_HEADER_ORG, data->org_id); + struct curl_slist *tmp = curl_slist_append(headers, org_header); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + + // Set headers if needed (S3 pre-signed URLs usually do not require additional headers) + tmp = curl_slist_append(headers, "Content-Type: application/octet-stream"); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + + if (!headers) goto cleanup; + if (curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers) != CURLE_OK) goto cleanup; + + // Set HTTP PUT method + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); + + // Set the size of the blob + curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)blob_size); + + // Provide the data using a custom read callback + network_read_data rdata = { + .data = (const char *)blob, + .size = blob_size, + .read_pos = 0 + }; + + curl_easy_setopt(curl, CURLOPT_READFUNCTION, network_read_callback); + curl_easy_setopt(curl, CURLOPT_READDATA, &rdata); + + // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + + // perform the upload + rc = curl_easy_perform(curl); + if (curl) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + if (rc == CURLE_OK) result = true; + +cleanup: + #ifdef CLOUDSYNC_NETWORK_TRACE + network_trace_log_curl(data, "PUT", endpoint, response_code, + result ? CLOUDSYNC_NETWORK_OK : CLOUDSYNC_NETWORK_ERROR, + (size_t)blob_size, + result ? (size_t)blob_size : 0, + curl, pooled, network_trace_now_ms() - trace_start_ms); + #endif + if (curl && !pooled) curl_easy_cleanup(curl); + if (headers) curl_slist_free_all(headers); + return result; +} +#endif + +int network_set_sqlite_result (sqlite3_context *context, NETWORK_RESULT *result) { + int rc = 0; + switch (result->code) { + case CLOUDSYNC_NETWORK_OK: + sqlite3_result_error_code(context, SQLITE_OK); + sqlite3_result_int(context, 0); + rc = 0; + break; + + case CLOUDSYNC_NETWORK_ERROR: + sqlite3_result_error(context, (result->buffer) ? result->buffer : "Memory error.", -1); + sqlite3_result_error_code(context, SQLITE_ERROR); + rc = -1; + break; + + case CLOUDSYNC_NETWORK_BUFFER: + sqlite3_result_error_code(context, SQLITE_OK); + sqlite3_result_text(context, result->buffer, (int)result->blen, SQLITE_TRANSIENT); + rc = (int)result->blen; + break; + } + return rc; +} + +// If err_out is non-NULL, cloudsync_payload_apply failures are returned via +// *err_out (malloc'd, caller must cloudsync_memory_free) instead of being raised +// on the sqlite3_context. This lets composite callers (cloudsync_network_sync) +// surface apply errors as structured JSON. Endpoint/network errors always raise +// a SQL error regardless of err_out. +int network_download_changes (sqlite3_context *context, const char *download_url, int *pnrows, char **err_out, int64_t checkpoint_db_version, int64_t checkpoint_seq, int64_t *pbytes) { + DEBUG_FUNCTION("network_download_changes"); + + if (pbytes) *pbytes = 0; + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata) { + sqlite3_result_error(context, "Unable to retrieve network CloudSync context.", -1); + return -1; + } + + NETWORK_RESULT result = network_receive_buffer(netdata, download_url, NULL, false, false, NULL, NULL, 0); + + int rc = SQLITE_OK; + if (result.code == CLOUDSYNC_NETWORK_BUFFER) { + if (pbytes) *pbytes = (int64_t)result.blen; + rc = cloudsync_payload_apply(data, result.buffer, (int)result.blen, pnrows, checkpoint_db_version, checkpoint_seq); + if (rc != DBRES_OK) { + const char *msg = cloudsync_errmsg(data); + if (!msg || !msg[0]) msg = "cloudsync_payload_apply failed"; + if (err_out) *err_out = cloudsync_string_dup(msg); + else sqlite3_result_error(context, msg, -1); + if (pnrows) *pnrows = 0; + } + } else if (result.code == CLOUDSYNC_NETWORK_ERROR) { + network_set_sqlite_result(context, &result); + rc = -1; + if (pnrows) *pnrows = 0; + } else { + // CLOUDSYNC_NETWORK_OK — no data, not an error + if (pnrows) *pnrows = 0; + } + network_result_cleanup(&result); + + return rc; +} + +static int network_apply_payload_buffer(sqlite3_context *context, const char *payload, int payload_size, + int *pnrows, char **err_out, + int64_t checkpoint_db_version, int64_t checkpoint_seq, + const char *error_prefix) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_payload_apply(data, payload, payload_size, pnrows, checkpoint_db_version, checkpoint_seq); + if (rc != DBRES_OK) { + const char *msg = cloudsync_errmsg(data); + if (!msg || !msg[0]) msg = error_prefix ? error_prefix : "cloudsync_payload_apply failed"; + if (err_out) *err_out = cloudsync_string_dup(msg); + else sqlite3_result_error(context, msg, -1); + if (pnrows) *pnrows = 0; + } + return rc; +} + +char *network_authentication_token (const char *key, const char *value) { + size_t len = strlen(key) + strlen(value) + 64; + char *buffer = cloudsync_memory_zeroalloc(len); + if (!buffer) return NULL; + + // build new token + // we don't need a prefix because the token alreay include a prefix "sqa_" + snprintf(buffer, len, "%s", value); + return buffer; +} + +// MARK: - JSON helpers (jsmn) - + +static bool jsmn_token_eq(const char *json, const jsmntok_t *tok, const char *s) { + return (tok->type == JSMN_STRING && + (int)strlen(s) == tok->end - tok->start && + strncmp(json + tok->start, s, tok->end - tok->start) == 0); +} + +static int jsmn_find_key(const char *json, const jsmntok_t *tokens, int ntokens, const char *key) { + for (int i = 1; i + 1 < ntokens; i++) { + if (jsmn_token_eq(json, &tokens[i], key)) return i; + } + return -1; +} + +static int jsmn_token_span(const jsmntok_t *tokens, int ntokens, int index) { + if (!tokens || index < 0 || index >= ntokens) return 0; + int start = tokens[index].start; + int end = tokens[index].end; + int i = index + 1; + while (i < ntokens && tokens[i].start >= start && tokens[i].end <= end) i++; + return i - index; +} + +static bool jsmn_find_object_value(const char *json, const jsmntok_t *tokens, int ntokens, + int object_index, const char *key, int *value_index) { + if (!json || !tokens || !key || object_index < 0 || object_index >= ntokens || + tokens[object_index].type != JSMN_OBJECT) { + return false; + } + + int i = object_index + 1; + for (int pair = 0; pair < tokens[object_index].size && i + 1 < ntokens; pair++) { + if (jsmn_token_eq(json, &tokens[i], key)) { + if (value_index) *value_index = i + 1; + return true; + } + i++; + i += jsmn_token_span(tokens, ntokens, i); + } + return false; +} + +static jsmntok_t *json_parse_tokens_alloc(const char *json, size_t json_len, int *ntokens_out) { + if (ntokens_out) *ntokens_out = 0; + if (!json || json_len == 0 || !ntokens_out) return NULL; + + jsmn_parser parser; + jsmn_init(&parser); + int ntokens = jsmn_parse(&parser, json, json_len, NULL, 0); + if (ntokens < 1) return NULL; + + jsmntok_t *tokens = cloudsync_memory_alloc((uint64_t)ntokens * sizeof(jsmntok_t)); + if (!tokens) return NULL; + + jsmn_init(&parser); + int parsed = jsmn_parse(&parser, json, json_len, tokens, (unsigned int)ntokens); + if (parsed < 1) { + cloudsync_memory_free(tokens); + return NULL; + } + + *ntokens_out = parsed; + return tokens; +} + +static char *json_unescape_string(const char *src, int len) { + char *out = cloudsync_memory_zeroalloc(len + 1); + if (!out) return NULL; + + int j = 0; + for (int i = 0; i < len; ) { + if (src[i] == '\\' && i + 1 < len) { + char c = src[i + 1]; + if (c == '"' || c == '\\' || c == '/') { out[j++] = c; i += 2; } + else if (c == 'n') { out[j++] = '\n'; i += 2; } + else if (c == 'r') { out[j++] = '\r'; i += 2; } + else if (c == 't') { out[j++] = '\t'; i += 2; } + else if (c == 'b') { out[j++] = '\b'; i += 2; } + else if (c == 'f') { out[j++] = '\f'; i += 2; } + else if (c == 'u' && i + 5 < len) { + unsigned int cp = 0; + for (int k = 0; k < 4; k++) { + char h = src[i + 2 + k]; + cp <<= 4; + if (h >= '0' && h <= '9') cp |= h - '0'; + else if (h >= 'a' && h <= 'f') cp |= 10 + h - 'a'; + else if (h >= 'A' && h <= 'F') cp |= 10 + h - 'A'; + } + if (cp < 0x80) { out[j++] = (char)cp; } + else { out[j++] = '?'; } // non-ASCII: replace + i += 6; + } + else { out[j++] = src[i]; i++; } + } else { + out[j++] = src[i]; i++; + } + } + out[j] = '\0'; + return out; +} + +static char *json_extract_string(const char *json, size_t json_len, const char *key) { + if (!json || json_len == 0 || !key) return NULL; + + int ntokens = 0; + jsmntok_t *tokens = json_parse_tokens_alloc(json, json_len, &ntokens); + if (!tokens) return NULL; + + char *result = NULL; + int i = jsmn_find_key(json, tokens, ntokens, key); + if (i >= 0 && i + 1 < ntokens) { + jsmntok_t *val = &tokens[i + 1]; + if (val->type == JSMN_STRING) + result = json_unescape_string(json + val->start, val->end - val->start); + } + cloudsync_memory_free(tokens); + return result; +} + +static int64_t json_extract_int(const char *json, size_t json_len, const char *key, int64_t default_value) { + if (!json || json_len == 0 || !key) return default_value; + + int ntokens = 0; + jsmntok_t *tokens = json_parse_tokens_alloc(json, json_len, &ntokens); + if (!tokens) return default_value; + + int64_t result = default_value; + if (tokens[0].type == JSMN_OBJECT) { + int i = jsmn_find_key(json, tokens, ntokens, key); + if (i >= 0 && i + 1 < ntokens && tokens[i + 1].type == JSMN_PRIMITIVE) + result = strtoll(json + tokens[i + 1].start, NULL, 10); + } + cloudsync_memory_free(tokens); + return result; +} + +static bool json_extract_bool(const char *json, size_t json_len, const char *key, bool default_value) { + if (!json || json_len == 0 || !key) return default_value; + + int ntokens = 0; + jsmntok_t *tokens = json_parse_tokens_alloc(json, json_len, &ntokens); + if (!tokens) return default_value; + + bool result = default_value; + if (tokens[0].type == JSMN_OBJECT) { + int i = jsmn_find_key(json, tokens, ntokens, key); + if (i >= 0 && i + 1 < ntokens && tokens[i + 1].type == JSMN_PRIMITIVE) { + // JSON booleans (true/false) and numeric flags (1/0) are both accepted. + char c = json[tokens[i + 1].start]; + if (c == 't' || c == 'T') result = true; + else if (c == 'f' || c == 'F' || c == 'n' || c == 'N') result = false; + else result = strtoll(json + tokens[i + 1].start, NULL, 10) != 0; + } + } + cloudsync_memory_free(tokens); + return result; +} + +static int json_extract_array_size(const char *json, size_t json_len, const char *key) { + if (!json || json_len == 0 || !key) return -1; + + int ntokens = 0; + jsmntok_t *tokens = json_parse_tokens_alloc(json, json_len, &ntokens); + if (!tokens) return -1; + + int result = -1; + if (tokens[0].type == JSMN_OBJECT) { + int i = jsmn_find_key(json, tokens, ntokens, key); + if (i >= 0 && i + 1 < ntokens && tokens[i + 1].type == JSMN_ARRAY) + result = tokens[i + 1].size; + } + cloudsync_memory_free(tokens); + return result; +} + +// Escape a string for safe embedding as a JSON string value (without surrounding quotes). +// Caller must free with cloudsync_memory_free. +static char *json_escape_string(const char *src) { + if (!src) return NULL; + size_t len = strlen(src); + // worst case: every char becomes \uXXXX (6 bytes) + char *out = cloudsync_memory_zeroalloc(len * 6 + 1); + if (!out) return NULL; + size_t j = 0; + for (size_t i = 0; i < len; i++) { + unsigned char c = (unsigned char)src[i]; + switch (c) { + case '"': out[j++] = '\\'; out[j++] = '"'; break; + case '\\': out[j++] = '\\'; out[j++] = '\\'; break; + case '\b': out[j++] = '\\'; out[j++] = 'b'; break; + case '\f': out[j++] = '\\'; out[j++] = 'f'; break; + case '\n': out[j++] = '\\'; out[j++] = 'n'; break; + case '\r': out[j++] = '\\'; out[j++] = 'r'; break; + case '\t': out[j++] = '\\'; out[j++] = 't'; break; + default: + if (c < 0x20) { + static const char hex[] = "0123456789abcdef"; + out[j++] = '\\'; out[j++] = 'u'; + out[j++] = '0'; out[j++] = '0'; + out[j++] = hex[(c >> 4) & 0xf]; + out[j++] = hex[c & 0xf]; + } else { + out[j++] = (char)c; + } + } + } + out[j] = '\0'; + return out; +} + +// Returns a malloc'd copy of the raw JSON substring for an object-valued key +// (found at any depth). Caller must free with cloudsync_memory_free. +static char *json_extract_object_raw(const char *json, size_t json_len, const char *key) { + if (!json || json_len == 0 || !key) return NULL; + + int ntokens = 0; + jsmntok_t *tokens = json_parse_tokens_alloc(json, json_len, &ntokens); + if (!tokens) return NULL; + + char *out = NULL; + int i = jsmn_find_key(json, tokens, ntokens, key); + if (i >= 0 && i + 1 < ntokens) { + jsmntok_t *val = &tokens[i + 1]; + int len = val->end - val->start; + if (val->type == JSMN_OBJECT && len > 0) { + out = cloudsync_memory_zeroalloc(len + 1); + if (out) { + memcpy(out, json + val->start, len); + out[len] = '\0'; + } + } + } + cloudsync_memory_free(tokens); + return out; +} + +int network_extract_query_param (const char *query, const char *key, char *output, size_t output_size) { + if (!query || !key || !output || output_size == 0) { + return -1; // Invalid input + } + + size_t key_len = strlen(key); + const char *p = query; + #ifdef SQLITE_WASM_EXTRA_INIT + if (*p == '?') p++; + #endif + + while (p && *p) { + // Find the start of a key=value pair + const char *key_start = p; + const char *eq = strchr(key_start, '='); + if (!eq) break; // No '=' found, malformed query string + + size_t current_key_len = eq - key_start; + + // Check if the key matches (ensuring it's the full key) + if (current_key_len == key_len && strncmp(key_start, key, key_len) == 0) { + // Extract the value + const char *value_start = eq + 1; + const char *end = strchr(value_start, '&'); // Find end of value + + size_t value_len = (end) ? (size_t)(end - value_start) : strlen(value_start); + if (value_len >= output_size) { + return -2; // Output buffer too small + } + + strncpy(output, value_start, value_len); + output[value_len] = '\0'; // Null-terminate + return 0; // Success + } + + // Move to the next parameter + p = strchr(p, '&'); + if (p) p++; // Skip '&' + } + + return -3; // Key not found +} + +static bool network_compute_endpoints_with_address (sqlite3_context *context, network_data *data, const char *address, const char *managedDatabaseId) { + if (!managedDatabaseId || managedDatabaseId[0] == '\0') { + sqlite3_result_error(context, "managedDatabaseId cannot be empty", -1); + sqlite3_result_error_code(context, SQLITE_ERROR); + return false; + } + + if (!address || address[0] == '\0') { + sqlite3_result_error(context, "address cannot be empty", -1); + sqlite3_result_error_code(context, SQLITE_ERROR); + return false; + } + + // build endpoints: {address}/v2/cloudsync/databases/{managedDatabaseId}/{siteId}/{action} + size_t requested = strlen(address) + 1 + + strlen(CLOUDSYNC_ENDPOINT_PREFIX) + 1 + strlen(managedDatabaseId) + 1 + + UUID_STR_MAXLEN + 1 + 16; + char *check_endpoint = (char *)cloudsync_memory_zeroalloc(requested); + char *upload_endpoint = (char *)cloudsync_memory_zeroalloc(requested); + char *apply_endpoint = (char *)cloudsync_memory_zeroalloc(requested); + char *status_endpoint = (char *)cloudsync_memory_zeroalloc(requested); + + if (!check_endpoint || !upload_endpoint || !apply_endpoint || !status_endpoint) { + sqlite3_result_error_code(context, SQLITE_NOMEM); + if (check_endpoint) cloudsync_memory_free(check_endpoint); + if (upload_endpoint) cloudsync_memory_free(upload_endpoint); + if (apply_endpoint) cloudsync_memory_free(apply_endpoint); + if (status_endpoint) cloudsync_memory_free(status_endpoint); + return false; + } + + // format: {address}/v2/cloudsync/databases/{managedDatabaseID}/{siteId}/{action} + snprintf(check_endpoint, requested, "%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_CHECK); + snprintf(upload_endpoint, requested, "%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_UPLOAD); + snprintf(apply_endpoint, requested, "%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_APPLY); + snprintf(status_endpoint, requested, "%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_STATUS); + + network_data_clear_ticket(data); + + if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); + data->check_endpoint = check_endpoint; + + if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); + data->upload_endpoint = upload_endpoint; + + if (data->apply_endpoint) cloudsync_memory_free(data->apply_endpoint); + data->apply_endpoint = apply_endpoint; + + if (data->status_endpoint) cloudsync_memory_free(data->status_endpoint); + data->status_endpoint = status_endpoint; + + return true; +} + +void network_result_to_sqlite_error (sqlite3_context *context, NETWORK_RESULT res, const char *default_error_message) { + sqlite3_result_error(context, ((res.code == CLOUDSYNC_NETWORK_ERROR) && (res.buffer)) ? res.buffer : default_error_message, -1); + sqlite3_result_error_code(context, SQLITE_ERROR); +} + +// MARK: - Init / Cleanup - + +network_data *cloudsync_network_data (sqlite3_context *context) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (netdata) return netdata; + + netdata = (network_data *)cloudsync_memory_zeroalloc(sizeof(network_data)); + if (netdata) cloudsync_set_auxdata(data, netdata); + return netdata; +} + +static void cloudsync_network_init_internal (sqlite3_context *context, const char *address, const char *managedDatabaseId) { + #ifndef CLOUDSYNC_OMIT_CURL + curl_global_init(CURL_GLOBAL_ALL); + #endif + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = cloudsync_network_data(context); + if (!netdata) goto abort_memory; + + // init context + uint8_t *site_id = (uint8_t *)cloudsync_context_init(data); + if (!site_id) goto abort_siteid; + + // save site_id string representation: 01957493c6c07e14803727e969f1d2cc + cloudsync_uuid_v7_stringify(site_id, netdata->site_id, false); + + // compute endpoints + // authentication can be set later via cloudsync_network_set_token/cloudsync_network_set_apikey + if (network_compute_endpoints_with_address(context, netdata, address, managedDatabaseId) == false) { + goto abort_cleanup; + } + + cloudsync_set_auxdata(data, netdata); + sqlite3_result_int(context, SQLITE_OK); + return; + +abort_memory: + sqlite3_result_error(context, "Unable to allocate memory in cloudsync_network_init.", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + goto abort_cleanup; + +abort_siteid: + sqlite3_result_error(context, "Unable to compute/retrieve site_id.", -1); + sqlite3_result_error_code(context, SQLITE_MISUSE); + goto abort_cleanup; + +abort_cleanup: + cloudsync_set_auxdata(data, NULL); + network_data_free(netdata); +} + +void cloudsync_network_init (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_init"); + const char *managedDatabaseId = (const char *)sqlite3_value_text(argv[0]); + cloudsync_network_init_internal(context, CLOUDSYNC_DEFAULT_ADDRESS, managedDatabaseId); +} + +void cloudsync_network_init_custom (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_init_custom"); + const char *address = (const char *)sqlite3_value_text(argv[0]); + const char *managedDatabaseId = (const char *)sqlite3_value_text(argv[1]); + cloudsync_network_init_internal(context, address, managedDatabaseId); +} + +void cloudsync_network_cleanup_internal (sqlite3_context *context) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = cloudsync_network_data(context); + cloudsync_set_auxdata(data, NULL); + network_data_free(netdata); + + #ifndef CLOUDSYNC_OMIT_CURL + curl_global_cleanup(); + #endif +} + +void cloudsync_network_cleanup (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_cleanup"); + + cloudsync_network_cleanup_internal(context); + sqlite3_result_int(context, SQLITE_OK); +} + +// MARK: - Public - + +bool cloudsync_network_set_authentication_token (sqlite3_context *context, const char *value, bool is_token) { + network_data *data = cloudsync_network_data(context); + if (!data) return false; + + const char *key = (is_token) ? "token" : "apikey"; + char *new_auth_token = network_authentication_token(key, value); + if (!new_auth_token) return false; + + if (data->authentication) cloudsync_memory_free(data->authentication); + network_data_clear_ticket(data); + data->authentication = new_auth_token; + + return true; +} + +void cloudsync_network_set_token (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_set_token"); + + const char *value = (const char *)sqlite3_value_text(argv[0]); + bool result = cloudsync_network_set_authentication_token(context, value, true); + (result) ? sqlite3_result_int(context, SQLITE_OK) : sqlite3_result_error_code(context, SQLITE_NOMEM); +} + +void cloudsync_network_set_apikey (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_set_apikey"); + + const char *value = (const char *)sqlite3_value_text(argv[0]); + bool result = cloudsync_network_set_authentication_token(context, value, false); + (result) ? sqlite3_result_int(context, SQLITE_OK) : sqlite3_result_error_code(context, SQLITE_NOMEM); +} + +// Returns a malloc'd JSON array string like '["tasks","users"]', or NULL on error/no results. +// Caller must free with cloudsync_memory_free. +static char *network_get_affected_tables(sqlite3 *db, int64_t since_db_version) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT json_group_array(DISTINCT tbl) FROM cloudsync_changes WHERE db_version > ?", + -1, &stmt, NULL); + if (rc != SQLITE_OK) return NULL; + sqlite3_bind_int64(stmt, 1, since_db_version); + + char *result = NULL; + if (sqlite3_step(stmt) == SQLITE_ROW) { + const char *json = (const char *)sqlite3_column_text(stmt, 0); + if (json) result = cloudsync_string_dup(json); + } + sqlite3_finalize(stmt); + return result; +} + +// MARK: - Sync result +// +// Error-handling contract for send/check/sync functions: +// - Endpoint/network errors (server unreachable, auth failure, bad URL) +// always raise a SQL error via sqlite3_result_error. +// - cloudsync_payload_apply failures (unknown schema hash, invalid checksum, +// decompression error) are returned as structured JSON via receive.error. +// - Server-reported failures from the SyncStatusResponse failures object are +// forwarded as send.lastFailure (failures.apply) and receive.lastFailure +// (failures.check). Per-function scoping: send_changes emits send.lastFailure +// only; receive_changes emits receive.lastFailure only; sync emits both. +// - A non-retryable failures.check (retryable:false) is a permanent +// configuration/authorization problem: the receive drain stops polling +// immediately rather than waiting it out. receive_changes raises it as a SQL +// error (fail fast — no send block to preserve); sync still emits structured +// JSON with receive.lastFailure so its send block survives. +// +// Callers that receive JSON can trust that the server was reachable. +// A SQL error means connectivity or configuration is broken. + +typedef struct { + int64_t server_version; // lastOptimisticVersion + int64_t local_version; // new_db_version (max local) + const char *status; // computed status string + int rows_received; // rows from check + char *tables_json; // JSON array of affected table names, caller must cloudsync_memory_free + char *apply_failure_json; // raw JSON object for server-reported failures.apply, caller must cloudsync_memory_free + char *check_failure_json; // raw JSON object for server-reported failures.check, caller must cloudsync_memory_free + // Per-call chunked-download signals written by cloudsync_network_check_internal. + bool page_delivered; // one or more url/inline chunks were fetched + applied this call (rc==OK) + bool more_pending; // this was a non-final chunked batch with a next cursor (drain should continue) + bool defer_tables; // input: when true, check_internal skips its per-call tables query + // so a multi-chunk drain can compute the union once at the end + int64_t bytes_received; // serialized payload bytes received this call (0 on 202/no page) + int chunks_received; // payload chunks received this call (0 on 202/no page) + // Send-path transport volume, written by cloudsync_network_send_changes_internal. + int send_chunks; // number of payload chunks sent this call + int64_t send_bytes; // serialized payload bytes sent this call +} sync_result; + +// Returns a malloc'd raw JSON copy of failures. ("apply" or "check"), +// or NULL when the field is missing or is JSON null. Caller frees with cloudsync_memory_free. +static char *json_extract_failure_stage(const char *json, size_t json_len, const char *stage_key) { + if (!json || json_len == 0 || !stage_key) return NULL; + + char *failures = json_extract_object_raw(json, json_len, "failures"); + if (!failures) return NULL; + + char *stage = json_extract_object_raw(failures, strlen(failures), stage_key); + cloudsync_memory_free(failures); + return stage; +} + +static char *network_base64_encode(const unsigned char *src, size_t len) { + static const char table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + if (!src && len > 0) return NULL; + if (len > (SIZE_MAX - 1) / 4 * 3) return NULL; + + size_t out_len = 4 * ((len + 2) / 3); + char *out = cloudsync_memory_alloc((uint64_t)out_len + 1); + if (!out) return NULL; + + size_t i = 0; + size_t j = 0; + while (i < len) { + uint32_t octet_a = i < len ? src[i++] : 0; + uint32_t octet_b = i < len ? src[i++] : 0; + uint32_t octet_c = i < len ? src[i++] : 0; + uint32_t triple = (octet_a << 16) | (octet_b << 8) | octet_c; + + out[j++] = table[(triple >> 18) & 0x3f]; + out[j++] = table[(triple >> 12) & 0x3f]; + out[j++] = table[(triple >> 6) & 0x3f]; + out[j++] = table[triple & 0x3f]; + } + + if (len % 3 == 1) { + out[out_len - 1] = '='; + out[out_len - 2] = '='; + } else if (len % 3 == 2) { + out[out_len - 1] = '='; + } + + out[out_len] = '\0'; + return out; +} + +static int network_base64_value(char c) { + if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (c == '+') return 62; + if (c == '/') return 63; + return -1; +} + +static unsigned char *network_base64_decode(const char *src, size_t len, size_t *out_len) { + if (!src || !out_len) return NULL; + *out_len = 0; + + size_t effective_len = 0; + for (size_t i = 0; i < len; ++i) { + if (!isspace((unsigned char)src[i])) ++effective_len; + } + if (effective_len == 0 || effective_len % 4 != 0) return NULL; + + size_t max_out_len = (effective_len / 4) * 3; + unsigned char *out = cloudsync_memory_alloc((uint64_t)max_out_len); + if (!out) return NULL; + + int quartet[4]; + int q = 0; + size_t j = 0; + bool seen_padding = false; + + for (size_t i = 0; i < len; ++i) { + unsigned char c = (unsigned char)src[i]; + if (isspace(c)) continue; + + if (c == '=') { + quartet[q++] = -2; + seen_padding = true; + } else { + int v = network_base64_value((char)c); + if (v < 0 || seen_padding) goto invalid; + quartet[q++] = v; + } + + if (q == 4) { + if (quartet[0] < 0 || quartet[1] < 0) goto invalid; + if (quartet[2] == -2 && quartet[3] != -2) goto invalid; + + uint32_t triple = ((uint32_t)quartet[0] << 18) | ((uint32_t)quartet[1] << 12); + out[j++] = (unsigned char)((triple >> 16) & 0xff); + + if (quartet[2] >= 0) { + triple |= (uint32_t)quartet[2] << 6; + out[j++] = (unsigned char)((triple >> 8) & 0xff); + } + if (quartet[3] >= 0) { + triple |= (uint32_t)quartet[3]; + out[j++] = (unsigned char)(triple & 0xff); + } + q = 0; + } + } + + if (q != 0) goto invalid; + *out_len = j; + return out; + +invalid: + cloudsync_memory_free(out); + *out_len = 0; + return NULL; +} + +static int network_apply_check_chunk(sqlite3_context *context, const char *chunk_json, size_t chunk_json_len, + bool final_chunk, int *pnrows, char **err_out, int64_t *pbytes) { + if (pnrows) *pnrows = 0; + if (pbytes) *pbytes = 0; + + char *download_url = json_extract_string(chunk_json, chunk_json_len, "url"); + char *inline_payload = download_url ? NULL : json_extract_string(chunk_json, chunk_json_len, "payload"); + if (!download_url && !inline_payload) { + sqlite3_result_error(context, "cloudsync_network_receive_changes: check chunk is missing url or payload.", -1); + return SQLITE_ERROR; + } + + // A non-final chunk must never advance the receive cursor (see cloudsync.h): + // landing mid-db_version would let the next /check skip the unapplied + // remainder. Only the final chunk advances -- to the explicit watermark, or + // the legacy last-applied fallback when it is absent. + int64_t watermark = json_extract_int(chunk_json, chunk_json_len, "watermark", -1); + int64_t checkpoint_db_version; + if (!final_chunk) { + checkpoint_db_version = CLOUDSYNC_CHECKPOINT_NONE; + } else { + checkpoint_db_version = (watermark < 0) ? CLOUDSYNC_CHECKPOINT_LAST_APPLIED : watermark; + } + int64_t checkpoint_seq = 0; + + int rc = SQLITE_OK; + if (download_url) { + rc = network_download_changes(context, download_url, pnrows, err_out, + checkpoint_db_version, checkpoint_seq, pbytes); + } else { + size_t decoded_size = 0; + unsigned char *decoded = network_base64_decode(inline_payload, strlen(inline_payload), &decoded_size); + if (!decoded || decoded_size > INT_MAX) { + if (decoded) cloudsync_memory_free(decoded); + sqlite3_result_error(context, "cloudsync_network_receive_changes: invalid inline payload in check response.", -1); + rc = SQLITE_ERROR; + } else { + if (pbytes) *pbytes = (int64_t)decoded_size; + rc = network_apply_payload_buffer(context, (const char *)decoded, (int)decoded_size, pnrows, err_out, + checkpoint_db_version, checkpoint_seq, + "cloudsync_network_receive_changes: inline payload apply failed"); + cloudsync_memory_free(decoded); + } + } + + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] check chunk transport=%s chunked=%d final=%d rc=%d\n", + download_url ? "url" : (inline_payload ? "inline" : "none"), + (watermark >= 0), final_chunk, rc); + #endif + + if (download_url) cloudsync_memory_free(download_url); + if (inline_payload) cloudsync_memory_free(inline_payload); + return rc; +} + +char *network_apply_json_payload(const char *transport_key, const char *transport_value, + int64_t db_version_min, int64_t db_version_max, + const char *batch_id, int chunk_index, bool is_final) { + if (!transport_key || !transport_value || !batch_id) return NULL; + + char *escaped_value = json_escape_string(transport_value); + if (!escaped_value) return NULL; + + size_t requested = strlen(transport_key) + strlen(escaped_value) + strlen(batch_id) + 192; + char *json_payload = cloudsync_memory_alloc((uint64_t)requested); + if (!json_payload) { + cloudsync_memory_free(escaped_value); + return NULL; + } + + snprintf(json_payload, requested, + "{\"%s\":\"%s\", \"dbVersionMin\":%" PRId64 ", \"dbVersionMax\":%" PRId64 + ", \"batchId\":\"%s\", \"chunkIndex\":%d, \"isFinal\":%s}", + transport_key, escaped_value, db_version_min, db_version_max, + batch_id, chunk_index, is_final ? "true" : "false"); + + cloudsync_memory_free(escaped_value); + return json_payload; +} + +static int network_send_payload_to_apply(sqlite3_context *context, network_data *netdata, + const void *blob, int blob_size, + int64_t db_version_min, int64_t db_version_max, + const char *batch_id, int chunk_index, bool is_final, + NETWORK_RESULT *res_out) { + memset(res_out, 0, sizeof(*res_out)); + if (!blob || blob_size <= 0) { + sqlite3_result_error(context, "cloudsync_network_send_changes: invalid empty payload chunk.", -1); + return SQLITE_ERROR; + } + + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] send_changes chunk_size=%d fast-lane:%s db_version_min=%" PRId64 " db_version_max=%" PRId64 + " batch_id=%s chunk_index=%d is_final=%d\n", + blob_size, + blob_size <= CLOUDSYNC_NETWORK_FAST_LANE_MAX_BLOB_SIZE ? "true" : "false", + db_version_min, + db_version_max, + batch_id, + chunk_index, + is_final ? 1 : 0); + #endif + + if (blob_size <= CLOUDSYNC_NETWORK_FAST_LANE_MAX_BLOB_SIZE) { + char *blob_base64 = network_base64_encode((const unsigned char *)blob, (size_t)blob_size); + if (!blob_base64) { + sqlite3_result_error(context, "cloudsync_network_send_changes: unable to encode payload chunk.", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + return SQLITE_NOMEM; + } + + char *json_payload = network_apply_json_payload("blob", blob_base64, db_version_min, db_version_max, + batch_id, chunk_index, is_final); + cloudsync_memory_free(blob_base64); + if (!json_payload) { + sqlite3_result_error(context, "cloudsync_network_send_changes: unable to allocate apply request payload.", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + return SQLITE_NOMEM; + } + + *res_out = network_receive_buffer(netdata, netdata->apply_endpoint, netdata->authentication, true, true, + json_payload, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + cloudsync_memory_free(json_payload); + return SQLITE_OK; + } + + NETWORK_RESULT upload_res = network_receive_buffer(netdata, netdata->upload_endpoint, netdata->authentication, true, false, + NULL, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + if (upload_res.code != CLOUDSYNC_NETWORK_BUFFER) { + network_result_to_sqlite_error(context, upload_res, "cloudsync_network_send_changes unable to receive upload URL"); + network_result_cleanup(&upload_res); + return SQLITE_ERROR; + } + + char *s3_url = json_extract_string(upload_res.buffer, upload_res.blen, "url"); + if (!s3_url) { + sqlite3_result_error(context, "cloudsync_network_send_changes: missing 'url' in upload response.", -1); + network_result_cleanup(&upload_res); + return SQLITE_ERROR; + } + + bool sent = network_send_buffer(netdata, s3_url, NULL, blob, blob_size); + if (sent == false) { + cloudsync_memory_free(s3_url); + network_result_to_sqlite_error(context, upload_res, "cloudsync_network_send_changes unable to upload payload chunk to remote host."); + network_result_cleanup(&upload_res); + return SQLITE_ERROR; + } + + char *json_payload = network_apply_json_payload("url", s3_url, db_version_min, db_version_max, + batch_id, chunk_index, is_final); + cloudsync_memory_free(s3_url); + if (!json_payload) { + sqlite3_result_error(context, "cloudsync_network_send_changes: unable to allocate apply request payload.", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + network_result_cleanup(&upload_res); + return SQLITE_NOMEM; + } + + network_result_cleanup(&upload_res); + *res_out = network_receive_buffer(netdata, netdata->apply_endpoint, netdata->authentication, true, true, + json_payload, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + cloudsync_memory_free(json_payload); + return SQLITE_OK; +} + +void network_sync_state_update_from_response(NETWORK_RESULT *res, + int64_t *last_optimistic_version, + int64_t *last_confirmed_version, + int *gaps_size, + char **apply_failure_json, + char **check_failure_json) { + if (!res || res->code != CLOUDSYNC_NETWORK_BUFFER || !res->buffer) return; + + // Take the latest valid (>= 0) value, not the max: the server can move these + // BACKWARD on a rollback when a later send chunk fails, and lastOptimisticVersion + // becomes the durable send checkpoint — masking a decrease would advance the + // checkpoint past the rolled-back changes and silently drop them. + int64_t parsed_optimistic = json_extract_int(res->buffer, res->blen, "lastOptimisticVersion", -1); + if (parsed_optimistic >= 0) *last_optimistic_version = parsed_optimistic; + int64_t parsed_confirmed = json_extract_int(res->buffer, res->blen, "lastConfirmedVersion", -1); + if (parsed_confirmed >= 0) *last_confirmed_version = parsed_confirmed; + int parsed_gaps_size = json_extract_array_size(res->buffer, res->blen, "gaps"); + if (parsed_gaps_size >= 0) *gaps_size = parsed_gaps_size; + + char *apply_failure = json_extract_failure_stage(res->buffer, res->blen, "apply"); + if (apply_failure) { + if (*apply_failure_json) cloudsync_memory_free(*apply_failure_json); + *apply_failure_json = apply_failure; + } + + char *check_failure = json_extract_failure_stage(res->buffer, res->blen, "check"); + if (check_failure) { + if (*check_failure_json) cloudsync_memory_free(*check_failure_json); + *check_failure_json = check_failure; + } + + #ifdef CLOUDSYNC_NETWORK_TRACE + // Full endpoint response body that the sync-state fields above were parsed from. + // The buffer is not guaranteed NUL-terminated, so bound the print with its length. + fprintf(stderr, "[cloudsync-network] sync_state response=%.*s\n", (int)res->blen, res->buffer); + #endif +} + +const char *network_compute_status(int64_t last_optimistic, int64_t last_confirmed, + int gaps_size, int64_t local_version) { + if (last_optimistic < 0 || last_confirmed < 0) return "error"; + if (gaps_size > 0 || last_optimistic < local_version) return "out-of-sync"; + if (last_optimistic == last_confirmed) return "synced"; + return "syncing"; +} + +// MARK: - + +void cloudsync_network_has_unsent_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3 *db = sqlite3_context_db_handle(context); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata) {sqlite3_result_error(context, "Unable to retrieve CloudSync network context.", -1); return;} + + char *sql = "SELECT max(db_version) FROM cloudsync_changes WHERE site_id == (SELECT site_id FROM cloudsync_site_id WHERE rowid=0)"; + int64_t last_local_change = 0; + int rc = database_select_int(data, sql, &last_local_change); + if (rc != DBRES_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + sqlite3_result_error_code(context, rc); + return; + } + + if (last_local_change == 0) { + sqlite3_result_int(context, 0); + return; + } + + NETWORK_RESULT res = network_receive_buffer(netdata, netdata->status_endpoint, netdata->authentication, true, false, NULL, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + + int64_t last_optimistic_version = -1; + + if (res.code == CLOUDSYNC_NETWORK_BUFFER && res.buffer) { + last_optimistic_version = json_extract_int(res.buffer, res.blen, "lastOptimisticVersion", -1); + } else if (res.code != CLOUDSYNC_NETWORK_OK) { + network_result_to_sqlite_error(context, res, "unable to retrieve current status from remote host."); + network_result_cleanup(&res); + return; + } + + network_result_cleanup(&res); + sqlite3_result_int(context, (last_optimistic_version >= 0 && last_optimistic_version < last_local_change)); +} + +int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc, sqlite3_value **argv, sync_result *out) { + DEBUG_FUNCTION("cloudsync_network_send_changes"); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + // retrieve global context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata) {sqlite3_result_error(context, "Unable to retrieve CloudSync network context.", -1); return SQLITE_ERROR;} + + int64_t db_version = dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_SEND_DBVERSION); + if (db_version < 0) { + sqlite3_result_error(context, "Unable to retrieve db_version.", -1); + return SQLITE_ERROR; + } + + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_stmt *stmt = NULL; + const char *chunk_sql = + "SELECT payload, payload_size, watermark_db_version, is_final " + "FROM cloudsync_payload_chunks WHERE since_db_version = ?"; + int rc = sqlite3_prepare_v2(db, chunk_sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + sqlite3_result_error_code(context, rc); + return rc; + } + sqlite3_bind_int64(stmt, 1, db_version); + + int64_t new_db_version = db_version; + int64_t last_optimistic_version = -1; + int64_t last_confirmed_version = -1; + int gaps_size = -1; + char *apply_failure_json = NULL; + char *check_failure_json = NULL; + bool sent_any = false; + int sent_chunks = 0; // payload chunks sent this call + int64_t sent_bytes = 0; // serialized payload bytes sent this call + // One send call = one all-or-nothing batch: every chunk announces the same + // global window [send_checkpoint+1 .. watermark] (holes from non-local-change + // db_versions included) plus batchId/chunkIndex/isFinal. The server advances + // optimistic on the final chunk and confirms the whole window only when every + // chunk of the batch applied; a failed batch is re-sent whole under a new id. + char batch_id[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_string(batch_id, true); + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + const void *blob = sqlite3_column_blob(stmt, 0); + int blob_size = sqlite3_column_bytes(stmt, 0); + int64_t payload_size = sqlite3_column_int64(stmt, 1); + int64_t watermark = sqlite3_column_int64(stmt, 2); + bool is_final = sqlite3_column_int(stmt, 3) != 0; + + if (!blob || blob_size <= 0 || payload_size != blob_size || payload_size > INT_MAX || + watermark <= db_version) { + sqlite3_result_error(context, "cloudsync_network_send_changes: invalid payload chunk generated.", -1); + rc = SQLITE_ERROR; + goto cleanup; + } + + NETWORK_RESULT res = {0}; + rc = network_send_payload_to_apply(context, netdata, blob, blob_size, db_version + 1, watermark, + batch_id, sent_chunks, is_final, &res); + if (rc != SQLITE_OK) goto cleanup; + + if (res.code == CLOUDSYNC_NETWORK_BUFFER && res.buffer) { + network_sync_state_update_from_response(&res, &last_optimistic_version, &last_confirmed_version, &gaps_size, + &apply_failure_json, &check_failure_json); + } else if (res.code != CLOUDSYNC_NETWORK_OK) { + network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to apply changes to remote host."); + network_result_cleanup(&res); + rc = SQLITE_ERROR; + goto cleanup; + } + network_result_cleanup(&res); + + sent_any = true; + sent_chunks++; + sent_bytes += payload_size; + if (watermark > new_db_version) new_db_version = watermark; + } + if (rc != SQLITE_DONE) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + sqlite3_result_error_code(context, rc); + goto cleanup; + } + sqlite3_finalize(stmt); + stmt = NULL; + + if (!sent_any) { + // Empty local db with no server state: preserve the previous fast no-op path. + if (db_version == 0) { + if (out) { + out->server_version = 0; + out->local_version = 0; + out->status = network_compute_status(0, 0, 0, 0); + } + rc = SQLITE_OK; + goto cleanup; + } + + NETWORK_RESULT res = network_receive_buffer(netdata, netdata->status_endpoint, netdata->authentication, true, false, + NULL, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + if (res.code == CLOUDSYNC_NETWORK_BUFFER && res.buffer) { + network_sync_state_update_from_response(&res, &last_optimistic_version, &last_confirmed_version, &gaps_size, + &apply_failure_json, &check_failure_json); + } else if (res.code != CLOUDSYNC_NETWORK_OK) { + network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to apply changes to remote host."); + network_result_cleanup(&res); + rc = SQLITE_ERROR; + goto cleanup; + } + network_result_cleanup(&res); + } + if (gaps_size < 0) gaps_size = 0; + + // update db_version in settings + char buf[256]; + if (last_optimistic_version >= 0) { + if (last_optimistic_version != db_version) { + snprintf(buf, sizeof(buf), "%" PRId64, last_optimistic_version); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_SEND_DBVERSION, buf); + } + } else if (new_db_version != db_version) { + snprintf(buf, sizeof(buf), "%" PRId64, new_db_version); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_SEND_DBVERSION, buf); + } + + // populate sync result + if (out) { + out->server_version = last_optimistic_version; + out->local_version = new_db_version; + out->status = network_compute_status(last_optimistic_version, last_confirmed_version, gaps_size, new_db_version); + out->send_chunks = sent_chunks; + out->send_bytes = sent_bytes; + out->apply_failure_json = apply_failure_json; + out->check_failure_json = check_failure_json; + apply_failure_json = NULL; + check_failure_json = NULL; + } + if (apply_failure_json) cloudsync_memory_free(apply_failure_json); + if (check_failure_json) cloudsync_memory_free(check_failure_json); + return SQLITE_OK; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (apply_failure_json) cloudsync_memory_free(apply_failure_json); + if (check_failure_json) cloudsync_memory_free(check_failure_json); + return rc == SQLITE_DONE ? SQLITE_OK : rc; +} + +void cloudsync_network_send_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_send_changes"); + + // send-scoped: emits send.lastFailure (from failures.apply) only. + // failures.check arriving in the same response is parsed but discarded here. + sync_result sr = {.server_version = -1}; + int rc = cloudsync_network_send_changes_internal(context, argc, argv, &sr); + if (rc != SQLITE_OK) { + if (sr.apply_failure_json) cloudsync_memory_free(sr.apply_failure_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); + return; + } + + char *buf; + if (sr.apply_failure_json) { + buf = cloudsync_memory_mprintf( + "{\"send\":{\"status\":\"%s\",\"localVersion\":%lld,\"serverVersion\":%lld,\"chunks\":%d,\"bytes\":%lld,\"lastFailure\":%s}}", + sr.status ? sr.status : "error", + (long long)sr.local_version, (long long)sr.server_version, + sr.send_chunks, (long long)sr.send_bytes, + sr.apply_failure_json); + } else { + buf = cloudsync_memory_mprintf( + "{\"send\":{\"status\":\"%s\",\"localVersion\":%lld,\"serverVersion\":%lld,\"chunks\":%d,\"bytes\":%lld}}", + sr.status ? sr.status : "error", + (long long)sr.local_version, (long long)sr.server_version, + sr.send_chunks, (long long)sr.send_bytes); + } + sqlite3_result_text(context, buf, -1, cloudsync_memory_free); + if (sr.apply_failure_json) cloudsync_memory_free(sr.apply_failure_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); +} + +int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync_result *out, char **err_out, int request_max_chunks) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata) {sqlite3_result_error(context, "Unable to retrieve CloudSync network context.", -1); return -1;} + + int64_t db_version = dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_CHECK_DBVERSION); + if (db_version<0) {sqlite3_result_error(context, "Unable to retrieve db_version.", -1); return -1;} + + int seq = dbutils_settings_get_int_value(data, CLOUDSYNC_KEY_CHECK_SEQ); + if (seq<0) {sqlite3_result_error(context, "Unable to retrieve seq.", -1); return -1;} + + // Restart paging whenever the durable receive window changes: the page cursor + // is only meaningful within a single drain (check_dbversion held at "since"). + if (netdata->check_cursor_since != db_version) { + netdata->check_cursor = 0; + netdata->check_cursor_since = db_version; + } + + // Capture local db_version before download so we can query cloudsync_changes afterwards + int64_t prev_dbv = cloudsync_dbversion(data); + + // "cursor" is the spool page to serve. "maxChunks" lets cursor-spool servers + // batch several contiguous prepared pages in one response when present. + // Old/legacy servers ignore the unknown fields and omit them from the response; + // the client then never self-pages (check_cursor stays 0), preserving current behavior. + char json_payload[2048]; + if (request_max_chunks > 0) { + snprintf(json_payload, sizeof(json_payload), "{\"dbVersion\":%lld, \"seq\":%d, \"cursor\":%lld, \"maxChunks\":%d}", + (long long)db_version, seq, (long long)netdata->check_cursor, request_max_chunks); + } else { + snprintf(json_payload, sizeof(json_payload), "{\"dbVersion\":%lld, \"seq\":%d, \"cursor\":%lld}", + (long long)db_version, seq, (long long)netdata->check_cursor); + } + + if (out) { out->page_delivered = false; out->more_pending = false; out->bytes_received = 0; out->chunks_received = 0; } + + NETWORK_RESULT result = network_receive_buffer(netdata, netdata->check_endpoint, netdata->authentication, true, true, json_payload, cloudsync_check_headers, ARRAY_LEN(cloudsync_check_headers)); + int rc = SQLITE_OK; + if (result.code == CLOUDSYNC_NETWORK_BUFFER) { + // The /check endpoint returns one of these shapes: + // HTTP 200 -> {"url": "..."} (legacy artifact URL) + // HTTP 200 -> {"data":{"payload": "...", ...}} (single inline payload) + // HTTP 200 -> {"data":{"chunks":[...], ...}} (cursor-spool batch) + // HTTP 202 -> SyncStatusResponse (no artifact yet/status) + // Branch on the presence of transport fields rather than HTTP status; all + // shapes arrive as BUFFER. Newer servers wrap page metadata in "data"; + // legacy responses put "url" at the top level. + char *data_json = json_extract_object_raw(result.buffer, result.blen, "data"); + const char *check_json = data_json ? data_json : result.buffer; + size_t check_json_len = data_json ? strlen(data_json) : result.blen; + + int rows_total = 0; + int chunks_total = 0; + int64_t bytes_total = 0; + bool delivered = false; + bool more_pending = false; + int64_t next_cursor = json_extract_int(check_json, check_json_len, "nextCursor", -1); + + int ntokens = 0; + int chunks_index = -1; + jsmntok_t *tokens = json_parse_tokens_alloc(check_json, check_json_len, &ntokens); + bool has_chunks = tokens && tokens[0].type == JSMN_OBJECT && + jsmn_find_object_value(check_json, tokens, ntokens, 0, "chunks", &chunks_index) && + chunks_index >= 0 && chunks_index < ntokens && + tokens[chunks_index].type == JSMN_ARRAY; + + if (!tokens) { + // The body is BUFFER (non-empty) but unparseable (malformed, truncated, + // or token allocation failed). Without this, a chunks batch we failed to + // tokenize would fall through to the single-payload branch, find no + // url/payload, and be misreported as an empty "up to date" response, + // silently dropping the pending batch. + sqlite3_result_error(context, "cloudsync_network_receive_changes: unable to parse check response.", -1); + rc = SQLITE_ERROR; + } else if (has_chunks && request_max_chunks > 0 && tokens[chunks_index].size > request_max_chunks) { + sqlite3_result_error(context, "cloudsync_network_receive_changes: check response exceeded requested maxChunks.", -1); + rc = SQLITE_ERROR; + } else if (has_chunks && tokens[chunks_index].size > 0) { + bool final_batch = json_extract_bool(check_json, check_json_len, "final", true); + int64_t last_cursor = -1; + int child = chunks_index + 1; + + for (int i = 0; rc == SQLITE_OK && i < tokens[chunks_index].size; i++) { + if (child >= ntokens || tokens[child].type != JSMN_OBJECT) { + sqlite3_result_error(context, "cloudsync_network_receive_changes: invalid check chunks response.", -1); + rc = SQLITE_ERROR; + break; + } + + const char *chunk_json = check_json + tokens[child].start; + size_t chunk_json_len = (size_t)(tokens[child].end - tokens[child].start); + bool final_chunk = final_batch && (i == tokens[chunks_index].size - 1); + int chunk_rows = 0; + int64_t chunk_bytes = 0; + rc = network_apply_check_chunk(context, chunk_json, chunk_json_len, final_chunk, + &chunk_rows, err_out, &chunk_bytes); + if (rc == SQLITE_OK) { + rows_total += chunk_rows; + bytes_total += chunk_bytes; + chunks_total++; + delivered = true; + last_cursor = json_extract_int(chunk_json, chunk_json_len, "cursor", last_cursor); + netdata->check_cursor = final_chunk ? 0 : last_cursor + 1; + } + child += jsmn_token_span(tokens, ntokens, child); + } + + if (rc == SQLITE_OK) { + if (!final_batch && next_cursor < 0 && last_cursor >= 0) next_cursor = last_cursor + 1; + if (!final_batch && next_cursor < 0) { + // Non-final batch with no resumable cursor. The server contract + // always supplies nextCursor (or a per-chunk cursor) on a non-final + // batch, so a missing one would otherwise make us silently drop the + // rest of the stream and report a false "complete". Fail loudly. + sqlite3_result_error(context, "cloudsync_network_receive_changes: non-final check batch missing next cursor.", -1); + rc = SQLITE_ERROR; + } else { + more_pending = !final_batch && next_cursor >= 0; + } + } + } else if (has_chunks) { + // Explicit empty cursor-spool batch: no artifact in flight. + netdata->check_cursor = 0; + if (pnrows) *pnrows = 0; + } else { + char *download_url = json_extract_string(check_json, check_json_len, "url"); + char *inline_payload = download_url ? NULL : json_extract_string(check_json, check_json_len, "payload"); + if (download_url || inline_payload) { + bool final_chunk = json_extract_bool(check_json, check_json_len, "final", true); + int chunk_rows = 0; + int64_t chunk_bytes = 0; + rc = network_apply_check_chunk(context, check_json, check_json_len, final_chunk, + &chunk_rows, err_out, &chunk_bytes); + if (rc == SQLITE_OK && !final_chunk && next_cursor < 0) { + // Symmetric with the chunks-array path: a non-final response + // with no resumable cursor would otherwise silently drop the + // rest of the stream and report a false "complete". Fail loudly. + sqlite3_result_error(context, "cloudsync_network_receive_changes: non-final check response missing next cursor.", -1); + rc = SQLITE_ERROR; + } else if (rc == SQLITE_OK) { + rows_total = chunk_rows; + bytes_total = chunk_bytes; + chunks_total = 1; + delivered = true; + more_pending = !final_chunk && next_cursor >= 0; + } + } else { + // 202 / "up to date": no artifact in flight -> reset paging. + netdata->check_cursor = 0; + if (pnrows) *pnrows = 0; + } + if (download_url) cloudsync_memory_free(download_url); + if (inline_payload) cloudsync_memory_free(inline_payload); + } + + if (tokens) cloudsync_memory_free(tokens); + + if (rc == SQLITE_OK && delivered) { + // Finalize cursor state after the returned batch is applied/staged. + // Batched responses advance the in-memory spool cursor after each + // successful chunk, so a later failure retries from the failed chunk. + netdata->check_cursor = more_pending ? next_cursor : 0; + if (pnrows) *pnrows = rows_total; + if (out) { + out->page_delivered = true; + out->more_pending = more_pending; + out->bytes_received = bytes_total; + out->chunks_received = chunks_total; + } + } else { + if (pnrows) *pnrows = 0; + } + if (data_json) cloudsync_memory_free(data_json); + // failures.check may appear in either shape; extract opportunistically. + if (out) { + char *check_failure = json_extract_failure_stage(result.buffer, result.blen, "check"); + if (check_failure) { + if (out->check_failure_json) cloudsync_memory_free(out->check_failure_json); + out->check_failure_json = check_failure; + } + } + } else if (result.code == CLOUDSYNC_NETWORK_ERROR) { + network_set_sqlite_result(context, &result); + rc = -1; + } else { + // CLOUDSYNC_NETWORK_OK — no body (older server) — not an error + rc = 0; + } + + if (out && pnrows) out->rows_received = *pnrows; + + // Query cloudsync_changes for affected tables after successful download. + // Skipped when the caller defers it (a multi-chunk drain computes the union once + // at the end from a db_version captured before the whole drain started). + if (out && !out->defer_tables && rc == SQLITE_OK && pnrows && *pnrows > 0) { + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + out->tables_json = network_get_affected_tables(db, prev_dbv); + } + + network_result_cleanup(&result); + return rc; +} + +// Result of a receive drain (see network_drain_changes). +typedef struct { + int rows; // cumulative rows applied across the drain + int chunks; // payload chunks applied this drain + int64_t bytes; // serialized payload bytes received this drain + bool complete; // true iff the receive stream is fully drained (nothing pending) + bool check_permanent_failure; // server reported a non-retryable failures.check: polling stopped early + char *receive_err; // owned by the caller; client-side apply error, or NULL +} drain_result; + +// Drains chunked /check responses into the local database. Chunks that are already +// available are fetched back-to-back with no delay; wait_ms/max_retries are spent +// only while the server payload is not yet ready (HTTP 202). max_chunks > 0 caps how +// many chunks are applied this call (caller-driven progress/traffic control); +// max_chunks <= 0 drains everything available. The in-memory page cursor persists on +// the network context, so a capped drain resumes where it left off on the next call. +// Fills *dr and sr->tables_json (the union of tables touched across the drain). +static int network_drain_changes (sqlite3_context *context, sync_result *sr, + int wait_ms, int max_retries, int max_chunks, + drain_result *dr) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + // Capture the local db_version once before draining so the affected-tables union + // across all drained chunks can be computed with a single query at the end. + int64_t drain_prev_dbv = cloudsync_dbversion(data); + sr->defer_tables = true; + + int ntries = 0; // counts only "nothing ready" (202) polls + int nrows_total = 0; // cumulative rows applied across the whole drain + int nchunks = 0; // payload chunks applied this call + int64_t bytes_total = 0; // serialized payload bytes received this call + bool complete = true; // false iff the stream is known to have more pending + bool check_permanent_failure = false; // server reported a non-retryable failures.check + char *receive_err = NULL; + int rc = SQLITE_OK; + for (;;) { + if (receive_err) { cloudsync_memory_free(receive_err); receive_err = NULL; } + + int request_max_chunks = 0; + if (max_chunks > 0) { + int remaining = max_chunks - nchunks; + if (remaining <= 0) break; + request_max_chunks = remaining; + } + int safety_remaining = CLOUDSYNC_CHECK_MAX_CHUNKS_PER_DRAIN - nchunks; + if (safety_remaining <= 0) { complete = false; break; } + if (request_max_chunks > 0) { + if (safety_remaining < request_max_chunks) request_max_chunks = safety_remaining; + } else if (nchunks > 0) { + request_max_chunks = safety_remaining; + } + + int nrows = 0; + rc = cloudsync_network_check_internal(context, &nrows, sr, &receive_err, request_max_chunks); + // a receive error (network or apply) won't fix itself across retries + if (rc != SQLITE_OK) { complete = false; break; } + + if (sr->page_delivered) { + nrows_total += nrows; // a staged (incomplete) fragment contributes 0 + bytes_total += sr->bytes_received; + nchunks += sr->chunks_received; + complete = !sr->more_pending; // reflects whether the stream is finished + if (!sr->more_pending) break; // final batch -> drained + if (max_chunks > 0 && nchunks >= max_chunks) break; // caller cap: more pending + if (nchunks >= CLOUDSYNC_CHECK_MAX_CHUNKS_PER_DRAIN) break; // safety bound: more pending + continue; // keep draining immediately + } + + // A non-retryable server-side check failure (e.g. a permission/authorization + // error) won't clear by waiting: stop draining now instead of polling out the + // remaining retries. The failure object stays in sr->check_failure_json so the + // caller can surface it (receive.lastFailure / a raised error). + if (sr->check_failure_json && + !json_extract_bool(sr->check_failure_json, strlen(sr->check_failure_json), "retryable", true)) { + check_permanent_failure = true; + complete = false; + break; + } + + // nothing delivered (202 / up to date): preserve the polling-for-changes semantics. + // complete is left as-is (true if no page was ever delivered; false if the last + // delivered page was non-final), so a 202 after partial pages reports incomplete. + if (ntries + 1 >= max_retries) break; + ntries++; + sqlite3_sleep(wait_ms); + } + + // Compute the affected-tables union once, over the whole drain window. + if (!receive_err && rc == SQLITE_OK && nrows_total > 0) { + sr->tables_json = network_get_affected_tables(db, drain_prev_dbv); + } + + dr->rows = nrows_total; + dr->chunks = nchunks; + dr->bytes = bytes_total; + dr->complete = complete; + dr->check_permanent_failure = check_permanent_failure; + dr->receive_err = receive_err; + return rc; +} + +void cloudsync_network_sync (sqlite3_context *context, int wait_ms, int max_retries) { + sync_result sr = {.server_version = -1}; + int rc = cloudsync_network_send_changes_internal(context, 0, NULL, &sr); + if (rc != SQLITE_OK) { + if (sr.apply_failure_json) cloudsync_memory_free(sr.apply_failure_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); + return; + } + + // Drain the whole receive stream in one call (max_chunks=0 => unlimited). + drain_result dr = {0}; + rc = network_drain_changes(context, &sr, wait_ms, max_retries, 0, &dr); + char *receive_err = dr.receive_err; + int nrows_total = dr.rows; + + // If the receive phase failed, still emit structured JSON so the caller + // sees that the send phase completed and understands why receive did not. + if (rc != SQLITE_OK && !receive_err) { + receive_err = cloudsync_string_dup("receive failed"); + } + if (receive_err) { + rc = SQLITE_OK; + nrows_total = 0; + if (sr.tables_json) { cloudsync_memory_free(sr.tables_json); sr.tables_json = NULL; } + } + + const char *tables = sr.tables_json ? sr.tables_json : "[]"; + const char *complete_str = dr.complete ? "true" : "false"; + const char *status = sr.status ? sr.status : "error"; + char *escaped_err = receive_err ? json_escape_string(receive_err) : NULL; + + // Build send and receive blocks separately to avoid combinatorial explosion + // across optional fields (send.lastFailure, receive.error, receive.lastFailure). + char *send_part = sr.apply_failure_json + ? cloudsync_memory_mprintf( + "\"send\":{\"status\":\"%s\",\"localVersion\":%lld,\"serverVersion\":%lld,\"chunks\":%d,\"bytes\":%lld,\"lastFailure\":%s}", + status, (long long)sr.local_version, (long long)sr.server_version, sr.send_chunks, (long long)sr.send_bytes, sr.apply_failure_json) + : cloudsync_memory_mprintf( + "\"send\":{\"status\":\"%s\",\"localVersion\":%lld,\"serverVersion\":%lld,\"chunks\":%d,\"bytes\":%lld}", + status, (long long)sr.local_version, (long long)sr.server_version, sr.send_chunks, (long long)sr.send_bytes); + + char *recv_part; + if (escaped_err && sr.check_failure_json) { + recv_part = cloudsync_memory_mprintf( + "\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\",\"lastFailure\":%s}", + nrows_total, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped_err, sr.check_failure_json); + } else if (escaped_err) { + recv_part = cloudsync_memory_mprintf( + "\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\"}", + nrows_total, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped_err); + } else if (sr.check_failure_json) { + recv_part = cloudsync_memory_mprintf( + "\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"lastFailure\":%s}", + nrows_total, tables, dr.chunks, (long long)dr.bytes, complete_str, sr.check_failure_json); + } else { + recv_part = cloudsync_memory_mprintf( + "\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s}", + nrows_total, tables, dr.chunks, (long long)dr.bytes, complete_str); + } + + char *buf = cloudsync_memory_mprintf("{%s,%s}", send_part, recv_part); + cloudsync_memory_free(send_part); + cloudsync_memory_free(recv_part); + + sqlite3_result_text(context, buf, -1, cloudsync_memory_free); + if (escaped_err) cloudsync_memory_free(escaped_err); + if (receive_err) cloudsync_memory_free(receive_err); + if (sr.tables_json) cloudsync_memory_free(sr.tables_json); + if (sr.apply_failure_json) cloudsync_memory_free(sr.apply_failure_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); +} + +void cloudsync_network_sync0 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_sync2"); + + cloudsync_network_sync(context, DEFAULT_SYNC_WAIT_MS, DEFAULT_SYNC_MAX_RETRIES); +} + + +void cloudsync_network_sync2 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_sync2"); + + int wait_ms = sqlite3_value_int(argv[0]); + int max_retries = sqlite3_value_int(argv[1]); + + cloudsync_network_sync(context, wait_ms, max_retries); +} + + +// Shared implementation for cloudsync_network_receive_changes() and its deprecated +// alias cloudsync_network_check_changes(). Drains available chunks (no polling-wait +// for server preparation: wait_ms=0, max_retries=1) and emits a receive-only JSON +// block. max_chunks <= 0 drains everything available; max_chunks > 0 caps the chunks +// applied this call so a caller can drive progress/traffic — the in-memory page +// cursor persists on the network context, so the next call resumes the drain. +static void network_receive_changes_impl (sqlite3_context *context, int max_chunks) { + // check-scoped: emits receive.error (client-side apply) and/or + // receive.lastFailure (server-side failures.check) only — never a send block. + sync_result sr = {.server_version = -1}; + drain_result dr = {0}; + int rc = network_drain_changes(context, &sr, 0, 1, max_chunks, &dr); + char *receive_err = dr.receive_err; + int nrows = dr.rows; + + // Endpoint/network errors already raised a SQL error on the context + if (rc != SQLITE_OK && !receive_err) { + if (sr.tables_json) cloudsync_memory_free(sr.tables_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); + return; + } + + // A non-retryable server-side check failure is a permanent configuration/ + // authorization problem, not a transient "nothing ready yet": raise a SQL error so + // a polling caller fails fast instead of looping until it times out. (receive is a + // pure receive op with no send block to preserve, so a raised error is unambiguous.) + if (dr.check_permanent_failure && sr.check_failure_json) { + char *code = json_extract_string(sr.check_failure_json, strlen(sr.check_failure_json), "code"); + char *message = json_extract_string(sr.check_failure_json, strlen(sr.check_failure_json), "message"); + char *err = cloudsync_memory_mprintf( + "cloudsync_network_receive_changes: server rejected check (non-retryable): %s%s%s", + code ? code : "check failed", + message ? " - " : "", + message ? message : ""); + sqlite3_result_error(context, err ? err : "cloudsync_network_receive_changes: server rejected check (non-retryable).", -1); + if (err) cloudsync_memory_free(err); + if (code) cloudsync_memory_free(code); + if (message) cloudsync_memory_free(message); + if (receive_err) cloudsync_memory_free(receive_err); + if (sr.tables_json) cloudsync_memory_free(sr.tables_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); + return; + } + + // Apply errors → structured JSON with receive.error + if (receive_err) { + nrows = 0; + if (sr.tables_json) { cloudsync_memory_free(sr.tables_json); sr.tables_json = NULL; } + } + + const char *tables = sr.tables_json ? sr.tables_json : "[]"; + const char *complete_str = dr.complete ? "true" : "false"; + char *escaped = receive_err ? json_escape_string(receive_err) : NULL; + char *buf; + if (escaped && sr.check_failure_json) { + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\",\"lastFailure\":%s}}", + nrows, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped, sr.check_failure_json); + } else if (escaped) { + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\"}}", + nrows, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped); + } else if (sr.check_failure_json) { + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"lastFailure\":%s}}", + nrows, tables, dr.chunks, (long long)dr.bytes, complete_str, sr.check_failure_json); + } else { + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s}}", + nrows, tables, dr.chunks, (long long)dr.bytes, complete_str); + } + sqlite3_result_text(context, buf, -1, cloudsync_memory_free); + if (escaped) cloudsync_memory_free(escaped); + if (receive_err) cloudsync_memory_free(receive_err); + if (sr.tables_json) cloudsync_memory_free(sr.tables_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); +} + +void cloudsync_network_receive_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_receive_changes"); + network_receive_changes_impl(context, 0); +} + +void cloudsync_network_receive_changes1 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_receive_changes"); + int max_chunks = sqlite3_value_int(argv[0]); + network_receive_changes_impl(context, max_chunks); +} + +// Deprecated: use cloudsync_network_receive_changes(). Kept as a thin alias for +// backward compatibility; the name will be removed in a future major version. +void cloudsync_network_check_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_check_changes"); + network_receive_changes_impl(context, 0); +} + +void cloudsync_network_check_changes1 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_check_changes"); + int max_chunks = sqlite3_value_int(argv[0]); + network_receive_changes_impl(context, max_chunks); +} + +void cloudsync_network_reset_sync_version (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_reset_sync_version"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + char *buf = "0"; + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_DBVERSION, buf); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_SEQ, buf); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_SEND_DBVERSION, buf); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_SEND_SEQ, buf); +} + +/** + * Cleanup all local data from cloudsync-enabled tables, so the database can be safely reused + * by another user without exposing any data from the previous session. + * + * Warning: this function deletes all data from the tables. Use with caution. + */ +void cloudsync_network_logout (sqlite3_context *context, int argc, sqlite3_value **argv) { + bool savepoint_created = false; + bool completed = false; + char *errmsg = NULL; + int rc = SQLITE_ERROR; + sqlite3 *db = sqlite3_context_db_handle(context); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // if the network layer is enabled, remove the token or apikey + sqlite3_exec(db, "SELECT cloudsync_network_set_token('');", NULL, NULL, NULL); + + // get the list of cloudsync-enabled tables + char *sql = "SELECT tbl_name, key, value FROM cloudsync_table_settings;"; + char **result = NULL; + int nrows, ncols; + rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, NULL); + if (rc != SQLITE_OK) { + errmsg = cloudsync_memory_mprintf("Unable to get current cloudsync configuration %s", sqlite3_errmsg(db)); + goto finalize; + } + + // run everything in a savepoint + rc = database_begin_savepoint(data, "cloudsync_logout_savepoint"); + if (rc != SQLITE_OK) { + errmsg = cloudsync_memory_mprintf("Unable to create cloudsync_logout savepoint %s", cloudsync_errmsg(data)); + goto finalize; + } + savepoint_created = true; + + rc = cloudsync_cleanup_all(data); + if (rc != SQLITE_OK) { + errmsg = cloudsync_memory_mprintf("Unable to cleanup current database %s", cloudsync_errmsg(data)); + goto finalize; + } + + // delete all the local data for each previously enabled table + // re-enable cloudsync on previously enabled tables + for (int i = 1; i <= nrows; i++) { + char *tbl_name = result[i * ncols + 0]; + char *key = result[i * ncols + 1]; + char *value = result[i * ncols + 2]; + + if (strcmp(key, "algo") != 0) continue; + + sql = cloudsync_memory_mprintf("DELETE FROM \"%w\";", tbl_name); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + cloudsync_memory_free(sql); + if (rc != SQLITE_OK) { + errmsg = cloudsync_memory_mprintf("Unable to delete data from table %s. %s", tbl_name, sqlite3_errmsg(db)); + goto finalize; + } + + sql = cloudsync_memory_mprintf("SELECT cloudsync_init('%q', '%q', 1);", tbl_name, value); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + cloudsync_memory_free(sql); + if (rc != SQLITE_OK) { + errmsg = cloudsync_memory_mprintf("Unable to enable cloudsync on table %s. %s", tbl_name, sqlite3_errmsg(db)); + goto finalize; + } + } + + completed = true; + +finalize: + if (completed) { + database_commit_savepoint(data, "cloudsync_logout_savepoint"); + cloudsync_network_cleanup_internal(context); + sqlite3_result_int(context, SQLITE_OK); + } else { + // cleanup: + // ROLLBACK TO command reverts the state of the database back to what it was just after the corresponding SAVEPOINT + // then RELEASE to remove the SAVEPOINT from the transaction stack + if (savepoint_created) database_rollback_savepoint(data, "cloudsync_logout_savepoint"); + sqlite3_result_error(context, errmsg, -1); + sqlite3_result_error_code(context, rc); + } + sqlite3_free_table(result); + cloudsync_memory_free(errmsg); +} + +void cloudsync_network_status (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_status"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata) { + sqlite3_result_error(context, "Unable to retrieve CloudSync network context.", -1); + return; + } + + NETWORK_RESULT res = network_receive_buffer(netdata, netdata->status_endpoint, netdata->authentication, true, false, NULL, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + network_set_sqlite_result(context, &res); + network_result_cleanup(&res); +} + +// MARK: - + +int cloudsync_network_register (sqlite3 *db, char **pzErrMsg, void *ctx) { + const int DEFAULT_FLAGS = SQLITE_UTF8 | SQLITE_INNOCUOUS; + int rc = SQLITE_OK; + + rc = sqlite3_create_function(db, "cloudsync_network_init", 1, DEFAULT_FLAGS, ctx, cloudsync_network_init, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_create_function(db, "cloudsync_network_init_custom", 2, DEFAULT_FLAGS, ctx, cloudsync_network_init_custom, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_cleanup", 0, DEFAULT_FLAGS, ctx, cloudsync_network_cleanup, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_set_token", 1, DEFAULT_FLAGS, ctx, cloudsync_network_set_token, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_set_apikey", 1, DEFAULT_FLAGS, ctx, cloudsync_network_set_apikey, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_has_unsent_changes", 0, DEFAULT_FLAGS, ctx, cloudsync_network_has_unsent_changes, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_send_changes", 0, DEFAULT_FLAGS, ctx, cloudsync_network_send_changes, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_receive_changes", 0, DEFAULT_FLAGS, ctx, cloudsync_network_receive_changes, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_receive_changes", 1, DEFAULT_FLAGS, ctx, cloudsync_network_receive_changes1, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + // Deprecated alias of cloudsync_network_receive_changes(); kept for backward compatibility. + rc = sqlite3_create_function(db, "cloudsync_network_check_changes", 0, DEFAULT_FLAGS, ctx, cloudsync_network_check_changes, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_check_changes", 1, DEFAULT_FLAGS, ctx, cloudsync_network_check_changes1, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_sync", 0, DEFAULT_FLAGS, ctx, cloudsync_network_sync0, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_sync", 2, DEFAULT_FLAGS, ctx, cloudsync_network_sync2, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_reset_sync_version", 0, DEFAULT_FLAGS, ctx, cloudsync_network_reset_sync_version, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_logout", 0, DEFAULT_FLAGS, ctx, cloudsync_network_logout, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_status", 0, DEFAULT_FLAGS, ctx, cloudsync_network_status, NULL, NULL); + if (rc != SQLITE_OK) return rc; + +cleanup: + if ((rc != SQLITE_OK) && (pzErrMsg)) { + *pzErrMsg = sqlite3_mprintf("Error creating function in cloudsync_network_register: %s", sqlite3_errmsg(db)); + } + + return rc; +} + +#endif diff --git a/src/network.h b/src/network/network.h similarity index 68% rename from src/network.h rename to src/network/network.h index 73b7c792..0c7e7de4 100644 --- a/src/network.h +++ b/src/network/network.h @@ -8,7 +8,13 @@ #ifndef __CLOUDSYNC_NETWORK__ #define __CLOUDSYNC_NETWORK__ -#include "cloudsync.h" +#include "../cloudsync.h" + +#ifndef SQLITE_CORE +#include "sqlite3ext.h" +#else +#include "sqlite3.h" +#endif int cloudsync_network_register (sqlite3 *db, char **pzErrMsg, void *ctx); diff --git a/src/network.m b/src/network/network.m similarity index 50% rename from src/network.m rename to src/network/network.m index 6f4d2c1c..89c00f98 100644 --- a/src/network.m +++ b/src/network/network.m @@ -13,63 +13,18 @@ void network_buffer_cleanup (void *xdata) { if (xdata) CFRelease(xdata); } -bool network_compute_endpoints (sqlite3_context *context, network_data *data, const char *conn_string) { - NSString *conn = [NSString stringWithUTF8String:conn_string]; - NSString *conn_string_https = nil; - - if ([conn hasPrefix:@"sqlitecloud://"]) { - conn_string_https = [conn stringByReplacingCharactersInRange:NSMakeRange(0, [@"sqlitecloud://" length]) withString:@"https://"]; - } else { - conn_string_https = conn; - } - - NSURL *url = [NSURL URLWithString:conn_string_https]; - if (!url) return false; - - NSString *scheme = url.scheme; // "https" - if (!scheme) return false; - NSString *host = url.host; // "cn5xiooanz.global3.ryujaz.sqlite.cloud" - if (!host) return false; - - NSString *port = url.port.stringValue; - NSString *database = url.path; // "/chinook-cloudsync.sqlite" - if (!database) return false; - - NSString *query = url.query; // "apikey=hWDanFolRT9WDK0p54lufNrIyfgLZgtMw6tb6fbPmpo" (OPTIONAL) - NSString *authentication = nil; - - if (query) { - NSURLComponents *components = [NSURLComponents componentsWithString:[@"http://dummy?" stringByAppendingString:query]]; - NSArray *items = components.queryItems; - for (NSURLQueryItem *item in items) { - // build new token - // apikey: just write the key for retrocompatibility - // other keys, like token: add a prefix, i.e. token= - - if ([item.name isEqualToString:@"apikey"]) { - authentication = item.value; - break; - } - if ([item.name isEqualToString:@"token"]) { - authentication = [NSString stringWithFormat:@"%@=%@", item.name, item.value]; - break; - } - } - } - - char *site_id = network_data_get_siteid(data); - char *port_or_default = (port && strcmp(port.UTF8String, "8860") != 0) ? (char *)port.UTF8String : CLOUDSYNC_DEFAULT_ENDPOINT_PORT; - - NSString *check_endpoint = [NSString stringWithFormat:@"%s://%s:%s/%s%s/%s", scheme.UTF8String, host.UTF8String, port_or_default, CLOUDSYNC_ENDPOINT_PREFIX, database.UTF8String, site_id]; - NSString *upload_endpoint = [NSString stringWithFormat: @"%s://%s:%s/%s%s/%s/%s", scheme.UTF8String, host.UTF8String, port_or_default, CLOUDSYNC_ENDPOINT_PREFIX, database.UTF8String, site_id, CLOUDSYNC_ENDPOINT_UPLOAD]; - - return network_data_set_endpoints(data, (char *)authentication.UTF8String, (char *)check_endpoint.UTF8String, (char *)upload_endpoint.UTF8String, true); -} - bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { +#ifdef CLOUDSYNC_NETWORK_TRACE + double trace_start_ms = network_trace_now_ms(); +#endif NSString *urlString = [NSString stringWithUTF8String:endpoint]; NSURL *url = [NSURL URLWithString:urlString]; - if (!url) return false; + if (!url) { + #ifdef CLOUDSYNC_NETWORK_TRACE + network_trace_log(data, "PUT", endpoint, 0, CLOUDSYNC_NETWORK_ERROR, (size_t)blob_size, 0, network_trace_now_ms() - trace_start_ms); + #endif + return false; + } NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url]; [request setHTTPMethod:@"PUT"]; @@ -81,21 +36,27 @@ bool network_send_buffer(network_data *data, const char *endpoint, const char *a [request setValue:authString forHTTPHeaderField:@"Authorization"]; } + char *org_id = network_data_get_orgid(data); + if (org_id) { + [request setValue:[NSString stringWithUTF8String:org_id] forHTTPHeaderField:@CLOUDSYNC_HEADER_ORG]; + } + NSData *bodyData = [NSData dataWithBytes:blob length:blob_size]; [request setHTTPBody:bodyData]; __block bool success = false; + __block NSInteger statusCode = 0; dispatch_semaphore_t sema = dispatch_semaphore_create(0); NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; NSURLSession *session = [NSURLSession sessionWithConfiguration:config]; NSURLSessionDataTask *task = [session dataTaskWithRequest:request - completionHandler:^(NSData * _Nullable data, + completionHandler:^(NSData * _Nullable responseBody, NSURLResponse * _Nullable response, NSError * _Nullable error) { if (!error && [response isKindOfClass:[NSHTTPURLResponse class]]) { - NSInteger statusCode = [(NSHTTPURLResponse *)response statusCode]; + statusCode = [(NSHTTPURLResponse *)response statusCode]; success = (statusCode >= 200 && statusCode < 300); } dispatch_semaphore_signal(sema); @@ -103,12 +64,30 @@ bool network_send_buffer(network_data *data, const char *endpoint, const char *a [task resume]; dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER); + [session finishTasksAndInvalidate]; + + #ifdef CLOUDSYNC_NETWORK_TRACE + network_trace_log(data, "PUT", endpoint, (long)statusCode, + success ? CLOUDSYNC_NETWORK_OK : CLOUDSYNC_NETWORK_ERROR, + (size_t)blob_size, + success ? (size_t)blob_size : 0, + network_trace_now_ms() - trace_start_ms); + #endif return success; } -NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header) { +NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char **extra_headers, int nextra_headers) { +#ifdef CLOUDSYNC_NETWORK_TRACE + double trace_start_ms = network_trace_now_ms(); + size_t request_bytes = json_payload ? strlen(json_payload) : 0; +#endif + const char *method = (json_payload || is_post_request) ? "POST" : "GET"; + bool using_ticket = network_data_should_use_ticket(data, endpoint, authentication); +#ifndef CLOUDSYNC_NETWORK_TRACE + (void)method; +#endif NSString *urlString = [NSString stringWithUTF8String:endpoint]; NSURL *url = [NSURL URLWithString:urlString]; @@ -119,24 +98,38 @@ NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, result.buffer = (char *)msg.UTF8String; result.xdata = (void *)CFBridgingRetain(msg); result.xfree = network_buffer_cleanup; + #ifdef CLOUDSYNC_NETWORK_TRACE + network_trace_log(data, method, endpoint, 0, result.code, request_bytes, 0, network_trace_now_ms() - trace_start_ms); + #endif return result; } NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url]; request.HTTPMethod = (json_payload || is_post_request) ? @"POST" : @"GET"; - if (custom_header) { - NSString *header = [NSString stringWithUTF8String:custom_header]; - NSArray *parts = [header componentsSeparatedByString:@": "]; - if (parts.count == 2) { - [request setValue:parts[1] forHTTPHeaderField:parts[0]]; + for (int i = 0; i < nextra_headers; i++) { + NSString *header = [NSString stringWithUTF8String:extra_headers[i]]; + NSRange sep = [header rangeOfString:@": "]; + if (sep.location != NSNotFound) { + NSString *name = [header substringToIndex:sep.location]; + NSString *value = [header substringFromIndex:sep.location + sep.length]; + [request setValue:value forHTTPHeaderField:name]; } } + char *org_id = network_data_get_orgid(data); + if (org_id) { + [request setValue:[NSString stringWithUTF8String:org_id] forHTTPHeaderField:@CLOUDSYNC_HEADER_ORG]; + } + if (authentication) { NSString *authString = [NSString stringWithFormat:@"Bearer %s", authentication]; [request setValue:authString forHTTPHeaderField:@"Authorization"]; } + if (using_ticket) { + char *ticket = network_data_get_ticket(data); + [request setValue:[NSString stringWithUTF8String:ticket] forHTTPHeaderField:@CLOUDSYNC_HEADER_TICKET]; + } if (json_payload) { [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"]; @@ -150,29 +143,57 @@ NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, __block NSString *responseError = nil; __block NSInteger statusCode = 0; __block NSInteger errorCode = 0; + __block NSString *responseTicket = nil; + __block NSString *responseTicketExpiresAt = nil; dispatch_semaphore_t sema = dispatch_semaphore_create(0); - NSURLSession *session = [NSURLSession sharedSession]; - NSURLSessionDataTask *task = [session dataTaskWithRequest:request completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) { - responseData = data; + NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; + NSURLSession *session = [NSURLSession sessionWithConfiguration:config]; + NSURLSessionDataTask *task = [session dataTaskWithRequest:request completionHandler:^(NSData *responseBody, NSURLResponse *response, NSError *error) { + responseData = responseBody; if (error) { responseError = [error localizedDescription]; errorCode = [error code]; } if ([response isKindOfClass:[NSHTTPURLResponse class]]) { - statusCode = [(NSHTTPURLResponse *)response statusCode]; + NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response; + statusCode = [httpResponse statusCode]; + NSDictionary *headers = [httpResponse allHeaderFields]; + for (id key in headers) { + NSString *name = [key description]; + NSString *value = [[headers objectForKey:key] description]; + if ([name caseInsensitiveCompare:@CLOUDSYNC_HEADER_TICKET] == NSOrderedSame) { + responseTicket = value; + } else if ([name caseInsensitiveCompare:@CLOUDSYNC_HEADER_TICKET_EXPIRES_AT] == NSOrderedSame) { + responseTicketExpiresAt = value; + } + } } dispatch_semaphore_signal(sema); }]; [task resume]; dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER); + [session finishTasksAndInvalidate]; + + if (!responseError && (statusCode >= 200 && statusCode < 300) && responseTicket && [responseTicket length] > 0) { + network_data_update_ticket(data, [responseTicket UTF8String], + responseTicketExpiresAt ? [responseTicketExpiresAt UTF8String] : NULL); + } if (!responseError && (statusCode >= 200 && statusCode < 300)) { // check if OK should be returned if (responseData == nil || [responseData length] == 0) { - return (NETWORK_RESULT){CLOUDSYNC_NETWORK_OK, NULL, 0, NULL, NULL}; + NETWORK_RESULT result = {CLOUDSYNC_NETWORK_OK, NULL, 0, NULL, NULL}; + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] endpoint=%s using_ticket=%s\n", + network_trace_endpoint_name(data, endpoint), + using_ticket ? "true" : "false"); + network_trace_log(data, method, endpoint, (long)statusCode, result.code, request_bytes, 0, network_trace_now_ms() - trace_start_ms); + #endif + return result; } // otherwise return a buffer @@ -180,6 +201,14 @@ NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, result.code = CLOUDSYNC_NETWORK_BUFFER; if (zero_terminated) { NSString *utf8String = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding]; + if (!utf8String) { + NSString *msg = @"Response is not valid UTF-8"; + NETWORK_RESULT error_result = {CLOUDSYNC_NETWORK_ERROR, (char *)msg.UTF8String, 0, (void *)CFBridgingRetain(msg), network_buffer_cleanup}; + #ifdef CLOUDSYNC_NETWORK_TRACE + network_trace_log(data, method, endpoint, (long)statusCode, error_result.code, request_bytes, 0, network_trace_now_ms() - trace_start_ms); + #endif + return error_result; + } result.buffer = (char *)utf8String.UTF8String; result.xdata = (void *)CFBridgingRetain(utf8String); } else { @@ -189,6 +218,13 @@ NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, result.blen = [responseData length]; result.xfree = network_buffer_cleanup; + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] endpoint=%s using_ticket=%s\n", + network_trace_endpoint_name(data, endpoint), + using_ticket ? "true" : "false"); + network_trace_log(data, method, endpoint, (long)statusCode, result.code, request_bytes, result.blen, network_trace_now_ms() - trace_start_ms); + #endif return result; } @@ -212,5 +248,12 @@ NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, result.xfree = network_buffer_cleanup; result.blen = responseError ? (size_t)errorCode : (size_t)statusCode; + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] endpoint=%s using_ticket=%s\n", + network_trace_endpoint_name(data, endpoint), + using_ticket ? "true" : "false"); + network_trace_log(data, method, endpoint, (long)statusCode, result.code, request_bytes, 0, network_trace_now_ms() - trace_start_ms); + #endif return result; } diff --git a/src/network/network_private.h b/src/network/network_private.h new file mode 100644 index 00000000..21a46fd4 --- /dev/null +++ b/src/network/network_private.h @@ -0,0 +1,65 @@ +// +// network_private.h +// cloudsync +// +// Created by Marco Bambini on 23/05/25. +// + +#ifndef __CLOUDSYNC_NETWORK_PRIVATE__ +#define __CLOUDSYNC_NETWORK_PRIVATE__ + +#include +#include + +#define CLOUDSYNC_DEFAULT_ADDRESS "https://cloudsync.sqlite.ai" +#define CLOUDSYNC_ENDPOINT_PREFIX "v2/cloudsync/databases" +#define CLOUDSYNC_ENDPOINT_UPLOAD "upload" +#define CLOUDSYNC_ENDPOINT_CHECK "check" +#define CLOUDSYNC_ENDPOINT_APPLY "apply" +#define CLOUDSYNC_ENDPOINT_STATUS "status" +#define CLOUDSYNC_HEADER_ORG "X-CloudSync-Org" +#define CLOUDSYNC_HEADER_VERSION "X-CloudSync-Version" +#define CLOUDSYNC_HEADER_TICKET "X-CloudSync-Ticket" +#define CLOUDSYNC_HEADER_TICKET_EXPIRES_AT "X-CloudSync-Ticket-Expires-At" +// CLOUDSYNC_VERSION is defined in cloudsync.h — include it before this header at use sites. +#define CLOUDSYNC_HEADER_VERSION_LINE CLOUDSYNC_HEADER_VERSION ": " CLOUDSYNC_VERSION +#define CLOUDSYNC_HEADER_CHECK_CAPABILITIES "X-CloudSync-Capabilities: check-status-response, check-chunks" + +#define CLOUDSYNC_NETWORK_OK 1 +#define CLOUDSYNC_NETWORK_ERROR 2 +#define CLOUDSYNC_NETWORK_BUFFER 3 + +typedef struct network_data network_data; + +typedef struct { + int code; // network code: OK, ERROR, BUFFER + char *buffer; // network buffer + size_t blen; // blen if code is SQLITE_OK, rc in case of error + void *xdata; // optional custom external data + void (*xfree) (void *); // optional custom free callback +} NETWORK_RESULT; + +char *network_data_get_siteid (network_data *data); +char *network_data_get_orgid (network_data *data); +char *network_data_get_ticket (network_data *data); +bool network_data_should_use_ticket (network_data *data, const char *endpoint, const char *authentication); +void network_data_update_ticket (network_data *data, const char *ticket, const char *expires_at); +bool network_data_set_endpoints (network_data *data, char *auth, char *check, char *upload, char *apply, char *status); + +bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size); +NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char **extra_headers, int nextra_headers); + +// Exposed (non-static) for the network unit test; otherwise internal to network.c. +void network_sync_state_update_from_response(NETWORK_RESULT *res, int64_t *last_optimistic_version, int64_t *last_confirmed_version, int *gaps_size, char **apply_failure_json, char **check_failure_json); +const char *network_compute_status(int64_t last_optimistic, int64_t last_confirmed, int gaps_size, int64_t local_version); +char *network_apply_json_payload(const char *transport_key, const char *transport_value, int64_t db_version_min, int64_t db_version_max, const char *batch_id, int chunk_index, bool is_final); + +#ifdef CLOUDSYNC_NETWORK_TRACE +const char *network_trace_endpoint_name(network_data *data, const char *endpoint); +const char *network_trace_result_name(int code); +void network_trace_log(network_data *data, const char *method, const char *endpoint, long http_status, int result_code, size_t request_bytes, size_t bytes, double elapsed_ms); +double network_trace_now_ms(void); +#endif + + +#endif diff --git a/src/network_private.h b/src/network_private.h deleted file mode 100644 index cd970c5f..00000000 --- a/src/network_private.h +++ /dev/null @@ -1,39 +0,0 @@ -// -// network_private.h -// cloudsync -// -// Created by Marco Bambini on 23/05/25. -// - -#ifndef __CLOUDSYNC_NETWORK_PRIVATE__ -#define __CLOUDSYNC_NETWORK_PRIVATE__ - -#define CLOUDSYNC_ENDPOINT_PREFIX "v1/cloudsync" -#define CLOUDSYNC_ENDPOINT_UPLOAD "upload" -#define CLOUDSYNC_ENDPOINT_CHECK "check" -#define CLOUDSYNC_DEFAULT_ENDPOINT_PORT "443" -#define CLOUDSYNC_HEADER_SQLITECLOUD "Accept: sqlc/plain" - -#define CLOUDSYNC_NETWORK_OK 1 -#define CLOUDSYNC_NETWORK_ERROR 2 -#define CLOUDSYNC_NETWORK_BUFFER 3 - -typedef struct network_data network_data; - -typedef struct { - int code; // network code: OK, ERROR, BUFFER - char *buffer; // network buffer - size_t blen; // blen if code is SQLITE_OK, rc in case of error - void *xdata; // optional custom external data - void (*xfree) (void *); // optional custom free callback -} NETWORK_RESULT; - -char *network_data_get_siteid (network_data *data); -bool network_data_set_endpoints (network_data *data, char *auth, char *check, char *upload, bool duplicate); - -bool network_compute_endpoints (sqlite3_context *context, network_data *data, const char *conn_string); -bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size); -NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header); - - -#endif diff --git a/src/pk.c b/src/pk.c index ae605d15..dcc8ca67 100644 --- a/src/pk.c +++ b/src/pk.c @@ -7,11 +7,14 @@ #include "pk.h" #include "utils.h" +#include "cloudsync_endian.h" +#include "cloudsync.h" -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT3 -#endif - +#include +#include +#include +#include + /* The pk_encode and pk_decode functions are designed to serialize and deserialize an array of values (sqlite_value structures) @@ -67,45 +70,50 @@ SQLITE_EXTENSION_INIT3 * Versatility: The ability to handle multiple data types and variable-length data makes this solution suitable for complex data structures. * Simplicity: The functions are designed to be straightforward to use, with clear memory management responsibilities. + Notes + + * Floating point values are encoded as IEEE754 double, 64-bit, big-endian byte order. + */ // Three bits are reserved for the type field, so only values in the 0..7 range can be used (8 values) // SQLITE already reserved values from 1 to 5 -// #define SQLITE_INTEGER 1 -// #define SQLITE_FLOAT 2 -// #define SQLITE_TEXT 3 -// #define SQLITE_BLOB 4 -// #define SQLITE_NULL 5 -#define SQLITE_NEGATIVE_INTEGER 0 -#define SQLITE_MAX_NEGATIVE_INTEGER 6 -#define SQLITE_NEGATIVE_FLOAT 7 +// #define SQLITE_INTEGER 1 // now DBTYPE_INTEGER +// #define SQLITE_FLOAT 2 // now DBTYPE_FLOAT +// #define SQLITE_TEXT 3 // now DBTYPE_TEXT +// #define SQLITE_BLOB 4 // now DBTYPE_BLOB +// #define SQLITE_NULL 5 // now DBTYPE_NULL +#define DATABASE_TYPE_NEGATIVE_INTEGER 0 // was SQLITE_NEGATIVE_INTEGER +#define DATABASE_TYPE_MAX_NEGATIVE_INTEGER 6 // was SQLITE_MAX_NEGATIVE_INTEGER +#define DATABASE_TYPE_NEGATIVE_FLOAT 7 // was SQLITE_NEGATIVE_FLOAT -// MARK: - Decoding - +char * const PRIKEY_NULL_CONSTRAINT_ERROR = "PRIKEY_NULL_CONSTRAINT_ERROR"; + +// MARK: - Public Callbacks - int pk_decode_bind_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { - // default decode callback used to bind values to a sqlite3_stmt vm + // default decode callback used to bind values to a dbvm_t vm - sqlite3_stmt *vm = (sqlite3_stmt *)xdata; - int rc = SQLITE_OK; + int rc = DBRES_OK; switch (type) { - case SQLITE_INTEGER: - rc = sqlite3_bind_int64(vm, index+1, ival); + case DBTYPE_INTEGER: + rc = databasevm_bind_int(xdata, index+1, ival); break; - case SQLITE_FLOAT: - rc = sqlite3_bind_double(vm, index+1, dval); + case DBTYPE_FLOAT: + rc = databasevm_bind_double(xdata, index+1, dval); break; - case SQLITE_NULL: - rc = sqlite3_bind_null(vm, index+1); + case DBTYPE_NULL: + rc = databasevm_bind_null(xdata, index+1); break; - case SQLITE_TEXT: - rc = sqlite3_bind_text(vm, index+1, pval, (int)ival, SQLITE_STATIC); + case DBTYPE_TEXT: + rc = databasevm_bind_text(xdata, index+1, pval, (int)ival); break; - case SQLITE_BLOB: - rc = sqlite3_bind_blob64(vm, index+1, (const void *)pval, (sqlite3_uint64)ival, SQLITE_STATIC); + case DBTYPE_BLOB: + rc = databasevm_bind_blob(xdata, index+1, (const void *)pval, ival); break; } @@ -114,108 +122,198 @@ int pk_decode_bind_callback (void *xdata, int index, int type, int64_t ival, dou int pk_decode_print_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { switch (type) { - case SQLITE_INTEGER: - printf("%d\tINTEGER:\t%lld\n", index, (long long)ival); + case DBTYPE_INTEGER: + printf("%d\tINTEGER:\t%" PRId64 "\n", index, ival); break; - case SQLITE_FLOAT: + case DBTYPE_FLOAT: printf("%d\tFLOAT:\t%.5f\n", index, dval); break; - case SQLITE_NULL: + case DBTYPE_NULL: printf("%d\tNULL\n", index); break; - case SQLITE_TEXT: - printf("%d\tTEXT:\t%s\n", index, pval); + case DBTYPE_TEXT: + printf("%d\tTEXT:\t%.*s\n", index, (int)ival, pval); break; - case SQLITE_BLOB: - printf("%d\tBLOB:\t%lld bytes\n", index, (long long)ival); + case DBTYPE_BLOB: + printf("%d\tBLOB:\t%" PRId64 " bytes\n", index, ival); break; } - return SQLITE_OK; + return DBRES_OK; +} + +uint64_t pk_checksum (const char *buffer, size_t blen) { + const uint8_t *p = (const uint8_t *)buffer; + uint64_t h = 14695981039346656037ULL; + for (size_t i = 0; i < blen; i++) { + h ^= p[i]; + h *= 1099511628211ULL; + } + return h; } -uint8_t pk_decode_u8 (char *buffer, size_t *bseek) { - uint8_t value = buffer[*bseek]; +// MARK: - Decoding - + +static inline int pk_decode_check_bounds (size_t bseek, size_t blen, size_t need) { + // bounds check helper for decoding + if (bseek > blen) return 0; + return need <= (blen - bseek); +} + +int pk_decode_u8 (const uint8_t *buffer, size_t blen, size_t *bseek, uint8_t *out) { + if (!pk_decode_check_bounds(*bseek, blen, 1)) return 0; + *out = buffer[*bseek]; *bseek += 1; - return value; + return 1; } -int64_t pk_decode_int64 (char *buffer, size_t *bseek, size_t nbytes) { - int64_t value = 0; +static int pk_decode_uint64 (const uint8_t *buffer, size_t blen, size_t *bseek, size_t nbytes, uint64_t *out) { + if (nbytes > 8) return 0; + if (!pk_decode_check_bounds(*bseek, blen, nbytes)) return 0; // decode bytes in big-endian order (most significant byte first) + uint64_t v = 0; for (size_t i = 0; i < nbytes; i++) { - value = (value << 8) | (uint8_t)buffer[*bseek]; + v = (v << 8) | (uint64_t)buffer[*bseek]; (*bseek)++; } - return value; + *out = v; + return 1; } -char *pk_decode_data (char *buffer, size_t *bseek, int32_t blen) { - char *value = buffer + *bseek; - *bseek += blen; +static int pk_decode_data (const uint8_t *buffer, size_t blen, size_t *bseek, size_t n, const uint8_t **out) { + if (!pk_decode_check_bounds(*bseek, blen, n)) return 0; + *out = buffer + *bseek; + *bseek += n; - return value; + return 1; } -double pk_decode_double (char *buffer, size_t *bseek) { - double value = 0; - int64_t int64value = pk_decode_int64(buffer, bseek, sizeof(int64_t)); - memcpy(&value, &int64value, sizeof(int64_t)); +int pk_decode_double (const uint8_t *buffer, size_t blen, size_t *bseek, double *out) { + // Doubles are encoded as IEEE754 64-bit, big-endian. + // Convert back to host order before memcpy into double. + + uint64_t bits_be = 0; + if (!pk_decode_uint64(buffer, blen, bseek, sizeof(uint64_t), &bits_be)) return 0; - return value; + uint64_t bits = be64_to_host(bits_be); + double value = 0.0; + memcpy(&value, &bits, sizeof(bits)); + *out = value; + return 1; } -int pk_decode(char *buffer, size_t blen, int count, size_t *seek, int (*cb) (void *xdata, int index, int type, int64_t ival, double dval, char *pval), void *xdata) { +int pk_decode (char *buffer, size_t blen, int count, size_t *seek, int skip_decode_idx, pk_decode_callback cb, void *xdata) { + const uint8_t *ubuf = (const uint8_t *)buffer; size_t bseek = (seek) ? *seek : 0; - if (count == -1) count = pk_decode_u8(buffer, &bseek); - - for (size_t i = 0; i < (size_t)count; i++) { - uint8_t type_byte = (uint8_t)pk_decode_u8(buffer, &bseek); - int type = (int)(type_byte & 0x07); - size_t nbytes = (type_byte >> 3) & 0x1F; + if (count == -1) { + uint8_t c = 0; + if (!pk_decode_u8(ubuf, blen, &bseek, &c)) return -1; + count = (int)c; + } - switch (type) { - case SQLITE_MAX_NEGATIVE_INTEGER: { - int64_t value = INT64_MIN; - type = SQLITE_INTEGER; - if (cb) if (cb(xdata, (int)i, type, value, 0.0, NULL) != SQLITE_OK) return -1; + for (size_t i = 0; i < (size_t)count; i++) { + uint8_t type_byte = 0; + if (!pk_decode_u8(ubuf, blen, &bseek, &type_byte)) return -1; + int raw_type = (int)(type_byte & 0x07); + size_t nbytes = (size_t)((type_byte >> 3) & 0x1F); + + // skip_decode wants the raw encoded slice (type_byte + optional len/int + payload) + // we still must parse with the *raw* type to know how much to skip + bool skip_decode = ((skip_decode_idx >= 0) && (i == (size_t)skip_decode_idx)); + size_t initial_bseek = bseek - 1; // points to type_byte + + switch (raw_type) { + case DATABASE_TYPE_MAX_NEGATIVE_INTEGER: { + // must not carry length bits + if (nbytes != 0) return -1; + if (skip_decode) { + size_t slice_len = bseek - initial_bseek; + if (cb) if (cb(xdata, (int)i, DBTYPE_BLOB, (int64_t)slice_len, 0.0, (char *)(buffer + initial_bseek)) != DBRES_OK) return -1; + } else { + int64_t value = INT64_MIN; + if (cb) if (cb(xdata, (int)i, DBTYPE_INTEGER, value, 0.0, NULL) != DBRES_OK) return -1; + } } break; - case SQLITE_NEGATIVE_INTEGER: - case SQLITE_INTEGER: { - int64_t value = pk_decode_int64(buffer, &bseek, nbytes); - if (type == SQLITE_NEGATIVE_INTEGER) {value = -value; type = SQLITE_INTEGER;} - if (cb) if (cb(xdata, (int)i, type, value, 0.0, NULL) != SQLITE_OK) return -1; + case DATABASE_TYPE_NEGATIVE_INTEGER: + case DBTYPE_INTEGER: { + // validate nbytes to avoid UB/overreads + if (nbytes < 1 || nbytes > 8) return -1; + uint64_t u = 0; + if (!pk_decode_uint64(ubuf, blen, &bseek, nbytes, &u)) return -1; + + if (skip_decode) { + size_t slice_len = bseek - initial_bseek; + if (cb) if (cb(xdata, (int)i, DBTYPE_BLOB, (int64_t)slice_len, 0.0, (char *)(buffer + initial_bseek)) != DBRES_OK) return -1; + } else { + int64_t value = (int64_t)u; + if (raw_type == DATABASE_TYPE_NEGATIVE_INTEGER) value = -value; + if (cb) if (cb(xdata, (int)i, DBTYPE_INTEGER, value, 0.0, NULL) != DBRES_OK) return -1; + } } break; - case SQLITE_NEGATIVE_FLOAT: - case SQLITE_FLOAT: { - double value = pk_decode_double(buffer, &bseek); - if (type == SQLITE_NEGATIVE_FLOAT) {value = -value; type = SQLITE_FLOAT;} - if (cb) if (cb(xdata, (int)i, type, 0, value, NULL) != SQLITE_OK) return -1; + case DATABASE_TYPE_NEGATIVE_FLOAT: + case DBTYPE_FLOAT: { + // encoder stores float type with no length bits, so enforce nbytes==0 + if (nbytes != 0) return -1; + double value = 0.0; + if (!pk_decode_double(ubuf, blen, &bseek, &value)) return -1; + + if (skip_decode) { + size_t slice_len = bseek - initial_bseek; + if (cb) if (cb(xdata, (int)i, DBTYPE_BLOB, (int64_t)slice_len, 0.0, (char *)(buffer + initial_bseek)) != DBRES_OK) return -1; + } else { + if (raw_type == DATABASE_TYPE_NEGATIVE_FLOAT) value = -value; + if (cb) if (cb(xdata, (int)i, DBTYPE_FLOAT, 0, value, NULL) != DBRES_OK) return -1; + } } break; - case SQLITE_TEXT: - case SQLITE_BLOB: { - int64_t length = pk_decode_int64(buffer, &bseek, nbytes); - char *value = pk_decode_data(buffer, &bseek, (int32_t)length); - if (cb) if (cb(xdata, (int)i, type, length, 0.0, value) != SQLITE_OK) return -1; + case DBTYPE_TEXT: + case DBTYPE_BLOB: { + // validate nbytes for length field + if (nbytes < 1 || nbytes > 8) return -1; + uint64_t ulen = 0; + if (!pk_decode_uint64(ubuf, blen, &bseek, nbytes, &ulen)) return -1; + + // ensure ulen fits in size_t on this platform + if (ulen > (uint64_t)SIZE_MAX) return -1; + size_t len = (size_t)ulen; + const uint8_t *p = NULL; + if (!pk_decode_data(ubuf, blen, &bseek, len, &p)) return -1; + + if (skip_decode) { + // return the full encoded slice (type_byte + len bytes + payload) + size_t slice_len = bseek - initial_bseek; + if (cb) if (cb(xdata, (int)i, DBTYPE_BLOB, (int64_t)slice_len, 0.0, (char *)(buffer + initial_bseek)) != DBRES_OK) return -1; + } else { + if (cb) if (cb(xdata, (int)i, raw_type, (int64_t)len, 0.0, (char *)p) != DBRES_OK) return -1; + } } break; - case SQLITE_NULL: { - if (cb) if (cb(xdata, (int)i, type, 0, 0.0, NULL) != SQLITE_OK) return -1; + case DBTYPE_NULL: { + if (nbytes != 0) return -1; + if (skip_decode) { + size_t slice_len = bseek - initial_bseek; + if (cb) if (cb(xdata, (int)i, DBTYPE_BLOB, (int64_t)slice_len, 0.0, (char *)(buffer + initial_bseek)) != DBRES_OK) return -1; + } else { + if (cb) if (cb(xdata, (int)i, DBTYPE_NULL, 0, 0.0, NULL) != DBRES_OK) return -1; + } } break; + + default: + // should never reach this point + return -1; } } @@ -223,55 +321,86 @@ int pk_decode(char *buffer, size_t blen, int count, size_t *seek, int (*cb) (voi return count; } -int pk_decode_prikey (char *buffer, size_t blen, int (*cb) (void *xdata, int index, int type, int64_t ival, double dval, char *pval), void *xdata) { +int pk_decode_prikey (char *buffer, size_t blen, pk_decode_callback cb, void *xdata) { + const uint8_t *ubuf = (const uint8_t *)buffer; size_t bseek = 0; - uint8_t count = pk_decode_u8(buffer, &bseek); - return pk_decode(buffer, blen, count, &bseek, cb, xdata); + uint8_t count = 0; + if (!pk_decode_u8(ubuf, blen, &bseek, &count)) return -1; + return pk_decode(buffer, blen, count, &bseek, -1, cb, xdata); } // MARK: - Encoding - size_t pk_encode_nbytes_needed (int64_t value) { - if (value <= 0x7F) return 1; // 7 bits - if (value <= 0x7FFF) return 2; // 15 bits - if (value <= 0x7FFFFF) return 3; // 23 bits - if (value <= 0x7FFFFFFF) return 4; // 31 bits - if (value <= 0x7FFFFFFFFF) return 5; // 39 bits - if (value <= 0x7FFFFFFFFFFF) return 6; // 47 bits - if (value <= 0x7FFFFFFFFFFFFF) return 7; // 55 bits - return 8; // Larger than 7-byte range, needs 8 bytes + uint64_t v = (uint64_t)value; + if (v <= 0xFFULL) return 1; + if (v <= 0xFFFFULL) return 2; + if (v <= 0xFFFFFFULL) return 3; + if (v <= 0xFFFFFFFFULL) return 4; + if (v <= 0xFFFFFFFFFFULL) return 5; + if (v <= 0xFFFFFFFFFFFFULL) return 6; + if (v <= 0xFFFFFFFFFFFFFFULL) return 7; + return 8; } -size_t pk_encode_size (sqlite3_value **argv, int argc, int reserved) { +static inline int pk_encode_add_overflow_size (size_t a, size_t b, size_t *out) { + // safe size_t addition helper (prevents overflow) + if (b > (SIZE_MAX - a)) return 1; + *out = a + b; + return 0; +} + +size_t pk_encode_size (dbvalue_t **argv, int argc, int reserved, int skip_idx) { // estimate the required buffer size size_t required = reserved; size_t nbytes; - int64_t val, len; + int64_t val; for (int i = 0; i < argc; i++) { - switch (sqlite3_value_type(argv[i])) { - case SQLITE_INTEGER: - val = sqlite3_value_int64(argv[i]); + switch (database_value_type(argv[i])) { + case DBTYPE_INTEGER: { + val = database_value_int(argv[i]); if (val == INT64_MIN) { - required += 1; + if (pk_encode_add_overflow_size(required, 1, &required)) return SIZE_MAX; break; } if (val < 0) val = -val; nbytes = pk_encode_nbytes_needed(val); - required += 1 + nbytes; - break; - case SQLITE_FLOAT: - required += 1 + sizeof(int64_t); - break; - case SQLITE_TEXT: - case SQLITE_BLOB: - len = (int32_t)sqlite3_value_bytes(argv[i]); - nbytes = pk_encode_nbytes_needed(len); - required += 1 + len + nbytes; - break; - case SQLITE_NULL: - required += 1; - break; + + size_t tmp = 0; + if (pk_encode_add_overflow_size(1, nbytes, &tmp)) return SIZE_MAX; + if (pk_encode_add_overflow_size(required, tmp, &required)) return SIZE_MAX; + } break; + + case DBTYPE_FLOAT: { + size_t tmp = 0; + if (pk_encode_add_overflow_size(1, sizeof(uint64_t), &tmp)) return SIZE_MAX; + if (pk_encode_add_overflow_size(required, tmp, &required)) return SIZE_MAX; + } break; + + case DBTYPE_TEXT: + case DBTYPE_BLOB: { + size_t len_sz = (size_t)database_value_bytes(argv[i]); + if (i == skip_idx) { + if (pk_encode_add_overflow_size(required, len_sz, &required)) return SIZE_MAX; + break; + } + + // Ensure length can be represented by encoder (we encode length with up to 8 bytes) + // pk_encode_nbytes_needed expects int64-ish values; clamp-check here. + if (len_sz > (size_t)INT64_MAX) return SIZE_MAX; + nbytes = pk_encode_nbytes_needed((int64_t)len_sz); + + size_t tmp = 0; + // 1(type) + nbytes(len) + len_sz(payload) + if (pk_encode_add_overflow_size(1, nbytes, &tmp)) return SIZE_MAX; + if (pk_encode_add_overflow_size(tmp, len_sz, &tmp)) return SIZE_MAX; + if (pk_encode_add_overflow_size(required, tmp, &required)) return SIZE_MAX; + } break; + + case DBTYPE_NULL: { + if (pk_encode_add_overflow_size(required, 1, &required)) return SIZE_MAX; + } break; } } @@ -283,9 +412,9 @@ size_t pk_encode_u8 (char *buffer, size_t bseek, uint8_t value) { return bseek; } -size_t pk_encode_int64 (char *buffer, size_t bseek, int64_t value, size_t nbytes) { +static size_t pk_encode_uint64 (char *buffer, size_t bseek, uint64_t value, size_t nbytes) { for (size_t i = 0; i < nbytes; i++) { - buffer[bseek++] = (uint8_t)((value >> (8 * (nbytes - 1 - i))) & 0xFF); + buffer[bseek++] = (uint8_t)((value >> (8 * (nbytes - 1 - i))) & 0xFFu); } return bseek; } @@ -294,70 +423,162 @@ size_t pk_encode_data (char *buffer, size_t bseek, char *data, size_t datalen) { memcpy(buffer + bseek, data, datalen); return bseek + datalen; } + +size_t pk_encode_raw_size (int type, int64_t len_or_value) { + switch (type) { + case DBTYPE_INTEGER: { + if (len_or_value == INT64_MIN) return 1; + if (len_or_value < 0) len_or_value = -len_or_value; + return 1 + pk_encode_nbytes_needed(len_or_value); + } + case DBTYPE_FLOAT: + return 1 + sizeof(uint64_t); + case DBTYPE_TEXT: + case DBTYPE_BLOB: { + if (len_or_value < 0) return SIZE_MAX; + size_t nbytes = pk_encode_nbytes_needed(len_or_value); + return 1 + nbytes + (size_t)len_or_value; + } + case DBTYPE_NULL: + return 1; + } + return SIZE_MAX; +} + +size_t pk_encode_raw_int (char *buffer, int64_t value) { + int type = DBTYPE_INTEGER; + size_t bseek = 0; + if (value == INT64_MIN) { + return pk_encode_u8(buffer, bseek, DATABASE_TYPE_MAX_NEGATIVE_INTEGER); + } + if (value < 0) { value = -value; type = DATABASE_TYPE_NEGATIVE_INTEGER; } + size_t nbytes = pk_encode_nbytes_needed(value); + uint8_t type_byte = (uint8_t)((nbytes << 3) | type); + bseek = pk_encode_u8(buffer, bseek, type_byte); + return pk_encode_uint64(buffer, bseek, (uint64_t)value, nbytes); +} + +size_t pk_encode_raw_text (char *buffer, const char *value, size_t len) { + size_t nbytes = pk_encode_nbytes_needed((int64_t)len); + uint8_t type_byte = (uint8_t)((nbytes << 3) | DBTYPE_TEXT); + size_t bseek = pk_encode_u8(buffer, 0, type_byte); + bseek = pk_encode_uint64(buffer, bseek, (uint64_t)len, nbytes); + return pk_encode_data(buffer, bseek, (char *)value, len); +} + +size_t pk_encode_raw_blob (char *buffer, const void *value, size_t len) { + size_t nbytes = pk_encode_nbytes_needed((int64_t)len); + uint8_t type_byte = (uint8_t)((nbytes << 3) | DBTYPE_BLOB); + size_t bseek = pk_encode_u8(buffer, 0, type_byte); + bseek = pk_encode_uint64(buffer, bseek, (uint64_t)len, nbytes); + return pk_encode_data(buffer, bseek, (char *)value, len); +} -char *pk_encode (sqlite3_value **argv, int argc, char *b, bool is_prikey, size_t *bsize) { +char *pk_encode (dbvalue_t **argv, int argc, char *b, bool is_prikey, size_t *bsize, int skip_idx) { size_t bseek = 0; - size_t blen = 0; char *buffer = b; + // always compute blen (even if it is not a primary key) + size_t blen = pk_encode_size(argv, argc, (is_prikey) ? 1 : 0, skip_idx); + if (blen == SIZE_MAX) return NULL; + if (argc < 0) return NULL; + // in primary-key encoding the number of items must be explicitly added to the encoded buffer if (is_prikey) { - // 1 is the number of items in the serialization (always 1 byte so max 255 primary keys, even if there is an hard SQLite limit of 128) - blen = pk_encode_size(argv, argc, 1); + if (!bsize) return NULL; + // must fit in a single byte + if (argc > 255) return NULL; + + // if schema does not enforce NOT NULL on primary keys, check at runtime + #ifndef CLOUDSYNC_CHECK_NOTNULL_PRIKEYS + for (int i = 0; i < argc; i++) { + if (database_value_type(argv[i]) == DBTYPE_NULL) return PRIKEY_NULL_CONSTRAINT_ERROR; + } + #endif + + // 1 is the number of items in the serialization + // always 1 byte so max 255 primary keys, even if there is an hard SQLite limit of 128 size_t blen_curr = *bsize; - buffer = (blen > blen_curr || b == NULL) ? cloudsync_memory_alloc((sqlite3_uint64)blen) : b; + buffer = (blen > blen_curr || b == NULL) ? cloudsync_memory_alloc((uint64_t)blen) : b; if (!buffer) return NULL; // the first u8 value is the total number of items in the primary key(s) - bseek = pk_encode_u8(buffer, 0, argc); + bseek = pk_encode_u8(buffer, 0, (uint8_t)argc); + } else { + // ensure buffer exists and is large enough also in non-prikey mode + size_t curr = (bsize) ? *bsize : 0; + if (buffer == NULL || curr < blen) return NULL; } for (int i = 0; i < argc; i++) { - int type = sqlite3_value_type(argv[i]); + int type = database_value_type(argv[i]); switch (type) { - case SQLITE_INTEGER: { - int64_t value = sqlite3_value_int64(argv[i]); + case DBTYPE_INTEGER: { + int64_t value = database_value_int(argv[i]); if (value == INT64_MIN) { - bseek = pk_encode_u8(buffer, bseek, SQLITE_MAX_NEGATIVE_INTEGER); + bseek = pk_encode_u8(buffer, bseek, DATABASE_TYPE_MAX_NEGATIVE_INTEGER); break; } - if (value < 0) {value = -value; type = SQLITE_NEGATIVE_INTEGER;} + if (value < 0) {value = -value; type = DATABASE_TYPE_NEGATIVE_INTEGER;} size_t nbytes = pk_encode_nbytes_needed(value); - uint8_t type_byte = (nbytes << 3) | type; + uint8_t type_byte = (uint8_t)((nbytes << 3) | type); bseek = pk_encode_u8(buffer, bseek, type_byte); - bseek = pk_encode_int64(buffer, bseek, value, nbytes); + bseek = pk_encode_uint64(buffer, bseek, (uint64_t)value, nbytes); } break; - case SQLITE_FLOAT: { - double value = sqlite3_value_double(argv[i]); - if (value < 0) {value = -value; type = SQLITE_NEGATIVE_FLOAT;} - int64_t net_double; - memcpy(&net_double, &value, sizeof(int64_t)); - bseek = pk_encode_u8(buffer, bseek, type); - bseek = pk_encode_int64(buffer, bseek, net_double, sizeof(int64_t)); + case DBTYPE_FLOAT: { + // Encode doubles as IEEE754 64-bit, big-endian + double value = database_value_double(argv[i]); + if (value < 0) {value = -value; type = DATABASE_TYPE_NEGATIVE_FLOAT;} + uint64_t bits; + memcpy(&bits, &value, sizeof(bits)); + bits = host_to_be64(bits); + bseek = pk_encode_u8(buffer, bseek, (uint8_t)type); + bseek = pk_encode_uint64(buffer, bseek, bits, sizeof(bits)); } break; - case SQLITE_TEXT: - case SQLITE_BLOB: { - int32_t len = (int32_t)sqlite3_value_bytes(argv[i]); - size_t nbytes = pk_encode_nbytes_needed(len); - uint8_t type_byte = (nbytes << 3) | sqlite3_value_type(argv[i]); + case DBTYPE_TEXT: + case DBTYPE_BLOB: { + size_t len = (size_t)database_value_bytes(argv[i]); + if (i == skip_idx) { + memcpy(buffer + bseek, (char *)database_value_blob(argv[i]), len); + bseek += len; + break; + } + + if (len > (size_t)INT64_MAX) return NULL; + size_t nbytes = pk_encode_nbytes_needed((int64_t)len); + uint8_t type_byte = (uint8_t)((nbytes << 3) | database_value_type(argv[i])); bseek = pk_encode_u8(buffer, bseek, type_byte); - bseek = pk_encode_int64(buffer, bseek, len, nbytes); - bseek = pk_encode_data(buffer, bseek, (char *)sqlite3_value_blob(argv[i]), len); + bseek = pk_encode_uint64(buffer, bseek, (uint64_t)len, nbytes); + bseek = pk_encode_data(buffer, bseek, (char *)database_value_blob(argv[i]), len); } break; - case SQLITE_NULL: { - bseek = pk_encode_u8(buffer, bseek, SQLITE_NULL); + case DBTYPE_NULL: { + bseek = pk_encode_u8(buffer, bseek, DBTYPE_NULL); } break; } } - if (bsize) *bsize = blen; + // return actual bytes written; for prikey it's equal to blen, but safer to report bseek + if (bsize) *bsize = bseek; return buffer; } -char *pk_encode_prikey (sqlite3_value **argv, int argc, char *b, size_t *bsize) { - return pk_encode(argv, argc, b, true, bsize); +char *pk_encode_prikey (dbvalue_t **argv, int argc, char *b, size_t *bsize) { + return pk_encode(argv, argc, b, true, bsize, -1); +} + +char *pk_encode_value (dbvalue_t *value, size_t *bsize) { + dbvalue_t *argv[1] = {value}; + + size_t blen = pk_encode_size(argv, 1, 0, -1); + if (blen == SIZE_MAX) return NULL; + + char *buffer = cloudsync_memory_alloc((uint64_t)blen); + if (!buffer) return NULL; + + *bsize = blen; + return pk_encode(argv, 1, buffer, false, bsize, -1); } diff --git a/src/pk.h b/src/pk.h index ebcc0743..ab61b9f7 100644 --- a/src/pk.h +++ b/src/pk.h @@ -8,23 +8,27 @@ #ifndef __CLOUDSYNC_PK__ #define __CLOUDSYNC_PK__ -#include #include -#include +#include #include +#include "database.h" -#ifndef SQLITE_CORE -#include "sqlite3ext.h" -#else -#include "sqlite3.h" -#endif +typedef int (*pk_decode_callback) (void *xdata, int index, int type, int64_t ival, double dval, char *pval); + +extern char * const PRIKEY_NULL_CONSTRAINT_ERROR; -char *pk_encode_prikey (sqlite3_value **argv, int argc, char *b, size_t *bsize); -char *pk_encode (sqlite3_value **argv, int argc, char *b, bool is_prikey, size_t *bsize); -int pk_decode_prikey (char *buffer, size_t blen, int (*cb) (void *xdata, int index, int type, int64_t ival, double dval, char *pval), void *xdata); -int pk_decode(char *buffer, size_t blen, int count, size_t *seek, int (*cb) (void *xdata, int index, int type, int64_t ival, double dval, char *pval), void *xdata); -int pk_decode_bind_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval); -int pk_decode_print_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval); -size_t pk_encode_size (sqlite3_value **argv, int argc, int reserved); +char *pk_encode_prikey (dbvalue_t **argv, int argc, char *b, size_t *bsize); +char *pk_encode_value (dbvalue_t *value, size_t *bsize); +char *pk_encode (dbvalue_t **argv, int argc, char *b, bool is_prikey, size_t *bsize, int skip_idx); +int pk_decode_prikey (char *buffer, size_t blen, pk_decode_callback cb, void *xdata); +int pk_decode (char *buffer, size_t blen, int count, size_t *seek, int skip_decode_idx, pk_decode_callback cb, void *xdata); +int pk_decode_bind_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval); +int pk_decode_print_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval); +size_t pk_encode_size (dbvalue_t **argv, int argc, int reserved, int skip_idx); +uint64_t pk_checksum (const char *buffer, size_t blen); +size_t pk_encode_raw_size (int type, int64_t len_or_value); +size_t pk_encode_raw_int (char *buffer, int64_t value); +size_t pk_encode_raw_text (char *buffer, const char *value, size_t len); +size_t pk_encode_raw_blob (char *buffer, const void *value, size_t len); #endif diff --git a/src/postgresql/cloudsync.sql.in b/src/postgresql/cloudsync.sql.in new file mode 100644 index 00000000..92bb6f42 --- /dev/null +++ b/src/postgresql/cloudsync.sql.in @@ -0,0 +1,367 @@ +-- CloudSync Extension for PostgreSQL +-- Version @EXTVERSION@ + +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cloudsync" to load this file. \quit + +-- ============================================================================ +-- Public Functions +-- ============================================================================ + +-- Get extension version +CREATE OR REPLACE FUNCTION cloudsync_version() +RETURNS text +AS 'MODULE_PATHNAME', 'cloudsync_version' +LANGUAGE C IMMUTABLE STRICT; + +-- Get site identifier (UUID) +CREATE OR REPLACE FUNCTION cloudsync_siteid() +RETURNS bytea +AS 'MODULE_PATHNAME', 'pg_cloudsync_siteid' +LANGUAGE C STABLE; + +-- Generate a new UUID +CREATE OR REPLACE FUNCTION cloudsync_uuid() +RETURNS uuid +AS 'MODULE_PATHNAME', 'cloudsync_uuid' +LANGUAGE C VOLATILE; + +-- Get current database version +CREATE OR REPLACE FUNCTION cloudsync_db_version() +RETURNS bigint +AS 'MODULE_PATHNAME', 'cloudsync_db_version' +LANGUAGE C STABLE; + +-- Get next database version (with optional merging version) +CREATE OR REPLACE FUNCTION cloudsync_db_version_next() +RETURNS bigint +AS 'MODULE_PATHNAME', 'cloudsync_db_version_next' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_db_version_next(merging_version bigint) +RETURNS bigint +AS 'MODULE_PATHNAME', 'cloudsync_db_version_next' +LANGUAGE C VOLATILE; + +-- Initialize CloudSync for a table (3 variants for 1-3 arguments) +-- Returns site_id as bytea +CREATE OR REPLACE FUNCTION cloudsync_init(table_name text) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_init' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_init(table_name text, algo text) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_init' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_init(table_name text, algo text, init_flags integer) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_init' +LANGUAGE C VOLATILE; + +-- Enable sync for a table +CREATE OR REPLACE FUNCTION cloudsync_enable(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_enable' +LANGUAGE C VOLATILE; + +-- Disable sync for a table +CREATE OR REPLACE FUNCTION cloudsync_disable(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_disable' +LANGUAGE C VOLATILE; + +-- Check if table is sync-enabled +CREATE OR REPLACE FUNCTION cloudsync_is_enabled(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_is_enabled' +LANGUAGE C STABLE; + +-- Cleanup orphaned metadata for a table +CREATE OR REPLACE FUNCTION cloudsync_cleanup(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_cleanup' +LANGUAGE C VOLATILE; + +-- Terminate CloudSync +CREATE OR REPLACE FUNCTION cloudsync_terminate() +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_terminate' +LANGUAGE C VOLATILE; + +-- Set global configuration +CREATE OR REPLACE FUNCTION cloudsync_set(key text, value text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_set' +LANGUAGE C VOLATILE; + +-- Set table-level configuration +CREATE OR REPLACE FUNCTION cloudsync_set_table(table_name text, key text, value text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_set_table' +LANGUAGE C VOLATILE; + +-- Set row-level filter for conditional sync +CREATE OR REPLACE FUNCTION cloudsync_set_filter(table_name text, filter_expr text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_set_filter' +LANGUAGE C VOLATILE; + +-- Clear row-level filter +CREATE OR REPLACE FUNCTION cloudsync_clear_filter(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_clear_filter' +LANGUAGE C VOLATILE; + +-- Set column-level configuration +CREATE OR REPLACE FUNCTION cloudsync_set_column(table_name text, column_name text, key text, value text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_set_column' +LANGUAGE C VOLATILE; + +-- Begin schema alteration +CREATE OR REPLACE FUNCTION cloudsync_begin_alter(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_begin_alter' +LANGUAGE C VOLATILE; + +-- Commit schema alteration +CREATE OR REPLACE FUNCTION cloudsync_commit_alter(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_commit_alter' +LANGUAGE C VOLATILE; + +-- Payload encoding (aggregate function) +CREATE OR REPLACE FUNCTION cloudsync_payload_encode_transfn(state internal, tbl text, pk bytea, col_name text, col_value bytea, col_version bigint, db_version bigint, site_id bytea, cl bigint, seq bigint) +RETURNS internal +AS 'MODULE_PATHNAME', 'cloudsync_payload_encode_transfn' +LANGUAGE C; + +CREATE OR REPLACE FUNCTION cloudsync_payload_encode_finalfn(state internal) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_payload_encode_finalfn' +LANGUAGE C; + +CREATE OR REPLACE AGGREGATE cloudsync_payload_encode(text, bytea, text, bytea, bigint, bigint, bytea, bigint, bigint) ( + SFUNC = cloudsync_payload_encode_transfn, + STYPE = internal, + FINALFUNC = cloudsync_payload_encode_finalfn +); + +CREATE OR REPLACE FUNCTION cloudsync_payload_chunks( + since_db_version bigint DEFAULT NULL, + filter_site_id bytea DEFAULT NULL, + until_db_version bigint DEFAULT NULL, + exclude_filter_site_id boolean DEFAULT false, + resume_db_version bigint DEFAULT NULL, + resume_seq bigint DEFAULT NULL, + resume_frag_offset bigint DEFAULT NULL +) +RETURNS TABLE ( + payload bytea, + chunk_index bigint, + payload_size bigint, + rows bigint, + db_version_min bigint, + db_version_max bigint, + watermark_db_version bigint, + next_db_version bigint, + next_seq bigint, + next_frag_offset bigint, + is_final boolean +) +AS 'MODULE_PATHNAME', 'cloudsync_payload_chunks' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_payload_blob_checked( + since_db_version bigint, + since_seq bigint, + filter_site_id bytea, + exclude_filter_site_id boolean, + max_estimated_payload_size bigint +) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_payload_blob_checked' +LANGUAGE C VOLATILE; + +-- UUID binary <-> canonical string helpers (the changes table stores site_id as +-- the 16-byte binary UUID; these let string-based callers such as the /check +-- endpoint pass a site_id to cloudsync_payload_chunks). +CREATE OR REPLACE FUNCTION cloudsync_uuid_text(uuid bytea, dash_format boolean DEFAULT true) +RETURNS text +AS 'MODULE_PATHNAME', 'cloudsync_uuid_text' +LANGUAGE C IMMUTABLE; + +CREATE OR REPLACE FUNCTION cloudsync_uuid_blob(uuid text) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_uuid_blob' +LANGUAGE C IMMUTABLE; + +-- Payload decoding and application +CREATE OR REPLACE FUNCTION cloudsync_payload_decode(payload bytea) +RETURNS integer +AS 'MODULE_PATHNAME', 'cloudsync_payload_decode' +LANGUAGE C VOLATILE; + +-- Alias for payload_decode +CREATE OR REPLACE FUNCTION cloudsync_payload_apply(payload bytea) +RETURNS integer +AS 'MODULE_PATHNAME', 'pg_cloudsync_payload_apply' +LANGUAGE C VOLATILE; + +-- ============================================================================ +-- Private/Internal Functions +-- ============================================================================ + +-- Check if table has sync metadata +CREATE OR REPLACE FUNCTION cloudsync_is_sync(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_is_sync' +LANGUAGE C STABLE; + +-- Internal insert handler (variadic for multiple PK columns) +CREATE OR REPLACE FUNCTION cloudsync_insert(table_name text, VARIADIC pk_values "any") +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_insert' +LANGUAGE C VOLATILE; + +-- Internal delete handler (variadic for multiple PK columns) +CREATE OR REPLACE FUNCTION cloudsync_delete(table_name text, VARIADIC pk_values "any") +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_delete' +LANGUAGE C VOLATILE; + +-- Internal update tracking (aggregate function) +CREATE OR REPLACE FUNCTION cloudsync_update_transfn(state internal, table_name text, new_value anyelement, old_value anyelement) +RETURNS internal +AS 'MODULE_PATHNAME', 'cloudsync_update_transfn' +LANGUAGE C; + +CREATE OR REPLACE FUNCTION cloudsync_update_finalfn(state internal) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_update_finalfn' +LANGUAGE C; + +CREATE AGGREGATE cloudsync_update(text, anyelement, anyelement) ( + SFUNC = cloudsync_update_transfn, + STYPE = internal, + FINALFUNC = cloudsync_update_finalfn +); + +-- Get sequence number +CREATE OR REPLACE FUNCTION cloudsync_seq() +RETURNS integer +AS 'MODULE_PATHNAME', 'cloudsync_seq' +LANGUAGE C VOLATILE; + +-- Encode primary key (variadic for multiple columns) +CREATE OR REPLACE FUNCTION cloudsync_pk_encode(VARIADIC pk_values "any") +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_pk_encode' +LANGUAGE C IMMUTABLE STRICT; + +-- Decode primary key component +CREATE OR REPLACE FUNCTION cloudsync_pk_decode(encoded_pk bytea, index integer) +RETURNS text +AS 'MODULE_PATHNAME', 'cloudsync_pk_decode' +LANGUAGE C IMMUTABLE STRICT; + +-- ============================================================================ +-- Changes Functions +-- ============================================================================ + +CREATE OR REPLACE FUNCTION cloudsync_encode_value(anyelement) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_encode_value' +LANGUAGE C IMMUTABLE; + +-- Encoded column value helper (PG): returns cloudsync-encoded bytea +CREATE OR REPLACE FUNCTION cloudsync_col_value( + table_name text, + col_name text, + pk bytea +) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_col_value' +LANGUAGE C STABLE; + +-- SetReturningFunction: To implement SELECT FROM cloudsync_changes +CREATE FUNCTION cloudsync_changes_select( + min_db_version bigint DEFAULT 0, + filter_site_id bytea DEFAULT NULL +) +RETURNS TABLE ( + tbl text, + pk bytea, + col_name text, + col_value bytea, -- pk_encoded value bytes + col_version bigint, + db_version bigint, + site_id bytea, + cl bigint, + seq bigint +) +AS 'MODULE_PATHNAME', 'cloudsync_changes_select' +LANGUAGE C STABLE; + +-- View con lo stesso nome della vtab SQLite +CREATE OR REPLACE VIEW cloudsync_changes AS +SELECT * FROM cloudsync_changes_select(0, NULL); + +-- Trigger function to implement INSERT on the cloudsync_changes view +CREATE FUNCTION cloudsync_changes_insert_trigger() +RETURNS trigger +AS 'MODULE_PATHNAME', 'cloudsync_changes_insert_trigger' +LANGUAGE C; + +CREATE OR REPLACE TRIGGER cloudsync_changes_insert +INSTEAD OF INSERT ON cloudsync_changes +FOR EACH ROW +EXECUTE FUNCTION cloudsync_changes_insert_trigger(); + +-- Set current schema name +CREATE OR REPLACE FUNCTION cloudsync_set_schema(schema text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_set_schema' +LANGUAGE C VOLATILE; + +-- Get current schema name (if any) +CREATE OR REPLACE FUNCTION cloudsync_schema() +RETURNS text +AS 'MODULE_PATHNAME', 'pg_cloudsync_schema' +LANGUAGE C VOLATILE; + +-- Get current schema name (if any) +CREATE OR REPLACE FUNCTION cloudsync_table_schema(table_name text) +RETURNS text +AS 'MODULE_PATHNAME', 'pg_cloudsync_table_schema' +LANGUAGE C VOLATILE; + +-- ============================================================================ +-- Block-level LWW Functions +-- ============================================================================ + +-- Materialize block-level column into base table +CREATE OR REPLACE FUNCTION cloudsync_text_materialize(table_name text, col_name text, VARIADIC pk_values "any") +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_text_materialize' +LANGUAGE C VOLATILE; + +-- ============================================================================ +-- Type Casts +-- ============================================================================ + +-- Cast function: converts bigint to boolean (0 = false, non-zero = true) +-- Required because BOOLEAN values are encoded as INT8 in sync payloads, +-- but PostgreSQL has no built-in cast from bigint to boolean. +CREATE FUNCTION cloudsync_int8_to_bool(bigint) RETURNS boolean AS $$ + SELECT $1 <> 0 +$$ LANGUAGE SQL IMMUTABLE STRICT; + +-- ASSIGNMENT cast: auto-applies in INSERT/UPDATE context only +-- This enables BOOLEAN column sync where values are encoded as INT8. +-- Using ASSIGNMENT (not IMPLICIT) to avoid unintended conversions in WHERE clauses. +CREATE CAST (bigint AS boolean) + WITH FUNCTION cloudsync_int8_to_bool(bigint) + AS ASSIGNMENT; diff --git a/src/postgresql/cloudsync_postgresql.c b/src/postgresql/cloudsync_postgresql.c new file mode 100644 index 00000000..3f62d2f6 --- /dev/null +++ b/src/postgresql/cloudsync_postgresql.c @@ -0,0 +1,3621 @@ +// +// cloudsync_postgresql.c +// cloudsync +// +// Created by Claude Code on 18/12/25. +// + +// Define POSIX feature test macros before any includes +#define _POSIX_C_SOURCE 200809L + +// PostgreSQL requires postgres.h to be included FIRST +#include "postgres.h" +#include "utils/datum.h" +#include "access/xact.h" +#include "catalog/pg_type.h" +#include "catalog/namespace.h" +#include "executor/spi.h" +#include "utils/lsyscache.h" +#include "utils/array.h" +#include "fmgr.h" +#include "funcapi.h" +#include "pgvalue.h" +#include "storage/ipc.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/hsearch.h" +#include "utils/memutils.h" +#include "utils/uuid.h" +#include "nodes/nodeFuncs.h" // exprTypmod, exprCollation +#include "nodes/pg_list.h" // linitial +#include "nodes/primnodes.h" // FuncExpr + +// CloudSync headers (after PostgreSQL headers) +#include "../cloudsync.h" +#include "../block.h" +#include "../database.h" +#include "../dbutils.h" +#include "../pk.h" +#include "../utils.h" + +// Note: network.h is not needed for PostgreSQL implementation + +PG_MODULE_MAGIC; + +// Note: PG_FUNCTION_INFO_V1 macros are declared before each function implementation below +// They should NOT be duplicated here to avoid redefinition errors + +#ifndef UNUSED_PARAMETER +#define UNUSED_PARAMETER(X) (void)(X) +#endif + +#define CLOUDSYNC_RLS_RESTRICTED_VALUE_BYTEA "E'\\\\x0b095f5f5b524c535d5f5f'::bytea" +#define CLOUDSYNC_NULL_VALUE_BYTEA "E'\\\\x05'::bytea" + +// External declaration +Datum database_column_datum (dbvm_t *vm, int index); + +// MARK: - Context Management - + +// Global context stored per backend +static cloudsync_context *pg_cloudsync_context = NULL; + +static void cloudsync_pg_context_init (cloudsync_context *data) { + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + + PG_TRY(); + { + if (cloudsync_config_exists(data)) { + if (cloudsync_context_init(data) == NULL) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("An error occurred while trying to initialize context"))); + } + + // update schema hash if upgrading from an older version + if (dbutils_settings_check_version(data, NULL) != 0) { + cloudsync_update_schema_hash(data); + } + + // make sure to update internal version to current version + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_LIBVERSION, CLOUDSYNC_VERSION); + } + + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + SPI_finish(); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); +} + +// Get or create the CloudSync context for this backend +static cloudsync_context *get_cloudsync_context(void) { + if (pg_cloudsync_context == NULL) { + // Create context - db_t is not used in PostgreSQL mode + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + cloudsync_context *data = cloudsync_context_create(NULL); + MemoryContextSwitchTo(old); + if (!data) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Not enough memory to create a database context"))); + } + // Set early to prevent infinite recursion: during init, SQL queries may call + // cloudsync_schema() which calls get_cloudsync_context(). Without early assignment, + // each nested call sees NULL and tries to reinitialize, causing stack overflow. + pg_cloudsync_context = data; + PG_TRY(); + { + cloudsync_pg_context_init(data); + } + PG_CATCH(); + { + pg_cloudsync_context = NULL; + cloudsync_context_free(data); + PG_RE_THROW(); + } + PG_END_TRY(); + } + + return pg_cloudsync_context; +} + +// MARK: - Extension Entry Points - + +void _PG_init (void) { + // Extension initialization + // SPI will be connected per-function call + elog(DEBUG1, "CloudSync extension loading"); + + // Initialize memory debugger (NOOP in production) + cloudsync_memory_init(1); + + // Set fractional-indexing allocator to use cloudsync memory + block_init_allocator(); +} + +void _PG_fini (void) { + // Extension cleanup + elog(DEBUG1, "CloudSync extension unloading"); + + // Free global context if it exists + if (pg_cloudsync_context) { + cloudsync_context_free(pg_cloudsync_context); + pg_cloudsync_context = NULL; + } +} + +// MARK: - Public SQL Functions - + +// cloudsync_version() - Returns extension version +PG_FUNCTION_INFO_V1(cloudsync_version); +Datum cloudsync_version (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + PG_RETURN_TEXT_P(cstring_to_text(CLOUDSYNC_VERSION)); +} + +// cloudsync_siteid() - Get site identifier (UUID) +PG_FUNCTION_INFO_V1(pg_cloudsync_siteid); +Datum pg_cloudsync_siteid (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + + cloudsync_context *data = get_cloudsync_context(); + const void *siteid = cloudsync_siteid(data); + + if (!siteid) { + PG_RETURN_NULL(); + } + + // Return as bytea (binary UUID) + bytea *result = (bytea *)palloc(VARHDRSZ + UUID_LEN); + SET_VARSIZE(result, VARHDRSZ + UUID_LEN); + memcpy(VARDATA(result), siteid, UUID_LEN); + + PG_RETURN_BYTEA_P(result); +} + +// cloudsync_uuid() - Generate a new UUID +PG_FUNCTION_INFO_V1(cloudsync_uuid); +Datum cloudsync_uuid (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + + uint8_t uuid_bytes[UUID_LEN]; + cloudsync_uuid_v7(uuid_bytes); + + // Format as text with dashes (matches SQLite implementation) + char uuid_str[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_stringify(uuid_bytes, uuid_str, true); + + // Parse into PostgreSQL UUID type + Datum uuid_datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + PG_RETURN_DATUM(uuid_datum); +} + +// cloudsync_uuid_text(bytea, [dash_format]) - 16-byte UUID -> canonical string +PG_FUNCTION_INFO_V1(cloudsync_uuid_text); +Datum cloudsync_uuid_text (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) PG_RETURN_NULL(); + bytea *b = PG_GETARG_BYTEA_PP(0); + if (VARSIZE_ANY_EXHDR(b) != UUID_LEN) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_uuid_text: expected a 16-byte value"))); + } + bool dash_format = PG_ARGISNULL(1) ? true : PG_GETARG_BOOL(1); + char uuid_str[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_stringify((uint8_t *)VARDATA_ANY(b), uuid_str, dash_format); + PG_RETURN_TEXT_P(cstring_to_text(uuid_str)); +} + +// cloudsync_uuid_blob(text) - UUID string -> 16-byte value (dashed/undashed) +PG_FUNCTION_INFO_V1(cloudsync_uuid_blob); +Datum cloudsync_uuid_blob (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) PG_RETURN_NULL(); + text *t = PG_GETARG_TEXT_PP(0); + uint8_t uuid[UUID_LEN]; + if (cloudsync_uuid_v7_parse(VARDATA_ANY(t), (int)VARSIZE_ANY_EXHDR(t), uuid) != 0) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_uuid_blob: malformed UUID string"))); + } + bytea *result = (bytea *)palloc(VARHDRSZ + UUID_LEN); + SET_VARSIZE(result, VARHDRSZ + UUID_LEN); + memcpy(VARDATA(result), uuid, UUID_LEN); + PG_RETURN_BYTEA_P(result); +} + +// cloudsync_db_version() - Get current database version +PG_FUNCTION_INFO_V1(cloudsync_db_version); +Datum cloudsync_db_version (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + + cloudsync_context *data = get_cloudsync_context(); + int64_t version = 0; + bool spi_connected = false; + + // Connect SPI for database operations + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + int rc = cloudsync_dbversion_check_uptodate(data); + if (rc != DBRES_OK) { + // When cloudsync_init was never called, data_version_stmt is NULL + // and database_errmsg() is empty, producing the unhelpful "Unable + // to retrieve db_version ()". Detect the uninitialized state and + // return an actionable message instead. The extra check only runs + // on the error branch, so it costs nothing on the sync hot path. + if (!cloudsync_context_is_initialized(data)) { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cloudsync is not initialized: call SELECT cloudsync_init('') to enable sync on a table before calling cloudsync_db_version()."))); + } + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to retrieve db_version (%s)", database_errmsg(data)))); + } + + version = cloudsync_dbversion(data); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_INT64(version); +} + +// cloudsync_db_version_next([merging_version]) - Get next database version +PG_FUNCTION_INFO_V1(cloudsync_db_version_next); +Datum cloudsync_db_version_next (PG_FUNCTION_ARGS) { + cloudsync_context *data = get_cloudsync_context(); + int64_t next_version = 0; + bool spi_connected = false; + + int64_t merging_version = CLOUDSYNC_VALUE_NOTSET; + if (PG_NARGS() == 1 && !PG_ARGISNULL(0)) { + merging_version = PG_GETARG_INT64(0); + } + + // Connect SPI for database operations + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + next_version = cloudsync_dbversion_next(data, merging_version); + if (next_version == -1) { + // Previously this path silently returned -1, which is worse than + // an error because callers cannot distinguish a bogus version + // number from a real one. Emit an actionable message when the + // root cause is that cloudsync_init has never been called. + if (!cloudsync_context_is_initialized(data)) { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cloudsync is not initialized: call SELECT cloudsync_init('') to enable sync on a table before calling cloudsync_db_version_next()."))); + } + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Unable to retrieve next_db_version (%s)", database_errmsg(data)))); + } + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_INT64(next_version); +} + +// MARK: - Table Initialization - + +// Internal helper for cloudsync_init - replicates dbsync_init logic from SQLite +// Returns site_id as bytea on success, raises error on failure +static bytea *cloudsync_init_internal (cloudsync_context *data, const char *table, const char *algo, CLOUDSYNC_INIT_FLAG init_flags) { + bytea *result = NULL; + + // Connect SPI for database operations + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + + PG_TRY(); + { + // Begin savepoint for transactional init + int rc = database_begin_savepoint(data, "cloudsync_init"); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to create cloudsync_init savepoint: %s", database_errmsg(data)))); + } + + // Initialize table for sync + rc = cloudsync_init_table(data, table, algo, init_flags); + ereport(DEBUG1, (errmsg("cloudsync_init_internal cloudsync_init_table %d", rc))); + + if (rc == DBRES_OK) { + rc = database_commit_savepoint(data, "cloudsync_init"); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to release cloudsync_init savepoint: %s", database_errmsg(data)))); + } + + // Persist schema to settings now that the settings table exists + const char *cur_schema = cloudsync_schema(data); + if (cur_schema) { + dbutils_settings_set_key_value(data, "schema", cur_schema); + } + } else { + // In case of error, rollback sub-transaction and raise. + // We intentionally avoid database_rollback_savepoint() here + // because it calls database_refresh_snapshot() which pushes a + // new active snapshot. Pushing a snapshot after rollback and + // before ereport(ERROR) leaves portal->portalSnapshot non-NULL + // when PL/pgSQL's exception handler later calls + // EnsurePortalSnapshotExists(), triggering + // Assert(portal->portalSnapshot == NULL) on debug builds. + char err[1024]; + snprintf(err, sizeof(err), "%s", cloudsync_errmsg(data)); + if (GetCurrentTransactionNestLevel() > 1) { + RollbackAndReleaseCurrentSubTransaction(); + } + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", err))); + } + + cloudsync_update_schema_hash(data); + + // Build site_id as bytea to return + // Use SPI_palloc so the allocation survives SPI_finish + result = (bytea *)SPI_palloc(UUID_LEN + VARHDRSZ); + SET_VARSIZE(result, UUID_LEN + VARHDRSZ); + memcpy(VARDATA(result), cloudsync_siteid(data), UUID_LEN); + + SPI_finish(); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + return result; +} + +// cloudsync_init(table_name, [algo], [init_flags]) - Initialize table for sync +// Supports 1-3 arguments with defaults: algo=NULL, init_flags=CLOUDSYNC_INIT_FLAG_NONE +PG_FUNCTION_INFO_V1(cloudsync_init); +Datum cloudsync_init (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + // Default values + const char *algo = NULL; + int init_flags = CLOUDSYNC_INIT_FLAG_NONE; + + // Handle optional arguments + int nargs = PG_NARGS(); + + if (nargs >= 2 && !PG_ARGISNULL(1)) { + algo = text_to_cstring(PG_GETARG_TEXT_PP(1)); + } + + if (nargs >= 3 && !PG_ARGISNULL(2)) { + init_flags = PG_GETARG_INT32(2); + } + + cloudsync_context *data = get_cloudsync_context(); + + // Call internal helper and return site_id as bytea + bytea *result = cloudsync_init_internal(data, table, algo, init_flags); + PG_RETURN_BYTEA_P(result); +} + +// MARK: - Table Enable/Disable Functions - + +// Internal helper for enable/disable +static void cloudsync_enable_disable (const char *table_name, bool value) { + cloudsync_context *data = get_cloudsync_context(); + cloudsync_table_context *table = table_lookup(data, table_name); + if (table) table_set_enabled(table, value); +} + +// cloudsync_enable - Enable sync for a table +PG_FUNCTION_INFO_V1(cloudsync_enable); +Datum cloudsync_enable (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_enable_disable(table, true); + PG_RETURN_BOOL(true); +} + +// cloudsync_disable - Disable sync for a table +PG_FUNCTION_INFO_V1(cloudsync_disable); +Datum cloudsync_disable (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_enable_disable(table, false); + PG_RETURN_BOOL(true); +} + +// cloudsync_is_enabled - Check if table is sync-enabled +PG_FUNCTION_INFO_V1(cloudsync_is_enabled); +Datum cloudsync_is_enabled (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + cloudsync_context *data = get_cloudsync_context(); + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_table_context *table = table_lookup(data, table_name); + + bool result = (table && table_enabled(table)); + PG_RETURN_BOOL(result); +} + +// MARK: - Cleanup and Termination - + +// cloudsync_cleanup - Cleanup orphaned metadata for a table +PG_FUNCTION_INFO_V1(pg_cloudsync_cleanup); +Datum pg_cloudsync_cleanup (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + rc = cloudsync_cleanup(data, table); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + if (spi_connected) SPI_finish(); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + + PG_RETURN_BOOL(true); +} + +// cloudsync_terminate - Terminate CloudSync +PG_FUNCTION_INFO_V1(pg_cloudsync_terminate); +Datum pg_cloudsync_terminate (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + rc = cloudsync_terminate(data); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(rc == DBRES_OK); +} + +// MARK: - Settings Functions - + +// cloudsync_set - Set global configuration +PG_FUNCTION_INFO_V1(cloudsync_set); +Datum cloudsync_set (PG_FUNCTION_ARGS) { + const char *key = NULL; + const char *value = NULL; + + if (!PG_ARGISNULL(0)) { + key = text_to_cstring(PG_GETARG_TEXT_PP(0)); + } + if (!PG_ARGISNULL(1)) { + value = text_to_cstring(PG_GETARG_TEXT_PP(1)); + } + + // Silently fail if key is NULL (matches SQLite behavior) + if (key == NULL) { + PG_RETURN_BOOL(true); + } + + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + dbutils_settings_set_key_value(data, key, value); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(true); +} + +// cloudsync_set_table - Set table-level configuration +PG_FUNCTION_INFO_V1(cloudsync_set_table); +Datum cloudsync_set_table (PG_FUNCTION_ARGS) { + const char *tbl = NULL; + const char *key = NULL; + const char *value = NULL; + + if (!PG_ARGISNULL(0)) { + tbl = text_to_cstring(PG_GETARG_TEXT_PP(0)); + } + if (!PG_ARGISNULL(1)) { + key = text_to_cstring(PG_GETARG_TEXT_PP(1)); + } + if (!PG_ARGISNULL(2)) { + value = text_to_cstring(PG_GETARG_TEXT_PP(2)); + } + + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + dbutils_table_settings_set_key_value(data, tbl, "*", key, value); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(true); +} + +// cloudsync_set_column - Set column-level configuration +PG_FUNCTION_INFO_V1(cloudsync_set_column); +Datum cloudsync_set_column (PG_FUNCTION_ARGS) { + const char *tbl = NULL; + const char *col = NULL; + const char *key = NULL; + const char *value = NULL; + + if (!PG_ARGISNULL(0)) { + tbl = text_to_cstring(PG_GETARG_TEXT_PP(0)); + } + if (!PG_ARGISNULL(1)) { + col = text_to_cstring(PG_GETARG_TEXT_PP(1)); + } + if (!PG_ARGISNULL(2)) { + key = text_to_cstring(PG_GETARG_TEXT_PP(2)); + } + if (!PG_ARGISNULL(3)) { + value = text_to_cstring(PG_GETARG_TEXT_PP(3)); + } + + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + // Handle block column setup: cloudsync_set_column('tbl', 'col', 'algo', 'block') + if (key && value && strcmp(key, "algo") == 0 && strcmp(value, "block") == 0) { + int rc = cloudsync_setup_block_column(data, tbl, col, NULL, true); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + } else { + // Handle delimiter setting: cloudsync_set_column('tbl', 'col', 'delimiter', '\n\n') + if (key && strcmp(key, "delimiter") == 0) { + cloudsync_table_context *table = table_lookup(data, tbl); + if (table) { + int col_idx = table_col_index(table, col); + if (col_idx >= 0 && table_col_algo(table, col_idx) == col_algo_block) { + table_set_col_delimiter(table, col_idx, value); + } + } + } + dbutils_table_settings_set_key_value(data, tbl, col, key, value); + } + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(true); +} + +// MARK: - Row Filter - + +// cloudsync_set_filter - Set a row-level filter for conditional sync +PG_FUNCTION_INFO_V1(cloudsync_set_filter); +Datum cloudsync_set_filter (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_set_filter: table and filter expression required"))); + } + + const char *tbl = text_to_cstring(PG_GETARG_TEXT_PP(0)); + const char *filter_expr = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + // Guard against calling set_filter before the target table has been + // set up for sync: without this, we'd drop and fail to recreate + // triggers, emitting ten+ noisy "does not exist, skipping" NOTICEs + // followed by a generic "error recreating triggers" message that + // does not point at the real cause. + if (!cloudsync_context_is_initialized(data) || !table_lookup(data, tbl)) { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cloudsync_set_filter: table '%s' is not configured for sync. Call SELECT cloudsync_init('%s') first.", tbl, tbl))); + } + + // Store filter in table settings + dbutils_table_settings_set_key_value(data, tbl, "*", "filter", filter_expr); + + // Read current algo + table_algo algo = dbutils_table_settings_get_algo(data, tbl); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + + // Drop triggers + database_delete_triggers(data, tbl); + + // Reconnect SPI so that the catalog changes from DROP are visible + SPI_finish(); + spi_connected = false; + spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + // Recreate triggers with filter + int rc = database_create_triggers(data, tbl, algo, filter_expr); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cloudsync_set_filter: error recreating triggers"))); + } + + // Clean and refill metatable with the new filter + rc = cloudsync_reset_metatable(data, tbl); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cloudsync_set_filter: error resetting metatable"))); + } + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(true); +} + +// cloudsync_clear_filter - Remove the row-level filter for a table +PG_FUNCTION_INFO_V1(cloudsync_clear_filter); +Datum cloudsync_clear_filter (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_clear_filter: table name required"))); + } + + const char *tbl = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + // Guard against calling clear_filter before the target table has + // been set up for sync — see cloudsync_set_filter for the same + // rationale. + if (!cloudsync_context_is_initialized(data) || !table_lookup(data, tbl)) { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cloudsync_clear_filter: table '%s' is not configured for sync. Call SELECT cloudsync_init('%s') first.", tbl, tbl))); + } + + // Remove filter from settings + dbutils_table_settings_set_key_value(data, tbl, "*", "filter", NULL); + + // Read current algo + table_algo algo = dbutils_table_settings_get_algo(data, tbl); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + + // Drop triggers + database_delete_triggers(data, tbl); + + // Reconnect SPI so that the catalog changes from DROP are visible + SPI_finish(); + spi_connected = false; + spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + // Recreate triggers without filter + int rc = database_create_triggers(data, tbl, algo, NULL); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cloudsync_clear_filter: error recreating triggers"))); + } + + // Clean and refill metatable without filter (all rows) + rc = cloudsync_reset_metatable(data, tbl); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cloudsync_clear_filter: error resetting metatable"))); + } + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(true); +} + +// MARK: - Schema Alteration - + +// cloudsync_begin_alter - Begin schema alteration +PG_FUNCTION_INFO_V1(pg_cloudsync_begin_alter); +Datum pg_cloudsync_begin_alter (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + + if (SPI_connect() != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed"))); + } + + PG_TRY(); + { + database_begin_savepoint(data, "cloudsync_alter"); + rc = cloudsync_begin_alter(data, table_name); + if (rc != DBRES_OK) { + database_rollback_savepoint(data, "cloudsync_alter"); + } + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + SPI_finish(); + if (rc != DBRES_OK) { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("%s", cloudsync_errmsg(data)))); + } + PG_RETURN_BOOL(true); +} + +// cloudsync_commit_alter - Commit schema alteration +// +// This wrapper manages SPI in two phases to avoid the PostgreSQL warning +// "subtransaction left non-empty SPI stack". The subtransaction was opened +// by a prior cloudsync_begin_alter call, so SPI_connect() here creates a +// connection at the subtransaction level. We must disconnect SPI before +// cloudsync_commit_alter releases that subtransaction, then reconnect +// for post-commit work (cloudsync_update_schema_hash). +// Prepared statements survive SPI_finish via SPI_keepplan/TopMemoryContext. +PG_FUNCTION_INFO_V1(pg_cloudsync_commit_alter); +Datum pg_cloudsync_commit_alter (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + + // Phase 1: SPI work before savepoint release + if (SPI_connect() != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed"))); + } + + PG_TRY(); + { + rc = cloudsync_commit_alter(data, table_name); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + // Disconnect SPI before savepoint boundary + SPI_finish(); + + if (rc != DBRES_OK) { + // Rollback savepoint (SPI disconnected, no warning) + database_rollback_savepoint(data, "cloudsync_alter"); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + + // Release savepoint (SPI disconnected, no warning) + rc = database_commit_savepoint(data, "cloudsync_alter"); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to release cloudsync_alter savepoint: %s", database_errmsg(data)))); + } + + // Phase 2: reconnect SPI for post-commit work + if (SPI_connect() != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed after savepoint release"))); + } + + PG_TRY(); + { + cloudsync_update_schema_hash(data); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + SPI_finish(); + PG_RETURN_BOOL(true); +} + +// MARK: - Payload Functions - + +// Aggregate function: cloudsync_payload_encode transition function +PG_FUNCTION_INFO_V1(cloudsync_payload_encode_transfn); +Datum cloudsync_payload_encode_transfn (PG_FUNCTION_ARGS) { + MemoryContext aggContext; + cloudsync_payload_context *payload = NULL; + + if (!AggCheckCallContext(fcinfo, &aggContext)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_payload_encode_transfn called in non-aggregate context"))); + } + + // Get or allocate aggregate state + if (PG_ARGISNULL(0)) { + MemoryContext oldContext = MemoryContextSwitchTo(aggContext); + payload = (cloudsync_payload_context *)palloc0(cloudsync_payload_context_size(NULL)); + MemoryContextSwitchTo(oldContext); + } else { + payload = (cloudsync_payload_context *)PG_GETARG_POINTER(0); + } + + int argc = 0; + cloudsync_context *data = get_cloudsync_context(); + pgvalue_t **argv = pgvalues_from_args(fcinfo, 1, &argc); + + // Wrap variadic args into pgvalue_t so pk/payload helpers can read types safely. + if (argc > 0) { + int rc = cloudsync_payload_encode_step(payload, data, argc, (dbvalue_t **)argv); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + } + + // payload_encode_step does not retain pgvalue_t*, free transient wrappers now + for (int i = 0; i < argc; i++) { + pgvalue_free(argv[i]); + } + if (argv) cloudsync_memory_free(argv); + + PG_RETURN_POINTER(payload); +} + +// Aggregate function: cloudsync_payload_encode finalize function +PG_FUNCTION_INFO_V1(cloudsync_payload_encode_finalfn); +Datum cloudsync_payload_encode_finalfn (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + PG_RETURN_NULL(); + } + + cloudsync_payload_context *payload = (cloudsync_payload_context *)PG_GETARG_POINTER(0); + cloudsync_context *data = get_cloudsync_context(); + + int rc = cloudsync_payload_encode_final(payload, data); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + + int64_t blob_size = 0; + char *blob = cloudsync_payload_blob(payload, &blob_size, NULL); + + if (!blob) { + PG_RETURN_NULL(); + } + + bytea *result = (bytea *)palloc(VARHDRSZ + blob_size); + SET_VARSIZE(result, VARHDRSZ + blob_size); + memcpy(VARDATA(result), blob, blob_size); + + cloudsync_memory_free(blob); + + PG_RETURN_BYTEA_P(result); +} + +typedef struct { + Portal portal; + TupleDesc outdesc; + SPITupleTable *current_tuptable; + // Context for the per-row fields below that must outlive a single SRF call: + // a single oversized value emits its fragments over multiple SRF_PERCALL + // invocations and re-reads tbl/pk/col_name/col_value/site_id on each. The + // SRF sets this to funcctx->multi_call_memory_ctx (the only context the SRF + // protocol guarantees survives between calls). Non-SRF callers leave it NULL + // and allocate in the current context, freeing each row as they iterate. + MemoryContext value_ctx; + bool spi_connected; + bool has_current; + bool eof; + int64 chunk_index; + int64 watermark; + int max_size; + int frag_target; + + char *tbl; + bytea *pk; + char *col_name; + bytea *col_value; + bool col_value_owned; + int64 col_version; + int64 db_version; + bytea *site_id; + int64 cl; + int64 seq; + + bool frag_active; + int frag_part; + int frag_count; + int64 frag_offset; + int64 frag_total; + uint64 frag_checksum; +} PayloadChunksState; + +static void payload_chunks_free_current(PayloadChunksState *st) { + if (!st) return; + if (st->tbl) pfree(st->tbl); + if (st->pk) pfree(st->pk); + if (st->col_name) pfree(st->col_name); + if (st->col_value && st->col_value_owned) pfree(st->col_value); + if (st->site_id) pfree(st->site_id); + if (st->current_tuptable) SPI_freetuptable(st->current_tuptable); + st->tbl = NULL; + st->pk = NULL; + st->col_name = NULL; + st->col_value = NULL; + st->col_value_owned = false; + st->site_id = NULL; + st->current_tuptable = NULL; + st->has_current = false; +} + +static bool payload_chunks_fetch_current(PayloadChunksState *st) { + if (st->has_current) return true; + if (st->eof) return false; + SPI_cursor_fetch(st->portal, true, 1); + if (SPI_processed == 0) { + if (SPI_tuptable) { SPI_freetuptable(SPI_tuptable); SPI_tuptable = NULL; } + st->eof = true; + return false; + } + + st->current_tuptable = SPI_tuptable; + HeapTuple tup = SPI_tuptable->vals[0]; + TupleDesc td = SPI_tuptable->tupdesc; + bool isnull = false; + Datum d; + + // These fields are re-read on later SRF calls while emitting fragments of a + // single oversized value, so they must be allocated in a context that + // survives between calls (value_ctx == multi_call_memory_ctx for the SRF). + // Includes any bytea detoasted by DatumGetByteaPP below. Non-SRF callers + // (value_ctx == NULL) keep the prior per-call allocation behavior. + MemoryContext old_value_ctx = st->value_ctx ? MemoryContextSwitchTo(st->value_ctx) : NULL; + + d = SPI_getbinval(tup, td, 1, &isnull); + st->tbl = isnull ? pstrdup("") : text_to_cstring(DatumGetTextPP(d)); + d = SPI_getbinval(tup, td, 2, &isnull); + if (!isnull) { + bytea *b = DatumGetByteaPP(d); + size_t n = VARSIZE_ANY(b); + st->pk = (bytea *)palloc(n); + memcpy(st->pk, b, n); + // DatumGetByteaPP returns a fresh copy when the datum was toasted; free + // it after the memcpy so a scan with toasted pks does not retain one + // detoast temp per row in value_ctx until the SRF ends. + if ((Pointer) b != DatumGetPointer(d)) pfree(b); + } + d = SPI_getbinval(tup, td, 3, &isnull); + st->col_name = isnull ? pstrdup("") : text_to_cstring(DatumGetTextPP(d)); + d = SPI_getbinval(tup, td, 4, &isnull); + if (!isnull) { + bytea *b = DatumGetByteaPP(d); + st->col_value = b; + st->col_value_owned = ((Pointer) b != DatumGetPointer(d)); + } + d = SPI_getbinval(tup, td, 5, &isnull); st->col_version = isnull ? 0 : DatumGetInt64(d); + d = SPI_getbinval(tup, td, 6, &isnull); st->db_version = isnull ? 0 : DatumGetInt64(d); + d = SPI_getbinval(tup, td, 7, &isnull); + if (!isnull) { + bytea *b = DatumGetByteaPP(d); + size_t n = VARSIZE_ANY(b); + st->site_id = (bytea *)palloc(n); + memcpy(st->site_id, b, n); + if ((Pointer) b != DatumGetPointer(d)) pfree(b); + } + d = SPI_getbinval(tup, td, 8, &isnull); st->cl = isnull ? 0 : DatumGetInt64(d); + d = SPI_getbinval(tup, td, 9, &isnull); st->seq = isnull ? 0 : DatumGetInt64(d); + + if (old_value_ctx) MemoryContextSwitchTo(old_value_ctx); + + SPI_tuptable = NULL; + st->has_current = true; + return true; +} + +static void payload_chunks_make_pgvalues(PayloadChunksState *st, pgvalue_t **vals, text **owned_texts) { + owned_texts[0] = cstring_to_text(st->tbl); + owned_texts[1] = cstring_to_text(st->col_name); + vals[0] = pgvalue_create(PointerGetDatum(owned_texts[0]), TEXTOID, -1, InvalidOid, false); + vals[1] = pgvalue_create(PointerGetDatum(st->pk), BYTEAOID, -1, InvalidOid, false); + vals[2] = pgvalue_create(PointerGetDatum(owned_texts[1]), TEXTOID, -1, InvalidOid, false); + vals[3] = pgvalue_create(PointerGetDatum(st->col_value), BYTEAOID, -1, InvalidOid, false); + vals[4] = pgvalue_create(Int64GetDatum(st->col_version), INT8OID, -1, InvalidOid, false); + vals[5] = pgvalue_create(Int64GetDatum(st->db_version), INT8OID, -1, InvalidOid, false); + vals[6] = pgvalue_create(PointerGetDatum(st->site_id), BYTEAOID, -1, InvalidOid, false); + vals[7] = pgvalue_create(Int64GetDatum(st->cl), INT8OID, -1, InvalidOid, false); + vals[8] = pgvalue_create(Int64GetDatum(st->seq), INT8OID, -1, InvalidOid, false); +} + +static void payload_chunks_free_pgvalues(pgvalue_t **vals, text **owned_texts) { + for (int i = 0; i < 9; ++i) if (vals[i]) pgvalue_free(vals[i]); + if (owned_texts[0]) pfree(owned_texts[0]); + if (owned_texts[1]) pfree(owned_texts[1]); +} + +static bytea *payload_chunks_emit_pg_fragment(PayloadChunksState *st, cloudsync_context *data, + int64 *rows, int64 *dbv_min, int64 *dbv_max) { + int64 remaining = st->frag_total - st->frag_offset; + int frag_len = remaining > st->frag_target ? st->frag_target : (int)remaining; + if (frag_len <= 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("invalid payload fragment size"))); + const char *src = VARDATA_ANY(st->col_value) + st->frag_offset; + + cloudsync_payload_context *payload = cloudsync_memory_zeroalloc((uint64_t)cloudsync_payload_context_size(NULL)); + if (!payload) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + int rc = cloudsync_payload_encode_fragment_step(payload, data, + st->tbl, -1, + VARDATA_ANY(st->pk), VARSIZE_ANY_EXHDR(st->pk), + st->col_name, -1, + src, frag_len, + st->col_version, st->db_version, + VARDATA_ANY(st->site_id), VARSIZE_ANY_EXHDR(st->site_id), + st->cl, st->seq, + st->frag_checksum, st->frag_total, st->frag_part, st->frag_count); + if (rc != DBRES_OK) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + rc = cloudsync_payload_encode_final(payload, data); + if (rc != DBRES_OK) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + int64 blob_size = 0; + char *blob = cloudsync_payload_blob(payload, &blob_size, rows); + bytea *result = (bytea *)palloc(VARHDRSZ + blob_size); + SET_VARSIZE(result, VARHDRSZ + blob_size); + memcpy(VARDATA(result), blob, blob_size); + cloudsync_memory_free(blob); + cloudsync_memory_free(payload); + + *dbv_min = st->db_version; + *dbv_max = st->db_version; + st->frag_offset += frag_len; + st->frag_part++; + if (st->frag_part >= st->frag_count) { + st->frag_active = false; + payload_chunks_free_current(st); + } + return result; +} + +// Set up fragment state for the currently-fetched oversized value so +// emit_pg_fragment can stream it. start_offset is the byte offset within the value +// to resume from (0 when first reaching it; >0 when a positional cursor resumes +// mid-value). frag_part is derived from the offset so a streamed and a resumed +// fragment carry the same part index. The plan (frag_target/frag_count) is a +// deterministic function of the row, so a resumed fragment tiles identically. +static void payload_chunks_pg_begin_fragment(PayloadChunksState *st, cloudsync_context *data, int64 start_offset) { + st->frag_total = VARSIZE_ANY_EXHDR(st->col_value); + st->frag_offset = start_offset; + st->frag_target = cloudsync_payload_fragment_data_size(data, + st->tbl, -1, + VARDATA_ANY(st->pk), VARSIZE_ANY_EXHDR(st->pk), + st->col_name, -1, + st->col_version, st->db_version, + VARDATA_ANY(st->site_id), VARSIZE_ANY_EXHDR(st->site_id), + st->cl, st->seq, + st->frag_total, 0, 1); + if (st->frag_target <= 0) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg(CLOUDSYNC_ERRCODE_CHUNK_TOO_LARGE "payload fragment metadata exceeds max chunk size"))); + for (int i = 0; i < CLOUDSYNC_PAYLOAD_FRAGMENT_SIZE_FIXPOINT_ITERATIONS; ++i) { + int count = cloudsync_payload_fragment_count(st->frag_total, st->frag_target); + if (count <= 0) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg(CLOUDSYNC_ERRCODE_CHUNK_TOO_LARGE "payload requires too many fragments"))); + int planned = cloudsync_payload_fragment_data_size(data, + st->tbl, -1, + VARDATA_ANY(st->pk), VARSIZE_ANY_EXHDR(st->pk), + st->col_name, -1, + st->col_version, st->db_version, + VARDATA_ANY(st->site_id), VARSIZE_ANY_EXHDR(st->site_id), + st->cl, st->seq, + st->frag_total, count - 1, count); + if (planned <= 0) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg(CLOUDSYNC_ERRCODE_CHUNK_TOO_LARGE "payload fragment metadata exceeds max chunk size"))); + if (planned == st->frag_target) break; + st->frag_target = planned; + } + st->frag_count = cloudsync_payload_fragment_count(st->frag_total, st->frag_target); + if (st->frag_count <= 0) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("payload requires too many fragments"))); + st->frag_part = (st->frag_target > 0) ? (int)(start_offset / st->frag_target) : 0; + st->frag_checksum = pk_checksum(VARDATA_ANY(st->col_value), (size_t)st->frag_total); + st->frag_active = true; +} + +static bytea *payload_chunks_build_pg_next(PayloadChunksState *st, cloudsync_context *data, + int64 *rows, int64 *dbv_min, int64 *dbv_max) { + *rows = *dbv_min = *dbv_max = 0; + if (st->frag_active) return payload_chunks_emit_pg_fragment(st, data, rows, dbv_min, dbv_max); + if (!payload_chunks_fetch_current(st)) return NULL; + + size_t header_size = 0; + cloudsync_payload_context_size(&header_size); + cloudsync_payload_context *payload = cloudsync_memory_zeroalloc((uint64_t)cloudsync_payload_context_size(NULL)); + if (!payload) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + + while (payload_chunks_fetch_current(st)) { + size_t row_size = 0; + { + pgvalue_t *vals[9] = {0}; + text *owned_texts[2] = {0}; + payload_chunks_make_pgvalues(st, vals, owned_texts); + row_size = pk_encode_size((dbvalue_t **)vals, 9, 0, 3); + payload_chunks_free_pgvalues(vals, owned_texts); + } + if (row_size == SIZE_MAX) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg(CLOUDSYNC_ERRCODE_ROW_TOO_LARGE "payload row too large"))); + + if ((int64)row_size + (int64)header_size + CLOUDSYNC_PAYLOAD_CHUNK_SAFETY_MARGIN > st->max_size) { + if (cloudsync_payload_context_nrows(payload) > 0) break; + payload_chunks_pg_begin_fragment(st, data, 0); + cloudsync_memory_free(payload); + return payload_chunks_emit_pg_fragment(st, data, rows, dbv_min, dbv_max); + } + + if (cloudsync_payload_context_nrows(payload) > 0 && cloudsync_payload_context_bused(payload) + row_size > (size_t)st->max_size) break; + + pgvalue_t *vals[9] = {0}; + text *owned_texts[2] = {0}; + payload_chunks_make_pgvalues(st, vals, owned_texts); + int rc = cloudsync_payload_encode_step(payload, data, 9, (dbvalue_t **)vals); + payload_chunks_free_pgvalues(vals, owned_texts); + if (rc != DBRES_OK) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + + if (cloudsync_payload_context_nrows(payload) == 1) *dbv_min = st->db_version; + *dbv_max = st->db_version; + payload_chunks_free_current(st); + } + + if (cloudsync_payload_context_nrows(payload) == 0) { + cloudsync_memory_free(payload); + return NULL; + } + int rc = cloudsync_payload_encode_final(payload, data); + if (rc != DBRES_OK) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + int64 blob_size = 0; + char *blob = cloudsync_payload_blob(payload, &blob_size, rows); + bytea *result = (bytea *)palloc(VARHDRSZ + blob_size); + SET_VARSIZE(result, VARHDRSZ + blob_size); + memcpy(VARDATA(result), blob, blob_size); + cloudsync_memory_free(blob); + cloudsync_memory_free(payload); + return result; +} + +PG_FUNCTION_INFO_V1(cloudsync_payload_chunks); +Datum cloudsync_payload_chunks(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + cloudsync_context *data = get_cloudsync_context(); + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + MemoryContext oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + PayloadChunksState *st = palloc0(sizeof(*st)); + st->chunk_index = 0; + // Per-row fields that span multiple SRF_PERCALL calls (fragment emission) + // must be allocated here, not in the transient per-call context. + st->value_ctx = funcctx->multi_call_memory_ctx; + + if (SPI_connect() != SPI_OK_CONNECT) ereport(ERROR, (errmsg("SPI_connect failed"))); + st->spi_connected = true; + st->max_size = cloudsync_payload_max_chunk_size(data); + size_t header_size_tmp = 0; + cloudsync_payload_context_size(&header_size_tmp); + st->frag_target = st->max_size - (int)header_size_tmp - CLOUDSYNC_PAYLOAD_CHUNK_SAFETY_MARGIN; + if (st->frag_target < 1024) st->frag_target = 1024; + + int64 since = PG_ARGISNULL(0) ? dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_SEND_DBVERSION) : PG_GETARG_INT64(0); + bytea *site_id = PG_ARGISNULL(1) ? NULL : PG_GETARG_BYTEA_PP(1); + bool exclude = PG_ARGISNULL(3) ? false : PG_GETARG_BOOL(3); + // Positional resume cursor: when resume_db_version is given the scan starts + // at (resume_db_version, resume_seq) inclusive and the first chunk resumes a + // mid-value fragment at resume_frag_offset, instead of replaying from `since`. + // Lets the /check job page one chunk per round-trip with an O(1) seek and no + // spool table. + bool positional = !PG_ARGISNULL(4); + int64 resume_dbv = PG_ARGISNULL(4) ? 0 : PG_GETARG_INT64(4); + int64 resume_seq = PG_ARGISNULL(5) ? 0 : PG_GETARG_INT64(5); + int64 resume_frag = PG_ARGISNULL(6) ? 0 : PG_GETARG_INT64(6); + // Site filter resolution: + // exclude=true -> all sites except filter_site_id (CHECK path); site required + // filter given -> only that site + // default -> local site (send path, unchanged) + if (exclude && !site_id) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_payload_chunks: exclude_filter_site_id requires a non-NULL filter_site_id"))); + } + if (!exclude && !site_id) { + site_id = (bytea *)palloc(VARHDRSZ + UUID_LEN); + SET_VARSIZE(site_id, VARHDRSZ + UUID_LEN); + memcpy(VARDATA(site_id), cloudsync_siteid(data), UUID_LEN); + } + + int64 until = PG_ARGISNULL(2) ? 0 : PG_GETARG_INT64(2); + if (until == 0) { + Oid mt[1] = {BYTEAOID}; + Datum mv[1] = {PointerGetDatum(site_id)}; + char mn[1] = {' '}; + const char *mxq = exclude + ? "SELECT COALESCE(MAX(db_version),0) FROM cloudsync_changes_select(0,NULL) WHERE site_id <> $1" + : "SELECT COALESCE(MAX(db_version),0) FROM cloudsync_changes_select(0,$1)"; + int mrc = SPI_execute_with_args(mxq, 1, mt, mv, mn, true, 1); + if (mrc == SPI_OK_SELECT && SPI_processed > 0) { + bool isnull = false; + Datum d = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); + until = isnull ? 0 : DatumGetInt64(d); + } + if (SPI_tuptable) { SPI_freetuptable(SPI_tuptable); SPI_tuptable = NULL; } + } + st->watermark = until; + + StringInfoData q; + initStringInfo(&q); + if (positional) { + // Inclusive positional lower bound (db_version, seq) >= (resume_dbv, + // resume_seq) within db_version <= until. $1=site, $2=until, $3=resume_dbv, + // $4=resume_seq. (seq >= matches the SQLite vtab's exact tiling; contrast + // with payload_blob_checked's exclusive seq > for its last-applied cursor.) + if (exclude) { + appendStringInfoString(&q, + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes_select(0,NULL) " + "WHERE site_id <> $1 AND db_version <= $2 AND (db_version > $3 OR (db_version = $3 AND seq >= $4)) " + "ORDER BY db_version, seq ASC"); + } else { + appendStringInfoString(&q, + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes_select(0,$1) " + "WHERE db_version <= $2 AND (db_version > $3 OR (db_version = $3 AND seq >= $4)) " + "ORDER BY db_version, seq ASC"); + } + Oid argtypes[4] = {BYTEAOID, INT8OID, INT8OID, INT8OID}; + Datum values[4] = {PointerGetDatum(site_id), Int64GetDatum(until), Int64GetDatum(resume_dbv), Int64GetDatum(resume_seq)}; + char nulls[4] = {' ', ' ', ' ', ' '}; + st->portal = SPI_cursor_open_with_args(NULL, q.data, 4, argtypes, values, nulls, true, 0); + } else { + if (exclude) { + // $1=since (into changes_select), $2=site to exclude, $3=until watermark + appendStringInfoString(&q, + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes_select($1,NULL) WHERE site_id <> $2 AND db_version <= $3 ORDER BY db_version, seq ASC"); + } else { + appendStringInfoString(&q, + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes_select($1,$2) WHERE db_version <= $3 ORDER BY db_version, seq ASC"); + } + Oid argtypes[3] = {INT8OID, BYTEAOID, INT8OID}; + Datum values[3] = {Int64GetDatum(since), PointerGetDatum(site_id), Int64GetDatum(until)}; + char nulls[3] = {' ', ' ', ' '}; + st->portal = SPI_cursor_open_with_args(NULL, q.data, 3, argtypes, values, nulls, true, 0); + } + pfree(q.data); + if (!st->portal) ereport(ERROR, (errmsg("SPI_cursor_open failed"))); + + // Resuming inside a value that was fragmented across chunks: the first row is + // that value; re-establish the fragment plan and skip to resume_frag. + if (positional && resume_frag > 0 && payload_chunks_fetch_current(st)) { + payload_chunks_pg_begin_fragment(st, data, resume_frag); + } + + TupleDesc outdesc; + if (get_call_result_type(fcinfo, NULL, &outdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errmsg("return type must be composite"))); + st->outdesc = BlessTupleDesc(outdesc); + funcctx->user_fctx = st; + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + PayloadChunksState *st = (PayloadChunksState *)funcctx->user_fctx; + + int64 rows = 0, dbv_min = 0, dbv_max = 0; + bytea *payload = payload_chunks_build_pg_next(st, data, &rows, &dbv_min, &dbv_max); + if (!payload) { + if (st->portal) SPI_cursor_close(st->portal); + st->portal = NULL; + if (st->spi_connected) SPI_finish(); + st->spi_connected = false; + payload_chunks_free_current(st); + MemoryContextSwitchTo(fcinfo->flinfo->fn_mcxt); + SRF_RETURN_DONE(funcctx); + } + + // Resume point a stateless caller passes back to continue after this chunk. + // frag_active -> same value, next byte offset; otherwise peek the next row + // (buffered for the following build call): a row -> its (db_version, seq); + // end of stream -> this was the final chunk. + int64 next_dbv, next_seq, next_frag; + bool is_final; + if (st->frag_active) { + next_dbv = st->db_version; next_seq = st->seq; next_frag = st->frag_offset; is_final = false; + } else if (payload_chunks_fetch_current(st)) { + next_dbv = st->db_version; next_seq = st->seq; next_frag = 0; is_final = false; + } else { + next_dbv = st->watermark; next_seq = 0; next_frag = 0; is_final = true; + } + + Datum outvals[11]; + bool outnulls[11] = {false,false,false,false,false,false,false,false,false,false,false}; + outvals[0] = PointerGetDatum(payload); + outvals[1] = Int64GetDatum(st->chunk_index++); + outvals[2] = Int64GetDatum(VARSIZE_ANY_EXHDR(payload)); + outvals[3] = Int64GetDatum(rows); + outvals[4] = Int64GetDatum(dbv_min); + outvals[5] = Int64GetDatum(dbv_max); + outvals[6] = Int64GetDatum(st->watermark); + outvals[7] = Int64GetDatum(next_dbv); + outvals[8] = Int64GetDatum(next_seq); + outvals[9] = Int64GetDatum(next_frag); + outvals[10] = BoolGetDatum(is_final); + HeapTuple outtup = heap_form_tuple(st->outdesc, outvals, outnulls); + SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(outtup)); +} + +static void payload_blob_checked_pg_add(int64 *acc, int64 value) { + if (value < 0 || *acc > PG_INT64_MAX - value) { + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg(CLOUDSYNC_ERRCODE_PAYLOAD_TOO_LARGE "cloudsync_payload_blob_checked: payload estimate is too large"))); + } + *acc += value; +} + +static int64 payload_blob_checked_pg_estimate(cloudsync_context *data, int64 since, int64 since_seq, + bytea *site_id, bool exclude, int64 *watermark) { + PayloadChunksState st = {0}; + int64 estimated = 0; + bool has_rows = false; + + int64 until = 0; + Oid mt[1] = {BYTEAOID}; + Datum mv[1] = {PointerGetDatum(site_id)}; + char mn[1] = {' '}; + const char *mxq = exclude + ? "SELECT COALESCE(MAX(db_version),0) FROM cloudsync_changes_select(0,NULL) WHERE site_id <> $1" + : "SELECT COALESCE(MAX(db_version),0) FROM cloudsync_changes_select(0,$1)"; + int mrc = SPI_execute_with_args(mxq, 1, mt, mv, mn, true, 1); + if (mrc == SPI_OK_SELECT && SPI_processed > 0) { + bool isnull = false; + Datum d = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); + until = isnull ? 0 : DatumGetInt64(d); + } else { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cloudsync_payload_blob_checked: failed to capture payload watermark"))); + } + if (SPI_tuptable) { SPI_freetuptable(SPI_tuptable); SPI_tuptable = NULL; } + if (watermark) *watermark = until; + + if (until >= since) { + StringInfoData q; + initStringInfo(&q); + if (exclude) { + appendStringInfoString(&q, + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes_select(0,NULL) " + "WHERE site_id <> $2 AND db_version <= $3 AND (db_version > $1 OR (db_version = $1 AND seq > $4)) " + "ORDER BY db_version, seq ASC"); + } else { + appendStringInfoString(&q, + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes_select(0,$2) " + "WHERE db_version <= $3 AND (db_version > $1 OR (db_version = $1 AND seq > $4)) " + "ORDER BY db_version, seq ASC"); + } + Oid argtypes[4] = {INT8OID, BYTEAOID, INT8OID, INT8OID}; + Datum values[4] = {Int64GetDatum(since), PointerGetDatum(site_id), Int64GetDatum(until), Int64GetDatum(since_seq)}; + char nulls[4] = {' ', ' ', ' ', ' '}; + st.portal = SPI_cursor_open_with_args(NULL, q.data, 4, argtypes, values, nulls, true, 0); + pfree(q.data); + if (!st.portal) ereport(ERROR, (errmsg("SPI_cursor_open failed"))); + + size_t header_size = 0; + cloudsync_payload_context_size(&header_size); + if (header_size > (size_t)PG_INT64_MAX) { + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg(CLOUDSYNC_ERRCODE_PAYLOAD_TOO_LARGE "cloudsync_payload_blob_checked: payload estimate is too large"))); + } + + while (payload_chunks_fetch_current(&st)) { + pgvalue_t *vals[9] = {0}; + text *owned_texts[2] = {0}; + payload_chunks_make_pgvalues(&st, vals, owned_texts); + size_t row_size = pk_encode_size((dbvalue_t **)vals, 9, 0, 3); + payload_chunks_free_pgvalues(vals, owned_texts); + if (row_size == SIZE_MAX || row_size > (size_t)PG_INT64_MAX) { + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg(CLOUDSYNC_ERRCODE_ROW_TOO_LARGE "cloudsync_payload_blob_checked: payload row is too large"))); + } + if (!has_rows) { + payload_blob_checked_pg_add(&estimated, (int64)header_size); + has_rows = true; + } + payload_blob_checked_pg_add(&estimated, (int64)row_size); + payload_chunks_free_current(&st); + } + } + + if (st.portal) SPI_cursor_close(st.portal); + payload_chunks_free_current(&st); + return estimated; +} + +PG_FUNCTION_INFO_V1(cloudsync_payload_blob_checked); +Datum cloudsync_payload_blob_checked(PG_FUNCTION_ARGS) { + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + Portal portal = NULL; + PayloadChunksState encode_st = {0}; + cloudsync_payload_context *payload = NULL; + bytea *result = NULL; + bytea *site_id = PG_ARGISNULL(2) ? NULL : PG_GETARG_BYTEA_PP(2); + bool exclude = PG_ARGISNULL(3) ? false : PG_GETARG_BOOL(3); + + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(4)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_payload_blob_checked: since_db_version, since_seq, and max_estimated_payload_size are required"))); + } + int64 since = PG_GETARG_INT64(0); + int64 since_seq = PG_GETARG_INT64(1); + int64 max_estimated_size = PG_GETARG_INT64(4); + if (max_estimated_size <= 0) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_payload_blob_checked: max_estimated_payload_size must be positive"))); + } + if (exclude && !site_id) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_payload_blob_checked: exclude_filter_site_id requires a non-NULL filter_site_id"))); + } + if (!exclude && !site_id) { + site_id = (bytea *)palloc(VARHDRSZ + UUID_LEN); + SET_VARSIZE(site_id, VARHDRSZ + UUID_LEN); + memcpy(VARDATA(site_id), cloudsync_siteid(data), UUID_LEN); + } + + if (SPI_connect() != SPI_OK_CONNECT) ereport(ERROR, (errmsg("SPI_connect failed"))); + spi_connected = true; + + int64 watermark = 0; + int64 estimated = payload_blob_checked_pg_estimate(data, since, since_seq, site_id, exclude, &watermark); + if (estimated == 0) { + if (spi_connected) SPI_finish(); + PG_RETURN_NULL(); + } + if (estimated > max_estimated_size) { + if (spi_connected) SPI_finish(); + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg(CLOUDSYNC_ERRCODE_PAYLOAD_TOO_LARGE "cloudsync_payload_blob_checked: estimated payload size %lld exceeds max_estimated_payload_size %lld", + (long long)estimated, (long long)max_estimated_size))); + } + + PG_TRY(); + { + StringInfoData q; + initStringInfo(&q); + if (exclude) { + appendStringInfoString(&q, + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes_select(0,NULL) " + "WHERE site_id <> $2 AND db_version <= $3 AND (db_version > $1 OR (db_version = $1 AND seq > $4)) " + "ORDER BY db_version, seq ASC"); + } else { + appendStringInfoString(&q, + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes_select(0,$2) " + "WHERE db_version <= $3 AND (db_version > $1 OR (db_version = $1 AND seq > $4)) " + "ORDER BY db_version, seq ASC"); + } + Oid argtypes[4] = {INT8OID, BYTEAOID, INT8OID, INT8OID}; + Datum values[4] = {Int64GetDatum(since), PointerGetDatum(site_id), Int64GetDatum(watermark), Int64GetDatum(since_seq)}; + char nulls[4] = {' ', ' ', ' ', ' '}; + portal = SPI_cursor_open_with_args(NULL, q.data, 4, argtypes, values, nulls, true, 0); + pfree(q.data); + if (!portal) ereport(ERROR, (errmsg("SPI_cursor_open failed"))); + encode_st.portal = portal; + + payload = cloudsync_memory_zeroalloc((uint64_t)cloudsync_payload_context_size(NULL)); + if (!payload) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + + while (payload_chunks_fetch_current(&encode_st)) { + pgvalue_t *vals[9] = {0}; + text *owned_texts[2] = {0}; + payload_chunks_make_pgvalues(&encode_st, vals, owned_texts); + int rc = cloudsync_payload_encode_step(payload, data, 9, (dbvalue_t **)vals); + payload_chunks_free_pgvalues(vals, owned_texts); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + payload_chunks_free_current(&encode_st); + } + SPI_cursor_close(portal); + portal = NULL; + encode_st.portal = NULL; + + int rc = cloudsync_payload_encode_final(payload, data); + if (rc != DBRES_OK) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + int64 blob_size = 0; + char *blob = cloudsync_payload_blob(payload, &blob_size, NULL); + if (spi_connected) { + SPI_finish(); + spi_connected = false; + } + // NOTE: PG_RETURN_* expands to `return`, so returning from inside the + // PG_TRY block would skip PG_END_TRY() and leave PG_exception_stack + // pointing at this (now-dead) frame; a later ereport(ERROR) in the same + // query then siglongjmp()s into freed stack and segfaults. Compute the + // result here, return it after PG_END_TRY(). result == NULL means the + // empty-blob path (return SQL NULL). + if (blob) { + result = (bytea *)palloc(VARHDRSZ + blob_size); + SET_VARSIZE(result, VARHDRSZ + blob_size); + memcpy(VARDATA(result), blob, blob_size); + } + cloudsync_payload_context_free(payload); + payload = NULL; + } + PG_CATCH(); + { + if (portal) SPI_cursor_close(portal); + payload_chunks_free_current(&encode_st); + if (payload) cloudsync_payload_context_free(payload); + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + // Return outside the PG_TRY so PG_END_TRY() always restores PG_exception_stack. + if (!result) PG_RETURN_NULL(); + PG_RETURN_BYTEA_P(result); +} + +// Payload decode - Apply changes from payload +PG_FUNCTION_INFO_V1(cloudsync_payload_decode); +Datum cloudsync_payload_decode (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("payload cannot be NULL"))); + } + + bytea *payload_data = PG_GETARG_BYTEA_P_COPY(0); + int blen = VARSIZE(payload_data) - VARHDRSZ; + + // Sanity check payload size + size_t header_size = 0; + cloudsync_payload_context_size(&header_size); + if (blen < (int)header_size) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Invalid payload size"))); + } + + const char *payload = VARDATA(payload_data); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + int nrows = 0; + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + // PostgreSQL applies a complete monolithic payload: legacy last-applied + // checkpoint (ends on a db_version boundary, so it is safe). + rc = cloudsync_payload_apply(data, payload, blen, &nrows, CLOUDSYNC_CHECKPOINT_LAST_APPLIED, 0); + } + PG_CATCH(); + { + if (payload_data) pfree(payload_data); + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + if (rc != DBRES_OK) { + if (payload_data) pfree(payload_data); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + if (payload_data) pfree(payload_data); + PG_RETURN_INT32(nrows); +} + +// Alias for payload_decode +PG_FUNCTION_INFO_V1(pg_cloudsync_payload_apply); +Datum pg_cloudsync_payload_apply (PG_FUNCTION_ARGS) { + return cloudsync_payload_decode(fcinfo); +} + +// MARK: - Private/Internal Functions - + +typedef struct cloudsync_pg_cleanup_state { + char *pk; + char pk_buffer[1024]; + pgvalue_t **argv; + int argc; + bool spi_connected; +} cloudsync_pg_cleanup_state; + +static void cloudsync_pg_cleanup(int code, Datum arg) { + cloudsync_pg_cleanup_state *state = (cloudsync_pg_cleanup_state *)DatumGetPointer(arg); + if (!state) return; + UNUSED_PARAMETER(code); + + if (state->pk && state->pk != state->pk_buffer) { + cloudsync_memory_free(state->pk); + } + state->pk = NULL; + + for (int i = 0; i < state->argc; i++) { + pgvalue_free(state->argv[i]); + } + if (state->argv) cloudsync_memory_free(state->argv); + state->argv = NULL; + state->argc = 0; + + if (state->spi_connected) { + SPI_finish(); + state->spi_connected = false; + } +} + +// cloudsync_is_sync - Check if table has sync metadata +PG_FUNCTION_INFO_V1(cloudsync_is_sync); +Datum cloudsync_is_sync (PG_FUNCTION_ARGS) { + cloudsync_context *data = get_cloudsync_context(); + + if (cloudsync_insync(data)) { + PG_RETURN_BOOL(true); + } + + if (PG_ARGISNULL(0)) { + PG_RETURN_BOOL(false); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_table_context *table = table_lookup(data, table_name); + + bool result = (table && (table_enabled(table) == 0)); + PG_RETURN_BOOL(result); +} + +typedef struct cloudsync_update_payload { + pgvalue_t *table_name; + pgvalue_t **new_values; + pgvalue_t **old_values; + int count; + int capacity; + MemoryContext mcxt; + // Context-owned callback info for early-exit cleanup. + // We null the payload pointer on normal finalization to avoid double-free. + struct cloudsync_mcxt_cb_info *mcxt_cb_info; +} cloudsync_update_payload; + +static void cloudsync_update_payload_free (cloudsync_update_payload *payload); + +typedef struct cloudsync_mcxt_cb_info { + MemoryContext mcxt; + const char *name; + cloudsync_update_payload *payload; +} cloudsync_mcxt_cb_info; + +static void cloudsync_mcxt_reset_cb (void *arg) { + cloudsync_mcxt_cb_info *info = (cloudsync_mcxt_cb_info *)arg; + if (!info) return; + if (!info->payload) return; + + // Context reset means the aggregate state would be lost; clean it here. + cloudsync_update_payload_free(info->payload); + info->payload = NULL; +} + +static void cloudsync_update_payload_free (cloudsync_update_payload *payload) { + if (!payload) return; + + if (payload->mcxt_cb_info) { + // Normal finalize path: prevent the reset callback from double-free. + payload->mcxt_cb_info->payload = NULL; + } + + for (int i = 0; i < payload->count; i++) { + pgvalue_free(payload->new_values[i]); + pgvalue_free(payload->old_values[i]); + } + if (payload->new_values) pfree(payload->new_values); + if (payload->old_values) pfree(payload->old_values); + if (payload->table_name) pgvalue_free(payload->table_name); + + payload->new_values = NULL; + payload->old_values = NULL; + payload->table_name = NULL; + payload->count = 0; + payload->capacity = 0; + payload->mcxt = NULL; + payload->mcxt_cb_info = NULL; +} + +static bool cloudsync_update_payload_append (cloudsync_update_payload *payload, pgvalue_t *table_name, pgvalue_t *new_value, pgvalue_t *old_value) { + if (!payload) return false; + if (!payload->mcxt || !MemoryContextIsValid(payload->mcxt)) { + elog(DEBUG1, "cloudsync_update_payload_append invalid payload context payload=%p mcxt=%p", payload, payload->mcxt); + return false; + } + if (payload->count < 0 || payload->capacity < 0) { + elog(DEBUG1, "cloudsync_update_payload_append invalid counters payload=%p count=%d cap=%d", payload, payload->count, payload->capacity); + return false; + } + + if (payload->count >= payload->capacity) { + int newcap = payload->capacity ? payload->capacity * 2 : 128; + elog(DEBUG1, "cloudsync_update_payload_append newcap=%d", newcap); + MemoryContext old = MemoryContextSwitchTo(payload->mcxt); + if (payload->capacity == 0) { + payload->new_values = (pgvalue_t **)palloc0(newcap * sizeof(*payload->new_values)); + payload->old_values = (pgvalue_t **)palloc0(newcap * sizeof(*payload->old_values)); + } else { + payload->new_values = (pgvalue_t **)repalloc(payload->new_values, newcap * sizeof(*payload->new_values)); + payload->old_values = (pgvalue_t **)repalloc(payload->old_values, newcap * sizeof(*payload->old_values)); + } + payload->capacity = newcap; + MemoryContextSwitchTo(old); + } + + if (payload->count >= payload->capacity) { + elog(DEBUG1, + "cloudsync_update_payload_append count>=capacity payload=%p count=%d " + "cap=%d new_values=%p old_values=%p", + payload, payload->count, payload->capacity, payload->new_values, + payload->old_values); + return false; + } + + int index = payload->count; + if (payload->table_name == NULL) { + payload->table_name = table_name; + } else { + // Compare within the payload context so any lazy text/detoast buffers + // are allocated in a stable context (not ExprContext). + MemoryContext old = MemoryContextSwitchTo(payload->mcxt); + int cmp = dbutils_value_compare((dbvalue_t *)payload->table_name, (dbvalue_t *)table_name); + MemoryContextSwitchTo(old); + if (cmp != 0) { + return false; + } + pgvalue_free(table_name); + } + + payload->new_values[index] = new_value; + payload->old_values[index] = old_value; + payload->count++; + + return true; +} + +// cloudsync_seq - Get sequence number +PG_FUNCTION_INFO_V1(cloudsync_seq); +Datum cloudsync_seq (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + + cloudsync_context *data = get_cloudsync_context(); + int seq = cloudsync_bumpseq(data); + + PG_RETURN_INT32(seq); +} + +// cloudsync_pk_encode - Encode primary key from variadic arguments +PG_FUNCTION_INFO_V1(cloudsync_pk_encode); +Datum cloudsync_pk_encode (PG_FUNCTION_ARGS) { + int argc = 0; + pgvalue_t **argv = NULL; + + // Signature is VARIADIC "any", so extract all arguments starting from index 0 + argv = pgvalues_from_args(fcinfo, 0, &argc); + if (!argv || argc == 0) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_pk_encode requires at least one primary key value"))); + } + + // Normalize all values to text for consistent PK encoding + // (PG triggers cast PK values to ::text; SQL callers must match) + pgvalues_normalize_to_text(argv, argc); + + size_t pklen = 0; + char *encoded = pk_encode_prikey((dbvalue_t **)argv, argc, NULL, &pklen); + if (!encoded || encoded == PRIKEY_NULL_CONSTRAINT_ERROR) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_pk_encode failed to encode primary key"))); + } + + bytea *result = (bytea *)palloc(pklen + VARHDRSZ); + SET_VARSIZE(result, pklen + VARHDRSZ); + memcpy(VARDATA(result), encoded, pklen); + cloudsync_memory_free(encoded); + + for (int i = 0; i < argc; i++) { + pgvalue_free(argv[i]); + } + if (argv) cloudsync_memory_free(argv); + + PG_RETURN_BYTEA_P(result); +} + +typedef struct cloudsync_pk_decode_ctx { + int target_index; + text *result; + bool found; +} cloudsync_pk_decode_ctx; + +static int cloudsync_pk_decode_set_result (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { + cloudsync_pk_decode_ctx *ctx = (cloudsync_pk_decode_ctx *)xdata; + if (!ctx || ctx->found || (index + 1) != ctx->target_index) return DBRES_OK; + + switch (type) { + case DBTYPE_INTEGER: { + char *cstr = DatumGetCString(DirectFunctionCall1(int8out, Int64GetDatum(ival))); + ctx->result = cstring_to_text(cstr); + pfree(cstr); + break; + } + case DBTYPE_FLOAT: { + char *cstr = DatumGetCString(DirectFunctionCall1(float8out, Float8GetDatum(dval))); + ctx->result = cstring_to_text(cstr); + pfree(cstr); + break; + } + case DBTYPE_TEXT: { + ctx->result = cstring_to_text_with_len(pval, (int)ival); + break; + } + case DBTYPE_BLOB: { + bytea *ba = (bytea *)palloc(ival + VARHDRSZ); + SET_VARSIZE(ba, ival + VARHDRSZ); + memcpy(VARDATA(ba), pval, (size_t)ival); + char *cstr = DatumGetCString(DirectFunctionCall1(byteaout, PointerGetDatum(ba))); + ctx->result = cstring_to_text(cstr); + pfree(cstr); + pfree(ba); + break; + } + case DBTYPE_NULL: + default: + ctx->result = NULL; + break; + } + + ctx->found = true; + return DBRES_OK; +} + +// cloudsync_pk_decode - Decode primary key component at given index +PG_FUNCTION_INFO_V1(cloudsync_pk_decode); +Datum cloudsync_pk_decode (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) { + PG_RETURN_NULL(); + } + + bytea *ba = PG_GETARG_BYTEA_P(0); + int index = PG_GETARG_INT32(1); + if (index < 1) PG_RETURN_NULL(); + + cloudsync_pk_decode_ctx ctx = { + .target_index = index, + .result = NULL, + .found = false + }; + + char *buffer = VARDATA(ba); + size_t blen = (size_t)(VARSIZE(ba) - VARHDRSZ); + if (pk_decode_prikey(buffer, blen, cloudsync_pk_decode_set_result, &ctx) < 0) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_pk_decode failed to decode primary key"))); + } + + if (!ctx.found || ctx.result == NULL) PG_RETURN_NULL(); + PG_RETURN_TEXT_P(ctx.result); +} + +// cloudsync_insert - Internal insert handler +// Signature: cloudsync_insert(table_name text, VARIADIC pk_values anyarray) +PG_FUNCTION_INFO_V1(cloudsync_insert); +Datum cloudsync_insert (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + cloudsync_pg_cleanup_state cleanup = {0}; + + // Connect SPI for database operations + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + cleanup.spi_connected = true; + + PG_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + { + // Lookup table (load from settings if needed) + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + char meta_name[1024]; + snprintf(meta_name, sizeof(meta_name), "%s_cloudsync", table_name); + if (!database_table_exists(data, meta_name, cloudsync_schema(data))) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_insert", table_name))); + } + + table_algo algo = dbutils_table_settings_get_algo(data, table_name); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + if (!table_add_to_context(data, algo, table_name)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to load table context for %s", table_name))); + } + + table = table_lookup(data, table_name); + if (!table) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_insert", table_name))); + } + } + + // Extract PK values from VARIADIC "any" (args starting from index 1) + cleanup.argv = pgvalues_from_args(fcinfo, 1, &cleanup.argc); + + // Normalize PK values to text for consistent encoding + pgvalues_normalize_to_text(cleanup.argv, cleanup.argc); + + // Verify we have the correct number of PK columns + int expected_pks = table_count_pks(table); + if (cleanup.argc != expected_pks) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Expected %d primary key values, got %d", expected_pks, cleanup.argc))); + } + + // Encode the primary key values into a buffer + size_t pklen = sizeof(cleanup.pk_buffer); + cleanup.pk = pk_encode_prikey((dbvalue_t **)cleanup.argv, cleanup.argc, cleanup.pk_buffer, &pklen); + + if (!cleanup.pk) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Not enough memory to encode the primary key(s)"))); + } + if (cleanup.pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + cleanup.pk = NULL; + ereport(ERROR, (errcode(ERRCODE_NOT_NULL_VIOLATION), errmsg("Insert aborted because primary key in table %s contains NULL values", table_name))); + } + + // Compute the next database version for tracking changes + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + // Check if a row with the same primary key already exists + // (if so, this might be a previously deleted sentinel) + bool pk_exists = table_pk_exists(table, cleanup.pk, pklen); + int rc = DBRES_OK; + + if (table_count_cols(table) == 0) { + // If there are no columns other than primary keys, insert a sentinel record + rc = local_mark_insert_sentinel_meta(table, cleanup.pk, pklen, db_version, cloudsync_bumpseq(data)); + } else if (pk_exists) { + // If a row with the same primary key already exists, update the sentinel record + rc = local_update_sentinel(table, cleanup.pk, pklen, db_version, cloudsync_bumpseq(data)); + } + + if (rc == DBRES_OK) { + // Process each non-primary key column for insert or update + for (int i = 0; i < table_count_cols(table); i++) { + if (table_col_algo(table, i) == col_algo_block) { + // Block column: read value from base table, split into blocks, store each block + dbvm_t *val_vm = table_column_lookup(table, table_colname(table, i), false, NULL); + if (!val_vm) { rc = DBRES_ERROR; break; } + + int bind_rc = pk_decode_prikey(cleanup.pk, pklen, pk_decode_bind_callback, (void *)val_vm); + if (bind_rc < 0) { databasevm_reset(val_vm); rc = DBRES_ERROR; break; } + + int step_rc = databasevm_step(val_vm); + if (step_rc == DBRES_ROW) { + const char *text = database_column_text(val_vm, 0); + const char *delim = table_col_delimiter(table, i); + const char *col = table_colname(table, i); + + block_list_t *blocks = block_split(text ? text : "", delim); + if (blocks) { + char **positions = block_initial_positions(blocks->count); + if (positions) { + for (int b = 0; b < blocks->count; b++) { + char *block_cn = block_build_colname(col, positions[b]); + if (block_cn) { + rc = local_mark_insert_or_update_meta(table, cleanup.pk, pklen, block_cn, db_version, cloudsync_bumpseq(data)); + + // Store block value in blocks table + dbvm_t *wvm = table_block_value_write_stmt(table); + if (wvm && rc == DBRES_OK) { + databasevm_bind_blob(wvm, 1, cleanup.pk, (int)pklen); + databasevm_bind_text(wvm, 2, block_cn, -1); + databasevm_bind_text(wvm, 3, blocks->entries[b].content, -1); + databasevm_step(wvm); + databasevm_reset(wvm); + } + + cloudsync_memory_free(block_cn); + } + cloudsync_memory_free(positions[b]); + if (rc != DBRES_OK) break; + } + cloudsync_memory_free(positions); + } + block_list_free(blocks); + } + } + databasevm_reset(val_vm); + if (step_rc == DBRES_ROW || step_rc == DBRES_DONE) { if (rc == DBRES_OK) continue; } + if (rc != DBRES_OK) break; + } else { + rc = local_mark_insert_or_update_meta(table, cleanup.pk, pklen, table_colname(table, i), db_version, cloudsync_bumpseq(data)); + if (rc != DBRES_OK) break; + } + } + } + + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", database_errmsg(data)))); + } + } + PG_END_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + + cloudsync_pg_cleanup(0, PointerGetDatum(&cleanup)); + PG_RETURN_BOOL(true); +} + +// cloudsync_delete - Internal delete handler +// Signature: cloudsync_delete(table_name text, VARIADIC pk_values anyarray) +PG_FUNCTION_INFO_V1(cloudsync_delete); +Datum cloudsync_delete (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + cloudsync_pg_cleanup_state cleanup = {0}; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + cleanup.spi_connected = true; + + PG_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + { + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + char meta_name[1024]; + snprintf(meta_name, sizeof(meta_name), "%s_cloudsync", table_name); + if (!database_table_exists(data, meta_name, cloudsync_schema(data))) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_delete", table_name))); + } + + table_algo algo = dbutils_table_settings_get_algo(data, table_name); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + if (!table_add_to_context(data, algo, table_name)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to load table context for %s", table_name))); + } + + table = table_lookup(data, table_name); + if (!table) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_delete", table_name))); + } + } + + // Extract PK values from VARIADIC "any" (args starting from index 1) + cleanup.argv = pgvalues_from_args(fcinfo, 1, &cleanup.argc); + + // Normalize PK values to text for consistent encoding + pgvalues_normalize_to_text(cleanup.argv, cleanup.argc); + + int expected_pks = table_count_pks(table); + if (cleanup.argc != expected_pks) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Expected %d primary key values, got %d", expected_pks, cleanup.argc))); + } + int rc = DBRES_OK; + + size_t pklen = sizeof(cleanup.pk_buffer); + cleanup.pk = pk_encode_prikey((dbvalue_t **)cleanup.argv, cleanup.argc, cleanup.pk_buffer, &pklen); + if (!cleanup.pk) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Not enough memory to encode the primary key(s)"))); + } + if (cleanup.pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + cleanup.pk = NULL; + ereport(ERROR, (errcode(ERRCODE_NOT_NULL_VIOLATION), errmsg("Delete aborted because primary key in table %s contains NULL values", table_name))); + } + + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + rc = local_mark_delete_meta(table, cleanup.pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc == DBRES_OK) { + rc = local_drop_meta(table, cleanup.pk, pklen); + } + + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", database_errmsg(data)))); + } + } + PG_END_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + + cloudsync_pg_cleanup(0, PointerGetDatum(&cleanup)); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(cloudsync_update_transfn); +Datum cloudsync_update_transfn (PG_FUNCTION_ARGS) { + MemoryContext aggContext; + MemoryContext allocContext = NULL; + cloudsync_update_payload *payload = NULL; + + if (!AggCheckCallContext(fcinfo, &aggContext)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_update_transfn called in non-aggregate context"))); + } + + allocContext = aggContext; + if (aggContext && aggContext->name && strcmp(aggContext->name, "ExprContext") == 0 && aggContext->parent) { + allocContext = aggContext->parent; + } + + if (PG_ARGISNULL(0)) { + MemoryContext old = MemoryContextSwitchTo(allocContext); + payload = (cloudsync_update_payload *)palloc0(sizeof(cloudsync_update_payload)); + payload->mcxt = allocContext; + MemoryContextSwitchTo(old); + } else { + payload = (cloudsync_update_payload *)PG_GETARG_POINTER(0); + if (payload->mcxt == NULL || payload->mcxt != allocContext) { + elog(DEBUG1, "cloudsync_update_transfn repairing payload context payload=%p old_mcxt=%p new_mcxt=%p", payload, payload->mcxt, allocContext); + payload->mcxt = allocContext; + } + } + + if (!payload) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_update_transfn payload is null"))); + } + + if (payload->mcxt_cb_info && payload->mcxt_cb_info->mcxt != allocContext) { + payload->mcxt_cb_info->payload = NULL; + payload->mcxt_cb_info = NULL; + } + + if (!payload->mcxt_cb_info) { + MemoryContext old = MemoryContextSwitchTo(allocContext); + // info and cb are automatically freed when that context is reset or deleted + cloudsync_mcxt_cb_info *info = (cloudsync_mcxt_cb_info *)palloc0(sizeof(*info)); + info->mcxt = allocContext; + info->name = allocContext ? allocContext->name : ""; + info->payload = payload; + MemoryContextCallback *cb = (MemoryContextCallback *)palloc0(sizeof(*cb)); + cb->func = cloudsync_mcxt_reset_cb; + cb->arg = info; + MemoryContextRegisterResetCallback(allocContext, cb); + payload->mcxt_cb_info = info; + MemoryContextSwitchTo(old); + } + + if (payload->count < 0 || payload->capacity < 0 ||payload->count > payload->capacity) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_update_transfn invalid payload state: count=%d cap=%d", payload->count, payload->capacity))); + } + + elog(DEBUG1, + "cloudsync_update_transfn contexts current=%p name=%s agg=%p name=%s " + "alloc=%p name=%s", + CurrentMemoryContext, + CurrentMemoryContext ? CurrentMemoryContext->name : "", aggContext, + aggContext ? aggContext->name : "", allocContext, + allocContext ? allocContext->name : ""); + + Oid table_type = get_fn_expr_argtype(fcinfo->flinfo, 1); + bool table_null = PG_ARGISNULL(1); + Datum table_datum = table_null ? (Datum)0 : PG_GETARG_DATUM(1); + Oid new_type = get_fn_expr_argtype(fcinfo->flinfo, 2); + bool new_null = PG_ARGISNULL(2); + Datum new_datum = new_null ? (Datum)0 : PG_GETARG_DATUM(2); + Oid old_type = get_fn_expr_argtype(fcinfo->flinfo, 3); + bool old_null = PG_ARGISNULL(3); + Datum old_datum = old_null ? (Datum)0 : PG_GETARG_DATUM(3); + + if (!OidIsValid(table_type) || !OidIsValid(new_type) || !OidIsValid(old_type)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_update_transfn invalid argument types"))); + } + + MemoryContext old_ctx = MemoryContextSwitchTo(allocContext); + // debug code + // MemoryContextStats(allocContext); + pgvalue_t *table_name = pgvalue_create(table_datum, table_type, -1, fcinfo->fncollation, table_null); + pgvalue_t *new_value = pgvalue_create(new_datum, new_type, -1, fcinfo->fncollation, new_null); + pgvalue_t *old_value = pgvalue_create(old_datum, old_type, -1, fcinfo->fncollation, old_null); + if (table_name) pgvalue_ensure_detoast(table_name); + if (new_value) pgvalue_ensure_detoast(new_value); + if (old_value) pgvalue_ensure_detoast(old_value); + MemoryContextSwitchTo(old_ctx); + + if (!table_name || !new_value || !old_value) { + if (table_name) pgvalue_free(table_name); + if (new_value) pgvalue_free(new_value); + if (old_value) pgvalue_free(old_value); + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("cloudsync_update_transfn failed to allocate values"))); + } + + if (!cloudsync_update_payload_append(payload, table_name, new_value, old_value)) { + if (table_name && payload->table_name != table_name) pgvalue_free(table_name); + if (new_value) pgvalue_free(new_value); + if (old_value) pgvalue_free(old_value); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_update_transfn failed to append payload"))); + } + + PG_RETURN_POINTER(payload); +} + +PG_FUNCTION_INFO_V1(cloudsync_update_finalfn); +Datum cloudsync_update_finalfn (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + PG_RETURN_BOOL(true); + } + + cloudsync_update_payload *payload = (cloudsync_update_payload *)PG_GETARG_POINTER(0); + if (!payload || payload->count == 0) { + PG_RETURN_BOOL(true); + } + + cloudsync_context *data = get_cloudsync_context(); + cloudsync_table_context *table = NULL; + int rc = DBRES_OK; + bool spi_connected = false; + char buffer[1024]; + char buffer2[1024]; + size_t pklen = sizeof(buffer); + size_t oldpklen = sizeof(buffer2); + char *pk = NULL; + char *oldpk = NULL; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + const char *table_name = database_value_text((dbvalue_t *)payload->table_name); + table = table_lookup(data, table_name); + if (!table) { + char meta_name[1024]; + snprintf(meta_name, sizeof(meta_name), "%s_cloudsync", table_name); + if (!database_table_exists(data, meta_name, cloudsync_schema(data))) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_update", table_name))); + } + + table_algo algo = dbutils_table_settings_get_algo(data, table_name); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + if (!table_add_to_context(data, algo, table_name)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to load table context for %s", table_name))); + } + + table = table_lookup(data, table_name); + if (!table) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_update", table_name))); + } + } + + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + int pk_count = table_count_pks(table); + if (payload->count < pk_count) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Not enough primary key values in cloudsync_update payload"))); + } + int max_expected = pk_count + table_count_cols(table); + if (payload->count > max_expected) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Too many values in cloudsync_update payload: got " + "%d expected <= %d", + payload->count, max_expected))); + } + + bool prikey_changed = false; + for (int i = 0; i < pk_count; i++) { + if (dbutils_value_compare((dbvalue_t *)payload->old_values[i], (dbvalue_t *)payload->new_values[i]) != 0) { + prikey_changed = true; + break; + } + } + + pk = pk_encode_prikey((dbvalue_t **)payload->new_values, pk_count, buffer, &pklen); + if (!pk) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Not enough memory to encode the primary key(s)"))); + } + if (pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + pk = NULL; + ereport(ERROR, (errcode(ERRCODE_NOT_NULL_VIOLATION), errmsg("Update aborted because primary key in table %s contains NULL values", table_name))); + } + if (prikey_changed) { + oldpk = pk_encode_prikey((dbvalue_t **)payload->old_values, pk_count, buffer2, &oldpklen); + if (!oldpk) { + rc = DBRES_NOMEM; + goto cleanup; + } + + rc = local_mark_delete_meta(table, oldpk, oldpklen, db_version, cloudsync_bumpseq(data)); + if (rc != DBRES_OK) goto cleanup; + + rc = local_update_move_meta(table, pk, pklen, oldpk, oldpklen, db_version); + if (rc != DBRES_OK) goto cleanup; + + rc = local_mark_insert_sentinel_meta(table, pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc != DBRES_OK) goto cleanup; + } + + for (int i = 0; i < table_count_cols(table); i++) { + int col_index = pk_count + i; + if (col_index >= payload->count) break; + + if (dbutils_value_compare((dbvalue_t *)payload->old_values[col_index], (dbvalue_t *)payload->new_values[col_index]) != 0) { + if (table_col_algo(table, i) == col_algo_block) { + // Block column: diff old and new text, emit per-block metadata changes + const char *new_text = (const char *)database_value_text(payload->new_values[col_index]); + const char *delim = table_col_delimiter(table, i); + const char *col = table_colname(table, i); + + // Read existing blocks from blocks table + block_list_t *old_blocks = block_list_create_empty(); + char *like_pattern = block_build_colname(col, "%"); + if (like_pattern && old_blocks) { + char *list_sql = cloudsync_memory_mprintf( + "SELECT col_name, col_value FROM %s WHERE pk = $1 AND col_name LIKE $2 ORDER BY col_name COLLATE \"C\"", + table_blocks_ref(table)); + if (list_sql) { + dbvm_t *list_vm = NULL; + if (databasevm_prepare(data, list_sql, &list_vm, 0) == DBRES_OK) { + databasevm_bind_blob(list_vm, 1, pk, (int)pklen); + databasevm_bind_text(list_vm, 2, like_pattern, -1); + while (databasevm_step(list_vm) == DBRES_ROW) { + const char *bcn = database_column_text(list_vm, 0); + const char *bval = database_column_text(list_vm, 1); + const char *pos = block_extract_position_id(bcn); + if (pos && old_blocks) { + block_list_add(old_blocks, bval ? bval : "", pos); + } + } + databasevm_finalize(list_vm); + } + cloudsync_memory_free(list_sql); + } + } + + // Split new text into parts (NULL text = all blocks removed) + block_list_t *new_blocks = new_text ? block_split(new_text, delim) : block_list_create_empty(); + if (new_blocks && old_blocks) { + // Build array of new content strings (NULL when count is 0) + const char **new_parts = NULL; + if (new_blocks->count > 0) { + new_parts = (const char **)cloudsync_memory_alloc( + (uint64_t)(new_blocks->count * sizeof(char *))); + if (new_parts) { + for (int b = 0; b < new_blocks->count; b++) { + new_parts[b] = new_blocks->entries[b].content; + } + } + } + + if (new_parts || new_blocks->count == 0) { + block_diff_t *diff = block_diff(old_blocks->entries, old_blocks->count, + new_parts, new_blocks->count); + if (diff) { + for (int d = 0; d < diff->count; d++) { + block_diff_entry_t *de = &diff->entries[d]; + char *block_cn = block_build_colname(col, de->position_id); + if (!block_cn) continue; + + if (de->type == BLOCK_DIFF_ADDED || de->type == BLOCK_DIFF_MODIFIED) { + rc = local_mark_insert_or_update_meta(table, pk, pklen, block_cn, + db_version, cloudsync_bumpseq(data)); + // Store block value + if (rc == DBRES_OK && table_block_value_write_stmt(table)) { + dbvm_t *wvm = table_block_value_write_stmt(table); + databasevm_bind_blob(wvm, 1, pk, (int)pklen); + databasevm_bind_text(wvm, 2, block_cn, -1); + databasevm_bind_text(wvm, 3, de->content, -1); + databasevm_step(wvm); + databasevm_reset(wvm); + } + } else if (de->type == BLOCK_DIFF_REMOVED) { + // Mark block as deleted in metadata (even col_version) + rc = local_mark_delete_block_meta(table, pk, pklen, block_cn, + db_version, cloudsync_bumpseq(data)); + // Remove from blocks table + if (rc == DBRES_OK) { + block_delete_value_external(data, table, pk, pklen, block_cn); + } + } + cloudsync_memory_free(block_cn); + if (rc != DBRES_OK) break; + } + block_diff_free(diff); + } + if (new_parts) cloudsync_memory_free((void *)new_parts); + } + } + if (new_blocks) block_list_free(new_blocks); + if (old_blocks) block_list_free(old_blocks); + if (like_pattern) cloudsync_memory_free(like_pattern); + if (rc != DBRES_OK) goto cleanup; + } else { + rc = local_mark_insert_or_update_meta(table, pk, pklen, table_colname(table, i), db_version, cloudsync_bumpseq(data)); + if (rc != DBRES_OK) goto cleanup; + } + } + } + +cleanup: + if (pk != buffer) cloudsync_memory_free(pk); + if (oldpk && (oldpk != buffer2)) cloudsync_memory_free(oldpk); + } + PG_CATCH(); + { + if (payload) { + cloudsync_update_payload_free(payload); + } + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (payload) { + cloudsync_update_payload_free(payload); + } + if (spi_connected) SPI_finish(); + + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", database_errmsg(data)))); + } + + PG_RETURN_BOOL(true); +} + +// Placeholder - not implemented yet +PG_FUNCTION_INFO_V1(cloudsync_payload_encode); +Datum cloudsync_payload_encode (PG_FUNCTION_ARGS) { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cloudsync_payload_encode should not be called directly - use aggregate version"))); + PG_RETURN_NULL(); +} + +// MARK: - Schema - + +PG_FUNCTION_INFO_V1(pg_cloudsync_set_schema); +Datum pg_cloudsync_set_schema (PG_FUNCTION_ARGS) { + const char *schema = NULL; + + if (!PG_ARGISNULL(0)) { + schema = text_to_cstring(PG_GETARG_TEXT_PP(0)); + } + + cloudsync_context *data = get_cloudsync_context(); + cloudsync_set_schema(data, schema); + + // Persist schema to settings so it is restored on context re-initialization. + // Only persist if settings table exists (it may not exist before cloudsync_init). + int spi_rc = SPI_connect(); + if (spi_rc == SPI_OK_CONNECT) { + if (database_internal_table_exists(data, CLOUDSYNC_SETTINGS_NAME)) { + dbutils_settings_set_key_value(data, "schema", schema); + } + SPI_finish(); + } + + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_schema); +Datum pg_cloudsync_schema (PG_FUNCTION_ARGS) { + cloudsync_context *data = get_cloudsync_context(); + const char *schema = cloudsync_schema(data); + + if (!schema) { + PG_RETURN_NULL(); + } + + PG_RETURN_TEXT_P(cstring_to_text(schema)); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_table_schema); +Datum pg_cloudsync_table_schema (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + const char *schema = cloudsync_table_schema(data, table_name); + + if (!schema) { + PG_RETURN_NULL(); + } + + PG_RETURN_TEXT_P(cstring_to_text(schema)); +} + +// MARK: - Changes - + +// Encode a single value using cloudsync pk encoding +static bytea *cloudsync_encode_value_from_datum (Datum val, Oid typeid, int32 typmod, Oid collation, bool isnull) { + pgvalue_t *v = pgvalue_create(val, typeid, typmod, collation, isnull); + if (!v) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("cloudsync: failed to allocate value"))); + } + if (!isnull) { + pgvalue_ensure_detoast(v); + } + + size_t encoded_len = pk_encode_size((dbvalue_t **)&v, 1, 0, -1); + bytea *out = (bytea *)palloc(VARHDRSZ + encoded_len); + if (!out) { + pgvalue_free(v); + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("cloudsync: failed to allocate encoding buffer"))); + } + + pk_encode((dbvalue_t **)&v, 1, VARDATA(out), false, &encoded_len, -1); + SET_VARSIZE(out, VARHDRSZ + encoded_len); + + pgvalue_free(v); + return out; +} + +// Encode a NULL value using cloudsync pk encoding +static bytea *cloudsync_encode_null_value (void) { + return cloudsync_encode_value_from_datum((Datum)0, TEXTOID, -1, InvalidOid, true); +} + +// Hold a decoded pk-encoded value with its original type +typedef struct { + int dbtype; + int64_t ival; + double dval; + char *pval; + int64_t len; + bool isnull; +} cloudsync_decoded_value; + +// Decode a single pk-encoded value into a typed representation +static int cloudsync_decode_value_cb (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { + cloudsync_decoded_value *out = (cloudsync_decoded_value *)xdata; + if (!out || index != 0) return DBRES_ERROR; + + out->dbtype = type; + out->isnull = false; + out->ival = 0; + out->dval = 0.0; + out->pval = NULL; + out->len = 0; + + switch (type) { + case DBTYPE_INTEGER: + out->ival = ival; + break; + case DBTYPE_FLOAT: + out->dval = dval; + break; + case DBTYPE_TEXT: + out->pval = pnstrdup(pval, (int)ival); + out->len = ival; + break; + case DBTYPE_BLOB: + if (ival > 0) { + out->pval = (char *)palloc((size_t)ival); + memcpy(out->pval, pval, (size_t)ival); + } + out->len = ival; + break; + case DBTYPE_NULL: + out->isnull = true; + break; + default: + return DBRES_ERROR; + } + return DBRES_OK; +} + +// Map a column Oid to the decoded type Oid that would be used for non-NULL values. +// This ensures NULL and non-NULL values use consistent types for SPI plan caching. +// The mapping must match pgvalue_dbtype() in pgvalue.c which determines encode/decode types. +// For example, INT4OID columns decode to INT8OID, UUIDOID columns decode to TEXTOID. +static Oid map_column_oid_to_decoded_oid(Oid col_oid) { + switch (col_oid) { + // Integer types → INT8OID (all integers decode to int64) + // Must match DBTYPE_INTEGER cases in pgvalue_dbtype() + case INT2OID: + case INT4OID: + case INT8OID: + case BOOLOID: // BOOLEAN encodes/decodes as INTEGER + case CHAROID: // "char" encodes/decodes as INTEGER + case OIDOID: // OID encodes/decodes as INTEGER + return INT8OID; + // Float types → FLOAT8OID (all floats decode to double) + // Must match DBTYPE_FLOAT cases in pgvalue_dbtype() + case FLOAT4OID: + case FLOAT8OID: + case NUMERICOID: + return FLOAT8OID; + // Binary types → BYTEAOID + // Must match DBTYPE_BLOB cases in pgvalue_dbtype() + case BYTEAOID: + return BYTEAOID; + // All other types (text, varchar, uuid, json, date, timestamp, etc.) → TEXTOID + // These all encode/decode as DBTYPE_TEXT + default: + return TEXTOID; + } +} + +// Get the Oid of a column from the system catalog. +// Requires SPI to be connected. Returns InvalidOid if not found. +static Oid get_column_oid(const char *schema, const char *table_name, const char *column_name) { + if (!table_name || !column_name) return InvalidOid; + + const char *query = + "SELECT a.atttypid " + "FROM pg_attribute a " + "JOIN pg_class c ON c.oid = a.attrelid " + "LEFT JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE c.relname = $1 " + "AND a.attname = $2 " + "AND a.attnum > 0 " + "AND NOT a.attisdropped " + "AND (n.nspname = $3 OR $3 IS NULL)"; + + Oid argtypes[3] = {TEXTOID, TEXTOID, TEXTOID}; + Datum values[3]; + char nulls[3] = {' ', ' ', schema ? ' ' : 'n'}; + + values[0] = CStringGetTextDatum(table_name); + values[1] = CStringGetTextDatum(column_name); + values[2] = schema ? CStringGetTextDatum(schema) : (Datum)0; + + int ret = SPI_execute_with_args(query, 3, argtypes, values, nulls, true, 1); + + pfree(DatumGetPointer(values[0])); + pfree(DatumGetPointer(values[1])); + if (schema) pfree(DatumGetPointer(values[2])); + + if (ret != SPI_OK_SELECT || SPI_processed == 0) { + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return InvalidOid; + } + + bool isnull; + Datum col_oid = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); + Oid result = isnull ? InvalidOid : DatumGetObjectId(col_oid); + SPI_freetuptable(SPI_tuptable); + return result; +} + +// Decode encoded bytea into a pgvalue_t with the decoded base type. +// Type casting to the target column type is handled by the SQL statement. +static pgvalue_t *cloudsync_decode_bytea_to_pgvalue (bytea *encoded, bool *out_isnull) { + // Decode input guardrails. + if (out_isnull) *out_isnull = true; + if (!encoded) return NULL; + + // Decode bytea into C types with dbtype info. + cloudsync_decoded_value dv = {.isnull = true}; + int blen = (int)VARSIZE_ANY_EXHDR(encoded); + int decoded = pk_decode((char *)VARDATA_ANY(encoded), (size_t)blen, 1, NULL, -1, cloudsync_decode_value_cb, &dv); + if (decoded != 1) ereport(ERROR, (errmsg("cloudsync: failed to decode encoded value"))); + if (out_isnull) *out_isnull = dv.isnull; + if (dv.isnull) return NULL; + + // Map decoded C types into a PostgreSQL Datum with the base type. + // The SQL statement handles casting to the target column type via $n::typename. + Oid typoid = TEXTOID; + Datum datum; + + switch (dv.dbtype) { + case DBTYPE_INTEGER: + typoid = INT8OID; + datum = Int64GetDatum(dv.ival); + break; + case DBTYPE_FLOAT: + typoid = FLOAT8OID; + datum = Float8GetDatum(dv.dval); + break; + case DBTYPE_TEXT: { + typoid = TEXTOID; + Size tlen = dv.pval ? (Size)dv.len : 0; + text *t = (text *)palloc(VARHDRSZ + tlen); + SET_VARSIZE(t, VARHDRSZ + tlen); + if (tlen > 0) memmove(VARDATA(t), dv.pval, tlen); + datum = PointerGetDatum(t); + } break; + case DBTYPE_BLOB: { + typoid = BYTEAOID; + bytea *ba = (bytea *)palloc(VARHDRSZ + dv.len); + SET_VARSIZE(ba, VARHDRSZ + dv.len); + if (dv.len > 0) memcpy(VARDATA(ba), dv.pval, (size_t)dv.len); + datum = PointerGetDatum(ba); + } break; + case DBTYPE_NULL: + if (out_isnull) *out_isnull = true; + if (dv.pval) pfree(dv.pval); + return NULL; + default: + if (dv.pval) pfree(dv.pval); + ereport(ERROR, (errmsg("cloudsync: unsupported decoded type"))); + } + + if (dv.pval) pfree(dv.pval); + + return pgvalue_create(datum, typoid, -1, InvalidOid, false); +} + +PG_FUNCTION_INFO_V1(cloudsync_encode_value); +Datum cloudsync_encode_value(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + bytea *null_encoded = cloudsync_encode_null_value(); + PG_RETURN_BYTEA_P(null_encoded); + } + + Oid typeoid = get_fn_expr_argtype(fcinfo->flinfo, 0); + int32 typmod = -1; + Oid collid = PG_GET_COLLATION(); + + if (!OidIsValid(typeoid) || typeoid == ANYELEMENTOID) { + if (fcinfo->flinfo->fn_expr && IsA(fcinfo->flinfo->fn_expr, FuncExpr)) { + FuncExpr *fexpr = (FuncExpr *) fcinfo->flinfo->fn_expr; + if (fexpr->args && list_length(fexpr->args) >= 1) { + Node *arg = (Node *) linitial(fexpr->args); + typeoid = exprType(arg); + typmod = exprTypmod(arg); + collid = exprCollation(arg); + } + } + } + + if (!OidIsValid(typeoid) || typeoid == ANYELEMENTOID) { + ereport(ERROR, (errmsg("cloudsync_encode_any: unable to resolve argument type"))); + } + + Datum val = PG_GETARG_DATUM(0); + bytea *result = cloudsync_encode_value_from_datum(val, typeoid, typmod, collid, false); + PG_RETURN_BYTEA_P(result); +} + +PG_FUNCTION_INFO_V1(cloudsync_col_value); +Datum cloudsync_col_value(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cloudsync_col_value arguments cannot be NULL"))); + } + + // argv[0] -> table name + // argv[1] -> column name + // argv[2] -> encoded pk + + char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *col_name = text_to_cstring(PG_GETARG_TEXT_PP(1)); + bytea *encoded_pk = PG_GETARG_BYTEA_P(2); + + // check for special tombstone value + if (strcmp(col_name, CLOUDSYNC_TOMBSTONE_VALUE) == 0) { + bytea *null_encoded = cloudsync_encode_null_value(); + PG_RETURN_BYTEA_P(null_encoded); + } + + cloudsync_context *data = get_cloudsync_context(); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + ereport(ERROR, (errmsg("Unable to retrieve table name %s in clousdsync_col_value.", table_name))); + } + + // Block column: if col_name contains \x1F, read from blocks table + if (block_is_block_colname(col_name) && table_has_block_cols(table)) { + dbvm_t *bvm = table_block_value_read_stmt(table); + if (!bvm) { + bytea *null_encoded = cloudsync_encode_null_value(); + PG_RETURN_BYTEA_P(null_encoded); + } + + bytea *encoded_pk_b = PG_GETARG_BYTEA_P(2); + size_t b_pk_len = (size_t)VARSIZE_ANY_EXHDR(encoded_pk_b); + int brc = databasevm_bind_blob(bvm, 1, VARDATA_ANY(encoded_pk_b), (uint64_t)b_pk_len); + if (brc != DBRES_OK) { databasevm_reset(bvm); ereport(ERROR, (errmsg("cloudsync_col_value block bind error"))); } + brc = databasevm_bind_text(bvm, 2, col_name, -1); + if (brc != DBRES_OK) { databasevm_reset(bvm); ereport(ERROR, (errmsg("cloudsync_col_value block bind error"))); } + + brc = databasevm_step(bvm); + if (brc == DBRES_ROW) { + size_t blob_len = 0; + const void *blob = database_column_blob(bvm, 0, &blob_len); + bytea *result = NULL; + if (blob && blob_len > 0) { + result = (bytea *)palloc(VARHDRSZ + blob_len); + SET_VARSIZE(result, VARHDRSZ + blob_len); + memcpy(VARDATA(result), blob, blob_len); + } + databasevm_reset(bvm); + if (result) PG_RETURN_BYTEA_P(result); + PG_RETURN_NULL(); + } else { + databasevm_reset(bvm); + bytea *null_encoded = cloudsync_encode_null_value(); + PG_RETURN_BYTEA_P(null_encoded); + } + } + + // extract the right col_value vm associated to the column name + dbvm_t *vm = table_column_lookup(table, col_name, false, NULL); + if (!vm) { + ereport(ERROR, (errmsg("Unable to retrieve column value precompiled statement in clousdsync_col_value."))); + } + + // bind primary key values + size_t pk_len = (size_t)VARSIZE_ANY_EXHDR(encoded_pk); + int count = pk_decode_prikey((char *)VARDATA_ANY(encoded_pk), pk_len, pk_decode_bind_callback, (void *)vm); + if (count <= 0) { + ereport(ERROR, (errmsg("Unable to decode primary key value in clousdsync_col_value."))); + } + + // execute vm + int rc = databasevm_step(vm); + if (rc == DBRES_DONE) { + databasevm_reset(vm); + // row not found (RLS or genuinely missing) — return the RLS sentinel as bytea + const char *rls = CLOUDSYNC_RLS_RESTRICTED_VALUE; + size_t rls_len = strlen(rls); + bytea *result = (bytea *)palloc(VARHDRSZ + rls_len); + SET_VARSIZE(result, VARHDRSZ + rls_len); + memcpy(VARDATA(result), rls, rls_len); + PG_RETURN_BYTEA_P(result); + } else if (rc == DBRES_ROW) { + // copy value before reset invalidates SPI tuple memory + size_t blob_len = 0; + const void *blob = database_column_blob(vm, 0, &blob_len); + bytea *result = NULL; + if (blob && blob_len > 0) { + result = (bytea *)palloc(VARHDRSZ + blob_len); + SET_VARSIZE(result, VARHDRSZ + blob_len); + memcpy(VARDATA(result), blob, blob_len); + } + databasevm_reset(vm); + if (result) PG_RETURN_BYTEA_P(result); + PG_RETURN_NULL(); + } + + databasevm_reset(vm); + ereport(ERROR, (errmsg("cloudsync_col_value error: %s", cloudsync_errmsg(data)))); + PG_RETURN_NULL(); // unreachable, silences compiler +} + +// MARK: - Block-level LWW - + +PG_FUNCTION_INFO_V1(cloudsync_text_materialize); +Datum cloudsync_text_materialize (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_text_materialize: table_name and col_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + const char *col_name = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + cloudsync_context *data = get_cloudsync_context(); + cloudsync_pg_cleanup_state cleanup = {0}; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + cleanup.spi_connected = true; + + PG_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + { + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Unable to retrieve table name %s in cloudsync_text_materialize", table_name))); + } + + int col_idx = table_col_index(table, col_name); + if (col_idx < 0 || table_col_algo(table, col_idx) != col_algo_block) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Column %s in table %s is not configured as block-level", col_name, table_name))); + } + + // Extract PK values from VARIADIC "any" (args starting from index 2) + cleanup.argv = pgvalues_from_args(fcinfo, 2, &cleanup.argc); + + // Normalize PK values to text for consistent encoding + pgvalues_normalize_to_text(cleanup.argv, cleanup.argc); + + int expected_pks = table_count_pks(table); + if (cleanup.argc != expected_pks) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Expected %d primary key values, got %d", expected_pks, cleanup.argc))); + } + + size_t pklen = sizeof(cleanup.pk_buffer); + cleanup.pk = pk_encode_prikey((dbvalue_t **)cleanup.argv, cleanup.argc, cleanup.pk_buffer, &pklen); + if (!cleanup.pk || cleanup.pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + if (cleanup.pk == PRIKEY_NULL_CONSTRAINT_ERROR) cleanup.pk = NULL; + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Failed to encode primary key(s)"))); + } + + int rc = block_materialize_column(data, table, cleanup.pk, (int)pklen, col_name); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("%s", cloudsync_errmsg(data)))); + } + } + PG_END_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + + cloudsync_pg_cleanup(0, PointerGetDatum(&cleanup)); + PG_RETURN_BOOL(true); +} + +// Track SRF execution state across calls +typedef struct { + Portal portal; + TupleDesc outdesc; + bool spi_connected; +} SRFState; + +// Build the UNION ALL SQL for cloudsync_changes SRF +static char * build_union_sql (void) { + char *result = NULL; + MemoryContext caller_ctx = CurrentMemoryContext; + + if (SPI_connect() != SPI_OK_CONNECT) { + ereport(ERROR, (errmsg("cloudsync: SPI_connect failed"))); + } + + PG_TRY(); + { + const char *sql = + "SELECT n.nspname, c.relname " + "FROM pg_class c " + "JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE c.relkind = 'r' " + " AND n.nspname NOT IN ('pg_catalog','information_schema') " + " AND c.relname LIKE '%\\_cloudsync' ESCAPE '\\' " + "ORDER BY n.nspname, c.relname"; + + int rc = SPI_execute(sql, true, 0); + if (rc != SPI_OK_SELECT || !SPI_tuptable) { + ereport(ERROR, (errmsg("cloudsync: SPI_execute failed while listing *_cloudsync"))); + } + + StringInfoData buf; + initStringInfo(&buf); + + uint64 ntables = SPI_processed; + char **nsp_list = NULL; + char **rel_list = NULL; + if (ntables > 0) { + nsp_list = (char **)palloc0(sizeof(char *) * ntables); + rel_list = (char **)palloc0(sizeof(char *) * ntables); + } + for (uint64 i = 0; i < ntables; i++) { + HeapTuple tup = SPI_tuptable->vals[i]; + TupleDesc td = SPI_tuptable->tupdesc; + char *nsp = SPI_getvalue(tup, td, 1); + char *rel = SPI_getvalue(tup, td, 2); + if (!nsp || !rel) { + if (nsp) pfree(nsp); + if (rel) pfree(rel); + continue; + } + nsp_list[i] = pstrdup(nsp); + rel_list[i] = pstrdup(rel); + pfree(nsp); + pfree(rel); + } + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + + bool first = true; + for (uint64 i = 0; i < ntables; i++) { + char *nsp = nsp_list ? nsp_list[i] : NULL; + char *rel = rel_list ? rel_list[i] : NULL; + if (!nsp || !rel) { + if (nsp) pfree(nsp); + if (rel) pfree(rel); + continue; + } + + size_t rlen = strlen(rel); + if (rlen <= 10) {pfree(nsp); pfree(rel); continue;} /* "_cloudsync" */ + + char *base = pstrdup(rel); + base[rlen - 10] = '\0'; + + char *quoted_base = quote_literal_cstr(base); + const char *quoted_nsp = quote_identifier(nsp); + const char *quoted_rel = quote_identifier(rel); + + if (!first) appendStringInfoString(&buf, " UNION ALL "); + first = false; + + + /* + * Build a single SELECT per table that: + * - reads change rows from
_cloudsync (t1) + * - joins the base table (b) using decoded PK components + * - computes col_value in-SQL with a CASE over col_name + * + * This avoids calling cloudsync_col_value() (and therefore avoids + * executing extra SPI queries per row), while still honoring RLS: + * if the base row is not visible, the LEFT JOIN yields NULL and we + * return the restricted sentinel value (then filtered out). + */ + + char *nsp_lit = quote_literal_cstr(nsp); + char *base_lit = quote_literal_cstr(base); + + /* Collect PK columns (name + SQL type) */ + StringInfoData pkq; + initStringInfo(&pkq); + appendStringInfo(&pkq, + "SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS typ " + "FROM pg_index i " + "JOIN pg_class c ON c.oid = i.indrelid " + "JOIN pg_namespace n ON n.oid = c.relnamespace " + "JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = ANY(i.indkey) " + "WHERE i.indisprimary AND n.nspname = %s AND c.relname = %s " + "ORDER BY array_position(i.indkey, a.attnum)", + nsp_lit, base_lit + ); + int pkrc = SPI_execute(pkq.data, true, 0); + pfree(pkq.data); + if (pkrc != SPI_OK_SELECT || (SPI_processed == 0) || (!SPI_tuptable)) { + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + ereport(ERROR, (errmsg("cloudsync: unable to resolve primary key for %s.%s", nsp, base))); + } + uint64 npk = SPI_processed; + + StringInfoData joincond; + initStringInfo(&joincond); + for (uint64 k = 0; k < npk; k++) { + HeapTuple pkt = SPI_tuptable->vals[k]; + TupleDesc pkd = SPI_tuptable->tupdesc; + char *pkname = SPI_getvalue(pkt, pkd, 1); + char *pktype = SPI_getvalue(pkt, pkd, 2); + if (!pkname || !pktype) { + if (pkname) pfree(pkname); + if (pktype) pfree(pktype); + pfree(joincond.data); + SPI_freetuptable(SPI_tuptable); + ereport(ERROR, (errmsg("cloudsync: invalid pk metadata for %s.%s", nsp, base))); + } + + if (k > 0) appendStringInfoString(&joincond, " AND "); + appendStringInfo(&joincond, + "b.%s = cloudsync_pk_decode(t1.pk, %llu)::%s", + quote_identifier(pkname), + (unsigned long long)(k + 1), + pktype + ); + pfree(pkname); + pfree(pktype); + } + SPI_freetuptable(SPI_tuptable); + + // Check if blocks table exists for this table + char blocks_tbl_name[1024]; + snprintf(blocks_tbl_name, sizeof(blocks_tbl_name), "%s_cloudsync_blocks", base); + StringInfoData btq; + initStringInfo(&btq); + appendStringInfo(&btq, + "SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE c.relname = %s AND n.nspname = %s AND c.relkind = 'r'", + quote_literal_cstr(blocks_tbl_name), nsp_lit); + int btrc = SPI_execute(btq.data, true, 1); + bool has_blocks_table = (btrc == SPI_OK_SELECT && SPI_processed > 0); + if (SPI_tuptable) { SPI_freetuptable(SPI_tuptable); SPI_tuptable = NULL; } + pfree(btq.data); + + /* Collect all base-table columns to build CASE over t1.col_name */ + StringInfoData colq; + initStringInfo(&colq); + appendStringInfo(&colq, + "SELECT a.attname " + "FROM pg_attribute a " + "JOIN pg_class c ON c.oid = a.attrelid " + "JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE a.attnum > 0 AND NOT a.attisdropped " + " AND n.nspname = %s AND c.relname = %s " + "ORDER BY a.attnum", + nsp_lit, base_lit + ); + int colrc = SPI_execute(colq.data, true, 0); + pfree(colq.data); + if (colrc != SPI_OK_SELECT || !SPI_tuptable) { + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + ereport(ERROR, (errmsg("cloudsync: unable to resolve columns for %s.%s", nsp, base))); + } + uint64 ncols = SPI_processed; + + StringInfoData caseexpr; + initStringInfo(&caseexpr); + appendStringInfoString(&caseexpr, + "CASE " + "WHEN t1.col_name = '" CLOUDSYNC_TOMBSTONE_VALUE "' THEN " CLOUDSYNC_NULL_VALUE_BYTEA " " + "WHEN b.ctid IS NULL THEN " CLOUDSYNC_RLS_RESTRICTED_VALUE_BYTEA " " + ); + if (has_blocks_table) { + appendStringInfo(&caseexpr, + "WHEN t1.col_name LIKE '%%' || chr(31) || '%%' THEN " + "(SELECT cloudsync_encode_value(blk.col_value) FROM %s.\"%s_cloudsync_blocks\" blk " + "WHERE blk.pk = t1.pk AND blk.col_name = t1.col_name) ", + quote_identifier(nsp), base); + } + appendStringInfoString(&caseexpr, + "ELSE CASE t1.col_name " + ); + + for (uint64 k = 0; k < ncols; k++) { + HeapTuple ct = SPI_tuptable->vals[k]; + TupleDesc cd = SPI_tuptable->tupdesc; + char *cname = SPI_getvalue(ct, cd, 1); + if (!cname) continue; + + appendStringInfo(&caseexpr, + "WHEN %s THEN cloudsync_encode_value(b.%s) ", + quote_literal_cstr(cname), + quote_identifier(cname) + ); + pfree(cname); + } + SPI_freetuptable(SPI_tuptable); + + appendStringInfoString(&caseexpr, + "ELSE " CLOUDSYNC_RLS_RESTRICTED_VALUE_BYTEA " END END" + ); + + const char *quoted_base_ident = quote_identifier(base); + + appendStringInfo(&buf, + "SELECT * FROM (" + "SELECT %s AS tbl, t1.pk, t1.col_name, " + "%s AS col_value, " + "t1.col_version, t1.db_version, site_tbl.site_id, " + "COALESCE(t2.col_version, 1) AS cl, t1.seq " + "FROM %s.%s t1 " + "LEFT JOIN cloudsync_site_id site_tbl ON t1.site_id = site_tbl.id " + "LEFT JOIN %s.%s t2 " + " ON t1.pk = t2.pk AND t2.col_name = '%s' " + "LEFT JOIN %s.%s b ON %s " + ") s WHERE s.col_value IS DISTINCT FROM %s", + quoted_base, + caseexpr.data, + quoted_nsp, quoted_rel, + quoted_nsp, quoted_rel, + CLOUDSYNC_TOMBSTONE_VALUE, + quoted_nsp, quoted_base_ident, + joincond.data, + CLOUDSYNC_RLS_RESTRICTED_VALUE_BYTEA + ); + + // Only free quoted identifiers if they're different from the input + // (quote_identifier returns input pointer if no quoting needed) + if (quoted_base_ident != base) pfree((void*)quoted_base_ident); + pfree(joincond.data); + pfree(caseexpr.data); + + pfree(base); + pfree(base_lit); + + pfree(quoted_base); + pfree(nsp_lit); + bool nsp_was_quoted = (quoted_nsp != nsp); + pfree(nsp); + if (nsp_was_quoted) pfree((void *)quoted_nsp); + bool rel_was_quoted = (quoted_rel != rel); + pfree(rel); + if (rel_was_quoted) pfree((void *)quoted_rel); + } + if (nsp_list) pfree(nsp_list); + if (rel_list) pfree(rel_list); + + // Ensure result survives SPI_finish by allocating in the caller context. + MemoryContext old_ctx = MemoryContextSwitchTo(caller_ctx); + if (first) { + result = pstrdup( + "SELECT NULL::text AS tbl, NULL::bytea AS pk, NULL::text AS col_name, NULL::bytea AS col_value, " + "NULL::bigint AS col_version, NULL::bigint AS db_version, NULL::bytea AS site_id, " + "NULL::bigint AS cl, NULL::bigint AS seq WHERE false" + ); + } else { + result = pstrdup(buf.data); + } + MemoryContextSwitchTo(old_ctx); + + SPI_finish(); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + return result; +} + +PG_FUNCTION_INFO_V1(cloudsync_changes_select); +Datum cloudsync_changes_select(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx = NULL; + SRFState *st_local = NULL; + bool spi_connected_local = false; + bool srf_done = false; + Datum srf_result = (Datum) 0; + + // NOTE: the SRF_RETURN_* macros expand to `return`, so they must run AFTER + // PG_END_TRY(). Returning from inside the PG_TRY block skips PG_END_TRY and + // leaves PG_exception_stack pointing at this (now-dead) frame; a later + // ereport(ERROR) in the same query then siglongjmp()s into freed stack and + // segfaults. Compute the result inside the guarded block, return outside it. + PG_TRY(); + { + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + MemoryContext oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + int64 min_db_version = PG_GETARG_INT64(0); + bool site_is_null = PG_ARGISNULL(1); + bytea *filter_site_id = site_is_null ? NULL : PG_GETARG_BYTEA_PP(1); + + char *union_sql = build_union_sql(); + + StringInfoData q; + initStringInfo(&q); + appendStringInfo(&q, + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM ( %s ) u " + "WHERE db_version > $1 " + " AND ($2 IS NULL OR site_id = $2) " + "ORDER BY db_version, seq ASC", + union_sql + ); + + if (SPI_connect() != SPI_OK_CONNECT) { + ereport(ERROR, (errmsg("cloudsync: SPI_connect failed in SRF"))); + } + spi_connected_local = true; + + Oid argtypes[2] = {INT8OID, BYTEAOID}; + Datum values[2]; + char nulls[2] = {' ', ' '}; + + values[0] = Int64GetDatum(min_db_version); + if (site_is_null) { nulls[1] = 'n'; values[1] = (Datum)0; } + else values[1] = PointerGetDatum(filter_site_id); + + Portal portal = SPI_cursor_open_with_args(NULL, q.data, 2, argtypes, values, nulls, true, 0); + if (!portal) { + ereport(ERROR, (errmsg("cloudsync: SPI_cursor_open failed in SRF"))); + } + + TupleDesc outdesc; + if (get_call_result_type(fcinfo, NULL, &outdesc) != TYPEFUNC_COMPOSITE) { + ereport(ERROR, (errmsg("cloudsync: return type must be composite"))); + } + outdesc = BlessTupleDesc(outdesc); + + SRFState *st = palloc0(sizeof(SRFState)); + st->portal = portal; + st->outdesc = outdesc; + st->spi_connected = true; + funcctx->user_fctx = st; + st_local = st; + + pfree(union_sql); + pfree(q.data); + + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + SRFState *st = (SRFState *) funcctx->user_fctx; + st_local = st; + + SPI_cursor_fetch(st->portal, true, 1); + if (SPI_processed == 0) { + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + SPI_cursor_close(st->portal); + st->portal = NULL; + + SPI_finish(); + st->spi_connected = false; + + // SPI operations may leave us in multi_call_memory_ctx + // Must switch to a safe context before SRF_RETURN_DONE deletes it + MemoryContextSwitchTo(fcinfo->flinfo->fn_mcxt); + + srf_done = true; + } else { + HeapTuple tup = SPI_tuptable->vals[0]; + TupleDesc td = SPI_tuptable->tupdesc; + + Datum outvals[9]; + bool outnulls[9]; + for (int i = 0; i < 9; i++) { + outvals[i] = SPI_getbinval(tup, td, i+1, &outnulls[i]); + if (!outnulls[i]) { + Form_pg_attribute att = TupleDescAttr(td, i); + outvals[i] = datumCopy(outvals[i], att->attbyval, att->attlen); + } + } + + HeapTuple outtup = heap_form_tuple(st->outdesc, outvals, outnulls); + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + srf_result = HeapTupleGetDatum(outtup); + } + } + PG_CATCH(); + { + // Switch to function's context (safe, won't be deleted) + // Avoids assertion if we're currently in multi_call_memory_ctx + MemoryContextSwitchTo(fcinfo->flinfo->fn_mcxt); + + if (st_local && st_local->portal) { + SPI_cursor_close(st_local->portal); + st_local->portal = NULL; + } + + if (st_local && st_local->spi_connected) { + SPI_finish(); + st_local->spi_connected = false; + spi_connected_local = false; + } else if (spi_connected_local) { + SPI_finish(); + spi_connected_local = false; + } + + PG_RE_THROW(); + } + PG_END_TRY(); + + // Return outside the PG_TRY so PG_END_TRY() always restores PG_exception_stack. + if (srf_done) SRF_RETURN_DONE(funcctx); + SRF_RETURN_NEXT(funcctx, srf_result); +} + +// Trigger INSERT + +PG_FUNCTION_INFO_V1(cloudsync_changes_insert_trigger); +Datum cloudsync_changes_insert_trigger (PG_FUNCTION_ARGS) { + // sanity check + bool spi_connected = false; + TriggerData *trigdata = (TriggerData *) fcinfo->context; + if (!CALLED_AS_TRIGGER(fcinfo)) ereport(ERROR, (errmsg("cloudsync_changes_insert_trigger must be called as trigger"))); + if (!TRIGGER_FIRED_BY_INSERT(trigdata->tg_event)) ereport(ERROR, (errmsg("Only INSERT allowed on cloudsync_changes"))); + + HeapTuple newtup = trigdata->tg_trigtuple; + pgvalue_t *col_value = NULL; + PG_TRY(); + { + TupleDesc desc = trigdata->tg_relation->rd_att; + bool isnull; + + char *insert_tbl = text_to_cstring((text*) DatumGetPointer(heap_getattr(newtup, 1, desc, &isnull))); + if (isnull) ereport(ERROR, (errmsg("tbl cannot be NULL"))); + + bytea *insert_pk = (bytea*) DatumGetPointer(heap_getattr(newtup, 2, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("pk cannot be NULL"))); + int insert_pk_len = (int)(VARSIZE_ANY_EXHDR(insert_pk)); + + Datum insert_name_datum = heap_getattr(newtup, 3, desc, &isnull); + char *insert_name = NULL; + bool insert_name_owned = false; + if (isnull) { + insert_name = CLOUDSYNC_TOMBSTONE_VALUE; + } else { + insert_name = text_to_cstring((text*) DatumGetPointer(insert_name_datum)); + insert_name_owned = true; + } + bool is_tombstone = (strcmp(insert_name, CLOUDSYNC_TOMBSTONE_VALUE) == 0); + + // raw_insert_value is declared as bytea in the view (cloudsync-encoded value) + bytea *insert_value_encoded = (bytea*) DatumGetPointer(heap_getattr(newtup, 4, desc, &isnull)); + + int64 insert_col_version = DatumGetInt64(heap_getattr(newtup, 5, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("col_version cannot be NULL"))); + + int64 insert_db_version = DatumGetInt64(heap_getattr(newtup, 6, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("db_version cannot be NULL"))); + + bytea *insert_site_id = (bytea*) DatumGetPointer(heap_getattr(newtup, 7, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("site_id cannot be NULL"))); + int insert_site_id_len = (int)(VARSIZE_ANY_EXHDR(insert_site_id)); + + int64 insert_cl = DatumGetInt64(heap_getattr(newtup, 8, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("cl cannot be NULL"))); + + int64 insert_seq = DatumGetInt64(heap_getattr(newtup, 9, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("seq cannot be NULL"))); + + // lookup algo in cloudsync_tables + cloudsync_context *data = get_cloudsync_context(); + cloudsync_table_context *table = table_lookup(data, insert_tbl); + if (!table) ereport(ERROR, (errmsg("Unable to find table"))); + + if (SPI_connect() != SPI_OK_CONNECT) ereport(ERROR, (errmsg("cloudsync: SPI_connect failed in trigger"))); + spi_connected = true; + + // Decode value to base type; SQL statement handles type casting via $n::typename. + // For non-NULL values, we get the decoded base type (INT8OID for integers, TEXTOID for text/UUID, etc). + // For NULL values, we must use the SAME decoded type that non-NULL values would use. + // This ensures type consistency across all calls, as SPI caches parameter types on first prepare. + if (!is_tombstone) { + bool value_is_null = false; + col_value = cloudsync_decode_bytea_to_pgvalue(insert_value_encoded, &value_is_null); + + // When value is NULL, create a typed NULL pgvalue with the decoded type. + // We map the column's actual Oid to the corresponding decoded Oid (e.g., INT4OID → INT8OID). + if (!col_value && value_is_null) { + Oid col_oid = get_column_oid(table_schema(table), insert_tbl, insert_name); + if (OidIsValid(col_oid)) { + Oid decoded_oid = map_column_oid_to_decoded_oid(col_oid); + col_value = pgvalue_create((Datum)0, decoded_oid, -1, InvalidOid, true); + } + } + } + + int rc = DBRES_OK; + int64_t rowid = 0; + if (table_algo_isgos(table)) { + rc = merge_insert_col(data, table, VARDATA_ANY(insert_pk), insert_pk_len, insert_name, col_value, (int64_t)insert_col_version, (int64_t)insert_db_version, VARDATA_ANY(insert_site_id), insert_site_id_len, (int64_t)insert_seq, &rowid); + } else { + rc = merge_insert (data, table, VARDATA_ANY(insert_pk), insert_pk_len, insert_cl, insert_name, col_value, insert_col_version, insert_db_version, VARDATA_ANY(insert_site_id), insert_site_id_len, insert_seq, &rowid); + } + if (rc != DBRES_OK) { + ereport(ERROR, (errmsg("Error during merge_insert: %s", database_errmsg(data)))); + } + + pgvalue_free(col_value); + pfree(insert_tbl); + if (insert_name_owned) pfree(insert_name); + + SPI_finish(); + spi_connected = false; + } + PG_CATCH(); + { + pgvalue_free(col_value); + if (spi_connected) { + SPI_finish(); + spi_connected = false; + } + PG_RE_THROW(); + } + PG_END_TRY(); + + return PointerGetDatum(newtup); +} diff --git a/src/postgresql/database_postgresql.c b/src/postgresql/database_postgresql.c new file mode 100644 index 00000000..3fc6310f --- /dev/null +++ b/src/postgresql/database_postgresql.c @@ -0,0 +1,3076 @@ +// +// database_postgresql.c +// cloudsync +// +// Created by Marco Bambini on 03/12/25. +// + +// PostgreSQL requires postgres.h to be included FIRST +// It sets up the entire environment including platform compatibility +#include "postgres.h" + +#include +#include +#include + +#include "../cloudsync.h" +#include "../database.h" +#include "../dbutils.h" +#include "../sql.h" +#include "../utils.h" + +// PostgreSQL SPI and other headers +#include "access/xact.h" +#include "catalog/pg_type.h" +#include "executor/spi.h" +#include "funcapi.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/snapmgr.h" + +#include "pgvalue.h" + +// ============================================================================ +// SPI CONNECTION REQUIREMENTS +// ============================================================================ +// +// IMPORTANT: This implementation requires an active SPI connection to function. +// The Extension Function that calls these functions MUST: +// +// 1. Call SPI_connect() before using any database functions +// 2. Call SPI_finish() before returning from the extension function +// +// ============================================================================ + +// MARK: - PREPARED STATEMENTS - + +// PostgreSQL SPI handles require knowing parameter count and types upfront. +// Solution: Defer actual SPI_prepare until first step(), after all bindings are set. +#define MAX_PARAMS 32 + +typedef struct { + // Prepared plan + SPIPlanPtr plan; + bool plan_is_prepared; + + // Cursor execution + Portal portal; // owned by statement + bool portal_open; + + // Current fetched batch (we fetch 1 row at a time, but SPI still returns a tuptable) + SPITupleTable *last_tuptable; // must SPI_freetuptable() before next fetch + HeapTuple current_tuple; + TupleDesc current_tupdesc; + + // Params + int nparams; + Oid types[MAX_PARAMS]; + Oid prepared_types[MAX_PARAMS]; // types used when plan was SPI_prepare'd + int prepared_nparams; // nparams at prepare time + Datum values[MAX_PARAMS]; + char nulls[MAX_PARAMS]; + bool executed_nonselect; // non-select executed already + + // Memory + MemoryContext stmt_mcxt; // lifetime = pg_stmt_t + MemoryContext bind_mcxt; // resettable region for parameters (cleared on clear_bindings/reset) + MemoryContext row_mcxt; // per-row scratch (cleared each step after consumer copies) + + // Context + const char *sql; + cloudsync_context *data; +} pg_stmt_t; + +static int database_refresh_snapshot (void); + +// MARK: - SQL - + +static char *sql_escape_character (const char *name, char *buffer, size_t bsize, char c) { + if (!name || !buffer || bsize < 1) { + if (buffer && bsize > 0) buffer[0] = '\0'; + return NULL; + } + + size_t i = 0, j = 0; + + while (name[i]) { + if (name[i] == c) { + // Need space for 2 chars (escaped c) + null + if (j >= bsize - 2) { + elog(WARNING, "Identifier name too long for buffer, truncated: %s", name); + break; + } + buffer[j++] = c; + buffer[j++] = c; + } else { + // Need space for 1 char + null + if (j >= bsize - 1) { + elog(WARNING, "Identifier name too long for buffer, truncated: %s", name); + break; + } + buffer[j++] = name[i]; + } + i++; + } + + buffer[j] = '\0'; + return buffer; +} + +static char *sql_escape_identifier (const char *name, char *buffer, size_t bsize) { + // PostgreSQL identifier escaping: double any embedded double quotes + // Does NOT add surrounding quotes (caller's responsibility) + // Similar to SQLite's %q behavior for escaping + return sql_escape_character(name, buffer, bsize, '"'); +} + +static char *sql_escape_literal (const char *name, char *buffer, size_t bsize) { + // Escapes single quotes for use inside SQL string literals: ' → '' + // Does NOT add surrounding quotes (caller's responsibility) + return sql_escape_character(name, buffer, bsize, '\''); +} + +char *sql_build_drop_table (const char *table_name, char *buffer, int bsize, bool is_meta) { + // Escape the table name (doubles any embedded quotes) + char escaped[512]; + sql_escape_identifier(table_name, escaped, sizeof(escaped)); + + // Add the surrounding quotes in the format string + if (is_meta) { + snprintf(buffer, bsize, "DROP TABLE IF EXISTS \"%s_cloudsync\";", escaped); + } else { + snprintf(buffer, bsize, "DROP TABLE IF EXISTS \"%s\";", escaped); + } + + return buffer; +} + +char *sql_build_select_nonpk_by_pk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(data); + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + char *sql = cloudsync_memory_mprintf(SQL_BUILD_SELECT_NONPK_COLS_BY_PK, qualified); + cloudsync_memory_free(qualified); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_delete_by_pk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(data); + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + char *sql = cloudsync_memory_mprintf(SQL_BUILD_DELETE_ROW_BY_PK, qualified); + cloudsync_memory_free(qualified); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_insert_pk_ignore (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(data); + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + char *sql = cloudsync_memory_mprintf(SQL_BUILD_INSERT_PK_IGNORE, qualified); + cloudsync_memory_free(qualified); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_upsert_pk_and_col (cloudsync_context *data, const char *table_name, const char *colname, const char *schema) { + UNUSED_PARAMETER(data); + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + char *sql = cloudsync_memory_mprintf(SQL_BUILD_UPSERT_PK_AND_COL, qualified, colname, colname); + cloudsync_memory_free(qualified); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_upsert_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema) { + if (ncolnames <= 0 || !colnames) return NULL; + + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + // Build VALUES list for column names: ('col_a',1),('col_b',2) + // Column names are SQL literals here, so escape single quotes + size_t values_cap = (size_t)ncolnames * 128 + 1; + char *col_values = cloudsync_memory_alloc(values_cap); + if (!col_values) { cloudsync_memory_free(qualified); return NULL; } + + size_t vpos = 0; + for (int i = 0; i < ncolnames; i++) { + char esc[1024]; + sql_escape_literal(colnames[i], esc, sizeof(esc)); + vpos += snprintf(col_values + vpos, values_cap - vpos, "%s('%s'::text,%d)", + i > 0 ? "," : "", esc, i + 1); + } + + // Build meta-query that generates the final INSERT...ON CONFLICT SQL with proper types + char *meta_sql = cloudsync_memory_mprintf( + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + "), " + "pk_count AS (SELECT count(*) AS n FROM pk), " + "cols AS (" + " SELECT u.colname, format_type(a.atttypid, a.atttypmod) AS coltype, u.ord " + " FROM (VALUES %s) AS u(colname, ord) " + " JOIN pg_attribute a ON a.attrelid = (SELECT oid FROM tbl) AND a.attname = u.colname " + " WHERE a.attnum > 0 AND NOT a.attisdropped" + ") " + "SELECT " + " 'INSERT INTO ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' (' || (SELECT string_agg(format('%%I', attname), ',' ORDER BY ord) FROM pk)" + " || ',' || (SELECT string_agg(format('%%I', colname), ',' ORDER BY ord) FROM cols) || ')'" + " || ' VALUES (' || (SELECT string_agg(format('$%%s::%%s', ord, coltype), ',' ORDER BY ord) FROM pk)" + " || ',' || (SELECT string_agg(format('$%%s::%%s', (SELECT n FROM pk_count) + ord, coltype), ',' ORDER BY ord) FROM cols) || ')'" + " || ' ON CONFLICT (' || (SELECT string_agg(format('%%I', attname), ',' ORDER BY ord) FROM pk) || ')'" + " || ' DO UPDATE SET ' || (SELECT string_agg(format('%%I=EXCLUDED.%%I', colname, colname), ',' ORDER BY ord) FROM cols)" + " || ';';", + qualified, col_values + ); + + cloudsync_memory_free(qualified); + cloudsync_memory_free(col_values); + if (!meta_sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, meta_sql, &query); + cloudsync_memory_free(meta_sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_update_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema) { + if (ncolnames <= 0 || !colnames) return NULL; + + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + // Build VALUES list for column names: ('col_a',1),('col_b',2) + size_t values_cap = (size_t)ncolnames * 128 + 1; + char *col_values = cloudsync_memory_alloc(values_cap); + if (!col_values) { cloudsync_memory_free(qualified); return NULL; } + + size_t vpos = 0; + for (int i = 0; i < ncolnames; i++) { + char esc[1024]; + sql_escape_literal(colnames[i], esc, sizeof(esc)); + vpos += snprintf(col_values + vpos, values_cap - vpos, "%s('%s'::text,%d)", + i > 0 ? "," : "", esc, i + 1); + } + + // Build meta-query that generates UPDATE ... SET col=$ WHERE pk=$ + char *meta_sql = cloudsync_memory_mprintf( + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + "), " + "pk_count AS (SELECT count(*) AS n FROM pk), " + "cols AS (" + " SELECT u.colname, format_type(a.atttypid, a.atttypmod) AS coltype, u.ord " + " FROM (VALUES %s) AS u(colname, ord) " + " JOIN pg_attribute a ON a.attrelid = (SELECT oid FROM tbl) AND a.attname = u.colname " + " WHERE a.attnum > 0 AND NOT a.attisdropped" + ") " + "SELECT " + " 'UPDATE ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' SET ' || (SELECT string_agg(format('%%I=$%%s::%%s', colname, (SELECT n FROM pk_count) + ord, coltype), ',' ORDER BY ord) FROM cols)" + " || ' WHERE ' || (SELECT string_agg(format('%%I=$%%s::%%s', attname, ord, coltype), ' AND ' ORDER BY ord) FROM pk)" + " || ';';", + qualified, col_values + ); + + cloudsync_memory_free(qualified); + cloudsync_memory_free(col_values); + if (!meta_sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, meta_sql, &query); + cloudsync_memory_free(meta_sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_select_cols_by_pk (cloudsync_context *data, const char *table_name, const char *colname, const char *schema) { + UNUSED_PARAMETER(data); + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + char *sql = cloudsync_memory_mprintf(SQL_BUILD_SELECT_COLS_BY_PK_FMT, qualified, colname); + cloudsync_memory_free(qualified); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_rekey_pk_and_reset_version_except_col (cloudsync_context *data, const char *table_name, const char *except_col) { + char *meta_ref = database_build_meta_ref(cloudsync_schema(data), table_name); + if (!meta_ref) return NULL; + + char *result = cloudsync_memory_mprintf(SQL_CLOUDSYNC_REKEY_PK_AND_RESET_VERSION_EXCEPT_COL, meta_ref, except_col, meta_ref, meta_ref, except_col); + cloudsync_memory_free(meta_ref); + return result; +} + +char *database_table_schema (const char *table_name) { + if (!table_name) return NULL; + + // Build metadata table name + char meta_table[256]; + snprintf(meta_table, sizeof(meta_table), "%s_cloudsync", table_name); + + // Query system catalogs to find the schema of the metadata table. + // Rationale: The metadata table is created in the same schema as the base table, + // so finding its location tells us which schema the table belongs to. + const char *query = + "SELECT n.nspname " + "FROM pg_class c " + "JOIN pg_namespace n ON c.relnamespace = n.oid " + "WHERE c.relname = $1 " + "AND c.relkind = 'r'"; // 'r' = ordinary table + + char *schema = NULL; + + if (SPI_connect() != SPI_OK_CONNECT) { + return NULL; + } + + Oid argtypes[1] = {TEXTOID}; + Datum values[1] = {CStringGetTextDatum(meta_table)}; + char nulls[1] = {' '}; + + int rc = SPI_execute_with_args(query, 1, argtypes, values, nulls, true, 1); + + if (rc == SPI_OK_SELECT && SPI_processed > 0) { + TupleDesc tupdesc = SPI_tuptable->tupdesc; + HeapTuple tuple = SPI_tuptable->vals[0]; + bool isnull; + + Datum datum = SPI_getbinval(tuple, tupdesc, 1, &isnull); + if (!isnull) { + // pg_namespace.nspname is type 'name', not 'text' + Name nspname = DatumGetName(datum); + schema = cloudsync_string_dup(NameStr(*nspname)); + } + } + + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + } + pfree(DatumGetPointer(values[0])); + SPI_finish(); + + // Returns NULL if metadata table doesn't exist yet (during initialization). + // Caller should fall back to cloudsync_schema() in this case. + return schema; +} + +char *database_build_meta_ref (const char *schema, const char *table_name) { + char escaped_table[512]; + sql_escape_identifier(table_name, escaped_table, sizeof(escaped_table)); + if (schema) { + char escaped_schema[512]; + sql_escape_identifier(schema, escaped_schema, sizeof(escaped_schema)); + return cloudsync_memory_mprintf("\"%s\".\"%s_cloudsync\"", escaped_schema, escaped_table); + } + return cloudsync_memory_mprintf("\"%s_cloudsync\"", escaped_table); +} + +char *database_build_base_ref (const char *schema, const char *table_name) { + char escaped_table[512]; + sql_escape_identifier(table_name, escaped_table, sizeof(escaped_table)); + if (schema) { + char escaped_schema[512]; + sql_escape_identifier(schema, escaped_schema, sizeof(escaped_schema)); + return cloudsync_memory_mprintf("\"%s\".\"%s\"", escaped_schema, escaped_table); + } + return cloudsync_memory_mprintf("\"%s\"", escaped_table); +} + +char *database_build_blocks_ref (const char *schema, const char *table_name) { + char escaped_table[512]; + sql_escape_identifier(table_name, escaped_table, sizeof(escaped_table)); + if (schema) { + char escaped_schema[512]; + sql_escape_identifier(schema, escaped_schema, sizeof(escaped_schema)); + return cloudsync_memory_mprintf("\"%s\".\"%s_cloudsync_blocks\"", escaped_schema, escaped_table); + } + return cloudsync_memory_mprintf("\"%s_cloudsync_blocks\"", escaped_table); +} + +// Schema-aware SQL builder for PostgreSQL: deletes columns not in schema or pkcol. +// Schema parameter: pass empty string to fall back to current_schema() via SQL. +char *sql_build_delete_cols_not_in_schema_query (const char *schema, const char *table_name, const char *meta_ref, const char *pkcol) { + const char *schema_param = schema ? schema : ""; + + char esc_table[1024], esc_schema[1024]; + sql_escape_literal(table_name, esc_table, sizeof(esc_table)); + sql_escape_literal(schema_param, esc_schema, sizeof(esc_schema)); + + return cloudsync_memory_mprintf( + "DELETE FROM %s WHERE col_name NOT IN (" + "SELECT column_name FROM information_schema.columns WHERE table_name = '%s' " + "AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "UNION SELECT '%s'" + ");", + meta_ref, esc_table, esc_schema, pkcol + ); +} + +// Builds query to get comma-separated list of primary key column names. +char *sql_build_pk_collist_query (const char *schema, const char *table_name) { + const char *schema_param = schema ? schema : ""; + + char esc_table[1024], esc_schema[1024]; + sql_escape_literal(table_name, esc_table, sizeof(esc_table)); + sql_escape_literal(schema_param, esc_schema, sizeof(esc_schema)); + + return cloudsync_memory_mprintf( + "SELECT string_agg(quote_ident(column_name), ',') " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND constraint_name LIKE '%%_pkey';", + esc_table, esc_schema + ); +} + +// Builds query to get SELECT list of decoded primary key columns. +char *sql_build_pk_decode_selectlist_query (const char *schema, const char *table_name) { + const char *schema_param = schema ? schema : ""; + + char esc_table[1024], esc_schema[1024]; + sql_escape_literal(table_name, esc_table, sizeof(esc_table)); + sql_escape_literal(schema_param, esc_schema, sizeof(esc_schema)); + + return cloudsync_memory_mprintf( + "SELECT string_agg(" + "'cloudsync_pk_decode(pk, ' || ordinal_position || ') AS ' || quote_ident(column_name), ',' ORDER BY ordinal_position" + ") " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND constraint_name LIKE '%%_pkey';", + esc_table, esc_schema + ); +} + +// Builds query to get qualified (schema.table.column) primary key column list. +char *sql_build_pk_qualified_collist_query (const char *schema, const char *table_name) { + const char *schema_param = schema ? schema : ""; + + char esc_table[1024], esc_schema[1024]; + sql_escape_literal(table_name, esc_table, sizeof(esc_table)); + sql_escape_literal(schema_param, esc_schema, sizeof(esc_schema)); + + return cloudsync_memory_mprintf( + "SELECT string_agg(quote_ident(column_name), ',' ORDER BY ordinal_position) " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND constraint_name LIKE '%%_pkey';", esc_table, esc_schema + ); +} + +char *sql_build_insert_missing_pks_query(const char *schema, const char *table_name, + const char *pkvalues_identifiers, + const char *base_ref, const char *meta_ref, + const char *filter) { + UNUSED_PARAMETER(schema); + + char esc_table[1024]; + sql_escape_literal(table_name, esc_table, sizeof(esc_table)); + + // PostgreSQL: Use NOT EXISTS with cloudsync_pk_encode to avoid EXCEPT type mismatch. + // + // CRITICAL: Pass PK columns directly to VARIADIC functions (NOT wrapped in ARRAY[]). + // This preserves each column's actual type (TEXT, INTEGER, etc.) for correct pk_encode. + // Using ARRAY[] would require all elements to be the same type, causing errors with + // mixed-type composite PKs (e.g., TEXT + INTEGER). + // + // Example: cloudsync_insert('table', col1, col2) where col1=TEXT, col2=INTEGER + // PostgreSQL's VARIADIC handling preserves each type and matches SQLite's encoding. + if (filter) { + return cloudsync_memory_mprintf( + "SELECT cloudsync_insert('%s', %s) " + "FROM %s b " + "WHERE (%s) AND NOT EXISTS (" + " SELECT 1 FROM %s m WHERE m.pk = cloudsync_pk_encode(%s)" + ");", + esc_table, pkvalues_identifiers, base_ref, filter, meta_ref, pkvalues_identifiers + ); + } + return cloudsync_memory_mprintf( + "SELECT cloudsync_insert('%s', %s) " + "FROM %s b " + "WHERE NOT EXISTS (" + " SELECT 1 FROM %s m WHERE m.pk = cloudsync_pk_encode(%s)" + ");", + esc_table, pkvalues_identifiers, base_ref, meta_ref, pkvalues_identifiers + ); +} + +// MARK: - HELPER FUNCTIONS - + +// Map SPI result codes to DBRES +static int map_spi_result (int rc) { + switch (rc) { + case SPI_OK_SELECT: + case SPI_OK_INSERT: + case SPI_OK_UPDATE: + case SPI_OK_DELETE: + case SPI_OK_UTILITY: + return DBRES_OK; + case SPI_OK_INSERT_RETURNING: + case SPI_OK_UPDATE_RETURNING: + case SPI_OK_DELETE_RETURNING: + return DBRES_ROW; + default: + return DBRES_ERROR; + } +} + +static void clear_fetch_batch (pg_stmt_t *stmt) { + if (!stmt) return; + if (stmt->last_tuptable) { + SPI_freetuptable(stmt->last_tuptable); + stmt->last_tuptable = NULL; + } + stmt->current_tuple = NULL; + stmt->current_tupdesc = NULL; + if (stmt->row_mcxt) MemoryContextReset(stmt->row_mcxt); +} + +static void close_portal (pg_stmt_t *stmt) { + if (!stmt) return; + + // Always clear portal_open first to maintain consistent state + stmt->portal_open = false; + + if (!stmt->portal) return; + + PG_TRY(); + { + SPI_cursor_close(stmt->portal); + } + PG_CATCH(); + { + // Log but don't throw - we're cleaning up + FlushErrorState(); + } + PG_END_TRY(); + stmt->portal = NULL; +} + +static inline Datum get_datum (pg_stmt_t *stmt, int col /* 0-based */, bool *isnull, Oid *type) { + if (!stmt || !stmt->current_tuple || !stmt->current_tupdesc) { + if (isnull) *isnull = true; + if (type) *type = 0; + return (Datum) 0; + } + if (type) *type = SPI_gettypeid(stmt->current_tupdesc, col + 1); + return SPI_getbinval(stmt->current_tuple, stmt->current_tupdesc, col + 1, isnull); +} + +// MARK: - PRIVATE - + +int database_select1_value (cloudsync_context *data, const char *sql, char **ptr_value, int64_t *int_value, DBTYPE expected_type) { + cloudsync_reset_error(data); + + // init values and sanity check expected_type + if (ptr_value) *ptr_value = NULL; + if (int_value) *int_value = 0; + if (expected_type != DBTYPE_INTEGER && expected_type != DBTYPE_TEXT && expected_type != DBTYPE_BLOB) { + return cloudsync_set_error(data, "Invalid expected_type", DBRES_MISUSE); + } + + int rc = SPI_execute(sql, true, 0); + if (rc < 0) { + rc = cloudsync_set_error(data, "SPI_execute failed in database_select1_value", DBRES_ERROR); + goto cleanup; + } + + // ensure at least one column + if (!SPI_tuptable || !SPI_tuptable->tupdesc) { + rc = cloudsync_set_error(data, "No result table", DBRES_ERROR); + goto cleanup; + } + if (SPI_tuptable->tupdesc->natts < 1) { + rc = cloudsync_set_error(data, "No columns in result", DBRES_ERROR); + goto cleanup; + } + + // no rows OK + if (SPI_processed == 0) { + rc = DBRES_OK; + goto cleanup; + } + + HeapTuple tuple = SPI_tuptable->vals[0]; + bool isnull; + Datum datum = SPI_getbinval(tuple, SPI_tuptable->tupdesc, 1, &isnull); + + // NULL value is OK + if (isnull) { + rc = DBRES_OK; + goto cleanup; + } + + // Get type info + Oid typeid = SPI_gettypeid(SPI_tuptable->tupdesc, 1); + + if (expected_type == DBTYPE_INTEGER) { + switch (typeid) { + case INT2OID: + *int_value = (int64_t)DatumGetInt16(datum); + break; + case INT4OID: + *int_value = (int64_t)DatumGetInt32(datum); + break; + case INT8OID: + *int_value = DatumGetInt64(datum); + break; + default: + rc = cloudsync_set_error(data, "Type mismatch: expected integer", DBRES_ERROR); + goto cleanup; + } + } else if (expected_type == DBTYPE_TEXT) { + char *val = SPI_getvalue(tuple, SPI_tuptable->tupdesc, 1); + if (val) { + size_t len = strlen(val); + char *ptr = cloudsync_memory_alloc(len + 1); + if (!ptr) { + pfree(val); + rc = cloudsync_set_error(data, "Memory allocation failed", DBRES_NOMEM); + goto cleanup; + } + memcpy(ptr, val, len); + ptr[len] = '\0'; + if (ptr_value) *ptr_value = ptr; + if (int_value) *int_value = (int64_t)len; + pfree(val); + } + } else if (expected_type == DBTYPE_BLOB) { + bytea *ba = DatumGetByteaP(datum); + int len = VARSIZE(ba) - VARHDRSZ; + if (len > 0) { + char *ptr = cloudsync_memory_alloc(len); + if (!ptr) { + rc = cloudsync_set_error(data, "Memory allocation failed", DBRES_NOMEM); + goto cleanup; + } + memcpy(ptr, VARDATA(ba), len); + if (ptr_value) *ptr_value = ptr; + if (int_value) *int_value = len; + } + } + + rc = DBRES_OK; + +cleanup: + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return rc; +} + +int database_select2_values (cloudsync_context *data, const char *sql, char **value, int64_t *len, int64_t *value2) { + cloudsync_reset_error(data); + + // init values + *value = NULL; + *value2 = 0; + *len = 0; + + int rc = SPI_execute(sql, true, 0); + if (rc < 0) { + rc = cloudsync_set_error(data, "SPI_execute failed in database_select2_values", DBRES_ERROR); + goto cleanup; + } + + if (!SPI_tuptable || !SPI_tuptable->tupdesc) { + rc = cloudsync_set_error(data, "No result table in database_select2_values", DBRES_ERROR); + goto cleanup; + } + if (SPI_tuptable->tupdesc->natts < 2) { + rc = cloudsync_set_error(data, "Result has fewer than 2 columns in database_select2_values", DBRES_ERROR); + goto cleanup; + } + if (SPI_processed == 0) { + rc = DBRES_OK; + goto cleanup; + } + + HeapTuple tuple = SPI_tuptable->vals[0]; + bool isnull; + + // First column - text/blob + Datum datum1 = SPI_getbinval(tuple, SPI_tuptable->tupdesc, 1, &isnull); + if (!isnull) { + Oid typeid = SPI_gettypeid(SPI_tuptable->tupdesc, 1); + if (typeid == BYTEAOID) { + bytea *ba = DatumGetByteaP(datum1); + int blob_len = VARSIZE(ba) - VARHDRSZ; + if (blob_len > 0) { + char *ptr = cloudsync_memory_alloc(blob_len); + if (!ptr) { + rc = DBRES_NOMEM; + goto cleanup; + } + + memcpy(ptr, VARDATA(ba), blob_len); + *value = ptr; + *len = blob_len; + } + } else { + text *txt = DatumGetTextP(datum1); + int text_len = VARSIZE(txt) - VARHDRSZ; + if (text_len > 0) { + char *ptr = cloudsync_memory_alloc(text_len + 1); + if (!ptr) { + rc = DBRES_NOMEM; + goto cleanup; + } + + memcpy(ptr, VARDATA(txt), text_len); + ptr[text_len] = '\0'; + *value = ptr; + *len = text_len; + } + } + } + + // Second column - int + Datum datum2 = SPI_getbinval(tuple, SPI_tuptable->tupdesc, 2, &isnull); + if (!isnull) { + Oid typeid = SPI_gettypeid(SPI_tuptable->tupdesc, 2); + if (typeid == INT8OID) { + *value2 = DatumGetInt64(datum2); + } else if (typeid == INT4OID) { + *value2 = (int64_t)DatumGetInt32(datum2); + } + } + + rc = DBRES_OK; + +cleanup: + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return rc; +} + +static bool database_system_exists (cloudsync_context *data, const char *name, const char *type, bool force_public, const char *schema) { + if (!name || !type) return false; + cloudsync_reset_error(data); + + bool exists = false; + const char *query; + // Schema parameter: pass empty string to fall back to current_schema() via SQL + const char *schema_param = (schema && schema[0]) ? schema : ""; + + if (strcmp(type, "table") == 0) { + if (force_public) { + query = "SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND tablename = $1"; + } else { + query = "SELECT 1 FROM pg_tables WHERE schemaname = COALESCE(NULLIF($2, ''), current_schema()) AND tablename = $1"; + } + } else if (strcmp(type, "trigger") == 0) { + query = "SELECT 1 FROM pg_trigger WHERE tgname = $1"; + } else { + return false; + } + + bool need_schema_param = !force_public && strcmp(type, "trigger") != 0; + Datum datum_name = CStringGetTextDatum(name); + Datum datum_schema = need_schema_param ? CStringGetTextDatum(schema_param) : (Datum)0; + + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + if (!need_schema_param) { + // force_public or trigger: only need table/trigger name parameter + Oid argtypes[1] = {TEXTOID}; + Datum values[1] = {datum_name}; + char nulls[1] = {' '}; + int rc = SPI_execute_with_args(query, 1, argtypes, values, nulls, true, 0); + exists = (rc >= 0 && SPI_processed > 0); + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + } else { + // table with schema parameter + Oid argtypes[2] = {TEXTOID, TEXTOID}; + Datum values[2] = {datum_name, datum_schema}; + char nulls[2] = {' ', ' '}; + int rc = SPI_execute_with_args(query, 2, argtypes, values, nulls, true, 0); + exists = (rc >= 0 && SPI_processed > 0); + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + } + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + exists = false; + } + PG_END_TRY(); + + pfree(DatumGetPointer(datum_name)); + if (need_schema_param) pfree(DatumGetPointer(datum_schema)); + + elog(DEBUG1, "database_system_exists %s: %d", name, exists); + return exists; + } + +// MARK: - GENERAL - + +int database_exec (cloudsync_context *data, const char *sql) { + if (!sql) return cloudsync_set_error(data, "SQL statement is NULL", DBRES_ERROR); + cloudsync_reset_error(data); + + int rc; + bool is_error = false; + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + rc = SPI_execute(sql, false, 0); + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + } + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + } + is_error = true; + } + PG_END_TRY(); + + if (is_error) return rc; + + // Increment command counter to make changes visible + if (rc >= 0) { + database_refresh_snapshot(); + return map_spi_result(rc); + } + + return cloudsync_set_error(data, "SPI_execute failed", DBRES_ERROR); +} + +int database_exec_callback (cloudsync_context *data, const char *sql, int (*callback)(void *xdata, int argc, char **values, char **names), void *xdata) { + if (!sql) return cloudsync_set_error(data, "SQL statement is NULL", DBRES_ERROR); + cloudsync_reset_error(data); + + int rc; + bool is_error = false; + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + rc = SPI_execute(sql, true, 0); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + is_error = true; + } + PG_END_TRY(); + + if (is_error) return rc; + if (rc < 0) return cloudsync_set_error(data, "SPI_execute failed", DBRES_ERROR); + + // Call callback for each row if provided + if (callback && SPI_tuptable) { + TupleDesc tupdesc = SPI_tuptable->tupdesc; + if (!tupdesc) { + SPI_freetuptable(SPI_tuptable); + return cloudsync_set_error(data, "Invalid tuple descriptor", DBRES_ERROR); + } + + int ncols = tupdesc->natts; + if (ncols <= 0) { + SPI_freetuptable(SPI_tuptable); + return DBRES_OK; + } + + // IMPORTANT: Save SPI state before any callback can modify it. + // Callbacks may execute SPI queries which overwrite global SPI_tuptable. + // We must copy all data we need BEFORE calling any callbacks. + uint64 nrows = SPI_processed; + SPITupleTable *saved_tuptable = SPI_tuptable; + + // No rows to process - free tuptable and return success + if (nrows == 0) { + SPI_freetuptable(saved_tuptable); + return DBRES_OK; + } + + // Allocate array for column names (shared across all rows) + char **names = cloudsync_memory_alloc(ncols * sizeof(char*)); + if (!names) { + SPI_freetuptable(saved_tuptable); + return DBRES_NOMEM; + } + + // Get column names - make copies to avoid pointing to internal memory + for (int i = 0; i < ncols; i++) { + Form_pg_attribute attr = TupleDescAttr(tupdesc, i); + if (attr) { + names[i] = cloudsync_string_dup(NameStr(attr->attname)); + } else { + names[i] = NULL; + } + } + + // Pre-extract ALL row values before calling any callbacks. + // This prevents SPI state corruption when callbacks run queries. + char ***all_values = cloudsync_memory_alloc(nrows * sizeof(char**)); + if (!all_values) { + for (int i = 0; i < ncols; i++) { + if (names[i]) cloudsync_memory_free(names[i]); + } + cloudsync_memory_free(names); + SPI_freetuptable(saved_tuptable); + return DBRES_NOMEM; + } + + // Extract values from all tuples + for (uint64 row = 0; row < nrows; row++) { + HeapTuple tuple = saved_tuptable->vals[row]; + all_values[row] = cloudsync_memory_alloc(ncols * sizeof(char*)); + if (!all_values[row]) { + // Cleanup already allocated rows + for (uint64 r = 0; r < row; r++) { + for (int c = 0; c < ncols; c++) { + if (all_values[r][c]) pfree(all_values[r][c]); + } + cloudsync_memory_free(all_values[r]); + } + cloudsync_memory_free(all_values); + for (int i = 0; i < ncols; i++) { + if (names[i]) cloudsync_memory_free(names[i]); + } + cloudsync_memory_free(names); + SPI_freetuptable(saved_tuptable); + return DBRES_NOMEM; + } + + if (!tuple) { + for (int i = 0; i < ncols; i++) all_values[row][i] = NULL; + continue; + } + + for (int i = 0; i < ncols; i++) { + bool isnull; + SPI_getbinval(tuple, tupdesc, i + 1, &isnull); + all_values[row][i] = (isnull) ? NULL : SPI_getvalue(tuple, tupdesc, i + 1); + } + } + + // Free SPI_tuptable BEFORE calling callbacks - we have all data we need + SPI_freetuptable(saved_tuptable); + SPI_tuptable = NULL; + + // Now process each row - callbacks can safely run SPI queries + int result = DBRES_OK; + for (uint64 row = 0; row < nrows; row++) { + int cb_rc = callback(xdata, ncols, all_values[row], names); + + if (cb_rc != 0) { + char errmsg[1024]; + snprintf(errmsg, sizeof(errmsg), "database_exec_callback aborted %d", cb_rc); + result = cloudsync_set_error(data, errmsg, DBRES_ABORT); + break; + } + } + + // Cleanup all extracted values + for (uint64 row = 0; row < nrows; row++) { + for (int i = 0; i < ncols; i++) { + if (all_values[row][i]) pfree(all_values[row][i]); + } + cloudsync_memory_free(all_values[row]); + } + cloudsync_memory_free(all_values); + + // Free column names + for (int i = 0; i < ncols; i++) { + if (names[i]) cloudsync_memory_free(names[i]); + } + cloudsync_memory_free(names); + + return result; + } + + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return DBRES_OK; +} + +int database_write (cloudsync_context *data, const char *sql, const char **bind_values, DBTYPE bind_types[], int bind_lens[], int bind_count) { + if (!sql) return cloudsync_set_error(data, "Invalid parameters to database_write", DBRES_ERROR); + cloudsync_reset_error(data); + + // Prepare statement + dbvm_t *stmt; + int rc = databasevm_prepare(data, sql, &stmt, 0); + if (rc != DBRES_OK) return rc; + + // Bind parameters + for (int i = 0; i < bind_count; i++) { + int param_idx = i + 1; + + switch (bind_types[i]) { + case DBTYPE_NULL: + rc = databasevm_bind_null(stmt, param_idx); + break; + case DBTYPE_INTEGER: { + int64_t val = strtoll(bind_values[i], NULL, 0); + rc = databasevm_bind_int(stmt, param_idx, val); + break; + } + case DBTYPE_FLOAT: { + double val = strtod(bind_values[i], NULL); + rc = databasevm_bind_double(stmt, param_idx, val); + break; + } + case DBTYPE_TEXT: + rc = databasevm_bind_text(stmt, param_idx, bind_values[i], bind_lens[i]); + break; + case DBTYPE_BLOB: + rc = databasevm_bind_blob(stmt, param_idx, bind_values[i], bind_lens[i]); + break; + default: + rc = DBRES_ERROR; + break; + } + + if (rc != DBRES_OK) { + databasevm_finalize(stmt); + return rc; + } + } + + // Execute + rc = databasevm_step(stmt); + databasevm_finalize(stmt); + + return (rc == DBRES_DONE) ? DBRES_OK : rc; +} + +int database_select_int (cloudsync_context *data, const char *sql, int64_t *value) { + return database_select1_value(data, sql, NULL, value, DBTYPE_INTEGER); +} + +int database_select_text (cloudsync_context *data, const char *sql, char **value) { + int64_t len = 0; + return database_select1_value(data, sql, value, &len, DBTYPE_TEXT); +} + +int database_select_blob (cloudsync_context *data, const char *sql, char **value, int64_t *len) { + return database_select1_value(data, sql, value, len, DBTYPE_BLOB); +} + +int database_select_blob_int (cloudsync_context *data, const char *sql, char **value, int64_t *len, int64_t *value2) { + return database_select2_values(data, sql, value, len, value2); +} + +int database_cleanup (cloudsync_context *data) { + // NOOP + return DBRES_OK; +} + +// MARK: - STATUS - +int database_errcode (cloudsync_context *data) { + return cloudsync_errcode(data); +} + +const char *database_errmsg (cloudsync_context *data) { + return cloudsync_errmsg(data); +} + +bool database_in_transaction (cloudsync_context *data) { + // In SPI context, we're always in a transaction + return IsTransactionState(); +} + +bool database_table_exists (cloudsync_context *data, const char *name, const char *schema) { + return database_system_exists(data, name, "table", false, schema); +} + +bool database_internal_table_exists (cloudsync_context *data, const char *name) { + // Internal tables always in public schema + return database_system_exists(data, name, "table", true, NULL); +} + +bool database_trigger_exists (cloudsync_context *data, const char *name) { + // Triggers: extract table name to get schema + // Trigger names follow pattern:
_cloudsync_ + // For now, pass NULL to use current_schema() + return database_system_exists(data, name, "trigger", false, NULL); +} + +// MARK: - SCHEMA INFO - + +static int64_t database_count_bind (cloudsync_context *data, const char *sql, const char *table_name, const char *schema) { + // Schema parameter: pass empty string to fall back to current_schema() via SQL + const char *schema_param = (schema && schema[0]) ? schema : ""; + + Oid argtypes[2] = {TEXTOID, TEXTOID}; + Datum values[2] = {CStringGetTextDatum(table_name), CStringGetTextDatum(schema_param)}; + char nulls[2] = {' ', ' '}; + + int64_t count = 0; + int rc = SPI_execute_with_args(sql, 2, argtypes, values, nulls, true, 0); + if (rc >= 0 && SPI_processed > 0 && SPI_tuptable) { + bool isnull; + Datum d = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); + if (!isnull) count = DatumGetInt64(d); + } + + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + pfree(DatumGetPointer(values[0])); + pfree(DatumGetPointer(values[1])); + return count; +} + +int database_count_pk (cloudsync_context *data, const char *table_name, bool not_null, const char *schema) { + const char *sql = + "SELECT COUNT(*) FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "WHERE tc.table_name = $1 AND tc.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY'"; + + return (int)database_count_bind(data, sql, table_name, schema); +} + +int database_count_nonpk (cloudsync_context *data, const char *table_name, const char *schema) { + const char *sql = + "SELECT COUNT(*) FROM information_schema.columns c " + "WHERE c.table_name = $1 " + "AND c.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + "AND c.column_name NOT IN (" + " SELECT kcu.column_name FROM information_schema.table_constraints tc " + " JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + " WHERE tc.table_name = $1 AND tc.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + " AND tc.constraint_type = 'PRIMARY KEY'" + ")"; + + return (int)database_count_bind(data, sql, table_name, schema); +} + +int database_count_int_pk (cloudsync_context *data, const char *table_name, const char *schema) { + const char *sql = + "SELECT COUNT(*) FROM information_schema.columns c " + "JOIN information_schema.key_column_usage kcu ON c.column_name = kcu.column_name AND c.table_schema = kcu.table_schema AND c.table_name = kcu.table_name " + "JOIN information_schema.table_constraints tc ON kcu.constraint_name = tc.constraint_name AND kcu.table_schema = tc.table_schema " + "WHERE c.table_name = $1 AND c.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY' " + "AND c.data_type IN ('smallint', 'integer', 'bigint')"; + + return (int)database_count_bind(data, sql, table_name, schema); +} + +int database_count_notnull_without_default (cloudsync_context *data, const char *table_name, const char *schema) { + const char *sql = + "SELECT COUNT(*) FROM information_schema.columns c " + "WHERE c.table_name = $1 " + "AND c.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + "AND c.is_nullable = 'NO' " + "AND c.column_default IS NULL " + "AND c.column_name NOT IN (" + " SELECT kcu.column_name FROM information_schema.table_constraints tc " + " JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + " WHERE tc.table_name = $1 AND tc.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + " AND tc.constraint_type = 'PRIMARY KEY'" + ")"; + + return (int)database_count_bind(data, sql, table_name, schema); +} + +/* +int database_debug (db_t *db, bool print_result) { + // PostgreSQL debug information + if (print_result) { + elog(DEBUG1, "PostgreSQL SPI debug info:"); + elog(DEBUG1, " SPI_processed: %lu", (unsigned long)SPI_processed); + elog(DEBUG1, " In transaction: %d", IsTransactionState()); + } + return DBRES_OK; +} + */ + +// MARK: - METADATA TABLES - + +int database_create_metatable (cloudsync_context *data, const char *table_name) { + int rc; + const char *schema = cloudsync_schema(data); + + char *meta_ref = database_build_meta_ref(schema, table_name); + if (!meta_ref) return DBRES_NOMEM; + + char *sql2 = cloudsync_memory_mprintf( + "CREATE TABLE IF NOT EXISTS %s (" + "pk BYTEA NOT NULL," + "col_name TEXT NOT NULL," + "col_version BIGINT," + "db_version BIGINT NOT NULL DEFAULT 0," + "seq INTEGER NOT NULL DEFAULT 0," + "site_id BIGINT NOT NULL DEFAULT 0," + "PRIMARY KEY (pk, col_name)" + ");", + meta_ref); + if (!sql2) { cloudsync_memory_free(meta_ref); return DBRES_NOMEM; } + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + if (rc != DBRES_OK) { cloudsync_memory_free(meta_ref); return rc; } + + // Create indices for performance + { + char escaped_tbl[512], escaped_sch[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + if (schema) { + sql_escape_identifier(schema, escaped_sch, sizeof(escaped_sch)); + sql2 = cloudsync_memory_mprintf( + "CREATE INDEX IF NOT EXISTS \"%s_cloudsync_db_version_idx\" " + "ON \"%s\".\"%s_cloudsync\" (db_version);", + escaped_tbl, escaped_sch, escaped_tbl); + } else { + sql2 = cloudsync_memory_mprintf( + "CREATE INDEX IF NOT EXISTS \"%s_cloudsync_db_version_idx\" " + "ON \"%s_cloudsync\" (db_version);", + escaped_tbl, escaped_tbl); + } + } + cloudsync_memory_free(meta_ref); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + return rc; +} + +// MARK: - TRIGGERS - + +static int database_create_insert_trigger_internal (cloudsync_context *data, const char *table_name, char *trigger_when, const char *schema) { + if (!table_name) return DBRES_MISUSE; + + const char *schema_param = (schema && schema[0]) ? schema : ""; + + char trigger_name[1024]; + char func_name[1024]; + char escaped_tbl[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + snprintf(trigger_name, sizeof(trigger_name), "cloudsync_after_insert_%s", escaped_tbl); + snprintf(func_name, sizeof(func_name), "cloudsync_after_insert_%s_fn", escaped_tbl); + + if (database_trigger_exists(data, trigger_name)) return DBRES_OK; + + char esc_tbl_literal[1024], esc_schema_literal[1024]; + sql_escape_literal(table_name, esc_tbl_literal, sizeof(esc_tbl_literal)); + sql_escape_literal(schema_param, esc_schema_literal, sizeof(esc_schema_literal)); + + char sql[2048]; + snprintf(sql, sizeof(sql), + "SELECT string_agg('NEW.' || quote_ident(kcu.column_name) || '::text', ',' ORDER BY kcu.ordinal_position) " + "FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu " + " ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "WHERE tc.table_name = '%s' AND tc.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY';", + esc_tbl_literal, esc_schema_literal); + + char *pk_list = NULL; + int rc = database_select_text(data, sql, &pk_list); + if (rc != DBRES_OK) return rc; + if (!pk_list || pk_list[0] == '\0') { + if (pk_list) cloudsync_memory_free(pk_list); + return cloudsync_set_error(data, "No primary key columns found for table", DBRES_ERROR); + } + + char *sql2 = cloudsync_memory_mprintf( + "CREATE OR REPLACE FUNCTION \"%s\"() RETURNS trigger AS $$ " + "BEGIN " + " IF cloudsync_is_sync('%s') THEN RETURN NEW; END IF; " + " PERFORM cloudsync_insert('%s', %s); " + " RETURN NEW; " + "END; " + "$$ LANGUAGE plpgsql;", + func_name, esc_tbl_literal, esc_tbl_literal, pk_list); + cloudsync_memory_free(pk_list); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + if (rc != DBRES_OK) return rc; + + char *base_ref = database_build_base_ref(schema, table_name); + if (!base_ref) return DBRES_NOMEM; + + sql2 = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%s\" AFTER INSERT ON %s %s " + "EXECUTE FUNCTION \"%s\"();", + trigger_name, base_ref, trigger_when ? trigger_when : "", func_name); + cloudsync_memory_free(base_ref); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + return rc; +} + +static int database_create_update_trigger_gos_internal (cloudsync_context *data, const char *table_name, const char *schema) { + if (!table_name) return DBRES_MISUSE; + + char trigger_name[1024]; + char func_name[1024]; + char escaped_tbl[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + snprintf(trigger_name, sizeof(trigger_name), "cloudsync_before_update_%s", escaped_tbl); + snprintf(func_name, sizeof(func_name), "cloudsync_before_update_%s_fn", escaped_tbl); + + if (database_trigger_exists(data, trigger_name)) return DBRES_OK; + + char esc_tbl_literal[1024]; + sql_escape_literal(table_name, esc_tbl_literal, sizeof(esc_tbl_literal)); + + char *sql = cloudsync_memory_mprintf( + "CREATE OR REPLACE FUNCTION \"%s\"() RETURNS trigger AS $$ " + "BEGIN " + " RAISE EXCEPTION 'Error: UPDATE operation is not allowed on table %s.'; " + "END; " + "$$ LANGUAGE plpgsql;", + func_name, esc_tbl_literal); + if (!sql) return DBRES_NOMEM; + + int rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + + char *base_ref = database_build_base_ref(schema, table_name); + if (!base_ref) return DBRES_NOMEM; + + sql = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%s\" BEFORE UPDATE ON %s " + "FOR EACH ROW WHEN (cloudsync_is_enabled('%s') = true) " + "EXECUTE FUNCTION \"%s\"();", + trigger_name, base_ref, esc_tbl_literal, func_name); + cloudsync_memory_free(base_ref); + if (!sql) return DBRES_NOMEM; + + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + return rc; +} + +static int database_create_update_trigger_internal (cloudsync_context *data, const char *table_name, const char *trigger_when, const char *schema) { + if (!table_name) return DBRES_MISUSE; + + const char *schema_param = (schema && schema[0]) ? schema : ""; + + char trigger_name[1024]; + char func_name[1024]; + char escaped_tbl[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + snprintf(trigger_name, sizeof(trigger_name), "cloudsync_after_update_%s", escaped_tbl); + snprintf(func_name, sizeof(func_name), "cloudsync_after_update_%s_fn", escaped_tbl); + + if (database_trigger_exists(data, trigger_name)) return DBRES_OK; + + char esc_tbl_literal[1024], esc_schema_literal[1024]; + sql_escape_literal(table_name, esc_tbl_literal, sizeof(esc_tbl_literal)); + sql_escape_literal(schema_param, esc_schema_literal, sizeof(esc_schema_literal)); + + char sql[2048]; + snprintf(sql, sizeof(sql), + "SELECT string_agg(" + " '(''%s'', NEW.' || quote_ident(kcu.column_name) || '::text, OLD.' || " + "quote_ident(kcu.column_name) || '::text)', " + " ', ' ORDER BY kcu.ordinal_position" + ") " + "FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu " + " ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "WHERE tc.table_name = '%s' AND tc.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY';", + esc_tbl_literal, esc_tbl_literal, esc_schema_literal); + + char *pk_values_list = NULL; + int rc = database_select_text(data, sql, &pk_values_list); + if (rc != DBRES_OK) return rc; + if (!pk_values_list || pk_values_list[0] == '\0') { + if (pk_values_list) cloudsync_memory_free(pk_values_list); + return cloudsync_set_error(data, "No primary key columns found for table", DBRES_ERROR); + } + + snprintf(sql, sizeof(sql), + "SELECT string_agg(" + " '(''%s'', NEW.' || quote_ident(c.column_name) || '::text, OLD.' || " + "quote_ident(c.column_name) || '::text)', " + " ', ' ORDER BY c.ordinal_position" + ") " + "FROM information_schema.columns c " + "WHERE c.table_name = '%s' " + "AND c.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND NOT EXISTS (" + " SELECT 1 FROM information_schema.table_constraints tc " + " JOIN information_schema.key_column_usage kcu " + " ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + " WHERE tc.table_name = c.table_name " + " AND tc.table_schema = c.table_schema " + " AND tc.constraint_type = 'PRIMARY KEY' " + " AND kcu.column_name = c.column_name" + ");", + esc_tbl_literal, esc_tbl_literal, esc_schema_literal); + + char *col_values_list = NULL; + rc = database_select_text(data, sql, &col_values_list); + if (rc != DBRES_OK) { + if (pk_values_list) cloudsync_memory_free(pk_values_list); + return rc; + } + + char *values_query = NULL; + if (col_values_list && col_values_list[0] != '\0') { + values_query = cloudsync_memory_mprintf("VALUES %s, %s", pk_values_list, col_values_list); + } else { + values_query = cloudsync_memory_mprintf("VALUES %s", pk_values_list); + } + + if (pk_values_list) cloudsync_memory_free(pk_values_list); + if (col_values_list) cloudsync_memory_free(col_values_list); + if (!values_query) return DBRES_NOMEM; + + char *sql2 = cloudsync_memory_mprintf( + "CREATE OR REPLACE FUNCTION \"%s\"() RETURNS trigger AS $$ " + "BEGIN " + " IF cloudsync_is_sync('%s') THEN RETURN NEW; END IF; " + " PERFORM cloudsync_update(table_name, new_value, old_value) " + " FROM (%s) AS v(table_name, new_value, old_value); " + " RETURN NEW; " + "END; " + "$$ LANGUAGE plpgsql;", + func_name, esc_tbl_literal, values_query); + cloudsync_memory_free(values_query); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + if (rc != DBRES_OK) return rc; + + char *base_ref = database_build_base_ref(schema, table_name); + if (!base_ref) return DBRES_NOMEM; + + sql2 = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%s\" AFTER UPDATE ON %s %s " + "EXECUTE FUNCTION \"%s\"();", + trigger_name, base_ref, trigger_when ? trigger_when : "", func_name); + cloudsync_memory_free(base_ref); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + return rc; +} + +static int database_create_delete_trigger_gos_internal (cloudsync_context *data, const char *table_name, const char *schema) { + if (!table_name) return DBRES_MISUSE; + + char trigger_name[1024]; + char func_name[1024]; + char escaped_tbl[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + snprintf(trigger_name, sizeof(trigger_name), "cloudsync_before_delete_%s", escaped_tbl); + snprintf(func_name, sizeof(func_name), "cloudsync_before_delete_%s_fn", escaped_tbl); + + if (database_trigger_exists(data, trigger_name)) return DBRES_OK; + + char esc_tbl_literal[1024]; + sql_escape_literal(table_name, esc_tbl_literal, sizeof(esc_tbl_literal)); + + char *sql = cloudsync_memory_mprintf( + "CREATE OR REPLACE FUNCTION \"%s\"() RETURNS trigger AS $$ " + "BEGIN " + " RAISE EXCEPTION 'Error: DELETE operation is not allowed on table %s.'; " + "END; " + "$$ LANGUAGE plpgsql;", + func_name, esc_tbl_literal); + if (!sql) return DBRES_NOMEM; + + int rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + + char *base_ref = database_build_base_ref(schema, table_name); + if (!base_ref) return DBRES_NOMEM; + + sql = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%s\" BEFORE DELETE ON %s " + "FOR EACH ROW WHEN (cloudsync_is_enabled('%s') = true) " + "EXECUTE FUNCTION \"%s\"();", + trigger_name, base_ref, esc_tbl_literal, func_name); + cloudsync_memory_free(base_ref); + if (!sql) return DBRES_NOMEM; + + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + return rc; +} + +static int database_create_delete_trigger_internal (cloudsync_context *data, const char *table_name, const char *trigger_when, const char *schema) { + if (!table_name) return DBRES_MISUSE; + + const char *schema_param = (schema && schema[0]) ? schema : ""; + + char trigger_name[1024]; + char func_name[1024]; + char escaped_tbl[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + snprintf(trigger_name, sizeof(trigger_name), "cloudsync_after_delete_%s", escaped_tbl); + snprintf(func_name, sizeof(func_name), "cloudsync_after_delete_%s_fn", escaped_tbl); + + if (database_trigger_exists(data, trigger_name)) return DBRES_OK; + + char esc_tbl_literal[1024], esc_schema_literal[1024]; + sql_escape_literal(table_name, esc_tbl_literal, sizeof(esc_tbl_literal)); + sql_escape_literal(schema_param, esc_schema_literal, sizeof(esc_schema_literal)); + + char sql[2048]; + snprintf(sql, sizeof(sql), + "SELECT string_agg('OLD.' || quote_ident(kcu.column_name) || '::text', ',' ORDER BY kcu.ordinal_position) " + "FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu " + " ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "WHERE tc.table_name = '%s' AND tc.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY';", + esc_tbl_literal, esc_schema_literal); + + char *pk_list = NULL; + int rc = database_select_text(data, sql, &pk_list); + if (rc != DBRES_OK) return rc; + if (!pk_list || pk_list[0] == '\0') { + if (pk_list) cloudsync_memory_free(pk_list); + return cloudsync_set_error(data, "No primary key columns found for table", DBRES_ERROR); + } + + char *sql2 = cloudsync_memory_mprintf( + "CREATE OR REPLACE FUNCTION \"%s\"() RETURNS trigger AS $$ " + "BEGIN " + " IF cloudsync_is_sync('%s') THEN RETURN OLD; END IF; " + " PERFORM cloudsync_delete('%s', %s); " + " RETURN OLD; " + "END; " + "$$ LANGUAGE plpgsql;", + func_name, esc_tbl_literal, esc_tbl_literal, pk_list); + cloudsync_memory_free(pk_list); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + if (rc != DBRES_OK) return rc; + + char *base_ref = database_build_base_ref(schema, table_name); + if (!base_ref) return DBRES_NOMEM; + + sql2 = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%s\" AFTER DELETE ON %s %s " + "EXECUTE FUNCTION \"%s\"();", + trigger_name, base_ref, trigger_when ? trigger_when : "", func_name); + cloudsync_memory_free(base_ref); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + return rc; +} + +// Build trigger WHEN clauses, optionally incorporating a row-level filter. +// INSERT/UPDATE use NEW-prefixed filter, DELETE uses OLD-prefixed filter. +static void database_build_trigger_when( + cloudsync_context *data, const char *table_name, const char *filter, + const char *schema, + char *when_new, size_t when_new_size, + char *when_old, size_t when_old_size) +{ + char *new_filter_str = NULL; + char *old_filter_str = NULL; + + if (filter) { + const char *schema_param = (schema && schema[0]) ? schema : ""; + char esc_tbl[1024], esc_schema[1024]; + sql_escape_literal(table_name, esc_tbl, sizeof(esc_tbl)); + sql_escape_literal(schema_param, esc_schema, sizeof(esc_schema)); + + char col_sql[2048]; + snprintf(col_sql, sizeof(col_sql), + "SELECT column_name::text FROM information_schema.columns " + "WHERE table_name = '%s' AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "ORDER BY ordinal_position;", + esc_tbl, esc_schema); + + char *col_names[256]; + int ncols = 0; + + dbvm_t *col_vm = NULL; + int crc = databasevm_prepare(data, col_sql, &col_vm, 0); + if (crc == DBRES_OK) { + while (databasevm_step(col_vm) == DBRES_ROW && ncols < 256) { + const char *name = database_column_text(col_vm, 0); + if (name) col_names[ncols++] = cloudsync_memory_mprintf("%s", name); + } + databasevm_finalize(col_vm); + } + + if (ncols > 0) { + new_filter_str = cloudsync_filter_add_row_prefix(filter, "NEW", col_names, ncols); + old_filter_str = cloudsync_filter_add_row_prefix(filter, "OLD", col_names, ncols); + for (int i = 0; i < ncols; ++i) cloudsync_memory_free(col_names[i]); + } + } + + char esc_tbl[512]; + sql_escape_literal(table_name, esc_tbl, sizeof(esc_tbl)); + + if (new_filter_str) { + snprintf(when_new, when_new_size, + "FOR EACH ROW WHEN (cloudsync_is_sync('%s') = false AND (%s))", + esc_tbl, new_filter_str); + } else { + snprintf(when_new, when_new_size, + "FOR EACH ROW WHEN (cloudsync_is_sync('%s') = false)", + esc_tbl); + } + + if (old_filter_str) { + snprintf(when_old, when_old_size, + "FOR EACH ROW WHEN (cloudsync_is_sync('%s') = false AND (%s))", + esc_tbl, old_filter_str); + } else { + snprintf(when_old, when_old_size, + "FOR EACH ROW WHEN (cloudsync_is_sync('%s') = false)", + esc_tbl); + } + + if (new_filter_str) cloudsync_memory_free(new_filter_str); + if (old_filter_str) cloudsync_memory_free(old_filter_str); +} + +int database_create_triggers (cloudsync_context *data, const char *table_name, table_algo algo, const char *filter) { + if (!table_name) return DBRES_MISUSE; + + // Detect schema from metadata table if it exists, otherwise use cloudsync_schema() + // This is called before table_add_to_context(), so we can't rely on table lookup. + char *detected_schema = database_table_schema(table_name); + const char *schema = detected_schema ? detected_schema : cloudsync_schema(data); + + char trigger_when_new[4096]; + char trigger_when_old[4096]; + database_build_trigger_when(data, table_name, filter, schema, + trigger_when_new, sizeof(trigger_when_new), + trigger_when_old, sizeof(trigger_when_old)); + + int rc = database_create_insert_trigger_internal(data, table_name, trigger_when_new, schema); + if (rc != DBRES_OK) { + if (detected_schema) cloudsync_memory_free(detected_schema); + return rc; + } + + if (algo == table_algo_crdt_gos) { + rc = database_create_update_trigger_gos_internal(data, table_name, schema); + } else { + rc = database_create_update_trigger_internal(data, table_name, trigger_when_new, schema); + } + if (rc != DBRES_OK) { + if (detected_schema) cloudsync_memory_free(detected_schema); + return rc; + } + + if (algo == table_algo_crdt_gos) { + rc = database_create_delete_trigger_gos_internal(data, table_name, schema); + } else { + rc = database_create_delete_trigger_internal(data, table_name, trigger_when_old, schema); + } + + if (detected_schema) cloudsync_memory_free(detected_schema); + return rc; +} + +int database_delete_triggers (cloudsync_context *data, const char *table) { + char *base_ref = database_build_base_ref(cloudsync_schema(data), table); + if (!base_ref) return DBRES_NOMEM; + + char escaped_tbl[512]; + sql_escape_identifier(table, escaped_tbl, sizeof(escaped_tbl)); + + char *sql = cloudsync_memory_mprintf( + "DROP TRIGGER IF EXISTS \"cloudsync_after_insert_%s\" ON %s;", + escaped_tbl, base_ref); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP FUNCTION IF EXISTS \"cloudsync_after_insert_%s_fn\"() CASCADE;", + escaped_tbl); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP TRIGGER IF EXISTS \"cloudsync_after_update_%s\" ON %s;", + escaped_tbl, base_ref); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP TRIGGER IF EXISTS \"cloudsync_before_update_%s\" ON %s;", + escaped_tbl, base_ref); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP FUNCTION IF EXISTS \"cloudsync_after_update_%s_fn\"() CASCADE;", + escaped_tbl); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP FUNCTION IF EXISTS \"cloudsync_before_update_%s_fn\"() CASCADE;", + escaped_tbl); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP TRIGGER IF EXISTS \"cloudsync_after_delete_%s\" ON %s;", + escaped_tbl, base_ref); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP TRIGGER IF EXISTS \"cloudsync_before_delete_%s\" ON %s;", + escaped_tbl, base_ref); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP FUNCTION IF EXISTS \"cloudsync_after_delete_%s_fn\"() CASCADE;", + escaped_tbl); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP FUNCTION IF EXISTS \"cloudsync_before_delete_%s_fn\"() CASCADE;", + escaped_tbl); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + cloudsync_memory_free(base_ref); + return DBRES_OK; +} + +// MARK: - SCHEMA VERSIONING - + +int64_t database_schema_version (cloudsync_context *data) { + int64_t value = 0; + int rc = database_select_int(data, SQL_SCHEMA_VERSION, &value); + return (rc == DBRES_OK) ? value : 0; +} + +uint64_t database_schema_hash (cloudsync_context *data) { + int64_t value = 0; + int rc = database_select_int(data, "SELECT hash FROM cloudsync_schema_versions ORDER BY seq DESC LIMIT 1;", &value); + return (rc == DBRES_OK) ? (uint64_t)value : 0; +} + +bool database_check_schema_hash (cloudsync_context *data, uint64_t hash) { + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT 1 FROM cloudsync_schema_versions WHERE hash = %" PRId64, (int64_t)hash); + + int64_t value = 0; + database_select_int(data, sql, &value); + return (value == 1); +} + +int database_update_schema_hash (cloudsync_context *data, uint64_t *hash) { + // Build normalized schema string using only: column name (lowercase), type (SQLite affinity), pk flag + // Format: tablename:colname:affinity:pk,... (ordered by table name, then column ordinal position) + // This makes the hash resilient to formatting, quoting, case differences and portable across databases + // + // PostgreSQL type to SQLite affinity mapping: + // - integer, smallint, bigint, boolean → 'integer' + // - bytea → 'blob' + // - real, double precision → 'real' + // - numeric, decimal → 'numeric' + // - Everything else → 'text' (default) + // This includes: text, varchar, char, uuid, timestamp, timestamptz, date, time, + // interval, json, jsonb, inet, cidr, macaddr, geometric types, xml, enums, + // and any custom/extension types. Using 'text' as default ensures compatibility + // since most types serialize to text representation and SQLite stores unknown + // types as TEXT affinity. + + char *schema = NULL; + int rc = database_select_text(data, + "SELECT string_agg(" + " LOWER(c.table_name) || ':' || LOWER(c.column_name) || ':' || " + " CASE " + // Integer types (including boolean as 0/1) + " WHEN c.data_type IN ('integer', 'smallint', 'bigint', 'boolean') THEN 'integer' " + // Blob type + " WHEN c.data_type = 'bytea' THEN 'blob' " + // Real/float types + " WHEN c.data_type IN ('real', 'double precision') THEN 'real' " + // Numeric types (explicit precision/scale) + " WHEN c.data_type IN ('numeric', 'decimal') THEN 'numeric' " + // Default to text for everything else: + // - String types: text, character varying, character, name, uuid + // - Date/time: timestamp, date, time, interval, etc. + // - JSON: json, jsonb + // - Network: inet, cidr, macaddr + // - Geometric: point, line, box, etc. + // - Custom/extension types + " ELSE 'text' " + " END || ':' || " + " CASE WHEN kcu.column_name IS NOT NULL THEN '1' ELSE '0' END, " + " ',' ORDER BY c.table_name, c.ordinal_position" + ") " + "FROM information_schema.columns c " + "JOIN cloudsync_table_settings cts ON LOWER(c.table_name) = LOWER(cts.tbl_name) " + "LEFT JOIN information_schema.table_constraints tc " + " ON tc.table_name = c.table_name " + " AND tc.table_schema = c.table_schema " + " AND tc.constraint_type = 'PRIMARY KEY' " + "LEFT JOIN information_schema.key_column_usage kcu " + " ON kcu.table_name = c.table_name " + " AND kcu.column_name = c.column_name " + " AND kcu.table_schema = c.table_schema " + " AND kcu.constraint_name = tc.constraint_name " + "WHERE c.table_schema = COALESCE(cloudsync_schema(), current_schema())", + &schema); + + if (rc != DBRES_OK || !schema) return cloudsync_set_error(data, "database_update_schema_hash error 1", DBRES_ERROR); + + size_t schema_len = strlen(schema); + DEBUG_MERGE("database_update_schema_hash len %zu schema %s", schema_len, schema); + uint64_t h = fnv1a_hash(schema, schema_len); + cloudsync_memory_free(schema); + if (hash && *hash == h) return cloudsync_set_error(data, "database_update_schema_hash constraint", DBRES_CONSTRAINT); + + char sql[1024]; + snprintf(sql, sizeof(sql), + "INSERT INTO cloudsync_schema_versions (hash, seq) " + "VALUES (%" PRId64 ", COALESCE((SELECT MAX(seq) FROM cloudsync_schema_versions), 0) + 1) " + "ON CONFLICT(hash) DO UPDATE SET " + "seq = (SELECT COALESCE(MAX(seq), 0) + 1 FROM cloudsync_schema_versions);", + (int64_t)h); + rc = database_exec(data, sql); + if (rc == DBRES_OK) { + if (hash) *hash = h; + return rc; + } + + return cloudsync_set_error(data, "database_update_schema_hash error 2", DBRES_ERROR); +} + +// MARK: - PRIMARY KEY - + +int database_pk_rowid (cloudsync_context *data, const char *table_name, char ***names, int *count) { + // PostgreSQL doesn't have rowid concept like SQLite + // Use OID or primary key columns instead + return database_pk_names(data, table_name, names, count); +} + +int database_pk_names (cloudsync_context *data, const char *table_name, char ***names, int *count) { + if (!table_name || !names || !count) return DBRES_MISUSE; + + const char *sql = + "SELECT kcu.column_name FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "WHERE tc.table_name = $1 AND tc.table_schema = COALESCE(cloudsync_schema(), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY' " + "ORDER BY kcu.ordinal_position"; + + Oid argtypes[1] = { TEXTOID }; + Datum values[1] = { CStringGetTextDatum(table_name) }; + char nulls[1] = { ' ' }; + + int rc = SPI_execute_with_args(sql, 1, argtypes, values, nulls, true, 0); + pfree(DatumGetPointer(values[0])); + + if (rc != SPI_OK_SELECT || SPI_processed == 0) { + *names = NULL; + *count = 0; + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return DBRES_OK; + } + + uint64_t n = SPI_processed; + char **pk_names = cloudsync_memory_zeroalloc(n * sizeof(char*)); + if (!pk_names) { + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return DBRES_NOMEM; + } + + for (uint64_t i = 0; i < n; i++) { + HeapTuple tuple = SPI_tuptable->vals[i]; + bool isnull; + SPI_getbinval(tuple, SPI_tuptable->tupdesc, 1, &isnull); + if (!isnull) { + // SPI_getvalue returns a palloc'd string regardless of column type + char *name = SPI_getvalue(tuple, SPI_tuptable->tupdesc, 1); + pk_names[i] = (name) ? cloudsync_string_dup(name) : NULL; + if (name) pfree(name); + } + + // Cleanup on allocation failure + if (!isnull && pk_names[i] == NULL) { + for (uint64_t j = 0; j < i; j++) { + if (pk_names[j]) cloudsync_memory_free(pk_names[j]); + } + cloudsync_memory_free(pk_names); + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return DBRES_NOMEM; + } + } + + *names = pk_names; + *count = (int)n; + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return DBRES_OK; +} + +// MARK: - VM - + +int databasevm_prepare (cloudsync_context *data, const char *sql, dbvm_t **vm, int flags) { + if (!sql || !vm) { + return cloudsync_set_error(data, "Invalid parameters to databasevm_prepare", DBRES_ERROR); + } + *vm = NULL; + cloudsync_reset_error(data); + + // sanity check number of parameters + // int counter = count_params(sql); + // if (counter > MAX_PARAMS) return cloudsync_set_error(data, "Maximum number of parameters reached", DBRES_MISUSE); + + // create PostgreSQL VM statement + pg_stmt_t *stmt = (pg_stmt_t *)cloudsync_memory_zeroalloc(sizeof(pg_stmt_t)); + if (!stmt) return cloudsync_set_error(data, "Not enough memory to allocate a dbvm_t struct", DBRES_NOMEM); + stmt->data = data; + + int rc = DBRES_OK; + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + MemoryContext parent = (flags & DBFLAG_PERSISTENT) ? TopMemoryContext : CurrentMemoryContext; + stmt->stmt_mcxt = AllocSetContextCreate(parent, "cloudsync stmt", ALLOCSET_DEFAULT_SIZES); + if (!stmt->stmt_mcxt) { + cloudsync_memory_free(stmt); + ereport(ERROR, (errmsg("Failed to create statement memory context"))); + } + stmt->bind_mcxt = AllocSetContextCreate(stmt->stmt_mcxt, "cloudsync binds", ALLOCSET_DEFAULT_SIZES); + stmt->row_mcxt = AllocSetContextCreate(stmt->stmt_mcxt, "cloudsync row", ALLOCSET_DEFAULT_SIZES); + + MemoryContext old = MemoryContextSwitchTo(stmt->stmt_mcxt); + stmt->sql = pstrdup(sql); + MemoryContextSwitchTo(old); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + if (stmt->stmt_mcxt) MemoryContextDelete(stmt->stmt_mcxt); + cloudsync_memory_free(stmt); + rc = DBRES_NOMEM; + stmt = NULL; + } + PG_END_TRY(); + + if (stmt) databasevm_clear_bindings((dbvm_t*)stmt); + *vm = (dbvm_t*)stmt; + + return rc; +} + +int databasevm_step0 (pg_stmt_t *stmt) { + cloudsync_context *data = stmt->data; + if (!data) return DBRES_ERROR; + + int rc = DBRES_OK; + MemoryContext oldcontext = CurrentMemoryContext; + + PG_TRY(); + { + if (!stmt->sql) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("databasevm_step0 invalid sql pointer"))); + } + + stmt->plan = SPI_prepare(stmt->sql, stmt->nparams, stmt->types); + if (stmt->plan == NULL) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Unable to prepare SQL statement"))); + } + + SPI_keepplan(stmt->plan); + stmt->plan_is_prepared = true; + + // Save the types used for this plan so we can detect type changes + memcpy(stmt->prepared_types, stmt->types, sizeof(Oid) * stmt->nparams); + stmt->prepared_nparams = stmt->nparams; + } + PG_CATCH(); + { + // Switch to safe context for CopyErrorData (can't be ErrorContext) + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + + // Clean up partially prepared plan if needed + if (stmt->plan != NULL && !stmt->plan_is_prepared) { + PG_TRY(); + { + SPI_freeplan(stmt->plan); + } + PG_CATCH(); + { + FlushErrorState(); // Swallow errors during cleanup + } + PG_END_TRY(); + stmt->plan = NULL; + } + } + PG_END_TRY(); + + return rc; +} + +int databasevm_step (dbvm_t *vm) { + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt) return DBRES_MISUSE; + + cloudsync_context *data = stmt->data; + cloudsync_reset_error(data); + + // If plan is prepared but parameter types have changed since preparation, + // free the old plan and re-prepare with new types. This happens when the same + // prepared statement is reused with different PK encodings (e.g., integer vs text). + if (stmt->plan_is_prepared && stmt->plan) { + bool types_changed = (stmt->nparams != stmt->prepared_nparams); + if (!types_changed) { + for (int i = 0; i < stmt->nparams; i++) { + if (stmt->types[i] != stmt->prepared_types[i]) { + types_changed = true; + break; + } + } + } + if (types_changed) { + SPI_freeplan(stmt->plan); + stmt->plan = NULL; + stmt->plan_is_prepared = false; + } + } + + if (!stmt->plan_is_prepared) { + int rc = databasevm_step0(stmt); + if (rc != DBRES_OK) return rc; + } + if (!stmt->plan_is_prepared || !stmt->plan) return DBRES_ERROR; + + int rc = DBRES_DONE; + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + do { + // if portal is open, we fetch one row + if (stmt->portal_open) { + // free prior fetched row batch + clear_fetch_batch(stmt); + + SPI_cursor_fetch(stmt->portal, true, 1); + + if (SPI_processed == 0) { + clear_fetch_batch(stmt); + close_portal(stmt); + rc = DBRES_DONE; + break; + } + + // null check for SPI_tuptable + if (!SPI_tuptable || !SPI_tuptable->tupdesc || !SPI_tuptable->vals) { + clear_fetch_batch(stmt); + close_portal(stmt); + rc = cloudsync_set_error(data, "SPI_cursor_fetch returned invalid tuptable", DBRES_ERROR); + break; + } + + MemoryContextReset(stmt->row_mcxt); + + stmt->last_tuptable = SPI_tuptable; + stmt->current_tupdesc = stmt->last_tuptable->tupdesc; + stmt->current_tuple = stmt->last_tuptable->vals[0]; + rc = DBRES_ROW; + break; + } + + // First step: decide whether to use portal. + // Even for INSERT/UPDATE/DELETE ... RETURNING you WANT a portal. + // Strategy: + // - Only open a cursor if the plan supports it (avoid "cannot open INSERT query as cursor"). + // - Otherwise execute once as a non-row-returning statement. + if (!stmt->executed_nonselect) { + if (SPI_is_cursor_plan(stmt->plan)) { + // try cursor open + stmt->portal = NULL; + if (stmt->nparams == 0) stmt->portal = SPI_cursor_open(NULL, stmt->plan, NULL, NULL, false); + else stmt->portal = SPI_cursor_open(NULL, stmt->plan, stmt->values, stmt->nulls, false); + + if (stmt->portal != NULL) { + // Don't set portal_open until we successfully fetch first row + + // fetch first row + clear_fetch_batch(stmt); + SPI_cursor_fetch(stmt->portal, true, 1); + + if (SPI_processed == 0) { + // No rows - close portal, don't set portal_open + clear_fetch_batch(stmt); + close_portal(stmt); + rc = DBRES_DONE; + break; + } + + // null check for SPI_tuptable + if (!SPI_tuptable || !SPI_tuptable->tupdesc || !SPI_tuptable->vals) { + clear_fetch_batch(stmt); + close_portal(stmt); + rc = cloudsync_set_error(data, "SPI_cursor_fetch returned invalid tuptable", DBRES_ERROR); + break; + } + + MemoryContextReset(stmt->row_mcxt); + + stmt->last_tuptable = SPI_tuptable; + stmt->current_tupdesc = stmt->last_tuptable->tupdesc; + stmt->current_tuple = stmt->last_tuptable->vals[0]; + + // Only set portal_open AFTER everything succeeded + stmt->portal_open = true; + + rc = DBRES_ROW; + break; + } + } + + // Execute once (non-row-returning or cursor open failed). + int spi_rc; + if (stmt->nparams == 0) spi_rc = SPI_execute_plan(stmt->plan, NULL, NULL, false, 0); + else spi_rc = SPI_execute_plan(stmt->plan, stmt->values, stmt->nulls, false, 0); + if (spi_rc < 0) { + rc = cloudsync_set_error(data, "SPI_execute_plan failed", DBRES_ERROR); + break; + } + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + + stmt->executed_nonselect = true; + rc = DBRES_DONE; + break; + } + + rc = DBRES_DONE; + } while (0); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + int err = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + + // free resources + clear_fetch_batch(stmt); + close_portal(stmt); + + rc = err; + } + PG_END_TRY(); + return rc; +} + +void databasevm_finalize (dbvm_t *vm) { + if (!vm) return; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + + PG_TRY(); + { + clear_fetch_batch(stmt); + close_portal(stmt); + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + + if (stmt->plan_is_prepared && stmt->plan) { + SPI_freeplan(stmt->plan); + stmt->plan = NULL; + stmt->plan_is_prepared = false; + } + } + PG_CATCH(); + { + /* don't throw from finalize; just swallow */ + FlushErrorState(); + } + PG_END_TRY(); + + if (stmt->stmt_mcxt) MemoryContextDelete(stmt->stmt_mcxt); + cloudsync_memory_free(stmt); +} + +void databasevm_reset (dbvm_t *vm) { + if (!vm) return; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + + // Close any open cursor and clear fetched data + clear_fetch_batch(stmt); + close_portal(stmt); + + // Clear global SPI tuple table if any + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + + // Reset execution state + stmt->executed_nonselect = false; + + // Reset parameter values but keep the plan, types, and nparams intact. + // The prepared plan can be reused with new values of the same types, + // avoiding the cost of re-planning on every iteration. + if (stmt->bind_mcxt) MemoryContextReset(stmt->bind_mcxt); + for (int i = 0; i < stmt->nparams; i++) { + stmt->values[i] = (Datum) 0; + stmt->nulls[i] = 'n'; + } +} + +void databasevm_clear_bindings (dbvm_t *vm) { + if (!vm) return; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + + // Only clear the bound parameter values. + // Do NOT close portals, free fetch batches, or free the plan — + // those are execution state, not bindings. + if (stmt->bind_mcxt) MemoryContextReset(stmt->bind_mcxt); + stmt->nparams = 0; + + // Reset params array to defaults + for (int i = 0; i < MAX_PARAMS; i++) { + stmt->types[i] = UNKNOWNOID; + stmt->values[i] = (Datum) 0; + stmt->nulls[i] = 'n'; // default NULL + } +} + +const char *databasevm_sql (dbvm_t *vm) { + if (!vm) return NULL; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + return (char *)stmt->sql; +} + +// MARK: - BINDING - + +static int databasevm_bind_null_type (dbvm_t *vm, int index, Oid t) { + int rc = databasevm_bind_null(vm, index); + if (rc != DBRES_OK) return rc; + int idx = index - 1; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + stmt->types[idx] = t; + return rc; +} + +int databasevm_bind_blob (dbvm_t *vm, int index, const void *value, uint64_t size) { + if (!vm || index < 1) return DBRES_ERROR; + if (!value) return databasevm_bind_null_type(vm, index, BYTEAOID); + + // validate size fits Size and won't overflow + if (size > (uint64) (MaxAllocSize - VARHDRSZ)) return DBRES_NOMEM; + + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + MemoryContext old = MemoryContextSwitchTo(stmt->bind_mcxt); + + // Convert binary data to PostgreSQL bytea + bytea *ba = (bytea*)palloc(size + VARHDRSZ); + SET_VARSIZE(ba, size + VARHDRSZ); + memcpy(VARDATA(ba), value, size); + + stmt->values[idx] = PointerGetDatum(ba); + stmt->types[idx] = BYTEAOID; + stmt->nulls[idx] = ' '; + + MemoryContextSwitchTo(old); + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +int databasevm_bind_double (dbvm_t *vm, int index, double value) { + if (!vm || index < 1) return DBRES_ERROR; + + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + stmt->values[idx] = Float8GetDatum(value); + stmt->types[idx] = FLOAT8OID; + stmt->nulls[idx] = ' '; + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +int databasevm_bind_int (dbvm_t *vm, int index, int64_t value) { + if (!vm || index < 1) return DBRES_ERROR; + + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + stmt->values[idx] = Int64GetDatum(value); + stmt->types[idx] = INT8OID; + stmt->nulls[idx] = ' '; + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +int databasevm_bind_null (dbvm_t *vm, int index) { + if (!vm || index < 1) return DBRES_ERROR; + + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + stmt->values[idx] = (Datum)0; + stmt->types[idx] = TEXTOID; // TEXTOID has casts to most types + stmt->nulls[idx] = 'n'; + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +int databasevm_bind_text (dbvm_t *vm, int index, const char *value, int size) { + if (!vm || index < 1) return DBRES_ERROR; + if (!value) return databasevm_bind_null_type(vm, index, TEXTOID); + + // validate size fits Size and won't overflow + if (size < 0) size = (int)strlen(value); + if ((Size)size > MaxAllocSize - VARHDRSZ) return DBRES_NOMEM; + + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + MemoryContext old = MemoryContextSwitchTo(stmt->bind_mcxt); + + text *t = cstring_to_text_with_len(value, size); + stmt->values[idx] = PointerGetDatum(t); + stmt->types[idx] = TEXTOID; + stmt->nulls[idx] = ' '; + + MemoryContextSwitchTo(old); + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +int databasevm_bind_value (dbvm_t *vm, int index, dbvalue_t *value) { + if (!vm) return DBRES_ERROR; + if (!value) return databasevm_bind_null_type(vm, index, TEXTOID); + + // validate index bounds properly (1-based index) + if (index < 1) return DBRES_ERROR; + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) { + stmt->values[idx] = (Datum)0; + // Use the actual column type if available, otherwise default to TEXTOID + stmt->types[idx] = (v && OidIsValid(v->typeid)) ? v->typeid : TEXTOID; + stmt->nulls[idx] = 'n'; + } else { + int16 typlen; + bool typbyval; + + get_typlenbyval(v->typeid, &typlen, &typbyval); + MemoryContext old = MemoryContextSwitchTo(stmt->bind_mcxt); + + Datum dcopy; + if (typbyval) { + // Pass-by-value: direct copy is safe + dcopy = v->datum; + } else { + // Pass-by-reference: need to copy the actual data + // Handle variable-length types (typlen == -1) and cstrings (typlen == -2) + if (typlen == -1) { + // Variable-length type (varlena): use VARSIZE_ANY to handle + // both short (1-byte) and regular (4-byte) varlena headers + Size len = VARSIZE_ANY(DatumGetPointer(v->datum)); + dcopy = PointerGetDatum(palloc(len)); + memcpy(DatumGetPointer(dcopy), DatumGetPointer(v->datum), len); + } else if (typlen == -2) { + // Null-terminated cstring + dcopy = CStringGetDatum(pstrdup(DatumGetCString(v->datum))); + } else { + // Fixed-length pass-by-reference + dcopy = datumCopy(v->datum, false, typlen); + } + } + + stmt->values[idx] = dcopy; + MemoryContextSwitchTo(old); + stmt->types[idx] = OidIsValid(v->typeid) ? v->typeid : TEXTOID; + stmt->nulls[idx] = ' '; + } + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +// MARK: - COLUMN - + +Datum database_column_datum (dbvm_t *vm, int index) { + if (!vm) return (Datum)0; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return (Datum)0; + if (index < 0 || index >= stmt->current_tupdesc->natts) return (Datum)0; + + bool isnull = true; + Datum d = get_datum(stmt, index, &isnull, NULL); + return (isnull) ? (Datum)0 : d; +} + +const void *database_column_blob (dbvm_t *vm, int index, size_t *len) { + if (!vm) return NULL; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return NULL; + if (index < 0 || index >= stmt->current_tupdesc->natts) return NULL; + + bool isnull = true; + Datum d = get_datum(stmt, index, &isnull, NULL); + if (isnull) return NULL; + + MemoryContext old = MemoryContextSwitchTo(stmt->row_mcxt); + bytea *ba = DatumGetByteaP(d); + + // Validate VARSIZE before computing length + Size varsize = VARSIZE(ba); + if (varsize < VARHDRSZ) { + // Corrupt or invalid bytea - VARSIZE should always be >= VARHDRSZ + MemoryContextSwitchTo(old); + elog(WARNING, "database_column_blob: invalid bytea VARSIZE %zu", varsize); + return NULL; + } + + Size blen = VARSIZE(ba) - VARHDRSZ; + void *out = palloc(blen); + if (!out) { + MemoryContextSwitchTo(old); + return NULL; + } + + memcpy(out, VARDATA(ba), (size_t)blen); + MemoryContextSwitchTo(old); + + if (len) *len = (size_t)blen; + return out; +} + +double database_column_double (dbvm_t *vm, int index) { + if (!vm) return 0.0; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return 0.0; + if (index < 0 || index >= stmt->current_tupdesc->natts) return 0.0; + + bool isnull = true; + Oid type = 0; + Datum d = get_datum(stmt, index, &isnull, &type); + if (isnull) return 0.0; + + switch (type) { + case FLOAT4OID: return (double)DatumGetFloat4(d); + case FLOAT8OID: return (double)DatumGetFloat8(d); + case NUMERICOID: return DatumGetFloat8(DirectFunctionCall1(numeric_float8_no_overflow, d)); + case INT2OID: return (double)DatumGetInt16(d); + case INT4OID: return (double)DatumGetInt32(d); + case INT8OID: return (double)DatumGetInt64(d); + case BOOLOID: return (double)DatumGetBool(d); + } + + return 0.0; +} + +int64_t database_column_int (dbvm_t *vm, int index) { + if (!vm) return 0; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return 0; + if (index < 0 || index >= stmt->current_tupdesc->natts) return 0; + + bool isnull = true; + Oid type = 0; + Datum d = get_datum(stmt, index, &isnull, &type); + if (isnull) return 0; + + switch (type) { + case FLOAT4OID: return (int64_t)DatumGetFloat4(d); + case FLOAT8OID: return (int64_t)DatumGetFloat8(d); + case INT2OID: return (int64_t)DatumGetInt16(d); + case INT4OID: return (int64_t)DatumGetInt32(d); + case INT8OID: return (int64_t)DatumGetInt64(d); + case BOOLOID: return (int64_t)DatumGetBool(d); + } + + return 0; +} + +const char *database_column_text (dbvm_t *vm, int index) { + if (!vm) return NULL; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return NULL; + if (index < 0 || index >= stmt->current_tupdesc->natts) return NULL; + + bool isnull = true; + Oid type = 0; + Datum d = get_datum(stmt, index, &isnull, &type); + if (isnull) return NULL; + + MemoryContext old = MemoryContextSwitchTo(stmt->row_mcxt); + char *out = NULL; + + if (type == BYTEAOID) { + bytea *b = DatumGetByteaP(d); + int len = VARSIZE(b) - VARHDRSZ; + out = palloc(len + 1); + memcpy(out, VARDATA(b), len); + out[len] = 0; + } else if (type == TEXTOID || type == VARCHAROID || type == BPCHAROID) { + text *t = DatumGetTextP(d); + int len = VARSIZE(t) - VARHDRSZ; + out = palloc(len + 1); + memcpy(out, VARDATA(t), len); + out[len] = 0; + } else { + MemoryContextSwitchTo(old); + return NULL; + } + + MemoryContextSwitchTo(old); + + return out; +} + +dbvalue_t *database_column_value (dbvm_t *vm, int index) { + if (!vm) return NULL; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return NULL; + if (index < 0 || index >= stmt->current_tupdesc->natts) return NULL; + + bool isnull = true; + Oid type = 0; + Datum d = get_datum(stmt, index, &isnull, &type); + int32 typmod = TupleDescAttr(stmt->current_tupdesc, index)->atttypmod; + Oid collation = TupleDescAttr(stmt->current_tupdesc, index)->attcollation; + + pgvalue_t *v = pgvalue_create(d, type, typmod, collation, isnull); + if (v) pgvalue_ensure_detoast(v); + return (dbvalue_t*)v; +} + +int database_column_bytes (dbvm_t *vm, int index) { + if (!vm) return 0; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return 0; + if (index < 0 || index >= stmt->current_tupdesc->natts) return 0; + + bool isnull = true; + Oid type = 0; + Datum d = get_datum(stmt, index, &isnull, &type); + if (isnull) return 0; + + MemoryContext old = MemoryContextSwitchTo(stmt->row_mcxt); + + int bytes = 0; + if (type == BYTEAOID) { + // BLOB case + bytea *ba = DatumGetByteaP(d); + bytes = (int)(VARSIZE(ba) - VARHDRSZ); + } else if (type != TEXTOID && type != VARCHAROID && type != BPCHAROID) { + // any non-TEXT case should be discarded + bytes = 0; + } else { + // for text, return string length + text *txt = DatumGetTextP(d); + bytes = (int)(VARSIZE(txt) - VARHDRSZ); + } + MemoryContextSwitchTo(old); + + return bytes; +} + +int database_column_type (dbvm_t *vm, int index) { + if (!vm) return DBTYPE_NULL; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return DBTYPE_NULL; + if (index < 0 || index >= stmt->current_tupdesc->natts) return DBTYPE_NULL; + + bool isnull = true; + Oid type = 0; + get_datum(stmt, index, &isnull, &type); + if (isnull) return DBTYPE_NULL; + + switch (type) { + case INT2OID: + case INT4OID: + case INT8OID: + return DBTYPE_INTEGER; + + case FLOAT4OID: + case FLOAT8OID: + case NUMERICOID: + return DBTYPE_FLOAT; + + case TEXTOID: + case VARCHAROID: + case BPCHAROID: + return DBTYPE_TEXT; + + case BYTEAOID: + return DBTYPE_BLOB; + } + + return DBTYPE_TEXT; +} + +// MARK: - VALUE - + +const void *database_value_blob (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) return NULL; + + // Text types reuse blob accessor (pk encode reads text bytes directly). + // Exclude JSONB: its internal format is binary, not text — + // it must go through OidOutputFunctionCall to get the JSON text. + if (pgvalue_is_text_type(v->typeid) && v->typeid != JSONBOID) { + pgvalue_ensure_detoast(v); + text *txt = (text *)DatumGetPointer(v->datum); + return VARDATA_ANY(txt); + } + + if (v->typeid == BYTEAOID) { + pgvalue_ensure_detoast(v); + bytea *ba = (bytea *)DatumGetPointer(v->datum); + return VARDATA_ANY(ba); + } + + // For unmapped types and JSONB (mapped to DBTYPE_TEXT), + // convert to text representation via the type's output function + const char *cstr = database_value_text(value); + if (cstr) return cstr; + + return NULL; +} + +double database_value_double (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) return 0.0; + + switch (v->typeid) { + case FLOAT4OID: + return (double)DatumGetFloat4(v->datum); + case FLOAT8OID: + return DatumGetFloat8(v->datum); + case NUMERICOID: + return DatumGetFloat8(DirectFunctionCall1(numeric_float8_no_overflow, v->datum)); + case INT2OID: + return (double)DatumGetInt16(v->datum); + case INT4OID: + return (double)DatumGetInt32(v->datum); + case INT8OID: + return (double)DatumGetInt64(v->datum); + case BOOLOID: + return DatumGetBool(v->datum) ? 1.0 : 0.0; + default: + return 0.0; + } +} + +int64_t database_value_int (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) return 0; + + switch (v->typeid) { + case INT2OID: + return (int64_t)DatumGetInt16(v->datum); + case INT4OID: + return (int64_t)DatumGetInt32(v->datum); + case INT8OID: + return DatumGetInt64(v->datum); + case BOOLOID: + return DatumGetBool(v->datum) ? 1 : 0; + default: + return 0; + } +} + +const char *database_value_text (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) return NULL; + + if (!v->cstring && !v->owns_cstring) { + PG_TRY(); + { + if (pgvalue_is_text_type(v->typeid) && v->typeid != JSONBOID) { + pgvalue_ensure_detoast(v); + v->cstring = text_to_cstring((text *)DatumGetPointer(v->datum)); + } else { + // Type output function for JSONB and other non-text types + Oid outfunc; + bool isvarlena; + getTypeOutputInfo(v->typeid, &outfunc, &isvarlena); + v->cstring = OidOutputFunctionCall(outfunc, v->datum); + } + v->owns_cstring = true; + } + PG_CATCH(); + { + MemoryContextSwitchTo(CurrentMemoryContext); + ErrorData *edata = CopyErrorData(); + elog(WARNING, "database_value_text: conversion failed for type %u: %s", v->typeid, edata->message); + FreeErrorData(edata); + FlushErrorState(); + v->cstring = NULL; + v->owns_cstring = true; // prevents retry of failed conversion + } + PG_END_TRY(); + } + + return v->cstring; +} + +int database_value_bytes (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) return 0; + + // Exclude JSONB: binary internal format, must use OidOutputFunctionCall + if (pgvalue_is_text_type(v->typeid) && v->typeid != JSONBOID) { + pgvalue_ensure_detoast(v); + text *txt = (text *)DatumGetPointer(v->datum); + return VARSIZE_ANY_EXHDR(txt); + } + if (v->typeid == BYTEAOID) { + pgvalue_ensure_detoast(v); + bytea *ba = (bytea *)DatumGetPointer(v->datum); + return VARSIZE_ANY_EXHDR(ba); + } + // For unmapped types and JSONB (mapped to DBTYPE_TEXT), + // ensure the text representation is materialized + database_value_text(value); + if (v->cstring) { + return (int)strlen(v->cstring); + } + return 0; +} + +int database_value_type (dbvalue_t *value) { + return pgvalue_dbtype((pgvalue_t *)value); +} + +void database_value_free (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + pgvalue_free(v); +} + +void *database_value_dup (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v) return NULL; + + pgvalue_t *copy = pgvalue_create(v->datum, v->typeid, v->typmod, v->collation, v->isnull); + + // Deep-copy pass-by-reference (varlena) datum data into TopMemoryContext + // so the copy survives SPI_finish() which destroys the caller's SPI context. + bool is_varlena = (v->typeid == BYTEAOID) || pgvalue_is_text_type(v->typeid); + if (is_varlena && !v->isnull) { + void *src = v->owned_detoast ? v->owned_detoast : DatumGetPointer(v->datum); + Size len = VARSIZE_ANY(src); + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + copy->owned_detoast = palloc(len); + MemoryContextSwitchTo(old); + memcpy(copy->owned_detoast, src, len); + copy->datum = PointerGetDatum(copy->owned_detoast); + copy->detoasted = true; + } + if (v->cstring) { + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + copy->cstring = pstrdup(v->cstring); + MemoryContextSwitchTo(old); + copy->owns_cstring = true; + } + return (void*)copy; +} + +// MARK: - SAVEPOINTS - + +static int database_refresh_snapshot (void) { + // Only manipulate snapshots in a valid transaction + if (!IsTransactionState()) { + return DBRES_OK; // Not in transaction, nothing to do + } + + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + CommandCounterIncrement(); + + // Pop existing snapshot if any + if (ActiveSnapshotSet()) { + PopActiveSnapshot(); + } + + // Push fresh snapshot + PushActiveSnapshot(GetTransactionSnapshot()); + } + PG_CATCH(); + { + // Snapshot refresh failed - log warning but don't fail operation + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + elog(WARNING, "refresh_snapshot_after_command failed: %s", edata->message); + FreeErrorData(edata); + FlushErrorState(); + return DBRES_ERROR; + } + PG_END_TRY(); + + return DBRES_OK; +} + +int database_begin_savepoint (cloudsync_context *data, const char *savepoint_name) { + cloudsync_reset_error(data); + int rc = DBRES_OK; + + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + BeginInternalSubTransaction(NULL); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + } + PG_END_TRY(); + + return rc; +} + +int database_commit_savepoint (cloudsync_context *data, const char *savepoint_name) { + cloudsync_reset_error(data); + if (GetCurrentTransactionNestLevel() <= 1) return DBRES_OK; + int rc = DBRES_OK; + + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + ReleaseCurrentSubTransaction(); + database_refresh_snapshot(); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + rc = DBRES_ERROR; + } + PG_END_TRY(); + + return rc; +} + +int database_rollback_savepoint (cloudsync_context *data, const char *savepoint_name) { + cloudsync_reset_error(data); + if (GetCurrentTransactionNestLevel() <= 1) return DBRES_OK; + int rc = DBRES_OK; + + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + RollbackAndReleaseCurrentSubTransaction(); + database_refresh_snapshot(); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + rc = DBRES_ERROR; + } + PG_END_TRY(); + + return rc; +} + +// MARK: - MEMORY - +// Use palloc in TopMemoryContext for PostgreSQL memory management integration. +// This provides memory tracking, debugging support, and proper cleanup on connection end. + +void *dbmem_alloc (uint64_t size) { + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + void *ptr = palloc(size); + MemoryContextSwitchTo(old); + return ptr; +} + +void *dbmem_zeroalloc (uint64_t size) { + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + void *ptr = palloc0(size); + MemoryContextSwitchTo(old); + return ptr; +} + +void *dbmem_realloc (void *ptr, uint64_t new_size) { + // repalloc doesn't accept NULL, unlike realloc + if (!ptr) return dbmem_alloc(new_size); + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + void *newptr = repalloc(ptr, new_size); + MemoryContextSwitchTo(old); + return newptr; +} + +char *dbmem_mprintf (const char *format, ...) { + if (!format) return NULL; + + va_list args; + va_start(args, format); + + // Calculate required buffer size + va_list args_copy; + va_copy(args_copy, args); + int len = vsnprintf(NULL, 0, format, args_copy); + va_end(args_copy); + + if (len < 0) { + va_end(args); + return NULL; + } + + // Allocate buffer in TopMemoryContext and format string + char *result = dbmem_alloc(len + 1); + if (!result) {va_end(args); return NULL;} + vsnprintf(result, len + 1, format, args); + + va_end(args); + return result; +} + +char *dbmem_vmprintf (const char *format, va_list list) { + if (!format) return NULL; + + // Calculate required buffer size + va_list args_copy; + va_copy(args_copy, list); + int len = vsnprintf(NULL, 0, format, args_copy); + va_end(args_copy); + + if (len < 0) return NULL; + + // Allocate buffer in TopMemoryContext and format string + char *result = dbmem_alloc(len + 1); + if (!result) return NULL; + vsnprintf(result, len + 1, format, list); + + return result; +} + +void dbmem_free (void *ptr) { + if (ptr) { + pfree(ptr); + } +} + +uint64_t dbmem_size (void *ptr) { + // palloc doesn't expose allocated size directly + // Return 0 as a safe default + return 0; +} + + diff --git a/src/postgresql/migrations/README.md b/src/postgresql/migrations/README.md new file mode 100644 index 00000000..497bf779 --- /dev/null +++ b/src/postgresql/migrations/README.md @@ -0,0 +1,74 @@ +# CloudSync PostgreSQL Migration Scripts + +This directory holds PostgreSQL extension upgrade scripts of the form: + + cloudsync----.sql + +PostgreSQL uses these to execute `ALTER EXTENSION cloudsync UPDATE` by chaining +one or more files to reach the target version. + +## Versioning model + +The PostgreSQL extension version (`default_version` in `cloudsync.control`) is +**`MAJOR.MINOR`** only — derived from the first two components of +`CLOUDSYNC_VERSION` in `src/cloudsync.h`. The full semver of the compiled +binary is reported by the `cloudsync_version()` SQL function. + +| Release kind | Example | EXTVERSION moves? | Upgrade script? | User action | +| -------------------------------- | --------------- | ----------------- | --------------- | ------------------ | +| PATCH bump (binary only) | 1.0.16 → 1.0.17 | No (stays `1.0`) | Not required | Swap the `.so`. | +| MINOR bump (SQL surface changes) | 1.0.x → 1.1.0 | Yes (`1.0` → `1.1`) | Required | `ALTER EXTENSION cloudsync UPDATE;` | +| MAJOR bump | 1.x → 2.0.0 | Yes (`1.x` → `2.0`) | Required | `ALTER EXTENSION cloudsync UPDATE;` | + +CI enforces this contract via `scripts/check-postgres-migration.sh`: + +- PATCH releases: the script diffs the current `cloudsync.sql.in` against the + previous release's install script. If they differ, the build fails — + accidental SQL surface drift in a PATCH release would silently break users + whose `pg_extension.extversion` would otherwise stay at the old EXTVERSION. +- MINOR/MAJOR releases: the script requires a matching + `cloudsync----.sql` in this directory. + +## When to add an upgrade script + +Add one **only when `EXTVERSION` changes** — i.e. when you bump MINOR or MAJOR +in `src/cloudsync.h`. The filename is literally +`cloudsync----.sql`, not the full semver. + +Examples: + + 1.0.17 -> 1.1.0 → cloudsync--1.0--1.1.sql + 1.1.5 -> 2.0.0 → cloudsync--1.1--2.0.sql + +PATCH-level releases (1.0.16 → 1.0.17 → 1.0.18 → …) require **no file** — the +catalog's `installed_version` stays at the MAJOR.MINOR, and the `.so` swap is +a transparent binary upgrade. + +## Rules for upgrade script content + +- Use `CREATE OR REPLACE` for every function — the underlying C symbol may + have changed even when the SQL signature didn't. +- Drop removed objects explicitly (`DROP FUNCTION IF EXISTS ...`). +- Never run `CREATE EXTENSION`-style bootstrap inside an upgrade script. +- Objects created inside an upgrade script are automatically attached to the + extension via `pg_depend`. +- Scripts are packaged and installed by `make postgres-install` / + `make postgres-package` via a wildcard; no need to list them anywhere. + +## Verifying the chain + +After rebuilding, inside the PG container: + + SELECT * FROM pg_extension_update_paths('cloudsync'); + +All `source -> target` rows should show a non-NULL `path`. + +## I intended a PATCH but the CI check says "SQL surface drift" + +Either: + +1. You meant to change SQL. Bump MINOR in `src/cloudsync.h` and add + `cloudsync----.sql`. +2. You didn't. Revert the change in `src/postgresql/cloudsync.sql.in`. + +The CI error message prints a unified diff to help you decide which. diff --git a/src/postgresql/migrations/cloudsync--1.0--1.1.sql b/src/postgresql/migrations/cloudsync--1.0--1.1.sql new file mode 100644 index 00000000..1acb4810 --- /dev/null +++ b/src/postgresql/migrations/cloudsync--1.0--1.1.sql @@ -0,0 +1,81 @@ +-- CloudSync PostgreSQL extension upgrade: 1.0 -> 1.1 +-- +-- Adds the chunked-payload SQL surface introduced in 1.1: +-- * cloudsync_payload_chunks() set-returning function (with the +-- exclude_filter_site_id flag used by the /check download path) +-- * cloudsync_payload_blob_checked() scalar helper for guarded legacy +-- monolithic payload materialization +-- * cloudsync_uuid_text() / cloudsync_uuid_blob() UUID conversion helpers +-- * cloudsync_payload_fragments spool table for oversized v3 fragment +-- values +-- +-- Run automatically by: ALTER EXTENSION cloudsync UPDATE; + +-- Spool table for oversized v3 fragment values. Created here (as the +-- extension owner) because on an already-configured database the first role +-- to touch it may be a sync-only role without CREATE privileges; fresh +-- setups get it from dbutils_settings_init like the other cloudsync tables. +-- Detached from the extension right after creation so DROP EXTENSION and +-- pg_dump treat it exactly like those runtime-created tables. The DO block +-- skips both steps when the table already exists (detach would fail on a +-- non-member table). +DO $$ +BEGIN + IF to_regclass('cloudsync_payload_fragments') IS NULL THEN + CREATE TABLE cloudsync_payload_fragments ( + value_id TEXT NOT NULL, part_index BIGINT NOT NULL, part_count BIGINT NOT NULL, total_size BIGINT NOT NULL, + checksum TEXT NOT NULL, created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())::bigint), + tbl TEXT NOT NULL, pk BYTEA NOT NULL, col_name TEXT NOT NULL, col_version BIGINT NOT NULL, db_version BIGINT NOT NULL, + site_id BYTEA NOT NULL, cl BIGINT NOT NULL, seq BIGINT NOT NULL, fragment BYTEA NOT NULL, + PRIMARY KEY(value_id, part_index) + ); + ALTER EXTENSION cloudsync DROP TABLE cloudsync_payload_fragments; + END IF; +END $$; + +CREATE OR REPLACE FUNCTION cloudsync_payload_chunks( + since_db_version bigint DEFAULT NULL, + filter_site_id bytea DEFAULT NULL, + until_db_version bigint DEFAULT NULL, + exclude_filter_site_id boolean DEFAULT false, + resume_db_version bigint DEFAULT NULL, + resume_seq bigint DEFAULT NULL, + resume_frag_offset bigint DEFAULT NULL +) +RETURNS TABLE ( + payload bytea, + chunk_index bigint, + payload_size bigint, + rows bigint, + db_version_min bigint, + db_version_max bigint, + watermark_db_version bigint, + next_db_version bigint, + next_seq bigint, + next_frag_offset bigint, + is_final boolean +) +AS 'MODULE_PATHNAME', 'cloudsync_payload_chunks' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_payload_blob_checked( + since_db_version bigint, + since_seq bigint, + filter_site_id bytea, + exclude_filter_site_id boolean, + max_estimated_payload_size bigint +) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_payload_blob_checked' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_uuid_text(uuid bytea, dash_format boolean DEFAULT true) +RETURNS text +AS 'MODULE_PATHNAME', 'cloudsync_uuid_text' +LANGUAGE C IMMUTABLE; + +CREATE OR REPLACE FUNCTION cloudsync_uuid_blob(uuid text) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_uuid_blob' +LANGUAGE C IMMUTABLE; + diff --git a/src/postgresql/pgvalue.c b/src/postgresql/pgvalue.c new file mode 100644 index 00000000..69fd626c --- /dev/null +++ b/src/postgresql/pgvalue.c @@ -0,0 +1,198 @@ +// +// pgvalue.c +// PostgreSQL-specific dbvalue_t helpers +// + +#include "pgvalue.h" + +#include "catalog/pg_type.h" +#include "utils/lsyscache.h" +#include "utils/builtins.h" +#include "../utils.h" + +bool pgvalue_is_text_type(Oid typeid) { + switch (typeid) { + case TEXTOID: + case VARCHAROID: + case BPCHAROID: + case NAMEOID: + case JSONOID: + case JSONBOID: + case XMLOID: + return true; + default: + return false; + } +} + +static bool pgvalue_is_varlena(Oid typeid) { + return (typeid == BYTEAOID) || pgvalue_is_text_type(typeid); +} + +pgvalue_t *pgvalue_create(Datum datum, Oid typeid, int32 typmod, Oid collation, bool isnull) { + pgvalue_t *v = cloudsync_memory_zeroalloc(sizeof(pgvalue_t)); + if (!v) return NULL; + + v->datum = datum; + v->typeid = typeid; + v->typmod = typmod; + v->collation = collation; + v->isnull = isnull; + return v; +} + +void pgvalue_free (pgvalue_t *v) { + if (!v) return; + + if (v->owned_detoast) { + pfree(v->owned_detoast); + } + if (v->owns_cstring && v->cstring) { + pfree(v->cstring); + } + cloudsync_memory_free(v); +} + +void pgvalue_ensure_detoast(pgvalue_t *v) { + if (!v || v->detoasted) return; + if (!pgvalue_is_varlena(v->typeid) || v->isnull) return; + + v->owned_detoast = (void *)PG_DETOAST_DATUM_COPY(v->datum); + v->datum = PointerGetDatum(v->owned_detoast); + v->detoasted = true; +} + +int pgvalue_dbtype(pgvalue_t *v) { + if (!v || v->isnull) return DBTYPE_NULL; + switch (v->typeid) { + case INT2OID: + case INT4OID: + case INT8OID: + case BOOLOID: + case CHAROID: + case OIDOID: + return DBTYPE_INTEGER; + case FLOAT4OID: + case FLOAT8OID: + case NUMERICOID: + return DBTYPE_FLOAT; + case BYTEAOID: + return DBTYPE_BLOB; + default: + if (pgvalue_is_text_type(v->typeid)) { + return DBTYPE_TEXT; + } + return DBTYPE_TEXT; + } +} + +static bool pgvalue_vec_push(pgvalue_t ***arr, int *count, int *cap, pgvalue_t *val) { + if (*cap == 0) { + *cap = 8; + *arr = (pgvalue_t **)cloudsync_memory_zeroalloc(sizeof(pgvalue_t *) * (*cap)); + if (*arr == NULL) return false; + } else if (*count >= *cap) { + *cap *= 2; + *arr = (pgvalue_t **)cloudsync_memory_realloc(*arr, sizeof(pgvalue_t *) * (*cap)); + if (*arr == NULL) return false; + } + (*arr)[(*count)++] = val; + return true; +} + +pgvalue_t **pgvalues_from_array(ArrayType *array, int *out_count) { + if (out_count) *out_count = 0; + if (!array) return NULL; + + Oid elem_type = ARR_ELEMTYPE(array); + int16 elmlen; + bool elmbyval; + char elmalign; + get_typlenbyvalalign(elem_type, &elmlen, &elmbyval, &elmalign); + + Datum *elems = NULL; + bool *nulls = NULL; + int nelems = 0; + + deconstruct_array(array, elem_type, elmlen, elmbyval, elmalign, &elems, &nulls, &nelems); + + pgvalue_t **values = NULL; + int count = 0; + int cap = 0; + + for (int i = 0; i < nelems; i++) { + pgvalue_t *v = pgvalue_create(elems[i], elem_type, -1, InvalidOid, nulls ? nulls[i] : false); + pgvalue_vec_push(&values, &count, &cap, v); + } + + if (elems) pfree(elems); + if (nulls) pfree(nulls); + + if (out_count) *out_count = count; + return values; +} + +pgvalue_t **pgvalues_from_args(FunctionCallInfo fcinfo, int start_arg, int *out_count) { + if (out_count) *out_count = 0; + if (!fcinfo) return NULL; + + pgvalue_t **values = NULL; + int count = 0; + int cap = 0; + + for (int i = start_arg; i < PG_NARGS(); i++) { + Oid argtype = get_fn_expr_argtype(fcinfo->flinfo, i); + bool isnull = PG_ARGISNULL(i); + + // If the argument is an array (used for VARIADIC pk functions), expand it. + Oid elemtype = InvalidOid; + if (OidIsValid(argtype)) { + elemtype = get_element_type(argtype); + } + + if (OidIsValid(elemtype) && !isnull) { + ArrayType *array = PG_GETARG_ARRAYTYPE_P(i); + int subcount = 0; + pgvalue_t **subvals = pgvalues_from_array(array, &subcount); + for (int j = 0; j < subcount; j++) { + pgvalue_vec_push(&values, &count, &cap, subvals[j]); + } + if (subvals) cloudsync_memory_free(subvals); + continue; + } + + Datum datum = isnull ? (Datum)0 : PG_GETARG_DATUM(i); + pgvalue_t *v = pgvalue_create(datum, argtype, -1, fcinfo->fncollation, isnull); + pgvalue_vec_push(&values, &count, &cap, v); + } + + if (out_count) *out_count = count; + return values; +} + +void pgvalues_normalize_to_text(pgvalue_t **values, int count) { + // Convert all non-text pgvalues to text representation. + // This ensures PK encoding is consistent regardless of whether the caller + // passes native types (e.g., integer 1) or text representations (e.g., '1'). + // The UPDATE trigger casts all values to ::text, so INSERT trigger and + // SQL functions must do the same for PK encoding consistency. + if (!values) return; + + for (int i = 0; i < count; i++) { + pgvalue_t *v = values[i]; + if (!v || v->isnull) continue; + if (pgvalue_is_text_type(v->typeid)) continue; + + // Convert to text using the type's output function + const char *cstr = database_value_text((dbvalue_t *)v); + if (!cstr) continue; + + // Create a new text datum + text *t = cstring_to_text(cstr); + pgvalue_t *new_v = pgvalue_create(PointerGetDatum(t), TEXTOID, -1, v->collation, false); + if (new_v) { + pgvalue_free(v); + values[i] = new_v; + } + } +} diff --git a/src/postgresql/pgvalue.h b/src/postgresql/pgvalue.h new file mode 100644 index 00000000..3fbd28b4 --- /dev/null +++ b/src/postgresql/pgvalue.h @@ -0,0 +1,44 @@ +// pgvalue.h +// PostgreSQL-specific dbvalue_t wrapper + +#ifndef CLOUDSYNC_PGVALUE_H +#define CLOUDSYNC_PGVALUE_H + +// Define POSIX feature test macros before any includes +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "postgres.h" +#include "fmgr.h" +#include "utils/memutils.h" +#include "utils/array.h" +#include "../database.h" + +// dbvalue_t representation for PostgreSQL. We capture Datum + type metadata so +// value helpers can resolve type/length/ownership without relying on fcinfo lifetime. +typedef struct pgvalue_t { + Datum datum; + Oid typeid; + int32 typmod; + Oid collation; + bool isnull; + bool detoasted; + void *owned_detoast; + char *cstring; + bool owns_cstring; +} pgvalue_t; + +pgvalue_t *pgvalue_create(Datum datum, Oid typeid, int32 typmod, Oid collation, bool isnull); +void pgvalue_free (pgvalue_t *v); +void pgvalue_ensure_detoast(pgvalue_t *v); +bool pgvalue_is_text_type(Oid typeid); +int pgvalue_dbtype(pgvalue_t *v); +pgvalue_t **pgvalues_from_array(ArrayType *array, int *out_count); +pgvalue_t **pgvalues_from_args(FunctionCallInfo fcinfo, int start_arg, int *out_count); +void pgvalues_normalize_to_text(pgvalue_t **values, int count); + +#endif // CLOUDSYNC_PGVALUE_H diff --git a/src/postgresql/postgresql_log.h b/src/postgresql/postgresql_log.h new file mode 100644 index 00000000..94e0e12f --- /dev/null +++ b/src/postgresql/postgresql_log.h @@ -0,0 +1,27 @@ +// +// postgresql_log.h +// cloudsync +// +// PostgreSQL-specific logging implementation using elog() +// +// Note: This header requires _POSIX_C_SOURCE and _GNU_SOURCE to be defined +// before any includes. These are set as compiler flags in Makefile.postgresql. +// + +#ifndef __POSTGRESQL_LOG__ +#define __POSTGRESQL_LOG__ + +// setjmp.h is needed before postgres.h for sigjmp_buf type +#include + +// Include PostgreSQL headers +#include "postgres.h" +#include "utils/elog.h" + +// PostgreSQL logging macros using elog() +// DEBUG1 is the highest priority debug level in PostgreSQL +// LOG is for informational messages +#define CLOUDSYNC_LOG_DEBUG(...) elog(DEBUG1, __VA_ARGS__) +#define CLOUDSYNC_LOG_INFO(...) elog(LOG, __VA_ARGS__) + +#endif diff --git a/src/postgresql/sql_postgresql.c b/src/postgresql/sql_postgresql.c new file mode 100644 index 00000000..9106b1b1 --- /dev/null +++ b/src/postgresql/sql_postgresql.c @@ -0,0 +1,488 @@ +// +// sql_postgresql.c +// cloudsync +// +// PostgreSQL-specific SQL queries +// Created by Claude Code on 22/12/25. +// + +#include "../sql.h" + +// MARK: Settings + +const char * const SQL_SETTINGS_GET_VALUE = + "SELECT value FROM cloudsync_settings WHERE key=$1;"; + +const char * const SQL_SETTINGS_SET_KEY_VALUE_REPLACE = + "INSERT INTO cloudsync_settings (key, value) VALUES ($1, $2) " + "ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value;"; + +const char * const SQL_SETTINGS_SET_KEY_VALUE_DELETE = + "DELETE FROM cloudsync_settings WHERE key = $1;"; + +const char * const SQL_TABLE_SETTINGS_GET_VALUE = + "SELECT value FROM cloudsync_table_settings WHERE (tbl_name=$1 AND col_name=$2 AND key=$3);"; + +const char * const SQL_TABLE_SETTINGS_DELETE_ALL_FOR_TABLE = + "DELETE FROM cloudsync_table_settings WHERE tbl_name=$1;"; + +const char * const SQL_TABLE_SETTINGS_REPLACE = + "INSERT INTO cloudsync_table_settings (tbl_name, col_name, key, value) VALUES ($1, $2, $3, $4) " + "ON CONFLICT (tbl_name, col_name, key) DO UPDATE SET value = EXCLUDED.value;"; + +const char * const SQL_TABLE_SETTINGS_DELETE_ONE = + "DELETE FROM cloudsync_table_settings WHERE (tbl_name=$1 AND col_name=$2 AND key=$3);"; + +const char * const SQL_TABLE_SETTINGS_COUNT_TABLES = + "SELECT count(*) FROM cloudsync_table_settings WHERE key='algo';"; + +const char * const SQL_SETTINGS_LOAD_GLOBAL = + "SELECT key, value FROM cloudsync_settings;"; + +const char * const SQL_SETTINGS_LOAD_TABLE = + "SELECT lower(tbl_name), lower(col_name), key, value FROM cloudsync_table_settings ORDER BY tbl_name, col_name;"; + +const char * const SQL_CREATE_SETTINGS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_settings (key TEXT PRIMARY KEY NOT NULL, value TEXT);" + "CREATE TABLE IF NOT EXISTS public.app_schema_version (" + "version BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY" + ");" + "CREATE OR REPLACE FUNCTION bump_app_schema_version() " + "RETURNS event_trigger AS $$ " + "BEGIN " + "INSERT INTO public.app_schema_version DEFAULT VALUES; " + "END;" + "$$ LANGUAGE plpgsql;" + "DROP EVENT TRIGGER IF EXISTS app_schema_change;" + "CREATE EVENT TRIGGER app_schema_change " + "ON ddl_command_end " + "EXECUTE FUNCTION bump_app_schema_version();"; + +// format strings (snprintf) are also static SQL templates +const char * const SQL_INSERT_SETTINGS_STR_FORMAT = + "INSERT INTO cloudsync_settings (key, value) VALUES ('%s', '%s');"; + +const char * const SQL_INSERT_SETTINGS_INT_FORMAT = + "INSERT INTO cloudsync_settings (key, value) VALUES ('%s', %lld);"; + +const char * const SQL_CREATE_SITE_ID_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_site_id (" + "id BIGSERIAL PRIMARY KEY, " + "site_id BYTEA UNIQUE NOT NULL" + ");"; + +const char * const SQL_INSERT_SITE_ID_ROWID = + "INSERT INTO cloudsync_site_id (id, site_id) VALUES ($1, $2);"; + +const char * const SQL_CREATE_TABLE_SETTINGS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_table_settings (tbl_name TEXT NOT NULL, col_name TEXT NOT NULL, key TEXT NOT NULL, value TEXT, PRIMARY KEY(tbl_name,col_name,key));"; + +const char * const SQL_CREATE_SCHEMA_VERSIONS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_schema_versions (hash BIGINT PRIMARY KEY, seq INTEGER NOT NULL)"; + +const char * const SQL_SETTINGS_CLEANUP_DROP_ALL = + "DROP TABLE IF EXISTS cloudsync_settings CASCADE; " + "DROP TABLE IF EXISTS cloudsync_site_id CASCADE; " + "DROP TABLE IF EXISTS cloudsync_table_settings CASCADE; " + "DROP TABLE IF EXISTS cloudsync_schema_versions CASCADE; " + "DROP TABLE IF EXISTS cloudsync_payload_fragments CASCADE;"; + +// MARK: CloudSync + +const char * const SQL_DBVERSION_BUILD_QUERY = + "WITH table_names AS (" + "SELECT quote_ident(schemaname) || '.' || quote_ident(tablename) as tbl_name " + "FROM pg_tables " + "WHERE tablename LIKE '%_cloudsync'" + "), " + "query_parts AS (" + "SELECT tbl_name, " + "format('SELECT COALESCE(MAX(db_version), 0) FROM %s', tbl_name) as part " + "FROM table_names" + ") " + "SELECT string_agg(part, ' UNION ALL ') FROM query_parts;"; + +const char * const SQL_CHANGES_INSERT_ROW = + "INSERT INTO cloudsync_changes(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) " + "VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9);"; + +const char * const SQL_PAYLOAD_FRAGMENTS_CREATE_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_payload_fragments (" + "value_id TEXT NOT NULL, part_index BIGINT NOT NULL, part_count BIGINT NOT NULL, total_size BIGINT NOT NULL, " + "checksum TEXT NOT NULL, created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())::bigint), " + "tbl TEXT NOT NULL, pk BYTEA NOT NULL, col_name TEXT NOT NULL, col_version BIGINT NOT NULL, db_version BIGINT NOT NULL, " + "site_id BYTEA NOT NULL, cl BIGINT NOT NULL, seq BIGINT NOT NULL, fragment BYTEA NOT NULL, " + "PRIMARY KEY(value_id, part_index));"; + +const char * const SQL_PAYLOAD_FRAGMENTS_UPSERT = + "INSERT INTO cloudsync_payload_fragments " + "(value_id, part_index, part_count, total_size, checksum, created_at, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, fragment) " + "VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) " + "ON CONFLICT (value_id, part_index) DO UPDATE SET " + "part_count=EXCLUDED.part_count,total_size=EXCLUDED.total_size,checksum=EXCLUDED.checksum,created_at=EXCLUDED.created_at," + "tbl=EXCLUDED.tbl,pk=EXCLUDED.pk," + "col_name=EXCLUDED.col_name,col_version=EXCLUDED.col_version,db_version=EXCLUDED.db_version," + "site_id=EXCLUDED.site_id,cl=EXCLUDED.cl,seq=EXCLUDED.seq,fragment=EXCLUDED.fragment;"; + +const char * const SQL_PAYLOAD_FRAGMENTS_COUNT = + "SELECT COUNT(*), MIN(part_count), MAX(part_count), MIN(total_size), MAX(total_size), " + "MIN(checksum), MAX(checksum), MIN(part_index), MAX(part_index) " + "FROM cloudsync_payload_fragments WHERE value_id=$1;"; + +const char * const SQL_PAYLOAD_FRAGMENTS_SELECT = + "SELECT fragment, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, checksum " + "FROM cloudsync_payload_fragments WHERE value_id=$1 ORDER BY part_index ASC;"; + +const char * const SQL_PAYLOAD_FRAGMENTS_DELETE = + "DELETE FROM cloudsync_payload_fragments WHERE value_id=$1;"; + +const char * const SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE = + "DELETE FROM cloudsync_payload_fragments " + "WHERE created_at < $1 AND value_id IN (" + "SELECT value_id FROM cloudsync_payload_fragments GROUP BY value_id " + "HAVING COUNT(*) < MAX(part_count));"; + +// MARK: Additional SQL constants for PostgreSQL + +const char * const SQL_SITEID_SELECT_ROWID0 = + "SELECT site_id FROM cloudsync_site_id WHERE id = 0;"; + +const char * const SQL_DATA_VERSION = + "SELECT txid_snapshot_xmin(txid_current_snapshot());"; // was "PRAGMA data_version" + +const char * const SQL_SCHEMA_VERSION = + "SELECT COALESCE(max(version), 0) FROM app_schema_version;"; // was "PRAGMA schema_version" + +const char * const SQL_SITEID_GETSET_ROWID_BY_SITEID = + "INSERT INTO cloudsync_site_id (site_id) VALUES ($1) " + "ON CONFLICT(site_id) DO UPDATE SET site_id = EXCLUDED.site_id " + "RETURNING id;"; + +const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_ROWID = + "SELECT string_agg(quote_ident(column_name), ',' ORDER BY ordinal_position) " + "FROM information_schema.columns " + "WHERE table_name = $1 AND column_name NOT IN (" + "SELECT column_name FROM information_schema.key_column_usage " + "WHERE table_name = $1 AND constraint_name LIKE '%_pkey'" + ");"; // TODO: build full SELECT ... WHERE ctid=? analog with ordered columns like SQLite + +const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_PK = + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + "), " + "nonpk AS (" + " SELECT a.attname " + " FROM pg_attribute a " + " JOIN tbl t ON t.oid = a.attrelid " + " WHERE a.attnum > 0 AND NOT a.attisdropped " + " AND a.attnum NOT IN (" + " SELECT k.attnum " + " FROM pg_index x " + " JOIN tbl t2 ON t2.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) AS k(attnum) ON true " + " WHERE x.indisprimary" + " ) " + " ORDER BY a.attnum" + ") " + "SELECT " + " 'SELECT '" + " || (SELECT string_agg(format('%%I', attname), ',') FROM nonpk)" + " || ' FROM ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' WHERE '" + " || (SELECT string_agg(format('%%I=$%%s::%%s', attname, ord, coltype), ' AND ' ORDER BY ord) FROM pk)" + " || ';';"; + +const char * const SQL_DELETE_ROW_BY_ROWID = + "DELETE FROM %s WHERE ctid = $1;"; // TODO: consider using PK-based deletion; ctid is unstable + +const char * const SQL_BUILD_DELETE_ROW_BY_PK = + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + ") " + "SELECT " + " 'DELETE FROM ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' WHERE '" + " || (SELECT string_agg(format('%%I=$%%s::%%s', attname, ord, coltype), ' AND ' ORDER BY ord) FROM pk)" + " || ';';"; + +const char * const SQL_INSERT_ROWID_IGNORE = + "INSERT INTO %s DEFAULT VALUES ON CONFLICT DO NOTHING;"; // TODO: adapt to explicit PK inserts (no rowid in PG) + +const char * const SQL_UPSERT_ROWID_AND_COL_BY_ROWID = + "INSERT INTO %s (ctid, %s) VALUES ($1, $2) " + "ON CONFLICT DO UPDATE SET %s = $2;"; // TODO: align with SQLite upsert by rowid; avoid ctid + +const char * const SQL_BUILD_INSERT_PK_IGNORE = + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + ") " + "SELECT " + " 'INSERT INTO ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' (' || (SELECT string_agg(format('%%I', attname), ',') FROM pk) || ')'" + " || ' VALUES (' || (SELECT string_agg(format('$%%s::%%s', ord, coltype), ',') FROM pk) || ')'" + " || ' ON CONFLICT DO NOTHING;';"; + +const char * const SQL_BUILD_UPSERT_PK_AND_COL = + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + "), " + "pk_count AS (" + " SELECT count(*) AS n FROM pk" + "), " + "col AS (" + " SELECT '%s'::text AS colname, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_attribute a " + " JOIN tbl t ON t.oid = a.attrelid " + " WHERE a.attname = '%s' AND a.attnum > 0 AND NOT a.attisdropped" + ") " + "SELECT " + " 'INSERT INTO ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' (' || (SELECT string_agg(format('%%I', attname), ',') FROM pk)" + " || ',' || (SELECT format('%%I', colname) FROM col) || ')'" + " || ' VALUES (' || (SELECT string_agg(format('$%%s::%%s', ord, coltype), ',') FROM pk)" + " || ',' || (SELECT format('$%%s::%%s', (SELECT n FROM pk_count) + 1, coltype) FROM col) || ')'" + " || ' ON CONFLICT (' || (SELECT string_agg(format('%%I', attname), ',') FROM pk) || ')'" + " || ' DO UPDATE SET ' || (SELECT format('%%I', colname) FROM col)" + " || '=' || (SELECT format('$%%s::%%s', (SELECT n FROM pk_count) + 2, coltype) FROM col) || ';';"; + +const char * const SQL_SELECT_COLS_BY_ROWID_FMT = + "SELECT %s%s%s FROM %s WHERE ctid = $1;"; // TODO: align with PK/rowid selection builder + +const char * const SQL_BUILD_SELECT_COLS_BY_PK_FMT = + "WITH tbl AS (" + " SELECT to_regclass('%s') AS tblreg" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.tblreg = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + "), " + "col AS (" + " SELECT '%s'::text AS colname" + ") " + "SELECT " + " 'SELECT ' || (SELECT format('%%I', colname) FROM col) " + " || ' FROM ' || (SELECT tblreg::text FROM tbl)" + " || ' WHERE '" + " || (SELECT string_agg(format('%%I=$%%s::%%s', attname, ord, coltype), ' AND ' ORDER BY ord) FROM pk)" + " || ';';"; + +const char * const SQL_CLOUDSYNC_ROW_EXISTS_BY_PK = + "SELECT EXISTS(SELECT 1 FROM %s WHERE pk = $1 LIMIT 1);"; + +const char * const SQL_CLOUDSYNC_UPDATE_COL_BUMP_VERSION = + "UPDATE %s " + "SET col_version = CASE col_version %% 2 WHEN 0 THEN col_version + 1 ELSE col_version + 2 END, " + "db_version = $1, seq = $2, site_id = 0 " + "WHERE pk = $3 AND col_name = '%s';"; + +const char * const SQL_CLOUDSYNC_UPSERT_COL_INIT_OR_BUMP_VERSION = + "INSERT INTO %s (pk, col_name, col_version, db_version, seq, site_id) " + "VALUES ($1, '%s', 1, $2, $3, 0) " + "ON CONFLICT (pk, col_name) DO UPDATE SET " + "col_version = CASE %s.col_version %% 2 WHEN 0 THEN %s.col_version + 1 ELSE %s.col_version + 2 END, " + "db_version = $2, seq = $3, site_id = 0;"; + +const char * const SQL_CLOUDSYNC_UPSERT_RAW_COLVERSION = + "INSERT INTO %s (pk, col_name, col_version, db_version, seq, site_id) " + "VALUES ($1, $2, $3, $4, $5, 0) " + "ON CONFLICT (pk, col_name) DO UPDATE SET " + "col_version = %s.col_version + 1, db_version = $6, seq = $7, site_id = 0;"; + +const char * const SQL_CLOUDSYNC_DELETE_PK_EXCEPT_COL = + "DELETE FROM %s WHERE pk = $1 AND col_name != '%s';"; // TODO: match SQLite delete semantics + +const char * const SQL_CLOUDSYNC_REKEY_PK_AND_RESET_VERSION_EXCEPT_COL = + "WITH moved AS (" + " SELECT col_name " + " FROM %s WHERE pk = $3 AND col_name != '%s'" + "), " + "upserted AS (" + " INSERT INTO %s (pk, col_name, col_version, db_version, seq, site_id) " + " SELECT $1, col_name, 1, $2, cloudsync_seq(), 0 " + " FROM moved " + " ON CONFLICT (pk, col_name) DO UPDATE SET " + " col_version = 1, db_version = $2, seq = cloudsync_seq(), site_id = 0" + ") " + "DELETE FROM %s WHERE pk = $3 AND col_name != '%s';"; + +const char * const SQL_CLOUDSYNC_GET_COL_VERSION_OR_ROW_EXISTS = + "SELECT COALESCE(" + "(SELECT col_version FROM %s WHERE pk = $1 AND col_name = '%s'), " + "(SELECT 1 FROM %s WHERE pk = $1 LIMIT 1)" + ");"; + +const char * const SQL_CLOUDSYNC_INSERT_RETURN_CHANGE_ID = + "INSERT INTO %s " + "(pk, col_name, col_version, db_version, seq, site_id) " + "VALUES ($1, $2, $3, cloudsync_db_version_next($4), $5, $6) " + "ON CONFLICT (pk, col_name) DO UPDATE SET " + "col_version = EXCLUDED.col_version, " + "db_version = cloudsync_db_version_next($4), " + "seq = EXCLUDED.seq, " + "site_id = EXCLUDED.site_id " + "RETURNING ((db_version::bigint << 30) | seq);"; // TODO: align RETURNING and bump logic with SQLite (version increments on conflict) + +const char * const SQL_CLOUDSYNC_TOMBSTONE_PK_EXCEPT_COL = + "UPDATE %s " + "SET col_version = 0, db_version = cloudsync_db_version_next($1) " + "WHERE pk = $2 AND col_name != '%s';"; // TODO: confirm tombstone semantics match SQLite + +const char * const SQL_CLOUDSYNC_SELECT_COL_VERSION_BY_PK_COL = + "SELECT col_version FROM %s WHERE pk = $1 AND col_name = $2;"; + +const char * const SQL_CLOUDSYNC_SELECT_SITE_ID_BY_PK_COL = + "SELECT site_id FROM %s WHERE pk = $1 AND col_name = $2;"; + +const char * const SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID = + "SELECT c.column_name, c.ordinal_position " + "FROM information_schema.columns c " + "WHERE c.table_name = '%s' " + "AND c.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND c.column_name NOT IN (" + " SELECT kcu.column_name FROM information_schema.table_constraints tc " + " JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + " WHERE tc.table_name = '%s' AND tc.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + " AND tc.constraint_type = 'PRIMARY KEY'" + ") " + "ORDER BY ordinal_position;"; + +const char * const SQL_DROP_CLOUDSYNC_TABLE = + "DROP TABLE IF EXISTS %s CASCADE;"; + +const char * const SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE = + "DELETE FROM %s;"; + +const char * const SQL_CLOUDSYNC_DELETE_COLS_NOT_IN_SCHEMA_OR_PKCOL = + "DELETE FROM %s WHERE col_name NOT IN (" + "SELECT column_name FROM information_schema.columns WHERE table_name = '%s' " + "AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "UNION SELECT '%s'" + ");"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_QUALIFIED_COLLIST_FMT = + "SELECT string_agg(quote_ident(column_name), ',' ORDER BY ordinal_position) " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(cloudsync_schema(), current_schema()) " + "AND constraint_name LIKE '%%_pkey';"; + +const char * const SQL_CLOUDSYNC_GC_DELETE_ORPHANED_PK = + "DELETE FROM %s " + "WHERE (col_name != '%s' OR (col_name = '%s' AND col_version %% 2 != 0)) " + "AND NOT EXISTS (" + "SELECT 1 FROM %s " + "WHERE %s.pk = cloudsync_pk_encode(%s) LIMIT 1" + ");"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_COLLIST = + "SELECT string_agg(quote_ident(column_name), ',') " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(cloudsync_schema(), current_schema()) " + "AND constraint_name LIKE '%%_pkey';"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_DECODE_SELECTLIST = + "SELECT string_agg(" + "'cloudsync_pk_decode(pk, ' || ordinal_position || ') AS ' || quote_ident(column_name), ',' ORDER BY ordinal_position" + ") " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(cloudsync_schema(), current_schema()) " + "AND constraint_name LIKE '%%_pkey';"; + +const char * const SQL_CLOUDSYNC_INSERT_MISSING_PKS_FROM_BASE_EXCEPT_SYNC = + "SELECT cloudsync_insert('%s', %s) " + "FROM (SELECT %s FROM %s EXCEPT SELECT %s FROM %s);"; + +const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL = + "WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM %s) " + "SELECT _cstemp1.pk FROM _cstemp1 " + "WHERE NOT EXISTS (" + "SELECT 1 FROM %s _cstemp2 " + "WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = $1" + ");"; + +const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL_FILTERED = + "WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM %s WHERE (%s)) " + "SELECT _cstemp1.pk FROM _cstemp1 " + "WHERE NOT EXISTS (" + "SELECT 1 FROM %s _cstemp2 " + "WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = $1" + ");"; + +// MARK: Blocks (block-level LWW) + +const char * const SQL_BLOCKS_CREATE_TABLE = + "CREATE TABLE IF NOT EXISTS %s (" + "pk BYTEA NOT NULL, " + "col_name TEXT COLLATE \"C\" NOT NULL, " + "col_value TEXT, " + "PRIMARY KEY (pk, col_name))"; + +const char * const SQL_BLOCKS_UPSERT = + "INSERT INTO %s (pk, col_name, col_value) VALUES ($1, $2, $3) " + "ON CONFLICT (pk, col_name) DO UPDATE SET col_value = EXCLUDED.col_value"; + +const char * const SQL_BLOCKS_SELECT = + "SELECT col_value FROM %s WHERE pk = $1 AND col_name = $2"; + +const char * const SQL_BLOCKS_DELETE = + "DELETE FROM %s WHERE pk = $1 AND col_name = $2"; + +const char * const SQL_BLOCKS_LIST_ALIVE = + "SELECT b.col_value FROM %s b " + "JOIN %s m ON b.pk = m.pk AND b.col_name = m.col_name " + "WHERE b.pk = $1 AND b.col_name LIKE $2 " + "AND m.pk = $3 AND m.col_name LIKE $4 AND m.col_version %% 2 = 1 " + "ORDER BY b.col_name COLLATE \"C\""; + +const char * const SQL_BLOCKS_INSERT_IGNORE = + "INSERT INTO %s (pk, col_name, col_value) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING"; + +const char * const SQL_META_SCAN_COL_FOR_MIGRATION = + "SELECT DISTINCT m.pk FROM %s m " + "WHERE m.col_name = $1 AND m.col_version %% 2 = 1 " + "AND NOT EXISTS (SELECT 1 FROM %s b WHERE b.pk = m.pk AND b.col_name LIKE $2)"; + +const char * const SQL_META_INSERT_BLOCK_IGNORE = + "INSERT INTO %s (pk, col_name, col_version, db_version, seq, site_id) " + "VALUES ($1, $2, $3, $4, $5, 0) ON CONFLICT DO NOTHING"; diff --git a/src/sql.h b/src/sql.h new file mode 100644 index 00000000..68371218 --- /dev/null +++ b/src/sql.h @@ -0,0 +1,87 @@ +// +// sql.h +// cloudsync +// +// Created by Marco Bambini on 17/12/25. +// + +#ifndef __CLOUDSYNC_SQL__ +#define __CLOUDSYNC_SQL__ + +// SETTINGS +extern const char * const SQL_SETTINGS_GET_VALUE; +extern const char * const SQL_SETTINGS_SET_KEY_VALUE_REPLACE; +extern const char * const SQL_SETTINGS_SET_KEY_VALUE_DELETE; +extern const char * const SQL_TABLE_SETTINGS_GET_VALUE; +extern const char * const SQL_TABLE_SETTINGS_DELETE_ALL_FOR_TABLE; +extern const char * const SQL_TABLE_SETTINGS_REPLACE; +extern const char * const SQL_TABLE_SETTINGS_DELETE_ONE; +extern const char * const SQL_TABLE_SETTINGS_COUNT_TABLES; +extern const char * const SQL_SETTINGS_LOAD_GLOBAL; +extern const char * const SQL_SETTINGS_LOAD_TABLE; +extern const char * const SQL_CREATE_SETTINGS_TABLE; +extern const char * const SQL_INSERT_SETTINGS_STR_FORMAT; +extern const char * const SQL_INSERT_SETTINGS_INT_FORMAT; +extern const char * const SQL_CREATE_SITE_ID_TABLE; +extern const char * const SQL_INSERT_SITE_ID_ROWID; +extern const char * const SQL_CREATE_TABLE_SETTINGS_TABLE; +extern const char * const SQL_CREATE_SCHEMA_VERSIONS_TABLE; +extern const char * const SQL_SETTINGS_CLEANUP_DROP_ALL; + +// CLOUDSYNC +extern const char * const SQL_DBVERSION_BUILD_QUERY; +extern const char * const SQL_SITEID_SELECT_ROWID0; +extern const char * const SQL_DATA_VERSION; +extern const char * const SQL_SCHEMA_VERSION; +extern const char * const SQL_SITEID_GETSET_ROWID_BY_SITEID; +extern const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_ROWID; +extern const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_PK; +extern const char * const SQL_DELETE_ROW_BY_ROWID; +extern const char * const SQL_BUILD_DELETE_ROW_BY_PK; +extern const char * const SQL_INSERT_ROWID_IGNORE; +extern const char * const SQL_UPSERT_ROWID_AND_COL_BY_ROWID; +extern const char * const SQL_BUILD_INSERT_PK_IGNORE; +extern const char * const SQL_BUILD_UPSERT_PK_AND_COL; +extern const char * const SQL_SELECT_COLS_BY_ROWID_FMT; +extern const char * const SQL_BUILD_SELECT_COLS_BY_PK_FMT; +extern const char * const SQL_CLOUDSYNC_ROW_EXISTS_BY_PK; +extern const char * const SQL_CLOUDSYNC_UPDATE_COL_BUMP_VERSION; +extern const char * const SQL_CLOUDSYNC_UPSERT_COL_INIT_OR_BUMP_VERSION; +extern const char * const SQL_CLOUDSYNC_UPSERT_RAW_COLVERSION; +extern const char * const SQL_CLOUDSYNC_DELETE_PK_EXCEPT_COL; +extern const char * const SQL_CLOUDSYNC_REKEY_PK_AND_RESET_VERSION_EXCEPT_COL; +extern const char * const SQL_CLOUDSYNC_GET_COL_VERSION_OR_ROW_EXISTS; +extern const char * const SQL_CLOUDSYNC_INSERT_RETURN_CHANGE_ID; +extern const char * const SQL_CLOUDSYNC_TOMBSTONE_PK_EXCEPT_COL; +extern const char * const SQL_CLOUDSYNC_SELECT_COL_VERSION_BY_PK_COL; +extern const char * const SQL_CLOUDSYNC_SELECT_SITE_ID_BY_PK_COL; +extern const char * const SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID; +extern const char * const SQL_DROP_CLOUDSYNC_TABLE; +extern const char * const SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE; +extern const char * const SQL_CLOUDSYNC_DELETE_COLS_NOT_IN_SCHEMA_OR_PKCOL; +extern const char * const SQL_PRAGMA_TABLEINFO_PK_QUALIFIED_COLLIST_FMT; +extern const char * const SQL_CLOUDSYNC_GC_DELETE_ORPHANED_PK; +extern const char * const SQL_PRAGMA_TABLEINFO_PK_COLLIST; +extern const char * const SQL_PRAGMA_TABLEINFO_PK_DECODE_SELECTLIST; +extern const char * const SQL_CLOUDSYNC_INSERT_MISSING_PKS_FROM_BASE_EXCEPT_SYNC; +extern const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL; +extern const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL_FILTERED; +extern const char * const SQL_CHANGES_INSERT_ROW; +extern const char * const SQL_PAYLOAD_FRAGMENTS_CREATE_TABLE; +extern const char * const SQL_PAYLOAD_FRAGMENTS_UPSERT; +extern const char * const SQL_PAYLOAD_FRAGMENTS_COUNT; +extern const char * const SQL_PAYLOAD_FRAGMENTS_SELECT; +extern const char * const SQL_PAYLOAD_FRAGMENTS_DELETE; +extern const char * const SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE; + +// BLOCKS (block-level LWW) +extern const char * const SQL_BLOCKS_CREATE_TABLE; +extern const char * const SQL_BLOCKS_UPSERT; +extern const char * const SQL_BLOCKS_SELECT; +extern const char * const SQL_BLOCKS_DELETE; +extern const char * const SQL_BLOCKS_LIST_ALIVE; +extern const char * const SQL_BLOCKS_INSERT_IGNORE; +extern const char * const SQL_META_SCAN_COL_FOR_MIGRATION; +extern const char * const SQL_META_INSERT_BLOCK_IGNORE; + +#endif diff --git a/src/vtab.c b/src/sqlite/cloudsync_changes_sqlite.c similarity index 73% rename from src/vtab.c rename to src/sqlite/cloudsync_changes_sqlite.c index 09c2fb64..5cd8a145 100644 --- a/src/vtab.c +++ b/src/sqlite/cloudsync_changes_sqlite.c @@ -1,5 +1,5 @@ // -// vtab.c +// cloudsync_changes_sqlite.c // cloudsync // // Created by Marco Bambini on 23/09/24. @@ -7,10 +7,10 @@ #include #include -#include "vtab.h" -#include "utils.h" -#include "dbutils.h" -#include "cloudsync.h" + +#include "cloudsync_changes_sqlite.h" +#include "../utils.h" +#include "../dbutils.h" #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT3 @@ -19,7 +19,7 @@ SQLITE_EXTENSION_INIT3 typedef struct cloudsync_changes_vtab { sqlite3_vtab base; // base class, must be first sqlite3 *db; - void *aux; + cloudsync_context *data; } cloudsync_changes_vtab; typedef struct cloudsync_changes_cursor { @@ -49,7 +49,18 @@ bool force_vtab_filter_abort = false; // MARK: - -const char *opname_from_value (int value) { +int vtab_set_error (sqlite3_vtab *vtab, const char *format, ...) { + va_list arg; + va_start (arg, format); + char *err = sqlite3_vmprintf(format, arg); + va_end (arg); + + if (vtab->zErrMsg) sqlite3_free(vtab->zErrMsg); + vtab->zErrMsg = err; + return SQLITE_ERROR; +} + +const char *vtab_opname_from_value (int value) { switch (value) { case SQLITE_INDEX_CONSTRAINT_EQ: return "="; case SQLITE_INDEX_CONSTRAINT_GT: return ">"; @@ -79,7 +90,7 @@ const char *opname_from_value (int value) { return NULL; } -int colname_is_legal (const char *name) { +int vtab_colname_is_legal (const char *name) { int count = sizeof(cloudsync_changes_columns) / sizeof (char *); for (int i=0; idb = db; - vnew->aux = aux; + vnew->data = aux; *vtab = (sqlite3_vtab *)vnew; } @@ -284,8 +296,8 @@ int cloudsync_changesvtab_best_index (sqlite3_vtab *vtab, sqlite3_index_info *id int idx = constraint->iColumn; uint8_t op = constraint->op; - const char *colname = (idx > 0) ? COLNAME_FROM_INDEX(idx) : "rowid"; - const char *opname = opname_from_value(op); + const char *colname = (idx >= 0 && idx < CLOUDSYNC_CHANGES_NCOLS) ? COLNAME_FROM_INDEX(idx) : "rowid"; + const char *opname = vtab_opname_from_value(op); if (!opname) continue; // build next constraint @@ -318,8 +330,8 @@ int cloudsync_changesvtab_best_index (sqlite3_vtab *vtab, sqlite3_index_info *id if (i > 0) sindex += snprintf(s+sindex, slen-sindex, ", "); int idx = orderby->iColumn; - const char *colname = COLNAME_FROM_INDEX(idx); - if (!colname_is_legal(colname)) orderconsumed = 0; + const char *colname = (idx >= 0 && idx < CLOUDSYNC_CHANGES_NCOLS) ? COLNAME_FROM_INDEX(idx) : "rowid"; + if (!vtab_colname_is_legal(colname)) orderconsumed = 0; sindex += snprintf(s+sindex, slen-sindex, "%s %s", colname, orderby->desc ? " DESC" : " ASC"); } @@ -383,11 +395,41 @@ int cloudsync_changesvtab_best_index (sqlite3_vtab *vtab, sqlite3_index_info *id int cloudsync_changesvtab_filter (sqlite3_vtab_cursor *cursor, int idxn, const char *idxs, int argc, sqlite3_value **argv) { DEBUG_VTAB("cloudsync_changesvtab_filter"); - + cloudsync_changes_cursor *c = (cloudsync_changes_cursor *)cursor; + cloudsync_context *data = c->vtab->data; sqlite3 *db = c->vtab->db; - char *sql = build_changes_sql(db, idxs); - if (sql == NULL) return SQLITE_NOMEM; + char *sql = vtab_build_changes_sql(data, idxs); + if (sql == NULL) { + // vtab_build_changes_sql returns NULL when no *_cloudsync meta-tables + // exist (cloudsync_init was never called, or the last configured table + // was cleaned up): the inner GROUP_CONCAT produces a NULL row and the + // outer SELECT yields a NULL final string. Distinguish this from a + // genuine OOM by checking whether cloudsync is configured, so the user + // gets an actionable message instead of "out of memory". + if (!cloudsync_config_exists(data) || dbutils_table_settings_count_tables(data) == 0) { + return vtab_set_error((sqlite3_vtab *)c->vtab, + "cloudsync has no tables configured for sync. Call " + "SELECT cloudsync_init('') to enable sync on a " + "table before querying cloudsync_changes."); + } + // settings rows exist but no *_cloudsync tables do (e.g. a dump/restore + // that skipped the meta tables): still not an OOM + sqlite3_stmt *vm = NULL; + bool meta_missing = false; + if (sqlite3_prepare_v2(db, "SELECT 1 FROM sqlite_master WHERE type='table' AND tbl_name LIKE '%_cloudsync' LIMIT 1;", -1, &vm, NULL) == SQLITE_OK) { + meta_missing = (sqlite3_step(vm) != SQLITE_ROW); + } + if (vm) sqlite3_finalize(vm); + if (meta_missing) { + return vtab_set_error((sqlite3_vtab *)c->vtab, + "cloudsync settings reference tables whose sync metadata is " + "missing (no *_cloudsync tables found). Re-run " + "SELECT cloudsync_init('') for each configured " + "table to rebuild it."); + } + return SQLITE_NOMEM; + } // the xFilter method may be called multiple times on the same sqlite3_vtab_cursor* if (c->vm) sqlite3_finalize(c->vm); @@ -472,39 +514,94 @@ int cloudsync_changesvtab_rowid (sqlite3_vtab_cursor *cursor, sqlite3_int64 *row return SQLITE_OK; } +int cloudsync_changesvtab_insert_gos (sqlite3_vtab *vtab, cloudsync_context *data, cloudsync_table_context *table, const char *insert_pk, int insert_pk_len, const char *insert_name, sqlite3_value *insert_value, sqlite3_int64 insert_col_version, sqlite3_int64 insert_db_version, const char *insert_site_id, int insert_site_id_len, sqlite3_int64 insert_seq, int64_t *rowid) { + DEBUG_VTAB("cloudsync_changesvtab_insert_gos"); + + // Grow-Only Set (GOS) Algorithm: Only insertions are allowed, deletions and updates are prevented from a trigger. + int rc = merge_insert_col(data, table, insert_pk, insert_pk_len, insert_name, insert_value, (int64_t)insert_col_version, (int64_t)insert_db_version, insert_site_id, insert_site_id_len, (int64_t)insert_seq, rowid); + + if (rc != SQLITE_OK) { + vtab_set_error(vtab, "%s", cloudsync_errmsg(data)); + } + + return rc; +} + +int cloudsync_changesvtab_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, sqlite3_int64 *rowid) { + DEBUG_VTAB("cloudsync_changesvtab_insert"); + + // this function performs the merging logic for an insert in a cloud-synchronized table. It handles + // different scenarios including conflicts, causal lengths, delete operations, and resurrecting rows + // based on the incoming data (from remote nodes or clients) and the local database state + + // this function handles different CRDT algorithms (GOS, DWS, AWS, and CLS). + // the merging strategy is determined based on the table->algo value. + + // meta table declaration: + // tbl TEXT NOT NULL, pk BLOB NOT NULL, col_name TEXT NOT NULL," + // "col_value ANY, col_version INTEGER NOT NULL, db_version INTEGER NOT NULL," + // "site_id BLOB NOT NULL, cl INTEGER NOT NULL, seq INTEGER NOT NULL + + // meta information to retrieve from arguments: + // argv[0] -> table name (TEXT) + // argv[1] -> primary key (BLOB) + // argv[2] -> column name (TEXT or NULL if sentinel) + // argv[3] -> column value (ANY) + // argv[4] -> column version (INTEGER) + // argv[5] -> database version (INTEGER) + // argv[6] -> site ID (BLOB, identifies the origin of the update) + // argv[7] -> causal length (INTEGER, tracks the order of operations) + // argv[8] -> sequence number (INTEGER, unique per operation) + + // extract table name + const char *insert_tbl = (const char *)sqlite3_value_text(argv[0]); + + // lookup table + cloudsync_context *data = (cloudsync_context *)(((cloudsync_changes_vtab *)vtab)->data); + cloudsync_table_context *table = table_lookup(data, insert_tbl); + if (!table) return vtab_set_error(vtab, "Unable to find table %s,", insert_tbl); + + // extract the remaining fields from the input values + const char *insert_pk = (const char *)sqlite3_value_blob(argv[1]); + int insert_pk_len = sqlite3_value_bytes(argv[1]); + const char *insert_name = (sqlite3_value_type(argv[2]) == SQLITE_NULL) ? CLOUDSYNC_TOMBSTONE_VALUE : (const char *)sqlite3_value_text(argv[2]); + sqlite3_value *insert_value = argv[3]; + int64_t insert_col_version = (int64_t)sqlite3_value_int(argv[4]); + int64_t insert_db_version = (int64_t)sqlite3_value_int(argv[5]); + const char *insert_site_id = (const char *)sqlite3_value_blob(argv[6]); + int insert_site_id_len = sqlite3_value_bytes(argv[6]); + int64_t insert_cl = (int64_t)sqlite3_value_int(argv[7]); + int64_t insert_seq = (int64_t)sqlite3_value_int(argv[8]); + + // perform different logic for each different table algorithm + if (table_algo_isgos(table)) return cloudsync_changesvtab_insert_gos(vtab, data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, (int64_t *)rowid); + + int rc = merge_insert (data, table, insert_pk, insert_pk_len, insert_cl, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, (int64_t *)rowid); + if (rc != SQLITE_OK) { + return vtab_set_error(vtab, "%s", cloudsync_errmsg(data)); + } + + return SQLITE_OK; +} + int cloudsync_changesvtab_update (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, sqlite3_int64 *rowid) { DEBUG_VTAB("cloudsync_changesvtab_update"); // only INSERT statements are allowed bool is_insert = (argc > 1 && sqlite3_value_type(argv[0]) == SQLITE_NULL); if (!is_insert) { - cloudsync_vtab_set_error(vtab, "Only INSERT and SELECT statements are allowed against the cloudsync_changes table"); + vtab_set_error(vtab, "Only INSERT and SELECT statements are allowed against the cloudsync_changes table"); return SQLITE_MISUSE; } // argv[0] is set only in case of DELETE statement (it contains the rowid of a row in the virtual table to be deleted) // argv[1] is the rowid of a new row to be inserted into the virtual table (always NULL in our case) // so reduce the number of meaningful arguments by 2 - return cloudsync_merge_insert(vtab, argc-2, &argv[2], rowid); + return cloudsync_changesvtab_insert(vtab, argc-2, &argv[2], rowid); } // MARK: - -cloudsync_context *cloudsync_vtab_get_context (sqlite3_vtab *vtab) { - return (cloudsync_context *)(((cloudsync_changes_vtab *)vtab)->aux); -} - -int cloudsync_vtab_set_error (sqlite3_vtab *vtab, const char *format, ...) { - va_list arg; - va_start (arg, format); - char *err = cloudsync_memory_vmprintf(format, arg); - va_end (arg); - - if (vtab->zErrMsg) cloudsync_memory_free(vtab->zErrMsg); - vtab->zErrMsg = err; - return SQLITE_ERROR; -} - int cloudsync_vtab_register_changes (sqlite3 *db, cloudsync_context *xdata) { static sqlite3_module cloudsync_changes_module = { /* iVersion */ 0, diff --git a/src/sqlite/cloudsync_changes_sqlite.h b/src/sqlite/cloudsync_changes_sqlite.h new file mode 100644 index 00000000..d6c284d3 --- /dev/null +++ b/src/sqlite/cloudsync_changes_sqlite.h @@ -0,0 +1,21 @@ +// +// cloudsync_changes_sqlite.h +// cloudsync +// +// Created by Marco Bambini on 23/09/24. +// + +#ifndef __CLOUDSYNC_CHANGES_SQLITE__ +#define __CLOUDSYNC_CHANGES_SQLITE__ + +#include "../cloudsync.h" + +#ifndef SQLITE_CORE +#include "sqlite3ext.h" +#else +#include "sqlite3.h" +#endif + +int cloudsync_vtab_register_changes (sqlite3 *db, cloudsync_context *xdata); + +#endif diff --git a/src/sqlite/cloudsync_sqlite.c b/src/sqlite/cloudsync_sqlite.c new file mode 100644 index 00000000..c416b015 --- /dev/null +++ b/src/sqlite/cloudsync_sqlite.c @@ -0,0 +1,2281 @@ +// +// cloudsync_sqlite.c +// cloudsync +// +// Created by Marco Bambini on 05/12/25. +// + +#include "cloudsync_sqlite.h" +#include "cloudsync_changes_sqlite.h" +#include "../pk.h" +#include "../cloudsync.h" +#include "../block.h" +#include "../database.h" +#include "../dbutils.h" +#include "../sql.h" +#include +#include + +#ifndef CLOUDSYNC_OMIT_NETWORK +#include "../network/network.h" +#endif + +#ifndef SQLITE_CORE +SQLITE_EXTENSION_INIT1 +#endif + +#ifndef UNUSED_PARAMETER +#define UNUSED_PARAMETER(X) (void)(X) +#endif + +#ifdef _WIN32 +#define APIEXPORT __declspec(dllexport) +#else +#define APIEXPORT +#endif + +typedef struct { + sqlite3_context *context; + int index; +} cloudsync_pk_decode_context; + +typedef struct { + sqlite3_value *table_name; + sqlite3_value **new_values; + sqlite3_value **old_values; + int count; + int capacity; +} cloudsync_update_payload; + +void dbsync_set_error (sqlite3_context *context, const char *format, ...) { + char buffer[2048]; + + va_list arg; + va_start (arg, format); + vsnprintf(buffer, sizeof(buffer), format, arg); + va_end (arg); + + if (context) sqlite3_result_error(context, buffer, -1); +} + +// MARK: - Public - + +void dbsync_version (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_version"); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + sqlite3_result_text(context, CLOUDSYNC_VERSION, -1, SQLITE_STATIC); +} + +void dbsync_siteid (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_siteid"); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + sqlite3_result_blob(context, cloudsync_siteid(data), UUID_LEN, SQLITE_STATIC); +} + +void dbsync_db_version (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_db_version"); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = cloudsync_dbversion_check_uptodate(data); + if (rc != SQLITE_OK) { + // When cloudsync_init was never called, data_version_stmt is NULL and + // database_errmsg() falls back to "not an error", producing the + // confusing "Unable to retrieve db_version (not an error)". Detect the + // uninitialized state and return an actionable message instead. The + // extra check only runs on the error branch, so it costs nothing on + // the sync hot path (merge operations keep going through the normal + // path where rc == SQLITE_OK). + if (!cloudsync_context_is_initialized(data)) { + dbsync_set_error(context, + "cloudsync is not initialized: call SELECT cloudsync_init('') " + "to enable sync on a table before calling cloudsync_db_version()."); + } else { + dbsync_set_error(context, "Unable to retrieve db_version (%s).", database_errmsg(data)); + } + return; + } + + sqlite3_result_int64(context, cloudsync_dbversion(data)); +} + +void dbsync_db_version_next (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_db_version_next"); + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + sqlite3_int64 merging_version = (argc == 1) ? database_value_int(argv[0]) : CLOUDSYNC_VALUE_NOTSET; + sqlite3_int64 value = cloudsync_dbversion_next(data, merging_version); + if (value == -1) { + if (!cloudsync_context_is_initialized(data)) { + dbsync_set_error(context, + "cloudsync is not initialized: call SELECT cloudsync_init('') " + "to enable sync on a table before calling cloudsync_db_version_next()."); + } else { + dbsync_set_error(context, "Unable to retrieve next_db_version (%s).", database_errmsg(data)); + } + return; + } + + sqlite3_result_int64(context, value); +} + +void dbsync_seq (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_seq"); + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + sqlite3_result_int(context, cloudsync_bumpseq(data)); +} + +void dbsync_uuid (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_uuid"); + + char value[UUID_STR_MAXLEN]; + char *uuid = cloudsync_uuid_v7_string(value, true); + sqlite3_result_text(context, uuid, -1, SQLITE_TRANSIENT); +} + +// cloudsync_uuid_text(blob, [dash_format]) -> canonical UUID string +void dbsync_uuid_text (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_uuid_text"); + + if (sqlite3_value_type(argv[0]) == SQLITE_NULL) { sqlite3_result_null(context); return; } + if (sqlite3_value_type(argv[0]) != SQLITE_BLOB || sqlite3_value_bytes(argv[0]) != UUID_LEN) { + sqlite3_result_error(context, "cloudsync_uuid_text: expected a 16-byte BLOB.", -1); + return; + } + bool dash_format = (argc > 1) ? (sqlite3_value_int(argv[1]) != 0) : true; + char value[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_stringify((uint8_t *)sqlite3_value_blob(argv[0]), value, dash_format); + sqlite3_result_text(context, value, -1, SQLITE_TRANSIENT); +} + +// cloudsync_uuid_blob(text) -> 16-byte UUID blob (accepts dashed/undashed) +void dbsync_uuid_blob (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_uuid_blob"); + + if (sqlite3_value_type(argv[0]) == SQLITE_NULL) { sqlite3_result_null(context); return; } + const char *str = (const char *)sqlite3_value_text(argv[0]); + int len = sqlite3_value_bytes(argv[0]); + uint8_t uuid[UUID_LEN]; + if (!str || cloudsync_uuid_v7_parse(str, len, uuid) != 0) { + sqlite3_result_error(context, "cloudsync_uuid_blob: malformed UUID string.", -1); + return; + } + sqlite3_result_blob(context, uuid, UUID_LEN, SQLITE_TRANSIENT); +} + +// MARK: - + +void dbsync_set (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_set"); + + // sanity check parameters + const char *key = (const char *)database_value_text(argv[0]); + const char *value = (const char *)database_value_text(argv[1]); + + // silently fails + if (key == NULL) return; + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + dbutils_settings_set_key_value(data, key, value); +} + +void dbsync_set_column (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_set_column"); + + const char *tbl = (const char *)database_value_text(argv[0]); + const char *col = (const char *)database_value_text(argv[1]); + const char *key = (const char *)database_value_text(argv[2]); + const char *value = (const char *)database_value_text(argv[3]); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // Handle block column setup: cloudsync_set_column('tbl', 'col', 'algo', 'block') + if (key && value && strcmp(key, "algo") == 0 && strcmp(value, "block") == 0) { + int rc = cloudsync_setup_block_column(data, tbl, col, NULL, true); + if (rc != DBRES_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + } + return; + } + + // Handle delimiter setting: cloudsync_set_column('tbl', 'col', 'delimiter', '\n\n') + if (key && strcmp(key, "delimiter") == 0) { + cloudsync_table_context *table = table_lookup(data, tbl); + if (table) { + int col_idx = table_col_index(table, col); + if (col_idx >= 0 && table_col_algo(table, col_idx) == col_algo_block) { + table_set_col_delimiter(table, col_idx, value); + } + } + } + + dbutils_table_settings_set_key_value(data, tbl, col, key, value); +} + +void dbsync_set_table (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_set_table"); + + const char *tbl = (const char *)database_value_text(argv[0]); + const char *key = (const char *)database_value_text(argv[1]); + const char *value = (const char *)database_value_text(argv[2]); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + dbutils_table_settings_set_key_value(data, tbl, "*", key, value); +} + +void dbsync_set_schema (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_set_schema"); + + const char *schema = (const char *)database_value_text(argv[0]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + cloudsync_set_schema(data, schema); +} + +void dbsync_schema (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_schema"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + const char *schema = cloudsync_schema(data); + (schema) ? sqlite3_result_text(context, schema, -1, NULL) : sqlite3_result_null(context); +} + +void dbsync_table_schema (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_table_schema"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + const char *table_name = (const char *)database_value_text(argv[0]); + const char *schema = cloudsync_table_schema(data, table_name); + (schema) ? sqlite3_result_text(context, schema, -1, NULL) : sqlite3_result_null(context); +} + +void dbsync_is_sync (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_is_sync"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + if (cloudsync_insync(data)) { + sqlite3_result_int(context, 1); + return; + } + + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_table_context *table = table_lookup(data, table_name); + sqlite3_result_int(context, (table) ? (table_enabled(table) == 0) : 0); +} + +void dbsync_col_value (sqlite3_context *context, int argc, sqlite3_value **argv) { + // DEBUG_FUNCTION("cloudsync_col_value"); + + // argv[0] -> table name + // argv[1] -> column name + // argv[2] -> encoded pk + + // retrieve column name + const char *col_name = (const char *)database_value_text(argv[1]); + if (!col_name) { + dbsync_set_error(context, "Column name cannot be NULL"); + return; + } + + // check for special tombstone value + if (strcmp(col_name, CLOUDSYNC_TOMBSTONE_VALUE) == 0) { + sqlite3_result_null(context); + return; + } + + // lookup table + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + dbsync_set_error(context, "Unable to retrieve table name %s in clousdsync_colvalue.", table_name); + return; + } + + // Block column: if col_name contains \x1F, read from blocks table + if (block_is_block_colname(col_name) && table_has_block_cols(table)) { + dbvm_t *bvm = table_block_value_read_stmt(table); + if (!bvm) { + sqlite3_result_null(context); + return; + } + int rc = databasevm_bind_blob(bvm, 1, database_value_blob(argv[2]), database_value_bytes(argv[2])); + if (rc != DBRES_OK) { databasevm_reset(bvm); sqlite3_result_error(context, database_errmsg(data), -1); return; } + rc = databasevm_bind_text(bvm, 2, col_name, -1); + if (rc != DBRES_OK) { databasevm_reset(bvm); sqlite3_result_error(context, database_errmsg(data), -1); return; } + + rc = databasevm_step(bvm); + if (rc == SQLITE_ROW) { + sqlite3_result_value(context, database_column_value(bvm, 0)); + } else if (rc == SQLITE_DONE) { + sqlite3_result_null(context); + } else { + sqlite3_result_error(context, database_errmsg(data), -1); + } + databasevm_reset(bvm); + return; + } + + // extract the right col_value vm associated to the column name + sqlite3_stmt *vm = table_column_lookup(table, col_name, false, NULL); + if (!vm) { + sqlite3_result_error(context, "Unable to retrieve column value precompiled statement in clousdsync_colvalue.", -1); + return; + } + + // bind primary key values + int rc = pk_decode_prikey((char *)database_value_blob(argv[2]), (size_t)database_value_bytes(argv[2]), pk_decode_bind_callback, (void *)vm); + if (rc < 0) goto cleanup; + + // execute vm + rc = databasevm_step(vm); + if (rc == SQLITE_DONE) { + rc = SQLITE_OK; + sqlite3_result_text(context, CLOUDSYNC_RLS_RESTRICTED_VALUE, -1, SQLITE_STATIC); + } else if (rc == SQLITE_ROW) { + // store value result + rc = SQLITE_OK; + sqlite3_result_value(context, database_column_value(vm, 0)); + } + +cleanup: + if (rc != SQLITE_OK) { + sqlite3_result_error(context, database_errmsg(data), -1); + } + databasevm_reset(vm); +} + +void dbsync_pk_encode (sqlite3_context *context, int argc, sqlite3_value **argv) { + size_t bsize = 0; + char *buffer = pk_encode_prikey((dbvalue_t **)argv, argc, NULL, &bsize); + if (!buffer || buffer == PRIKEY_NULL_CONSTRAINT_ERROR) { + sqlite3_result_null(context); + return; + } + sqlite3_result_blob(context, (const void *)buffer, (int)bsize, SQLITE_TRANSIENT); + cloudsync_memory_free(buffer); +} + +int dbsync_pk_decode_set_result_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { + cloudsync_pk_decode_context *decode_context = (cloudsync_pk_decode_context *)xdata; + // decode_context->index is 1 based + // index is 0 based + if (decode_context->index != index+1) return SQLITE_OK; + + int rc = 0; + sqlite3_context *context = decode_context->context; + switch (type) { + case SQLITE_INTEGER: + sqlite3_result_int64(context, ival); + break; + + case SQLITE_FLOAT: + sqlite3_result_double(context, dval); + break; + + case SQLITE_NULL: + sqlite3_result_null(context); + break; + + case SQLITE_TEXT: + sqlite3_result_text(context, pval, (int)ival, SQLITE_TRANSIENT); + break; + + case SQLITE_BLOB: + sqlite3_result_blob(context, pval, (int)ival, SQLITE_TRANSIENT); + break; + } + + return rc; +} + + +void dbsync_pk_decode (sqlite3_context *context, int argc, sqlite3_value **argv) { + const char *pk = (const char *)database_value_blob(argv[0]); + int pk_len = database_value_bytes(argv[0]); + int i = (int)database_value_int(argv[1]); + + cloudsync_pk_decode_context xdata = {.context = context, .index = i}; + pk_decode_prikey((char *)pk, (size_t)pk_len, dbsync_pk_decode_set_result_callback, &xdata); +} + +// MARK: - + +void dbsync_insert (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_insert %s", database_value_text(argv[0])); + // debug_values(argc-1, &argv[1]); + + // argv[0] is table name + // argv[1]..[N] is primary key(s) + + // table_cloudsync + // pk -> encode(argc-1, &argv[1]) + // col_name -> name + // col_version -> 0/1 +1 + // db_version -> check + // site_id 0 + // seq -> sqlite_master + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // lookup table + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + dbsync_set_error(context, "Unable to retrieve table name %s in cloudsync_insert.", table_name); + return; + } + + // encode the primary key values into a buffer + char buffer[1024]; + size_t pklen = sizeof(buffer); + char *pk = pk_encode_prikey((dbvalue_t **)&argv[1], table_count_pks(table), buffer, &pklen); + if (!pk) { + sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); + return; + } + if (pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + dbsync_set_error(context, "Insert aborted because primary key in table %s contains NULL values.", table_name); + return; + } + + // compute the next database version for tracking changes + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + // check if a row with the same primary key already exists + // if so, this means the row might have been previously deleted (sentinel) + bool pk_exists = table_pk_exists(table, pk, pklen); + int rc = SQLITE_OK; + + if (table_count_cols(table) == 0) { + // if there are no columns other than primary keys, insert a sentinel record + rc = local_mark_insert_sentinel_meta(table, pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + } else if (pk_exists){ + // if a row with the same primary key already exists, update the sentinel record + rc = local_update_sentinel(table, pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + } + + // process each non-primary key column for insert or update + for (int i=0; icount); + if (positions) { + for (int b = 0; b < blocks->count; b++) { + char *block_cn = block_build_colname(col, positions[b]); + if (block_cn) { + rc = local_mark_insert_or_update_meta(table, pk, pklen, block_cn, db_version, cloudsync_bumpseq(data)); + + // Store block value in blocks table + dbvm_t *wvm = table_block_value_write_stmt(table); + if (wvm && rc == SQLITE_OK) { + databasevm_bind_blob(wvm, 1, pk, (int)pklen); + databasevm_bind_text(wvm, 2, block_cn, -1); + databasevm_bind_text(wvm, 3, blocks->entries[b].content, -1); + databasevm_step(wvm); + databasevm_reset(wvm); + } + + cloudsync_memory_free(block_cn); + } + cloudsync_memory_free(positions[b]); + if (rc != SQLITE_OK) break; + } + cloudsync_memory_free(positions); + } + block_list_free(blocks); + } + } + databasevm_reset((dbvm_t *)val_vm); + if (rc == DBRES_ROW || rc == DBRES_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto cleanup; + } else { + // Regular column: mark as inserted or updated in the metadata + rc = local_mark_insert_or_update_meta(table, pk, pklen, table_colname(table, i), db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + } + } + +cleanup: + if (rc != SQLITE_OK) sqlite3_result_error(context, database_errmsg(data), -1); + // free memory if the primary key was dynamically allocated + if (pk != buffer) cloudsync_memory_free(pk); +} + +void dbsync_delete (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_delete %s", database_value_text(argv[0])); + // debug_values(argc-1, &argv[1]); + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // lookup table + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + dbsync_set_error(context, "Unable to retrieve table name %s in cloudsync_delete.", table_name); + return; + } + + // compute the next database version for tracking changes + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + int rc = SQLITE_OK; + + // encode the primary key values into a buffer + char buffer[1024]; + size_t pklen = sizeof(buffer); + char *pk = pk_encode_prikey((dbvalue_t **)&argv[1], table_count_pks(table), buffer, &pklen); + if (!pk) { + sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); + return; + } + + if (pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + dbsync_set_error(context, "Delete aborted because primary key in table %s contains NULL values.", table_name); + return; + } + + // mark the row as deleted by inserting a delete sentinel into the metadata + rc = local_mark_delete_meta(table, pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + + // remove any metadata related to the old rows associated with this primary key + rc = local_drop_meta(table, pk, pklen); + if (rc != SQLITE_OK) goto cleanup; + +cleanup: + if (rc != SQLITE_OK) sqlite3_result_error(context, database_errmsg(data), -1); + // free memory if the primary key was dynamically allocated + if (pk != buffer) cloudsync_memory_free(pk); +} + +// MARK: - + +void dbsync_update_payload_free (cloudsync_update_payload *payload) { + for (int i=0; icount; i++) { + database_value_free(payload->new_values[i]); + database_value_free(payload->old_values[i]); + } + cloudsync_memory_free(payload->new_values); + cloudsync_memory_free(payload->old_values); + database_value_free(payload->table_name); + payload->new_values = NULL; + payload->old_values = NULL; + payload->table_name = NULL; + payload->count = 0; + payload->capacity = 0; +} + +int dbsync_update_payload_append (cloudsync_update_payload *payload, sqlite3_value *v1, sqlite3_value *v2, sqlite3_value *v3) { + if (payload->count >= payload->capacity) { + int newcap = payload->capacity ? payload->capacity * 2 : 128; + + sqlite3_value **new_values_2 = (sqlite3_value **)cloudsync_memory_realloc(payload->new_values, newcap * sizeof(*new_values_2)); + if (!new_values_2) return SQLITE_NOMEM; + + sqlite3_value **old_values_2 = (sqlite3_value **)cloudsync_memory_realloc(payload->old_values, newcap * sizeof(*old_values_2)); + if (!old_values_2) { + // new_values_2 succeeded but old_values failed; keep new_values_2 pointer + // (it's still valid, just larger) but don't update capacity + payload->new_values = new_values_2; + return SQLITE_NOMEM; + } + + payload->new_values = new_values_2; + payload->old_values = old_values_2; + payload->capacity = newcap; + } + + int index = payload->count; + if (payload->table_name == NULL) payload->table_name = database_value_dup(v1); + else if (dbutils_value_compare(payload->table_name, v1) != 0) return SQLITE_NOMEM; + + payload->new_values[index] = database_value_dup(v2); + payload->old_values[index] = database_value_dup(v3); + + // sanity check memory allocations before committing count + bool v1_can_be_null = (database_value_type(v1) == SQLITE_NULL); + bool v2_can_be_null = (database_value_type(v2) == SQLITE_NULL); + bool v3_can_be_null = (database_value_type(v3) == SQLITE_NULL); + + bool oom = false; + if ((payload->table_name == NULL) && (!v1_can_be_null)) oom = true; + if ((payload->new_values[index] == NULL) && (!v2_can_be_null)) oom = true; + if ((payload->old_values[index] == NULL) && (!v3_can_be_null)) oom = true; + + if (oom) { + // clean up partial allocations at this index to prevent leaks + if (payload->new_values[index]) { database_value_free(payload->new_values[index]); payload->new_values[index] = NULL; } + if (payload->old_values[index]) { database_value_free(payload->old_values[index]); payload->old_values[index] = NULL; } + return SQLITE_NOMEM; + } + + payload->count++; + return SQLITE_OK; +} + +void dbsync_update_step (sqlite3_context *context, int argc, sqlite3_value **argv) { + // argv[0] => table_name + // argv[1] => new_column_value + // argv[2] => old_column_value + + // allocate/get the update payload + cloudsync_update_payload *payload = (cloudsync_update_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_update_payload)); + if (!payload) {sqlite3_result_error_nomem(context); return;} + + if (dbsync_update_payload_append(payload, argv[0], argv[1], argv[2]) != SQLITE_OK) { + sqlite3_result_error_nomem(context); + } +} + +void dbsync_update_final (sqlite3_context *context) { + cloudsync_update_payload *payload = (cloudsync_update_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_update_payload)); + if (!payload || payload->count == 0) return; + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // lookup table + const char *table_name = (const char *)database_value_text(payload->table_name); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + dbsync_set_error(context, "Unable to retrieve table name %s in cloudsync_update.", table_name); + dbsync_update_payload_free(payload); + return; + } + + // compute the next database version for tracking changes + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + int rc = SQLITE_OK; + + // Check if the primary key(s) have changed + bool prikey_changed = false; + for (int i=0; iold_values[i], payload->new_values[i]) != 0) { + prikey_changed = true; + break; + } + } + + // encode the NEW primary key values into a buffer (used later for indexing) + char buffer[1024]; + char buffer2[1024]; + size_t pklen = sizeof(buffer); + size_t oldpklen = sizeof(buffer2); + char *oldpk = NULL; + + char *pk = pk_encode_prikey((dbvalue_t **)payload->new_values, table_count_pks(table), buffer, &pklen); + if (!pk) { + sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); + dbsync_update_payload_free(payload); + return; + } + if (pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + dbsync_set_error(context, "Update aborted because primary key in table %s contains NULL values.", table_name); + dbsync_update_payload_free(payload); + return; + } + + if (prikey_changed) { + // if the primary key has changed, we need to handle the row differently: + // 1. mark the old row (OLD primary key) as deleted + // 2. create a new row (NEW primary key) + + // encode the OLD primary key into a buffer + oldpk = pk_encode_prikey((dbvalue_t **)payload->old_values, table_count_pks(table), buffer2, &oldpklen); + if (!oldpk) { + // no check here about PRIKEY_NULL_CONSTRAINT_ERROR because by design oldpk cannot contain NULL values + if (pk != buffer) cloudsync_memory_free(pk); + sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); + dbsync_update_payload_free(payload); + return; + } + + // mark the rows with the old primary key as deleted in the metadata (old row handling) + rc = local_mark_delete_meta(table, oldpk, oldpklen, db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + + // move non-sentinel metadata entries from OLD primary key to NEW primary key + // handles the case where some metadata is retained across primary key change + // see https://github.com/sqliteai/sqlite-sync/blob/main/docs/PriKey.md for more details + rc = local_update_move_meta(table, pk, pklen, oldpk, oldpklen, db_version); + if (rc != SQLITE_OK) goto cleanup; + + // mark a new sentinel row with the new primary key in the metadata + rc = local_mark_insert_sentinel_meta(table, pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + + // free memory if the OLD primary key was dynamically allocated + if (oldpk != buffer2) cloudsync_memory_free(oldpk); + oldpk = NULL; + } + + // compare NEW and OLD values (excluding primary keys) to handle column updates + for (int i=0; iold_values[col_index], payload->new_values[col_index]) != 0) { + if (table_col_algo(table, i) == col_algo_block) { + // Block column: diff old and new text, emit per-block metadata changes + const char *new_text = (const char *)database_value_text(payload->new_values[col_index]); + const char *delim = table_col_delimiter(table, i); + const char *col = table_colname(table, i); + + // Read existing blocks from blocks table + block_list_t *old_blocks = block_list_create_empty(); + if (table_block_list_stmt(table)) { + char *like_pattern = block_build_colname(col, "%"); + if (like_pattern) { + // Query blocks table directly for existing block names and values + char *list_sql = cloudsync_memory_mprintf( + "SELECT col_name, col_value FROM %s WHERE pk = ?1 AND col_name LIKE ?2 ORDER BY col_name", + table_blocks_ref(table)); + if (list_sql) { + dbvm_t *list_vm = NULL; + if (databasevm_prepare(data, list_sql, &list_vm, 0) == DBRES_OK) { + databasevm_bind_blob(list_vm, 1, pk, (int)pklen); + databasevm_bind_text(list_vm, 2, like_pattern, -1); + while (databasevm_step(list_vm) == DBRES_ROW) { + const char *bcn = database_column_text(list_vm, 0); + const char *bval = database_column_text(list_vm, 1); + const char *pos = block_extract_position_id(bcn); + if (pos && old_blocks) { + block_list_add(old_blocks, bval ? bval : "", pos); + } + } + databasevm_finalize(list_vm); + } + cloudsync_memory_free(list_sql); + } + cloudsync_memory_free(like_pattern); + } + } + + // Split new text into parts (NULL text = all blocks removed) + block_list_t *new_blocks = new_text ? block_split(new_text, delim) : block_list_create_empty(); + if (new_blocks && old_blocks) { + // Build array of new content strings (NULL when count is 0) + const char **new_parts = NULL; + if (new_blocks->count > 0) { + new_parts = (const char **)cloudsync_memory_alloc( + (uint64_t)(new_blocks->count * sizeof(char *))); + if (new_parts) { + for (int b = 0; b < new_blocks->count; b++) { + new_parts[b] = new_blocks->entries[b].content; + } + } + } + + if (new_parts || new_blocks->count == 0) { + block_diff_t *diff = block_diff(old_blocks->entries, old_blocks->count, + new_parts, new_blocks->count); + if (diff) { + for (int d = 0; d < diff->count; d++) { + block_diff_entry_t *de = &diff->entries[d]; + char *block_cn = block_build_colname(col, de->position_id); + if (!block_cn) continue; + + if (de->type == BLOCK_DIFF_ADDED || de->type == BLOCK_DIFF_MODIFIED) { + rc = local_mark_insert_or_update_meta(table, pk, pklen, block_cn, + db_version, cloudsync_bumpseq(data)); + // Store block value + if (rc == SQLITE_OK && table_block_value_write_stmt(table)) { + dbvm_t *wvm = table_block_value_write_stmt(table); + databasevm_bind_blob(wvm, 1, pk, (int)pklen); + databasevm_bind_text(wvm, 2, block_cn, -1); + databasevm_bind_text(wvm, 3, de->content, -1); + databasevm_step(wvm); + databasevm_reset(wvm); + } + } else if (de->type == BLOCK_DIFF_REMOVED) { + // Mark block as deleted in metadata (even col_version) + rc = local_mark_delete_block_meta(table, pk, pklen, block_cn, + db_version, cloudsync_bumpseq(data)); + // Remove from blocks table + if (rc == SQLITE_OK) { + block_delete_value_external(data, table, pk, pklen, block_cn); + } + } + cloudsync_memory_free(block_cn); + if (rc != SQLITE_OK) break; + } + block_diff_free(diff); + } + if (new_parts) cloudsync_memory_free((void *)new_parts); + } + } + if (new_blocks) block_list_free(new_blocks); + if (old_blocks) block_list_free(old_blocks); + if (rc != SQLITE_OK) goto cleanup; + } else { + // Regular column: mark as updated in the metadata (columns are in cid order) + rc = local_mark_insert_or_update_meta(table, pk, pklen, table_colname(table, i), db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + } + } + } + +cleanup: + if (rc != SQLITE_OK) sqlite3_result_error(context, database_errmsg(data), -1); + if (pk != buffer) cloudsync_memory_free(pk); + if (oldpk && (oldpk != buffer2)) cloudsync_memory_free(oldpk); + + dbsync_update_payload_free(payload); +} + +// MARK: - + +void dbsync_cleanup (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_cleanup"); + + const char *table = (const char *)database_value_text(argv[0]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = cloudsync_cleanup(data, table); + if (rc != DBRES_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + } +} + +void dbsync_enable_disable (sqlite3_context *context, const char *table_name, bool value) { + DEBUG_FUNCTION("cloudsync_enable_disable"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) return; + + table_set_enabled(table, value); +} + +void dbsync_enable (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_enable"); + + const char *table = (const char *)database_value_text(argv[0]); + dbsync_enable_disable(context, table, true); +} + +void dbsync_disable (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_disable"); + + const char *table = (const char *)database_value_text(argv[0]); + dbsync_enable_disable(context, table, false); +} + +void dbsync_is_enabled (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_is_enabled"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_table_context *table = table_lookup(data, table_name); + + int result = (table && table_enabled(table)) ? 1 : 0; + sqlite3_result_int(context, result); +} + +void dbsync_terminate (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_terminate"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_terminate(data); + sqlite3_result_int(context, rc); +} + +// MARK: - + +void dbsync_init (sqlite3_context *context, const char *table, const char *algo, CLOUDSYNC_INIT_FLAG init_flags) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = database_begin_savepoint(data, "cloudsync_init"); + if (rc != SQLITE_OK) { + dbsync_set_error(context, "Unable to create cloudsync_init savepoint. %s", database_errmsg(data)); + sqlite3_result_error_code(context, rc); + return; + } + + rc = cloudsync_init_table(data, table, algo, init_flags); + if (rc == SQLITE_OK) { + rc = database_commit_savepoint(data, "cloudsync_init"); + if (rc != SQLITE_OK) { + dbsync_set_error(context, "Unable to release cloudsync_init savepoint. %s", database_errmsg(data)); + sqlite3_result_error_code(context, rc); + } + } else { + // in case of error, rollback transaction + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + database_rollback_savepoint(data, "cloudsync_init"); + return; + } + + cloudsync_update_schema_hash(data); + + // returns site_id as TEXT + char buffer[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_stringify(cloudsync_siteid(data), buffer, false); + sqlite3_result_text(context, buffer, -1, SQLITE_TRANSIENT); +} + +void dbsync_init3 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_init2"); + + const char *table = (const char *)database_value_text(argv[0]); + const char *algo = (const char *)database_value_text(argv[1]); + int init_flags = database_value_int(argv[2]); + dbsync_init(context, table, algo, init_flags); +} + +void dbsync_init2 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_init2"); + + const char *table = (const char *)database_value_text(argv[0]); + const char *algo = (const char *)database_value_text(argv[1]); + dbsync_init(context, table, algo, CLOUDSYNC_INIT_FLAG_NONE); +} + +void dbsync_init1 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_init1"); + + const char *table = (const char *)database_value_text(argv[0]); + dbsync_init(context, table, NULL, CLOUDSYNC_INIT_FLAG_NONE); +} + +// MARK: - + +void dbsync_begin_alter (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_begin_alter"); + + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = database_begin_savepoint(data, "cloudsync_alter"); + if (rc != DBRES_OK) { + sqlite3_result_error(context, "Unable to create cloudsync_alter savepoint", -1); + sqlite3_result_error_code(context, rc); + return; + } + + rc = cloudsync_begin_alter(data, table_name); + if (rc != DBRES_OK) { + database_rollback_savepoint(data, "cloudsync_alter"); + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + } +} + +void dbsync_commit_alter (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_commit_alter"); + + //retrieve table argument + const char *table_name = (const char *)database_value_text(argv[0]); + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = cloudsync_commit_alter(data, table_name); + if (rc != DBRES_OK) { + database_rollback_savepoint(data, "cloudsync_alter"); + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + + rc = database_commit_savepoint(data, "cloudsync_alter"); + if (rc != DBRES_OK) { + sqlite3_result_error(context, database_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + + cloudsync_update_schema_hash(data); +} + +// MARK: - Payload - + +void dbsync_payload_encode_step (sqlite3_context *context, int argc, sqlite3_value **argv) { + // allocate/get the session context + cloudsync_payload_context *payload = (cloudsync_payload_context *)sqlite3_aggregate_context(context, (int)cloudsync_payload_context_size(NULL)); + if (!payload) { + sqlite3_result_error(context, "Not enough memory to allocate payload session context", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + return; + } + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = cloudsync_payload_encode_step(payload, data, argc, (dbvalue_t **)argv); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + } +} + +void dbsync_payload_encode_final (sqlite3_context *context) { + // get the session context + cloudsync_payload_context *payload = (cloudsync_payload_context *)sqlite3_aggregate_context(context, (int)cloudsync_payload_context_size(NULL)); + if (!payload) { + sqlite3_result_error(context, "Unable to extract payload session context", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + return; + } + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = cloudsync_payload_encode_final(payload, data); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + + // result is OK so get BLOB and returns it + int64_t blob_size = 0; + char *blob = cloudsync_payload_blob (payload, &blob_size, NULL); + if (!blob) { + sqlite3_result_null(context); + } else { + sqlite3_result_blob64(context, blob, blob_size, SQLITE_TRANSIENT); + cloudsync_memory_free(blob); + } + + // from: https://sqlite.org/c3ref/aggregate_context.html + // SQLite automatically frees the memory allocated by sqlite3_aggregate_context() when the aggregate query concludes. +} + +void dbsync_payload_decode (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_payload_decode"); + //debug_values(argc, argv); + + // sanity check payload type + if (database_value_type(argv[0]) != SQLITE_BLOB) { + sqlite3_result_error(context, "Error on cloudsync_payload_decode: value must be a BLOB.", -1); + sqlite3_result_error_code(context, SQLITE_MISUSE); + return; + } + + // sanity check payload size + int blen = database_value_bytes(argv[0]); + size_t header_size = 0; + cloudsync_payload_context_size(&header_size); + if (blen < (int)header_size) { + sqlite3_result_error(context, "Error on cloudsync_payload_decode: invalid input size.", -1); + sqlite3_result_error_code(context, SQLITE_MISUSE); + return; + } + + // obtain payload + const char *payload = (const char *)database_value_blob(argv[0]); + + // apply changes + // The public SQL function applies a single complete payload: advance the + // receive cursor to its last applied (db_version, seq) (legacy behavior, safe + // for a payload that ends on a db_version boundary). The chunked-download + // receive path gates cursor advancement on stream completion via the C-level + // checkpoint argument instead (see cloudsync_payload_apply in cloudsync.h). + int nrows = 0; + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_payload_apply(data, payload, blen, &nrows, CLOUDSYNC_CHECKPOINT_LAST_APPLIED, 0); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + + // returns number of applied rows + sqlite3_result_int(context, nrows); +} + +typedef struct { + sqlite3_vtab base; + sqlite3 *db; + cloudsync_context *data; +} cloudsync_payload_chunks_vtab; + +typedef struct { + sqlite3_vtab_cursor base; + cloudsync_payload_chunks_vtab *vtab; + sqlite3_stmt *src; + bool eof; + bool has_row; + int chunk_index; + char *payload; + int64_t payload_size; + int64_t rows; + int64_t dbv_min; + int64_t dbv_max; + int64_t watermark; + bool frag_active; + int frag_part; + int frag_count; + int frag_target; + int64_t frag_offset; + int64_t frag_total; + uint64_t frag_checksum; + char value_header[16]; + int value_header_len; + const char *value_data; + int64_t value_data_len; + // Positional-cursor outputs: the resume point AFTER the chunk currently held. + // These live in the per-scan reset region (after eof) so xFilter's bulk memset + // clears them. next_* is the (db_version, seq, frag_offset) a follow-up call + // passes back as resume_* to continue exactly where this chunk stopped. + int64_t next_dbv; + int64_t next_seq; + int64_t next_frag_offset; + bool is_final; +} cloudsync_payload_chunks_cursor; + +static int payload_chunks_connect(sqlite3 *db, void *aux, int argc, const char *const *argv, sqlite3_vtab **vtab, char **err) { + UNUSED_PARAMETER(argc); UNUSED_PARAMETER(argv); UNUSED_PARAMETER(err); + int rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(payload BLOB, chunk_index INTEGER, payload_size INTEGER, rows INTEGER, " + "db_version_min INTEGER, db_version_max INTEGER, watermark_db_version INTEGER, " + "since_db_version HIDDEN, site_id HIDDEN, until_db_version HIDDEN, exclude_filter_site_id HIDDEN, " + // Positional-cursor outputs (cols 11..14): the resume point after the + // emitted chunk, plus a final-chunk flag. A stateless /check passes these + // back as the resume_* inputs (cols 15..17) to continue the drain without + // a spool table — O(1) seek per chunk instead of replaying from since. + "next_db_version INTEGER, next_seq INTEGER, next_frag_offset INTEGER, is_final INTEGER, " + "resume_db_version HIDDEN, resume_seq HIDDEN, resume_frag_offset HIDDEN)"); + if (rc != SQLITE_OK) return rc; + cloudsync_payload_chunks_vtab *p = sqlite3_malloc64(sizeof(*p)); + if (!p) return SQLITE_NOMEM; + memset(p, 0, sizeof(*p)); + p->db = db; + p->data = (cloudsync_context *)aux; + *vtab = (sqlite3_vtab *)p; + return SQLITE_OK; +} + +static int payload_chunks_disconnect(sqlite3_vtab *vtab) { + sqlite3_free(vtab); + return SQLITE_OK; +} + +static int payload_chunks_open(sqlite3_vtab *vtab, sqlite3_vtab_cursor **cursor) { + cloudsync_payload_chunks_cursor *c = cloudsync_memory_zeroalloc(sizeof(*c)); + if (!c) return SQLITE_NOMEM; + c->vtab = (cloudsync_payload_chunks_vtab *)vtab; + *cursor = (sqlite3_vtab_cursor *)c; + return SQLITE_OK; +} + +static int payload_chunks_close(sqlite3_vtab_cursor *cursor) { + cloudsync_payload_chunks_cursor *c = (cloudsync_payload_chunks_cursor *)cursor; + if (c->src) sqlite3_finalize(c->src); + if (c->payload) cloudsync_memory_free(c->payload); + cloudsync_memory_free(c); + return SQLITE_OK; +} + +static int payload_chunks_best_index(sqlite3_vtab *vtab, sqlite3_index_info *idxinfo) { + UNUSED_PARAMETER(vtab); + // Assign argvIndex in a canonical hidden-column order so xFilter can read argv + // in a fixed order regardless of how SQLite presents constraints. idxNum bit k + // is set when handled_cols[k] is bound; xFilter reads argv in this same order. + // bit0=since_db_version(7) bit1=site_id(8) bit2=until_db_version(9) + // bit3=exclude_filter_site_id(10) bit4=resume_db_version(15) + // bit5=resume_seq(16) bit6=resume_frag_offset(17) + static const int handled_cols[] = {7, 8, 9, 10, 15, 16, 17}; + int argv_index = 1; + int idxnum = 0; + for (size_t k = 0; k < sizeof(handled_cols) / sizeof(handled_cols[0]); ++k) { + int col = handled_cols[k]; + for (int i = 0; i < idxinfo->nConstraint; ++i) { + struct sqlite3_index_constraint *cn = &idxinfo->aConstraint[i]; + if (!cn->usable || cn->op != SQLITE_INDEX_CONSTRAINT_EQ || cn->iColumn != col) continue; + idxinfo->aConstraintUsage[i].argvIndex = argv_index++; + idxinfo->aConstraintUsage[i].omit = 1; + idxnum |= (1 << k); + break; // at most one constraint consumed per hidden column + } + } + idxinfo->idxNum = idxnum; + idxinfo->estimatedCost = 10.0; + idxinfo->estimatedRows = 10; + return SQLITE_OK; +} + +static int payload_chunks_step_source(cloudsync_payload_chunks_cursor *c) { + int rc = sqlite3_step(c->src); + if (rc == SQLITE_ROW) { c->has_row = true; return SQLITE_OK; } + c->has_row = false; + if (rc == SQLITE_DONE) return SQLITE_OK; + // copy the inner statement's message onto this vtab or SQLite surfaces the + // error as a bare "SQL logic error" + if (c->vtab->base.zErrMsg) sqlite3_free(c->vtab->base.zErrMsg); + c->vtab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(c->vtab->db)); + return rc; +} + +static int payload_chunks_plan_fragment(cloudsync_payload_chunks_cursor *c) { + cloudsync_context *data = c->vtab->data; + int target = cloudsync_payload_fragment_data_size(data, + (const char *)sqlite3_column_text(c->src, 0), sqlite3_column_bytes(c->src, 0), + sqlite3_column_blob(c->src, 1), sqlite3_column_bytes(c->src, 1), + (const char *)sqlite3_column_text(c->src, 2), sqlite3_column_bytes(c->src, 2), + sqlite3_column_int64(c->src, 4), sqlite3_column_int64(c->src, 5), + sqlite3_column_blob(c->src, 6), sqlite3_column_bytes(c->src, 6), + sqlite3_column_int64(c->src, 7), sqlite3_column_int64(c->src, 8), + c->frag_total, 0, 1); + if (target <= 0) return SQLITE_TOOBIG; + + int count = 0; + for (int i = 0; i < CLOUDSYNC_PAYLOAD_FRAGMENT_SIZE_FIXPOINT_ITERATIONS; ++i) { + count = cloudsync_payload_fragment_count(c->frag_total, target); + if (count <= 0) return SQLITE_TOOBIG; + int planned = cloudsync_payload_fragment_data_size(data, + (const char *)sqlite3_column_text(c->src, 0), sqlite3_column_bytes(c->src, 0), + sqlite3_column_blob(c->src, 1), sqlite3_column_bytes(c->src, 1), + (const char *)sqlite3_column_text(c->src, 2), sqlite3_column_bytes(c->src, 2), + sqlite3_column_int64(c->src, 4), sqlite3_column_int64(c->src, 5), + sqlite3_column_blob(c->src, 6), sqlite3_column_bytes(c->src, 6), + sqlite3_column_int64(c->src, 7), sqlite3_column_int64(c->src, 8), + c->frag_total, count - 1, count); + if (planned <= 0) return SQLITE_TOOBIG; + if (planned == target) break; + target = planned; + } + + c->frag_target = target; + c->frag_count = cloudsync_payload_fragment_count(c->frag_total, target); + if (c->frag_count <= 0) return SQLITE_TOOBIG; + return SQLITE_OK; +} + +// Set up fragment state for the current source row (a single value larger than +// max_chunk_size) so emit_fragment can stream it. start_offset is the byte offset +// within the encoded value to resume from (0 when first reaching the value; +// >0 when a positional cursor resumes mid-value). frag_part is derived from the +// offset so the fragment's part index is consistent whether reached by streaming +// or by a seek. The plan (frag_target/frag_count) is a deterministic function of +// the row, so a resumed fragment tiles identically to a streamed one. +static int payload_chunks_begin_fragment(cloudsync_payload_chunks_cursor *c, int64_t start_offset) { + dbvalue_t *col_value = (dbvalue_t *)sqlite3_column_value(c->src, 3); + int type = database_value_type(col_value); + if (type != DBTYPE_TEXT && type != DBTYPE_BLOB) return SQLITE_TOOBIG; + int64_t raw_len = 0; + int header_len = cloudsync_payload_encoded_value_header(col_value, c->value_header, sizeof(c->value_header), &raw_len); + if (header_len <= 0) return SQLITE_ERROR; + c->value_header_len = header_len; + c->value_data = (const char *)database_value_blob(col_value); + c->value_data_len = raw_len; + c->frag_total = header_len + raw_len; + c->frag_offset = start_offset; + int rc = payload_chunks_plan_fragment(c); + if (rc != SQLITE_OK) return rc; + c->frag_part = (c->frag_target > 0) ? (int)(start_offset / c->frag_target) : 0; + c->frag_checksum = cloudsync_payload_encoded_value_checksum(col_value); + c->frag_active = true; + return SQLITE_OK; +} + +static int payload_chunks_emit_fragment(cloudsync_payload_chunks_cursor *c) { + cloudsync_context *data = c->vtab->data; + if (c->payload) { cloudsync_memory_free(c->payload); c->payload = NULL; } + int64_t remaining = c->frag_total - c->frag_offset; + int frag_len = remaining > c->frag_target ? c->frag_target : (int)remaining; + if (frag_len <= 0) return SQLITE_CORRUPT; + char *frag = cloudsync_memory_alloc((uint64_t)frag_len); + if (!frag) return SQLITE_NOMEM; + int copied = 0; + int64_t off = c->frag_offset; + if (off < c->value_header_len) { + int n = c->value_header_len - (int)off; + if (n > frag_len) n = frag_len; + memcpy(frag, c->value_header + off, (size_t)n); + copied += n; + off += n; + } + if (copied < frag_len) { + int64_t data_off = off - c->value_header_len; + memcpy(frag + copied, c->value_data + data_off, (size_t)(frag_len - copied)); + } + + cloudsync_payload_context *payload = cloudsync_memory_zeroalloc((uint64_t)cloudsync_payload_context_size(NULL)); + if (!payload) { cloudsync_memory_free(frag); return SQLITE_NOMEM; } + int rc = cloudsync_payload_encode_fragment_step(payload, data, + (const char *)sqlite3_column_text(c->src, 0), sqlite3_column_bytes(c->src, 0), + sqlite3_column_blob(c->src, 1), sqlite3_column_bytes(c->src, 1), + (const char *)sqlite3_column_text(c->src, 2), sqlite3_column_bytes(c->src, 2), + frag, frag_len, + sqlite3_column_int64(c->src, 4), sqlite3_column_int64(c->src, 5), + sqlite3_column_blob(c->src, 6), sqlite3_column_bytes(c->src, 6), + sqlite3_column_int64(c->src, 7), sqlite3_column_int64(c->src, 8), + c->frag_checksum, c->frag_total, c->frag_part, c->frag_count); + cloudsync_memory_free(frag); + if (rc != SQLITE_OK) { cloudsync_memory_free(payload); return rc; } + rc = cloudsync_payload_encode_final(payload, data); + if (rc != SQLITE_OK) { cloudsync_memory_free(payload); return rc; } + c->payload = cloudsync_payload_blob(payload, &c->payload_size, &c->rows); + cloudsync_memory_free(payload); + c->dbv_min = sqlite3_column_int64(c->src, 5); + c->dbv_max = c->dbv_min; + c->chunk_index++; + c->frag_offset += frag_len; + c->frag_part++; + if (c->frag_part >= c->frag_count) { + c->frag_active = false; + rc = payload_chunks_step_source(c); + } + return rc; +} + +static int payload_chunks_build_next(cloudsync_payload_chunks_cursor *c) { + cloudsync_context *data = c->vtab->data; + int rc = SQLITE_OK; + if (c->payload) { cloudsync_memory_free(c->payload); c->payload = NULL; } + c->payload_size = c->rows = c->dbv_min = c->dbv_max = 0; + if (c->frag_active) return payload_chunks_emit_fragment(c); + if (!c->has_row) { c->eof = true; return SQLITE_OK; } + + int max_size = cloudsync_payload_max_chunk_size(data); + size_t payload_header_size = 0; + cloudsync_payload_context_size(&payload_header_size); + cloudsync_payload_context *payload = cloudsync_memory_zeroalloc((uint64_t)cloudsync_payload_context_size(NULL)); + if (!payload) return SQLITE_NOMEM; + while (c->has_row) { + sqlite3_value *rowv[9]; + for (int i = 0; i < 9; ++i) rowv[i] = sqlite3_column_value(c->src, i); + size_t row_size = pk_encode_size((dbvalue_t **)rowv, 9, 0, -1); + if (row_size == SIZE_MAX) { cloudsync_memory_free(payload); return SQLITE_NOMEM; } + + if ((int64_t)row_size + (int64_t)payload_header_size + CLOUDSYNC_PAYLOAD_CHUNK_SAFETY_MARGIN > max_size) { + if (cloudsync_payload_context_nrows(payload) > 0) break; + cloudsync_memory_free(payload); + rc = payload_chunks_begin_fragment(c, 0); + if (rc != SQLITE_OK) return rc; + return payload_chunks_emit_fragment(c); + } + + if (cloudsync_payload_context_nrows(payload) > 0 && cloudsync_payload_context_bused(payload) + row_size > (size_t)max_size) break; + rc = cloudsync_payload_encode_step(payload, data, 9, (dbvalue_t **)rowv); + if (rc != SQLITE_OK) { cloudsync_memory_free(payload); return rc; } + int64_t dbv = sqlite3_column_int64(c->src, 5); + if (cloudsync_payload_context_nrows(payload) == 1) c->dbv_min = dbv; + c->dbv_max = dbv; + rc = payload_chunks_step_source(c); + if (rc != SQLITE_OK) { cloudsync_memory_free(payload); return rc; } + } + + if (cloudsync_payload_context_nrows(payload) == 0) { cloudsync_memory_free(payload); c->eof = true; return SQLITE_OK; } + rc = cloudsync_payload_encode_final(payload, data); + if (rc != SQLITE_OK) { cloudsync_memory_free(payload); return rc; } + c->payload = cloudsync_payload_blob(payload, &c->payload_size, &c->rows); + cloudsync_memory_free(payload); + c->chunk_index++; + return SQLITE_OK; +} + +// Record the resume point a stateless caller passes back to continue after the +// chunk just built. Reads the source statement, which is positioned at the next +// unconsumed row (or the same row when a value is still mid-fragment). Must be +// called only after build_next produced a chunk (i.e. !eof). +static void payload_chunks_set_next_cursor(cloudsync_payload_chunks_cursor *c) { + if (c->frag_active) { + // Mid-value: resume the same row at the next byte offset. + c->next_dbv = sqlite3_column_int64(c->src, 5); + c->next_seq = sqlite3_column_int64(c->src, 8); + c->next_frag_offset = c->frag_offset; + c->is_final = false; + } else if (c->has_row) { + // Row boundary: the next chunk starts at the current (unconsumed) row. + c->next_dbv = sqlite3_column_int64(c->src, 5); + c->next_seq = sqlite3_column_int64(c->src, 8); + c->next_frag_offset = 0; + c->is_final = false; + } else { + // Stream exhausted: this was the last chunk of the window. + c->next_dbv = c->watermark; + c->next_seq = 0; + c->next_frag_offset = 0; + c->is_final = true; + } +} + +static int payload_chunks_advance(cloudsync_payload_chunks_cursor *c) { + int rc = payload_chunks_build_next(c); + if (rc == SQLITE_OK && !c->eof) payload_chunks_set_next_cursor(c); + return rc; +} + +static int payload_chunks_filter(sqlite3_vtab_cursor *cursor, int idxnum, const char *idxstr, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(idxstr); UNUSED_PARAMETER(argc); + cloudsync_payload_chunks_cursor *c = (cloudsync_payload_chunks_cursor *)cursor; + cloudsync_context *data = c->vtab->data; + if (!cloudsync_context_is_initialized(data)) { + c->vtab->base.zErrMsg = sqlite3_mprintf( + "cloudsync is not initialized: call SELECT cloudsync_init('') " + "to enable sync on a table before querying cloudsync_payload_chunks."); + return SQLITE_ERROR; + } + if (c->src) { sqlite3_finalize(c->src); c->src = NULL; } + if (c->payload) { cloudsync_memory_free(c->payload); c->payload = NULL; } + // Contract: all per-scan state that can be bulk-reset here must live at or + // after eof. Fields before eof are cursor lifetime state preserved across + // xFilter calls. + memset(&c->eof, 0, sizeof(*c) - offsetof(cloudsync_payload_chunks_cursor, eof)); + + int argi = 0; + int64_t since = dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_SEND_DBVERSION); + const void *site_id = NULL; + int site_id_len = 0; + bool site_id_given = false; + int64_t until = 0; + bool exclude = false; + // Positional resume cursor (cols 15..17): when resume_db_version is bound the + // scan starts at (resume_db_version, resume_seq) inclusive and the first chunk + // resumes a mid-value fragment at resume_frag_offset, instead of replaying the + // whole window from `since`. Lets a stateless /check page the stream with an + // O(1) seek per call and no spool table. + bool positional = false; + int64_t resume_dbv = 0, resume_seq = 0, resume_frag = 0; + if (idxnum & 1) since = sqlite3_value_int64(argv[argi++]); + if (idxnum & 2) { + if (sqlite3_value_type(argv[argi]) != SQLITE_NULL) { + site_id = sqlite3_value_blob(argv[argi]); + site_id_len = sqlite3_value_bytes(argv[argi]); + site_id_given = true; + } + argi++; + } + if (idxnum & 4) until = sqlite3_value_int64(argv[argi++]); + if (idxnum & 8) exclude = (sqlite3_value_int(argv[argi++]) != 0); + if (idxnum & 16) { resume_dbv = sqlite3_value_int64(argv[argi++]); positional = true; } + if (idxnum & 32) resume_seq = sqlite3_value_int64(argv[argi++]); + if (idxnum & 64) resume_frag = sqlite3_value_int64(argv[argi++]); + + // Resolve the site filter: + // exclude=true -> all sites except filter_site_id (CHECK path); site required + // filter given -> only that site + // default -> local site (send path, unchanged) + const char *site_op; + if (exclude) { + if (!site_id_given) { + c->vtab->base.zErrMsg = sqlite3_mprintf( + "cloudsync_payload_chunks: exclude_filter_site_id requires a non-NULL site_id"); + return SQLITE_ERROR; + } + site_op = "<>"; + } else { + site_op = "="; + if (!site_id_given) { site_id = cloudsync_siteid(data); site_id_len = UUID_LEN; } + } + + if (until == 0) { + char *mxsql = sqlite3_mprintf( + "SELECT COALESCE(MAX(db_version),0) FROM cloudsync_changes WHERE site_id%s?", site_op); + if (!mxsql) return SQLITE_NOMEM; + sqlite3_stmt *mx = NULL; + int rc = sqlite3_prepare_v2(c->vtab->db, mxsql, -1, &mx, NULL); + sqlite3_free(mxsql); + if (rc != SQLITE_OK) return rc; + sqlite3_bind_blob(mx, 1, site_id, site_id_len, SQLITE_TRANSIENT); + // MAX() yields exactly one row, so anything else is an error: propagate + // it instead of silently scanning an empty window with until=0 + rc = sqlite3_step(mx); + if (rc == SQLITE_ROW) until = sqlite3_column_int64(mx, 0); + sqlite3_finalize(mx); + if (rc != SQLITE_ROW) { + if (c->vtab->base.zErrMsg) sqlite3_free(c->vtab->base.zErrMsg); + c->vtab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(c->vtab->db)); + return rc == SQLITE_DONE ? SQLITE_ERROR : rc; + } + } + c->watermark = until; + + // Window upper bound is always `until`. The lower bound is either the legacy + // exclusive `since` (db_version > since) or the inclusive positional cursor + // (db_version, seq) >= (resume_dbv, resume_seq). + char *sql; + if (positional) { + sql = sqlite3_mprintf( + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes WHERE db_version<=? AND site_id%s? AND " + "(db_version>? OR (db_version=? AND seq>=?)) ORDER BY db_version, seq ASC", + site_op); + } else { + sql = sqlite3_mprintf( + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes WHERE db_version>? AND site_id%s? AND db_version<=? ORDER BY db_version, seq ASC", + site_op); + } + if (!sql) return SQLITE_NOMEM; + int rc = sqlite3_prepare_v2(c->vtab->db, sql, -1, &c->src, NULL); + sqlite3_free(sql); + if (rc != SQLITE_OK) return rc; + if (positional) { + sqlite3_bind_int64(c->src, 1, until); + sqlite3_bind_blob(c->src, 2, site_id, site_id_len, SQLITE_TRANSIENT); + sqlite3_bind_int64(c->src, 3, resume_dbv); + sqlite3_bind_int64(c->src, 4, resume_dbv); + sqlite3_bind_int64(c->src, 5, resume_seq); + } else { + sqlite3_bind_int64(c->src, 1, since); + sqlite3_bind_blob(c->src, 2, site_id, site_id_len, SQLITE_TRANSIENT); + sqlite3_bind_int64(c->src, 3, until); + } + rc = payload_chunks_step_source(c); + if (rc != SQLITE_OK) return rc; + // Resuming inside a value that was fragmented across chunks: the first row is + // that value; re-establish the fragment plan and skip to resume_frag. + if (positional && resume_frag > 0 && c->has_row) { + rc = payload_chunks_begin_fragment(c, resume_frag); + if (rc != SQLITE_OK) return rc; + } + return payload_chunks_advance(c); +} + +static int payload_chunks_next(sqlite3_vtab_cursor *cursor) { + return payload_chunks_advance((cloudsync_payload_chunks_cursor *)cursor); +} + +static int payload_chunks_eof(sqlite3_vtab_cursor *cursor) { + return ((cloudsync_payload_chunks_cursor *)cursor)->eof; +} + +static int payload_chunks_column(sqlite3_vtab_cursor *cursor, sqlite3_context *ctx, int col) { + cloudsync_payload_chunks_cursor *c = (cloudsync_payload_chunks_cursor *)cursor; + switch (col) { + case 0: sqlite3_result_blob64(ctx, c->payload, (sqlite3_uint64)c->payload_size, SQLITE_TRANSIENT); break; + case 1: sqlite3_result_int(ctx, c->chunk_index - 1); break; + case 2: sqlite3_result_int64(ctx, c->payload_size); break; + case 3: sqlite3_result_int64(ctx, c->rows); break; + case 4: sqlite3_result_int64(ctx, c->dbv_min); break; + case 5: sqlite3_result_int64(ctx, c->dbv_max); break; + case 6: sqlite3_result_int64(ctx, c->watermark); break; + case 11: sqlite3_result_int64(ctx, c->next_dbv); break; + case 12: sqlite3_result_int64(ctx, c->next_seq); break; + case 13: sqlite3_result_int64(ctx, c->next_frag_offset); break; + case 14: sqlite3_result_int(ctx, c->is_final ? 1 : 0); break; + default: sqlite3_result_null(ctx); break; + } + return SQLITE_OK; +} + +static int payload_chunks_rowid(sqlite3_vtab_cursor *cursor, sqlite3_int64 *rowid) { + *rowid = ((cloudsync_payload_chunks_cursor *)cursor)->chunk_index; + return SQLITE_OK; +} + +static sqlite3_module cloudsync_payload_chunks_module = { + /* iVersion */ 0, + /* xCreate */ NULL, + /* xConnect */ payload_chunks_connect, + /* xBestIndex */ payload_chunks_best_index, + /* xDisconnect */ payload_chunks_disconnect, + /* xDestroy */ NULL, + /* xOpen */ payload_chunks_open, + /* xClose */ payload_chunks_close, + /* xFilter */ payload_chunks_filter, + /* xNext */ payload_chunks_next, + /* xEof */ payload_chunks_eof, + /* xColumn */ payload_chunks_column, + /* xRowid */ payload_chunks_rowid, + /* xUpdate */ NULL, + /* xBegin */ NULL, + /* xSync */ NULL, + /* xCommit */ NULL, + /* xRollback */ NULL, + /* xFindMethod */ NULL, + /* xRename */ NULL, + /* xSavepoint */ NULL, + /* xRelease */ NULL, + /* xRollbackTo */ NULL, + /* xShadowName */ NULL, + /* xIntegrity */ NULL +}; + +static int payload_estimated_size_add(sqlite3_int64 *acc, sqlite3_int64 value) { + if (value < 0 || *acc > INT64_MAX - value) return SQLITE_TOOBIG; + *acc += value; + return SQLITE_OK; +} + +static int payload_blob_checked_estimate(sqlite3 *db, const void *site_id, int site_id_len, + sqlite3_int64 since, sqlite3_int64 since_seq, + bool exclude, sqlite3_int64 *estimated, + sqlite3_int64 *watermark) { + sqlite3_stmt *stmt = NULL; + sqlite3_stmt *mx = NULL; + int rc = SQLITE_OK; + sqlite3_int64 until = 0; + size_t header_size = 0; + bool has_rows = false; + const char *site_op = exclude ? "<>" : "="; + *estimated = 0; + + char *mxsql = sqlite3_mprintf( + "SELECT COALESCE(MAX(db_version),0) FROM cloudsync_changes WHERE site_id%s?", site_op); + if (!mxsql) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(db, mxsql, -1, &mx, NULL); + sqlite3_free(mxsql); + if (rc != SQLITE_OK) goto error; + sqlite3_bind_blob(mx, 1, site_id, site_id_len, SQLITE_TRANSIENT); + rc = sqlite3_step(mx); + if (rc == SQLITE_ROW) until = sqlite3_column_int64(mx, 0); + else if (rc != SQLITE_DONE) goto error; + sqlite3_finalize(mx); + mx = NULL; + if (watermark) *watermark = until; + + if (until < since) return SQLITE_OK; + + char *sql = sqlite3_mprintf( + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes WHERE (db_version>? OR (db_version=? AND seq>?)) " + "AND site_id%s? AND db_version<=? ORDER BY db_version, seq ASC", + site_op); + if (!sql) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + sqlite3_free(sql); + if (rc != SQLITE_OK) goto error; + sqlite3_bind_int64(stmt, 1, since); + sqlite3_bind_int64(stmt, 2, since); + sqlite3_bind_int64(stmt, 3, since_seq); + sqlite3_bind_blob(stmt, 4, site_id, site_id_len, SQLITE_TRANSIENT); + sqlite3_bind_int64(stmt, 5, until); + + cloudsync_payload_context_size(&header_size); + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + sqlite3_value *rowv[9]; + for (int i = 0; i < 9; ++i) rowv[i] = sqlite3_column_value(stmt, i); + size_t row_size = pk_encode_size((dbvalue_t **)rowv, 9, 0, -1); + if (row_size == SIZE_MAX || row_size > (size_t)INT64_MAX) { + rc = SQLITE_TOOBIG; + goto error; + } + if (!has_rows) { + if (header_size > (size_t)INT64_MAX) { + rc = SQLITE_TOOBIG; + goto error; + } + rc = payload_estimated_size_add(estimated, (sqlite3_int64)header_size); + if (rc != SQLITE_OK) goto error; + has_rows = true; + } + rc = payload_estimated_size_add(estimated, (sqlite3_int64)row_size); + if (rc != SQLITE_OK) goto error; + } + if (rc != SQLITE_DONE) goto error; + sqlite3_finalize(stmt); + return SQLITE_OK; + +error: + if (stmt) sqlite3_finalize(stmt); + if (mx) sqlite3_finalize(mx); + return rc; +} + +void dbsync_payload_blob_checked(sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_payload_blob_checked"); + UNUSED_PARAMETER(argc); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_stmt *stmt = NULL; + cloudsync_payload_context *payload = NULL; + int rc = SQLITE_OK; + sqlite3_int64 since = 0; + sqlite3_int64 since_seq = 0; + sqlite3_int64 max_estimated_size = 0; + sqlite3_int64 estimated = 0; + sqlite3_int64 watermark = 0; + const void *site_id = NULL; + int site_id_len = 0; + bool exclude = false; + + if (sqlite3_value_type(argv[0]) != SQLITE_NULL) since = sqlite3_value_int64(argv[0]); + if (sqlite3_value_type(argv[1]) != SQLITE_NULL) since_seq = sqlite3_value_int64(argv[1]); + if (sqlite3_value_type(argv[2]) != SQLITE_NULL) { + site_id = sqlite3_value_blob(argv[2]); + site_id_len = sqlite3_value_bytes(argv[2]); + } + exclude = sqlite3_value_type(argv[3]) != SQLITE_NULL && sqlite3_value_int(argv[3]) != 0; + if (sqlite3_value_type(argv[4]) == SQLITE_NULL || sqlite3_value_int64(argv[4]) <= 0) { + sqlite3_result_error(context, "cloudsync_payload_blob_checked: max_estimated_payload_size must be positive", -1); + return; + } + max_estimated_size = sqlite3_value_int64(argv[4]); + + if (exclude && !site_id) { + sqlite3_result_error(context, + "cloudsync_payload_blob_checked: exclude_filter_site_id requires a non-NULL site_id", -1); + return; + } + if (!exclude && !site_id) { + site_id = cloudsync_siteid(data); + site_id_len = UUID_LEN; + } + + rc = payload_blob_checked_estimate(db, site_id, site_id_len, since, since_seq, exclude, &estimated, &watermark); + if (rc != SQLITE_OK) goto error; + if (estimated == 0) { + sqlite3_result_null(context); + return; + } + if (estimated > max_estimated_size) { + dbsync_set_error(context, + CLOUDSYNC_ERRCODE_PAYLOAD_TOO_LARGE "cloudsync_payload_blob_checked: estimated payload size %" PRId64 " exceeds max_estimated_payload_size %" PRId64, + (int64_t)estimated, (int64_t)max_estimated_size); + return; + } + + const char *site_op = exclude ? "<>" : "="; + char *sql = sqlite3_mprintf( + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM cloudsync_changes WHERE (db_version>? OR (db_version=? AND seq>?)) " + "AND site_id%s? AND db_version<=? ORDER BY db_version, seq ASC", + site_op); + if (!sql) { rc = SQLITE_NOMEM; goto error; } + rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + sqlite3_free(sql); + if (rc != SQLITE_OK) goto error; + sqlite3_bind_int64(stmt, 1, since); + sqlite3_bind_int64(stmt, 2, since); + sqlite3_bind_int64(stmt, 3, since_seq); + sqlite3_bind_blob(stmt, 4, site_id, site_id_len, SQLITE_TRANSIENT); + sqlite3_bind_int64(stmt, 5, watermark); + + payload = cloudsync_memory_zeroalloc((uint64_t)cloudsync_payload_context_size(NULL)); + if (!payload) { rc = SQLITE_NOMEM; goto error; } + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + sqlite3_value *rowv[9]; + for (int i = 0; i < 9; ++i) rowv[i] = sqlite3_column_value(stmt, i); + rc = cloudsync_payload_encode_step(payload, data, 9, (dbvalue_t **)rowv); + if (rc != SQLITE_OK) goto error; + } + if (rc != SQLITE_DONE) goto error; + sqlite3_finalize(stmt); + stmt = NULL; + + rc = cloudsync_payload_encode_final(payload, data); + if (rc != SQLITE_OK) goto error; + int64_t blob_size = 0; + char *blob = cloudsync_payload_blob(payload, &blob_size, NULL); + if (!blob) { + sqlite3_result_null(context); + cloudsync_payload_context_free(payload); + } else { + sqlite3_result_blob64(context, blob, (sqlite3_uint64)blob_size, cloudsync_memory_free); + cloudsync_memory_free(payload); + } + return; + +error: + if (stmt) sqlite3_finalize(stmt); + if (payload) cloudsync_payload_context_free(payload); + if (rc == SQLITE_NOMEM) sqlite3_result_error_nomem(context); + else if (rc == SQLITE_TOOBIG) sqlite3_result_error(context, CLOUDSYNC_ERRCODE_PAYLOAD_TOO_LARGE "cloudsync_payload_blob_checked: payload estimate is too large", -1); + else sqlite3_result_error(context, sqlite3_errmsg(db), -1); +} + +#ifdef CLOUDSYNC_DESKTOP_OS +void dbsync_payload_save (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_payload_save"); + + // sanity check argument + if (database_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(context, "Unable to retrieve file path.", -1); + return; + } + + // retrieve full path to file + const char *payload_path = (const char *)database_value_text(argv[0]); + + // retrieve global context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int blob_size = 0; + int rc = cloudsync_payload_save(data, payload_path, &blob_size); + if (rc == SQLITE_OK) { + // if OK then returns blob size + sqlite3_result_int64(context, (sqlite3_int64)blob_size); + return; + } + + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); +} + +void dbsync_payload_load (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_payload_load"); + + // sanity check argument + if (database_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(context, "Unable to retrieve file path.", -1); + return; + } + + // retrieve full path to file + const char *path = (const char *)database_value_text(argv[0]); + + int64_t payload_size = 0; + char *payload = cloudsync_file_read(path, &payload_size); + if (!payload) { + if (payload_size < 0) { + sqlite3_result_error(context, "Unable to read payload from file path.", -1); + sqlite3_result_error_code(context, SQLITE_IOERR); + return; + } + // no rows affected but no error either + sqlite3_result_int(context, 0); + return; + } + + int nrows = 0; + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + // File-based load applies a complete monolithic payload: legacy last-applied + // checkpoint (ends on a db_version boundary, so it is safe). + int rc = cloudsync_payload_apply (data, payload, (int)payload_size, &nrows, CLOUDSYNC_CHECKPOINT_LAST_APPLIED, 0); + if (payload) cloudsync_memory_free(payload); + + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + + // returns number of applied rows + sqlite3_result_int(context, nrows); +} +#endif + +// MARK: - Register - + +int dbsync_register_with_flags (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, int flags, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + + int rc = sqlite3_create_function_v2(db, name, nargs, flags, ctx, xfunc, xstep, xfinal, ctx_free); + + if (rc != SQLITE_OK) { + if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("Error creating function %s: %s", name, sqlite3_errmsg(db)); + return rc; + } + return SQLITE_OK; +} + +int dbsync_register (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + const int FLAGS_VOLATILE = SQLITE_UTF8; + DEBUG_DBFUNCTION("dbsync_register %s", name); + return dbsync_register_with_flags(db, name, xfunc, xstep, xfinal, nargs, FLAGS_VOLATILE, pzErrMsg, ctx, ctx_free); +} + +int dbsync_register_function (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + DEBUG_DBFUNCTION("dbsync_register_function %s", name); + return dbsync_register(db, name, xfunc, NULL, NULL, nargs, pzErrMsg, ctx, ctx_free); +} + +int dbsync_register_pure_function (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + const int FLAGS_PURE = SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC; + DEBUG_DBFUNCTION("dbsync_register_pure_function %s", name); + return dbsync_register_with_flags(db, name, xfunc, NULL, NULL, nargs, FLAGS_PURE, pzErrMsg, ctx, ctx_free); +} + +int dbsync_register_trigger_function (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + const int FLAGS_TRIGGER = SQLITE_UTF8 | SQLITE_INNOCUOUS; + DEBUG_DBFUNCTION("dbsync_register_trigger_function %s", name); + return dbsync_register_with_flags(db, name, xfunc, NULL, NULL, nargs, FLAGS_TRIGGER, pzErrMsg, ctx, ctx_free); +} + +int dbsync_register_aggregate (sqlite3 *db, const char *name, void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + DEBUG_DBFUNCTION("dbsync_register_aggregate %s", name); + return dbsync_register(db, name, NULL, xstep, xfinal, nargs, pzErrMsg, ctx, ctx_free); +} + +int dbsync_register_trigger_aggregate (sqlite3 *db, const char *name, void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + const int FLAGS_TRIGGER = SQLITE_UTF8 | SQLITE_INNOCUOUS; + DEBUG_DBFUNCTION("dbsync_register_trigger_aggregate %s", name); + return dbsync_register_with_flags(db, name, NULL, xstep, xfinal, nargs, FLAGS_TRIGGER, pzErrMsg, ctx, ctx_free); +} + +// MARK: - Block-level LWW - + +void dbsync_text_materialize (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_text_materialize"); + + // argv[0] -> table name + // argv[1] -> column name + // argv[2..N] -> primary key values + + if (argc < 3) { + sqlite3_result_error(context, "cloudsync_text_materialize requires at least 3 arguments: table, column, pk...", -1); + return; + } + + const char *table_name = (const char *)database_value_text(argv[0]); + const char *col_name = (const char *)database_value_text(argv[1]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + dbsync_set_error(context, "Unable to retrieve table name %s in cloudsync_text_materialize.", table_name); + return; + } + + int col_idx = table_col_index(table, col_name); + if (col_idx < 0 || table_col_algo(table, col_idx) != col_algo_block) { + dbsync_set_error(context, "Column %s in table %s is not configured as block-level.", col_name, table_name); + return; + } + + // Encode primary keys + int npks = table_count_pks(table); + if (argc - 2 != npks) { + sqlite3_result_error(context, "Wrong number of primary key values for cloudsync_text_materialize.", -1); + return; + } + + char buffer[1024]; + size_t pklen = sizeof(buffer); + char *pk = pk_encode_prikey((dbvalue_t **)&argv[2], npks, buffer, &pklen); + if (!pk || pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + sqlite3_result_error(context, "Failed to encode primary key(s).", -1); + return; + } + + // Materialize the column + int rc = block_materialize_column(data, table, pk, (int)pklen, col_name); + if (rc != DBRES_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + } else { + sqlite3_result_int(context, 1); + } + + if (pk != buffer) cloudsync_memory_free(pk); +} + +// MARK: - Row Filter - + +void dbsync_set_filter (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_set_filter"); + + const char *tbl = (const char *)database_value_text(argv[0]); + const char *filter_expr = (const char *)database_value_text(argv[1]); + if (!tbl || !filter_expr) { + dbsync_set_error(context, "cloudsync_set_filter: table and filter expression required"); + return; + } + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // Guard against calling set_filter before the target table has been set + // up for sync: without this, we'd hit "no such table: + // cloudsync_table_settings" or "no such table: main." deep inside + // the trigger recreation path, which is not actionable. + if (!cloudsync_context_is_initialized(data) || !table_lookup(data, tbl)) { + dbsync_set_error(context, + "cloudsync_set_filter: table '%s' is not configured for sync. " + "Call SELECT cloudsync_init('%s') first.", tbl, tbl); + return; + } + + // Store filter in table settings + dbutils_table_settings_set_key_value(data, tbl, "*", "filter", filter_expr); + + // Read current algo + table_algo algo = dbutils_table_settings_get_algo(data, tbl); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + + // Drop and recreate triggers with the filter + database_delete_triggers(data, tbl); + int rc = database_create_triggers(data, tbl, algo, filter_expr); + if (rc != DBRES_OK) { + dbsync_set_error(context, "cloudsync_set_filter: error recreating triggers"); + sqlite3_result_error_code(context, rc); + return; + } + + // Clean and refill metatable with the new filter + rc = cloudsync_reset_metatable(data, tbl); + if (rc != DBRES_OK) { + dbsync_set_error(context, "cloudsync_set_filter: error resetting metatable"); + sqlite3_result_error_code(context, rc); + return; + } + + sqlite3_result_int(context, 1); +} + +void dbsync_clear_filter (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_clear_filter"); + + const char *tbl = (const char *)database_value_text(argv[0]); + if (!tbl) { + dbsync_set_error(context, "cloudsync_clear_filter: table name required"); + return; + } + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // Guard against calling clear_filter before the target table has been set + // up for sync — see dbsync_set_filter for the same rationale. + if (!cloudsync_context_is_initialized(data) || !table_lookup(data, tbl)) { + dbsync_set_error(context, + "cloudsync_clear_filter: table '%s' is not configured for sync. " + "Call SELECT cloudsync_init('%s') first.", tbl, tbl); + return; + } + + // Remove filter from table settings (set to NULL/empty) + dbutils_table_settings_set_key_value(data, tbl, "*", "filter", NULL); + + // Read current algo + table_algo algo = dbutils_table_settings_get_algo(data, tbl); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + + // Drop and recreate triggers without filter + database_delete_triggers(data, tbl); + int rc = database_create_triggers(data, tbl, algo, NULL); + if (rc != DBRES_OK) { + dbsync_set_error(context, "cloudsync_clear_filter: error recreating triggers"); + sqlite3_result_error_code(context, rc); + return; + } + + // Clean and refill metatable without filter (all rows) + rc = cloudsync_reset_metatable(data, tbl); + if (rc != DBRES_OK) { + dbsync_set_error(context, "cloudsync_clear_filter: error resetting metatable"); + sqlite3_result_error_code(context, rc); + return; + } + + sqlite3_result_int(context, 1); +} + +int dbsync_register_functions (sqlite3 *db, char **pzErrMsg) { + int rc = SQLITE_OK; + + // there's no built-in way to verify if sqlite3_cloudsync_init has already been called + // for this specific database connection, we use a workaround: we attempt to retrieve the + // cloudsync_version and check for an error, an error indicates that initialization has not been performed + if (sqlite3_exec(db, "SELECT cloudsync_version();", NULL, NULL, NULL) == SQLITE_OK) return SQLITE_OK; + + // init memory debugger (NOOP in production) + cloudsync_memory_init(1); + + // set fractional-indexing allocator to use cloudsync memory + block_init_allocator(); + + // init context + void *ctx = cloudsync_context_create(db); + if (!ctx) { + if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("Not enought memory to create a database context"); + return SQLITE_NOMEM; + } + + // register functions + + // PUBLIC functions + rc = dbsync_register_pure_function(db, "cloudsync_version", dbsync_version, 0, pzErrMsg, ctx, cloudsync_context_free); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_init", dbsync_init1, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_init", dbsync_init2, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_init", dbsync_init3, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_enable", dbsync_enable, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_disable", dbsync_disable, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_is_enabled", dbsync_is_enabled, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_cleanup", dbsync_cleanup, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_terminate", dbsync_terminate, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_set", dbsync_set, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_set_table", dbsync_set_table, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_set_filter", dbsync_set_filter, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_clear_filter", dbsync_clear_filter, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_set_schema", dbsync_set_schema, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_schema", dbsync_schema, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_table_schema", dbsync_table_schema, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_set_column", dbsync_set_column, 4, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_siteid", dbsync_siteid, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_db_version", dbsync_db_version, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_db_version_next", dbsync_db_version_next, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_db_version_next", dbsync_db_version_next, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_begin_alter", dbsync_begin_alter, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_commit_alter", dbsync_commit_alter, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_uuid", dbsync_uuid, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_uuid_text", dbsync_uuid_text, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_uuid_text", dbsync_uuid_text, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_uuid_blob", dbsync_uuid_blob, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + // PAYLOAD + rc = dbsync_register_aggregate(db, "cloudsync_payload_encode", dbsync_payload_encode_step, dbsync_payload_encode_final, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + // alias + rc = dbsync_register_function(db, "cloudsync_payload_decode", dbsync_payload_decode, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_payload_apply", dbsync_payload_decode, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_module(db, "cloudsync_payload_chunks", &cloudsync_payload_chunks_module, (void *)ctx); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_payload_blob_checked", dbsync_payload_blob_checked, 5, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + #ifdef CLOUDSYNC_DESKTOP_OS + rc = dbsync_register_function(db, "cloudsync_payload_save", dbsync_payload_save, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_payload_load", dbsync_payload_load, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + #endif + + // PRIVATE functions (used inside triggers — require SQLITE_INNOCUOUS) + rc = dbsync_register_trigger_function(db, "cloudsync_is_sync", dbsync_is_sync, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_trigger_function(db, "cloudsync_insert", dbsync_insert, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_trigger_aggregate(db, "cloudsync_update", dbsync_update_step, dbsync_update_final, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_trigger_function(db, "cloudsync_delete", dbsync_delete, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_col_value", dbsync_col_value, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_pure_function(db, "cloudsync_pk_encode", dbsync_pk_encode, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_pure_function(db, "cloudsync_pk_decode", dbsync_pk_decode, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_seq", dbsync_seq, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_text_materialize", dbsync_text_materialize, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + // NETWORK LAYER + #ifndef CLOUDSYNC_OMIT_NETWORK + rc = cloudsync_network_register(db, pzErrMsg, ctx); + if (rc != SQLITE_OK) return rc; + #endif + + cloudsync_context *data = (cloudsync_context *)ctx; + sqlite3_commit_hook(db, cloudsync_commit_hook, ctx); + sqlite3_rollback_hook(db, cloudsync_rollback_hook, ctx); + + // register eponymous only changes virtual table + rc = cloudsync_vtab_register_changes (db, data); + if (rc != SQLITE_OK) { + if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("Error creating changes virtual table: %s", sqlite3_errmsg(db)); + return rc; + } + + // load config, if exists + if (cloudsync_config_exists(data)) { + if (cloudsync_context_init(data) == NULL) { + // Do not free ctx here: it is already owned by the cloudsync_version + // function (registered above with cloudsync_context_free as its + // destructor). SQLite will release it when the connection is closed. + // Freeing it manually would cause a double-free on sqlite3_close. + if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("An error occurred while trying to initialize context"); + return SQLITE_ERROR; + } + + // update schema hash if upgrading from an older version + if (dbutils_settings_check_version(data, NULL) != 0) { + cloudsync_update_schema_hash(data); + } + + // make sure to update internal version to current version + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_LIBVERSION, CLOUDSYNC_VERSION); + } + + return SQLITE_OK; +} + +// MARK: - Main Entrypoint - + +APIEXPORT int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + DEBUG_FUNCTION("sqlite3_cloudsync_init"); + + #ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT2(pApi); + #endif + + return dbsync_register_functions(db, pzErrMsg); +} diff --git a/src/sqlite/cloudsync_sqlite.h b/src/sqlite/cloudsync_sqlite.h new file mode 100644 index 00000000..12127e45 --- /dev/null +++ b/src/sqlite/cloudsync_sqlite.h @@ -0,0 +1,19 @@ +// +// cloudsync_sqlite.h +// cloudsync +// +// Created by Marco Bambini on 05/12/25. +// + +#ifndef __CLOUDSYNC_SQLITE__ +#define __CLOUDSYNC_SQLITE__ + +#ifndef SQLITE_CORE +#include "sqlite3ext.h" +#else +#include "sqlite3.h" +#endif + +int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); + +#endif diff --git a/src/sqlite/database_sqlite.c b/src/sqlite/database_sqlite.c new file mode 100644 index 00000000..b7864bbd --- /dev/null +++ b/src/sqlite/database_sqlite.c @@ -0,0 +1,1387 @@ +// +// database_sqlite.c +// cloudsync +// +// Created by Marco Bambini on 03/12/25. +// + +#include "../cloudsync.h" +#include "../database.h" +#include "../dbutils.h" +#include "../utils.h" +#include "../sql.h" + +#include +#include +#include + +#ifndef SQLITE_CORE +#include "sqlite3ext.h" +#else +#include "sqlite3.h" +#endif + +#ifndef SQLITE_CORE +SQLITE_EXTENSION_INIT3 +#endif + +// MARK: - SQL - + +char *sql_build_drop_table (const char *table_name, char *buffer, int bsize, bool is_meta) { + char *sql = NULL; + + if (is_meta) { + sql = sqlite3_snprintf(bsize, buffer, "DROP TABLE IF EXISTS \"%w_cloudsync\";", table_name); + } else { + sql = sqlite3_snprintf(bsize, buffer, "DROP TABLE IF EXISTS \"%w\";", table_name); + } + + return sql; +} + +char *sql_escape_identifier (const char *name, char *buffer, size_t bsize) { + return sqlite3_snprintf((int)bsize, buffer, "%q", name); +} + +char *sql_build_select_nonpk_by_pk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char *sql = NULL; + + /* + This SQL statement dynamically generates a SELECT query for a specified table. + It uses Common Table Expressions (CTEs) to construct the column names and + primary key conditions based on the table schema, which is obtained through + the `pragma_table_info` function. + + 1. `col_names` CTE: + - Retrieves a comma-separated list of non-primary key column names from + the specified table's schema. + + 2. `pk_where` CTE: + - Retrieves a condition string representing the primary key columns in the + format: "column1=? AND column2=? AND ...", used to create the WHERE clause + for selecting rows based on primary key values. + + 3. Final SELECT: + - Constructs the complete SELECT statement as a string, combining: + - Column names from `col_names`. + - The target table name. + - The WHERE clause conditions from `pk_where`. + + The resulting query can be used to select rows from the table based on primary + key values, and can be executed within the application to retrieve data dynamically. + */ + + // Unfortunately in SQLite column names (or table names) cannot be bound parameters in a SELECT statement + // otherwise we should have used something like SELECT 'SELECT ? FROM %w WHERE rowid=?'; + char buffer[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + + #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES + if (table->rowid_only) { + sql = memory_mprintf(SQL_BUILD_SELECT_NONPK_COLS_BY_ROWID, table->name, table->name); + goto process_process; + } + #endif + + sql = cloudsync_memory_mprintf(SQL_BUILD_SELECT_NONPK_COLS_BY_PK, table_name, table_name, singlequote_escaped_table_name); + +#if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES +process_process: +#endif + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_delete_by_pk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + char *sql = cloudsync_memory_mprintf(SQL_BUILD_DELETE_ROW_BY_PK, table_name, singlequote_escaped_table_name); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_insert_pk_ignore (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + char *sql = cloudsync_memory_mprintf(SQL_BUILD_INSERT_PK_IGNORE, table_name, table_name, singlequote_escaped_table_name); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_upsert_pk_and_col (cloudsync_context *data, const char *table_name, const char *colname, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char buffer2[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + char *singlequote_escaped_col_name = sql_escape_identifier(colname, buffer2, sizeof(buffer2)); + char *sql = cloudsync_memory_mprintf( + SQL_BUILD_UPSERT_PK_AND_COL, + table_name, + table_name, + singlequote_escaped_table_name, + singlequote_escaped_col_name, + singlequote_escaped_col_name + ); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_upsert_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema) { + UNUSED_PARAMETER(schema); + if (ncolnames <= 0 || !colnames) return NULL; + + // Get PK column names via pragma_table_info (same approach as database_pk_names) + char **pk_names = NULL; + int npks = 0; + int rc = database_pk_names(data, table_name, &pk_names, &npks); + if (rc != DBRES_OK || npks <= 0 || !pk_names) return NULL; + + // Build column list: "pk1","pk2","col_a","col_b" + char *col_list = cloudsync_memory_mprintf("\"%w\"", pk_names[0]); + if (!col_list) goto fail; + for (int i = 1; i < npks; i++) { + char *prev = col_list; + col_list = cloudsync_memory_mprintf("%s,\"%w\"", prev, pk_names[i]); + cloudsync_memory_free(prev); + if (!col_list) goto fail; + } + for (int i = 0; i < ncolnames; i++) { + char *prev = col_list; + col_list = cloudsync_memory_mprintf("%s,\"%w\"", prev, colnames[i]); + cloudsync_memory_free(prev); + if (!col_list) goto fail; + } + + // Build bind list: ?,?,?,? + int total = npks + ncolnames; + char *binds = (char *)cloudsync_memory_alloc(total * 2); + if (!binds) { cloudsync_memory_free(col_list); goto fail; } + int pos = 0; + for (int i = 0; i < total; i++) { + if (i > 0) binds[pos++] = ','; + binds[pos++] = '?'; + } + binds[pos] = '\0'; + + // Build excluded set: "col_a"=EXCLUDED."col_a","col_b"=EXCLUDED."col_b" + char *excl = cloudsync_memory_mprintf("\"%w\"=EXCLUDED.\"%w\"", colnames[0], colnames[0]); + if (!excl) { cloudsync_memory_free(col_list); cloudsync_memory_free(binds); goto fail; } + for (int i = 1; i < ncolnames; i++) { + char *prev = excl; + excl = cloudsync_memory_mprintf("%s,\"%w\"=EXCLUDED.\"%w\"", prev, colnames[i], colnames[i]); + cloudsync_memory_free(prev); + if (!excl) { cloudsync_memory_free(col_list); cloudsync_memory_free(binds); goto fail; } + } + + // Assemble final SQL + char *sql = cloudsync_memory_mprintf( + "INSERT INTO \"%w\" (%s) VALUES (%s) ON CONFLICT DO UPDATE SET %s;", + table_name, col_list, binds, excl + ); + + cloudsync_memory_free(col_list); + cloudsync_memory_free(binds); + cloudsync_memory_free(excl); + for (int i = 0; i < npks; i++) cloudsync_memory_free(pk_names[i]); + cloudsync_memory_free(pk_names); + return sql; + +fail: + if (pk_names) { + for (int i = 0; i < npks; i++) cloudsync_memory_free(pk_names[i]); + cloudsync_memory_free(pk_names); + } + return NULL; +} + +char *sql_build_update_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema) { + UNUSED_PARAMETER(schema); + if (ncolnames <= 0 || !colnames) return NULL; + + // Get PK column names + char **pk_names = NULL; + int npks = 0; + int rc = database_pk_names(data, table_name, &pk_names, &npks); + if (rc != DBRES_OK || npks <= 0 || !pk_names) return NULL; + + // Build SET clause: "col_a"=?npks+1,"col_b"=?npks+2 + // Uses numbered parameters to match merge_flush_pending bind order: + // positions 1..npks are PKs, npks+1..npks+ncolnames are column values. + char *set_clause = cloudsync_memory_mprintf("\"%w\"=?%d", colnames[0], npks + 1); + if (!set_clause) goto fail; + for (int i = 1; i < ncolnames; i++) { + char *prev = set_clause; + set_clause = cloudsync_memory_mprintf("%s,\"%w\"=?%d", prev, colnames[i], npks + 1 + i); + cloudsync_memory_free(prev); + if (!set_clause) goto fail; + } + + // Build WHERE clause: "pk1"=?1 AND "pk2"=?2 + char *where_clause = cloudsync_memory_mprintf("\"%w\"=?%d", pk_names[0], 1); + if (!where_clause) { cloudsync_memory_free(set_clause); goto fail; } + for (int i = 1; i < npks; i++) { + char *prev = where_clause; + where_clause = cloudsync_memory_mprintf("%s AND \"%w\"=?%d", prev, pk_names[i], 1 + i); + cloudsync_memory_free(prev); + if (!where_clause) { cloudsync_memory_free(set_clause); goto fail; } + } + + // Assemble: UPDATE "table" SET ... WHERE ... + char *sql = cloudsync_memory_mprintf( + "UPDATE \"%w\" SET %s WHERE %s;", + table_name, set_clause, where_clause + ); + + cloudsync_memory_free(set_clause); + cloudsync_memory_free(where_clause); + for (int i = 0; i < npks; i++) cloudsync_memory_free(pk_names[i]); + cloudsync_memory_free(pk_names); + return sql; + +fail: + if (pk_names) { + for (int i = 0; i < npks; i++) cloudsync_memory_free(pk_names[i]); + cloudsync_memory_free(pk_names); + } + return NULL; +} + +char *sql_build_select_cols_by_pk (cloudsync_context *data, const char *table_name, const char *colname, const char *schema) { + UNUSED_PARAMETER(schema); + char *colnamequote = "\""; + char buffer[1024]; + char buffer2[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + char *singlequote_escaped_col_name = sql_escape_identifier(colname, buffer2, sizeof(buffer2)); + char *sql = cloudsync_memory_mprintf( + SQL_BUILD_SELECT_COLS_BY_PK_FMT, + table_name, + colnamequote, + singlequote_escaped_col_name, + colnamequote, + singlequote_escaped_table_name + ); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_rekey_pk_and_reset_version_except_col (cloudsync_context *data, const char *table_name, const char *except_col) { + UNUSED_PARAMETER(data); + + char *meta_ref = database_build_meta_ref(NULL, table_name); + if (!meta_ref) return NULL; + + char *result = cloudsync_memory_mprintf(SQL_CLOUDSYNC_REKEY_PK_AND_RESET_VERSION_EXCEPT_COL, meta_ref, except_col); + cloudsync_memory_free(meta_ref); + return result; +} + +char *database_table_schema (const char *table_name) { + return NULL; +} + +char *database_build_meta_ref (const char *schema, const char *table_name) { + // schema unused in SQLite + return cloudsync_memory_mprintf("%s_cloudsync", table_name); +} + +char *database_build_base_ref (const char *schema, const char *table_name) { + // schema unused in SQLite + return cloudsync_string_dup(table_name); +} + +char *database_build_blocks_ref (const char *schema, const char *table_name) { + // schema unused in SQLite + return cloudsync_memory_mprintf("%s_cloudsync_blocks", table_name); +} + +// SQLite version: schema parameter unused (SQLite has no schemas). +char *sql_build_delete_cols_not_in_schema_query (const char *schema, const char *table_name, const char *meta_ref, const char *pkcol) { + UNUSED_PARAMETER(schema); + return cloudsync_memory_mprintf( + "DELETE FROM \"%w\" WHERE col_name NOT IN (" + "SELECT name FROM pragma_table_info('%q') " + "UNION SELECT '%s'" + ");", + meta_ref, table_name, pkcol + ); +} + +char *sql_build_pk_collist_query (const char *schema, const char *table_name) { + UNUSED_PARAMETER(schema); + return cloudsync_memory_mprintf( + "SELECT group_concat('\"' || format('%%w', name) || '\"', ',') " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", + table_name + ); +} + +char *sql_build_pk_decode_selectlist_query (const char *schema, const char *table_name) { + UNUSED_PARAMETER(schema); + return cloudsync_memory_mprintf( + "SELECT group_concat(" + "'cloudsync_pk_decode(pk, ' || pk || ') AS ' || '\"' || format('%%w', name) || '\"', ','" + ") " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", + table_name + ); +} + +char *sql_build_pk_qualified_collist_query (const char *schema, const char *table_name) { + UNUSED_PARAMETER(schema); + + char buffer[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + if (!singlequote_escaped_table_name) return NULL; + + return cloudsync_memory_mprintf( + "SELECT group_concat('\"%w\".\"' || format('%%w', name) || '\"', ',') " + "FROM pragma_table_info('%s') WHERE pk>0 ORDER BY pk;", singlequote_escaped_table_name, singlequote_escaped_table_name + ); +} + +char *sql_build_insert_missing_pks_query(const char *schema, const char *table_name, + const char *pkvalues_identifiers, + const char *base_ref, const char *meta_ref, + const char *filter) { + UNUSED_PARAMETER(schema); + + // SQLite: Use NOT EXISTS with cloudsync_pk_encode (same approach as PostgreSQL). + // This avoids needing pk_decode select list which requires executing a query. + if (filter) { + return cloudsync_memory_mprintf( + "SELECT cloudsync_insert('%q', %s) " + "FROM \"%w\" " + "WHERE (%s) AND NOT EXISTS (" + " SELECT 1 FROM \"%w\" WHERE pk = cloudsync_pk_encode(%s)" + ");", + table_name, pkvalues_identifiers, base_ref, filter, meta_ref, pkvalues_identifiers + ); + } + return cloudsync_memory_mprintf( + "SELECT cloudsync_insert('%q', %s) " + "FROM \"%w\" " + "WHERE NOT EXISTS (" + " SELECT 1 FROM \"%w\" WHERE pk = cloudsync_pk_encode(%s)" + ");", + table_name, pkvalues_identifiers, base_ref, meta_ref, pkvalues_identifiers + ); +} + +// MARK: - PRIVATE - + +static int database_select1_value (cloudsync_context *data, const char *sql, char **ptr_value, int64_t *int_value, DBTYPE expected_type) { + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + + // init values and sanity check expected_type + if (ptr_value) *ptr_value = NULL; + *int_value = 0; + if (expected_type != DBTYPE_INTEGER && expected_type != DBTYPE_TEXT && expected_type != DBTYPE_BLOB) return SQLITE_MISUSE; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2((sqlite3 *)db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup_select; + + // ensure at least one column + if (sqlite3_column_count(vm) < 1) {rc = SQLITE_MISMATCH; goto cleanup_select;} + + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) {rc = SQLITE_OK; goto cleanup_select;} // no rows OK + if (rc != SQLITE_ROW) goto cleanup_select; + + // sanity check column type + DBTYPE type = (DBTYPE)sqlite3_column_type(vm, 0); + if (type == SQLITE_NULL) {rc = SQLITE_OK; goto cleanup_select;} + if (type != expected_type) {rc = SQLITE_MISMATCH; goto cleanup_select;} + + if (expected_type == DBTYPE_INTEGER) { + *int_value = (int64_t)sqlite3_column_int64(vm, 0); + } else { + const void *value = (expected_type == DBTYPE_TEXT) ? (const void *)sqlite3_column_text(vm, 0) : (const void *)sqlite3_column_blob(vm, 0); + int len = sqlite3_column_bytes(vm, 0); + if (len) { + char *ptr = cloudsync_memory_alloc(len + 1); + if (!ptr) {rc = SQLITE_NOMEM; goto cleanup_select;} + + if (len > 0 && value) memcpy(ptr, value, len); + if (expected_type == DBTYPE_TEXT) ptr[len] = 0; // NULL terminate in case of TEXT + + *ptr_value = ptr; + *int_value = len; + } + } + rc = SQLITE_OK; + +cleanup_select: + if (vm) sqlite3_finalize(vm); + return rc; +} + +static int database_select2_values (cloudsync_context *data, const char *sql, char **value, int64_t *len, int64_t *value2) { + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + + // init values and sanity check expected_type + *value = NULL; + *value2 = 0; + *len = 0; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2((sqlite3 *)db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup_select; + + // ensure column count + if (sqlite3_column_count(vm) < 2) {rc = SQLITE_MISMATCH; goto cleanup_select;} + + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) {rc = SQLITE_OK; goto cleanup_select;} // no rows OK + if (rc != SQLITE_ROW) goto cleanup_select; + + // sanity check column types + if (sqlite3_column_type(vm, 0) != SQLITE_BLOB) {rc = SQLITE_MISMATCH; goto cleanup_select;} + if (sqlite3_column_type(vm, 1) != SQLITE_INTEGER) {rc = SQLITE_MISMATCH; goto cleanup_select;} + + // 1st column is BLOB + const void *blob = (const void *)sqlite3_column_blob(vm, 0); + int blob_len = sqlite3_column_bytes(vm, 0); + if (blob_len) { + char *ptr = cloudsync_memory_alloc(blob_len); + if (!ptr) {rc = SQLITE_NOMEM; goto cleanup_select;} + + if (blob_len > 0 && blob) memcpy(ptr, blob, blob_len); + *value = ptr; + *len = blob_len; + } + + // 2nd column is INTEGER + *value2 = (int64_t)sqlite3_column_int64(vm, 1); + + rc = SQLITE_OK; + +cleanup_select: + if (vm) sqlite3_finalize(vm); + return rc; +} + +bool database_system_exists (cloudsync_context *data, const char *name, const char *type) { + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + sqlite3_stmt *vm = NULL; + bool result = false; + + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type='%s' AND name=?1 COLLATE NOCASE);", type); + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_bind_text(vm, 1, name, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + result = (bool)sqlite3_column_int(vm, 0); + rc = SQLITE_OK; + } + +finalize: + if (rc != SQLITE_OK) DEBUG_ALWAYS("Error executing %s in dbutils_system_exists for type %s name %s (%s).", sql, type, name, sqlite3_errmsg(db)); + if (vm) sqlite3_finalize(vm); + return result; +} + +// MARK: - GENERAL - + +int database_exec (cloudsync_context *data, const char *sql) { + return sqlite3_exec((sqlite3 *)cloudsync_db(data), sql, NULL, NULL, NULL); +} + +int database_exec_callback (cloudsync_context *data, const char *sql, int (*callback)(void *xdata, int argc, char **values, char **names), void *xdata) { + return sqlite3_exec((sqlite3 *)cloudsync_db(data), sql, callback, xdata, NULL); +} + +int database_write (cloudsync_context *data, const char *sql, const char **bind_values, DBTYPE bind_types[], int bind_lens[], int bind_count) { + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2((sqlite3 *)db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup_write; + + for (int i=0; i0 AND \"notnull\"=1;", table_name); + } else { + sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk>0;", table_name); + } + + int64_t count = 0; + int rc = database_select_int(data, sql, &count); + if (rc != DBRES_OK) return -1; + return (int)count; +} + +int database_count_nonpk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char *sql = NULL; + + sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk=0;", table_name); + int64_t count = 0; + int rc = database_select_int(data, sql, &count); + if (rc != DBRES_OK) return -1; + return (int)count; +} + +int database_count_int_pk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char *sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk=1 AND \"type\" LIKE '%%INT%%';", table_name); + + int64_t count = 0; + int rc = database_select_int(data, sql, &count); + if (rc != DBRES_OK) return -1; + return (int)count; +} + +int database_count_notnull_without_default (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char *sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk=0 AND \"notnull\"=1 AND \"dflt_value\" IS NULL;", table_name); + + int64_t count = 0; + int rc = database_select_int(data, sql, &count); + if (rc != DBRES_OK) return -1; + return (int)count; +} + +int database_cleanup (cloudsync_context *data) { + char *sql = "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'cloudsync_%' AND name NOT LIKE '%_cloudsync';"; + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + + char **result = NULL; + char *errmsg = NULL; + int nrows, ncols; + int rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, &errmsg); + if (rc != SQLITE_OK) { + cloudsync_set_error(data, (errmsg) ? errmsg : "Error retrieving augmented tables", rc); + goto exit_cleanup; + } + + for (int i = ncols; i < nrows+ncols; i+=ncols) { + int rc2 = cloudsync_cleanup(data, result[i]); + if (rc2 != SQLITE_OK) {rc = rc2; goto exit_cleanup;} + } + +exit_cleanup: + if (result) sqlite3_free_table(result); + if (errmsg) sqlite3_free(errmsg); + return rc; +} + +// MARK: - TRIGGERS and META - + +int database_create_metatable (cloudsync_context *data, const char *table_name) { + DEBUG_DBFUNCTION("database_create_metatable %s", table_name); + + // table_name cannot be longer than 512 characters so static buffer size is computed accordling to that value + char buffer[2048]; + + // WITHOUT ROWID is available starting from SQLite version 3.8.2 (2013-12-06) and later + char *sql = sqlite3_snprintf(sizeof(buffer), buffer, "CREATE TABLE IF NOT EXISTS \"%w_cloudsync\" (pk BLOB NOT NULL, col_name TEXT NOT NULL, col_version INTEGER, db_version INTEGER, site_id INTEGER DEFAULT 0, seq INTEGER, PRIMARY KEY (pk, col_name)) WITHOUT ROWID; CREATE INDEX IF NOT EXISTS \"%w_cloudsync_db_idx\" ON \"%w_cloudsync\" (db_version);", table_name, table_name, table_name); + + int rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + return rc; +} + +int database_create_insert_trigger (cloudsync_context *data, const char *table_name, char *trigger_when) { + // NEW.prikey1, NEW.prikey2... + char buffer[1024]; + char *trigger_name = sqlite3_snprintf(sizeof(buffer), buffer, "cloudsync_after_insert_%s", table_name); + if (database_trigger_exists(data, trigger_name)) return SQLITE_OK; + + char buffer2[2048]; + char *sql2 = sqlite3_snprintf(sizeof(buffer2), buffer2, "SELECT group_concat('NEW.\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name); + + char *pkclause = NULL; + int rc = database_select_text(data, sql2, &pkclause); + if (rc != SQLITE_OK) return rc; + char *pkvalues = (pkclause) ? pkclause : "NEW.rowid"; + + char *sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" AFTER INSERT ON \"%w\" %s BEGIN SELECT cloudsync_insert('%q', %s); END", trigger_name, table_name, trigger_when, table_name, pkvalues); + if (pkclause) cloudsync_memory_free(pkclause); + if (!sql) return SQLITE_NOMEM; + + rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + cloudsync_memory_free(sql); + return rc; +} + +int database_create_update_trigger_gos (cloudsync_context *data, const char *table_name) { + // Grow Only Set + // In a grow-only set, the update operation is not allowed. + // A grow-only set is a type of CRDT (Conflict-free Replicated Data Type) where the only permissible operation is to add elements to the set, + // without ever removing or modifying them. + // Once an element is added to the set, it remains there permanently, which guarantees that the set only grows over time. + char buffer[1024]; + char *trigger_name = sqlite3_snprintf(sizeof(buffer), buffer, "cloudsync_before_update_%s", table_name); + if (database_trigger_exists(data, trigger_name)) return SQLITE_OK; + + char buffer2[2048+512]; + char *sql = sqlite3_snprintf(sizeof(buffer2), buffer2, "CREATE TRIGGER \"%w\" BEFORE UPDATE ON \"%w\" FOR EACH ROW WHEN cloudsync_is_enabled('%q') = 1 BEGIN SELECT RAISE(ABORT, 'Error: UPDATE operation is not allowed on table %w.'); END", trigger_name, table_name, table_name, table_name); + + int rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + return rc; +} + +int database_create_update_trigger (cloudsync_context *data, const char *table_name, const char *trigger_when) { + // NEW.prikey1, NEW.prikey2, OLD.prikey1, OLD.prikey2, NEW.col1, OLD.col1, NEW.col2, OLD.col2... + + char buffer[1024]; + char *trigger_name = sqlite3_snprintf(sizeof(buffer), buffer, "cloudsync_after_update_%s", table_name); + if (database_trigger_exists(data, trigger_name)) return SQLITE_OK; + + // generate VALUES clause for all columns using a CTE to avoid compound SELECT limits + // first, get all primary key columns in order + char buffer2[2048]; + char *sql2 = sqlite3_snprintf(sizeof(buffer2), buffer2, "SELECT group_concat('('||quote('%q')||', NEW.\"' || format('%%w', name) || '\", OLD.\"' || format('%%w', name) || '\")', ', ') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name, table_name); + + char *pk_values_list = NULL; + int rc = database_select_text(data, sql2, &pk_values_list); + if (rc != SQLITE_OK) return rc; + + // then get all regular columns in order + sql2 = sqlite3_snprintf(sizeof(buffer2), buffer2, "SELECT group_concat('('||quote('%q')||', NEW.\"' || format('%%w', name) || '\", OLD.\"' || format('%%w', name) || '\")', ', ') FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid;", table_name, table_name); + + char *col_values_list = NULL; + rc = database_select_text(data, sql2, &col_values_list); + if (rc != SQLITE_OK) { + if (pk_values_list) cloudsync_memory_free(pk_values_list); + return rc; + } + + // build the complete VALUES query + char *values_query = NULL; + if (col_values_list && strlen(col_values_list) > 0) { + // Table has both primary keys and regular columns + values_query = cloudsync_memory_mprintf( + "WITH column_data(table_name, new_value, old_value) AS (VALUES %s, %s) " + "SELECT table_name, new_value, old_value FROM column_data", + pk_values_list, col_values_list); + } else { + // Table has only primary keys + values_query = cloudsync_memory_mprintf( + "WITH column_data(table_name, new_value, old_value) AS (VALUES %s) " + "SELECT table_name, new_value, old_value FROM column_data", + pk_values_list); + } + + if (pk_values_list) cloudsync_memory_free(pk_values_list); + if (col_values_list) cloudsync_memory_free(col_values_list); + if (!values_query) return SQLITE_NOMEM; + + // create the trigger with aggregate function + char *sql = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%w\" AFTER UPDATE ON \"%w\" %s BEGIN " + "SELECT cloudsync_update(table_name, new_value, old_value) FROM (%s); " + "END", + trigger_name, table_name, trigger_when, values_query); + + cloudsync_memory_free(values_query); + if (!sql) return SQLITE_NOMEM; + + rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + cloudsync_memory_free(sql); + return rc; +} + +int database_create_delete_trigger_gos (cloudsync_context *data, const char *table_name) { + // Grow Only Set + // In a grow-only set, the delete operation is not allowed. + + char buffer[1024]; + char *trigger_name = sqlite3_snprintf(sizeof(buffer), buffer, "cloudsync_before_delete_%s", table_name); + if (database_trigger_exists(data, trigger_name)) return SQLITE_OK; + + char buffer2[2048+512]; + char *sql = sqlite3_snprintf(sizeof(buffer2), buffer2, "CREATE TRIGGER \"%w\" BEFORE DELETE ON \"%w\" FOR EACH ROW WHEN cloudsync_is_enabled('%q') = 1 BEGIN SELECT RAISE(ABORT, 'Error: DELETE operation is not allowed on table %w.'); END", trigger_name, table_name, table_name, table_name); + + int rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + return rc; +} + +int database_create_delete_trigger (cloudsync_context *data, const char *table_name, const char *trigger_when) { + // OLD.prikey1, OLD.prikey2... + + char buffer[1024]; + char *trigger_name = sqlite3_snprintf(sizeof(buffer), buffer, "cloudsync_after_delete_%s", table_name); + if (database_trigger_exists(data, trigger_name)) return SQLITE_OK; + + char buffer2[1024]; + char *sql2 = sqlite3_snprintf(sizeof(buffer2), buffer2, "SELECT group_concat('OLD.\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name); + + char *pkclause = NULL; + int rc = database_select_text(data, sql2, &pkclause); + if (rc != SQLITE_OK) return rc; + char *pkvalues = (pkclause) ? pkclause : "OLD.rowid"; + + char *sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" AFTER DELETE ON \"%w\" %s BEGIN SELECT cloudsync_delete('%q',%s); END", trigger_name, table_name, trigger_when, table_name, pkvalues); + if (pkclause) cloudsync_memory_free(pkclause); + if (!sql) return SQLITE_NOMEM; + + rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + cloudsync_memory_free(sql); + return rc; +} + +// Build trigger WHEN clauses, optionally incorporating a row-level filter. +// INSERT/UPDATE use NEW-prefixed filter, DELETE uses OLD-prefixed filter. +// Returns dynamically-allocated strings that must be freed with cloudsync_memory_free. +static void database_build_trigger_when( + cloudsync_context *data, const char *table_name, const char *filter, + char **when_new_out, char **when_old_out) +{ + char *new_filter_str = NULL; + char *old_filter_str = NULL; + + if (filter) { + char sql_cols[1024]; + sqlite3_snprintf(sizeof(sql_cols), sql_cols, + "SELECT name FROM pragma_table_info('%q') ORDER BY cid;", table_name); + + char *col_names[256]; + int ncols = 0; + + sqlite3_stmt *col_vm = NULL; + int col_rc = sqlite3_prepare_v2((sqlite3 *)cloudsync_db(data), sql_cols, -1, &col_vm, NULL); + if (col_rc == SQLITE_OK) { + while (sqlite3_step(col_vm) == SQLITE_ROW && ncols < 256) { + const char *name = (const char *)sqlite3_column_text(col_vm, 0); + if (name) { + char *dup = cloudsync_memory_mprintf("%s", name); + if (!dup) break; + col_names[ncols++] = dup; + } + } + sqlite3_finalize(col_vm); + } + + if (ncols > 0) { + new_filter_str = cloudsync_filter_add_row_prefix(filter, "NEW", col_names, ncols); + old_filter_str = cloudsync_filter_add_row_prefix(filter, "OLD", col_names, ncols); + for (int i = 0; i < ncols; ++i) cloudsync_memory_free(col_names[i]); + } + } + + if (new_filter_str) { + *when_new_out = cloudsync_memory_mprintf( + "FOR EACH ROW WHEN cloudsync_is_sync('%q') = 0 AND (%s)", table_name, new_filter_str); + } else { + *when_new_out = cloudsync_memory_mprintf( + "FOR EACH ROW WHEN cloudsync_is_sync('%q') = 0", table_name); + } + + if (old_filter_str) { + *when_old_out = cloudsync_memory_mprintf( + "FOR EACH ROW WHEN cloudsync_is_sync('%q') = 0 AND (%s)", table_name, old_filter_str); + } else { + *when_old_out = cloudsync_memory_mprintf( + "FOR EACH ROW WHEN cloudsync_is_sync('%q') = 0", table_name); + } + + if (new_filter_str) cloudsync_memory_free(new_filter_str); + if (old_filter_str) cloudsync_memory_free(old_filter_str); +} + +int database_create_triggers (cloudsync_context *data, const char *table_name, table_algo algo, const char *filter) { + DEBUG_DBFUNCTION("database_create_triggers %s", table_name); + + if (dbutils_settings_check_version(data, "0.8.25") <= 0) { + database_delete_triggers(data, table_name); + } + + char *trigger_when_new = NULL; + char *trigger_when_old = NULL; + database_build_trigger_when(data, table_name, filter, + &trigger_when_new, &trigger_when_old); + + if (!trigger_when_new || !trigger_when_old) { + if (trigger_when_new) cloudsync_memory_free(trigger_when_new); + if (trigger_when_old) cloudsync_memory_free(trigger_when_old); + return SQLITE_NOMEM; + } + + // INSERT TRIGGER (uses NEW prefix) + int rc = database_create_insert_trigger(data, table_name, trigger_when_new); + if (rc != SQLITE_OK) goto done; + + // UPDATE TRIGGER (uses NEW prefix) + if (algo == table_algo_crdt_gos) rc = database_create_update_trigger_gos(data, table_name); + else rc = database_create_update_trigger(data, table_name, trigger_when_new); + if (rc != SQLITE_OK) goto done; + + // DELETE TRIGGER (uses OLD prefix) + if (algo == table_algo_crdt_gos) rc = database_create_delete_trigger_gos(data, table_name); + else rc = database_create_delete_trigger(data, table_name, trigger_when_old); + +done: + if (rc != SQLITE_OK) DEBUG_ALWAYS("database_create_triggers error %s (%d)", sqlite3_errmsg(cloudsync_db(data)), rc); + cloudsync_memory_free(trigger_when_new); + cloudsync_memory_free(trigger_when_old); + return rc; +} + +int database_delete_triggers (cloudsync_context *data, const char *table) { + DEBUG_DBFUNCTION("database_delete_triggers %s", table); + + // from cloudsync_table_sanity_check we already know that 2048 is OK + char buffer[2048]; + size_t blen = sizeof(buffer); + int rc = SQLITE_ERROR; + + char *sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_before_update_%w\";", table); + rc = database_exec(data, sql); + if (rc != SQLITE_OK) goto finalize; + + sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_before_delete_%w\";", table); + rc = database_exec(data, sql); + if (rc != SQLITE_OK) goto finalize; + + sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_insert_%w\";", table); + rc = database_exec(data, sql); + if (rc != SQLITE_OK) goto finalize; + + sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_update_%w\";", table); + rc = database_exec(data, sql); + if (rc != SQLITE_OK) goto finalize; + + sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_delete_%w\";", table); + rc = database_exec(data, sql); + if (rc != SQLITE_OK) goto finalize; + +finalize: + if (rc != SQLITE_OK) DEBUG_ALWAYS("dbutils_delete_triggers error %s (%s)", database_errmsg(data), sql); + return rc; +} + +// MARK: - SCHEMA - + +int64_t database_schema_version (cloudsync_context *data) { + int64_t value = 0; + int rc = database_select_int(data, SQL_SCHEMA_VERSION, &value); + return (rc == DBRES_OK) ? value : 0; +} + +uint64_t database_schema_hash (cloudsync_context *data) { + int64_t value = 0; + int rc = database_select_int(data, "SELECT hash FROM cloudsync_schema_versions ORDER BY seq DESC limit 1;", &value); + return (rc == DBRES_OK) ? (uint64_t)value : 0; +} + +bool database_check_schema_hash (cloudsync_context *data, uint64_t hash) { + // a change from the current version of the schema or from previous known schema can be applied + // a change from a newer schema version not yet applied to this peer cannot be applied + // so a schema hash is valid if it exists in the cloudsync_schema_versions table + + // the idea is to allow changes on stale peers and to be able to apply these changes on peers with newer schema, + // but it requires alter table operation on augmented tables only add new columns and never drop columns for backward compatibility + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT 1 FROM cloudsync_schema_versions WHERE hash = (%" PRId64 ")", (int64_t)hash); + + int64_t value = 0; + database_select_int(data, sql, &value); + return (value == 1); +} + +int database_update_schema_hash (cloudsync_context *data, uint64_t *hash) { + // Build normalized schema string using only: column name (lowercase), type (SQLite affinity), pk flag + // Format: tablename:colname:affinity:pk,... (ordered by table name, then column id) + // This makes the hash resilient to formatting, quoting, case differences and portable across databases + // + // Type mapping (simplified from SQLite affinity rules for cross-database compatibility): + // - Types containing 'INT' → 'integer' + // - Types containing 'CHAR', 'CLOB', 'TEXT' → 'text' + // - Types containing 'BLOB' or empty → 'blob' + // - Types containing 'REAL', 'FLOA', 'DOUB' → 'real' + // - Types exactly 'NUMERIC' or 'DECIMAL' → 'numeric' + // - Everything else → 'text' (default) + // + // NOTE: This deviates from SQLite's actual affinity rules where unknown types get NUMERIC affinity. + // We use 'text' as default to improve cross-database compatibility with PostgreSQL, where types + // like TIMESTAMPTZ, UUID, JSON, etc. are commonly used and map to 'text' in the PostgreSQL + // implementation. This ensures schemas with PostgreSQL-specific type names in SQLite DDL + // will hash consistently across both databases. + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + + char **tables = NULL; + int ntables, tcols; + int rc = sqlite3_get_table(db, "SELECT DISTINCT tbl_name FROM cloudsync_table_settings ORDER BY tbl_name;", + &tables, &ntables, &tcols, NULL); + if (rc != SQLITE_OK || ntables == 0) { + if (tables) sqlite3_free_table(tables); + return SQLITE_ERROR; + } + + char *schema = NULL; + size_t schema_len = 0; + size_t schema_cap = 0; + + for (int t = 1; t <= ntables; t++) { + const char *tbl_name = tables[t]; + + // Query pragma_table_info for this table with normalized type + char *col_sql = cloudsync_memory_mprintf( + "SELECT LOWER(name), " + "CASE " + " WHEN UPPER(type) LIKE '%%INT%%' THEN 'integer' " + " WHEN UPPER(type) LIKE '%%CHAR%%' OR UPPER(type) LIKE '%%CLOB%%' OR UPPER(type) LIKE '%%TEXT%%' THEN 'text' " + " WHEN UPPER(type) LIKE '%%BLOB%%' OR type = '' THEN 'blob' " + " WHEN UPPER(type) LIKE '%%REAL%%' OR UPPER(type) LIKE '%%FLOA%%' OR UPPER(type) LIKE '%%DOUB%%' THEN 'real' " + " WHEN UPPER(type) IN ('NUMERIC', 'DECIMAL') THEN 'numeric' " + " ELSE 'text' " + "END, " + "CASE WHEN pk > 0 THEN '1' ELSE '0' END " + "FROM pragma_table_info('%q') ORDER BY cid;", tbl_name); + + if (!col_sql) { + if (schema) cloudsync_memory_free(schema); + sqlite3_free_table(tables); + return SQLITE_NOMEM; + } + + char **cols = NULL; + int nrows, ncols; + rc = sqlite3_get_table(db, col_sql, &cols, &nrows, &ncols, NULL); + cloudsync_memory_free(col_sql); + + if (rc != SQLITE_OK || ncols != 3) { + if (cols) sqlite3_free_table(cols); + if (schema) cloudsync_memory_free(schema); + sqlite3_free_table(tables); + return SQLITE_ERROR; + } + + // Append each column: tablename:colname:affinity:pk + for (int r = 1; r <= nrows; r++) { + const char *col_name = cols[r * 3]; + const char *col_type = cols[r * 3 + 1]; + const char *col_pk = cols[r * 3 + 2]; + + // Calculate required size: tbl_name:col_name:col_type:col_pk, + size_t entry_len = strlen(tbl_name) + 1 + strlen(col_name) + 1 + strlen(col_type) + 1 + strlen(col_pk) + 1; + + if (schema_len + entry_len + 1 > schema_cap) { + schema_cap = (schema_cap == 0) ? 1024 : schema_cap * 2; + if (schema_cap < schema_len + entry_len + 1) schema_cap = schema_len + entry_len + 1; + char *new_schema = cloudsync_memory_realloc(schema, schema_cap); + if (!new_schema) { + if (schema) cloudsync_memory_free(schema); + sqlite3_free_table(cols); + sqlite3_free_table(tables); + return SQLITE_NOMEM; + } + schema = new_schema; + } + + int written = snprintf(schema + schema_len, schema_cap - schema_len, "%s:%s:%s:%s,", + tbl_name, col_name, col_type, col_pk); + schema_len += written; + } + + sqlite3_free_table(cols); + } + + sqlite3_free_table(tables); + + if (!schema || schema_len == 0) return SQLITE_ERROR; + + // Remove trailing comma + if (schema_len > 0 && schema[schema_len - 1] == ',') { + schema[schema_len - 1] = '\0'; + schema_len--; + } + + DEBUG_MERGE("database_update_schema_hash len %zu schema %s", schema_len, schema); + sqlite3_uint64 h = fnv1a_hash(schema, schema_len); + cloudsync_memory_free(schema); + if (hash && *hash == h) return SQLITE_CONSTRAINT; + + char sql[1024]; + snprintf(sql, sizeof(sql), "INSERT INTO cloudsync_schema_versions (hash, seq) " + "VALUES (%" PRId64 ", COALESCE((SELECT MAX(seq) FROM cloudsync_schema_versions), 0) + 1) " + "ON CONFLICT(hash) DO UPDATE SET " + " seq = (SELECT COALESCE(MAX(seq), 0) + 1 FROM cloudsync_schema_versions);", (int64_t)h); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc == SQLITE_OK && hash) *hash = h; + return rc; +} + +// MARK: - VM - + +int databasevm_prepare (cloudsync_context *data, const char *sql, dbvm_t **vm, int flags) { + return sqlite3_prepare_v3((sqlite3 *)cloudsync_db(data), sql, -1, flags, (sqlite3_stmt **)vm, NULL); +} + +int databasevm_step (dbvm_t *vm) { + return sqlite3_step((sqlite3_stmt *)vm); +} + +void databasevm_finalize (dbvm_t *vm) { + sqlite3_finalize((sqlite3_stmt *)vm); +} + +void databasevm_reset (dbvm_t *vm) { + sqlite3_reset((sqlite3_stmt *)vm); +} + +void databasevm_clear_bindings (dbvm_t *vm) { + sqlite3_clear_bindings((sqlite3_stmt *)vm); +} + +const char *databasevm_sql (dbvm_t *vm) { + return sqlite3_sql((sqlite3_stmt *)vm); + // the following allocates memory that needs to be freed + // return sqlite3_expanded_sql((sqlite3_stmt *)vm); +} + +static int database_pk_rowid (sqlite3 *db, const char *table_name, char ***names, int *count) { + char buffer[2048]; + char *sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT rowid FROM %Q LIMIT 0;", table_name); + if (!sql) return SQLITE_NOMEM; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + { + char **r = (char**)cloudsync_memory_alloc(sizeof(char*)); + if (!r) {rc = SQLITE_NOMEM; goto cleanup;} + r[0] = cloudsync_string_dup("rowid"); + if (!r[0]) {cloudsync_memory_free(r); rc = SQLITE_NOMEM; goto cleanup;} + *names = r; + *count = 1; + } + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + +int database_pk_names (cloudsync_context *data, const char *table_name, char ***names, int *count) { + char buffer[2048]; + char *sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT name FROM pragma_table_info(%Q) WHERE pk > 0 ORDER BY pk;", table_name); + if (!sql) return SQLITE_NOMEM; + + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + sqlite3_stmt *vm = NULL; + + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // count PK columns + int rows = 0; + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) rows++; + if (rc != SQLITE_DONE) goto cleanup; + + if (rows == 0) { + sqlite3_finalize(vm); + // no declared PKs so check for rowid availability + return database_pk_rowid(db, table_name, names, count); + } + + // reset vm to read PKs again + rc = sqlite3_reset(vm); + if (rc != SQLITE_OK) goto cleanup; + + // allocate array + char **r = (char**)cloudsync_memory_zeroalloc(sizeof(char*) * rows); + if (!r) {rc = SQLITE_NOMEM; goto cleanup;} + + int i = 0; + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { + const char *txt = (const char*)sqlite3_column_text(vm, 0); + if (!txt) {rc = SQLITE_ERROR; goto cleanup_r;} + r[i] = cloudsync_string_dup(txt); + if (!r[i]) { rc = SQLITE_NOMEM; goto cleanup_r;} + i++; + } + if (rc == SQLITE_DONE) rc = SQLITE_OK; + + *names = r; + *count = rows; + goto cleanup; + +cleanup_r: + for (int j = 0; j < i; j++) { + if (r[j]) cloudsync_memory_free(r[j]); + } + cloudsync_memory_free(r); + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + +// MARK: - BINDING - + +int databasevm_bind_blob (dbvm_t *vm, int index, const void *value, uint64_t size) { + return sqlite3_bind_blob64((sqlite3_stmt *)vm, index, value, size, SQLITE_STATIC); +} + +int databasevm_bind_double (dbvm_t *vm, int index, double value) { + return sqlite3_bind_double((sqlite3_stmt *)vm, index, value); +} + +int databasevm_bind_int (dbvm_t *vm, int index, int64_t value) { + return sqlite3_bind_int64((sqlite3_stmt *)vm, index, value); +} + +int databasevm_bind_null (dbvm_t *vm, int index) { + return sqlite3_bind_null((sqlite3_stmt *)vm, index); +} + +int databasevm_bind_text (dbvm_t *vm, int index, const char *value, int size) { + return sqlite3_bind_text((sqlite3_stmt *)vm, index, value, size, SQLITE_STATIC); +} + +int databasevm_bind_value (dbvm_t *vm, int index, dbvalue_t *value) { + return sqlite3_bind_value((sqlite3_stmt *)vm, index, (const sqlite3_value *)value); +} + +// MARK: - VALUE - + +const void *database_value_blob (dbvalue_t *value) { + return sqlite3_value_blob((sqlite3_value *)value); +} + +double database_value_double (dbvalue_t *value) { + return sqlite3_value_double((sqlite3_value *)value); +} + +int64_t database_value_int (dbvalue_t *value) { + return (int64_t)sqlite3_value_int64((sqlite3_value *)value); +} + +const char *database_value_text (dbvalue_t *value) { + return (const char *)sqlite3_value_text((sqlite3_value *)value); +} + +int database_value_bytes (dbvalue_t *value) { + return sqlite3_value_bytes((sqlite3_value *)value); +} + +int database_value_type (dbvalue_t *value) { + return sqlite3_value_type((sqlite3_value *)value); +} + +void database_value_free (dbvalue_t *value) { + sqlite3_value_free((sqlite3_value *)value); +} + +void *database_value_dup (dbvalue_t *value) { + return sqlite3_value_dup((const sqlite3_value *)value); +} + + +// MARK: - COLUMN - + +const void *database_column_blob (dbvm_t *vm, int index, size_t *len) { + if (len) *len = sqlite3_column_bytes((sqlite3_stmt *)vm, index); + return sqlite3_column_blob((sqlite3_stmt *)vm, index); +} + +double database_column_double (dbvm_t *vm, int index) { + return sqlite3_column_double((sqlite3_stmt *)vm, index); +} + +int64_t database_column_int (dbvm_t *vm, int index) { + return (int64_t)sqlite3_column_int64((sqlite3_stmt *)vm, index); +} + +const char *database_column_text (dbvm_t *vm, int index) { + return (const char *)sqlite3_column_text((sqlite3_stmt *)vm, index); +} + +dbvalue_t *database_column_value (dbvm_t *vm, int index) { + return (dbvalue_t *)sqlite3_column_value((sqlite3_stmt *)vm, index); +} + +int database_column_bytes (dbvm_t *vm, int index) { + return sqlite3_column_bytes((sqlite3_stmt *)vm, index); +} + +int database_column_type (dbvm_t *vm, int index) { + return sqlite3_column_type((sqlite3_stmt *)vm, index); +} + +// MARK: - SAVEPOINT - + +int database_begin_savepoint (cloudsync_context *data, const char *savepoint_name) { + char sql[1024]; + snprintf(sql, sizeof(sql), "SAVEPOINT %s;", savepoint_name); + return database_exec(data, sql); +} + +int database_commit_savepoint (cloudsync_context *data, const char *savepoint_name) { + char sql[1024]; + snprintf(sql, sizeof(sql), "RELEASE %s;", savepoint_name); + return database_exec(data, sql); +} + +int database_rollback_savepoint (cloudsync_context *data, const char *savepoint_name) { + char sql[1024]; + snprintf(sql, sizeof(sql), "ROLLBACK TO %s; RELEASE %s;", savepoint_name, savepoint_name); + return database_exec(data, sql); +} + +// MARK: - MEMORY - + +void *dbmem_alloc (uint64_t size) { + return sqlite3_malloc64((sqlite3_uint64)size); +} + +void *dbmem_zeroalloc (uint64_t size) { + void *ptr = (void *)dbmem_alloc(size); + if (!ptr) return NULL; + + memset(ptr, 0, (size_t)size); + return ptr; +} + +void *dbmem_realloc (void *ptr, uint64_t new_size) { + return sqlite3_realloc64(ptr, (sqlite3_uint64)new_size); +} + +char *dbmem_vmprintf (const char *format, va_list list) { + return sqlite3_vmprintf(format, list); +} + +char *dbmem_mprintf(const char *format, ...) { + va_list ap; + char *z; + + va_start(ap, format); + z = dbmem_vmprintf(format, ap); + va_end(ap); + + return z; +} + +void dbmem_free (void *ptr) { + sqlite3_free(ptr); +} + +uint64_t dbmem_size (void *ptr) { + return (uint64_t)sqlite3_msize(ptr); +} + + diff --git a/src/sqlite/sql_sqlite.c b/src/sqlite/sql_sqlite.c new file mode 100644 index 00000000..a6b1d7ac --- /dev/null +++ b/src/sqlite/sql_sqlite.c @@ -0,0 +1,350 @@ +// +// sql_sqlite.c +// cloudsync +// +// Created by Marco Bambini on 17/12/25. +// + +#include "../sql.h" + +// MARK: Settings + +const char * const SQL_SETTINGS_GET_VALUE = + "SELECT value FROM cloudsync_settings WHERE key=?1;"; + +const char * const SQL_SETTINGS_SET_KEY_VALUE_REPLACE = + "REPLACE INTO cloudsync_settings (key, value) VALUES (?1, ?2);"; + +const char * const SQL_SETTINGS_SET_KEY_VALUE_DELETE = + "DELETE FROM cloudsync_settings WHERE key = ?1;"; + +const char * const SQL_TABLE_SETTINGS_GET_VALUE = + "SELECT value FROM cloudsync_table_settings WHERE (tbl_name=?1 AND col_name=?2 AND key=?3);"; + +const char * const SQL_TABLE_SETTINGS_DELETE_ALL_FOR_TABLE = + "DELETE FROM cloudsync_table_settings WHERE tbl_name=?1;"; + +const char * const SQL_TABLE_SETTINGS_REPLACE = + "REPLACE INTO cloudsync_table_settings (tbl_name, col_name, key, value) VALUES (?1, ?2, ?3, ?4);"; + +const char * const SQL_TABLE_SETTINGS_DELETE_ONE = + "DELETE FROM cloudsync_table_settings WHERE (tbl_name=?1 AND col_name=?2 AND key=?3);"; + +const char * const SQL_TABLE_SETTINGS_COUNT_TABLES = + "SELECT count(*) FROM cloudsync_table_settings WHERE key='algo';"; + +const char * const SQL_SETTINGS_LOAD_GLOBAL = + "SELECT key, value FROM cloudsync_settings;"; + +const char * const SQL_SETTINGS_LOAD_TABLE = + "SELECT lower(tbl_name), lower(col_name), key, value FROM cloudsync_table_settings ORDER BY tbl_name, col_name;"; + +const char * const SQL_CREATE_SETTINGS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_settings (key TEXT PRIMARY KEY NOT NULL COLLATE NOCASE, value TEXT);"; + +// format strings (sqlite3_snprintf) are also static SQL templates +const char * const SQL_INSERT_SETTINGS_STR_FORMAT = + "INSERT INTO cloudsync_settings (key, value) VALUES ('%q', '%q');"; + +const char * const SQL_INSERT_SETTINGS_INT_FORMAT = + "INSERT INTO cloudsync_settings (key, value) VALUES ('%s', %lld);"; + +const char * const SQL_CREATE_SITE_ID_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_site_id (site_id BLOB UNIQUE NOT NULL);"; + +const char * const SQL_INSERT_SITE_ID_ROWID = + "INSERT INTO cloudsync_site_id (rowid, site_id) VALUES (?, ?);"; + +const char * const SQL_CREATE_TABLE_SETTINGS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_table_settings (tbl_name TEXT NOT NULL COLLATE NOCASE, col_name TEXT NOT NULL COLLATE NOCASE, key TEXT, value TEXT, PRIMARY KEY(tbl_name,col_name,key));"; + +const char * const SQL_CREATE_SCHEMA_VERSIONS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_schema_versions (hash INTEGER PRIMARY KEY, seq INTEGER NOT NULL)"; + +const char * const SQL_SETTINGS_CLEANUP_DROP_ALL = + "DROP TABLE IF EXISTS cloudsync_settings; " + "DROP TABLE IF EXISTS cloudsync_site_id; " + "DROP TABLE IF EXISTS cloudsync_table_settings; " + "DROP TABLE IF EXISTS cloudsync_schema_versions; " + "DROP TABLE IF EXISTS cloudsync_payload_fragments; "; + +// MARK: CloudSync + +const char * const SQL_DBVERSION_BUILD_QUERY = + "WITH table_names AS (" + "SELECT format('%w', name) as tbl_name " + "FROM sqlite_master " + "WHERE type='table' " + "AND name LIKE '%_cloudsync'" + "), " + "query_parts AS (" + "SELECT 'SELECT max(db_version) as version FROM \"' || tbl_name || '\"' as part FROM table_names" + "), " + "combined_query AS (" + "SELECT GROUP_CONCAT(part, ' UNION ALL ') || ' UNION SELECT value as version FROM cloudsync_settings WHERE key = ''pre_alter_dbversion''' as full_query FROM query_parts" + ") " + "SELECT 'SELECT max(version) as version FROM (' || full_query || ');' FROM combined_query;"; + +const char * const SQL_SITEID_SELECT_ROWID0 = + "SELECT site_id FROM cloudsync_site_id WHERE rowid=0;"; + +const char * const SQL_DATA_VERSION = + "PRAGMA data_version;"; + +const char * const SQL_SCHEMA_VERSION = + "PRAGMA schema_version;"; + +const char * const SQL_SITEID_GETSET_ROWID_BY_SITEID = + "INSERT INTO cloudsync_site_id (site_id) VALUES (?) " + "ON CONFLICT(site_id) DO UPDATE SET site_id = site_id " + "RETURNING rowid;"; + +// Format +const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_ROWID = + "WITH col_names AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"', ',') AS cols " + "FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid" + ") " + "SELECT 'SELECT ' || (SELECT cols FROM col_names) || ' FROM \"%w\" WHERE rowid=?;'"; + +const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_PK = + "WITH col_names AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"', ',') AS cols " + "FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid" + "), " + "pk_where AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + ") " + "SELECT 'SELECT ' || (SELECT cols FROM col_names) || ' FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'"; + +const char * const SQL_DELETE_ROW_BY_ROWID = + "DELETE FROM \"%w\" WHERE rowid=?;"; + +const char * const SQL_BUILD_DELETE_ROW_BY_PK = + "WITH pk_where AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + ") " + "SELECT 'DELETE FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'"; + +const char * const SQL_INSERT_ROWID_IGNORE = + "INSERT OR IGNORE INTO \"%w\" (rowid) VALUES (?);"; + +const char * const SQL_UPSERT_ROWID_AND_COL_BY_ROWID = + "INSERT INTO \"%w\" (rowid, \"%w\") VALUES (?, ?) ON CONFLICT DO UPDATE SET \"%w\"=?;"; + +const char * const SQL_BUILD_INSERT_PK_IGNORE = + "WITH pk_where AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"') AS pk_clause " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + "), " + "pk_bind AS (" + "SELECT group_concat('?') AS pk_binding " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + ") " + "SELECT 'INSERT OR IGNORE INTO \"%w\" (' || (SELECT pk_clause FROM pk_where) || ') VALUES (' || (SELECT pk_binding FROM pk_bind) || ');'"; + +const char * const SQL_BUILD_UPSERT_PK_AND_COL = + "WITH pk_where AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"') AS pk_clause " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + "), " + "pk_bind AS (" + "SELECT group_concat('?') AS pk_binding " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + ") " + "SELECT 'INSERT INTO \"%w\" (' || (SELECT pk_clause FROM pk_where) || ',\"%w\") VALUES (' || (SELECT pk_binding FROM pk_bind) || ',?) ON CONFLICT DO UPDATE SET \"%w\"=?;'"; + +const char * const SQL_SELECT_COLS_BY_ROWID_FMT = + "SELECT %s%w%s FROM \"%w\" WHERE rowid=?;"; + +const char * const SQL_BUILD_SELECT_COLS_BY_PK_FMT = + "WITH pk_where AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + ") " + "SELECT 'SELECT %s%w%s FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'"; + +const char * const SQL_CLOUDSYNC_ROW_EXISTS_BY_PK = + "SELECT EXISTS(SELECT 1 FROM \"%w\" WHERE pk = ? LIMIT 1);"; + +const char * const SQL_CLOUDSYNC_UPDATE_COL_BUMP_VERSION = + "UPDATE \"%w\" " + "SET col_version = CASE col_version %% 2 WHEN 0 THEN col_version + 1 ELSE col_version + 2 END, " + "db_version = ?, seq = ?, site_id = 0 " + "WHERE pk = ? AND col_name = '%s';"; + +const char * const SQL_CLOUDSYNC_UPSERT_COL_INIT_OR_BUMP_VERSION = + "INSERT INTO \"%w\" (pk, col_name, col_version, db_version, seq, site_id) " + "SELECT ?, '%s', 1, ?, ?, 0 " + "WHERE 1 " + "ON CONFLICT DO UPDATE SET " + "col_version = CASE col_version %% 2 WHEN 0 THEN col_version + 1 ELSE col_version + 2 END, " + "db_version = ?, seq = ?, site_id = 0;"; + +const char * const SQL_CLOUDSYNC_UPSERT_RAW_COLVERSION = + "INSERT INTO \"%w\" (pk, col_name, col_version, db_version, seq, site_id ) " + "SELECT ?, ?, ?, ?, ?, 0 " + "WHERE 1 " + "ON CONFLICT DO UPDATE SET " + "col_version = \"%w\".col_version + 1, db_version = ?, seq = ?, site_id = 0;"; + +const char * const SQL_CLOUDSYNC_DELETE_PK_EXCEPT_COL = + "DELETE FROM \"%w\" WHERE pk=? AND col_name!='%s';"; + +const char * const SQL_CLOUDSYNC_REKEY_PK_AND_RESET_VERSION_EXCEPT_COL = + "UPDATE OR REPLACE \"%w\" " + "SET pk=?, db_version=?, col_version=1, seq=cloudsync_seq(), site_id=0 " + "WHERE (pk=? AND col_name!='%s');"; + +const char * const SQL_CLOUDSYNC_GET_COL_VERSION_OR_ROW_EXISTS = + "SELECT COALESCE(" + "(SELECT col_version FROM \"%w\" WHERE pk=? AND col_name='%s'), " + "(SELECT 1 FROM \"%w\" WHERE pk=?)" + ");"; + +const char * const SQL_CLOUDSYNC_INSERT_RETURN_CHANGE_ID = + "INSERT OR REPLACE INTO \"%w\" " + "(pk, col_name, col_version, db_version, seq, site_id) " + "VALUES (?, ?, ?, cloudsync_db_version_next(?), ?, ?) " + "RETURNING ((db_version << 30) | seq);"; + +const char * const SQL_CLOUDSYNC_TOMBSTONE_PK_EXCEPT_COL = + "UPDATE \"%w\" " + "SET col_version = 0, db_version = cloudsync_db_version_next(?) " + "WHERE pk=? AND col_name!='%s';"; + +const char * const SQL_CLOUDSYNC_SELECT_COL_VERSION_BY_PK_COL = + "SELECT col_version FROM \"%w\" WHERE pk=? AND col_name=?;"; + +const char * const SQL_CLOUDSYNC_SELECT_SITE_ID_BY_PK_COL = + "SELECT site_id FROM \"%w\" WHERE pk=? AND col_name=?;"; + +const char * const SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID = + "SELECT name, cid FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid;"; + +const char * const SQL_DROP_CLOUDSYNC_TABLE = + "DROP TABLE IF EXISTS \"%w\";"; + +const char * const SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE = + "DELETE FROM \"%w\";"; + +const char * const SQL_CLOUDSYNC_DELETE_COLS_NOT_IN_SCHEMA_OR_PKCOL = + "DELETE FROM \"%w\" WHERE \"col_name\" NOT IN (" + "SELECT name FROM pragma_table_info('%q') UNION SELECT '%s'" + ")"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_QUALIFIED_COLLIST_FMT = + "SELECT group_concat('\"%w\".\"' || format('%%w', name) || '\"', ',') " + "FROM pragma_table_info('%s') WHERE pk>0 ORDER BY pk;"; + +const char * const SQL_CLOUDSYNC_GC_DELETE_ORPHANED_PK = + "DELETE FROM \"%w\" " + "WHERE (\"col_name\" != '%s' OR (\"col_name\" = '%s' AND col_version %% 2 != 0)) " + "AND NOT EXISTS (" + "SELECT 1 FROM \"%w\" " + "WHERE \"%w\".pk = cloudsync_pk_encode(%s) LIMIT 1" + ");"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_COLLIST = + "SELECT group_concat('\"' || format('%%w', name) || '\"', ',') " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_DECODE_SELECTLIST = + "SELECT group_concat(" + "'cloudsync_pk_decode(pk, ' || pk || ') AS ' || '\"' || format('%%w', name) || '\"', ','" + ") " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;"; + +const char * const SQL_CLOUDSYNC_INSERT_MISSING_PKS_FROM_BASE_EXCEPT_SYNC = + "SELECT cloudsync_insert('%q', %s) " + "FROM (SELECT %s FROM \"%w\" EXCEPT SELECT %s FROM \"%w\");"; + +const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL = + "WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM \"%w\") " + "SELECT _cstemp1.pk FROM _cstemp1 " + "WHERE NOT EXISTS (" + "SELECT 1 FROM \"%w\" _cstemp2 " + "WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = ?" + ");"; + +const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL_FILTERED = + "WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM \"%w\" WHERE (%s)) " + "SELECT _cstemp1.pk FROM _cstemp1 " + "WHERE NOT EXISTS (" + "SELECT 1 FROM \"%w\" _cstemp2 " + "WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = ?" + ");"; + +const char * const SQL_CHANGES_INSERT_ROW = + "INSERT INTO cloudsync_changes(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) " + "VALUES (?,?,?,?,?,?,?,?,?);"; + +const char * const SQL_PAYLOAD_FRAGMENTS_CREATE_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_payload_fragments (" + "value_id TEXT NOT NULL, part_index INTEGER NOT NULL, part_count INTEGER NOT NULL, total_size INTEGER NOT NULL, " + "checksum TEXT NOT NULL, created_at INTEGER NOT NULL DEFAULT (unixepoch()), " + "tbl TEXT NOT NULL, pk BLOB NOT NULL, col_name TEXT NOT NULL, col_version INTEGER NOT NULL, db_version INTEGER NOT NULL, " + "site_id BLOB NOT NULL, cl INTEGER NOT NULL, seq INTEGER NOT NULL, fragment BLOB NOT NULL, " + "PRIMARY KEY(value_id, part_index)) WITHOUT ROWID;"; + +const char * const SQL_PAYLOAD_FRAGMENTS_UPSERT = + "INSERT OR REPLACE INTO cloudsync_payload_fragments " + "(value_id, part_index, part_count, total_size, checksum, created_at, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, fragment) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);"; + +const char * const SQL_PAYLOAD_FRAGMENTS_COUNT = + "SELECT COUNT(*), MIN(part_count), MAX(part_count), MIN(total_size), MAX(total_size), " + "MIN(checksum), MAX(checksum), MIN(part_index), MAX(part_index) " + "FROM cloudsync_payload_fragments WHERE value_id=?;"; + +const char * const SQL_PAYLOAD_FRAGMENTS_SELECT = + "SELECT fragment, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, checksum " + "FROM cloudsync_payload_fragments WHERE value_id=? ORDER BY part_index ASC;"; + +const char * const SQL_PAYLOAD_FRAGMENTS_DELETE = + "DELETE FROM cloudsync_payload_fragments WHERE value_id=?;"; + +const char * const SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE = + "DELETE FROM cloudsync_payload_fragments " + "WHERE created_at < ? AND value_id IN (" + "SELECT value_id FROM cloudsync_payload_fragments GROUP BY value_id " + "HAVING COUNT(*) < MAX(part_count));"; + +// MARK: Blocks (block-level LWW) + +const char * const SQL_BLOCKS_CREATE_TABLE = + "CREATE TABLE IF NOT EXISTS %s (" + "pk BLOB NOT NULL, " + "col_name TEXT NOT NULL, " + "col_value BLOB, " + "PRIMARY KEY (pk, col_name)) WITHOUT ROWID"; + +const char * const SQL_BLOCKS_UPSERT = + "INSERT OR REPLACE INTO %s (pk, col_name, col_value) VALUES (?1, ?2, ?3)"; + +const char * const SQL_BLOCKS_SELECT = + "SELECT col_value FROM %s WHERE pk = ?1 AND col_name = ?2"; + +const char * const SQL_BLOCKS_DELETE = + "DELETE FROM %s WHERE pk = ?1 AND col_name = ?2"; + +const char * const SQL_BLOCKS_LIST_ALIVE = + "SELECT b.col_value FROM %s b " + "JOIN %s m ON b.pk = m.pk AND b.col_name = m.col_name " + "WHERE b.pk = ?1 AND b.col_name LIKE ?2 " + "AND m.pk = ?3 AND m.col_name LIKE ?4 AND m.col_version %% 2 = 1 " + "ORDER BY b.col_name"; + +const char * const SQL_BLOCKS_INSERT_IGNORE = + "INSERT OR IGNORE INTO %s (pk, col_name, col_value) VALUES (?1, ?2, ?3)"; + +const char * const SQL_META_SCAN_COL_FOR_MIGRATION = + "SELECT DISTINCT m.pk FROM %s m " + "WHERE m.col_name = ?1 AND m.col_version %% 2 = 1 " + "AND NOT EXISTS (SELECT 1 FROM %s b WHERE b.pk = m.pk AND b.col_name LIKE ?2)"; + +const char * const SQL_META_INSERT_BLOCK_IGNORE = + "INSERT OR IGNORE INTO %s (pk, col_name, col_version, db_version, seq, site_id) " + "VALUES (?1, ?2, ?3, ?4, ?5, 0)"; diff --git a/src/sqlite3ext.h b/src/sqlite/sqlite3ext.h similarity index 100% rename from src/sqlite3ext.h rename to src/sqlite/sqlite3ext.h diff --git a/src/utils.c b/src/utils.c index b5e0de74..371e29c9 100644 --- a/src/utils.c +++ b/src/utils.c @@ -18,7 +18,7 @@ #define file_close _close #else #include -#if defined(__APPLE__) +#if defined(__APPLE__) && !defined(CLOUDSYNC_POSTGRESQL_BUILD) #include #elif !defined(__ANDROID__) #include @@ -33,15 +33,11 @@ #include #endif -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT3 -#endif - #define FNV_OFFSET_BASIS 0xcbf29ce484222325ULL #define FNV_PRIME 0x100000001b3ULL #define HASH_CHAR(_c) do { h ^= (uint8_t)(_c); h *= FNV_PRIME; h_final = h;} while (0) -// MARK: UUIDv7 - +// MARK: - UUIDv7 - /* UUIDv7 is a 128-bit unique identifier like it's older siblings, such as the widely used UUIDv4. @@ -61,9 +57,12 @@ int cloudsync_uuid_v7 (uint8_t value[UUID_LEN]) { // fill the buffer with high-quality random data #ifdef _WIN32 if (BCryptGenRandom(NULL, (BYTE*)value, UUID_LEN, BCRYPT_USE_SYSTEM_PREFERRED_RNG) != STATUS_SUCCESS) return -1; - #elif defined(__APPLE__) + #elif defined(__APPLE__) && !defined(CLOUDSYNC_POSTGRESQL_BUILD) // Use SecRandomCopyBytes for macOS/iOS if (SecRandomCopyBytes(kSecRandomDefault, UUID_LEN, value) != errSecSuccess) return -1; + #elif defined(__APPLE__) && defined(CLOUDSYNC_POSTGRESQL_BUILD) + // PostgreSQL build: use getentropy to avoid Security.framework type conflicts + if (getentropy(value, UUID_LEN) != 0) return -1; #elif defined(__ANDROID__) //arc4random_buf doesn't have a return value to check for success arc4random_buf(value, UUID_LEN); @@ -113,15 +112,46 @@ char *cloudsync_uuid_v7_stringify (uint8_t uuid[UUID_LEN], char value[UUID_STR_M char *cloudsync_uuid_v7_string (char value[UUID_STR_MAXLEN], bool dash_format) { uint8_t uuid[UUID_LEN]; + if (cloudsync_uuid_v7(uuid) != 0) return NULL; - return cloudsync_uuid_v7_stringify(uuid, value, dash_format); } +static int cloudsync_hex_nibble (char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +int cloudsync_uuid_v7_parse (const char *str, int len, uint8_t out[UUID_LEN]) { + if (!str || !out) return -1; + if (len < 0) len = (int)strlen(str); + + // Accept the canonical dashed form (8-4-4-4-12) or bare 32-hex; dashes, + // if present, must be at the canonical positions. Parse 32 hex nibbles. + int nibbles = 0; + for (int i = 0; i < len; ++i) { + char c = str[i]; + if (c == '-') continue; + int hi = cloudsync_hex_nibble(c); + if (hi < 0) return -1; + if (i + 1 >= len) return -1; + int lo = cloudsync_hex_nibble(str[i + 1]); + if (lo < 0) return -1; + if (nibbles >= UUID_LEN) return -1; + out[nibbles++] = (uint8_t)((hi << 4) | lo); + ++i; // consumed the low nibble too + } + return (nibbles == UUID_LEN) ? 0 : -1; +} + int cloudsync_uuid_v7_compare (uint8_t value1[UUID_LEN], uint8_t value2[UUID_LEN]) { // reconstruct the timestamp by reversing the bit shifts and combining the bytes - uint64_t t1 = ((uint64_t)value1[0] << 40) | ((uint64_t)value1[1] << 32) | ((uint64_t)value1[2] << 24) | ((uint64_t)value1[3] << 16) | ((uint64_t)value1[4] << 8) | ((uint64_t)value1[5]); - uint64_t t2 = ((uint64_t)value2[0] << 40) | ((uint64_t)value2[1] << 32) | ((uint64_t)value2[2] << 24) | ((uint64_t)value2[3] << 16) | ((uint64_t)value2[4] << 8) | ((uint64_t)value2[5]); + uint64_t t1 = ((uint64_t)value1[0] << 40) | ((uint64_t)value1[1] << 32) | ((uint64_t)value1[2] << 24) | + ((uint64_t)value1[3] << 16) | ((uint64_t)value1[4] << 8) | ((uint64_t)value1[5]); + uint64_t t2 = ((uint64_t)value2[0] << 40) | ((uint64_t)value2[1] << 32) | ((uint64_t)value2[2] << 24) | + ((uint64_t)value2[3] << 16) | ((uint64_t)value2[4] << 8) | ((uint64_t)value2[5]); if (t1 == t2) return memcmp(value1, value2, UUID_LEN); return (t1 > t2) ? 1 : -1; @@ -129,24 +159,16 @@ int cloudsync_uuid_v7_compare (uint8_t value1[UUID_LEN], uint8_t value2[UUID_LEN // MARK: - General - -void *cloudsync_memory_zeroalloc (uint64_t size) { - void *ptr = (void *)cloudsync_memory_alloc((sqlite3_uint64)size); - if (!ptr) return NULL; - - memset(ptr, 0, (size_t)size); - return ptr; -} - -char *cloudsync_string_ndup (const char *str, size_t len, bool lowercase) { +char *cloudsync_string_ndup_v2 (const char *str, size_t len, bool lowercase) { if (str == NULL) return NULL; - char *s = (char *)cloudsync_memory_alloc((sqlite3_uint64)(len + 1)); + char *s = (char *)cloudsync_memory_alloc((uint64_t)(len + 1)); if (!s) return NULL; if (lowercase) { // convert each character to lowercase and copy it to the new string for (size_t i = 0; i < len; i++) { - s[i] = tolower(str[i]); + s[i] = (char)tolower(str[i]); } } else { memcpy(s, str, len); @@ -158,35 +180,42 @@ char *cloudsync_string_ndup (const char *str, size_t len, bool lowercase) { return s; } -char *cloudsync_string_dup (const char *str, bool lowercase) { - if (str == NULL) return NULL; - - size_t len = strlen(str); - return cloudsync_string_ndup(str, len, lowercase); +char *cloudsync_string_ndup (const char *str, size_t len) { + return cloudsync_string_ndup_v2(str, len, false); +} + +char *cloudsync_string_ndup_lowercase (const char *str, size_t len) { + return cloudsync_string_ndup_v2(str, len, true); +} + +char *cloudsync_string_dup (const char *str) { + return cloudsync_string_ndup_v2(str, (str) ? strlen(str) : 0, false); +} + +char *cloudsync_string_dup_lowercase (const char *str) { + return cloudsync_string_ndup_v2(str, (str) ? strlen(str) : 0, true); } int cloudsync_blob_compare(const char *blob1, size_t size1, const char *blob2, size_t size2) { - if (size1 != size2) { - return (int)(size1 - size2); // Blobs are different if sizes are different - } - return memcmp(blob1, blob2, size1); // Use memcmp for byte-by-byte comparison + if (size1 != size2) return (size1 > size2) ? 1 : -1; // blobs are different if sizes are different + return memcmp(blob1, blob2, size1); // use memcmp for byte-by-byte comparison } -void cloudsync_rowid_decode (sqlite3_int64 rowid, sqlite3_int64 *db_version, sqlite3_int64 *seq) { +void cloudsync_rowid_decode (int64_t rowid, int64_t *db_version, int64_t *seq) { // use unsigned 64-bit integer for intermediate calculations // when db_version is large enough, it can cause overflow, leading to negative values // to handle this correctly, we need to ensure the calculations are done in an unsigned 64-bit integer context - // before converting back to sqlite3_int64 as needed + // before converting back to int64_t as needed uint64_t urowid = (uint64_t)rowid; // define the bit mask for seq (30 bits) const uint64_t SEQ_MASK = 0x3FFFFFFF; // (2^30 - 1) // extract seq by masking the lower 30 bits - *seq = (sqlite3_int64)(urowid & SEQ_MASK); + *seq = (int64_t)(urowid & SEQ_MASK); // extract db_version by shifting 30 bits to the right - *db_version = (sqlite3_int64)(urowid >> 30); + *db_version = (int64_t)(urowid >> 30); } char *cloudsync_string_replace_prefix(const char *input, char *prefix, char *replacement) { @@ -196,13 +225,13 @@ char *cloudsync_string_replace_prefix(const char *input, char *prefix, char *rep size_t replacement_len = strlen(replacement); if (strncmp(input, prefix, prefix_len) == 0) { - // Allocate memory for new string + // allocate memory for new string size_t input_len = strlen(input); size_t new_len = input_len - prefix_len + replacement_len; char *result = cloudsync_memory_alloc(new_len + 1); // +1 for null terminator if (!result) return NULL; - // Copy replacement and the rest of the input string + // copy replacement and the rest of the input string strcpy(result, replacement); strcpy(result + replacement_len, input + prefix_len); return result; @@ -213,7 +242,7 @@ char *cloudsync_string_replace_prefix(const char *input, char *prefix, char *rep } /* - Compute a normalized hash of a SQLite CREATE TABLE statement. + Compute a normalized hash of a CREATE TABLE statement. * Normalization: * - Skips comments (-- and / * ) @@ -322,7 +351,7 @@ static bool cloudsync_file_read_all (int fd, char *buf, size_t n) { return true; } -char *cloudsync_file_read (const char *path, sqlite3_int64 *len) { +char *cloudsync_file_read (const char *path, int64_t *len) { int fd = -1; char *buffer = NULL; @@ -409,31 +438,6 @@ bool cloudsync_file_write (const char *path, const char *buffer, size_t len) { #endif -// MARK: - CRDT algos - - -table_algo crdt_algo_from_name (const char *algo_name) { - if (algo_name == NULL) return table_algo_none; - - if ((strcasecmp(algo_name, "CausalLengthSet") == 0) || (strcasecmp(algo_name, "cls") == 0)) return table_algo_crdt_cls; - if ((strcasecmp(algo_name, "GrowOnlySet") == 0) || (strcasecmp(algo_name, "gos") == 0)) return table_algo_crdt_gos; - if ((strcasecmp(algo_name, "DeleteWinsSet") == 0) || (strcasecmp(algo_name, "dws") == 0)) return table_algo_crdt_dws; - if ((strcasecmp(algo_name, "AddWinsSet") == 0) || (strcasecmp(algo_name, "aws") == 0)) return table_algo_crdt_aws; - - // if nothing is found - return table_algo_none; -} - -const char *crdt_algo_name (table_algo algo) { - switch (algo) { - case table_algo_crdt_cls: return "cls"; - case table_algo_crdt_gos: return "gos"; - case table_algo_crdt_dws: return "dws"; - case table_algo_crdt_aws: return "aws"; - case table_algo_none: return NULL; - } - return NULL; -} - // MARK: - Memory Debugger - #if CLOUDSYNC_DEBUG_MEMORY @@ -620,10 +624,10 @@ void memdebug_finalize (void) { } } -void *memdebug_alloc (sqlite3_uint64 size) { - void *ptr = sqlite3_malloc64(size); +void *memdebug_alloc (uint64_t size) { + void *ptr = dbmem_alloc(size); if (!ptr) { - BUILD_ERROR("Unable to allocated a block of %lld bytes", size); + BUILD_ERROR("Unable to allocated a block of %" PRIu64" bytes", size); BUILD_STACK(n, stack); memdebug_report(current_error, stack, n, NULL); return NULL; @@ -632,7 +636,15 @@ void *memdebug_alloc (sqlite3_uint64 size) { return ptr; } -void *memdebug_realloc (void *ptr, sqlite3_uint64 new_size) { +void *memdebug_zeroalloc (uint64_t size) { + void *ptr = memdebug_alloc(size); + if (!ptr) return NULL; + + memset(ptr, 0, (size_t)size); + return ptr; +} + +void *memdebug_realloc (void *ptr, uint64_t new_size) { if (!ptr) return memdebug_alloc(new_size); mem_slot *slot = _ptr_lookup(ptr); @@ -644,9 +656,9 @@ void *memdebug_realloc (void *ptr, sqlite3_uint64 new_size) { } void *back_ptr = ptr; - void *new_ptr = sqlite3_realloc64(ptr, new_size); + void *new_ptr = dbmem_realloc(ptr, new_size); if (!new_ptr) { - BUILD_ERROR("Unable to reallocate a block of %lld bytes.", new_size); + BUILD_ERROR("Unable to reallocate a block of %" PRIu64 " bytes.", new_size); BUILD_STACK(n, stack); memdebug_report(current_error, stack, n, slot); return NULL; @@ -657,15 +669,15 @@ void *memdebug_realloc (void *ptr, sqlite3_uint64 new_size) { } char *memdebug_vmprintf (const char *format, va_list list) { - char *ptr = sqlite3_vmprintf(format, list); + char *ptr = dbmem_vmprintf(format, list); if (!ptr) { - BUILD_ERROR("Unable to allocated for sqlite3_vmprintf with format %s", format); + BUILD_ERROR("Unable to allocated for dbmem_vmprintf with format %s", format); BUILD_STACK(n, stack); memdebug_report(current_error, stack, n, NULL); return NULL; } - _ptr_add(ptr, sqlite3_msize(ptr)); + _ptr_add(ptr, dbmem_size(ptr)); return ptr; } @@ -680,8 +692,8 @@ char *memdebug_mprintf(const char *format, ...) { return z; } -sqlite3_uint64 memdebug_msize (void *ptr) { - return sqlite3_msize(ptr); +uint64_t memdebug_msize (void *ptr) { + return dbmem_size(ptr); } void memdebug_free (void *ptr) { @@ -709,7 +721,7 @@ void memdebug_free (void *ptr) { } _ptr_remove(ptr); - sqlite3_free(ptr); + dbmem_free(ptr); } #endif diff --git a/src/utils.h b/src/utils.h index d526d86d..f71b1fe6 100644 --- a/src/utils.h +++ b/src/utils.h @@ -14,6 +14,7 @@ #include #include #include +#include "database.h" // CLOUDSYNC_DESKTOP_OS = 1 if compiling for macOS, Linux (desktop), or Windows // Not set for iOS, Android, WebAssembly, or other platforms @@ -28,12 +29,6 @@ #define CLOUDSYNC_DESKTOP_OS 1 #endif -#ifndef SQLITE_CORE -#include "sqlite3ext.h" -#else -#include "sqlite3.h" -#endif - #define CLOUDSYNC_DEBUG_FUNCTIONS 0 #define CLOUDSYNC_DEBUG_DBFUNCTIONS 0 #define CLOUDSYNC_DEBUG_SETTINGS 0 @@ -43,49 +38,60 @@ #define CLOUDSYNC_DEBUG_STMT 0 #define CLOUDSYNC_DEBUG_MERGE 0 -#define DEBUG_RUNTIME(...) do {if (data->debug) printf(__VA_ARGS__ );} while (0) -#define DEBUG_PRINTLN(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) -#define DEBUG_ALWAYS(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) -#define DEBUG_PRINT(...) do {printf(__VA_ARGS__ );} while (0) +// Debug macros - platform-specific logging +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + // PostgreSQL build - use elog() for logging + #include "postgresql/postgresql_log.h" + #define DEBUG_RUNTIME(...) do {if (data->debug) CLOUDSYNC_LOG_DEBUG(__VA_ARGS__ );} while (0) + #define DEBUG_PRINTLN(...) CLOUDSYNC_LOG_DEBUG(__VA_ARGS__) + #define DEBUG_ALWAYS(...) CLOUDSYNC_LOG_INFO(__VA_ARGS__) + #define DEBUG_PRINT(...) CLOUDSYNC_LOG_DEBUG(__VA_ARGS__) +#else + // SQLite and other platforms use printf() + #define DEBUG_RUNTIME(...) do {if (data->debug) printf(__VA_ARGS__ );} while (0) + #define DEBUG_PRINTLN(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) + #define DEBUG_ALWAYS(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) + #define DEBUG_PRINT(...) do {printf(__VA_ARGS__ );} while (0) +#endif #if CLOUDSYNC_DEBUG_FUNCTIONS -#define DEBUG_FUNCTION(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) +#define DEBUG_FUNCTION(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_FUNCTION(...) #endif -#if CLOUDSYNC_DEBUG_DBFUNCTION -#define DEBUG_DBFUNCTION(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) +#if CLOUDSYNC_DEBUG_DBFUNCTIONS +#define DEBUG_DBFUNCTION(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_DBFUNCTION(...) #endif #if CLOUDSYNC_DEBUG_SETTINGS -#define DEBUG_SETTINGS(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) +#define DEBUG_SETTINGS(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_SETTINGS(...) #endif #if CLOUDSYNC_DEBUG_SQL -#define DEBUG_SQL(...) do {printf(__VA_ARGS__ );printf("\n\n");} while (0) +#define DEBUG_SQL(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_SQL(...) #endif #if CLOUDSYNC_DEBUG_VTAB -#define DEBUG_VTAB(...) do {printf(__VA_ARGS__ );printf("\n\n");} while (0) +#define DEBUG_VTAB(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_VTAB(...) #endif #if CLOUDSYNC_DEBUG_STMT -#define DEBUG_STMT(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) +#define DEBUG_STMT(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_STMT(...) #endif #if CLOUDSYNC_DEBUG_MERGE -#define DEBUG_MERGE(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) +#define DEBUG_MERGE(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_MERGE(...) #endif @@ -94,64 +100,59 @@ #define cloudsync_memory_init(_once) memdebug_init(_once) #define cloudsync_memory_finalize memdebug_finalize #define cloudsync_memory_alloc memdebug_alloc +#define cloudsync_memory_zeroalloc memdebug_zeroalloc #define cloudsync_memory_free memdebug_free #define cloudsync_memory_realloc memdebug_realloc #define cloudsync_memory_size memdebug_msize -#define cloudsync_memory_vmprintf memdebug_vmprintf #define cloudsync_memory_mprintf memdebug_mprintf void memdebug_init (int once); void memdebug_finalize (void); -void *memdebug_alloc (sqlite3_uint64 size); -void *memdebug_realloc (void *ptr, sqlite3_uint64 new_size); +void *memdebug_alloc (uint64_t size); +void *memdebug_zeroalloc (uint64_t size); +void *memdebug_realloc (void *ptr, uint64_t new_size); char *memdebug_vmprintf (const char *format, va_list list); char *memdebug_mprintf(const char *format, ...); void memdebug_free (void *ptr); -sqlite3_uint64 memdebug_msize (void *ptr); +uint64_t memdebug_msize (void *ptr); #else #define cloudsync_memory_init(_once) #define cloudsync_memory_finalize() -#define cloudsync_memory_alloc sqlite3_malloc64 -#define cloudsync_memory_free sqlite3_free -#define cloudsync_memory_realloc sqlite3_realloc64 -#define cloudsync_memory_size sqlite3_msize -#define cloudsync_memory_vmprintf sqlite3_vmprintf -#define cloudsync_memory_mprintf sqlite3_mprintf +#define cloudsync_memory_alloc dbmem_alloc +#define cloudsync_memory_zeroalloc dbmem_zeroalloc +#define cloudsync_memory_free dbmem_free +#define cloudsync_memory_realloc dbmem_realloc +#define cloudsync_memory_size dbmem_size +#define cloudsync_memory_mprintf dbmem_mprintf #endif #define UUID_STR_MAXLEN 37 #define UUID_LEN 16 -// The type of CRDT chosen for a table controls what rows are included or excluded when merging tables together from different databases -typedef enum { - table_algo_none = 0, - table_algo_crdt_cls = 100, // CausalLengthSet - table_algo_crdt_gos, // GrowOnlySet - table_algo_crdt_dws, // DeleteWinsSet - table_algo_crdt_aws // AddWinsSet -} table_algo; - -table_algo crdt_algo_from_name (const char *name); -const char *crdt_algo_name (table_algo algo); - int cloudsync_uuid_v7 (uint8_t value[UUID_LEN]); int cloudsync_uuid_v7_compare (uint8_t value1[UUID_LEN], uint8_t value2[UUID_LEN]); char *cloudsync_uuid_v7_string (char value[UUID_STR_MAXLEN], bool dash_format); char *cloudsync_uuid_v7_stringify (uint8_t uuid[UUID_LEN], char value[UUID_STR_MAXLEN], bool dash_format); -char *cloudsync_string_replace_prefix(const char *input, char *prefix, char *replacement); +// Parse a UUID string into its UUID_LEN binary form. Tolerant: accepts the +// canonical dashed form (36 chars) and the bare 32-hex form, case-insensitive. +// Returns 0 on success, -1 on malformed input. +int cloudsync_uuid_v7_parse (const char *str, int len, uint8_t out[UUID_LEN]); uint64_t fnv1a_hash(const char *data, size_t len); -void *cloudsync_memory_zeroalloc (uint64_t size); -char *cloudsync_string_ndup (const char *str, size_t len, bool lowercase); -char *cloudsync_string_dup (const char *str, bool lowercase); +char *cloudsync_string_replace_prefix(const char *input, char *prefix, char *replacement); +char *cloudsync_string_dup (const char *str); +char *cloudsync_string_dup_lowercase (const char *str); +char *cloudsync_string_ndup (const char *str, size_t len); +char *cloudsync_string_ndup_lowercase (const char *str, size_t len); + int cloudsync_blob_compare(const char *blob1, size_t size1, const char *blob2, size_t size2); -void cloudsync_rowid_decode (sqlite3_int64 rowid, sqlite3_int64 *db_version, sqlite3_int64 *seq); +void cloudsync_rowid_decode (int64_t rowid, int64_t *db_version, int64_t *seq); -// available only on Desktop OS +// available only on Desktop OS (no WASM, no mobile) #ifdef CLOUDSYNC_DESKTOP_OS bool cloudsync_file_delete (const char *path); -char *cloudsync_file_read (const char *path, sqlite3_int64 *len); +char *cloudsync_file_read (const char *path, int64_t *len); bool cloudsync_file_write (const char *path, const char *buffer, size_t len); #endif diff --git a/src/vtab.h b/src/vtab.h deleted file mode 100644 index 0c9bd643..00000000 --- a/src/vtab.h +++ /dev/null @@ -1,18 +0,0 @@ -// -// vtab.h -// cloudsync -// -// Created by Marco Bambini on 23/09/24. -// - -#ifndef __CLOUDSYNC_VTAB__ -#define __CLOUDSYNC_VTAB__ - -#include "cloudsync.h" -#include "cloudsync_private.h" - -int cloudsync_vtab_register_changes (sqlite3 *db, cloudsync_context *xdata); -cloudsync_context *cloudsync_vtab_get_context (sqlite3_vtab *vtab); -int cloudsync_vtab_set_error (sqlite3_vtab *vtab, const char *format, ...); - -#endif diff --git a/test/chunk_bench.c b/test/chunk_bench.c new file mode 100644 index 00000000..4321ac2f --- /dev/null +++ b/test/chunk_bench.c @@ -0,0 +1,177 @@ +// +// chunk_bench.c +// cloudsync +// +// Local-only benchmark for the positional /check drain: build a window of N +// chunks and time paging the whole window one chunk per call via the +// (resume_db_version, resume_seq, resume_frag_offset) cursor on +// cloudsync_payload_chunks. Reports wall time and per-chunk cost so the +// computational growth of the drain (currently O(N^2): each resume re-scans +// cloudsync_changes) can be tracked — e.g. to confirm a future indexed +// (db_version, seq) seek flattens it to O(N). +// +// Env: CHUNK_BENCH_ROWS (default 400), CHUNK_BENCH_ROW_BYTES (default 60000), +// CHUNK_BENCH_TXNS (default 1; rows split across this many db_versions), +// CHUNK_BENCH_REPEATS (default 5), CHUNK_BENCH_CHUNK_SIZE (default 262144). +// + +#include +#include +#include +#include +#include +#include "sqlite3.h" + +#define DB_PATH "dist/chunk-bench.sqlite" +#define EXT_PATH "./dist/cloudsync" + +static double monotonic_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((double)ts.tv_sec * 1000.0) + ((double)ts.tv_nsec / 1000000.0); +} + +static int env_int(const char *name, int dflt) { + const char *v = getenv(name); + if (!v || !*v) return dflt; + char *end = NULL; + long p = strtol(v, &end, 10); + if (!end || *end != '\0' || p <= 0) return dflt; + return (int)p; +} + +static int db_exec(sqlite3 *db, const char *sql) { + char *err = NULL; + int rc = sqlite3_exec(db, sql, NULL, NULL, &err); + if (rc != SQLITE_OK) { + fprintf(stderr, "exec failed: %s: %s\n", sql, err ? err : sqlite3_errmsg(db)); + sqlite3_free(err); + } + return rc; +} + +// Drain the whole window via the positional cursor, one chunk per query. Returns +// the chunk count and accumulates total payload bytes touched into *bytes. +static int drain_positional(sqlite3 *db, int *chunks_out, long long *bytes_out) { + const char *first_sql = + "SELECT payload, next_db_version, next_seq, next_frag_offset, is_final, watermark_db_version " + "FROM cloudsync_payload_chunks WHERE since_db_version=0 LIMIT 1;"; + const char *resume_sql = + "SELECT payload, next_db_version, next_seq, next_frag_offset, is_final " + "FROM cloudsync_payload_chunks " + "WHERE until_db_version=?1 AND resume_db_version=?2 AND resume_seq=?3 AND resume_frag_offset=?4 LIMIT 1;"; + sqlite3_stmt *first = NULL, *resume = NULL; + int rc = sqlite3_prepare_v2(db, first_sql, -1, &first, NULL); + if (rc != SQLITE_OK) goto done; + rc = sqlite3_prepare_v2(db, resume_sql, -1, &resume, NULL); + if (rc != SQLITE_OK) goto done; + + int chunks = 0; + long long bytes = 0; + long long watermark = 0, rdbv = 0, rseq = 0, rfrag = 0; + bool is_final = false; + + rc = sqlite3_step(first); + if (rc == SQLITE_ROW) { + bytes += sqlite3_column_bytes(first, 0); + rdbv = sqlite3_column_int64(first, 1); + rseq = sqlite3_column_int64(first, 2); + rfrag = sqlite3_column_int64(first, 3); + is_final = sqlite3_column_int(first, 4) != 0; + watermark = sqlite3_column_int64(first, 5); + chunks++; + } else if (rc == SQLITE_DONE) { + rc = SQLITE_OK; + goto done; // empty window + } else { + goto done; + } + + while (!is_final) { + sqlite3_reset(resume); + sqlite3_bind_int64(resume, 1, watermark); + sqlite3_bind_int64(resume, 2, rdbv); + sqlite3_bind_int64(resume, 3, rseq); + sqlite3_bind_int64(resume, 4, rfrag); + rc = sqlite3_step(resume); + if (rc != SQLITE_ROW) { if (rc == SQLITE_DONE) rc = SQLITE_OK; break; } + bytes += sqlite3_column_bytes(resume, 0); + rdbv = sqlite3_column_int64(resume, 1); + rseq = sqlite3_column_int64(resume, 2); + rfrag = sqlite3_column_int64(resume, 3); + is_final = sqlite3_column_int(resume, 4) != 0; + chunks++; + } + rc = SQLITE_OK; + *chunks_out = chunks; + *bytes_out = bytes; + +done: + if (first) sqlite3_finalize(first); + if (resume) sqlite3_finalize(resume); + return rc; +} + +int main(void) { + int rows = env_int("CHUNK_BENCH_ROWS", 400); + int row_bytes = env_int("CHUNK_BENCH_ROW_BYTES", 60000); + int repeats = env_int("CHUNK_BENCH_REPEATS", 5); + int chunk_size = env_int("CHUNK_BENCH_CHUNK_SIZE", 262144); + + remove(DB_PATH); + sqlite3 *db = NULL; + if (sqlite3_open(DB_PATH, &db) != SQLITE_OK) { fprintf(stderr, "open failed\n"); return 1; } + if (sqlite3_enable_load_extension(db, 1) != SQLITE_OK) return 1; + if (db_exec(db, "SELECT load_extension('" EXT_PATH "');") != SQLITE_OK) return 1; + + char setup[256]; + snprintf(setup, sizeof(setup), + "CREATE TABLE chunk_bench (id TEXT PRIMARY KEY, body BLOB);" + "SELECT cloudsync_init('chunk_bench');" + "SELECT cloudsync_set('payload_max_chunk_size', '%d');", chunk_size); + if (db_exec(db, setup) != SQLITE_OK) return 1; + + // Split the rows across CHUNK_BENCH_TXNS transactions: each is one db_version, + // so TXNS=1 is the pathological single-version window and TXNS=rows is the + // many-versions case a real /check window resembles. Incompressible bodies keep + // the window many-chunked. + int txns = env_int("CHUNK_BENCH_TXNS", 1); + if (txns < 1) txns = 1; + if (txns > rows) txns = rows; + int idbase = 0; + for (int t = 0; t < txns; ++t) { + int n = rows / txns + (t < rows % txns ? 1 : 0); + if (n <= 0) continue; + char insert[256]; + snprintf(insert, sizeof(insert), + "WITH RECURSIVE c(i) AS (SELECT %d UNION ALL SELECT i+1 FROM c WHERE i < %d) " + "INSERT INTO chunk_bench(id, body) SELECT printf('row-%%06d', i), randomblob(%d) FROM c;", + idbase + 1, idbase + n, row_bytes); + if (db_exec(db, insert) != SQLITE_OK) return 1; + idbase += n; + } + + int chunks = 0; + long long bytes = 0; + double best = 1e18, sum = 0; + for (int r = 0; r < repeats; ++r) { + double t0 = monotonic_ms(); + if (drain_positional(db, &chunks, &bytes) != SQLITE_OK) { fprintf(stderr, "positional drain failed\n"); return 1; } + double dt = monotonic_ms() - t0; + sum += dt; if (dt < best) best = dt; + } + + double mean = sum / repeats; + printf("\nPositional /check drain benchmark (local SQLite, no network)\n"); + printf("rows: %d row_bytes: %d txns: %d chunk_size: %d repeats: %d\n", + rows, row_bytes, txns, chunk_size, repeats); + printf("chunks: %d payload_bytes: %lld\n", chunks, bytes); + printf("drain: best=%.2f ms mean=%.2f ms\n", best, mean); + if (chunks > 0) + printf("per-chunk: best=%.3f ms throughput: %.1f MB/s\n", + best / chunks, (double)bytes / 1024.0 / 1024.0 / (best / 1000.0)); + + sqlite3_close(db); + remove(DB_PATH); + return 0; +} diff --git a/test/integration.c b/test/integration.c new file mode 100644 index 00000000..27fadea8 --- /dev/null +++ b/test/integration.c @@ -0,0 +1,1895 @@ +// +// integration.c +// cloudsync +// +// Created by Gioele Cantoni on 05/06/25. +// Set INTEGRATION_TEST_OFFLINE_DATABASE_ID and INTEGRATION_TEST_DATABASE_ID environment variables before running this test. +// Set INTEGRATION_TEST_CHUNKED_DATABASE_ID to enable the chunked-payload e2e test against an isolated remote database. +// Set INTEGRATION_TEST_WEBLITE_ADDRESS to enable the token-auth e2e test (mints a user token from the gateway /v2/tokens endpoint). +// + +#include +#include +#include +#include +#include +#include +#include "utils.h" +#include "sqlite3.h" + +// Define the number of simulated peers, when it's 0 it skips the peer test. +#if defined(__linux__) && !defined(__ANDROID__) +#define PEERS 0 +#else +#define PEERS 5 +#endif + +#ifdef PEERS +#ifdef _WIN32 +#include +#else +#include +#endif +#endif // PEERS + +#ifdef _WIN32 +#define popen _popen +#define pclose _pclose +#endif + +#ifdef CLOUDSYNC_LOAD_FROM_SOURCES +#include "cloudsync.h" +#include "cloudsync_sqlite.h" +#endif + +#define DB_PATH "health-track.sqlite" +#define EXT_PATH "./dist/cloudsync" +#define TEST_SKIPPED 100001 +#define RCHECK if (rc != SQLITE_OK) goto abort_test; +#define ERROR_MSG if (rc != SQLITE_OK) printf("Error: %s\n", sqlite3_errmsg(db)); +#define TERMINATE if (db) { db_exec(db, "SELECT cloudsync_terminate();"); } +#define ABORT_TEST abort_test: ERROR_MSG TERMINATE if (db) sqlite3_close(db); return rc; + +typedef enum { PRINT, NOPRINT, INTGR, GT0, STR } expected_type; + +typedef struct { + expected_type type; + union { + int i; + const char *s; // for future use, if needed + } value; +} expected_t; + +int open_load_ext(const char *db_path, sqlite3 **out_db); + +static int callback(void *data, int argc, char **argv, char **names) { + expected_t *expect = (expected_t *)data; + + switch(expect->type) { + case NOPRINT: break; + case PRINT: + for (int i = 0; i < argc; i++) { + printf("%s: %s ", names[i], argv[i] ? argv[i] : "NULL"); + } + printf("\n"); + return SQLITE_OK; + + case INTGR: + if(argc == 1){ + int res = atoi(argv[0]); + + if(res != expect->value.i){ + printf("Error: expected from %s: %d, got %d\n", names[0], expect->value.i, res); + return SQLITE_ERROR; + } + + } else goto multiple_columns; + break; + + case GT0: + if(argc == 1){ + int res = atoi(argv[0]); + + if(!(res > 0)){ + printf("Error: expected from %s: to be greater than 0, got %d\n", names[0], res); + return SQLITE_ERROR; + } + + } else goto multiple_columns; + break; + + case STR: + if(argc == 1){ + if(!argv[0] || strcmp(argv[0], expect->value.s) != 0){ + printf("Error: expected from %s: \"%s\", got \"%s\"\n", names[0], expect->value.s, argv[0] ? argv[0] : "NULL"); + return SQLITE_ERROR; + } + } else goto multiple_columns; + break; + + default: + printf("Error: unknown expect type\n"); + return SQLITE_ERROR; + } + + return SQLITE_OK; + +multiple_columns: + printf("Error: expected 1 column, got %d\n", argc); + return SQLITE_ERROR; +} + +int db_exec (sqlite3 *db, const char *sql) { + expected_t data; + data.type = NOPRINT; + + int rc = sqlite3_exec(db, sql, callback, &data, NULL); + if (rc != SQLITE_OK) printf("Error while executing %s: %s\n", sql, sqlite3_errmsg(db)); + return rc; +} + +int db_print (sqlite3 *db, const char *sql) { + expected_t data; + data.type = PRINT; + + int rc = sqlite3_exec(db, sql, callback, &data, NULL); + if (rc != SQLITE_OK) printf("Error while executing %s: %s\n", sql, sqlite3_errmsg(db)); + return rc; +} + +int db_expect_int (sqlite3 *db, const char *sql, int expect) { + expected_t data; + data.type = INTGR; + data.value.i = expect; + + int rc = sqlite3_exec(db, sql, callback, &data, NULL); + if (rc != SQLITE_OK) printf("Error while executing %s: %s\n", sql, sqlite3_errmsg(db)); + return rc; +} + +int db_expect_gt0 (sqlite3 *db, const char *sql) { + expected_t data; + data.type = GT0; + + int rc = sqlite3_exec(db, sql, callback, &data, NULL); + if (rc != SQLITE_OK) printf("Error while executing %s: %s\n", sql, sqlite3_errmsg(db)); + return rc; +} + +int db_expect_str (sqlite3 *db, const char *sql, const char *expect) { + expected_t data; + data.type = STR; + data.value.s = expect; + + int rc = sqlite3_exec(db, sql, callback, &data, NULL); + if (rc != SQLITE_OK) printf("Error while executing %s: %s\n", sql, sqlite3_errmsg(db)); + return rc; +} + +int db_select_int (sqlite3 *db, const char *sql, int *out) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + printf("Error while preparing %s: %s\n", sql, sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { + printf("Error while executing %s: expected one row, got rc=%d\n", sql, rc); + sqlite3_finalize(stmt); + return SQLITE_ERROR; + } + + *out = sqlite3_column_int(stmt, 0); + + rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + printf("Error while executing %s: expected one row only, got rc=%d\n", sql, rc); + sqlite3_finalize(stmt); + return SQLITE_ERROR; + } + + return sqlite3_finalize(stmt); +} + +// Reads the receive probe — chunks (int), complete (int), error (text) — from a +// single row of one network call. error_out is set to "" when receive.error is +// absent (the common success case) and to the message otherwise, so a swallowed +// client-side apply failure surfaces instead of timing out as "not received". +// The call must be a bare/read-only SELECT (e.g. a subquery), never +// CREATE TABLE ... AS SELECT, or the apply path cannot open its savepoint +// ("SQL statements in progress") and the download is silently dropped. +int db_select_receive (sqlite3 *db, const char *sql, int *chunks, int *complete, char *error_out, int error_len) { + if (error_len > 0) error_out[0] = '\0'; + + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + printf("Error while preparing %s: %s\n", sql, sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { + printf("Error while executing %s: expected one row, got rc=%d\n", sql, rc); + sqlite3_finalize(stmt); + return SQLITE_ERROR; + } + + *chunks = sqlite3_column_int(stmt, 0); + *complete = sqlite3_column_int(stmt, 1); + const unsigned char *err = sqlite3_column_text(stmt, 2); + if (err && error_len > 0) snprintf(error_out, error_len, "%s", (const char *)err); + + rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + printf("Error while executing %s: expected one row only, got rc=%d\n", sql, rc); + sqlite3_finalize(stmt); + return SQLITE_ERROR; + } + + return sqlite3_finalize(stmt); +} + +int db_expect_min (sqlite3 *db, const char *sql, int expect_min) { + int value = 0; + int rc = db_select_int(db, sql, &value); + if (rc != SQLITE_OK) return rc; + if (value < expect_min) { + printf("Error: expected %s to be >= %d, got %d\n", sql, expect_min, value); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +// Run a send that is expected to succeed and assert it didn't fail at the protocol +// level: status is not "error" and the server reported no per-chunk failure +// (send.lastFailure is omitted on success). Catches a server-reported apply/check +// failure that doesn't raise a SQL error — invisible to a plain db_exec of the send. +int db_send_ok (sqlite3 *db) { + return db_expect_int(db, + "SELECT (j ->> '$.send.status') <> 'error' AND (j ->> '$.send.lastFailure') IS NULL " + "FROM (SELECT cloudsync_network_send_changes() AS j);", 1); +} + +// Send, then poll until the server's optimistic version (send.serverVersion) catches +// up to send.localVersion — the change is durably covered with no gap. Robust to the +// server's asynchronous apply: the first call sends, later calls are no-ops that +// re-read status. Fails if it has not converged within max_attempts. +int db_send_await_converge (sqlite3 *db, int max_attempts, int delay_ms) { + const char *sql = + "SELECT (j ->> '$.send.serverVersion') >= (j ->> '$.send.localVersion') " + "FROM (SELECT cloudsync_network_send_changes() AS j);"; + for (int i = 0; i < max_attempts; i++) { + int converged = 0; + int rc = db_select_int(db, sql, &converged); + if (rc != SQLITE_OK) return rc; + if (converged) return SQLITE_OK; + if (i + 1 < max_attempts) sqlite3_sleep(delay_ms); + } + printf("Error: send did not converge (serverVersion < localVersion) after %d attempts\n", max_attempts); + return SQLITE_ERROR; +} + +// Bring a fresh receiver up to date with the tenant's accumulated history before a +// capped-receive test: send a sentinel row from the sender, then drain (uncapped) +// until it arrives. A fresh site's first check makes the server spool the ENTIRE +// tenant history — which grows with every run — answering 202 while it prepares, and +// a 202 yields rows=0/complete=1, indistinguishable from "already drained", so only +// the sentinel's arrival proves the replay actually finished. This keeps the capped +// attempt budgets O(this test's batch) instead of O(tenant history). +int chunked_receiver_catch_up (sqlite3 *sender, sqlite3 *receiver, const char *sentinel_id) { + char sql[512]; + snprintf(sql, sizeof(sql), + "INSERT INTO chunked_payload_items (id, body) VALUES ('%s', 'sentinel');", sentinel_id); + int rc = db_exec(sender, sql); if (rc != SQLITE_OK) return rc; + rc = db_send_ok(sender); if (rc != SQLITE_OK) return rc; + + for (int i = 0; i < 120; i++) { + int matches = 0; + rc = db_exec(receiver, "SELECT cloudsync_network_receive_changes();"); + if (rc != SQLITE_OK) return rc; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM chunked_payload_items WHERE id='%s';", sentinel_id); + rc = db_select_int(receiver, sql, &matches); if (rc != SQLITE_OK) return rc; + if (matches == 1) return SQLITE_OK; + sqlite3_sleep(500); + } + printf("Error: receiver did not catch up with tenant history (sentinel %s missing).\n", sentinel_id); + return SQLITE_ERROR; +} + +int integration_network_init_base(sqlite3 *db, const char *database_id, char *network_init, size_t network_init_len) { + if (!database_id) { + fprintf(stderr, "Error: integration database ID not set.\n"); + return SQLITE_ERROR; + } + + const char* custom_address = getenv("INTEGRATION_TEST_CLOUDSYNC_ADDRESS"); + if (custom_address) { + snprintf(network_init, network_init_len, + "SELECT cloudsync_network_init_custom('%s', '%s');", custom_address, database_id); + } else { + snprintf(network_init, network_init_len, + "SELECT cloudsync_network_init('%s');", database_id); + } + + return db_exec(db, network_init); +} + +int integration_network_init(sqlite3 *db, const char *database_id, char *network_init, size_t network_init_len) { + int rc = integration_network_init_base(db, database_id, network_init, network_init_len); + if (rc != SQLITE_OK) return rc; + + const char* apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, set_apikey); + } + + return rc; +} + +// Fixed, obviously-synthetic user id ("e2e test") used for every minted token. +#define TOKEN_TEST_USER_ID "e2e7e57e-0000-4000-8000-000000000001" + +// Some platforms that run the tests have no curl CLI (e.g. Android emulator images). +int curl_cli_available (void) { +#ifdef _WIN32 + FILE *pipe = popen("curl --version 2>nul", "r"); +#else + FILE *pipe = popen("curl --version 2>/dev/null", "r"); +#endif + if (!pipe) return 0; + char buffer[64]; + size_t nread = fread(buffer, 1, sizeof(buffer), pipe); + pclose(pipe); + return nread > 0; +} + +// Mint a user token from the gateway (POST /v2/tokens) via the curl CLI: this binary +// is built without networking, HTTP lives only in the loaded extension. +// Returns TEST_SKIPPED when the gateway address or apikey is not configured, +// or when there is no curl CLI to mint the token with. +int fetch_gateway_token (char *token, size_t token_len) { + const char *gateway = getenv("INTEGRATION_TEST_WEBLITE_ADDRESS"); + const char *apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (!gateway || !*gateway || !apikey || !*apikey) return TEST_SKIPPED; + if (!curl_cli_available()) return TEST_SKIPPED; + + // expire the token in 24h so repeated runs don't leave live credentials behind + time_t expires_time = time(NULL) + 24*60*60; + char expires[32]; + strftime(expires, sizeof(expires), "%Y-%m-%d %H:%M:%S", gmtime(&expires_time)); + + char command[1024]; + snprintf(command, sizeof(command), + "curl -s -X POST \"%s/v2/tokens\"" + " -H \"Authorization: Bearer %s\"" + " -H \"Content-Type: application/json; charset=utf-8\"" + " -d \"{\\\"userId\\\":\\\"%s\\\",\\\"name\\\":\\\"integration-test-token\\\",\\\"expiresAt\\\":\\\"%s\\\"}\"", + gateway, apikey, TOKEN_TEST_USER_ID, expires); + + FILE *pipe = popen(command, "r"); + if (!pipe) { + printf("Error: unable to run curl to mint the gateway token\n"); + return SQLITE_ERROR; + } + char response[4096]; + size_t nread = fread(response, 1, sizeof(response) - 1, pipe); + pclose(pipe); + response[nread] = '\0'; + + const char *marker = "\"token\":\""; + char *start = strstr(response, marker); + if (!start) { + printf("Error: no token in gateway response: %s\n", response); + return SQLITE_ERROR; + } + start += strlen(marker); + char *end = strchr(start, '"'); + if (!end || (size_t)(end - start) >= token_len) { + printf("Error: malformed token in gateway response: %s\n", response); + return SQLITE_ERROR; + } + memcpy(token, start, (size_t)(end - start)); + token[end - start] = '\0'; + return SQLITE_OK; +} + +int test_chunked_schema_init(sqlite3 *db) { + int rc = db_exec(db, + "CREATE TABLE IF NOT EXISTS chunked_payload_items (" + "id TEXT PRIMARY KEY NOT NULL," + "body TEXT NOT NULL DEFAULT ''" + ");"); + if (rc != SQLITE_OK) return rc; + + return db_exec(db, "SELECT cloudsync_init('chunked_payload_items');"); +} + +int test_chunked_failure_schema_init(sqlite3 *db) { + int rc = db_exec(db, + "CREATE TABLE IF NOT EXISTS chunked_payload_failure_items (" + "id TEXT PRIMARY KEY NOT NULL," + "body TEXT NOT NULL DEFAULT ''" + ");"); + if (rc != SQLITE_OK) return rc; + + return db_exec(db, "SELECT cloudsync_init('chunked_payload_failure_items');"); +} + +int test_chunked_sender_open(sqlite3 **sender, char *network_init, size_t network_init_len) { + const char* test_db_id = getenv("INTEGRATION_TEST_CHUNKED_DATABASE_ID"); + if (!test_db_id || !*test_db_id) { + return TEST_SKIPPED; + } + + int rc = open_load_ext(":memory:", sender); + if (rc != SQLITE_OK) return rc; + rc = integration_network_init(*sender, test_db_id, network_init, network_init_len); + if (rc != SQLITE_OK) return rc; + + return rc; +} + +int test_chunked_pair_open(sqlite3 **sender, sqlite3 **receiver, char *network_init, size_t network_init_len) { + const char* test_db_id = getenv("INTEGRATION_TEST_CHUNKED_DATABASE_ID"); + if (!test_db_id || !*test_db_id) { + return TEST_SKIPPED; + } + + int rc = open_load_ext(":memory:", sender); + if (rc != SQLITE_OK) return rc; + rc = test_chunked_schema_init(*sender); + if (rc != SQLITE_OK) return rc; + rc = integration_network_init(*sender, test_db_id, network_init, network_init_len); + if (rc != SQLITE_OK) return rc; + + rc = open_load_ext(":memory:", receiver); + if (rc != SQLITE_OK) return rc; + rc = test_chunked_schema_init(*receiver); + if (rc != SQLITE_OK) return rc; + rc = db_exec(*receiver, network_init); + if (rc != SQLITE_OK) return rc; + + const char* apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(*receiver, set_apikey); + } + + return rc; +} + +void test_chunked_pair_close(sqlite3 *sender, sqlite3 *receiver) { + if (sender) { + db_exec(sender, "SELECT cloudsync_terminate();"); + sqlite3_close(sender); + } + if (receiver) { + db_exec(receiver, "SELECT cloudsync_terminate();"); + sqlite3_close(receiver); + } +} + +void test_chunked_sender_close(sqlite3 *sender) { + if (sender) { + db_exec(sender, "SELECT cloudsync_terminate();"); + sqlite3_close(sender); + } +} + +int open_load_ext(const char *db_path, sqlite3 **out_db) { + sqlite3 *db = NULL; + int rc = sqlite3_open(db_path, &db); + RCHECK + +#ifdef CLOUDSYNC_LOAD_FROM_SOURCES + rc = sqlite3_cloudsync_init(db, NULL, NULL); +#else + // enable load extension + rc = sqlite3_enable_load_extension(db, 1); + RCHECK + + rc = db_exec(db, "SELECT load_extension('"EXT_PATH"');"); + RCHECK +#endif + + *out_db = db; + return rc; + +ABORT_TEST +} + +// MARK: - + +int db_init (sqlite3 *db){ + + int rc = db_exec(db, "\ + CREATE TABLE IF NOT EXISTS users (\ + id TEXT PRIMARY KEY NOT NULL,\ + name TEXT UNIQUE NOT NULL DEFAULT ''\ + );\ + CREATE TABLE IF NOT EXISTS activities (\ + id TEXT PRIMARY KEY NOT NULL,\ + user_id TEXT,\ + km REAL,\ + bpm INTEGER,\ + time TEXT,\ + activity_type TEXT NOT NULL DEFAULT 'running',\ + FOREIGN KEY(user_id) REFERENCES users(id)\ + );\ + CREATE TABLE IF NOT EXISTS workouts (\ + id TEXT PRIMARY KEY NOT NULL,\ + assigned_user_id TEXT,\ + day_of_week TEXT,\ + km REAL,\ + max_time TEXT\ + );\ + "); + +ERROR_MSG + return rc; + +} + +int test_init (const char *db_path, int init) { + int rc = SQLITE_OK; + + sqlite3 *db = NULL; + rc = open_load_ext(db_path, &db); RCHECK + + if(init){ + rc = db_init(db); + RCHECK + } + + rc = db_exec(db, "SELECT cloudsync_init('users');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('activities');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('workouts');"); RCHECK + + // init network + char network_init[1024]; + const char* test_db_id = getenv("INTEGRATION_TEST_DATABASE_ID"); + if (!test_db_id) { + fprintf(stderr, "Error: INTEGRATION_TEST_DATABASE_ID not set.\n"); + exit(1); + } + const char* custom_address = getenv("INTEGRATION_TEST_CLOUDSYNC_ADDRESS"); + if (custom_address) { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init_custom('%s', '%s');", custom_address, test_db_id); + } else { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init('%s');", test_db_id); + } + rc = db_exec(db, network_init); RCHECK + + const char* apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, set_apikey); RCHECK + } + + rc = db_expect_int(db, "SELECT COUNT(*) as count FROM activities;", 0); RCHECK + rc = db_expect_int(db, "SELECT COUNT(*) as count FROM workouts;", 0); RCHECK + char value[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_string(value, true); + char sql[256]; + snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s', '%s');", value, value); + rc = db_exec(db, sql); RCHECK + rc = db_expect_int(db, "SELECT COUNT(*) as count FROM users;", 1); RCHECK + rc = db_expect_gt0(db, "SELECT cloudsync_network_sync(250,10) ->> '$.receive.rows';"); RCHECK + rc = db_expect_gt0(db, "SELECT COUNT(*) as count FROM users;"); RCHECK + rc = db_expect_gt0(db, "SELECT COUNT(*) as count FROM activities;"); RCHECK + rc = db_expect_int(db, "SELECT COUNT(*) as count FROM workouts;", 0); RCHECK + rc = db_exec(db, "SELECT cloudsync_terminate();"); + +ABORT_TEST +} + +int test_is_enabled(const char *db_path) { + sqlite3 *db = NULL; + int rc = open_load_ext(db_path, &db); + + rc = db_expect_int(db, "SELECT cloudsync_is_enabled('users');", 1); RCHECK + rc = db_expect_int(db, "SELECT cloudsync_is_enabled('activities');", 1); RCHECK + rc = db_expect_int(db, "SELECT cloudsync_is_enabled('workouts');", 1); + +ABORT_TEST +} + +int test_db_version(const char *db_path) { + sqlite3 *db = NULL; + int rc = open_load_ext(db_path, &db); + + rc = db_expect_gt0(db, "SELECT cloudsync_db_version();"); RCHECK + rc = db_expect_gt0(db, "SELECT cloudsync_db_version_next();"); + +ABORT_TEST +} + +int test_enable_disable(const char *db_path) { + sqlite3 *db = NULL; + int rc = open_load_ext(db_path, &db); RCHECK + + char value[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_string(value, true); + char sql[256]; + + rc = db_exec(db, "SELECT cloudsync_init('users');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('activities');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('workouts');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_disable('users');"); RCHECK + + snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s', '%s');", value, value); + //rc = db_exec(db, sql); RCHECK + + rc = db_exec(db, "SELECT cloudsync_enable('users');"); RCHECK + + snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s-should-sync', '%s-should-sync');", value, value); + rc = db_exec(db, sql); RCHECK + + // init network + char network_init[1024]; + const char* test_db_id = getenv("INTEGRATION_TEST_DATABASE_ID"); + if (!test_db_id) { + fprintf(stderr, "Error: INTEGRATION_TEST_DATABASE_ID not set.\n"); + exit(1); + } + const char* custom_address = getenv("INTEGRATION_TEST_CLOUDSYNC_ADDRESS"); + if (custom_address) { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init_custom('%s', '%s');", custom_address, test_db_id); + } else { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init('%s');", test_db_id); + } + rc = db_exec(db, network_init); RCHECK + + const char* apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, set_apikey); RCHECK + } + + rc = db_send_ok(db); RCHECK + rc = db_exec(db, "SELECT cloudsync_cleanup('users');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_cleanup('activities');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_cleanup('workouts');"); RCHECK + + // give the server the time to apply the latest sent changes, it is an async job + sqlite3_sleep(5000); + + sqlite3 *db2 = NULL; + rc = open_load_ext(":memory:", &db2); RCHECK + rc = db_init(db2); RCHECK + + rc = db_exec(db2, "SELECT cloudsync_init('users');"); RCHECK + rc = db_exec(db2, "SELECT cloudsync_init('activities');"); RCHECK + rc = db_exec(db2, "SELECT cloudsync_init('workouts');"); RCHECK + + // init network with connection string + apikey + rc = db_exec(db2, network_init); RCHECK + + if (apikey) { + char set_apikey2[512]; + snprintf(set_apikey2, sizeof(set_apikey2), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db2, set_apikey2); RCHECK + } + + rc = db_expect_gt0(db2, "SELECT cloudsync_network_sync(250,10) ->> '$.receive.rows';"); RCHECK + + snprintf(sql, sizeof(sql), "SELECT COUNT(*) FROM users WHERE name='%s';", value); + rc = db_expect_int(db2, sql, 0); RCHECK + + snprintf(sql, sizeof(sql), "SELECT COUNT(*) FROM users WHERE name='%s-should-sync';", value); + rc = db_expect_int(db2, sql, 1); RCHECK + + rc = db_exec(db2, "SELECT cloudsync_terminate();"); RCHECK + + sqlite3_close(db2); + +ABORT_TEST +} + +// Reproduces the spurious-gap bug in the send path: when the local db_version clock +// has been advanced past the site's own changes — as happens when applied remote +// changes bump the clock — the send announces only the change's own db_version range, +// so the skipped versions stay a gap in the server's per-site coverage and +// lastOptimisticVersion can never reach localVersion. cloudsync_db_version_next() +// forces the jump deterministically, no second database required. Expected to FAIL +// until the send announces the covered window [last_sent+1 .. watermark]. +int test_send_gap_from_clock_hole(const char *db_path) { + sqlite3 *db = NULL; + int rc = open_load_ext(db_path, &db); RCHECK + rc = db_init(db); RCHECK // create users/activities/workouts (this db is fresh) + + char value[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_string(value, true); + char sql[256]; + + rc = db_exec(db, "SELECT cloudsync_init('users');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('activities');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('workouts');"); RCHECK + + // Force the next local change to land at db_version 10, leaving 1..9 with no + // local-site change (the "hole" that merging applied remote changes would create). + rc = db_exec(db, "SELECT cloudsync_db_version_next(10);"); RCHECK + + snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s', '%s');", value, value); + rc = db_exec(db, sql); RCHECK + + // sanity: the change really landed at db_version 10, so there is a leading hole + rc = db_expect_int(db, "SELECT cloudsync_db_version();", 10); RCHECK + + // init network + char network_init[1024]; + const char* test_db_id = getenv("INTEGRATION_TEST_DATABASE_ID"); + if (!test_db_id) { + fprintf(stderr, "Error: INTEGRATION_TEST_DATABASE_ID not set.\n"); + exit(1); + } + const char* custom_address = getenv("INTEGRATION_TEST_CLOUDSYNC_ADDRESS"); + if (custom_address) { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init_custom('%s', '%s');", custom_address, test_db_id); + } else { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init('%s');", test_db_id); + } + rc = db_exec(db, network_init); RCHECK + + const char* apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, set_apikey); RCHECK + } + + // Send, then poll until the server's optimistic version (serverVersion) reaches + // localVersion (10). With contiguous coverage it converges; with the gap bug it + // never does — serverVersion stays at 0 because db_versions 1..9 are reported + // missing. Polling absorbs the server's asynchronous apply. + rc = db_send_await_converge(db, 8, 1000); RCHECK + + rc = db_exec(db, "SELECT cloudsync_cleanup('users');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_cleanup('activities');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_cleanup('workouts');"); RCHECK + +ABORT_TEST +} + +// Same roundtrip as test_init, but authenticated with a gateway-minted user token +// (cloudsync_network_set_token) instead of the apikey. +int test_token_auth (void) { + char token[512]; + int rc = fetch_gateway_token(token, sizeof(token)); + if (rc != SQLITE_OK) return rc; + + sqlite3 *db = NULL; + rc = open_load_ext(":memory:", &db); RCHECK + rc = db_init(db); RCHECK + + rc = db_exec(db, "SELECT cloudsync_init('users');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('activities');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('workouts');"); RCHECK + + char network_init[1024]; + rc = integration_network_init_base(db, getenv("INTEGRATION_TEST_DATABASE_ID"), network_init, sizeof(network_init)); RCHECK + + char set_token[1024]; + snprintf(set_token, sizeof(set_token), "SELECT cloudsync_network_set_token('%s');", token); + rc = db_exec(db, set_token); RCHECK + + char value[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_string(value, true); + char sql[256]; + snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s', '%s');", value, value); + rc = db_exec(db, sql); RCHECK + rc = db_expect_gt0(db, "SELECT cloudsync_network_sync(250,10) ->> '$.receive.rows';"); RCHECK + rc = db_exec(db, "SELECT cloudsync_terminate();"); + +ABORT_TEST +} + +int test_chunked_payload_paths(void) { + int rc = SQLITE_OK; + sqlite3 *sender = NULL; + sqlite3 *receiver = NULL; + char network_init[1024]; + char row_id[UUID_STR_MAXLEN]; + char sql[1024]; + bool found = false; + bool cleanup_remote_row = false; + + rc = test_chunked_pair_open(&sender, &receiver, network_init, sizeof(network_init)); + if (rc == TEST_SKIPPED) return TEST_SKIPPED; + if (rc != SQLITE_OK) goto cleanup; + + cloudsync_uuid_v7_string(row_id, true); + rc = db_exec(sender, "SELECT cloudsync_set('payload_max_chunk_size', '262144');"); if (rc != SQLITE_OK) goto cleanup; + snprintf(sql, sizeof(sql), + "INSERT INTO chunked_payload_items (id, body) " + "VALUES ('%s', lower(hex(zeroblob(360000))));", + row_id); + rc = db_exec(sender, sql); if (rc != SQLITE_OK) goto cleanup; + + rc = db_expect_min(sender, "SELECT COUNT(*) FROM cloudsync_payload_chunks();", 2); if (rc != SQLITE_OK) goto cleanup; + rc = db_expect_min(sender, "SELECT COUNT(*) FROM cloudsync_payload_chunks() WHERE hex(substr(payload,5,1))='03';", 2); if (rc != SQLITE_OK) goto cleanup; + + rc = db_send_ok(sender); if (rc != SQLITE_OK) goto cleanup; + cleanup_remote_row = true; + + for (int attempt = 0; attempt < 30; ++attempt) { + int matches = 0; + + // Exercises the deprecated cloudsync_network_check_changes() alias on purpose + // (backward-compatibility coverage); cloudsync_network_receive_changes() is the + // canonical name and is covered by the rowset and capped-drain tests. + rc = db_exec(receiver, "SELECT cloudsync_network_check_changes();"); + if (rc != SQLITE_OK) goto cleanup; + + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM chunked_payload_items " + "WHERE id='%s' " + "AND length(body)=720000 " + "AND body=lower(hex(zeroblob(360000)));", + row_id); + rc = db_select_int(receiver, sql, &matches); + if (rc != SQLITE_OK) goto cleanup; + if (matches == 1) { + found = true; + break; + } + + sqlite3_sleep(500); + } + + if (!found) { + printf("Error: chunked e2e row %s was not received.\n", row_id); + rc = SQLITE_ERROR; + goto cleanup; + } + +cleanup: + if (cleanup_remote_row && sender) { + snprintf(sql, sizeof(sql), "DELETE FROM chunked_payload_items WHERE id='%s';", row_id); + if (db_exec(sender, sql) == SQLITE_OK) { + db_exec(sender, "SELECT cloudsync_network_send_changes();"); + } + } + test_chunked_pair_close(sender, receiver); + return rc; +} + +int test_chunked_payload_rowset_path(void) { + int rc = SQLITE_OK; + sqlite3 *sender = NULL; + sqlite3 *receiver = NULL; + char network_init[1024]; + char batch_id[UUID_STR_MAXLEN]; + char sql[1024]; + bool found = false; + bool cleanup_remote_rows = false; + const int row_count = 500; + const int body_bytes = 1600; + + rc = test_chunked_pair_open(&sender, &receiver, network_init, sizeof(network_init)); + if (rc == TEST_SKIPPED) return TEST_SKIPPED; + if (rc != SQLITE_OK) goto cleanup; + + cloudsync_uuid_v7_string(batch_id, true); + rc = db_exec(sender, "SELECT cloudsync_set('payload_max_chunk_size', '262144');"); if (rc != SQLITE_OK) goto cleanup; + snprintf(sql, sizeof(sql), + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM c WHERE i < %d) " + "INSERT INTO chunked_payload_items (id, body) " + "SELECT '%s-' || printf('%%03d', i), lower(hex(zeroblob(%d))) FROM c;", + row_count, batch_id, body_bytes); + rc = db_exec(sender, sql); if (rc != SQLITE_OK) goto cleanup; + + rc = db_expect_min(sender, "SELECT COUNT(*) FROM cloudsync_payload_chunks();", 2); if (rc != SQLITE_OK) goto cleanup; + rc = db_expect_int(sender, "SELECT COUNT(*) FROM cloudsync_payload_chunks() WHERE hex(substr(payload,5,1))='03';", 0); if (rc != SQLITE_OK) goto cleanup; + + rc = db_send_ok(sender); if (rc != SQLITE_OK) goto cleanup; + cleanup_remote_rows = true; + + for (int attempt = 0; attempt < 30; ++attempt) { + int matches = 0; + + rc = db_exec(receiver, "SELECT cloudsync_network_receive_changes();"); + if (rc != SQLITE_OK) goto cleanup; + + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM chunked_payload_items " + "WHERE id LIKE '%s-%%' " + "AND length(body)=%d " + "AND body=lower(hex(zeroblob(%d)));", + batch_id, body_bytes * 2, body_bytes); + rc = db_select_int(receiver, sql, &matches); + if (rc != SQLITE_OK) goto cleanup; + if (matches == row_count) { + found = true; + break; + } + + sqlite3_sleep(500); + } + + if (!found) { + printf("Error: chunked rowset e2e batch %s was not received.\n", batch_id); + rc = SQLITE_ERROR; + goto cleanup; + } + +cleanup: + if (cleanup_remote_rows && sender) { + snprintf(sql, sizeof(sql), "DELETE FROM chunked_payload_items WHERE id LIKE '%s-%%';", batch_id); + if (db_exec(sender, sql) == SQLITE_OK) { + db_exec(sender, "SELECT cloudsync_network_send_changes();"); + } + } + test_chunked_pair_close(sender, receiver); + return rc; +} + +// Verifies that a single cloudsync_network_sync() call drains an entire multi-chunk +// download (no per-chunk extra sync() calls), and that the new receive.chunks / +// receive.complete fields report the drain. A large batch of incompressible rows is +// sent so the server's check response spans several pages; one sync() on the receiver +// must pull them all and report chunks>1 with complete=true. +int test_chunked_payload_single_sync_drain(void) { + int rc = SQLITE_OK; + sqlite3 *sender = NULL; + sqlite3 *receiver = NULL; + char network_init[1024]; + char batch_id[UUID_STR_MAXLEN]; + char sql[1024]; + bool found = false; + bool observed_multi_chunk = false; // saw chunks>1 && complete=1 in a SINGLE sync() call + bool cleanup_remote_rows = false; + // INCOMPRESSIBLE random bodies (distinct per row) so the payload can't compress + // below the server's page size. Multi-page splitting is driven by the TENANT's + // payload_max_chunk_size (the server-side check-response page size, distinct from + // the client upload chunk size set below), which must be small on + // INTEGRATION_TEST_CHUNKED_DATABASE_ID — set it to 262144 to match the client. + // ~1 MB of incompressible data over a 256 KB page cap yields several pages + // regardless of tenant backlog, so one sync() must drain chunks>1. (zeroblob bodies + // would compress to ~nothing and collapse back to a single page.) + const int row_count = 500; + const int body_rand_bytes = 2048; // 4096-char bodies; ~1 MB random / ~2 MB serialized + + rc = test_chunked_pair_open(&sender, &receiver, network_init, sizeof(network_init)); + if (rc == TEST_SKIPPED) return TEST_SKIPPED; + if (rc != SQLITE_OK) goto cleanup; + + cloudsync_uuid_v7_string(batch_id, true); + rc = db_exec(sender, "SELECT cloudsync_set('payload_max_chunk_size', '262144');"); if (rc != SQLITE_OK) goto cleanup; + snprintf(sql, sizeof(sql), + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM c WHERE i < %d) " + "INSERT INTO chunked_payload_items (id, body) " + "SELECT '%s-' || printf('%%03d', i), lower(hex(randomblob(%d))) FROM c;", + row_count, batch_id, body_rand_bytes); + rc = db_exec(sender, sql); if (rc != SQLITE_OK) goto cleanup; + + // Sender splits into multiple non-fragment chunks. + rc = db_expect_min(sender, "SELECT COUNT(*) FROM cloudsync_payload_chunks();", 2); if (rc != SQLITE_OK) goto cleanup; + + rc = db_send_ok(sender); if (rc != SQLITE_OK) goto cleanup; + cleanup_remote_rows = true; + + for (int attempt = 0; attempt < 40; ++attempt) { + int chunks = 0, complete = 0, matches = 0; + char recv_err[512]; + + // Run exactly one sync() and read both receive fields from that single call. + // The call must stay a read-only SELECT (here a subquery): wrapping it in + // CREATE TABLE ... AS SELECT would keep the outer statement stepping while the + // apply path tries to open its savepoint, which SQLite rejects ("SQL statements + // in progress") — silently leaving the download unapplied. + rc = db_select_receive(receiver, + "SELECT j ->> '$.receive.chunks', j ->> '$.receive.complete', j ->> '$.receive.error' " + "FROM (SELECT cloudsync_network_sync(250, 30) AS j);", + &chunks, &complete, recv_err, sizeof(recv_err)); + if (rc != SQLITE_OK) goto cleanup; + // A client-side apply error is swallowed into receive.error; fail loudly + // instead of looping until the "not received" timeout. + if (recv_err[0]) { + printf("Error: chunked single-sync drain batch %s reported receive.error: %s\n", batch_id, recv_err); + rc = SQLITE_ERROR; + goto cleanup; + } + if (chunks > 1 && complete == 1) observed_multi_chunk = true; + + // Random bodies can't be matched by value; assert the full set arrived with + // each row at its expected length (content correctness is covered elsewhere). + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM chunked_payload_items " + "WHERE id LIKE '%s-%%' " + "AND length(body)=%d;", + batch_id, body_rand_bytes * 2); + rc = db_select_int(receiver, sql, &matches); if (rc != SQLITE_OK) goto cleanup; + if (matches == row_count) { found = true; break; } + + sqlite3_sleep(500); + } + + if (!found) { + printf("Error: chunked single-sync drain batch %s was not received.\n", batch_id); + rc = SQLITE_ERROR; + goto cleanup; + } + if (!observed_multi_chunk) { + printf("Error: a single cloudsync_network_sync() did not drain a multi-chunk stream (chunks>1, complete=1) for batch %s.\n", batch_id); + rc = SQLITE_ERROR; + goto cleanup; + } + +cleanup: + if (cleanup_remote_rows && sender) { + snprintf(sql, sizeof(sql), "DELETE FROM chunked_payload_items WHERE id LIKE '%s-%%';", batch_id); + if (db_exec(sender, sql) == SQLITE_OK) { + db_exec(sender, "SELECT cloudsync_network_send_changes();"); + } + } + test_chunked_pair_close(sender, receiver); + return rc; +} + +// Verifies the opt-in max_chunks cap on cloudsync_network_receive_changes(): each call +// applies at most one chunk, reports complete=false while more is pending, and resumes +// across calls (the in-memory page cursor persists) until the whole batch is received. +int test_chunked_payload_capped_receive(void) { + int rc = SQLITE_OK; + sqlite3 *sender = NULL; + sqlite3 *receiver = NULL; + char network_init[1024]; + char batch_id[UUID_STR_MAXLEN]; + char sentinel_id[UUID_STR_MAXLEN]; + char sql[1024]; + bool found = false; + bool observed_capped_partial = false; // saw chunks==1 && complete=0 from a receive(1) call + bool cleanup_remote_rows = false; + // INCOMPRESSIBLE random bodies (distinct per row) so the payload can't compress + // below the server's page size. Multi-page splitting is driven by the TENANT's + // payload_max_chunk_size (the server-side check-response page size, distinct from + // the client upload chunk size set below), which must be small on + // INTEGRATION_TEST_CHUNKED_DATABASE_ID — set it to 262144 to match the client — so + // the batch spans several pages and receive_changes(1) leaves a partial. (zeroblob + // bodies would compress away and collapse to one page.) + const int row_count = 500; + const int body_rand_bytes = 2048; // 4096-char bodies; ~1 MB random / ~2 MB serialized + + rc = test_chunked_pair_open(&sender, &receiver, network_init, sizeof(network_init)); + if (rc == TEST_SKIPPED) return TEST_SKIPPED; + if (rc != SQLITE_OK) goto cleanup; + + cloudsync_uuid_v7_string(batch_id, true); + cloudsync_uuid_v7_string(sentinel_id, true); + + // Replay accumulated tenant history up front so the 80x1-chunk budget below only + // has to fetch this test's batch (see chunked_receiver_catch_up). + rc = chunked_receiver_catch_up(sender, receiver, sentinel_id); + if (rc != SQLITE_OK) goto cleanup; + cleanup_remote_rows = true; + + rc = db_exec(sender, "SELECT cloudsync_set('payload_max_chunk_size', '262144');"); if (rc != SQLITE_OK) goto cleanup; + snprintf(sql, sizeof(sql), + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM c WHERE i < %d) " + "INSERT INTO chunked_payload_items (id, body) " + "SELECT '%s-' || printf('%%03d', i), lower(hex(randomblob(%d))) FROM c;", + row_count, batch_id, body_rand_bytes); + rc = db_exec(sender, sql); if (rc != SQLITE_OK) goto cleanup; + + rc = db_expect_min(sender, "SELECT COUNT(*) FROM cloudsync_payload_chunks();", 2); if (rc != SQLITE_OK) goto cleanup; + + rc = db_send_ok(sender); if (rc != SQLITE_OK) goto cleanup; + + for (int attempt = 0; attempt < 80; ++attempt) { + int chunks = 0, complete = 0, matches = 0; + char recv_err[512]; + + // Cap each call to a single chunk; read both fields from that one call. Must + // stay a read-only SELECT (subquery): CREATE TABLE ... AS SELECT would block + // the apply savepoint ("SQL statements in progress") and silently drop the + // downloaded chunk. + rc = db_select_receive(receiver, + "SELECT j ->> '$.receive.chunks', j ->> '$.receive.complete', j ->> '$.receive.error' " + "FROM (SELECT cloudsync_network_receive_changes(1) AS j);", + &chunks, &complete, recv_err, sizeof(recv_err)); + if (rc != SQLITE_OK) goto cleanup; + // Surface a swallowed client-side apply error instead of looping to timeout. + if (recv_err[0]) { + printf("Error: capped-receive batch %s reported receive.error: %s\n", batch_id, recv_err); + rc = SQLITE_ERROR; + goto cleanup; + } + // The cap must never apply more than one chunk per call. + if (chunks > 1) { + printf("Error: cloudsync_network_receive_changes(1) applied %d chunks (cap violated) for batch %s.\n", chunks, batch_id); + rc = SQLITE_ERROR; + goto cleanup; + } + if (chunks == 1 && complete == 0) observed_capped_partial = true; + + // Random bodies can't be matched by value; assert the full set arrived with + // each row at its expected length (content correctness is covered elsewhere). + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM chunked_payload_items " + "WHERE id LIKE '%s-%%' " + "AND length(body)=%d;", + batch_id, body_rand_bytes * 2); + rc = db_select_int(receiver, sql, &matches); if (rc != SQLITE_OK) goto cleanup; + if (matches == row_count) { found = true; break; } + + sqlite3_sleep(300); + } + + if (!found) { + printf("Error: capped-receive batch %s was not fully received.\n", batch_id); + rc = SQLITE_ERROR; + goto cleanup; + } + if (!observed_capped_partial) { + printf("Error: cloudsync_network_receive_changes(1) never reported a capped partial drain (chunks=1, complete=0) for batch %s.\n", batch_id); + rc = SQLITE_ERROR; + goto cleanup; + } + +cleanup: + if (cleanup_remote_rows && sender) { + snprintf(sql, sizeof(sql), "DELETE FROM chunked_payload_items WHERE id LIKE '%s-%%' OR id='%s';", batch_id, sentinel_id); + if (db_exec(sender, sql) == SQLITE_OK) { + db_exec(sender, "SELECT cloudsync_network_send_changes();"); + } + } + test_chunked_pair_close(sender, receiver); + return rc; +} + +// Verifies the batched cursor-spool response shape with an explicit maxChunks cap +// greater than one. The old single-page client parser ignores data.chunks[] and +// would report chunks=0 forever against the batched server response. +int test_chunked_payload_batched_receive(void) { + int rc = SQLITE_OK; + sqlite3 *sender = NULL; + sqlite3 *receiver = NULL; + char network_init[1024]; + char batch_id[UUID_STR_MAXLEN]; + char sentinel_id[UUID_STR_MAXLEN]; + char sql[1024]; + bool found = false; + bool observed_batched_partial = false; // saw chunks==2 && complete=0 from receive(2) + bool cleanup_remote_rows = false; + const int row_count = 500; + const int body_rand_bytes = 2048; + + rc = test_chunked_pair_open(&sender, &receiver, network_init, sizeof(network_init)); + if (rc == TEST_SKIPPED) return TEST_SKIPPED; + if (rc != SQLITE_OK) goto cleanup; + + cloudsync_uuid_v7_string(batch_id, true); + cloudsync_uuid_v7_string(sentinel_id, true); + + // Replay accumulated tenant history up front so the capped budget below only has + // to fetch this test's batch (see chunked_receiver_catch_up). + rc = chunked_receiver_catch_up(sender, receiver, sentinel_id); + if (rc != SQLITE_OK) goto cleanup; + cleanup_remote_rows = true; + + rc = db_exec(sender, "SELECT cloudsync_set('payload_max_chunk_size', '262144');"); if (rc != SQLITE_OK) goto cleanup; + snprintf(sql, sizeof(sql), + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM c WHERE i < %d) " + "INSERT INTO chunked_payload_items (id, body) " + "SELECT '%s-' || printf('%%03d', i), lower(hex(randomblob(%d))) FROM c;", + row_count, batch_id, body_rand_bytes); + rc = db_exec(sender, sql); if (rc != SQLITE_OK) goto cleanup; + + rc = db_expect_min(sender, "SELECT COUNT(*) FROM cloudsync_payload_chunks();", 3); if (rc != SQLITE_OK) goto cleanup; + + rc = db_send_ok(sender); if (rc != SQLITE_OK) goto cleanup; + + for (int attempt = 0; attempt < 80; ++attempt) { + int chunks = 0, complete = 0, matches = 0; + char recv_err[512]; + + rc = db_select_receive(receiver, + "SELECT j ->> '$.receive.chunks', j ->> '$.receive.complete', j ->> '$.receive.error' " + "FROM (SELECT cloudsync_network_receive_changes(2) AS j);", + &chunks, &complete, recv_err, sizeof(recv_err)); + if (rc != SQLITE_OK) goto cleanup; + if (recv_err[0]) { + printf("Error: batched-receive batch %s reported receive.error: %s\n", batch_id, recv_err); + rc = SQLITE_ERROR; + goto cleanup; + } + if (chunks > 2) { + printf("Error: cloudsync_network_receive_changes(2) applied %d chunks (cap violated) for batch %s.\n", chunks, batch_id); + rc = SQLITE_ERROR; + goto cleanup; + } + if (chunks == 2 && complete == 0) observed_batched_partial = true; + + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM chunked_payload_items " + "WHERE id LIKE '%s-%%' " + "AND length(body)=%d;", + batch_id, body_rand_bytes * 2); + rc = db_select_int(receiver, sql, &matches); if (rc != SQLITE_OK) goto cleanup; + if (matches == row_count) { found = true; break; } + + sqlite3_sleep(300); + } + + if (!found) { + printf("Error: batched-receive batch %s was not fully received.\n", batch_id); + rc = SQLITE_ERROR; + goto cleanup; + } + if (!observed_batched_partial) { + printf("Error: cloudsync_network_receive_changes(2) never reported a batched partial drain (chunks=2, complete=0) for batch %s.\n", batch_id); + rc = SQLITE_ERROR; + goto cleanup; + } + +cleanup: + if (cleanup_remote_rows && sender) { + snprintf(sql, sizeof(sql), "DELETE FROM chunked_payload_items WHERE id LIKE '%s-%%' OR id='%s';", batch_id, sentinel_id); + if (db_exec(sender, sql) == SQLITE_OK) { + db_exec(sender, "SELECT cloudsync_network_send_changes();"); + } + } + test_chunked_pair_close(sender, receiver); + return rc; +} + +int test_chunked_send_failure_preserves_checkpoint(void) { + int rc = SQLITE_OK; + sqlite3 *sender = NULL; + char network_init[1024]; + char batch_id[UUID_STR_MAXLEN]; + char sql[1024]; + char *errmsg = NULL; + int send_dbversion_before = 0; + int send_dbversion_after = -1; + const int row_count = 500; + const int body_bytes = 1600; + + rc = test_chunked_sender_open(&sender, network_init, sizeof(network_init)); + if (rc == TEST_SKIPPED) return TEST_SKIPPED; + if (rc != SQLITE_OK) goto cleanup; + + rc = test_chunked_failure_schema_init(sender); if (rc != SQLITE_OK) goto cleanup; + rc = db_exec(sender, "SELECT cloudsync_set('payload_max_chunk_size', '262144');"); if (rc != SQLITE_OK) goto cleanup; + + cloudsync_uuid_v7_string(batch_id, true); + snprintf(sql, sizeof(sql), + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM c WHERE i < %d) " + "INSERT INTO chunked_payload_failure_items (id, body) " + "SELECT '%s-' || printf('%%03d', i), lower(hex(zeroblob(%d))) FROM c;", + row_count, batch_id, body_bytes); + rc = db_exec(sender, sql); if (rc != SQLITE_OK) goto cleanup; + + rc = db_expect_min(sender, "SELECT COUNT(*) FROM cloudsync_payload_chunks();", 2); if (rc != SQLITE_OK) goto cleanup; + rc = db_expect_int(sender, "SELECT COUNT(*) FROM cloudsync_payload_chunks() WHERE hex(substr(payload,5,1))='03';", 0); if (rc != SQLITE_OK) goto cleanup; + + rc = db_select_int(sender, + "SELECT CAST(COALESCE((SELECT value FROM cloudsync_settings WHERE key='send_dbversion'), '0') AS INTEGER);", + &send_dbversion_before); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(sender, "SELECT cloudsync_network_send_changes();", NULL, NULL, &errmsg); + if (rc == SQLITE_OK) { + printf("Error: chunked send failure test expected cloudsync_network_send_changes to fail.\n"); + rc = SQLITE_ERROR; + goto cleanup; + } + if (errmsg) { + sqlite3_free(errmsg); + errmsg = NULL; + } + + rc = db_select_int(sender, + "SELECT CAST(COALESCE((SELECT value FROM cloudsync_settings WHERE key='send_dbversion'), '0') AS INTEGER);", + &send_dbversion_after); + if (rc != SQLITE_OK) goto cleanup; + + if (send_dbversion_after != send_dbversion_before) { + printf("Error: send_dbversion advanced after failed chunked send (before=%d after=%d).\n", + send_dbversion_before, send_dbversion_after); + rc = SQLITE_ERROR; + goto cleanup; + } + + rc = SQLITE_OK; + +cleanup: + if (errmsg) sqlite3_free(errmsg); + test_chunked_sender_close(sender); + return rc; +} + +// Regression test for the chunked-check stale negative-cache bug +// (see cloudsync/docs/chunked-check-negative-cache.md). A receiver that drains +// the tenant to empty must not be permanently pinned to "up to date": an empty +// /check result advances no cursor, so the receiver keeps polling the same +// (dbVersion, seq) key. A server that *caches* that empty chunk at the key would +// keep answering "no changes" even after another client commits changes at the +// same key, hiding them until the artifact TTL (up to 24h) expires. +// +// This drives the exact sequence that exposes the bug end-to-end: +// 1. the receiver drains the whole tenant until it is provably caught up; +// 2. it keeps polling for a few more seconds and asserts it stays at 0 rows +// (steady empty state — this is where the stale empty chunk would be cached); +// 3. a second client (the sender) inserts a row and sends it at that same key; +// 4. the receiver must observe that row on a subsequent receive. Against a +// server that caches the negative result this never arrives and the test +// fails; against the fixed server it is delivered. +// +// "Caught up" cannot be detected from row counts alone: a server that is still +// preparing a page replies 202, which the client surfaces as receive.rows=0, +// complete=1 — indistinguishable from a genuinely empty result. (An earlier +// version of this test mistook three preparation 202s for "drained" and then +// had ~71k backlog rows land during phase 2.) So we anchor on a sentinel: the +// sender commits a marker row *after* any pre-existing backlog, and phase 1 is +// only considered drained once the receiver has both applied that sentinel +// (the whole backlog is therefore behind it) and then seen a 0-row poll. +int test_chunked_negative_cache_invalidation(void) { + int rc = SQLITE_OK; + sqlite3 *sender = NULL; + sqlite3 *receiver = NULL; + char network_init[1024]; + char sentinel_id[UUID_STR_MAXLEN]; + char row_id[UUID_STR_MAXLEN]; + char sql[1024]; + bool drained = false; + bool found = false; + bool cleanup_sentinel = false; + bool cleanup_remote_row = false; + + rc = test_chunked_pair_open(&sender, &receiver, network_init, sizeof(network_init)); + if (rc == TEST_SKIPPED) return TEST_SKIPPED; + if (rc != SQLITE_OK) goto cleanup; + + // Commit a sentinel row that sorts after any pre-existing tenant backlog, so + // observing it on the receiver proves the entire backlog has been drained. + cloudsync_uuid_v7_string(sentinel_id, true); + snprintf(sql, sizeof(sql), + "INSERT INTO chunked_payload_items (id, body) VALUES ('%s', 'negative-cache-sentinel');", + sentinel_id); + rc = db_exec(sender, sql); if (rc != SQLITE_OK) goto cleanup; + rc = db_send_ok(sender); if (rc != SQLITE_OK) goto cleanup; + cleanup_sentinel = true; + + // Phase 1: drain the receiver until it is provably caught up. Each bare + // receive_changes() applies everything currently ready and returns at the first + // 202, so a large backlog is pulled across several iterations. Termination + // requires the sentinel to be present (backlog fully drained) AND a subsequent + // 0-row poll, so a mid-preparation 202 can never be mistaken for "caught up". + bool sentinel_seen = false; + for (int attempt = 0; attempt < 200 && !drained; ++attempt) { + int rows = 0, complete = 0; + char recv_err[512]; + + rc = db_select_receive(receiver, + "SELECT j ->> '$.receive.rows', j ->> '$.receive.complete', j ->> '$.receive.error' " + "FROM (SELECT cloudsync_network_receive_changes() AS j);", + &rows, &complete, recv_err, sizeof(recv_err)); + if (rc != SQLITE_OK) goto cleanup; + if (recv_err[0]) { + printf("Error: negative-cache drain reported receive.error: %s\n", recv_err); + rc = SQLITE_ERROR; + goto cleanup; + } + + if (!sentinel_seen) { + int matches = 0; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM chunked_payload_items WHERE id='%s';", sentinel_id); + rc = db_select_int(receiver, sql, &matches); if (rc != SQLITE_OK) goto cleanup; + if (matches == 1) sentinel_seen = true; + } + + // Caught up only once the backlog (including the sentinel) is fully applied + // and a further poll delivers nothing. + if (sentinel_seen && rows == 0) drained = true; + + sqlite3_sleep(300); + } + + if (!drained) { + printf("Error: negative-cache receiver never caught up to the sentinel row.\n"); + rc = SQLITE_ERROR; + goto cleanup; + } + + // Phase 2: keep polling for a few seconds with the tenant idle. The receiver is + // caught up, so every poll must continue to report 0 rows. (This is the window + // where a buggy server caches the empty chunk at the receiver's key.) + for (int i = 0; i < 6; ++i) { + int rows = 0, complete = 0; + char recv_err[512]; + + rc = db_select_receive(receiver, + "SELECT j ->> '$.receive.rows', j ->> '$.receive.complete', j ->> '$.receive.error' " + "FROM (SELECT cloudsync_network_receive_changes() AS j);", + &rows, &complete, recv_err, sizeof(recv_err)); + if (rc != SQLITE_OK) goto cleanup; + if (recv_err[0]) { + printf("Error: negative-cache idle poll reported receive.error: %s\n", recv_err); + rc = SQLITE_ERROR; + goto cleanup; + } + if (rows != 0) { + printf("Error: negative-cache receiver applied %d unexpected rows while the tenant was idle.\n", rows); + rc = SQLITE_ERROR; + goto cleanup; + } + + sqlite3_sleep(500); + } + + // Phase 3: a second client commits a change at the same (dbVersion, seq) key + // the receiver has been polling against. + cloudsync_uuid_v7_string(row_id, true); + snprintf(sql, sizeof(sql), + "INSERT INTO chunked_payload_items (id, body) VALUES ('%s', 'negative-cache');", + row_id); + rc = db_exec(sender, sql); if (rc != SQLITE_OK) goto cleanup; + rc = db_send_ok(sender); if (rc != SQLITE_OK) goto cleanup; + cleanup_remote_row = true; + + // Phase 4: the receiver must now pick up the change on a subsequent receive. + // A stale negative cache would keep answering "no changes" at the unchanged key + // and this row would never arrive. + for (int attempt = 0; attempt < 60; ++attempt) { + int matches = 0; + char recv_err[512]; + int rows = 0, complete = 0; + + rc = db_select_receive(receiver, + "SELECT j ->> '$.receive.rows', j ->> '$.receive.complete', j ->> '$.receive.error' " + "FROM (SELECT cloudsync_network_receive_changes() AS j);", + &rows, &complete, recv_err, sizeof(recv_err)); + if (rc != SQLITE_OK) goto cleanup; + if (recv_err[0]) { + printf("Error: negative-cache post-send poll reported receive.error: %s\n", recv_err); + rc = SQLITE_ERROR; + goto cleanup; + } + + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM chunked_payload_items WHERE id='%s' AND body='negative-cache';", + row_id); + rc = db_select_int(receiver, sql, &matches); if (rc != SQLITE_OK) goto cleanup; + if (matches == 1) { found = true; break; } + + sqlite3_sleep(500); + } + + if (!found) { + printf("Error: stale negative cache — receiver never received row %s after the sender committed it.\n", row_id); + rc = SQLITE_ERROR; + goto cleanup; + } + +cleanup: + if ((cleanup_sentinel || cleanup_remote_row) && sender) { + if (cleanup_remote_row) { + snprintf(sql, sizeof(sql), "DELETE FROM chunked_payload_items WHERE id='%s';", row_id); + db_exec(sender, sql); + } + if (cleanup_sentinel) { + snprintf(sql, sizeof(sql), "DELETE FROM chunked_payload_items WHERE id='%s';", sentinel_id); + db_exec(sender, sql); + } + db_exec(sender, "SELECT cloudsync_network_send_changes();"); + } + test_chunked_pair_close(sender, receiver); + return rc; +} + +int test_offline_error(const char *db_path) { + sqlite3 *db = NULL; + int rc = open_load_ext(db_path, &db); + RCHECK + + rc = db_exec(db, "CREATE TABLE IF NOT EXISTS test_table (id TEXT PRIMARY KEY NOT NULL, value TEXT NOT NULL DEFAULT '', created_at TEXT DEFAULT CURRENT_TIMESTAMP);"); + RCHECK + + rc = db_exec(db, "SELECT cloudsync_init('test_table');"); + RCHECK + + rc = db_exec(db, "INSERT INTO test_table (id, value) VALUES (cloudsync_uuid(), 'test1'), (cloudsync_uuid(), 'test2');"); + RCHECK + + // Initialize network with offline database ID + const char* offline_db_id = getenv("INTEGRATION_TEST_OFFLINE_DATABASE_ID"); + if (!offline_db_id || !*offline_db_id) { + // Clean up the db opened above and return TEST_SKIPPED directly (going through + // abort_test would trip ERROR_MSG on the non-zero rc and print a spurious error). + if (db) { db_exec(db, "SELECT cloudsync_terminate();"); sqlite3_close(db); } + return TEST_SKIPPED; + } + + char network_init[512]; + const char* custom_address = getenv("INTEGRATION_TEST_CLOUDSYNC_ADDRESS"); + if (custom_address) { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init_custom('%s', '%s');", custom_address, offline_db_id); + } else { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init('%s');", offline_db_id); + } + rc = db_exec(db, network_init); + RCHECK + + const char* apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, set_apikey); + RCHECK + } + + // Try to sync - this should fail with the expected error + char *errmsg = NULL; + rc = sqlite3_exec(db, "SELECT cloudsync_network_sync();", NULL, NULL, &errmsg); + + if (rc == SQLITE_OK) { + printf("Error: Expected network sync to fail, but it succeeded.\n"); + rc = SQLITE_ERROR; + goto abort_test; + } + + // Verify the error JSON contains expected fields using SQLite JSON extraction + if (!errmsg) { + printf("Error: Expected an error message, but got NULL\n"); + rc = SQLITE_ERROR; + goto abort_test; + } + + char verify_sql[1024]; + snprintf(verify_sql, sizeof(verify_sql), + "SELECT json_extract('%s', '$.errors[0].status');", errmsg); + rc = db_expect_str(db, verify_sql, "503"); + if (rc != SQLITE_OK) { printf("Offline error: unexpected status in: %s\n", errmsg); sqlite3_free(errmsg); goto abort_test; } + + snprintf(verify_sql, sizeof(verify_sql), + "SELECT json_extract('%s', '$.errors[0].code');", errmsg); + rc = db_expect_str(db, verify_sql, "database_paused"); + if (rc != SQLITE_OK) { printf("Offline error: unexpected code in: %s\n", errmsg); sqlite3_free(errmsg); goto abort_test; } + + snprintf(verify_sql, sizeof(verify_sql), + "SELECT json_extract('%s', '$.errors[0].title');", errmsg); + rc = db_expect_str(db, verify_sql, "Database paused"); + if (rc != SQLITE_OK) { printf("Offline error: unexpected title in: %s\n", errmsg); sqlite3_free(errmsg); goto abort_test; } + + sqlite3_free(errmsg); + rc = SQLITE_OK; + +ABORT_TEST +} + +int test_double_empty_network_init(const char *db_path) { + sqlite3 *db = NULL; + int rc = open_load_ext(db_path, &db); + RCHECK + + // First call with empty string - should return error + char *errmsg1 = NULL; + rc = sqlite3_exec(db, "SELECT cloudsync_network_init('');", NULL, NULL, &errmsg1); + if (rc == SQLITE_OK) { + printf("Error: First cloudsync_network_init('') should have failed but succeeded\n"); + if (errmsg1) sqlite3_free(errmsg1); + rc = SQLITE_ERROR; + goto abort_test; + } + if (errmsg1) sqlite3_free(errmsg1); + + // Second call with empty string - should also return error (not segfault) + char *errmsg2 = NULL; + rc = sqlite3_exec(db, "SELECT cloudsync_network_init('');", NULL, NULL, &errmsg2); + if (rc == SQLITE_OK) { + printf("Error: Second cloudsync_network_init('') should have failed but succeeded\n"); + if (errmsg2) sqlite3_free(errmsg2); + rc = SQLITE_ERROR; + goto abort_test; + } + if (errmsg2) sqlite3_free(errmsg2); + + rc = SQLITE_OK; + +ABORT_TEST +} + +// Failure-path integration test. +// +// Targets a cloudsync database (INTEGRATION_TEST_FAILURE_DATABASE_ID) +// configured server-side to fail apply and check jobs. Verifies that the +// new failures.{apply,check} response shape is correctly parsed and emitted as +// send.lastFailure (cloudsync_network_send_changes) and receive.lastFailure +// (cloudsync_network_receive_changes), and that cloudsync_network_sync surfaces +// at least one of them. +// +// First invocation primes the server (sends data, queues a check) — server-side +// async jobs may not have failed yet. After a sleep, the second invocation must +// see lastFailure populated. +int test_failure_path (const char *db_path) { + int rc = SQLITE_OK; + sqlite3 *db = NULL; + + const char *test_db_id = getenv("INTEGRATION_TEST_FAILURE_DATABASE_ID"); + if (!test_db_id || !*test_db_id) { + return TEST_SKIPPED; + } + const char *custom_address = getenv("INTEGRATION_TEST_CLOUDSYNC_ADDRESS"); + if (!custom_address || !*custom_address) { + return TEST_SKIPPED; + } + + rc = open_load_ext(db_path, &db); RCHECK + + rc = db_exec(db, "CREATE TABLE IF NOT EXISTS failure_users (id TEXT PRIMARY KEY NOT NULL, name TEXT NOT NULL DEFAULT '', value BLOB);"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('failure_users');"); RCHECK + + char network_init[1024]; + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init_custom('%s', '%s');", custom_address, test_db_id); + rc = db_exec(db, network_init); RCHECK + + const char *apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, set_apikey); RCHECK + } + + // Insert a row so cloudsync_network_send_changes has a payload to upload. + // Insert a 1MB value to skip the fast-lane and force using the normal s3 path with async job, + // otherwise the error would be immediately returned by the apply endpoint. + char value[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_string(value, true); + char sql[256]; + snprintf(sql, sizeof(sql), "INSERT INTO failure_users (id, name, value) VALUES ('%s', '%s', randomblob(1048576));", value, value); + rc = db_exec(db, sql); RCHECK + + // First invocation — primes the server. Failures may not yet be reported. + rc = db_exec(db, "SELECT cloudsync_network_send_changes();"); RCHECK + rc = db_exec(db, "SELECT cloudsync_network_receive_changes();"); RCHECK + rc = db_exec(db, "SELECT cloudsync_network_sync(250, 1);"); RCHECK + + // Give the server time to process and fail the queued apply/check jobs. + sqlite3_sleep(5000); + + // Second invocation — failures must surface now. The tenant database is not + // configured for cloudsync on the node, so every surfaced failure must carry + // the actionable "cloudsync is not initialized" message (instr on the message + // doubles as the lastFailure existence check: NULL fails GT0). + rc = db_expect_gt0(db, + "SELECT instr(cloudsync_network_send_changes() ->> '$.send.lastFailure.message', 'cloudsync is not initialized');"); RCHECK + + // The server reports the failed check job as non-retryable, so + // receive_changes raises a SQL error instead of returning JSON. + char *errmsg = NULL; + if (sqlite3_exec(db, "SELECT cloudsync_network_receive_changes();", NULL, NULL, &errmsg) == SQLITE_OK) { + printf("Error: expected cloudsync_network_receive_changes to fail, but it succeeded\n"); + rc = SQLITE_ERROR; + goto abort_test; + } + if (!errmsg || !strstr(errmsg, "cloudsync is not initialized")) { + printf("Error: unexpected receive_changes error: %s\n", errmsg ? errmsg : "(null)"); + sqlite3_free(errmsg); + rc = SQLITE_ERROR; + goto abort_test; + } + sqlite3_free(errmsg); + + // sync keeps emitting structured JSON so its send block survives; the + // forwarded message must appear in at least one lastFailure object. + rc = db_expect_gt0(db, + "SELECT instr(cloudsync_network_sync(250, 1), 'cloudsync is not initialized');"); RCHECK + + rc = db_exec(db, "SELECT cloudsync_terminate();"); + +ABORT_TEST +} + +int version(void){ + sqlite3 *db = NULL; + int rc = open_load_ext(":memory:", &db); + + rc = db_print(db, "SELECT cloudsync_version();"); + RCHECK + +ABORT_TEST +} + +// MARK: - + +int test_report(const char *description, int rc){ + const char *result = (rc == TEST_SKIPPED) ? "SKIPPED" : (rc ? "FAILED" : "OK"); + printf("%-32s %s\n", description, result); + return (rc == TEST_SKIPPED) ? 0 : rc; // a skipped test is not a failure +} + +#ifdef PEERS +#ifdef _WIN32 +DWORD WINAPI worker(LPVOID arg) { +#else +void* worker(void* arg) { +#endif + int thread_id = *(int*)arg; + int result = 0; + + char description[32]; + snprintf(description, sizeof(description), "%d/%d Peer Test", thread_id+1, PEERS); + result = test_init(":memory:", 1); + if(test_report(description, result)){ + printf("PEER %d FAIL.\n", thread_id+1); + // Return error code instead of exiting entire process +#ifdef _WIN32 + return (DWORD)(intptr_t)(thread_id+1); +#else + return (void*)(intptr_t)(thread_id+1); +#endif + } + +#ifdef _WIN32 + return 0; +#else + return NULL; +#endif +} +#endif // PEERS + +int main (void) { + int rc = SQLITE_OK; + remove(DB_PATH); // remove the database file if it exists + + cloudsync_memory_init(1); + + printf("\n\nIntegration Test "); + rc += version(); + printf("===========================================\n"); + test_report("Version Test:", rc); + + sqlite3 *db = NULL; + rc += open_load_ext(DB_PATH, &db); + rc += db_init(db); + if (db) sqlite3_close(db); + + rc += test_report("Init+Sync Test:", test_init(DB_PATH, 0)); + rc += test_report("Is Enabled Test:", test_is_enabled(DB_PATH)); + rc += test_report("DB Version Test:", test_db_version(DB_PATH)); + rc += test_report("Enable Disable Test:", test_enable_disable(DB_PATH)); + rc += test_report("Send Gap From Clock Hole Test:", test_send_gap_from_clock_hole(":memory:")); + rc += test_report("Token Auth Test:", test_token_auth()); + + // Chunked payload tests run only when INTEGRATION_TEST_CHUNKED_DATABASE_ID points at a + // tenant with a small payload_max_chunk_size; state the skip reason once for the group. + const char *chunked_id = getenv("INTEGRATION_TEST_CHUNKED_DATABASE_ID"); + printf("\n-- Chunked Payload Tests%s --\n", + (chunked_id && *chunked_id) ? "" : " (INTEGRATION_TEST_CHUNKED_DATABASE_ID not set, skipping)"); + rc += test_report("Chunked Paths Test:", test_chunked_payload_paths()); + rc += test_report("Chunked Rowset Test:", test_chunked_payload_rowset_path()); + rc += test_report("Chunked Single-Sync Drain Test:", test_chunked_payload_single_sync_drain()); + rc += test_report("Chunked Capped Receive Test:", test_chunked_payload_capped_receive()); + rc += test_report("Chunked Batched Receive Test:", test_chunked_payload_batched_receive()); + rc += test_report("Chunked Failure Test:", test_chunked_send_failure_preserves_checkpoint()); + rc += test_report("Chunked Negative Cache Test:", test_chunked_negative_cache_invalidation()); + printf("\n"); + + rc += test_report("Offline Error Test:", test_offline_error(":memory:")); + rc += test_report("Double Empty Init Test:", test_double_empty_network_init(":memory:")); + rc += test_report("Failure Path Test:", test_failure_path(":memory:")); + + remove(DB_PATH); // remove the database file + + #ifdef PEERS + #ifdef _WIN32 + HANDLE threads[PEERS]; + #else + pthread_t threads[PEERS]; + #endif + int thread_ids[PEERS]; + int threads_created = 0; + int thread_errors = 0; + + // Initialize threads array to invalid values for cleanup + #ifdef _WIN32 + for (int i = 0; i < PEERS; i++) { + threads[i] = NULL; + } + #else + memset(threads, 0, sizeof(threads)); + #endif + + // Create threads with proper error handling + for (int i = 0; i < PEERS; i++) { + thread_ids[i] = i; + #ifdef _WIN32 + threads[i] = CreateThread(NULL, 0, worker, &thread_ids[i], 0, NULL); + if (threads[i] == NULL) { + fprintf(stderr, "CreateThread failed for thread %d: %lu\n", i, GetLastError()); + thread_errors++; + break; // Stop creating more threads on failure + } + #else + int pthread_result = pthread_create(&threads[i], NULL, worker, &thread_ids[i]); + if (pthread_result != 0) { + fprintf(stderr, "pthread_create failed for thread %d: %s\n", i, strerror(pthread_result)); + threads[i] = 0; // Mark as invalid + thread_errors++; + break; // Stop creating more threads on failure + } + #endif + threads_created++; + } + + // Wait for all successfully created threads to finish and collect results + #ifdef _WIN32 + if (threads_created > 0) { + DWORD wait_result = WaitForMultipleObjects(threads_created, threads, TRUE, INFINITE); + if (wait_result == WAIT_FAILED) { + fprintf(stderr, "WaitForMultipleObjects failed: %lu\n", GetLastError()); + thread_errors++; + } + } + #endif + + // Join threads and collect exit codes + for (int i = 0; i < threads_created; i++) { + #ifdef _WIN32 + if (threads[i] != NULL) { + DWORD exit_code; + if (GetExitCodeThread(threads[i], &exit_code) && exit_code != 0) { + thread_errors++; + printf("Thread %d failed with exit code %lu\n", i, exit_code); + } + CloseHandle(threads[i]); + threads[i] = NULL; + } + #else + if (threads[i] != 0) { + void* thread_result = NULL; + int join_result = pthread_join(threads[i], &thread_result); + if (join_result != 0) { + fprintf(stderr, "pthread_join failed for thread %d: %s\n", i, strerror(join_result)); + thread_errors++; + } else if (thread_result != NULL) { + int exit_code = (int)(intptr_t)thread_result; + thread_errors++; + printf("Thread %d failed with exit code %d\n", i, exit_code); + } + threads[i] = 0; + } + #endif + } + + // Update return code if any thread errors occurred + if (thread_errors > 0) { + printf("Threading test failed: %d thread(s) had errors\n", thread_errors); + rc += thread_errors; + } + #endif // PEERS + + cloudsync_memory_finalize(); + + printf("\n"); + return rc; +} diff --git a/test/main.c b/test/main.c deleted file mode 100644 index a540c9f2..00000000 --- a/test/main.c +++ /dev/null @@ -1,565 +0,0 @@ -// -// main.c -// cloudsync -// -// Created by Gioele Cantoni on 05/06/25. -// Set CONNECTION_STRING, APIKEY and WEBLITE environment variables before running this test. -// - -#include -#include -#include -#include -#include -#include "utils.h" -#include "sqlite3.h" - -// Define the number of simulated peers, when it's 0 it skips the peer test. -#if defined(__linux__) && !defined(__ANDROID__) -#define PEERS 0 -#else -#define PEERS 5 -#endif - -#ifdef PEERS -#ifdef _WIN32 -#include -#else -#include -#endif -#endif // PEERS - -#ifdef CLOUDSYNC_LOAD_FROM_SOURCES -#include "cloudsync.h" -#endif - -#define DB_PATH "health-track.sqlite" -#define EXT_PATH "./dist/cloudsync" -#define RCHECK if (rc != SQLITE_OK) goto abort_test; -#define ERROR_MSG if (rc != SQLITE_OK) printf("Error: %s\n", sqlite3_errmsg(db)); -#define TERMINATE if (db) { db_exec(db, "SELECT cloudsync_terminate();"); } -#define ABORT_TEST abort_test: ERROR_MSG TERMINATE if (db) sqlite3_close(db); return rc; - -typedef enum { PRINT, NOPRINT, INTGR, GT0 } expected_type; - -typedef struct { - expected_type type; - union { - int i; - const char *s; // for future use, if needed - } value; -} expected_t; - -static int callback(void *data, int argc, char **argv, char **names) { - expected_t *expect = (expected_t *)data; - - switch(expect->type) { - case NOPRINT: break; - case PRINT: - for (int i = 0; i < argc; i++) { - printf("%s: %s ", names[i], argv[i] ? argv[i] : "NULL"); - } - printf("\n"); - return SQLITE_OK; - - case INTGR: - if(argc == 1){ - int res = atoi(argv[0]); - - if(res != expect->value.i){ - printf("Error: expected from %s: %d, got %d\n", names[0], expect->value.i, res); - return SQLITE_ERROR; - } - - } else goto multiple_columns; - break; - - case GT0: - if(argc == 1){ - int res = atoi(argv[0]); - - if(!(res > 0)){ - printf("Error: expected from %s: to be greater than 0, got %d\n", names[0], res); - return SQLITE_ERROR; - } - - } else goto multiple_columns; - break; - - default: - printf("Error: unknown expect type\n"); - return SQLITE_ERROR; - } - - return SQLITE_OK; - -multiple_columns: - printf("Error: expected 1 column, got %d\n", argc); - return SQLITE_ERROR; -} - -int db_exec (sqlite3 *db, const char *sql) { - expected_t data; - data.type = NOPRINT; - - int rc = sqlite3_exec(db, sql, callback, &data, NULL); - if (rc != SQLITE_OK) printf("Error while executing %s: %s\n", sql, sqlite3_errmsg(db)); - return rc; -} - -int db_print (sqlite3 *db, const char *sql) { - expected_t data; - data.type = PRINT; - - int rc = sqlite3_exec(db, sql, callback, &data, NULL); - if (rc != SQLITE_OK) printf("Error while executing %s: %s\n", sql, sqlite3_errmsg(db)); - return rc; -} - -int db_expect_int (sqlite3 *db, const char *sql, int expect) { - expected_t data; - data.type = INTGR; - data.value.i = expect; - - int rc = sqlite3_exec(db, sql, callback, &data, NULL); - if (rc != SQLITE_OK) printf("Error while executing %s: %s\n", sql, sqlite3_errmsg(db)); - return rc; -} - -int db_expect_gt0 (sqlite3 *db, const char *sql) { - expected_t data; - data.type = GT0; - - int rc = sqlite3_exec(db, sql, callback, &data, NULL); - if (rc != SQLITE_OK) printf("Error while executing %s: %s\n", sql, sqlite3_errmsg(db)); - return rc; -} - -int open_load_ext(const char *db_path, sqlite3 **out_db) { - sqlite3 *db = NULL; - int rc = sqlite3_open(db_path, &db); - RCHECK - -#ifdef CLOUDSYNC_LOAD_FROM_SOURCES - rc = sqlite3_cloudsync_init(db, NULL, NULL); -#else - // enable load extension - rc = sqlite3_enable_load_extension(db, 1); - RCHECK - - rc = db_exec(db, "SELECT load_extension('"EXT_PATH"');"); - RCHECK -#endif - - *out_db = db; - return rc; - -ABORT_TEST -} - -// MARK: - - -int db_init (sqlite3 *db){ - - int rc = db_exec(db, "\ - CREATE TABLE IF NOT EXISTS users (\ - id TEXT PRIMARY KEY NOT NULL,\ - name TEXT UNIQUE NOT NULL DEFAULT ''\ - );\ - CREATE TABLE IF NOT EXISTS activities (\ - id TEXT PRIMARY KEY NOT NULL,\ - user_id TEXT,\ - km REAL,\ - bpm INTEGER,\ - time TEXT,\ - activity_type TEXT NOT NULL DEFAULT 'running',\ - FOREIGN KEY(user_id) REFERENCES users(id)\ - );\ - CREATE TABLE IF NOT EXISTS workouts (\ - id TEXT PRIMARY KEY NOT NULL,\ - assigned_user_id TEXT,\ - day_of_week TEXT,\ - km REAL,\ - max_time TEXT\ - );\ - "); - -ERROR_MSG - return rc; - -} - -int test_init (const char *db_path, int init) { - int rc = SQLITE_OK; - - sqlite3 *db = NULL; - rc = open_load_ext(db_path, &db); RCHECK - - if(init){ - rc = db_init(db); - RCHECK - } - - rc = db_exec(db, "SELECT cloudsync_init('users');"); RCHECK - rc = db_exec(db, "SELECT cloudsync_init('activities');"); RCHECK - rc = db_exec(db, "SELECT cloudsync_init('workouts');"); RCHECK - - // init network with connection string + apikey - char network_init[512]; - const char* conn_str = getenv("CONNECTION_STRING"); - const char* apikey = getenv("APIKEY"); - if (!conn_str || !apikey) { - fprintf(stderr, "Error: CONNECTION_STRING or APIKEY not set.\n"); - exit(1); - } - snprintf(network_init, sizeof(network_init), "SELECT cloudsync_network_init('%s?apikey=%s');", conn_str, apikey); - rc = db_exec(db, network_init); RCHECK - - rc = db_expect_int(db, "SELECT COUNT(*) as count FROM activities;", 0); RCHECK - rc = db_expect_int(db, "SELECT COUNT(*) as count FROM workouts;", 0); RCHECK - char value[UUID_STR_MAXLEN]; - cloudsync_uuid_v7_string(value, true); - char sql[256]; - snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s', '%s');", value, value); - rc = db_exec(db, sql); RCHECK - rc = db_expect_int(db, "SELECT COUNT(*) as count FROM users;", 1); RCHECK - rc = db_expect_gt0(db, "SELECT cloudsync_network_sync(250,10);"); RCHECK - rc = db_expect_gt0(db, "SELECT COUNT(*) as count FROM users;"); RCHECK - rc = db_expect_gt0(db, "SELECT COUNT(*) as count FROM activities;"); RCHECK - rc = db_expect_int(db, "SELECT COUNT(*) as count FROM workouts;", 0); RCHECK - rc = db_exec(db, "SELECT cloudsync_terminate();"); - -ABORT_TEST -} - -int test_is_enabled(const char *db_path) { - sqlite3 *db = NULL; - int rc = open_load_ext(db_path, &db); - - rc = db_expect_int(db, "SELECT cloudsync_is_enabled('users');", 1); RCHECK - rc = db_expect_int(db, "SELECT cloudsync_is_enabled('activities');", 1); RCHECK - rc = db_expect_int(db, "SELECT cloudsync_is_enabled('workouts');", 1); - -ABORT_TEST -} - -int test_db_version(const char *db_path) { - sqlite3 *db = NULL; - int rc = open_load_ext(db_path, &db); - - rc = db_expect_gt0(db, "SELECT cloudsync_db_version();"); RCHECK - rc = db_expect_gt0(db, "SELECT cloudsync_db_version_next();"); - -ABORT_TEST -} - -int test_enable_disable(const char *db_path) { - sqlite3 *db = NULL; - int rc = open_load_ext(db_path, &db); RCHECK - - char value[UUID_STR_MAXLEN]; - cloudsync_uuid_v7_string(value, true); - char sql[256]; - - rc = db_exec(db, "SELECT cloudsync_init('*');"); RCHECK - rc = db_exec(db, "SELECT cloudsync_disable('users');"); RCHECK - - snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s', '%s');", value, value); - rc = db_exec(db, sql); RCHECK - - rc = db_exec(db, "SELECT cloudsync_enable('users');"); RCHECK - - snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s-should-sync', '%s-should-sync');", value, value); - rc = db_exec(db, sql); RCHECK - - // init network with connection string + apikey - char network_init[512]; - const char* conn_str = getenv("CONNECTION_STRING"); - const char* apikey = getenv("APIKEY"); - if (!conn_str || !apikey) { - fprintf(stderr, "Error: CONNECTION_STRING or APIKEY not set.\n"); - exit(1); - } - snprintf(network_init, sizeof(network_init), "SELECT cloudsync_network_init('%s?apikey=%s');", conn_str, apikey); - rc = db_exec(db, network_init); RCHECK - - rc = db_exec(db, "SELECT cloudsync_network_send_changes();"); RCHECK - rc = db_exec(db, "SELECT cloudsync_cleanup('*');"); - - // give the server the time to apply the latest sent changes, it is an async job - sqlite3_sleep(5000); - - sqlite3 *db2 = NULL; - rc = open_load_ext(":memory:", &db2); RCHECK - rc = db_init(db2); RCHECK - - rc = db_exec(db2, "SELECT cloudsync_init('*');"); RCHECK - - // init network with connection string + apikey - rc = db_exec(db2, network_init); RCHECK - - rc = db_expect_gt0(db2, "SELECT cloudsync_network_sync(250,10);"); RCHECK - - snprintf(sql, sizeof(sql), "SELECT COUNT(*) FROM users WHERE name='%s';", value); - rc = db_expect_int(db2, sql, 0); RCHECK - - snprintf(sql, sizeof(sql), "SELECT COUNT(*) FROM users WHERE name='%s-should-sync';", value); - rc = db_expect_int(db2, sql, 1); RCHECK - - rc = db_exec(db2, "SELECT cloudsync_terminate();"); RCHECK - - sqlite3_close(db2); - -ABORT_TEST -} - -int test_offline_error(const char *db_path) { - sqlite3 *db = NULL; - int rc = open_load_ext(db_path, &db); - RCHECK - - rc = db_exec(db, "CREATE TABLE IF NOT EXISTS test_table (id TEXT PRIMARY KEY NOT NULL, value TEXT NOT NULL DEFAULT '', created_at TEXT DEFAULT CURRENT_TIMESTAMP);"); - RCHECK - - rc = db_exec(db, "SELECT cloudsync_init('test_table');"); - RCHECK - - rc = db_exec(db, "INSERT INTO test_table (id, value) VALUES (cloudsync_uuid(), 'test1'), (cloudsync_uuid(), 'test2');"); - RCHECK - - // Initialize network with offline connection string - const char* offline_conn_str = getenv("CONNECTION_STRING_OFFLINE_PROJECT"); - if (!offline_conn_str) { - printf("Skipping offline error test: CONNECTION_STRING_OFFLINE_PROJECT not set.\n"); - rc = SQLITE_OK; - goto abort_test; - } - - char network_init[512]; - snprintf(network_init, sizeof(network_init), "SELECT cloudsync_network_init('%s');", offline_conn_str); - rc = db_exec(db, network_init); - RCHECK - - // Try to sync - this should fail with the expected error - char *errmsg = NULL; - rc = sqlite3_exec(db, "SELECT cloudsync_network_sync();", NULL, NULL, &errmsg); - - if (rc == SQLITE_OK) { - printf("Error: Expected network sync to fail, but it succeeded.\n"); - rc = SQLITE_ERROR; - goto abort_test; - } - - // Verify the error message contains the expected text - const char *expected_error = "cloudsync_network_send_changes unable to upload BLOB changes to remote host"; - if (!errmsg || strstr(errmsg, expected_error) == NULL) { - printf("Error: Expected error message containing '%s', but got '%s'\n", - expected_error, errmsg ? errmsg : "NULL"); - if (errmsg) sqlite3_free(errmsg); - rc = SQLITE_ERROR; - goto abort_test; - } - - if (errmsg) sqlite3_free(errmsg); - rc = SQLITE_OK; - -ABORT_TEST -} - -int test_double_empty_network_init(const char *db_path) { - sqlite3 *db = NULL; - int rc = open_load_ext(db_path, &db); - RCHECK - - // First call with empty string - should return error - char *errmsg1 = NULL; - rc = sqlite3_exec(db, "SELECT cloudsync_network_init('');", NULL, NULL, &errmsg1); - if (rc == SQLITE_OK) { - printf("Error: First cloudsync_network_init('') should have failed but succeeded\n"); - if (errmsg1) sqlite3_free(errmsg1); - rc = SQLITE_ERROR; - goto abort_test; - } - if (errmsg1) sqlite3_free(errmsg1); - - // Second call with empty string - should also return error (not segfault) - char *errmsg2 = NULL; - rc = sqlite3_exec(db, "SELECT cloudsync_network_init('');", NULL, NULL, &errmsg2); - if (rc == SQLITE_OK) { - printf("Error: Second cloudsync_network_init('') should have failed but succeeded\n"); - if (errmsg2) sqlite3_free(errmsg2); - rc = SQLITE_ERROR; - goto abort_test; - } - if (errmsg2) sqlite3_free(errmsg2); - - rc = SQLITE_OK; - -ABORT_TEST -} - -int version(void){ - sqlite3 *db = NULL; - int rc = open_load_ext(":memory:", &db); - - rc = db_print(db, "SELECT cloudsync_version();"); - RCHECK - -ABORT_TEST -} - -// MARK: - - -int test_report(const char *description, int rc){ - printf("%-24s %s\n", description, rc ? "FAILED" : "OK"); - return rc; -} - -#ifdef PEERS -#ifdef _WIN32 -DWORD WINAPI worker(LPVOID arg) { -#else -void* worker(void* arg) { -#endif - int thread_id = *(int*)arg; - int result = 0; - - char description[32]; - snprintf(description, sizeof(description), "%d/%d Peer Test", thread_id+1, PEERS); - result = test_init(":memory:", 1); - if(test_report(description, result)){ - printf("PEER %d FAIL.\n", thread_id+1); - // Return error code instead of exiting entire process -#ifdef _WIN32 - return (DWORD)(intptr_t)(thread_id+1); -#else - return (void*)(intptr_t)(thread_id+1); -#endif - } - -#ifdef _WIN32 - return 0; -#else - return NULL; -#endif -} -#endif // PEERS - -int main (void) { - int rc = SQLITE_OK; - remove(DB_PATH); // remove the database file if it exists - - cloudsync_memory_init(1); - - printf("\n\nIntegration Test "); - rc += version(); - printf("===========================================\n"); - test_report("Version Test:", rc); - - sqlite3 *db = NULL; - rc += open_load_ext(DB_PATH, &db); - rc += db_init(db); - if (db) sqlite3_close(db); - - rc += test_report("Init+Sync Test:", test_init(DB_PATH, 0)); - rc += test_report("Is Enabled Test:", test_is_enabled(DB_PATH)); - rc += test_report("DB Version Test:", test_db_version(DB_PATH)); - rc += test_report("Enable Disable Test:", test_enable_disable(DB_PATH)); - rc += test_report("Offline Error Test:", test_offline_error(":memory:")); - rc += test_report("Double Empty Init Test:", test_double_empty_network_init(":memory:")); - - remove(DB_PATH); // remove the database file - - #ifdef PEERS - #ifdef _WIN32 - HANDLE threads[PEERS]; - #else - pthread_t threads[PEERS]; - #endif - int thread_ids[PEERS]; - int threads_created = 0; - int thread_errors = 0; - - // Initialize threads array to invalid values for cleanup - #ifdef _WIN32 - for (int i = 0; i < PEERS; i++) { - threads[i] = NULL; - } - #else - memset(threads, 0, sizeof(threads)); - #endif - - // Create threads with proper error handling - for (int i = 0; i < PEERS; i++) { - thread_ids[i] = i; - #ifdef _WIN32 - threads[i] = CreateThread(NULL, 0, worker, &thread_ids[i], 0, NULL); - if (threads[i] == NULL) { - fprintf(stderr, "CreateThread failed for thread %d: %lu\n", i, GetLastError()); - thread_errors++; - break; // Stop creating more threads on failure - } - #else - int pthread_result = pthread_create(&threads[i], NULL, worker, &thread_ids[i]); - if (pthread_result != 0) { - fprintf(stderr, "pthread_create failed for thread %d: %s\n", i, strerror(pthread_result)); - threads[i] = 0; // Mark as invalid - thread_errors++; - break; // Stop creating more threads on failure - } - #endif - threads_created++; - } - - // Wait for all successfully created threads to finish and collect results - #ifdef _WIN32 - if (threads_created > 0) { - DWORD wait_result = WaitForMultipleObjects(threads_created, threads, TRUE, INFINITE); - if (wait_result == WAIT_FAILED) { - fprintf(stderr, "WaitForMultipleObjects failed: %lu\n", GetLastError()); - thread_errors++; - } - } - #endif - - // Join threads and collect exit codes - for (int i = 0; i < threads_created; i++) { - #ifdef _WIN32 - if (threads[i] != NULL) { - DWORD exit_code; - if (GetExitCodeThread(threads[i], &exit_code) && exit_code != 0) { - thread_errors++; - printf("Thread %d failed with exit code %lu\n", i, exit_code); - } - CloseHandle(threads[i]); - threads[i] = NULL; - } - #else - if (threads[i] != 0) { - void* thread_result = NULL; - int join_result = pthread_join(threads[i], &thread_result); - if (join_result != 0) { - fprintf(stderr, "pthread_join failed for thread %d: %s\n", i, strerror(join_result)); - thread_errors++; - } else if (thread_result != NULL) { - int exit_code = (int)(intptr_t)thread_result; - thread_errors++; - printf("Thread %d failed with exit code %d\n", i, exit_code); - } - threads[i] = 0; - } - #endif - } - - // Update return code if any thread errors occurred - if (thread_errors > 0) { - printf("Threading test failed: %d thread(s) had errors\n", thread_errors); - rc += thread_errors; - } - #endif // PEERS - - cloudsync_memory_finalize(); - - printf("\n"); - return rc; -} diff --git a/test/network_unit.c b/test/network_unit.c new file mode 100644 index 00000000..f4ccb27c --- /dev/null +++ b/test/network_unit.c @@ -0,0 +1,139 @@ +// +// network_unit.c +// cloudsync +// +// Unit tests for the network layer's pure response-handling logic. Built with +// networking ENABLED (unlike dist/unit, which is -DCLOUDSYNC_OMIT_NETWORK), so it +// can call the internal functions directly on crafted in-memory NETWORK_RESULT +// buffers — no server, no sockets. +// + +#include +#include +#include +#include +#include +#include "utils.h" +#include "network_private.h" + +static int failures = 0; + +static void check(const char *name, bool ok) { + printf("%-64s %s\n", name, ok ? "OK" : "FAIL"); + if (!ok) failures++; +} + +static NETWORK_RESULT json_buffer(char *json) { + NETWORK_RESULT r = {0}; + r.code = CLOUDSYNC_NETWORK_BUFFER; + r.buffer = json; + r.blen = strlen(json); + return r; +} + +// Regression: lastOptimisticVersion must track the LATEST valid value, including a +// decrease. The server can roll the optimistic version back when a later send chunk +// fails; since it becomes the durable send checkpoint, a monotonic "max" would mask +// the rollback and skip the rolled-back changes on the next send. +static bool test_optimistic_version_rollback(void) { + int64_t optimistic = -1, confirmed = -1; + int gaps = -1; + char *apply = NULL, *check_fail = NULL; + bool ok = true; + + char j1[] = "{\"lastOptimisticVersion\":50,\"lastConfirmedVersion\":10}"; + NETWORK_RESULT r1 = json_buffer(j1); + network_sync_state_update_from_response(&r1, &optimistic, &confirmed, &gaps, &apply, &check_fail); + ok = ok && optimistic == 50 && confirmed == 10; + + char j2[] = "{\"lastOptimisticVersion\":100,\"lastConfirmedVersion\":20}"; + NETWORK_RESULT r2 = json_buffer(j2); + network_sync_state_update_from_response(&r2, &optimistic, &confirmed, &gaps, &apply, &check_fail); + ok = ok && optimistic == 100 && confirmed == 20; + + // Server rolls back on a later chunk error: the value must DECREASE to 50. + char j3[] = "{\"lastOptimisticVersion\":50,\"lastConfirmedVersion\":20}"; + NETWORK_RESULT r3 = json_buffer(j3); + network_sync_state_update_from_response(&r3, &optimistic, &confirmed, &gaps, &apply, &check_fail); + ok = ok && optimistic == 50; + + // A response missing the field (parsed -1) must NOT clobber the current value. + char j4[] = "{\"lastConfirmedVersion\":20}"; + NETWORK_RESULT r4 = json_buffer(j4); + network_sync_state_update_from_response(&r4, &optimistic, &confirmed, &gaps, &apply, &check_fail); + ok = ok && optimistic == 50; + + ok = ok && apply == NULL && check_fail == NULL; // no failures object in these responses + return ok; +} + +// A non-BUFFER result (or NULL buffer) must leave the accumulators untouched. +static bool test_non_buffer_is_noop(void) { + int64_t optimistic = 7, confirmed = 3; + int gaps = 0; + char *apply = NULL, *check_fail = NULL; + + NETWORK_RESULT err = {0}; + err.code = CLOUDSYNC_NETWORK_ERROR; + network_sync_state_update_from_response(&err, &optimistic, &confirmed, &gaps, &apply, &check_fail); + return optimistic == 7 && confirmed == 3 && gaps == 0; +} + +// One send call = one all-or-nothing batch: every chunk of it announces the same +// global window [checkpoint+1 .. watermark] plus batchId/chunkIndex/isFinal, so the +// server confirms the whole window only when every chunk of the batch applied and a +// failed batch is re-sent whole under a new id. +static bool test_apply_json_payload_batch(void) { + bool ok = true; + + char *j0 = network_apply_json_payload("blob", "QUJD", 1, 10, "batch-uuid-1", 0, false); + ok = ok && j0 != NULL; + if (j0) { + ok = ok && strstr(j0, "\"blob\":\"QUJD\"") != NULL; + ok = ok && strstr(j0, "\"dbVersionMin\":1") != NULL; + ok = ok && strstr(j0, "\"dbVersionMax\":10") != NULL; + ok = ok && strstr(j0, "\"batchId\":\"batch-uuid-1\"") != NULL; + ok = ok && strstr(j0, "\"chunkIndex\":0") != NULL; + ok = ok && strstr(j0, "\"isFinal\":false") != NULL; + cloudsync_memory_free(j0); + } + + // last chunk, url transport: same window, higher index, isFinal true + char *j1 = network_apply_json_payload("url", "https://s3/part", 1, 10, "batch-uuid-1", 3, true); + ok = ok && j1 != NULL; + if (j1) { + ok = ok && strstr(j1, "\"url\":\"https://s3/part\"") != NULL; + ok = ok && strstr(j1, "\"dbVersionMin\":1") != NULL; + ok = ok && strstr(j1, "\"dbVersionMax\":10") != NULL; + ok = ok && strstr(j1, "\"batchId\":\"batch-uuid-1\"") != NULL; + ok = ok && strstr(j1, "\"chunkIndex\":3") != NULL; + ok = ok && strstr(j1, "\"isFinal\":true") != NULL; + cloudsync_memory_free(j1); + } + + // a batch id is mandatory for the chunked send path + ok = ok && network_apply_json_payload("blob", "QUJD", 1, 10, NULL, 0, false) == NULL; + + return ok; +} + +static bool test_compute_status(void) { + bool ok = true; + ok = ok && strcmp(network_compute_status(100, 100, 0, 100), "synced") == 0; + ok = ok && strcmp(network_compute_status(100, 50, 0, 100), "syncing") == 0; + ok = ok && strcmp(network_compute_status(100, 100, 1, 100), "out-of-sync") == 0; // gaps + ok = ok && strcmp(network_compute_status(90, 90, 0, 100), "out-of-sync") == 0; // behind local + ok = ok && strcmp(network_compute_status(-1, 100, 0, 100), "error") == 0; // unparsed + return ok; +} + +int main(void) { + printf("\nNetwork unit tests\n"); + check("optimistic/confirmed version folds latest-valid (allows rollback):", test_optimistic_version_rollback()); + check("non-buffer response is a no-op:", test_non_buffer_is_noop()); + check("send batch /apply payload (window / batchId / chunkIndex / isFinal):", test_apply_json_payload_batch()); + check("network_compute_status:", test_compute_status()); + if (failures) { printf("\n%d test(s) FAILED\n", failures); return 1; } + printf("\nAll network unit tests passed\n"); + return 0; +} diff --git a/test/postgresql/01_unittest.sql b/test/postgresql/01_unittest.sql new file mode 100644 index 00000000..ad14d72b --- /dev/null +++ b/test/postgresql/01_unittest.sql @@ -0,0 +1,322 @@ +-- 'Unittest' + +\set testid '01' + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_test_1; +CREATE DATABASE cloudsync_test_1; + +\connect cloudsync_test_1 +\ir helper_psql_conn_setup.sql + +-- Reset extension and install +-- DROP EXTENSION IF EXISTS cloudsync CASCADE; +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- 'Test version visibility' +SELECT cloudsync_version() AS version \gset +SELECT current_setting('server_version') AS pg_version \gset +\echo [PASS] (:testid) Test cloudsync_version: :version (PostgreSQL :pg_version) + +-- Test uuid generation +SELECT cloudsync_uuid() AS uuid1 \gset +SELECT pg_sleep(0.1) \gset +SELECT cloudsync_uuid() AS uuid2 \gset + +-- Test 1: Format check (UUID v7 has standard format: xxxxxxxx-xxxx-7xxx-xxxx-xxxxxxxxxxxx) +SELECT (:'uuid1' ~ '^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$') AS uuid_format_ok \gset +\if :uuid_format_ok +\echo [PASS] (:testid) UUID format valid (UUIDv7 pattern) +\else +\echo [FAIL] (:testid) UUID format invalid - Got: :uuid1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: Uniqueness check +SELECT (:'uuid1' != :'uuid2') AS uuid_unique_ok \gset +\if :uuid_unique_ok +\echo [PASS] (:testid) UUID uniqueness (two calls generated different UUIDs) +\else +\echo [FAIL] (:testid) UUID uniqueness - Both calls returned: :uuid1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Monotonicity check (UUIDv7 should be sortable by timestamp) +SELECT (:'uuid1' < :'uuid2') AS uuid_monotonic_ok \gset +\if :uuid_monotonic_ok +\echo [PASS] (:testid) UUID monotonicity (UUIDs are time-ordered) +\else +\echo [FAIL] (:testid) UUID monotonicity - uuid1: :uuid1, uuid2: :uuid2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 4: Type check (ensure it's actually UUID type, not text) +SELECT (pg_typeof(cloudsync_uuid())::text = 'uuid') AS uuid_type_ok \gset +\if :uuid_type_ok +\echo [PASS] (:testid) UUID type is correct (uuid, not text or bytea) +\else +\echo [FAIL] (:testid) UUID type incorrect - Got: (pg_typeof(cloudsync_uuid())::text) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test init on a simple table' +SELECT cloudsync_cleanup('smoke_tbl') AS _cleanup_ok \gset +SELECT (cloudsync_is_sync('smoke_tbl') = false) AS init_cleanup_ok \gset +\if :init_cleanup_ok +\echo [PASS] (:testid) Test init cleanup +\else +\echo [FAIL] (:testid) Test init cleanup +SELECT (:fail::int + 1) AS fail \gset +\endif +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id \gset +SELECT (to_regclass('public.smoke_tbl_cloudsync') IS NOT NULL) AS init_create_ok \gset +\if :init_create_ok +\echo [PASS] (:testid) Test init create +\else +\echo [FAIL] (:testid) Test init create +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test insert metadata row creation' +SELECT cloudsync_uuid() AS smoke_id \gset +INSERT INTO smoke_tbl (id, val) VALUES (:'smoke_id', 'hello'); +SELECT (COUNT(*) = 1) AS insert_meta_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = 'val' \gset +\if :insert_meta_ok +\echo [PASS] (:testid) Test insert metadata row creation +\else +\echo [FAIL] (:testid) Test insert metadata row creation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test insert metadata fields' +SELECT (db_version > 0 AND seq >= 0) AS insert_meta_fields_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = 'val' \gset +\if :insert_meta_fields_ok +\echo [PASS] (:testid) Test insert metadata fields +\else +\echo [FAIL] (:testid) Test insert metadata fields +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test update val only' +SELECT col_version AS val_ver_before +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = 'val' \gset +UPDATE smoke_tbl SET val = 'hello2' WHERE id = :'smoke_id'; +SELECT col_version AS val_ver_after +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = 'val' \gset +SELECT (:val_ver_after::bigint > :val_ver_before::bigint) AS update_val_ok \gset +\if :update_val_ok +\echo [PASS] (:testid) Test update val only +\else +\echo [FAIL] (:testid) Test update val only +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test update id only' +SELECT cloudsync_uuid() AS smoke_id2 \gset +UPDATE smoke_tbl SET id = :'smoke_id2' WHERE id = :'smoke_id'; +SELECT (COUNT(*) = 1) AS update_id_old_tombstone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = '__[RIP]__' \gset +\if :update_id_old_tombstone_ok +\echo [PASS] (:testid) Test update id only (old tombstone) +\else +\echo [FAIL] (:testid) Test update id only (old tombstone) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 0) AS update_id_old_val_gone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = 'val' \gset +\if :update_id_old_val_gone_ok +\echo [PASS] (:testid) Test update id only (old val gone) +\else +\echo [FAIL] (:testid) Test update id only (old val gone) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 1) AS update_id_new_val_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id2']::text[]) + AND col_name = 'val' \gset +\if :update_id_new_val_ok +\echo [PASS] (:testid) Test update id only (new val) +\else +\echo [FAIL] (:testid) Test update id only (new val) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 1) AS update_id_new_tombstone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id2']::text[]) + AND col_name = '__[RIP]__' \gset +\if :update_id_new_tombstone_ok +\echo [PASS] (:testid) Test update id only (new tombstone) +\else +\echo [FAIL] (:testid) Test update id only (new tombstone) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test update id and val' +SELECT cloudsync_uuid() AS smoke_id3 \gset +UPDATE smoke_tbl SET id = :'smoke_id3', val = 'hello3' WHERE id = :'smoke_id2'; +SELECT (COUNT(*) = 1) AS update_both_old_tombstone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id2']::text[]) + AND col_name = '__[RIP]__' \gset +\if :update_both_old_tombstone_ok +\echo [PASS] (:testid) Test update id and val (old tombstone) +\else +\echo [FAIL] (:testid) Test update id and val (old tombstone) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 0) AS update_both_old_val_gone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id2']::text[]) + AND col_name = 'val' \gset +\if :update_both_old_val_gone_ok +\echo [PASS] (:testid) Test update id and val (old val gone) +\else +\echo [FAIL] (:testid) Test update id and val (old val gone) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 1) AS update_both_new_val_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id3']::text[]) + AND col_name = 'val' \gset +\if :update_both_new_val_ok +\echo [PASS] (:testid) Test update id and val (new val) +\else +\echo [FAIL] (:testid) Test update id and val (new val) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 1) AS update_both_new_tombstone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id3']::text[]) + AND col_name = '__[RIP]__' \gset +\if :update_both_new_tombstone_ok +\echo [PASS] (:testid) Test update id and val (new tombstone) +\else +\echo [FAIL] (:testid) Test update id and val (new tombstone) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test delete metadata tombstone' +DELETE FROM smoke_tbl WHERE id = :'smoke_id3'; +SELECT (COUNT(*) = 1) AS delete_meta_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id3']::text[]) + AND col_name = '__[RIP]__' \gset +\if :delete_meta_ok +\echo [PASS] (:testid) Test delete metadata tombstone +\else +\echo [FAIL] (:testid) Test delete metadata tombstone +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test delete metadata fields' +SELECT (db_version > 0 AND seq >= 0) AS delete_meta_fields_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id3']::text[]) + AND col_name = '__[RIP]__' \gset +\if :delete_meta_fields_ok +\echo [PASS] (:testid) Test delete metadata fields +\else +\echo [FAIL] (:testid) Test delete metadata fields +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test delete removes non-tombstone metadata' +SELECT (COUNT(*) = 0) AS delete_meta_only_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id3']::text[]) + AND col_name != '__[RIP]__' \gset +\if :delete_meta_only_ok +\echo [PASS] (:testid) Test delete removes non-tombstone metadata +\else +\echo [FAIL] (:testid) Test delete removes non-tombstone metadata +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view write' +SELECT cloudsync_uuid() AS smoke_id4 \gset +INSERT INTO cloudsync_changes (tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) +VALUES ( + 'smoke_tbl', + cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id4']::text[]), + 'val', + -- "change_write" encoded as cloudsync text value (type 0x61 + len 0x0c) + decode('0b0c6368616e67655f7772697465', 'hex'), + 1, + cloudsync_db_version_next(), + cloudsync_siteid(), + 1, + 0 +); +SELECT (COUNT(*) = 1) AS changes_write_row_ok +FROM smoke_tbl +WHERE id = :'smoke_id4' AND val = 'change_write' \gset +\if :changes_write_row_ok +\echo [PASS] (:testid) Test cloudsync_changes view write +\else +\echo [FAIL] (:testid) Test cloudsync_changes view write +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view read' +SELECT COUNT(*) AS changes_view_count +FROM cloudsync_changes +WHERE tbl = 'smoke_tbl' \gset +SELECT COUNT(*) AS changes_meta_count +FROM smoke_tbl_cloudsync \gset +SELECT (:changes_view_count::int = :changes_meta_count::int) AS changes_read_ok \gset +\if :changes_read_ok +\echo [PASS] (:testid) Test cloudsync_changes view read +\else +\echo [FAIL] (:testid) Test cloudsync_changes view read +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test site id visibility' +SELECT cloudsync_siteid() AS site_id \gset +\echo [PASS] (:testid) Test site id visibility :site_id + +-- 'Test site id encoding' +SELECT (length(encode(cloudsync_siteid()::bytea, 'hex')) > 0) AS sid_ok \gset +\if :sid_ok +\echo [PASS] (:testid) Test site id encoding +\else +\echo [FAIL] (:testid) Test site id encoding +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test double init no-op' +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id2 \gset +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id3 \gset +\echo [PASS] (:testid) Test double init no-op + +-- 'Test payload encode signature' +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash +FROM smoke_tbl \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset +SELECT (length(:'payload_hex') > 0 AND substring(:'payload_hex' from 1 for 8) = '434c5359') AS payload_sig_ok \gset +\if :payload_sig_ok +\echo [PASS] (:testid) Test payload encode signature +\else +\echo [FAIL] (:testid) Test payload encode signature +SELECT (:fail::int + 1) AS fail \gset +\endif \ No newline at end of file diff --git a/test/postgresql/02_roundtrip.sql b/test/postgresql/02_roundtrip.sql new file mode 100644 index 00000000..5a7ecbfa --- /dev/null +++ b/test/postgresql/02_roundtrip.sql @@ -0,0 +1,38 @@ +-- '2 db roundtrip test' + +\set testid '02' +\ir helper_test_init.sql + +\connect cloudsync_test_1 +\ir helper_psql_conn_setup.sql +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +DROP DATABASE IF EXISTS cloudsync_test_2; +CREATE DATABASE cloudsync_test_2; +\connect cloudsync_test_2 +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS _apply_ok \gset +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset +SELECT (:'smoke_hash' = :'smoke_hash_b') AS payload_roundtrip_ok \gset +\if :payload_roundtrip_ok +\echo [PASS] (:testid) Test payload roundtrip to another database +\else +\echo [FAIL] (:testid) Test payload roundtrip to another database +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_1; +DROP DATABASE IF EXISTS cloudsync_test_2; +\else +\echo [INFO] !!!!! +\endif \ No newline at end of file diff --git a/test/postgresql/03_multiple_roundtrip.sql b/test/postgresql/03_multiple_roundtrip.sql new file mode 100644 index 00000000..18541d3f --- /dev/null +++ b/test/postgresql/03_multiple_roundtrip.sql @@ -0,0 +1,302 @@ +-- 'Test multi-db roundtrip with concurrent updates' + +\set testid '03' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_03_a; +DROP DATABASE IF EXISTS cloudsync_test_03_b; +DROP DATABASE IF EXISTS cloudsync_test_03_c; +CREATE DATABASE cloudsync_test_03_a; +CREATE DATABASE cloudsync_test_03_b; +CREATE DATABASE cloudsync_test_03_c; + +\connect cloudsync_test_03_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_03_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_03_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: independent inserts on each database +\connect cloudsync_test_03_a +INSERT INTO smoke_tbl VALUES ('id1', 'a1'); +INSERT INTO smoke_tbl VALUES ('id2', 'a2'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_b +INSERT INTO smoke_tbl VALUES ('id3', 'b3'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_c +INSERT INTO smoke_tbl VALUES ('id4', 'c4'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 1 apply: fan-out changes +\connect cloudsync_test_03_a +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif + +\connect cloudsync_test_03_b +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif + +\connect cloudsync_test_03_c +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif + +-- Round 2: concurrent updates on the same row + mixed operations +\connect cloudsync_test_03_a +UPDATE smoke_tbl SET val = 'a1_a' WHERE id = 'id1'; +DELETE FROM smoke_tbl WHERE id = 'id2'; +INSERT INTO smoke_tbl VALUES ('id5', 'a5'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_b +UPDATE smoke_tbl SET val = 'a1_b' WHERE id = 'id1'; +UPDATE smoke_tbl SET val = 'b3_b' WHERE id = 'id3'; +INSERT INTO smoke_tbl VALUES ('id6', 'b6'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_c +UPDATE smoke_tbl SET val = 'a1_c' WHERE id = 'id1'; +DELETE FROM smoke_tbl WHERE id = 'id4'; +INSERT INTO smoke_tbl VALUES ('id7', 'c7'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 2 apply: fan-out changes +\connect cloudsync_test_03_a +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif + +\connect cloudsync_test_03_b +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif + +\connect cloudsync_test_03_c +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif + +-- Round 3: additional operations to force another sync cycle +\connect cloudsync_test_03_a +UPDATE smoke_tbl SET val = 'b3_a' WHERE id = 'id3'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_b +DELETE FROM smoke_tbl WHERE id = 'id5'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_c +UPDATE smoke_tbl SET val = 'b6_c' WHERE id = 'id6'; +INSERT INTO smoke_tbl VALUES ('id8', 'c8'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 3 apply: final fan-out +\connect cloudsync_test_03_a +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif + +\connect cloudsync_test_03_b +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif + +\connect cloudsync_test_03_c +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_03_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_03_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_03_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Test multi-db roundtrip with concurrent updates +\else +\echo [FAIL] (:testid) Test multi-db roundtrip with concurrent updates +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_03_a; +DROP DATABASE IF EXISTS cloudsync_test_03_b; +DROP DATABASE IF EXISTS cloudsync_test_03_c; +\endif \ No newline at end of file diff --git a/test/postgresql/04_colversion_skew.sql b/test/postgresql/04_colversion_skew.sql new file mode 100644 index 00000000..b2d1000b --- /dev/null +++ b/test/postgresql/04_colversion_skew.sql @@ -0,0 +1,325 @@ +-- 'Test multi-db roundtrip with skewed col_version updates' +-- - concurrent update pattern where A/B/C perform 2/1/3 updates respectively on id1 before syncing. +-- - It follows the same apply order as the existing 3‑DB test and verifies final convergence across all three databases + +\set testid '04' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_04_a; +DROP DATABASE IF EXISTS cloudsync_test_04_b; +DROP DATABASE IF EXISTS cloudsync_test_04_c; +CREATE DATABASE cloudsync_test_04_a; +CREATE DATABASE cloudsync_test_04_b; +CREATE DATABASE cloudsync_test_04_c; + +\connect cloudsync_test_04_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_04_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_04_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed id1 on a single database, then sync +\connect cloudsync_test_04_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_a INSERT id1=seed_a1' +\endif +INSERT INTO smoke_tbl VALUES ('id1', 'seed_a1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_04_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_04_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 1 apply: fan-out changes +\connect cloudsync_test_04_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_04_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_04_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_04_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_04_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_04_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_04_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_04_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_04_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 2: skewed concurrent updates on id1 +\connect cloudsync_test_04_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_a UPDATE id1=a1_u1' +\endif +UPDATE smoke_tbl SET val = 'a1_u1' WHERE id = 'id1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_a UPDATE id1=a1_u2' +\endif +UPDATE smoke_tbl SET val = 'a1_u2' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_04_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_b UPDATE id1=b1_u1' +\endif +UPDATE smoke_tbl SET val = 'b1_u1' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_04_c +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_c UPDATE id1=c1_u1' +\endif +UPDATE smoke_tbl SET val = 'c1_u1' WHERE id = 'id1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_c UPDATE id1=c1_u2' +\endif +UPDATE smoke_tbl SET val = 'c1_u2' WHERE id = 'id1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_c UPDATE id1=c1_u3' +\endif +UPDATE smoke_tbl SET val = 'c1_u3' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 2 apply: fan-out changes +\connect cloudsync_test_04_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_04_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_04_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_04_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_04_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_04_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_04_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_04_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_04_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_04_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_04_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_04_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Test multi-db roundtrip with skewed col_version updates +\else +\echo [FAIL] (:testid) Test multi-db roundtrip with skewed col_version updates +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_04_a; +DROP DATABASE IF EXISTS cloudsync_test_04_b; +DROP DATABASE IF EXISTS cloudsync_test_04_c; +\endif \ No newline at end of file diff --git a/test/postgresql/05_delete_recreate_cycle.sql b/test/postgresql/05_delete_recreate_cycle.sql new file mode 100644 index 00000000..0e3a3130 --- /dev/null +++ b/test/postgresql/05_delete_recreate_cycle.sql @@ -0,0 +1,784 @@ +-- 'Test delete/recreate/update/delete/reinsert cycle across multiple DBs' +-- 1. A inserts +-- 2. B deletes +-- 3. C recreates with new value +-- 4. A updates +-- 5. B deletes again +-- 6. C reinserts with another value + + +\set testid '05' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_05_a; +DROP DATABASE IF EXISTS cloudsync_test_05_b; +DROP DATABASE IF EXISTS cloudsync_test_05_c; +CREATE DATABASE cloudsync_test_05_a; +CREATE DATABASE cloudsync_test_05_b; +CREATE DATABASE cloudsync_test_05_c; + +\connect cloudsync_test_05_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_05_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_05_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed row on A, sync to B/C +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_a INSERT id1=seed_v1' +\endif +INSERT INTO smoke_tbl VALUES ('id1', 'seed_v1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 2: B deletes id1, sync +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_b DELETE id1' +\endif +DELETE FROM smoke_tbl WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 3: C recreates id1, sync +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_c INSERT id1=recreate_v2' +\endif +INSERT INTO smoke_tbl VALUES ('id1', 'recreate_v2'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 4: A updates id1, sync +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_a UPDATE id1=update_v3' +\endif +UPDATE smoke_tbl SET val = 'update_v3' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r4, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r4_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r4, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r4_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r4, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r4_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r4', 3), 'hex')) AS _apply_a_r4_b \gset +\else +SELECT 0 AS _apply_a_r4_b \gset +\endif +\if :payload_c_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r4', 3), 'hex')) AS _apply_a_r4_c \gset +\else +SELECT 0 AS _apply_a_r4_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r4', 3), 'hex')) AS _apply_b_r4_a \gset +\else +SELECT 0 AS _apply_b_r4_a \gset +\endif +\if :payload_c_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r4', 3), 'hex')) AS _apply_b_r4_c \gset +\else +SELECT 0 AS _apply_b_r4_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r4', 3), 'hex')) AS _apply_c_r4_a \gset +\else +SELECT 0 AS _apply_c_r4_a \gset +\endif +\if :payload_b_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r4', 3), 'hex')) AS _apply_c_r4_b \gset +\else +SELECT 0 AS _apply_c_r4_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 5: B deletes id1, sync +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_b DELETE id1 (round5)' +\endif +DELETE FROM smoke_tbl WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r5, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r5_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r5, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r5_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r5, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r5_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r5', 3), 'hex')) AS _apply_a_r5_b \gset +\else +SELECT 0 AS _apply_a_r5_b \gset +\endif +\if :payload_c_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r5', 3), 'hex')) AS _apply_a_r5_c \gset +\else +SELECT 0 AS _apply_a_r5_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r5', 3), 'hex')) AS _apply_b_r5_a \gset +\else +SELECT 0 AS _apply_b_r5_a \gset +\endif +\if :payload_c_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r5', 3), 'hex')) AS _apply_b_r5_c \gset +\else +SELECT 0 AS _apply_b_r5_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r5', 3), 'hex')) AS _apply_c_r5_a \gset +\else +SELECT 0 AS _apply_c_r5_a \gset +\endif +\if :payload_b_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r5', 3), 'hex')) AS _apply_c_r5_b \gset +\else +SELECT 0 AS _apply_c_r5_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 6: C re-inserts id1, sync +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_c INSERT id1=reinsert_v4' +\endif +INSERT INTO smoke_tbl VALUES ('id1', 'reinsert_v4'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r6, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r6_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r6, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r6_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r6, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r6_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r6', 3), 'hex')) AS _apply_a_r6_b \gset +\else +SELECT 0 AS _apply_a_r6_b \gset +\endif +\if :payload_c_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r6', 3), 'hex')) AS _apply_a_r6_c \gset +\else +SELECT 0 AS _apply_a_r6_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r6', 3), 'hex')) AS _apply_b_r6_a \gset +\else +SELECT 0 AS _apply_b_r6_a \gset +\endif +\if :payload_c_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r6', 3), 'hex')) AS _apply_b_r6_c \gset +\else +SELECT 0 AS _apply_b_r6_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r6', 3), 'hex')) AS _apply_c_r6_a \gset +\else +SELECT 0 AS _apply_c_r6_a \gset +\endif +\if :payload_b_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r6', 3), 'hex')) AS _apply_c_r6_b \gset +\else +SELECT 0 AS _apply_c_r6_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_05_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_05_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_05_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Test delete/recreate/update/delete/reinsert cycle +\else +\echo [FAIL] (:testid) Test delete/recreate/update/delete/reinsert cycle +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_05_a; +DROP DATABASE IF EXISTS cloudsync_test_05_b; +DROP DATABASE IF EXISTS cloudsync_test_05_c; +\endif \ No newline at end of file diff --git a/test/postgresql/06_out_of_order_delivery.sql b/test/postgresql/06_out_of_order_delivery.sql new file mode 100644 index 00000000..607fd251 --- /dev/null +++ b/test/postgresql/06_out_of_order_delivery.sql @@ -0,0 +1,288 @@ +-- 'Test out-of-order payload delivery across multiple DBs' +-- - Seeds id1 +-- - Produces round2 and round3 concurrent updates +-- - Applies round3 before round2 on C, while A/B apply round2 then round3 +-- - Verifies convergence across all three DBs + +\set testid '06' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_06_a; +DROP DATABASE IF EXISTS cloudsync_test_06_b; +DROP DATABASE IF EXISTS cloudsync_test_06_c; +CREATE DATABASE cloudsync_test_06_a; +CREATE DATABASE cloudsync_test_06_b; +CREATE DATABASE cloudsync_test_06_c; + +\connect cloudsync_test_06_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_06_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_06_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed row on A, sync to B/C +\connect cloudsync_test_06_a +INSERT INTO smoke_tbl VALUES ('id1', 'seed_v1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_a +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif + +\connect cloudsync_test_06_b +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif + +\connect cloudsync_test_06_c +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif + +-- Round 2: concurrent updates +\connect cloudsync_test_06_a +UPDATE smoke_tbl SET val = 'a1_r2' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_b +UPDATE smoke_tbl SET val = 'b1_r2' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_c +UPDATE smoke_tbl SET val = 'c1_r2' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 3: further updates (newer payloads) +\connect cloudsync_test_06_a +UPDATE smoke_tbl SET val = 'a1_r3' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_b +UPDATE smoke_tbl SET val = 'b1_r3' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_c +UPDATE smoke_tbl SET val = 'c1_r3' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Out-of-order apply: apply round3 before round2 on C, and round2 before round3 on A/B +\connect cloudsync_test_06_a +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif + +\connect cloudsync_test_06_b +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif + +\connect cloudsync_test_06_c +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_06_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_06_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_06_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Test out-of-order payload delivery +\else +\echo [FAIL] (:testid) Test out-of-order payload delivery +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_06_a; +DROP DATABASE IF EXISTS cloudsync_test_06_b; +DROP DATABASE IF EXISTS cloudsync_test_06_c; +\endif \ No newline at end of file diff --git a/test/postgresql/07_delete_vs_update.sql b/test/postgresql/07_delete_vs_update.sql new file mode 100644 index 00000000..cfa3f758 --- /dev/null +++ b/test/postgresql/07_delete_vs_update.sql @@ -0,0 +1,290 @@ +-- Concurrent delete vs update +-- Steps: +-- 1) Seed id1 on A, sync to B/C +-- 2) B deletes id1 while C updates id1, then sync +-- 3) A updates id1 after merge, then sync + +\set testid '07' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_07_a; +DROP DATABASE IF EXISTS cloudsync_test_07_b; +DROP DATABASE IF EXISTS cloudsync_test_07_c; +CREATE DATABASE cloudsync_test_07_a; +CREATE DATABASE cloudsync_test_07_b; +CREATE DATABASE cloudsync_test_07_c; + +\connect cloudsync_test_07_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_07_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_07_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed id1 on A, sync to B/C +\connect cloudsync_test_07_a +INSERT INTO smoke_tbl VALUES ('id1', 'seed_v1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_a +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif + +\connect cloudsync_test_07_b +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif + +\connect cloudsync_test_07_c +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif + +-- Round 2: B deletes id1, C updates id1, then sync +\connect cloudsync_test_07_b +DELETE FROM smoke_tbl WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_c +UPDATE smoke_tbl SET val = 'c1_update' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_a +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif + +\connect cloudsync_test_07_b +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif + +\connect cloudsync_test_07_c +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif + +-- Round 3: A updates id1 after merge, then sync +\connect cloudsync_test_07_a +UPDATE smoke_tbl SET val = 'a1_post_merge' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_a +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif + +\connect cloudsync_test_07_b +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif + +\connect cloudsync_test_07_c +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_07_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_07_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_07_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Concurrent delete vs update +\else +\echo [FAIL] (:testid) Concurrent delete vs update +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_07_a; +DROP DATABASE IF EXISTS cloudsync_test_07_b; +DROP DATABASE IF EXISTS cloudsync_test_07_c; +\endif \ No newline at end of file diff --git a/test/postgresql/08_resurrect_delayed_delete.sql b/test/postgresql/08_resurrect_delayed_delete.sql new file mode 100644 index 00000000..c714a1c6 --- /dev/null +++ b/test/postgresql/08_resurrect_delayed_delete.sql @@ -0,0 +1,363 @@ +-- Resurrect after delete with delayed payload +-- Steps: +-- 1) Seed id1 on A, sync to B/C +-- 2) A deletes id1 and generates delete payload (do not apply yet on B) +-- 3) B recreates id1 with new value, sync to A/C +-- 4) Apply delayed delete payload from A to B/C +-- 5) Verify convergence + +\set testid '08' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_08_a; +DROP DATABASE IF EXISTS cloudsync_test_08_b; +DROP DATABASE IF EXISTS cloudsync_test_08_c; +CREATE DATABASE cloudsync_test_08_a; +CREATE DATABASE cloudsync_test_08_b; +CREATE DATABASE cloudsync_test_08_c; + +\connect cloudsync_test_08_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_08_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_08_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed id1 on A, sync to B/C +\connect cloudsync_test_08_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_08_a INSERT id1=seed_v1' +\endif +INSERT INTO smoke_tbl VALUES ('id1', 'seed_v1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_08_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_08_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_08_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_08_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 2: A deletes id1 (payload delayed for B/C) +\connect cloudsync_test_08_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_08_a DELETE id1' +\endif +DELETE FROM smoke_tbl WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 3: B recreates id1, sync to A/C (but A's delete still not applied on B/C) +\connect cloudsync_test_08_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_08_b UPSERT id1=recreate_v2' +\endif +INSERT INTO smoke_tbl (id, val) +VALUES ('id1', 'recreate_v2') +ON CONFLICT (id) DO UPDATE SET val = EXCLUDED.val; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_08_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_08_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_08_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_08_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 4: apply delayed delete payload from A to B/C +\connect cloudsync_test_08_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 before merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply delayed a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r4_a_delayed \gset +\else +SELECT 0 AS _apply_b_r4_a_delayed \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 after merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_08_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 before merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply delayed a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r4_a_delayed \gset +\else +SELECT 0 AS _apply_c_r4_a_delayed \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 after merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_08_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_08_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_08_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Resurrect after delete with delayed payload +\else +\echo [FAIL] (:testid) Resurrect after delete with delayed payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_08_a; +DROP DATABASE IF EXISTS cloudsync_test_08_b; +DROP DATABASE IF EXISTS cloudsync_test_08_c; +\endif \ No newline at end of file diff --git a/test/postgresql/09_multicol_concurrent_edits.sql b/test/postgresql/09_multicol_concurrent_edits.sql new file mode 100644 index 00000000..b3142f33 --- /dev/null +++ b/test/postgresql/09_multicol_concurrent_edits.sql @@ -0,0 +1,217 @@ +-- Multi-column concurrent edits +-- Steps: +-- 1) Create table with two data columns, seed row on A, sync to B/C +-- 2) B updates col_a while C updates col_b concurrently +-- 3) Sync and verify both columns are preserved on all DBs + +\set testid '09' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_09_a; +DROP DATABASE IF EXISTS cloudsync_test_09_b; +DROP DATABASE IF EXISTS cloudsync_test_09_c; +CREATE DATABASE cloudsync_test_09_a; +CREATE DATABASE cloudsync_test_09_b; +CREATE DATABASE cloudsync_test_09_c; + +\connect cloudsync_test_09_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, col_a TEXT, col_b TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_09_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, col_a TEXT, col_b TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_09_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, col_a TEXT, col_b TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed row on A, sync to B/C +\connect cloudsync_test_09_a +INSERT INTO smoke_tbl VALUES ('id1', 'a0', 'b0'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_09_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_09_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_09_a +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif + +\connect cloudsync_test_09_b +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif + +\connect cloudsync_test_09_c +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif + +-- Round 2: concurrent edits on different columns +\connect cloudsync_test_09_b +UPDATE smoke_tbl SET col_a = 'a1' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_09_c +UPDATE smoke_tbl SET col_b = 'b1' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_09_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Apply round 2 payloads +\connect cloudsync_test_09_a +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif + +\connect cloudsync_test_09_b +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif + +\connect cloudsync_test_09_c +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif + +-- Final consistency check across all three databases (both columns) +\connect cloudsync_test_09_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(col_a, '') || ':' || COALESCE(col_b, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_09_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(col_a, '') || ':' || COALESCE(col_b, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_09_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(col_a, '') || ':' || COALESCE(col_b, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Multi-column concurrent edits +\else +\echo [FAIL] (:testid) Multi-column concurrent edits +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_09_a; +DROP DATABASE IF EXISTS cloudsync_test_09_b; +DROP DATABASE IF EXISTS cloudsync_test_09_c; +\endif \ No newline at end of file diff --git a/test/postgresql/10_empty_payload_noop.sql b/test/postgresql/10_empty_payload_noop.sql new file mode 100644 index 00000000..ba65f0da --- /dev/null +++ b/test/postgresql/10_empty_payload_noop.sql @@ -0,0 +1,217 @@ +-- Empty payload + no-op merge +-- Steps: +-- 1) Setup three DBs and table +-- 2) Attempt to encode/apply empty payloads +-- 3) Verify data unchanged and hashes match + +\set testid '10' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_10_a; +DROP DATABASE IF EXISTS cloudsync_test_10_b; +DROP DATABASE IF EXISTS cloudsync_test_10_c; +CREATE DATABASE cloudsync_test_10_a; +CREATE DATABASE cloudsync_test_10_b; +CREATE DATABASE cloudsync_test_10_c; + +\connect cloudsync_test_10_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_10_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_10_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Seed a stable row so hashes are meaningful +\connect cloudsync_test_10_a +INSERT INTO smoke_tbl VALUES ('id1', 'seed_v1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_seed, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_seed_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_10_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_seed, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_seed_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_10_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_seed, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_seed_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Apply seed payloads so all DBs start in sync +\connect cloudsync_test_10_a +\if :payload_b_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_seed', 3), 'hex')) AS _apply_a_seed_b \gset +\else +SELECT 0 AS _apply_a_seed_b \gset +\endif +\if :payload_c_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_seed', 3), 'hex')) AS _apply_a_seed_c \gset +\else +SELECT 0 AS _apply_a_seed_c \gset +\endif + +\connect cloudsync_test_10_b +\if :payload_a_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_seed', 3), 'hex')) AS _apply_b_seed_a \gset +\else +SELECT 0 AS _apply_b_seed_a \gset +\endif +\if :payload_c_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_seed', 3), 'hex')) AS _apply_b_seed_c \gset +\else +SELECT 0 AS _apply_b_seed_c \gset +\endif + +\connect cloudsync_test_10_c +\if :payload_a_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_seed', 3), 'hex')) AS _apply_c_seed_a \gset +\else +SELECT 0 AS _apply_c_seed_a \gset +\endif +\if :payload_b_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_seed', 3), 'hex')) AS _apply_c_seed_b \gset +\else +SELECT 0 AS _apply_c_seed_b \gset +\endif + +-- Encode payloads with no changes (expected empty) +\connect cloudsync_test_10_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_empty, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_empty_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_10_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_empty, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_empty_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_10_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_empty, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_empty_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Apply empty payloads (should be no-ops) +\connect cloudsync_test_10_a +\if :payload_b_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_empty', 3), 'hex')) AS _apply_a_empty_b \gset +\else +SELECT 0 AS _apply_a_empty_b \gset +\endif +\if :payload_c_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_empty', 3), 'hex')) AS _apply_a_empty_c \gset +\else +SELECT 0 AS _apply_a_empty_c \gset +\endif + +\connect cloudsync_test_10_b +\if :payload_a_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_empty', 3), 'hex')) AS _apply_b_empty_a \gset +\else +SELECT 0 AS _apply_b_empty_a \gset +\endif +\if :payload_c_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_empty', 3), 'hex')) AS _apply_b_empty_c \gset +\else +SELECT 0 AS _apply_b_empty_c \gset +\endif + +\connect cloudsync_test_10_c +\if :payload_a_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_empty', 3), 'hex')) AS _apply_c_empty_a \gset +\else +SELECT 0 AS _apply_c_empty_a \gset +\endif +\if :payload_b_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_empty', 3), 'hex')) AS _apply_c_empty_b \gset +\else +SELECT 0 AS _apply_c_empty_b \gset +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_10_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_10_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_10_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Empty payload + no-op merge +\else +\echo [FAIL] (:testid) Empty payload + no-op merge +SELECT (:fail::int + 1) AS fail \gset +\endif + + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_10_a; +DROP DATABASE IF EXISTS cloudsync_test_10_b; +DROP DATABASE IF EXISTS cloudsync_test_10_c; +\endif \ No newline at end of file diff --git a/test/postgresql/11_multi_table_multi_columns_rounds.sql b/test/postgresql/11_multi_table_multi_columns_rounds.sql new file mode 100644 index 00000000..a94550f1 --- /dev/null +++ b/test/postgresql/11_multi_table_multi_columns_rounds.sql @@ -0,0 +1,717 @@ +-- 'Test multi-table multi-db roundtrip' +-- simulate the sport tracker app from examples +-- Steps: +-- 1) Create three databases, initialize users/activities/workouts and cloudsync +-- 2) Round 1: seed base data on A and sync to B/C +-- 3) Round 2: concurrent updates/inserts on A/B/C, then sync +-- 4) Round 3: more concurrent edits, then sync +-- 5) Verify convergence per table across all three databases + +\set testid '11' +\ir helper_test_init.sql + +-- Step 1: setup databases and schema +\if :{?DEBUG_MERGE} +\echo '[STEP 1] Setup databases and schema' +\endif +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_11_a; +DROP DATABASE IF EXISTS cloudsync_test_11_b; +DROP DATABASE IF EXISTS cloudsync_test_11_c; +CREATE DATABASE cloudsync_test_11_a; +CREATE DATABASE cloudsync_test_11_b; +CREATE DATABASE cloudsync_test_11_c; + +\connect cloudsync_test_11_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS workouts; +DROP TABLE IF EXISTS activities; +DROP TABLE IF EXISTS users; +CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL DEFAULT '' +); +CREATE TABLE IF NOT EXISTS activities ( + id TEXT PRIMARY KEY NOT NULL, + type TEXT NOT NULL DEFAULT 'runnning', + duration INTEGER, + distance DOUBLE PRECISION, + calories INTEGER, + date TEXT, + notes TEXT, + user_id TEXT REFERENCES users (id) +); +CREATE TABLE IF NOT EXISTS workouts ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + type TEXT, + duration INTEGER, + exercises TEXT, + date TEXT, + completed INTEGER DEFAULT 0, + user_id TEXT +); +SELECT cloudsync_init('users', 'CLS', 1) AS _init_users_a \gset +SELECT cloudsync_init('activities', 'CLS', 1) AS _init_activities_a \gset +SELECT cloudsync_init('workouts', 'CLS', 1) AS _init_workouts_a \gset + +\connect cloudsync_test_11_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS workouts; +DROP TABLE IF EXISTS activities; +DROP TABLE IF EXISTS users; +CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL DEFAULT '' +); +CREATE TABLE IF NOT EXISTS activities ( + id TEXT PRIMARY KEY NOT NULL, + type TEXT NOT NULL DEFAULT 'runnning', + duration INTEGER, + distance DOUBLE PRECISION, + calories INTEGER, + date TEXT, + notes TEXT, + user_id TEXT REFERENCES users (id) +); +CREATE TABLE IF NOT EXISTS workouts ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + type TEXT, + duration INTEGER, + exercises TEXT, + date TEXT, + completed INTEGER DEFAULT 0, + user_id TEXT +); +SELECT cloudsync_init('users', 'CLS', 1) AS _init_users_b \gset +SELECT cloudsync_init('activities', 'CLS', 1) AS _init_activities_b \gset +SELECT cloudsync_init('workouts', 'CLS', 1) AS _init_workouts_b \gset + +\connect cloudsync_test_11_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS workouts; +DROP TABLE IF EXISTS activities; +DROP TABLE IF EXISTS users; +CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL DEFAULT '' +); +CREATE TABLE IF NOT EXISTS activities ( + id TEXT PRIMARY KEY NOT NULL, + type TEXT NOT NULL DEFAULT 'runnning', + duration INTEGER, + distance DOUBLE PRECISION, + calories INTEGER, + date TEXT, + notes TEXT, + user_id TEXT REFERENCES users (id) +); +CREATE TABLE IF NOT EXISTS workouts ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + type TEXT, + duration INTEGER, + exercises TEXT, + date TEXT, + completed INTEGER DEFAULT 0, + user_id TEXT +); +SELECT cloudsync_init('users', 'CLS', 1) AS _init_users_c \gset +SELECT cloudsync_init('activities', 'CLS', 1) AS _init_activities_c \gset +SELECT cloudsync_init('workouts', 'CLS', 1) AS _init_workouts_c \gset + +-- Step 2: Round 1 seed base data on A, sync to B/C +\if :{?DEBUG_MERGE} +\echo '[STEP 2] Round 1 seed base data on A, sync to B/C' +\endif +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a INSERT users u1=alice' +\endif +INSERT INTO users (id, name) VALUES ('u1', 'alice'); +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a INSERT activities act1' +\endif +INSERT INTO activities (id, type, duration, distance, calories, date, notes, user_id) +VALUES ('act1', 'running', 30, 5.0, 200, '2026-01-01', 'seed', 'u1'); +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a INSERT workouts w1' +\endif +INSERT INTO workouts (id, name, type, duration, exercises, date, completed, user_id) +VALUES ('w1', 'base', 'cardio', 30, 'run', '2026-01-01', 0, 'u1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> b' +\echo ### payload_a_r1: :payload_a_r1 +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +-- Step 3: Round 2 concurrent updates and inserts across nodes +\if :{?DEBUG_MERGE} +\echo '[STEP 3] Round 2 concurrent updates and inserts across nodes' +\endif +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a UPDATE users u1=alice_a2' +\endif +UPDATE users SET name = 'alice_a2' WHERE id = 'u1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a UPDATE activities act1 duration/calories' +\endif +UPDATE activities SET duration = 35, calories = 220 WHERE id = 'act1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a INSERT workouts w2' +\endif +INSERT INTO workouts (id, name, type, duration, exercises, date, completed, user_id) +VALUES ('w2', 'tempo', 'cardio', 40, 'run', '2026-01-02', 0, 'u1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_b UPDATE users u1=alice_b2' +\endif +UPDATE users SET name = 'alice_b2' WHERE id = 'u1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_b UPDATE workouts w1 completed=1' +\endif +UPDATE workouts SET completed = 1 WHERE id = 'w1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_b INSERT users u2=bob' +\endif +INSERT INTO users (id, name) VALUES ('u2', 'bob'); +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_b INSERT activities act2' +\endif +INSERT INTO activities (id, type, duration, distance, calories, date, notes, user_id) +VALUES ('act2', 'cycling', 60, 20.0, 500, '2026-01-02', 'b_seed', 'u2'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_c +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_c UPDATE activities act1 notes=c_note' +\endif +UPDATE activities SET notes = 'c_note' WHERE id = 'act1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_c UPDATE workouts w1 type=strength' +\endif +UPDATE workouts SET type = 'strength' WHERE id = 'w1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_c INSERT workouts w3' +\endif +INSERT INTO workouts (id, name, type, duration, exercises, date, completed, user_id) +VALUES ('w3', 'lift', 'strength', 45, 'squat', '2026-01-02', 0, 'u1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +-- Step 4: Round 3 more concurrent edits +\if :{?DEBUG_MERGE} +\echo '[STEP 4] Round 3 more concurrent edits' +\endif +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a UPDATE workouts w2 completed=1' +\endif +UPDATE workouts SET completed = 1 WHERE id = 'w2'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_b UPDATE activities act1 distance=6.5' +\endif +UPDATE activities SET distance = 6.5 WHERE id = 'act1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_c +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_c UPDATE users u1=alice_c3' +\endif +UPDATE users SET name = 'alice_c3' WHERE id = 'u1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_c INSERT activities act3' +\endif +INSERT INTO activities (id, type, duration, distance, calories, date, notes, user_id) +VALUES ('act3', 'yoga', 45, 0.0, 150, '2026-01-03', 'c_seed', 'u1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +-- Step 5: final consistency check across all three databases +\if :{?DEBUG_MERGE} +\echo '[STEP 5] Final consistency check across all three databases' +\endif +\connect cloudsync_test_11_a +SELECT md5(COALESCE(string_agg(id || ':' || name, ',' ORDER BY id), '')) AS users_hash_a +FROM users \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(type, '') || ':' || COALESCE(duration::text, '') || ':' || + COALESCE(distance::text, '') || ':' || COALESCE(calories::text, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(notes, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS activities_hash_a +FROM activities \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(name, '') || ':' || COALESCE(type, '') || ':' || + COALESCE(duration::text, '') || ':' || COALESCE(exercises, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(completed::text, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS workouts_hash_a +FROM workouts \gset + +\connect cloudsync_test_11_b +SELECT md5(COALESCE(string_agg(id || ':' || name, ',' ORDER BY id), '')) AS users_hash_b +FROM users \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(type, '') || ':' || COALESCE(duration::text, '') || ':' || + COALESCE(distance::text, '') || ':' || COALESCE(calories::text, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(notes, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS activities_hash_b +FROM activities \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(name, '') || ':' || COALESCE(type, '') || ':' || + COALESCE(duration::text, '') || ':' || COALESCE(exercises, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(completed::text, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS workouts_hash_b +FROM workouts \gset + +\connect cloudsync_test_11_c +SELECT md5(COALESCE(string_agg(id || ':' || name, ',' ORDER BY id), '')) AS users_hash_c +FROM users \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(type, '') || ':' || COALESCE(duration::text, '') || ':' || + COALESCE(distance::text, '') || ':' || COALESCE(calories::text, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(notes, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS activities_hash_c +FROM activities \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(name, '') || ':' || COALESCE(type, '') || ':' || + COALESCE(duration::text, '') || ':' || COALESCE(exercises, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(completed::text, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS workouts_hash_c +FROM workouts \gset + +SELECT (:'users_hash_a' = :'users_hash_b' AND :'users_hash_a' = :'users_hash_c') AS users_ok \gset +\if :users_ok +\echo [PASS] (:testid) Multi-table users convergence +\else +\echo [FAIL] (:testid) Multi-table users convergence +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:'activities_hash_a' = :'activities_hash_b' AND :'activities_hash_a' = :'activities_hash_c') AS activities_ok \gset +\if :activities_ok +\echo [PASS] (:testid) Multi-table activities convergence +\else +\echo [FAIL] (:testid) Multi-table activities convergence +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:'workouts_hash_a' = :'workouts_hash_b' AND :'workouts_hash_a' = :'workouts_hash_c') AS workouts_ok \gset +\if :workouts_ok +\echo [PASS] (:testid) Multi-table workouts convergence +\else +\echo [FAIL] (:testid) Multi-table workouts convergence +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_11_a; +DROP DATABASE IF EXISTS cloudsync_test_11_b; +DROP DATABASE IF EXISTS cloudsync_test_11_c; +\endif \ No newline at end of file diff --git a/test/postgresql/12_repeated_table_multi_schemas.sql b/test/postgresql/12_repeated_table_multi_schemas.sql new file mode 100644 index 00000000..d3628243 --- /dev/null +++ b/test/postgresql/12_repeated_table_multi_schemas.sql @@ -0,0 +1,228 @@ +\set testid '12' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_test_12; +CREATE DATABASE cloudsync_test_12; + +\connect cloudsync_test_12 +\ir helper_psql_conn_setup.sql + +-- Reset extension and install +DROP EXTENSION IF EXISTS cloudsync CASCADE; +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- 'Test multi-schema table init (setup)' +CREATE SCHEMA IF NOT EXISTS test_schema; +DROP TABLE IF EXISTS public.repeated_table; +DROP TABLE IF EXISTS test_schema.repeated_table; +CREATE TABLE public.repeated_table (id TEXT PRIMARY KEY, data TEXT); +CREATE TABLE test_schema.repeated_table (id TEXT PRIMARY KEY, data TEXT); + +-- Reset the connection to test if we load the configuration correctly +\connect cloudsync_test_12 +\ir helper_psql_conn_setup.sql + +-- 'Test init on table that exists in multiple schemas (default: public)' +SELECT cloudsync_cleanup('repeated_table') AS _cleanup_repeated \gset +SELECT cloudsync_init('repeated_table', 'CLS', 1) AS _init_repeated_public \gset +SELECT cloudsync_table_schema('repeated_table') AS repeated_schema_public \gset +SELECT (:'repeated_schema_public' = 'public') AS repeated_schema_public_ok \gset +\if :repeated_schema_public_ok +\echo [PASS] (:testid) Test cloudsync_table_schema returns public for repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_table_schema returns public for repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (to_regclass('public.repeated_table_cloudsync') IS NOT NULL) AS init_repeated_public_ok \gset +\if :init_repeated_public_ok +\echo [PASS] (:testid) Test init on repeated_table in public schema +\else +\echo [FAIL] (:testid) Test init on repeated_table in public schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test insert on repeated_table in public schema' +SELECT cloudsync_uuid() AS repeated_id1 \gset +INSERT INTO public.repeated_table (id, data) VALUES (:'repeated_id1', 'public_data'); +SELECT (COUNT(*) = 1) AS insert_repeated_public_ok +FROM public.repeated_table_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'repeated_id1']::text[]) + AND col_name = 'data' \gset +\if :insert_repeated_public_ok +\echo [PASS] (:testid) Test insert metadata on repeated_table in public +\else +\echo [FAIL] (:testid) Test insert metadata on repeated_table in public +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view read for public.repeated_table' +SELECT COUNT(*) AS changes_view_repeated_count +FROM cloudsync_changes +WHERE tbl = 'repeated_table' \gset +SELECT COUNT(*) AS changes_meta_repeated_count +FROM public.repeated_table_cloudsync \gset +SELECT (:changes_view_repeated_count::int = :changes_meta_repeated_count::int) AS changes_read_repeated_ok \gset +\if :changes_read_repeated_ok +\echo [PASS] (:testid) Test cloudsync_changes view read for public.repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_changes view read for public.repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view write for public.repeated_table' +SELECT cloudsync_uuid() AS repeated_id2 \gset +INSERT INTO cloudsync_changes (tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) +VALUES ( + 'repeated_table', + cloudsync_pk_encode(VARIADIC ARRAY[:'repeated_id2']::text[]), + 'data', + -- "public_write" encoded as cloudsync text value (type 0x0b + len 0x0c) + decode('0b0c7075626c69635f7772697465', 'hex'), + 1, + cloudsync_db_version_next(), + cloudsync_siteid(), + 1, + 0 +); +SELECT (COUNT(*) = 1) AS changes_write_repeated_ok +FROM public.repeated_table +WHERE id = :'repeated_id2' AND data = 'public_write' \gset +\if :changes_write_repeated_ok +\echo [PASS] (:testid) Test cloudsync_changes view write for public.repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_changes view write for public.repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cleanup on table with ambiguous name' +SELECT cloudsync_cleanup('repeated_table') AS _cleanup_repeated2 \gset +SELECT (to_regclass('public.repeated_table_cloudsync') IS NULL) AS cleanup_repeated_ok \gset +\if :cleanup_repeated_ok +\echo [PASS] (:testid) Test cleanup on repeated_table +\else +\echo [FAIL] (:testid) Test cleanup on repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_set_schema and init on test_schema' +SELECT cloudsync_set_schema('test_schema') AS _set_schema \gset +SELECT cloudsync_init('repeated_table', 'CLS', 1) AS _init_repeated_test_schema \gset +SELECT cloudsync_table_schema('repeated_table') AS repeated_schema_test_schema \gset +SELECT (:'repeated_schema_test_schema' = 'test_schema') AS repeated_schema_test_schema_ok \gset +\if :repeated_schema_test_schema_ok +\echo [PASS] (:testid) Test cloudsync_table_schema returns test_schema for repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_table_schema returns test_schema for repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (to_regclass('test_schema.repeated_table_cloudsync') IS NOT NULL) AS init_repeated_test_schema_ok \gset +\if :init_repeated_test_schema_ok +\echo [PASS] (:testid) Test init on repeated_table in test_schema +\else +\echo [FAIL] (:testid) Test init on repeated_table in test_schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test that public.repeated_table_cloudsync was not recreated' +SELECT (to_regclass('public.repeated_table_cloudsync') IS NULL) AS public_still_clean_ok \gset +\if :public_still_clean_ok +\echo [PASS] (:testid) Test public.repeated_table_cloudsync still cleaned up +\else +\echo [FAIL] (:testid) Test public.repeated_table_cloudsync should not exist +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- reset the current schema to check if the next connection load the correct configuration +SELECT cloudsync_set_schema('public') AS _reset_schema \gset + +-- Reset the connection to test if if loads the correct configuration for the table on the correct schema +\connect cloudsync_test_12 +\ir helper_psql_conn_setup.sql + +-- 'Test insert on repeated_table in test_schema' +SELECT cloudsync_uuid() AS repeated_id3 \gset +INSERT INTO test_schema.repeated_table (id, data) VALUES (:'repeated_id3', 'test_schema_data'); +SELECT (COUNT(*) = 1) AS insert_repeated_test_schema_ok +FROM test_schema.repeated_table_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'repeated_id3']::text[]) + AND col_name = 'data' \gset +\if :insert_repeated_test_schema_ok +\echo [PASS] (:testid) Test insert metadata on repeated_table in test_schema +\else +\echo [FAIL] (:testid) Test insert metadata on repeated_table in test_schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view read for test_schema.repeated_table' +SELECT COUNT(*) AS changes_view_test_schema_count +FROM cloudsync_changes +WHERE tbl = 'repeated_table' \gset +SELECT COUNT(*) AS changes_meta_test_schema_count +FROM test_schema.repeated_table_cloudsync \gset +SELECT (:changes_view_test_schema_count::int = :changes_meta_test_schema_count::int) AS changes_read_test_schema_ok \gset +\if :changes_read_test_schema_ok +\echo [PASS] (:testid) Test cloudsync_changes view read for test_schema.repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_changes view read for test_schema.repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view write for test_schema.repeated_table' +SELECT cloudsync_uuid() AS repeated_id4 \gset +INSERT INTO cloudsync_changes (tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) +VALUES ( + 'repeated_table', + cloudsync_pk_encode(VARIADIC ARRAY[:'repeated_id4']::text[]), + 'data', + -- "testschema_write" encoded as cloudsync text value (type 0x0b + len 0x10) + decode('0b1074657374736368656d615f7772697465', 'hex'), + 1, + cloudsync_db_version_next(), + cloudsync_siteid(), + 1, + 0 +); +SELECT (COUNT(*) = 1) AS changes_write_test_schema_ok +FROM test_schema.repeated_table +WHERE id = :'repeated_id4' AND data = 'testschema_write' \gset +\if :changes_write_test_schema_ok +\echo [PASS] (:testid) Test cloudsync_changes view write for test_schema.repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_changes view write for test_schema.repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cleanup on repeated_table on test_schema' +SELECT cloudsync_cleanup('repeated_table') AS _cleanup_repeated3 \gset +SELECT (to_regclass('test_schema.repeated_table_cloudsync') IS NULL) AS cleanup_repeated3_ok \gset +\if :cleanup_repeated3_ok +\echo [PASS] (:testid) Test cleanup on repeated_table on test_schema +\else +\echo [FAIL] (:testid) Test cleanup on repeated_table on test_schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Reset schema to public for subsequent tests' +SELECT cloudsync_set_schema('public') AS _reset_schema \gset +SELECT current_schema() AS current_schema_after_reset \gset +SELECT (:'current_schema_after_reset' = 'public') AS schema_reset_ok \gset +\if :schema_reset_ok +\echo [PASS] (:testid) Test schema reset to public +\else +\echo [FAIL] (:testid) Test schema reset to public +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :{?DEBUG_MERGE} +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_12; +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_12; +\endif \ No newline at end of file diff --git a/test/postgresql/13_per_table_schema_tracking.sql b/test/postgresql/13_per_table_schema_tracking.sql new file mode 100644 index 00000000..8729fe68 --- /dev/null +++ b/test/postgresql/13_per_table_schema_tracking.sql @@ -0,0 +1,234 @@ +-- Per-Table Schema Tracking Tests +-- Tests from plans/PLAN_per_table_schema_tracking.md + +\set testid '13' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup any existing test databases and schemas +DROP DATABASE IF EXISTS cloudsync_test_13; +CREATE DATABASE cloudsync_test_13; + +\connect cloudsync_test_13 +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- ============================================================================ +-- Test 1: Basic Schema Detection +-- ============================================================================ + +CREATE SCHEMA test_schema; +CREATE TABLE test_schema.products (id TEXT PRIMARY KEY, name TEXT NOT NULL DEFAULT ''); +SELECT cloudsync_set_schema('test_schema') AS _set_schema \gset +SELECT cloudsync_init('products', 'CLS', 1) AS _init_products \gset + +-- Test: Verify schema is detected correctly +SELECT cloudsync_table_schema('products') AS detected_schema \gset +SELECT (:'detected_schema' = 'test_schema') AS basic_schema_detection_ok \gset +\if :basic_schema_detection_ok +\echo [PASS] (:testid) Basic Schema Detection +\else +\echo [FAIL] (:testid) Basic Schema Detection - Expected 'test_schema', got '':detected_schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 2: Same-Connection Duplicate Prevention +-- ============================================================================ + +CREATE TABLE public.users (id TEXT PRIMARY KEY, name TEXT NOT NULL DEFAULT ''); + +SELECT cloudsync_set_schema('public') AS _set_public \gset +SELECT cloudsync_init('users', 'CLS', 1) AS _init_users_public \gset + +-- Attempt to init again in same connection (should FAIL) +DO $$ +BEGIN + PERFORM cloudsync_init('users', 'CLS', 1); + RAISE EXCEPTION 'Expected error but init succeeded'; +EXCEPTION + WHEN OTHERS THEN + -- Expected to fail + NULL; +END $$; + +\echo [PASS] (:testid) Same-Connection Duplicate Prevention + +-- ============================================================================ +-- Test 3: Schema Not Found (NULL handling) +-- ============================================================================ + +CREATE TABLE orphan_table (id TEXT PRIMARY KEY); + +-- Test: Querying schema of non-initialized table should return NULL or empty +DO $$ +DECLARE + orphan_schema TEXT; +BEGIN + SELECT cloudsync_table_schema('orphan_table') INTO orphan_schema; + IF orphan_schema IS NOT NULL THEN + RAISE EXCEPTION 'Expected NULL for orphan table schema, got %', orphan_schema; + END IF; +END $$; + +\echo [PASS] (:testid) Schema Not Found (NULL handling) + +-- ============================================================================ +-- Test 4: Schema Setting Does Not Affect Existing Tables +-- ============================================================================ + +CREATE SCHEMA schema_a; +CREATE SCHEMA schema_b; +CREATE TABLE schema_a.orders (id TEXT PRIMARY KEY, total TEXT NOT NULL DEFAULT '0'); +CREATE TABLE schema_b.products_b (id TEXT PRIMARY KEY, name TEXT NOT NULL DEFAULT ''); + +-- Initialize in schema_a +SELECT cloudsync_set_schema('schema_a') AS _set_schema_a \gset +SELECT cloudsync_init('orders', 'CLS', 1) AS _init_orders \gset + +-- Verify schema +SELECT cloudsync_table_schema('orders') AS orders_schema_before \gset +SELECT (:'orders_schema_before' = 'schema_a') AS orders_schema_before_ok \gset +\if :orders_schema_before_ok +\echo [PASS] (:testid) Schema Setting - Initial schema correct (schema_a) +\else +\echo [FAIL] (:testid) Schema Setting - Initial schema incorrect. Expected 'schema_a', got '':orders_schema_before +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Change global schema setting +SELECT cloudsync_set_schema('schema_b') AS _set_schema_b \gset + +-- Test: Existing table still uses original schema +SELECT cloudsync_table_schema('orders') AS orders_schema_after \gset +SELECT (:'orders_schema_after' = 'schema_a') AS orders_schema_unchanged_ok \gset +\if :orders_schema_unchanged_ok +\echo [PASS] (:testid) Schema Setting Does Not Affect Existing Tables +\else +\echo [FAIL] (:testid) Schema Setting Does Not Affect Existing Tables - Expected 'schema_a', got '':orders_schema_after +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test: New initialization uses new schema +SELECT cloudsync_init('products_b', 'CLS', 1) AS _init_products_b \gset +SELECT cloudsync_table_schema('products_b') AS products_schema \gset +SELECT (:'products_schema' = 'schema_b') AS new_table_uses_new_schema_ok \gset +\if :new_table_uses_new_schema_ok +\echo [PASS] (:testid) New table uses new schema setting (schema_b) +\else +\echo [FAIL] (:testid) New table uses new schema setting - Expected 'schema_b', got '':products_schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 5: Multi-Schema Database with Different Table Names +-- ============================================================================ + +CREATE SCHEMA sales; +CREATE SCHEMA analytics; + +CREATE TABLE sales.orders_sales (id TEXT PRIMARY KEY, total TEXT NOT NULL DEFAULT '0'); +CREATE TABLE analytics.reports (id TEXT PRIMARY KEY, data TEXT NOT NULL DEFAULT '{}'); + +SELECT cloudsync_set_schema('sales') AS _set_sales \gset +SELECT cloudsync_init('orders_sales', 'CLS', 1) AS _init_orders_sales \gset + +SELECT cloudsync_set_schema('analytics') AS _set_analytics \gset +SELECT cloudsync_init('reports', 'CLS', 1) AS _init_reports \gset + +-- Both should work independently +INSERT INTO sales.orders_sales VALUES (cloudsync_uuid()::text, '100.00'); +INSERT INTO analytics.reports VALUES (cloudsync_uuid()::text, '{"type":"summary"}'); + +-- Verify changes tracked in correct schemas +SELECT + (SELECT COUNT(*) > 0 FROM sales.orders_sales_cloudsync) AND + (SELECT COUNT(*) > 0 FROM analytics.reports_cloudsync) AS multi_schema_ok \gset +\if :multi_schema_ok +\echo [PASS] (:testid) Multi-Schema Database with Different Table Names +\else +\echo [FAIL] (:testid) Multi-Schema Database with Different Table Names +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 6: System Tables in Public Schema +-- ============================================================================ + +CREATE SCHEMA custom_schema; +SELECT cloudsync_set_schema('custom_schema') AS _set_custom \gset +CREATE TABLE custom_schema.test_table (id TEXT PRIMARY KEY, val TEXT NOT NULL DEFAULT ''); +SELECT cloudsync_init('test_table', 'CLS', 1) AS _init_test_table \gset + +-- System tables should still be in public +SELECT COUNT(*) AS system_tables_in_public FROM pg_tables +WHERE tablename IN ('cloudsync_settings', 'cloudsync_site_id', 'cloudsync_table_settings', 'cloudsync_schema_versions') + AND schemaname = 'public' \gset + +-- Metadata table should be in custom_schema +SELECT schemaname AS metadata_schema FROM pg_tables +WHERE tablename = 'test_table_cloudsync' \gset + +SELECT (:system_tables_in_public = 4 AND :'metadata_schema' = 'custom_schema') AS system_tables_ok \gset +\if :system_tables_ok +\echo [PASS] (:testid) System Tables in Public Schema +\else +\echo [FAIL] (:testid) System Tables in Public Schema - System tables: :system_tables_in_public/4, metadata schema: ':metadata_schema' +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 7: Metadata Table Location Detection +-- ============================================================================ + +-- Verify that metadata tables are created in the correct schemas +SELECT COUNT(*) AS products_meta FROM pg_tables +WHERE tablename = 'products_cloudsync' AND schemaname = 'test_schema' \gset + +SELECT COUNT(*) AS users_meta FROM pg_tables +WHERE tablename = 'users_cloudsync' AND schemaname = 'public' \gset + +SELECT COUNT(*) AS orders_meta FROM pg_tables +WHERE tablename = 'orders_cloudsync' AND schemaname = 'schema_a' \gset + +SELECT (:products_meta = 1 AND :users_meta = 1 AND :orders_meta = 1) AS metadata_locations_ok \gset +\if :metadata_locations_ok +\echo [PASS] (:testid) Metadata Table Location Detection +\else +\echo [FAIL] (:testid) Metadata Table Location Detection - products: :products_meta, users: :users_meta, orders: :orders_meta +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 8: Schema-Qualified Queries Work Correctly +-- ============================================================================ + +-- Insert data into different schemas and verify they're independent +INSERT INTO test_schema.products VALUES (cloudsync_uuid()::text, 'Product A'); +INSERT INTO public.users VALUES (cloudsync_uuid()::text, 'User A'); +INSERT INTO schema_a.orders VALUES (cloudsync_uuid()::text, '50.00'); + +-- Count rows in each table +SELECT COUNT(*) AS products_count FROM test_schema.products \gset +SELECT COUNT(*) AS users_count FROM public.users \gset +SELECT COUNT(*) AS orders_count FROM schema_a.orders \gset + +-- All should have at least one row +SELECT (:products_count > 0 AND :users_count > 0 AND :orders_count > 0) AS qualified_queries_ok \gset +\if :qualified_queries_ok +\echo [PASS] (:testid) Schema-Qualified Queries Work Correctly +\else +\echo [FAIL] (:testid) Schema-Qualified Queries Work Correctly - products: :products_count, users: :users_count, orders: :orders_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_13; +\endif diff --git a/test/postgresql/14_datatype_roundtrip.sql b/test/postgresql/14_datatype_roundtrip.sql new file mode 100644 index 00000000..a1393109 --- /dev/null +++ b/test/postgresql/14_datatype_roundtrip.sql @@ -0,0 +1,404 @@ +-- DBTYPE Roundtrip Test +-- Tests encoding/decoding of all DBTYPEs (INTEGER, FLOAT, TEXT, BLOB, NULL) +-- in the internal value representation during database synchronization + +\set testid '14' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_14a; +DROP DATABASE IF EXISTS cloudsync_test_14b; +CREATE DATABASE cloudsync_test_14a; +CREATE DATABASE cloudsync_test_14b; + +-- ============================================================================ +-- Setup Database A with comprehensive table +-- ============================================================================ + +\connect cloudsync_test_14a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with composite primary key and all data types +CREATE TABLE all_types ( + -- Composite primary key (TEXT columns as required by CloudSync) + id1 TEXT NOT NULL, + id2 TEXT NOT NULL, + PRIMARY KEY (id1, id2), + + -- INTEGER columns + col_int_notnull INTEGER NOT NULL DEFAULT 0, + col_int_nullable INTEGER, + + -- FLOAT columns (using DOUBLE PRECISION in PostgreSQL) + col_float_notnull DOUBLE PRECISION NOT NULL DEFAULT 0.0, + col_float_nullable DOUBLE PRECISION, + + -- TEXT columns + col_text_notnull TEXT NOT NULL DEFAULT '', + col_text_nullable TEXT, + + -- BLOB columns (BYTEA in PostgreSQL) + col_blob_notnull BYTEA NOT NULL DEFAULT E'\\x00', + col_blob_nullable BYTEA +); + +-- Initialize CloudSync +SELECT cloudsync_init('all_types', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Insert test data with various values for each type +-- ============================================================================ + +-- Row 1: All non-null values +INSERT INTO all_types VALUES ( + 'pk1', 'pk2', + -- INTEGER + 42, 100, + -- FLOAT + 3.14159, 2.71828, + -- TEXT + 'hello world', 'test string', + -- BLOB + E'\\xDEADBEEF', E'\\xCAFEBABE' +); + +-- Row 2: Mix of null and non-null +INSERT INTO all_types (id1, id2, col_int_notnull, col_float_notnull, col_text_notnull, col_blob_notnull) +VALUES ( + 'pk3', 'pk4', + -999, + -123.456, + 'only required fields', + E'\\x0102030405' +); + +-- Row 3: Edge cases - zeros, empty strings, single byte blob +INSERT INTO all_types VALUES ( + 'pk5', 'pk6', + 0, 0, + 0.0, 0.0, + '', '', + E'\\x00', E'\\x00' +); + +-- Row 4: Large values +INSERT INTO all_types VALUES ( + 'pk7', 'pk8', + 2147483647, -2147483648, -- INT max and min + 1.7976931348623157e+308, -1.7976931348623157e+308, -- Large floats + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. ' || + 'Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', + 'Another long text with special chars: café, naïve, 日本語, emoji: 🚀', + E'\\xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', -- Large blob (16 bytes of 0xFF) + E'\\x0102030405060708090A0B0C0D0E0F10' -- Sequential bytes +); + +-- Row 5: Special characters in text +INSERT INTO all_types VALUES ( + 'pk9', 'pk10', + 1, 2, + 1.5, 2.5, + E'Special\nchars:\t\r\nand\\backslash', -- Escaped characters + E'Quote''s and "double" quotes', + E'\\x5C00', -- Backslash byte and null byte + E'\\x0D0A' -- CR LF +); + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id1 || ':' || id2 || ':' || + COALESCE(col_int_notnull::text, 'NULL') || ':' || + COALESCE(col_int_nullable::text, 'NULL') || ':' || + COALESCE(col_float_notnull::text, 'NULL') || ':' || + COALESCE(col_float_nullable::text, 'NULL') || ':' || + COALESCE(col_text_notnull, 'NULL') || ':' || + COALESCE(col_text_nullable, 'NULL') || ':' || + COALESCE(encode(col_blob_notnull, 'hex'), 'NULL') || ':' || + COALESCE(encode(col_blob_nullable, 'hex'), 'NULL'), + '|' ORDER BY id1, id2 + ), + '' + ) +) AS hash_a FROM all_types \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_14b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create identical table schema +CREATE TABLE all_types ( + id1 TEXT NOT NULL, + id2 TEXT NOT NULL, + PRIMARY KEY (id1, id2), + col_int_notnull INTEGER NOT NULL DEFAULT 0, + col_int_nullable INTEGER, + col_float_notnull DOUBLE PRECISION NOT NULL DEFAULT 0.0, + col_float_nullable DOUBLE PRECISION, + col_text_notnull TEXT NOT NULL DEFAULT '', + col_text_nullable TEXT, + col_blob_notnull BYTEA NOT NULL DEFAULT E'\\x00', + col_blob_nullable BYTEA +); + +-- Initialize CloudSync +SELECT cloudsync_init('all_types', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +-- Compute hash of Database B data (should match Database A) +SELECT md5( + COALESCE( + string_agg( + id1 || ':' || id2 || ':' || + COALESCE(col_int_notnull::text, 'NULL') || ':' || + COALESCE(col_int_nullable::text, 'NULL') || ':' || + COALESCE(col_float_notnull::text, 'NULL') || ':' || + COALESCE(col_float_nullable::text, 'NULL') || ':' || + COALESCE(col_text_notnull, 'NULL') || ':' || + COALESCE(col_text_nullable, 'NULL') || ':' || + COALESCE(encode(col_blob_notnull, 'hex'), 'NULL') || ':' || + COALESCE(encode(col_blob_nullable, 'hex'), 'NULL'), + '|' ORDER BY id1, id2 + ), + '' + ) +) AS hash_b FROM all_types \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +-- Compare hashes +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_a FROM all_types \gset +\connect cloudsync_test_14a +SELECT COUNT(*) AS count_a_orig FROM all_types \gset + +\connect cloudsync_test_14b +SELECT (:count_a = :count_a_orig) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_a rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a_orig, Database B: :count_a +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test specific data type preservation +-- ============================================================================ + +-- Verify INTEGER values +SELECT + (SELECT col_int_notnull FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') = 42 AND + (SELECT col_int_nullable FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') = 100 AND + (SELECT col_int_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS integers_ok \gset +\if :integers_ok +\echo [PASS] (:testid) INTEGER type preservation +\else +\echo [FAIL] (:testid) INTEGER type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify FLOAT values (with tolerance for floating point) +SELECT + ABS((SELECT col_float_notnull FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') - 3.14159) < 0.00001 AND + ABS((SELECT col_float_nullable FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') - 2.71828) < 0.00001 AND + (SELECT col_float_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS floats_ok \gset +\if :floats_ok +\echo [PASS] (:testid) FLOAT type preservation +\else +\echo [FAIL] (:testid) FLOAT type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify TEXT values +SELECT + (SELECT col_text_notnull FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') = 'hello world' AND + (SELECT col_text_nullable FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') = 'test string' AND + (SELECT col_text_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL AND + (SELECT col_text_notnull FROM all_types WHERE id1 = 'pk5' AND id2 = 'pk6') = '' +AS text_ok \gset +\if :text_ok +\echo [PASS] (:testid) TEXT type preservation +\else +\echo [FAIL] (:testid) TEXT type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify BLOB values +SELECT + encode((SELECT col_blob_notnull FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2'), 'hex') = 'deadbeef' AND + encode((SELECT col_blob_nullable FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2'), 'hex') = 'cafebabe' AND + (SELECT col_blob_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS blobs_ok \gset +\if :blobs_ok +\echo [PASS] (:testid) BLOB type preservation +\else +\echo [FAIL] (:testid) BLOB type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify NULL handling +SELECT + (SELECT col_int_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL AND + (SELECT col_float_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL AND + (SELECT col_text_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL AND + (SELECT col_blob_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS nulls_ok \gset +\if :nulls_ok +\echo [PASS] (:testid) NULL type preservation +\else +\echo [FAIL] (:testid) NULL type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test special characters and edge cases +-- ============================================================================ + +-- Verify special characters in TEXT +SELECT + (SELECT col_text_notnull FROM all_types WHERE id1 = 'pk9' AND id2 = 'pk10') = E'Special\nchars:\t\r\nand\\backslash' +AS special_chars_ok \gset +\if :special_chars_ok +\echo [PASS] (:testid) Special characters in TEXT preserved +\else +\echo [FAIL] (:testid) Special characters in TEXT not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify zero values +SELECT + (SELECT col_int_notnull FROM all_types WHERE id1 = 'pk5' AND id2 = 'pk6') = 0 AND + (SELECT col_float_notnull FROM all_types WHERE id1 = 'pk5' AND id2 = 'pk6') = 0.0 +AS zero_values_ok \gset +\if :zero_values_ok +\echo [PASS] (:testid) Zero values preserved +\else +\echo [FAIL] (:testid) Zero values not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test composite primary key encoding +-- ============================================================================ + +-- Verify all primary key combinations are present +SELECT COUNT(DISTINCT (id1, id2)) = 5 AS pk_count_ok FROM all_types \gset +\if :pk_count_ok +\echo [PASS] (:testid) Composite primary keys preserved +\else +\echo [FAIL] (:testid) Composite primary keys not all preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +\connect cloudsync_test_14b + +-- Add a new row in Database B +INSERT INTO all_types VALUES ( + 'pkB1', 'pkB2', + 999, 888, + 9.99, 8.88, + 'from database B', 'bidirectional test', + E'\\xBBBBBBBB', E'\\xAAAAAAAA' +); + +-- Encode payload from Database B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply to Database A +\connect cloudsync_test_14a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +-- Verify the new row exists in Database A +SELECT COUNT(*) = 1 AS bidirectional_ok +FROM all_types +WHERE id1 = 'pkB1' AND id2 = 'pkB2' AND col_text_notnull = 'from database B' \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_14a; +DROP DATABASE IF EXISTS cloudsync_test_14b; +\endif diff --git a/test/postgresql/15_datatype_roundtrip_unmapped.sql b/test/postgresql/15_datatype_roundtrip_unmapped.sql new file mode 100644 index 00000000..cfc2fcf9 --- /dev/null +++ b/test/postgresql/15_datatype_roundtrip_unmapped.sql @@ -0,0 +1,388 @@ +-- DBTYPE Roundtrip Test (Unmapped PostgreSQL Types) +-- Tests encoding/decoding for types that are not explicitly mapped to +-- DBTYPE_INTEGER/FLOAT/TEXT/BLOB/NULL in the common layer. + +\set testid '15' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_15a; +DROP DATABASE IF EXISTS cloudsync_test_15b; +CREATE DATABASE cloudsync_test_15a; +CREATE DATABASE cloudsync_test_15b; + +-- ============================================================================ +-- Setup Database A with unmapped types table +-- ============================================================================ + +\connect cloudsync_test_15a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with composite primary key and unmapped types +CREATE TABLE unmapped_types ( + -- Composite primary key (TEXT columns as required by CloudSync) + id1 TEXT NOT NULL, + id2 TEXT NOT NULL, + PRIMARY KEY (id1, id2), + + -- JSONB columns + col_jsonb_notnull JSONB NOT NULL DEFAULT '{}'::jsonb, + col_jsonb_nullable JSONB, + + -- UUID columns + col_uuid_notnull UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + col_uuid_nullable UUID, + + -- INET columns + col_inet_notnull INET NOT NULL DEFAULT '0.0.0.0', + col_inet_nullable INET, + + -- CIDR columns + col_cidr_notnull CIDR NOT NULL DEFAULT '0.0.0.0/0', + col_cidr_nullable CIDR, + + -- RANGE columns + col_int4range_notnull INT4RANGE NOT NULL DEFAULT 'empty'::int4range, + col_int4range_nullable INT4RANGE +); + +-- Initialize CloudSync +SELECT cloudsync_init('unmapped_types', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Insert test data with various values for each type +-- ============================================================================ + +-- Row 1: All non-null values +INSERT INTO unmapped_types VALUES ( + 'pk1', 'pk2', + '{"a":1,"b":[1,2]}'::jsonb, '{"k":"v"}'::jsonb, + '11111111-1111-1111-1111-111111111111', '22222222-2222-2222-2222-222222222222', + '192.168.1.10', '10.1.2.3', + '10.0.0.0/24', '192.168.0.0/16', + '[1,10)', '[20,30)' +); + +-- Row 2: Mix of null and non-null +INSERT INTO unmapped_types ( + id1, id2, + col_jsonb_notnull, + col_uuid_notnull, + col_inet_notnull, + col_cidr_notnull, + col_int4range_notnull +) VALUES ( + 'pk3', 'pk4', + '{"only":"required"}'::jsonb, + '33333333-3333-3333-3333-333333333333', + '127.0.0.1', + '127.0.0.0/8', + '[0,1)' +); + +-- Row 3: Edge cases - empty JSON, empty range +INSERT INTO unmapped_types VALUES ( + 'pk5', 'pk6', + '{}'::jsonb, '[]'::jsonb, + '44444444-4444-4444-4444-444444444444', '55555555-5555-5555-5555-555555555555', + '0.0.0.0', '255.255.255.255', + '0.0.0.0/0', '255.255.255.0/24', + 'empty'::int4range, 'empty'::int4range +); + +-- Row 4: IPv6 + negative range +INSERT INTO unmapped_types VALUES ( + 'pk7', 'pk8', + '{"ipv6":true}'::jsonb, '{"note":"range"}'::jsonb, + '66666666-6666-6666-6666-666666666666', '77777777-7777-7777-7777-777777777777', + '2001:db8::1', '2001:db8::2', + '2001:db8::/32', '2001:db8:abcd::/48', + '[-5,5]', '[-10,10)' +); + +-- Row 5: Nested JSON +INSERT INTO unmapped_types VALUES ( + 'pk9', 'pk10', + '{"obj":{"x":1,"y":[2,3]}}'::jsonb, '{"arr":[{"a":1},{"b":2}]}'::jsonb, + '88888888-8888-8888-8888-888888888888', '99999999-9999-9999-9999-999999999999', + '172.16.0.1', '172.16.0.2', + '172.16.0.0/12', '172.16.1.0/24', + '[100,200)', '[200,300)' +); + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id1 || ':' || id2 || ':' || + COALESCE(col_jsonb_notnull::text, 'NULL') || ':' || + COALESCE(col_jsonb_nullable::text, 'NULL') || ':' || + COALESCE(col_uuid_notnull::text, 'NULL') || ':' || + COALESCE(col_uuid_nullable::text, 'NULL') || ':' || + COALESCE(col_inet_notnull::text, 'NULL') || ':' || + COALESCE(col_inet_nullable::text, 'NULL') || ':' || + COALESCE(col_cidr_notnull::text, 'NULL') || ':' || + COALESCE(col_cidr_nullable::text, 'NULL') || ':' || + COALESCE(col_int4range_notnull::text, 'NULL') || ':' || + COALESCE(col_int4range_nullable::text, 'NULL'), + '|' ORDER BY id1, id2 + ), + '' + ) +) AS hash_a FROM unmapped_types \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_15b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create identical table schema +CREATE TABLE unmapped_types ( + id1 TEXT NOT NULL, + id2 TEXT NOT NULL, + PRIMARY KEY (id1, id2), + col_jsonb_notnull JSONB NOT NULL DEFAULT '{}'::jsonb, + col_jsonb_nullable JSONB, + col_uuid_notnull UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + col_uuid_nullable UUID, + col_inet_notnull INET NOT NULL DEFAULT '0.0.0.0', + col_inet_nullable INET, + col_cidr_notnull CIDR NOT NULL DEFAULT '0.0.0.0/0', + col_cidr_nullable CIDR, + col_int4range_notnull INT4RANGE NOT NULL DEFAULT 'empty'::int4range, + col_int4range_nullable INT4RANGE +); + +-- Initialize CloudSync +SELECT cloudsync_init('unmapped_types', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +-- Compute hash of Database B data (should match Database A) +SELECT md5( + COALESCE( + string_agg( + id1 || ':' || id2 || ':' || + COALESCE(col_jsonb_notnull::text, 'NULL') || ':' || + COALESCE(col_jsonb_nullable::text, 'NULL') || ':' || + COALESCE(col_uuid_notnull::text, 'NULL') || ':' || + COALESCE(col_uuid_nullable::text, 'NULL') || ':' || + COALESCE(col_inet_notnull::text, 'NULL') || ':' || + COALESCE(col_inet_nullable::text, 'NULL') || ':' || + COALESCE(col_cidr_notnull::text, 'NULL') || ':' || + COALESCE(col_cidr_nullable::text, 'NULL') || ':' || + COALESCE(col_int4range_notnull::text, 'NULL') || ':' || + COALESCE(col_int4range_nullable::text, 'NULL'), + '|' ORDER BY id1, id2 + ), + '' + ) +) AS hash_b FROM unmapped_types \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +-- Compare hashes +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM unmapped_types \gset +\connect cloudsync_test_15a +SELECT COUNT(*) AS count_a_orig FROM unmapped_types \gset + +\connect cloudsync_test_15b +SELECT (:count_b = :count_a_orig) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_b rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a_orig, Database B: :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test specific data type preservation +-- ============================================================================ + +-- JSONB values +SELECT + (SELECT col_jsonb_notnull FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '{"a":1,"b":[1,2]}'::jsonb AND + (SELECT col_jsonb_nullable FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '{"k":"v"}'::jsonb AND + (SELECT col_jsonb_nullable FROM unmapped_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS jsonb_ok \gset +\if :jsonb_ok +\echo [PASS] (:testid) JSONB type preservation +\else +\echo [FAIL] (:testid) JSONB type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- UUID values +SELECT + (SELECT col_uuid_notnull FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '11111111-1111-1111-1111-111111111111'::uuid AND + (SELECT col_uuid_nullable FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '22222222-2222-2222-2222-222222222222'::uuid AND + (SELECT col_uuid_nullable FROM unmapped_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS uuid_ok \gset +\if :uuid_ok +\echo [PASS] (:testid) UUID type preservation +\else +\echo [FAIL] (:testid) UUID type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- INET values +SELECT + (SELECT col_inet_notnull FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '192.168.1.10'::inet AND + (SELECT col_inet_nullable FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '10.1.2.3'::inet AND + (SELECT col_inet_nullable FROM unmapped_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS inet_ok \gset +\if :inet_ok +\echo [PASS] (:testid) INET type preservation +\else +\echo [FAIL] (:testid) INET type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- CIDR values +SELECT + (SELECT col_cidr_notnull FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '10.0.0.0/24'::cidr AND + (SELECT col_cidr_nullable FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '192.168.0.0/16'::cidr AND + (SELECT col_cidr_nullable FROM unmapped_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS cidr_ok \gset +\if :cidr_ok +\echo [PASS] (:testid) CIDR type preservation +\else +\echo [FAIL] (:testid) CIDR type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- RANGE values +SELECT + (SELECT col_int4range_notnull FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '[1,10)'::int4range AND + (SELECT col_int4range_nullable FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '[20,30)'::int4range AND + (SELECT col_int4range_nullable FROM unmapped_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS ranges_ok \gset +\if :ranges_ok +\echo [PASS] (:testid) RANGE type preservation +\else +\echo [FAIL] (:testid) RANGE type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test composite primary key encoding +-- ============================================================================ + +-- Verify all primary key combinations are present +SELECT COUNT(DISTINCT (id1, id2)) = 5 AS pk_count_ok FROM unmapped_types \gset +\if :pk_count_ok +\echo [PASS] (:testid) Composite primary keys preserved +\else +\echo [FAIL] (:testid) Composite primary keys not all preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +\connect cloudsync_test_15b + +-- Add a new row in Database B +INSERT INTO unmapped_types VALUES ( + 'pkB1', 'pkB2', + '{"from":"database B"}'::jsonb, '{"bidirectional":true}'::jsonb, + 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', + '10.10.10.10', '10.10.10.11', + '10.10.0.0/16', '10.10.10.0/24', + '[50,60)', '[60,70)' +); + +-- Encode payload from Database B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply to Database A +\connect cloudsync_test_15a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +-- Verify the new row exists in Database A +SELECT COUNT(*) = 1 AS bidirectional_ok +FROM unmapped_types +WHERE id1 = 'pkB1' AND id2 = 'pkB2' AND col_jsonb_notnull = '{"from":"database B"}'::jsonb \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_15a; +DROP DATABASE IF EXISTS cloudsync_test_15b; +\endif diff --git a/test/postgresql/16_composite_pk_text_int_roundtrip.sql b/test/postgresql/16_composite_pk_text_int_roundtrip.sql new file mode 100644 index 00000000..c2fcb8ec --- /dev/null +++ b/test/postgresql/16_composite_pk_text_int_roundtrip.sql @@ -0,0 +1,217 @@ +-- Composite PK Roundtrip Test (TEXT + INTEGER) +-- Tests roundtrip with a composite primary key that mixes TEXT and INTEGER. + +\set testid '16' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_16a; +DROP DATABASE IF EXISTS cloudsync_test_16b; +CREATE DATABASE cloudsync_test_16a; +CREATE DATABASE cloudsync_test_16b; + +-- ============================================================================ +-- Setup Database A with composite PK (TEXT + INTEGER) +-- ============================================================================ + +\connect cloudsync_test_16a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE mixed_pk ( + id_text TEXT NOT NULL, + id_int INTEGER NOT NULL, + PRIMARY KEY (id_text, id_int), + + col_text TEXT NOT NULL DEFAULT '', + col_int INTEGER NOT NULL DEFAULT 0, + col_float DOUBLE PRECISION NOT NULL DEFAULT 0.0, + col_blob BYTEA +); + +-- Initialize CloudSync (skip int pk check for this test) +SELECT cloudsync_init('mixed_pk', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Insert test data +-- ============================================================================ + +INSERT INTO mixed_pk VALUES ('pkA', 1, 'hello', 42, 3.14, E'\\xDEADBEEF'); +INSERT INTO mixed_pk VALUES ('pkA', 2, 'world', 7, 2.71, NULL); +INSERT INTO mixed_pk VALUES ('pkB', 1, '', 0, 0.0, E'\\x00'); +INSERT INTO mixed_pk VALUES ('pkC', 10, 'edge', -1, -1.5, E'\\xCAFEBABE'); +INSERT INTO mixed_pk VALUES ('pkD', 20, 'more', 999, 123.456, E'\\x010203'); + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id_text || ':' || id_int::text || ':' || + COALESCE(col_text, 'NULL') || ':' || + COALESCE(col_int::text, 'NULL') || ':' || + COALESCE(col_float::text, 'NULL') || ':' || + COALESCE(encode(col_blob, 'hex'), 'NULL'), + '|' ORDER BY id_text, id_int + ), + '' + ) +) AS hash_a FROM mixed_pk \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_16b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE mixed_pk ( + id_text TEXT NOT NULL, + id_int INTEGER NOT NULL, + PRIMARY KEY (id_text, id_int), + col_text TEXT NOT NULL DEFAULT '', + col_int INTEGER NOT NULL DEFAULT 0, + col_float DOUBLE PRECISION NOT NULL DEFAULT 0.0, + col_blob BYTEA +); + +-- Initialize CloudSync (skip int pk check for this test) +SELECT cloudsync_init('mixed_pk', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id_text || ':' || id_int::text || ':' || + COALESCE(col_text, 'NULL') || ':' || + COALESCE(col_int::text, 'NULL') || ':' || + COALESCE(col_float::text, 'NULL') || ':' || + COALESCE(encode(col_blob, 'hex'), 'NULL'), + '|' ORDER BY id_text, id_int + ), + '' + ) +) AS hash_b FROM mixed_pk \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM mixed_pk \gset +\connect cloudsync_test_16a +SELECT COUNT(*) AS count_a_orig FROM mixed_pk \gset + +\connect cloudsync_test_16b +SELECT (:count_b = :count_a_orig) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_b rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a_orig, Database B: :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test composite primary key encoding +-- ============================================================================ + +SELECT COUNT(DISTINCT (id_text, id_int)) = 5 AS pk_count_ok FROM mixed_pk \gset +\if :pk_count_ok +\echo [PASS] (:testid) Composite primary keys preserved +\else +\echo [FAIL] (:testid) Composite primary keys not all preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +\connect cloudsync_test_16b + +INSERT INTO mixed_pk VALUES ('pkB', 99, 'from B', 123, 9.99, E'\\xBEEF'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_16a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +SELECT COUNT(*) = 1 AS bidirectional_ok +FROM mixed_pk +WHERE id_text = 'pkB' AND id_int = 99 AND col_text = 'from B' \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_16a; +DROP DATABASE IF EXISTS cloudsync_test_16b; +\endif diff --git a/test/postgresql/17_uuid_pk_roundtrip.sql b/test/postgresql/17_uuid_pk_roundtrip.sql new file mode 100644 index 00000000..ceb79528 --- /dev/null +++ b/test/postgresql/17_uuid_pk_roundtrip.sql @@ -0,0 +1,230 @@ +-- UUID Primary Key Roundtrip Test +-- Tests roundtrip with a UUID primary key (single column). + +\set testid '17' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_17a; +DROP DATABASE IF EXISTS cloudsync_test_17b; +CREATE DATABASE cloudsync_test_17a; +CREATE DATABASE cloudsync_test_17b; + +-- ============================================================================ +-- Setup Database A with UUID primary key +-- ============================================================================ + +\connect cloudsync_test_17a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE products ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + stock INTEGER NOT NULL DEFAULT 0, + metadata BYTEA +); + +-- Initialize CloudSync +SELECT cloudsync_init('products', 'CLS', 0) AS _init_a \gset + +-- ============================================================================ +-- Insert test data with UUIDs +-- ============================================================================ + +INSERT INTO products VALUES ('550e8400-e29b-41d4-a716-446655440000', 'Product A', 99.99, 100, E'\\xDEADBEEF'); +INSERT INTO products VALUES ('6ba7b810-9dad-11d1-80b4-00c04fd430c8', 'Product B', 49.50, 50, NULL); +INSERT INTO products VALUES ('6ba7b811-9dad-11d1-80b4-00c04fd430c8', 'Product C', 0.0, 0, E'\\x00'); +INSERT INTO products VALUES ('6ba7b812-9dad-11d1-80b4-00c04fd430c8', 'Product D', 123.45, 999, E'\\xCAFEBABE'); +INSERT INTO products VALUES ('6ba7b813-9dad-11d1-80b4-00c04fd430c8', '', -1.0, -1, E'\\x010203'); + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(stock::text, 'NULL') || ':' || + COALESCE(encode(metadata, 'hex'), 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM products \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_17b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE products ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + stock INTEGER NOT NULL DEFAULT 0, + metadata BYTEA +); + +-- Initialize CloudSync +SELECT cloudsync_init('products', 'CLS', 0) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(stock::text, 'NULL') || ':' || + COALESCE(encode(metadata, 'hex'), 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM products \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM products \gset +\connect cloudsync_test_17a +SELECT COUNT(*) AS count_a_orig FROM products \gset + +\connect cloudsync_test_17b +SELECT (:count_b = :count_a_orig) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_b rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a_orig, Database B: :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify UUID primary keys preserved +-- ============================================================================ + +SELECT COUNT(DISTINCT id) = 5 AS uuid_count_ok FROM products \gset +\if :uuid_count_ok +\echo [PASS] (:testid) UUID primary keys preserved +\else +\echo [FAIL] (:testid) UUID primary keys not all preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test specific UUID values +-- ============================================================================ + +SELECT COUNT(*) = 1 AS uuid_test_ok +FROM products +WHERE id = '550e8400-e29b-41d4-a716-446655440000' + AND name = 'Product A' + AND price = 99.99 \gset +\if :uuid_test_ok +\echo [PASS] (:testid) Specific UUID record verified +\else +\echo [FAIL] (:testid) Specific UUID record not found or incorrect +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +\connect cloudsync_test_17b + +INSERT INTO products VALUES ('7ba7b814-9dad-11d1-80b4-00c04fd430c8', 'From B', 77.77, 777, E'\\xBEEF'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_17a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +SELECT COUNT(*) = 1 AS bidirectional_ok +FROM products +WHERE id = '7ba7b814-9dad-11d1-80b4-00c04fd430c8' + AND name = 'From B' + AND price = 77.77 \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_17a; +DROP DATABASE IF EXISTS cloudsync_test_17b; +\endif diff --git a/test/postgresql/18_bulk_insert_performance.sql b/test/postgresql/18_bulk_insert_performance.sql new file mode 100644 index 00000000..89132216 --- /dev/null +++ b/test/postgresql/18_bulk_insert_performance.sql @@ -0,0 +1,244 @@ +-- Bulk Insert Performance Roundtrip Test +-- Tests roundtrip with 1000 rows and measures time for each operation. + +\set testid '18' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_18a; +DROP DATABASE IF EXISTS cloudsync_test_18b; +CREATE DATABASE cloudsync_test_18a; +CREATE DATABASE cloudsync_test_18b; + +-- ============================================================================ +-- Setup Database A +-- ============================================================================ + +\connect cloudsync_test_18a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE items ( + id UUID NOT NULL PRIMARY KEY DEFAULT cloudsync_uuid(), + name TEXT NOT NULL DEFAULT '', + value DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0, + description TEXT +); + +-- Initialize CloudSync +SELECT cloudsync_init('items', 'CLS', 0) AS _init_a \gset + +-- ============================================================================ +-- Record start time +-- ============================================================================ + +SELECT clock_timestamp() AS test_start_time \gset +\echo [INFO] (:testid) Test started at :test_start_time + +-- ============================================================================ +-- Insert 1000 rows and measure time +-- ============================================================================ + +SELECT clock_timestamp() AS insert_start_time \gset + +INSERT INTO items (name, value, quantity, description) +SELECT + 'Item ' || i, + (random() * 1000)::DOUBLE PRECISION, + (random() * 100)::INTEGER, + 'Description for item ' || i || ' with some additional text to simulate real data' +FROM generate_series(1, 1000) AS i; + +SELECT clock_timestamp() AS insert_end_time \gset + +SELECT EXTRACT(EPOCH FROM (:'insert_end_time'::timestamp - :'insert_start_time'::timestamp)) * 1000 AS insert_time_ms \gset +\echo [INFO] (:testid) Insert 1000 rows: :insert_time_ms ms + +-- ============================================================================ +-- Verify row count in Database A +-- ============================================================================ + +SELECT COUNT(*) AS count_a FROM items \gset +SELECT (:count_a = 1000) AS insert_count_ok \gset +\if :insert_count_ok +\echo [PASS] (:testid) Inserted 1000 rows successfully +\else +\echo [FAIL] (:testid) Expected 1000 rows, got :count_a +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(description, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM items \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A and measure time +-- ============================================================================ + +SELECT clock_timestamp() AS encode_start_time \gset + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT clock_timestamp() AS encode_end_time \gset + +SELECT EXTRACT(EPOCH FROM (:'encode_end_time'::timestamp - :'encode_start_time'::timestamp)) * 1000 AS encode_time_ms \gset +\echo [INFO] (:testid) Encode payload: :encode_time_ms ms + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Report payload size +SELECT length(:'payload_a_hex') / 2 AS payload_size_bytes \gset +\echo [INFO] (:testid) Payload size: :payload_size_bytes bytes + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_18b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE items ( + id UUID NOT NULL PRIMARY KEY DEFAULT cloudsync_uuid(), + name TEXT NOT NULL DEFAULT '', + value DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0, + description TEXT +); + +-- Initialize CloudSync +SELECT cloudsync_init('items', 'CLS', 0) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B and measure time +-- ============================================================================ + +SELECT clock_timestamp() AS apply_start_time \gset + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +SELECT clock_timestamp() AS apply_end_time \gset + +SELECT EXTRACT(EPOCH FROM (:'apply_end_time'::timestamp - :'apply_start_time'::timestamp)) * 1000 AS apply_time_ms \gset +\echo [INFO] (:testid) Apply payload: :apply_time_ms ms + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(description, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM items \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM items \gset + +SELECT (:count_b = :count_a) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_b rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a, Database B: :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify sample data integrity +-- ============================================================================ + +SELECT COUNT(*) = 1 AS sample_check_ok +FROM items +WHERE name = 'Item 500' \gset +\if :sample_check_ok +\echo [PASS] (:testid) Sample row verified (name='Item 500') +\else +\echo [FAIL] (:testid) Sample row verification failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Calculate and report total elapsed time +-- ============================================================================ + +SELECT clock_timestamp() AS test_end_time \gset + +SELECT EXTRACT(EPOCH FROM (:'test_end_time'::timestamp - :'test_start_time'::timestamp)) * 1000 AS total_time_ms \gset + +\echo [INFO] (:testid) Performance summary: +\echo [INFO] (:testid) - Insert 1000 rows: :insert_time_ms ms +\echo [INFO] (:testid) - Encode payload: :encode_time_ms ms +\echo [INFO] (:testid) - Apply payload: :apply_time_ms ms +\echo [INFO] (:testid) - Total elapsed time: :total_time_ms ms + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_18a; +DROP DATABASE IF EXISTS cloudsync_test_18b; +\endif diff --git a/test/postgresql/19_uuid_pk_with_unmapped_cols.sql b/test/postgresql/19_uuid_pk_with_unmapped_cols.sql new file mode 100644 index 00000000..9111c8e4 --- /dev/null +++ b/test/postgresql/19_uuid_pk_with_unmapped_cols.sql @@ -0,0 +1,681 @@ +-- UUID PK with Unmapped Non-PK Columns Test +-- Tests comprehensive CRUD operations with UUID primary key and unmapped PostgreSQL types. +-- Covers: INSERT, UPDATE non-PK, UPDATE mapped cols, UPDATE PK, DELETE, RESURRECT, bidirectional sync. + +\set testid '19' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_19a; +DROP DATABASE IF EXISTS cloudsync_test_19b; +CREATE DATABASE cloudsync_test_19a; +CREATE DATABASE cloudsync_test_19b; + +-- ============================================================================ +-- Setup Database A with UUID PK and unmapped non-PK columns +-- ============================================================================ + +\connect cloudsync_test_19a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE items ( + id UUID PRIMARY KEY, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + related_id UUID, + ip_address INET NOT NULL DEFAULT '0.0.0.0', + network CIDR, + name TEXT NOT NULL DEFAULT '', + count INTEGER NOT NULL DEFAULT 0 +); + +-- Initialize CloudSync +SELECT cloudsync_init('items', 'CLS', 0) AS _init_a \gset + +-- ============================================================================ +-- ROUND 1: Initial INSERT (A -> B) +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 1: Initial INSERT (A -> B) === + +INSERT INTO items VALUES ( + '11111111-1111-1111-1111-111111111111', + '{"type":"widget","tags":["a","b"]}'::jsonb, + 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', + '192.168.1.1', + '192.168.0.0/16', + 'Widget One', + 100 +); + +INSERT INTO items VALUES ( + '22222222-2222-2222-2222-222222222222', + '{"type":"gadget"}'::jsonb, + NULL, + '10.0.0.1', + '10.0.0.0/8', + 'Gadget Two', + 200 +); + +INSERT INTO items VALUES ( + '33333333-3333-3333-3333-333333333333', + '{}'::jsonb, + 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', + '127.0.0.1', + NULL, + '', + 0 +); + +-- Compute hash for round 1 +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r1 FROM items \gset + +\echo [INFO] (:testid) Round 1 - Database A hash: :hash_a_r1 + +-- Encode and sync to B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_r1 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Setup Database B +\connect cloudsync_test_19b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE items ( + id UUID PRIMARY KEY, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + related_id UUID, + ip_address INET NOT NULL DEFAULT '0.0.0.0', + network CIDR, + name TEXT NOT NULL DEFAULT '', + count INTEGER NOT NULL DEFAULT 0 +); + +SELECT cloudsync_init('items', 'CLS', 0) AS _init_b \gset + +-- Apply round 1 payload +SELECT cloudsync_payload_apply(decode(:'payload_a_r1', 'hex')) AS apply_r1 \gset + +SELECT (:apply_r1 >= 0) AS r1_applied \gset +\if :r1_applied +\echo [PASS] (:testid) Round 1 payload applied +\else +\echo [FAIL] (:testid) Round 1 payload apply failed: :apply_r1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify hash match +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r1 FROM items \gset + +SELECT (:'hash_a_r1' = :'hash_b_r1') AS r1_match \gset +\if :r1_match +\echo [PASS] (:testid) Round 1 hashes match +\else +\echo [FAIL] (:testid) Round 1 hash mismatch: A=:hash_a_r1 B=:hash_b_r1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 2: UPDATE unmapped non-PK columns (A -> B) +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 2: UPDATE unmapped non-PK columns (A -> B) === + +\connect cloudsync_test_19a +\ir helper_psql_conn_setup.sql + +-- Update JSONB +UPDATE items SET metadata = '{"type":"widget","tags":["a","b","c"],"updated":true}'::jsonb +WHERE id = '11111111-1111-1111-1111-111111111111'; + +-- Update UUID non-PK +UPDATE items SET related_id = 'cccccccc-cccc-cccc-cccc-cccccccccccc' +WHERE id = '22222222-2222-2222-2222-222222222222'; + +-- Update INET +UPDATE items SET ip_address = '172.16.0.1' +WHERE id = '33333333-3333-3333-3333-333333333333'; + +-- Update CIDR +UPDATE items SET network = '172.16.0.0/12' +WHERE id = '33333333-3333-3333-3333-333333333333'; + +-- Compute hash +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r2 FROM items \gset + +\echo [INFO] (:testid) Round 2 - Database A hash: :hash_a_r2 + +-- Sync to B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_r2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19b +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_a_r2', 'hex')) AS apply_r2 \gset + +SELECT (:apply_r2 >= 0) AS r2_applied \gset +\if :r2_applied +\echo [PASS] (:testid) Round 2 payload applied +\else +\echo [FAIL] (:testid) Round 2 payload apply failed: :apply_r2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r2 FROM items \gset + +SELECT (:'hash_a_r2' = :'hash_b_r2') AS r2_match \gset +\if :r2_match +\echo [PASS] (:testid) Round 2 hashes match (unmapped cols updated) +\else +\echo [FAIL] (:testid) Round 2 hash mismatch: A=:hash_a_r2 B=:hash_b_r2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 3: UPDATE mapped columns (B -> A) - Bidirectional +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 3: UPDATE mapped columns (B -> A) === + +-- Update TEXT and INTEGER columns in B +UPDATE items SET name = 'Updated Widget', count = 150 +WHERE id = '11111111-1111-1111-1111-111111111111'; + +UPDATE items SET name = 'Updated Gadget', count = 250 +WHERE id = '22222222-2222-2222-2222-222222222222'; + +-- Compute hash +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r3 FROM items \gset + +\echo [INFO] (:testid) Round 3 - Database B hash: :hash_b_r3 + +-- Sync B -> A +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_r3 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19a +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_b_r3', 'hex')) AS apply_r3 \gset + +SELECT (:apply_r3 >= 0) AS r3_applied \gset +\if :r3_applied +\echo [PASS] (:testid) Round 3 payload applied (B -> A) +\else +\echo [FAIL] (:testid) Round 3 payload apply failed: :apply_r3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r3 FROM items \gset + +SELECT (:'hash_a_r3' = :'hash_b_r3') AS r3_match \gset +\if :r3_match +\echo [PASS] (:testid) Round 3 hashes match (mapped cols updated B->A) +\else +\echo [FAIL] (:testid) Round 3 hash mismatch: A=:hash_a_r3 B=:hash_b_r3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 4: DELETE row (A -> B) +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 4: DELETE row (A -> B) === + +DELETE FROM items WHERE id = '33333333-3333-3333-3333-333333333333'; + +SELECT COUNT(*) AS count_a_r4 FROM items \gset +\echo [INFO] (:testid) Round 4 - Database A row count: :count_a_r4 + +-- Sync deletion +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_r4 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19b +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_a_r4', 'hex')) AS apply_r4 \gset + +SELECT (:apply_r4 >= 0) AS r4_applied \gset +\if :r4_applied +\echo [PASS] (:testid) Round 4 payload applied +\else +\echo [FAIL] (:testid) Round 4 payload apply failed: :apply_r4 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT COUNT(*) AS count_b_r4 FROM items \gset + +SELECT (:count_a_r4 = :count_b_r4) AS r4_count_match \gset +\if :r4_count_match +\echo [PASS] (:testid) Round 4 row counts match after DELETE (:count_b_r4 rows) +\else +\echo [FAIL] (:testid) Round 4 row count mismatch: A=:count_a_r4 B=:count_b_r4 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify deleted row is gone +SELECT COUNT(*) = 0 AS deleted_row_gone +FROM items WHERE id = '33333333-3333-3333-3333-333333333333' \gset +\if :deleted_row_gone +\echo [PASS] (:testid) Deleted row not present in Database B +\else +\echo [FAIL] (:testid) Deleted row still exists in Database B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 5: RESURRECT row (B -> A) +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 5: RESURRECT row (B -> A) === + +-- Re-insert the deleted row with different values +INSERT INTO items VALUES ( + '33333333-3333-3333-3333-333333333333', + '{"resurrected":true}'::jsonb, + 'dddddddd-dddd-dddd-dddd-dddddddddddd', + '8.8.8.8', + '8.8.0.0/16', + 'Resurrected Item', + 999 +); + +SELECT COUNT(*) AS count_b_r5 FROM items \gset +\echo [INFO] (:testid) Round 5 - Database B row count: :count_b_r5 + +-- Compute hash +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r5 FROM items \gset + +\echo [INFO] (:testid) Round 5 - Database B hash: :hash_b_r5 + +-- Sync resurrection B -> A +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_r5 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19a +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_b_r5', 'hex')) AS apply_r5 \gset + +SELECT (:apply_r5 >= 0) AS r5_applied \gset +\if :r5_applied +\echo [PASS] (:testid) Round 5 payload applied (resurrection) +\else +\echo [FAIL] (:testid) Round 5 payload apply failed: :apply_r5 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r5 FROM items \gset + +SELECT (:'hash_a_r5' = :'hash_b_r5') AS r5_match \gset +\if :r5_match +\echo [PASS] (:testid) Round 5 hashes match (row resurrected) +\else +\echo [FAIL] (:testid) Round 5 hash mismatch: A=:hash_a_r5 B=:hash_b_r5 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify resurrected row exists with correct values +SELECT COUNT(*) = 1 AS resurrected_ok +FROM items +WHERE id = '33333333-3333-3333-3333-333333333333' + AND metadata = '{"resurrected":true}'::jsonb + AND name = 'Resurrected Item' + AND count = 999 \gset +\if :resurrected_ok +\echo [PASS] (:testid) Resurrected row verified with correct values +\else +\echo [FAIL] (:testid) Resurrected row has incorrect values +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 6: UPDATE primary key (A -> B) +-- Note: PK update = DELETE old row + INSERT new row in CRDT systems +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 6: UPDATE primary key (A -> B) === + +-- Change the UUID PK of the resurrected row +-- This should result in old PK being deleted and new PK being inserted +UPDATE items SET id = '55555555-5555-5555-5555-555555555555' +WHERE id = '33333333-3333-3333-3333-333333333333'; + +SELECT COUNT(*) AS count_a_r6 FROM items \gset +\echo [INFO] (:testid) Round 6 - Database A row count after PK update: :count_a_r6 + +-- Verify old PK is gone and new PK exists in A +SELECT COUNT(*) = 0 AS old_pk_gone_a +FROM items WHERE id = '33333333-3333-3333-3333-333333333333' \gset + +SELECT COUNT(*) = 1 AS new_pk_exists_a +FROM items WHERE id = '55555555-5555-5555-5555-555555555555' \gset + +-- Compute hash +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r6 FROM items \gset + +\echo [INFO] (:testid) Round 6 - Database A hash: :hash_a_r6 + +-- Sync PK update A -> B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_r6 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19b +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_a_r6', 'hex')) AS apply_r6 \gset + +SELECT (:apply_r6 >= 0) AS r6_applied \gset +\if :r6_applied +\echo [PASS] (:testid) Round 6 payload applied (PK update) +\else +\echo [FAIL] (:testid) Round 6 payload apply failed: :apply_r6 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify old PK is gone in B +SELECT COUNT(*) = 0 AS old_pk_gone_b +FROM items WHERE id = '33333333-3333-3333-3333-333333333333' \gset +\if :old_pk_gone_b +\echo [PASS] (:testid) Old UUID PK removed from Database B +\else +\echo [FAIL] (:testid) Old UUID PK still exists in Database B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify new PK exists in B with correct data +SELECT COUNT(*) = 1 AS new_pk_exists_b +FROM items +WHERE id = '55555555-5555-5555-5555-555555555555' + AND metadata = '{"resurrected":true}'::jsonb + AND name = 'Resurrected Item' + AND count = 999 \gset +\if :new_pk_exists_b +\echo [PASS] (:testid) New UUID PK exists with correct data in Database B +\else +\echo [FAIL] (:testid) New UUID PK missing or has incorrect data in Database B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify hash match +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r6 FROM items \gset + +SELECT (:'hash_a_r6' = :'hash_b_r6') AS r6_match \gset +\if :r6_match +\echo [PASS] (:testid) Round 6 hashes match (PK updated) +\else +\echo [FAIL] (:testid) Round 6 hash mismatch: A=:hash_a_r6 B=:hash_b_r6 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 7: INSERT new row (A -> B) - Final verification +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 7: INSERT new row (A -> B) === + +\connect cloudsync_test_19a +\ir helper_psql_conn_setup.sql + +INSERT INTO items VALUES ( + '44444444-4444-4444-4444-444444444444', + '{"final":"test"}'::jsonb, + 'eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee', + '1.1.1.1', + '1.0.0.0/8', + 'Final Item', + 444 +); + +-- Compute hash for round 7 +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r7 FROM items \gset + +\echo [INFO] (:testid) Round 7 - Database A hash: :hash_a_r7 + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_r7 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19b +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_a_r7', 'hex')) AS apply_r7 \gset + +SELECT (:apply_r7 >= 0) AS r7_applied \gset +\if :r7_applied +\echo [PASS] (:testid) Round 7 payload applied +\else +\echo [FAIL] (:testid) Round 7 payload apply failed: :apply_r7 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r7 FROM items \gset + +SELECT (:'hash_a_r7' = :'hash_b_r7') AS r7_match \gset +\if :r7_match +\echo [PASS] (:testid) Round 7 hashes match - all CRUD operations verified +\else +\echo [FAIL] (:testid) Round 7 hash mismatch: A=:hash_a_r7 B=:hash_b_r7 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Final row count verification +SELECT COUNT(*) AS final_count FROM items \gset +SELECT (:final_count = 4) AS final_count_ok \gset +\if :final_count_ok +\echo [PASS] (:testid) Final row count correct (:final_count rows) +\else +\echo [FAIL] (:testid) Final row count incorrect: expected 4, got :final_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_19a; +DROP DATABASE IF EXISTS cloudsync_test_19b; +\endif diff --git a/test/postgresql/20_init_with_existing_data.sql b/test/postgresql/20_init_with_existing_data.sql new file mode 100644 index 00000000..57e249e0 --- /dev/null +++ b/test/postgresql/20_init_with_existing_data.sql @@ -0,0 +1,298 @@ +-- Init With Existing Data Test +-- Tests cloudsync_init on a table that already contains data. +-- This verifies that cloudsync_refill_metatable correctly creates +-- metadata entries for pre-existing rows. + +\set testid '20' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_20a; +DROP DATABASE IF EXISTS cloudsync_test_20b; +CREATE DATABASE cloudsync_test_20a; +CREATE DATABASE cloudsync_test_20b; + +-- ============================================================================ +-- Setup Database A - INSERT DATA BEFORE cloudsync_init +-- ============================================================================ + +\connect cloudsync_test_20a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with UUID primary key (required for CRDT replication) +CREATE TABLE items ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0, + metadata JSONB +); + +-- ============================================================================ +-- INSERT DATA BEFORE CALLING cloudsync_init +-- This is the key difference from other tests - data exists before sync setup +-- ============================================================================ + +INSERT INTO items VALUES ('11111111-1111-1111-1111-111111111111', 'Pre-existing Item 1', 10.99, 100, '{"pre": true}'); +INSERT INTO items VALUES ('22222222-2222-2222-2222-222222222222', 'Pre-existing Item 2', 20.50, 200, '{"pre": true, "id": 2}'); +INSERT INTO items VALUES ('33333333-3333-3333-3333-333333333333', 'Pre-existing Item 3', 30.00, 300, NULL); +INSERT INTO items VALUES ('44444444-4444-4444-4444-444444444444', 'Pre-existing Item 4', 0.0, 0, '[]'); +INSERT INTO items VALUES ('55555555-5555-5555-5555-555555555555', 'Pre-existing Item 5', -5.50, -10, '{"nested": {"key": "value"}}'); + +-- Verify data exists before init +SELECT COUNT(*) AS pre_init_count FROM items \gset +\echo [INFO] (:testid) Rows before cloudsync_init: :pre_init_count + +-- ============================================================================ +-- NOW call cloudsync_init - this should trigger cloudsync_refill_metatable +-- ============================================================================ + +SELECT cloudsync_init('items', 'CLS', 0) AS _init_a \gset + +-- ============================================================================ +-- Verify metadata was created for existing rows +-- ============================================================================ + +-- Check that metadata table exists and has entries +SELECT COUNT(*) AS metadata_count FROM items_cloudsync \gset + +SELECT (:metadata_count > 0) AS metadata_created \gset +\if :metadata_created +\echo [PASS] (:testid) Metadata table populated after init (:metadata_count entries) +\else +\echo [FAIL] (:testid) Metadata table empty after init - cloudsync_refill_metatable may have failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(metadata::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM items \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A (pre-existing data) +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema (empty) +-- ============================================================================ + +\connect cloudsync_test_20b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE items ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0, + metadata JSONB +); + +-- Initialize CloudSync on empty table +SELECT cloudsync_init('items', 'CLS', 0) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(metadata::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM items \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM items \gset +\connect cloudsync_test_20a +SELECT COUNT(*) AS count_a_orig FROM items \gset + +\connect cloudsync_test_20b +SELECT (:count_b = :count_a_orig) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_b rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a_orig, Database B: :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify specific pre-existing data was synced correctly +-- ============================================================================ + +SELECT COUNT(*) = 1 AS item1_ok +FROM items +WHERE id = '11111111-1111-1111-1111-111111111111' + AND name = 'Pre-existing Item 1' + AND price = 10.99 + AND quantity = 100 \gset +\if :item1_ok +\echo [PASS] (:testid) Pre-existing item 1 synced correctly +\else +\echo [FAIL] (:testid) Pre-existing item 1 not found or incorrect +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify JSONB data +SELECT COUNT(*) = 1 AS jsonb_ok +FROM items +WHERE id = '55555555-5555-5555-5555-555555555555' AND metadata = '{"nested": {"key": "value"}}'::jsonb \gset +\if :jsonb_ok +\echo [PASS] (:testid) JSONB data synced correctly +\else +\echo [FAIL] (:testid) JSONB data not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test: Add new data AFTER init, verify it also syncs +-- ============================================================================ + +\connect cloudsync_test_20a + +-- Add new row after init +INSERT INTO items VALUES ('66666666-6666-6666-6666-666666666666', 'Post-init Item', 66.66, 666, '{"post": true}'); + +-- Encode new changes +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_20b +SELECT cloudsync_payload_apply(decode(:'payload_a2_hex', 'hex')) AS apply_result2 \gset + +SELECT COUNT(*) = 1 AS post_init_ok +FROM items +WHERE id = '66666666-6666-6666-6666-666666666666' AND name = 'Post-init Item' \gset +\if :post_init_ok +\echo [PASS] (:testid) Post-init data syncs correctly +\else +\echo [FAIL] (:testid) Post-init data failed to sync +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +INSERT INTO items VALUES ('77777777-7777-7777-7777-777777777777', 'From B', 77.77, 777, '{"from": "B"}'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_20a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +SELECT COUNT(*) = 1 AS bidirectional_ok +FROM items +WHERE id = '77777777-7777-7777-7777-777777777777' AND name = 'From B' \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Final verification: total row count should be 7 in both databases +-- ============================================================================ + +SELECT COUNT(*) AS final_count_a FROM items \gset +\connect cloudsync_test_20b +SELECT COUNT(*) AS final_count_b FROM items \gset + +SELECT (:final_count_a = 7 AND :final_count_b = 7) AS final_counts_ok \gset +\if :final_counts_ok +\echo [PASS] (:testid) Final row counts correct (7 rows each) +\else +\echo [FAIL] (:testid) Final row counts incorrect - A: :final_count_a, B: :final_count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_20a; +DROP DATABASE IF EXISTS cloudsync_test_20b; +\endif diff --git a/test/postgresql/21_null_value_sync.sql b/test/postgresql/21_null_value_sync.sql new file mode 100644 index 00000000..8b75d18d --- /dev/null +++ b/test/postgresql/21_null_value_sync.sql @@ -0,0 +1,194 @@ +-- Test: NULL Value Sync Parameter Binding +-- This test verifies that syncing NULL values works correctly in all scenarios: +-- 1. Insert with NULL value first, then non-NULL +-- 2. Update existing row to NULL +-- +-- ISSUE: When a NULL value is synced first, PostgreSQL SPI prepares a statement with +-- only the PK parameters. Subsequent non-NULL syncs fail with "there is no parameter $3". +-- +-- The test uses payload_encode/payload_apply to simulate cross-database sync. + +\set testid '21' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_21a; +DROP DATABASE IF EXISTS cloudsync_test_21b; +CREATE DATABASE cloudsync_test_21a; +CREATE DATABASE cloudsync_test_21b; + +-- ============================================================================ +-- Setup Database A - Source database +-- ============================================================================ + +\connect cloudsync_test_21a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create a simple table with a nullable column +CREATE TABLE test_null_sync ( + id TEXT NOT NULL PRIMARY KEY, + value TEXT -- Nullable column +); + +-- Initialize CloudSync +SELECT cloudsync_init('test_null_sync', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Setup Database B - Target database (same schema) +-- ============================================================================ + +\connect cloudsync_test_21b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE test_null_sync ( + id TEXT NOT NULL PRIMARY KEY, + value TEXT +); + +SELECT cloudsync_init('test_null_sync', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- Test 1: Insert NULL value first, then sync to B +-- ============================================================================ + +\connect cloudsync_test_21a + +-- Insert row with NULL value +INSERT INTO test_null_sync (id, value) VALUES ('row1', NULL); + +-- Encode payload from Database A +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_null_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_21b + +-- Apply payload with NULL value +SELECT cloudsync_payload_apply(decode(:'payload_null_hex', 'hex')) AS apply_null_result \gset + +SELECT (:apply_null_result >= 0) AS null_applied \gset +\if :null_applied +\echo [PASS] (:testid) NULL value payload applied successfully +\else +\echo [FAIL] (:testid) NULL value payload failed to apply +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify the NULL value was synced +SELECT COUNT(*) = 1 AS null_row_exists FROM test_null_sync WHERE id = 'row1' AND value IS NULL \gset +\if :null_row_exists +\echo [PASS] (:testid) NULL value synced correctly +\else +\echo [FAIL] (:testid) NULL value not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 2: Insert non-NULL value, then sync to B +-- ============================================================================ + +\connect cloudsync_test_21a + +-- Insert row with non-NULL value +INSERT INTO test_null_sync (id, value) VALUES ('row2', 'hello world'); + +-- Encode payload from Database A (includes new row) +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_nonnull_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_21b + +-- Apply payload with non-NULL value +SELECT cloudsync_payload_apply(decode(:'payload_nonnull_hex', 'hex')) AS apply_nonnull_result \gset + +SELECT (:apply_nonnull_result >= 0) AS nonnull_applied \gset +\if :nonnull_applied +\echo [PASS] (:testid) Non-NULL value payload applied successfully after NULL +\else +\echo [FAIL] (:testid) Non-NULL value payload failed to apply (parameter binding issue) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify the non-NULL value was synced +SELECT COUNT(*) = 1 AS nonnull_row_exists FROM test_null_sync WHERE id = 'row2' AND value = 'hello world' \gset +\if :nonnull_row_exists +\echo [PASS] (:testid) Non-NULL value synced correctly +\else +\echo [FAIL] (:testid) Non-NULL value not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 3: Verify both rows exist in Database B +-- ============================================================================ + +SELECT COUNT(*) AS total_rows FROM test_null_sync \gset +SELECT (:total_rows = 2) AS both_rows_exist \gset +\if :both_rows_exist +\echo [PASS] (:testid) Both rows synced successfully +\else +\echo [FAIL] (:testid) Expected 2 rows, found :total_rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 4: Update existing row to NULL, then sync to B +-- This tests that updating a column from non-NULL to NULL works correctly. +-- ============================================================================ + +\connect cloudsync_test_21a + +-- Update row2 to set value to NULL +UPDATE test_null_sync SET value = NULL WHERE id = 'row2'; + +-- Encode payload from Database A +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_update_null_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_21b + +-- Apply payload with updated NULL value +SELECT cloudsync_payload_apply(decode(:'payload_update_null_hex', 'hex')) AS apply_update_null_result \gset + +SELECT (:apply_update_null_result >= 0) AS update_null_applied \gset +\if :update_null_applied +\echo [PASS] (:testid) Update to NULL payload applied successfully +\else +\echo [FAIL] (:testid) Update to NULL payload failed to apply +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify the update to NULL was synced +SELECT COUNT(*) = 1 AS update_null_synced FROM test_null_sync WHERE id = 'row2' AND value IS NULL \gset +\if :update_null_synced +\echo [PASS] (:testid) Update to NULL synced correctly +\else +\echo [FAIL] (:testid) Update to NULL not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_21a; +DROP DATABASE IF EXISTS cloudsync_test_21b; +\endif diff --git a/test/postgresql/22_null_column_roundtrip.sql b/test/postgresql/22_null_column_roundtrip.sql new file mode 100644 index 00000000..0a601a98 --- /dev/null +++ b/test/postgresql/22_null_column_roundtrip.sql @@ -0,0 +1,347 @@ +-- Test: NULL Column Roundtrip +-- This test verifies that syncing rows with various NULL column combinations works correctly. +-- Tests all permutations: NULL in first column, second column, both, and neither. + +\set testid '22' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_22a; +DROP DATABASE IF EXISTS cloudsync_test_22b; +CREATE DATABASE cloudsync_test_22a; +CREATE DATABASE cloudsync_test_22b; + +-- ============================================================================ +-- Setup Database A - Source database +-- ============================================================================ + +\connect cloudsync_test_22a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with nullable columns (no DEFAULT values) +CREATE TABLE null_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + value INTEGER +); + +-- Initialize CloudSync +SELECT cloudsync_init('null_sync_test', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Insert test data with various NULL combinations +-- ============================================================================ + +-- Row 1: NULL in value column only +INSERT INTO null_sync_test (id, name, value) VALUES ('pg1', 'name1', NULL); + +-- Row 2: NULL in name column only +INSERT INTO null_sync_test (id, name, value) VALUES ('pg2', NULL, 42); + +-- Row 3: NULL in both columns +INSERT INTO null_sync_test (id, name, value) VALUES ('pg3', NULL, NULL); + +-- Row 4: No NULLs (both columns have values) +INSERT INTO null_sync_test (id, name, value) VALUES ('pg4', 'name4', 100); + +-- Row 5: Empty string (not NULL) and zero +INSERT INTO null_sync_test (id, name, value) VALUES ('pg5', '', 0); + +-- Row 6: Another NULL in value +INSERT INTO null_sync_test (id, name, value) VALUES ('pg6', 'name6', NULL); + +-- ============================================================================ +-- Verify source data +-- ============================================================================ + +SELECT COUNT(*) = 6 AS source_row_count_ok FROM null_sync_test \gset +\if :source_row_count_ok +\echo [PASS] (:testid) Source database has 6 rows +\else +\echo [FAIL] (:testid) Source database row count incorrect +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM null_sync_test \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_22b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create identical table schema +CREATE TABLE null_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + value INTEGER +); + +-- Initialize CloudSync +SELECT cloudsync_init('null_sync_test', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B (result: :apply_result) +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +-- Compute hash of Database B data (should match Database A) +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM null_sync_test \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +-- Compare hashes +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM null_sync_test \gset +SELECT (:count_b = 6) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (6 rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Expected 6, got :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify specific NULL patterns +-- ============================================================================ + +-- pg1: name='name1', value=NULL +SELECT (SELECT name = 'name1' AND value IS NULL FROM null_sync_test WHERE id = 'pg1') AS pg1_ok \gset +\if :pg1_ok +\echo [PASS] (:testid) pg1: name='name1', value=NULL preserved +\else +\echo [FAIL] (:testid) pg1: NULL in value column not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg2: name=NULL, value=42 +SELECT (SELECT name IS NULL AND value = 42 FROM null_sync_test WHERE id = 'pg2') AS pg2_ok \gset +\if :pg2_ok +\echo [PASS] (:testid) pg2: name=NULL, value=42 preserved +\else +\echo [FAIL] (:testid) pg2: NULL in name column not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg3: name=NULL, value=NULL +SELECT (SELECT name IS NULL AND value IS NULL FROM null_sync_test WHERE id = 'pg3') AS pg3_ok \gset +\if :pg3_ok +\echo [PASS] (:testid) pg3: name=NULL, value=NULL preserved +\else +\echo [FAIL] (:testid) pg3: Both NULLs not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg4: name='name4', value=100 (no NULLs) +SELECT (SELECT name = 'name4' AND value = 100 FROM null_sync_test WHERE id = 'pg4') AS pg4_ok \gset +\if :pg4_ok +\echo [PASS] (:testid) pg4: name='name4', value=100 preserved +\else +\echo [FAIL] (:testid) pg4: Non-NULL values not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg5: name='', value=0 (empty string and zero, not NULL) +SELECT (SELECT name = '' AND value = 0 FROM null_sync_test WHERE id = 'pg5') AS pg5_ok \gset +\if :pg5_ok +\echo [PASS] (:testid) pg5: empty string and zero preserved (not NULL) +\else +\echo [FAIL] (:testid) pg5: Empty string or zero incorrectly converted +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg6: name='name6', value=NULL +SELECT (SELECT name = 'name6' AND value IS NULL FROM null_sync_test WHERE id = 'pg6') AS pg6_ok \gset +\if :pg6_ok +\echo [PASS] (:testid) pg6: name='name6', value=NULL preserved +\else +\echo [FAIL] (:testid) pg6: NULL in value column not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +-- Add a new row in Database B with NULLs +INSERT INTO null_sync_test (id, name, value) VALUES ('pgB1', NULL, 999); + +-- Encode payload from Database B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply to Database A +\connect cloudsync_test_22a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +-- Verify the new row exists in Database A with correct NULL +SELECT (SELECT name IS NULL AND value = 999 FROM null_sync_test WHERE id = 'pgB1') AS bidirectional_ok \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A with NULL) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test UPDATE to NULL +-- ============================================================================ + +-- Update pg4 to set name to NULL +UPDATE null_sync_test SET name = NULL WHERE id = 'pg4'; + +-- Encode and sync to B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_update_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_22b +SELECT cloudsync_payload_apply(decode(:'payload_update_hex', 'hex')) AS apply_update \gset + +-- Verify pg4 now has NULL name +SELECT (SELECT name IS NULL AND value = 100 FROM null_sync_test WHERE id = 'pg4') AS update_to_null_ok \gset +\if :update_to_null_ok +\echo [PASS] (:testid) UPDATE to NULL synced correctly +\else +\echo [FAIL] (:testid) UPDATE to NULL failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test UPDATE from NULL to value +-- ============================================================================ + +-- Update pg3 to set both columns to non-NULL values +\connect cloudsync_test_22a +UPDATE null_sync_test SET name = 'updated', value = 123 WHERE id = 'pg3'; + +-- Encode and sync to B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_update2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_22b +SELECT cloudsync_payload_apply(decode(:'payload_update2_hex', 'hex')) AS apply_update2 \gset + +-- Verify pg3 now has non-NULL values +SELECT (SELECT name = 'updated' AND value = 123 FROM null_sync_test WHERE id = 'pg3') AS update_from_null_ok \gset +\if :update_from_null_ok +\echo [PASS] (:testid) UPDATE from NULL to value synced correctly +\else +\echo [FAIL] (:testid) UPDATE from NULL to value failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Final verification - both databases should have 7 rows with matching content +-- ============================================================================ + +SELECT COUNT(*) AS final_count_b FROM null_sync_test \gset +\connect cloudsync_test_22a +SELECT COUNT(*) AS final_count_a FROM null_sync_test \gset + +\connect cloudsync_test_22b +SELECT (:final_count_a = 7 AND :final_count_b = 7) AS final_counts_ok \gset +\if :final_counts_ok +\echo [PASS] (:testid) Final row counts correct (7 rows each) +\else +\echo [FAIL] (:testid) Final row counts incorrect - A: :final_count_a, B: :final_count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_22a; +DROP DATABASE IF EXISTS cloudsync_test_22b; +\endif diff --git a/test/postgresql/23_uuid_column_roundtrip.sql b/test/postgresql/23_uuid_column_roundtrip.sql new file mode 100644 index 00000000..38adbdc8 --- /dev/null +++ b/test/postgresql/23_uuid_column_roundtrip.sql @@ -0,0 +1,359 @@ +-- Test: UUID Column Roundtrip +-- This test verifies that syncing rows with UUID columns (not as PK) works correctly. +-- Tests various combinations of NULL and non-NULL UUID values alongside other nullable columns. +-- +-- IMPORTANT: This test is structured to isolate whether NULL UUID values trigger encoding bugs: +-- Step 1: Sync a single row with non-NULL UUID only (baseline) +-- Step 2: Sync a row with NULL UUID, then a row with non-NULL UUID (test NULL trigger) +-- Step 3: Sync remaining rows with mixed NULL/non-NULL UUIDs + +\set testid '23' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_23a; +DROP DATABASE IF EXISTS cloudsync_test_23b; +CREATE DATABASE cloudsync_test_23a; +CREATE DATABASE cloudsync_test_23b; + +-- ============================================================================ +-- Setup Database A - Source database +-- ============================================================================ + +\connect cloudsync_test_23a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with UUID column and other nullable columns +CREATE TABLE uuid_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + value INTEGER, + id2 UUID +); + +-- Initialize CloudSync +SELECT cloudsync_init('uuid_sync_test', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Setup Database B with same schema (before any inserts) +-- ============================================================================ + +\connect cloudsync_test_23b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE uuid_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + value INTEGER, + id2 UUID +); + +SELECT cloudsync_init('uuid_sync_test', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- STEP 1: Sync a single row with non-NULL UUID only (baseline test) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 1: Single row with non-NULL UUID === + +\connect cloudsync_test_23a + +-- Insert only one row with a non-NULL UUID +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('step1', 'baseline', 1, 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'); + +-- Encode payload +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step1_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT max(db_version) AS db_version FROM uuid_sync_test_cloudsync \gset + +-- Apply to Database B +\connect cloudsync_test_23b +SELECT cloudsync_payload_apply(decode(:'payload_step1_hex', 'hex')) AS apply_step1 \gset + +-- Verify step 1 +SELECT (SELECT id2 = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' FROM uuid_sync_test WHERE id = 'step1') AS step1_ok \gset +\if :step1_ok +\echo [PASS] (:testid) Step 1: Single non-NULL UUID preserved correctly +\else +\echo [FAIL] (:testid) Step 1: Single non-NULL UUID NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT id, name, value, id2::text FROM uuid_sync_test WHERE id = 'step1'; +\endif + +-- ============================================================================ +-- STEP 2: Sync NULL UUID row, then non-NULL UUID row (test NULL trigger) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 2: NULL UUID followed by non-NULL UUID === + +\connect cloudsync_test_23a + +-- Insert a row with NULL UUID first +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('step2a', 'null_uuid', 2, NULL); + +-- Then insert a row with non-NULL UUID +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('step2b', 'after_null', 3, 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb'); + +-- Encode payload (should contain both rows) +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM uuid_sync_test_cloudsync \gset + +-- Apply to Database B +\connect cloudsync_test_23b +SELECT cloudsync_payload_apply(decode(:'payload_step2_hex', 'hex')) AS apply_step2 \gset + +-- Verify step 2a (NULL UUID) +SELECT (SELECT id2 IS NULL FROM uuid_sync_test WHERE id = 'step2a') AS step2a_ok \gset +\if :step2a_ok +\echo [PASS] (:testid) Step 2a: NULL UUID preserved correctly +\else +\echo [FAIL] (:testid) Step 2a: NULL UUID NOT preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify step 2b (non-NULL UUID after NULL) +SELECT (SELECT id2 = 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb' FROM uuid_sync_test WHERE id = 'step2b') AS step2b_ok \gset +\if :step2b_ok +\echo [PASS] (:testid) Step 2b: Non-NULL UUID after NULL preserved correctly +\else +\echo [FAIL] (:testid) Step 2b: Non-NULL UUID after NULL NOT preserved (NULL may have triggered bug) +SELECT (:fail::int + 1) AS fail \gset +SELECT id, name, value, id2::text FROM uuid_sync_test WHERE id = 'step2b'; +\endif + +-- ============================================================================ +-- STEP 3: Sync remaining rows with mixed NULL/non-NULL UUIDs +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 3: Mixed NULL/non-NULL UUIDs === + +\connect cloudsync_test_23a + +-- Row with NULL in value and id2 +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg1', 'name1', NULL, NULL); + +-- Row with NULL in name, has UUID +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg2', NULL, 42, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'); + +-- Row with all nullable columns NULL +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg3', NULL, NULL, NULL); + +-- Row with no NULLs - all columns have values +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg4', 'name4', 100, 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22'); + +-- Row with only id2 NULL +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg5', 'name5', 55, NULL); + +-- Row with only name NULL, has different UUID +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg6', NULL, 66, 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a33'); + +-- ============================================================================ +-- Verify source data +-- ============================================================================ + +SELECT COUNT(*) = 9 AS source_row_count_ok FROM uuid_sync_test \gset +\if :source_row_count_ok +\echo [PASS] (:testid) Source database has 9 rows +\else +\echo [FAIL] (:testid) Source database row count incorrect +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL') || ':' || + COALESCE(id2::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM uuid_sync_test \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A (step 3 rows only) +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step3_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM uuid_sync_test_cloudsync \gset + +-- Verify payload was created +SELECT (length(:'payload_step3_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +\connect cloudsync_test_23b + +SELECT cloudsync_payload_apply(decode(:'payload_step3_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B (result: :apply_result) +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +-- Compute hash of Database B data (should match Database A) +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL') || ':' || + COALESCE(id2::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM uuid_sync_test \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +-- Compare hashes +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM uuid_sync_test \gset +SELECT (:count_b = 9) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (9 rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Expected 9, got :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify specific UUID and NULL patterns +-- ============================================================================ + +-- pg1: name='name1', value=NULL, id2=NULL +SELECT (SELECT name = 'name1' AND value IS NULL AND id2 IS NULL FROM uuid_sync_test WHERE id = 'pg1') AS pg1_ok \gset +\if :pg1_ok +\echo [PASS] (:testid) pg1: name='name1', value=NULL, id2=NULL preserved +\else +\echo [FAIL] (:testid) pg1: NULL values not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg2: name=NULL, value=42, id2='a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11' +SELECT (SELECT name IS NULL AND value = 42 AND id2 = 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11' FROM uuid_sync_test WHERE id = 'pg2') AS pg2_ok \gset +\if :pg2_ok +\echo [PASS] (:testid) pg2: name=NULL, value=42, UUID preserved +\else +\echo [FAIL] (:testid) pg2: UUID or NULL not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg3: all nullable columns NULL +SELECT (SELECT name IS NULL AND value IS NULL AND id2 IS NULL FROM uuid_sync_test WHERE id = 'pg3') AS pg3_ok \gset +\if :pg3_ok +\echo [PASS] (:testid) pg3: all NULLs preserved +\else +\echo [FAIL] (:testid) pg3: all NULLs not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg4: no NULLs, id2='b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22' +SELECT (SELECT name = 'name4' AND value = 100 AND id2 = 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22' FROM uuid_sync_test WHERE id = 'pg4') AS pg4_ok \gset +\if :pg4_ok +\echo [PASS] (:testid) pg4: all values including UUID preserved +\else +\echo [FAIL] (:testid) pg4: values not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg5: name='name5', value=55, id2=NULL +SELECT (SELECT name = 'name5' AND value = 55 AND id2 IS NULL FROM uuid_sync_test WHERE id = 'pg5') AS pg5_ok \gset +\if :pg5_ok +\echo [PASS] (:testid) pg5: values with NULL UUID preserved +\else +\echo [FAIL] (:testid) pg5: NULL UUID not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg6: name=NULL, value=66, id2='c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a33' +SELECT (SELECT name IS NULL AND value = 66 AND id2 = 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a33' FROM uuid_sync_test WHERE id = 'pg6') AS pg6_ok \gset +\if :pg6_ok +\echo [PASS] (:testid) pg6: NULL name with UUID preserved +\else +\echo [FAIL] (:testid) pg6: UUID c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a33 not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Show actual data for debugging if there are failures +-- ============================================================================ + +\if :{?DEBUG} +\echo [INFO] (:testid) Database A data: +\connect cloudsync_test_23a +SELECT id, name, value, id2::text FROM uuid_sync_test ORDER BY id; + +\echo [INFO] (:testid) Database B data: +\connect cloudsync_test_23b +SELECT id, name, value, id2::text FROM uuid_sync_test ORDER BY id; +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_23a; +DROP DATABASE IF EXISTS cloudsync_test_23b; +\endif diff --git a/test/postgresql/24_nullable_types_roundtrip.sql b/test/postgresql/24_nullable_types_roundtrip.sql new file mode 100644 index 00000000..2ac4082c --- /dev/null +++ b/test/postgresql/24_nullable_types_roundtrip.sql @@ -0,0 +1,495 @@ +-- Test: Nullable Types Roundtrip +-- This test verifies that syncing rows with various nullable column types works correctly. +-- Tests the type mapping for NULL values: INT2/4/8 → INT8, FLOAT4/8/NUMERIC → FLOAT8, BYTEA → BYTEA, others → TEXT +-- +-- IMPORTANT: This test inserts NULL values FIRST to trigger SPI plan caching with decoded types, +-- then inserts non-NULL values to verify the type mapping is consistent. + +\set testid '24' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_24a; +DROP DATABASE IF EXISTS cloudsync_test_24b; +CREATE DATABASE cloudsync_test_24a; +CREATE DATABASE cloudsync_test_24b; + +-- ============================================================================ +-- Setup Database A - Source database +-- ============================================================================ + +\connect cloudsync_test_24a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with various nullable column types +-- NOTE: BOOLEAN is excluded because it encodes as INTEGER but PostgreSQL can't cast INT8 to BOOLEAN. +-- This is a known limitation that requires changes to the encoding layer. +CREATE TABLE types_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + -- Integer types (all map to INT8OID in decoding) + col_int2 SMALLINT, + col_int4 INTEGER, + col_int8 BIGINT, + -- Float types (all map to FLOAT8OID in decoding) + col_float4 REAL, + col_float8 DOUBLE PRECISION, + col_numeric NUMERIC(10,2), + -- Binary type (maps to BYTEAOID in decoding) + col_bytea BYTEA, + -- Text types (all map to TEXTOID in decoding) + col_text TEXT, + col_varchar VARCHAR(100), + col_char CHAR(10), + -- Other types that map to TEXTOID + col_uuid UUID, + col_json JSON, + col_jsonb JSONB, + col_date DATE, + col_timestamp TIMESTAMP +); + +-- Initialize CloudSync +SELECT cloudsync_init('types_sync_test', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Setup Database B with same schema (before any inserts) +-- ============================================================================ + +\connect cloudsync_test_24b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE types_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + col_int2 SMALLINT, + col_int4 INTEGER, + col_int8 BIGINT, + col_float4 REAL, + col_float8 DOUBLE PRECISION, + col_numeric NUMERIC(10,2), + col_bytea BYTEA, + col_text TEXT, + col_varchar VARCHAR(100), + col_char CHAR(10), + col_uuid UUID, + col_json JSON, + col_jsonb JSONB, + col_date DATE, + col_timestamp TIMESTAMP +); + +SELECT cloudsync_init('types_sync_test', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- STEP 1: Insert row with ALL NULL values first (triggers SPI plan caching) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 1: Insert row with ALL NULL values === + +\connect cloudsync_test_24a + +INSERT INTO types_sync_test ( + id, col_int2, col_int4, col_int8, col_float4, col_float8, col_numeric, + col_bytea, col_text, col_varchar, col_char, col_uuid, col_json, col_jsonb, + col_date, col_timestamp +) VALUES ( + 'null_row', NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL +); + +-- Encode payload +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step1_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT max(db_version) AS db_version FROM types_sync_test_cloudsync \gset + +-- Apply to Database B +\connect cloudsync_test_24b +SELECT cloudsync_payload_apply(decode(:'payload_step1_hex', 'hex')) AS apply_step1 \gset + +-- Verify step 1: all values should be NULL +SELECT (SELECT + col_int2 IS NULL AND col_int4 IS NULL AND col_int8 IS NULL AND + col_float4 IS NULL AND col_float8 IS NULL AND col_numeric IS NULL AND + col_bytea IS NULL AND col_text IS NULL AND col_varchar IS NULL AND + col_char IS NULL AND col_uuid IS NULL AND col_json IS NULL AND + col_jsonb IS NULL AND col_date IS NULL AND col_timestamp IS NULL +FROM types_sync_test WHERE id = 'null_row') AS step1_ok \gset + +\if :step1_ok +\echo [PASS] (:testid) Step 1: All NULL values preserved correctly +\else +\echo [FAIL] (:testid) Step 1: NULL values NOT preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 2: Insert row with ALL non-NULL values (tests type consistency) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 2: Insert row with ALL non-NULL values === + +\connect cloudsync_test_24a + +INSERT INTO types_sync_test ( + id, col_int2, col_int4, col_int8, col_float4, col_float8, col_numeric, + col_bytea, col_text, col_varchar, col_char, col_uuid, col_json, col_jsonb, + col_date, col_timestamp +) VALUES ( + 'full_row', + 32767, -- INT2 max + 2147483647, -- INT4 max + 9223372036854775807, -- INT8 max + 3.14159, -- FLOAT4 + 3.141592653589793, -- FLOAT8 + 12345.67, -- NUMERIC + '\xDEADBEEF', -- BYTEA + 'Hello, World!', -- TEXT + 'varchar_val', -- VARCHAR + 'char_val', -- CHAR (will be padded) + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', -- UUID + '{"key": "value"}', -- JSON + '{"nested": {"array": [1, 2, 3]}}', -- JSONB + '2024-01-15', -- DATE + '2024-01-15 10:30:00' -- TIMESTAMP +); + +-- Encode payload (only new rows) +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM types_sync_test_cloudsync \gset + +-- Apply to Database B +\connect cloudsync_test_24b +SELECT cloudsync_payload_apply(decode(:'payload_step2_hex', 'hex')) AS apply_step2 \gset + +-- Verify step 2: Integer types +SELECT (SELECT col_int2 = 32767 FROM types_sync_test WHERE id = 'full_row') AS int2_ok \gset +\if :int2_ok +\echo [PASS] (:testid) INT2 (SMALLINT) value preserved: 32767 +\else +\echo [FAIL] (:testid) INT2 (SMALLINT) value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_int2 FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_int4 = 2147483647 FROM types_sync_test WHERE id = 'full_row') AS int4_ok \gset +\if :int4_ok +\echo [PASS] (:testid) INT4 (INTEGER) value preserved: 2147483647 +\else +\echo [FAIL] (:testid) INT4 (INTEGER) value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_int4 FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_int8 = 9223372036854775807 FROM types_sync_test WHERE id = 'full_row') AS int8_ok \gset +\if :int8_ok +\echo [PASS] (:testid) INT8 (BIGINT) value preserved: 9223372036854775807 +\else +\echo [FAIL] (:testid) INT8 (BIGINT) value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_int8 FROM types_sync_test WHERE id = 'full_row'; +\endif + +-- Verify step 2: Float types (use approximate comparison for floats) +SELECT (SELECT abs(col_float4 - 3.14159) < 0.0001 FROM types_sync_test WHERE id = 'full_row') AS float4_ok \gset +\if :float4_ok +\echo [PASS] (:testid) FLOAT4 (REAL) value preserved: ~3.14159 +\else +\echo [FAIL] (:testid) FLOAT4 (REAL) value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_float4 FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT abs(col_float8 - 3.141592653589793) < 0.000000000001 FROM types_sync_test WHERE id = 'full_row') AS float8_ok \gset +\if :float8_ok +\echo [PASS] (:testid) FLOAT8 (DOUBLE PRECISION) value preserved: ~3.141592653589793 +\else +\echo [FAIL] (:testid) FLOAT8 (DOUBLE PRECISION) value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_float8 FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_numeric = 12345.67 FROM types_sync_test WHERE id = 'full_row') AS numeric_ok \gset +\if :numeric_ok +\echo [PASS] (:testid) NUMERIC value preserved: 12345.67 +\else +\echo [FAIL] (:testid) NUMERIC value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_numeric FROM types_sync_test WHERE id = 'full_row'; +\endif + +-- Verify step 2: BYTEA type +SELECT (SELECT col_bytea = '\xDEADBEEF' FROM types_sync_test WHERE id = 'full_row') AS bytea_ok \gset +\if :bytea_ok +\echo [PASS] (:testid) BYTEA value preserved: DEADBEEF +\else +\echo [FAIL] (:testid) BYTEA value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT encode(col_bytea, 'hex') FROM types_sync_test WHERE id = 'full_row'; +\endif + +-- Verify step 2: Text types +SELECT (SELECT col_text = 'Hello, World!' FROM types_sync_test WHERE id = 'full_row') AS text_ok \gset +\if :text_ok +\echo [PASS] (:testid) TEXT value preserved: Hello, World! +\else +\echo [FAIL] (:testid) TEXT value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_text FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_varchar = 'varchar_val' FROM types_sync_test WHERE id = 'full_row') AS varchar_ok \gset +\if :varchar_ok +\echo [PASS] (:testid) VARCHAR value preserved: varchar_val +\else +\echo [FAIL] (:testid) VARCHAR value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_varchar FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT trim(col_char) = 'char_val' FROM types_sync_test WHERE id = 'full_row') AS char_ok \gset +\if :char_ok +\echo [PASS] (:testid) CHAR value preserved: char_val +\else +\echo [FAIL] (:testid) CHAR value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_char FROM types_sync_test WHERE id = 'full_row'; +\endif + +-- Verify step 2: Other types mapped to TEXT +SELECT (SELECT col_uuid = 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11' FROM types_sync_test WHERE id = 'full_row') AS uuid_ok \gset +\if :uuid_ok +\echo [PASS] (:testid) UUID value preserved: a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +\else +\echo [FAIL] (:testid) UUID value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_uuid FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_json::text = '{"key": "value"}' FROM types_sync_test WHERE id = 'full_row') AS json_ok \gset +\if :json_ok +\echo [PASS] (:testid) JSON value preserved: {"key": "value"} +\else +\echo [FAIL] (:testid) JSON value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_json FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_jsonb @> '{"nested": {"array": [1, 2, 3]}}' FROM types_sync_test WHERE id = 'full_row') AS jsonb_ok \gset +\if :jsonb_ok +\echo [PASS] (:testid) JSONB value preserved: {"nested": {"array": [1, 2, 3]}} +\else +\echo [FAIL] (:testid) JSONB value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_jsonb FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_date = '2024-01-15' FROM types_sync_test WHERE id = 'full_row') AS date_ok \gset +\if :date_ok +\echo [PASS] (:testid) DATE value preserved: 2024-01-15 +\else +\echo [FAIL] (:testid) DATE value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_date FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_timestamp = '2024-01-15 10:30:00' FROM types_sync_test WHERE id = 'full_row') AS timestamp_ok \gset +\if :timestamp_ok +\echo [PASS] (:testid) TIMESTAMP value preserved: 2024-01-15 10:30:00 +\else +\echo [FAIL] (:testid) TIMESTAMP value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_timestamp FROM types_sync_test WHERE id = 'full_row'; +\endif + +-- ============================================================================ +-- STEP 3: Insert row with mixed NULL/non-NULL values +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 3: Insert row with mixed NULL/non-NULL values === + +\connect cloudsync_test_24a + +INSERT INTO types_sync_test ( + id, col_int2, col_int4, col_int8, col_float4, col_float8, col_numeric, + col_bytea, col_text, col_varchar, col_char, col_uuid, col_json, col_jsonb, + col_date, col_timestamp +) VALUES ( + 'mixed_row', + NULL, -- INT2 NULL + 42, -- INT4 non-NULL + NULL, -- INT8 NULL + NULL, -- FLOAT4 NULL + 2.718281828, -- FLOAT8 non-NULL (e) + NULL, -- NUMERIC NULL + '\xCAFEBABE', -- BYTEA non-NULL + NULL, -- TEXT NULL + 'mixed', -- VARCHAR non-NULL + NULL, -- CHAR NULL + 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22', -- UUID non-NULL + NULL, -- JSON NULL + '{"mixed": true}', -- JSONB non-NULL + NULL, -- DATE NULL + '2024-06-15 14:00:00' -- TIMESTAMP non-NULL +); + +-- Encode payload (only new rows) +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step3_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM types_sync_test_cloudsync \gset + +-- Apply to Database B +\connect cloudsync_test_24b +SELECT cloudsync_payload_apply(decode(:'payload_step3_hex', 'hex')) AS apply_step3 \gset + +-- Verify mixed row +SELECT (SELECT + col_int2 IS NULL AND + col_int4 = 42 AND + col_int8 IS NULL AND + col_float4 IS NULL AND + abs(col_float8 - 2.718281828) < 0.000001 AND + col_numeric IS NULL AND + col_bytea = '\xCAFEBABE' AND + col_text IS NULL AND + col_varchar = 'mixed' AND + col_char IS NULL AND + col_uuid = 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22' AND + col_json IS NULL AND + col_jsonb @> '{"mixed": true}' AND + col_date IS NULL AND + col_timestamp = '2024-06-15 14:00:00' +FROM types_sync_test WHERE id = 'mixed_row') AS mixed_ok \gset + +\if :mixed_ok +\echo [PASS] (:testid) Mixed NULL/non-NULL row preserved correctly +\else +\echo [FAIL] (:testid) Mixed NULL/non-NULL row NOT preserved correctly +SELECT (:fail::int + 1) AS fail \gset +SELECT * FROM types_sync_test WHERE id = 'mixed_row'; +\endif + +-- ============================================================================ +-- STEP 4: Verify data integrity with hash comparison +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 4: Verify data integrity === + +\connect cloudsync_test_24a + +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(col_int2::text, 'NULL') || ':' || + COALESCE(col_int4::text, 'NULL') || ':' || + COALESCE(col_int8::text, 'NULL') || ':' || + COALESCE(col_float8::text, 'NULL') || ':' || + COALESCE(col_numeric::text, 'NULL') || ':' || + COALESCE(encode(col_bytea, 'hex'), 'NULL') || ':' || + COALESCE(col_text, 'NULL') || ':' || + COALESCE(col_varchar, 'NULL') || ':' || + COALESCE(col_uuid::text, 'NULL') || ':' || + COALESCE(col_jsonb::text, 'NULL') || ':' || + COALESCE(col_date::text, 'NULL') || ':' || + COALESCE(col_timestamp::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM types_sync_test \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +\connect cloudsync_test_24b + +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(col_int2::text, 'NULL') || ':' || + COALESCE(col_int4::text, 'NULL') || ':' || + COALESCE(col_int8::text, 'NULL') || ':' || + COALESCE(col_float8::text, 'NULL') || ':' || + COALESCE(col_numeric::text, 'NULL') || ':' || + COALESCE(encode(col_bytea, 'hex'), 'NULL') || ':' || + COALESCE(col_text, 'NULL') || ':' || + COALESCE(col_varchar, 'NULL') || ':' || + COALESCE(col_uuid::text, 'NULL') || ':' || + COALESCE(col_jsonb::text, 'NULL') || ':' || + COALESCE(col_date::text, 'NULL') || ':' || + COALESCE(col_timestamp::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM types_sync_test \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - hashes do not match +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row count +SELECT COUNT(*) AS count_b FROM types_sync_test \gset +SELECT (:count_b = 3) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (3 rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Expected 3, got :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Show actual data for debugging if there are failures +-- ============================================================================ + +\if :{?DEBUG} +\echo [INFO] (:testid) Database A data: +\connect cloudsync_test_24a +SELECT id, col_int2, col_int4, col_int8, col_float4, col_float8, col_numeric FROM types_sync_test ORDER BY id; +SELECT id, encode(col_bytea, 'hex') as col_bytea, col_text, col_varchar, trim(col_char) as col_char FROM types_sync_test ORDER BY id; +SELECT id, col_uuid, col_json, col_jsonb, col_date, col_timestamp FROM types_sync_test ORDER BY id; + +\echo [INFO] (:testid) Database B data: +\connect cloudsync_test_24b +SELECT id, col_int2, col_int4, col_int8, col_float4, col_float8, col_numeric FROM types_sync_test ORDER BY id; +SELECT id, encode(col_bytea, 'hex') as col_bytea, col_text, col_varchar, trim(col_char) as col_char FROM types_sync_test ORDER BY id; +SELECT id, col_uuid, col_json, col_jsonb, col_date, col_timestamp FROM types_sync_test ORDER BY id; +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_24a; +DROP DATABASE IF EXISTS cloudsync_test_24b; +\endif diff --git a/test/postgresql/25_boolean_type_issue.sql b/test/postgresql/25_boolean_type_issue.sql new file mode 100644 index 00000000..b9fb6bb1 --- /dev/null +++ b/test/postgresql/25_boolean_type_issue.sql @@ -0,0 +1,241 @@ +-- Test: BOOLEAN Type Roundtrip +-- This test verifies that BOOLEAN columns sync correctly. +-- BOOLEAN values are encoded as INT8 in sync payloads. The cloudsync extension +-- provides a custom cast (bigint AS boolean) to enable this. +-- +-- See plans/ANALYSIS_BOOLEAN_TYPE_CONVERSION.md for details. + +\set testid '25' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_test_25a; +DROP DATABASE IF EXISTS cloudsync_test_25b; +CREATE DATABASE cloudsync_test_25a; +CREATE DATABASE cloudsync_test_25b; + +-- Setup Database A +\connect cloudsync_test_25a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE bool_test ( + id TEXT PRIMARY KEY NOT NULL, + flag BOOLEAN, + name TEXT +); + +SELECT cloudsync_init('bool_test', 'CLS', 1) AS _init_a \gset + +-- Setup Database B +\connect cloudsync_test_25b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE bool_test ( + id TEXT PRIMARY KEY NOT NULL, + flag BOOLEAN, + name TEXT +); + +SELECT cloudsync_init('bool_test', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- STEP 1: Insert NULL BOOLEAN first (triggers SPI plan caching) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 1: NULL BOOLEAN === + +\connect cloudsync_test_25a +INSERT INTO bool_test (id, flag, name) VALUES ('row1', NULL, 'null_flag'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload1_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT max(db_version) AS db_version FROM bool_test_cloudsync \gset + +\connect cloudsync_test_25b +SELECT cloudsync_payload_apply(decode(:'payload1_hex', 'hex')) AS apply1 \gset + +SELECT (SELECT flag IS NULL AND name = 'null_flag' FROM bool_test WHERE id = 'row1') AS step1_ok \gset +\if :step1_ok +\echo [PASS] (:testid) Step 1: NULL BOOLEAN preserved +\else +\echo [FAIL] (:testid) Step 1: NULL BOOLEAN not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 2: Insert TRUE BOOLEAN (tests INT8 -> BOOLEAN cast after NULL) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 2: TRUE BOOLEAN after NULL === + +\connect cloudsync_test_25a +INSERT INTO bool_test (id, flag, name) VALUES ('row2', true, 'true_flag'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM bool_test_cloudsync \gset + +\connect cloudsync_test_25b +SELECT cloudsync_payload_apply(decode(:'payload2_hex', 'hex')) AS apply2 \gset + +SELECT (SELECT flag = true AND name = 'true_flag' FROM bool_test WHERE id = 'row2') AS step2_ok \gset +\if :step2_ok +\echo [PASS] (:testid) Step 2: TRUE BOOLEAN preserved after NULL +\else +\echo [FAIL] (:testid) Step 2: TRUE BOOLEAN not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 3: Insert FALSE BOOLEAN +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 3: FALSE BOOLEAN === + +\connect cloudsync_test_25a +INSERT INTO bool_test (id, flag, name) VALUES ('row3', false, 'false_flag'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload3_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM bool_test_cloudsync \gset + +\connect cloudsync_test_25b +SELECT cloudsync_payload_apply(decode(:'payload3_hex', 'hex')) AS apply3 \gset + +SELECT (SELECT flag = false AND name = 'false_flag' FROM bool_test WHERE id = 'row3') AS step3_ok \gset +\if :step3_ok +\echo [PASS] (:testid) Step 3: FALSE BOOLEAN preserved +\else +\echo [FAIL] (:testid) Step 3: FALSE BOOLEAN not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 4: Update TRUE to FALSE +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 4: Update TRUE to FALSE === + +\connect cloudsync_test_25a +UPDATE bool_test SET flag = false WHERE id = 'row2'; + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload4_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM bool_test_cloudsync \gset + +\connect cloudsync_test_25b +SELECT cloudsync_payload_apply(decode(:'payload4_hex', 'hex')) AS apply4 \gset + +SELECT (SELECT flag = false FROM bool_test WHERE id = 'row2') AS step4_ok \gset +\if :step4_ok +\echo [PASS] (:testid) Step 4: Update TRUE to FALSE synced +\else +\echo [FAIL] (:testid) Step 4: Update TRUE to FALSE not synced +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 5: Update NULL to TRUE +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 5: Update NULL to TRUE === + +\connect cloudsync_test_25a +UPDATE bool_test SET flag = true WHERE id = 'row1'; + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload5_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM bool_test_cloudsync \gset + +\connect cloudsync_test_25b +SELECT cloudsync_payload_apply(decode(:'payload5_hex', 'hex')) AS apply5 \gset + +SELECT (SELECT flag = true FROM bool_test WHERE id = 'row1') AS step5_ok \gset +\if :step5_ok +\echo [PASS] (:testid) Step 5: Update NULL to TRUE synced +\else +\echo [FAIL] (:testid) Step 5: Update NULL to TRUE not synced +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 6: Verify final state with hash comparison +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 6: Verify data integrity === + +\connect cloudsync_test_25a +SELECT md5( + COALESCE( + string_agg( + id || ':' || COALESCE(flag::text, 'NULL') || ':' || COALESCE(name, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM bool_test \gset + +\connect cloudsync_test_25b +SELECT md5( + COALESCE( + string_agg( + id || ':' || COALESCE(flag::text, 'NULL') || ':' || COALESCE(name, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM bool_test \gset + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT COUNT(*) AS count_b FROM bool_test \gset +SELECT (:count_b = 3) AS count_ok \gset +\if :count_ok +\echo [PASS] (:testid) Row count correct (3 rows) +\else +\echo [FAIL] (:testid) Row count incorrect - expected 3, got :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_25a; +DROP DATABASE IF EXISTS cloudsync_test_25b; +\endif diff --git a/test/postgresql/26_row_filter.sql b/test/postgresql/26_row_filter.sql new file mode 100644 index 00000000..01c9dde5 --- /dev/null +++ b/test/postgresql/26_row_filter.sql @@ -0,0 +1,105 @@ +-- 'Row-level filter (conditional sync) test' + +\set testid '26' +\ir helper_test_init.sql + +-- Create first database +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_26_a; +CREATE DATABASE cloudsync_test_26_a; + +\connect cloudsync_test_26_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table, init, set filter +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('tasks') AS _init_site_id_a \gset +SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _set_filter_ok \gset + +-- Insert matching rows (user_id = 1) and non-matching rows (user_id = 2) +INSERT INTO tasks VALUES ('a', 'Task A', 1); +INSERT INTO tasks VALUES ('b', 'Task B', 2); +INSERT INTO tasks VALUES ('c', 'Task C', 1); + +-- Test 1: Verify only matching rows are tracked in _cloudsync metadata +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 2) AS filter_insert_ok \gset +\if :filter_insert_ok +\echo [PASS] (:testid) Only matching rows tracked after INSERT (2 of 3) +\else +\echo [FAIL] (:testid) Expected 2 tracked PKs after INSERT, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: Update non-matching row → no metadata change +SELECT COUNT(*) AS meta_before FROM tasks_cloudsync \gset +UPDATE tasks SET title = 'Task B Updated' WHERE id = 'b'; +SELECT COUNT(*) AS meta_after FROM tasks_cloudsync \gset +SELECT (:meta_before = :meta_after) AS filter_update_nonmatch_ok \gset +\if :filter_update_nonmatch_ok +\echo [PASS] (:testid) Non-matching UPDATE did not change metadata +\else +\echo [FAIL] (:testid) Non-matching UPDATE changed metadata (:meta_before -> :meta_after) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Delete non-matching row → no metadata change +SELECT COUNT(*) AS meta_before FROM tasks_cloudsync \gset +DELETE FROM tasks WHERE id = 'b'; +SELECT COUNT(*) AS meta_after FROM tasks_cloudsync \gset +SELECT (:meta_before = :meta_after) AS filter_delete_nonmatch_ok \gset +\if :filter_delete_nonmatch_ok +\echo [PASS] (:testid) Non-matching DELETE did not change metadata +\else +\echo [FAIL] (:testid) Non-matching DELETE changed metadata (:meta_before -> :meta_after) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 4: Roundtrip - sync to second database, verify only filtered rows transfer +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_26_b; +CREATE DATABASE cloudsync_test_26_b; + +\connect cloudsync_test_26_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('tasks') AS _init_site_id_b \gset +SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _set_filter_b_ok \gset +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS _apply_ok \gset + +-- Verify: db2 should have only the matching rows +SELECT COUNT(*) AS task_count FROM tasks \gset +SELECT (:task_count = 2) AS roundtrip_count_ok \gset +\if :roundtrip_count_ok +\echo [PASS] (:testid) Roundtrip: correct number of rows synced (2) +\else +\echo [FAIL] (:testid) Roundtrip: expected 2 rows, got :task_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row 'c' exists with user_id = 1 +SELECT COUNT(*) AS c_exists FROM tasks WHERE id = 'c' AND user_id = 1 \gset +SELECT (:c_exists = 1) AS roundtrip_row_ok \gset +\if :roundtrip_row_ok +\echo [PASS] (:testid) Roundtrip: task 'c' with user_id=1 present +\else +\echo [FAIL] (:testid) Roundtrip: task 'c' with user_id=1 not found +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_26_a; +DROP DATABASE IF EXISTS cloudsync_test_26_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/27_rls_batch_merge.sql b/test/postgresql/27_rls_batch_merge.sql new file mode 100644 index 00000000..2ab51bfd --- /dev/null +++ b/test/postgresql/27_rls_batch_merge.sql @@ -0,0 +1,356 @@ +-- 'RLS batch merge test' +-- Verifies that the deferred column-batch merge produces complete rows +-- that work correctly with PostgreSQL Row Level Security policies. +-- +-- Tests 1-3: cloudsync_payload_apply runs as superuser (service-role pattern). +-- RLS is enforced at the query layer when users access data. +-- +-- Tests 4-6: cloudsync_payload_apply runs as non-superuser (authenticated-role +-- pattern). RLS is enforced during the write itself. + +\set testid '27' +\ir helper_test_init.sql + +\set USER1 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' +\set USER2 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb' + +-- ============================================================ +-- DB A: source database (no RLS) +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_27_a; +CREATE DATABASE cloudsync_test_27_a; + +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE documents ( + id TEXT PRIMARY KEY NOT NULL, + user_id UUID, + title TEXT, + content TEXT +); +SELECT cloudsync_init('documents') AS _init_site_id_a \gset + +-- ============================================================ +-- DB B: target database (with RLS) +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_27_b; +CREATE DATABASE cloudsync_test_27_b; + +-- Create non-superuser role (ignore error if it already exists) +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'test_rls_user') THEN + CREATE ROLE test_rls_user LOGIN; + END IF; +END $$; + +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE documents ( + id TEXT PRIMARY KEY NOT NULL, + user_id UUID, + title TEXT, + content TEXT +); +SELECT cloudsync_init('documents') AS _init_site_id_b \gset + +-- Auth mock: auth.uid() reads from session variable app.current_user_id +CREATE SCHEMA IF NOT EXISTS auth; +CREATE OR REPLACE FUNCTION auth.uid() RETURNS UUID + LANGUAGE sql STABLE +AS $$ SELECT NULLIF(current_setting('app.current_user_id', true), '')::UUID; $$; + +-- Enable RLS +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "select_own" ON documents FOR SELECT + USING (auth.uid() = user_id); +CREATE POLICY "insert_own" ON documents FOR INSERT + WITH CHECK (auth.uid() = user_id); +CREATE POLICY "update_own" ON documents FOR UPDATE + USING (auth.uid() = user_id) + WITH CHECK (auth.uid() = user_id); +CREATE POLICY "delete_own" ON documents FOR DELETE + USING (auth.uid() = user_id); + +-- Grant permissions to test_rls_user +GRANT USAGE ON SCHEMA public TO test_rls_user; +GRANT ALL ON ALL TABLES IN SCHEMA public TO test_rls_user; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO test_rls_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO test_rls_user; +GRANT USAGE ON SCHEMA auth TO test_rls_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA auth TO test_rls_user; + +-- ============================================================ +-- Test 1: Batch merge produces complete row — user1 doc synced +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +INSERT INTO documents VALUES ('doc1', :'USER1'::UUID, 'Title 1', 'Content 1'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_1 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Save high-water mark so subsequent encodes only pick up new changes +SELECT COALESCE(max(db_version), 0) AS max_dbv_1 FROM cloudsync_changes \gset + +-- Apply as superuser (service-role pattern) +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_hex_1', 'hex')) AS apply_1 \gset + +-- 1 row × 3 non-PK columns = 3 column-change entries +SELECT (:apply_1::int = 3) AS apply_1_ok \gset +\if :apply_1_ok +\echo [PASS] (:testid) RLS: apply returned :apply_1 +\else +\echo [FAIL] (:testid) RLS: apply returned :apply_1 (expected 3) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify complete row written (all columns present) +SELECT COUNT(*) AS doc1_count FROM documents WHERE id = 'doc1' AND title = 'Title 1' AND content = 'Content 1' AND user_id = :'USER1'::UUID \gset +SELECT (:doc1_count::int = 1) AS test1_ok \gset +\if :test1_ok +\echo [PASS] (:testid) RLS: batch merge writes complete row +\else +\echo [FAIL] (:testid) RLS: batch merge writes complete row — got :doc1_count matching rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Sync user2 doc, then verify RLS hides it from user1 +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +INSERT INTO documents VALUES ('doc2', :'USER2'::UUID, 'Title 2', 'Content 2'); + +-- Encode only changes newer than test 1 (doc2 only) +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_1 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_2 FROM cloudsync_changes \gset + +-- Apply as superuser +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_hex_2', 'hex')) AS apply_2 \gset + +-- 1 row × 3 non-PK columns = 3 entries +SELECT (:apply_2::int = 3) AS apply_2_ok \gset +\if :apply_2_ok +\echo [PASS] (:testid) RLS: apply returned :apply_2 +\else +\echo [FAIL] (:testid) RLS: apply returned :apply_2 (expected 3) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify doc2 exists (superuser sees all) +SELECT COUNT(*) AS doc2_exists FROM documents WHERE id = 'doc2' \gset + +-- Now check as user1: RLS should hide doc2 (owned by user2) +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT COUNT(*) AS doc2_visible FROM documents WHERE id = 'doc2' \gset +RESET ROLE; + +SELECT (:doc2_exists::int = 1 AND :doc2_visible::int = 0) AS test2_ok \gset +\if :test2_ok +\echo [PASS] (:testid) RLS: user2 doc synced but hidden from user1 +\else +\echo [FAIL] (:testid) RLS: user2 doc synced but hidden from user1 — exists=:doc2_exists visible=:doc2_visible +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Update doc1, verify user1 sees update via RLS +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +UPDATE documents SET title = 'Title 1 Updated' WHERE id = 'doc1'; + +-- Encode only changes newer than test 2 (doc1 update only) +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_3 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_2 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_3 FROM cloudsync_changes \gset + +-- Apply as superuser +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_hex_3', 'hex')) AS apply_3 \gset + +-- 1 row × 1 changed column (title) = 1 entry +SELECT (:apply_3::int = 1) AS apply_3_ok \gset +\if :apply_3_ok +\echo [PASS] (:testid) RLS: apply returned :apply_3 +\else +\echo [FAIL] (:testid) RLS: apply returned :apply_3 (expected 1) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify update applied (superuser check) +SELECT COUNT(*) AS doc1_updated FROM documents WHERE id = 'doc1' AND title = 'Title 1 Updated' \gset + +-- Verify user1 can see the updated row via RLS +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT COUNT(*) AS doc1_visible FROM documents WHERE id = 'doc1' AND title = 'Title 1 Updated' \gset +RESET ROLE; + +SELECT (:doc1_updated::int = 1 AND :doc1_visible::int = 1) AS test3_ok \gset +\if :test3_ok +\echo [PASS] (:testid) RLS: update synced and visible to owner +\else +\echo [FAIL] (:testid) RLS: update synced and visible to owner — updated=:doc1_updated visible=:doc1_visible +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Authenticated insert allowed (own row) +-- cloudsync_payload_apply as non-superuser with matching user_id +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +INSERT INTO documents VALUES ('doc3', :'USER1'::UUID, 'Title 3', 'Content 3'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_4 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_3 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_4 FROM cloudsync_changes \gset + +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_4', 'hex')) AS apply_4 \gset +RESET ROLE; + +-- 1 row × 3 non-PK columns = 3 entries +SELECT (:apply_4::int = 3) AS apply_4_ok \gset +\if :apply_4_ok +\echo [PASS] (:testid) RLS auth: apply returned :apply_4 +\else +\echo [FAIL] (:testid) RLS auth: apply returned :apply_4 (expected 3) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify doc3 exists with all columns correct +SELECT COUNT(*) AS doc3_count FROM documents WHERE id = 'doc3' AND title = 'Title 3' AND content = 'Content 3' AND user_id = :'USER1'::UUID \gset +SELECT (:doc3_count::int = 1) AS test4_ok \gset +\if :test4_ok +\echo [PASS] (:testid) RLS auth: insert own row allowed +\else +\echo [FAIL] (:testid) RLS auth: insert own row allowed — got :doc3_count matching rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Authenticated insert denied (other user's row) +-- cloudsync_payload_apply as non-superuser with mismatched user_id +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +INSERT INTO documents VALUES ('doc4', :'USER2'::UUID, 'Title 4', 'Content 4'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_5 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_4 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_5 FROM cloudsync_changes \gset + +-- Apply as test_rls_user with USER1 identity — should be denied (doc4 owned by USER2) +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_5', 'hex')) AS apply_5 \gset + +-- Reconnect for clean state after expected RLS denial +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql + +-- 1 row × 3 non-PK columns = 3 entries (returned even if denied) +SELECT (:apply_5::int = 3) AS apply_5_ok \gset +\if :apply_5_ok +\echo [PASS] (:testid) RLS auth: denied apply returned :apply_5 +\else +\echo [FAIL] (:testid) RLS auth: denied apply returned :apply_5 (expected 3) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify doc4 does NOT exist (superuser check) +SELECT COUNT(*) AS doc4_count FROM documents WHERE id = 'doc4' \gset +SELECT (:doc4_count::int = 0) AS test5_ok \gset +\if :test5_ok +\echo [PASS] (:testid) RLS auth: insert other user row denied +\else +\echo [FAIL] (:testid) RLS auth: insert other user row denied — got :doc4_count rows (expected 0) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Authenticated update allowed (own row) +-- cloudsync_payload_apply as non-superuser updating own row +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +UPDATE documents SET title = 'Title 3 Updated' WHERE id = 'doc3'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_6 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_5 \gset + +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_6', 'hex')) AS apply_6 \gset +RESET ROLE; + +-- 1 row × 1 changed column (title) = 1 entry +SELECT (:apply_6::int = 1) AS apply_6_ok \gset +\if :apply_6_ok +\echo [PASS] (:testid) RLS auth: apply returned :apply_6 +\else +\echo [FAIL] (:testid) RLS auth: apply returned :apply_6 (expected 1) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify doc3 title was updated +SELECT COUNT(*) AS doc3_updated FROM documents WHERE id = 'doc3' AND title = 'Title 3 Updated' \gset +SELECT (:doc3_updated::int = 1) AS test6_ok \gset +\if :test6_ok +\echo [PASS] (:testid) RLS auth: update own row allowed +\else +\echo [FAIL] (:testid) RLS auth: update own row allowed — got :doc3_updated matching rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_27_a; +DROP DATABASE IF EXISTS cloudsync_test_27_b; +DROP ROLE IF EXISTS test_rls_user; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/28_db_version_tracking.sql b/test/postgresql/28_db_version_tracking.sql new file mode 100644 index 00000000..2baa69bf --- /dev/null +++ b/test/postgresql/28_db_version_tracking.sql @@ -0,0 +1,275 @@ +-- Test db_version/seq tracking in cloudsync_changes after payload apply +-- PostgreSQL equivalent of SQLite unit tests: +-- "Merge Test db_version 1" (do_test_merge_check_db_version) +-- "Merge Test db_version 2" (do_test_merge_check_db_version_2) + +\set testid '28' +\ir helper_test_init.sql + +-- ============================================================ +-- Setup: create databases A and B with the todo table +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_28_a; +DROP DATABASE IF EXISTS cloudsync_test_28_b; +CREATE DATABASE cloudsync_test_28_a; +CREATE DATABASE cloudsync_test_28_b; + +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE todo (id TEXT PRIMARY KEY NOT NULL, title TEXT, status TEXT); +SELECT cloudsync_init('todo', 'CLS', 1) AS _init_a \gset + +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE todo (id TEXT PRIMARY KEY NOT NULL, title TEXT, status TEXT); +SELECT cloudsync_init('todo', 'CLS', 1) AS _init_b \gset + +-- ============================================================ +-- Test 1: One-way merge (A -> B), mixed insert patterns +-- Mirrors do_test_merge_check_db_version from test/unit.c +-- ============================================================ + +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql + +-- Autocommit insert (db_version 1) +INSERT INTO todo VALUES ('ID1', 'Buy groceries', 'in_progress1'); + +-- Multi-row insert (db_version 2 — single statement) +INSERT INTO todo VALUES ('ID2', 'Buy bananas', 'in_progress2'), ('ID3', 'Buy vegetables', 'in_progress3'); + +-- Autocommit insert (db_version 3) +INSERT INTO todo VALUES ('ID4', 'Buy apples', 'in_progress4'); + +-- Transaction with 3 inserts (db_version 4 — one transaction) +BEGIN; +INSERT INTO todo VALUES ('ID5', 'Buy oranges', 'in_progress5'); +INSERT INTO todo VALUES ('ID6', 'Buy lemons', 'in_progress6'); +INSERT INTO todo VALUES ('ID7', 'Buy pizza', 'in_progress7'); +COMMIT; + +-- Encode payload +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_t1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_t1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Apply to B +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +\if :payload_a_t1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_t1', 3), 'hex')) AS _apply_t1 \gset +\endif + +-- Verify data matches +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(title, '') || ':' || COALESCE(status, ''), ',' ORDER BY id), '')) AS hash_b_t1 +FROM todo \gset + +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(title, '') || ':' || COALESCE(status, ''), ',' ORDER BY id), '')) AS hash_a_t1 +FROM todo \gset + +SELECT (:'hash_a_t1' = :'hash_b_t1') AS t1_data_ok \gset +\if :t1_data_ok +\echo [PASS] (:testid) db_version test 1: data roundtrip matches +\else +\echo [FAIL] (:testid) db_version test 1: data roundtrip mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify no repeated (db_version, seq) tuples on B +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +SELECT COUNT(*) AS dup_count_b_t1 +FROM ( + SELECT db_version, seq, COUNT(*) AS cnt + FROM cloudsync_changes + GROUP BY db_version, seq + HAVING COUNT(*) > 1 +) AS dups \gset + +SELECT (:dup_count_b_t1::int = 0) AS t1_no_dups_b \gset +\if :t1_no_dups_b +\echo [PASS] (:testid) db_version test 1: no duplicate (db_version, seq) on B +\else +\echo [FAIL] (:testid) db_version test 1: duplicate (db_version, seq) on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row count +SELECT COUNT(*) AS row_count_b_t1 FROM todo \gset +SELECT (:row_count_b_t1::int = 7) AS t1_count_ok \gset +\if :t1_count_ok +\echo [PASS] (:testid) db_version test 1: row count correct (7) +\else +\echo [FAIL] (:testid) db_version test 1: expected 7 rows, got :row_count_b_t1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Bidirectional merge (A -> B, B -> A), mixed patterns +-- Mirrors do_test_merge_check_db_version_2 from test/unit.c +-- ============================================================ + +-- Reset: drop and recreate databases +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_28_a; +DROP DATABASE IF EXISTS cloudsync_test_28_b; +CREATE DATABASE cloudsync_test_28_a; +CREATE DATABASE cloudsync_test_28_b; + +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE todo (id TEXT PRIMARY KEY NOT NULL, title TEXT, status TEXT); +SELECT cloudsync_init('todo', 'CLS', 1) AS _init_a2 \gset + +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE todo (id TEXT PRIMARY KEY NOT NULL, title TEXT, status TEXT); +SELECT cloudsync_init('todo', 'CLS', 1) AS _init_b2 \gset + +-- DB A: two autocommit inserts (db_version 1, 2) +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +INSERT INTO todo VALUES ('ID1', 'Buy groceries', 'in_progress'); +INSERT INTO todo VALUES ('ID2', 'Foo', 'Bar'); + +-- DB B: two autocommit inserts + one transaction with 2 inserts +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +INSERT INTO todo VALUES ('ID3', 'Foo3', 'Bar3'); +INSERT INTO todo VALUES ('ID4', 'Foo4', 'Bar4'); +BEGIN; +INSERT INTO todo VALUES ('ID5', 'Foo5', 'Bar5'); +INSERT INTO todo VALUES ('ID6', 'Foo6', 'Bar6'); +COMMIT; + +-- Encode A's payload +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_t2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_t2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Encode B's payload +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_t2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_t2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Apply A -> B +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +\if :payload_a_t2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_t2', 3), 'hex')) AS _apply_a_to_b \gset +\endif + +-- Apply B -> A +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +\if :payload_b_t2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_t2', 3), 'hex')) AS _apply_b_to_a \gset +\endif + +-- Verify data matches between A and B +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(title, '') || ':' || COALESCE(status, ''), ',' ORDER BY id), '')) AS hash_a_t2 +FROM todo \gset + +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(title, '') || ':' || COALESCE(status, ''), ',' ORDER BY id), '')) AS hash_b_t2 +FROM todo \gset + +SELECT (:'hash_a_t2' = :'hash_b_t2') AS t2_data_ok \gset +\if :t2_data_ok +\echo [PASS] (:testid) db_version test 2: bidirectional data matches +\else +\echo [FAIL] (:testid) db_version test 2: bidirectional data mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row count (6 rows: ID1-ID6) +SELECT COUNT(*) AS row_count_t2 FROM todo \gset +SELECT (:row_count_t2::int = 6) AS t2_count_ok \gset +\if :t2_count_ok +\echo [PASS] (:testid) db_version test 2: row count correct (6) +\else +\echo [FAIL] (:testid) db_version test 2: expected 6 rows, got :row_count_t2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify no repeated (db_version, seq) tuples on A +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +SELECT COUNT(*) AS dup_count_a_t2 +FROM ( + SELECT db_version, seq, COUNT(*) AS cnt + FROM cloudsync_changes + GROUP BY db_version, seq + HAVING COUNT(*) > 1 +) AS dups \gset + +SELECT (:dup_count_a_t2::int = 0) AS t2_no_dups_a \gset +\if :t2_no_dups_a +\echo [PASS] (:testid) db_version test 2: no duplicate (db_version, seq) on A +\else +\echo [FAIL] (:testid) db_version test 2: duplicate (db_version, seq) on A +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify no repeated (db_version, seq) tuples on B +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +SELECT COUNT(*) AS dup_count_b_t2 +FROM ( + SELECT db_version, seq, COUNT(*) AS cnt + FROM cloudsync_changes + GROUP BY db_version, seq + HAVING COUNT(*) > 1 +) AS dups \gset + +SELECT (:dup_count_b_t2::int = 0) AS t2_no_dups_b \gset +\if :t2_no_dups_b +\echo [PASS] (:testid) db_version test 2: no duplicate (db_version, seq) on B +\else +\echo [FAIL] (:testid) db_version test 2: duplicate (db_version, seq) on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +-- DROP DATABASE IF EXISTS cloudsync_test_28_a; +-- DROP DATABASE IF EXISTS cloudsync_test_28_b; +\endif diff --git a/test/postgresql/29_rls_multicol.sql b/test/postgresql/29_rls_multicol.sql new file mode 100644 index 00000000..de8f3047 --- /dev/null +++ b/test/postgresql/29_rls_multicol.sql @@ -0,0 +1,435 @@ +-- 'RLS multi-column batch merge test' +-- Extends test 27 with more column types (INTEGER, BOOLEAN) and additional +-- test cases: update-denied, mixed payloads (per-PK savepoint isolation), +-- and NULL handling. +-- +-- Tests 1-2: superuser (service-role pattern) +-- Tests 3-8: authenticated-role pattern + +\set testid '29' +\ir helper_test_init.sql + +\set USER1 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' +\set USER2 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb' + +-- ============================================================ +-- DB A: source database (no RLS) +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_29_a; +CREATE DATABASE cloudsync_test_29_a; + +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE tasks ( + id TEXT PRIMARY KEY NOT NULL, + user_id UUID, + title TEXT, + description TEXT, + priority INTEGER, + is_complete BOOLEAN +); +SELECT cloudsync_init('tasks') AS _init_site_id_a \gset + +-- ============================================================ +-- DB B: target database (with RLS) +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_29_b; +CREATE DATABASE cloudsync_test_29_b; + +-- Create non-superuser role (ignore error if it already exists) +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'test_rls_user') THEN + CREATE ROLE test_rls_user LOGIN; + END IF; +END $$; + +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE tasks ( + id TEXT PRIMARY KEY NOT NULL, + user_id UUID, + title TEXT, + description TEXT, + priority INTEGER, + is_complete BOOLEAN +); +SELECT cloudsync_init('tasks') AS _init_site_id_b \gset + +-- Auth mock: auth.uid() reads from session variable app.current_user_id +CREATE SCHEMA IF NOT EXISTS auth; +CREATE OR REPLACE FUNCTION auth.uid() RETURNS UUID + LANGUAGE sql STABLE +AS $$ SELECT NULLIF(current_setting('app.current_user_id', true), '')::UUID; $$; + +-- Enable RLS +ALTER TABLE tasks ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "select_own" ON tasks FOR SELECT + USING (auth.uid() = user_id); +CREATE POLICY "insert_own" ON tasks FOR INSERT + WITH CHECK (auth.uid() = user_id); +CREATE POLICY "update_own" ON tasks FOR UPDATE + USING (auth.uid() = user_id) + WITH CHECK (auth.uid() = user_id); +CREATE POLICY "delete_own" ON tasks FOR DELETE + USING (auth.uid() = user_id); + +-- Grant permissions to test_rls_user +GRANT USAGE ON SCHEMA public TO test_rls_user; +GRANT ALL ON ALL TABLES IN SCHEMA public TO test_rls_user; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO test_rls_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO test_rls_user; +GRANT USAGE ON SCHEMA auth TO test_rls_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA auth TO test_rls_user; + +-- ============================================================ +-- Test 1: Superuser multi-row insert with varied types +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +INSERT INTO tasks VALUES ('t1', :'USER1'::UUID, 'Task 1', 'Desc 1', 3, false); +INSERT INTO tasks VALUES ('t2', :'USER1'::UUID, 'Task 2', 'Desc 2', 1, true); +INSERT INTO tasks VALUES ('t3', :'USER2'::UUID, 'Task 3', 'Desc 3', 5, false); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_1 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_1 FROM cloudsync_changes \gset + +-- Apply as superuser +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_hex_1', 'hex')) AS apply_1 \gset + +-- 3 rows × 5 non-PK columns = 15 column-change entries +SELECT (:apply_1::int = 15) AS apply_1_ok \gset +\if :apply_1_ok +\echo [PASS] (:testid) RLS multicol: superuser multi-row apply returned :apply_1 +\else +\echo [FAIL] (:testid) RLS multicol: superuser multi-row apply returned :apply_1 (expected 15) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify all 3 rows with correct column values +SELECT COUNT(*) AS t1_ok FROM tasks WHERE id = 't1' AND user_id = :'USER1'::UUID AND title = 'Task 1' AND description = 'Desc 1' AND priority = 3 AND is_complete = false \gset +SELECT COUNT(*) AS t2_ok FROM tasks WHERE id = 't2' AND user_id = :'USER1'::UUID AND title = 'Task 2' AND description = 'Desc 2' AND priority = 1 AND is_complete = true \gset +SELECT COUNT(*) AS t3_ok FROM tasks WHERE id = 't3' AND user_id = :'USER2'::UUID AND title = 'Task 3' AND description = 'Desc 3' AND priority = 5 AND is_complete = false \gset +SELECT (:t1_ok::int = 1 AND :t2_ok::int = 1 AND :t3_ok::int = 1) AS test1_ok \gset +\if :test1_ok +\echo [PASS] (:testid) RLS multicol: superuser multi-row insert with varied types +\else +\echo [FAIL] (:testid) RLS multicol: superuser multi-row insert with varied types — t1=:t1_ok t2=:t2_ok t3=:t3_ok +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Superuser multi-column partial update +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +UPDATE tasks SET title = 'Task 1 Updated', priority = 10, is_complete = true WHERE id = 't1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_1 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_2 FROM cloudsync_changes \gset + +-- Apply as superuser +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_hex_2', 'hex')) AS apply_2 \gset + +-- 1 row × 3 changed columns (title, priority, is_complete) = 3 entries +SELECT (:apply_2::int = 3) AS apply_2_ok \gset +\if :apply_2_ok +\echo [PASS] (:testid) RLS multicol: superuser partial update apply returned :apply_2 +\else +\echo [FAIL] (:testid) RLS multicol: superuser partial update apply returned :apply_2 (expected 3) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify updated columns changed and description preserved +SELECT COUNT(*) AS t1_updated FROM tasks WHERE id = 't1' AND title = 'Task 1 Updated' AND description = 'Desc 1' AND priority = 10 AND is_complete = true \gset +SELECT (:t1_updated::int = 1) AS test2_ok \gset +\if :test2_ok +\echo [PASS] (:testid) RLS multicol: superuser partial update preserves unchanged columns +\else +\echo [FAIL] (:testid) RLS multicol: superuser partial update preserves unchanged columns — got :t1_updated +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Authenticated insert own row (all columns) +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +INSERT INTO tasks VALUES ('t4', :'USER1'::UUID, 'Task 4', 'Desc 4', 2, false); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_3 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_2 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_3 FROM cloudsync_changes \gset + +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_3', 'hex')) AS apply_3 \gset +RESET ROLE; + +-- 1 row × 5 non-PK columns = 5 entries +SELECT (:apply_3::int = 5) AS apply_3_ok \gset +\if :apply_3_ok +\echo [PASS] (:testid) RLS multicol auth: insert own row apply returned :apply_3 +\else +\echo [FAIL] (:testid) RLS multicol auth: insert own row apply returned :apply_3 (expected 5) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row exists with all columns correct +SELECT COUNT(*) AS t4_count FROM tasks WHERE id = 't4' AND user_id = :'USER1'::UUID AND title = 'Task 4' AND description = 'Desc 4' AND priority = 2 AND is_complete = false \gset +SELECT (:t4_count::int = 1) AS test3_ok \gset +\if :test3_ok +\echo [PASS] (:testid) RLS multicol auth: insert own row allowed +\else +\echo [FAIL] (:testid) RLS multicol auth: insert own row allowed — got :t4_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Authenticated insert denied (other user's row) +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +INSERT INTO tasks VALUES ('t5', :'USER2'::UUID, 'Task 5', 'Desc 5', 7, true); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_4 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_3 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_4 FROM cloudsync_changes \gset + +-- Apply as test_rls_user with USER1 identity — should be denied (t5 owned by USER2) +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_4', 'hex')) AS apply_4 \gset + +-- Reconnect for clean state after expected RLS denial +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql + +-- 1 row × 5 columns = 5 entries in payload (returned even if denied) +SELECT (:apply_4::int = 5) AS apply_4_ok \gset +\if :apply_4_ok +\echo [PASS] (:testid) RLS multicol auth: denied insert apply returned :apply_4 +\else +\echo [FAIL] (:testid) RLS multicol auth: denied insert apply returned :apply_4 (expected 5) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify t5 does NOT exist (superuser check) +SELECT COUNT(*) AS t5_count FROM tasks WHERE id = 't5' \gset +SELECT (:t5_count::int = 0) AS test4_ok \gset +\if :test4_ok +\echo [PASS] (:testid) RLS multicol auth: insert other user row denied +\else +\echo [FAIL] (:testid) RLS multicol auth: insert other user row denied — got :t5_count rows (expected 0) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Authenticated update own row (multiple columns) +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +UPDATE tasks SET title = 'Task 4 Updated', priority = 9 WHERE id = 't4'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_5 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_4 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_5 FROM cloudsync_changes \gset + +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_5', 'hex')) AS apply_5 \gset +RESET ROLE; + +-- 1 row × 2 changed columns (title, priority) = 2 entries +SELECT (:apply_5::int = 2) AS apply_5_ok \gset +\if :apply_5_ok +\echo [PASS] (:testid) RLS multicol auth: update own row apply returned :apply_5 +\else +\echo [FAIL] (:testid) RLS multicol auth: update own row apply returned :apply_5 (expected 2) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify both columns changed, others preserved +SELECT COUNT(*) AS t4_updated FROM tasks WHERE id = 't4' AND title = 'Task 4 Updated' AND description = 'Desc 4' AND priority = 9 AND is_complete = false \gset +SELECT (:t4_updated::int = 1) AS test5_ok \gset +\if :test5_ok +\echo [PASS] (:testid) RLS multicol auth: update own row allowed +\else +\echo [FAIL] (:testid) RLS multicol auth: update own row allowed — got :t4_updated +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Authenticated update denied (other user's row) +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +-- t3 is owned by USER2, update it on A +UPDATE tasks SET title = 'Task 3 Hacked', priority = 99 WHERE id = 't3'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_6 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_5 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_6 FROM cloudsync_changes \gset + +-- Apply as test_rls_user with USER1 identity — should be denied (t3 owned by USER2) +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_6', 'hex')) AS apply_6 \gset + +-- Reconnect for clean state after expected RLS denial +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql + +-- 1 row × 2 changed columns (title, priority) = 2 entries in payload +SELECT (:apply_6::int = 2) AS apply_6_ok \gset +\if :apply_6_ok +\echo [PASS] (:testid) RLS multicol auth: denied update apply returned :apply_6 +\else +\echo [FAIL] (:testid) RLS multicol auth: denied update apply returned :apply_6 (expected 2) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify t3 still has original values (superuser check) +SELECT COUNT(*) AS t3_unchanged FROM tasks WHERE id = 't3' AND title = 'Task 3' AND priority = 5 \gset +SELECT (:t3_unchanged::int = 1) AS test6_ok \gset +\if :test6_ok +\echo [PASS] (:testid) RLS multicol auth: update other user row denied +\else +\echo [FAIL] (:testid) RLS multicol auth: update other user row denied — got :t3_unchanged (expected 1 unchanged) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 7: Mixed payload — own + other user's rows (per-PK savepoint) +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +INSERT INTO tasks VALUES ('t6', :'USER1'::UUID, 'Task 6', 'Desc 6', 4, false); +INSERT INTO tasks VALUES ('t7', :'USER2'::UUID, 'Task 7', 'Desc 7', 8, true); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_7 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_6 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_7 FROM cloudsync_changes \gset + +-- Apply as test_rls_user with USER1 identity +-- Per-PK savepoint: t6 (USER1) should succeed, t7 (USER2) should be denied +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_7', 'hex')) AS apply_7 \gset + +-- Reconnect for clean verification as superuser +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql + +-- 2 rows × 5 columns = 10 entries in payload +SELECT (:apply_7::int = 10) AS apply_7_ok \gset +\if :apply_7_ok +\echo [PASS] (:testid) RLS multicol auth: mixed payload apply returned :apply_7 +\else +\echo [FAIL] (:testid) RLS multicol auth: mixed payload apply returned :apply_7 (expected 10) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- t6 (own row) should exist, t7 (other's row) should NOT +SELECT COUNT(*) AS t6_exists FROM tasks WHERE id = 't6' AND user_id = :'USER1'::UUID AND title = 'Task 6' \gset +SELECT COUNT(*) AS t7_exists FROM tasks WHERE id = 't7' \gset +SELECT (:t6_exists::int = 1 AND :t7_exists::int = 0) AS test7_ok \gset +\if :test7_ok +\echo [PASS] (:testid) RLS multicol auth: mixed payload — per-PK savepoint isolation +\else +\echo [FAIL] (:testid) RLS multicol auth: mixed payload — t6=:t6_exists (expect 1) t7=:t7_exists (expect 0) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 8: NULL in non-ownership columns +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +INSERT INTO tasks VALUES ('t8', :'USER1'::UUID, 'Task 8', NULL, NULL, false); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_8 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_7 \gset + +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_8', 'hex')) AS apply_8 \gset +RESET ROLE; + +-- 1 row × 5 non-PK columns = 5 entries +SELECT (:apply_8::int = 5) AS apply_8_ok \gset +\if :apply_8_ok +\echo [PASS] (:testid) RLS multicol auth: NULL columns apply returned :apply_8 +\else +\echo [FAIL] (:testid) RLS multicol auth: NULL columns apply returned :apply_8 (expected 5) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify NULLs preserved +SELECT COUNT(*) AS t8_count FROM tasks WHERE id = 't8' AND user_id = :'USER1'::UUID AND title = 'Task 8' AND description IS NULL AND priority IS NULL AND is_complete = false \gset +SELECT (:t8_count::int = 1) AS test8_ok \gset +\if :test8_ok +\echo [PASS] (:testid) RLS multicol auth: NULL in non-ownership columns preserved +\else +\echo [FAIL] (:testid) RLS multicol auth: NULL in non-ownership columns preserved — got :t8_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_29_a; +DROP DATABASE IF EXISTS cloudsync_test_29_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/30_null_prikey_insert.sql b/test/postgresql/30_null_prikey_insert.sql new file mode 100644 index 00000000..4e0bc087 --- /dev/null +++ b/test/postgresql/30_null_prikey_insert.sql @@ -0,0 +1,68 @@ +-- Test: NULL Primary Key Insert Rejection +-- Verifies that inserting a NULL primary key into a cloudsync-enabled table fails +-- and that the metatable only contains rows for valid inserts. + +\set testid '30' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test database +DROP DATABASE IF EXISTS cloudsync_test_30; +CREATE DATABASE cloudsync_test_30; + +\connect cloudsync_test_30 +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with primary key and init cloudsync +CREATE TABLE t_null_pk ( + id TEXT NOT NULL PRIMARY KEY, + value TEXT +); + +SELECT cloudsync_init('t_null_pk', 'CLS', 1) AS _init \gset + +-- Test 1: INSERT with NULL primary key should fail +DO $$ +BEGIN + INSERT INTO t_null_pk (id, value) VALUES (NULL, 'test'); + RAISE EXCEPTION 'INSERT with NULL PK should have failed'; +EXCEPTION WHEN not_null_violation THEN + -- Expected +END $$; + +SELECT (COUNT(*) = 0) AS null_pk_rejected FROM t_null_pk \gset +\if :null_pk_rejected +\echo [PASS] (:testid) NULL PK insert rejected +\else +\echo [FAIL] (:testid) NULL PK insert was not rejected +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: INSERT with valid (non-NULL) primary key should succeed +INSERT INTO t_null_pk (id, value) VALUES ('valid_id', 'test'); + +SELECT (COUNT(*) = 1) AS valid_insert_ok FROM t_null_pk WHERE id = 'valid_id' \gset +\if :valid_insert_ok +\echo [PASS] (:testid) Valid PK insert succeeded +\else +\echo [FAIL] (:testid) Valid PK insert failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Metatable should have exactly 1 row (from the valid insert only) +SELECT (COUNT(*) = 1) AS meta_row_ok FROM t_null_pk_cloudsync \gset +\if :meta_row_ok +\echo [PASS] (:testid) Metatable has exactly 1 row +\else +\echo [FAIL] (:testid) Metatable row count mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_30; +\endif diff --git a/test/postgresql/31_alter_table_sync.sql b/test/postgresql/31_alter_table_sync.sql new file mode 100644 index 00000000..365c3568 --- /dev/null +++ b/test/postgresql/31_alter_table_sync.sql @@ -0,0 +1,383 @@ +-- Alter Table Sync Test +-- Tests cloudsync_begin_alter and cloudsync_commit_alter functions. +-- Verifies that schema changes (add column) are handled correctly +-- and data syncs after alteration. + +\set testid '31' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_31a; +DROP DATABASE IF EXISTS cloudsync_test_31b; +CREATE DATABASE cloudsync_test_31a; +CREATE DATABASE cloudsync_test_31b; + +-- ============================================================================ +-- Setup Database A +-- ============================================================================ + +\connect cloudsync_test_31a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE products ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0 +); + +SELECT cloudsync_init('products', 'CLS', 0) AS _init_a \gset + +INSERT INTO products VALUES ('11111111-1111-1111-1111-111111111111', 'Product A1', 10.99, 100); +INSERT INTO products VALUES ('22222222-2222-2222-2222-222222222222', 'Product A2', 20.50, 200); + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_31b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE products ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0 +); + +SELECT cloudsync_init('products', 'CLS', 0) AS _init_b \gset + +INSERT INTO products VALUES ('33333333-3333-3333-3333-333333333333', 'Product B1', 30.00, 300); +INSERT INTO products VALUES ('44444444-4444-4444-4444-444444444444', 'Product B2', 40.75, 400); + +-- ============================================================================ +-- Initial Sync: A -> B and B -> A +-- ============================================================================ + +\echo [INFO] (:testid) === Initial Sync Before ALTER === + +-- Encode payload from A +\connect cloudsync_test_31a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply A's payload to B, encode B's payload +\connect cloudsync_test_31b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_a_to_b \gset + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply B's payload to A, verify initial sync +\connect cloudsync_test_31a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +SELECT COUNT(*) AS count_a_initial FROM products \gset + +\connect cloudsync_test_31b +\ir helper_psql_conn_setup.sql +SELECT COUNT(*) AS count_b_initial FROM products \gset + +SELECT (:count_a_initial = 4 AND :count_b_initial = 4) AS initial_sync_ok \gset +\if :initial_sync_ok +\echo [PASS] (:testid) Initial sync complete - both databases have 4 rows +\else +\echo [FAIL] (:testid) Initial sync failed - A: :count_a_initial, B: :count_b_initial +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ALTER TABLE on Database A (begin_alter + ALTER + commit_alter on SAME connection) +-- ============================================================================ + +\echo [INFO] (:testid) === ALTER TABLE on Database A === + +\connect cloudsync_test_31a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset + +SELECT cloudsync_begin_alter('products') AS begin_alter_a \gset +\if :begin_alter_a +\echo [PASS] (:testid) cloudsync_begin_alter succeeded on Database A +\else +\echo [FAIL] (:testid) cloudsync_begin_alter failed on Database A +SELECT (:fail::int + 1) AS fail \gset +\endif + +ALTER TABLE products ADD COLUMN description TEXT NOT NULL DEFAULT ''; + +SELECT cloudsync_commit_alter('products') AS commit_alter_a \gset +\if :commit_alter_a +\echo [PASS] (:testid) cloudsync_commit_alter succeeded on Database A +\else +\echo [FAIL] (:testid) cloudsync_commit_alter failed on Database A +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Insert and update post-ALTER data on A +INSERT INTO products (id, name, price, quantity, description) +VALUES ('55555555-5555-5555-5555-555555555555', 'New Product A', 55.55, 555, 'Added after alter on A'); + +UPDATE products SET description = 'Updated on A' WHERE id = '11111111-1111-1111-1111-111111111111'; +UPDATE products SET quantity = 150 WHERE id = '11111111-1111-1111-1111-111111111111'; + +-- Encode post-ALTER payload from A +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT (length(:'payload_a2_hex') > 0) AS payload_a2_created \gset +\if :payload_a2_created +\echo [PASS] (:testid) Post-alter payload encoded from Database A +\else +\echo [FAIL] (:testid) Post-alter payload empty from Database A +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ALTER TABLE on Database B (begin_alter + ALTER + commit_alter on SAME connection) +-- Apply A's payload, insert/update, encode B's payload +-- ============================================================================ + +\echo [INFO] (:testid) === ALTER TABLE on Database B === + +\connect cloudsync_test_31b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset + +SELECT cloudsync_begin_alter('products') AS begin_alter_b \gset +\if :begin_alter_b +\echo [PASS] (:testid) cloudsync_begin_alter succeeded on Database B +\else +\echo [FAIL] (:testid) cloudsync_begin_alter failed on Database B +SELECT (:fail::int + 1) AS fail \gset +\endif + +ALTER TABLE products ADD COLUMN description TEXT NOT NULL DEFAULT ''; + +SELECT cloudsync_commit_alter('products') AS commit_alter_b \gset +\if :commit_alter_b +\echo [PASS] (:testid) cloudsync_commit_alter succeeded on Database B +\else +\echo [FAIL] (:testid) cloudsync_commit_alter failed on Database B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Insert and update post-ALTER data on B +INSERT INTO products (id, name, price, quantity, description) +VALUES ('66666666-6666-6666-6666-666666666666', 'New Product B', 66.66, 666, 'Added after alter on B'); + +UPDATE products SET description = 'Updated on B' WHERE id = '33333333-3333-3333-3333-333333333333'; +UPDATE products SET quantity = 350 WHERE id = '33333333-3333-3333-3333-333333333333'; + +-- Apply A's post-alter payload to B +SELECT cloudsync_payload_apply(decode(:'payload_a2_hex', 'hex')) AS apply_a2_to_b \gset + +SELECT (:apply_a2_to_b >= 0) AS apply_a2_ok \gset +\if :apply_a2_ok +\echo [PASS] (:testid) Post-alter payload from A applied to B +\else +\echo [FAIL] (:testid) Post-alter payload from A failed to apply to B: :apply_a2_to_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Encode post-ALTER payload from B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- ============================================================================ +-- Apply B's payload to A, then verify final state +-- ============================================================================ + +\echo [INFO] (:testid) === Apply B payload to A and verify === + +\connect cloudsync_test_31a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_b2_hex', 'hex')) AS apply_b2_to_a \gset + +SELECT (:apply_b2_to_a >= 0) AS apply_b2_ok \gset +\if :apply_b2_ok +\echo [PASS] (:testid) Post-alter payload from B applied to A +\else +\echo [FAIL] (:testid) Post-alter payload from B failed to apply to A: :apply_b2_to_a +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify final state +-- ============================================================================ + +\echo [INFO] (:testid) === Verify Final State === + +-- Compute hash of Database A +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(description, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_final FROM products \gset + +\echo [INFO] (:testid) Database A final hash: :hash_a_final + +-- Row count on A +SELECT COUNT(*) AS count_a_final FROM products \gset + +-- Verify new row from B exists in A +SELECT COUNT(*) = 1 AS new_row_b_ok +FROM products +WHERE id = '66666666-6666-6666-6666-666666666666' + AND name = 'New Product B' + AND price = 66.66 + AND quantity = 666 + AND description = 'Added after alter on B' \gset + +-- Verify updated row from B synced to A +SELECT COUNT(*) = 1 AS updated_row_b_ok +FROM products +WHERE id = '33333333-3333-3333-3333-333333333333' + AND description = 'Updated on B' + AND quantity = 350 \gset + +\connect cloudsync_test_31b +\ir helper_psql_conn_setup.sql + +-- Compute hash of Database B +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(description, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_final FROM products \gset + +\echo [INFO] (:testid) Database B final hash: :hash_b_final + +-- Row count on B +SELECT COUNT(*) AS count_b_final FROM products \gset + +-- Verify new row from A exists in B +SELECT COUNT(*) = 1 AS new_row_a_ok +FROM products +WHERE id = '55555555-5555-5555-5555-555555555555' + AND name = 'New Product A' + AND price = 55.55 + AND quantity = 555 + AND description = 'Added after alter on A' \gset + +-- Verify updated row from A synced to B +SELECT COUNT(*) = 1 AS updated_row_a_ok +FROM products +WHERE id = '11111111-1111-1111-1111-111111111111' + AND description = 'Updated on A' + AND quantity = 150 \gset + +-- Verify new column exists +SELECT COUNT(*) = 1 AS description_column_exists +FROM information_schema.columns +WHERE table_name = 'products' AND column_name = 'description' \gset + +-- ============================================================================ +-- Report results +-- ============================================================================ + +-- Compare final hashes +SELECT (:'hash_a_final' = :'hash_b_final') AS final_hashes_match \gset +\if :final_hashes_match +\echo [PASS] (:testid) Final data integrity verified - hashes match after ALTER +\else +\echo [FAIL] (:testid) Final data integrity check failed - A: :hash_a_final, B: :hash_b_final +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:count_a_final = 6 AND :count_b_final = 6) AS row_counts_ok \gset +\if :row_counts_ok +\echo [PASS] (:testid) Row counts match (6 rows each) +\else +\echo [FAIL] (:testid) Row counts mismatch - A: :count_a_final, B: :count_b_final +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :new_row_a_ok +\echo [PASS] (:testid) New row from A synced to B with new schema +\else +\echo [FAIL] (:testid) New row from A not found or incorrect in B +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :new_row_b_ok +\echo [PASS] (:testid) New row from B synced to A with new schema +\else +\echo [FAIL] (:testid) New row from B not found or incorrect in A +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :updated_row_a_ok +\echo [PASS] (:testid) Updated row from A synced with new column values +\else +\echo [FAIL] (:testid) Updated row from A not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :updated_row_b_ok +\echo [PASS] (:testid) Updated row from B synced with new column values +\else +\echo [FAIL] (:testid) Updated row from B not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :description_column_exists +\echo [PASS] (:testid) Added column 'description' exists +\else +\echo [FAIL] (:testid) Added column 'description' not found +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_31a; +DROP DATABASE IF EXISTS cloudsync_test_31b; +\endif diff --git a/test/postgresql/32_block_lww.sql b/test/postgresql/32_block_lww.sql new file mode 100644 index 00000000..389408f3 --- /dev/null +++ b/test/postgresql/32_block_lww.sql @@ -0,0 +1,146 @@ +-- 'Block-level LWW test' + +\set testid '32' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_test_a; +CREATE DATABASE cloudsync_block_test_a; + +\connect cloudsync_block_test_a +\ir helper_psql_conn_setup.sql + +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create a table with a text column for block-level LWW +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); + +-- Initialize cloudsync for the table +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset + +-- Configure body column as block-level +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol \gset + +-- Test 1: INSERT text, verify blocks table populated +INSERT INTO docs (id, body) VALUES ('doc1', 'line1 +line2 +line3'); + +-- Verify blocks table was created +SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'docs_cloudsync_blocks') AS blocks_table_exists \gset +\if :blocks_table_exists +\echo [PASS] (:testid) Blocks table created +\else +\echo [FAIL] (:testid) Blocks table not created +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify blocks have been stored (3 lines = 3 blocks) +SELECT count(*) AS block_count FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:block_count::int = 3) AS insert_blocks_ok \gset +\if :insert_blocks_ok +\echo [PASS] (:testid) Block insert: 3 blocks created +\else +\echo [FAIL] (:testid) Block insert: expected 3 blocks, got :block_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify metadata has block entries (col_name contains \x1F separator) +SELECT count(*) AS meta_block_count FROM docs_cloudsync WHERE col_name LIKE 'body' || chr(31) || '%' \gset +SELECT (:meta_block_count::int = 3) AS meta_blocks_ok \gset +\if :meta_blocks_ok +\echo [PASS] (:testid) Block metadata: 3 block entries in _cloudsync +\else +\echo [FAIL] (:testid) Block metadata: expected 3 entries, got :meta_block_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: UPDATE text (modify one line, add one line) +UPDATE docs SET body = 'line1 +line2_modified +line3 +line4' WHERE id = 'doc1'; + +-- Verify blocks updated (should now have 4 blocks) +SELECT count(*) AS block_count2 FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:block_count2::int = 4) AS update_blocks_ok \gset +\if :update_blocks_ok +\echo [PASS] (:testid) Block update: 4 blocks after update +\else +\echo [FAIL] (:testid) Block update: expected 4 blocks, got :block_count2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Materialize and verify round-trip +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat \gset +SELECT body AS materialized_body FROM docs WHERE id = 'doc1' \gset + +SELECT (:'materialized_body' = 'line1 +line2_modified +line3 +line4') AS materialize_ok \gset +\if :materialize_ok +\echo [PASS] (:testid) Text materialize: reconstructed text matches +\else +\echo [FAIL] (:testid) Text materialize: text mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 4: Verify col_value works for block entries +SELECT count(*) AS col_value_count FROM docs_cloudsync +WHERE col_name LIKE 'body' || chr(31) || '%' +AND cloudsync_col_value('docs', col_name, pk) IS NOT NULL \gset +SELECT (:col_value_count::int > 0) AS col_value_ok \gset +\if :col_value_ok +\echo [PASS] (:testid) col_value works for block entries +\else +\echo [FAIL] (:testid) col_value returned NULL for block entries +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 5: Sync roundtrip - encode payload from db A before disconnecting +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS block_payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_test_b; +CREATE DATABASE cloudsync_block_test_b; +\connect cloudsync_block_test_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_b \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_b \gset + +SELECT cloudsync_payload_apply(decode(:'block_payload_hex', 'hex')) AS _apply_b \gset + +-- Materialize on db B +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat_b \gset +SELECT body AS body_b FROM docs WHERE id = 'doc1' \gset + +SELECT (:'body_b' = 'line1 +line2_modified +line3 +line4') AS sync_ok \gset +\if :sync_ok +\echo [PASS] (:testid) Block sync roundtrip: text matches after apply + materialize +\else +\echo [FAIL] (:testid) Block sync roundtrip: text mismatch on db B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_block_test_a; +DROP DATABASE IF EXISTS cloudsync_block_test_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/33_block_lww_extended.sql b/test/postgresql/33_block_lww_extended.sql new file mode 100644 index 00000000..8b5de48d --- /dev/null +++ b/test/postgresql/33_block_lww_extended.sql @@ -0,0 +1,339 @@ +-- 'Block-level LWW extended tests: DELETE, empty text, multi-update, conflict' + +\set testid '33' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_ext_a; +DROP DATABASE IF EXISTS cloudsync_block_ext_b; +CREATE DATABASE cloudsync_block_ext_a; +CREATE DATABASE cloudsync_block_ext_b; + +-- ============================================================ +-- Setup db A +-- ============================================================ +\connect cloudsync_block_ext_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_a \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_a \gset + +-- ============================================================ +-- Test 1: DELETE marks tombstone, block metadata dropped +-- ============================================================ +INSERT INTO docs (id, body) VALUES ('doc1', 'line1 +line2 +line3'); + +-- Verify 3 block metadata entries exist +SELECT count(*) AS meta_before FROM docs_cloudsync WHERE col_name LIKE 'body' || chr(31) || '%' \gset +SELECT (:meta_before::int = 3) AS meta_before_ok \gset +\if :meta_before_ok +\echo [PASS] (:testid) Delete pre-check: 3 block metadata entries +\else +\echo [FAIL] (:testid) Delete pre-check: expected 3 metadata, got :meta_before +SELECT (:fail::int + 1) AS fail \gset +\endif + +DELETE FROM docs WHERE id = 'doc1'; + +-- Tombstone should exist with even version (deleted) +SELECT count(*) AS tombstone_count FROM docs_cloudsync WHERE col_name = '__[RIP]__' AND col_version % 2 = 0 \gset +SELECT (:tombstone_count::int = 1) AS tombstone_ok \gset +\if :tombstone_ok +\echo [PASS] (:testid) Delete: tombstone exists with even version +\else +\echo [FAIL] (:testid) Delete: expected 1 tombstone, got :tombstone_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Block metadata should be dropped +SELECT count(*) AS meta_after FROM docs_cloudsync WHERE col_name LIKE 'body' || chr(31) || '%' \gset +SELECT (:meta_after::int = 0) AS meta_dropped_ok \gset +\if :meta_dropped_ok +\echo [PASS] (:testid) Delete: block metadata dropped +\else +\echo [FAIL] (:testid) Delete: expected 0 metadata after delete, got :meta_after +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Row should be gone from base table +SELECT count(*) AS row_after FROM docs WHERE id = 'doc1' \gset +SELECT (:row_after::int = 0) AS row_gone_ok \gset +\if :row_gone_ok +\echo [PASS] (:testid) Delete: row removed from base table +\else +\echo [FAIL] (:testid) Delete: row still in base table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Empty text creates single block +-- ============================================================ +INSERT INTO docs (id, body) VALUES ('doc_empty', ''); + +SELECT count(*) AS empty_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_empty') \gset +SELECT (:empty_blocks::int = 1) AS empty_block_ok \gset +\if :empty_block_ok +\echo [PASS] (:testid) Empty text: 1 block created +\else +\echo [FAIL] (:testid) Empty text: expected 1 block, got :empty_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update from empty to multi-line +UPDATE docs SET body = 'NewLine1 +NewLine2' WHERE id = 'doc_empty'; + +SELECT count(*) AS updated_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_empty') \gset +SELECT (:updated_blocks::int = 2) AS update_from_empty_ok \gset +\if :update_from_empty_ok +\echo [PASS] (:testid) Empty text: 2 blocks after update +\else +\echo [FAIL] (:testid) Empty text: expected 2 blocks after update, got :updated_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Multi-update block counts +-- ============================================================ +INSERT INTO docs (id, body) VALUES ('doc_multi', 'A +B +C'); + +-- Update 1: remove middle line +UPDATE docs SET body = 'A +C' WHERE id = 'doc_multi'; + +SELECT count(*) AS blocks1 FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_multi') \gset +SELECT (:blocks1::int = 2) AS multi1_ok \gset +\if :multi1_ok +\echo [PASS] (:testid) Multi-update: 2 blocks after removing middle +\else +\echo [FAIL] (:testid) Multi-update: expected 2, got :blocks1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update 2: add two lines +UPDATE docs SET body = 'A +X +C +Y' WHERE id = 'doc_multi'; + +SELECT count(*) AS blocks2 FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_multi') \gset +SELECT (:blocks2::int = 4) AS multi2_ok \gset +\if :multi2_ok +\echo [PASS] (:testid) Multi-update: 4 blocks after adding lines +\else +\echo [FAIL] (:testid) Multi-update: expected 4, got :blocks2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update 3: collapse to single line +UPDATE docs SET body = 'SINGLE' WHERE id = 'doc_multi'; + +SELECT count(*) AS blocks3 FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_multi') \gset +SELECT (:blocks3::int = 1) AS multi3_ok \gset +\if :multi3_ok +\echo [PASS] (:testid) Multi-update: 1 block after collapse +\else +\echo [FAIL] (:testid) Multi-update: expected 1, got :blocks3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Materialize and verify +SELECT cloudsync_text_materialize('docs', 'body', 'doc_multi') AS _mat_multi \gset +SELECT body AS multi_body FROM docs WHERE id = 'doc_multi' \gset +SELECT (:'multi_body' = 'SINGLE') AS multi_mat_ok \gset +\if :multi_mat_ok +\echo [PASS] (:testid) Multi-update: materialize matches +\else +\echo [FAIL] (:testid) Multi-update: materialize mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Two-database conflict on same block +-- ============================================================ + +-- Setup db B +\connect cloudsync_block_ext_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_b \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_b \gset + +-- Insert initial doc on db A +\connect cloudsync_block_ext_a +INSERT INTO docs (id, body) VALUES ('doc_conflict', 'Same +Middle +End'); + +-- Sync A -> B (round 1) +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a_r1 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_ext_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_a_r1', 'hex')) AS _apply_b_r1 \gset + +-- Materialize on B to get body +SELECT cloudsync_text_materialize('docs', 'body', 'doc_conflict') AS _mat_b_init \gset + +-- Verify B has the initial doc +SELECT body AS body_b_init FROM docs WHERE id = 'doc_conflict' \gset +SELECT (:'body_b_init' = 'Same +Middle +End') AS init_sync_ok \gset +\if :init_sync_ok +\echo [PASS] (:testid) Conflict: initial sync to B matches +\else +\echo [FAIL] (:testid) Conflict: initial sync to B mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Site A edits first line +\connect cloudsync_block_ext_a +UPDATE docs SET body = 'SiteA +Middle +End' WHERE id = 'doc_conflict'; + +-- Site B edits first line (conflict!) +\connect cloudsync_block_ext_b +UPDATE docs SET body = 'SiteB +Middle +End' WHERE id = 'doc_conflict'; + +-- Collect payloads from both sites +\connect cloudsync_block_ext_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a_r2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_ext_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_b_r2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply A's changes to B +SELECT cloudsync_payload_apply(decode(:'payload_a_r2', 'hex')) AS _apply_b_r2 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_conflict') AS _mat_b_r2 \gset + +-- Apply B's changes to A +\connect cloudsync_block_ext_a +SELECT cloudsync_payload_apply(decode(:'payload_b_r2', 'hex')) AS _apply_a_r2 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_conflict') AS _mat_a_r2 \gset + +-- Both should converge +SELECT body AS body_a_final FROM docs WHERE id = 'doc_conflict' \gset + +\connect cloudsync_block_ext_b +SELECT body AS body_b_final FROM docs WHERE id = 'doc_conflict' \gset + +-- Bodies must match (convergence) +SELECT (:'body_a_final' = :'body_b_final') AS converge_ok \gset +\if :converge_ok +\echo [PASS] (:testid) Conflict: databases converge after sync +\else +\echo [FAIL] (:testid) Conflict: databases diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Unchanged lines must be preserved +SELECT (position('Middle' in :'body_a_final') > 0) AS has_middle \gset +\if :has_middle +\echo [PASS] (:testid) Conflict: unchanged line 'Middle' preserved +\else +\echo [FAIL] (:testid) Conflict: 'Middle' missing from result +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('End' in :'body_a_final') > 0) AS has_end \gset +\if :has_end +\echo [PASS] (:testid) Conflict: unchanged line 'End' preserved +\else +\echo [FAIL] (:testid) Conflict: 'End' missing from result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- One of the conflicting edits must win +SELECT (position('SiteA' in :'body_a_final') > 0 OR position('SiteB' in :'body_a_final') > 0) AS has_winner \gset +\if :has_winner +\echo [PASS] (:testid) Conflict: one site edit won (LWW) +\else +\echo [FAIL] (:testid) Conflict: neither SiteA nor SiteB in result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: DELETE then re-INSERT (reinsert) +-- ============================================================ +\connect cloudsync_block_ext_a + +INSERT INTO docs (id, body) VALUES ('doc_reinsert', 'Old1 +Old2'); +DELETE FROM docs WHERE id = 'doc_reinsert'; + +-- Block metadata should be dropped after delete +SELECT count(*) AS meta_reinsert_del FROM docs_cloudsync +WHERE pk = cloudsync_pk_encode('doc_reinsert') +AND col_name LIKE 'body' || chr(31) || '%' \gset +SELECT (:meta_reinsert_del::int = 0) AS reinsert_meta_del_ok \gset +\if :reinsert_meta_del_ok +\echo [PASS] (:testid) Reinsert: metadata dropped after delete +\else +\echo [FAIL] (:testid) Reinsert: expected 0 metadata, got :meta_reinsert_del +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Re-insert with new content +INSERT INTO docs (id, body) VALUES ('doc_reinsert', 'New1 +New2 +New3'); + +SELECT count(*) AS meta_reinsert_new FROM docs_cloudsync +WHERE pk = cloudsync_pk_encode('doc_reinsert') +AND col_name LIKE 'body' || chr(31) || '%' \gset +SELECT (:meta_reinsert_new::int = 3) AS reinsert_meta_ok \gset +\if :reinsert_meta_ok +\echo [PASS] (:testid) Reinsert: 3 block metadata after re-insert +\else +\echo [FAIL] (:testid) Reinsert: expected 3 metadata, got :meta_reinsert_new +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync to B and materialize +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_reinsert +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_ext_b +SELECT cloudsync_payload_apply(decode(:'payload_reinsert', 'hex')) AS _apply_reinsert \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_reinsert') AS _mat_reinsert \gset +SELECT body AS body_reinsert FROM docs WHERE id = 'doc_reinsert' \gset + +SELECT (:'body_reinsert' = 'New1 +New2 +New3') AS reinsert_sync_ok \gset +\if :reinsert_sync_ok +\echo [PASS] (:testid) Reinsert: sync roundtrip matches +\else +\echo [FAIL] (:testid) Reinsert: sync mismatch on db B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_block_ext_a; +DROP DATABASE IF EXISTS cloudsync_block_ext_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/34_block_lww_advanced.sql b/test/postgresql/34_block_lww_advanced.sql new file mode 100644 index 00000000..f8753ffa --- /dev/null +++ b/test/postgresql/34_block_lww_advanced.sql @@ -0,0 +1,698 @@ +-- 'Block-level LWW advanced tests: noconflict, add+edit, three-way, mixed cols, NULL->text, interleaved, custom delimiter, large text, rapid updates' + +\set testid '34' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_adv_a; +DROP DATABASE IF EXISTS cloudsync_block_adv_b; +DROP DATABASE IF EXISTS cloudsync_block_adv_c; +CREATE DATABASE cloudsync_block_adv_a; +CREATE DATABASE cloudsync_block_adv_b; +CREATE DATABASE cloudsync_block_adv_c; + +-- ============================================================ +-- Test 1: Non-conflicting edits on different blocks +-- Site A edits line 1, Site B edits line 3 — BOTH should survive +-- ============================================================ +\connect cloudsync_block_adv_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_a \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_a \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_b \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_b \gset + +-- Insert initial on A +\connect cloudsync_block_adv_a +INSERT INTO docs (id, body) VALUES ('doc1', 'Line1 +Line2 +Line3'); + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_init +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_init', 'hex')) AS _apply_init \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat_init \gset + +-- Site A: edit first line +\connect cloudsync_block_adv_a +UPDATE docs SET body = 'EditedByA +Line2 +Line3' WHERE id = 'doc1'; + +-- Site B: edit third line (no conflict — different block) +\connect cloudsync_block_adv_b +UPDATE docs SET body = 'Line1 +Line2 +EditedByB' WHERE id = 'doc1'; + +-- Collect payloads +\connect cloudsync_block_adv_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +-- Apply A -> B, B -> A +SELECT cloudsync_payload_apply(decode(:'payload_a', 'hex')) AS _apply_ab \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat_b \gset + +\connect cloudsync_block_adv_a +SELECT cloudsync_payload_apply(decode(:'payload_b', 'hex')) AS _apply_ba \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat_a \gset + +-- Both should converge +SELECT body AS body_a FROM docs WHERE id = 'doc1' \gset +\connect cloudsync_block_adv_b +SELECT body AS body_b FROM docs WHERE id = 'doc1' \gset + +SELECT (:'body_a' = :'body_b') AS converge_ok \gset +\if :converge_ok +\echo [PASS] (:testid) NoConflict: databases converge +\else +\echo [FAIL] (:testid) NoConflict: databases diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Both edits should be preserved +SELECT (position('EditedByA' in :'body_a') > 0) AS has_a_edit \gset +\if :has_a_edit +\echo [PASS] (:testid) NoConflict: Site A edit preserved +\else +\echo [FAIL] (:testid) NoConflict: Site A edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('EditedByB' in :'body_a') > 0) AS has_b_edit \gset +\if :has_b_edit +\echo [PASS] (:testid) NoConflict: Site B edit preserved +\else +\echo [FAIL] (:testid) NoConflict: Site B edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('Line2' in :'body_a') > 0) AS has_middle \gset +\if :has_middle +\echo [PASS] (:testid) NoConflict: unchanged line preserved +\else +\echo [FAIL] (:testid) NoConflict: unchanged line missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Concurrent add + edit +-- Site A adds a line, Site B modifies an existing line +-- ============================================================ +\connect cloudsync_block_adv_a +INSERT INTO docs (id, body) VALUES ('doc2', 'Alpha +Bravo'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_d2_init +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_d2_init', 'hex')) AS _apply_d2 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc2') AS _mat_d2 \gset + +-- Site A: add a new line at end +\connect cloudsync_block_adv_a +UPDATE docs SET body = 'Alpha +Bravo +Charlie' WHERE id = 'doc2'; + +-- Site B: modify first line +\connect cloudsync_block_adv_b +UPDATE docs SET body = 'AlphaEdited +Bravo' WHERE id = 'doc2'; + +\connect cloudsync_block_adv_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_d2a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_d2b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +SELECT cloudsync_payload_apply(decode(:'payload_d2a', 'hex')) AS _apply_d2ab \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc2') AS _mat_d2b \gset + +\connect cloudsync_block_adv_a +SELECT cloudsync_payload_apply(decode(:'payload_d2b', 'hex')) AS _apply_d2ba \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc2') AS _mat_d2a \gset + +SELECT body AS body_d2a FROM docs WHERE id = 'doc2' \gset +\connect cloudsync_block_adv_b +SELECT body AS body_d2b FROM docs WHERE id = 'doc2' \gset + +SELECT (:'body_d2a' = :'body_d2b') AS d2_converge \gset +\if :d2_converge +\echo [PASS] (:testid) Add+Edit: databases converge +\else +\echo [FAIL] (:testid) Add+Edit: databases diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('Charlie' in :'body_d2a') > 0) AS has_charlie \gset +\if :has_charlie +\echo [PASS] (:testid) Add+Edit: added line Charlie preserved +\else +\echo [FAIL] (:testid) Add+Edit: added line Charlie missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('Bravo' in :'body_d2a') > 0) AS has_bravo \gset +\if :has_bravo +\echo [PASS] (:testid) Add+Edit: unchanged Bravo preserved +\else +\echo [FAIL] (:testid) Add+Edit: Bravo missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Three-way sync — 3 databases, each edits a different line +-- ============================================================ +\connect cloudsync_block_adv_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_c \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_c \gset + +-- Insert initial on A +\connect cloudsync_block_adv_a +INSERT INTO docs (id, body) VALUES ('doc3', 'L1 +L2 +L3 +L4'); + +-- Sync A -> B, A -> C +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3init +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3init', 'hex')) AS _apply_3b \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat_3b \gset + +\connect cloudsync_block_adv_c +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3init', 'hex')) AS _apply_3c \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat_3c \gset + +-- A edits line 1 +\connect cloudsync_block_adv_a +UPDATE docs SET body = 'S0 +L2 +L3 +L4' WHERE id = 'doc3'; + +-- B edits line 2 +\connect cloudsync_block_adv_b +UPDATE docs SET body = 'L1 +S1 +L3 +L4' WHERE id = 'doc3'; + +-- C edits line 4 +\connect cloudsync_block_adv_c +UPDATE docs SET body = 'L1 +L2 +L3 +S2' WHERE id = 'doc3'; + +-- Collect all payloads +\connect cloudsync_block_adv_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_c +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3c +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +-- Full mesh apply: each site receives from the other two +\connect cloudsync_block_adv_a +SELECT cloudsync_payload_apply(decode(:'payload_3b', 'hex')) AS _3ab \gset +SELECT cloudsync_payload_apply(decode(:'payload_3c', 'hex')) AS _3ac \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat_3a_final \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3a', 'hex')) AS _3ba \gset +SELECT cloudsync_payload_apply(decode(:'payload_3c', 'hex')) AS _3bc \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat_3b_final \gset + +\connect cloudsync_block_adv_c +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3a', 'hex')) AS _3ca \gset +SELECT cloudsync_payload_apply(decode(:'payload_3b', 'hex')) AS _3cb \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat_3c_final \gset + +-- All three should converge +\connect cloudsync_block_adv_a +SELECT body AS body_3a FROM docs WHERE id = 'doc3' \gset +\connect cloudsync_block_adv_b +SELECT body AS body_3b FROM docs WHERE id = 'doc3' \gset +\connect cloudsync_block_adv_c +SELECT body AS body_3c FROM docs WHERE id = 'doc3' \gset + +SELECT (:'body_3a' = :'body_3b' AND :'body_3b' = :'body_3c') AS three_converge \gset +\if :three_converge +\echo [PASS] (:testid) Three-way: all 3 databases converge +\else +\echo [FAIL] (:testid) Three-way: databases diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('S0' in :'body_3a') > 0) AS has_s0 \gset +\if :has_s0 +\echo [PASS] (:testid) Three-way: Site A edit preserved +\else +\echo [FAIL] (:testid) Three-way: Site A edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('S1' in :'body_3a') > 0) AS has_s1 \gset +\if :has_s1 +\echo [PASS] (:testid) Three-way: Site B edit preserved +\else +\echo [FAIL] (:testid) Three-way: Site B edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('S2' in :'body_3a') > 0) AS has_s2 \gset +\if :has_s2 +\echo [PASS] (:testid) Three-way: Site C edit preserved +\else +\echo [FAIL] (:testid) Three-way: Site C edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Mixed block + normal columns +-- ============================================================ +\connect cloudsync_block_adv_a +DROP TABLE IF EXISTS notes; +CREATE TABLE notes (id TEXT PRIMARY KEY NOT NULL, body TEXT, title TEXT); +SELECT cloudsync_init('notes', 'CLS', 1) AS _init_notes_a \gset +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block') AS _setcol_notes_a \gset + +\connect cloudsync_block_adv_b +DROP TABLE IF EXISTS notes; +CREATE TABLE notes (id TEXT PRIMARY KEY NOT NULL, body TEXT, title TEXT); +SELECT cloudsync_init('notes', 'CLS', 1) AS _init_notes_b \gset +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block') AS _setcol_notes_b \gset + +\connect cloudsync_block_adv_a +INSERT INTO notes (id, body, title) VALUES ('n1', 'Line1 +Line2 +Line3', 'My Title'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_notes_init +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'notes' \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_notes_init', 'hex')) AS _apply_notes \gset +SELECT cloudsync_text_materialize('notes', 'body', 'n1') AS _mat_notes \gset + +-- A: edit block line 1 + title +\connect cloudsync_block_adv_a +UPDATE notes SET body = 'EditedLine1 +Line2 +Line3', title = 'Title From A' WHERE id = 'n1'; + +-- B: edit block line 3 + title (title conflicts via normal LWW) +\connect cloudsync_block_adv_b +UPDATE notes SET body = 'Line1 +Line2 +EditedLine3', title = 'Title From B' WHERE id = 'n1'; + +\connect cloudsync_block_adv_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_notes_a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'notes' \gset + +\connect cloudsync_block_adv_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_notes_b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'notes' \gset + +SELECT cloudsync_payload_apply(decode(:'payload_notes_a', 'hex')) AS _apply_notes_ab \gset +SELECT cloudsync_text_materialize('notes', 'body', 'n1') AS _mat_notes_b \gset + +\connect cloudsync_block_adv_a +SELECT cloudsync_payload_apply(decode(:'payload_notes_b', 'hex')) AS _apply_notes_ba \gset +SELECT cloudsync_text_materialize('notes', 'body', 'n1') AS _mat_notes_a \gset + +SELECT body AS notes_body_a FROM notes WHERE id = 'n1' \gset +SELECT title AS notes_title_a FROM notes WHERE id = 'n1' \gset +\connect cloudsync_block_adv_b +SELECT body AS notes_body_b FROM notes WHERE id = 'n1' \gset +SELECT title AS notes_title_b FROM notes WHERE id = 'n1' \gset + +SELECT (:'notes_body_a' = :'notes_body_b') AS mixed_body_ok \gset +\if :mixed_body_ok +\echo [PASS] (:testid) MixedCols: body converges +\else +\echo [FAIL] (:testid) MixedCols: body diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('EditedLine1' in :'notes_body_a') > 0 AND position('EditedLine3' in :'notes_body_a') > 0) AS both_edits \gset +\if :both_edits +\echo [PASS] (:testid) MixedCols: both block edits preserved +\else +\echo [FAIL] (:testid) MixedCols: block edits missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:'notes_title_a' = :'notes_title_b') AS mixed_title_ok \gset +\if :mixed_title_ok +\echo [PASS] (:testid) MixedCols: title converges (normal LWW) +\else +\echo [FAIL] (:testid) MixedCols: title diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: NULL to text transition +-- ============================================================ +\connect cloudsync_block_adv_a +INSERT INTO docs (id, body) VALUES ('doc_null', NULL); + +-- Verify 1 block for NULL +SELECT count(*) AS null_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_null') \gset +SELECT (:null_blocks::int = 1) AS null_block_ok \gset +\if :null_block_ok +\echo [PASS] (:testid) NULL->Text: 1 block for NULL body +\else +\echo [FAIL] (:testid) NULL->Text: expected 1 block, got :null_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update to multi-line +UPDATE docs SET body = 'Hello +World +Foo' WHERE id = 'doc_null'; + +SELECT count(*) AS text_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_null') \gset +SELECT (:text_blocks::int = 3) AS text_block_ok \gset +\if :text_block_ok +\echo [PASS] (:testid) NULL->Text: 3 blocks after update +\else +\echo [FAIL] (:testid) NULL->Text: expected 3 blocks, got :text_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync and verify +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_null +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_null', 'hex')) AS _apply_null \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_null') AS _mat_null \gset + +SELECT body AS body_null FROM docs WHERE id = 'doc_null' \gset +SELECT (:'body_null' = 'Hello +World +Foo') AS null_text_ok \gset +\if :null_text_ok +\echo [PASS] (:testid) NULL->Text: sync roundtrip matches +\else +\echo [FAIL] (:testid) NULL->Text: sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Interleaved inserts — multiple rounds between existing lines +-- ============================================================ +\connect cloudsync_block_adv_a +INSERT INTO docs (id, body) VALUES ('doc_inter', 'A +B'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_inter_init +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_inter_init', 'hex')) AS _apply_inter \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_inter') AS _mat_inter \gset + +-- Round 1: A inserts between A and B +\connect cloudsync_block_adv_a +UPDATE docs SET body = 'A +C +B' WHERE id = 'doc_inter'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_inter_r1 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_inter_r1', 'hex')) AS _r1 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_inter') AS _mat_r1 \gset + +-- Round 2: B inserts between A and C +\connect cloudsync_block_adv_b +UPDATE docs SET body = 'A +D +C +B' WHERE id = 'doc_inter'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_inter_r2 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset +\connect cloudsync_block_adv_a +SELECT cloudsync_payload_apply(decode(:'payload_inter_r2', 'hex')) AS _r2 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_inter') AS _mat_r2 \gset + +-- Round 3: A inserts between D and C +\connect cloudsync_block_adv_a +UPDATE docs SET body = 'A +D +E +C +B' WHERE id = 'doc_inter'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_inter_r3 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_inter_r3', 'hex')) AS _r3 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_inter') AS _mat_r3 \gset + +\connect cloudsync_block_adv_a +SELECT body AS inter_body_a FROM docs WHERE id = 'doc_inter' \gset +\connect cloudsync_block_adv_b +SELECT body AS inter_body_b FROM docs WHERE id = 'doc_inter' \gset + +SELECT (:'inter_body_a' = :'inter_body_b') AS inter_converge \gset +\if :inter_converge +\echo [PASS] (:testid) Interleaved: databases converge +\else +\echo [FAIL] (:testid) Interleaved: databases diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT count(*) AS inter_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_inter') \gset +SELECT (:inter_blocks::int = 5) AS inter_count_ok \gset +\if :inter_count_ok +\echo [PASS] (:testid) Interleaved: 5 blocks after 3 rounds +\else +\echo [FAIL] (:testid) Interleaved: expected 5 blocks, got :inter_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 7: Custom delimiter (paragraph separator: double newline) +-- ============================================================ +\connect cloudsync_block_adv_a +DROP TABLE IF EXISTS paragraphs; +CREATE TABLE paragraphs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('paragraphs', 'CLS', 1) AS _init_para \gset +SELECT cloudsync_set_column('paragraphs', 'body', 'algo', 'block') AS _setcol_para \gset +SELECT cloudsync_set_column('paragraphs', 'body', 'delimiter', E'\n\n') AS _setdelim \gset + +\connect cloudsync_block_adv_b +DROP TABLE IF EXISTS paragraphs; +CREATE TABLE paragraphs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('paragraphs', 'CLS', 1) AS _init_para_b \gset +SELECT cloudsync_set_column('paragraphs', 'body', 'algo', 'block') AS _setcol_para_b \gset +SELECT cloudsync_set_column('paragraphs', 'body', 'delimiter', E'\n\n') AS _setdelim_b \gset + +\connect cloudsync_block_adv_a +INSERT INTO paragraphs (id, body) VALUES ('p1', E'Para one line1\nline2\n\nPara two\n\nPara three'); + +-- Should produce 3 blocks (3 paragraphs) +SELECT count(*) AS para_blocks FROM paragraphs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('p1') \gset +SELECT (:para_blocks::int = 3) AS para_ok \gset +\if :para_ok +\echo [PASS] (:testid) CustomDelim: 3 paragraph blocks +\else +\echo [FAIL] (:testid) CustomDelim: expected 3 blocks, got :para_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync and verify roundtrip +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_para +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'paragraphs' \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_para', 'hex')) AS _apply_para \gset +SELECT cloudsync_text_materialize('paragraphs', 'body', 'p1') AS _mat_para \gset + +SELECT body AS para_body FROM paragraphs WHERE id = 'p1' \gset +SELECT (:'para_body' = E'Para one line1\nline2\n\nPara two\n\nPara three') AS para_roundtrip \gset +\if :para_roundtrip +\echo [PASS] (:testid) CustomDelim: sync roundtrip matches +\else +\echo [FAIL] (:testid) CustomDelim: sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 8: Large text — 200 lines +-- ============================================================ +\connect cloudsync_block_adv_a +\ir helper_psql_conn_setup.sql +INSERT INTO docs (id, body) +SELECT 'bigdoc', string_agg('Line ' || lpad(i::text, 3, '0') || ' content', E'\n' ORDER BY i) +FROM generate_series(0, 199) AS s(i); + +SELECT count(*) AS big_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('bigdoc') \gset +SELECT (:big_blocks::int = 200) AS big_ok \gset +\if :big_ok +\echo [PASS] (:testid) LargeText: 200 blocks created +\else +\echo [FAIL] (:testid) LargeText: expected 200 blocks, got :big_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- All positions unique +SELECT count(DISTINCT col_name) AS big_distinct FROM docs_cloudsync +WHERE col_name LIKE 'body' || chr(31) || '%' +AND pk = cloudsync_pk_encode('bigdoc') \gset +SELECT (:big_distinct::int = 200) AS big_unique \gset +\if :big_unique +\echo [PASS] (:testid) LargeText: 200 unique position IDs +\else +\echo [FAIL] (:testid) LargeText: expected 200 unique positions, got :big_distinct +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync and verify +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_big +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_big', 'hex')) AS _apply_big \gset +SELECT cloudsync_text_materialize('docs', 'body', 'bigdoc') AS _mat_big \gset + +SELECT body AS big_body_b FROM docs WHERE id = 'bigdoc' \gset +\connect cloudsync_block_adv_a +SELECT body AS big_body_a FROM docs WHERE id = 'bigdoc' \gset + +SELECT (:'big_body_a' = :'big_body_b') AS big_match \gset +\if :big_match +\echo [PASS] (:testid) LargeText: sync roundtrip matches +\else +\echo [FAIL] (:testid) LargeText: sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 9: Rapid sequential updates — 50 updates on same row +-- ============================================================ +\connect cloudsync_block_adv_a +\ir helper_psql_conn_setup.sql +INSERT INTO docs (id, body) VALUES ('rapid', 'Start'); + +DO $$ +DECLARE + i INT; + new_body TEXT := ''; +BEGIN + FOR i IN 0..49 LOOP + IF i > 0 THEN new_body := new_body || E'\n'; END IF; + new_body := new_body || 'Update' || i; + UPDATE docs SET body = new_body WHERE id = 'rapid'; + END LOOP; +END $$; + +SELECT count(*) AS rapid_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('rapid') \gset +SELECT (:rapid_blocks::int = 50) AS rapid_ok \gset +\if :rapid_ok +\echo [PASS] (:testid) RapidUpdates: 50 blocks after 50 updates +\else +\echo [FAIL] (:testid) RapidUpdates: expected 50 blocks, got :rapid_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync and verify +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_rapid +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_rapid', 'hex')) AS _apply_rapid \gset +SELECT cloudsync_text_materialize('docs', 'body', 'rapid') AS _mat_rapid \gset + +SELECT body AS rapid_body_b FROM docs WHERE id = 'rapid' \gset +\connect cloudsync_block_adv_a +SELECT body AS rapid_body_a FROM docs WHERE id = 'rapid' \gset + +SELECT (:'rapid_body_a' = :'rapid_body_b') AS rapid_match \gset +\if :rapid_match +\echo [PASS] (:testid) RapidUpdates: sync roundtrip matches +\else +\echo [FAIL] (:testid) RapidUpdates: sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('Update0' in :'rapid_body_a') > 0) AS has_first \gset +\if :has_first +\echo [PASS] (:testid) RapidUpdates: first update present +\else +\echo [FAIL] (:testid) RapidUpdates: first update missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('Update49' in :'rapid_body_a') > 0) AS has_last \gset +\if :has_last +\echo [PASS] (:testid) RapidUpdates: last update present +\else +\echo [FAIL] (:testid) RapidUpdates: last update missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_block_adv_a; +DROP DATABASE IF EXISTS cloudsync_block_adv_b; +DROP DATABASE IF EXISTS cloudsync_block_adv_c; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/35_block_lww_edge_cases.sql b/test/postgresql/35_block_lww_edge_cases.sql new file mode 100644 index 00000000..362d063b --- /dev/null +++ b/test/postgresql/35_block_lww_edge_cases.sql @@ -0,0 +1,420 @@ +-- 'Block-level LWW edge cases: unicode, special chars, delete vs edit, two block cols, text->NULL, payload sync, idempotent, ordering' + +\set testid '35' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_edge_a; +DROP DATABASE IF EXISTS cloudsync_block_edge_b; +CREATE DATABASE cloudsync_block_edge_a; +CREATE DATABASE cloudsync_block_edge_b; + +-- ============================================================ +-- Test 1: Unicode / multibyte content (emoji, CJK, accented) +-- ============================================================ +\connect cloudsync_block_edge_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc \gset + +-- Insert unicode text on A +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc1', E'Hello \U0001F600\nBonjour caf\u00e9\n\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8'); + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload1 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload1', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat \gset + +SELECT (body LIKE E'Hello %') AS unicode_ok FROM docs WHERE id = 'doc1' \gset +\if :unicode_ok +\echo [PASS] (:testid) Unicode: body starts with Hello +\else +\echo [FAIL] (:testid) Unicode: body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Check line count (3 lines = 2 newlines) +SELECT (length(body) - length(replace(body, E'\n', '')) = 2) AS unicode_lines FROM docs WHERE id = 'doc1' \gset +\if :unicode_lines +\echo [PASS] (:testid) Unicode: 3 lines present +\else +\echo [FAIL] (:testid) Unicode: wrong line count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Special characters (tabs, backslashes, quotes) +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc2', E'line\twith\ttabs\nback\\\\slash\nO''Brien said "hi"'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload2 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc2') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload2', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc2') AS _mat \gset + +SELECT (body LIKE E'%\t%') AS special_tabs FROM docs WHERE id = 'doc2' \gset +\if :special_tabs +\echo [PASS] (:testid) SpecialChars: tabs preserved +\else +\echo [FAIL] (:testid) SpecialChars: tabs lost +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE '%Brien%') AS special_quotes FROM docs WHERE id = 'doc2' \gset +\if :special_quotes +\echo [PASS] (:testid) SpecialChars: quotes preserved +\else +\echo [FAIL] (:testid) SpecialChars: quotes lost +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Delete vs edit — A deletes block 1, B edits block 2 +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc3', E'Alpha\nBeta\nGamma'); + +-- Sync initial to B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload3i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc3') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload3i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat \gset + +-- A: remove first line +\connect cloudsync_block_edge_a +UPDATE docs SET body = E'Beta\nGamma' WHERE id = 'doc3'; + +-- B: edit second line +\connect cloudsync_block_edge_b +UPDATE docs SET body = E'Alpha\nBetaEdited\nGamma' WHERE id = 'doc3'; + +-- Sync A -> B +\connect cloudsync_block_edge_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload3a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc3') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload3a', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat \gset + +-- B should have: Alpha removed (A wins), BetaEdited kept (B's edit) +SELECT (body NOT LIKE '%Alpha%') AS dve_no_alpha FROM docs WHERE id = 'doc3' \gset +\if :dve_no_alpha +\echo [PASS] (:testid) DelVsEdit: Alpha removed +\else +\echo [FAIL] (:testid) DelVsEdit: Alpha still present +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE '%BetaEdited%') AS dve_beta FROM docs WHERE id = 'doc3' \gset +\if :dve_beta +\echo [PASS] (:testid) DelVsEdit: BetaEdited present +\else +\echo [FAIL] (:testid) DelVsEdit: BetaEdited missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE '%Gamma%') AS dve_gamma FROM docs WHERE id = 'doc3' \gset +\if :dve_gamma +\echo [PASS] (:testid) DelVsEdit: Gamma present +\else +\echo [FAIL] (:testid) DelVsEdit: Gamma missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Two block columns on the same table (body + notes) +-- ============================================================ +\connect cloudsync_block_edge_a +DROP TABLE IF EXISTS articles; +CREATE TABLE articles (id TEXT PRIMARY KEY NOT NULL, body TEXT, notes TEXT); +SELECT cloudsync_init('articles', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('articles', 'body', 'algo', 'block') AS _sc1 \gset +SELECT cloudsync_set_column('articles', 'notes', 'algo', 'block') AS _sc2 \gset + +\connect cloudsync_block_edge_b +DROP TABLE IF EXISTS articles; +CREATE TABLE articles (id TEXT PRIMARY KEY NOT NULL, body TEXT, notes TEXT); +SELECT cloudsync_init('articles', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('articles', 'body', 'algo', 'block') AS _sc1 \gset +SELECT cloudsync_set_column('articles', 'notes', 'algo', 'block') AS _sc2 \gset + +-- Insert on A +\connect cloudsync_block_edge_a +INSERT INTO articles (id, body, notes) VALUES ('art1', E'Body line 1\nBody line 2', E'Note 1\nNote 2'); + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload4 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'articles' \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload4', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('articles', 'body', 'art1') AS _mb \gset +SELECT cloudsync_text_materialize('articles', 'notes', 'art1') AS _mn \gset + +SELECT (body = E'Body line 1\nBody line 2') AS twocol_body FROM articles WHERE id = 'art1' \gset +\if :twocol_body +\echo [PASS] (:testid) TwoBlockCols: body matches +\else +\echo [FAIL] (:testid) TwoBlockCols: body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (notes = E'Note 1\nNote 2') AS twocol_notes FROM articles WHERE id = 'art1' \gset +\if :twocol_notes +\echo [PASS] (:testid) TwoBlockCols: notes matches +\else +\echo [FAIL] (:testid) TwoBlockCols: notes mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit body on A, notes on B — then sync +\connect cloudsync_block_edge_a +UPDATE articles SET body = E'Body EDITED\nBody line 2' WHERE id = 'art1'; + +\connect cloudsync_block_edge_b +UPDATE articles SET notes = E'Note 1\nNote EDITED' WHERE id = 'art1'; + +-- Sync A -> B +\connect cloudsync_block_edge_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload4b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'articles' \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload4b', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('articles', 'body', 'art1') AS _mb \gset +SELECT cloudsync_text_materialize('articles', 'notes', 'art1') AS _mn \gset + +SELECT (body LIKE '%Body EDITED%') AS twocol_body_ed FROM articles WHERE id = 'art1' \gset +\if :twocol_body_ed +\echo [PASS] (:testid) TwoBlockCols: body edited +\else +\echo [FAIL] (:testid) TwoBlockCols: body edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (notes LIKE '%Note EDITED%') AS twocol_notes_ed FROM articles WHERE id = 'art1' \gset +\if :twocol_notes_ed +\echo [PASS] (:testid) TwoBlockCols: notes kept +\else +\echo [FAIL] (:testid) TwoBlockCols: notes edit lost +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Text -> NULL (update to NULL removes all blocks) +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc5', E'Line1\nLine2\nLine3'); + +-- Verify blocks created +SELECT (count(*) = 3) AS blk_ok FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc5') \gset +\if :blk_ok +\echo [PASS] (:testid) TextToNull: 3 blocks created +\else +\echo [FAIL] (:testid) TextToNull: wrong initial block count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update to NULL +UPDATE docs SET body = NULL WHERE id = 'doc5'; + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload5 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc5') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload5', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc5') AS _mat \gset + +SELECT (body IS NULL) AS null_remote FROM docs WHERE id = 'doc5' \gset +\if :null_remote +\echo [PASS] (:testid) TextToNull: body is NULL on remote +\else +\echo [FAIL] (:testid) TextToNull: body not NULL on remote +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Payload-based sync with non-conflicting edits +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc6', E'First\nSecond\nThird'); + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload6i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc6') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload6i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc6') AS _mat \gset + +-- A edits line 1 +\connect cloudsync_block_edge_a +UPDATE docs SET body = E'FirstEdited\nSecond\nThird' WHERE id = 'doc6'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload6a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc6') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload6a', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc6') AS _mat \gset + +SELECT (body = E'FirstEdited\nSecond\nThird') AS payload_ok FROM docs WHERE id = 'doc6' \gset +\if :payload_ok +\echo [PASS] (:testid) PayloadSync: body matches +\else +\echo [FAIL] (:testid) PayloadSync: body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 7: Idempotent apply — same payload twice is a no-op +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc7', E'AAA\nBBB\nCCC'); + +-- Sync initial +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload7i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc7') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload7i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc7') AS _mat \gset + +-- A edits +\connect cloudsync_block_edge_a +UPDATE docs SET body = E'AAA-edited\nBBB\nCCC' WHERE id = 'doc7'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload7e +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc7') \gset + +-- Apply TWICE to B +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload7e', 'hex')) AS _app1 \gset +SELECT cloudsync_payload_apply(decode(:'payload7e', 'hex')) AS _app2 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc7') AS _mat \gset + +SELECT (body LIKE '%AAA-edited%') AS idemp_ok FROM docs WHERE id = 'doc7' \gset +\if :idemp_ok +\echo [PASS] (:testid) Idempotent: body matches after double apply +\else +\echo [FAIL] (:testid) Idempotent: body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 8: Block position ordering — sequential inserts preserve order after sync +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc8', E'Top\nBottom'); + +-- Sync initial to B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload8i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc8') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload8i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc8') AS _mat \gset + +-- A: add two lines between Top and Bottom +\connect cloudsync_block_edge_a +UPDATE docs SET body = E'Top\nMiddle1\nMiddle2\nBottom' WHERE id = 'doc8'; + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload8a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc8') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload8a', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc8') AS _mat \gset + +SELECT (body LIKE 'Top%') AS ord_top FROM docs WHERE id = 'doc8' \gset +\if :ord_top +\echo [PASS] (:testid) Ordering: Top first +\else +\echo [FAIL] (:testid) Ordering: Top not first +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE '%Bottom') AS ord_bottom FROM docs WHERE id = 'doc8' \gset +\if :ord_bottom +\echo [PASS] (:testid) Ordering: Bottom last +\else +\echo [FAIL] (:testid) Ordering: Bottom not last +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Middle1 should come before Middle2 +SELECT (position('Middle1' IN body) < position('Middle2' IN body)) AS ord_correct FROM docs WHERE id = 'doc8' \gset +\if :ord_correct +\echo [PASS] (:testid) Ordering: Middle1 before Middle2 +\else +\echo [FAIL] (:testid) Ordering: wrong order +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body = E'Top\nMiddle1\nMiddle2\nBottom') AS ord_exact FROM docs WHERE id = 'doc8' \gset +\if :ord_exact +\echo [PASS] (:testid) Ordering: exact match +\else +\echo [FAIL] (:testid) Ordering: content mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_block_edge_a; +DROP DATABASE IF EXISTS cloudsync_block_edge_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/36_block_lww_round3.sql b/test/postgresql/36_block_lww_round3.sql new file mode 100644 index 00000000..196ca801 --- /dev/null +++ b/test/postgresql/36_block_lww_round3.sql @@ -0,0 +1,476 @@ +-- 'Block-level LWW round 3: composite PK, empty vs null, delete+reinsert, integer PK, multi-row, non-overlapping add, long line, whitespace' + +\set testid '36' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_r3_a; +DROP DATABASE IF EXISTS cloudsync_block_r3_b; +CREATE DATABASE cloudsync_block_r3_a; +CREATE DATABASE cloudsync_block_r3_b; + +-- ============================================================ +-- Test 1: Composite primary key (text + int) with block column +-- ============================================================ +\connect cloudsync_block_r3_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (owner TEXT NOT NULL, seq INTEGER NOT NULL, body TEXT, PRIMARY KEY(owner, seq)); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (owner TEXT NOT NULL, seq INTEGER NOT NULL, body TEXT, PRIMARY KEY(owner, seq)); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc \gset + +-- Insert on A +\connect cloudsync_block_r3_a +INSERT INTO docs (owner, seq, body) VALUES ('alice', 1, E'Line1\nLine2\nLine3'); + +SELECT count(*) AS cpk_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('alice', 1) \gset +SELECT (:'cpk_blocks'::int = 3) AS cpk_blk_ok \gset +\if :cpk_blk_ok +\echo [PASS] (:testid) CompositePK: 3 blocks created +\else +\echo [FAIL] (:testid) CompositePK: expected 3 blocks, got :cpk_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload1 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload1', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'alice', 1) AS _mat \gset + +SELECT (body = E'Line1\nLine2\nLine3') AS cpk_body_ok FROM docs WHERE owner = 'alice' AND seq = 1 \gset +\if :cpk_body_ok +\echo [PASS] (:testid) CompositePK: body matches on B +\else +\echo [FAIL] (:testid) CompositePK: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit on B, sync back +UPDATE docs SET body = E'Line1\nEdited2\nLine3' WHERE owner = 'alice' AND seq = 1; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload1b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' \gset + +\connect cloudsync_block_r3_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload1b', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'alice', 1) AS _mat \gset + +SELECT (body = E'Line1\nEdited2\nLine3') AS cpk_rev_ok FROM docs WHERE owner = 'alice' AND seq = 1 \gset +\if :cpk_rev_ok +\echo [PASS] (:testid) CompositePK: reverse sync body matches +\else +\echo [FAIL] (:testid) CompositePK: reverse sync body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Empty string vs NULL +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS edocs; +CREATE TABLE edocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('edocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('edocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS edocs; +CREATE TABLE edocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('edocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('edocs', 'body', 'algo', 'block') AS _sc \gset + +-- Insert empty string on A +\connect cloudsync_block_r3_a +INSERT INTO edocs (id, body) VALUES ('doc1', ''); + +SELECT count(*) AS evn_blocks FROM edocs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'evn_blocks'::int = 1) AS evn_blk_ok \gset +\if :evn_blk_ok +\echo [PASS] (:testid) EmptyVsNull: 1 block for empty string +\else +\echo [FAIL] (:testid) EmptyVsNull: expected 1 block, got :evn_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync to B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload2 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'edocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload2', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('edocs', 'body', 'doc1') AS _mat \gset + +SELECT (body IS NOT NULL AND body = '') AS evn_empty_ok FROM edocs WHERE id = 'doc1' \gset +\if :evn_empty_ok +\echo [PASS] (:testid) EmptyVsNull: body is empty string (not NULL) +\else +\echo [FAIL] (:testid) EmptyVsNull: body should be empty string +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: DELETE row then re-insert with different content +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS rdocs; +CREATE TABLE rdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('rdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('rdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS rdocs; +CREATE TABLE rdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('rdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('rdocs', 'body', 'algo', 'block') AS _sc \gset + +-- Insert and sync +\connect cloudsync_block_r3_a +INSERT INTO rdocs (id, body) VALUES ('doc1', E'Old1\nOld2\nOld3'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload3i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'rdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload3i', 'hex')) AS _app \gset + +-- Delete on A +\connect cloudsync_block_r3_a +DELETE FROM rdocs WHERE id = 'doc1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload3d +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'rdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload3d', 'hex')) AS _app \gset + +SELECT (count(*) = 0) AS dr_deleted FROM rdocs WHERE id = 'doc1' \gset +\if :dr_deleted +\echo [PASS] (:testid) DelReinsert: row deleted on B +\else +\echo [FAIL] (:testid) DelReinsert: row not deleted on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Re-insert with different content on A +\connect cloudsync_block_r3_a +INSERT INTO rdocs (id, body) VALUES ('doc1', E'New1\nNew2'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload3r +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'rdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload3r', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('rdocs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'New1\nNew2') AS dr_body_ok FROM rdocs WHERE id = 'doc1' \gset +\if :dr_body_ok +\echo [PASS] (:testid) DelReinsert: body matches after re-insert +\else +\echo [FAIL] (:testid) DelReinsert: body mismatch after re-insert +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: INTEGER primary key with block column +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS notes; +CREATE TABLE notes (id INTEGER PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('notes', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS notes; +CREATE TABLE notes (id INTEGER PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('notes', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_a +INSERT INTO notes (id, body) VALUES (42, E'First\nSecond\nThird'); + +SELECT count(*) AS ipk_blocks FROM notes_cloudsync_blocks WHERE pk = cloudsync_pk_encode(42) \gset +SELECT (:'ipk_blocks'::int = 3) AS ipk_blk_ok \gset +\if :ipk_blk_ok +\echo [PASS] (:testid) IntegerPK: 3 blocks created +\else +\echo [FAIL] (:testid) IntegerPK: expected 3 blocks, got :ipk_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload4 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'notes' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload4', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('notes', 'body', 42) AS _mat \gset + +SELECT (body = E'First\nSecond\nThird') AS ipk_body_ok FROM notes WHERE id = 42 \gset +\if :ipk_body_ok +\echo [PASS] (:testid) IntegerPK: body matches on B +\else +\echo [FAIL] (:testid) IntegerPK: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Multiple rows with block columns in a single sync +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS mdocs; +CREATE TABLE mdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('mdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('mdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS mdocs; +CREATE TABLE mdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('mdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('mdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_a +INSERT INTO mdocs (id, body) VALUES ('r1', E'R1-Line1\nR1-Line2'); +INSERT INTO mdocs (id, body) VALUES ('r2', E'R2-Alpha\nR2-Beta\nR2-Gamma'); +INSERT INTO mdocs (id, body) VALUES ('r3', 'R3-Only'); +UPDATE mdocs SET body = E'R1-Edited\nR1-Line2' WHERE id = 'r1'; +UPDATE mdocs SET body = 'R3-Changed' WHERE id = 'r3'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload5 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'mdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload5', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('mdocs', 'body', 'r1') AS _m1 \gset +SELECT cloudsync_text_materialize('mdocs', 'body', 'r2') AS _m2 \gset +SELECT cloudsync_text_materialize('mdocs', 'body', 'r3') AS _m3 \gset + +SELECT (body = E'R1-Edited\nR1-Line2') AS mr_r1 FROM mdocs WHERE id = 'r1' \gset +\if :mr_r1 +\echo [PASS] (:testid) MultiRow: r1 matches +\else +\echo [FAIL] (:testid) MultiRow: r1 mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body = E'R2-Alpha\nR2-Beta\nR2-Gamma') AS mr_r2 FROM mdocs WHERE id = 'r2' \gset +\if :mr_r2 +\echo [PASS] (:testid) MultiRow: r2 matches +\else +\echo [FAIL] (:testid) MultiRow: r2 mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body = 'R3-Changed') AS mr_r3 FROM mdocs WHERE id = 'r3' \gset +\if :mr_r3 +\echo [PASS] (:testid) MultiRow: r3 matches +\else +\echo [FAIL] (:testid) MultiRow: r3 mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Concurrent add at non-overlapping positions (top vs bottom) +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS ndocs; +CREATE TABLE ndocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('ndocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('ndocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS ndocs; +CREATE TABLE ndocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('ndocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('ndocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_a +INSERT INTO ndocs (id, body) VALUES ('doc1', E'A\nB\nC'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload6i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'ndocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload6i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('ndocs', 'body', 'doc1') AS _mat \gset + +-- A: add at top -> X A B C +\connect cloudsync_block_r3_a +UPDATE ndocs SET body = E'X\nA\nB\nC' WHERE id = 'doc1'; + +-- B: add at bottom -> A B C Y +\connect cloudsync_block_r3_b +UPDATE ndocs SET body = E'A\nB\nC\nY' WHERE id = 'doc1'; + +-- Sync A -> B +\connect cloudsync_block_r3_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload6a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'ndocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload6a', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('ndocs', 'body', 'doc1') AS _mat \gset + +SELECT (body LIKE '%X%') AS no_x FROM ndocs WHERE id = 'doc1' \gset +\if :no_x +\echo [PASS] (:testid) NonOverlap: X present +\else +\echo [FAIL] (:testid) NonOverlap: X missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE '%Y%') AS no_y FROM ndocs WHERE id = 'doc1' \gset +\if :no_y +\echo [PASS] (:testid) NonOverlap: Y present +\else +\echo [FAIL] (:testid) NonOverlap: Y missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE 'X%' OR body LIKE E'%\nX\n%') AS no_x_before FROM ndocs WHERE id = 'doc1' \gset +\if :no_x_before +\echo [PASS] (:testid) NonOverlap: X before A +\else +\echo [FAIL] (:testid) NonOverlap: X not before A +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 7: Very long single line (10K chars) +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS ldocs; +CREATE TABLE ldocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('ldocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('ldocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS ldocs; +CREATE TABLE ldocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('ldocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('ldocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_a +INSERT INTO ldocs (id, body) VALUES ('doc1', repeat('ABCDEFGHIJ', 1000)); + +SELECT count(*) AS ll_blocks FROM ldocs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'ll_blocks'::int = 1) AS ll_blk_ok \gset +\if :ll_blk_ok +\echo [PASS] (:testid) LongLine: 1 block for 10K char line +\else +\echo [FAIL] (:testid) LongLine: expected 1 block, got :ll_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload7 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'ldocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload7', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('ldocs', 'body', 'doc1') AS _mat \gset + +SELECT (body = repeat('ABCDEFGHIJ', 1000)) AS ll_body_ok FROM ldocs WHERE id = 'doc1' \gset +\if :ll_body_ok +\echo [PASS] (:testid) LongLine: body matches on B +\else +\echo [FAIL] (:testid) LongLine: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 8: Whitespace and empty lines (delimiter edge cases) +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS wdocs; +CREATE TABLE wdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('wdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('wdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS wdocs; +CREATE TABLE wdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('wdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('wdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_a +-- Text: "Line1\n\n spaces \n\t\ttabs\n\nLine6\n" = 7 blocks +INSERT INTO wdocs (id, body) VALUES ('doc1', E'Line1\n\n spaces \n\t\ttabs\n\nLine6\n'); + +SELECT count(*) AS ws_blocks FROM wdocs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'ws_blocks'::int = 7) AS ws_blk_ok \gset +\if :ws_blk_ok +\echo [PASS] (:testid) Whitespace: 7 blocks with empty/whitespace lines +\else +\echo [FAIL] (:testid) Whitespace: expected 7 blocks, got :ws_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload8 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'wdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload8', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('wdocs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Line1\n\n spaces \n\t\ttabs\n\nLine6\n') AS ws_body_ok FROM wdocs WHERE id = 'doc1' \gset +\if :ws_body_ok +\echo [PASS] (:testid) Whitespace: body matches with whitespace preserved +\else +\echo [FAIL] (:testid) Whitespace: body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit: remove empty lines +\connect cloudsync_block_r3_a +UPDATE wdocs SET body = E'Line1\n spaces \n\t\ttabs\nLine6' WHERE id = 'doc1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload8b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'wdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload8b', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('wdocs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Line1\n spaces \n\t\ttabs\nLine6') AS ws_edit_ok FROM wdocs WHERE id = 'doc1' \gset +\if :ws_edit_ok +\echo [PASS] (:testid) Whitespace: edited body matches +\else +\echo [FAIL] (:testid) Whitespace: edited body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_block_r3_a; +DROP DATABASE IF EXISTS cloudsync_block_r3_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/37_block_lww_round4.sql b/test/postgresql/37_block_lww_round4.sql new file mode 100644 index 00000000..b04713e6 --- /dev/null +++ b/test/postgresql/37_block_lww_round4.sql @@ -0,0 +1,500 @@ +-- 'Block-level LWW round 4: UUID PK, RLS+blocks, multi-table, 3-site convergence, custom delimiter sync, mixed column updates' + +\set testid '37' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_r4_a; +DROP DATABASE IF EXISTS cloudsync_block_r4_b; +DROP DATABASE IF EXISTS cloudsync_block_r4_c; +DROP DATABASE IF EXISTS cloudsync_block_3s_a; +DROP DATABASE IF EXISTS cloudsync_block_3s_b; +DROP DATABASE IF EXISTS cloudsync_block_3s_c; +CREATE DATABASE cloudsync_block_r4_a; +CREATE DATABASE cloudsync_block_r4_b; +CREATE DATABASE cloudsync_block_r4_c; + +-- ============================================================ +-- Test 1: UUID primary key with block column +-- ============================================================ +\connect cloudsync_block_r4_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS uuid_docs; +CREATE TABLE uuid_docs (id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), body TEXT); +SELECT cloudsync_init('uuid_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('uuid_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS uuid_docs; +CREATE TABLE uuid_docs (id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), body TEXT); +SELECT cloudsync_init('uuid_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('uuid_docs', 'body', 'algo', 'block') AS _sc \gset + +-- Insert on A with explicit UUID +\connect cloudsync_block_r4_a +INSERT INTO uuid_docs (id, body) VALUES ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', E'UUID-Line1\nUUID-Line2\nUUID-Line3'); + +SELECT count(*) AS uuid_blocks FROM uuid_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11') \gset +SELECT (:'uuid_blocks'::int = 3) AS uuid_blk_ok \gset +\if :uuid_blk_ok +\echo [PASS] (:testid) UUID_PK: 3 blocks created +\else +\echo [FAIL] (:testid) UUID_PK: expected 3 blocks, got :uuid_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_uuid +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'uuid_docs' \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_uuid', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('uuid_docs', 'body', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11') AS _mat \gset + +SELECT (body = E'UUID-Line1\nUUID-Line2\nUUID-Line3') AS uuid_body_ok FROM uuid_docs WHERE id = 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11' \gset +\if :uuid_body_ok +\echo [PASS] (:testid) UUID_PK: body matches on B +\else +\echo [FAIL] (:testid) UUID_PK: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit on B, reverse sync +\connect cloudsync_block_r4_b +UPDATE uuid_docs SET body = E'UUID-Line1\nUUID-Edited\nUUID-Line3' WHERE id = 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_uuid_r +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'uuid_docs' \gset + +\connect cloudsync_block_r4_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_uuid_r', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('uuid_docs', 'body', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11') AS _mat \gset + +SELECT (body = E'UUID-Line1\nUUID-Edited\nUUID-Line3') AS uuid_rev_ok FROM uuid_docs WHERE id = 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11' \gset +\if :uuid_rev_ok +\echo [PASS] (:testid) UUID_PK: reverse sync matches +\else +\echo [FAIL] (:testid) UUID_PK: reverse sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: RLS filter + block columns +-- Only rows matching filter should have block tracking +-- ============================================================ +\connect cloudsync_block_r4_a +DROP TABLE IF EXISTS rls_docs; +CREATE TABLE rls_docs (id TEXT PRIMARY KEY NOT NULL, owner_id INTEGER, body TEXT); +SELECT cloudsync_init('rls_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('rls_docs', 'body', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_filter('rls_docs', 'owner_id = 1') AS _sf \gset + +\connect cloudsync_block_r4_b +DROP TABLE IF EXISTS rls_docs; +CREATE TABLE rls_docs (id TEXT PRIMARY KEY NOT NULL, owner_id INTEGER, body TEXT); +SELECT cloudsync_init('rls_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('rls_docs', 'body', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_filter('rls_docs', 'owner_id = 1') AS _sf \gset + +-- Insert matching row (owner_id=1) and non-matching row (owner_id=2) +\connect cloudsync_block_r4_a +INSERT INTO rls_docs (id, owner_id, body) VALUES ('match1', 1, E'Filtered-Line1\nFiltered-Line2'); +INSERT INTO rls_docs (id, owner_id, body) VALUES ('nomatch', 2, E'Hidden-Line1\nHidden-Line2'); + +-- Check: matching row has blocks, non-matching does not +SELECT count(*) AS rls_match_blocks FROM rls_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('match1') \gset +SELECT count(*) AS rls_nomatch_blocks FROM rls_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('nomatch') \gset + +SELECT (:'rls_match_blocks'::int = 2) AS rls_match_ok \gset +\if :rls_match_ok +\echo [PASS] (:testid) RLS+Blocks: matching row has 2 blocks +\else +\echo [FAIL] (:testid) RLS+Blocks: expected 2 blocks for matching row, got :rls_match_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:'rls_nomatch_blocks'::int = 0) AS rls_nomatch_ok \gset +\if :rls_nomatch_ok +\echo [PASS] (:testid) RLS+Blocks: non-matching row has 0 blocks +\else +\echo [FAIL] (:testid) RLS+Blocks: expected 0 blocks for non-matching row, got :rls_nomatch_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync: only matching row should appear in changes +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_rls +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'rls_docs' \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_rls', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('rls_docs', 'body', 'match1') AS _mat \gset + +SELECT (body = E'Filtered-Line1\nFiltered-Line2') AS rls_sync_ok FROM rls_docs WHERE id = 'match1' \gset +\if :rls_sync_ok +\echo [PASS] (:testid) RLS+Blocks: matching row synced with correct body +\else +\echo [FAIL] (:testid) RLS+Blocks: matching row body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- non-matching row should NOT exist on B +SELECT (count(*) = 0) AS rls_norow_ok FROM rls_docs WHERE id = 'nomatch' \gset +\if :rls_norow_ok +\echo [PASS] (:testid) RLS+Blocks: non-matching row not synced +\else +\echo [FAIL] (:testid) RLS+Blocks: non-matching row should not exist on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Multi-table blocks — two tables with block columns in same payload +-- ============================================================ +\connect cloudsync_block_r4_a +DROP TABLE IF EXISTS articles; +DROP TABLE IF EXISTS comments; +CREATE TABLE articles (id TEXT PRIMARY KEY NOT NULL, content TEXT); +CREATE TABLE comments (id TEXT PRIMARY KEY NOT NULL, text_body TEXT); +SELECT cloudsync_init('articles', 'CLS', 1) AS _init \gset +SELECT cloudsync_init('comments', 'CLS', 1) AS _init2 \gset +SELECT cloudsync_set_column('articles', 'content', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_column('comments', 'text_body', 'algo', 'block') AS _sc2 \gset + +\connect cloudsync_block_r4_b +DROP TABLE IF EXISTS articles; +DROP TABLE IF EXISTS comments; +CREATE TABLE articles (id TEXT PRIMARY KEY NOT NULL, content TEXT); +CREATE TABLE comments (id TEXT PRIMARY KEY NOT NULL, text_body TEXT); +SELECT cloudsync_init('articles', 'CLS', 1) AS _init \gset +SELECT cloudsync_init('comments', 'CLS', 1) AS _init2 \gset +SELECT cloudsync_set_column('articles', 'content', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_column('comments', 'text_body', 'algo', 'block') AS _sc2 \gset + +\connect cloudsync_block_r4_a +INSERT INTO articles (id, content) VALUES ('art1', E'Para1\nPara2\nPara3'); +INSERT INTO comments (id, text_body) VALUES ('cmt1', E'Comment-Line1\nComment-Line2'); +UPDATE articles SET content = E'Para1-Edited\nPara2\nPara3' WHERE id = 'art1'; + +-- Single payload containing changes from both tables +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_mt +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_mt', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('articles', 'content', 'art1') AS _m1 \gset +SELECT cloudsync_text_materialize('comments', 'text_body', 'cmt1') AS _m2 \gset + +SELECT (content = E'Para1-Edited\nPara2\nPara3') AS mt_art_ok FROM articles WHERE id = 'art1' \gset +\if :mt_art_ok +\echo [PASS] (:testid) MultiTable: articles content matches +\else +\echo [FAIL] (:testid) MultiTable: articles content mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (text_body = E'Comment-Line1\nComment-Line2') AS mt_cmt_ok FROM comments WHERE id = 'cmt1' \gset +\if :mt_cmt_ok +\echo [PASS] (:testid) MultiTable: comments text_body matches +\else +\echo [FAIL] (:testid) MultiTable: comments text_body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Three-site convergence with block columns +-- All three sites make different edits, pairwise sync, verify convergence +-- Uses dedicated databases so all 3 have identical schema +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_block_3s_a; +DROP DATABASE IF EXISTS cloudsync_block_3s_b; +DROP DATABASE IF EXISTS cloudsync_block_3s_c; +CREATE DATABASE cloudsync_block_3s_a; +CREATE DATABASE cloudsync_block_3s_b; +CREATE DATABASE cloudsync_block_3s_c; + +\connect cloudsync_block_3s_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE tdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('tdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('tdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_3s_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE tdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('tdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('tdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_3s_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE tdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('tdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('tdocs', 'body', 'algo', 'block') AS _sc \gset + +-- Initial insert on A, sync to B and C +\connect cloudsync_block_3s_a +INSERT INTO tdocs (id, body) VALUES ('doc1', E'Line1\nLine2\nLine3\nLine4\nLine5'); + +-- Full changes from A (includes schema info) +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3s_init +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'tdocs' \gset + +\connect cloudsync_block_3s_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3s_init', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('tdocs', 'body', 'doc1') AS _mat \gset + +\connect cloudsync_block_3s_c +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3s_init', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('tdocs', 'body', 'doc1') AS _mat \gset + +-- Each site edits a DIFFERENT line (no conflicts) +-- A edits line 1 +\connect cloudsync_block_3s_a +UPDATE tdocs SET body = E'Line1-A\nLine2\nLine3\nLine4\nLine5' WHERE id = 'doc1'; + +-- B edits line 3 +\connect cloudsync_block_3s_b +UPDATE tdocs SET body = E'Line1\nLine2\nLine3-B\nLine4\nLine5' WHERE id = 'doc1'; + +-- C edits line 5 +\connect cloudsync_block_3s_c +UPDATE tdocs SET body = E'Line1\nLine2\nLine3\nLine4\nLine5-C' WHERE id = 'doc1'; + +-- Collect ALL changes from each site (not filtered by site_id) +-- This includes the schema info that recipients need +\connect cloudsync_block_3s_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3s_a +FROM cloudsync_changes WHERE tbl = 'tdocs' \gset + +\connect cloudsync_block_3s_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3s_b +FROM cloudsync_changes WHERE tbl = 'tdocs' \gset + +\connect cloudsync_block_3s_c +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3s_c +FROM cloudsync_changes WHERE tbl = 'tdocs' \gset + +-- Apply all to A (B's and C's changes) +\connect cloudsync_block_3s_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3s_b', 'hex')) AS _app_b \gset +SELECT cloudsync_payload_apply(decode(:'payload_3s_c', 'hex')) AS _app_c \gset +SELECT cloudsync_text_materialize('tdocs', 'body', 'doc1') AS _mat \gset + +-- Apply all to B (A's and C's changes) +\connect cloudsync_block_3s_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3s_a', 'hex')) AS _app_a \gset +SELECT cloudsync_payload_apply(decode(:'payload_3s_c', 'hex')) AS _app_c \gset +SELECT cloudsync_text_materialize('tdocs', 'body', 'doc1') AS _mat \gset + +-- Apply all to C (A's and B's changes) +\connect cloudsync_block_3s_c +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3s_a', 'hex')) AS _app_a \gset +SELECT cloudsync_payload_apply(decode(:'payload_3s_b', 'hex')) AS _app_b \gset +SELECT cloudsync_text_materialize('tdocs', 'body', 'doc1') AS _mat \gset + +-- All three should converge +\connect cloudsync_block_3s_a +SELECT body AS body_a FROM tdocs WHERE id = 'doc1' \gset +\connect cloudsync_block_3s_b +SELECT body AS body_b FROM tdocs WHERE id = 'doc1' \gset +\connect cloudsync_block_3s_c +SELECT body AS body_c FROM tdocs WHERE id = 'doc1' \gset + +SELECT (:'body_a' = :'body_b') AS ab_match \gset +SELECT (:'body_b' = :'body_c') AS bc_match \gset + +\if :ab_match +\echo [PASS] (:testid) 3-Site: A and B converge +\else +\echo [FAIL] (:testid) 3-Site: A and B diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :bc_match +\echo [PASS] (:testid) 3-Site: B and C converge +\else +\echo [FAIL] (:testid) 3-Site: B and C diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- All edits should be present (non-conflicting) +SELECT (position('Line1-A' in :'body_a') > 0) AS has_a \gset +SELECT (position('Line3-B' in :'body_a') > 0) AS has_b \gset +SELECT (position('Line5-C' in :'body_a') > 0) AS has_c \gset + +\if :has_a +\echo [PASS] (:testid) 3-Site: Site A edit present +\else +\echo [FAIL] (:testid) 3-Site: Site A edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :has_b +\echo [PASS] (:testid) 3-Site: Site B edit present +\else +\echo [FAIL] (:testid) 3-Site: Site B edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :has_c +\echo [PASS] (:testid) 3-Site: Site C edit present +\else +\echo [FAIL] (:testid) 3-Site: Site C edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Custom delimiter sync roundtrip +-- Uses paragraph delimiter (double newline), edits, syncs +-- ============================================================ +\connect cloudsync_block_r4_a +DROP TABLE IF EXISTS para_docs; +CREATE TABLE para_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('para_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('para_docs', 'body', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_column('para_docs', 'body', 'delimiter', E'\n\n') AS _sd \gset + +\connect cloudsync_block_r4_b +DROP TABLE IF EXISTS para_docs; +CREATE TABLE para_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('para_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('para_docs', 'body', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_column('para_docs', 'body', 'delimiter', E'\n\n') AS _sd \gset + +\connect cloudsync_block_r4_a +INSERT INTO para_docs (id, body) VALUES ('doc1', E'First paragraph.\n\nSecond paragraph.\n\nThird paragraph.'); + +SELECT count(*) AS pd_blocks FROM para_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'pd_blocks'::int = 3) AS pd_blk_ok \gset +\if :pd_blk_ok +\echo [PASS] (:testid) CustomDelimSync: 3 paragraph blocks +\else +\echo [FAIL] (:testid) CustomDelimSync: expected 3 blocks, got :pd_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_pd +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'para_docs' \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_pd', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('para_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'First paragraph.\n\nSecond paragraph.\n\nThird paragraph.') AS pd_sync_ok FROM para_docs WHERE id = 'doc1' \gset +\if :pd_sync_ok +\echo [PASS] (:testid) CustomDelimSync: body matches on B +\else +\echo [FAIL] (:testid) CustomDelimSync: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit paragraph 2 on B, sync back +\connect cloudsync_block_r4_b +UPDATE para_docs SET body = E'First paragraph.\n\nEdited second paragraph.\n\nThird paragraph.' WHERE id = 'doc1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_pd_r +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'para_docs' \gset + +\connect cloudsync_block_r4_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_pd_r', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('para_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'First paragraph.\n\nEdited second paragraph.\n\nThird paragraph.') AS pd_rev_ok FROM para_docs WHERE id = 'doc1' \gset +\if :pd_rev_ok +\echo [PASS] (:testid) CustomDelimSync: reverse sync matches +\else +\echo [FAIL] (:testid) CustomDelimSync: reverse sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Block column + regular LWW column — mixed update +-- Single UPDATE changes both block col and regular col +-- ============================================================ +\connect cloudsync_block_r4_a +DROP TABLE IF EXISTS mixed_docs; +CREATE TABLE mixed_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT, title TEXT); +SELECT cloudsync_init('mixed_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('mixed_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r4_b +DROP TABLE IF EXISTS mixed_docs; +CREATE TABLE mixed_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT, title TEXT); +SELECT cloudsync_init('mixed_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('mixed_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r4_a +INSERT INTO mixed_docs (id, body, title) VALUES ('doc1', E'Body-Line1\nBody-Line2', 'Original Title'); + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_mix_i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'mixed_docs' \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_mix_i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('mixed_docs', 'body', 'doc1') AS _mat \gset + +-- Update BOTH columns simultaneously on A +\connect cloudsync_block_r4_a +UPDATE mixed_docs SET body = E'Body-Edited1\nBody-Line2', title = 'New Title' WHERE id = 'doc1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_mix_u +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'mixed_docs' \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_mix_u', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('mixed_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Body-Edited1\nBody-Line2') AS mix_body_ok FROM mixed_docs WHERE id = 'doc1' \gset +\if :mix_body_ok +\echo [PASS] (:testid) MixedUpdate: block column body matches +\else +\echo [FAIL] (:testid) MixedUpdate: block column body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (title = 'New Title') AS mix_title_ok FROM mixed_docs WHERE id = 'doc1' \gset +\if :mix_title_ok +\echo [PASS] (:testid) MixedUpdate: regular column title matches +\else +\echo [FAIL] (:testid) MixedUpdate: regular column title mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_block_r4_a; +DROP DATABASE IF EXISTS cloudsync_block_r4_b; +DROP DATABASE IF EXISTS cloudsync_block_r4_c; +DROP DATABASE IF EXISTS cloudsync_block_3s_a; +DROP DATABASE IF EXISTS cloudsync_block_3s_b; +DROP DATABASE IF EXISTS cloudsync_block_3s_c; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/38_block_lww_round5.sql b/test/postgresql/38_block_lww_round5.sql new file mode 100644 index 00000000..93b6a16f --- /dev/null +++ b/test/postgresql/38_block_lww_round5.sql @@ -0,0 +1,433 @@ +-- 'Block-level LWW round 5: large blocks, payload idempotency composite PK, init with existing data, drop/re-add block config, delimiter-in-content' + +\set testid '38' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_r5_a; +DROP DATABASE IF EXISTS cloudsync_block_r5_b; +CREATE DATABASE cloudsync_block_r5_a; +CREATE DATABASE cloudsync_block_r5_b; + +-- ============================================================ +-- Test 7: Large number of blocks (200+ lines) +-- Verify diff and materialize work correctly at scale +-- ============================================================ +\connect cloudsync_block_r5_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS big_docs; +CREATE TABLE big_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('big_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('big_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS big_docs; +CREATE TABLE big_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('big_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('big_docs', 'body', 'algo', 'block') AS _sc \gset + +-- Generate 250-line text +\connect cloudsync_block_r5_a +INSERT INTO big_docs (id, body) +SELECT 'doc1', string_agg('Line-' || gs::text, E'\n' ORDER BY gs) +FROM generate_series(1, 250) gs; + +SELECT count(*) AS big_blocks FROM big_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'big_blocks'::int = 250) AS big_blk_ok \gset +\if :big_blk_ok +\echo [PASS] (:testid) LargeBlocks: 250 blocks created +\else +\echo [FAIL] (:testid) LargeBlocks: expected 250 blocks, got :big_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit a few lines scattered through the document +UPDATE big_docs SET body = ( + SELECT string_agg( + CASE + WHEN gs = 50 THEN 'EDITED-50' + WHEN gs = 150 THEN 'EDITED-150' + WHEN gs = 200 THEN 'EDITED-200' + ELSE 'Line-' || gs::text + END, + E'\n' ORDER BY gs + ) FROM generate_series(1, 250) gs +) WHERE id = 'doc1'; + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_big +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'big_docs' \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_big', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('big_docs', 'body', 'doc1') AS _mat \gset + +-- Verify edited lines are present +SELECT (position('EDITED-50' in body) > 0) AS big_e50 FROM big_docs WHERE id = 'doc1' \gset +SELECT (position('EDITED-150' in body) > 0) AS big_e150 FROM big_docs WHERE id = 'doc1' \gset +SELECT (position('EDITED-200' in body) > 0) AS big_e200 FROM big_docs WHERE id = 'doc1' \gset + +\if :big_e50 +\echo [PASS] (:testid) LargeBlocks: EDITED-50 present +\else +\echo [FAIL] (:testid) LargeBlocks: EDITED-50 missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :big_e150 +\echo [PASS] (:testid) LargeBlocks: EDITED-150 present +\else +\echo [FAIL] (:testid) LargeBlocks: EDITED-150 missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :big_e200 +\echo [PASS] (:testid) LargeBlocks: EDITED-200 present +\else +\echo [FAIL] (:testid) LargeBlocks: EDITED-200 missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify block count still 250 (edits don't change count) +SELECT count(*) AS big_blocks2 FROM big_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'big_blocks2'::int = 250) AS big_cnt_ok \gset +\if :big_cnt_ok +\echo [PASS] (:testid) LargeBlocks: block count stable after sync +\else +\echo [FAIL] (:testid) LargeBlocks: expected 250 blocks after sync, got :big_blocks2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 8: Payload idempotency with composite PK +-- Apply same payload twice, verify no duplication or corruption +-- ============================================================ +\connect cloudsync_block_r5_a +DROP TABLE IF EXISTS idem_docs; +CREATE TABLE idem_docs (owner TEXT NOT NULL, seq INTEGER NOT NULL, body TEXT, PRIMARY KEY(owner, seq)); +SELECT cloudsync_init('idem_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('idem_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r5_b +DROP TABLE IF EXISTS idem_docs; +CREATE TABLE idem_docs (owner TEXT NOT NULL, seq INTEGER NOT NULL, body TEXT, PRIMARY KEY(owner, seq)); +SELECT cloudsync_init('idem_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('idem_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r5_a +INSERT INTO idem_docs (owner, seq, body) VALUES ('bob', 1, E'Idem-Line1\nIdem-Line2\nIdem-Line3'); +UPDATE idem_docs SET body = E'Idem-Line1\nIdem-Edited\nIdem-Line3' WHERE owner = 'bob' AND seq = 1; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_idem +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'idem_docs' \gset + +-- Apply on B — first time +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_idem', 'hex')) AS _app1 \gset +SELECT cloudsync_text_materialize('idem_docs', 'body', 'bob', 1) AS _mat1 \gset + +SELECT (body = E'Idem-Line1\nIdem-Edited\nIdem-Line3') AS idem1_ok FROM idem_docs WHERE owner = 'bob' AND seq = 1 \gset +\if :idem1_ok +\echo [PASS] (:testid) Idempotent: first apply correct +\else +\echo [FAIL] (:testid) Idempotent: first apply mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT count(*) AS idem_meta1 FROM idem_docs_cloudsync WHERE pk = cloudsync_pk_encode('bob', 1) \gset + +-- Apply SAME payload again — second time (idempotent) +SELECT cloudsync_payload_apply(decode(:'payload_idem', 'hex')) AS _app2 \gset +SELECT cloudsync_text_materialize('idem_docs', 'body', 'bob', 1) AS _mat2 \gset + +SELECT (body = E'Idem-Line1\nIdem-Edited\nIdem-Line3') AS idem2_ok FROM idem_docs WHERE owner = 'bob' AND seq = 1 \gset +\if :idem2_ok +\echo [PASS] (:testid) Idempotent: second apply still correct +\else +\echo [FAIL] (:testid) Idempotent: body corrupted after double apply +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Metadata count should not change +SELECT count(*) AS idem_meta2 FROM idem_docs_cloudsync WHERE pk = cloudsync_pk_encode('bob', 1) \gset +SELECT (:'idem_meta1' = :'idem_meta2') AS idem_meta_ok \gset +\if :idem_meta_ok +\echo [PASS] (:testid) Idempotent: metadata count unchanged after double apply +\else +\echo [FAIL] (:testid) Idempotent: metadata count changed (:idem_meta1 vs :idem_meta2) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 9: Init with pre-existing data, then enable block column +-- Table has rows before cloudsync_set_column algo=block +-- ============================================================ +\connect cloudsync_block_r5_a +DROP TABLE IF EXISTS predata; +CREATE TABLE predata (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('predata', 'CLS', 1) AS _init \gset + +-- Insert rows BEFORE enabling block algorithm +INSERT INTO predata (id, body) VALUES ('pre1', E'Pre-Line1\nPre-Line2'); +INSERT INTO predata (id, body) VALUES ('pre2', E'Pre-Alpha\nPre-Beta\nPre-Gamma'); + +-- Now enable block on the column +SELECT cloudsync_set_column('predata', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r5_b +DROP TABLE IF EXISTS predata; +CREATE TABLE predata (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('predata', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('predata', 'body', 'algo', 'block') AS _sc \gset + +-- Update a pre-existing row on A to trigger block creation +\connect cloudsync_block_r5_a +UPDATE predata SET body = E'Pre-Line1\nPre-Edited2' WHERE id = 'pre1'; + +SELECT count(*) AS pre_blocks FROM predata_cloudsync_blocks WHERE pk = cloudsync_pk_encode('pre1') \gset +SELECT (:'pre_blocks'::int >= 2) AS pre_blk_ok \gset +\if :pre_blk_ok +\echo [PASS] (:testid) PreExisting: blocks created after update +\else +\echo [FAIL] (:testid) PreExisting: expected >= 2 blocks, got :pre_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync to B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_pre +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'predata' \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_pre', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('predata', 'body', 'pre1') AS _mat \gset + +SELECT (body = E'Pre-Line1\nPre-Edited2') AS pre_sync_ok FROM predata WHERE id = 'pre1' \gset +\if :pre_sync_ok +\echo [PASS] (:testid) PreExisting: synced body matches after late block enable +\else +\echo [FAIL] (:testid) PreExisting: synced body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pre2 should also sync (as regular LWW or with insert sentinel) +SELECT (count(*) = 1) AS pre2_exists FROM predata WHERE id = 'pre2' \gset +\if :pre2_exists +\echo [PASS] (:testid) PreExisting: pre2 row synced +\else +\echo [FAIL] (:testid) PreExisting: pre2 row missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 10: Remove block algo then re-add +-- ============================================================ +\connect cloudsync_block_r5_a +DROP TABLE IF EXISTS toggle_docs; +CREATE TABLE toggle_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('toggle_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('toggle_docs', 'body', 'algo', 'block') AS _sc1 \gset + +\connect cloudsync_block_r5_b +DROP TABLE IF EXISTS toggle_docs; +CREATE TABLE toggle_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('toggle_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('toggle_docs', 'body', 'algo', 'block') AS _sc1 \gset + +-- Insert with blocks on A +\connect cloudsync_block_r5_a +INSERT INTO toggle_docs (id, body) VALUES ('doc1', E'Toggle-Line1\nToggle-Line2'); + +SELECT count(*) AS tog_blocks1 FROM toggle_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'tog_blocks1'::int = 2) AS tog_blk1_ok \gset +\if :tog_blk1_ok +\echo [PASS] (:testid) Toggle: blocks created initially +\else +\echo [FAIL] (:testid) Toggle: expected 2 blocks initially, got :tog_blocks1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Remove block algo (set to default LWW) +SELECT cloudsync_set_column('toggle_docs', 'body', 'algo', 'lww') AS _sc2 \gset + +-- Update while in LWW mode — should NOT create new blocks +UPDATE toggle_docs SET body = E'Toggle-LWW-Updated' WHERE id = 'doc1'; + +-- Re-enable block algo +SELECT cloudsync_set_column('toggle_docs', 'body', 'algo', 'block') AS _sc3 \gset + +-- Update with blocks re-enabled +UPDATE toggle_docs SET body = E'Toggle-Block-Again1\nToggle-Block-Again2\nToggle-Block-Again3' WHERE id = 'doc1'; + +SELECT count(*) AS tog_blocks2 FROM toggle_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'tog_blocks2'::int = 3) AS tog_blk2_ok \gset +\if :tog_blk2_ok +\echo [PASS] (:testid) Toggle: 3 blocks after re-enable +\else +\echo [FAIL] (:testid) Toggle: expected 3 blocks after re-enable, got :tog_blocks2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync to B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_tog +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'toggle_docs' \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_tog', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('toggle_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Toggle-Block-Again1\nToggle-Block-Again2\nToggle-Block-Again3') AS tog_sync_ok FROM toggle_docs WHERE id = 'doc1' \gset +\if :tog_sync_ok +\echo [PASS] (:testid) Toggle: body matches after re-enable and sync +\else +\echo [FAIL] (:testid) Toggle: body mismatch after re-enable and sync +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 11: Text containing the delimiter character as content +-- Default delimiter is \n — content has no real structure, just embedded newlines +-- ============================================================ +\connect cloudsync_block_r5_a +DROP TABLE IF EXISTS delim_docs; +CREATE TABLE delim_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('delim_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('delim_docs', 'body', 'algo', 'block') AS _sc \gset +-- Use paragraph delimiter (double newline) +SELECT cloudsync_set_column('delim_docs', 'body', 'delimiter', E'\n\n') AS _sd \gset + +\connect cloudsync_block_r5_b +DROP TABLE IF EXISTS delim_docs; +CREATE TABLE delim_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('delim_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('delim_docs', 'body', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_column('delim_docs', 'body', 'delimiter', E'\n\n') AS _sd \gset + +-- Content with single newlines inside paragraphs (not delimiters) +\connect cloudsync_block_r5_a +INSERT INTO delim_docs (id, body) VALUES ('doc1', E'Paragraph one\nstill paragraph one.\n\nParagraph two\nstill para two.\n\nParagraph three.'); + +-- Should be 3 blocks (split by double newline) +SELECT count(*) AS dc_blocks FROM delim_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'dc_blocks'::int = 3) AS dc_blk_ok \gset +\if :dc_blk_ok +\echo [PASS] (:testid) DelimContent: 3 paragraph blocks (single newlines inside) +\else +\echo [FAIL] (:testid) DelimContent: expected 3 blocks, got :dc_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_dc +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'delim_docs' \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_dc', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('delim_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Paragraph one\nstill paragraph one.\n\nParagraph two\nstill para two.\n\nParagraph three.') AS dc_sync_ok FROM delim_docs WHERE id = 'doc1' \gset +\if :dc_sync_ok +\echo [PASS] (:testid) DelimContent: body matches on B (embedded newlines preserved) +\else +\echo [FAIL] (:testid) DelimContent: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit paragraph 2 on B (change only the second paragraph), sync back +\connect cloudsync_block_r5_b +UPDATE delim_docs SET body = E'Paragraph one\nstill paragraph one.\n\nEdited paragraph two.\n\nParagraph three.' WHERE id = 'doc1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_dc_r +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'delim_docs' \gset + +\connect cloudsync_block_r5_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_dc_r', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('delim_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Paragraph one\nstill paragraph one.\n\nEdited paragraph two.\n\nParagraph three.') AS dc_rev_ok FROM delim_docs WHERE id = 'doc1' \gset +\if :dc_rev_ok +\echo [PASS] (:testid) DelimContent: reverse sync matches (paragraph edit) +\else +\echo [FAIL] (:testid) DelimContent: reverse sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Concurrent edit: A edits para 1, B edits para 3 +\connect cloudsync_block_r5_a +UPDATE delim_docs SET body = E'Edited para one by A.\n\nEdited paragraph two.\n\nParagraph three.' WHERE id = 'doc1'; + +\connect cloudsync_block_r5_b +UPDATE delim_docs SET body = E'Paragraph one\nstill paragraph one.\n\nEdited paragraph two.\n\nEdited para three by B.' WHERE id = 'doc1'; + +\connect cloudsync_block_r5_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_dc_a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'delim_docs' \gset + +\connect cloudsync_block_r5_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_dc_b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'delim_docs' \gset + +-- Apply cross +\connect cloudsync_block_r5_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_dc_b', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('delim_docs', 'body', 'doc1') AS _mat \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_dc_a', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('delim_docs', 'body', 'doc1') AS _mat \gset + +-- Both should converge and both edits should be present +\connect cloudsync_block_r5_a +SELECT md5(body) AS dc_md5_a FROM delim_docs WHERE id = 'doc1' \gset +\connect cloudsync_block_r5_b +SELECT md5(body) AS dc_md5_b FROM delim_docs WHERE id = 'doc1' \gset + +SELECT (:'dc_md5_a' = :'dc_md5_b') AS dc_converge \gset +\if :dc_converge +\echo [PASS] (:testid) DelimContent: concurrent paragraph edits converge +\else +\echo [FAIL] (:testid) DelimContent: concurrent paragraph edits diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +\connect cloudsync_block_r5_a +SELECT (position('Edited para one by A.' in body) > 0) AS dc_has_a FROM delim_docs WHERE id = 'doc1' \gset +SELECT (position('Edited para three by B.' in body) > 0) AS dc_has_b FROM delim_docs WHERE id = 'doc1' \gset + +\if :dc_has_a +\echo [PASS] (:testid) DelimContent: site A paragraph edit present +\else +\echo [FAIL] (:testid) DelimContent: site A paragraph edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :dc_has_b +\echo [PASS] (:testid) DelimContent: site B paragraph edit present +\else +\echo [FAIL] (:testid) DelimContent: site B paragraph edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_block_r5_a; +DROP DATABASE IF EXISTS cloudsync_block_r5_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/39_concurrent_write_apply.sql b/test/postgresql/39_concurrent_write_apply.sql new file mode 100644 index 00000000..d84397ed --- /dev/null +++ b/test/postgresql/39_concurrent_write_apply.sql @@ -0,0 +1,179 @@ +-- 'Test concurrent write lock during payload apply' +-- NOTE: The lock-contention portion requires dblink with table access. +-- On environments where dblink cannot lock the table (e.g. Supabase), +-- the lock test is skipped and only apply + consistency are verified. + +\set testid '39' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_39_a; +DROP DATABASE IF EXISTS cloudsync_test_39_b; +CREATE DATABASE cloudsync_test_39_a; +CREATE DATABASE cloudsync_test_39_b; + +-- Setup db_a +\connect cloudsync_test_39_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE concurrent_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('concurrent_tbl', 'CLS', 1) AS _init_a \gset + +-- Setup db_b +\connect cloudsync_test_39_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE concurrent_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('concurrent_tbl', 'CLS', 1) AS _init_b \gset + +-- Insert row1 on db_a and sync to db_b +\connect cloudsync_test_39_a +INSERT INTO concurrent_tbl VALUES ('row1', 'val_a'); + +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_init, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_init_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, + db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_39_b +\if :payload_init_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_init', 3), 'hex')) AS _apply_init \gset +\endif + +-- Update row1 on db_a +\connect cloudsync_test_39_a +UPDATE concurrent_tbl SET val = 'val_a_updated' WHERE id = 'row1'; + +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_upd, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_upd_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, + db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Try to set up dblink and acquire a table lock +\connect cloudsync_test_39_b +CREATE EXTENSION IF NOT EXISTS dblink; + +SELECT dblink_connect('locker', 'dbname=cloudsync_test_39_b') AS _conn \gset +SELECT dblink_exec('locker', 'BEGIN') AS _begin \gset + +-- Try to acquire EXCLUSIVE lock — if this fails (e.g. permission denied on +-- Supabase), _lock won't be set and we skip the lock-contention test +\unset _lock +SELECT dblink_exec('locker', 'LOCK TABLE concurrent_tbl IN EXCLUSIVE MODE') AS _lock \gset + +\if :{?_lock} +-- ===== Lock acquired — run lock-contention test ===== + +BEGIN; +\set ON_ERROR_ROLLBACK on +SET LOCAL lock_timeout = '500ms'; + +\if :payload_upd_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_upd', 3), 'hex')) AS _blocked_apply \gset +\endif + +COMMIT; +\set ON_ERROR_ROLLBACK off + +-- row1 should still have the OLD value because the apply was blocked +SELECT val AS row1_val_check FROM concurrent_tbl WHERE id = 'row1' \gset +SELECT (:'row1_val_check' = 'val_a') AS blocked_ok \gset +\if :blocked_ok +\echo [PASS] (:testid) Apply correctly blocked by concurrent table lock +\else +\echo [FAIL] (:testid) Expected val_a (blocked), got :'row1_val_check' +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Release the table lock +SELECT dblink_exec('locker', 'COMMIT') AS _release \gset +SELECT dblink_disconnect('locker') AS _disconn \gset + +-- Retry apply — should succeed now +\if :payload_upd_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_upd', 3), 'hex')) AS _apply_retry \gset +\endif + +SELECT val AS row1_val FROM concurrent_tbl WHERE id = 'row1' \gset +SELECT (:'row1_val' = 'val_a_updated') AS retry_ok \gset +\if :retry_ok +\echo [PASS] (:testid) Apply succeeded after lock released +\else +\echo [FAIL] (:testid) Apply after unlock - expected val_a_updated, got :'row1_val' +SELECT (:fail::int + 1) AS fail \gset +\endif + +\else +-- ===== Lock failed — skip contention test, apply directly ===== +\echo [SKIP] (:testid) Lock-contention test skipped (dblink cannot lock table) + +-- Clean up the dblink connection (transaction is aborted) +SELECT dblink_exec('locker', 'ROLLBACK') AS _rollback \gset +SELECT dblink_disconnect('locker') AS _disconn \gset + +\if :payload_upd_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_upd', 3), 'hex')) AS _apply_direct \gset +\endif + +SELECT val AS row1_val FROM concurrent_tbl WHERE id = 'row1' \gset +SELECT (:'row1_val' = 'val_a_updated') AS direct_ok \gset +\if :direct_ok +\echo [PASS] (:testid) Apply succeeded (no lock contention) +\else +\echo [FAIL] (:testid) Apply failed - expected val_a_updated, got :'row1_val' +SELECT (:fail::int + 1) AS fail \gset +\endif + +\endif + +-- Full cross-sync for consistency +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_final, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_final_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, + db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_39_a +\if :payload_b_final_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_final', 3), 'hex')) AS _apply_final \gset +\endif + +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS hash_a +FROM concurrent_tbl \gset + +\connect cloudsync_test_39_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS hash_b +FROM concurrent_tbl \gset + +SELECT (:'hash_a' = :'hash_b') AS consistency_ok \gset +\if :consistency_ok +\echo [PASS] (:testid) Cross-database consistency verified +\else +\echo [FAIL] (:testid) Consistency failed (hash_a=:'hash_a' hash_b=:'hash_b') +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_39_a; +DROP DATABASE IF EXISTS cloudsync_test_39_b; +\endif diff --git a/test/postgresql/40_unsupported_algorithms.sql b/test/postgresql/40_unsupported_algorithms.sql new file mode 100644 index 00000000..fcbf0f60 --- /dev/null +++ b/test/postgresql/40_unsupported_algorithms.sql @@ -0,0 +1,80 @@ +-- Test unsupported CRDT algorithms (DWS, AWS) +-- Verifies that cloudsync_init rejects DWS and AWS with clear errors +-- and that no metadata tables are created. + +\set testid '40' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_40; +CREATE DATABASE cloudsync_test_40; + +\connect cloudsync_test_40 +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE test_dws (id TEXT PRIMARY KEY, val TEXT); +CREATE TABLE test_aws (id TEXT PRIMARY KEY, val TEXT); + +-- Test DWS rejection +DO $$ +BEGIN + PERFORM cloudsync_init('test_dws', 'dws', 1); + RAISE EXCEPTION 'cloudsync_init with dws should have failed'; +EXCEPTION WHEN OTHERS THEN + IF SQLERRM NOT LIKE '%not yet supported%' THEN + RAISE EXCEPTION 'Unexpected error for dws: %', SQLERRM; + END IF; +END $$; + +-- Verify no companion table was created for DWS +SELECT COUNT(*) = 0 AS no_dws_meta +FROM information_schema.tables +WHERE table_name = 'test_dws_cloudsync' \gset +\if :no_dws_meta +\echo [PASS] (:testid) DWS rejected - no metadata table created +\else +\echo [FAIL] (:testid) DWS metadata table should not exist +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test AWS rejection +DO $$ +BEGIN + PERFORM cloudsync_init('test_aws', 'aws', 1); + RAISE EXCEPTION 'cloudsync_init with aws should have failed'; +EXCEPTION WHEN OTHERS THEN + IF SQLERRM NOT LIKE '%not yet supported%' THEN + RAISE EXCEPTION 'Unexpected error for aws: %', SQLERRM; + END IF; +END $$; + +-- Verify no companion table was created for AWS +SELECT COUNT(*) = 0 AS no_aws_meta +FROM information_schema.tables +WHERE table_name = 'test_aws_cloudsync' \gset +\if :no_aws_meta +\echo [PASS] (:testid) AWS rejected - no metadata table created +\else +\echo [FAIL] (:testid) AWS metadata table should not exist +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify CLS still works (sanity check) +SELECT cloudsync_init('test_dws', 'cls', 1) AS _init_cls \gset +SELECT COUNT(*) = 1 AS cls_meta_ok +FROM information_schema.tables +WHERE table_name = 'test_dws_cloudsync' \gset +\if :cls_meta_ok +\echo [PASS] (:testid) CLS init works after DWS/AWS rejection +\else +\echo [FAIL] (:testid) CLS init should work +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_40; +\endif diff --git a/test/postgresql/41_corrupted_payload.sql b/test/postgresql/41_corrupted_payload.sql new file mode 100644 index 00000000..235c79dc --- /dev/null +++ b/test/postgresql/41_corrupted_payload.sql @@ -0,0 +1,130 @@ +-- Test corrupted payload handling +-- Verifies that cloudsync_payload_apply rejects corrupted payloads +-- without crashing or corrupting state. + +\set testid '41' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_41_src; +DROP DATABASE IF EXISTS cloudsync_test_41_dst; +CREATE DATABASE cloudsync_test_41_src; +CREATE DATABASE cloudsync_test_41_dst; + +-- Setup source database with data +\connect cloudsync_test_41_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_src \gset +INSERT INTO test_tbl VALUES ('id1', 'value1'); +INSERT INTO test_tbl VALUES ('id2', 'value2'); + +-- Get a valid payload +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS valid_payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Setup destination database +\connect cloudsync_test_41_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_dst \gset + +-- Record initial state +SELECT COUNT(*) AS initial_count FROM test_tbl \gset + +-- Test 1: Empty blob (zero bytes) +DO $$ +BEGIN + PERFORM cloudsync_payload_apply(''::bytea); + -- If it returns without error with 0 rows, that's also acceptable +EXCEPTION WHEN OTHERS THEN + -- Expected: error on empty payload + NULL; +END $$; + +SELECT COUNT(*) AS count_after_empty FROM test_tbl \gset +SELECT (:count_after_empty::int = :initial_count::int) AS empty_blob_ok \gset +\if :empty_blob_ok +\echo [PASS] (:testid) Empty blob rejected - table unchanged +\else +\echo [FAIL] (:testid) Empty blob corrupted table state +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: Random garbage bytes +DO $$ +BEGIN + PERFORM cloudsync_payload_apply(decode('deadbeefcafebabe0102030405060708', 'hex')); +EXCEPTION WHEN OTHERS THEN + -- Expected: error on garbage payload + NULL; +END $$; + +SELECT COUNT(*) AS count_after_garbage FROM test_tbl \gset +SELECT (:count_after_garbage::int = :initial_count::int) AS garbage_ok \gset +\if :garbage_ok +\echo [PASS] (:testid) Garbage bytes rejected - table unchanged +\else +\echo [FAIL] (:testid) Garbage bytes corrupted table state +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Truncated payload (first 10 bytes of valid payload) +-- Build truncated hex at top level using psql variable interpolation +SELECT substr(:'valid_payload_hex', 1, 20) AS truncated_hex \gset +-- Expected: error on truncated payload — locally disable ON_ERROR_STOP +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'truncated_hex', 'hex')) AS _apply_truncated \gset +\set ON_ERROR_STOP on + +SELECT COUNT(*) AS count_after_truncated FROM test_tbl \gset +SELECT (:count_after_truncated::int = :initial_count::int) AS truncated_ok \gset +\if :truncated_ok +\echo [PASS] (:testid) Truncated payload rejected - table unchanged +\else +\echo [FAIL] (:testid) Truncated payload corrupted table state +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 4: Valid payload with flipped byte in the middle +-- Compute corrupted payload at top level: flip one byte via XOR with FF +SELECT + substr(:'valid_payload_hex', 1, length(:'valid_payload_hex') / 2 - 1) + || lpad(to_hex(get_byte(decode(substr(:'valid_payload_hex', length(:'valid_payload_hex') / 2, 2), 'hex'), 0) # 255), 2, '0') + || substr(:'valid_payload_hex', length(:'valid_payload_hex') / 2 + 2) + AS corrupted_hex \gset +-- Expected: error on corrupted payload — locally disable ON_ERROR_STOP +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'corrupted_hex', 'hex')) AS _apply_corrupted \gset +\set ON_ERROR_STOP on + +SELECT COUNT(*) AS count_after_flipped FROM test_tbl \gset +SELECT (:count_after_flipped::int = :initial_count::int) AS flipped_ok \gset +\if :flipped_ok +\echo [PASS] (:testid) Flipped-byte payload rejected - table unchanged +\else +\echo [FAIL] (:testid) Flipped-byte payload corrupted table state +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 5: Now apply the VALID payload to confirm it still works +SELECT cloudsync_payload_apply(decode(:'valid_payload_hex', 'hex')) AS valid_apply \gset +SELECT COUNT(*) AS count_after_valid FROM test_tbl \gset +SELECT (:count_after_valid::int = 2) AS valid_ok \gset +\if :valid_ok +\echo [PASS] (:testid) Valid payload applied successfully after corrupted attempts +\else +\echo [FAIL] (:testid) Valid payload failed after corrupted attempts - count: :count_after_valid +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_41_src; +DROP DATABASE IF EXISTS cloudsync_test_41_dst; +\endif diff --git a/test/postgresql/42_payload_idempotency.sql b/test/postgresql/42_payload_idempotency.sql new file mode 100644 index 00000000..86b62623 --- /dev/null +++ b/test/postgresql/42_payload_idempotency.sql @@ -0,0 +1,88 @@ +-- Test payload apply idempotency +-- Applying the same payload multiple times must produce identical results. + +\set testid '42' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_42_src; +DROP DATABASE IF EXISTS cloudsync_test_42_dst; +CREATE DATABASE cloudsync_test_42_src; +CREATE DATABASE cloudsync_test_42_dst; + +-- Setup source with data +\connect cloudsync_test_42_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT, num INTEGER); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_src \gset +INSERT INTO test_tbl VALUES ('id1', 'hello', 10); +INSERT INTO test_tbl VALUES ('id2', 'world', 20); +UPDATE test_tbl SET val = 'hello_updated' WHERE id = 'id1'; + +-- Encode payload +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Setup destination +\connect cloudsync_test_42_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT, num INTEGER); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_dst \gset + +-- Apply #1 +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS apply_1 \gset +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, '') || ':' || COALESCE(num::text, ''), ',' ORDER BY id), '')) AS hash_1 +FROM test_tbl \gset +SELECT COUNT(*) AS count_1 FROM test_tbl \gset + +-- Apply #2 +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS apply_2 \gset +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, '') || ':' || COALESCE(num::text, ''), ',' ORDER BY id), '')) AS hash_2 +FROM test_tbl \gset +SELECT COUNT(*) AS count_2 FROM test_tbl \gset + +-- Apply #3 +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS apply_3 \gset +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, '') || ':' || COALESCE(num::text, ''), ',' ORDER BY id), '')) AS hash_3 +FROM test_tbl \gset +SELECT COUNT(*) AS count_3 FROM test_tbl \gset + +-- Verify row count stays constant +SELECT (:count_1::int = :count_2::int AND :count_2::int = :count_3::int) AS count_stable \gset +\if :count_stable +\echo [PASS] (:testid) Row count stable across 3 applies (:count_1 rows) +\else +\echo [FAIL] (:testid) Row count changed: :count_1 -> :count_2 -> :count_3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify data hash is identical after each apply +SELECT (:'hash_1' = :'hash_2' AND :'hash_2' = :'hash_3') AS hash_stable \gset +\if :hash_stable +\echo [PASS] (:testid) Data hash identical across 3 applies +\else +\echo [FAIL] (:testid) Data hash changed: :hash_1 -> :hash_2 -> :hash_3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify data values are correct +SELECT COUNT(*) = 1 AS data_ok +FROM test_tbl +WHERE id = 'id1' AND val = 'hello_updated' AND num = 10 \gset +\if :data_ok +\echo [PASS] (:testid) Data values correct after idempotent applies +\else +\echo [FAIL] (:testid) Data values incorrect +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_42_src; +DROP DATABASE IF EXISTS cloudsync_test_42_dst; +\endif diff --git a/test/postgresql/43_delete_resurrect_ordering.sql b/test/postgresql/43_delete_resurrect_ordering.sql new file mode 100644 index 00000000..a95f27a3 --- /dev/null +++ b/test/postgresql/43_delete_resurrect_ordering.sql @@ -0,0 +1,147 @@ +-- Test delete/resurrect with out-of-order payload delivery +-- Verifies CRDT causal length parity handles resurrection correctly +-- even when payloads arrive in non-causal order. + +\set testid '43' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_43_a; +DROP DATABASE IF EXISTS cloudsync_test_43_b; +DROP DATABASE IF EXISTS cloudsync_test_43_c; +CREATE DATABASE cloudsync_test_43_a; +CREATE DATABASE cloudsync_test_43_b; +CREATE DATABASE cloudsync_test_43_c; + +-- Setup all three databases +\connect cloudsync_test_43_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_a \gset + +\connect cloudsync_test_43_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_b \gset + +\connect cloudsync_test_43_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_c \gset + +-- Round 1: A inserts row, sync to all +\connect cloudsync_test_43_a +INSERT INTO test_tbl VALUES ('row1', 'original'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_43_b +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply \gset +\endif + +\connect cloudsync_test_43_c +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply \gset +\endif + +-- Round 2: A deletes row (CL goes 1->2) +\connect cloudsync_test_43_a +DELETE FROM test_tbl WHERE id = 'row1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Sync delete to B +\connect cloudsync_test_43_b +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply \gset +\endif + +-- Round 3: B re-inserts (CL goes 2->3, resurrection) +\connect cloudsync_test_43_b +INSERT INTO test_tbl VALUES ('row1', 'resurrected_by_b'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- C receives payloads in REVERSE order: B's resurrection FIRST, then A's delete +\connect cloudsync_test_43_c +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_b \gset +\endif +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_a \gset +\endif + +-- A receives B's resurrection +\connect cloudsync_test_43_a +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_b \gset +\endif + +-- Final convergence check: all three should have row1 with 'resurrected_by_b' +\connect cloudsync_test_43_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS hash_a +FROM test_tbl \gset + +\connect cloudsync_test_43_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS hash_b +FROM test_tbl \gset + +\connect cloudsync_test_43_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS hash_c +FROM test_tbl \gset + +SELECT (:'hash_a' = :'hash_b' AND :'hash_b' = :'hash_c') AS all_converge \gset +\if :all_converge +\echo [PASS] (:testid) All 3 databases converge after out-of-order delete/resurrect +\else +\echo [FAIL] (:testid) Databases diverged - A: :hash_a, B: :hash_b, C: :hash_c +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify the row exists (resurrection won) +SELECT COUNT(*) = 1 AS row_exists +FROM test_tbl +WHERE id = 'row1' \gset +\if :row_exists +\echo [PASS] (:testid) Resurrected row exists on C (received out of order) +\else +\echo [FAIL] (:testid) Resurrected row missing on C +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_43_a; +DROP DATABASE IF EXISTS cloudsync_test_43_b; +DROP DATABASE IF EXISTS cloudsync_test_43_c; +\endif diff --git a/test/postgresql/44_large_composite_pk.sql b/test/postgresql/44_large_composite_pk.sql new file mode 100644 index 00000000..316b9f07 --- /dev/null +++ b/test/postgresql/44_large_composite_pk.sql @@ -0,0 +1,142 @@ +-- Test large composite primary key (5 columns) +-- Verifies pk_encode/pk_decode handles complex multi-column PKs correctly. + +\set testid '44' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_44_a; +DROP DATABASE IF EXISTS cloudsync_test_44_b; +CREATE DATABASE cloudsync_test_44_a; +CREATE DATABASE cloudsync_test_44_b; + +-- Setup Database A +\connect cloudsync_test_44_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE composite_pk_tbl ( + pk_text1 TEXT NOT NULL, + pk_int1 INTEGER NOT NULL, + pk_text2 TEXT NOT NULL, + pk_int2 INTEGER NOT NULL, + pk_text3 TEXT NOT NULL, + data_col TEXT, + num_col INTEGER, + PRIMARY KEY (pk_text1, pk_int1, pk_text2, pk_int2, pk_text3) +); + +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _init_a \gset + +INSERT INTO composite_pk_tbl VALUES ('alpha', 1, 'beta', 100, 'gamma', 'data_a1', 42); +INSERT INTO composite_pk_tbl VALUES ('alpha', 2, 'beta', 200, 'delta', 'data_a2', 84); +INSERT INTO composite_pk_tbl VALUES ('x', 999, 'y', -1, 'z', 'edge_case', 0); + +-- Setup Database B +\connect cloudsync_test_44_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE composite_pk_tbl ( + pk_text1 TEXT NOT NULL, + pk_int1 INTEGER NOT NULL, + pk_text2 TEXT NOT NULL, + pk_int2 INTEGER NOT NULL, + pk_text3 TEXT NOT NULL, + data_col TEXT, + num_col INTEGER, + PRIMARY KEY (pk_text1, pk_int1, pk_text2, pk_int2, pk_text3) +); + +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _init_b \gset + +INSERT INTO composite_pk_tbl VALUES ('alpha', 1, 'beta', 100, 'gamma', 'data_b1', 99); +INSERT INTO composite_pk_tbl VALUES ('foo', 3, 'bar', 300, 'baz', 'data_b2', 77); + +-- Encode and exchange payloads +\connect cloudsync_test_44_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _reinit \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_44_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _reinit \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_b +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply A -> B +SELECT cloudsync_payload_apply(decode(:'payload_a', 'hex')) AS apply_a_to_b \gset + +-- Apply B -> A +\connect cloudsync_test_44_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_b', 'hex')) AS apply_b_to_a \gset + +-- Update a row on A +UPDATE composite_pk_tbl SET data_col = 'updated_on_a' WHERE pk_text1 = 'foo' AND pk_int1 = 3 AND pk_text2 = 'bar' AND pk_int2 = 300 AND pk_text3 = 'baz'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_44_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_a2', 'hex')) AS apply_a2_to_b \gset + +-- Final hash comparison +SELECT md5(COALESCE(string_agg( + pk_text1 || ':' || pk_int1::text || ':' || pk_text2 || ':' || pk_int2::text || ':' || pk_text3 || ':' || + COALESCE(data_col, 'NULL') || ':' || COALESCE(num_col::text, 'NULL'), + '|' ORDER BY pk_text1, pk_int1, pk_text2, pk_int2, pk_text3 +), '')) AS hash_b FROM composite_pk_tbl \gset + +\connect cloudsync_test_44_a +\ir helper_psql_conn_setup.sql +SELECT md5(COALESCE(string_agg( + pk_text1 || ':' || pk_int1::text || ':' || pk_text2 || ':' || pk_int2::text || ':' || pk_text3 || ':' || + COALESCE(data_col, 'NULL') || ':' || COALESCE(num_col::text, 'NULL'), + '|' ORDER BY pk_text1, pk_int1, pk_text2, pk_int2, pk_text3 +), '')) AS hash_a FROM composite_pk_tbl \gset + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Large composite PK (5 cols) roundtrip and update +\else +\echo [FAIL] (:testid) Hash mismatch - A: :hash_a, B: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row count +SELECT COUNT(*) AS row_count FROM composite_pk_tbl \gset +SELECT (:row_count::int = 4) AS count_ok \gset +\if :count_ok +\echo [PASS] (:testid) Row count correct (4 rows with 5-col composite PK) +\else +\echo [FAIL] (:testid) Expected 4 rows, got :row_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify update propagated +SELECT COUNT(*) = 1 AS update_ok +FROM composite_pk_tbl +WHERE pk_text1 = 'foo' AND pk_int1 = 3 AND data_col = 'updated_on_a' \gset +\if :update_ok +\echo [PASS] (:testid) Update propagated correctly for composite PK row +\else +\echo [FAIL] (:testid) Update not propagated +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_44_a; +DROP DATABASE IF EXISTS cloudsync_test_44_b; +\endif diff --git a/test/postgresql/45_pg_specific_types.sql b/test/postgresql/45_pg_specific_types.sql new file mode 100644 index 00000000..70ff7018 --- /dev/null +++ b/test/postgresql/45_pg_specific_types.sql @@ -0,0 +1,176 @@ +-- Test PostgreSQL-specific type roundtrips +-- Covers JSONB, TIMESTAMPTZ, NUMERIC with precision, BYTEA + +\set testid '45' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_45_a; +DROP DATABASE IF EXISTS cloudsync_test_45_b; +CREATE DATABASE cloudsync_test_45_a; +CREATE DATABASE cloudsync_test_45_b; + +-- Setup Database A +\connect cloudsync_test_45_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE typed_tbl ( + id TEXT PRIMARY KEY, + json_col JSONB, + ts_col TIMESTAMPTZ, + num_col NUMERIC(18, 6), + bin_col BYTEA +); + +SELECT cloudsync_init('typed_tbl', 'CLS', 1) AS _init_a \gset + +INSERT INTO typed_tbl VALUES ( + 'row1', + '{"key": "value", "nested": {"arr": [1, 2, 3]}}', + '2025-01-15 10:30:00+00', + 123456.789012, + '\x48656c6c6f' +); + +INSERT INTO typed_tbl VALUES ( + 'row2', + '[1, "two", null, true, false]', + '2024-06-30 23:59:59.999999+05:30', + -999999.123456, + '\xdeadbeef' +); + +INSERT INTO typed_tbl VALUES ( + 'row3', + 'null', + '1970-01-01 00:00:00+00', + 0.000000, + '\x' +); + +-- Setup Database B +\connect cloudsync_test_45_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE typed_tbl ( + id TEXT PRIMARY KEY, + json_col JSONB, + ts_col TIMESTAMPTZ, + num_col NUMERIC(18, 6), + bin_col BYTEA +); + +SELECT cloudsync_init('typed_tbl', 'CLS', 1) AS _init_b \gset + +-- Encode and apply A -> B +\connect cloudsync_test_45_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('typed_tbl', 'CLS', 1) AS _reinit \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_45_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('typed_tbl', 'CLS', 1) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_a', 'hex')) AS apply_a_to_b \gset + +-- Verify row count +SELECT COUNT(*) AS count_b FROM typed_tbl \gset +SELECT (:count_b::int = 3) AS count_ok \gset +\if :count_ok +\echo [PASS] (:testid) All 3 rows synced to B +\else +\echo [FAIL] (:testid) Expected 3 rows, got :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify JSONB roundtrip +SELECT COUNT(*) = 1 AS jsonb_ok +FROM typed_tbl +WHERE id = 'row1' + AND json_col @> '{"key": "value"}' + AND json_col -> 'nested' -> 'arr' = '[1, 2, 3]'::jsonb \gset +\if :jsonb_ok +\echo [PASS] (:testid) JSONB roundtrip correct +\else +\echo [FAIL] (:testid) JSONB data mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify TIMESTAMPTZ roundtrip +SELECT COUNT(*) = 1 AS ts_ok +FROM typed_tbl +WHERE id = 'row1' + AND ts_col = '2025-01-15 10:30:00+00'::timestamptz \gset +\if :ts_ok +\echo [PASS] (:testid) TIMESTAMPTZ roundtrip correct +\else +\echo [FAIL] (:testid) TIMESTAMPTZ data mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify NUMERIC roundtrip +SELECT COUNT(*) = 1 AS num_ok +FROM typed_tbl +WHERE id = 'row1' + AND num_col = 123456.789012 \gset +\if :num_ok +\echo [PASS] (:testid) NUMERIC(18,6) roundtrip correct +\else +\echo [FAIL] (:testid) NUMERIC data mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify BYTEA roundtrip +SELECT COUNT(*) = 1 AS bytea_ok +FROM typed_tbl +WHERE id = 'row1' + AND bin_col = '\x48656c6c6f'::bytea \gset +\if :bytea_ok +\echo [PASS] (:testid) BYTEA roundtrip correct +\else +\echo [FAIL] (:testid) BYTEA data mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify full hash match +\connect cloudsync_test_45_a +\ir helper_psql_conn_setup.sql +SELECT md5(COALESCE(string_agg( + id || ':' || + COALESCE(json_col::text, 'NULL') || ':' || + COALESCE(ts_col::text, 'NULL') || ':' || + COALESCE(num_col::text, 'NULL') || ':' || + COALESCE(encode(bin_col, 'hex'), 'NULL'), + '|' ORDER BY id +), '')) AS hash_a FROM typed_tbl \gset + +\connect cloudsync_test_45_b +\ir helper_psql_conn_setup.sql +SELECT md5(COALESCE(string_agg( + id || ':' || + COALESCE(json_col::text, 'NULL') || ':' || + COALESCE(ts_col::text, 'NULL') || ':' || + COALESCE(num_col::text, 'NULL') || ':' || + COALESCE(encode(bin_col, 'hex'), 'NULL'), + '|' ORDER BY id +), '')) AS hash_b FROM typed_tbl \gset + +SELECT (:'hash_a' = :'hash_b') AS hash_match \gset +\if :hash_match +\echo [PASS] (:testid) Full data hash matches for PG-specific types +\else +\echo [FAIL] (:testid) Hash mismatch - A: :hash_a, B: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_45_a; +DROP DATABASE IF EXISTS cloudsync_test_45_b; +\endif diff --git a/test/postgresql/46_schema_hash_mismatch.sql b/test/postgresql/46_schema_hash_mismatch.sql new file mode 100644 index 00000000..6aa16023 --- /dev/null +++ b/test/postgresql/46_schema_hash_mismatch.sql @@ -0,0 +1,95 @@ +-- Test schema hash mismatch during merge +-- Verifies detection when ALTER TABLE is done without cloudsync_begin/commit_alter. + +\set testid '46' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_46_src; +DROP DATABASE IF EXISTS cloudsync_test_46_dst; +CREATE DATABASE cloudsync_test_46_src; +CREATE DATABASE cloudsync_test_46_dst; + +-- Setup source +\connect cloudsync_test_46_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_src \gset +INSERT INTO test_tbl VALUES ('id1', 'value1'); + +-- Setup destination with same schema +\connect cloudsync_test_46_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_dst \gset + +-- Initial sync to get both in sync +\connect cloudsync_test_46_src +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _reinit \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_initial +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_46_dst +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_initial', 'hex')) AS _apply_initial \gset + +-- Now ALTER TABLE on destination WITHOUT using cloudsync_begin/commit_alter +ALTER TABLE test_tbl ADD COLUMN extra TEXT DEFAULT 'default'; + +-- Insert new data on source +\connect cloudsync_test_46_src +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _reinit \gset +INSERT INTO test_tbl VALUES ('id2', 'value2'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_post_alter +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply payload from pre-alter source to post-alter destination +-- This should detect schema mismatch +\connect cloudsync_test_46_dst +\ir helper_psql_conn_setup.sql + +-- Reinit to pick up new schema +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _reinit_dst \gset + +-- The apply may error due to schema mismatch, or succeed silently. +-- Either outcome is acceptable — the key is no corruption (verified below). +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'payload_post_alter', 'hex')) AS _apply_mismatch \gset +\set ON_ERROR_STOP on + +-- Verify database is in a consistent state (not corrupted) +SELECT COUNT(*) AS final_count FROM test_tbl \gset +SELECT (:final_count::int >= 1) AS state_ok \gset +\if :state_ok +\echo [PASS] (:testid) Database consistent after schema mismatch scenario (rows: :final_count) +\else +\echo [FAIL] (:testid) Database corrupted after schema mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify original data is intact +SELECT COUNT(*) = 1 AS original_ok +FROM test_tbl +WHERE id = 'id1' AND val = 'value1' \gset +\if :original_ok +\echo [PASS] (:testid) Original data intact after schema mismatch +\else +\echo [FAIL] (:testid) Original data corrupted +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_46_src; +DROP DATABASE IF EXISTS cloudsync_test_46_dst; +\endif diff --git a/test/postgresql/47_row_filter_advanced.sql b/test/postgresql/47_row_filter_advanced.sql new file mode 100644 index 00000000..c5c89964 --- /dev/null +++ b/test/postgresql/47_row_filter_advanced.sql @@ -0,0 +1,194 @@ +-- 'Row-level filter advanced tests (clear, complex expressions, row transitions, filter change)' + +\set testid '47' +\ir helper_test_init.sql + +-- Create database +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_47_a; +CREATE DATABASE cloudsync_test_47_a; + +\connect cloudsync_test_47_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- ============================================================ +-- Test 1: cloudsync_clear_filter lifecycle +-- ============================================================ +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('tasks') AS _init \gset +SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _sf \gset + +INSERT INTO tasks VALUES ('a', 'Task A', 1); +INSERT INTO tasks VALUES ('b', 'Task B', 2); +INSERT INTO tasks VALUES ('c', 'Task C', 1); + +-- Only matching rows tracked +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 2) AS clear_t1a_ok \gset +\if :clear_t1a_ok +\echo [PASS] (:testid) clear_filter: 2 PKs tracked before clear +\else +\echo [FAIL] (:testid) clear_filter: expected 2 tracked PKs before clear, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Clear filter +SELECT cloudsync_clear_filter('tasks') AS _cf \gset + +-- Insert non-matching row — should now be tracked (no filter) +-- clear_filter refilled metatable with all 3 existing rows (a, b, c) + insert d = 4 +INSERT INTO tasks VALUES ('d', 'Task D', 2); +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 4) AS clear_t1b_ok \gset +\if :clear_t1b_ok +\echo [PASS] (:testid) clear_filter: non-matching row tracked after clear (4 PKs) +\else +\echo [FAIL] (:testid) clear_filter: expected 4 PKs after clear+insert, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update row 'b' — already tracked by clear_filter refill, meta count unchanged +UPDATE tasks SET title = 'Task B Updated' WHERE id = 'b'; +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 4) AS clear_t1c_ok \gset +\if :clear_t1c_ok +\echo [PASS] (:testid) clear_filter: update on 'b' still 4 PKs +\else +\echo [FAIL] (:testid) clear_filter: expected 4 PKs after update on 'b', got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Complex filter — AND + comparison operators +-- ============================================================ +DROP TABLE IF EXISTS items; +CREATE TABLE items (id TEXT PRIMARY KEY NOT NULL, status TEXT, priority INTEGER, category TEXT, user_id INTEGER); +SELECT cloudsync_init('items') AS _init \gset +SELECT cloudsync_set_filter('items', 'user_id = 1 AND priority > 3') AS _sf \gset + +INSERT INTO items VALUES ('a', 'active', 5, 'work', 1); -- matches +INSERT INTO items VALUES ('b', 'active', 2, 'work', 1); -- fails priority +INSERT INTO items VALUES ('c', 'active', 5, 'work', 2); -- fails user_id + +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM items_cloudsync \gset +SELECT (:meta_pk_count = 1) AS complex_t2_ok \gset +\if :complex_t2_ok +\echo [PASS] (:testid) complex_filter: AND+comparison tracked 1 of 3 rows +\else +\echo [FAIL] (:testid) complex_filter: expected 1 tracked PK, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: IS NULL filter +-- ============================================================ +SELECT cloudsync_clear_filter('items') AS _cf \gset +SELECT cloudsync_set_filter('items', 'category IS NULL') AS _sf \gset + +SELECT COUNT(DISTINCT pk) AS meta_before FROM items_cloudsync \gset +INSERT INTO items VALUES ('f', 'x', 1, NULL, 1); -- matches +INSERT INTO items VALUES ('g', 'x', 1, 'work', 1); -- fails +SELECT COUNT(DISTINCT pk) AS meta_after FROM items_cloudsync \gset +SELECT ((:meta_after::int - :meta_before::int) = 1) AS null_t3_ok \gset +\if :null_t3_ok +\echo [PASS] (:testid) IS NULL filter: only NULL-category row tracked +\else +\echo [FAIL] (:testid) IS NULL filter: expected 1 new PK, got (:meta_after - :meta_before) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Row exits filter (matching -> non-matching via UPDATE) +-- ============================================================ +DROP TABLE IF EXISTS trans; +CREATE TABLE trans (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('trans') AS _init \gset +SELECT cloudsync_set_filter('trans', 'user_id = 1') AS _sf \gset + +INSERT INTO trans VALUES ('a', 'Task A', 1); + +SELECT COUNT(*) AS meta_before FROM trans_cloudsync \gset +UPDATE trans SET user_id = 2 WHERE id = 'a'; +SELECT COUNT(*) AS meta_after FROM trans_cloudsync \gset +SELECT (:meta_before = :meta_after) AS exit_t4_ok \gset +\if :exit_t4_ok +\echo [PASS] (:testid) row_exit: UPDATE out of filter did not change metadata +\else +\echo [FAIL] (:testid) row_exit: UPDATE out of filter changed metadata (:meta_before -> :meta_after) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Row enters filter (non-matching -> matching via UPDATE) +-- ============================================================ +INSERT INTO trans VALUES ('b', 'Task B', 2); + +SELECT COUNT(DISTINCT pk) AS meta_before FROM trans_cloudsync \gset +UPDATE trans SET user_id = 1 WHERE id = 'b'; +SELECT COUNT(DISTINCT pk) AS meta_after FROM trans_cloudsync \gset +SELECT (:meta_after::int > :meta_before::int) AS enter_t5_ok \gset +\if :enter_t5_ok +\echo [PASS] (:testid) row_enter: UPDATE into filter created metadata +\else +\echo [FAIL] (:testid) row_enter: UPDATE into filter did not create metadata (:meta_before -> :meta_after) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Filter change after data +-- ============================================================ +DROP TABLE IF EXISTS fchange; +CREATE TABLE fchange (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('fchange') AS _init \gset +SELECT cloudsync_set_filter('fchange', 'user_id = 1') AS _sf \gset + +INSERT INTO fchange VALUES ('a', 'A', 1); -- matches +INSERT INTO fchange VALUES ('b', 'B', 2); -- non-matching +INSERT INTO fchange VALUES ('c', 'C', 1); -- matches + +SELECT COUNT(DISTINCT pk) AS meta_count FROM fchange_cloudsync \gset +SELECT (:meta_count = 2) AS change_t6a_ok \gset +\if :change_t6a_ok +\echo [PASS] (:testid) filter_change: 2 PKs under initial filter +\else +\echo [FAIL] (:testid) filter_change: expected 2 PKs under initial filter, got :meta_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Change filter — resets metatable to only rows matching new filter (user_id = 2) +-- Only 'b' (user_id=2) matches new filter → 1 PK from refill, then insert d → 2 +SELECT cloudsync_set_filter('fchange', 'user_id = 2') AS _sf2 \gset + +INSERT INTO fchange VALUES ('d', 'D', 2); -- matches new filter +INSERT INTO fchange VALUES ('e', 'E', 1); -- non-matching under new filter + +SELECT COUNT(DISTINCT pk) AS meta_count FROM fchange_cloudsync \gset +SELECT (:meta_count = 2) AS change_t6b_ok \gset +\if :change_t6b_ok +\echo [PASS] (:testid) filter_change: 2 PKs after filter change (metatable reset) +\else +\echo [FAIL] (:testid) filter_change: expected 2 PKs after filter change, got :meta_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update 'a' (user_id=1) should NOT generate new metadata under new filter (user_id=2) +SELECT COUNT(*) AS meta_before FROM fchange_cloudsync \gset +UPDATE fchange SET title = 'A Updated' WHERE id = 'a'; +SELECT COUNT(*) AS meta_after FROM fchange_cloudsync \gset +SELECT (:meta_before = :meta_after) AS change_t6c_ok \gset +\if :change_t6c_ok +\echo [PASS] (:testid) filter_change: update on 'a' not tracked under new filter +\else +\echo [FAIL] (:testid) filter_change: update on 'a' changed metadata (:meta_before -> :meta_after) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_47_a; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/48_row_filter_multi_table.sql b/test/postgresql/48_row_filter_multi_table.sql new file mode 100644 index 00000000..c2a05902 --- /dev/null +++ b/test/postgresql/48_row_filter_multi_table.sql @@ -0,0 +1,166 @@ +-- 'Row-level filter: multi-table with different filters and composite primary keys' + +\set testid '48' +\ir helper_test_init.sql + +-- Create databases +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_48_a; +DROP DATABASE IF EXISTS cloudsync_test_48_b; +CREATE DATABASE cloudsync_test_48_a; +CREATE DATABASE cloudsync_test_48_b; + +-- ============================================================ +-- Setup Database A +-- ============================================================ +\connect cloudsync_test_48_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Table 1: composite PK with simple filter +CREATE TABLE projects (org_id INTEGER NOT NULL, proj_id INTEGER NOT NULL, name TEXT, PRIMARY KEY(org_id, proj_id)); +SELECT cloudsync_init('projects') AS _init \gset +SELECT cloudsync_set_filter('projects', 'org_id = 1') AS _sf \gset + +-- Table 2: composite PK with multi-column filter including string literal +CREATE TABLE members (org_id INTEGER NOT NULL, user_id INTEGER NOT NULL, role TEXT, PRIMARY KEY(org_id, user_id)); +SELECT cloudsync_init('members') AS _init \gset +SELECT cloudsync_set_filter('members', 'org_id = 1 AND role = ''admin''') AS _sf \gset + +-- ============================================================ +-- Test 1: Composite PK — only matching rows tracked +-- ============================================================ +INSERT INTO projects VALUES (1, 1, 'Proj A'); -- matches +INSERT INTO projects VALUES (2, 1, 'Proj B'); -- fails org_id +INSERT INTO projects VALUES (1, 2, 'Proj C'); -- matches + +SELECT COUNT(DISTINCT pk) AS proj_meta FROM projects_cloudsync \gset +SELECT (:proj_meta = 2) AS t1_proj_ok \gset +\if :t1_proj_ok +\echo [PASS] (:testid) composite_pk: 2 of 3 projects tracked +\else +\echo [FAIL] (:testid) composite_pk: expected 2 project PKs, got :proj_meta +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Multi-column filter with string literal — different table +-- ============================================================ +INSERT INTO members VALUES (1, 10, 'admin'); -- matches both conditions +INSERT INTO members VALUES (1, 20, 'viewer'); -- fails role +INSERT INTO members VALUES (2, 10, 'admin'); -- fails org_id + +SELECT COUNT(DISTINCT pk) AS mem_meta FROM members_cloudsync \gset +SELECT (:mem_meta = 1) AS t2_mem_ok \gset +\if :t2_mem_ok +\echo [PASS] (:testid) multi_filter: 1 of 3 members tracked +\else +\echo [FAIL] (:testid) multi_filter: expected 1 member PK, got :mem_meta +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Roundtrip sync — only matching rows per table transfer +-- ============================================================ +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_48_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE projects (org_id INTEGER NOT NULL, proj_id INTEGER NOT NULL, name TEXT, PRIMARY KEY(org_id, proj_id)); +SELECT cloudsync_init('projects') AS _init \gset +SELECT cloudsync_set_filter('projects', 'org_id = 1') AS _sf \gset + +CREATE TABLE members (org_id INTEGER NOT NULL, user_id INTEGER NOT NULL, role TEXT, PRIMARY KEY(org_id, user_id)); +SELECT cloudsync_init('members') AS _init \gset +SELECT cloudsync_set_filter('members', 'org_id = 1 AND role = ''admin''') AS _sf \gset + +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS _apply \gset + +-- Verify projects +SELECT COUNT(*) AS proj_count FROM projects \gset +SELECT (:proj_count = 2) AS t3_proj_ok \gset +\if :t3_proj_ok +\echo [PASS] (:testid) roundtrip: 2 projects synced to db_b +\else +\echo [FAIL] (:testid) roundtrip: expected 2 projects in db_b, got :proj_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify members +SELECT COUNT(*) AS mem_count FROM members \gset +SELECT (:mem_count = 1) AS t3_mem_ok \gset +\if :t3_mem_ok +\echo [PASS] (:testid) roundtrip: 1 member synced to db_b +\else +\echo [FAIL] (:testid) roundtrip: expected 1 member in db_b, got :mem_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify correct member identity +SELECT COUNT(*) AS admin_exists FROM members WHERE org_id = 1 AND user_id = 10 AND role = 'admin' \gset +SELECT (:admin_exists = 1) AS t3_admin_ok \gset +\if :t3_admin_ok +\echo [PASS] (:testid) roundtrip: correct admin member (1,10) present +\else +\echo [FAIL] (:testid) roundtrip: admin member (1,10) not found +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Non-matching rows should NOT exist +SELECT COUNT(*) AS bad_proj FROM projects WHERE org_id = 2 \gset +SELECT (:bad_proj = 0) AS t3_no_bad_proj \gset +\if :t3_no_bad_proj +\echo [PASS] (:testid) roundtrip: no org_id=2 projects in db_b +\else +\echo [FAIL] (:testid) roundtrip: unexpected org_id=2 projects found (:bad_proj) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Update and delete on composite PK, then re-sync +-- ============================================================ +\connect cloudsync_test_48_a +\ir helper_psql_conn_setup.sql + +UPDATE projects SET name = 'Proj A Updated' WHERE org_id = 1 AND proj_id = 1; +DELETE FROM projects WHERE org_id = 1 AND proj_id = 2; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_48_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload2_hex', 'hex')) AS _apply2 \gset + +SELECT COUNT(*) AS proj_count FROM projects \gset +SELECT (:proj_count = 1) AS t4_count_ok \gset +\if :t4_count_ok +\echo [PASS] (:testid) update_delete: 1 project remaining after sync +\else +\echo [FAIL] (:testid) update_delete: expected 1 project, got :proj_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT COUNT(*) AS updated_exists FROM projects WHERE org_id = 1 AND proj_id = 1 AND name = 'Proj A Updated' \gset +SELECT (:updated_exists = 1) AS t4_updated_ok \gset +\if :t4_updated_ok +\echo [PASS] (:testid) update_delete: 'Proj A Updated' present in db_b +\else +\echo [FAIL] (:testid) update_delete: 'Proj A Updated' not found in db_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_48_a; +DROP DATABASE IF EXISTS cloudsync_test_48_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/49_row_filter_prefill.sql b/test/postgresql/49_row_filter_prefill.sql new file mode 100644 index 00000000..a5708566 --- /dev/null +++ b/test/postgresql/49_row_filter_prefill.sql @@ -0,0 +1,192 @@ +-- 'Row-level filter with pre-existing data (prefill tests)' +-- Tests that cloudsync_refill_metatable correctly handles the filter +-- when rows exist before cloudsync_init and cloudsync_set_filter. + +\set testid '49' +\ir helper_test_init.sql + +-- Create databases +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_49_a; +DROP DATABASE IF EXISTS cloudsync_test_49_b; +CREATE DATABASE cloudsync_test_49_a; +CREATE DATABASE cloudsync_test_49_b; + +-- ============================================================ +-- Setup Database A — insert data BEFORE cloudsync_init +-- ============================================================ + +\connect cloudsync_test_49_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); + +-- Pre-existing rows: 3 matching (user_id=1), 2 non-matching +INSERT INTO tasks VALUES ('a', 'Task A', 1); +INSERT INTO tasks VALUES ('b', 'Task B', 2); +INSERT INTO tasks VALUES ('c', 'Task C', 1); +INSERT INTO tasks VALUES ('d', 'Task D', 3); +INSERT INTO tasks VALUES ('e', 'Task E', 1); + +-- Init and set filter AFTER data exists +SELECT cloudsync_init('tasks') AS _init_a \gset +SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _sf_a \gset + +-- ============================================================ +-- Test 1: set_filter resets metatable to only matching rows +-- cloudsync_init filled all 5, then set_filter cleaned and refilled → 3 matching (a, c, e) +-- ============================================================ + +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 3) AS prefill_t1_ok \gset +\if :prefill_t1_ok +\echo [PASS] (:testid) prefill: 3 matching rows have metadata after set_filter +\else +\echo [FAIL] (:testid) prefill: expected 3 tracked PKs after set_filter, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: New matching insert IS tracked +-- ============================================================ + +INSERT INTO tasks VALUES ('f', 'Task F', 1); +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 4) AS prefill_t2_ok \gset +\if :prefill_t2_ok +\echo [PASS] (:testid) prefill: new matching insert tracked (4 PKs) +\else +\echo [FAIL] (:testid) prefill: expected 4 PKs after matching insert, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: New non-matching insert is NOT tracked +-- ============================================================ + +INSERT INTO tasks VALUES ('g', 'Task G', 2); +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 4) AS prefill_t3_ok \gset +\if :prefill_t3_ok +\echo [PASS] (:testid) prefill: new non-matching insert not tracked (still 4 PKs) +\else +\echo [FAIL] (:testid) prefill: expected still 4 PKs after non-matching insert, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Sync roundtrip — pre-existing + matching new rows transfer +-- ============================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Setup Database B (empty) +\connect cloudsync_test_49_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('tasks') AS _init_b \gset +SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _sf_b \gset + +-- Apply payload +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS _apply \gset + +-- Only matching rows (a, c, e, f) should arrive; non-matching (b, d, g) should not +SELECT COUNT(*) AS row_count FROM tasks \gset +SELECT (:row_count = 4) AS prefill_t4a_ok \gset +\if :prefill_t4a_ok +\echo [PASS] (:testid) prefill_sync: 4 matching rows synced to Database B +\else +\echo [FAIL] (:testid) prefill_sync: expected 4 rows in Database B, got :row_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify non-matching post-filter row 'g' did NOT sync +SELECT COUNT(*) AS g_count FROM tasks WHERE id = 'g' \gset +SELECT (:g_count = 0) AS prefill_t4b_ok \gset +\if :prefill_t4b_ok +\echo [PASS] (:testid) prefill_sync: non-matching post-filter row 'g' not synced +\else +\echo [FAIL] (:testid) prefill_sync: row 'g' should not have synced +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify pre-existing non-matching row 'b' did NOT sync (metadata removed by set_filter) +SELECT COUNT(*) AS b_count FROM tasks WHERE id = 'b' AND user_id = 2 \gset +SELECT (:b_count = 0) AS prefill_t4c_ok \gset +\if :prefill_t4c_ok +\echo [PASS] (:testid) prefill_sync: pre-existing non-matching row 'b' not synced +\else +\echo [FAIL] (:testid) prefill_sync: pre-existing non-matching row 'b' should not have synced +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Composite PK with pre-existing data +-- ============================================================ + +\connect cloudsync_test_49_a + +CREATE TABLE projects ( + org_id INTEGER NOT NULL, + proj_id INTEGER NOT NULL, + name TEXT, + status TEXT, + PRIMARY KEY(org_id, proj_id) +); + +-- Pre-existing rows: 2 matching (org_id=1), 2 non-matching +INSERT INTO projects VALUES (1, 1, 'Alpha', 'active'); +INSERT INTO projects VALUES (1, 2, 'Beta', 'active'); +INSERT INTO projects VALUES (2, 1, 'Gamma', 'active'); +INSERT INTO projects VALUES (2, 2, 'Delta', 'active'); + +SELECT cloudsync_init('projects') AS _init_proj \gset +SELECT cloudsync_set_filter('projects', 'org_id = 1') AS _sf_proj \gset + +-- set_filter resets metatable: only 2 matching rows (org_id=1) should have metadata +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM projects_cloudsync \gset +SELECT (:meta_pk_count = 2) AS prefill_t5_ok \gset +\if :prefill_t5_ok +\echo [PASS] (:testid) prefill_composite: 2 matching rows have metadata after set_filter +\else +\echo [FAIL] (:testid) prefill_composite: expected 2 tracked PKs, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- New matching insert tracked +INSERT INTO projects VALUES (1, 3, 'Epsilon', 'active'); +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM projects_cloudsync \gset +SELECT (:meta_pk_count = 3) AS prefill_t5b_ok \gset +\if :prefill_t5b_ok +\echo [PASS] (:testid) prefill_composite: new matching row tracked (3 PKs) +\else +\echo [FAIL] (:testid) prefill_composite: expected 3 PKs after matching insert, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- New non-matching insert NOT tracked +INSERT INTO projects VALUES (3, 1, 'Zeta', 'active'); +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM projects_cloudsync \gset +SELECT (:meta_pk_count = 3) AS prefill_t5c_ok \gset +\if :prefill_t5c_ok +\echo [PASS] (:testid) prefill_composite: new non-matching row not tracked (still 3 PKs) +\else +\echo [FAIL] (:testid) prefill_composite: expected still 3 PKs, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_49_a; +DROP DATABASE IF EXISTS cloudsync_test_49_b; +\endif diff --git a/test/postgresql/50_block_lww_existing_data.sql b/test/postgresql/50_block_lww_existing_data.sql new file mode 100644 index 00000000..485a43e8 --- /dev/null +++ b/test/postgresql/50_block_lww_existing_data.sql @@ -0,0 +1,92 @@ +-- 'Block-level LWW: migration of existing tracked rows when algo=block is enabled' +-- Mirrors the SQLite unit test: Block LWW Existing Data + +\set testid '50' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_existing_a; +CREATE DATABASE cloudsync_block_existing_a; + +\connect cloudsync_block_existing_a +\ir helper_psql_conn_setup.sql + +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create a table and init cloudsync WITHOUT block algo first +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset + +-- Insert rows BEFORE enabling block algorithm (they will be tracked as regular CLS rows) +INSERT INTO docs (id, body) VALUES ('d1', E'Line1\nLine2\nLine3'); +INSERT INTO docs (id, body) VALUES ('d2', E'Alpha\nBeta'); + +-- Now enable block algo on the column that already has data +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc \gset + +-- Test 1: Blocks table should have 5 entries (3 for d1, 2 for d2) immediately after set_column +SELECT count(*) AS block_count FROM docs_cloudsync_blocks \gset +SELECT (:block_count::int = 5) AS block_count_ok \gset +\if :block_count_ok +\echo [PASS] (:testid) Migration: 5 block entries after set_column on existing data +\else +\echo [FAIL] (:testid) Migration: expected 5 block entries, got :block_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: Metadata should have 5 alive block entries +SELECT count(*) AS meta_count FROM docs_cloudsync +WHERE col_name LIKE 'body' || chr(31) || '%' AND col_version % 2 = 1 \gset +SELECT (:meta_count::int = 5) AS meta_count_ok \gset +\if :meta_count_ok +\echo [PASS] (:testid) Migration: 5 alive block metadata entries +\else +\echo [FAIL] (:testid) Migration: expected 5 alive metadata entries, got :meta_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Calling set_column again should be idempotent (count stays at 5) +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc2 \gset + +SELECT count(*) AS block_count2 FROM docs_cloudsync_blocks \gset +SELECT (:block_count2::int = 5) AS idempotent_ok \gset +\if :idempotent_ok +\echo [PASS] (:testid) Migration: idempotent (still 5 blocks after second set_column) +\else +\echo [FAIL] (:testid) Migration: idempotency broken, got :block_count2 blocks (expected 5) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 4: UPDATE on d1 should still work correctly after migration +UPDATE docs SET body = E'Line1\nLine2-edited\nLine3' WHERE id = 'd1'; + +SELECT count(*) AS block_count3 FROM docs_cloudsync_blocks \gset +SELECT (:block_count3::int = 5) AS update_count_ok \gset +\if :update_count_ok +\echo [PASS] (:testid) Migration: 5 blocks after UPDATE (d1 edited in-place) +\else +\echo [FAIL] (:testid) Migration: expected 5 blocks after update, got :block_count3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 5: Materialized value should reflect the update +SELECT cloudsync_text_materialize('docs', 'body', 'd1') AS _mat \gset + +SELECT (body = E'Line1\nLine2-edited\nLine3') AS mat_ok FROM docs WHERE id = 'd1' \gset +\if :mat_ok +\echo [PASS] (:testid) Migration: materialized value correct after update +\else +\echo [FAIL] (:testid) Migration: materialized value mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_block_existing_a; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/51_stale_table_settings_dropped_meta.sql b/test/postgresql/51_stale_table_settings_dropped_meta.sql new file mode 100644 index 00000000..6d4880a4 --- /dev/null +++ b/test/postgresql/51_stale_table_settings_dropped_meta.sql @@ -0,0 +1,123 @@ +-- 'Stale cloudsync_table_settings with dropped meta-table' +-- Mirrors the SQLite unit test: Stale Table Settings Dropped Meta +-- +-- When a user drops a tracked table and its
_cloudsync meta-table +-- manually (without calling cloudsync_cleanup), cloudsync_table_settings is +-- left with stale rows while pg_tables no longer has any matching +-- *_cloudsync table. Before the fix in cloudsync_dbversion_rebuild, opening a +-- new backend and calling any cloudsync function caused +-- cloudsync_dbversion_build_query to produce a NULL SQL string (string_agg +-- over zero rows), which was misreported as DBRES_NOMEM, making +-- cloudsync_context_init fail and every cloudsync_* call ereport ERROR. + +\set testid '51' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_test_51; +CREATE DATABASE cloudsync_test_51; + +\connect cloudsync_test_51 +\ir helper_psql_conn_setup.sql + +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Phase 1: create a tracked table and initialize cloudsync on it. +DROP TABLE IF EXISTS stale_doc CASCADE; +CREATE TABLE stale_doc (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('stale_doc', 'CLS', 1) AS _init \gset + +-- Sanity: the meta-table exists and cloudsync_table_settings has a row for it. +SELECT count(*) AS meta_exists FROM pg_tables WHERE tablename = 'stale_doc_cloudsync' \gset +SELECT (:meta_exists::int = 1) AS meta_exists_ok \gset +\if :meta_exists_ok +\echo [PASS] (:testid) stale_doc_cloudsync meta-table created +\else +\echo [FAIL] (:testid) expected stale_doc_cloudsync to exist +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT count(*) AS settings_rows FROM cloudsync_table_settings WHERE tbl_name = 'stale_doc' \gset +SELECT (:settings_rows::int > 0) AS settings_rows_ok \gset +\if :settings_rows_ok +\echo [PASS] (:testid) cloudsync_table_settings has row(s) for stale_doc +\else +\echo [FAIL] (:testid) expected cloudsync_table_settings row for stale_doc +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Phase 2: drop BOTH the base table and the meta-table without calling +-- cloudsync_cleanup. cloudsync_table_settings still references stale_doc, +-- but pg_tables has no *_cloudsync tables at all now. +DROP TABLE stale_doc; +DROP TABLE stale_doc_cloudsync; + +SELECT count(*) AS cloudsync_meta_tables FROM pg_tables WHERE tablename LIKE '%_cloudsync' \gset +SELECT (:cloudsync_meta_tables::int = 0) AS no_meta_ok \gset +\if :no_meta_ok +\echo [PASS] (:testid) no *_cloudsync meta-tables remain in pg_tables +\else +\echo [FAIL] (:testid) expected zero *_cloudsync tables, got :cloudsync_meta_tables +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Phase 3: reconnect to force a fresh backend. pg_cloudsync_context is a +-- static per-process pointer, so a new backend means +-- cloudsync_pg_context_init runs again on the next cloudsync call — which +-- is exactly what used to fail under this bug. +\connect cloudsync_test_51 +\ir helper_psql_conn_setup.sql + +-- cloudsync_version is a pure function that does not touch the context, so +-- this call cannot fail even with the bug present. It's here only as a +-- trivial smoke check that the extension is still loadable. +SELECT cloudsync_version() IS NOT NULL AS version_ok \gset +\if :version_ok +\echo [PASS] (:testid) cloudsync_version() reachable after reopen +\else +\echo [FAIL] (:testid) cloudsync_version() failed after reopen +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- The real check: calling a function that goes through get_cloudsync_context() +-- must succeed. Before the fix, cloudsync_dbversion_rebuild returned +-- DBRES_NOMEM here because SQL_DBVERSION_BUILD_QUERY's string_agg over zero +-- rows produced a NULL SQL string, and the whole init path would ereport +-- ERROR — any cloudsync_* call below would abort the script. +CREATE TABLE stale_doc2 (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('stale_doc2', 'CLS', 1) AS _init2 \gset + +-- The new table's meta-table exists. If cloudsync_init failed (pre-fix +-- behavior) this count will be 0, covering both the init-rc check and the +-- meta-table creation in a single assertion. +SELECT count(*) AS meta2_exists FROM pg_tables WHERE tablename = 'stale_doc2_cloudsync' \gset +SELECT (:meta2_exists::int = 1) AS meta2_exists_ok \gset +\if :meta2_exists_ok +\echo [PASS] (:testid) cloudsync_init succeeded and stale_doc2_cloudsync was created +\else +\echo [FAIL] (:testid) expected stale_doc2_cloudsync to exist after cloudsync_init +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- An insert via the new cloudsync_init'd table should produce a cloudsync +-- metadata entry — confirming the context is fully functional. +INSERT INTO stale_doc2 (id, body) VALUES ('a', 'hello'); + +SELECT count(*) AS meta_rows FROM stale_doc2_cloudsync \gset +SELECT (:meta_rows::int > 0) AS meta_rows_ok \gset +\if :meta_rows_ok +\echo [PASS] (:testid) stale_doc2_cloudsync has metadata after insert +\else +\echo [FAIL] (:testid) expected metadata rows in stale_doc2_cloudsync +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_51; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/52_payload_chunks.sql b/test/postgresql/52_payload_chunks.sql new file mode 100644 index 00000000..ecdd3648 --- /dev/null +++ b/test/postgresql/52_payload_chunks.sql @@ -0,0 +1,332 @@ +-- Payload chunks and transparent large-value fragmentation + +\set testid '52-chunks' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_52_chunks_src; +DROP DATABASE IF EXISTS cloudsync_test_52_chunks_dst; +DROP DATABASE IF EXISTS cloudsync_test_52_chunks_legacy; +CREATE DATABASE cloudsync_test_52_chunks_src; +CREATE DATABASE cloudsync_test_52_chunks_dst; +CREATE DATABASE cloudsync_test_52_chunks_legacy; + +\connect cloudsync_test_52_chunks_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE payload_chunk_test ( + id TEXT PRIMARY KEY, + note TEXT DEFAULT '', + data BYTEA DEFAULT '\x'::bytea +); +SELECT cloudsync_init('payload_chunk_test', 'CLS', 1) AS _init_src \gset +SELECT cloudsync_set('payload_max_chunk_size', '1'); + +INSERT INTO payload_chunk_test(id, note, data) +SELECT + 'big', + (SELECT string_agg(md5(i::text), '') FROM generate_series(1, 22500) AS g(i)), + decode((SELECT string_agg(md5((i * 17)::text), '') FROM generate_series(1, 23000) AS g(i)), 'hex'); + +INSERT INTO payload_chunk_test(id, note, data) +VALUES + ('same-a', 'same payload a', decode(repeat('ab', 360000), 'hex')), + ('same-b', 'same payload b', decode(repeat('ab', 360000), 'hex')); + +INSERT INTO payload_chunk_test(id, note, data) +SELECT + format('row-%s', lpad(i::text, 3, '0')), + format('small-%s-%s', i, repeat(md5(i::text), 24)), + decode(repeat(md5((i * 31)::text), 16), 'hex') +FROM generate_series(1, 260) AS g(i); + +SELECT + count(*) AS chunk_count, + count(*) FILTER (WHERE get_byte(payload, 4) = 3) AS v3_chunk_count, + bool_and(octet_length(payload) <= 262144) AS chunks_within_limit, + max(octet_length(payload)) AS max_chunk_len, + sum(rows) AS chunk_rows +FROM cloudsync_payload_chunks() \gset + +\if :chunks_within_limit +\echo [PASS] (:testid) Generated chunks respect the 256KB technical minimum - max: :max_chunk_len +\else +\echo [FAIL] (:testid) Generated chunk exceeds 256KB - max: :max_chunk_len +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:chunk_count::int >= 5 AND :v3_chunk_count::int >= 2) AS chunk_shape_ok \gset +\if :chunk_shape_ok +\echo [PASS] (:testid) Rowset and large-value fragmentation produced multiple chunks (:chunk_count total, :v3_chunk_count v3) +\else +\echo [FAIL] (:testid) Expected multiple chunks and v3 fragments, got :chunk_count total and :v3_chunk_count v3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT count(*) AS explicit_arg_chunk_count +FROM cloudsync_payload_chunks(NULL, cloudsync_siteid(), NULL) \gset + +SELECT (:explicit_arg_chunk_count::int = :chunk_count::int) AS explicit_args_ok \gset +\if :explicit_args_ok +\echo [PASS] (:testid) Optional cloudsync_payload_chunks arguments work +\else +\echo [FAIL] (:testid) Optional cloudsync_payload_chunks arguments changed result count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- exclude_filter_site_id flag: every change here originates from the local +-- site, so excluding it must yield zero chunks while including it yields the +-- full set. This proves the predicate flips between "= site" and "<> site". +SELECT count(*) AS excl_local_chunks +FROM cloudsync_payload_chunks(0, cloudsync_siteid(), NULL, true) \gset + +SELECT count(*) AS incl_local_chunks +FROM cloudsync_payload_chunks(0, cloudsync_siteid(), NULL, false) \gset + +SELECT + cloudsync_payload_blob_checked(0, 0, cloudsync_siteid(), true, 10000000) IS NULL AS excl_local_blob_is_null, + octet_length(cloudsync_payload_blob_checked(0, 0, cloudsync_siteid(), false, 10000000)) AS incl_local_blob_size \gset + +SELECT (:excl_local_chunks::int = 0 AND :incl_local_chunks::int > 0) AS exclude_flag_ok \gset +\if :exclude_flag_ok +\echo [PASS] (:testid) exclude_filter_site_id flips the site filter (exclude local -> 0, include -> :incl_local_chunks) +\else +\echo [FAIL] (:testid) exclude_filter_site_id did not flip the filter (exclude=:excl_local_chunks include=:incl_local_chunks) +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:'excl_local_blob_is_null'::bool AND :incl_local_blob_size::bigint > 0) AS blob_checked_exclude_flag_ok \gset +\if :blob_checked_exclude_flag_ok +\echo [PASS] (:testid) cloudsync_payload_blob_checked honors include/exclude site filters +\else +\echo [FAIL] (:testid) cloudsync_payload_blob_checked include/exclude mismatch (exclude_null=:excl_local_blob_is_null include=:incl_local_blob_size) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- exclude=true without a filter_site_id must raise an error +CREATE TEMP TABLE _excl_err(ok bool); +DO $$ +BEGIN + PERFORM 1 FROM cloudsync_payload_chunks(0, NULL, NULL, true); + INSERT INTO _excl_err VALUES (false); +EXCEPTION WHEN OTHERS THEN + INSERT INTO _excl_err VALUES (true); +END $$; +SELECT ok AS exclude_no_site_errors FROM _excl_err \gset +DROP TABLE _excl_err; +\if :exclude_no_site_errors +\echo [PASS] (:testid) exclude_filter_site_id without a site_id raises an error +\else +\echo [FAIL] (:testid) exclude_filter_site_id without a site_id did not error +SELECT (:fail::int + 1) AS fail \gset +\endif + +CREATE TEMP TABLE _blob_checked_excl_err(ok bool); +DO $$ +BEGIN + PERFORM cloudsync_payload_blob_checked(0, 0, NULL, true, 10000000); + INSERT INTO _blob_checked_excl_err VALUES (false); +EXCEPTION WHEN OTHERS THEN + INSERT INTO _blob_checked_excl_err VALUES (true); +END $$; +SELECT ok AS blob_checked_exclude_no_site_errors FROM _blob_checked_excl_err \gset +DROP TABLE _blob_checked_excl_err; +\if :blob_checked_exclude_no_site_errors +\echo [PASS] (:testid) cloudsync_payload_blob_checked exclude without a site_id raises an error +\else +\echo [FAIL] (:testid) cloudsync_payload_blob_checked exclude without a site_id did not error +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- UUID text<->blob roundtrip, including the /check string-argument path +SELECT ( + cloudsync_uuid_blob(cloudsync_uuid_text(cloudsync_siteid())) = cloudsync_siteid() + AND cloudsync_uuid_blob(cloudsync_uuid_text(cloudsync_siteid(), false)) = cloudsync_siteid() + AND cloudsync_uuid_blob(upper(cloudsync_uuid_text(cloudsync_siteid()))) = cloudsync_siteid() + AND length(cloudsync_uuid_text(cloudsync_siteid())) = 36 + AND length(cloudsync_uuid_text(cloudsync_siteid(), false)) = 32 +) AS uuid_conv_ok \gset +\if :uuid_conv_ok +\echo [PASS] (:testid) cloudsync_uuid_text/cloudsync_uuid_blob roundtrip (dashed, undashed, uppercase) +\else +\echo [FAIL] (:testid) UUID conversion roundtrip mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- A string-derived site_id (as the /check endpoint would pass it) selects the +-- same chunks as the binary site_id. +SELECT count(*) AS str_arg_chunks +FROM cloudsync_payload_chunks(0, cloudsync_uuid_blob(cloudsync_uuid_text(cloudsync_siteid())), NULL, false) \gset +SELECT (:str_arg_chunks::int = :incl_local_chunks::int) AS str_arg_ok \gset +\if :str_arg_ok +\echo [PASS] (:testid) String-derived site_id via cloudsync_uuid_blob matches the binary site_id +\else +\echo [FAIL] (:testid) String-derived site_id mismatch (:str_arg_chunks vs :incl_local_chunks) +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT + md5(string_agg(id || ':' || note || ':' || encode(data, 'hex'), '|' ORDER BY id)) AS src_hash, + count(*) AS src_count +FROM payload_chunk_test \gset + +SELECT string_agg(encode(payload, 'hex'), ',' ORDER BY chunk_index) AS chunks_hex +FROM cloudsync_payload_chunks() \gset + +SELECT + encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS legacy_payload_hex, + octet_length(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq)) AS legacy_payload_len +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT + octet_length(cloudsync_payload_blob_checked(0, 0, cloudsync_siteid(), false, 10000000)) AS checked_payload_len, + cloudsync_payload_blob_checked(999999, 0, cloudsync_siteid(), false, 10000000) IS NULL AS checked_empty_is_null \gset + +SELECT (:legacy_payload_len::int > 262144) AS legacy_payload_large_ok \gset +\if :legacy_payload_large_ok +\echo [PASS] (:testid) Legacy monolithic payload is larger than local chunk setting (:legacy_payload_len bytes) +\else +\echo [FAIL] (:testid) Legacy monolithic payload was expected to exceed the chunk setting +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:checked_payload_len::bigint = :legacy_payload_len::bigint AND :'checked_empty_is_null'::bool) AS blob_checked_ok \gset +\if :blob_checked_ok +\echo [PASS] (:testid) cloudsync_payload_blob_checked returns the legacy payload and empty windows return NULL +\else +\echo [FAIL] (:testid) cloudsync_payload_blob_checked mismatch (checked=:checked_payload_len legacy=:legacy_payload_len empty=:checked_empty_is_null) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- This step also guards a backend crash regression: the size-limit ereport() +-- fires right after the internal cloudsync_changes_select() cursor has run and +-- unwound. If that SRF ever again returns from inside its PG_TRY (leaving +-- PG_exception_stack dangling), this ereport longjmps into freed stack and +-- segfaults instead of raising -- which drops the connection and fails the suite. +CREATE TEMP TABLE _blob_checked_limit_err(ok bool); +DO $$ +BEGIN + PERFORM cloudsync_payload_blob_checked(0, 0, cloudsync_siteid(), false, 1); + INSERT INTO _blob_checked_limit_err VALUES (false); +EXCEPTION WHEN OTHERS THEN + INSERT INTO _blob_checked_limit_err VALUES (true); +END $$; +SELECT ok AS blob_checked_limit_errors FROM _blob_checked_limit_err \gset +DROP TABLE _blob_checked_limit_err; +\if :blob_checked_limit_errors +\echo [PASS] (:testid) cloudsync_payload_blob_checked raises a limit-exceeded error +\else +\echo [FAIL] (:testid) cloudsync_payload_blob_checked did not raise a limit-exceeded error +SELECT (:fail::int + 1) AS fail \gset +\endif + +\connect cloudsync_test_52_chunks_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE payload_chunk_test ( + id TEXT PRIMARY KEY, + note TEXT DEFAULT '', + data BYTEA DEFAULT '\x'::bytea +); +SELECT cloudsync_init('payload_chunk_test', 'CLS', 1) AS _init_dst \gset +SELECT cloudsync_set('payload_max_chunk_size', '262144'); + +CREATE TEMP TABLE chunk_transport(ord INT, payload BYTEA); +INSERT INTO chunk_transport(ord, payload) +SELECT ord::int, decode(chunk_hex, 'hex') +FROM unnest(string_to_array(:'chunks_hex', ',')) WITH ORDINALITY AS t(chunk_hex, ord); + +-- Stale-fragment GC: on this fresh connection no fragment has been applied yet, +-- so the first applied v3 fragment triggers cleanup of the old incomplete group. +CREATE TABLE IF NOT EXISTS cloudsync_payload_fragments ( + value_id TEXT NOT NULL, part_index BIGINT NOT NULL, part_count BIGINT NOT NULL, + total_size BIGINT NOT NULL, checksum TEXT NOT NULL, + created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())::bigint), + tbl TEXT NOT NULL, pk BYTEA NOT NULL, col_name TEXT NOT NULL, col_version BIGINT NOT NULL, + db_version BIGINT NOT NULL, site_id BYTEA NOT NULL, cl BIGINT NOT NULL, seq BIGINT NOT NULL, + fragment BYTEA NOT NULL, PRIMARY KEY(value_id, part_index) +); +INSERT INTO cloudsync_payload_fragments +(value_id, part_index, part_count, total_size, checksum, created_at, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, fragment) +VALUES ('stale-incomplete', 0, 2, 10, '0000000000000000', 0, 'payload_chunk_test', '\x01', 'data', 1, 1, decode(repeat('00', 16), 'hex'), 1, 1, '\x00'); + +SELECT cloudsync_payload_apply(payload) AS stale_cleanup_apply +FROM chunk_transport WHERE get_byte(payload, 4) = 3 ORDER BY ord LIMIT 1 \gset + +SELECT (COUNT(*) = 0) AS stale_cleanup_ok +FROM cloudsync_payload_fragments WHERE value_id = 'stale-incomplete' \gset +\if :stale_cleanup_ok +\echo [PASS] (:testid) Stale incomplete fragment cleanup works (first apply on a fresh connection) +\else +\echo [FAIL] (:testid) Stale incomplete fragment cleanup failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT coalesce(sum(cloudsync_payload_apply(payload)), 0) AS chunk_apply_rows +FROM (SELECT payload FROM chunk_transport ORDER BY ord DESC) AS ordered_chunks \gset + +SELECT + md5(string_agg(id || ':' || note || ':' || encode(data, 'hex'), '|' ORDER BY id)) AS dst_hash, + count(*) AS dst_count +FROM payload_chunk_test \gset + +SELECT (:'dst_hash' = :'src_hash' AND :dst_count::int = :src_count::int) AS chunk_apply_ok \gset +\if :chunk_apply_ok +\echo [PASS] (:testid) Chunked payloads apply correctly, including reverse-order v3 fragments and identical large values +\else +\echo [FAIL] (:testid) Chunked payload apply mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Throttle check: the full apply above already ran the stale GC, so an old +-- group inserted now must NOT be removed by the next immediate fragment apply. +-- This proves the GC is not re-scanned on every applied fragment (the O(n^2) fix). +INSERT INTO cloudsync_payload_fragments +(value_id, part_index, part_count, total_size, checksum, created_at, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, fragment) +VALUES ('stale-incomplete-2', 0, 2, 10, '0000000000000000', 0, 'payload_chunk_test', '\x02', 'data', 1, 1, decode(repeat('00', 16), 'hex'), 1, 1, '\x00'); + +SELECT cloudsync_payload_apply(payload) AS throttle_apply +FROM chunk_transport WHERE get_byte(payload, 4) = 3 ORDER BY ord LIMIT 1 \gset + +SELECT (COUNT(*) = 1) AS stale_throttle_ok +FROM cloudsync_payload_fragments WHERE value_id = 'stale-incomplete-2' \gset +\if :stale_throttle_ok +\echo [PASS] (:testid) Stale GC is throttled (not re-run on every applied fragment) +\else +\echo [FAIL] (:testid) Stale GC was not throttled (removed a fresh-inserted old group on the next apply) +SELECT (:fail::int + 1) AS fail \gset +\endif + +\connect cloudsync_test_52_chunks_legacy +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE payload_chunk_test ( + id TEXT PRIMARY KEY, + note TEXT DEFAULT '', + data BYTEA DEFAULT '\x'::bytea +); +SELECT cloudsync_init('payload_chunk_test', 'CLS', 1) AS _init_legacy \gset +SELECT cloudsync_set('payload_max_chunk_size', '262144'); +SELECT cloudsync_payload_apply(decode(:'legacy_payload_hex', 'hex')) AS legacy_apply_rows \gset + +SELECT + md5(string_agg(id || ':' || note || ':' || encode(data, 'hex'), '|' ORDER BY id)) AS legacy_hash, + count(*) AS legacy_count +FROM payload_chunk_test \gset + +SELECT (:'legacy_hash' = :'src_hash' AND :legacy_count::int = :src_count::int) AS legacy_apply_ok \gset +\if :legacy_apply_ok +\echo [PASS] (:testid) Legacy monolithic payload applies even when larger than local chunk setting +\else +\echo [FAIL] (:testid) Legacy monolithic payload apply mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_52_chunks_src; +DROP DATABASE IF EXISTS cloudsync_test_52_chunks_dst; +DROP DATABASE IF EXISTS cloudsync_test_52_chunks_legacy; +\endif diff --git a/test/postgresql/53_payload_blob_checked_pg_try.sql b/test/postgresql/53_payload_blob_checked_pg_try.sql new file mode 100644 index 00000000..d2c64840 --- /dev/null +++ b/test/postgresql/53_payload_blob_checked_pg_try.sql @@ -0,0 +1,77 @@ +-- Regression: cloudsync_payload_blob_checked must restore PG_exception_stack on +-- its SUCCESS path. +-- +-- The function returns its result with PG_RETURN_BYTEA_P(result) (and +-- PG_RETURN_NULL on the empty-blob path) from INSIDE its PG_TRY()/PG_END_TRY() +-- block. PG_RETURN_* expands to `return`, so PG_END_TRY() never runs and +-- PG_exception_stack is left pointing at this function's now-dead stack frame. +-- A later ereport(ERROR) raised in the SAME query then siglongjmp()s into that +-- freed frame (before the enclosing portal's PG_END_TRY can restore the stack) +-- and the backend segfaults. This is the exact hazard the PR documents and +-- fixes in cloudsync_changes_select, reintroduced here. +-- +-- Reproduction: one top-level query that evaluates blob_checked (the divide's +-- operand forces it to run first) and then raises integer division-by-zero. +-- BUG PRESENT -> backend crash; psql reports "server closed the connection +-- unexpectedly" and the run aborts at the SELECT below. +-- BUG FIXED -> a clean "division by zero" SQL error; the session survives +-- and the [PASS] line is reached. + +\set testid '53-pg-try-stack' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_53_pg_try; +CREATE DATABASE cloudsync_test_53_pg_try; + +\connect cloudsync_test_53_pg_try +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE blob_checked_test ( + id TEXT PRIMARY KEY, + note TEXT DEFAULT '' +); +SELECT cloudsync_init('blob_checked_test', 'CLS', 1) AS _init \gset +INSERT INTO blob_checked_test(id, note) VALUES ('a', 'hello'), ('b', 'world'); + +-- Sanity: blob_checked returns a non-NULL payload, so the call exercises the +-- success return path that carries the bug. (A standalone call is healed when +-- the statement completes and the portal's PG_END_TRY restores the stack, so +-- this line is safe on its own.) +SELECT cloudsync_payload_blob_checked(0, 0, cloudsync_siteid(), false, 1000000000) IS NOT NULL AS have_blob \gset +\if :have_blob +\echo [PASS] (:testid) blob_checked produced a payload (exercises the success return path) +\else +\echo [FAIL] (:testid) blob_checked returned NULL - cannot exercise the buggy path +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- The post-call error is expected; it must not abort the script so we can check +-- whether the session survived it. +\set ON_ERROR_STOP off + +-- Single top-level query: blob_checked runs first (operand of the divide), +-- leaving PG_exception_stack dangling on the buggy build, then int4div raises +-- ereport(ERROR) while that dangling frame is still the active longjmp target. +SELECT (cloudsync_payload_blob_checked(0, 0, cloudsync_siteid(), false, 1000000000) IS NOT NULL)::int / 0 AS boom; + +\set ON_ERROR_STOP on + +-- Reaching here means the backend is still alive: the dangling-frame siglongjmp +-- did NOT happen, i.e. PG_END_TRY ran on the success path. On the buggy build +-- the connection is already gone and psql never runs this. +SELECT 1 AS still_alive \gset +\if :{?still_alive} +\echo [PASS] (:testid) session survived a post-call error - PG_exception_stack was restored +\else +\echo [FAIL] (:testid) session did not survive a post-call error +SELECT (:fail::int + 1) AS fail \gset +\endif + +\ir helper_test_cleanup.sql +\if :should_cleanup +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_53_pg_try; +\endif diff --git a/test/postgresql/54_payload_chunks_fragment_state.sql b/test/postgresql/54_payload_chunks_fragment_state.sql new file mode 100644 index 00000000..d3fd2d06 --- /dev/null +++ b/test/postgresql/54_payload_chunks_fragment_state.sql @@ -0,0 +1,105 @@ +-- Reproduction probe for finding #2: cloudsync_payload_chunks() stages +-- fragment-spanning state (tbl/pk/col_name/col_value/site_id) that is palloc'd +-- by payload_chunks_fetch_current in the SRF's per-call memory context, then +-- read again on later SRF_PERCALL invocations when emitting fragments 1..N of a +-- single oversized value. Per the SRF protocol only multi_call_memory_ctx is +-- guaranteed to survive between calls, so reading that state on a later call is +-- a use-after-free in theory. +-- +-- This test forces a single column value large enough to span MANY fragments +-- (each ~256KB, the technical minimum), gives each fragment region a distinct +-- byte pattern so any cross-fragment corruption is visible, round-trips the +-- chunks through apply on a fresh database, and compares the recovered value to +-- the source byte-for-byte. If fragments 1..N read freed/clobbered memory the +-- reassembly checksum fails ("inconsistent v3 fragments") or the recovered value +-- differs -> [FAIL]. If the per-call state survives (current FROM-clause path) +-- it round-trips cleanly -> [PASS]. + +\set testid '54-frag-state' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_54_frag_src; +DROP DATABASE IF EXISTS cloudsync_test_54_frag_dst; +CREATE DATABASE cloudsync_test_54_frag_src; +CREATE DATABASE cloudsync_test_54_frag_dst; + +\connect cloudsync_test_54_frag_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE frag_test ( + id TEXT PRIMARY KEY, + note TEXT DEFAULT '' +); +SELECT cloudsync_init('frag_test', 'CLS', 1) AS _init_src \gset +SELECT cloudsync_set('payload_max_chunk_size', '1'); -- clamps to 256KB minimum + +-- One oversized value (~1.18 MB) whose fragments straddle distinct regions: +-- A...A | B...B | C...C | D...D | E...E. Cross-fragment corruption (reading the +-- wrong region after a per-call context reset) would change the recovered text. +INSERT INTO frag_test(id, note) +VALUES ('big', repeat('A', 262144) || repeat('B', 262144) || repeat('C', 262144) + || repeat('D', 262144) || repeat('E', 131072)); + +-- Confirm this single value actually fragments into several v3 chunks. +SELECT + count(*) FILTER (WHERE get_byte(payload, 4) = 3) AS v3_chunks, + count(*) AS total_chunks +FROM cloudsync_payload_chunks() \gset +\if :{?v3_chunks} +\endif +SELECT (:v3_chunks::int >= 3) AS multi_fragment_ok \gset +\if :multi_fragment_ok +\echo [PASS] (:testid) Oversized value fragmented into :v3_chunks v3 fragments (:total_chunks chunks total) +\else +\echo [FAIL] (:testid) Expected >=3 v3 fragments, got :v3_chunks (test cannot exercise multi-fragment state) +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT + md5(note) AS src_md5, + length(note) AS src_len +FROM frag_test WHERE id = 'big' \gset + +-- Capture every chunk for transport, in generation order. +SELECT string_agg(encode(payload, 'hex'), ',' ORDER BY chunk_index) AS chunks_hex +FROM cloudsync_payload_chunks() \gset + +\connect cloudsync_test_54_frag_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE frag_test ( + id TEXT PRIMARY KEY, + note TEXT DEFAULT '' +); +SELECT cloudsync_init('frag_test', 'CLS', 1) AS _init_dst \gset +SELECT cloudsync_set('payload_max_chunk_size', '262144'); + +CREATE TEMP TABLE chunk_transport(ord INT, payload BYTEA); +INSERT INTO chunk_transport(ord, payload) +SELECT ord::int, decode(chunk_hex, 'hex') +FROM unnest(string_to_array(:'chunks_hex', ',')) WITH ORDINALITY AS t(chunk_hex, ord); + +SELECT coalesce(sum(cloudsync_payload_apply(payload)), 0) AS applied_rows +FROM (SELECT payload FROM chunk_transport ORDER BY ord) AS ordered \gset + +SELECT + md5(note) AS dst_md5, + length(note) AS dst_len +FROM frag_test WHERE id = 'big' \gset + +SELECT (:'dst_md5' = :'src_md5' AND :dst_len::bigint = :src_len::bigint) AS frag_roundtrip_ok \gset +\if :frag_roundtrip_ok +\echo [PASS] (:testid) Multi-fragment oversized value round-trips byte-exact (:src_len bytes) +\else +\echo [FAIL] (:testid) Multi-fragment value corrupted across SRF calls (src_len=:src_len dst_len=:dst_len src_md5=:src_md5 dst_md5=:dst_md5) +SELECT (:fail::int + 1) AS fail \gset +\endif + +\ir helper_test_cleanup.sql +\if :should_cleanup +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_54_frag_src; +DROP DATABASE IF EXISTS cloudsync_test_54_frag_dst; +\endif diff --git a/test/postgresql/55_payload_chunks_positional_resume.sql b/test/postgresql/55_payload_chunks_positional_resume.sql new file mode 100644 index 00000000..c1c1f95f --- /dev/null +++ b/test/postgresql/55_payload_chunks_positional_resume.sql @@ -0,0 +1,164 @@ +-- Payload chunks positional-cursor resume +-- +-- Proves the positional cursor on cloudsync_payload_chunks tiles a window exactly: +-- resuming at any chunk's (next_db_version, next_seq, next_frag_offset) reproduces +-- the following chunk byte-for-byte, including boundaries that fall inside a single +-- committed db_version and inside a value larger than the chunk budget. No spool +-- table, no idempotent overlap. +-- +-- Part 2 is end-to-end: drain the whole window the way the /check job will (one +-- chunk per call via the positional cursor), apply that stream to a fresh database, +-- and assert the receiver's table content hashes identically to the source. + +\set testid '55-positional' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_55_positional; +DROP DATABASE IF EXISTS cloudsync_test_55_positional_dst; +CREATE DATABASE cloudsync_test_55_positional; +CREATE DATABASE cloudsync_test_55_positional_dst; + +\connect cloudsync_test_55_positional +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE split_test (id TEXT PRIMARY KEY, body BYTEA DEFAULT '\x'::bytea); +SELECT cloudsync_init('split_test', 'CLS', 1) AS _init \gset +SELECT cloudsync_set('payload_max_chunk_size', '262144'); + +-- tx1: many medium incompressible rows in one statement -> a single db_version +-- split across several chunks (row-boundary resumes, incl. resumes landing INSIDE +-- one committed version that the legacy since>db_version cursor could not express). +INSERT INTO split_test(id, body) +SELECT format('row-%s', lpad(i::text, 4, '0')), + decode((SELECT string_agg(md5((i * 1000 + j)::text), '') FROM generate_series(1, 88) AS s(j)), 'hex') +FROM generate_series(1, 500) AS g(i); + +-- tx2: one value larger than the chunk budget -> v3 fragments (mid-fragment resumes). +INSERT INTO split_test(id, body) +VALUES ('big', decode((SELECT string_agg(md5(j::text), '') FROM generate_series(1, 30000) AS s(j)), 'hex')); + +-- For each non-final chunk of the full-window scan, resume at its reported cursor +-- and fetch the first chunk; it must equal the next chunk of the full scan. The +-- correlated SRF subquery uses ORDER BY ... LIMIT 1 so each resume call drains +-- fully (no early-terminated value-per-call SRF). +WITH base AS ( + SELECT chunk_index, payload, next_db_version, next_seq, next_frag_offset, is_final, watermark_db_version + FROM cloudsync_payload_chunks(0, cloudsync_siteid(), NULL, false) +), +resumed AS ( + SELECT b.chunk_index, + (b.next_frag_offset > 0) AS is_frag_boundary, + (SELECT r.payload + FROM cloudsync_payload_chunks(NULL, cloudsync_siteid(), + (SELECT max(watermark_db_version) FROM base), false, + b.next_db_version, b.next_seq, b.next_frag_offset) r + ORDER BY r.chunk_index LIMIT 1) AS next_payload + FROM base b + WHERE NOT b.is_final +) +SELECT + (SELECT count(*) FROM base) AS base_count, + coalesce((SELECT bool_and(r.next_payload = b2.payload) + FROM resumed r JOIN base b2 ON b2.chunk_index = r.chunk_index + 1), false) AS chunks_identical, + coalesce((SELECT bool_or(is_frag_boundary) FROM resumed), false) AS saw_frag +\gset + +SELECT (:base_count::int >= 4) AS enough_chunks \gset +\if :enough_chunks +\echo [PASS] (:testid) window produced multiple chunks (:base_count) +\else +\echo [FAIL] (:testid) expected a multi-chunk window, got :base_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :chunks_identical +\echo [PASS] (:testid) positional resume reproduced every following chunk byte-for-byte +\else +\echo [FAIL] (:testid) a positional resume did not reproduce the next chunk +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :saw_frag +\echo [PASS] (:testid) mid-fragment resume exercised +\else +\echo [FAIL] (:testid) mid-fragment resume not exercised +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Part 2: end-to-end drain + apply round-trip. +-- +-- Drain the window exactly as the /check job will: start with the legacy +-- exclusive cursor (since=0), then step the positional cursor one chunk per call, +-- collecting payloads in drain order. ORDER BY chunk_index LIMIT 1 forces each +-- value-per-call SRF to run to completion (no early-terminated cursor). The drained +-- chunks are returned hex-joined so they can cross \connect into the receiver DB. +CREATE OR REPLACE FUNCTION _positional_drain_hex() RETURNS text LANGUAGE plpgsql AS $$ +DECLARE + rdbv bigint; rseq bigint; rfrag bigint; wm bigint := 0; + rec record; parts text[] := '{}'; guard int := 0; +BEGIN + LOOP + guard := guard + 1; + IF guard > 100000 THEN RAISE EXCEPTION 'positional drain did not terminate'; END IF; + IF wm = 0 THEN + SELECT * INTO rec FROM cloudsync_payload_chunks(0, cloudsync_siteid(), NULL, false) + ORDER BY chunk_index LIMIT 1; + IF NOT FOUND THEN EXIT; END IF; + wm := rec.watermark_db_version; + ELSE + SELECT * INTO rec FROM cloudsync_payload_chunks(NULL, cloudsync_siteid(), wm, false, rdbv, rseq, rfrag) + ORDER BY chunk_index LIMIT 1; + IF NOT FOUND THEN EXIT; END IF; + END IF; + parts := array_append(parts, encode(rec.payload, 'hex')); + rdbv := rec.next_db_version; rseq := rec.next_seq; rfrag := rec.next_frag_offset; + EXIT WHEN rec.is_final; + END LOOP; + RETURN array_to_string(parts, ','); +END $$; + +SELECT _positional_drain_hex() AS chunks_hex \gset +SELECT + md5(string_agg(id || ':' || encode(body, 'hex'), '|' ORDER BY id)) AS src_hash, + count(*) AS src_count +FROM split_test \gset + +\connect cloudsync_test_55_positional_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE split_test (id TEXT PRIMARY KEY, body BYTEA DEFAULT '\x'::bytea); +SELECT cloudsync_init('split_test', 'CLS', 1) AS _init_dst \gset +SELECT cloudsync_set('payload_max_chunk_size', '262144'); + +-- Reconstitute the drained chunks and apply them (reverse order on purpose: apply +-- must be order-independent and reassemble fragments regardless). +CREATE TEMP TABLE chunk_transport(ord int, payload bytea); +INSERT INTO chunk_transport(ord, payload) +SELECT ord::int, decode(chunk_hex, 'hex') +FROM unnest(string_to_array(:'chunks_hex', ',')) WITH ORDINALITY AS t(chunk_hex, ord); + +SELECT coalesce(sum(cloudsync_payload_apply(payload)), 0) AS applied_rows +FROM (SELECT payload FROM chunk_transport ORDER BY ord DESC) AS ordered \gset + +SELECT + md5(string_agg(id || ':' || encode(body, 'hex'), '|' ORDER BY id)) AS dst_hash, + count(*) AS dst_count +FROM split_test \gset + +SELECT (:'dst_hash' = :'src_hash' AND :dst_count::int = :src_count::int + AND :dst_count::int > 0) AS roundtrip_ok \gset +\if :roundtrip_ok +\echo [PASS] (:testid) positional drain applied to a fresh database reproduces the source (:dst_count rows) +\else +\echo [FAIL] (:testid) drain/apply mismatch (src_count=:src_count dst_count=:dst_count hashes :'src_hash' vs :'dst_hash') +SELECT (:fail::int + 1) AS fail \gset +\endif + +\ir helper_test_cleanup.sql +\if :should_cleanup +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_55_positional; +DROP DATABASE IF EXISTS cloudsync_test_55_positional_dst; +\endif diff --git a/test/postgresql/full_test.sql b/test/postgresql/full_test.sql new file mode 100644 index 00000000..c6f38b71 --- /dev/null +++ b/test/postgresql/full_test.sql @@ -0,0 +1,78 @@ +-- usage: +-- - normal: `psql postgresql://postgres:postgres@localhost:5432/cloudsync_test -f test/postgresql/smoke_test.sql` +-- - debug: `psql -v DEBUG=1 postgresql://postgres:postgres@localhost:5432/cloudsync_test -f test/postgresql/smoke_test.sql` + +\echo 'Running smoke_test...' + +\ir helper_psql_conn_setup.sql +-- \set ON_ERROR_STOP on +\set fail 0 + +\ir 01_unittest.sql +\ir 02_roundtrip.sql +\ir 03_multiple_roundtrip.sql +\ir 04_colversion_skew.sql +\ir 05_delete_recreate_cycle.sql +\ir 06_out_of_order_delivery.sql +\ir 07_delete_vs_update.sql +\ir 08_resurrect_delayed_delete.sql +\ir 09_multicol_concurrent_edits.sql +\ir 10_empty_payload_noop.sql +\ir 11_multi_table_multi_columns_rounds.sql +\ir 12_repeated_table_multi_schemas.sql +\ir 13_per_table_schema_tracking.sql +\ir 14_datatype_roundtrip.sql +\ir 15_datatype_roundtrip_unmapped.sql +\ir 16_composite_pk_text_int_roundtrip.sql +\ir 17_uuid_pk_roundtrip.sql +\ir 18_bulk_insert_performance.sql +\ir 19_uuid_pk_with_unmapped_cols.sql +\ir 20_init_with_existing_data.sql +\ir 21_null_value_sync.sql +\ir 22_null_column_roundtrip.sql +\ir 23_uuid_column_roundtrip.sql +\ir 24_nullable_types_roundtrip.sql +\ir 25_boolean_type_issue.sql +\ir 26_row_filter.sql +\ir 27_rls_batch_merge.sql +\ir 28_db_version_tracking.sql +\ir 29_rls_multicol.sql +\ir 30_null_prikey_insert.sql +\ir 31_alter_table_sync.sql +\ir 32_block_lww.sql +\ir 33_block_lww_extended.sql +\ir 34_block_lww_advanced.sql +\ir 35_block_lww_edge_cases.sql +\ir 36_block_lww_round3.sql +\ir 37_block_lww_round4.sql +\ir 38_block_lww_round5.sql +\ir 39_concurrent_write_apply.sql +\ir 40_unsupported_algorithms.sql +\ir 41_corrupted_payload.sql +\ir 42_payload_idempotency.sql +\ir 43_delete_resurrect_ordering.sql +\ir 44_large_composite_pk.sql +\ir 45_pg_specific_types.sql +\ir 46_schema_hash_mismatch.sql +\ir 47_row_filter_advanced.sql +\ir 48_row_filter_multi_table.sql +\ir 49_row_filter_prefill.sql +\ir 50_block_lww_existing_data.sql +\ir 51_stale_table_settings_dropped_meta.sql +\ir 52_payload_chunks.sql +\ir 53_payload_blob_checked_pg_try.sql +\ir 54_payload_chunks_fragment_state.sql +\ir 55_payload_chunks_positional_resume.sql + +-- 'Test summary' +\echo '\nTest summary:' +\echo - Failures: :fail +SELECT (:fail::int > 0) AS fail_any \gset +\if :fail_any +\echo smoke test failed: :fail test(s) failed +DO $$ BEGIN + RAISE EXCEPTION 'smoke test failed'; +END $$; +\else +\echo - Status: OK +\endif diff --git a/test/postgresql/helper_psql_conn_setup.sql b/test/postgresql/helper_psql_conn_setup.sql new file mode 100644 index 00000000..3205be43 --- /dev/null +++ b/test/postgresql/helper_psql_conn_setup.sql @@ -0,0 +1,11 @@ +\if :{?DEBUG} +\set QUIET 0 +SET client_min_messages = debug1; SET log_min_messages = debug1; SET log_error_verbosity = verbose; +\pset tuples_only off +\pset format aligned +\else +\set QUIET 1 +SET client_min_messages = warning; SET log_min_messages = warning; +\pset tuples_only on +\pset format unaligned +\endif diff --git a/test/postgresql/helper_test_cleanup.sql b/test/postgresql/helper_test_cleanup.sql new file mode 100644 index 00000000..f7f73581 --- /dev/null +++ b/test/postgresql/helper_test_cleanup.sql @@ -0,0 +1,24 @@ +-- Test Cleanup Helper +-- This script should be included at the end of each test file +-- It determines if cleanup should happen based on DEBUG mode and test failures +-- Usage: +-- \ir helper_test_cleanup.sql +-- \if :should_cleanup +-- DROP DATABASE IF EXISTS your_test_db; +-- \endif + +\connect postgres + +-- Determine if we should cleanup +\if :{?DEBUG} +\set should_cleanup false +\echo [INFO] (:testid) DEBUG mode enabled - keeping test databases for inspection +\else +SELECT (:fail::int > :initial_fail::int) AS has_test_failures \gset +\if :has_test_failures +\set should_cleanup false +\echo [INFO] (:testid) Test failures detected - keeping test databases for inspection +\else +\set should_cleanup true +\endif +\endif diff --git a/test/postgresql/helper_test_init.sql b/test/postgresql/helper_test_init.sql new file mode 100644 index 00000000..e7b2b4db --- /dev/null +++ b/test/postgresql/helper_test_init.sql @@ -0,0 +1,13 @@ +-- Test Initialization Helper +-- This script should be included at the beginning of each test file after setting testid +-- Usage: \ir helper_test_init.sql + +-- Initialize fail counter if not already set +\if :{?fail} +-- fail counter already exists from previous tests +\else +\set fail 0 +\endif + +-- Store initial fail count to detect failures in this test +SELECT :fail::int AS initial_fail \gset diff --git a/test/sync_bench.c b/test/sync_bench.c new file mode 100644 index 00000000..5fd496aa --- /dev/null +++ b/test/sync_bench.c @@ -0,0 +1,688 @@ +// +// sync_bench.c +// cloudsync +// +// Measures end-to-end sync latency from one local SQLite database to another. +// + +#include +#include +#include +#include +#ifdef CLOUDSYNC_NETWORK_TRACE +#include +#endif +#include "sqlite3.h" +#include "utils.h" + +#ifdef _WIN32 +#include +#else +#include +#endif + +#define DB_A_PATH "dist/sync-bench-a.sqlite" +#define DB_B_PATH "dist/sync-bench-b.sqlite" +#define EXT_PATH "./dist/cloudsync" +#define DEFAULT_POLL_DELAY_MS 250 +#define DEFAULT_MAX_POLLS 40 +#define DEFAULT_RANDOM_BLOB_SIZE_BYTES (100 * 1024) +#define DEFAULT_CLEANUP_OLDER_THAN_SECONDS (24 * 60 * 60) + +typedef struct { + const char *operation; + int attempt; + int sqlite_rc; + int rows_received; + double started_ms; + double ended_ms; + double elapsed_ms; + char *result_json; +} sync_bench_request; + +typedef struct { + int local_version; + int server_version; + char *status; +} sync_bench_send_summary; + +#ifdef CLOUDSYNC_NETWORK_TRACE +static void bench_trace(const char *fmt, ...) { + va_list args; + va_start(args, fmt); + fprintf(stderr, "[sync-bench] "); + vfprintf(stderr, fmt, args); + fprintf(stderr, "\n"); + va_end(args); +} +#else +#define bench_trace(...) ((void)0) +#endif + +static double monotonic_ms(void) { +#ifdef _WIN32 + LARGE_INTEGER freq; + LARGE_INTEGER counter; + QueryPerformanceFrequency(&freq); + QueryPerformanceCounter(&counter); + return ((double)counter.QuadPart * 1000.0) / (double)freq.QuadPart; +#else + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((double)ts.tv_sec * 1000.0) + ((double)ts.tv_nsec / 1000000.0); +#endif +} + +static char *str_dup(const char *value) { + if (!value) value = ""; + size_t len = strlen(value); + char *copy = (char *)malloc(len + 1); + if (!copy) return NULL; + memcpy(copy, value, len + 1); + return copy; +} + +static int env_int(const char *name, int default_value) { + const char *value = getenv(name); + if (!value || !*value) return default_value; + char *end = NULL; + long parsed = strtol(value, &end, 10); + if (!end || *end != '\0' || parsed < 0 || parsed > 1000000) return default_value; + return (int)parsed; +} + +static int db_exec(sqlite3 *db, const char *sql) { + char *errmsg = NULL; + int rc = sqlite3_exec(db, sql, NULL, NULL, &errmsg); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while executing %s: %s\n", sql, errmsg ? errmsg : sqlite3_errmsg(db)); + sqlite3_free(errmsg); + } + return rc; +} + +static int query_text(sqlite3 *db, const char *sql, char **out) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while preparing %s: %s\n", sql, sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + const unsigned char *value = sqlite3_column_text(stmt, 0); + *out = str_dup((const char *)value); + if (!*out) rc = SQLITE_NOMEM; + else rc = SQLITE_OK; + } else if (rc == SQLITE_DONE) { + *out = NULL; + rc = SQLITE_OK; + } else { + fprintf(stderr, "Error while stepping %s: %s\n", sql, sqlite3_errmsg(db)); + } + + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK && finalize_rc != SQLITE_OK) rc = finalize_rc; + return rc; +} + +static int timed_query_text(sqlite3 *db, const char *sql, char **out, double *started_ms, double *ended_ms) { + *started_ms = monotonic_ms(); + int rc = query_text(db, sql, out); + *ended_ms = monotonic_ms(); + return rc; +} + +// Runs a cloudsync_network_* scalar function. In trace mode it captures and prints +// the JSON the function returns; otherwise it executes without keeping the result. +static int db_exec_network(sqlite3 *db, const char *operation, const char *sql) { +#ifdef CLOUDSYNC_NETWORK_TRACE + char *result = NULL; + int rc = query_text(db, sql, &result); + bench_trace("response op=%s json=%s", operation, result ? result : "(null)"); + free(result); + return rc; +#else + (void)operation; + return db_exec(db, sql); +#endif +} + +static int open_load_ext(const char *db_path, sqlite3 **out_db) { + bench_trace("step=open-load-extension db_path=%s begin", db_path); + sqlite3 *db = NULL; + int rc = sqlite3_open(db_path, &db); + if (rc != SQLITE_OK) { + fprintf(stderr, "Unable to open %s: %s\n", db_path, db ? sqlite3_errmsg(db) : "unknown error"); + if (db) sqlite3_close(db); + bench_trace("step=open-load-extension db_path=%s end rc=%d", db_path, rc); + return rc; + } + + rc = sqlite3_enable_load_extension(db, 1); + if (rc != SQLITE_OK) { + fprintf(stderr, "Unable to enable load_extension for %s: %s\n", db_path, sqlite3_errmsg(db)); + sqlite3_close(db); + bench_trace("step=open-load-extension db_path=%s end rc=%d", db_path, rc); + return rc; + } + + rc = db_exec(db, "SELECT load_extension('" EXT_PATH "');"); + if (rc != SQLITE_OK) { + sqlite3_close(db); + bench_trace("step=open-load-extension db_path=%s end rc=%d", db_path, rc); + return rc; + } + + *out_db = db; + bench_trace("step=open-load-extension db_path=%s end rc=%d", db_path, SQLITE_OK); + return SQLITE_OK; +} + +static int init_schema(sqlite3 *db, const char *label) { + bench_trace("step=init-schema db=%s begin", label); + int rc = db_exec(db, + "CREATE TABLE IF NOT EXISTS sync_bench_items (" + "id TEXT PRIMARY KEY NOT NULL," + "payload TEXT NOT NULL DEFAULT ''," + "marker TEXT NOT NULL DEFAULT ''," + "random_blob BLOB NOT NULL DEFAULT X''," + "updated_at TEXT NOT NULL DEFAULT ''" + ");"); + if (rc != SQLITE_OK) { + bench_trace("step=init-schema db=%s end rc=%d", label, rc); + return rc; + } + + rc = db_exec(db, "SELECT cloudsync_init('sync_bench_items');"); + bench_trace("step=init-schema db=%s end rc=%d", label, rc); + return rc; +} + +static int init_network(sqlite3 *db, const char *label, const char *database_id, const char *address, const char *apikey) { + char sql[2048]; + if (address && *address) { + snprintf(sql, sizeof(sql), "SELECT cloudsync_network_init_custom('%s', '%s');", address, database_id); + } else { + snprintf(sql, sizeof(sql), "SELECT cloudsync_network_init('%s');", database_id); + } + bench_trace("step=network-init db=%s begin mode=%s", label, (address && *address) ? "custom-address" : "default-address"); + int rc = db_exec(db, sql); + bench_trace("step=network-init db=%s end rc=%d", label, rc); + if (rc != SQLITE_OK) return rc; + + if (apikey && *apikey) { + bench_trace("step=set-apikey db=%s begin", label); + snprintf(sql, sizeof(sql), "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, sql); + bench_trace("step=set-apikey db=%s end rc=%d", label, rc); + if (rc != SQLITE_OK) return rc; + } + + // Drain any pre-existing backlog so the measured send->apply later starts from a + // clean baseline. This is untimed warm-up, not part of the latency measurement. + bench_trace("step=pre-measure-sync db=%s begin sql=cloudsync_network_sync(500,4)", label); + rc = db_exec_network(db, "pre-measure-sync", "SELECT cloudsync_network_sync(500, 4);"); + bench_trace("step=pre-measure-sync db=%s end rc=%d", label, rc); + return rc; +} + +static int setup_database(const char *label, const char *path, const char *database_id, const char *address, const char *apikey, sqlite3 **out_db) { + bench_trace("step=setup-database db=%s path=%s begin", label, path); + int rc = open_load_ext(path, out_db); + if (rc != SQLITE_OK) { + bench_trace("step=setup-database db=%s end rc=%d", label, rc); + return rc; + } + + rc = init_schema(*out_db, label); + if (rc != SQLITE_OK) { + bench_trace("step=setup-database db=%s end rc=%d", label, rc); + return rc; + } + + rc = init_network(*out_db, label, database_id, address, apikey); + bench_trace("step=setup-database db=%s end rc=%d", label, rc); + return rc; +} + +static int verify_row(sqlite3 *db, const char *id, const char *payload, const char *marker, + const void *random_blob, int random_blob_size, bool *verified) { + sqlite3_stmt *stmt = NULL; + const char *sql = "SELECT payload, marker, random_blob FROM sync_bench_items WHERE id = ?;"; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while preparing verification query: %s\n", sqlite3_errmsg(db)); + return rc; + } + + sqlite3_bind_text(stmt, 1, id, -1, SQLITE_TRANSIENT); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + const char *actual_payload = (const char *)sqlite3_column_text(stmt, 0); + const char *actual_marker = (const char *)sqlite3_column_text(stmt, 1); + const void *actual_blob = sqlite3_column_blob(stmt, 2); + int actual_blob_size = sqlite3_column_bytes(stmt, 2); + bool blob_matches = (actual_blob_size == random_blob_size) && + (random_blob_size == 0 || + (actual_blob && random_blob && memcmp(actual_blob, random_blob, (size_t)random_blob_size) == 0)); + *verified = actual_payload && actual_marker && + strcmp(actual_payload, payload) == 0 && + strcmp(actual_marker, marker) == 0 && + blob_matches; + rc = SQLITE_OK; + } else if (rc == SQLITE_DONE) { + *verified = false; + rc = SQLITE_OK; + } else { + fprintf(stderr, "Error while verifying row: %s\n", sqlite3_errmsg(db)); + } + + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK && finalize_rc != SQLITE_OK) rc = finalize_rc; + return rc; +} + +static int insert_benchmark_row(sqlite3 *db, const char *id, const char *payload, const char *marker, + const void *random_blob, int random_blob_size) { + bench_trace("step=insert-source-row db=db_a row_id=%s random_blob_size_bytes=%d begin", id, random_blob_size); + sqlite3_stmt *stmt = NULL; + const char *sql = "INSERT INTO sync_bench_items (id, payload, marker, random_blob, updated_at) VALUES (?, ?, ?, ?, datetime('now'));"; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while preparing insert: %s\n", sqlite3_errmsg(db)); + bench_trace("step=insert-source-row db=db_a row_id=%s end rc=%d", id, rc); + return rc; + } + + rc = sqlite3_bind_text(stmt, 1, id, -1, SQLITE_TRANSIENT); + if (rc == SQLITE_OK) rc = sqlite3_bind_text(stmt, 2, payload, -1, SQLITE_TRANSIENT); + if (rc == SQLITE_OK) rc = sqlite3_bind_text(stmt, 3, marker, -1, SQLITE_TRANSIENT); + if (rc == SQLITE_OK) rc = sqlite3_bind_blob(stmt, 4, random_blob, random_blob_size, SQLITE_TRANSIENT); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while binding benchmark row: %s\n", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + bench_trace("step=insert-source-row db=db_a row_id=%s random_blob_size_bytes=%d end rc=%d", id, random_blob_size, rc); + return rc; + } + + rc = sqlite3_step(stmt); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + else fprintf(stderr, "Error while inserting benchmark row: %s\n", sqlite3_errmsg(db)); + + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK && finalize_rc != SQLITE_OK) rc = finalize_rc; + bench_trace("step=insert-source-row db=db_a row_id=%s random_blob_size_bytes=%d end rc=%d", id, random_blob_size, rc); + return rc; +} + +static int cleanup_old_benchmark_rows(sqlite3 *db, int older_than_seconds, int *deleted_count) { + if (deleted_count) *deleted_count = 0; + if (older_than_seconds <= 0) { + bench_trace("step=cleanup-old-source-rows db=db_a enabled=false"); + return SQLITE_OK; + } + + char modifier[64]; + snprintf(modifier, sizeof(modifier), "-%d seconds", older_than_seconds); + + bench_trace("step=cleanup-old-source-rows db=db_a older_than_seconds=%d begin", older_than_seconds); + sqlite3_stmt *stmt = NULL; + const char *sql = + "DELETE FROM sync_bench_items " + "WHERE marker LIKE 'sync-bench-%' " + "AND updated_at < datetime('now', ?);"; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while preparing cleanup delete: %s\n", sqlite3_errmsg(db)); + bench_trace("step=cleanup-old-source-rows db=db_a end rc=%d deleted=0", rc); + return rc; + } + + rc = sqlite3_bind_text(stmt, 1, modifier, -1, SQLITE_TRANSIENT); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while binding cleanup delete: %s\n", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + bench_trace("step=cleanup-old-source-rows db=db_a end rc=%d deleted=0", rc); + return rc; + } + + rc = sqlite3_step(stmt); + if (rc == SQLITE_DONE) { + rc = SQLITE_OK; + if (deleted_count) *deleted_count = sqlite3_changes(db); + } else { + fprintf(stderr, "Error while deleting old benchmark rows: %s\n", sqlite3_errmsg(db)); + } + + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK && finalize_rc != SQLITE_OK) rc = finalize_rc; + bench_trace("step=cleanup-old-source-rows db=db_a end rc=%d deleted=%d", rc, deleted_count ? *deleted_count : 0); + return rc; +} + +static int json_int_at_path(sqlite3 *db, const char *json, const char *path, int default_value) { + sqlite3_stmt *stmt = NULL; + int value = default_value; + int rc = sqlite3_prepare_v2(db, "SELECT json_extract(?, ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) return default_value; + sqlite3_bind_text(stmt, 1, json ? json : "", -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, path, -1, SQLITE_TRANSIENT); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW && sqlite3_column_type(stmt, 0) != SQLITE_NULL) value = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + return value; +} + +static char *json_text_at_path(sqlite3 *db, const char *json, const char *path) { + sqlite3_stmt *stmt = NULL; + char *value = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT json_extract(?, ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) return NULL; + sqlite3_bind_text(stmt, 1, json ? json : "", -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, path, -1, SQLITE_TRANSIENT); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW && sqlite3_column_type(stmt, 0) != SQLITE_NULL) { + value = str_dup((const char *)sqlite3_column_text(stmt, 0)); + } + sqlite3_finalize(stmt); + return value; +} + +static int timed_request(sqlite3 *db, sync_bench_request *request, const char *operation, int attempt, const char *sql) { + request->operation = operation; + request->attempt = attempt; + request->rows_received = -1; + request->result_json = NULL; + request->sqlite_rc = timed_query_text(db, sql, &request->result_json, &request->started_ms, &request->ended_ms); + request->elapsed_ms = request->ended_ms - request->started_ms; + if (strcmp(operation, "check") == 0 && request->result_json) { + request->rows_received = json_int_at_path(db, request->result_json, "$.receive.rows", -1); + } + // Surface the raw JSON returned by the network function (send/check) under trace. + bench_trace("response op=%s attempt=%d json=%s", operation, attempt, + request->result_json ? request->result_json : "(null)"); + return request->sqlite_rc; +} + +static void json_print_escaped(const char *value) { + putchar('"'); + if (value) { + for (const unsigned char *p = (const unsigned char *)value; *p; p++) { + switch (*p) { + case '\\': printf("\\\\"); break; + case '"': printf("\\\""); break; + case '\n': printf("\\n"); break; + case '\r': printf("\\r"); break; + case '\t': printf("\\t"); break; + default: + if (*p < 0x20) printf("\\u%04x", *p); + else putchar(*p); + } + } + } + putchar('"'); +} + +static void print_text_report(const char *database_id, int poll_delay_ms, int max_polls, int random_blob_size, + int cleanup_older_than_seconds, int cleanup_deleted_rows, + const char *row_id, bool applied, + int polls, double total_ms, double verify_ms, double request_ms, double poll_sleep_ms, + double measured_overhead_ms, sync_bench_send_summary send_summary, + sync_bench_request *requests, int request_count) { + printf("\nSync Performance Benchmark\n"); + printf("database_id: %s\n", database_id); + printf("poll_delay_ms: %d\n", poll_delay_ms); + printf("max_polls: %d\n", max_polls); + printf("random_blob_size_bytes: %d\n", random_blob_size); + printf("cleanup_older_than_seconds: %d\n", cleanup_older_than_seconds); + printf("cleanup_deleted_rows: %d\n", cleanup_deleted_rows); + printf("row_id: %s\n", row_id); + printf("\nRequests:\n"); + for (int i = 0; i < request_count; i++) { + if (strcmp(requests[i].operation, "check") == 0) { + printf("%s[%d] %.2f ms rc=%d rows=%d\n", requests[i].operation, requests[i].attempt, + requests[i].elapsed_ms, requests[i].sqlite_rc, requests[i].rows_received); + } else { + printf("%s[%d] %.2f ms rc=%d status=%s localVersion=%d serverVersion=%d\n", + requests[i].operation, requests[i].attempt, requests[i].elapsed_ms, requests[i].sqlite_rc, + send_summary.status ? send_summary.status : "unknown", + send_summary.local_version, send_summary.server_version); + } + } + printf("\nResult:\n"); + printf("applied: %s\n", applied ? "true" : "false"); + printf("polls: %d\n", polls); + printf("total_send_to_apply_check_end_ms: %.2f\n", total_ms); + printf("network_request_elapsed_ms: %.2f\n", request_ms); + printf("poll_sleep_elapsed_ms: %.2f\n", poll_sleep_ms); + printf("local_overhead_elapsed_ms: %.2f\n", measured_overhead_ms); + printf("verification_select_ms: %.2f\n", verify_ms); +} + +static void print_json_report(int poll_delay_ms, int max_polls, int random_blob_size, + int cleanup_older_than_seconds, int cleanup_deleted_rows, + const char *row_id, bool applied, + int polls, double total_ms, double verify_ms, double request_ms, double poll_sleep_ms, + double measured_overhead_ms, sync_bench_send_summary send_summary, + sync_bench_request *requests, int request_count) { + printf("{\n"); + printf(" \"applied\": %s,\n", applied ? "true" : "false"); + printf(" \"pollDelayMs\": %d,\n", poll_delay_ms); + printf(" \"maxPolls\": %d,\n", max_polls); + printf(" \"randomBlobSizeBytes\": %d,\n", random_blob_size); + printf(" \"cleanupOlderThanSeconds\": %d,\n", cleanup_older_than_seconds); + printf(" \"cleanupDeletedRows\": %d,\n", cleanup_deleted_rows); + printf(" \"polls\": %d,\n", polls); + printf(" \"rowId\": "); json_print_escaped(row_id); printf(",\n"); + printf(" \"totalSendToApplyCheckEndMs\": %.2f,\n", total_ms); + printf(" \"networkRequestElapsedMs\": %.2f,\n", request_ms); + printf(" \"pollSleepElapsedMs\": %.2f,\n", poll_sleep_ms); + printf(" \"localOverheadElapsedMs\": %.2f,\n", measured_overhead_ms); + printf(" \"verificationSelectMs\": %.2f,\n", verify_ms); + printf(" \"send\": {\"status\": "); json_print_escaped(send_summary.status); + printf(", \"localVersion\": %d, \"serverVersion\": %d},\n", send_summary.local_version, send_summary.server_version); + printf(" \"requests\": [\n"); + for (int i = 0; i < request_count; i++) { + printf(" {\"operation\": "); json_print_escaped(requests[i].operation); + printf(", \"attempt\": %d, \"sqliteRc\": %d, \"elapsedMs\": %.2f", requests[i].attempt, + requests[i].sqlite_rc, requests[i].elapsed_ms); + if (strcmp(requests[i].operation, "check") == 0) printf(", \"rows\": %d", requests[i].rows_received); + printf(", \"result\": "); json_print_escaped(requests[i].result_json); + printf("}%s\n", i + 1 == request_count ? "" : ","); + } + printf(" ]\n"); + printf("}\n"); +} + +static void free_requests(sync_bench_request *requests, int request_count) { + for (int i = 0; i < request_count; i++) free(requests[i].result_json); +} + +int main(void) { + int rc = SQLITE_OK; + sqlite3 *db_a = NULL; + sqlite3 *db_b = NULL; + sync_bench_request *requests = NULL; + int request_count = 0; + bool applied = false; + int polls = 0; + double total_ms = 0.0; + double verify_ms = 0.0; + double poll_sleep_ms = 0.0; + double request_ms = 0.0; + double measured_overhead_ms = 0.0; + sync_bench_send_summary send_summary = {-1, -1, NULL}; + int cleanup_deleted_rows = 0; + char row_id[UUID_STR_MAXLEN] = ""; + char marker[96] = ""; + char payload[128] = ""; + unsigned char empty_blob = 0; + void *random_blob = NULL; + + const char *database_id = getenv("SYNC_BENCH_DATABASE_ID"); + const char *address = getenv("SYNC_BENCH_CLOUDSYNC_ADDRESS"); + const char *apikey = getenv("SYNC_BENCH_APIKEY"); + const char *output = getenv("SYNC_BENCH_OUTPUT"); + int poll_delay_ms = env_int("SYNC_BENCH_POLL_DELAY_MS", DEFAULT_POLL_DELAY_MS); + int max_polls = env_int("SYNC_BENCH_MAX_POLLS", DEFAULT_MAX_POLLS); + int random_blob_size = env_int("SYNC_BENCH_RANDOM_BLOB_SIZE_BYTES", DEFAULT_RANDOM_BLOB_SIZE_BYTES); + int cleanup_older_than_seconds = env_int("SYNC_BENCH_CLEANUP_OLDER_THAN_SECONDS", DEFAULT_CLEANUP_OLDER_THAN_SECONDS); + + if (!database_id || !*database_id) { + fprintf(stderr, "Error: SYNC_BENCH_DATABASE_ID not set.\n"); + return SQLITE_MISUSE; + } + + requests = (sync_bench_request *)calloc((size_t)max_polls + 1, sizeof(sync_bench_request)); + if (!requests) return SQLITE_NOMEM; + + remove(DB_A_PATH); + remove(DB_B_PATH); + cloudsync_memory_init(1); + + // Step 1 - setup: open both local databases (sender A, receiver B), load the + // extension, create the schema, attach to the network, and drain to a clean baseline. + bench_trace("step=benchmark-setup begin database_id=%s poll_delay_ms=%d max_polls=%d", database_id, poll_delay_ms, max_polls); + rc = setup_database("db_a", DB_A_PATH, database_id, address, apikey, &db_a); + if (rc != SQLITE_OK) goto cleanup; + rc = setup_database("db_b", DB_B_PATH, database_id, address, apikey, &db_b); + if (rc != SQLITE_OK) goto cleanup; + bench_trace("step=benchmark-setup end rc=%d", rc); + + // Prune stale rows from prior runs and push those deletions so they don't pollute + // the receiver's backlog during the measured round-trip. + rc = cleanup_old_benchmark_rows(db_a, cleanup_older_than_seconds, &cleanup_deleted_rows); + if (rc != SQLITE_OK) goto cleanup; + if (cleanup_deleted_rows > 0) { + bench_trace("step=cleanup-send db=db_a deleted=%d begin sql=cloudsync_network_send_changes", cleanup_deleted_rows); + rc = db_exec_network(db_a, "cleanup-send", "SELECT cloudsync_network_send_changes();"); + bench_trace("step=cleanup-send db=db_a deleted=%d end rc=%d", cleanup_deleted_rows, rc); + if (rc != SQLITE_OK) goto cleanup; + } + + // Step 3 - build the unique benchmark row: a UUIDv7 id plus an incompressible + // random blob, so the payload survives compression and exercises a realistic size. + cloudsync_uuid_v7_string(row_id, true); + snprintf(marker, sizeof(marker), "sync-bench-%s", row_id); + snprintf(payload, sizeof(payload), "payload-%s", row_id); + + if (random_blob_size > 0) { + random_blob = malloc((size_t)random_blob_size); + if (!random_blob) { + rc = SQLITE_NOMEM; + goto cleanup; + } + sqlite3_randomness(random_blob_size, random_blob); + } else { + random_blob = &empty_blob; + } + + // Insert the row on the sender. This is the change whose propagation we time. + rc = insert_benchmark_row(db_a, row_id, payload, marker, random_blob, random_blob_size); + if (rc != SQLITE_OK) goto cleanup; + + // Step 4 - precondition: the row must not already exist on the receiver, or the + // measurement would be meaningless. + bench_trace("step=verify-before-send db=db_b row_id=%s begin", row_id); + rc = verify_row(db_b, row_id, payload, marker, random_blob, random_blob_size, &applied); + bench_trace("step=verify-before-send db=db_b row_id=%s end rc=%d applied=%s", row_id, rc, applied ? "true" : "false"); + if (rc != SQLITE_OK) goto cleanup; + if (applied) { + fprintf(stderr, "Error: benchmark row already exists in receiver before send.\n"); + rc = SQLITE_ERROR; + goto cleanup; + } + + // Step 5 - measured send: push the row from A and capture the send summary + // (status / localVersion / serverVersion). total_start_ms anchors the latency clock. + bench_trace("step=send db=db_a row_id=%s begin sql=cloudsync_network_send_changes", row_id); + rc = timed_request(db_a, &requests[request_count++], "send", 1, "SELECT cloudsync_network_send_changes();"); + bench_trace("step=send db=db_a row_id=%s end rc=%d elapsed_ms=%.2f", row_id, rc, requests[request_count - 1].elapsed_ms); + if (rc != SQLITE_OK) goto cleanup; + send_summary.status = json_text_at_path(db_a, requests[0].result_json, "$.send.status"); + send_summary.local_version = json_int_at_path(db_a, requests[0].result_json, "$.send.localVersion", -1); + send_summary.server_version = json_int_at_path(db_a, requests[0].result_json, "$.send.serverVersion", -1); + double total_start_ms = requests[0].started_ms; + + // Step 6 - poll loop: receive on B (sleep between attempts) until the row is + // applied and verified, or max_polls is exhausted. + for (int i = 0; i < max_polls; i++) { + if (i > 0 && poll_delay_ms > 0) { + bench_trace("step=poll-sleep attempt=%d delay_ms=%d begin", i + 1, poll_delay_ms); + double sleep_start_ms = monotonic_ms(); + sqlite3_sleep(poll_delay_ms); + double sleep_elapsed_ms = monotonic_ms() - sleep_start_ms; + poll_sleep_ms += sleep_elapsed_ms; + bench_trace("step=poll-sleep attempt=%d end elapsed_ms=%.2f", i + 1, sleep_elapsed_ms); + } + bench_trace("step=check db=db_b attempt=%d row_id=%s begin sql=cloudsync_network_receive_changes", i + 1, row_id); + rc = timed_request(db_b, &requests[request_count++], "check", i + 1, "SELECT cloudsync_network_receive_changes();"); + polls = i + 1; + bench_trace("step=check db=db_b attempt=%d row_id=%s end rc=%d rows=%d elapsed_ms=%.2f", i + 1, row_id, rc, requests[request_count - 1].rows_received, requests[request_count - 1].elapsed_ms); + if (rc != SQLITE_OK) goto cleanup; + + bench_trace("step=verify-after-check db=db_b attempt=%d row_id=%s begin", i + 1, row_id); + double verify_start_ms = monotonic_ms(); + rc = verify_row(db_b, row_id, payload, marker, random_blob, random_blob_size, &applied); + double verify_end_ms = monotonic_ms(); + verify_ms = verify_end_ms - verify_start_ms; + bench_trace("step=verify-after-check db=db_b attempt=%d row_id=%s end rc=%d applied=%s elapsed_ms=%.2f", i + 1, row_id, rc, applied ? "true" : "false", verify_ms); + if (rc != SQLITE_OK) goto cleanup; + + if (applied) { + total_ms = requests[request_count - 1].ended_ms - total_start_ms; + break; + } + } + + if (!applied) { + total_ms = request_count > 0 ? requests[request_count - 1].ended_ms - requests[0].started_ms : 0.0; + rc = SQLITE_BUSY; + } + + // Step 7 - aggregate timings: split the end-to-end latency into time spent in + // network requests, in poll sleeps, and the remaining local overhead. + for (int i = 0; i < request_count; i++) request_ms += requests[i].elapsed_ms; + measured_overhead_ms = total_ms - request_ms - poll_sleep_ms; + if (measured_overhead_ms < 0.0 && measured_overhead_ms > -0.01) measured_overhead_ms = 0.0; + +cleanup: + bench_trace("step=report begin rc=%d applied=%s request_count=%d", rc, applied ? "true" : "false", request_count); + if (output && strcmp(output, "json") == 0) { + print_json_report(poll_delay_ms, max_polls, random_blob_size, cleanup_older_than_seconds, cleanup_deleted_rows, + row_id, applied, polls, total_ms, verify_ms, + request_ms, poll_sleep_ms, measured_overhead_ms, send_summary, + requests, request_count); + } else { + print_text_report(database_id, poll_delay_ms, max_polls, random_blob_size, cleanup_older_than_seconds, cleanup_deleted_rows, + row_id, applied, polls, total_ms, verify_ms, + request_ms, poll_sleep_ms, measured_overhead_ms, send_summary, + requests, request_count); + } + bench_trace("step=report end rc=%d", rc); + + if (!applied && rc == SQLITE_BUSY) { + fprintf(stderr, "Error: row was not applied to receiver after %d polls.\n", max_polls); + } + + if (db_a) { + bench_trace("step=terminate db=db_a begin"); + db_exec(db_a, "SELECT cloudsync_terminate();"); + sqlite3_close(db_a); + bench_trace("step=terminate db=db_a end"); + } + if (db_b) { + bench_trace("step=terminate db=db_b begin"); + db_exec(db_b, "SELECT cloudsync_terminate();"); + sqlite3_close(db_b); + bench_trace("step=terminate db=db_b end"); + } + free_requests(requests, request_count); + free(send_summary.status); + if (random_blob && random_blob != &empty_blob) free(random_blob); + free(requests); + cloudsync_memory_finalize(); + return rc == SQLITE_OK ? 0 : rc; +} diff --git a/test/unit.c b/test/unit.c index f21614fa..ca7d6fc6 100644 --- a/test/unit.c +++ b/test/unit.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "sqlite3.h" @@ -21,28 +22,32 @@ #include "pk.h" #include "dbutils.h" +#include "database.h" #include "cloudsync.h" -#include "cloudsync_private.h" +#include "cloudsync_sqlite.h" // declared only if macro CLOUDSYNC_UNITTEST is defined extern char *OUT_OF_MEMORY_BUFFER; extern bool force_vtab_filter_abort; extern bool force_uncompressed_blob; -// private prototypes -sqlite3_stmt *stmt_reset (sqlite3_stmt *stmt); -int stmt_count (sqlite3_stmt *stmt, const char *value, size_t len, int type); -int stmt_execute (sqlite3_stmt *stmt, void *data); +void dbvm_reset (dbvm_t *stmt); +int dbvm_count (dbvm_t *stmt, const char *value, size_t len, int type); +int dbvm_execute (dbvm_t *stmt, void *data); -sqlite3_int64 dbutils_select (sqlite3 *db, const char *sql, const char **values, int types[], int lens[], int count, int expected_type); +int dbutils_settings_get_value (cloudsync_context *data, const char *key, char *buffer, size_t *blen, int64_t *intvalue); int dbutils_settings_table_load_callback (void *xdata, int ncols, char **values, char **names); -int dbutils_settings_check_version (sqlite3 *db, const char *version); -bool dbutils_migrate (sqlite3 *db); -const char *opname_from_value (int value); -int colname_is_legal (const char *name); -int binary_comparison (int x, int y); +int dbutils_settings_check_version (cloudsync_context *data, const char *version); +bool dbutils_settings_migrate (cloudsync_context *data); +const char *vtab_opname_from_value (int value); +int vtab_colname_is_legal (const char *name); +int dbutils_binary_comparison (int x, int y); sqlite3 *do_create_database (void); +int cloudsync_table_sanity_check (cloudsync_context *data, const char *name, CLOUDSYNC_INIT_FLAG init_flags); +bool database_system_exists (cloudsync_context *data, const char *name, const char *type); +int cloudsync_dbversion_rebuild (cloudsync_context *data); + static int stdout_backup = -1; // Backup file descriptor for stdout static int dev_null_fd = -1; // File descriptor for /dev/null static int test_counter = 1; @@ -88,6 +93,136 @@ static const char *query_changes = "QUERY_CHANGES"; // MARK: - +typedef struct { + int type; + int len; + int rc; + union { + sqlite3_int64 intValue; + double doubleValue; + char *stringValue; + } value; +} DATABASE_RESULT; + +DATABASE_RESULT unit_exec (cloudsync_context *data, const char *sql, const char **values, int types[], int lens[], int count, DATABASE_RESULT results[], int expected_types[], int result_count) { + DEBUG_DBFUNCTION("unit_exec %s", sql); + + sqlite3_stmt *pstmt = NULL; + bool is_write = (result_count == 0); + int type = 0; + + // compile sql + int rc = databasevm_prepare(data, sql, (void **)&pstmt, 0); + if (rc != SQLITE_OK) goto unitexec_finalize; + // check bindings + for (int i=0; iin_savepoint == true) { - if (s->is_approved) rc = sqlite3_exec(db, "RELEASE unittest_payload_apply_transaction", NULL, NULL, NULL); - else rc = sqlite3_exec(db, "ROLLBACK TO unittest_payload_apply_transaction; RELEASE unittest_payload_apply_transaction", NULL, NULL, NULL); - if (rc == SQLITE_OK) s->in_savepoint = false; - } - if (create_new) { - rc = sqlite3_exec(db, "SAVEPOINT unittest_payload_apply_transaction", NULL, NULL, NULL); - if (rc == SQLITE_OK) s->in_savepoint = true; - } - return rc; -} - -bool unittest_payload_apply_rls_callback(void **xdata, cloudsync_pk_decode_bind_context *d, sqlite3 *db, cloudsync_context *data, int step, int rc) { - bool is_approved = false; - unittest_payload_apply_rls_status *s; - if (*xdata) { - s = (unittest_payload_apply_rls_status *)*xdata; - } else { - s = cloudsync_memory_zeroalloc(sizeof(unittest_payload_apply_rls_status)); - s->is_approved = true; - *xdata = s; - } - - // extract context info - int64_t colname_len = 0; - char *colname = cloudsync_pk_context_colname(d, &colname_len); - - int64_t tbl_len = 0; - char *tbl = cloudsync_pk_context_tbl(d, &tbl_len); - - int64_t pk_len = 0; - void *pk = cloudsync_pk_context_pk(d, &pk_len); - - int64_t cl = cloudsync_pk_context_cl(d); - int64_t db_version = cloudsync_pk_context_dbversion(d); - - switch (step) { - case CLOUDSYNC_PAYLOAD_APPLY_WILL_APPLY: { - // if the tbl name or the prikey has changed, then verify if the row is valid - // must use strncmp because strings in xdata are not zero-terminated - bool tbl_changed = (s->last_tbl && (strlen(s->last_tbl) != (size_t)tbl_len || strncmp(s->last_tbl, tbl, (size_t)tbl_len) != 0)); - bool pk_changed = (s->last_pk && pk && cloudsync_blob_compare(s->last_pk, s->last_pk_len, pk, pk_len) != 0); - if (s->is_approved - && !s->last_is_delete - && (tbl_changed || pk_changed)) { - s->is_approved = unittest_validate_changed_row(db, data, s->last_tbl, s->last_pk, s->last_pk_len); - } - - s->last_is_delete = ((size_t)colname_len == strlen(CLOUDSYNC_TOMBSTONE_VALUE) && - strncmp(colname, CLOUDSYNC_TOMBSTONE_VALUE, (size_t)colname_len) == 0 - ) && cl % 2 == 0; - - // update the last_tbl value, if needed - if (!s->last_tbl || - !tbl || - (strlen(s->last_tbl) != (size_t)tbl_len) || - strncmp(s->last_tbl, tbl, (size_t)tbl_len) != 0) { - if (s->last_tbl) cloudsync_memory_free(s->last_tbl); - if (tbl && tbl_len > 0) s->last_tbl = cloudsync_string_ndup(tbl, tbl_len, false); - else s->last_tbl = NULL; - } - - // update the last_prikey and len values, if needed - if (!s->last_pk || !pk || cloudsync_blob_compare(s->last_pk, s->last_pk_len, pk, pk_len) != 0) { - if (s->last_pk) cloudsync_memory_free(s->last_pk); - if (pk && pk_len > 0) { - s->last_pk = cloudsync_memory_alloc(pk_len); - memcpy(s->last_pk, pk, pk_len); - s->last_pk_len = pk_len; - } else { - s->last_pk = NULL; - s->last_pk_len = 0; - } - } - - // commit the previous transaction, if any - // begin new transacion, if needed - if (s->last_db_version != db_version) { - rc = unittest_payload_apply_reset_transaction(db, s, true); - if (rc != SQLITE_OK) printf("unittest_payload_apply error in reset_transaction: (%d) %s\n", rc, sqlite3_errmsg(db)); - - // reset local variables - s->last_db_version = db_version; - s->is_approved = true; - } - - is_approved = s->is_approved; - break; - } - case CLOUDSYNC_PAYLOAD_APPLY_DID_APPLY: - is_approved = s->is_approved; - break; - case CLOUDSYNC_PAYLOAD_APPLY_CLEANUP: - if (s->is_approved && !s->last_is_delete) s->is_approved = unittest_validate_changed_row(db, data, s->last_tbl, s->last_pk, s->last_pk_len); - rc = unittest_payload_apply_reset_transaction(db, s, false); - if (s->last_tbl) cloudsync_memory_free(s->last_tbl); - if (s->last_pk) { - cloudsync_memory_free(s->last_pk); - s->last_pk_len = 0; - } - is_approved = s->is_approved; - - cloudsync_memory_free(s); - *xdata = NULL; - break; - } - - return is_approved; -} -#endif - // MARK: - #ifndef CLOUDSYNC_OMIT_PRINT_RESULT @@ -758,7 +731,7 @@ void do_delete (sqlite3 *db, int table_mask, bool print_result) { if (print_result) printf("TESTING DELETE on %s\n", table_name); char *sql = sqlite3_mprintf("DELETE FROM \"%w\" WHERE first_name='name5';", table_name); - int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); sqlite3_free(sql); if (rc != SQLITE_OK) goto finalize; @@ -772,7 +745,7 @@ void do_delete (sqlite3 *db, int table_mask, bool print_result) { const char *table_name = CUSTOMERS_NOCOLS_TABLE; if (print_result) printf("TESTING DELETE on %s\n", table_name); - int rc = sqlite3_exec(db, "DELETE FROM \"" CUSTOMERS_NOCOLS_TABLE "\" WHERE first_name='name100005';", NULL, NULL, NULL); + rc = sqlite3_exec(db, "DELETE FROM \"" CUSTOMERS_NOCOLS_TABLE "\" WHERE first_name='name100005';", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; rc = sqlite3_exec(db, "DELETE FROM \"" CUSTOMERS_NOCOLS_TABLE "\" WHERE first_name='name100007';", NULL, NULL, NULL); @@ -783,7 +756,7 @@ void do_delete (sqlite3 *db, int table_mask, bool print_result) { const char *table_name = "customers_noprikey"; if (print_result) printf("TESTING DELETE on %s\n", table_name); - int rc = sqlite3_exec(db, "DELETE FROM customers_noprikey WHERE first_name='name200005';", NULL, NULL, NULL); + rc = sqlite3_exec(db, "DELETE FROM customers_noprikey WHERE first_name='name200005';", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; rc = sqlite3_exec(db, "DELETE FROM customers_noprikey WHERE first_name='name200007';", NULL, NULL, NULL); @@ -876,7 +849,8 @@ bool do_test_vtab2 (void) { finalize: if (rc != SQLITE_OK) printf("do_test_vtab2 error: %s\n", sqlite3_errmsg(db)); - db = close_db(db); + close_db(db); + db = NULL; return result; } @@ -934,13 +908,13 @@ bool do_test_vtab(sqlite3 *db) { rc = sqlite3_exec(db, "SELECT tbl FROM cloudsync_changes WHERE db_version LIKE 1;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - const char *name = opname_from_value (666); + const char *name = vtab_opname_from_value (666); if (name != NULL) goto finalize; - rc = colname_is_legal("db_version"); + rc = vtab_colname_is_legal("db_version"); if (rc != 1) goto finalize; - rc = colname_is_legal("non_existing_column"); + rc = vtab_colname_is_legal("non_existing_column"); if (rc != 0) goto finalize; return do_test_vtab2(); @@ -951,28 +925,47 @@ bool do_test_vtab(sqlite3 *db) { } bool do_test_functions (sqlite3 *db, bool print_results) { - int size = 0, rc2; - char *site_id = dbutils_blob_select(db, "SELECT cloudsync_siteid();", &size, NULL, &rc2); - if (site_id == NULL || size != 16) goto abort_test_functions; + char *site_id = NULL; + int64_t len = 0; + cloudsync_context *data = cloudsync_context_create(db); + if (!data) return false; + + int rc = database_select_blob(data, "SELECT cloudsync_siteid();", &site_id, &len); + if (rc != DBRES_OK || site_id == NULL || len != 16) { + if (site_id) cloudsync_memory_free(site_id); + goto abort_test_functions; + } cloudsync_memory_free(site_id); - char *site_id_str = dbutils_text_select(db, "SELECT quote(cloudsync_siteid());"); - if (site_id_str == NULL) goto abort_test_functions; + char *site_id_str = NULL; + rc = database_select_text(data, "SELECT quote(cloudsync_siteid());", &site_id_str); + if (rc != DBRES_OK || site_id_str == NULL) { + if (site_id_str) cloudsync_memory_free(site_id_str); + goto abort_test_functions; + } if (print_results) printf("Site ID: %s\n", site_id_str); cloudsync_memory_free(site_id_str); - char *version = dbutils_text_select(db, "SELECT cloudsync_version();"); - if (version == NULL) goto abort_test_functions; + char *version = NULL; + rc = database_select_text(data, "SELECT cloudsync_version();", &version); + if (rc != DBRES_OK || version == NULL) { + if (version) cloudsync_memory_free(version); + goto abort_test_functions; + } if (print_results) printf("Lib Version: %s\n", version); cloudsync_memory_free(version); - sqlite3_int64 db_version = dbutils_int_select(db, "SELECT cloudsync_db_version();"); - if (print_results) printf("DB Version: %lld\n", db_version); + int64_t db_version = 0; + rc = database_select_int(data, "SELECT cloudsync_db_version();", &db_version); + if (rc != DBRES_OK) goto abort_test_functions; + if (print_results) printf("DB Version: %" PRId64 "\n", db_version); - sqlite3_int64 db_version_next = dbutils_int_select(db, "SELECT cloudsync_db_version_next();"); - if (print_results) printf("DB Version Next: %lld\n", db_version_next); + int64_t db_version_next = 0; + rc = database_select_int(data, "SELECT cloudsync_db_version_next();", &db_version); + if (rc != DBRES_OK) goto abort_test_functions; + if (print_results) printf("DB Version Next: %" PRId64 "\n", db_version_next); - int rc = sqlite3_exec(db, "CREATE TABLE tbl1 (col1 TEXT PRIMARY KEY NOT NULL, col2);", NULL, NULL, NULL); + rc = sqlite3_exec(db, "CREATE TABLE tbl1 (col1 TEXT PRIMARY KEY NOT NULL, col2);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; rc = sqlite3_exec(db, "CREATE TABLE tbl2 (col1 TEXT PRIMARY KEY NOT NULL, col2);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; @@ -980,31 +973,46 @@ bool do_test_functions (sqlite3 *db, bool print_results) { rc = sqlite3_exec(db, "DROP TABLE IF EXISTS rowid_table; DROP TABLE IF EXISTS nonnull_prikey_table;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - rc = sqlite3_exec(db, "SELECT cloudsync_init('*');", NULL, NULL, NULL); + // * disabled in 0.9.0 + rc = sqlite3_exec(db, "SELECT cloudsync_init('tbl1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto abort_test_functions; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('tbl2');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; rc = sqlite3_exec(db, "SELECT cloudsync_disable('tbl1');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - int v1 = (int)dbutils_int_select(db, "SELECT cloudsync_is_enabled('tbl1');"); + int64_t value = 0; + rc = database_select_int(data, "SELECT cloudsync_is_enabled('tbl1');", &value); + if (rc != DBRES_OK) goto abort_test_functions; + int v1 = (int)value; if (v1 == 1) goto abort_test_functions; - rc = sqlite3_exec(db, "SELECT cloudsync_disable('*');", NULL, NULL, NULL); + // * disabled in 0.9.0 + rc = sqlite3_exec(db, "SELECT cloudsync_disable('tbl2');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - int v2 = (int)dbutils_int_select(db, "SELECT cloudsync_is_enabled('tbl2');"); + rc = database_select_int(data, "SELECT cloudsync_is_enabled('tbl2');", &value); + if (rc != DBRES_OK) goto abort_test_functions; + int v2 = (int)value; if (v2 == 1) goto abort_test_functions; rc = sqlite3_exec(db, "SELECT cloudsync_enable('tbl1');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - int v3 = (int)dbutils_int_select(db, "SELECT cloudsync_is_enabled('tbl1');"); + rc = database_select_int(data, "SELECT cloudsync_is_enabled('tbl1');", &value); + if (rc != DBRES_OK) goto abort_test_functions; + int v3 = (int)value; if (v3 != 1) goto abort_test_functions; - rc = sqlite3_exec(db, "SELECT cloudsync_enable('*');", NULL, NULL, NULL); + // * disabled in 0.9.0 + rc = sqlite3_exec(db, "SELECT cloudsync_enable('tbl2');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - int v4 = (int)dbutils_int_select(db, "SELECT cloudsync_is_enabled('tbl2');"); + rc = database_select_int(data, "SELECT cloudsync_is_enabled('tbl2');", &value); + if (rc != DBRES_OK) goto abort_test_functions; + int v4 = (int)value; if (v4 != 1) goto abort_test_functions; rc = sqlite3_exec(db, "SELECT cloudsync_set('key1', 'value1');", NULL, NULL, NULL); @@ -1016,17 +1024,26 @@ bool do_test_functions (sqlite3 *db, bool print_results) { rc = sqlite3_exec(db, "SELECT cloudsync_set_column('tbl1', 'col1', 'key1', 'value1');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - rc = sqlite3_exec(db, "SELECT cloudsync_cleanup('*');", NULL, NULL, NULL); + // * disabled in 0.9.0 + rc = sqlite3_exec(db, "SELECT cloudsync_cleanup('tbl1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto abort_test_functions; + rc = sqlite3_exec(db, "SELECT cloudsync_cleanup('tbl2');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - char *uuid = dbutils_text_select(db, "SELECT cloudsync_uuid();"); - if (uuid == NULL) goto abort_test_functions; + char *uuid = NULL; + rc = database_select_text(data, "SELECT cloudsync_uuid();", &uuid); + if (rc != DBRES_OK || uuid == NULL) { + if (uuid) cloudsync_memory_free(uuid); + goto abort_test_functions; + } if (print_results) printf("New uuid: %s\n", uuid); cloudsync_memory_free(uuid); + cloudsync_context_free(data); return true; abort_test_functions: + cloudsync_context_free(data); printf("Error in do_test_functions: %s\n", sqlite3_errmsg(db)); return false; } @@ -1196,18 +1213,18 @@ bool do_augment_tables (int table_mask, sqlite3 *db, table_algo algo) { char sql[512]; if (table_mask & TEST_PRIKEYS) { - sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('%q', '%s');", CUSTOMERS_TABLE, crdt_algo_name(algo)); + sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('%q', '%s');", CUSTOMERS_TABLE, cloudsync_algo_name(algo)); int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_augment_tables; } if (table_mask & TEST_NOCOLS) { - sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('%q', '%s');", CUSTOMERS_NOCOLS_TABLE, crdt_algo_name(algo)); + sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('%q', '%s');", CUSTOMERS_NOCOLS_TABLE, cloudsync_algo_name(algo)); if (sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) goto abort_augment_tables; } if (table_mask & TEST_NOPRIKEYS) { - sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('customers_noprikey', '%s');", crdt_algo_name(algo)); + sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('customers_noprikey', '%s');", cloudsync_algo_name(algo)); if (sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) goto abort_augment_tables; } @@ -1307,7 +1324,7 @@ bool do_test_pk_single_value (sqlite3 *db, int type, int64_t ivalue, double dval pklist[0].type = type; if (type == SQLITE_INTEGER) { - snprintf(sql, sizeof(sql), "SELECT cloudsync_pk_encode(%lld);", ivalue); + snprintf(sql, sizeof(sql), "SELECT cloudsync_pk_encode(%" PRId64 ");", ivalue); pklist[0].ivalue = ivalue; } else if (type == SQLITE_FLOAT) { snprintf(sql, sizeof(sql), "SELECT cloudsync_pk_encode(%f);", dvalue); @@ -1357,7 +1374,7 @@ bool do_test_pk_single_value (sqlite3 *db, int type, int64_t ivalue, double dval exit(-666); } if (stmt) sqlite3_finalize(stmt); - dbutils_debug_stmt(db, true); + unit_debug(db, true); return result; } @@ -1414,7 +1431,38 @@ bool do_test_pkbind_callback (sqlite3 *db) { exit(-666); } if (stmt) sqlite3_finalize(stmt); - dbutils_debug_stmt(db, true); + unit_debug(db, true); + return result; +} + +bool do_test_single_pk (bool print_result) { + bool result = false; + + sqlite3 *db = NULL; + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) goto cleanup; + + // manually load extension + sqlite3_cloudsync_init(db, NULL, NULL); + + rc = sqlite3_exec(db, "CREATE TABLE single_pk_test (col1 INTEGER PRIMARY KEY NOT NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // the following function should fail + rc = sqlite3_exec(db, "SELECT cloudsync_init('single_pk_test');", NULL, NULL, NULL); + if (rc == SQLITE_OK) return false; + + // the following function should succedd + rc = sqlite3_exec(db, "SELECT cloudsync_init('single_pk_test', 'cls', 1);", NULL, NULL, NULL); + if (rc != SQLITE_OK) return false; + result = true; + + // cleanup newly created table + sqlite3_exec(db, "SELECT cloudsync_cleanup('single_pk_test');", NULL, NULL, NULL); + +cleanup: + if (rc != SQLITE_OK && print_result) printf("do_test_single_pk error: %s\n", sqlite3_errmsg(db)); + close_db(db); return result; } @@ -1502,9 +1550,9 @@ bool do_test_pk (sqlite3 *db, int ntest, bool print_result) { // cleanup memory sqlite3_finalize(stmt); stmt = NULL; - for (int i=0; i 0) goto finalize; - sqlite3_int64 db_version = dbutils_int_select(db, "SELECT cloudsync_db_version();"); + int64_t db_version = 0; + database_select_int(data, "SELECT cloudsync_db_version();", &db_version); char *site_id_blob; - int site_id_blob_size; - sqlite3_int64 dbver1, seq1; - rc = dbutils_blob_int_int_select(db, "SELECT cloudsync_siteid(), cloudsync_db_version(), cloudsync_seq();", &site_id_blob, &site_id_blob_size, &dbver1, &seq1); + int64_t site_id_blob_size; + int64_t dbver1; + rc = database_select_blob_int(data, "SELECT cloudsync_siteid(), cloudsync_db_version();", &site_id_blob, &site_id_blob_size, &dbver1); if (rc != SQLITE_OK || site_id_blob == NULL ||dbver1 != db_version) goto finalize; cloudsync_memory_free(site_id_blob); // force out-of-memory test - value1 = dbutils_settings_get_value(db, "key1", OUT_OF_MEMORY_BUFFER, 0); - if (value1 != NULL) goto finalize; + rc = dbutils_settings_get_value(data, "key1", NULL, 0, NULL); + if (rc != SQLITE_MISUSE) goto finalize; - value1 = dbutils_table_settings_get_value(db, "foo", NULL, "key1", OUT_OF_MEMORY_BUFFER, 0); - if (value1 != NULL) goto finalize; + rc = dbutils_table_settings_get_value(data, "foo", NULL, "key1", NULL, 0); + if (rc != DBRES_MISUSE) goto finalize; - char *p = NULL; - dbutils_select(db, "SELECT zeroblob(16);", NULL, NULL, NULL, 0, SQLITE_NOMEM); - if (p != NULL) goto finalize; + //char *p = NULL; + //dbutils_select(data, "SELECT zeroblob(16);", NULL, NULL, NULL, 0, SQLITE_BLOB); + //if (p != NULL) goto finalize; - dbutils_settings_set_key_value(db, NULL, CLOUDSYNC_KEY_LIBVERSION, "0.0.0"); - int cmp = dbutils_settings_check_version(db, NULL); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_LIBVERSION, "0.0.0"); + int cmp = dbutils_settings_check_version(data, NULL); if (cmp == 0) goto finalize; - dbutils_settings_set_key_value(db, NULL, CLOUDSYNC_KEY_LIBVERSION, CLOUDSYNC_VERSION); - cmp = dbutils_settings_check_version(db, NULL); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_LIBVERSION, CLOUDSYNC_VERSION); + cmp = dbutils_settings_check_version(data, NULL); if (cmp != 0) goto finalize; - cmp = dbutils_settings_check_version(db, "0.8.25"); + cmp = dbutils_settings_check_version(data, "0.8.25"); if (cmp <= 0) goto finalize; //dbutils_settings_table_load_callback(NULL, 0, NULL, NULL); - dbutils_migrate(NULL); + dbutils_settings_migrate(NULL); - dbutils_settings_cleanup(db); + dbutils_settings_cleanup(data); int n1 = 1; int n2 = 2; - cmp = binary_comparison(n1, n2); + cmp = dbutils_binary_comparison(n1, n2); if (cmp != -1) goto finalize; - cmp = binary_comparison(n2, n1); + cmp = dbutils_binary_comparison(n2, n1); if (cmp != 1) goto finalize; - cmp = binary_comparison(n1, n1); + cmp = dbutils_binary_comparison(n1, n1); if (cmp != 0) goto finalize; rc = SQLITE_OK; finalize: if (rc != SQLITE_OK) printf("%s\n", sqlite3_errmsg(db)); - db = close_db(db); + close_db(db); + db = NULL; + if (data) cloudsync_context_free(data); return (rc == SQLITE_OK); } @@ -1923,10 +1987,10 @@ bool do_test_others (sqlite3 *db) { // test unfinalized statement just to increase code coverage sqlite3_stmt *stmt = NULL; sqlite3_prepare_v2(db, "SELECT 1;", -1, &stmt, NULL); - int count = dbutils_debug_stmt(db, false); + int count = unit_debug(db, false); sqlite3_finalize(stmt); // to increase code coverage - dbutils_context_result_error(NULL, "Test is: %s", "Hello World"); + // dbutils_set_error(NULL, "Test is: %s", "Hello World"); return (count == 1); } @@ -1936,7 +2000,7 @@ bool do_test_error_cases (sqlite3 *db) { // test cloudsync_init missing table sqlite3_prepare_v2(db, "SELECT cloudsync_init('missing_table');", -1, &stmt, NULL); - int res = stmt_execute(stmt, NULL); + int res = dbvm_execute(stmt, NULL); sqlite3_finalize(stmt); if (res != -1) return false; @@ -1959,6 +2023,43 @@ bool do_test_error_cases (sqlite3 *db) { return true; } +bool do_test_null_prikey_insert (sqlite3 *db) { + // Create a table with a primary key that allows NULL (no NOT NULL constraint) + const char *sql = "CREATE TABLE IF NOT EXISTS t_null_pk (id TEXT PRIMARY KEY, value TEXT);" + "SELECT cloudsync_init('t_null_pk');"; + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) return false; + + // Attempt to insert a row with NULL primary key — should fail + char *errmsg = NULL; + sql = "INSERT INTO t_null_pk (id, value) VALUES (NULL, 'test');"; + rc = sqlite3_exec(db, sql, NULL, NULL, &errmsg); + if (rc == SQLITE_OK) return false; // should have failed + if (!errmsg) return false; + + // Verify the error message matches the expected format + const char *expected = "Insert aborted because primary key in table t_null_pk contains NULL values."; + bool match = (strcmp(errmsg, expected) == 0); + sqlite3_free(errmsg); + if (!match) return false; + + // Verify that a non-NULL primary key insert succeeds + sql = "INSERT INTO t_null_pk (id, value) VALUES ('valid_id', 'test');"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) return false; + + // Verify the metatable has exactly 1 row (only the valid insert) + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM t_null_pk_cloudsync;", -1, &stmt, NULL); + if (rc != SQLITE_OK) return false; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); return false; } + int count = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + if (count != 1) return false; + + return true; +} + bool do_test_internal_functions (void) { sqlite3 *db = NULL; sqlite3_stmt *vm = NULL; @@ -1986,12 +2087,12 @@ bool do_test_internal_functions (void) { rc = sqlite3_prepare(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto abort_test; - int res = stmt_count(vm, NULL, 0, 0); + int res = dbvm_count(vm, NULL, 0, 0); if (res != 0) goto abort_test; if (vm) sqlite3_finalize(vm); vm = NULL; - // TEST 2 (stmt_execute returns an error) + // TEST 2 (dbvm_execute returns an error) sql = "INSERT INTO foo (name, age) VALUES ('Name1', 22)"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test; @@ -2000,7 +2101,7 @@ bool do_test_internal_functions (void) { if (rc != SQLITE_OK) goto abort_test; // this statement must fail - res = stmt_execute(vm, NULL); + res = dbvm_execute(vm, NULL); if (res != -1) goto abort_test; if (vm) sqlite3_finalize(vm); vm = NULL; @@ -2017,1172 +2118,5326 @@ bool do_test_string_replace_prefix(void) { char *host = "rejfwkr.sqlite.cloud"; char *prefix = "sqlitecloud://"; char *replacement = "https://"; - + char string[512]; snprintf(string, sizeof(string), "%s%s", prefix, host); char expected[512]; snprintf(expected, sizeof(expected), "%s%s", replacement, host); - + char *replaced = cloudsync_string_replace_prefix(string, prefix, replacement); if (string == replaced || strcmp(replaced, expected) != 0) return false; if (string != replaced) cloudsync_memory_free(replaced); - + replaced = cloudsync_string_replace_prefix(expected, prefix, replacement); if (expected != replaced) return false; - + return true; } -bool do_test_many_columns (int ncols, sqlite3 *db) { - char sql_create[10000]; - int pos = 0; - pos += snprintf(sql_create+pos, sizeof(sql_create)-pos, "CREATE TABLE IF NOT EXISTS test_many_columns (id TEXT PRIMARY KEY NOT NULL"); - for (int i=1; i sqlite3_step reports a different result: %d != %d\n", rc1, rc2); - goto finalize; - } - // rc(s) are equals here - if (rc1 == SQLITE_DONE) break; - if (rc1 != SQLITE_ROW) goto finalize; - - // we have a ROW here - for (int i=0; i_cloudsync +// meta-table before reopening. With a stale cloudsync_table_settings row and +// no matching *_cloudsync meta-table in sqlite_master, the dbversion query +// builder produces an empty (NULL) SQL string, causing sqlite3_cloudsync_init +// to fail on reopen — previously crashing in some environments. +bool do_test_stale_table_settings_dropped_meta(bool cleanup_databases) { bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS; + char dbpath[256]; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i customers\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); + return SQLITE_OK; +} + +// Run a SQL statement expected to fail, and verify that its error message +// contains the stable substring "cloudsync_init" — which every uninitialized +// guard added by the fix points the caller at. Matching a single stable token +// rather than full text tolerates future rewordings of the user-facing string. +static bool expect_uninit_error (sqlite3 *db, const char *sql) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + // Prepare-time failures also land in sqlite3_errmsg — accept those. + const char *m = sqlite3_errmsg(db); + bool ok = (m != NULL && strstr(m, "cloudsync_init") != NULL); + if (stmt) sqlite3_finalize(stmt); + return ok; } - + rc = sqlite3_step(stmt); + bool failed = (rc != SQLITE_ROW && rc != SQLITE_DONE); + const char *msg = sqlite3_errmsg(db); + bool has_hint = (msg != NULL && strstr(msg, "cloudsync_init") != NULL); + sqlite3_finalize(stmt); + return failed && has_hint; +} + +// Regression test for the "uninit error messages" fix. Every function that +// previously leaked a misleading low-level symptom (out of memory, "not an +// error", silent -1, multi-line "no such table" dumps) must now point the +// caller at SELECT cloudsync_init(...). Match on a stable substring rather +// than full text so the test tolerates future rewordings. +bool do_test_uninit_error_messages (void) { + sqlite3 *db = NULL; + bool result = false; + + if (sqlite3_open(":memory:", &db) != SQLITE_OK) return false; + if (sqlite3_cloudsync_init(db, NULL, NULL) != SQLITE_OK) goto cleanup; + + // 96 bytes of zero padding — bigger than cloudsync_payload_header, so the + // size sanity check in dbsync_payload_decode passes and control reaches + // our guard inside cloudsync_payload_apply. + const char *payload_sql = + "SELECT cloudsync_payload_apply(zeroblob(96));"; + + if (!expect_uninit_error(db, "SELECT * FROM cloudsync_changes;")) goto cleanup; + if (!expect_uninit_error(db, "SELECT cloudsync_db_version();")) goto cleanup; + if (!expect_uninit_error(db, "SELECT cloudsync_db_version_next();")) goto cleanup; + if (!expect_uninit_error(db, "SELECT cloudsync_set_filter('foo','1=1');")) goto cleanup; + if (!expect_uninit_error(db, "SELECT cloudsync_clear_filter('foo');")) goto cleanup; + if (!expect_uninit_error(db, payload_sql)) goto cleanup; + + // Happy path: after cloudsync_init the same functions no longer fail with + // the uninitialized hint. cloudsync_db_version must now return a value. + if (sqlite3_exec(db, + "CREATE TABLE t (id TEXT PRIMARY KEY NOT NULL, v TEXT);" + "SELECT cloudsync_init('t');", + NULL, NULL, NULL) != SQLITE_OK) goto cleanup; + + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(db, "SELECT cloudsync_db_version();", -1, &stmt, NULL) != SQLITE_OK) goto cleanup; + int step_rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (step_rc != SQLITE_ROW) goto cleanup; + result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); - } - } - - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); - } + +cleanup: + if (db) { + sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + sqlite3_close(db); } return result; } -// Test a sequence of random different concurrent changes in various clients -// with changes to pk columns and non-pk columns. -bool do_test_merge_2 (int nclients, int table_mask, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; +// Verify that cloudsync_dbversion_rebuild surfaces a real failure from +// database_select_text(SQL_DBVERSION_BUILD_QUERY, ...) instead of silently +// treating it as "no *_cloudsync meta-tables present" — which would leave +// db_version_stmt unset and cause writes to fall back to CLOUDSYNC_MIN_DB_VERSION. +bool do_test_dbversion_rebuild_error(void) { + sqlite3 *db = NULL; + cloudsync_context *ctx = NULL; bool result = false; - int rc = SQLITE_OK; - int nrows = NINSERT; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 0 — the early-return-OK path is not taken). + rc = sqlite3_exec(db, "CREATE TABLE t (id TEXT PRIMARY KEY NOT NULL, v TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_exec(db, "SELECT cloudsync_init('t');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create a secondary context on the same db and initialize it. This + // context is independent from the one registered by sqlite3_cloudsync_init, + // so we can call cloudsync_dbversion_rebuild on it directly without + // disturbing the registered functions. + ctx = cloudsync_context_create(db); + if (!ctx) goto cleanup; + if (cloudsync_context_init(ctx) == NULL) goto cleanup; + + // Install an authorizer that denies reads of sqlite_master. New prepares + // (including the one SQL_DBVERSION_BUILD_QUERY triggers inside + // database_select_text) will fail with SQLITE_AUTH. Already-prepared + // statements are unaffected, so the registered cloudsync_* functions + // still work for cleanup. + sqlite3_set_authorizer(db, deny_sqlite_master_authorizer, NULL); + + // Expect a non-OK result now that the build query cannot be prepared. + // Before the review fix this would incorrectly return DBRES_OK and leave + // db_version_stmt == NULL, silently masking the failure. + int rebuild_rc = cloudsync_dbversion_rebuild(ctx); + + // Remove authorizer before any further work so cleanup can run normally. + sqlite3_set_authorizer(db, NULL, NULL); + + if (rebuild_rc == DBRES_OK) goto cleanup; + + // The error must have been recorded on the context via cloudsync_set_dberror. + if (cloudsync_errcode(ctx) == DBRES_OK) goto cleanup; + const char *msg = cloudsync_errmsg(ctx); + if (!msg || msg[0] == 0) goto cleanup; + + result = true; + +cleanup: + sqlite3_set_authorizer(db, NULL, NULL); + if (ctx) cloudsync_context_free(ctx); + if (db) { + sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + sqlite3_close(db); } - - // insert data in the last customer - do_insert(db[nclients-1], table_mask, NINSERT, print_result); - - // update some random data in all clients except the last one - for (int i=0; i 0) { + g_deny_insert_remaining--; + return SQLITE_OK; } + return SQLITE_DENY; } - - // deleta data in the first customer - do_delete(db[0], table_mask, print_result); - - // merge all changes - if (do_merge(db, nclients, false) == false) { - goto finalize; - } - - // compare results - for (int i=1; i " CUSTOMERS_TABLE "\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - if (table_mask & TEST_NOCOLS) { - printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "s\"\n"); - do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); - } - } - - result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); - } - } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); - } + return SQLITE_OK; +} + +// Test that the error cleanup path in table_add_to_context_cb doesn't crash +// when databasevm_prepare fails for the merge statement. +// Before the zeroalloc fix, col_value_stmt[index] contained uninitialized +// garbage and databasevm_finalize was called on it → SIGSEGV. +bool do_test_context_cb_error_cleanup(void) { + sqlite3 *db = NULL; + cloudsync_context *ctx = NULL; + bool result = false; + + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + sqlite3_cloudsync_init(db, NULL, NULL); + + // Create table with PK and non-PK columns + rc = sqlite3_exec(db, + "CREATE TABLE ctx_err (id TEXT PRIMARY KEY NOT NULL, val TEXT, num INTEGER);", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Init cloudsync on the table (creates meta table, triggers, settings) + rc = sqlite3_exec(db, "SELECT cloudsync_init('ctx_err');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create a fresh context — ctx_err is not in its table list, + // so table_add_to_context won't short-circuit at table_lookup. + ctx = cloudsync_context_create(db); + if (!ctx) goto cleanup; + + // Install authorizer that denies INSERT on the base table. + // Allow 1 INSERT (the sentinel prepared in table_add_stmts), + // then deny the next (the merge UPSERT prepared in the callback). + // This causes databasevm_prepare to fail inside table_add_to_context_cb, + // triggering the error cleanup path. + g_deny_insert_table = "ctx_err"; + g_deny_insert_remaining = 1; + sqlite3_set_authorizer(db, deny_insert_authorizer, NULL); + + // Must return false (callback error) without crashing + bool added = table_add_to_context(ctx, table_algo_crdt_cls, "ctx_err"); + + // Remove authorizer before any further operations + sqlite3_set_authorizer(db, NULL, NULL); + g_deny_insert_table = NULL; + + if (added) goto cleanup; // should have failed + + result = true; + +cleanup: + sqlite3_set_authorizer(db, NULL, NULL); + g_deny_insert_table = NULL; + if (ctx) cloudsync_context_free(ctx); + if (db) { + sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + sqlite3_close(db); } return result; } -// Test cloudsync_merge_insert where local col_version is equal to the insert col_version, -// the greater value must win. -bool do_test_merge_4 (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; +// Test cloudsync_terminate function +bool do_test_terminate(void) { + sqlite3 *db = NULL; bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS; - time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i " CUSTOMERS_TABLE "\n"); - sqlite3_snprintf(sizeof(buf), buf, "SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], buf, query_table); - } - + + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table + rc = sqlite3_exec(db, "CREATE TABLE term_test (id TEXT PRIMARY KEY NOT NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('term_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Call terminate + rc = sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i, col_name:age, db_version:2, col_version:2, site_id:1 (remote change) -// b. pk:, col_name:TOMBSTONE, db_version:3, col_version:2, site_id:0 (local change, delete old pk) -// c. pk:, col_name:TOMBSTONE, db_version:3, col_version:1, site_id:0 (local change, updated row with new pk) -// 5. merge changes from the second client to the first client: -// - if the second client sends only local changes, then the change at 4a is not sent so the first client will show NULL -// at column "age" for the row with the new pk -// - if the second client sends all the changes (not only local changes), then the change at 4a is sent along with 4b and 4c -// so the first client will show the correct value for the "age" column -bool do_test_merge_5 (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS; - time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i customers\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - - result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i size2 should give positive result + int r1 = cloudsync_blob_compare(blob1, 100, blob2, 1); + if (r1 <= 0) return false; + + // size1 < size2 should give negative result + int r2 = cloudsync_blob_compare(blob1, 1, blob2, 100); + if (r2 >= 0) return false; + + // Same size, different content + int r3 = cloudsync_blob_compare(blob1, 1, blob2, 1); + if (r3 == 0) return false; + + // Equal + int r4 = cloudsync_blob_compare(blob1, 1, blob1, 1); + if (r4 != 0) return false; + + return true; +} + +// Test that cloudsync_uuid() is non-deterministic (returns different values in same query) +bool do_test_deterministic_flags(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - time_t timestamp = time(NULL); - int saved_counter = test_counter++; - for (int i=0; i customers\n"); - char *sql = "SELECT * FROM cloudsync_changes;"; - do_query(db[1], sql, query_changes); - } - + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // SELECT cloudsync_uuid(), cloudsync_uuid() — both values should differ + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_uuid(), cloudsync_uuid();", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const char *u1 = (const char *)sqlite3_column_text(stmt, 0); + const char *u2 = (const char *)sqlite3_column_text(stmt, 1); + if (!u1 || !u2) goto cleanup; + + // Non-deterministic: same query, different results + if (strcmp(u1, u2) == 0) goto cleanup; + result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - time_t timestamp = time(NULL); - int saved_counter = test_counter++; - for (int i=0; i 1;"; - sqlite3_stmt *vm = NULL; - rc = sqlite3_prepare_v2(db[i], sql, -1, &vm, NULL); - if (rc != SQLITE_OK) goto finalize; - rc = sqlite3_step(vm); - if (rc != SQLITE_DONE) { - printf("cloudsync_changes should not have repeated db_version and seq values, got: db_version=%d, seq=%d, count=%d\n", sqlite3_column_int(vm, 0), sqlite3_column_int(vm, 1), sqlite3_column_int(vm, 2)); - if (vm) sqlite3_finalize(vm); - goto finalize; - } - if (vm) sqlite3_finalize(vm); + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table — use TEXT pk to avoid single INTEGER pk warning + rc = sqlite3_exec(db, "CREATE TABLE t1 (id TEXT PRIMARY KEY NOT NULL, name TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('t1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Get the schema hash value by reading cloudsync_schema_versions + { + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT hash FROM cloudsync_schema_versions ORDER BY seq DESC LIMIT 1;", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { sqlite3_finalize(stmt); goto cleanup; } + + int64_t hash = sqlite3_column_int64(stmt, 0); + sqlite3_finalize(stmt); + + // Verify the hash can be looked up using the same int64 representation + // This tests the PRId64 format consistency fix + char sql[256]; + snprintf(sql, sizeof(sql), "SELECT 1 FROM cloudsync_schema_versions WHERE hash = (%" PRId64 ")", hash); + + stmt = NULL; + rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { sqlite3_finalize(stmt); goto cleanup; } + + int found = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + if (found != 1) goto cleanup; } - - // check grouped values from cloudsync_changes - char *query_changes = "SELECT db_version, COUNT(distinct(seq)) AS cnt FROM cloudsync_changes GROUP BY db_version;"; - char *query_expected_results = "SELECT * FROM (VALUES (1,2),(2,2),(3,2),(4,2),(5,4));"; - if (do_compare_queries(db[0], query_changes, db[0], query_expected_results, -1, -1, print_result) == false) { - goto finalize; + + result = true; + +cleanup: + if (db) close_db(db); + return result; +} + +// Test cloudsync_blob_compare function +bool do_test_blob_compare(void) { + // Test same content, same size + const char blob1[] = {0x01, 0x02, 0x03, 0x04}; + const char blob2[] = {0x01, 0x02, 0x03, 0x04}; + int result = cloudsync_blob_compare(blob1, 4, blob2, 4); + if (result != 0) return false; + + // Test different sizes (line 168 in utils.c) + const char blob3[] = {0x01, 0x02, 0x03}; + result = cloudsync_blob_compare(blob1, 4, blob3, 3); + if (result == 0) return false; // Should be non-zero (different sizes) + + // Test different content, same size + const char blob4[] = {0x01, 0x02, 0x03, 0x05}; + result = cloudsync_blob_compare(blob1, 4, blob4, 4); + if (result == 0) return false; // Should be non-zero (different content) + + // Test empty blobs + result = cloudsync_blob_compare("", 0, "", 0); + if (result != 0) return false; + + return true; +} + +// Test string duplication functions more thoroughly +bool do_test_string_functions(void) { + // Test cloudsync_string_ndup (non-lowercase path) + const char *test_str = "Hello World"; + char *dup = cloudsync_string_ndup(test_str, 5); // "Hello" + if (!dup) return false; + if (strcmp(dup, "Hello") != 0) { + cloudsync_memory_free(dup); + return false; } - - if (print_result) { - printf("\n-> customers\n"); - char *sql = "SELECT * FROM cloudsync_changes();"; - do_query(db[1], sql, query_changes); + cloudsync_memory_free(dup); + + // Test cloudsync_string_dup + dup = cloudsync_string_dup("Test String"); + if (!dup) return false; + if (strcmp(dup, "Test String") != 0) { + cloudsync_memory_free(dup); + return false; } - - result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i local_col_version, even if db_version is smaller - rc = sqlite3_exec(db, "INSERT INTO cloudsync_changes (tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) VALUES ('todo',x'010B0133','status','this update must be applied',11,9,x'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA',1,0);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // compare results - char *sql = "SELECT * FROM todo ORDER BY id;"; - char *query_expected_results = "SELECT * FROM (VALUES ('1','Buy groceries','finalized'),('2','Buy bananas','in_progress2'),('3','New Item','this update must be applied'));"; - if (do_compare_queries(db, sql, db, query_expected_results, -1, -1, print_result) == false) { - goto finalize; - } - - // compare values in the changes vtab - sql = "SELECT col_name,col_value,col_version,db_version,cl,seq FROM cloudsync_changes;"; - query_expected_results = "SELECT * FROM (VALUES ('title','Buy groceries',1,1,1,0),('title','Buy bananas',1,2,1,0),('status','in_progress2',1,2,1,1),('title','New Item',10,10,1,0),('status','finalized',200,300,1,0),('status','this update must be applied',11,301,1,0));"; - if (do_compare_queries(db, sql, db, query_expected_results, -1, -1, print_result) == false) { - goto finalize; - } - - if (print_result) { - printf("\n-> customers\n"); - char *sql = "SELECT * FROM todo;"; - do_query(db, sql, query_changes); - } - result = true; - rc = SQLITE_OK; - -finalize: - if (rc != SQLITE_OK && db && (sqlite3_errcode(db) != SQLITE_OK)) printf("do_test_insert_cloudsync_changes: %s\n", sqlite3_errmsg(db)); + +cleanup: + if (stmt) sqlite3_finalize(stmt); if (db) close_db(db); - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, 0, timestamp, saved_counter); - file_delete_internal(buf); - } return result; } -#define SKIP_SCHEMA_CHECK 1 - -bool do_test_merge_alter_schema_1 (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; +// Test SQL-level pk_decode function +bool do_test_sql_pk_decode(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; - time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 3.15) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_pk_decode for BLOB (index 4) + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_decode(?, 4);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const unsigned char expected_blob[] = {0xDE, 0xAD, 0xBE, 0xEF}; + const void *blob_val = sqlite3_column_blob(stmt, 0); + int blob_len = sqlite3_column_bytes(stmt, 0); + if (blob_len != 4 || memcmp(blob_val, expected_blob, 4) != 0) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; - // merge all changes, now it must work fine - if (do_merge(db, nclients, only_locals) == false) { - goto finalize; - } - - // compare results - for (int i=1; i customers\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test negative integer encoding and decoding + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(-12345);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const void *pk = sqlite3_column_blob(stmt, 0); + int pk_len = sqlite3_column_bytes(stmt, 0); + char pk_copy[1024]; + memcpy(pk_copy, pk, pk_len); + + sqlite3_finalize(stmt); + stmt = NULL; + + // Decode and verify + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_decode(?, 1);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int64_t int_val = sqlite3_column_int64(stmt, 0); + if (int_val != -12345) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test negative float encoding and decoding + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(-3.14159);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + pk_len = sqlite3_column_bytes(stmt, 0); + memcpy(pk_copy, pk, pk_len); + + sqlite3_finalize(stmt); + stmt = NULL; + + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_decode(?, 1);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + double float_val = sqlite3_column_double(stmt, 0); + if (float_val > -3.14 || float_val < -3.15) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test INT64_MIN (maximum negative integer) + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(-9223372036854775808);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + pk_len = sqlite3_column_bytes(stmt, 0); + memcpy(pk_copy, pk, pk_len); + + sqlite3_finalize(stmt); + stmt = NULL; + + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_decode(?, 1);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int_val = sqlite3_column_int64(stmt, 0); + if (int_val != INT64_MIN) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test settings functions +bool do_test_settings_functions(void) { + sqlite3 *db = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_set + rc = sqlite3_exec(db, "SELECT cloudsync_set('test_key', 'test_value');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create a table and test table-level settings + rc = sqlite3_exec(db, "CREATE TABLE settings_test (id TEXT PRIMARY KEY NOT NULL, data TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('settings_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_set_table + rc = sqlite3_exec(db, "SELECT cloudsync_set_table('settings_test', 'table_key', 'table_value');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_set_column + rc = sqlite3_exec(db, "SELECT cloudsync_set_column('settings_test', 'data', 'col_key', 'col_value');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + result = true; + +cleanup: + if (db) close_db(db); + return result; +} + +// Test cloudsync_is_sync and cloudsync_is_enabled functions +bool do_test_sync_enabled_functions(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table + rc = sqlite3_exec(db, "CREATE TABLE sync_test (id TEXT PRIMARY KEY NOT NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('sync_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_is_enabled - should be 1 after init + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_enabled('sync_test');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int enabled = sqlite3_column_int(stmt, 0); + if (enabled != 1) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_is_sync - should return 0 (not in sync mode) + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_sync('sync_test');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + // Value depends on implementation + sqlite3_finalize(stmt); + stmt = NULL; + + // Disable sync + rc = sqlite3_exec(db, "SELECT cloudsync_disable('sync_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_is_enabled - should be 0 after disable + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_enabled('sync_test');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + enabled = sqlite3_column_int(stmt, 0); + if (enabled != 0) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Re-enable sync + rc = sqlite3_exec(db, "SELECT cloudsync_enable('sync_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_is_enabled - should be 1 again + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_enabled('sync_test');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + enabled = sqlite3_column_int(stmt, 0); + if (enabled != 1) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test with non-existent table + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_enabled('non_existent_table');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + enabled = sqlite3_column_int(stmt, 0); + if (enabled != 0) goto cleanup; // Should be 0 for non-existent table + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test cloudsync_uuid SQL function +bool do_test_sql_uuid_function(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_uuid() SQL function + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_uuid();", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const char *uuid1 = (const char *)sqlite3_column_text(stmt, 0); + if (!uuid1 || strlen(uuid1) != 36) goto cleanup; // UUID with dashes + + // Store the first UUID + char uuid1_copy[40]; + strncpy(uuid1_copy, uuid1, sizeof(uuid1_copy) - 1); + + sqlite3_finalize(stmt); + stmt = NULL; + + // Get another UUID - should be different + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_uuid();", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const char *uuid2 = (const char *)sqlite3_column_text(stmt, 0); + if (!uuid2 || strlen(uuid2) != 36) goto cleanup; + + // UUIDs should be different + if (strcmp(uuid1_copy, uuid2) == 0) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test pk_encode with empty values +bool do_test_pk_encode_edge_cases(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test encoding empty text + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode('');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const void *pk = sqlite3_column_blob(stmt, 0); + if (!pk) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test encoding empty blob + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(X'');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + if (!pk) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test encoding zero + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(0);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + if (!pk) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test encoding 0.0 + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(0.0);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + if (!pk) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test encoding large integers + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(9223372036854775807);", -1, &stmt, NULL); // INT64_MAX + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + if (!pk) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test cloudsync_col_value function +bool do_test_col_value_function(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table + rc = sqlite3_exec(db, "CREATE TABLE col_test (id TEXT PRIMARY KEY NOT NULL, data TEXT, num INTEGER);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('col_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Insert data + rc = sqlite3_exec(db, "INSERT INTO col_test (id, data, num) VALUES ('key1', 'value1', 42);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Get the pk for key1 + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode('key1');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const void *pk = sqlite3_column_blob(stmt, 0); + int pk_len = sqlite3_column_bytes(stmt, 0); + char pk_copy[256]; + memcpy(pk_copy, pk, pk_len); + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_col_value for text column + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_col_value('col_test', 'data', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const char *val = (const char *)sqlite3_column_text(stmt, 0); + if (!val || strcmp(val, "value1") != 0) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_col_value for integer column + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_col_value('col_test', 'num', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int num_val = sqlite3_column_int(stmt, 0); + if (num_val != 42) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_col_value with TOMBSTONE value (should return NULL) + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_col_value('col_test', '__[RIP]__', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + if (sqlite3_column_type(stmt, 0) != SQLITE_NULL) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test cloudsync_is_sync function +bool do_test_is_sync_function(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table + rc = sqlite3_exec(db, "CREATE TABLE sync_check (id TEXT PRIMARY KEY NOT NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('sync_check');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_is_sync + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_sync('sync_check');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + // Result depends on internal state + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_is_sync with non-existent table + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_sync('non_existent');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int is_sync = sqlite3_column_int(stmt, 0); + if (is_sync != 0) goto cleanup; // Should be 0 for non-existent table + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test cloudsync_db_version_next function +bool do_test_db_version_next(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and sync a table to properly initialize the context + rc = sqlite3_exec(db, "CREATE TABLE dbv_test (id TEXT PRIMARY KEY NOT NULL, val TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_sync('dbv_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_db_version_next without argument + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_db_version_next();", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int64_t v1 = sqlite3_column_int64(stmt, 0); + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_db_version_next with merging_version argument + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_db_version_next(100);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int64_t v2 = sqlite3_column_int64(stmt, 0); + + sqlite3_finalize(stmt); + stmt = NULL; + + // v2 should be greater or equal to v1 + if (v2 < v1) goto cleanup; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test various insert/update/delete scenarios through SQL +bool do_test_insert_update_delete_sql(void) { + sqlite3 *db = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table + rc = sqlite3_exec(db, "CREATE TABLE iud_test (id TEXT PRIMARY KEY NOT NULL, val TEXT, num REAL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('iud_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Insert data + rc = sqlite3_exec(db, "INSERT INTO iud_test (id, val, num) VALUES ('id1', 'initial', 1.5);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Update data + rc = sqlite3_exec(db, "UPDATE iud_test SET val = 'updated', num = 2.5 WHERE id = 'id1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Insert more data + rc = sqlite3_exec(db, "INSERT INTO iud_test (id, val, num) VALUES ('id2', 'second', 3.5);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Delete data + rc = sqlite3_exec(db, "DELETE FROM iud_test WHERE id = 'id2';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Check changes table + rc = sqlite3_exec(db, "SELECT COUNT(*) FROM cloudsync_changes;", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + result = true; + +cleanup: + if (db) close_db(db); + return result; +} + +// Test dbutils_binary_comparison function (already exposed for testing) +bool do_test_binary_comparison(void) { + // Test cases for dbutils_binary_comparison + int result1 = dbutils_binary_comparison(5, 3); // 5 > 3, should return 1 + if (result1 != 1) return false; + + int result2 = dbutils_binary_comparison(3, 5); // 3 < 5, should return -1 + if (result2 != -1) return false; + + int result3 = dbutils_binary_comparison(5, 5); // 5 == 5, should return 0 + if (result3 != 0) return false; + + int result4 = dbutils_binary_comparison(-10, 10); // -10 < 10 + if (result4 != -1) return false; + + int result5 = dbutils_binary_comparison(0, 0); // 0 == 0 + if (result5 != 0) return false; + + return true; +} + + +// Test pk_decode with various malformed inputs +bool do_test_pk_decode_malformed(void) { + // Test with empty buffer + int res = pk_decode_prikey(NULL, 0, NULL, NULL); + if (res != -1) return false; // Should fail for NULL buffer + + // Test with buffer but 0 length + char empty[1] = {0}; + res = pk_decode_prikey(empty, 0, NULL, NULL); + // This should also fail since count can't be read + if (res != -1) return false; + + // Test pk_decode with count specified but incomplete buffer + size_t seek = 0; + res = pk_decode(empty, 0, 5, &seek, -1, NULL, NULL); + if (res != -1) return false; // Should fail - can't read 5 elements from empty buffer + + return true; +} + + +bool do_test_many_columns (int ncols, sqlite3 *db) { + char sql_create[10000]; + int pos = 0; + pos += snprintf(sql_create+pos, sizeof(sql_create)-pos, "CREATE TABLE IF NOT EXISTS test_many_columns (id TEXT PRIMARY KEY NOT NULL"); + for (int i=1; i sqlite3_step reports a different result: %d != %d\n", rc1, rc2); + goto finalize; + } + // rc(s) are equals here + if (rc1 == SQLITE_DONE) break; + if (rc1 != SQLITE_ROW) goto finalize; + + // we have a ROW here + for (int i=0; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + int table_mask = TEST_PRIKEYS; + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i customers\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); + } + } + + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test a sequence of random different concurrent changes in various clients +// with changes to pk columns and non-pk columns. +bool do_test_merge_2 (int nclients, int table_mask, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int nrows = NINSERT; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i " CUSTOMERS_TABLE "\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + if (table_mask & TEST_NOCOLS) { + printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "s\"\n"); + do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test cloudsync_merge_insert where local col_version is equal to the insert col_version, +// the greater value must win. +bool do_test_merge_4 (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + int table_mask = TEST_PRIKEYS; + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i " CUSTOMERS_TABLE "\n"); + sqlite3_snprintf(sizeof(buf), buf, "SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], buf, query_table); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i, col_name:age, db_version:2, col_version:2, site_id:1 (remote change) +// b. pk:, col_name:TOMBSTONE, db_version:3, col_version:2, site_id:0 (local change, delete old pk) +// c. pk:, col_name:TOMBSTONE, db_version:3, col_version:1, site_id:0 (local change, updated row with new pk) +// 5. merge changes from the second client to the first client: +// - if the second client sends only local changes, then the change at 4a is not sent so the first client will show NULL +// at column "age" for the row with the new pk +// - if the second client sends all the changes (not only local changes), then the change at 4a is sent along with 4b and 4c +// so the first client will show the correct value for the "age" column +bool do_test_merge_5 (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + int table_mask = TEST_PRIKEYS; + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i customers\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + time_t timestamp = time(NULL); + int saved_counter = test_counter++; + for (int i=0; i customers\n"); + do_query(db[1], "SELECT * FROM cloudsync_changes;", query_changes); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + time_t timestamp = time(NULL); + int saved_counter = test_counter++; + for (int i=0; i 1;"; + sqlite3_stmt *vm = NULL; + rc = sqlite3_prepare_v2(db[i], sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(vm); + if (rc != SQLITE_DONE) { + printf("cloudsync_changes should not have repeated db_version and seq values, got: db_version=%d, seq=%d, count=%d\n", sqlite3_column_int(vm, 0), sqlite3_column_int(vm, 1), sqlite3_column_int(vm, 2)); + if (vm) sqlite3_finalize(vm); + goto finalize; + } + if (vm) sqlite3_finalize(vm); + } + + // check grouped values from cloudsync_changes + char *sql_changes = "SELECT db_version, COUNT(distinct(seq)) AS cnt FROM cloudsync_changes GROUP BY db_version;"; + char *query_expected_results = "SELECT * FROM (VALUES (1,2),(2,2),(3,2),(4,2),(5,4));"; + if (do_compare_queries(db[0], sql_changes, db[0], query_expected_results, -1, -1, print_result) == false) { + goto finalize; + } + + if (print_result) { + printf("\n-> customers\n"); + do_query(db[1], "SELECT * FROM cloudsync_changes();", query_changes); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i local_col_version, even if db_version is smaller + rc = sqlite3_exec(db, "INSERT INTO cloudsync_changes (tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) VALUES ('todo',x'010B0133','status','this update must be applied',11,9,x'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA',1,0);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; + + // compare results + char *sql = "SELECT * FROM todo ORDER BY id;"; + char *query_expected_results = "SELECT * FROM (VALUES ('1','Buy groceries','finalized'),('2','Buy bananas','in_progress2'),('3','New Item','this update must be applied'));"; + if (do_compare_queries(db, sql, db, query_expected_results, -1, -1, print_result) == false) { + goto finalize; + } + + // compare values in the changes vtab + sql = "SELECT col_name,col_value,col_version,db_version,cl,seq FROM cloudsync_changes;"; + query_expected_results = "SELECT * FROM (VALUES ('title','Buy groceries',1,1,1,0),('title','Buy bananas',1,2,1,0),('status','in_progress2',1,2,1,1),('title','New Item',10,10,1,0),('status','finalized',200,300,1,0),('status','this update must be applied',11,301,1,0));"; + if (do_compare_queries(db, sql, db, query_expected_results, -1, -1, print_result) == false) { + goto finalize; + } + + if (print_result) { + printf("\n-> customers\n"); + do_query(db, "SELECT * FROM todo;", query_changes); + } + + result = true; + rc = SQLITE_OK; + +finalize: + if (rc != SQLITE_OK && db && (sqlite3_errcode(db) != SQLITE_OK)) printf("do_test_insert_cloudsync_changes: %s\n", sqlite3_errmsg(db)); + if (db) close_db(db); + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, 0, timestamp, saved_counter); + file_delete_internal(buf); + } + return result; +} + +bool do_test_merge_alter_schema_1 (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i customers\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + int table_mask = TEST_PRIKEYS; + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i customers\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i " CUSTOMERS_TABLE "\n"); + sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + + printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); + do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_two_tables error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test conflicting primary key updates +bool do_test_merge_conflicting_pkeys (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i " CUSTOMERS_TABLE " (after conflict resolution)\n"); + sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_conflicting_pkeys error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test large dataset merge performance +bool do_test_merge_large_dataset (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + const int LARGE_N = 1000; // Insert 1000 records per client + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i Large dataset merge completed\n"); + char *sql = sqlite3_mprintf("SELECT COUNT(*) as total_records FROM \"%w\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_large_dataset error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test nested transactions before merge +bool do_test_merge_nested_transactions (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_nested_transactions error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test three-way merge pattern +bool do_test_merge_three_way (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients < 3) { + nclients = 3; + printf("Number of test merge increased to %d clients\n", 3); + } + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; iB, then B->C, then verify A==C + if (do_merge_using_payload(db[0], db[1], false, true) == false) { + goto finalize; + } + + if (do_merge_using_payload(db[1], db[2], false, true) == false) { + goto finalize; + } + + if (do_merge_using_payload(db[2], db[0], false, true) == false) { + goto finalize; + } + + // final full merge to ensure consistency + if (do_merge(db, nclients, false) == false) { + goto finalize; + } + + // verify all databases are identical + for (int i=1; i 0) { + result = false; + printf("do_test_merge_three_way error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test NULL value handling during merge +bool do_test_merge_null_values (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_null_values error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test BLOB data merge +bool do_test_merge_blob_data (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i BLOB test table\n"); + do_query(db[0], "SELECT id, HEX(data), description FROM blob_test ORDER BY id;", query_table); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_blob_data error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test mixed operations (INSERT/UPDATE/DELETE) in transactions +bool do_test_merge_mixed_operations (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_mixed_operations error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test hub-spoke merge pattern (central client merging with multiple peripherals) +bool do_test_merge_hub_spoke (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients < 4) { + nclients = 4; + printf("Number of test merge increased to %d clients\n", 4); + } + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i hub + for (int i=1; i all spokes + for (int i=1; i 0) { + result = false; + printf("do_test_merge_hub_spoke error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test timestamp precision conflicts +bool do_test_merge_timestamp_precision (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i Timestamp precision test\n"); + do_query(db[0], "SELECT id, created_at, data FROM timestamp_test ORDER BY id, created_at;", query_table); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_timestamp_precision error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test partial merge failure recovery +bool do_test_merge_partial_failure (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_partial_failure error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test transaction rollback scenarios +bool do_test_merge_rollback_scenarios (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + printf("Rollback failed - found %d rolled back records\n", rollback_count); + sqlite3_finalize(stmt); + goto finalize; + } + } + sqlite3_finalize(stmt); + } + + // merge changes + if (do_merge(db, nclients, false) == false) { + goto finalize; + } + + // verify consistency - only committed data should be present + for (int i=1; i 0) { + result = false; + printf("do_test_merge_rollback_scenarios error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test circular merge (A->B->C->A) +bool do_test_merge_circular (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients < 3) { + nclients = 3; + printf("Number of test merge increased to %d clients\n", 3); + } + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i1->2->0 + if (do_merge_using_payload(db[0], db[1], false, true) == false) { + goto finalize; + } + + if (do_merge_using_payload(db[1], db[2], false, true) == false) { + goto finalize; + } + + if (do_merge_using_payload(db[2], db[0], false, true) == false) { + goto finalize; + } + + // complete the circle and ensure consistency + if (do_merge(db, nclients, false) == false) { + goto finalize; + } + + // verify all clients have all data + for (int i=1; i Circular merge result\n"); + do_query(db[0], "SELECT * FROM circular_test ORDER BY origin_client;", query_table); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_circular error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test foreign key constraints during merge +bool do_test_merge_foreign_keys (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_foreign_keys error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test trigger interaction during merge +// Expected failure: TRIGGERs are not fully supported by this extension. +bool do_test_merge_triggers (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i Trigger test results\n"); + do_query(db[0], "SELECT id, data, update_count FROM trigger_test ORDER BY id;", query_table); + printf("\n-> Audit log\n"); + do_query(db[0], "SELECT COUNT(*) as audit_entries FROM audit_log;", query_table); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_triggers error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test index consistency during merge +bool do_test_merge_index_consistency (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_index_consistency error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test JSON column merge +bool do_test_merge_json_columns (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i JSON test results\n"); + do_query(db[0], "SELECT id, json_extract(metadata, '$.name') as name, json_extract(metadata, '$.preferences.theme') as theme FROM json_test ORDER BY id;", query_table); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_json_columns error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test concurrent merge attempts +bool do_test_payload_apply_concurrent_write (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + sqlite3 *db_target2 = NULL; + sqlite3_stmt *select_stmt = NULL; + sqlite3_stmt *apply_stmt = NULL; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients < 2) nclients = 2; + if (nclients > 2) nclients = 2; // this test uses exactly 2 databases + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + + // create two file-based databases: db[0]=src, db[1]=target + for (int i = 0; i < nclients; ++i) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (db[i] == NULL) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE concurrent_tbl (id TEXT PRIMARY KEY, val TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('concurrent_tbl', 'cls', 1);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + } + + // insert data on src (db[0]) + rc = sqlite3_exec(db[0], "INSERT INTO concurrent_tbl VALUES ('row1', 'hello');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "INSERT INTO concurrent_tbl VALUES ('row2', 'world');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // extract payload from db[0] + const char *encode_sql = "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes WHERE site_id=cloudsync_siteid();"; + rc = sqlite3_prepare_v2(db[0], encode_sql, -1, &select_stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_step(select_stmt); + if (rc != SQLITE_ROW) goto finalize; + + const void *payload_data = sqlite3_column_blob(select_stmt, 0); + int payload_size = sqlite3_column_bytes(select_stmt, 0); + if (payload_data == NULL || payload_size == 0) goto finalize; + + // copy payload since we'll need it after finalizing select_stmt + void *payload_copy = malloc(payload_size); + if (payload_copy == NULL) goto finalize; + memcpy(payload_copy, payload_data, payload_size); + sqlite3_finalize(select_stmt); + select_stmt = NULL; + + // open second connection to same file as db[1] (target) + { + char buf[256]; + do_build_database_path(buf, 1, timestamp, saved_counter + 1); + rc = sqlite3_open(buf, &db_target2); + if (rc != SQLITE_OK) { + printf("Error opening db_target2: %s\n", sqlite3_errmsg(db_target2)); + free(payload_copy); + goto finalize; + } + sqlite3_exec(db_target2, "PRAGMA journal_mode=WAL;", NULL, NULL, NULL); + sqlite3_cloudsync_init(db_target2, NULL, NULL); + } + + // on db[1] (target): begin immediate to hold write lock + rc = sqlite3_exec(db[1], "BEGIN IMMEDIATE;", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("BEGIN IMMEDIATE failed: %s\n", sqlite3_errmsg(db[1])); + free(payload_copy); + goto finalize; + } + rc = sqlite3_exec(db[1], "INSERT INTO concurrent_tbl VALUES ('blocker', 'blocking');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("Blocker INSERT failed: %s\n", sqlite3_errmsg(db[1])); + free(payload_copy); + goto finalize; + } + + // on db_target2: try to apply payload — should fail with BUSY + rc = sqlite3_prepare_v2(db_target2, "SELECT cloudsync_payload_decode(?);", -1, &apply_stmt, NULL); + if (rc != SQLITE_OK) { + printf("Prepare apply failed: %s\n", sqlite3_errmsg(db_target2)); + free(payload_copy); + goto finalize; + } + rc = sqlite3_bind_blob(apply_stmt, 1, payload_copy, payload_size, SQLITE_STATIC); + if (rc != SQLITE_OK) { + printf("Bind failed: %s\n", sqlite3_errmsg(db_target2)); + free(payload_copy); + goto finalize; + } + + // set a short busy timeout so it doesn't wait forever (0 = fail immediately) + sqlite3_busy_timeout(db_target2, 0); + + rc = sqlite3_step(apply_stmt); + if (rc == SQLITE_ROW || rc == SQLITE_DONE) { + printf("Expected BUSY error but apply succeeded (rc=%d)\n", rc); + free(payload_copy); + goto finalize; + } + + // verify we got a BUSY-related error + int errcode = sqlite3_errcode(db_target2); + if (errcode != SQLITE_BUSY && errcode != SQLITE_LOCKED) { + printf("Expected SQLITE_BUSY or SQLITE_LOCKED but got error %d: %s\n", errcode, sqlite3_errmsg(db_target2)); + free(payload_copy); + goto finalize; + } + + if (print_result) { + printf(" Step 1: Apply blocked as expected (errcode=%d: %s)\n", errcode, sqlite3_errmsg(db_target2)); + } + + // release the write lock on db[1] + rc = sqlite3_exec(db[1], "COMMIT;", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("COMMIT failed: %s\n", sqlite3_errmsg(db[1])); + free(payload_copy); + goto finalize; + } + + // retry: reset and step again — should succeed now + sqlite3_reset(apply_stmt); + sqlite3_busy_timeout(db_target2, 5000); // give it time now + + rc = sqlite3_step(apply_stmt); + if (rc != SQLITE_ROW) { + printf("Expected SQLITE_ROW on retry but got %d: %s\n", rc, sqlite3_errmsg(db_target2)); + free(payload_copy); + goto finalize; + } + + if (print_result) { + printf(" Step 2: Apply succeeded after lock released\n"); + } + + sqlite3_finalize(apply_stmt); + apply_stmt = NULL; + free(payload_copy); + payload_copy = NULL; + + // verify: db_target2 should have row1, row2 (from payload) + blocker (from db[1]) + { + sqlite3_stmt *count_stmt = NULL; + rc = sqlite3_prepare_v2(db_target2, "SELECT COUNT(*) FROM concurrent_tbl;", -1, &count_stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(count_stmt); + if (rc != SQLITE_ROW) { sqlite3_finalize(count_stmt); goto finalize; } + int count = sqlite3_column_int(count_stmt, 0); + sqlite3_finalize(count_stmt); + if (count != 3) { + printf("Expected 3 rows but got %d\n", count); + goto finalize; + } + if (print_result) { + printf(" Step 3: Target has %d rows (expected 3)\n", count); + } + } + + // full consistency: merge all databases using payload + // first close db_target2 to avoid lock conflicts during merge + close_db(db_target2); + db_target2 = NULL; + + // merge db[0] <-> db[1] in both directions + if (do_merge_using_payload(db[0], db[1], false, true) == false) { + printf("Merge src->target failed\n"); + goto finalize; + } + if (do_merge_using_payload(db[1], db[0], false, true) == false) { + printf("Merge target->src failed\n"); + goto finalize; + } + + // verify consistency + { + const char *sql = "SELECT * FROM concurrent_tbl ORDER BY id;"; + bool cmp = do_compare_queries(db[0], sql, db[1], sql, -1, -1, print_result); + if (!cmp) { + printf("Consistency check failed between src and target\n"); + goto finalize; + } + } + + if (print_result) { + printf(" Step 4: Full consistency verified\n"); + } + + result = true; + +finalize: + if (select_stmt) sqlite3_finalize(select_stmt); + if (apply_stmt) sqlite3_finalize(apply_stmt); + if (db_target2) close_db(db_target2); + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_payload_apply_concurrent_write error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) { + if (sqlite3_get_autocommit(db[i]) == 0) { + result = false; + printf("do_test_payload_apply_concurrent_write error: db %d is in transaction\n", i); + } + int counter = close_db(db[i]); + if (counter > 0) { + result = false; + printf("do_test_payload_apply_concurrent_write error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +bool do_test_merge_concurrent_attempts (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 3) { + nclients = 3; + printf("Number of test merge increased to %d clients\n", 3); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i1 and 1->2 simultaneously + bool merge1 = do_merge_using_payload(db[0], db[1], false, true); + bool merge2 = do_merge_using_payload(db[1], db[2], false, true); + + if (!merge1 || !merge2) { + printf("Concurrent merge phase 1 had issues\n"); + } + + // second: 2->0 and 0->1 simultaneously + bool merge3 = do_merge_using_payload(db[2], db[0], false, true); + bool merge4 = do_merge_using_payload(db[0], db[1], false, true); + + if (!merge3 || !merge4) { + printf("Concurrent merge phase 2 had issues\n"); + } + + // final consistency merge + if (do_merge(db, nclients, false) == false) { + printf("Final merge failed\n"); + } + + // verify all clients have same data count + int expected_count = nclients * 5; // each client inserted 5 records + for (int i=0; i 0) { + result = false; + printf("do_test_merge_concurrent_attempts error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test with 10 clients using a simple table with composite primary key +// Summary of the test function: +// 1. Simple table with composite primary key: Creates a table with columns (id1, id2) as the composite primary key and (data1, data2) as non-primary key columns +// 2. 10 database simulation: Forces exactly 10 clients regardless of the parameter passed +// 3. CloudSync initialization: Each database initializes the cloudsync library for the test table +// 4. Initial data: Each client inserts 3 initial rows with unique data +// 5. Multiple operation rounds: Performs 5 rounds of operations where each client performs: +// - Inserts new rows (20%) +// - Updates local data (15%) - data inserted by the same client +// - Updates remote data (50%) - data inserted by other clients, tests cross-client conflicts +// - Deletes both local and remote data (15%) - minimal deletions to preserve data for updates +// 6. Partial merging: After each round, clients merge with only 2-3 other clients (not all), simulating real-world scenarios where not all clients sync immediately +// 7. Final convergence: +// - Merges all changes from all clients to the first client (client 0) +// - Propagates the final state from client 0 to all other clients +// 8. Verification: Compares all databases to ensure they have converged to identical final states +bool do_test_merge_composite_pk_10_clients (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + const char *table_name = "simple_composite_test"; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); } else if (nclients < 2) { nclients = 2; printf("Number of test merge increased to %d clients\n", 2); } + // Force to 10 clients for this specific test + nclients = 10; + // create databases and tables - int table_mask = TEST_PRIKEYS; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 0) { + printf("Client %d updated %d local rows in round %d\n", i, updated_rows, round + 1); + } + } + break; + + case 2: // UPDATE remote data (more aggressive - creates conflicts) + { + int target_client = (i + 1 + round) % nclients; // Vary target client + char *sql; + int updated_rows = 0; + + // First try: update rows from target client's range + sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_targeting%d', data2 = %d WHERE id1 BETWEEN %d AND %d;", + table_name, round, i, target_client, round * 2000 + i, target_client * 100, (target_client + 1) * 100 - 1); + rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + sqlite3_free(sql); + updated_rows += sqlite3_changes(db[i]); + + // Second try: if no rows updated, target any rows not from current client + if (updated_rows == 0) { + sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_any', data2 = %d WHERE id1 NOT BETWEEN %d AND %d LIMIT 2;", + table_name, round, i, round * 2000 + i, i * 100, (i + 1) * 100 - 1); + rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + sqlite3_free(sql); + updated_rows += sqlite3_changes(db[i]); + } + + // Third try: if still no updates, target any available rows + if (updated_rows == 0) { + sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_fallback', data2 = %d WHERE rowid IN (SELECT rowid FROM %s LIMIT 2 OFFSET %d);", + table_name, round, i, round * 2000 + i, table_name, (i + round) % 10); + rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + sqlite3_free(sql); + updated_rows += sqlite3_changes(db[i]); + } + + if (rc != SQLITE_OK) { + printf("Update remote data failed on client %d targeting client %d: %s\n", i, target_client, sqlite3_errmsg(db[i])); + } else if (print_result && updated_rows > 0) { + printf("Client %d updated %d remote rows in round %d\n", i, updated_rows, round + 1); + } + } + break; + + case 3: // DELETE (favoring remote data deletion) + { + // 80% chance to delete remote data, 20% chance to delete local data + if ((i + round) % 5 == 0) { + // Delete local data (20% of delete operations) + char *sql = sqlite3_mprintf("DELETE FROM %s WHERE id1 = %d AND id2 = 'client%d_row1';", + table_name, i * 100 + 1, i); + rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + sqlite3_free(sql); + if (rc != SQLITE_OK) { + printf("Delete local data failed on client %d: %s\n", i, sqlite3_errmsg(db[i])); + } + } else { + // Delete remote data (80% of delete operations) + int target_client = (i + 2 + round) % nclients; // Vary target client + char *sql = sqlite3_mprintf("DELETE FROM %s WHERE id1 = %d AND id2 = 'client%d_row0';", + table_name, target_client * 100, target_client); + rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + sqlite3_free(sql); + if (rc != SQLITE_OK) { + printf("Delete remote data failed on client %d targeting client %d: %s\n", i, target_client, sqlite3_errmsg(db[i])); + } + } + } + break; + } } - if (do_augment_tables(TEST_PRIKEYS, db[i], table_algo_crdt_cls) == false) { - goto finalize; + // Partial merge: each client merges with 2-3 other random clients (not all) + for (int i = 0; i < nclients; ++i) { + int merge_targets = 2 + (round % 2); // 2 or 3 targets + for (int j = 0; j < merge_targets; ++j) { + int target = (i + j + 1 + round) % nclients; + if (target != i) { + if (do_merge_using_payload(db[i], db[target], true, true) == false) { + if (print_result) printf("Partial merge failed from client %d to %d\n", i, target); + goto finalize; + } + } + } } - } - - // insert, update and delete some data in the first client - do_insert(db[0], TEST_PRIKEYS, NINSERT, print_result); - - // alter table also on db0 - if (do_alter_tables(table_mask, db[0], 4) == false) { - goto finalize; - } - -#ifndef SKIP_SCHEMA_CHECK - // merge changes from db0 to db1, it should fail because db0 has a newer schema hash - if (do_merge_using_payload(db[0], db[1], only_locals, false) == true) { - goto finalize; - } -#endif - - // insert a new value on db1 - do_insert_val(db[1], TEST_PRIKEYS, 123456, print_result); - // merge changes from db1 to db0, it should work if columns are not removed - if (do_merge_using_payload(db[1], db[0], only_locals, false) == false) { - goto finalize; + if (print_result) printf("Completed round %d operations and partial merges\n", round + 1); } - // alter table also on db1 - if (do_alter_tables(table_mask, db[1], 4) == false) { - goto finalize; + // Final convergence: merge all changes to client 0 + if (print_result) printf("Starting final convergence to client 0\n"); + for (int i = 1; i < nclients; ++i) { + if (do_merge_using_payload(db[i], db[0], true, true) == false) { + if (print_result) printf("Final merge to client 0 failed from client %d\n", i); + goto finalize; + } } - // merge all changes, now it must work fine - if (do_merge(db, nclients, only_locals) == false) { - goto finalize; + // Merge changes from client 0 to all other clients + if (print_result) printf("Propagating final state from client 0 to all other clients\n"); + for (int i = 1; i < nclients; ++i) { + if (do_merge_using_payload(db[0], db[i], false, true) == false) { + if (print_result) printf("Final merge from client 0 failed to client %d\n", i); + goto finalize; + } } - - // compare results - for (int i=1; i customers\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); + printf("\nFinal converged state:\n"); + char *display_sql = sqlite3_mprintf("SELECT * FROM %s ORDER BY id1, id2;", table_name); + do_query(db[0], display_sql, NULL); + sqlite3_free(display_sql); } result = true; - rc = SQLITE_OK; finalize: - for (int i=0; i 0) { + result = false; + printf("do_test_merge_composite_pk_10_clients error: db %d has %d unterminated statements\n", i, counter); + } + } if (cleanup_databases) { char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); + do_build_database_path(buf, i, timestamp, saved_counter); file_delete_internal(buf); } } return result; } -bool do_test_merge_two_tables (int nclients, bool print_result, bool cleanup_databases) { +bool do_test_prikey (int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); if (nclients >= MAX_SIMULATED_CLIENTS) { @@ -3197,99 +7452,202 @@ bool do_test_merge_two_tables (int nclients, bool print_result, bool cleanup_dat time_t timestamp = time(NULL); int saved_counter = test_counter; for (int i=0; i foo (client1)\n"); + do_query(db[0], "SELECT * FROM foo ORDER BY a;", NULL); + } + + // send local changes to client1 + for (int i=1; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } - sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\", age, note, stamp) VALUES ('alice', 'jones', 25, 'another note', 'stamp2');", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[1], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; + // create databases and tables + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i " CUSTOMERS_TABLE "\n"); - sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - - printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); - do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + printf("\n-> log\n"); + do_query(db[0], "SELECT * FROM log ORDER BY id;", NULL); } result = true; - rc = SQLITE_OK; finalize: for (int i=0; i 0) { - result = false; - printf("do_test_merge_two_tables error: db %d has %d unterminated statements\n", i, counter); - } - } + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) printf("do_test_gos error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); + if (data[i]) cloudsync_context_free(data[i]); + if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3299,12 +7657,14 @@ bool do_test_merge_two_tables (int nclients, bool print_result, bool cleanup_dat return result; } -// Test conflicting primary key updates -bool do_test_merge_conflicting_pkeys (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - + +bool do_test_network_encode_decode (int nclients, bool print_result, bool cleanup_databases, bool force_uncompressed) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + cloudsync_context *data[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); if (nclients >= MAX_SIMULATED_CLIENTS) { @@ -3315,12 +7675,17 @@ bool do_test_merge_conflicting_pkeys (int nclients, bool print_result, bool clea printf("Number of test merge increased to %d clients\n", 2); } + // create databases and tables + int table_mask = TEST_PRIKEYS | TEST_NOCOLS; time_t timestamp = time(NULL); int saved_counter = test_counter; for (int i=0; i " CUSTOMERS_TABLE " (after conflict resolution)\n"); - sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + printf("\n-> customers\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); do_query(db[0], sql, query_table); sqlite3_free(sql); } result = true; + rc = SQLITE_OK; finalize: for (int i=0; i 0) { - result = false; - printf("do_test_merge_conflicting_pkeys error: db %d has %d unterminated statements\n", i, counter); - } - } + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) printf("do_test_merge error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); + if (data[i]) cloudsync_context_free(data[i]); + if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3398,13 +7762,12 @@ bool do_test_merge_conflicting_pkeys (int nclients, bool print_result, bool clea return result; } -// Test large dataset merge performance -bool do_test_merge_large_dataset (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - + +bool do_test_fill_initial_data(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; - const int LARGE_N = 1000; // Insert 1000 records per client memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); if (nclients >= MAX_SIMULATED_CLIENTS) { @@ -3415,6 +7778,8 @@ bool do_test_merge_large_dataset (int nclients, bool print_result, bool cleanup_ printf("Number of test merge increased to %d clients\n", 2); } + // create databases and tables + int table_mask = TEST_PRIKEYS | TEST_NOCOLS; time_t timestamp = time(NULL); int saved_counter = test_counter; for (int i=0; i Large dataset merge completed\n"); - char *sql = sqlite3_mprintf("SELECT COUNT(*) as total_records FROM \"%w\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); + if (table_mask & TEST_PRIKEYS) { + printf("\n-> " CUSTOMERS_TABLE "\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + if (table_mask & TEST_NOCOLS) { + printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); + do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } + if (table_mask & TEST_NOPRIKEYS) { + printf("\n-> customers_noprikey\n"); + do_query(db[0], "SELECT * FROM customers_noprikey ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } } result = true; + rc = SQLITE_OK; finalize: for (int i=0; i 0) { result = false; - printf("do_test_merge_large_dataset error: db %d has %d unterminated statements\n", i, counter); + printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); } } + if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3492,12 +7868,10 @@ bool do_test_merge_large_dataset (int nclients, bool print_result, bool cleanup_ return result; } -// Test nested transactions before merge -bool do_test_merge_nested_transactions (int nclients, bool print_result, bool cleanup_databases) { +bool do_test_alter(int nclients, int alter_version, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); if (nclients >= MAX_SIMULATED_CLIENTS) { @@ -3508,6 +7882,8 @@ bool do_test_merge_nested_transactions (int nclients, bool print_result, bool cl printf("Number of test merge increased to %d clients\n", 2); } + // create databases and tables + int table_mask = TEST_PRIKEYS | TEST_NOCOLS; time_t timestamp = time(NULL); int saved_counter = test_counter; for (int i=0; i " CUSTOMERS_TABLE "\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + if (table_mask & TEST_NOCOLS) { + printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); + do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } + if (table_mask & TEST_NOPRIKEYS) { + printf("\n-> customers_noprikey\n"); + do_query(db[0], "SELECT * FROM customers_noprikey ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } + } - rc = sqlite3_exec(db[0], "SAVEPOINT sp1;", NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; + result = true; + rc = SQLITE_OK; - sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\") VALUES ('tx2', 'user');", CUSTOMERS_NOCOLS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); +finalize: + for (int i=0; i= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i = 0; i < nclients; ++i) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + + // Create table + rc = sqlite3_exec(db[i], "CREATE TABLE tasks(id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Init cloudsync + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('tasks');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Set filter: only sync rows where user_id = 1 + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_filter('tasks', 'user_id = 1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + } + + // --- Test 1: Insert matching and non-matching rows on db[0] --- + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('a', 'Task A', 1);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - rc = sqlite3_exec(db[1], "RELEASE sp2;", NULL, NULL, NULL); + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('b', 'Task B', 2);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - rc = sqlite3_exec(db[1], "COMMIT;", NULL, NULL, NULL); + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('c', 'Task C', 1);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // merge changes - if (do_merge(db, nclients, false) == false) { - goto finalize; + + // Verify: tasks_cloudsync should only have metadata for user_id=1 rows ('a' and 'c') + { + // Count distinct PKs in the meta table + int64_t meta_count = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM tasks_cloudsync;"); + if (meta_count != 2) { + printf("do_test_row_filter: expected 2 tracked PKs after insert, got %" PRId64 "\n", meta_count); + goto finalize; + } } - - // verify consistency - for (int i=1; i %" PRId64 ")\n", before, after); + goto finalize; + } } - + + // --- Test 4: Delete non-matching row → NO metadata change --- + { + int64_t before = test_query_int(db[0], "SELECT COUNT(*) FROM tasks_cloudsync;"); + rc = sqlite3_exec(db[0], "DELETE FROM tasks WHERE id='b';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + int64_t after = test_query_int(db[0], "SELECT COUNT(*) FROM tasks_cloudsync;"); + if (after != before) { + printf("do_test_row_filter: non-matching DELETE changed meta count (%" PRId64 " -> %" PRId64 ")\n", before, after); + goto finalize; + } + } + + // --- Test 5: Delete matching row → metadata should update (tombstone) --- + rc = sqlite3_exec(db[0], "DELETE FROM tasks WHERE id='a';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // --- Test 6: Merge from db[0] to db[1] and verify only filtered rows transfer --- + if (do_merge_using_payload(db[0], db[1], true, true) == false) goto finalize; + + { + // db[1] should have 'c' (user_id=1) and the tombstone for 'a', but NOT 'b' + int64_t task_count = test_query_int(db[1], "SELECT COUNT(*) FROM tasks;"); + if (task_count != 1) { + printf("do_test_row_filter: expected 1 row in db[1] tasks after merge, got %" PRId64 "\n", task_count); + goto finalize; + } + // Verify it's 'c' + int64_t c_exists = test_query_int(db[1], "SELECT COUNT(*) FROM tasks WHERE id='c';"); + if (c_exists != 1) { + printf("do_test_row_filter: expected task 'c' in db[1], not found\n"); + goto finalize; + } + } + + if (print_result) { + printf("\n-> tasks (db[0])\n"); + do_query(db[0], "SELECT * FROM tasks ORDER BY id;", NULL); + printf("\n-> tasks_cloudsync (db[0])\n"); + do_query(db[0], "SELECT hex(pk), col_name, col_version, db_version FROM tasks_cloudsync ORDER BY pk, col_name;", NULL); + printf("\n-> tasks (db[1])\n"); + do_query(db[1], "SELECT * FROM tasks ORDER BY id;", NULL); + } + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_nested_transactions error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3611,89 +8226,100 @@ bool do_test_merge_nested_transactions (int nclients, bool print_result, bool cl return result; } -// Test three-way merge pattern -bool do_test_merge_three_way (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Clear Filter - + +bool do_test_row_filter_clear(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients < 3) { - nclients = 3; - printf("Number of test merge increased to %d clients\n", 3); - } - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; iB, then B->C, then verify A==C - if (do_merge_using_payload(db[0], db[1], false, true) == false) { - goto finalize; + + // Insert matching and non-matching rows on db[0] + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('a', 'Task A', 1);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('b', 'Task B', 2);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('c', 'Task C', 1);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Test 1: Only matching rows tracked + { + int64_t meta_count = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM tasks_cloudsync;"); + if (meta_count != 2) { + printf("do_test_row_filter_clear: expected 2 tracked PKs before clear, got %" PRId64 "\n", meta_count); + goto finalize; + } } - - if (do_merge_using_payload(db[1], db[2], false, true) == false) { - goto finalize; + + // Test 2: Clear filter on db[0] — resets metatable with all rows (a, b, c) + rc = sqlite3_exec(db[0], "SELECT cloudsync_clear_filter('tasks');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Test 3: Insert non-matching row — should be tracked (no filter) + // clear_filter refilled with all 3 existing rows + 'd' = 4 + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('d', 'Task D', 2);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + { + int64_t meta_count = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM tasks_cloudsync;"); + if (meta_count != 4) { + printf("do_test_row_filter_clear: expected 4 tracked PKs after clear+insert, got %" PRId64 "\n", meta_count); + goto finalize; + } } - - if (do_merge_using_payload(db[2], db[0], false, true) == false) { - goto finalize; + + // Test 4: Update row 'b' — already tracked by clear_filter refill, meta count unchanged + rc = sqlite3_exec(db[0], "UPDATE tasks SET title='Task B Updated' WHERE id='b';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + { + int64_t meta_count = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM tasks_cloudsync;"); + if (meta_count != 4) { + printf("do_test_row_filter_clear: expected 4 tracked PKs after update on 'b', got %" PRId64 "\n", meta_count); + goto finalize; + } } - - // final full merge to ensure consistency - if (do_merge(db, nclients, false) == false) { - goto finalize; + + // Test 5: Sync to db[1] (which still has filter active) + // Payload apply bypasses local triggers, so db[1] should receive all rows + if (do_merge_using_payload(db[0], db[1], true, true) == false) goto finalize; + { + int64_t task_count = test_query_int(db[1], "SELECT COUNT(*) FROM tasks;"); + if (task_count != 4) { + printf("do_test_row_filter_clear: expected 4 rows in db[1] after merge, got %" PRId64 "\n", task_count); + goto finalize; + } } - - // verify all databases are identical - for (int i=1; i tasks (db[0]) after clear:\n"); + do_query(db[0], "SELECT * FROM tasks ORDER BY id;", NULL); + printf("\n-> tasks (db[1]) after merge:\n"); + do_query(db[1], "SELECT * FROM tasks ORDER BY id;", NULL); } - + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_three_way error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_clear error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3703,179 +8329,144 @@ bool do_test_merge_three_way (int nclients, bool print_result, bool cleanup_data return result; } -// Test NULL value handling during merge -bool do_test_merge_null_values (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Complex Expressions - + +bool do_test_row_filter_complex_expressions(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i 3');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // update with non-NULL values in client 1 - sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\", age, note, stamp) VALUES ('null_test', 'user', 25, 'updated', 'timestamp');", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[1], sql, NULL, NULL, NULL); - sqlite3_free(sql); + + rc = sqlite3_exec(db[0], "INSERT INTO items VALUES('a', 'active', 5, 'work', 1);", NULL, NULL, NULL); // matches if (rc != SQLITE_OK) goto finalize; - - // update NULL to non-NULL in client 0 - sql = sqlite3_mprintf("UPDATE \"%w\" SET age = 30 WHERE first_name = 'null_test';", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); - sqlite3_free(sql); + rc = sqlite3_exec(db[0], "INSERT INTO items VALUES('b', 'active', 2, 'work', 1);", NULL, NULL, NULL); // fails priority if (rc != SQLITE_OK) goto finalize; - - // update non-NULL to NULL in client 1 - sql = sqlite3_mprintf("UPDATE \"%w\" SET note = NULL WHERE first_name = 'null_test';", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[1], sql, NULL, NULL, NULL); - sqlite3_free(sql); + rc = sqlite3_exec(db[0], "INSERT INTO items VALUES('c', 'active', 5, 'work', 2);", NULL, NULL, NULL); // fails user_id if (rc != SQLITE_OK) goto finalize; - - // merge changes - if (do_merge(db, nclients, false) == false) { - goto finalize; - } - - // verify consistency - for (int i=1; i 0) { - result = false; - printf("do_test_merge_null_values error: db %d has %d unterminated statements\n", i, counter); - } + + { + int64_t meta_count = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM items_cloudsync;"); + if (meta_count != 1) { + printf("do_test_row_filter_complex: Phase A expected 1 tracked PK, got %" PRId64 "\n", meta_count); + goto finalize; } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); + } + + // --- Phase B: String literal with escaped quotes --- + rc = sqlite3_exec(db[0], "SELECT cloudsync_clear_filter('items');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_filter('items', 'status = ''active'' AND user_id = 1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + { + int64_t before = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM items_cloudsync;"); + rc = sqlite3_exec(db[0], "INSERT INTO items VALUES('d', 'active', 1, 'home', 1);", NULL, NULL, NULL); // matches + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "INSERT INTO items VALUES('e', 'inactive', 1, 'home', 1);", NULL, NULL, NULL); // fails status + if (rc != SQLITE_OK) goto finalize; + int64_t after = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM items_cloudsync;"); + if (after - before != 1) { + printf("do_test_row_filter_complex: Phase B expected 1 new PK, got %" PRId64 "\n", after - before); + goto finalize; } } - return result; -} -// Test BLOB data merge -bool do_test_merge_blob_data (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); + // --- Phase C: IS NULL --- + rc = sqlite3_exec(db[0], "SELECT cloudsync_clear_filter('items');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_filter('items', 'category IS NULL');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + { + int64_t before = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM items_cloudsync;"); + rc = sqlite3_exec(db[0], "INSERT INTO items VALUES('f', 'x', 1, NULL, 1);", NULL, NULL, NULL); // matches + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "INSERT INTO items VALUES('g', 'x', 1, 'work', 1);", NULL, NULL, NULL); // fails + if (rc != SQLITE_OK) goto finalize; + int64_t after = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM items_cloudsync;"); + if (after - before != 1) { + printf("do_test_row_filter_complex: Phase C (IS NULL) expected 1 new PK, got %" PRId64 "\n", after - before); + goto finalize; + } } - - time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 3');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - const char *sql2 = "INSERT INTO blob_test (id, data, description) VALUES ('blob2', X'54657374204461746121', 'Test Data from client1');"; - rc = sqlite3_exec(db[1], sql2, NULL, NULL, NULL); + + // Insert on db[1] matching and non-matching + rc = sqlite3_exec(db[1], "INSERT INTO items VALUES('j', 'test', 5, 'cat', 1);", NULL, NULL, NULL); // matches if (rc != SQLITE_OK) goto finalize; - - // update BLOB data - const char *sql3 = "UPDATE blob_test SET data = X'5570646174656421' WHERE id = 'blob1';"; - rc = sqlite3_exec(db[0], sql3, NULL, NULL, NULL); + rc = sqlite3_exec(db[1], "INSERT INTO items VALUES('k', 'test', 1, 'cat', 1);", NULL, NULL, NULL); // fails priority if (rc != SQLITE_OK) goto finalize; - - // merge changes - if (do_merge(db, nclients, false) == false) { - goto finalize; - } - - // verify consistency - for (int i=1; i db[0] + if (do_merge_using_payload(db[1], db[0], true, true) == false) goto finalize; + { + int64_t j_exists = test_query_int(db[0], "SELECT COUNT(*) FROM items WHERE id='j';"); + if (j_exists != 1) { + printf("do_test_row_filter_complex: Phase E expected 'j' in db[0], not found\n"); + goto finalize; + } + int64_t k_exists = test_query_int(db[0], "SELECT COUNT(*) FROM items WHERE id='k';"); + if (k_exists != 0) { + printf("do_test_row_filter_complex: Phase E expected 'k' NOT in db[0], but found\n"); + goto finalize; + } } - + if (print_result) { - printf("\n-> BLOB test table\n"); - do_query(db[0], "SELECT id, HEX(data), description FROM blob_test ORDER BY id;", query_table); + printf("\n-> items (db[0]):\n"); + do_query(db[0], "SELECT * FROM items ORDER BY id;", NULL); + printf("\n-> items (db[1]):\n"); + do_query(db[1], "SELECT * FROM items ORDER BY id;", NULL); } - + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_blob_data error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_complex error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3885,125 +8476,132 @@ bool do_test_merge_blob_data (int nclients, bool print_result, bool cleanup_data return result; } -// Test mixed operations (INSERT/UPDATE/DELETE) in transactions -bool do_test_merge_mixed_operations (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Row Transition - + +bool do_test_row_filter_row_transition(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i non-matching) --- + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('a', 'Task A', 1);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + { + int64_t meta_before = test_query_int(db[0], "SELECT COUNT(*) FROM tasks_cloudsync;"); + // Update row to no longer match filter + rc = sqlite3_exec(db[0], "UPDATE tasks SET user_id = 2 WHERE id = 'a';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + int64_t meta_after = test_query_int(db[0], "SELECT COUNT(*) FROM tasks_cloudsync;"); + // UPDATE trigger should NOT fire (NEW.user_id=2 fails WHEN clause) + if (meta_after != meta_before) { + printf("do_test_row_filter_row_transition: Phase A - UPDATE out of filter changed meta (%" PRId64 " -> %" PRId64 ")\n", meta_before, meta_after); + goto finalize; } - - if (do_augment_tables(table_mask, db[i], table_algo_crdt_cls) == false) { - return false; + } + + // Sync to db[1]: payload reads current row values, so 'a' arrives with user_id=2 + // (the INSERT metadata triggers the sync, but the payload carries current column values) + if (do_merge_using_payload(db[0], db[1], true, true) == false) goto finalize; + { + int64_t a_uid = test_query_int(db[1], "SELECT user_id FROM tasks WHERE id='a';"); + if (a_uid != 2) { + printf("do_test_row_filter_row_transition: Phase A - db[1] expected user_id=2 for 'a', got %" PRId64 "\n", a_uid); + goto finalize; } } - - // initial data setup - char *sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\", age, note, stamp) VALUES ('initial', 'data', 25, 'original', 'stamp1');", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\", age, note, stamp) VALUES ('to_delete', 'data', 30, 'will_delete', 'stamp2');", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); - sqlite3_free(sql); + + // --- Phase B: Row enters filter (non-matching -> matching) --- + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('b', 'Task B', 2);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // sync initial data - if (do_merge(db, nclients, false) == false) { - goto finalize; + + { + int64_t meta_before = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM tasks_cloudsync;"); + // Update row to now match filter + rc = sqlite3_exec(db[0], "UPDATE tasks SET user_id = 1 WHERE id = 'b';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + int64_t meta_after = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM tasks_cloudsync;"); + // UPDATE trigger should fire (NEW.user_id=1 passes WHEN clause) + if (meta_after <= meta_before) { + printf("do_test_row_filter_row_transition: Phase B - UPDATE into filter did not create metadata (%" PRId64 " -> %" PRId64 ")\n", meta_before, meta_after); + goto finalize; + } } - - // mixed operations transaction in client 0 - rc = sqlite3_exec(db[0], "BEGIN TRANSACTION;", NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - // INSERT - sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\", age, note, stamp) VALUES ('new', 'record', 35, 'inserted', 'stamp3');", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - - // UPDATE - sql = sqlite3_mprintf("UPDATE \"%w\" SET age = 26, note = 'updated' WHERE first_name = 'initial';", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - - // DELETE - sql = sqlite3_mprintf("DELETE FROM \"%w\" WHERE first_name = 'to_delete';", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - - rc = sqlite3_exec(db[0], "COMMIT;", NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - // different mixed operations in client 1 - rc = sqlite3_exec(db[1], "BEGIN TRANSACTION;", NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\", age, note, stamp) VALUES ('another', 'record', 40, 'client1_insert', 'stamp4');", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[1], sql, NULL, NULL, NULL); - sqlite3_free(sql); + + // Sync to db[1]: should now get 'b' with user_id=1 + if (do_merge_using_payload(db[0], db[1], true, true) == false) goto finalize; + { + int64_t b_exists = test_query_int(db[1], "SELECT COUNT(*) FROM tasks WHERE id='b' AND user_id=1;"); + if (b_exists != 1) { + printf("do_test_row_filter_row_transition: Phase B - db[1] expected 'b' with user_id=1, not found\n"); + goto finalize; + } + } + + // --- Phase C: Bidirectional with transition --- + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('c', 'Task C', 1);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_mprintf("UPDATE \"%w\" SET note = 'client1_update' WHERE first_name = 'initial';", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[1], sql, NULL, NULL, NULL); - sqlite3_free(sql); + // Sync 'c' to db[1] + if (do_merge_using_payload(db[0], db[1], true, true) == false) goto finalize; + { + int64_t c_exists = test_query_int(db[1], "SELECT COUNT(*) FROM tasks WHERE id='c';"); + if (c_exists != 1) { + printf("do_test_row_filter_row_transition: Phase C - db[1] should have 'c'\n"); + goto finalize; + } + } + + // db[0]: move 'c' out of filter (no metadata generated) + rc = sqlite3_exec(db[0], "UPDATE tasks SET user_id = 2 WHERE id = 'c';", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - rc = sqlite3_exec(db[1], "COMMIT;", NULL, NULL, NULL); + + // db[1]: update 'c' title (still matching on db[1], metadata generated) + rc = sqlite3_exec(db[1], "UPDATE tasks SET title = 'Task C Edited' WHERE id = 'c';", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // merge all changes - if (do_merge(db, nclients, false) == false) { - goto finalize; + + // Sync db[1] -> db[0] + if (do_merge_using_payload(db[1], db[0], true, true) == false) goto finalize; + { + // db[0] should receive the title update from db[1] + int64_t c_title_match = test_query_int(db[0], "SELECT COUNT(*) FROM tasks WHERE id='c' AND title='Task C Edited';"); + if (c_title_match != 1) { + printf("do_test_row_filter_row_transition: Phase C - db[0] should have updated title for 'c'\n"); + goto finalize; + } } - - // verify consistency - for (int i=1; i tasks (db[0]):\n"); + do_query(db[0], "SELECT * FROM tasks ORDER BY id;", NULL); + printf("\n-> tasks (db[1]):\n"); + do_query(db[1], "SELECT * FROM tasks ORDER BY id;", NULL); } - + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_mixed_operations error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_row_transition error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4013,92 +8611,118 @@ bool do_test_merge_mixed_operations (int nclients, bool print_result, bool clean return result; } -// Test hub-spoke merge pattern (central client merging with multiple peripherals) -bool do_test_merge_hub_spoke (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Filter Change - + +bool do_test_row_filter_change(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients < 4) { - nclients = 4; - printf("Number of test merge increased to %d clients\n", 4); - } - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i hub - for (int i=1; i %" PRId64 ")\n", before, after); goto finalize; } } - - // hub inserts aggregated data - char *sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\", age, note, stamp) VALUES ('hub', 'aggregated', 50, 'hub_summary', 'hub_stamp');", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); - sqlite3_free(sql); + + // Test 4: Sync to db[1] — db[1] has filter 'user_id = 1' + rc = sqlite3_exec(db[1], "SELECT cloudsync_set_filter('tasks', 'user_id = 1');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // hub -> all spokes - for (int i=1; i tasks (db[0]):\n"); + do_query(db[0], "SELECT * FROM tasks ORDER BY id;", NULL); + printf("\n-> tasks_cloudsync (db[0]):\n"); + do_query(db[0], "SELECT hex(pk), col_name, col_version, db_version FROM tasks_cloudsync ORDER BY pk, col_name;", NULL); + printf("\n-> tasks (db[1]):\n"); + do_query(db[1], "SELECT * FROM tasks ORDER BY id;", NULL); } - + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_hub_spoke error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_change error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4108,89 +8732,135 @@ bool do_test_merge_hub_spoke (int nclients, bool print_result, bool cleanup_data return result; } -// Test timestamp precision conflicts -bool do_test_merge_timestamp_precision (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Composite PK + Multi-Table - + +bool do_test_row_filter_composite_pk_multi_table(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i Timestamp precision test\n"); - do_query(db[0], "SELECT id, created_at, data FROM timestamp_test ORDER BY id, created_at;", query_table); + printf("\n-> projects (db[0]):\n"); + do_query(db[0], "SELECT * FROM projects ORDER BY org_id, proj_id;", NULL); + printf("\n-> members (db[0]):\n"); + do_query(db[0], "SELECT * FROM members ORDER BY org_id, user_id;", NULL); + printf("\n-> projects (db[1]):\n"); + do_query(db[1], "SELECT * FROM projects ORDER BY org_id, proj_id;", NULL); + printf("\n-> members (db[1]):\n"); + do_query(db[1], "SELECT * FROM members ORDER BY org_id, user_id;", NULL); } - + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_timestamp_precision error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_composite error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4200,88 +8870,121 @@ bool do_test_merge_timestamp_precision (int nclients, bool print_result, bool cl return result; } -// Test partial merge failure recovery -bool do_test_merge_partial_failure (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Pre-existing Data (Prefill) - + +bool do_test_row_filter_prefill(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i tasks (db[0]):\n"); + do_query(db[0], "SELECT * FROM tasks ORDER BY id;", NULL); + printf("\n-> tasks_cloudsync (db[0]):\n"); + do_query(db[0], "SELECT hex(pk), col_name, col_version, db_version FROM tasks_cloudsync ORDER BY pk, col_name;", NULL); + printf("\n-> tasks (db[1]):\n"); + do_query(db[1], "SELECT * FROM tasks ORDER BY id;", NULL); + } + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_partial_failure error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_prefill error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4291,1396 +8994,4095 @@ bool do_test_merge_partial_failure (int nclients, bool print_result, bool cleanu return result; } -// Test transaction rollback scenarios -bool do_test_merge_rollback_scenarios (int nclients, bool print_result, bool cleanup_databases) { +// Test that BEFORE triggers with RAISE(ABORT) simulate RLS denial: +// per-PK savepoints isolate failures so allowed rows commit and denied rows roll back. +bool do_test_rls_trigger_denial (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); if (nclients >= MAX_SIMULATED_CLIENTS) { nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); } else if (nclients < 2) { nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); } - + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i 0) { - printf("Rollback failed - found %d rolled back records\n", rollback_count); - sqlite3_finalize(stmt); - goto finalize; - } + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES ('t5', 'user2', 'Task 5', 7);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Merge with partial-failure tolerance: cloudsync_payload_decode returns error + // when any PK is denied, but allowed PKs are already committed via per-PK savepoints. + { + sqlite3_stmt *sel = NULL, *ins = NULL; + const char *sel_sql = only_locals + ? "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes WHERE site_id=cloudsync_siteid();" + : "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes;"; + rc = sqlite3_prepare_v2(db[0], sel_sql, -1, &sel, NULL); + if (rc != SQLITE_OK) { sqlite3_finalize(sel); goto finalize; } + rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_decode(?);", -1, &ins, NULL); + if (rc != SQLITE_OK) { sqlite3_finalize(sel); sqlite3_finalize(ins); goto finalize; } + + while (sqlite3_step(sel) == SQLITE_ROW) { + sqlite3_value *v = sqlite3_column_value(sel, 0); + if (sqlite3_value_type(v) == SQLITE_NULL) continue; + sqlite3_bind_value(ins, 1, v); + sqlite3_step(ins); // partial failure expected — ignore rc + sqlite3_reset(ins); } + sqlite3_finalize(sel); + sqlite3_finalize(ins); + } + + // Verify: t4 present (user1 → allowed) + { + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT COUNT(*) FROM tasks WHERE id='t4';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } + int count = sqlite3_column_int(stmt, 0); sqlite3_finalize(stmt); + if (count != 1) { + printf("Phase 2: t4 expected 1 row, got %d\n", count); + goto finalize; + } } - - // merge changes - if (do_merge(db, nclients, false) == false) { - goto finalize; + + // Verify: t5 absent (user2 → denied) + { + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT COUNT(*) FROM tasks WHERE id='t5';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } + int count = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + if (count != 0) { + printf("Phase 2: t5 expected 0 rows, got %d\n", count); + goto finalize; + } + } + + // Verify: total 4 rows on B (t1, t2, t3 from phase 1 + t4) + { + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT COUNT(*) FROM tasks;", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } + int count = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + if (count != 4) { + printf("Phase 2: expected 4 total rows, got %d\n", count); + goto finalize; + } + } + + // --- Phase 3: UPDATE denial --- + rc = sqlite3_exec(db[0], "UPDATE tasks SET title='Task 1 Updated', priority=10 WHERE id='t1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "UPDATE tasks SET title='Task 2 Hacked', priority=99 WHERE id='t2';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Merge with partial-failure tolerance (same pattern as phase 2) + { + sqlite3_stmt *sel = NULL, *ins = NULL; + const char *sel_sql = only_locals + ? "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes WHERE site_id=cloudsync_siteid();" + : "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes;"; + rc = sqlite3_prepare_v2(db[0], sel_sql, -1, &sel, NULL); + if (rc != SQLITE_OK) { sqlite3_finalize(sel); goto finalize; } + rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_decode(?);", -1, &ins, NULL); + if (rc != SQLITE_OK) { sqlite3_finalize(sel); sqlite3_finalize(ins); goto finalize; } + + while (sqlite3_step(sel) == SQLITE_ROW) { + sqlite3_value *v = sqlite3_column_value(sel, 0); + if (sqlite3_value_type(v) == SQLITE_NULL) continue; + sqlite3_bind_value(ins, 1, v); + sqlite3_step(ins); // partial failure expected — ignore rc + sqlite3_reset(ins); + } + sqlite3_finalize(sel); + sqlite3_finalize(ins); + } + + // Verify: t1 updated (user1 → allowed) + { + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT title, priority FROM tasks WHERE id='t1';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } + const char *title = (const char *)sqlite3_column_text(stmt, 0); + int priority = sqlite3_column_int(stmt, 1); + bool ok = (strcmp(title, "Task 1 Updated") == 0) && (priority == 10); + sqlite3_finalize(stmt); + if (!ok) { + printf("Phase 3: t1 update not applied (title='%s', priority=%d)\n", title, priority); + goto finalize; + } + } + + // Verify: t2 unchanged (user2 → denied) + { + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT title, priority FROM tasks WHERE id='t2';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } + const char *title = (const char *)sqlite3_column_text(stmt, 0); + int priority = sqlite3_column_int(stmt, 1); + bool ok = (strcmp(title, "Task 2") == 0) && (priority == 5); + sqlite3_finalize(stmt); + if (!ok) { + printf("Phase 3: t2 should be unchanged (title='%s', priority=%d)\n", title, priority); + goto finalize; + } + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_rls_trigger_denial error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) { + if (sqlite3_get_autocommit(db[i]) == 0) { + result = false; + printf("do_test_rls_trigger_denial error: db %d is in transaction\n", i); + } + int counter = close_db(db[i]); + if (counter > 0) { + result = false; + printf("do_test_rls_trigger_denial error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// MARK: - Block-level LWW Tests - + +static int64_t do_select_int(sqlite3 *db, const char *sql) { + sqlite3_stmt *stmt = NULL; + int64_t val = -1; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + val = sqlite3_column_int64(stmt, 0); + } + } + if (stmt) sqlite3_finalize(stmt); + return val; +} + +// Regression: persisted block-column settings must not cause an infinite loop +// when the extension is reloaded. Without the fix, dbutils_settings_table_load_callback +// (driven by sqlite3_exec over cloudsync_table_settings) would re-invoke +// cloudsync_setup_block_column, which REPLACEd the same row mid-iteration and +// re-fed it back into the cursor. +bool do_test_block_column_reload(bool cleanup_databases) { + bool result = false; + char dbpath[256]; + time_t timestamp = time(NULL); + + #ifdef __ANDROID__ + snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockreload-%ld.sqlite", ".", timestamp); + #else + snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockreload-%ld.sqlite", getenv("HOME"), timestamp); + #endif + + // Phase 1: create database, table, init cloudsync, mark a column as block algo. + // Use a custom delimiter so both "algo" and "delimiter" rows get persisted. + sqlite3 *db = NULL; + int rc = sqlite3_open(dbpath, &db); + if (rc != SQLITE_OK) return false; + sqlite3_exec(db, "PRAGMA journal_mode=WAL;", NULL, NULL, NULL); + sqlite3_cloudsync_init(db, NULL, NULL); + + rc = sqlite3_exec(db, "CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_set_column('docs', 'body', 'delimiter', '\n');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Insert something so the blocks table is populated. + rc = sqlite3_exec(db, "INSERT INTO docs (id, body) VALUES ('doc1', 'AAA\nBBB\nCCC');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + int64_t settings_before = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='docs' AND col_name='body';"); + + sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + sqlite3_close(db); + db = NULL; + + // Phase 2: reopen — sqlite3_cloudsync_init triggers dbutils_settings_load, + // which iterates cloudsync_table_settings and replays the block-column setup. + // Without the fix this hangs (infinite REPLACE-during-iteration loop). + rc = sqlite3_open(dbpath, &db); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Settings table should be unchanged after reload (no duplicate rewrites). + int64_t settings_after = do_select_int(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='docs' AND col_name='body';"); + if (settings_after != settings_before) { + printf("block_column_reload: settings count changed across reload (before=%" PRId64 ", after=%" PRId64 ")\n", settings_before, settings_after); + goto cleanup; + } + + // Block column wiring must still be active after reload: an UPDATE should + // continue to write per-block metadata, not a single whole-column entry. + rc = sqlite3_exec(db, "UPDATE docs SET body='AAA\nXXX\nCCC' WHERE id='doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + int64_t whole_meta = do_select_int(db, "SELECT count(*) FROM docs_cloudsync WHERE col_name='body';"); + if (whole_meta != 0) { + printf("block_column_reload: expected 0 whole-column metadata entries after reload, got %" PRId64 "\n", whole_meta); + goto cleanup; + } + + result = true; + +cleanup: + if (db) { + sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + sqlite3_close(db); + } + if (cleanup_databases) { + file_delete_internal(dbpath); + char walpath[280]; + snprintf(walpath, sizeof(walpath), "%s-wal", dbpath); + file_delete_internal(walpath); + snprintf(walpath, sizeof(walpath), "%s-shm", dbpath); + file_delete_internal(walpath); + } + return result; +} + +static char *do_select_text(sqlite3 *db, const char *sql) { + sqlite3_stmt *stmt = NULL; + char *val = NULL; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + const char *t = (const char *)sqlite3_column_text(stmt, 0); + if (t) val = sqlite3_mprintf("%s", t); + } + } + if (stmt) sqlite3_finalize(stmt); + return val; +} + +// Test: enabling block-level LWW on a table that already contains rows migrates +// existing data into the blocks table so it is not silently ignored. +// Verifies: +// 1. Blocks table is populated for pre-existing rows after set_column('algo','block'). +// 2. A subsequent UPDATE correctly diffs against the migrated blocks, not from empty. +// 3. The migration is idempotent (calling set_column twice does not double the blocks). +bool do_test_block_lww_existing_data(bool cleanup_databases) { + sqlite3 *db = NULL; + bool result = false; + char dbpath[256]; + time_t timestamp = time(NULL); + + #ifdef __ANDROID__ + snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockexist-%ld.sqlite", ".", timestamp); + #else + snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockexist-%ld.sqlite", getenv("HOME"), timestamp); + #endif + + int rc = sqlite3_open(dbpath, &db); + if (rc != SQLITE_OK) return false; + sqlite3_exec(db, "PRAGMA journal_mode=WAL;", NULL, NULL, NULL); + sqlite3_cloudsync_init(db, NULL, NULL); + + // Create table and init sync BEFORE enabling block-level LWW + rc = sqlite3_exec(db, "CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_exec(db, "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Insert rows via the sync layer (populates regular column metadata) + rc = sqlite3_exec(db, "INSERT INTO docs (id, body) VALUES ('d1', 'line1\nline2\nline3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_exec(db, "INSERT INTO docs (id, body) VALUES ('d2', 'alpha\nbeta');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Sanity: no blocks table yet + int64_t blocks_table_before = do_select_int(db, + "SELECT count(*) FROM sqlite_master WHERE name='docs_cloudsync_blocks';"); + if (blocks_table_before != 0) { + printf("block_existing_data: blocks table should not exist before set_column\n"); + goto cleanup; + } + + // NOW enable block-level LWW — this should migrate the existing rows + rc = sqlite3_exec(db, "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_existing_data: set_column failed: %s\n", sqlite3_errmsg(db)); goto cleanup; } + + // Verify blocks table was created and is populated + int64_t blocks_table_after = do_select_int(db, + "SELECT count(*) FROM sqlite_master WHERE name='docs_cloudsync_blocks';"); + if (blocks_table_after != 1) { + printf("block_existing_data: blocks table not created after set_column\n"); + goto cleanup; + } + + // d1 has 3 lines → 3 block entries; d2 has 2 lines → 2 block entries = 5 total + int64_t block_count = do_select_int(db, "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (block_count != 5) { + printf("block_existing_data: expected 5 block entries after migration, got %" PRId64 "\n", block_count); + goto cleanup; + } + + // Metadata must contain the migrated block entries (alive, odd col_version) + int64_t meta_block_count = do_select_int(db, + "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body\x1f%' AND col_version % 2 = 1;"); + if (meta_block_count != 5) { + printf("block_existing_data: expected 5 block metadata entries, got %" PRId64 "\n", meta_block_count); + goto cleanup; + } + + // UPDATE d1: change one line — diff should reuse existing block positions + rc = sqlite3_exec(db, "UPDATE docs SET body='line1\nMODIFIED\nline3' WHERE id='d1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_existing_data: UPDATE failed: %s\n", sqlite3_errmsg(db)); goto cleanup; } + + // After the update: total alive blocks is still 5 (d1: 3, d2: 2) + int64_t alive_after_update = do_select_int(db, + "SELECT count(*) FROM docs_cloudsync_blocks b " + "JOIN docs_cloudsync m ON b.pk = m.pk AND b.col_name = m.col_name " + "WHERE b.col_name LIKE 'body\x1f%' AND m.col_version % 2 = 1;"); + if (alive_after_update != 5) { + printf("block_existing_data: expected 5 alive blocks after update, got %" PRId64 "\n", alive_after_update); + goto cleanup; + } + + // Verify materialized value is correct after update + char *val = do_select_text(db, "SELECT body FROM docs WHERE id='d1';"); + if (!val || strcmp(val, "line1\nMODIFIED\nline3") != 0) { + printf("block_existing_data: unexpected body after update: '%s'\n", val ? val : "(null)"); + if (val) sqlite3_free(val); + goto cleanup; } - - // verify consistency - only committed data should be present - for (int i=1; i 0) { - result = false; - printf("do_test_merge_rollback_scenarios error: db %d has %d unterminated statements\n", i, counter); - } - } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); - } + +cleanup: + if (db) { + sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + sqlite3_close(db); + } + if (cleanup_databases) { + file_delete_internal(dbpath); + char walpath[280]; + snprintf(walpath, sizeof(walpath), "%s-wal", dbpath); + file_delete_internal(walpath); + snprintf(walpath, sizeof(walpath), "%s-shm", dbpath); + file_delete_internal(walpath); } return result; } -// Test circular merge (A->B->C->A) -bool do_test_merge_circular (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients < 3) { - nclients = 3; - printf("Number of test merge increased to %d clients\n", 3); - } - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } - +bool do_test_block_lww_insert(int nclients, bool print_result, bool cleanup_databases) { + // Test: INSERT into a table with a block column properly splits text into blocks + sqlite3 *db[2] = {NULL, NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i1->2->0 - if (do_merge_using_payload(db[0], db[1], false, true) == false) { - goto finalize; - } - - if (do_merge_using_payload(db[1], db[2], false, true) == false) { - goto finalize; + if (!db[i]) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_insert: CREATE TABLE failed: %s\n", sqlite3_errmsg(db[i])); goto fail; } + + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_insert: cloudsync_init failed: %s\n", sqlite3_errmsg(db[i])); goto fail; } + + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_insert: set_column failed: %s\n", sqlite3_errmsg(db[i])); goto fail; } } - - if (do_merge_using_payload(db[2], db[0], false, true) == false) { - goto finalize; + + // Insert a document with 3 lines + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line 1\nLine 2\nLine 3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_insert: INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Verify blocks were created in the blocks table + int64_t block_count = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (block_count != 3) { + printf("block_insert: expected 3 blocks, got %" PRId64 "\n", block_count); + goto fail; } - - // complete the circle and ensure consistency - if (do_merge(db, nclients, false) == false) { - goto finalize; + + // Verify metadata entries for blocks (col_name contains \x1F) + int64_t meta_count = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + if (meta_count != 3) { + printf("block_insert: expected 3 block metadata entries, got %" PRId64 "\n", meta_count); + goto fail; } - - // verify all clients have all data - for (int i=1; i Circular merge result\n"); - do_query(db[0], "SELECT * FROM circular_test ORDER BY origin_client;", query_table); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +bool do_test_block_lww_update(int nclients, bool print_result, bool cleanup_databases) { + // Test: UPDATE on a block column performs block diff + sqlite3 *db[1] = {NULL}; + time_t timestamp = time(NULL); + int rc; + + db[0] = do_create_database_file(0, timestamp, test_counter++); + if (!db[0]) return false; + + rc = sqlite3_exec(db[0], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Insert initial text + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'AAA\nBBB\nCCC');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_update: INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + int64_t blocks_before = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + + // Update: change middle line and add a new line + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'AAA\nXXX\nCCC\nDDD' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_update: UPDATE failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + int64_t blocks_after = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + + // Should have 4 blocks after update (AAA, XXX, CCC, DDD) + if (blocks_after != 4) { + printf("block_update: expected 4 blocks after update, got %" PRId64 " (before: %" PRId64 ")\n", blocks_after, blocks_before); + goto fail; } - - result = true; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_circular error: db %d has %d unterminated statements\n", i, counter); - } + + close_db(db[0]); + return true; + +fail: + if (db[0]) close_db(db[0]); + return false; +} + +bool do_test_block_lww_sync(int nclients, bool print_result, bool cleanup_databases) { + // Test: Two sites edit different blocks of the same document; after sync, both edits are preserved + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Site 0 inserts the initial document + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line A\nLine B\nLine C');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_sync: INSERT db[0] failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Sync initial state: db[0] -> db[1] so both have the same document + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("block_sync: initial merge 0->1 failed\n"); goto fail; } + + // Site 0: edit first line + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'EDITED A\nLine B\nLine C' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_sync: UPDATE db[0] failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Site 1: edit third line + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Line A\nLine B\nEDITED C' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_sync: UPDATE db[1] failed: %s\n", sqlite3_errmsg(db[1])); goto fail; } + + // Sync: db[0] -> db[1] (send site 0's edits) + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("block_sync: merge 0->1 failed\n"); goto fail; } + // Sync: db[1] -> db[0] (send site 1's edits) + if (!do_merge_using_payload(db[1], db[0], true, true)) { printf("block_sync: merge 1->0 failed\n"); goto fail; } + + // Both databases should now have the merged result: "EDITED A\nLine B\nEDITED C" + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1) { + printf("block_sync: could not read body from one or both databases\n"); + ok = false; + } else if (strcmp(body0, body1) != 0) { + printf("block_sync: bodies don't match after sync:\n db[0]: %s\n db[1]: %s\n", body0, body1); + ok = false; + } else { + // Check that both edits were preserved + if (!strstr(body0, "EDITED A")) { + printf("block_sync: missing 'EDITED A' in result: %s\n", body0); + ok = false; } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); + if (!strstr(body0, "EDITED C")) { + printf("block_sync: missing 'EDITED C' in result: %s\n", body0); + ok = false; + } + if (!strstr(body0, "Line B")) { + printf("block_sync: missing 'Line B' in result: %s\n", body0); + ok = false; } } - return result; + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; } -// Test foreign key constraints during merge -bool do_test_merge_foreign_keys (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); +bool do_test_block_lww_delete(int nclients, bool print_result, bool cleanup_databases) { + // Test: DELETE on a row with block columns marks tombstone and block metadata is dropped + sqlite3 *db[1] = {NULL}; + time_t timestamp = time(NULL); + int rc; + + db[0] = do_create_database_file(0, timestamp, test_counter++); + if (!db[0]) return false; + + rc = sqlite3_exec(db[0], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Insert a document + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line A\nLine B\nLine C');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_delete: INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Verify blocks and metadata exist + int64_t blocks_before = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (blocks_before != 3) { + printf("block_delete: expected 3 blocks before delete, got %" PRId64 "\n", blocks_before); + goto fail; } - + int64_t meta_before = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + if (meta_before != 3) { + printf("block_delete: expected 3 block metadata before delete, got %" PRId64 "\n", meta_before); + goto fail; + } + + // Delete the row + rc = sqlite3_exec(db[0], "DELETE FROM docs WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_delete: DELETE failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Verify metadata tombstone exists (delete sentinel) + int64_t tombstone = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name = '__[RIP]__' AND col_version % 2 = 0;"); + if (tombstone != 1) { + printf("block_delete: expected 1 delete tombstone, got %" PRId64 "\n", tombstone); + goto fail; + } + + // Verify block metadata was dropped (local_drop_meta removes non-tombstone metadata) + int64_t meta_after = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + if (meta_after != 0) { + printf("block_delete: expected 0 block metadata after delete, got %" PRId64 "\n", meta_after); + goto fail; + } + + // Row should be gone from base table + int64_t row_count = do_select_int(db[0], "SELECT count(*) FROM docs WHERE id = 'doc1';"); + if (row_count != 0) { + printf("block_delete: row still in base table after delete\n"); + goto fail; + } + + close_db(db[0]); + return true; + +fail: + if (db[0]) close_db(db[0]); + return false; +} + +bool do_test_block_lww_materialize(int nclients, bool print_result, bool cleanup_databases) { + // Test: cloudsync_text_materialize reconstructs text from blocks after sync + // Sync to a second db where body column is empty, then materialize there + sqlite3 *db[2] = {NULL, NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 0) { - result = false; - printf("do_test_merge_foreign_keys error: db %d has %d unterminated statements\n", i, counter); - } + + // Update from empty to multi-line + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Line1\nLine2' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_empty: UPDATE from empty failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + int64_t updated_blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (updated_blocks != 2) { + printf("block_empty: expected 2 blocks after update from empty, got %" PRId64 "\n", updated_blocks); + goto fail; + } + + close_db(db[0]); + return true; + +fail: + if (db[0]) close_db(db[0]); + return false; +} + +bool do_test_block_lww_conflict(int nclients, bool print_result, bool cleanup_databases) { + // Test: Two sites edit the SAME line concurrently; LWW picks the later write + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Site 0 inserts initial document + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Same\nMiddle\nEnd');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_conflict: INSERT db[0] failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Sync initial state: db[0] -> db[1] + if (!do_merge_values(db[0], db[1], false)) { printf("block_conflict: initial merge failed\n"); goto fail; } + + // Site 0: edit first line + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Site0\nMiddle\nEnd' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_conflict: UPDATE db[0] failed\n"); goto fail; } + + // Site 1: also edit first line (conflict!) + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Site1\nMiddle\nEnd' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_conflict: UPDATE db[1] failed\n"); goto fail; } + + // Sync both ways using row-by-row merge + if (!do_merge_values(db[0], db[1], true)) { printf("block_conflict: merge 0->1 failed\n"); goto fail; } + if (!do_merge_values(db[1], db[0], true)) { printf("block_conflict: merge 1->0 failed\n"); goto fail; } + + // Materialize on both databases to reconstruct body from blocks + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_conflict: materialize db[0] failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_conflict: materialize db[1] failed: %s\n", sqlite3_errmsg(db[1])); goto fail; } + + // Both databases should converge (same value) + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1) { + printf("block_conflict: could not read body from databases\n"); + ok = false; + } else if (strcmp(body0, body1) != 0) { + printf("block_conflict: bodies don't match after sync:\n db[0]: %s\n db[1]: %s\n", body0, body1); + ok = false; + } else { + // Should contain either "Site0" or "Site1" (LWW picks one), plus unchanged lines + if (!strstr(body0, "Middle")) { + printf("block_conflict: missing 'Middle' in result: %s\n", body0); + ok = false; } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); + if (!strstr(body0, "End")) { + printf("block_conflict: missing 'End' in result: %s\n", body0); + ok = false; + } + // One of the conflicting edits should win + if (!strstr(body0, "Site0") && !strstr(body0, "Site1")) { + printf("block_conflict: neither 'Site0' nor 'Site1' in result: %s\n", body0); + ok = false; } } - return result; + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; } -// Test trigger interaction during merge -// Expected failure: TRIGGERs are not fully supported by this extension. -bool do_test_merge_triggers (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); +bool do_test_block_lww_multi_update(int nclients, bool print_result, bool cleanup_databases) { + // Test: Multiple successive updates correctly maintain block state + sqlite3 *db[1] = {NULL}; + time_t timestamp = time(NULL); + int rc; + + db[0] = do_create_database_file(0, timestamp, test_counter++); + if (!db[0]) return false; + + rc = sqlite3_exec(db[0], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Insert initial text (3 lines) + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'A\nB\nC');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_multi: INSERT failed\n"); goto fail; } + + // Update 1: remove middle line (3 -> 2 blocks) + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'A\nC' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_multi: UPDATE 1 failed\n"); goto fail; } + + int64_t blocks1 = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (blocks1 != 2) { printf("block_multi: expected 2 blocks after update 1, got %" PRId64 "\n", blocks1); goto fail; } + + // Update 2: add two lines (2 -> 4 blocks) + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'A\nX\nC\nY' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_multi: UPDATE 2 failed\n"); goto fail; } + + int64_t blocks2 = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (blocks2 != 4) { printf("block_multi: expected 4 blocks after update 2, got %" PRId64 "\n", blocks2); goto fail; } + + // Update 3: change everything to a single line (4 -> 1 block) + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'SINGLE' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_multi: UPDATE 3 failed\n"); goto fail; } + + int64_t blocks3 = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (blocks3 != 1) { printf("block_multi: expected 1 block after update 3, got %" PRId64 "\n", blocks3); goto fail; } + + // Materialize and verify + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_multi: materialize failed\n"); goto fail; } + + char *body = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, "SINGLE") != 0) { + printf("block_multi: expected 'SINGLE', got '%s'\n", body ? body : "NULL"); + if (body) sqlite3_free(body); + goto fail; } - + sqlite3_free(body); + + close_db(db[0]); + return true; + +fail: + if (db[0]) close_db(db[0]); + return false; +} + +bool do_test_block_lww_reinsert(int nclients, bool print_result, bool cleanup_databases) { + // Test: DELETE then re-INSERT recreates blocks properly + sqlite3 *db[2] = {NULL, NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i Trigger test results\n"); - do_query(db[0], "SELECT id, data, update_count FROM trigger_test ORDER BY id;", query_table); - printf("\n-> Audit log\n"); - do_query(db[0], "SELECT COUNT(*) as audit_entries FROM audit_log;", query_table); + + // Sync to db[1] and verify + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("block_reinsert: merge failed\n"); goto fail; } + + // Materialize on db[1] + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_reinsert: materialize on db[1] failed: %s\n", sqlite3_errmsg(db[1])); goto fail; } + + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body1 || strcmp(body1, "New1\nNew2\nNew3") != 0) { + printf("block_reinsert: body mismatch on db[1]: %s\n", body1 ? body1 : "NULL"); + if (body1) sqlite3_free(body1); + goto fail; } - - result = true; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_triggers error: db %d has %d unterminated statements\n", i, counter); - } - } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); + sqlite3_free(body1); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +bool do_test_block_lww_add_lines(int nclients, bool print_result, bool cleanup_databases) { + // Test: Both sites add lines at different positions; after sync, all lines are present + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Site 0 inserts initial doc + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line1\nLine2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync initial: 0 -> 1 + if (!do_merge_using_payload(db[0], db[1], false, true)) goto fail; + + // Site 0: append a line at the end + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Line1\nLine2\nAppended0' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: insert a line in the middle + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Line1\nInserted1\nLine2' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_using_payload(db[0], db[1], true, true)) goto fail; + if (!do_merge_using_payload(db[1], db[0], true, true)) goto fail; + + // Both should converge + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1) { + printf("block_add_lines: could not read body\n"); + ok = false; + } else if (strcmp(body0, body1) != 0) { + printf("block_add_lines: bodies don't match:\n db[0]: %s\n db[1]: %s\n", body0, body1); + ok = false; + } else { + // All original and added lines should be present + if (!strstr(body0, "Line1")) { printf("block_add_lines: missing Line1\n"); ok = false; } + if (!strstr(body0, "Line2")) { printf("block_add_lines: missing Line2\n"); ok = false; } + if (!strstr(body0, "Appended0")) { printf("block_add_lines: missing Appended0\n"); ok = false; } + if (!strstr(body0, "Inserted1")) { printf("block_add_lines: missing Inserted1\n"); ok = false; } + } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 1: Non-conflicting edits on different blocks — both edits preserved +bool do_test_block_lww_noconflict(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Site 0 inserts initial document with 3 lines + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line1\nLine2\nLine3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync initial: 0 -> 1 + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: edit first line only + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'EditedByA\nLine2\nLine3' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: edit third line only (no conflict — different block) + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Line1\nLine2\nEditedByB' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + // Materialize on both + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("noconflict: bodies diverged: [%s] vs [%s]\n", body0 ? body0 : "NULL", body1 ? body1 : "NULL"); + ok = false; + } else { + // BOTH edits should be preserved (this is the key value of block-level LWW) + if (!strstr(body0, "EditedByA")) { printf("noconflict: missing EditedByA\n"); ok = false; } + if (!strstr(body0, "Line2")) { printf("noconflict: missing Line2\n"); ok = false; } + if (!strstr(body0, "EditedByB")) { printf("noconflict: missing EditedByB\n"); ok = false; } + } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 2: Concurrent add + edit — Site A adds a line, Site B modifies an existing line +bool do_test_block_lww_add_and_edit(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Initial doc + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Alpha\nBravo');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: add a new line at the end + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Alpha\nBravo\nCharlie' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: modify first line + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'AlphaEdited\nBravo' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("add_and_edit: bodies diverged: [%s] vs [%s]\n", body0 ? body0 : "NULL", body1 ? body1 : "NULL"); + ok = false; + } else { + // The added line and the edit should both be present + if (!strstr(body0, "Charlie")) { printf("add_and_edit: missing Charlie (added line)\n"); ok = false; } + if (!strstr(body0, "Bravo")) { printf("add_and_edit: missing Bravo\n"); ok = false; } + // First line: either AlphaEdited wins (from site 1) or Alpha (from site 0) — depends on LWW + // But the added line Charlie must survive regardless + } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 3: Three-way sync — 3 databases with overlapping edits converge +bool do_test_block_lww_three_way(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[3] = {NULL, NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 3; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Site 0 creates initial doc + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'L1\nL2\nL3\nL4');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync 0 -> 1, 0 -> 2 + if (!do_merge_values(db[0], db[1], false)) goto fail; + if (!do_merge_values(db[0], db[2], false)) goto fail; + + // Site 0: edit line 1 + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'S0\nL2\nL3\nL4' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: edit line 2 + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'L1\nS1\nL3\nL4' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 2: edit line 4 + rc = sqlite3_exec(db[2], "UPDATE docs SET body = 'L1\nL2\nL3\nS2' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Full mesh sync: each site sends to every other site + for (int src = 0; src < 3; src++) { + for (int dst = 0; dst < 3; dst++) { + if (src == dst) continue; + if (!do_merge_values(db[src], db[dst], true)) { printf("three_way: merge %d->%d failed\n", src, dst); goto fail; } } } - return result; + + // Materialize all + for (int i = 0; i < 3; i++) { + rc = sqlite3_exec(db[i], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("three_way: materialize db[%d] failed\n", i); goto fail; } + } + + // All three should converge + char *body[3]; + for (int i = 0; i < 3; i++) { + body[i] = do_select_text(db[i], "SELECT body FROM docs WHERE id = 'doc1';"); + } + + bool ok = true; + if (!body[0] || !body[1] || !body[2]) { printf("three_way: NULL body\n"); ok = false; } + else if (strcmp(body[0], body[1]) != 0 || strcmp(body[1], body[2]) != 0) { + printf("three_way: not converged:\n [0]: %s\n [1]: %s\n [2]: %s\n", body[0], body[1], body[2]); + ok = false; + } else { + // All three non-conflicting edits should be preserved + if (!strstr(body[0], "S0")) { printf("three_way: missing S0\n"); ok = false; } + if (!strstr(body[0], "S1")) { printf("three_way: missing S1\n"); ok = false; } + if (!strstr(body[0], "L3")) { printf("three_way: missing L3\n"); ok = false; } + if (!strstr(body[0], "S2")) { printf("three_way: missing S2\n"); ok = false; } + } + + for (int i = 0; i < 3; i++) { if (body[i]) sqlite3_free(body[i]); } + for (int i = 0; i < 3; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 3; i++) if (db[i]) close_db(db[i]); + return false; } -// Test index consistency during merge -bool do_test_merge_index_consistency (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - +// Test 4: Mixed block + normal columns — both work independently +bool do_test_block_lww_mixed_columns(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 1 + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: edit block column (body line 1) AND normal column (title) + rc = sqlite3_exec(db[0], "UPDATE notes SET body = 'EditedLine1\nLine2\nLine3', title = 'Title From A' WHERE id = 'n1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: edit a different block (body line 3) AND normal column (title — will conflict via LWW) + rc = sqlite3_exec(db[1], "UPDATE notes SET body = 'Line1\nLine2\nEditedLine3', title = 'Title From B' WHERE id = 'n1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + // Materialize block column + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('notes', 'body', 'n1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('notes', 'body', 'n1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM notes WHERE id = 'n1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM notes WHERE id = 'n1';"); + char *title0 = do_select_text(db[0], "SELECT title FROM notes WHERE id = 'n1';"); + char *title1 = do_select_text(db[1], "SELECT title FROM notes WHERE id = 'n1';"); + + bool ok = true; + + // Bodies should converge + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("mixed_columns: body diverged\n"); + ok = false; + } else { + // Both non-conflicting block edits should be preserved + if (!strstr(body0, "EditedLine1")) { printf("mixed_columns: missing EditedLine1\n"); ok = false; } + if (!strstr(body0, "Line2")) { printf("mixed_columns: missing Line2\n"); ok = false; } + if (!strstr(body0, "EditedLine3")) { printf("mixed_columns: missing EditedLine3\n"); ok = false; } } - - // verify data consistency - for (int i=1; i 0) { - result = false; - printf("do_test_merge_index_consistency error: db %d has %d unterminated statements\n", i, counter); - } - } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); - } + + // Insert with NULL body on site 0 + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("null_to_text: INSERT NULL failed\n"); goto fail; } + + // Sync to site 1 + if (!do_merge_values(db[0], db[1], false)) { printf("null_to_text: initial sync failed\n"); goto fail; } + + // Update to multi-line text on site 0 + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Hello\nWorld\nFoo' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("null_to_text: UPDATE failed\n"); goto fail; } + + // Verify blocks created + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 3) { printf("null_to_text: expected 3 blocks, got %" PRId64 "\n", blocks); goto fail; } + + // Sync update to site 1 + if (!do_merge_values(db[0], db[1], true)) { printf("null_to_text: sync update failed\n"); goto fail; } + + // Materialize on site 1 + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("null_to_text: materialize failed\n"); goto fail; } + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, "Hello\nWorld\nFoo") != 0) { + printf("null_to_text: expected 'Hello\\nWorld\\nFoo', got '%s'\n", body ? body : "NULL"); + if (body) sqlite3_free(body); + goto fail; } - return result; + sqlite3_free(body); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; } -// Test JSON column merge -bool do_test_merge_json_columns (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); +// Test 6: Interleaved inserts — multiple rounds of inserting between existing lines +bool do_test_block_lww_interleaved(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Start with 2 lines + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'A\nB');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Round 1: Site 0 inserts between A and B + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'A\nC\nB' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[0], db[1], true)) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Round 2: Site 1 inserts between A and C + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'A\nD\nC\nB' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Round 3: Site 0 inserts between D and C + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'A\nD\nE\nC\nB' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[0], db[1], true)) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Verify final state on both sites + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("interleaved: diverged: [%s] vs [%s]\n", body0 ? body0 : "NULL", body1 ? body1 : "NULL"); + ok = false; + } else { + // All 5 lines should be present + if (!strstr(body0, "A")) { printf("interleaved: missing A\n"); ok = false; } + if (!strstr(body0, "D")) { printf("interleaved: missing D\n"); ok = false; } + if (!strstr(body0, "E")) { printf("interleaved: missing E\n"); ok = false; } + if (!strstr(body0, "C")) { printf("interleaved: missing C\n"); ok = false; } + if (!strstr(body0, "B")) { printf("interleaved: missing B\n"); ok = false; } + + // Verify 5 blocks + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 5) { printf("interleaved: expected 5 blocks, got %" PRId64 "\n", blocks); ok = false; } } - + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 7: Custom delimiter — paragraph separator instead of newline +bool do_test_block_lww_custom_delimiter(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 0) large_text[offset++] = '\n'; + offset += snprintf(large_text + offset, sizeof(large_text) - offset, "Line %03d content", i); } - - // verify JSON data consistency - for (int i=1; i 0) rapid_text[roff++] = '\n'; + roff += snprintf(rapid_text + roff, sizeof(rapid_text) - roff, "Update%d", i); + + sqlite3_bind_text(upd, 1, rapid_text, roff, SQLITE_STATIC); + rc = sqlite3_step(upd); + if (rc != SQLITE_DONE) { printf("rapid: UPDATE %d failed\n", i); sqlite3_finalize(upd); goto fail; } + sqlite3_reset(upd); + } + sqlite3_finalize(upd); + + // Verify final block count matches line count + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != RAPID_ROUNDS) { + printf("rapid: expected %d blocks, got %" PRId64 "\n", RAPID_ROUNDS, blocks); + goto fail; + } + + // Sync and verify roundtrip + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("rapid: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("rapid: materialize failed\n"); goto fail; } + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("rapid: roundtrip mismatch\n"); + ok = false; + } else { + // Check first and last lines + if (!strstr(body0, "Update0")) { printf("rapid: missing Update0\n"); ok = false; } + if (!strstr(body0, "Update49")) { printf("rapid: missing Update49\n"); ok = false; } } - - if (print_result) { - printf("\n-> JSON test results\n"); - do_query(db[0], "SELECT id, json_extract(metadata, '$.name') as name, json_extract(metadata, '$.preferences.theme') as theme FROM json_test ORDER BY id;", query_table); + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Unicode/multibyte content in blocks (emoji, CJK, accented chars) +bool do_test_block_lww_unicode(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; } - - result = true; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_json_columns error: db %d has %d unterminated statements\n", i, counter); - } - } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); - } + + // Insert multi-line text with unicode content + const char *unicode_text = "Hello \xC3\xA9\xC3\xA0\xC3\xBC" "\n" // accented: éàü + "\xE4\xB8\xAD\xE6\x96\x87\xE6\xB5\x8B\xE8\xAF\x95" "\n" // CJK: 中文测试 + "\xF0\x9F\x98\x80\xF0\x9F\x8E\x89\xF0\x9F\x9A\x80"; // emoji: 😀🎉🚀 + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto fail; + sqlite3_bind_text(stmt, 1, unicode_text, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) goto fail; + + // Should have 3 blocks + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 3) { printf("unicode: expected 3 blocks, got %" PRId64 "\n", blocks); goto fail; } + + // Sync and materialize + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("unicode: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("unicode: materialize failed\n"); goto fail; } + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, unicode_text) != 0) { + printf("unicode: roundtrip mismatch\n"); + if (body) sqlite3_free(body); + goto fail; } - return result; + + // Update: edit the emoji line + const char *updated_text = "Hello \xC3\xA9\xC3\xA0\xC3\xBC" "\n" + "\xE4\xB8\xAD\xE6\x96\x87\xE6\xB5\x8B\xE8\xAF\x95" "\n" + "\xF0\x9F\x92\xAF\xF0\x9F\x94\xA5"; // changed emoji: 💯🔥 + sqlite3_free(body); + + stmt = NULL; + rc = sqlite3_prepare_v2(db[0], "UPDATE docs SET body = ? WHERE id = 'doc1';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto fail; + sqlite3_bind_text(stmt, 1, updated_text, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) goto fail; + + // Sync update + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("unicode: sync update failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, updated_text) != 0) { + printf("unicode: update roundtrip mismatch\n"); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; } -// Test concurrent merge attempts -bool do_test_merge_concurrent_attempts (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 3) { - nclients = 3; - printf("Number of test merge increased to %d clients\n", 3); +// Test: Special characters (tabs, carriage returns, etc.) in blocks +bool do_test_block_lww_special_chars(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; } - + + // Text with tabs, carriage returns, and other special chars within lines + const char *special_text = "col1\tcol2\tcol3\n" // tabs within line + "line with\r\nembedded\n" // \r before \n delimiter + "back\\slash \"quotes\""; // backslash and quotes + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto fail; + sqlite3_bind_text(stmt, 1, special_text, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) goto fail; + + // Should split on \n: "col1\tcol2\tcol3", "line with\r", "embedded", "back\\slash \"quotes\"" + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 4) { printf("special: expected 4 blocks, got %" PRId64 "\n", blocks); goto fail; } + + // Sync and verify roundtrip + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("special: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, special_text) != 0) { + printf("special: roundtrip mismatch\n"); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Concurrent delete vs edit on different blocks +// Site A deletes the row, Site B edits a line. After sync, delete wins. +bool do_test_block_lww_delete_vs_edit(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 0 && rows1 > 0) { + sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (body0 && body1 && strcmp(body0, body1) != 0) { + printf("delete_vs_edit: bodies diverged\n"); + ok = false; } + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); } - - // simulate concurrent merge attempts by merging in different orders - // first: 0->1 and 1->2 simultaneously - bool merge1 = do_merge_using_payload(db[0], db[1], false, true); - bool merge2 = do_merge_using_payload(db[1], db[2], false, true); - - if (!merge1 || !merge2) { - printf("Concurrent merge phase 1 had issues\n"); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Two block columns on same table +bool do_test_block_lww_two_block_cols(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT, notes TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'notes', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert with both block columns + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body, notes) VALUES ('doc1', 'B1\nB2\nB3', 'N1\nN2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("two_block_cols: INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Verify blocks created for both columns + int64_t body_blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + int64_t notes_blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'notes' || x'1f' || '%';"); + if (body_blocks != 3) { printf("two_block_cols: expected 3 body blocks, got %" PRId64 "\n", body_blocks); goto fail; } + if (notes_blocks != 2) { printf("two_block_cols: expected 2 notes blocks, got %" PRId64 "\n", notes_blocks); goto fail; } + + // Sync to site 1 + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: edit body line 1 + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'B1_edited\nB2\nB3' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: edit notes line 2 + rc = sqlite3_exec(db[1], "UPDATE docs SET notes = 'N1\nN2_edited' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + // Materialize both columns on both sites + for (int i = 0; i < 2; i++) { + rc = sqlite3_exec(db[i], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("two_block_cols: materialize body db[%d] failed\n", i); goto fail; } + rc = sqlite3_exec(db[i], "SELECT cloudsync_text_materialize('docs', 'notes', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("two_block_cols: materialize notes db[%d] failed\n", i); goto fail; } + } + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + char *notes0 = do_select_text(db[0], "SELECT notes FROM docs WHERE id = 'doc1';"); + char *notes1 = do_select_text(db[1], "SELECT notes FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("two_block_cols: body diverged\n"); ok = false; + } else if (!strstr(body0, "B1_edited")) { + printf("two_block_cols: body edit missing\n"); ok = false; + } + + if (!notes0 || !notes1 || strcmp(notes0, notes1) != 0) { + printf("two_block_cols: notes diverged\n"); ok = false; + } else if (!strstr(notes0, "N2_edited")) { + printf("two_block_cols: notes edit missing\n"); ok = false; + } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + if (notes0) sqlite3_free(notes0); + if (notes1) sqlite3_free(notes1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Update text to NULL (text->NULL transition) +bool do_test_block_lww_text_to_null(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; } - - // second: 2->0 and 0->1 simultaneously - bool merge3 = do_merge_using_payload(db[2], db[0], false, true); - bool merge4 = do_merge_using_payload(db[0], db[1], false, true); - - if (!merge3 || !merge4) { - printf("Concurrent merge phase 2 had issues\n"); + + // Insert multi-line text + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line1\nLine2\nLine3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + int64_t blocks_before = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks_before != 3) { printf("text_to_null: expected 3 blocks before, got %" PRId64 "\n", blocks_before); goto fail; } + + // Update to NULL + rc = sqlite3_exec(db[0], "UPDATE docs SET body = NULL WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("text_to_null: UPDATE to NULL failed\n"); goto fail; } + + // Verify body is NULL + int64_t is_null = do_select_int(db[0], "SELECT body IS NULL FROM docs WHERE id = 'doc1';"); + if (is_null != 1) { printf("text_to_null: body not NULL after update\n"); goto fail; } + + // Sync and verify + if (!do_merge_values(db[0], db[1], false)) { printf("text_to_null: sync failed\n"); goto fail; } + + // Materialize on site 1 + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + int64_t is_null_b = do_select_int(db[1], "SELECT body IS NULL FROM docs WHERE id = 'doc1';"); + if (is_null_b != 1) { printf("text_to_null: body not NULL on site 1 after sync\n"); goto fail; } + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Payload-based sync for block columns (vs row-by-row do_merge_values) +bool do_test_block_lww_payload_sync(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert and first sync via payload + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Alpha\nBravo\nCharlie');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("payload_sync: initial merge failed\n"); goto fail; } + + // Edit on both sites + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Alpha_A\nBravo\nCharlie' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Alpha\nBravo\nCharlie_B' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync via payload both ways + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("payload_sync: merge 0->1 failed\n"); goto fail; } + if (!do_merge_using_payload(db[1], db[0], true, true)) { printf("payload_sync: merge 1->0 failed\n"); goto fail; } + + // Materialize + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("payload_sync: bodies diverged\n"); ok = false; + } else { + if (!strstr(body0, "Alpha_A")) { printf("payload_sync: missing Alpha_A\n"); ok = false; } + if (!strstr(body0, "Bravo")) { printf("payload_sync: missing Bravo\n"); ok = false; } + if (!strstr(body0, "Charlie_B")) { printf("payload_sync: missing Charlie_B\n"); ok = false; } } - - // final consistency merge - if (do_merge(db, nclients, false) == false) { - printf("Final merge failed\n"); + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Idempotent apply — applying the same payload twice is a no-op +bool do_test_block_lww_idempotent(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; } - - // verify all clients have same data count - int expected_count = nclients * 5; // each client inserted 5 records - for (int i=0; i A B X C E + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'A\nB\nX\nC\nE' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: insert Y between D and E -> A B C D Y E + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'A\nB\nC\nD\nY\nE' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("ordering: bodies diverged: [%s] vs [%s]\n", body0 ? body0 : "NULL", body1 ? body1 : "NULL"); + ok = false; + } else { + // Verify ordering: A must come before B, B before C, etc. + // All lines that survived should maintain relative order + const char *pA = strstr(body0, "A"); + const char *pB = strstr(body0, "B"); + const char *pC = strstr(body0, "C"); + const char *pE = strstr(body0, "E"); + + if (!pA || !pB || !pC || !pE) { + printf("ordering: missing original lines\n"); ok = false; + } else { + if (pA >= pB) { printf("ordering: A not before B\n"); ok = false; } + if (pB >= pC) { printf("ordering: B not before C\n"); ok = false; } + if (pC >= pE) { printf("ordering: C not before E\n"); ok = false; } } - } - - // test passes if we reach here without database corruption - result = true; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_concurrent_attempts error: db %d has %d unterminated statements\n", i, counter); - } + + // X (inserted between B and C) should appear between B and C + const char *pX = strstr(body0, "X"); + if (pX) { + if (pX <= pB || pX >= pC) { printf("ordering: X not between B and C\n"); ok = false; } } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); + + // Y should appear somewhere after C + const char *pY = strstr(body0, "Y"); + if (pY) { + if (pY <= pC) { printf("ordering: Y not after C\n"); ok = false; } } } - return result; + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Composite primary key (text + int) with block column +bool do_test_block_lww_composite_pk(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (owner TEXT NOT NULL, seq INTEGER NOT NULL, body TEXT, PRIMARY KEY(owner, seq));", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert on site 0 + rc = sqlite3_exec(db[0], "INSERT INTO docs (owner, seq, body) VALUES ('alice', 1, 'Line1\nLine2\nLine3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("composite_pk: INSERT failed\n"); goto fail; } + + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('alice', 1);"); + if (blocks != 3) { printf("composite_pk: expected 3 blocks, got %" PRId64 "\n", blocks); goto fail; } + + // Sync to site 1 + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("composite_pk: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'alice', 1);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("composite_pk: materialize failed: %s\n", sqlite3_errmsg(db[1])); goto fail; } + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE owner = 'alice' AND seq = 1;"); + if (!body || strcmp(body, "Line1\nLine2\nLine3") != 0) { + printf("composite_pk: body mismatch: [%s]\n", body ? body : "NULL"); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + // Edit on site 1, sync back + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Line1\nEdited2\nLine3' WHERE owner = 'alice' AND seq = 1;", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_using_payload(db[1], db[0], true, true)) { printf("composite_pk: reverse sync failed\n"); goto fail; } + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'alice', 1);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE owner = 'alice' AND seq = 1;"); + if (!body0 || strcmp(body0, "Line1\nEdited2\nLine3") != 0) { + printf("composite_pk: reverse body mismatch: [%s]\n", body0 ? body0 : "NULL"); + if (body0) sqlite3_free(body0); + goto fail; + } + sqlite3_free(body0); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; } -// Test with 10 clients using a simple table with composite primary key -// Summary of the test function: -// 1. Simple table with composite primary key: Creates a table with columns (id1, id2) as the composite primary key and (data1, data2) as non-primary key columns -// 2. 10 database simulation: Forces exactly 10 clients regardless of the parameter passed -// 3. CloudSync initialization: Each database initializes the cloudsync library for the test table -// 4. Initial data: Each client inserts 3 initial rows with unique data -// 5. Multiple operation rounds: Performs 5 rounds of operations where each client performs: -// - Inserts new rows (20%) -// - Updates local data (15%) - data inserted by the same client -// - Updates remote data (50%) - data inserted by other clients, tests cross-client conflicts -// - Deletes both local and remote data (15%) - minimal deletions to preserve data for updates -// 6. Partial merging: After each round, clients merge with only 2-3 other clients (not all), simulating real-world scenarios where not all clients sync immediately -// 7. Final convergence: -// - Merges all changes from all clients to the first client (client 0) -// - Propagates the final state from client 0 to all other clients -// 8. Verification: Compares all databases to ensure they have converged to identical final states -bool do_test_merge_composite_pk_10_clients (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - const char *table_name = "simple_composite_test"; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); +// Test: Empty string body (not NULL) — should produce 1 block with empty content +bool do_test_block_lww_empty_vs_null(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert empty string (NOT NULL) + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', '');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 1) { printf("empty_vs_null: expected 1 block for empty string, got %" PRId64 "\n", blocks); goto fail; } + + // Insert NULL + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc2', NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + int64_t blocks_null = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc2');"); + if (blocks_null != 1) { printf("empty_vs_null: expected 1 block for NULL, got %" PRId64 "\n", blocks_null); goto fail; } + + // Sync both to site 1 + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("empty_vs_null: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // doc1 (empty string): body should be empty string, NOT NULL + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + int64_t is_null1 = do_select_int(db[1], "SELECT body IS NULL FROM docs WHERE id = 'doc1';"); + if (is_null1 != 0) { printf("empty_vs_null: doc1 body should NOT be NULL\n"); if (body1) sqlite3_free(body1); goto fail; } + if (!body1 || strcmp(body1, "") != 0) { printf("empty_vs_null: doc1 body should be empty, got [%s]\n", body1 ? body1 : "NULL"); if (body1) sqlite3_free(body1); goto fail; } + sqlite3_free(body1); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: DELETE row then re-insert with different block content (resurrection) +bool do_test_block_lww_delete_reinsert(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; } - - // Force to 10 clients for this specific test - nclients = 10; - - // create databases and tables + + // Insert and sync + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Old1\nOld2\nOld3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_using_payload(db[0], db[1], false, true)) goto fail; + + // Delete the row + rc = sqlite3_exec(db[0], "DELETE FROM docs WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("del_reinsert: DELETE failed\n"); goto fail; } + + // Sync delete + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("del_reinsert: delete sync failed\n"); goto fail; } + + // Verify row gone on site 1 + int64_t count = do_select_int(db[1], "SELECT count(*) FROM docs WHERE id = 'doc1';"); + if (count != 0) { printf("del_reinsert: row should be deleted on site 1, count=%" PRId64 "\n", count); goto fail; } + + // Re-insert with different content + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'New1\nNew2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("del_reinsert: re-INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Sync re-insert + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("del_reinsert: reinsert sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, "New1\nNew2") != 0) { + printf("del_reinsert: body mismatch after reinsert: [%s]\n", body ? body : "NULL"); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: INTEGER primary key with block column +bool do_test_block_lww_integer_pk(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter++; - for (int i = 0; i < nclients; ++i) { - db[i] = do_create_database_file_v2(i, timestamp, saved_counter, true); - if (db[i] == false) return false; - - // Create simple table with composite primary key (id1, id2) and two non-pk columns (data1, data2) - char *sql = sqlite3_mprintf("CREATE TABLE %s (id1 INTEGER NOT NULL, id2 TEXT NOT NULL, data1 TEXT, data2 INTEGER, PRIMARY KEY(id1, id2));", table_name); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - - // Initialize cloudsync for the table - sql = sqlite3_mprintf("SELECT cloudsync_init('%s', 'cls', 1);", table_name); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE notes (id INTEGER NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("int_pk: CREATE TABLE failed on %d: %s\n", i, sqlite3_errmsg(db[i])); goto fail; } + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('notes', 'CLS', 1);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("int_pk: init failed on %d: %s\n", i, sqlite3_errmsg(db[i])); goto fail; } + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('notes', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("int_pk: set_column failed on %d: %s\n", i, sqlite3_errmsg(db[i])); goto fail; } + } + + // Insert on site 0 + rc = sqlite3_exec(db[0], "INSERT INTO notes (id, body) VALUES (42, 'First\nSecond\nThird');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("int_pk: INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM notes_cloudsync_blocks WHERE pk = cloudsync_pk_encode(42);"); + if (blocks != 3) { printf("int_pk: expected 3 blocks, got %" PRId64 "\n", blocks); goto fail; } + + // Sync to site 1 + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("int_pk: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('notes', 'body', 42);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("int_pk: materialize failed: %s\n", sqlite3_errmsg(db[1])); goto fail; } + + // Debug: check row exists + int64_t row_count = do_select_int(db[1], "SELECT count(*) FROM notes WHERE id = 42;"); + if (row_count != 1) { printf("int_pk: row not found on site 1, count=%" PRId64 "\n", row_count); goto fail; } + + char *body = do_select_text(db[1], "SELECT body FROM notes WHERE id = 42;"); + if (!body || strcmp(body, "First\nSecond\nThird") != 0) { + printf("int_pk: body mismatch: [%s]\n", body ? body : "NULL"); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + // Edit and sync back + rc = sqlite3_exec(db[1], "UPDATE notes SET body = 'First\nEdited\nThird' WHERE id = 42;", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("int_pk: UPDATE failed: %s\n", sqlite3_errmsg(db[1])); goto fail; } + if (!do_merge_using_payload(db[1], db[0], true, true)) { printf("int_pk: reverse sync failed\n"); goto fail; } + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('notes', 'body', 42);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("int_pk: reverse mat failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + char *body0 = do_select_text(db[0], "SELECT body FROM notes WHERE id = 42;"); + if (!body0 || strcmp(body0, "First\nEdited\nThird") != 0) { + printf("int_pk: reverse body mismatch: [%s]\n", body0 ? body0 : "NULL"); + if (body0) sqlite3_free(body0); + goto fail; + } + sqlite3_free(body0); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Multiple rows with block columns in a single sync +bool do_test_block_lww_multi_row(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert 3 rows + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('r1', 'R1-Line1\nR1-Line2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('r2', 'R2-Alpha\nR2-Beta\nR2-Gamma');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('r3', 'R3-Only');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Edit r1 and r3 + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'R1-Edited\nR1-Line2' WHERE id = 'r1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'R3-Changed' WHERE id = 'r3';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync all in one payload + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("multi_row: sync failed\n"); goto fail; } + + // Materialize all + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'r1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'r2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'r3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + bool ok = true; + char *b1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'r1';"); + if (!b1 || strcmp(b1, "R1-Edited\nR1-Line2") != 0) { printf("multi_row: r1 mismatch [%s]\n", b1 ? b1 : "NULL"); ok = false; } + if (b1) sqlite3_free(b1); + + char *b2 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'r2';"); + if (!b2 || strcmp(b2, "R2-Alpha\nR2-Beta\nR2-Gamma") != 0) { printf("multi_row: r2 mismatch [%s]\n", b2 ? b2 : "NULL"); ok = false; } + if (b2) sqlite3_free(b2); + + char *b3 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'r3';"); + if (!b3 || strcmp(b3, "R3-Changed") != 0) { printf("multi_row: r3 mismatch [%s]\n", b3 ? b3 : "NULL"); ok = false; } + if (b3) sqlite3_free(b3); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Concurrent add at non-overlapping positions (top vs bottom) +bool do_test_block_lww_nonoverlap_add(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert initial: A B C + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'A\nB\nC');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: add line at top -> X A B C + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'X\nA\nB\nC' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: add line at bottom -> A B C Y + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'A\nB\nC\nY' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Bidirectional sync + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("nonoverlap: bodies diverged: [%s] vs [%s]\n", body0 ? body0 : "NULL", body1 ? body1 : "NULL"); + ok = false; + } else { + // X should be present, Y should be present, original A B C should be present + if (!strstr(body0, "X")) { printf("nonoverlap: X missing\n"); ok = false; } + if (!strstr(body0, "Y")) { printf("nonoverlap: Y missing\n"); ok = false; } + if (!strstr(body0, "A")) { printf("nonoverlap: A missing\n"); ok = false; } + if (!strstr(body0, "B")) { printf("nonoverlap: B missing\n"); ok = false; } + if (!strstr(body0, "C")) { printf("nonoverlap: C missing\n"); ok = false; } + + // Order: X before A, Y after C + const char *pX = strstr(body0, "X"); + const char *pA = strstr(body0, "A"); + const char *pC = strstr(body0, "C"); + const char *pY = strstr(body0, "Y"); + if (pX && pA && pX >= pA) { printf("nonoverlap: X not before A\n"); ok = false; } + if (pC && pY && pY <= pC) { printf("nonoverlap: Y not after C\n"); ok = false; } + } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Very long single line (10K chars, single block) +bool do_test_block_lww_long_line(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Build a 10,000-char single line + { + char *long_line = (char *)malloc(10001); + if (!long_line) goto fail; + for (int i = 0; i < 10000; i++) long_line[i] = 'A' + (i % 26); + long_line[10000] = '\0'; + + char *sql = sqlite3_mprintf("INSERT INTO docs (id, body) VALUES ('doc1', '%q');", long_line); + rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - } - - if (print_result) printf("Created %d databases with composite primary key table '%s'\n", nclients, table_name); - - // Insert initial data in each client - for (int i = 0; i < nclients; ++i) { - for (int j = 0; j < 3; ++j) { - char *sql = sqlite3_mprintf("INSERT INTO %s (id1, id2, data1, data2) VALUES (%d, 'client%d_row%d', 'initial_data_%d_%d', %d);", - table_name, i * 100 + j, i, j, i, j, i * 10 + j); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - } + + if (rc != SQLITE_OK) { printf("long_line: INSERT failed: %s\n", sqlite3_errmsg(db[0])); free(long_line); goto fail; } + + // Should have 1 block (no newlines) + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 1) { printf("long_line: expected 1 block, got %" PRId64 "\n", blocks); free(long_line); goto fail; } + + // Sync to site 1 + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("long_line: sync failed\n"); free(long_line); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { free(long_line); goto fail; } + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + bool match = (body && strcmp(body, long_line) == 0); + if (!match) printf("long_line: body mismatch (len=%zu vs expected 10000)\n", body ? strlen(body) : 0); + if (body) sqlite3_free(body); + free(long_line); + if (!match) goto fail; } - - if (print_result) printf("Inserted initial data in all clients\n"); - - // Perform various operations across clients in multiple rounds - for (int round = 0; round < 20; ++round) { - if (print_result) printf("Starting round %d of operations\n", round + 1); - - // Each client performs different operations (favoring remote data operations for better conflict testing) - for (int i = 0; i < nclients; ++i) { - // Weighted operation selection: focus on updates, minimal deletes - int operation_selector = (i * 7 + round * 3) % 20; - int operation; - - if (operation_selector < 4) { - operation = 0; // INSERT (20%) - } else if (operation_selector < 7) { - operation = 1; // UPDATE local data (15%) - } else if (operation_selector < 17) { - operation = 2; // UPDATE remote data (50%) - } else { - operation = 3; // DELETE (15% - reduced from 30%) - } - - switch (operation) { - case 0: // INSERT - { - char *sql = sqlite3_mprintf("INSERT INTO %s (id1, id2, data1, data2) VALUES (%d, 'round%d_client%d', 'new_data_%d_%d', %d);", - table_name, 1000 + round * 100 + i, round, i, round, i, round * 100 + i); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK && rc != SQLITE_CONSTRAINT) { - printf("Insert failed on client %d: %s\n", i, sqlite3_errmsg(db[i])); - } - } - break; - - case 1: // UPDATE local data (more aggressive targeting) - { - // Try multiple approaches to find rows to update - char *sql; - int updated_rows = 0; - - // First try: update rows originally from this client - sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_local_round%d_client%d', data2 = %d WHERE id1 BETWEEN %d AND %d;", - table_name, round, i, round * 1000 + i, i * 100, (i + 1) * 100 - 1); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - updated_rows += sqlite3_changes(db[i]); - - // Second try: if no rows updated, target any available rows - if (updated_rows == 0) { - sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_local_round%d_client%d_any', data2 = %d WHERE rowid = (SELECT rowid FROM %s LIMIT 1 OFFSET %d);", - table_name, round, i, round * 1000 + i, table_name, i % 10); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - updated_rows += sqlite3_changes(db[i]); - } - - if (rc != SQLITE_OK) { - printf("Update local data failed on client %d: %s\n", i, sqlite3_errmsg(db[i])); - } else if (print_result && updated_rows > 0) { - printf("Client %d updated %d local rows in round %d\n", i, updated_rows, round + 1); - } - } - break; - - case 2: // UPDATE remote data (more aggressive - creates conflicts) - { - int target_client = (i + 1 + round) % nclients; // Vary target client - char *sql; - int updated_rows = 0; - - // First try: update rows from target client's range - sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_targeting%d', data2 = %d WHERE id1 BETWEEN %d AND %d;", - table_name, round, i, target_client, round * 2000 + i, target_client * 100, (target_client + 1) * 100 - 1); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - updated_rows += sqlite3_changes(db[i]); - - // Second try: if no rows updated, target any rows not from current client - if (updated_rows == 0) { - sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_any', data2 = %d WHERE id1 NOT BETWEEN %d AND %d LIMIT 2;", - table_name, round, i, round * 2000 + i, i * 100, (i + 1) * 100 - 1); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - updated_rows += sqlite3_changes(db[i]); - } - - // Third try: if still no updates, target any available rows - if (updated_rows == 0) { - sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_fallback', data2 = %d WHERE rowid IN (SELECT rowid FROM %s LIMIT 2 OFFSET %d);", - table_name, round, i, round * 2000 + i, table_name, (i + round) % 10); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - updated_rows += sqlite3_changes(db[i]); - } - - if (rc != SQLITE_OK) { - printf("Update remote data failed on client %d targeting client %d: %s\n", i, target_client, sqlite3_errmsg(db[i])); - } else if (print_result && updated_rows > 0) { - printf("Client %d updated %d remote rows in round %d\n", i, updated_rows, round + 1); - } - } - break; - - case 3: // DELETE (favoring remote data deletion) - { - // 80% chance to delete remote data, 20% chance to delete local data - if ((i + round) % 5 == 0) { - // Delete local data (20% of delete operations) - char *sql = sqlite3_mprintf("DELETE FROM %s WHERE id1 = %d AND id2 = 'client%d_row1';", - table_name, i * 100 + 1, i); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) { - printf("Delete local data failed on client %d: %s\n", i, sqlite3_errmsg(db[i])); - } - } else { - // Delete remote data (80% of delete operations) - int target_client = (i + 2 + round) % nclients; // Vary target client - char *sql = sqlite3_mprintf("DELETE FROM %s WHERE id1 = %d AND id2 = 'client%d_row0';", - table_name, target_client * 100, target_client); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) { - printf("Delete remote data failed on client %d targeting client %d: %s\n", i, target_client, sqlite3_errmsg(db[i])); - } - } - } - break; - } + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Whitespace and empty lines (delimiter edge cases) +bool do_test_block_lww_whitespace(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Text with empty lines, whitespace-only lines, trailing newline + const char *text = "Line1\n\n spaces \n\t\ttabs\n\nLine6\n"; + char *sql = sqlite3_mprintf("INSERT INTO docs (id, body) VALUES ('doc1', '%q');", text); + rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); + sqlite3_free(sql); + if (rc != SQLITE_OK) { printf("whitespace: INSERT failed\n"); goto fail; } + + // Count blocks: "Line1", "", " spaces ", "\t\ttabs", "", "Line6", "" (trailing newline produces empty last block) + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 7) { printf("whitespace: expected 7 blocks, got %" PRId64 "\n", blocks); goto fail; } + + // Sync to site 1 + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("whitespace: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, text) != 0) { + printf("whitespace: body mismatch: [%s] vs [%s]\n", body ? body : "NULL", text); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + // Edit: remove empty lines -> "Line1\n spaces \n\t\ttabs\nLine6" + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Line1\n spaces \n\t\ttabs\nLine6' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + if (!do_merge_using_payload(db[0], db[1], true, true)) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body2 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body2 || strcmp(body2, "Line1\n spaces \n\t\ttabs\nLine6") != 0) { + printf("whitespace: body2 mismatch: [%s]\n", body2 ? body2 : "NULL"); + if (body2) sqlite3_free(body2); + goto fail; + } + sqlite3_free(body2); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +bool do_test_block_lww_cleanup(bool print_result, bool cleanup_databases) { + // Test: cloudsync_cleanup removes both the meta-table and the blocks table + // when a table has block LWW columns configured. + sqlite3 *db = NULL; + time_t timestamp = time(NULL); + int rc; + bool result = false; + + db = do_create_database_file(0, timestamp, test_counter++); + if (!db) return false; + + rc = sqlite3_exec(db, "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_cleanup: CREATE TABLE failed: %s\n", sqlite3_errmsg(db)); goto fail; } + + rc = sqlite3_exec(db, "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_cleanup: cloudsync_init failed: %s\n", sqlite3_errmsg(db)); goto fail; } + + rc = sqlite3_exec(db, "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_cleanup: set_column failed: %s\n", sqlite3_errmsg(db)); goto fail; } + + // Insert a row to populate the blocks table + rc = sqlite3_exec(db, "INSERT INTO docs (id, body) VALUES ('doc1', 'Line 1\nLine 2\nLine 3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_cleanup: INSERT failed: %s\n", sqlite3_errmsg(db)); goto fail; } + + // Confirm docs_cloudsync and docs_cloudsync_blocks exist before cleanup + int64_t meta_exists = do_select_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE name='docs_cloudsync';"); + if (meta_exists != 1) { printf("block_cleanup: docs_cloudsync missing before cleanup\n"); goto fail; } + + int64_t blocks_exists = do_select_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE name='docs_cloudsync_blocks';"); + if (blocks_exists != 1) { printf("block_cleanup: docs_cloudsync_blocks missing before cleanup\n"); goto fail; } + + // Run cleanup + rc = sqlite3_exec(db, "SELECT cloudsync_cleanup('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_cleanup: cloudsync_cleanup failed: %s\n", sqlite3_errmsg(db)); goto fail; } + + // Both companion tables must be gone after cleanup + int64_t meta_after = do_select_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE name='docs_cloudsync';"); + if (meta_after != 0) { printf("block_cleanup: docs_cloudsync still exists after cleanup\n"); goto fail; } + + int64_t blocks_after = do_select_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE name='docs_cloudsync_blocks';"); + if (blocks_after != 0) { printf("block_cleanup: docs_cloudsync_blocks still exists after cleanup\n"); goto fail; } + + result = true; + +fail: + if (db) close_db(db); + return result; +} + +// MARK: - New edge-case tests + +bool do_test_unsupported_algorithms (sqlite3 *db) { + // Test that DWS and AWS algorithms are rejected with an error + const char *sql; + int rc; + + // Create tables for the test + sql = "CREATE TABLE IF NOT EXISTS test_dws (id TEXT PRIMARY KEY, val TEXT);" + "CREATE TABLE IF NOT EXISTS test_aws (id TEXT PRIMARY KEY, val TEXT);"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) return false; + + // DWS should fail + sql = "SELECT cloudsync_init('test_dws', 'dws');"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_ERROR) return false; + + // AWS should fail + sql = "SELECT cloudsync_init('test_aws', 'aws');"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_ERROR) return false; + + // Verify no companion tables were created + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM sqlite_master WHERE name='test_dws_cloudsync' OR name='test_aws_cloudsync';", -1, &stmt, NULL); + if (rc != SQLITE_OK) return false; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); return false; } + int count = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + if (count != 0) return false; + + // CLS should still work on the same table + sql = "SELECT cloudsync_init('test_dws', 'cls');"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) return false; + + return true; +} + +bool do_test_corrupted_payload (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + bool result = false; + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + + // Create source and destination databases + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + + int rc = sqlite3_exec(db[i], "CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('test_tbl');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + } + + // Insert data in source + sqlite3_exec(db[0], "INSERT INTO test_tbl VALUES ('id1', 'value1');", NULL, NULL, NULL); + sqlite3_exec(db[0], "INSERT INTO test_tbl VALUES ('id2', 'value2');", NULL, NULL, NULL); + + // Get valid payload as blob + sqlite3_stmt *enc_stmt = NULL; + int rc = sqlite3_prepare_v2(db[0], "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes WHERE site_id=cloudsync_siteid();", -1, &enc_stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_step(enc_stmt); + if (rc != SQLITE_ROW) { sqlite3_finalize(enc_stmt); goto finalize; } + + int valid_len = sqlite3_column_bytes(enc_stmt, 0); + const void *valid_blob = sqlite3_column_blob(enc_stmt, 0); + if (!valid_blob || valid_len < 20) { sqlite3_finalize(enc_stmt); goto finalize; } + + // Copy valid payload + char *payload_copy = (char *)malloc(valid_len); + if (!payload_copy) { sqlite3_finalize(enc_stmt); goto finalize; } + memcpy(payload_copy, valid_blob, valid_len); + sqlite3_finalize(enc_stmt); + + // Test 1: Empty blob + { + sqlite3_stmt *dec_stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_decode(?);", -1, &dec_stmt, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_blob(dec_stmt, 1, "", 0, SQLITE_STATIC); + rc = sqlite3_step(dec_stmt); + // Should either error or return without inserting + sqlite3_finalize(dec_stmt); } - - // Partial merge: each client merges with 2-3 other random clients (not all) - for (int i = 0; i < nclients; ++i) { - int merge_targets = 2 + (round % 2); // 2 or 3 targets - for (int j = 0; j < merge_targets; ++j) { - int target = (i + j + 1 + round) % nclients; - if (target != i) { - if (do_merge_using_payload(db[i], db[target], true, true) == false) { - if (print_result) printf("Partial merge failed from client %d to %d\n", i, target); - goto finalize; - } - } - } + } + + // Test 2: Random garbage + { + char garbage[16] = {0xDE, 0xAD, 0xBE, 0xEF, 0xCA, 0xFE, 0xBA, 0xBE, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}; + sqlite3_stmt *dec_stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_decode(?);", -1, &dec_stmt, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_blob(dec_stmt, 1, garbage, sizeof(garbage), SQLITE_STATIC); + rc = sqlite3_step(dec_stmt); + sqlite3_finalize(dec_stmt); } - - if (print_result) printf("Completed round %d operations and partial merges\n", round + 1); } - - // Final convergence: merge all changes to client 0 - if (print_result) printf("Starting final convergence to client 0\n"); - for (int i = 1; i < nclients; ++i) { - if (do_merge_using_payload(db[i], db[0], true, true) == false) { - if (print_result) printf("Final merge to client 0 failed from client %d\n", i); - goto finalize; + + // Test 3: Truncated payload (first 10 bytes) + { + sqlite3_stmt *dec_stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_decode(?);", -1, &dec_stmt, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_blob(dec_stmt, 1, payload_copy, 10, SQLITE_STATIC); + rc = sqlite3_step(dec_stmt); + sqlite3_finalize(dec_stmt); } } - - // Merge changes from client 0 to all other clients - if (print_result) printf("Propagating final state from client 0 to all other clients\n"); - for (int i = 1; i < nclients; ++i) { - if (do_merge_using_payload(db[0], db[i], false, true) == false) { - if (print_result) printf("Final merge from client 0 failed to client %d\n", i); - goto finalize; + + // Test 4: Valid payload with flipped byte in the middle + { + char *corrupted = (char *)malloc(valid_len); + memcpy(corrupted, payload_copy, valid_len); + corrupted[valid_len / 2] ^= 0xFF; + + sqlite3_stmt *dec_stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_decode(?);", -1, &dec_stmt, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_blob(dec_stmt, 1, corrupted, valid_len, SQLITE_STATIC); + rc = sqlite3_step(dec_stmt); + sqlite3_finalize(dec_stmt); } + free(corrupted); } - - // Verify all databases have converged to the same state - if (print_result) printf("Verifying convergence across all clients\n"); - char *verification_sql = sqlite3_mprintf("SELECT * FROM %s ORDER BY id1, id2;", table_name); - for (int i = 1; i < nclients; ++i) { - if (do_compare_queries(db[0], verification_sql, db[i], verification_sql, -1, -1, print_result) == false) { - if (print_result) printf("Convergence verification failed: client 0 vs client %d\n", i); - sqlite3_free(verification_sql); - goto finalize; + + // Verify destination table is still empty (no corrupted data inserted) + { + sqlite3_stmt *count_stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT COUNT(*) FROM test_tbl;", -1, &count_stmt, NULL); + if (rc != SQLITE_OK) { free(payload_copy); goto finalize; } + if (sqlite3_step(count_stmt) != SQLITE_ROW) { sqlite3_finalize(count_stmt); free(payload_copy); goto finalize; } + int count = sqlite3_column_int(count_stmt, 0); + sqlite3_finalize(count_stmt); + if (count != 0) { printf("corrupted_payload: expected 0 rows but got %d\n", count); free(payload_copy); goto finalize; } + } + + // Test 5: Valid payload should still work + if (!do_merge_using_payload(db[0], db[1], false, true)) { + printf("corrupted_payload: valid payload failed after corrupted attempts\n"); + free(payload_copy); + goto finalize; + } + + { + sqlite3_stmt *count_stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT COUNT(*) FROM test_tbl;", -1, &count_stmt, NULL); + if (rc != SQLITE_OK) { free(payload_copy); goto finalize; } + if (sqlite3_step(count_stmt) != SQLITE_ROW) { sqlite3_finalize(count_stmt); free(payload_copy); goto finalize; } + int count = sqlite3_column_int(count_stmt, 0); + sqlite3_finalize(count_stmt); + if (count != 2) { printf("corrupted_payload: expected 2 rows after valid apply but got %d\n", count); free(payload_copy); goto finalize; } + } + + free(payload_copy); + result = true; + +finalize: + for (int i = 0; i < 2; i++) { + if (db[i]) close_db(db[i]); + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +typedef struct { + void *data; + int len; +} test_payload_chunk; + +static void test_payload_chunks_free(test_payload_chunk *chunks, int count) { + if (!chunks) return; + for (int i = 0; i < count; ++i) { + if (chunks[i].data) free(chunks[i].data); + } + free(chunks); +} + +static bool test_payload_chunks_tables_equal(sqlite3 *src, sqlite3 *dst) { + sqlite3_stmt *s1 = NULL; + sqlite3_stmt *s2 = NULL; + bool result = false; + + const char *sql = "SELECT id, note, data FROM payload_chunk_test ORDER BY id;"; + int rc1 = sqlite3_prepare_v2(src, sql, -1, &s1, NULL); + int rc2 = sqlite3_prepare_v2(dst, sql, -1, &s2, NULL); + if (rc1 != SQLITE_OK || rc2 != SQLITE_OK) goto finalize; + + while (1) { + rc1 = sqlite3_step(s1); + rc2 = sqlite3_step(s2); + if (rc1 != rc2) goto finalize; + if (rc1 == SQLITE_DONE) break; + if (rc1 != SQLITE_ROW) goto finalize; + + for (int i = 0; i < 3; ++i) { + int t1 = sqlite3_column_type(s1, i); + int t2 = sqlite3_column_type(s2, i); + int n1 = sqlite3_column_bytes(s1, i); + int n2 = sqlite3_column_bytes(s2, i); + if (t1 != t2 || n1 != n2) goto finalize; + const void *v1 = (t1 == SQLITE_BLOB) ? sqlite3_column_blob(s1, i) : sqlite3_column_text(s1, i); + const void *v2 = (t2 == SQLITE_BLOB) ? sqlite3_column_blob(s2, i) : sqlite3_column_text(s2, i); + if (n1 > 0 && (!v1 || !v2 || memcmp(v1, v2, n1) != 0)) goto finalize; + } + } + + result = true; + +finalize: + if (s1) sqlite3_finalize(s1); + if (s2) sqlite3_finalize(s2); + return result; +} + +bool do_test_payload_chunks_large_values (bool print_result, bool cleanup_databases) { + sqlite3 *db[3] = {NULL, NULL, NULL}; + sqlite3_stmt *stmt = NULL; + sqlite3_stmt *apply = NULL; + test_payload_chunk *chunks = NULL; + int chunk_count = 0; + int chunk_cap = 0; + int v3_count = 0; + int first_v3_chunk = -1; + int legacy_payload_len = 0; + bool result = false; + int rc = SQLITE_OK; + const int max_chunk_size = CLOUDSYNC_PAYLOAD_CHUNK_MIN_SIZE; + + time_t timestamp = time(NULL); + int saved_counter = test_counter++; + + for (int i = 0; i < 3; ++i) { + db[i] = do_create_database_file(i, timestamp, saved_counter); + if (!db[i]) goto finalize; + + rc = sqlite3_exec(db[i], + "CREATE TABLE payload_chunk_test (" + "id TEXT PRIMARY KEY, " + "note TEXT DEFAULT '', " + "data BLOB DEFAULT x'');" + "SELECT cloudsync_init('payload_chunk_test');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + } + + rc = sqlite3_exec(db[0], + "SELECT cloudsync_set('payload_max_chunk_size', '262144');" + "INSERT INTO payload_chunk_test(id, note, data) " + "VALUES ('big', lower(hex(randomblob(360000))), randomblob(720000));" + "INSERT INTO payload_chunk_test(id, note, data) " + "VALUES ('same-a', 'same payload a', zeroblob(720000));" + "INSERT INTO payload_chunk_test(id, note, data) " + "VALUES ('same-b', 'same payload b', zeroblob(720000));" + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM c WHERE i < 260) " + "INSERT INTO payload_chunk_test(id, note, data) " + "SELECT printf('row-%03d', i), printf('small-%03d-%s', i, hex(randomblob(850))), randomblob(512) FROM c;", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_prepare_v2(db[0], + "SELECT payload, payload_size, rows FROM cloudsync_payload_chunks() ORDER BY chunk_index;", + -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + int len = sqlite3_column_bytes(stmt, 0); + const void *payload = sqlite3_column_blob(stmt, 0); + sqlite3_int64 payload_size = sqlite3_column_int64(stmt, 1); + sqlite3_int64 rows = sqlite3_column_int64(stmt, 2); + if (!payload || len <= 0 || payload_size != len || len > max_chunk_size || rows <= 0) goto finalize; + if (len > 4 && ((const unsigned char *)payload)[4] == 3) { + if (first_v3_chunk < 0) first_v3_chunk = chunk_count; + ++v3_count; + } + + if (chunk_count == chunk_cap) { + int new_cap = chunk_cap ? chunk_cap * 2 : 16; + test_payload_chunk *new_chunks = realloc(chunks, sizeof(*chunks) * new_cap); + if (!new_chunks) goto finalize; + memset(new_chunks + chunk_cap, 0, sizeof(*chunks) * (new_cap - chunk_cap)); + chunks = new_chunks; + chunk_cap = new_cap; + } + + chunks[chunk_count].data = malloc(len); + if (!chunks[chunk_count].data) goto finalize; + memcpy(chunks[chunk_count].data, payload, len); + chunks[chunk_count].len = len; + ++chunk_count; + } + if (rc != SQLITE_DONE) goto finalize; + sqlite3_finalize(stmt); + stmt = NULL; + + if (chunk_count < 5 || v3_count < 2) goto finalize; + + rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_apply(?);", -1, &apply, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Apply in reverse order to verify that v3 value fragments can be staged + // and completed independently from transport ordering. + for (int i = chunk_count - 1; i >= 0; --i) { + rc = sqlite3_bind_blob(apply, 1, chunks[i].data, chunks[i].len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(apply); + if (rc != SQLITE_ROW) goto finalize; + sqlite3_reset(apply); + sqlite3_clear_bindings(apply); + } + sqlite3_finalize(apply); + apply = NULL; + + if (!test_payload_chunks_tables_equal(db[0], db[1])) goto finalize; + + if (first_v3_chunk < 0) goto finalize; + + // Reopen db[1] so the v3-fragment stale-GC throttle starts fresh on this + // connection: the first fragment applied below runs the stale cleanup. + close_db(db[1]); + db[1] = do_create_database_file(1, timestamp, saved_counter); + if (!db[1]) goto finalize; + + rc = sqlite3_exec(db[1], + "CREATE TABLE IF NOT EXISTS cloudsync_payload_fragments (" + "value_id TEXT NOT NULL, part_index INTEGER NOT NULL, part_count INTEGER NOT NULL, total_size INTEGER NOT NULL, " + "checksum TEXT NOT NULL, created_at INTEGER NOT NULL DEFAULT (unixepoch()), " + "tbl TEXT NOT NULL, pk BLOB NOT NULL, col_name TEXT NOT NULL, col_version INTEGER NOT NULL, db_version INTEGER NOT NULL, " + "site_id BLOB NOT NULL, cl INTEGER NOT NULL, seq INTEGER NOT NULL, fragment BLOB NOT NULL, " + "PRIMARY KEY(value_id, part_index)) WITHOUT ROWID;" + "INSERT OR REPLACE INTO cloudsync_payload_fragments " + "(value_id, part_index, part_count, total_size, checksum, created_at, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, fragment) " + "VALUES ('stale-incomplete', 0, 2, 10, '0000000000000000', 0, 'payload_chunk_test', x'01', 'data', 1, 1, zeroblob(16), 1, 1, x'00');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // First fragment apply on the fresh connection -> stale GC runs -> removed. + rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_apply(?);", -1, &apply, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_bind_blob(apply, 1, chunks[first_v3_chunk].data, chunks[first_v3_chunk].len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(apply); + if (rc != SQLITE_ROW) goto finalize; + sqlite3_reset(apply); + sqlite3_clear_bindings(apply); + + rc = sqlite3_prepare_v2(db[1], "SELECT COUNT(*) FROM cloudsync_payload_fragments WHERE value_id='stale-incomplete';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW || sqlite3_column_int(stmt, 0) != 0) goto finalize; + sqlite3_finalize(stmt); + stmt = NULL; + + // Throttle check: an old group inserted now must NOT be removed by the very + // next fragment apply, because the GC just ran (within the throttle window). + // This proves the cleanup is not re-scanned on every applied fragment. + rc = sqlite3_exec(db[1], + "INSERT OR REPLACE INTO cloudsync_payload_fragments " + "(value_id, part_index, part_count, total_size, checksum, created_at, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, fragment) " + "VALUES ('stale-incomplete-2', 0, 2, 10, '0000000000000000', 0, 'payload_chunk_test', x'02', 'data', 1, 1, zeroblob(16), 1, 1, x'00');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_bind_blob(apply, 1, chunks[first_v3_chunk].data, chunks[first_v3_chunk].len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(apply); + if (rc != SQLITE_ROW) goto finalize; + sqlite3_finalize(apply); + apply = NULL; + + rc = sqlite3_prepare_v2(db[1], "SELECT COUNT(*) FROM cloudsync_payload_fragments WHERE value_id='stale-incomplete-2';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW || sqlite3_column_int(stmt, 0) != 1) goto finalize; // throttled -> still present + sqlite3_finalize(stmt); + stmt = NULL; + + // The legacy aggregate API must still produce an applyable monolithic payload. + rc = sqlite3_prepare_v2(db[0], + "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) " + "FROM cloudsync_changes WHERE site_id=cloudsync_siteid();", + -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto finalize; + legacy_payload_len = sqlite3_column_bytes(stmt, 0); + if (legacy_payload_len <= max_chunk_size) goto finalize; + rc = sqlite3_prepare_v2(db[2], "SELECT cloudsync_payload_apply(?);", -1, &apply, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_bind_value(apply, 1, sqlite3_column_value(stmt, 0)); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(apply); + if (rc != SQLITE_ROW) goto finalize; + sqlite3_finalize(apply); + apply = NULL; + sqlite3_finalize(stmt); + stmt = NULL; + + if (!test_payload_chunks_tables_equal(db[0], db[2])) goto finalize; + + rc = sqlite3_prepare_v2(db[0], "SELECT length(cloudsync_payload_blob_checked(0, 0, NULL, 0, 10000000));", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW || sqlite3_column_int(stmt, 0) != legacy_payload_len) goto finalize; + sqlite3_finalize(stmt); + stmt = NULL; + + rc = sqlite3_prepare_v2(db[0], "SELECT cloudsync_payload_blob_checked(999999, 0, NULL, 0, 10000000) IS NULL;", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW || sqlite3_column_int(stmt, 0) != 1) goto finalize; + sqlite3_finalize(stmt); + stmt = NULL; + + rc = sqlite3_prepare_v2(db[0], "SELECT cloudsync_payload_blob_checked(0, 0, NULL, 0, 1);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ERROR) goto finalize; + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +finalize: + if (!result && print_result) { + printf("do_test_payload_chunks_large_values error: %s / %s\n", + db[0] ? sqlite3_errmsg(db[0]) : "no source db", + db[1] ? sqlite3_errmsg(db[1]) : "no dest db"); + } + if (stmt) sqlite3_finalize(stmt); + if (apply) sqlite3_finalize(apply); + test_payload_chunks_free(chunks, chunk_count); + + for (int i = 0; i < 3; ++i) { + if (db[i]) close_db(db[i]); + } + + if (cleanup_databases) { + for (int i = 0; i < 3; ++i) { + char path[256], walpath[300], shmpath[300]; + do_build_database_path(path, i, timestamp, saved_counter); + snprintf(walpath, sizeof(walpath), "%s-wal", path); + snprintf(shmpath, sizeof(shmpath), "%s-shm", path); + file_delete_internal(path); + file_delete_internal(walpath); + file_delete_internal(shmpath); } } - sqlite3_free(verification_sql); - - if (print_result) { - printf("\nFinal converged state:\n"); - char *display_sql = sqlite3_mprintf("SELECT * FROM %s ORDER BY id1, id2;", table_name); - do_query(db[0], display_sql, NULL); - sqlite3_free(display_sql); + + return result; +} + +// Apply every payload chunk produced by `q` (column 0 = payload blob, run on +// src) into `dst`. Returns SQLITE_OK on success. +static int test_chunks_apply_all (sqlite3_stmt *q, sqlite3 *dst) { + sqlite3_stmt *apply = NULL; + int rc = sqlite3_prepare_v2(dst, "SELECT cloudsync_payload_apply(?);", -1, &apply, NULL); + if (rc != SQLITE_OK) return rc; + while ((rc = sqlite3_step(q)) == SQLITE_ROW) { + sqlite3_bind_blob(apply, 1, sqlite3_column_blob(q, 0), sqlite3_column_bytes(q, 0), SQLITE_STATIC); + if (sqlite3_step(apply) != SQLITE_ROW) { sqlite3_finalize(apply); return SQLITE_ERROR; } + sqlite3_reset(apply); + sqlite3_clear_bindings(apply); + } + sqlite3_finalize(apply); + return (rc == SQLITE_DONE) ? SQLITE_OK : rc; +} + +// Verify table t holds exactly the ids in expected_csv (sorted, '|'-joined). +static bool test_chunks_ids_equal (sqlite3 *db, const char *expected_csv) { + sqlite3_stmt *s = NULL; + if (sqlite3_prepare_v2(db, "SELECT coalesce(group_concat(id, '|'), '') FROM (SELECT id FROM t ORDER BY id);", -1, &s, NULL) != SQLITE_OK) return false; + bool ok = false; + if (sqlite3_step(s) == SQLITE_ROW) { + const char *v = (const char *)sqlite3_column_text(s, 0); + ok = v && strcmp(v, expected_csv) == 0; + } + sqlite3_finalize(s); + return ok; +} + +// Materialize all payload chunks returned by `sql` (column 0 = payload blob) on `db`. +static bool test_crdt_extract (sqlite3 *db, const char *sql, test_payload_chunk **out, int *count) { + *out = NULL; *count = 0; + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) return false; + test_payload_chunk *chunks = NULL; + int n = 0, cap = 0, rc; + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + int len = sqlite3_column_bytes(stmt, 0); + const void *p = sqlite3_column_blob(stmt, 0); + if (!p || len <= 0) { rc = SQLITE_ERROR; break; } + if (n == cap) { + int ncap = cap ? cap * 2 : 8; + test_payload_chunk *nc = realloc(chunks, sizeof(*chunks) * ncap); + if (!nc) { rc = SQLITE_ERROR; break; } + chunks = nc; cap = ncap; + } + chunks[n].data = malloc(len); + if (!chunks[n].data) { rc = SQLITE_ERROR; break; } + memcpy(chunks[n].data, p, len); + chunks[n].len = len; + ++n; + } + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) { test_payload_chunks_free(chunks, n); return false; } + *out = chunks; *count = n; + return true; +} + +// Apply one already-materialized chunk to `db`. +static bool test_crdt_apply (sqlite3 *db, const test_payload_chunk *c) { + sqlite3_stmt *apply = NULL; + if (sqlite3_prepare_v2(db, "SELECT cloudsync_payload_apply(?);", -1, &apply, NULL) != SQLITE_OK) return false; + bool ok = (sqlite3_bind_blob(apply, 1, c->data, c->len, SQLITE_STATIC) == SQLITE_OK && + sqlite3_step(apply) == SQLITE_ROW); + sqlite3_finalize(apply); + return ok; +} + +// Compare the single crdt_test row (a, b, c) between two databases. +static bool test_crdt_row_equal (sqlite3 *db1, sqlite3 *db2) { + sqlite3_stmt *s1 = NULL, *s2 = NULL; + const char *sql = "SELECT a, b, c FROM crdt_test WHERE id='r1';"; + bool result = false; + if (sqlite3_prepare_v2(db1, sql, -1, &s1, NULL) != SQLITE_OK) goto done; + if (sqlite3_prepare_v2(db2, sql, -1, &s2, NULL) != SQLITE_OK) goto done; + if (sqlite3_step(s1) != SQLITE_ROW || sqlite3_step(s2) != SQLITE_ROW) goto done; + for (int i = 0; i < 3; ++i) { + int n1 = sqlite3_column_bytes(s1, i), n2 = sqlite3_column_bytes(s2, i); + if (n1 != n2) goto done; + if (n1 > 0 && memcmp(sqlite3_column_blob(s1, i), sqlite3_column_blob(s2, i), n1) != 0) goto done; } - result = true; - -finalize: - for (int i = 0; i < nclients; ++i) { - if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) { - printf("do_test_merge_composite_pk_10_clients error: %s\n", sqlite3_errmsg(db[i])); - } - if (db[i]) { - if (sqlite3_get_autocommit(db[i]) == 0) { - result = false; - printf("do_test_merge_composite_pk_10_clients error: db %d is in transaction\n", i); - } - - int counter = close_db_v2(db[i]); - if (counter > 0) { - result = false; - printf("do_test_merge_composite_pk_10_clients error: db %d has %d unterminated statements\n", i, counter); - } - } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter); - file_delete_internal(buf); - } - } +done: + if (s1) sqlite3_finalize(s1); + if (s2) sqlite3_finalize(s2); return result; } -bool do_test_prikey (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; +// Read the single-row crdt_test.c blob into a malloc'd buffer. +static bool test_crdt_get_c (sqlite3 *db, void **out, int *outlen) { + *out = NULL; *outlen = 0; + sqlite3_stmt *s = NULL; + if (sqlite3_prepare_v2(db, "SELECT c FROM crdt_test WHERE id='r1';", -1, &s, NULL) != SQLITE_OK) return false; + bool ok = false; + if (sqlite3_step(s) == SQLITE_ROW) { + int n = sqlite3_column_bytes(s, 0); + const void *p = sqlite3_column_blob(s, 0); + if (n > 0 && p) { void *buf = malloc(n); if (buf) { memcpy(buf, p, n); *out = buf; *outlen = n; ok = true; } } + } + sqlite3_finalize(s); + return ok; +} + +// Evaluate a scalar-int SQL query on db, true iff the single result is 1. +static bool test_crdt_scalar_true (sqlite3 *db, const char *sql) { + sqlite3_stmt *s = NULL; + if (sqlite3_prepare_v2(db, sql, -1, &s, NULL) != SQLITE_OK) return false; + bool ok = (sqlite3_step(s) == SQLITE_ROW && sqlite3_column_int(s, 0) == 1); + sqlite3_finalize(s); + return ok; +} + +// CRDT convergence under chunked / fragmented, non-atomic, interleaved apply. +// Two sites concurrently change a small column and a big (fragmented) column of the +// same row, each in ONE transaction (one db_version, several seq). Both sites' chunks +// are applied to three targets in different orders (A-then-B, B-then-A, interleaved). +// Column-level LWW must make all three converge to the identical row: the mix +// a=from-A / b=from-B, with the big value being one site's value intact — fragment +// value_ids are per (site_id, col_version, checksum), so fragments never cross-mix. +bool do_test_payload_chunks_crdt_convergence (bool print_result, bool cleanup_databases) { + enum { SRC = 0, SITEA = 1, SITEB = 2, TGT1 = 3, TGT2 = 4, TGT3 = 5, NDB = 6 }; + sqlite3 *db[6] = {0}; + test_payload_chunk *base = NULL, *ca = NULL, *cb = NULL; + int nbase = 0, na = 0, nb = 0; + void *cA = NULL, *cB = NULL, *cT = NULL; int lA = 0, lB = 0, lT = 0; bool result = false; int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables + time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i= 0; --pass) + for (int j = 0, total = na + nb; j < total; ++j) { + if ((j & 1) != pass) continue; + const test_payload_chunk *c = (j < na) ? &ca[j] : &cb[j - na]; + if (!test_crdt_apply(db[TGT3], c)) goto finalize; + } + + // (1) Convergence: all three targets hold the identical row regardless of order. + if (!test_crdt_row_equal(db[TGT1], db[TGT2])) goto finalize; + if (!test_crdt_row_equal(db[TGT1], db[TGT3])) goto finalize; + + // (2) The row is the column-level LWW mix: a from A, b from B. + if (!test_crdt_scalar_true(db[TGT1], "SELECT a='AAA_from_A' AND b='BBB_from_B' FROM crdt_test WHERE id='r1';")) goto finalize; + + // (3) The big value is ONE site's value, fully reassembled and intact — fragments + // from the two sites never cross-contaminate (distinct value_ids). + if (!test_crdt_get_c(db[SITEA], &cA, &lA)) goto finalize; + if (!test_crdt_get_c(db[SITEB], &cB, &lB)) goto finalize; + if (!test_crdt_get_c(db[TGT1], &cT, &lT)) goto finalize; + if (lT != 720000) goto finalize; + if ((lT == lA && memcmp(cT, cA, lT) == 0) == (lT == lB && memcmp(cT, cB, lT) == 0)) goto finalize; // exactly one + + result = true; + +finalize: + if (!result && print_result) + printf("do_test_payload_chunks_crdt_convergence error: %s\n", db[SRC] ? sqlite3_errmsg(db[SRC]) : "no db"); + free(cA); free(cB); free(cT); + test_payload_chunks_free(base, nbase); + test_payload_chunks_free(ca, na); + test_payload_chunks_free(cb, nb); + for (int i = 0; i < NDB; ++i) if (db[i]) close_db(db[i]); + if (cleanup_databases) { + for (int i = 0; i < NDB; ++i) { + char path[256], walpath[300], shmpath[300]; + do_build_database_path(path, i, timestamp, saved_counter); + snprintf(walpath, sizeof(walpath), "%s-wal", path); + snprintf(shmpath, sizeof(shmpath), "%s-shm", path); + file_delete_internal(path); + file_delete_internal(walpath); + file_delete_internal(shmpath); } } - - // update primary key in all clients except client1 - sql = "UPDATE foo SET a=666 WHERE a=1;"; - for (int i=1; i foo (client1)\n"); - do_query(db[0], "SELECT * FROM foo ORDER BY a;", NULL); + + // db[0] = local site S0 changes; db[1] = remote site S1 changes + rc = sqlite3_exec(db[0], "INSERT INTO t(id, note) VALUES ('a0','n'),('b0','n');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[1], "INSERT INTO t(id, note) VALUES ('a1','n'),('b1','n');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // capture S1 (db[1]'s site id) + if (sqlite3_prepare_v2(db[1], "SELECT cloudsync_siteid();", -1, &stmt, NULL) != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ROW || sqlite3_column_bytes(stmt, 0) != 16) goto finalize; + memcpy(s1, sqlite3_column_blob(stmt, 0), 16); + sqlite3_finalize(stmt); stmt = NULL; + + // transfer db[1]'s changes into db[0] (now db[0] has S0 and S1 changes) + if (sqlite3_prepare_v2(db[1], "SELECT payload FROM cloudsync_payload_chunks WHERE since_db_version=0 ORDER BY chunk_index;", -1, &stmt, NULL) != SQLITE_OK) goto finalize; + if (test_chunks_apply_all(stmt, db[0]) != SQLITE_OK) goto finalize; + sqlite3_finalize(stmt); stmt = NULL; + if (!test_chunks_ids_equal(db[0], "a0|a1|b0|b1")) goto finalize; + + // exclude S1 -> only S0 changes (a0,b0) into db[2] + if (sqlite3_prepare_v2(db[0], "SELECT payload FROM cloudsync_payload_chunks WHERE since_db_version=0 AND site_id=? AND exclude_filter_site_id=1 ORDER BY chunk_index;", -1, &stmt, NULL) != SQLITE_OK) goto finalize; + sqlite3_bind_blob(stmt, 1, s1, 16, SQLITE_STATIC); + if (test_chunks_apply_all(stmt, db[2]) != SQLITE_OK) goto finalize; + sqlite3_finalize(stmt); stmt = NULL; + if (!test_chunks_ids_equal(db[2], "a0|b0")) goto finalize; + + // inclusive filter S1 -> only S1 changes (a1,b1) into db[3] + if (sqlite3_prepare_v2(db[0], "SELECT payload FROM cloudsync_payload_chunks WHERE since_db_version=0 AND site_id=? ORDER BY chunk_index;", -1, &stmt, NULL) != SQLITE_OK) goto finalize; + sqlite3_bind_blob(stmt, 1, s1, 16, SQLITE_STATIC); + if (test_chunks_apply_all(stmt, db[3]) != SQLITE_OK) goto finalize; + sqlite3_finalize(stmt); stmt = NULL; + if (!test_chunks_ids_equal(db[3], "a1|b1")) goto finalize; + + // exclude=true without a site_id must error + if (sqlite3_prepare_v2(db[0], "SELECT payload FROM cloudsync_payload_chunks WHERE exclude_filter_site_id=1;", -1, &stmt, NULL) != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ERROR) goto finalize; // expected: xFilter raises an error + sqlite3_finalize(stmt); stmt = NULL; + + if (sqlite3_prepare_v2(db[0], + "SELECT length(cloudsync_payload_blob_checked(0, 0, ?1, 1, 1000000)) > 0 " + "AND length(cloudsync_payload_blob_checked(0, 0, ?1, 0, 1000000)) > 0 " + "AND cloudsync_payload_blob_checked(999999, 0, ?1, 1, 1000000) IS NULL;", -1, &stmt, NULL) != SQLITE_OK) goto finalize; + sqlite3_bind_blob(stmt, 1, s1, 16, SQLITE_STATIC); + if (sqlite3_step(stmt) != SQLITE_ROW || sqlite3_column_int(stmt, 0) != 1) goto finalize; + sqlite3_finalize(stmt); stmt = NULL; + + if (sqlite3_prepare_v2(db[0], "SELECT cloudsync_payload_blob_checked(0, 0, NULL, 1, 1000000);", -1, &stmt, NULL) != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ERROR) goto finalize; + sqlite3_finalize(stmt); stmt = NULL; + + // UUID text<->blob roundtrip (dashed and undashed) must recover S1 + if (sqlite3_prepare_v2(db[0], + "SELECT cloudsync_uuid_blob(cloudsync_uuid_text(?1)) = ?1 " + "AND cloudsync_uuid_blob(cloudsync_uuid_text(?1, 0)) = ?1 " + "AND length(cloudsync_uuid_text(?1)) = 36 " + "AND length(cloudsync_uuid_text(?1, 0)) = 32;", -1, &stmt, NULL) != SQLITE_OK) goto finalize; + sqlite3_bind_blob(stmt, 1, s1, 16, SQLITE_STATIC); + if (sqlite3_step(stmt) != SQLITE_ROW || sqlite3_column_int(stmt, 0) != 1) goto finalize; + sqlite3_finalize(stmt); stmt = NULL; + + result = true; + +finalize: + if (!result && print_result) { + printf("do_test_payload_chunks_site_exclusion error: %s\n", db[0] ? sqlite3_errmsg(db[0]) : "no db"); } - - // send local changes to client1 - for (int i=1; i 0 && (!v1 || !v2 || memcmp(v1, v2, n1) != 0)) goto done; } } + result = true; +done: + if (s1) sqlite3_finalize(s1); + if (s2) sqlite3_finalize(s2); return result; } -bool do_test_double_init(int nclients, bool cleanup_databases) { - if (nclients<2) { - nclients = 2; - printf("Number of clients for test do_test_double_init increased to %d clients\n", 2); - } - +// Reproduces the chunked-download data-loss bug: a single source db_version split +// across >= 2 v2 transport chunks. Applying only the first chunk must NOT advance +// the durable receive cursor into the middle of that db_version, and a re-check +// (from the unchanged cursor) must re-deliver every row (no loss). +bool do_test_payload_chunks_split_dbversion (bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + sqlite3_stmt *stmt = NULL; + cloudsync_context *ctx = NULL; + test_payload_chunk *chunks = NULL; + int chunk_count = 0, chunk_cap = 0; + int64_t split_db_version = -1; + int64_t watermark = -1; + bool single_db_version = true; bool result = false; - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + int rc = SQLITE_OK; + time_t timestamp = time(NULL); - db[0] = do_create_database_file(0, timestamp, test_counter); + int saved_counter = test_counter++; - // configure cloudsync for a table on the first connection - int table_mask = TEST_PRIKEYS; - if (do_create_tables(table_mask, db[0]) == false) goto finalize; - if (do_augment_tables(table_mask, db[0], table_algo_crdt_cls) == false) goto finalize; - - // double load - sqlite3_cloudsync_init(db[0], NULL, NULL); - sqlite3_cloudsync_init(db[0], NULL, NULL); - - // double cloudsync-init - if (do_augment_tables(table_mask, db[0], table_algo_crdt_cls) == false) goto finalize; - if (do_augment_tables(table_mask, db[0], table_algo_crdt_cls) == false) goto finalize; + for (int i = 0; i < 2; ++i) { + db[i] = do_create_database_file(i, timestamp, saved_counter); + if (!db[i]) goto finalize; + rc = sqlite3_exec(db[i], + "CREATE TABLE split_test (id TEXT PRIMARY KEY, body TEXT DEFAULT '');" + "SELECT cloudsync_init('split_test');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + } - // double terminate - if (sqlite3_exec(db[0], "SELECT cloudsync_terminate();", NULL, NULL, NULL) != SQLITE_OK) goto finalize; - if (do_augment_tables(table_mask, db[0], table_algo_crdt_cls) == false) goto finalize; - if (sqlite3_exec(db[0], "SELECT cloudsync_terminate();", NULL, NULL, NULL) != SQLITE_OK) goto finalize; + // A single CTE INSERT is one transaction -> one source db_version. With a + // small chunk budget the ~500 medium rows split across multiple v2 chunks + // (every value is small, so no v3 fragments are produced). + rc = sqlite3_exec(db[0], + "SELECT cloudsync_set('payload_max_chunk_size', '262144');" + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM c WHERE i < 500) " + "INSERT INTO split_test(id, body) SELECT printf('row-%04d', i), hex(randomblob(700)) FROM c;", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; - db[1] = do_create_database_file(1, timestamp, test_counter); + // Collect the transport chunks for the whole window (since_db_version=0). + rc = sqlite3_prepare_v2(db[0], + "SELECT payload, db_version_min, db_version_max, watermark_db_version " + "FROM cloudsync_payload_chunks WHERE since_db_version=0 ORDER BY chunk_index;", + -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + int len = sqlite3_column_bytes(stmt, 0); + const void *payload = sqlite3_column_blob(stmt, 0); + int64_t dbv_min = sqlite3_column_int64(stmt, 1); + int64_t dbv_max = sqlite3_column_int64(stmt, 2); + int64_t wm = sqlite3_column_int64(stmt, 3); + if (!payload || len <= 0) goto finalize; + // every chunk must be a plain v2 payload (version byte at offset 4) + if (((const unsigned char *)payload)[4] != 2) goto finalize; + if (split_db_version < 0) split_db_version = dbv_min; + watermark = wm; + if (dbv_min != split_db_version || dbv_max != split_db_version) single_db_version = false; + + if (chunk_count == chunk_cap) { + int new_cap = chunk_cap ? chunk_cap * 2 : 8; + test_payload_chunk *nc = realloc(chunks, sizeof(*chunks) * new_cap); + if (!nc) goto finalize; + memset(nc + chunk_cap, 0, sizeof(*chunks) * (new_cap - chunk_cap)); + chunks = nc; + chunk_cap = new_cap; + } + chunks[chunk_count].data = malloc(len); + if (!chunks[chunk_count].data) goto finalize; + memcpy(chunks[chunk_count].data, payload, len); + chunks[chunk_count].len = len; + ++chunk_count; + } + if (rc != SQLITE_DONE) goto finalize; + sqlite3_finalize(stmt); stmt = NULL; + + // Preconditions for the scenario: a single db_version split into >= 2 chunks, + // and the watermark equals that db_version. + if (chunk_count < 2 || !single_db_version || watermark != split_db_version) goto finalize; + + // The server's chunk vtab resumes on db_version > since (no seq cursor). So + // advancing the cursor to split_db_version mid-stream would make the next + // /check (since=split_db_version) return NOTHING -> the un-applied chunks are + // lost. Prove that here: + rc = sqlite3_prepare_v2(db[0], + "SELECT count(*) FROM cloudsync_payload_chunks WHERE since_db_version=?;", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + sqlite3_bind_int64(stmt, 1, split_db_version); + if (sqlite3_step(stmt) != SQLITE_ROW || sqlite3_column_int(stmt, 0) != 0) goto finalize; + sqlite3_finalize(stmt); stmt = NULL; + + // The chunked-download receive cursor is controlled at the C level (the + // public cloudsync_payload_apply() SQL function stays single-argument). Drive + // cloudsync_payload_apply() directly on an initialized context to exercise the + // checkpoint gating exactly as the network /check path does. + ctx = cloudsync_context_create(db[1]); + if (!ctx || cloudsync_context_init(ctx) == NULL) goto finalize; + + // --- Apply ONLY the first chunk as a NON-FINAL chunk (checkpoint NONE) --- + { + int n = 0; + if (cloudsync_payload_apply(ctx, chunks[0].data, chunks[0].len, &n, CLOUDSYNC_CHECKPOINT_NONE, 0) != DBRES_OK) goto finalize; + } + + // Cursor MUST NOT have advanced into the incomplete db_version. + if (test_read_check_dbversion(db[1]) != 0) goto finalize; + // ...but the first chunk's rows are applied. + { + sqlite3_stmt *cnt = NULL; + int n = -1; + if (sqlite3_prepare_v2(db[1], "SELECT count(*) FROM split_test;", -1, &cnt, NULL) == SQLITE_OK && + sqlite3_step(cnt) == SQLITE_ROW) n = sqlite3_column_int(cnt, 0); + if (cnt) sqlite3_finalize(cnt); + if (n <= 0) goto finalize; + } + + // --- Re-check: cursor is still 0, so the server re-delivers the whole window. + // Apply every chunk; the first is re-applied (idempotent). The final chunk + // carries the watermark checkpoint, advancing the cursor only now. --- + for (int i = 0; i < chunk_count; ++i) { + bool final_chunk = (i == chunk_count - 1); + int n = 0; + int64_t checkpoint = final_chunk ? watermark : CLOUDSYNC_CHECKPOINT_NONE; + if (cloudsync_payload_apply(ctx, chunks[i].data, chunks[i].len, &n, checkpoint, 0) != DBRES_OK) goto finalize; + } + + // No data lost: destination matches source exactly. + if (!test_split_tables_equal(db[0], db[1])) goto finalize; + // Cursor advanced to the watermark only after the full stream was applied. + if (test_read_check_dbversion(db[1]) != watermark) goto finalize; result = true; - + finalize: - for (int i=0; idb_version cursor could not +// express) with a value larger than the chunk budget (mid-fragment resumes). +// Part 2 is end-to-end: the positionally-drained stream is applied to a fresh +// receiver and its table content is compared to the source (drain -> apply -> +// faithful replica), the real path the /check job will use. +bool do_test_payload_chunks_positional_resume (bool print_result, bool cleanup_databases) { + sqlite3 *db = NULL; + sqlite3 *db2 = NULL; + sqlite3_stmt *stmt = NULL; + sqlite3_stmt *apply = NULL; + test_payload_chunk *base = NULL; int base_count = 0, base_cap = 0; + test_payload_chunk *pos = NULL; int pos_count = 0, pos_cap = 0; + int64_t watermark = -1; bool result = false; int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables + time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i log\n"); - do_query(db[0], "SELECT * FROM log ORDER BY id;", NULL); + int saved_counter = test_counter++; + + db = do_create_database_file(0, timestamp, saved_counter); + if (!db) goto finalize; + rc = sqlite3_exec(db, + "CREATE TABLE split_test (id TEXT PRIMARY KEY, body TEXT DEFAULT '');" + "SELECT cloudsync_init('split_test');" + "SELECT cloudsync_set('payload_max_chunk_size', '262144');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // tx1: ~500 medium rows in one transaction -> one db_version split across + // several v2 chunks (row-boundary resumes within a single version). + rc = sqlite3_exec(db, + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM c WHERE i < 500) " + "INSERT INTO split_test(id, body) SELECT printf('row-%04d', i), hex(randomblob(700)) FROM c;", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // tx2: one value far larger than the chunk budget -> v3 fragments across + // several chunks (mid-fragment resumes inside a single value). + rc = sqlite3_exec(db, + "INSERT INTO split_test(id, body) VALUES ('big', hex(randomblob(900000)));", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Baseline: every chunk of the whole window, in order, via the legacy scan. + rc = sqlite3_prepare_v2(db, + "SELECT payload, watermark_db_version FROM cloudsync_payload_chunks " + "WHERE since_db_version=0 ORDER BY chunk_index;", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + int len = sqlite3_column_bytes(stmt, 0); + const void *payload = sqlite3_column_blob(stmt, 0); + if (!payload || len <= 0) goto finalize; + watermark = sqlite3_column_int64(stmt, 1); + if (base_count == base_cap) { + int nc = base_cap ? base_cap * 2 : 8; + test_payload_chunk *t = realloc(base, sizeof(*t) * nc); + if (!t) goto finalize; + memset(t + base_cap, 0, sizeof(*t) * (nc - base_cap)); + base = t; base_cap = nc; + } + base[base_count].data = malloc(len); + if (!base[base_count].data) goto finalize; + memcpy(base[base_count].data, payload, len); + base[base_count].len = len; + ++base_count; + } + if (rc != SQLITE_DONE) goto finalize; + sqlite3_finalize(stmt); stmt = NULL; + + // Scenario must actually exercise multiple chunks (and thus resumes). + if (base_count < 4 || watermark <= 0) goto finalize; + + // Positional drain: one chunk per call, seeking to the cursor the previous + // chunk reported. until is the frozen watermark from the baseline. + rc = sqlite3_prepare_v2(db, + "SELECT payload, next_db_version, next_seq, next_frag_offset, is_final " + "FROM cloudsync_payload_chunks " + "WHERE until_db_version=?1 AND resume_db_version=?2 AND resume_seq=?3 AND resume_frag_offset=?4 " + "LIMIT 1;", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + + int64_t rdbv = 0, rseq = 0, rfrag = 0; + bool done = false; + bool saw_frag_resume = false; // a follow-up call actually resumed mid-value + // Hard cap guards against a resume bug looping forever. + for (int guard = 0; !done && guard <= base_count + 2; ++guard) { + if (rfrag > 0) saw_frag_resume = true; + sqlite3_reset(stmt); + sqlite3_bind_int64(stmt, 1, watermark); + sqlite3_bind_int64(stmt, 2, rdbv); + sqlite3_bind_int64(stmt, 3, rseq); + sqlite3_bind_int64(stmt, 4, rfrag); + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto finalize; // every step before is_final must yield a chunk + int len = sqlite3_column_bytes(stmt, 0); + const void *payload = sqlite3_column_blob(stmt, 0); + if (!payload || len <= 0) goto finalize; + rdbv = sqlite3_column_int64(stmt, 1); + rseq = sqlite3_column_int64(stmt, 2); + rfrag = sqlite3_column_int64(stmt, 3); + done = sqlite3_column_int(stmt, 4) != 0; + if (pos_count == pos_cap) { + int nc = pos_cap ? pos_cap * 2 : 8; + test_payload_chunk *t = realloc(pos, sizeof(*t) * nc); + if (!t) goto finalize; + memset(t + pos_cap, 0, sizeof(*t) * (nc - pos_cap)); + pos = t; pos_cap = nc; + } + pos[pos_count].data = malloc(len); + if (!pos[pos_count].data) goto finalize; + memcpy(pos[pos_count].data, payload, len); + pos[pos_count].len = len; + ++pos_count; + } + sqlite3_finalize(stmt); stmt = NULL; + + // The positional drain must terminate exactly on is_final, reproduce the + // baseline chunk sequence byte-for-byte, and have actually exercised a + // mid-value (fragment) resume — not only row-boundary resumes. + if (!done || pos_count != base_count || !saw_frag_resume) goto finalize; + for (int i = 0; i < base_count; ++i) { + if (pos[i].len != base[i].len) goto finalize; + if (memcmp(pos[i].data, base[i].data, base[i].len) != 0) goto finalize; + } + + // End-to-end: apply the positionally-drained stream to a fresh receiver and + // assert its table content matches the source. This exercises the real /check + // path (positional drain -> apply -> faithful replica), not just byte-identity. + db2 = do_create_database_file(1, timestamp, saved_counter); + if (!db2) goto finalize; + rc = sqlite3_exec(db2, + "CREATE TABLE split_test (id TEXT PRIMARY KEY, body TEXT DEFAULT '');" + "SELECT cloudsync_init('split_test');", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_prepare_v2(db2, "SELECT cloudsync_payload_apply(?);", -1, &apply, NULL); + if (rc != SQLITE_OK) goto finalize; + // Apply in reverse drain order: apply must reassemble v3 fragments and merge + // rows independent of transport order. + for (int i = pos_count - 1; i >= 0; --i) { + rc = sqlite3_bind_blob(apply, 1, pos[i].data, pos[i].len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(apply); + if (rc != SQLITE_ROW) goto finalize; + sqlite3_reset(apply); + sqlite3_clear_bindings(apply); } - + sqlite3_finalize(apply); apply = NULL; + + if (!test_split_tables_equal(db, db2)) goto finalize; + result = true; - + finalize: - for (int i=0; i= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); + + db = do_create_database(); + if (!db) goto finalize; + + if (sqlite3_exec(db, "CREATE TABLE barang (id TEXT PRIMARY KEY, name TEXT);", NULL, NULL, NULL) != SQLITE_OK) goto finalize; + if (sqlite3_exec(db, "SELECT cloudsync_init('barang');", NULL, NULL, NULL) != SQLITE_OK) goto finalize; + if (sqlite3_exec(db, "INSERT INTO barang VALUES ('1','test');", NULL, NULL, NULL) != SQLITE_OK) goto finalize; + + // settings rows survive, meta table does not + if (sqlite3_exec(db, "DROP TABLE barang_cloudsync;", NULL, NULL, NULL) != SQLITE_OK) goto finalize; + + if (sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM cloudsync_changes;", -1, &stmt, NULL) != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ERROR) goto finalize; + if (!strstr(sqlite3_errmsg(db), "sync metadata")) goto finalize; + sqlite3_finalize(stmt); stmt = NULL; + + result = true; + +finalize: + if (!result && print_result) { + printf("do_test_changes_vtab_missing_meta_tables error: %s\n", db ? sqlite3_errmsg(db) : "no db"); } - - // create databases and tables - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +bool do_test_payload_idempotency (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + bool result = false; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i= 0 && count != prev_count) { + printf("payload_idempotency: row count changed from %d to %d on apply #%d\n", prev_count, count, apply + 1); + goto finalize; } + prev_count = count; } - - if (force_uncompressed) force_uncompressed_blob = false; - - // compare results - for (int i=1; i customers\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); + + // Verify data values are correct + { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db[1], "SELECT val FROM test_tbl WHERE id = 'id1';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } + const char *val = (const char *)sqlite3_column_text(stmt, 0); + if (!val || strcmp(val, "hello_updated") != 0) { + printf("payload_idempotency: expected 'hello_updated', got '%s'\n", val ? val : "NULL"); + sqlite3_finalize(stmt); + goto finalize; + } + sqlite3_finalize(stmt); } - - result = true; - rc = SQLITE_OK; - + + // Compare source and target + result = do_compare_queries(db[0], "SELECT * FROM test_tbl ORDER BY id;", + db[1], "SELECT * FROM test_tbl ORDER BY id;", + -1, -1, print_result); + finalize: - for (int i=0; i= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i " CUSTOMERS_TABLE "\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - if (table_mask & TEST_NOCOLS) { - printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); - do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); - } - if (table_mask & TEST_NOPRIKEYS) { - printf("\n-> customers_noprikey\n"); - do_query(db[0], "SELECT * FROM customers_noprikey ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); - } + + // All 3 must converge to the same value + const char *query = "SELECT val FROM test_tbl WHERE id = 'row1';"; + char *values[3] = {NULL, NULL, NULL}; + + for (int i = 0; i < 3; i++) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db[i], query, -1, &stmt, NULL); + if (rc != SQLITE_OK) goto tiebreak_finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto tiebreak_finalize; } + const char *text = (const char *)sqlite3_column_text(stmt, 0); + values[i] = text ? strdup(text) : NULL; + sqlite3_finalize(stmt); } - - result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); - } - } - + + // Check convergence + if (values[0] && values[1] && values[2] && + strcmp(values[0], values[1]) == 0 && strcmp(values[1], values[2]) == 0) { + result = true; + } else { + printf("causal_length_tiebreak: databases diverged: '%s', '%s', '%s'\n", + values[0] ? values[0] : "NULL", + values[1] ? values[1] : "NULL", + values[2] ? values[2] : "NULL"); + } + +tiebreak_finalize: + for (int i = 0; i < 3; i++) { + if (values[i]) free(values[i]); + } + +finalize: + for (int i = 0; i < 3; i++) { + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -5797,113 +13171,66 @@ bool do_test_fill_initial_data(int nclients, bool print_result, bool cleanup_dat return result; } -bool do_test_alter(int nclients, int alter_version, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; +bool do_test_delete_resurrect_ordering (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[3] = {NULL, NULL, NULL}; bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i2) + sqlite3_exec(db[0], "DELETE FROM test_tbl WHERE id = 'row1';", NULL, NULL, NULL); + + // Sync delete to B + do_merge_using_payload(db[0], db[1], true, true); + + // Site B: re-insert (CL 2->3, resurrection) + sqlite3_exec(db[1], "INSERT INTO test_tbl VALUES ('row1', 'resurrected_by_b');", NULL, NULL, NULL); + + // Site C receives payloads in REVERSE order: B's resurrection first, then A's delete + do_merge_using_payload(db[1], db[2], true, true); + do_merge_using_payload(db[0], db[2], true, true); + + // Site A receives B's resurrection + do_merge_using_payload(db[1], db[2], true, true); + do_merge_using_payload(db[1], db[0], true, true); + + // All 3 should converge: row exists + const char *query = "SELECT * FROM test_tbl ORDER BY id;"; + result = do_compare_queries(db[0], query, db[1], query, -1, -1, print_result); + if (result) result = do_compare_queries(db[0], query, db[2], query, -1, -1, print_result); + + // Verify the row exists (resurrection should win) + if (result) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db[2], "SELECT COUNT(*) FROM test_tbl WHERE id = 'row1';", -1, &stmt, NULL); + if (rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW) { + int count = sqlite3_column_int(stmt, 0); + if (count != 1) { + printf("delete_resurrect_ordering: expected row1 to exist on db[2], count=%d\n", count); + result = false; } - if (do_compare_queries(db[0], sql, db[i], sql, -1, -1, print_result) == false) goto finalize; - } - } - - if (print_result) { - if (table_mask & TEST_PRIKEYS) { - printf("\n-> " CUSTOMERS_TABLE "\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - if (table_mask & TEST_NOCOLS) { - printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); - do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); - } - if (table_mask & TEST_NOPRIKEYS) { - printf("\n-> customers_noprikey\n"); - do_query(db[0], "SELECT * FROM customers_noprikey ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); } + if (stmt) sqlite3_finalize(stmt); } - - result = true; - rc = SQLITE_OK; - + finalize: - for (int i=0; iText:", do_test_block_lww_null_to_text(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Interleave:", do_test_block_lww_interleaved(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW CustomDelim:", do_test_block_lww_custom_delimiter(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Large Text:", do_test_block_lww_large_text(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Rapid Upd:", do_test_block_lww_rapid_updates(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Unicode:", do_test_block_lww_unicode(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW SpecialChars:", do_test_block_lww_special_chars(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Del vs Edit:", do_test_block_lww_delete_vs_edit(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW TwoBlockCols:", do_test_block_lww_two_block_cols(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Text->NULL:", do_test_block_lww_text_to_null(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW PayloadSync:", do_test_block_lww_payload_sync(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Idempotent:", do_test_block_lww_idempotent(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Ordering:", do_test_block_lww_ordering(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW CompositePK:", do_test_block_lww_composite_pk(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW EmptyVsNull:", do_test_block_lww_empty_vs_null(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW DelReinsert:", do_test_block_lww_delete_reinsert(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW IntegerPK:", do_test_block_lww_integer_pk(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW MultiRow:", do_test_block_lww_multi_row(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW NonOverlap:", do_test_block_lww_nonoverlap_add(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW LongLine:", do_test_block_lww_long_line(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Whitespace:", do_test_block_lww_whitespace(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Cleanup:", do_test_block_lww_cleanup(print_result, cleanup_databases)); + + // edge-case tests + result += test_report("Corrupted Payload Test:", do_test_corrupted_payload(2, print_result, cleanup_databases)); + result += test_report("Payload Idempotency Test:", do_test_payload_idempotency(2, print_result, cleanup_databases)); + result += test_report("CL Tiebreak Test:", do_test_causal_length_tiebreak(3, print_result, cleanup_databases)); + result += test_report("Delete/Resurrect Order:", do_test_delete_resurrect_ordering(3, print_result, cleanup_databases)); + result += test_report("Large Composite PK Test:", do_test_large_composite_pk(2, print_result, cleanup_databases)); + result += test_report("Schema Hash Mismatch:", do_test_schema_hash_mismatch(2, print_result, cleanup_databases)); + result += test_report("Stale Table Settings:", do_test_stale_table_settings(cleanup_databases)); + result += test_report("Stale Table Settings Dropped Meta:", do_test_stale_table_settings_dropped_meta(cleanup_databases)); + result += test_report("DBVersion Rebuild Error:", do_test_dbversion_rebuild_error()); + result += test_report("Uninit Error Messages:", do_test_uninit_error_messages()); + result += test_report("Block LWW Existing Data:", do_test_block_lww_existing_data(cleanup_databases)); + result += test_report("Block Column Reload:", do_test_block_column_reload(cleanup_databases)); + result += test_report("CB Error Cleanup:", do_test_context_cb_error_cleanup()); + finalize: - printf("\n"); if (rc != SQLITE_OK) printf("%s (%d)\n", (db) ? sqlite3_errmsg(db) : "N/A", rc); - db = close_db(db); + close_db(db); + db = NULL; cloudsync_memory_finalize(); - sqlite3_int64 memory_used = sqlite3_memory_used(); + int64_t memory_used = (int64_t)sqlite3_memory_used(); + result += test_report("Memory Leaks Check:", memory_used == 0); if (memory_used > 0) { - printf("Memory leaked: %lld B\n", memory_used); + printf("\tleaked: %" PRId64 " B\n", memory_used); result++; }