diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed6769c34..642a7a3f8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -105,7 +105,15 @@ jobs: run: | docker pull cnosdb/cnosdb:community-latest docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest - until nc -z 127.0.0.1 8902 2>/dev/null; do sleep 1; done + for i in $(seq 1 30); do + if curl -sf -u "root:" -H "Content-Type: application/json" \ + "http://127.0.0.1:8902/api/v1/sql?db=public" -d "SELECT 1" >/dev/null 2>&1; then + echo "CnosDB is ready" + break + fi + echo "Waiting for CnosDB... ($i/30)" + sleep 10 + done - name: Run Tests run: | CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test diff --git a/src/sqlancer/cnosdb/CnosDBProvider.java b/src/sqlancer/cnosdb/CnosDBProvider.java index 8b69c53b3..1a69c5655 100644 --- a/src/sqlancer/cnosdb/CnosDBProvider.java +++ b/src/sqlancer/cnosdb/CnosDBProvider.java @@ -12,6 +12,7 @@ import sqlancer.StatementExecutor; import sqlancer.cnosdb.client.CnosDBClient; import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.cnosdb.client.CnosDBException; import sqlancer.cnosdb.gen.CnosDBInsertGenerator; import sqlancer.cnosdb.gen.CnosDBTableGenerator; import sqlancer.cnosdb.query.CnosDBOtherQuery; @@ -21,6 +22,9 @@ @AutoService(DatabaseProvider.class) public class CnosDBProvider extends ProviderAdapter { + private static final int DB_READY_RETRIES = 30; + private static final int DB_READY_SLEEP_MS = 1000; + protected String username; protected String password; protected String host; @@ -66,13 +70,17 @@ public CnosDBConnection createDatabase(CnosDBGlobalState globalState) throws Exc host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); databaseName = globalState.getDatabaseName(); - CnosDBClient client = new CnosDBClient(host, port, username, password, databaseName); - CnosDBConnection connection = new CnosDBConnection(client); - client.execute("DROP DATABASE IF EXISTS " + databaseName); + // Use a client connected to "public" for DROP/CREATE operations, + // since CnosDB cannot drop a database from its own connection context. + CnosDBClient adminClient = new CnosDBClient(host, port, username, password, "public"); + dropAndCreateDatabase(adminClient, databaseName); globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - client.execute("CREATE DATABASE " + databaseName); globalState.getState().logStatement("CREATE DATABASE " + databaseName); + adminClient.close(); + CnosDBClient client = new CnosDBClient(host, port, username, password, databaseName); + executeWithRetry(client, "SELECT 1"); + CnosDBConnection connection = new CnosDBConnection(client); return connection; } @@ -94,6 +102,54 @@ protected void prepareTables(CnosDBGlobalState globalState) throws Exception { se.executeStatements(); } + /** + * Drops and recreates a database, retrying the entire DROP+CREATE sequence on transient errors. + * + * CnosDB's Tskv index storage may not be fully ready even after the HTTP API responds to simple queries. DDL + * operations can fail with: {@code "grpc client request error: Tskv: Index: index storage error: Resource + * temporarily unavailable (os error 11)"}. DROP DATABASE may also appear to succeed but not yet propagate, causing + * CREATE DATABASE to fail with {@code "Database already exists"}. Retrying just the CREATE is futile in that case, + * so we retry the DROP+CREATE pair together. + */ + private static void dropAndCreateDatabase(CnosDBClient client, String databaseName) throws Exception { + String dropQuery = "DROP DATABASE IF EXISTS " + databaseName; + String createQuery = "CREATE DATABASE " + databaseName; + for (int i = 0; i < DB_READY_RETRIES; i++) { + try { + client.execute(dropQuery); + client.execute(createQuery); + return; + } catch (CnosDBException e) { + boolean isRetryable = e.getMessage().contains("Resource temporarily unavailable") + || e.getMessage().contains("already exists"); + if (!isRetryable || i == DB_READY_RETRIES - 1) { + throw e; + } + Thread.sleep(DB_READY_SLEEP_MS); + } + } + } + + /** + * Executes a query with retries to work around CnosDB storage layer initialization delays. + * + * A newly created database may not be immediately queryable, causing {@code "Database not found"} errors. + */ + private static void executeWithRetry(CnosDBClient client, String query) throws Exception { + for (int i = 0; i < DB_READY_RETRIES; i++) { + try { + client.execute(query); + return; + } catch (CnosDBException e) { + boolean isRetryable = e.getMessage().contains("Database not found"); + if (!isRetryable || i == DB_READY_RETRIES - 1) { + throw e; + } + Thread.sleep(DB_READY_SLEEP_MS); + } + } + } + @Override public String getDBMSName() { return "CnosDB".toLowerCase(); diff --git a/test/sqlancer/dbms/TestCnosDBNoREC.java b/test/sqlancer/dbms/TestCnosDBNoREC.java index 1a89a972a..42ece0733 100644 --- a/test/sqlancer/dbms/TestCnosDBNoREC.java +++ b/test/sqlancer/dbms/TestCnosDBNoREC.java @@ -15,8 +15,8 @@ public void testCnosDBNoREC() { // Run with 0 queries as current implementation is resulting in database crashes assertEquals(0, Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", - "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", - "--oracle", "NOREC" })); + "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", + "--num-threads", "1", "cnosdb", "--oracle", "NOREC" })); } } diff --git a/test/sqlancer/dbms/TestCnosDBTLP.java b/test/sqlancer/dbms/TestCnosDBTLP.java index 4b12aa409..2015e3af9 100644 --- a/test/sqlancer/dbms/TestCnosDBTLP.java +++ b/test/sqlancer/dbms/TestCnosDBTLP.java @@ -15,8 +15,8 @@ public void testCnosDBTLP() { // Run with 0 queries as current implementation is resulting in database crashes assertEquals(0, Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", - "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", - "--oracle", "QUERY_PARTITIONING" })); + "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", + "--num-threads", "1", "cnosdb", "--oracle", "QUERY_PARTITIONING" })); } }