#pragma clang diagnostic ignored "-Wdisabled-macro-expansion" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if USE_BUZZHOUSE #include #include #include #include #include namespace BuzzHouse { extern void loadFuzzerServerSettings(const FuzzConfig & fc); } #endif namespace DB { namespace Setting { extern const SettingsDialect dialect; } namespace ErrorCodes { extern const int CANNOT_PARSE_TEXT; extern const int NOT_IMPLEMENTED; extern const int SYNTAX_ERROR; extern const int MEMORY_LIMIT_EXCEEDED; extern const int TOO_DEEP_RECURSION; extern const int BUZZHOUSE; using ErrorCode = int; extern std::string_view getName(ErrorCode error_code); } bool Client::tryToReconnect(const uint32_t max_reconnection_attempts, const uint32_t time_to_sleep_between_reconnects) { chassert(max_reconnection_attempts); if (!connection->isConnected()) { // Try to reconnect after errors, for two reasons: // 1. We might not have realized that the server died, e.g. if // it sent us a trace and closed connection properly. // 2. The connection might have gotten into a wrong state and // the next query will get false positive about // "Unknown packet from server". for (uint32_t i = 0; i < max_reconnection_attempts; i++) { try { connection->forceConnected(connection_parameters.timeouts); break; } catch (...) { // Just report it, we'll terminate below. fmt::print(stderr, "Error while reconnecting to the server: {}\n", getCurrentExceptionMessage(true)); // The reconnection might fail, but we'll still be connected // in the sense of `connection->isConnected() = true`, // in case when the requested database doesn't exist. // Disconnect manually now, so that the following code doesn't // have any doubts, and the connection state is predictable. connection->disconnect(); if (i < max_reconnection_attempts - 1) { std::this_thread::sleep_for(std::chrono::milliseconds(time_to_sleep_between_reconnects)); } } } } if (!connection->isConnected()) { // Probably the server is dead because we found an assertion // failure. Fail fast. fmt::print(stderr, "Lost connection to the server.\n"); // Print the changed settings because they might be needed to // reproduce the error. printChangedSettings(); return false; } return true; } bool Client::processASTFuzzerStep(const String & query_to_execute, const ASTPtr & parsed_query) { bool async_insert = false; processParsedSingleQuery(query_to_execute, parsed_query, async_insert); const auto * exception = server_exception ? server_exception.get() : client_exception.get(); // Sometimes you may get TOO_DEEP_RECURSION from the server, // and TOO_DEEP_RECURSION should not fail the fuzzer check. // Similarly, MEMORY_LIMIT_EXCEEDED means the server correctly // rejected an expensive query, not that it died. if (have_error && (exception->code() == ErrorCodes::TOO_DEEP_RECURSION || exception->code() == ErrorCodes::MEMORY_LIMIT_EXCEEDED)) { have_error = false; server_exception.reset(); client_exception.reset(); return true; } if (have_error) { fmt::print(stderr, "Error on processing query '{}': {}\n", parsed_query->formatForErrorMessage(), exception->message()); } return tryToReconnect(1, 10); } /// Returns false when server is not available. bool Client::processWithASTFuzzer(std::string_view full_query) { ASTPtr orig_ast; try { const char * begin = full_query.data(); orig_ast = parseQuery( begin, begin + full_query.size(), client_context->getSettingsRef(), /*allow_multi_statements=*/true); } catch (const Exception & e) { if (e.code() != ErrorCodes::SYNTAX_ERROR && e.code() != ErrorCodes::TOO_DEEP_RECURSION) throw; } if (!orig_ast) { // Can't continue after a parsing error return true; } // `USE db` should not be executed // since this will break every query after `DROP db` if (orig_ast->as()) { return true; } // Kusto is not a subject for fuzzing (yet) if (client_context->getSettingsRef()[Setting::dialect] == DB::Dialect::kusto) { return true; } if (auto * q = orig_ast->as()) { if (auto * set_dialect = q->changes.tryGet("dialect"); set_dialect && set_dialect->safeGet() == "kusto") return true; } // Don't repeat: // - INSERT -- Because the tables may grow too big. // - CREATE -- Because first we run the unmodified query, it will succeed, // and the subsequent queries will fail. // When we run out of fuzzer errors, it may be interesting to // add fuzzing of create queries that wraps columns into // LowCardinality or Nullable. // Also there are other kinds of create queries such as CREATE // DICTIONARY, we could fuzz them as well. // - DROP -- No point in this (by the same reasons). // - SET -- The time to fuzz the settings has not yet come // (see comments in Client/QueryFuzzer.cpp) size_t this_query_runs = query_fuzzer_runs; ASTs queries_for_fuzzed_tables; if (orig_ast->as()) { this_query_runs = 1; } else if (const auto * create = orig_ast->as()) { if (QueryFuzzer::isSuitableForFuzzing(*create)) this_query_runs = create_query_fuzzer_runs; else this_query_runs = 1; } else if (const auto * /*insert*/ _ = orig_ast->as()) { this_query_runs = 1; queries_for_fuzzed_tables = fuzzer.getQueriesForFuzzedTables(full_query); } else if (const auto * /*optimize*/ _ = orig_ast->as()) { this_query_runs = 1; queries_for_fuzzed_tables = fuzzer.getQueriesForFuzzedTables(full_query); } else if (const auto * drop = orig_ast->as()) { this_query_runs = 1; queries_for_fuzzed_tables = fuzzer.getDropQueriesForFuzzedTables(*drop); } String query_to_execute; ASTPtr fuzz_base = orig_ast; #if USE_BUZZHOUSE BuzzHouse::PerformanceResult res1; BuzzHouse::PerformanceResult res2; const bool can_compare = fuzz_config && (fuzz_config->measure_performance || fuzz_config->compare_success_results) && external_integrations && external_integrations->hasClickHouseExtraServerConnection(); const bool try_measure_performance_in_loop = can_compare && fuzz_config->measure_performance && (orig_ast->as() || orig_ast->as()); auto insert_into = make_intrusive(); insert_into->table_function = makeASTFunction("file", make_intrusive("/dev/null"), make_intrusive("CSV")); #endif for (size_t fuzz_step = 0; fuzz_step < this_query_runs; ++fuzz_step) { #if USE_BUZZHOUSE bool peer_success = true; bool measure_performance = try_measure_performance_in_loop; ASTPtr old_settings = nullptr; ASTSelectQuery * select_query = nullptr; #endif fmt::print(stderr, "Fuzzing step {} out of {}\n", fuzz_step, this_query_runs); ASTPtr ast_to_process; try { auto base_before_fuzz = fuzz_base->formatForErrorMessage(); ast_to_process = fuzz_base->clone(); // Run the original query as well. if (fuzz_step > 0) { fuzzer.fuzzMain(ast_to_process); } query_to_execute = ast_to_process->formatForErrorMessage(); if (fuzz_step > 0 && query_to_execute == base_before_fuzz) { fmt::print(stderr, "Got boring AST\n"); continue; } #if USE_BUZZHOUSE if (measure_performance) { /// Add tag to find query later on auto * union_sel = ast_to_process->as(); if ((select_query = typeid_cast(union_sel ? union_sel->list_of_selects->children[0].get() : ast_to_process.get()))) { if (!select_query->settings()) { auto settings_query = make_intrusive(); SettingsChanges settings_changes; settings_changes.setSetting("log_comment", "measure_performance"); /// Sometimes change settings fuzzer.getRandomSettings(settings_changes); settings_query->changes = std::move(settings_changes); settings_query->is_standalone = false; select_query->setExpression(ASTSelectQuery::Expression::SETTINGS, std::move(settings_query)); } else { auto * set_query = select_query->settings()->as(); old_settings = set_query->clone(); set_query->changes.setSetting("log_comment", "measure_performance"); fuzzer.getRandomSettings(set_query->changes); } /// Dump into /dev/null, we are not interested in sending the results back to the client insert_into->select = ast_to_process; ast_to_process = insert_into; query_to_execute = ast_to_process->formatForErrorMessage(); } else { measure_performance = false; } } #endif #if 0 /// Somehow this code is not running /// `base_after_fuzz` should format from `ast_to_process` WriteBufferFromOwnString dump_before_fuzz; fuzz_base->dumpTree(dump_before_fuzz); auto base_after_fuzz = fuzz_base->formatForErrorMessage(); // Check that the source AST didn't change after fuzzing. This // helps debug AST cloning errors, where the cloned AST doesn't // clone all its children, and erroneously points to some source // child elements. if (base_before_fuzz != base_after_fuzz) { printChangedSettings(); fmt::print( stderr, "Base before fuzz: {}\n" "Base after fuzz: {}\n", base_before_fuzz, base_after_fuzz); fmt::print(stderr, "Dump before fuzz:\n{}\n", dump_before_fuzz.str()); fmt::print(stderr, "Dump of cloned AST:\n{}\n", dump_of_cloned_ast.str()); fmt::print(stderr, "Dump after fuzz:\n"); WriteBufferFromOStream cerr_buf(std::cerr, 4096); fuzz_base->dumpTree(cerr_buf); cerr_buf.finalize(); fmt::print( stderr, "Found error: IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its " "cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent " "node doesn't implement clone() correctly."); _exit(1); } #endif fmt::print(stdout, "Dump of fuzzed AST:\n{}\n", query_to_execute); if (const auto * insert_ast = ast_to_process->as(); insert_ast && insert_ast->hasInlinedData()) { /// Print insert data String bytes; auto read_buf = getReadBufferFromASTInsertQuery(ast_to_process); WriteBufferFromString write_buf(bytes); copyData(*read_buf, write_buf); fmt::print(stdout, "{}\n", bytes); } const auto res = processASTFuzzerStep(query_to_execute, ast_to_process); if (!res) return res; #if USE_BUZZHOUSE if (measure_performance) { /// Don't keep insert into in the AST ast_to_process = insert_into->select; /// Don't keep performance settings in AST if (select_query && old_settings) { select_query->setExpression(ASTSelectQuery::Expression::SETTINGS, std::move(old_settings)); } else if (select_query) { select_query->setExpression(ASTSelectQuery::Expression::SETTINGS, {}); } } #endif } catch (...) { if (!ast_to_process) fmt::print(stderr, "Error while forming new query: {}\n", getCurrentExceptionMessage(true)); // Some functions (e.g. protocol parsers) don't throw, but // set last_exception instead, so we'll also do it here for // uniformity. // Surprisingly, this is a client exception, because we get the // server exception w/o throwing (see onReceiveException()). client_exception = std::make_unique(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode()); have_error = true; } #if USE_BUZZHOUSE measure_performance &= !have_error; if (measure_performance) { measure_performance &= external_integrations->getPerformanceMetricsForLastQuery(BuzzHouse::PeerTableDatabase::None, res1); /// Replicate settings, so both servers have same configuration external_integrations->replicateSettings(BuzzHouse::PeerTableDatabase::ClickHouse); } if (can_compare) { /// Always run query on peer server fmt::print(stdout, "Running query on peer server\n"); peer_success &= !external_integrations->performQuery(BuzzHouse::PeerTableDatabase::ClickHouse, query_to_execute); } if (can_compare && fuzz_config->compare_success_results && peer_success != !have_error) { throw DB::Exception(DB::ErrorCodes::BUZZHOUSE, "AST Fuzzer: The peer server got a different success result"); } if (measure_performance) { measure_performance &= peer_success && external_integrations->getPerformanceMetricsForLastQuery(BuzzHouse::PeerTableDatabase::ClickHouse, res2); if (measure_performance) { fuzz_config->comparePerformanceResults("AST fuzzer", res1, res2); } } #endif // The server is still alive, so we're going to continue fuzzing. // Determine what we're going to use as the starting AST. if (have_error) { // Query completed with error, keep the previous starting AST. // Also discard the exception that we now know to be non-fatal, // so that it doesn't influence the exit code. server_exception.reset(); client_exception.reset(); fuzzer.notifyQueryFailed(ast_to_process); have_error = false; } else if (ast_to_process->formatForErrorMessage().size() > 2000) { // ast too long, start from original ast fmt::print(stderr, "Current AST is too long, discarding it and using the original AST as a start\n"); fuzz_base = orig_ast; } else { // fuzz starting from this successful query fmt::print(stderr, "Query succeeded, using this AST as a start\n"); fuzz_base = ast_to_process; } } for (const auto & query : queries_for_fuzzed_tables) { std::cout << std::endl; std::cout << query->formatWithSecretsOneLine() << std::endl; if (const auto * insert = query->as()) { /// For inserts with data it's really useful to have the data itself available in the logs if (insert->hasInlinedData()) { String bytes; { auto read_buf = getReadBufferFromASTInsertQuery(query); WriteBufferFromString write_buf(bytes); copyData(*read_buf, write_buf); } std::cout << bytes; } } std::cout << std::endl << std::endl; try { query_to_execute = query->formatForErrorMessage(); const auto res = processASTFuzzerStep(query_to_execute, query); if (!res) return res; } catch (...) { client_exception = std::make_unique(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode()); have_error = true; } if (have_error) { server_exception.reset(); client_exception.reset(); fuzzer.notifyQueryFailed(query); have_error = false; } #if USE_BUZZHOUSE if (can_compare) { const auto u = external_integrations->performQuery(BuzzHouse::PeerTableDatabase::ClickHouse, query_to_execute); UNUSED(u); } #endif } return true; } #if USE_BUZZHOUSE bool Client::processBuzzHouseQuery(const String & full_query) { static constexpr size_t max_query_bytes = 1 << 20; bool server_up = true; have_error = false; error_code = 0; if (full_query.size() > max_query_bytes) { have_error = true; error_code = ErrorCodes::CANNOT_PARSE_TEXT; LOG_WARNING(fuzz_config->log, "Skipping oversized query ({} bytes, limit {})", full_query.size(), max_query_bytes); } else if (!processQueryText(full_query)) { have_error = true; error_code = ErrorCodes::CANNOT_PARSE_TEXT; } if (error_code > 0) { if (fuzz_config->disallowed_error_codes.contains(error_code)) { throw Exception(ErrorCodes::BUZZHOUSE, "Found disallowed error code {} - {}", error_code, ErrorCodes::getName(error_code)); } server_up &= tryToReconnect(fuzz_config->max_reconnection_attempts, fuzz_config->time_to_sleep_between_reconnects); } return server_up; } bool Client::fuzzLoopReconnect() { connection->disconnect(); return tryToReconnect(fuzz_config->max_reconnection_attempts, fuzz_config->time_to_sleep_between_reconnects); } static void runExternalCommand( std::unique_ptr & external_integrations, const uint64_t seed, const bool async, const String & engine, const String & cname, const String & tname) { if (!external_integrations->performExternalCommand(seed, async, BuzzHouse::IntegrationCall::Dolor, engine, cname, tname)) { throw Exception(ErrorCodes::BUZZHOUSE, "External command failed for {} on catalog {}", tname, cname); } } static const String & restart_cmd = "--Reconnecting client"; static const String & external_cmd = "--External command "; static const String & health_check_cmd = "--Health check"; /// Encode a string as uppercase hex so it contains no whitespace or dots, /// making it safe to embed in the one-line external-command replay marker. static String markerHexEncode(const String & s) { static const char hex_digits[] = "0123456789ABCDEF"; String result; result.reserve(s.size() * 2); for (const unsigned char c : s) { result += hex_digits[c >> 4]; result += hex_digits[c & 0xF]; } return result; } /// Decode a hex string written by markerHexEncode. static String markerHexDecode(const String & s) { if (s.size() % 2 != 0) throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "markerHexDecode: odd-length input '{}'", s); auto nibble = [&](const char c) -> uint8_t { if (c >= '0' && c <= '9') return static_cast(c - '0'); if (c >= 'A' && c <= 'F') return static_cast(c - 'A' + 10); if (c >= 'a' && c <= 'f') return static_cast(c - 'a' + 10); throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "markerHexDecode: invalid hex character '{}' in '{}'", c, s); }; String result; result.reserve(s.size() / 2); for (size_t i = 0; i < s.size(); i += 2) result += static_cast((nibble(s[i]) << 4) | nibble(s[i + 1])); return result; } /// Returns false when server is not available. bool Client::buzzHouse() { String full_query; bool no_eof = true; bool server_up = true; bool no_timeout = true; static const String & rerun_database = "--External database "; static const RE2 rerun_database_re(R"((?i)^--External\s+database\s+(.*)$)"); static const String & rerun_table = "--External table "; static const RE2 rerun_table_re(R"((?i)^--External\s+table\s+(.*)$)"); static const RE2 extern_re( R"((?i)^--External\s+command\s+(?:(async)\s+)?with\s+seed\s+(\d+)\s+to\s+([^\s]+)\s+table\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)\s*$)"); /// Set time to run, but what if a query runs for too long? using clock = std::chrono::steady_clock; const auto deadline = fuzz_config->time_to_run > 0 ? std::optional(clock::now() + std::chrono::minutes(fuzz_config->time_to_run)) : std::nullopt; full_query.reserve(8192); if (fuzz_config->read_log) { std::ifstream infile(fuzz_config->log_path); while (server_up && (no_timeout = (!deadline || clock::now() < *deadline)) && (no_eof = static_cast(std::getline(infile, full_query)))) { String async_flag; String seed_str; String engine; String database; String table; if (full_query == restart_cmd) { server_up &= fuzzLoopReconnect(); } else if (startsWith(full_query, rerun_database) && RE2::FullMatch(full_query, rerun_database_re, &database)) { const auto x = external_integrations->reRunCreateDatabase(BuzzHouse::IntegrationCall::Dolor, database); UNUSED(x); } else if (startsWith(full_query, rerun_table) && RE2::FullMatch(full_query, rerun_table_re, &table)) { const auto x = external_integrations->reRunCreateTable(BuzzHouse::IntegrationCall::Dolor, table); UNUSED(x); } else if ( startsWith(full_query, external_cmd) && RE2::FullMatch(full_query, extern_re, &async_flag, &seed_str, &engine, &database, &table)) { uint64_t seed = 0; const auto * const first = seed_str.data(); const auto * const last = first + seed_str.size(); const auto x = std::from_chars(first, last, seed, 10); if (x.ec != std::errc{} || x.ptr != last) throw DB::Exception( DB::ErrorCodes::BUZZHOUSE, "Malformed external-command marker: cannot parse seed '{}' ({})", seed_str, x.ec == std::errc::result_out_of_range ? "out of range" : "invalid characters"); runExternalCommand( external_integrations, seed, !async_flag.empty(), engine, markerHexDecode(database), markerHexDecode(table)); } else if (startsWith(full_query, health_check_cmd)) { fuzz_config->validateClickHouseHealth(); } else { server_up &= processBuzzHouseQuery(full_query); } full_query.resize(0); } } else { String full_query2; std::vector peer_queries; bool has_cloud_features = true; BuzzHouse::RandomGenerator rg( fuzz_config->seed, fuzz_config->min_string_length, fuzz_config->max_string_length, fuzz_config->random_limited_values); BuzzHouse::SQLQuery sq1; BuzzHouse::SQLQuery sq2; BuzzHouse::SQLQuery sq3; BuzzHouse::SQLQuery sq4; std::vector intermediate_queries; uint32_t nsuccessfull_create_database = 0; uint32_t total_create_database_tries = 0; const uint32_t max_initial_databases = std::min(UINT32_C(3), fuzz_config->max_databases); uint32_t nsuccessfull_create_table = 0; uint32_t total_create_table_tries = 0; const uint32_t max_initial_tables = std::min(UINT32_C(10), fuzz_config->max_tables); GOOGLE_PROTOBUF_VERIFY_VERSION; has_cloud_features &= processTextAsSingleQuery("DROP DATABASE IF EXISTS fuzztest;"); has_cloud_features &= processTextAsSingleQuery("CREATE DATABASE fuzztest Engine=Shared;"); std::cout << "Cloud features " << (has_cloud_features ? "" : "not ") << "detected" << std::endl; const auto u = processTextAsSingleQuery("DROP DATABASE IF EXISTS fuzztest;"); UNUSED(u); fuzz_config->outf << "--Session seed: " << rg.getSeed() << std::endl; /// Load server configurations for the fuzzer fuzz_config->loadServerConfigurations(); loadFuzzerServerSettings(*fuzz_config); loadFuzzerTableSettings(*fuzz_config); loadSystemTables(*fuzz_config); if (fuzz_config->allow_client_restarts && fuzz_config->allow_query_oracles) { /// Create a dedicated oracle user and role for the row policy oracle. /// Row policies are created with `TO ` so they apply only to members /// of the role and do not affect the default admin session used for sq2. /// The oracle uses "EXECUTE AS " (allowed by the default /// access_control_improvements.allow_impersonate_user = true) to run sq1 with /// the row policy active. static const DB::Strings queries = { "CREATE USER IF NOT EXISTS " + BuzzHouse::FuzzConfig::oracleUser + " IDENTIFIED WITH no_password;", "CREATE ROLE IF NOT EXISTS " + BuzzHouse::FuzzConfig::oracleRole + ";", "GRANT SELECT ON *.* TO " + BuzzHouse::FuzzConfig::oracleRole + ";", "GRANT " + BuzzHouse::FuzzConfig::oracleRole + " TO " + BuzzHouse::FuzzConfig::oracleUser + ";", }; for (const String & q : queries) { fuzz_config->outf << q << std::endl; server_up &= processBuzzHouseQuery(q); } } full_query2.reserve(8192); BuzzHouse::StatementGenerator gen(rg, *fuzz_config, *external_integrations, has_cloud_features); BuzzHouse::QueryOracle qo(*fuzz_config); while (server_up && (no_timeout = (!deadline || clock::now() < *deadline))) { sq1.Clear(); full_query.resize(0); if (total_create_database_tries < 20 && nsuccessfull_create_database < max_initial_databases) { gen.generateNextCreateDatabase( rg, sq1.mutable_single_query()->mutable_explain()->mutable_inner_query()->mutable_create_database()); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); gen.updateGenerator(sq1, *external_integrations, !have_error); nsuccessfull_create_database += (have_error ? 0 : 1); total_create_database_tries++; } else if ( gen.collectionHas>(gen.attached_databases) && total_create_table_tries < 300 && nsuccessfull_create_table < max_initial_tables) { gen.generateNextCreateTable( rg, false, sq1.mutable_single_query()->mutable_explain()->mutable_inner_query()->mutable_create_table()); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); gen.updateGenerator(sq1, *external_integrations, !have_error); nsuccessfull_create_table += (have_error ? 0 : 1); total_create_table_tries++; } else { auto runDumpReadOracle = [&](auto dumpContent, auto dumpIntermediate, const char * oracle_name) { qo.resetOracleValues(); BuzzHouse::DumpOracleStrategy strategy = BuzzHouse::DumpOracleStrategy::REATTACH; rg.pickWeighted( {{20, [&]() { strategy = BuzzHouse::DumpOracleStrategy::REATTACH; }}, {5, [&]() { strategy = BuzzHouse::DumpOracleStrategy::BACKUP_RESTORE; }}}); full_query.resize(0); dumpContent(); full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processFirstOracleQueryResult(error_code, *external_integrations); dumpIntermediate(strategy); for (const auto & entry : intermediate_queries) { full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, entry); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.setIntermediateStepSuccess(!have_error); } full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq2); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processSecondOracleQueryResult(error_code, *external_integrations, oracle_name); }; rg.pickWeighted({ {20 * static_cast(fuzz_config->allow_query_oracles), [&]() { qo.resetOracleValues(); qo.generateCorrectnessTestFirstQuery(rg, gen, sq1); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processFirstOracleQueryResult(error_code, *external_integrations); sq2.Clear(); full_query.resize(0); qo.generateCorrectnessTestSecondQuery(sq1, sq2); BuzzHouse::SQLQueryToString(full_query, sq2); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processSecondOracleQueryResult(error_code, *external_integrations, "Correctness query"); }}, {30 * static_cast(fuzz_config->allow_query_oracles), [&]() { /// Test running query with different settings, but some times, call system commands qo.resetOracleValues(); const bool use_settings = qo.generateFirstSetting(rg, sq1); if (use_settings) { /// Run query only when something was generated BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.setIntermediateStepSuccess(!have_error); } sq2.Clear(); full_query.resize(0); qo.generateOracleSelectQuery(rg, BuzzHouse::PeerQuery::None, gen, sq2); BuzzHouse::SQLQueryToString(full_query, sq2); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processFirstOracleQueryResult(error_code, *external_integrations); sq3.Clear(); full_query.resize(0); qo.generateSecondSetting(rg, gen, use_settings, sq1, sq3); BuzzHouse::SQLQueryToString(full_query, sq3); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.setIntermediateStepSuccess(!have_error); sq4.Clear(); full_query.resize(0); qo.maybeUpdateOracleSelectQuery(rg, gen, sq2, sq4); BuzzHouse::SQLQueryToString(full_query, sq4); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processSecondOracleQueryResult(error_code, *external_integrations, "Multi setting query"); }}, {10 * static_cast( fuzz_config->allow_query_oracles && fuzz_config->use_dump_table_oracle > 0 && gen.collectionHas(gen.attached_tables_to_test_format)), [&]() { /// Test in and out formats /// When testing content, we have to export and import to the same table qo.resetOracleValues(); const bool test_content = fuzz_config->use_dump_table_oracle > 1 && rg.nextBool() && gen.collectionHas(gen.attached_tables_to_compare_content); const auto & tbl = rg.pickRandomly(gen.filterCollection( test_content ? gen.attached_tables_to_compare_content : gen.attached_tables_to_test_format)); const bool is_mt = tbl.get().isMergeTreeFamily(); BuzzHouse::DumpOracleStrategy strategy = BuzzHouse::DumpOracleStrategy::DO_NOTHING; rg.pickWeighted( {{15 * static_cast(test_content && tbl.get().can_run_merges), [&]() { strategy = BuzzHouse::DumpOracleStrategy::OPTIMIZE; }}, {25 * static_cast(test_content), [&]() { strategy = BuzzHouse::DumpOracleStrategy::REATTACH; }}, {10 * static_cast(fuzz_config->enable_backups && test_content), [&]() { strategy = BuzzHouse::DumpOracleStrategy::BACKUP_RESTORE; }}, {40 * static_cast(test_content), [&]() { strategy = BuzzHouse::DumpOracleStrategy::ALTER_TABLE; }}, {20 * static_cast(test_content), [&]() { strategy = BuzzHouse::DumpOracleStrategy::ALTER_UPDATE; }}, {25 * static_cast(test_content && tbl.get().areInsertsAppends(true)), [&]() { strategy = BuzzHouse::DumpOracleStrategy::INSERT_COUNT; }}, {10 * static_cast(fuzz_config->enable_renames && test_content), [&]() { strategy = BuzzHouse::DumpOracleStrategy::RENAME_BACK; }}, {15 * static_cast(test_content && is_mt), [&]() { strategy = BuzzHouse::DumpOracleStrategy::FREEZE_UNFREEZE; }}, {10 * static_cast(test_content && is_mt), [&]() { strategy = BuzzHouse::DumpOracleStrategy::MOVE_PARTITION; }}, {10 * static_cast(test_content && is_mt), [&]() { strategy = BuzzHouse::DumpOracleStrategy::REPLACE_PARTITION; }}, {15 * static_cast(test_content), [&]() { strategy = BuzzHouse::DumpOracleStrategy::ALTER_COLUMN; }}, {3 * static_cast( test_content && !tbl.get().isAnyS3Engine(true) && !tbl.get().isAnyAzureEngine(true)), [&]() { strategy = BuzzHouse::DumpOracleStrategy::TRUNCATE_COUNT; }}, {70 * static_cast(!tbl.get().isNotTruncableEngine()), [&]() { strategy = BuzzHouse::DumpOracleStrategy::REINSERT_TABLE; }}, {1, [&]() { /* Defensive line */ }}}); if (strategy != BuzzHouse::DumpOracleStrategy::DO_NOTHING) { if (test_content) { /// Dump table content and read it later to look for correctness full_query.resize(0); qo.dumpTableContent(rg, gen, strategy, test_content, tbl, sq1, sq2); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processFirstOracleQueryResult(error_code, *external_integrations); } qo.dumpOracleIntermediateSteps(rg, gen, tbl, strategy, test_content, intermediate_queries); for (const auto & entry : intermediate_queries) { /// Run each from the chosen strategy full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, entry); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.setIntermediateStepSuccess(!have_error); } if (test_content) { full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq2); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processSecondOracleQueryResult(error_code, *external_integrations, "Dump and read table"); } } }}, {5 * static_cast( fuzz_config->allow_query_oracles && fuzz_config->use_dump_table_oracle > 1 && gen.collectionHas(gen.attached_dictionaries_to_compare_content)), [&]() { const auto & dict = rg.pickRandomly( gen.filterCollection(gen.attached_dictionaries_to_compare_content)); BuzzHouse::SQLQuery reload; runDumpReadOracle( [&]() { qo.dumpDictionaryContent(rg, gen, dict, reload, sq1, sq2); full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, reload); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); }, [&](auto s) { qo.dumpObjectIntermediateSteps(rg, gen, dict, BuzzHouse::SQLObject::DICTIONARY, s, intermediate_queries); }, "Dump and read dictionary"); }}, {5 * static_cast( fuzz_config->allow_query_oracles && fuzz_config->use_dump_table_oracle > 1 && gen.collectionHas(gen.attached_views_to_compare_content)), [&]() { const auto & view = rg.pickRandomly(gen.filterCollection(gen.attached_views_to_compare_content)); runDumpReadOracle( [&]() { qo.dumpViewContent(rg, view, sq1, sq2); }, [&](auto s) { qo.dumpObjectIntermediateSteps(rg, gen, view, BuzzHouse::SQLObject::VIEW, s, intermediate_queries); }, "Dump and read view"); }}, {20 * static_cast( fuzz_config->allow_query_oracles && gen.collectionHas(gen.attached_tables_for_table_peer_oracle)), [&]() { /// Test results with peer tables qo.resetOracleValues(); int err_res = 0; BuzzHouse::PeerQuery nquery = ((!external_integrations->hasMySQLConnection() && !external_integrations->hasPostgreSQLConnection() && !external_integrations->hasSQLiteConnection()) || rg.nextBool()) && gen.collectionHas(gen.attached_tables_for_clickhouse_table_peer_oracle) ? BuzzHouse::PeerQuery::ClickHouseOnly : BuzzHouse::PeerQuery::AllPeers; const bool clickhouse_only = nquery == BuzzHouse::PeerQuery::ClickHouseOnly; sq2.Clear(); qo.generateOracleSelectQuery(rg, nquery, gen, sq1); qo.replaceQueryWithTablePeers(rg, sq1, gen, peer_queries, sq2); if (clickhouse_only) { external_integrations->replicateSettings(BuzzHouse::PeerTableDatabase::ClickHouse); } qo.truncatePeerTables(gen); for (const auto & entry : peer_queries) { full_query2.resize(0); BuzzHouse::SQLQueryToString(full_query2, entry); fuzz_config->outf << full_query2 << std::endl; server_up &= processBuzzHouseQuery(full_query2); qo.setIntermediateStepSuccess(!have_error); } qo.optimizePeerTables(gen); full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processFirstOracleQueryResult(error_code, *external_integrations); full_query2.resize(0); BuzzHouse::SQLQueryToString(full_query2, sq2); fuzz_config->outf << full_query2 << std::endl; if (clickhouse_only) { err_res = external_integrations->performQuery(BuzzHouse::PeerTableDatabase::ClickHouse, full_query2); } else { server_up &= processBuzzHouseQuery(full_query2); err_res = error_code; } qo.processSecondOracleQueryResult(err_res, *external_integrations, "Peer table query"); }}, {10 * static_cast(fuzz_config->allow_query_oracles), [&]() { /// Roundtrip oracle: check that encode/decode and encrypt/decrypt preserve data qo.resetOracleValues(); sq2.Clear(); qo.generateRoundtripOracleQueries(rg, gen, sq1, sq2); full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processFirstOracleQueryResult(error_code, *external_integrations); full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq2); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processSecondOracleQueryResult(error_code, *external_integrations, "Roundtrip oracle"); }}, {20 * static_cast(fuzz_config->allow_query_oracles), [&]() { /// ARRAY JOIN oracle: ARRAY JOIN clause vs arrayJoin function qo.resetOracleValues(); sq2.Clear(); qo.generateArrayJoinOracleQueries(rg, gen, sq1, sq2); full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processFirstOracleQueryResult(error_code, *external_integrations); full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq2); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processSecondOracleQueryResult(error_code, *external_integrations, "Array join oracle"); }}, {20 * static_cast(fuzz_config->allow_query_oracles), [&]() { /// COUNT(DISTINCT col) oracle: uniqExact aggregator vs DISTINCT + COUNT qo.resetOracleValues(); qo.generateCountDistinctFirstQuery(rg, gen, sq1); full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processFirstOracleQueryResult(error_code, *external_integrations); sq2.Clear(); qo.generateCountDistinctSecondQuery(sq1, sq2); full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq2); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processSecondOracleQueryResult(error_code, *external_integrations, "Count distinct oracle"); }}, {30 * static_cast( fuzz_config->allow_client_restarts && fuzz_config->allow_query_oracles && gen.collectionHas([&gen](const BuzzHouse::SQLPolicy & p) { return gen.rowPolicyForOracle(p); })), [&]() { /// Row policy oracle: an existing catalog row policy USING pred must be equivalent to WHERE pred. /// Q1: EXECUTE AS ; SELECT count() FROM db.t [FINAL] INTO OUTFILE /// (session switches to oracle user → policy active → filtered count) /// Reconnect: reset session back to default/admin user /// Q2: SELECT count() FROM db.t [FINAL] WHERE pred INTO OUTFILE (admin + explicit WHERE) qo.resetOracleValues(); sq2.Clear(); qo.generateRowPolicyOracleQueries(rg, gen, sq1, sq2); /// Step 1: EXECUTE AS oracle user — switches session; row policy applies for subsequent queries. /// Must be sent as a standalone statement; native TCP processes only one statement per request. full_query.resize(0); full_query += "EXECUTE AS '"; full_query += BuzzHouse::FuzzConfig::oracleUser; full_query += "'"; fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); /// Only proceed if EXECUTE AS succeeded. If it failed, processBuzzHouseQuery /// already reconnected (resetting the session to admin), so no cleanup needed. /// Skipping avoids comparing two admin-user queries which would be a false oracle. if (!error_code) { /// Step 2: SELECT count() FROM db.t [FINAL] INTO OUTFILE — runs as oracle user (no WHERE; policy filters rows). full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processFirstOracleQueryResult(error_code, *external_integrations); /// Step 3: Reconnect to reset session back to admin user before running the comparison query. fuzz_config->outf << restart_cmd << std::endl; server_up &= fuzzLoopReconnect(); /// Step 4: SELECT count() FROM db.t [FINAL] WHERE pred INTO OUTFILE — admin user + explicit WHERE predicate. full_query.resize(0); BuzzHouse::SQLQueryToString(full_query, sq2); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); qo.processSecondOracleQueryResult(error_code, *external_integrations, "Row policy oracle"); } /// if (!error_code) — EXECUTE AS succeeded }}, {1 * static_cast(fuzz_config->allow_client_restarts), [&]() { fuzz_config->outf << restart_cmd << std::endl; gen.setInTransaction(false); server_up &= fuzzLoopReconnect(); }}, {10 * static_cast(gen.collectionHas(gen.attached_tables_for_external_call)), [&]() { const uint64_t nseed = rg.nextInFullRange(); const auto & tbl = rg.pickRandomly(gen.filterCollection(gen.attached_tables_for_external_call)).get(); const auto & engine = tbl.isAnyIcebergEngine() ? "iceberg" : (tbl.isAnyDeltaLakeEngine() ? "deltalake" : (tbl.isAnyPaimonEngine() ? "paimon" : "kafka")); const auto & ndname = tbl.isKafkaEngine() ? tbl.getDatabaseName() : tbl.getSparkCatalogName(); const auto & ntname = tbl.getBaseName(false); const bool async = fuzz_config->allow_async_requests && rg.nextSmallNumber() < 4; chassert(tbl.isAnyLakeEngine() || tbl.isKafkaEngine()); fuzz_config->outf << external_cmd << (async ? "async " : "") << "with seed " << nseed << " to " << engine << " table " << markerHexEncode(ndname) << " " << markerHexEncode(ntname) << std::endl; runExternalCommand(external_integrations, nseed, async, engine, ndname, ntname); }}, {3 * static_cast(fuzz_config->allow_health_check), [&]() { fuzz_config->outf << health_check_cmd << std::endl; fuzz_config->validateClickHouseHealth(); }}, {910, [&]() { gen.generateNextStatement(rg, sq1); BuzzHouse::SQLQueryToString(full_query, sq1); fuzz_config->outf << full_query << std::endl; server_up &= processBuzzHouseQuery(full_query); gen.updateGenerator(sq1, *external_integrations, !have_error); }}, }); } } } if (!server_up) { LOG_INFO(fuzz_config->log, "The server is not responding, stopping fuzzing"); } if (!no_timeout) { /// Don't let the last query's error code become the exit code have_error = false; error_code = 0; server_exception.reset(); client_exception.reset(); LOG_INFO(fuzz_config->log, "The fuzzing time limit has been reached, stopping fuzzing"); } if (!no_eof) { LOG_INFO(fuzz_config->log, "End of fuzzing log file reached, stopping fuzzing"); } return server_up; } #else bool Client::buzzHouse() { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Clickhouse was compiled without BuzzHouse enabled"); } #endif }