From 6f115e76da42ffbcf39a5fbe1df298e6d6635f4d Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 7 Nov 2023 10:42:23 +0800 Subject: [PATCH 01/54] Fix ProducerBusy or ConsumerBusy error when configuring multiple brokers per connection (#337) ### Motivation This is a catch up for https://github.com/apache/pulsar/pull/21144 When a producer or consumer reconnects, a random number will be generated as the key suffix in `ConnectionPool` to create or get the `ClientConnection` object from the pool. https://github.com/apache/pulsar-client-cpp/blob/81cc562f7b366fad97e1b80c07ef9334a808390d/lib/ConnectionPool.cc#L75 If a new connection is created with the same producer or consumer name to the broker, the broker will respond with a `ProducerBusy` or `ConsumerBusy` error so that the reconnection will never succeed. ### Modifications - Add an overload of `ConnectionPool::getConnectionAsync` that accepts an integer parameter as the key suffix. If it's not specified, generate the random number as the suffix. In this method, choose the executor by `key suffix % size`. - Generate the random number and save it when creating the `HandlerBase` object. When connecting the owner broker of its topic, pass that index so that the reconnection will always reuse the same `ClientConnection` object. ### Verifying this change `ProducerTest.testReconnectMultiConnectionsPerBroker` is added to protected the change. --- lib/ClientImpl.cc | 6 +++--- lib/ClientImpl.h | 4 +++- lib/ConnectionPool.cc | 9 +++++---- lib/ConnectionPool.h | 14 +++++++++++++- lib/ExecutorService.cc | 4 ++-- lib/ExecutorService.h | 6 ++++-- lib/HandlerBase.cc | 4 +++- lib/HandlerBase.h | 1 + lib/ProducerImpl.cc | 2 +- tests/ProducerTest.cc | 15 +++++++++++++++ tests/PulsarFriend.h | 8 ++++++++ 11 files changed, 58 insertions(+), 15 deletions(-) diff --git a/lib/ClientImpl.cc b/lib/ClientImpl.cc index cb11b8e0..1032b79e 100644 --- a/lib/ClientImpl.cc +++ b/lib/ClientImpl.cc @@ -516,7 +516,7 @@ void ClientImpl::handleConsumerCreated(Result result, ConsumerImplBaseWeakPtr co } } -Future ClientImpl::getConnection(const std::string& topic) { +Future ClientImpl::getConnection(const std::string& topic, size_t key) { Promise promise; const auto topicNamePtr = TopicName::get(topic); @@ -528,12 +528,12 @@ Future ClientImpl::getConnection(const std::string& auto self = shared_from_this(); lookupServicePtr_->getBroker(*topicNamePtr) - .addListener([this, self, promise](Result result, const LookupService::LookupResult& data) { + .addListener([this, self, promise, key](Result result, const LookupService::LookupResult& data) { if (result != ResultOk) { promise.setFailed(result); return; } - pool_.getConnectionAsync(data.logicalAddress, data.physicalAddress) + pool_.getConnectionAsync(data.logicalAddress, data.physicalAddress, key) .addListener([promise](Result result, const ClientConnectionWeakPtr& weakCnx) { if (result == ResultOk) { auto cnx = weakCnx.lock(); diff --git a/lib/ClientImpl.h b/lib/ClientImpl.h index 0c6eeb40..f7b5a891 100644 --- a/lib/ClientImpl.h +++ b/lib/ClientImpl.h @@ -95,7 +95,7 @@ class ClientImpl : public std::enable_shared_from_this { void getPartitionsForTopicAsync(const std::string& topic, GetPartitionsCallback callback); - Future getConnection(const std::string& topic); + Future getConnection(const std::string& topic, size_t key); void closeAsync(CloseCallback callback); void shutdown(); @@ -123,6 +123,8 @@ class ClientImpl : public std::enable_shared_from_this { std::shared_ptr> getRequestIdGenerator() const { return requestIdGenerator_; } + ConnectionPool& getConnectionPool() noexcept { return pool_; } + friend class PulsarFriend; private: diff --git a/lib/ConnectionPool.cc b/lib/ConnectionPool.cc index 32325122..01285e9a 100644 --- a/lib/ConnectionPool.cc +++ b/lib/ConnectionPool.cc @@ -61,8 +61,9 @@ bool ConnectionPool::close() { return true; } -Future ConnectionPool::getConnectionAsync( - const std::string& logicalAddress, const std::string& physicalAddress) { +Future ConnectionPool::getConnectionAsync(const std::string& logicalAddress, + const std::string& physicalAddress, + size_t keySuffix) { if (closed_) { Promise promise; promise.setFailed(ResultAlreadyClosed); @@ -72,7 +73,7 @@ Future ConnectionPool::getConnectionAsync( std::unique_lock lock(mutex_); std::stringstream ss; - ss << logicalAddress << '-' << randomDistribution_(randomEngine_); + ss << logicalAddress << '-' << keySuffix; const std::string key = ss.str(); PoolMap::iterator cnxIt = pool_.find(key); @@ -95,7 +96,7 @@ Future ConnectionPool::getConnectionAsync( // No valid or pending connection found in the pool, creating a new one ClientConnectionPtr cnx; try { - cnx.reset(new ClientConnection(logicalAddress, physicalAddress, executorProvider_->get(), + cnx.reset(new ClientConnection(logicalAddress, physicalAddress, executorProvider_->get(keySuffix), clientConfiguration_, authentication_, clientVersion_, *this)); } catch (const std::runtime_error& e) { lock.unlock(); diff --git a/lib/ConnectionPool.h b/lib/ConnectionPool.h index c582dc9e..a51205b6 100644 --- a/lib/ConnectionPool.h +++ b/lib/ConnectionPool.h @@ -65,17 +65,29 @@ class PULSAR_PUBLIC ConnectionPool { * a proxy layer. Essentially, the pool is using the logical address as a way to * decide whether to reuse a particular connection. * + * There could be many connections to the same broker, so this pool uses an integer key as the suffix of + * the key that represents the connection. + * * @param logicalAddress the address to use as the broker tag * @param physicalAddress the real address where the TCP connection should be made + * @param keySuffix the key suffix to choose which connection on the same broker * @return a future that will produce the ClientCnx object */ Future getConnectionAsync(const std::string& logicalAddress, - const std::string& physicalAddress); + const std::string& physicalAddress, + size_t keySuffix); + + Future getConnectionAsync(const std::string& logicalAddress, + const std::string& physicalAddress) { + return getConnectionAsync(logicalAddress, physicalAddress, generateRandomIndex()); + } Future getConnectionAsync(const std::string& address) { return getConnectionAsync(address, address); } + size_t generateRandomIndex() { return randomDistribution_(randomEngine_); } + private: ClientConfiguration clientConfiguration_; ExecutorServiceProviderPtr executorProvider_; diff --git a/lib/ExecutorService.cc b/lib/ExecutorService.cc index a0dff0b6..53be8eaf 100644 --- a/lib/ExecutorService.cc +++ b/lib/ExecutorService.cc @@ -133,10 +133,10 @@ void ExecutorService::postWork(std::function task) { io_service_.pos ExecutorServiceProvider::ExecutorServiceProvider(int nthreads) : executors_(nthreads), executorIdx_(0), mutex_() {} -ExecutorServicePtr ExecutorServiceProvider::get() { +ExecutorServicePtr ExecutorServiceProvider::get(size_t idx) { + idx %= executors_.size(); Lock lock(mutex_); - int idx = executorIdx_++ % executors_.size(); if (!executors_[idx]) { executors_[idx] = ExecutorService::create(); } diff --git a/lib/ExecutorService.h b/lib/ExecutorService.h index 4717ccb5..a373c0af 100644 --- a/lib/ExecutorService.h +++ b/lib/ExecutorService.h @@ -88,7 +88,9 @@ class PULSAR_PUBLIC ExecutorServiceProvider { public: explicit ExecutorServiceProvider(int nthreads); - ExecutorServicePtr get(); + ExecutorServicePtr get() { return get(executorIdx_++); } + + ExecutorServicePtr get(size_t index); // See TimeoutProcessor for the semantics of the parameter. void close(long timeoutMs = 3000); @@ -96,7 +98,7 @@ class PULSAR_PUBLIC ExecutorServiceProvider { private: typedef std::vector ExecutorList; ExecutorList executors_; - int executorIdx_; + std::atomic_size_t executorIdx_; std::mutex mutex_; typedef std::unique_lock Lock; }; diff --git a/lib/HandlerBase.cc b/lib/HandlerBase.cc index 986063e0..b55f7510 100644 --- a/lib/HandlerBase.cc +++ b/lib/HandlerBase.cc @@ -32,6 +32,7 @@ namespace pulsar { HandlerBase::HandlerBase(const ClientImplPtr& client, const std::string& topic, const Backoff& backoff) : topic_(std::make_shared(topic)), client_(client), + connectionKeySuffix_(client->getConnectionPool().generateRandomIndex()), executor_(client->getIOExecutorProvider()->get()), mutex_(), creationTimestamp_(TimeUtils::now()), @@ -88,7 +89,8 @@ void HandlerBase::grabCnx() { return; } auto self = shared_from_this(); - client->getConnection(topic()).addListener([this, self](Result result, const ClientConnectionPtr& cnx) { + auto cnxFuture = client->getConnection(topic(), connectionKeySuffix_); + cnxFuture.addListener([this, self](Result result, const ClientConnectionPtr& cnx) { if (result == ResultOk) { LOG_DEBUG(getName() << "Connected to broker: " << cnx->cnxString()); connectionOpened(cnx).addListener([this, self](Result result, bool) { diff --git a/lib/HandlerBase.h b/lib/HandlerBase.h index 937b308d..f62c4df0 100644 --- a/lib/HandlerBase.h +++ b/lib/HandlerBase.h @@ -99,6 +99,7 @@ class HandlerBase : public std::enable_shared_from_this { protected: ClientImplWeakPtr client_; + const size_t connectionKeySuffix_; ExecutorServicePtr executor_; mutable std::mutex mutex_; std::mutex pendingReceiveMutex_; diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index a66fbfbd..80ee7358 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -966,7 +966,7 @@ bool ProducerImpl::encryptMessage(proto::MessageMetadata& metadata, SharedBuffer } void ProducerImpl::disconnectProducer() { - LOG_DEBUG("Broker notification of Closed producer: " << producerId_); + LOG_INFO("Broker notification of Closed producer: " << producerId_); resetCnx(); scheduleReconnection(); } diff --git a/tests/ProducerTest.cc b/tests/ProducerTest.cc index eeda2b47..9685cc8d 100644 --- a/tests/ProducerTest.cc +++ b/tests/ProducerTest.cc @@ -618,4 +618,19 @@ TEST(ProducerTest, testNoDeadlockWhenClosingPartitionedProducerAfterPartitionsUp client.close(); } +TEST(ProducerTest, testReconnectMultiConnectionsPerBroker) { + ClientConfiguration conf; + conf.setConnectionsPerBroker(10); + + Client client(serviceUrl, conf); + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer("producer-test-reconnect-twice", producer)); + + for (int i = 0; i < 5; i++) { + ASSERT_TRUE(PulsarFriend::reconnect(producer)) << "i: " << i; + } + + client.close(); +} + INSTANTIATE_TEST_CASE_P(Pulsar, ProducerTest, ::testing::Values(true, false)); diff --git a/tests/PulsarFriend.h b/tests/PulsarFriend.h index c25b1406..de82ce47 100644 --- a/tests/PulsarFriend.h +++ b/tests/PulsarFriend.h @@ -21,6 +21,7 @@ #include +#include "WaitUtils.h" #include "lib/ClientConnection.h" #include "lib/ClientImpl.h" #include "lib/ConsumerConfigurationImpl.h" @@ -197,6 +198,13 @@ class PulsarFriend { lookupData->setPartitions(newPartitions); partitionedProducer.handleGetPartitions(ResultOk, lookupData); } + + static bool reconnect(Producer producer) { + auto producerImpl = std::dynamic_pointer_cast(producer.impl_); + producerImpl->disconnectProducer(); + return waitUntil(std::chrono::seconds(3), + [producerImpl] { return !producerImpl->getCnx().expired(); }); + } }; } // namespace pulsar From 47fb809a3f92fe928f134687d980ffbae9bcdf72 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 9 Nov 2023 15:19:06 +0800 Subject: [PATCH 02/54] Use absolute path to find credential files in unit tests (#340) Fixes https://github.com/apache/pulsar-client-cpp/issues/339 ### Motivation Currently the relative path is used in unit tests to specify the token file (`.test-token.txt`) and the credential directory (`./test-conf`). If we don't run the `pulsar-tests` binary in a subdirectory, it won't be able to read these files. ### Modifications Add the macro `-DTOKEN_PATH="..." -DTEST_CONF_DIR="..."` to specify absolute paths according to the `PROJECT_SOURCE_DIR`, which is defined by CMake as the absolute path of the project directory. ### Verifications ```bash cmake -B build cmake --build build -j8 # Run the test in project directory ./build/tests/pulsar-tests --gtest_filter='AuthPlugin*:*testRSAEncryption:*testEncryptionFailure:*DecryptionFailedMessages' # Run the test in a subdirectory cd build ./tests/pulsar-tests --gtest_filter='AuthPlugin*:*testRSAEncryption:*testEncryptionFailure:*DecryptionFailedMessages' ``` --- CMakeLists.txt | 2 ++ pulsar-test-service-start.sh | 4 ++-- pulsar-test-service-stop.sh | 4 ++-- tests/AuthBasicTest.cc | 10 +++++++--- tests/AuthPluginTest.cc | 20 ++++++++++++++------ tests/AuthTokenTest.cc | 6 +++++- tests/BasicEndToEndTest.cc | 16 ++++++++++------ tests/CMakeLists.txt | 4 ++-- tests/ConsumerTest.cc | 8 ++++++-- tests/oauth2/Oauth2Test.cc | 6 +++--- 10 files changed, 53 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c1e7a79c..c8d6a00b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -395,6 +395,8 @@ if (BUILD_DYNAMIC_LIB) endif() if (BUILD_TESTS) + set(TOKEN_PATH "${PROJECT_SOURCE_DIR}/.test-token.txt") + set(TEST_CONF_DIR "${PROJECT_SOURCE_DIR}/test-conf") add_subdirectory(tests) endif() diff --git a/pulsar-test-service-start.sh b/pulsar-test-service-start.sh index 2b1a1e3b..23f70102 100755 --- a/pulsar-test-service-start.sh +++ b/pulsar-test-service-start.sh @@ -20,8 +20,8 @@ set -e -SRC_DIR=$(git rev-parse --show-toplevel) -cd $SRC_DIR +cd `dirname $0` +SRC_DIR=$PWD ./pulsar-test-service-stop.sh diff --git a/pulsar-test-service-stop.sh b/pulsar-test-service-stop.sh index 9d3bf4e2..aad069e6 100755 --- a/pulsar-test-service-stop.sh +++ b/pulsar-test-service-stop.sh @@ -20,8 +20,8 @@ set -e -SRC_DIR=$(git rev-parse --show-toplevel) -cd $SRC_DIR +cd `dirname $0` +SRC_DIR=$PWD CONTAINER_ID_PATH=".tests-container-id.txt" diff --git a/tests/AuthBasicTest.cc b/tests/AuthBasicTest.cc index 0b8b7f0c..0e181727 100644 --- a/tests/AuthBasicTest.cc +++ b/tests/AuthBasicTest.cc @@ -25,13 +25,17 @@ using namespace pulsar; +#ifndef TEST_CONF_DIR +#error "TEST_CONF_DIR is not specified" +#endif + static const std::string serviceUrl = "pulsar://localhost:6650"; static const std::string serviceUrlHttp = "http://localhost:8080"; static const std::string serviceUrlTls = "pulsar+ssl://localhost:6651"; static const std::string serviceUrlHttps = "https://localhost:8443"; -static const std::string caPath = "../test-conf/cacert.pem"; -static const std::string clientCertificatePath = "../test-conf/client-cert.pem"; -static const std::string clientPrivateKeyPath = "../test-conf/client-key.pem"; +static const std::string caPath = TEST_CONF_DIR "/cacert.pem"; +static const std::string clientCertificatePath = TEST_CONF_DIR "/client-cert.pem"; +static const std::string clientPrivateKeyPath = TEST_CONF_DIR "/client-key.pem"; TEST(AuthPluginBasic, testBasic) { ClientConfiguration config = ClientConfiguration(); diff --git a/tests/AuthPluginTest.cc b/tests/AuthPluginTest.cc index 2d23bf9f..1e16019d 100644 --- a/tests/AuthPluginTest.cc +++ b/tests/AuthPluginTest.cc @@ -37,15 +37,19 @@ int globalTestTlsMessagesCounter = 0; static const std::string serviceUrlTls = "pulsar+ssl://localhost:6651"; static const std::string serviceUrlHttps = "https://localhost:8443"; -static const std::string caPath = "../test-conf/cacert.pem"; -static const std::string clientPublicKeyPath = "../test-conf/client-cert.pem"; -static const std::string clientPrivateKeyPath = "../test-conf/client-key.pem"; +#ifndef TEST_CONF_DIR +#error "TEST_CONF_DIR is not specified" +#endif + +static const std::string caPath = TEST_CONF_DIR "/cacert.pem"; +static const std::string clientPublicKeyPath = TEST_CONF_DIR "/client-cert.pem"; +static const std::string clientPrivateKeyPath = TEST_CONF_DIR "/client-key.pem"; // Man in middle certificate which tries to act as a broker by sending its own valid certificate static const std::string mimServiceUrlTls = "pulsar+ssl://localhost:6653"; static const std::string mimServiceUrlHttps = "https://localhost:8444"; -static const std::string mimCaPath = "../test-conf/hn-verification/cacert.pem"; +static const std::string mimCaPath = TEST_CONF_DIR "/hn-verification/cacert.pem"; static void sendCallBackTls(Result r, const MessageId& msgId) { ASSERT_EQ(r, ResultOk); @@ -468,12 +472,16 @@ TEST(AuthPluginTest, testOauth2WrongSecret) { TEST(AuthPluginTest, testOauth2CredentialFile) { // test success get token from oauth2 server. pulsar::AuthenticationDataPtr data; - std::string params = R"({ + const char* paramsTemplate = R"({ "type": "client_credentials", "issuer_url": "https://dev-kt-aa9ne.us.auth0.com", - "private_key": "../test-conf/cpp_credentials_file.json", + "private_key": "%s/cpp_credentials_file.json", "audience": "https://dev-kt-aa9ne.us.auth0.com/api/v2/"})"; + char params[4096]; + int numWritten = snprintf(params, sizeof(params), paramsTemplate, TEST_CONF_DIR); + ASSERT_TRUE(numWritten < sizeof(params)); + int expectedTokenLength = 3379; LOG_INFO("PARAMS: " << params); pulsar::AuthenticationPtr auth = pulsar::AuthOauth2::create(params); diff --git a/tests/AuthTokenTest.cc b/tests/AuthTokenTest.cc index 8bdd2685..a04da085 100644 --- a/tests/AuthTokenTest.cc +++ b/tests/AuthTokenTest.cc @@ -37,7 +37,11 @@ using namespace pulsar; static const std::string serviceUrl = "pulsar://localhost:6650"; static const std::string serviceUrlHttp = "http://localhost:8080"; -static const std::string tokenPath = "../.test-token.txt"; +#ifndef TOKEN_PATH +#error "TOKEN_PATH is not specified" +#endif + +static const std::string tokenPath = TOKEN_PATH; static std::string getToken() { std::ifstream file(tokenPath); diff --git a/tests/BasicEndToEndTest.cc b/tests/BasicEndToEndTest.cc index 9ca2ab0c..0951dd48 100644 --- a/tests/BasicEndToEndTest.cc +++ b/tests/BasicEndToEndTest.cc @@ -54,6 +54,10 @@ DECLARE_LOG_OBJECT() using namespace pulsar; +#ifndef TEST_CONF_DIR +#error "TEST_CONF_DIR is not specified" +#endif + std::mutex mutex_; static int globalCount = 0; static long globalResendMessageCount = 0; @@ -1341,9 +1345,9 @@ TEST(BasicEndToEndTest, testRSAEncryption) { std::string subName = "my-sub-name"; Producer producer; - std::string PUBLIC_CERT_FILE_PATH = "../test-conf/public-key.client-rsa.pem"; + std::string PUBLIC_CERT_FILE_PATH = TEST_CONF_DIR "/public-key.client-rsa.pem"; - std::string PRIVATE_CERT_FILE_PATH = "../test-conf//private-key.client-rsa.pem"; + std::string PRIVATE_CERT_FILE_PATH = TEST_CONF_DIR "/private-key.client-rsa.pem"; std::shared_ptr keyReader = std::make_shared(PUBLIC_CERT_FILE_PATH, PRIVATE_CERT_FILE_PATH); @@ -1407,9 +1411,9 @@ TEST(BasicEndToEndTest, testEncryptionFailure) { std::string subName = "my-sub-name"; Producer producer; - std::string PUBLIC_CERT_FILE_PATH = "../test-conf/public-key.client-rsa-test.pem"; + std::string PUBLIC_CERT_FILE_PATH = TEST_CONF_DIR "/public-key.client-rsa-test.pem"; - std::string PRIVATE_CERT_FILE_PATH = "../test-conf//private-key.client-rsa-test.pem"; + std::string PRIVATE_CERT_FILE_PATH = TEST_CONF_DIR "/private-key.client-rsa-test.pem"; std::shared_ptr keyReader = std::make_shared(PUBLIC_CERT_FILE_PATH, PRIVATE_CERT_FILE_PATH); @@ -1449,9 +1453,9 @@ TEST(BasicEndToEndTest, testEncryptionFailure) { // 2. Add valid key { - PUBLIC_CERT_FILE_PATH = "../test-conf/public-key.client-rsa.pem"; + PUBLIC_CERT_FILE_PATH = TEST_CONF_DIR "/public-key.client-rsa.pem"; - PRIVATE_CERT_FILE_PATH = "../test-conf/private-key.client-rsa.pem"; + PRIVATE_CERT_FILE_PATH = TEST_CONF_DIR "/private-key.client-rsa.pem"; keyReader = std::make_shared(PUBLIC_CERT_FILE_PATH, PRIVATE_CERT_FILE_PATH); ProducerConfiguration prodConf; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b3731964..993c2fd0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -57,7 +57,7 @@ file(GLOB TEST_SOURCES *.cc c/*.cc) add_executable(pulsar-tests ${TEST_SOURCES} ${PROTO_SOURCES}) target_include_directories(pulsar-tests PRIVATE ${AUTOGEN_DIR}/lib) - +target_compile_options(pulsar-tests PRIVATE -DTOKEN_PATH="${TOKEN_PATH}" -DTEST_CONF_DIR="${TEST_CONF_DIR}") target_link_libraries(pulsar-tests ${CLIENT_LIBS} pulsarStatic $<$:${GMOCKD_LIBRARY_PATH}> $<$:${GTESTD_LIBRARY_PATH}> $<$>:${GMOCK_LIBRARY_PATH}> $<$>:${GTEST_LIBRARY_PATH}>) if (UNIX) @@ -69,7 +69,7 @@ add_executable(BrokerMetadataTest brokermetadata/BrokerMetadataTest.cc) target_link_libraries(BrokerMetadataTest ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) add_executable(Oauth2Test oauth2/Oauth2Test.cc) -target_compile_options(Oauth2Test PRIVATE "-DTEST_ROOT_PATH=\"${CMAKE_CURRENT_SOURCE_DIR}\"") +target_compile_options(Oauth2Test PRIVATE -DTEST_CONF_DIR="${TEST_CONF_DIR}") target_link_libraries(Oauth2Test ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) add_executable(ChunkDedupTest chunkdedup/ChunkDedupTest.cc) diff --git a/tests/ConsumerTest.cc b/tests/ConsumerTest.cc index 79175080..80c0a712 100644 --- a/tests/ConsumerTest.cc +++ b/tests/ConsumerTest.cc @@ -50,6 +50,10 @@ static const std::string adminUrl = "http://localhost:8080/"; DECLARE_LOG_OBJECT() +#ifndef TEST_CONF_DIR +#error "TEST_CONF_DIR is not specified" +#endif + namespace pulsar { class ConsumerStateEventListener : public ConsumerEventListener { @@ -960,8 +964,8 @@ TEST(ConsumerTest, testRedeliveryOfDecryptionFailedMessages) { std::string topicName = "testRedeliveryOfDecryptionFailedMessages" + std::to_string(time(nullptr)); std::string subName = "sub-test"; - std::string PUBLIC_CERT_FILE_PATH = "../test-conf/public-key.client-rsa.pem"; - std::string PRIVATE_CERT_FILE_PATH = "../test-conf/private-key.client-rsa.pem"; + std::string PUBLIC_CERT_FILE_PATH = TEST_CONF_DIR "/public-key.client-rsa.pem"; + std::string PRIVATE_CERT_FILE_PATH = TEST_CONF_DIR "/private-key.client-rsa.pem"; std::shared_ptr keyReader = std::make_shared(PUBLIC_CERT_FILE_PATH, PRIVATE_CERT_FILE_PATH); diff --git a/tests/oauth2/Oauth2Test.cc b/tests/oauth2/Oauth2Test.cc index 5f273d7e..719ae690 100644 --- a/tests/oauth2/Oauth2Test.cc +++ b/tests/oauth2/Oauth2Test.cc @@ -27,11 +27,11 @@ using namespace pulsar; -#ifndef TEST_ROOT_PATH -#define TEST_ROOT_PATH "." +#ifndef TEST_CONF_DIR +#error "TEST_CONF_DIR is not specified" #endif -static const std::string gKeyPath = std::string(TEST_ROOT_PATH) + "/../test-conf/cpp_credentials_file.json"; +static const std::string gKeyPath = TEST_CONF_DIR "/cpp_credentials_file.json"; static std::string gClientId; static std::string gClientSecret; static ParamMap gCommonParams; From a8402da957143b322364c048a8b19dc790891724 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 10 Nov 2023 10:54:58 +0800 Subject: [PATCH 03/54] Fix close() returns ResultAlreadyClosed after unsubscribe or close (#338) Fixes https://github.com/apache/pulsar-client-cpp/issues/88 ### Motivation When `close` is called if the consumer has already called `unsubscribe` or `close`, it should not fail. See https://github.com/apache/pulsar/blob/428c18c8d0c3d135189920740192982e11ffb2bf/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java#L1034 ### Modifications Use the same close logic with Java client. Add `testCloseAgainBeforeCloseDone` and `testCloseAfterUnsubscribe` to verify the new behaviors of `Consumer::close`. --- lib/ConsumerImpl.cc | 11 +++++++---- tests/BasicEndToEndTest.cc | 14 +++++++------- tests/ConsumerConfigurationTest.cc | 2 +- tests/ConsumerTest.cc | 25 +++++++++++++++++++++++++ tests/c/c_BasicEndToEndTest.cc | 2 +- 5 files changed, 41 insertions(+), 13 deletions(-) diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index 8dd334ea..b4666836 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -1238,10 +1238,12 @@ void ConsumerImpl::disconnectConsumer() { } void ConsumerImpl::closeAsync(ResultCallback originalCallback) { - auto callback = [this, originalCallback](Result result) { + auto callback = [this, originalCallback](Result result, bool alreadyClosed = false) { shutdown(); if (result == ResultOk) { - LOG_INFO(getName() << "Closed consumer " << consumerId_); + if (!alreadyClosed) { + LOG_INFO(getName() << "Closed consumer " << consumerId_); + } } else { LOG_WARN(getName() << "Failed to close consumer: " << result); } @@ -1250,8 +1252,9 @@ void ConsumerImpl::closeAsync(ResultCallback originalCallback) { } }; - if (state_ != Ready) { - callback(ResultAlreadyClosed); + auto state = state_.load(); + if (state == Closing || state == Closed) { + callback(ResultOk, true); return; } diff --git a/tests/BasicEndToEndTest.cc b/tests/BasicEndToEndTest.cc index 0951dd48..e2c6697f 100644 --- a/tests/BasicEndToEndTest.cc +++ b/tests/BasicEndToEndTest.cc @@ -244,7 +244,7 @@ TEST(BasicEndToEndTest, testProduceConsume) { consumer.receive(receivedMsg); ASSERT_EQ(content, receivedMsg.getDataAsString()); ASSERT_EQ(ResultOk, consumer.unsubscribe()); - ASSERT_EQ(ResultAlreadyClosed, consumer.close()); + ASSERT_EQ(ResultOk, consumer.close()); ASSERT_EQ(ResultOk, producer.close()); ASSERT_EQ(ResultOk, client.close()); } @@ -405,7 +405,7 @@ TEST(BasicEndToEndTest, testMultipleClientsMultipleSubscriptions) { ASSERT_EQ(ResultOk, producer1.close()); ASSERT_EQ(ResultOk, consumer1.close()); - ASSERT_EQ(ResultAlreadyClosed, consumer1.close()); + ASSERT_EQ(ResultOk, consumer1.close()); ASSERT_EQ(ResultConsumerNotInitialized, consumer2.close()); ASSERT_EQ(ResultOk, client1.close()); @@ -637,7 +637,7 @@ TEST(BasicEndToEndTest, testCompressionLZ4) { ASSERT_EQ(content2, receivedMsg.getDataAsString()); ASSERT_EQ(ResultOk, consumer.unsubscribe()); - ASSERT_EQ(ResultAlreadyClosed, consumer.close()); + ASSERT_EQ(ResultOk, consumer.close()); ASSERT_EQ(ResultOk, producer.close()); ASSERT_EQ(ResultOk, client.close()); } @@ -675,7 +675,7 @@ TEST(BasicEndToEndTest, testCompressionZLib) { ASSERT_EQ(content2, receivedMsg.getDataAsString()); ASSERT_EQ(ResultOk, consumer.unsubscribe()); - ASSERT_EQ(ResultAlreadyClosed, consumer.close()); + ASSERT_EQ(ResultOk, consumer.close()); ASSERT_EQ(ResultOk, producer.close()); ASSERT_EQ(ResultOk, client.close()); } @@ -750,7 +750,7 @@ TEST(BasicEndToEndTest, testConsumerClose) { Consumer consumer; ASSERT_EQ(ResultOk, client.subscribe(topicName, subName, consumer)); ASSERT_EQ(consumer.close(), ResultOk); - ASSERT_EQ(consumer.close(), ResultAlreadyClosed); + ASSERT_EQ(consumer.close(), ResultOk); } TEST(BasicEndToEndTest, testDuplicateConsumerCreationOnPartitionedTopic) { @@ -1398,7 +1398,7 @@ TEST(BasicEndToEndTest, testRSAEncryption) { } ASSERT_EQ(ResultOk, consumer.unsubscribe()); - ASSERT_EQ(ResultAlreadyClosed, consumer.close()); + ASSERT_EQ(ResultOk, consumer.close()); ASSERT_EQ(ResultOk, producer.close()); } ASSERT_EQ(ResultOk, client.close()); @@ -1617,7 +1617,7 @@ TEST(BasicEndToEndTest, testSeek) { ASSERT_EQ(expected.str(), msgReceived.getDataAsString()); ASSERT_EQ(ResultOk, consumer.acknowledge(msgReceived)); ASSERT_EQ(ResultOk, consumer.unsubscribe()); - ASSERT_EQ(ResultAlreadyClosed, consumer.close()); + ASSERT_EQ(ResultOk, consumer.close()); ASSERT_EQ(ResultOk, producer.close()); ASSERT_EQ(ResultOk, client.close()); } diff --git a/tests/ConsumerConfigurationTest.cc b/tests/ConsumerConfigurationTest.cc index f3785011..482f0fc5 100644 --- a/tests/ConsumerConfigurationTest.cc +++ b/tests/ConsumerConfigurationTest.cc @@ -314,7 +314,7 @@ TEST(ConsumerConfigurationTest, testSubscriptionInitialPosition) { ASSERT_EQ(content1, receivedMsg.getDataAsString()); ASSERT_EQ(ResultOk, consumer.unsubscribe()); - ASSERT_EQ(ResultAlreadyClosed, consumer.close()); + ASSERT_EQ(ResultOk, consumer.close()); ASSERT_EQ(ResultOk, producer.close()); ASSERT_EQ(ResultOk, client.close()); } diff --git a/tests/ConsumerTest.cc b/tests/ConsumerTest.cc index 80c0a712..0836fbf4 100644 --- a/tests/ConsumerTest.cc +++ b/tests/ConsumerTest.cc @@ -1405,4 +1405,29 @@ TEST(ConsumerTest, testNoListenerThreadBlocking) { client.close(); } +TEST(ConsumerTest, testCloseAfterUnsubscribe) { + Client client{lookupUrl}; + Consumer consumer; + ASSERT_EQ(ResultOk, client.subscribe("test-close-after-unsubscribe", "sub", consumer)); + ASSERT_EQ(ResultOk, consumer.unsubscribe()); + ASSERT_EQ(ResultOk, consumer.close()); +} + +TEST(ConsumerTest, testCloseAgainBeforeCloseDone) { + Client client{lookupUrl}; + Consumer consumer; + ASSERT_EQ(ResultOk, client.subscribe("test-close-again-before-close-done", "sub", consumer)); + auto done = std::make_shared(false); + auto result = std::make_shared>(ResultOk); + consumer.closeAsync([done, result](Result innerResult) { + result->store(innerResult); + done->store(true); + }); + ASSERT_EQ(ResultOk, consumer.close()); + ASSERT_FALSE(*done); + waitUntil(std::chrono::seconds(3), [done] { return done->load(); }); + ASSERT_EQ(ResultOk, *result); + ASSERT_TRUE(*done); +} + } // namespace pulsar diff --git a/tests/c/c_BasicEndToEndTest.cc b/tests/c/c_BasicEndToEndTest.cc index 04aa1dcd..e3ff19aa 100644 --- a/tests/c/c_BasicEndToEndTest.cc +++ b/tests/c/c_BasicEndToEndTest.cc @@ -109,7 +109,7 @@ TEST(c_BasicEndToEndTest, testAsyncProduceConsume) { delete receive_ctx.data; ASSERT_EQ(pulsar_result_Ok, pulsar_consumer_unsubscribe(consumer)); - ASSERT_EQ(pulsar_result_AlreadyClosed, pulsar_consumer_close(consumer)); + ASSERT_EQ(pulsar_result_Ok, pulsar_consumer_close(consumer)); ASSERT_EQ(pulsar_result_Ok, pulsar_producer_close(producer)); ASSERT_EQ(pulsar_result_Ok, pulsar_client_close(client)); From bb16f24bb68699c15eddba3163cfbef0b1282ebf Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 13 Nov 2023 10:02:01 +0800 Subject: [PATCH 04/54] Fix lazy partitioned producer might send duplicated messages (#342) Fixes https://github.com/apache/pulsar-client-cpp/issues/341 ### Motivation When a lazy partitioned producer sends two messages, the flow is: 1. `start` is called to grab the connection via `grab()`. 2. Generate 0 as the sequence id of the 1st message. 3. Add the 1st message into the queuea. 4. The connection is established, `msgSequenceGenerator_` is reset from 1 to 0. 5. When sending the 2nd message, 0 is also generated as the sequence id. Then two messages have the same sequence id. ### Modifications For lazy partitioned producers, if the internal producer is not started, sending the message in the callback of its future. Add `ChunkDedupTest#testLazyPartitionedProducer` to verify it since only the `tests/chunkdedup/docker-compose.yml` enables the deduplication. --- lib/PartitionedProducerImpl.cc | 22 +++++++++++++++++++--- lib/ProducerImpl.cc | 8 ++------ lib/ProducerImpl.h | 7 +++++-- tests/CMakeLists.txt | 2 +- tests/chunkdedup/ChunkDedupTest.cc | 26 ++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/lib/PartitionedProducerImpl.cc b/lib/PartitionedProducerImpl.cc index f7957d61..a799d5c8 100644 --- a/lib/PartitionedProducerImpl.cc +++ b/lib/PartitionedProducerImpl.cc @@ -198,7 +198,9 @@ void PartitionedProducerImpl::createLazyPartitionProducer(unsigned int partition // override void PartitionedProducerImpl::sendAsync(const Message& msg, SendCallback callback) { if (state_ != Ready) { - callback(ResultAlreadyClosed, msg.getMessageId()); + if (callback) { + callback(ResultAlreadyClosed, msg.getMessageId()); + } return; } @@ -209,7 +211,9 @@ void PartitionedProducerImpl::sendAsync(const Message& msg, SendCallback callbac LOG_ERROR("Got Invalid Partition for message from Router Policy, Partition - " << partition); // change me: abort or notify failure in callback? // change to appropriate error if callback - callback(ResultUnknownError, msg.getMessageId()); + if (callback) { + callback(ResultUnknownError, msg.getMessageId()); + } return; } // find a producer for that partition, index should start from 0 @@ -223,7 +227,19 @@ void PartitionedProducerImpl::sendAsync(const Message& msg, SendCallback callbac producersLock.unlock(); // send message on that partition - producer->sendAsync(msg, callback); + if (!conf_.getLazyStartPartitionedProducers() || producer->ready()) { + producer->sendAsync(msg, std::move(callback)); + } else { + // Wrapping the callback into a lambda has overhead, so we check if the producer is ready first + producer->getProducerCreatedFuture().addListener( + [msg, callback](Result result, ProducerImplBaseWeakPtr weakProducer) { + if (result == ResultOk) { + weakProducer.lock()->sendAsync(msg, std::move(callback)); + } else if (callback) { + callback(result, {}); + } + }); + } } // override diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index 80ee7358..76a999a3 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -60,8 +60,9 @@ ProducerImpl::ProducerImpl(ClientImplPtr client, const TopicName& topicName, userProvidedProducerName_(false), producerStr_("[" + topic() + ", " + producerName_ + "] "), producerId_(client->newProducerId()), - msgSequenceGenerator_(0), batchTimer_(executor_->createDeadlineTimer()), + lastSequenceIdPublished_(conf.getInitialSequenceId()), + msgSequenceGenerator_(lastSequenceIdPublished_ + 1), sendTimer_(executor_->createDeadlineTimer()), dataKeyRefreshTask_(*executor_, 4 * 60 * 60 * 1000), memoryLimitController_(client->getMemoryLimitController()), @@ -69,11 +70,6 @@ ProducerImpl::ProducerImpl(ClientImplPtr client, const TopicName& topicName, interceptors_(interceptors) { LOG_DEBUG("ProducerName - " << producerName_ << " Created producer on topic " << topic() << " id: " << producerId_); - - int64_t initialSequenceId = conf.getInitialSequenceId(); - lastSequenceIdPublished_ = initialSequenceId; - msgSequenceGenerator_ = initialSequenceId + 1; - if (!producerName_.empty()) { userProvidedProducerName_ = true; } diff --git a/lib/ProducerImpl.h b/lib/ProducerImpl.h index 91b95443..2fb0b886 100644 --- a/lib/ProducerImpl.h +++ b/lib/ProducerImpl.h @@ -19,6 +19,7 @@ #ifndef LIB_PRODUCERIMPL_H_ #define LIB_PRODUCERIMPL_H_ +#include #include #include #include @@ -103,6 +104,8 @@ class ProducerImpl : public HandlerBase, public ProducerImplBase { ProducerImplWeakPtr weak_from_this() noexcept { return shared_from_this(); } + bool ready() const { return producerCreatedPromise_.isComplete(); } + protected: ProducerStatsBasePtr producerStatsBasePtr_; @@ -169,13 +172,13 @@ class ProducerImpl : public HandlerBase, public ProducerImplBase { bool userProvidedProducerName_; std::string producerStr_; uint64_t producerId_; - int64_t msgSequenceGenerator_; std::unique_ptr batchMessageContainer_; DeadlineTimerPtr batchTimer_; PendingFailures batchMessageAndSend(const FlushCallback& flushCallback = nullptr); - volatile int64_t lastSequenceIdPublished_; + std::atomic lastSequenceIdPublished_; + std::atomic msgSequenceGenerator_; std::string schemaVersion_; DeadlineTimerPtr sendTimer_; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 993c2fd0..3bc5a156 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -72,5 +72,5 @@ add_executable(Oauth2Test oauth2/Oauth2Test.cc) target_compile_options(Oauth2Test PRIVATE -DTEST_CONF_DIR="${TEST_CONF_DIR}") target_link_libraries(Oauth2Test ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) -add_executable(ChunkDedupTest chunkdedup/ChunkDedupTest.cc) +add_executable(ChunkDedupTest chunkdedup/ChunkDedupTest.cc HttpHelper.cc) target_link_libraries(ChunkDedupTest ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) diff --git a/tests/chunkdedup/ChunkDedupTest.cc b/tests/chunkdedup/ChunkDedupTest.cc index 609511f2..4c9c28e7 100644 --- a/tests/chunkdedup/ChunkDedupTest.cc +++ b/tests/chunkdedup/ChunkDedupTest.cc @@ -18,7 +18,9 @@ */ #include #include +#include +#include "../HttpHelper.h" #include "lib/Latch.h" #include "lib/LogUtils.h" @@ -47,6 +49,30 @@ TEST(ChunkDedupTest, testSendChunks) { client.close(); } +TEST(ChunkDedupTest, testLazyPartitionedProducer) { + std::string topic = "test-lazy-partitioned-producer-" + std::to_string(time(nullptr)); + Client client{"pulsar://localhost:6650"}; + ProducerConfiguration conf; + conf.setLazyStartPartitionedProducers(true); + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer(topic, conf, producer)); + + constexpr int numPartitions = 3; + int res = + makePutRequest("http://localhost:8080/admin/v2/persistent/public/default/" + topic + "/partitions", + std::to_string(numPartitions)); + ASSERT_TRUE(res == 204 || res == 409) << "res: " << res; + + for (int i = 0; i < 10; i++) { + const auto key = std::to_string(i % numPartitions); + MessageId msgId; + producer.send(MessageBuilder().setPartitionKey(key).setContent("msg-" + std::to_string(i)).build(), + msgId); + ASSERT_TRUE(msgId.ledgerId() >= 0 && msgId.entryId() >= 0) << "i: " << i << ", msgId: " << msgId; + } + client.close(); +} + int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); From 8d32fd254e294d1fabba73aed70115a434b341ef Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 13 Nov 2023 19:55:45 +0800 Subject: [PATCH 05/54] Bumped version to 3.5.0-pre (#344) --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index ba21b572..b10ed391 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.4.0-pre +3.5.0-pre From 6d47e9495c071975d94fc284d6bfb6bbe9c7501a Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 17 Nov 2023 10:29:27 +0800 Subject: [PATCH 06/54] Fix crash when removing connection from the pool (#347) Fixes https://github.com/apache/pulsar-client-cpp/issues/346 ### Motivation https://github.com/apache/pulsar-client-cpp/pull/336 changes the key of the `ClientConnection` in `ConnectionPool`, while in `ClientConnection::close`, it still passes the old key (logical address) to `ConnectionPool::remove`, which results in the connection could never be removed and destroyed until being deleted as a stale connection. What's worse, if the key does not exist, the iterator returned by `std::map::find` will still be dereferenced, which might cause crash in some platforms. See https://github.com/apache/pulsar-client-cpp/blob/8d32fd254e294d1fabba73aed70115a434b341ef/lib/ConnectionPool.cc#L122-L123 ### Modifications - Avoid dereferencing the iterator if it's invalid in `ConnectionPool::remove`. - Store the key suffix in `ClientConnection` and pass the correct key to `ConnectionPool::remove` in `ClientConnection::close` - Add `ClientTest.testConnectionClose` to verify `ClientConnection::close` can remove itself from the pool and the connection will be destroyed eventually. --- lib/ClientConnection.cc | 9 +++++---- lib/ClientConnection.h | 4 +++- lib/ConnectionPool.cc | 7 ++++--- tests/ClientTest.cc | 41 +++++++++++++++++++++++++++++++++++++++++ tests/PulsarFriend.h | 2 ++ 5 files changed, 55 insertions(+), 8 deletions(-) diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index 8f42535c..fbcbe626 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -160,7 +160,7 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: ExecutorServicePtr executor, const ClientConfiguration& clientConfiguration, const AuthenticationPtr& authentication, const std::string& clientVersion, - ConnectionPool& pool) + ConnectionPool& pool, size_t poolIndex) : operationsTimeout_(seconds(clientConfiguration.getOperationTimeoutSeconds())), authentication_(authentication), serverProtocolVersion_(proto::ProtocolVersion_MIN), @@ -184,7 +184,8 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: consumerStatsRequestTimer_(executor_->createDeadlineTimer()), maxPendingLookupRequest_(clientConfiguration.getConcurrentLookupRequest()), clientVersion_(clientVersion), - pool_(pool) { + pool_(pool), + poolIndex_(poolIndex) { LOG_INFO(cnxString_ << "Create ClientConnection, timeout=" << clientConfiguration.getConnectionTimeout()); if (clientConfiguration.isUseTls()) { #if BOOST_VERSION >= 105400 @@ -265,7 +266,7 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: } ClientConnection::~ClientConnection() { - LOG_INFO(cnxString_ << "Destroyed connection to " << logicalAddress_); + LOG_INFO(cnxString_ << "Destroyed connection to " << logicalAddress_ << "-" << poolIndex_); } void ClientConnection::handlePulsarConnected(const proto::CommandConnected& cmdConnected) { @@ -1320,7 +1321,7 @@ void ClientConnection::close(Result result, bool detach) { } // Remove the connection from the pool before completing any promise if (detach) { - pool_.remove(logicalAddress_, this); // trigger the destructor + pool_.remove(logicalAddress_ + "-" + std::to_string(poolIndex_), this); } auto self = shared_from_this(); diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index 30ea8d82..965c6aec 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -129,7 +129,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this ConnectionPool::getConnectionAsync(const ClientConnectionPtr cnx; try { cnx.reset(new ClientConnection(logicalAddress, physicalAddress, executorProvider_->get(keySuffix), - clientConfiguration_, authentication_, clientVersion_, *this)); + clientConfiguration_, authentication_, clientVersion_, *this, + keySuffix)); } catch (const std::runtime_error& e) { lock.unlock(); LOG_ERROR("Failed to create connection: " << e.what()) @@ -106,7 +107,7 @@ Future ConnectionPool::getConnectionAsync(const return promise.getFuture(); } - LOG_INFO("Created connection for " << logicalAddress); + LOG_INFO("Created connection for " << key); Future future = cnx->getConnectFuture(); pool_.insert(std::make_pair(key, cnx)); @@ -120,7 +121,7 @@ Future ConnectionPool::getConnectionAsync(const void ConnectionPool::remove(const std::string& key, ClientConnection* value) { std::lock_guard lock(mutex_); auto it = pool_.find(key); - if (it->second.get() == value) { + if (it != pool_.end() && it->second.get() == value) { LOG_INFO("Remove connection for " << key); pool_.erase(it); } diff --git a/tests/ClientTest.cc b/tests/ClientTest.cc index 983e5d91..5b4dd2e8 100644 --- a/tests/ClientTest.cc +++ b/tests/ClientTest.cc @@ -400,3 +400,44 @@ TEST(ClientTest, testClientVersion) { client.close(); } + +TEST(ClientTest, testConnectionClose) { + std::vector clients; + clients.emplace_back(lookupUrl); + clients.emplace_back(lookupUrl, ClientConfiguration().setConnectionsPerBroker(5)); + + const auto topic = "client-test-connection-close"; + for (auto &client : clients) { + auto testClose = [&client](ClientConnectionWeakPtr weakCnx) { + auto cnx = weakCnx.lock(); + ASSERT_TRUE(cnx); + + auto numConnections = PulsarFriend::getConnections(client).size(); + LOG_INFO("Connection refcnt: " << cnx.use_count() << " before close"); + auto executor = PulsarFriend::getExecutor(*cnx); + // Simulate the close() happens in the event loop + executor->postWork([cnx, &client, numConnections] { + cnx->close(); + ASSERT_EQ(PulsarFriend::getConnections(client).size(), numConnections - 1); + LOG_INFO("Connection refcnt: " << cnx.use_count() << " after close"); + }); + cnx.reset(); + + // The ClientConnection could still be referred in a socket callback, wait until all these + // callbacks being cancelled due to the socket close. + ASSERT_TRUE(waitUntil( + std::chrono::seconds(1), [weakCnx] { return weakCnx.expired(); }, 1)); + }; + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer(topic, producer)); + testClose(PulsarFriend::getProducerImpl(producer).getCnx()); + producer.close(); + + Consumer consumer; + ASSERT_EQ(ResultOk, client.subscribe("client-test-connection-close", "sub", consumer)); + testClose(PulsarFriend::getConsumerImpl(consumer).getCnx()); + consumer.close(); + + client.close(); + } +} diff --git a/tests/PulsarFriend.h b/tests/PulsarFriend.h index de82ce47..c2863e8c 100644 --- a/tests/PulsarFriend.h +++ b/tests/PulsarFriend.h @@ -140,6 +140,8 @@ class PulsarFriend { return connections; } + static ExecutorServicePtr getExecutor(const ClientConnection& cnx) { return cnx.executor_; } + static std::vector getProducers(const ClientConnection& cnx) { std::vector producers; std::lock_guard lock(cnx.mutex_); From c771b12fd564c9d7c351e61fa3ced56a8f15ec1f Mon Sep 17 00:00:00 2001 From: erobot Date: Fri, 17 Nov 2023 16:03:37 +0800 Subject: [PATCH 07/54] Log topic lookup result (#351) --- lib/BinaryProtoLookupService.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/BinaryProtoLookupService.cc b/lib/BinaryProtoLookupService.cc index c8d6b769..87f02ea4 100644 --- a/lib/BinaryProtoLookupService.cc +++ b/lib/BinaryProtoLookupService.cc @@ -83,7 +83,8 @@ auto BinaryProtoLookupService::findBroker(const std::string& address, bool autho } }); } else { - LOG_DEBUG("Lookup response for " << topic << ", lookup-broker-url " << data->getBrokerUrl()); + LOG_INFO("Lookup response for " << topic << ", lookup-broker-url " << data->getBrokerUrl() + << ", from " << cnx->cnxString()); if (data->shouldProxyThroughServiceUrl()) { // logicalAddress is the proxy's address, we should still connect through proxy promise->setValue({responseBrokerAddress, address}); From 7bb94f45b917ed30b5302ac93ffa1f1942fc6313 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 17 Nov 2023 17:51:50 +0800 Subject: [PATCH 08/54] Fix bad_weak_ptr when close a ClientConnection during construction (#350) Fixes https://github.com/apache/pulsar-client-cpp/issues/348 Fixes https://github.com/apache/pulsar-client-cpp/issues/349 ### Motivation When `close` is called in `ClientConnection`'s constructor, `shared_from_this()` will be called, which results in a `std::bad_weak_ptr` error. This error does not happen before https://github.com/apache/pulsar-client-cpp/pull/317 because `shared_from_this()` could only be called when the `producers` or `consumers` field is not empty. ### Modifications Throw `ResultAuthenticationError` when there is anything wrong with authentication in `ClientConnection`'s constructor. Then catch the result in `ConnectionPool::getConnectionAsync` and returned the failed future. In addition, check `authentication_` even for non-TLS URLs. Otherwise, the null authentication will be used to construct `CommandConnect`. Add `testInvalidPlugin` and `testTlsConfigError` to verify the changes. --- lib/ClientConnection.cc | 21 +++++++++------------ lib/ClientConnection.h | 4 +--- lib/ConnectionPool.cc | 4 ++++ tests/AuthPluginTest.cc | 16 ++++++++++++++++ 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index fbcbe626..4086bdde 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -187,6 +187,12 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: pool_(pool), poolIndex_(poolIndex) { LOG_INFO(cnxString_ << "Create ClientConnection, timeout=" << clientConfiguration.getConnectionTimeout()); + if (!authentication_) { + LOG_ERROR("Invalid authentication plugin"); + throw ResultAuthenticationError; + return; + } + if (clientConfiguration.isUseTls()) { #if BOOST_VERSION >= 105400 boost::asio::ssl::context ctx(boost::asio::ssl::context::tlsv12_client); @@ -207,20 +213,13 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: ctx.load_verify_file(trustCertFilePath); } else { LOG_ERROR(trustCertFilePath << ": No such trustCertFile"); - close(ResultAuthenticationError, false); - return; + throw ResultAuthenticationError; } } else { ctx.set_default_verify_paths(); } } - if (!authentication_) { - LOG_ERROR("Invalid authentication plugin"); - close(ResultAuthenticationError, false); - return; - } - std::string tlsCertificates = clientConfiguration.getTlsCertificateFilePath(); std::string tlsPrivateKey = clientConfiguration.getTlsPrivateKeyFilePath(); @@ -231,13 +230,11 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: tlsPrivateKey = authData->getTlsPrivateKey(); if (!file_exists(tlsCertificates)) { LOG_ERROR(tlsCertificates << ": No such tlsCertificates"); - close(ResultAuthenticationError, false); - return; + throw ResultAuthenticationError; } if (!file_exists(tlsCertificates)) { LOG_ERROR(tlsCertificates << ": No such tlsCertificates"); - close(ResultAuthenticationError, false); - return; + throw ResultAuthenticationError; } ctx.use_private_key_file(tlsPrivateKey, boost::asio::ssl::context::pem); ctx.use_certificate_file(tlsCertificates, boost::asio::ssl::context::pem); diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index 965c6aec..ce2bd885 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -148,9 +148,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this ConnectionPool::getConnectionAsync(const cnx.reset(new ClientConnection(logicalAddress, physicalAddress, executorProvider_->get(keySuffix), clientConfiguration_, authentication_, clientVersion_, *this, keySuffix)); + } catch (Result result) { + Promise promise; + promise.setFailed(result); + return promise.getFuture(); } catch (const std::runtime_error& e) { lock.unlock(); LOG_ERROR("Failed to create connection: " << e.what()) diff --git a/tests/AuthPluginTest.cc b/tests/AuthPluginTest.cc index 1e16019d..9fd048b7 100644 --- a/tests/AuthPluginTest.cc +++ b/tests/AuthPluginTest.cc @@ -581,3 +581,19 @@ TEST(AuthPluginTest, testOauth2Failure) { ASSERT_EQ(client5.createProducer(topic, producer), ResultAuthenticationError); client5.close(); } + +TEST(AuthPluginTest, testInvalidPlugin) { + Client client("pulsar://localhost:6650", ClientConfiguration{}.setAuth(AuthFactory::create("invalid"))); + Producer producer; + ASSERT_EQ(ResultAuthenticationError, client.createProducer("my-topic", producer)); + client.close(); +} + +TEST(AuthPluginTest, testTlsConfigError) { + Client client(serviceUrlTls, ClientConfiguration{} + .setAuth(AuthTls::create(clientPublicKeyPath, clientPrivateKeyPath)) + .setTlsTrustCertsFilePath("invalid")); + Producer producer; + ASSERT_EQ(ResultAuthenticationError, client.createProducer("my-topic", producer)); + client.close(); +} From 37ea76977929d8fd2e74005a4fb7a238b83e3f1c Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 21 Nov 2023 10:42:01 +0800 Subject: [PATCH 09/54] Fix the flush callback might be called repeatedly (#353) Fixes https://github.com/apache/pulsar-client-cpp/issues/352 ### Motivation https://github.com/apache/pulsar-client-cpp/pull/303 adds the flush callback to the last `OpSendMsg` instead of adding to the batch message container. However, `batchMessageAndSend` will create an `OpSendMsg` and add it to the `pendingMessagesQueue_`. https://github.com/apache/pulsar-client-cpp/blob/7bb94f45b917ed30b5302ac93ffa1f1942fc6313/lib/ProducerImpl.cc#L384-L389 In the code above, `pendingMessagesQueue_` could never be empty and the callback will be added again by `opSendMsg->addTrackerCallback`. The 1st time it's added in `createOpSendMsg` or `createOpSendMsgs` called by `batchMessageAndSend`. ### Motivation Add the callback to the last `OpSendMsg only when the batch message container is empty. In `testFlushBatch`, replace the `flush` call with the `flushAsync` call and verify the callback is only called once after it's completed. --- lib/BatchMessageKeyBasedContainer.cc | 7 ++++- lib/ProducerImpl.cc | 40 +++++++++++++++++----------- tests/ProducerTest.cc | 20 ++++++++++++-- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/lib/BatchMessageKeyBasedContainer.cc b/lib/BatchMessageKeyBasedContainer.cc index 20067363..5b181843 100644 --- a/lib/BatchMessageKeyBasedContainer.cc +++ b/lib/BatchMessageKeyBasedContainer.cc @@ -77,11 +77,16 @@ std::vector> BatchMessageKeyBasedContainer::createOpS // Store raw pointers to use std::sort std::vector rawOpSendMsgs; for (auto& kv : batches_) { - rawOpSendMsgs.emplace_back(createOpSendMsgHelper(kv.second).release()); + if (!kv.second.empty()) { + rawOpSendMsgs.emplace_back(createOpSendMsgHelper(kv.second).release()); + } } std::sort(rawOpSendMsgs.begin(), rawOpSendMsgs.end(), [](const OpSendMsg* lhs, const OpSendMsg* rhs) { return lhs->sendArgs->sequenceId < rhs->sendArgs->sequenceId; }); + if (rawOpSendMsgs.empty()) { + return {}; + } rawOpSendMsgs.back()->addTrackerCallback(flushCallback); std::vector> opSendMsgs{rawOpSendMsgs.size()}; diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index 76a999a3..61b95bf0 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -376,29 +376,37 @@ void ProducerImpl::setMessageMetadata(const Message& msg, const uint64_t& sequen void ProducerImpl::flushAsync(FlushCallback callback) { if (state_ != Ready) { - callback(ResultAlreadyClosed); + if (callback) { + callback(ResultAlreadyClosed); + } return; } + + auto addCallbackToLastOp = [this, &callback] { + if (pendingMessagesQueue_.empty()) { + return false; + } + pendingMessagesQueue_.back()->addTrackerCallback(callback); + return true; + }; + if (batchMessageContainer_) { Lock lock(mutex_); - auto failures = batchMessageAndSend(callback); - if (!pendingMessagesQueue_.empty()) { - auto& opSendMsg = pendingMessagesQueue_.back(); - lock.unlock(); - failures.complete(); - opSendMsg->addTrackerCallback(callback); - } else { - lock.unlock(); - failures.complete(); - callback(ResultOk); + + if (batchMessageContainer_->isEmpty()) { + if (!addCallbackToLastOp() && callback) { + lock.unlock(); + callback(ResultOk); + } + return; } + + auto failures = batchMessageAndSend(callback); + lock.unlock(); + failures.complete(); } else { Lock lock(mutex_); - if (!pendingMessagesQueue_.empty()) { - auto& opSendMsg = pendingMessagesQueue_.back(); - lock.unlock(); - opSendMsg->addTrackerCallback(callback); - } else { + if (!addCallbackToLastOp() && callback) { lock.unlock(); callback(ResultOk); } diff --git a/tests/ProducerTest.cc b/tests/ProducerTest.cc index 9685cc8d..45bb3aae 100644 --- a/tests/ProducerTest.cc +++ b/tests/ProducerTest.cc @@ -476,7 +476,23 @@ TEST_P(ProducerTest, testFlushBatch) { producer.sendAsync(msg, cb); } - producer.flush(); + auto assertFlushCallbackOnce = [&producer] { + Latch latch{1}; + std::mutex mutex; + std::vector results; + producer.flushAsync([&](Result result) { + { + std::lock_guard lock{mutex}; + results.emplace_back(result); + } + latch.countdown(); + }); + latch.wait(); + std::lock_guard lock{mutex}; + ASSERT_EQ(results, (std::vector{ResultOk})); + }; + + assertFlushCallbackOnce(); ASSERT_EQ(needCallBack.load(), 0); producer.close(); @@ -494,7 +510,7 @@ TEST_P(ProducerTest, testFlushBatch) { producer.sendAsync(msg, cb2); } - producer.flush(); + assertFlushCallbackOnce(); ASSERT_EQ(needCallBack2.load(), 0); producer.close(); From f75b39bcbaad63ed96268fe7e09993e0e6a8a959 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 21 Nov 2023 20:45:54 +0800 Subject: [PATCH 10/54] Fix Protobuf symbols not found in libpulsarwithdeps.a when building on macOS (#354) ### Motivation https://github.com/apache/pulsar-client-cpp/pull/290 brings a regression that on macOS, Protobuf is always found with CMake Config mode, which does not set the `Protobuf_LIBRARIES` variable so that the libpulsarwithdeps.a misses the symbols of Protobuf. ### Modifications When `LINK_STATIC` is ON, use CMake Module mode to find the Protobuf. Add `build-static-library.sh` to build libraries with static dependencies and verify these libraries in PR workflow. Upload the pre-built binaries in the build workflow. --- .../workflows/ci-build-binary-artifacts.yaml | 31 +++ .github/workflows/ci-pr-validation.yaml | 22 ++ .gitignore | 2 + CMakeLists.txt | 16 +- dependencies.yaml | 2 +- pkg/mac/build-static-library.sh | 190 ++++++++++++++++++ 6 files changed, 258 insertions(+), 5 deletions(-) create mode 100755 pkg/mac/build-static-library.sh diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 586458d5..a2b7be84 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -192,3 +192,34 @@ jobs: with: name: ${{ matrix.triplet }}-Debug path: ${{ env.INSTALL_DIR }}-Debug + + package-macos: + name: Build macOS libraries + runs-on: macos-latest + timeout-minutes: 500 + + strategy: + fail-fast: false + matrix: + arch: [x86_64, arm64] + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Install dependencies + run: | + export ARCH=${{ matrix.arch }} + ./pkg/mac/build-static-library.sh + + - name: Zip artifact + run: | + cd ./pkg/mac/.install + zip -r macos-${{ matrix.arch }}.zip ./include/pulsar/* ./lib/* + cp macos-${{ matrix.arch }}.zip ../../../ + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: macos-${{ matrix.arch }}.zip + path: macos-${{ matrix.arch }}.zip diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 03d417fa..80e1a1d3 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -302,6 +302,28 @@ jobs: run: | cmake --build ./build-macos --parallel --config Release + cpp-build-macos-static: + timeout-minutes: 120 + name: Build CPP Client on macOS with static dependencies + runs-on: macos-12 + needs: unit-tests + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Build libraries + run: ./pkg/mac/build-static-library.sh + + - name: Test static libraries + run: | + export PULSAR_DIR=$PWD/pkg/mac/.install + echo "Build with static library" + clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a + ./static.out + echo "Build with dynamic library" + clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar + ./dynamic.out + # Job that will be required to complete and depends on all the other jobs check-completion: name: Check Completion diff --git a/.gitignore b/.gitignore index a569bda0..a1e5224a 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,5 @@ vcpkg_installed/ .tests-container-id.txt Testing .test-token.txt +pkg/mac/.build +pkg/mac/.install diff --git a/CMakeLists.txt b/CMakeLists.txt index c8d6a00b..fb4f1b12 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,9 +88,17 @@ find_package(Threads REQUIRED) MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT}) set(Boost_NO_BOOST_CMAKE ON) + +if (APPLE AND NOT LINK_STATIC) + # The latest Protobuf dependency on macOS requires the C++17 support and + # it could only be found by the CONFIG mode + set(LATEST_PROTOBUF TRUE) +else () + set(LATEST_PROTOBUF FALSE) +endif () + if (NOT CMAKE_CXX_STANDARD) - if (APPLE) - # The latest Protobuf dependency on macOS requires the C++17 support + if (LATEST_PROTOBUF) set(CMAKE_CXX_STANDARD 17) else () set(CMAKE_CXX_STANDARD 11) @@ -143,7 +151,7 @@ find_package(OpenSSL REQUIRED) message("OPENSSL_INCLUDE_DIR: " ${OPENSSL_INCLUDE_DIR}) message("OPENSSL_LIBRARIES: " ${OPENSSL_LIBRARIES}) -if (APPLE) +if (LATEST_PROTOBUF) # See https://github.com/apache/arrow/issues/35987 add_definitions(-DPROTOBUF_USE_DLLS) # Use Config mode to avoid FindProtobuf.cmake does not find the Abseil library @@ -318,7 +326,7 @@ set(COMMON_LIBS ${CMAKE_DL_LIBS} ) -if (APPLE) +if (LATEST_PROTOBUF) # Protobuf_LIBRARIES is empty when finding Protobuf in Config mode set(COMMON_LIBS ${COMMON_LIBS} protobuf::libprotobuf) else () diff --git a/dependencies.yaml b/dependencies.yaml index f206ccc4..18be4383 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -23,5 +23,5 @@ protobuf: 3.20.0 zlib: 1.2.12 zstd: 1.5.2 snappy: 1.1.9 -openssl: 1.1.1q +openssl: 1.1.1v curl: 8.4.0 diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh new file mode 100755 index 00000000..f0253fba --- /dev/null +++ b/pkg/mac/build-static-library.sh @@ -0,0 +1,190 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -ex +cd `dirname $0` + +pip3 install pyyaml + +MACOSX_DEPLOYMENT_TARGET=10.15 +if [[ -z ${ARCH} ]]; then + ARCH=`uname -m` +fi + +BUILD_DIR=$PWD/.build +INSTALL_DIR=$PWD/.install +PREFIX=$BUILD_DIR/install +mkdir -p $BUILD_DIR +cp -f ../../build-support/dep-version.py $BUILD_DIR/ +cp -f ../../dependencies.yaml $BUILD_DIR/ + +pushd $BUILD_DIR + +BOOST_VERSION=$(./dep-version.py boost) +ZLIB_VERSION=$(./dep-version.py zlib) +OPENSSL_VERSION=$(./dep-version.py openssl) +PROTOBUF_VERSION=$(./dep-version.py protobuf) +ZSTD_VERSION=$(./dep-version.py zstd) +SNAPPY_VERSION=$(./dep-version.py snappy) +CURL_VERSION=$(./dep-version.py curl) + +BOOST_VERSION_=${BOOST_VERSION//./_} +if [ ! -f boost/.done ]; then + echo "Building Boost $BOOST_VERSION" + curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz + tar zxf boost_${BOOST_VERSION_}.tar.gz + mkdir -p $PREFIX/include + cp -rf boost_${BOOST_VERSION_}/boost $PREFIX/include/ + mkdir -p boost + touch boost/.done +else + echo "Using cached Boost $BOOST_VERSION" +fi + +if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then + echo "Building ZLib $ZLIB_VERSION" + curl -O -L https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz + tar zxf zlib-${ZLIB_VERSION}.tar.gz + pushd zlib-$ZLIB_VERSION + CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX + make -j16 + make install + touch .done + popd +else + echo "Using cached ZLib $ZLIB_VERSION" +fi + +OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') +if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then + echo "Building OpenSSL $OPENSSL_VERSION" + curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz + tar zxf OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz + + pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} + if [[ $ARCH = 'arm64' ]]; then + PLATFORM=darwin64-arm64-cc + else + PLATFORM=darwin64-x86_64-cc + fi + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./Configure --prefix=$PREFIX no-shared no-unit-test $PLATFORM + make -j8 >/dev/null + make install_sw >/dev/null + popd + + touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done +else + echo "Using cached OpenSSL $OPENSSL_VERSION" +fi + +if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then + echo "Building Protobuf $PROTOBUF_VERSION" + curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz + tar zxf protobuf-cpp-${PROTOBUF_VERSION}.tar.gz + pushd protobuf-${PROTOBUF_VERSION} + pushd cmake/ + # Build protoc that can run on both x86 and arm architectures + cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch x86_64 -arch arm64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + -Dprotobuf_BUILD_TESTS=OFF \ + -DCMAKE_INSTALL_PREFIX=$PREFIX + cmake --build build -j16 --target install + popd + + # Retain the library for one architecture so that `ar` can work on the library + pushd $PREFIX/lib + mv libprotobuf.a libprotobuf_universal.a + lipo libprotobuf_universal.a -thin ${ARCH} -output libprotobuf.a + popd + touch .done + popd +else + echo "Using cached Protobuf $PROTOBUF_VERSION" +fi + +if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then + echo "Building ZStd $ZSTD_VERSION" + curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz + tar zxf zstd-${ZSTD_VERSION}.tar.gz + pushd zstd-${ZSTD_VERSION} + CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \ + make -j16 -C lib install-static install-includes + touch .done + popd +else + echo "Using cached ZStd $ZSTD_VERSION" +fi + +if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then + echo "Building Snappy $SNAPPY_VERSION" + curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz + tar zxf ${SNAPPY_VERSION}.tar.gz + pushd snappy-${SNAPPY_VERSION} + CXXFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF + make -j16 + make install + touch .done + popd +else + echo "Using cached Snappy $SNAPPY_VERSION" +fi + +if [ ! -f curl-${CURL_VERSION}/.done ]; then + echo "Building LibCurl $CURL_VERSION" + CURL_VERSION_=${CURL_VERSION//./_} + curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz + tar zxf curl-${CURL_VERSION}.tar.gz + pushd curl-${CURL_VERSION} + # Force the compiler to find the OpenSSL headers instead of the headers in the system path like /usr/local/include/openssl. + cp -rf $PREFIX/include/openssl include/ + CFLAGS="-I$PREFIX/include -fPIC -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./configure --with-ssl=$PREFIX \ + --without-nghttp2 \ + --without-libidn2 \ + --disable-ldap \ + --without-brotli \ + --without-secure-transport \ + --without-librtmp \ + --disable-ipv6 \ + --host=$ARCH-apple-darwin \ + --prefix=$PREFIX + make -j16 install + touch .done + popd +else + echo "Using cached LibCurl $CURL_VERSION" +fi + +popd # pkg/mac +cd ../../ # project root + +cmake -B build-static -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ + -DLINK_STATIC=ON \ + -DBUILD_TESTS=OFF \ + -DBUILD_DYNAMIC_LIB=ON \ + -DBUILD_STATIC_LIB=ON \ + -DCMAKE_OSX_ARCHITECTURES=${ARCH} \ + -DCMAKE_PREFIX_PATH=$PREFIX \ + -DOPENSSL_ROOT_DIR=$PREFIX \ + -DPROTOC_PATH=$PREFIX/bin/protoc \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ + -DCMAKE_BUILD_TYPE=Release +cmake --build build-static -j16 --target install From 0bbc15502905d19c630d237b5e102bfb996bb098 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 22 Nov 2023 12:04:22 +0800 Subject: [PATCH 11/54] Gather the macOS binaries when releasing (#355) --- build-support/stage-release.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build-support/stage-release.sh b/build-support/stage-release.sh index 2f26d383..c6665948 100755 --- a/build-support/stage-release.sh +++ b/build-support/stage-release.sh @@ -43,6 +43,11 @@ pushd "$DEST_PATH" tar cvzf x64-windows-static.tar.gz x64-windows-static tar cvzf x86-windows-static.tar.gz x86-windows-static rm -r x64-windows-static x86-windows-static +mv macos-arm64.zip macos-arm64 +mv macos-arm64/* . +mv macos-x86_64.zip macos-x86_64 +mv macos-x86_64/* . +rm -rf macos-x86_64/ macos-arm64/ popd # Sign all files From d2094828f9fe756b315ba194e1f8a69ca24ac6b4 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 24 Nov 2023 12:35:01 +0800 Subject: [PATCH 12/54] Fix HTTP lookup segfault when the redirect host cannot be resolved (#356) ### Motivation When the host of the redirect URL cannot be resolved, segmentation fault will happen: https://github.com/apache/pulsar-client-cpp/blob/0bbc15502905d19c630d237b5e102bfb996bb098/lib/CurlWrapper.h#L173-L175 In this case, `curl` will be `nullptr`. Assigning a nullptr to a `std::string` is an undefined behavior that might cause segfault. ### Modifications Check if `url` is nullptr in `CurlWrapper::get` and before assigning it to the `redirectUrl` field. Improve the `HTTPLookupService::sendHTTPRequest` method by configuring the `maxLookupRedirects` instead of a loop and print more detailed error messages. --- lib/CurlWrapper.h | 5 +- lib/HTTPLookupService.cc | 149 +++++++++++++++++---------------------- 2 files changed, 68 insertions(+), 86 deletions(-) diff --git a/lib/CurlWrapper.h b/lib/CurlWrapper.h index 89b7919d..749a79c3 100644 --- a/lib/CurlWrapper.h +++ b/lib/CurlWrapper.h @@ -172,7 +172,10 @@ inline CurlWrapper::Result CurlWrapper::get(const std::string& url, const std::s if (responseCode == 307 || responseCode == 302 || responseCode == 301) { char* url; curl_easy_getinfo(handle_, CURLINFO_REDIRECT_URL, &url); - result.redirectUrl = url; + // `url` is null when the host of the redirect URL cannot be resolved + if (url) { + result.redirectUrl = url; + } } return result; } diff --git a/lib/HTTPLookupService.cc b/lib/HTTPLookupService.cc index 4ec72c1e..0959af2a 100644 --- a/lib/HTTPLookupService.cc +++ b/lib/HTTPLookupService.cc @@ -191,98 +191,77 @@ Result HTTPLookupService::sendHTTPRequest(std::string completeUrl, std::string & Result HTTPLookupService::sendHTTPRequest(std::string completeUrl, std::string &responseData, long &responseCode) { - uint16_t reqCount = 0; - Result retResult = ResultOk; - while (++reqCount <= maxLookupRedirects_) { - // Authorization data - AuthenticationDataPtr authDataContent; - Result authResult = authenticationPtr_->getAuthData(authDataContent); - if (authResult != ResultOk) { - LOG_ERROR("Failed to getAuthData: " << authResult); - return authResult; - } + // Authorization data + AuthenticationDataPtr authDataContent; + Result authResult = authenticationPtr_->getAuthData(authDataContent); + if (authResult != ResultOk) { + LOG_ERROR("Failed to getAuthData: " << authResult); + return authResult; + } - CurlWrapper curl; - if (!curl.init()) { - LOG_ERROR("Unable to curl_easy_init for url " << completeUrl); - return ResultLookupError; - } + CurlWrapper curl; + if (!curl.init()) { + LOG_ERROR("Unable to curl_easy_init for url " << completeUrl); + return ResultLookupError; + } - std::unique_ptr tlsContext; - if (isUseTls_) { - tlsContext.reset(new CurlWrapper::TlsContext); - tlsContext->trustCertsFilePath = tlsTrustCertsFilePath_; - tlsContext->validateHostname = tlsValidateHostname_; - tlsContext->allowInsecure = tlsAllowInsecure_; - if (authDataContent->hasDataForTls()) { - tlsContext->certPath = authDataContent->getTlsCertificates(); - tlsContext->keyPath = authDataContent->getTlsPrivateKey(); - } else { - tlsContext->certPath = tlsCertificateFilePath_; - tlsContext->keyPath = tlsPrivateFilePath_; - } + std::unique_ptr tlsContext; + if (isUseTls_) { + tlsContext.reset(new CurlWrapper::TlsContext); + tlsContext->trustCertsFilePath = tlsTrustCertsFilePath_; + tlsContext->validateHostname = tlsValidateHostname_; + tlsContext->allowInsecure = tlsAllowInsecure_; + if (authDataContent->hasDataForTls()) { + tlsContext->certPath = authDataContent->getTlsCertificates(); + tlsContext->keyPath = authDataContent->getTlsPrivateKey(); + } else { + tlsContext->certPath = tlsCertificateFilePath_; + tlsContext->keyPath = tlsPrivateFilePath_; } + } - LOG_INFO("Curl [" << reqCount << "] Lookup Request sent for " << completeUrl); - CurlWrapper::Options options; - options.timeoutInSeconds = lookupTimeoutInSeconds_; - options.userAgent = std::string("Pulsar-CPP-v") + PULSAR_VERSION_STR; - options.maxLookupRedirects = 1; // redirection is implemented by the outer loop - auto result = curl.get(completeUrl, authDataContent->getHttpHeaders(), options, tlsContext.get()); - const auto &error = result.error; - if (!error.empty()) { - LOG_ERROR(completeUrl << " failed: " << error); - return ResultConnectError; - } + LOG_INFO("Curl Lookup Request sent for " << completeUrl); + CurlWrapper::Options options; + options.timeoutInSeconds = lookupTimeoutInSeconds_; + options.userAgent = std::string("Pulsar-CPP-v") + PULSAR_VERSION_STR; + options.maxLookupRedirects = maxLookupRedirects_; + auto result = curl.get(completeUrl, authDataContent->getHttpHeaders(), options, tlsContext.get()); + const auto &error = result.error; + if (!error.empty()) { + LOG_ERROR(completeUrl << " failed: " << error); + return ResultConnectError; + } - responseData = result.responseData; - responseCode = result.responseCode; - auto res = result.code; - LOG_INFO("Response received for url " << completeUrl << " responseCode " << responseCode - << " curl res " << res); - - const auto &redirectUrl = result.redirectUrl; - switch (res) { - case CURLE_OK: - if (responseCode == 200) { - retResult = ResultOk; - } else if (!redirectUrl.empty()) { - LOG_INFO("Response from url " << completeUrl << " to new url " << redirectUrl); - completeUrl = redirectUrl; - retResult = ResultLookupError; - } else { - retResult = ResultLookupError; - } - break; - case CURLE_COULDNT_CONNECT: - LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); - retResult = ResultRetryable; - break; - case CURLE_COULDNT_RESOLVE_PROXY: - case CURLE_COULDNT_RESOLVE_HOST: - case CURLE_HTTP_RETURNED_ERROR: - LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); - retResult = ResultConnectError; - break; - case CURLE_READ_ERROR: - LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); - retResult = ResultReadError; - break; - case CURLE_OPERATION_TIMEDOUT: - LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); - retResult = ResultTimeout; - break; - default: - LOG_ERROR("Response failed for url " << completeUrl << ". Error Code " << res); - retResult = ResultLookupError; - break; - } - if (redirectUrl.empty()) { - break; - } + responseData = result.responseData; + responseCode = result.responseCode; + auto res = result.code; + if (res == CURLE_OK) { + LOG_INFO("Response received for url " << completeUrl << " responseCode " << responseCode); + } else if (res == CURLE_TOO_MANY_REDIRECTS) { + LOG_ERROR("Response received for url " << completeUrl << ": " << curl_easy_strerror(res) + << ", curl error: " << result.serverError + << ", redirect URL: " << result.redirectUrl); + } else { + LOG_ERROR("Response failed for url " << completeUrl << ": " << curl_easy_strerror(res) + << ", curl error: " << result.serverError); } - return retResult; + switch (res) { + case CURLE_OK: + return ResultOk; + case CURLE_COULDNT_CONNECT: + return ResultRetryable; + case CURLE_COULDNT_RESOLVE_PROXY: + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_HTTP_RETURNED_ERROR: + return ResultConnectError; + case CURLE_READ_ERROR: + return ResultReadError; + case CURLE_OPERATION_TIMEDOUT: + return ResultTimeout; + default: + return ResultLookupError; + } } LookupDataResultPtr HTTPLookupService::parsePartitionData(const std::string &json) { From 751c807c1c3b6ea17e6611e5c8e436e8aac7a6a6 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 4 Dec 2023 10:25:34 +0800 Subject: [PATCH 13/54] Install Version.h when installing by CMakeLists (#361) ### Motivation The macOS pre-built binaries don't include the Version.h generated from CMake. It's because `cmake --target install` does not install the `Version.h`, which is generated by `configure_file`. ### Modifications Install the `Version.h`. --- lib/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index eca76380..8bd9749c 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -156,3 +156,4 @@ if (BUILD_DYNAMIC_LIB) endif() install(DIRECTORY "../include/pulsar" DESTINATION include) +install(FILES "${PROJECT_BINARY_DIR}/include/pulsar/Version.h" DESTINATION include/pulsar/) From 63d494f57df126cf633396c618886eeb79946408 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 4 Dec 2023 10:25:44 +0800 Subject: [PATCH 14/54] Add the Consumer::getConsumerName API (#360) ### Motivation C++ client supports setting the consumer name via `ConsumerConfiguration::setConsumerName` but it does not support getting the consumer name for a given consumer. ### Modifications Add the `getConsumerName` method to `ConsumerImplBase` to get the consumer name from the config and a public method with the same name to `Consumer`. Add `ConsumerTest.testConsumerName` to verify the consumer name is set correctly in the subscribe command. --- include/pulsar/Consumer.h | 7 +++++- lib/Consumer.cc | 4 ++++ lib/ConsumerImpl.cc | 7 +++--- lib/ConsumerImpl.h | 1 - lib/ConsumerImplBase.cc | 3 ++- lib/ConsumerImplBase.h | 4 ++++ tests/ClientTest.cc | 17 ++++---------- tests/ConsumerTest.cc | 48 ++++++++++++++++++++++++++++++++++++++- tests/PulsarAdminHelper.h | 42 ++++++++++++++++++++++++++++++++++ 9 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 tests/PulsarAdminHelper.h diff --git a/include/pulsar/Consumer.h b/include/pulsar/Consumer.h index 6596894d..63defdb8 100644 --- a/include/pulsar/Consumer.h +++ b/include/pulsar/Consumer.h @@ -48,10 +48,15 @@ class PULSAR_PUBLIC Consumer { const std::string& getTopic() const; /** - * @return the consumer name + * @return the subscription name */ const std::string& getSubscriptionName() const; + /** + * @return the consumer name + */ + const std::string& getConsumerName() const; + /** * Unsubscribe the current consumer from the topic. * diff --git a/lib/Consumer.cc b/lib/Consumer.cc index bda7f862..031cdff9 100644 --- a/lib/Consumer.cc +++ b/lib/Consumer.cc @@ -39,6 +39,10 @@ const std::string& Consumer::getSubscriptionName() const { return impl_ != NULL ? impl_->getSubscriptionName() : EMPTY_STRING; } +const std::string& Consumer::getConsumerName() const { + return impl_ ? impl_->getConsumerName() : EMPTY_STRING; +} + Result Consumer::unsubscribe() { if (!impl_) { return ResultConsumerNotInitialized; diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index b4666836..955fb325 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -83,7 +83,6 @@ ConsumerImpl::ConsumerImpl(const ClientImplPtr client, const std::string& topic, availablePermits_(0), receiverQueueRefillThreshold_(config_.getReceiverQueueSize() / 2), consumerId_(client->newConsumerId()), - consumerName_(config_.getConsumerName()), consumerStr_("[" + topic + ", " + subscriptionName + ", " + std::to_string(consumerId_) + "] "), messageListenerRunning_(true), negativeAcksTracker_(client, *this, conf), @@ -249,7 +248,7 @@ Future ConsumerImpl::connectionOpened(const ClientConnectionPtr& c ClientImplPtr client = client_.lock(); uint64_t requestId = client->newRequestId(); SharedBuffer cmd = Commands::newSubscribe( - topic(), subscription_, consumerId_, requestId, getSubType(), consumerName_, subscriptionMode_, + topic(), subscription_, consumerId_, requestId, getSubType(), getConsumerName(), subscriptionMode_, subscribeMessageId, readCompacted_, config_.getProperties(), config_.getSubscriptionProperties(), config_.getSchema(), getInitialPosition(), config_.isReplicateSubscriptionStateEnabled(), config_.getKeySharedPolicy(), config_.getPriorityLevel()); @@ -1780,7 +1779,7 @@ void ConsumerImpl::processPossibleToDLQ(const MessageId& messageId, ProcessDLQCa } if (result != ResultOk) { LOG_WARN("{" << self->topic() << "} {" << self->subscription_ << "} {" - << self->consumerName_ << "} Failed to acknowledge the message {" + << self->getConsumerName() << "} Failed to acknowledge the message {" << originMessageId << "} of the original topic but send to the DLQ successfully : " << result); @@ -1793,7 +1792,7 @@ void ConsumerImpl::processPossibleToDLQ(const MessageId& messageId, ProcessDLQCa }); } else { LOG_WARN("{" << self->topic() << "} {" << self->subscription_ << "} {" - << self->consumerName_ << "} Failed to send DLQ message to {" + << self->getConsumerName() << "} Failed to send DLQ message to {" << self->deadLetterPolicy_.getDeadLetterTopic() << "} for message id " << "{" << originMessageId << "} : " << res); cb(false); diff --git a/lib/ConsumerImpl.h b/lib/ConsumerImpl.h index 61d96b1c..5e9ef04e 100644 --- a/lib/ConsumerImpl.h +++ b/lib/ConsumerImpl.h @@ -216,7 +216,6 @@ class ConsumerImpl : public ConsumerImplBase { std::atomic_int availablePermits_; const int receiverQueueRefillThreshold_; uint64_t consumerId_; - std::string consumerName_; const std::string consumerStr_; int32_t partitionIndex_ = -1; Promise consumerCreatedPromise_; diff --git a/lib/ConsumerImplBase.cc b/lib/ConsumerImplBase.cc index 6c86aa44..39212371 100644 --- a/lib/ConsumerImplBase.cc +++ b/lib/ConsumerImplBase.cc @@ -33,7 +33,8 @@ ConsumerImplBase::ConsumerImplBase(ClientImplPtr client, const std::string& topi const ConsumerConfiguration& conf, ExecutorServicePtr listenerExecutor) : HandlerBase(client, topic, backoff), listenerExecutor_(listenerExecutor), - batchReceivePolicy_(conf.getBatchReceivePolicy()) { + batchReceivePolicy_(conf.getBatchReceivePolicy()), + consumerName_(conf.getConsumerName()) { auto userBatchReceivePolicy = conf.getBatchReceivePolicy(); if (userBatchReceivePolicy.getMaxNumMessages() > conf.getReceiverQueueSize()) { batchReceivePolicy_ = diff --git a/lib/ConsumerImplBase.h b/lib/ConsumerImplBase.h index 2f3420c6..1b7e86e1 100644 --- a/lib/ConsumerImplBase.h +++ b/lib/ConsumerImplBase.h @@ -82,6 +82,8 @@ class ConsumerImplBase : public HandlerBase { virtual const std::string& getName() const override = 0; virtual void hasMessageAvailableAsync(HasMessageAvailableCallback callback) = 0; + const std::string& getConsumerName() const noexcept { return consumerName_; } + protected: // overrided methods from HandlerBase Future connectionOpened(const ClientConnectionPtr& cnx) override { @@ -106,6 +108,8 @@ class ConsumerImplBase : public HandlerBase { virtual bool hasEnoughMessagesForBatchReceive() const = 0; private: + const std::string consumerName_; + virtual void setNegativeAcknowledgeEnabledForTesting(bool enabled) = 0; friend class MultiTopicsConsumerImpl; diff --git a/tests/ClientTest.cc b/tests/ClientTest.cc index 5b4dd2e8..8c5a32ab 100644 --- a/tests/ClientTest.cc +++ b/tests/ClientTest.cc @@ -21,13 +21,11 @@ #include #include -#include -#include #include #include #include -#include "HttpHelper.h" +#include "PulsarAdminHelper.h" #include "PulsarFriend.h" #include "WaitUtils.h" #include "lib/ClientConnection.h" @@ -335,18 +333,13 @@ class PulsarWrapper { // When `subscription` is empty, get client versions of the producers. // Otherwise, get client versions of the consumers under the subscribe. static std::vector getClientVersions(const std::string &topic, std::string subscription = "") { - const auto url = adminUrl + "admin/v2/persistent/public/default/" + topic + "/stats"; - std::string responseData; - int res = makeGetRequest(url, responseData); - if (res != 200) { - LOG_ERROR(url << " failed: " << res); + boost::property_tree::ptree root; + const auto error = getTopicStats(topic, root); + if (!error.empty()) { + LOG_ERROR(error); return {}; } - std::stringstream stream; - stream << responseData; - boost::property_tree::ptree root; - boost::property_tree::read_json(stream, root); std::vector versions; if (subscription.empty()) { for (auto &child : root.get_child("publishers")) { diff --git a/tests/ConsumerTest.cc b/tests/ConsumerTest.cc index 0836fbf4..3ca48cea 100644 --- a/tests/ConsumerTest.cc +++ b/tests/ConsumerTest.cc @@ -29,8 +29,8 @@ #include #include -#include "HttpHelper.h" #include "NoOpsCryptoKeyReader.h" +#include "PulsarAdminHelper.h" #include "PulsarFriend.h" #include "SynchronizedQueue.h" #include "WaitUtils.h" @@ -1430,4 +1430,50 @@ TEST(ConsumerTest, testCloseAgainBeforeCloseDone) { ASSERT_TRUE(*done); } +inline std::string getConsumerName(const std::string& topic) { + boost::property_tree::ptree root; + const auto error = getTopicStats(topic, root); + if (!error.empty()) { + LOG_INFO(error); + return {}; + } + return root.get_child("subscriptions") + .get_child("sub") + .get_child("consumers") + .front() + .second.get("consumerName"); +} + +TEST(ConsumerTest, testConsumerName) { + Client client{lookupUrl}; + Consumer consumer; + ASSERT_TRUE(consumer.getConsumerName().empty()); + const auto topic1 = "consumer-test-consumer-name-1"; + const auto topic2 = "consumer-test-consumer-name-2"; + + // Default consumer name + ASSERT_EQ(ResultOk, client.subscribe(topic1, "sub", consumer)); + LOG_INFO("Random consumer name: " << consumer.getConsumerName()); + ASSERT_FALSE(consumer.getConsumerName().empty()); // a random name + ASSERT_EQ(consumer.getConsumerName(), getConsumerName(topic1)); + consumer.close(); + + // Single-topic consumer + ConsumerConfiguration conf; + const std::string consumerName = "custom-consumer"; + conf.setConsumerName(consumerName); + ASSERT_EQ(ResultOk, client.subscribe(topic1, "sub", conf, consumer)); + ASSERT_EQ(consumerName, consumer.getConsumerName()); + ASSERT_EQ(consumerName, getConsumerName(topic1)); + consumer.close(); + + // Multi-topics consumer + ASSERT_EQ(ResultOk, client.subscribe(std::vector{topic1, topic2}, "sub", conf, consumer)); + ASSERT_EQ(consumerName, consumer.getConsumerName()); + ASSERT_EQ(consumerName, getConsumerName(topic1)); + ASSERT_EQ(consumerName, getConsumerName(topic2)); + + client.close(); +} + } // namespace pulsar diff --git a/tests/PulsarAdminHelper.h b/tests/PulsarAdminHelper.h new file mode 100644 index 00000000..944c7b8d --- /dev/null +++ b/tests/PulsarAdminHelper.h @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include + +#include "HttpHelper.h" + +namespace pulsar { + +inline std::string getTopicStats(const std::string& topic, boost::property_tree::ptree& root) { + const auto url = "http://localhost:8080/admin/v2/persistent/public/default/" + topic + "/stats"; + std::string responseData; + int code = makeGetRequest(url, responseData); + if (code != 200) { + return url + " failed: " + std::to_string(code); + } + + std::stringstream stream; + stream << responseData; + boost::property_tree::read_json(stream, root); + return ""; +} + +} // namespace pulsar From 24ab12c74127276c0e264ce869e553749b9170e8 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 6 Dec 2023 10:37:35 +0800 Subject: [PATCH 15/54] Fix accessing destroyed objects in the callback of async_wait (#362) Fixes https://github.com/apache/pulsar-client-cpp/issues/358 Fixes https://github.com/apache/pulsar-client-cpp/issues/359 ### Motivation `async_wait` is not used correctly in some places. A callback that captures the `this` pointer or reference to `this` is passed to `async_wait`, if this object is destroyed when the callback is called, an invalid memory access will happen. ### Modifications Use the following pattern in all `async_wait` calls. ```c++ std::weak_ptr weakSelf{shared_from_this()}; timer_->async_wait([weakSelf](/* ... */) { if (auto self = weakSelf.lock()) { self->foo(); } }); ``` --- lib/ConsumerImpl.cc | 12 +++++++----- lib/ConsumerImpl.h | 2 +- lib/MultiTopicsConsumerImpl.cc | 1 + lib/NegativeAcksTracker.cc | 7 ++++++- lib/NegativeAcksTracker.h | 2 +- lib/PatternMultiTopicsConsumerImpl.cc | 17 +++++++++++++---- lib/PatternMultiTopicsConsumerImpl.h | 4 ++++ lib/UnAckedMessageTrackerEnabled.cc | 19 ++++++++++--------- lib/UnAckedMessageTrackerEnabled.h | 19 +++++++++++-------- lib/UnAckedMessageTrackerInterface.h | 2 ++ tests/BasicEndToEndTest.cc | 2 ++ tests/ConsumerTest.cc | 4 +++- 12 files changed, 61 insertions(+), 30 deletions(-) diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index 955fb325..f9770d80 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -85,7 +85,7 @@ ConsumerImpl::ConsumerImpl(const ClientImplPtr client, const std::string& topic, consumerId_(client->newConsumerId()), consumerStr_("[" + topic + ", " + subscriptionName + ", " + std::to_string(consumerId_) + "] "), messageListenerRunning_(true), - negativeAcksTracker_(client, *this, conf), + negativeAcksTracker_(std::make_shared(client, *this, conf)), readCompacted_(conf.isReadCompacted()), startMessageId_(startMessageId), maxPendingChunkedMessage_(conf.getMaxPendingChunkedMessage()), @@ -104,6 +104,7 @@ ConsumerImpl::ConsumerImpl(const ClientImplPtr client, const std::string& topic, } else { unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerDisabled()); } + unAckedMessageTrackerPtr_->start(); // Setup stats reporter. unsigned int statsIntervalInSeconds = client->getClientConfig().getStatsIntervalInSeconds(); @@ -1227,7 +1228,7 @@ std::pair ConsumerImpl::prepareCumulativeAck(const MessageId& m void ConsumerImpl::negativeAcknowledge(const MessageId& messageId) { unAckedMessageTrackerPtr_->remove(messageId); - negativeAcksTracker_.add(messageId); + negativeAcksTracker_->add(messageId); } void ConsumerImpl::disconnectConsumer() { @@ -1265,7 +1266,7 @@ void ConsumerImpl::closeAsync(ResultCallback originalCallback) { if (ackGroupingTrackerPtr_) { ackGroupingTrackerPtr_->close(); } - negativeAcksTracker_.close(); + negativeAcksTracker_->close(); ClientConnectionPtr cnx = getCnx().lock(); if (!cnx) { @@ -1303,7 +1304,7 @@ void ConsumerImpl::shutdown() { if (client) { client->cleanupConsumer(this); } - negativeAcksTracker_.close(); + negativeAcksTracker_->close(); cancelTimers(); consumerCreatedPromise_.setFailed(ResultAlreadyClosed); failPendingReceiveCallback(); @@ -1608,7 +1609,7 @@ void ConsumerImpl::internalGetLastMessageIdAsync(const BackoffPtr& backoff, Time } void ConsumerImpl::setNegativeAcknowledgeEnabledForTesting(bool enabled) { - negativeAcksTracker_.setEnabledForTesting(enabled); + negativeAcksTracker_->setEnabledForTesting(enabled); } void ConsumerImpl::trackMessage(const MessageId& messageId) { @@ -1695,6 +1696,7 @@ void ConsumerImpl::cancelTimers() noexcept { boost::system::error_code ec; batchReceiveTimer_->cancel(ec); checkExpiredChunkedTimer_->cancel(ec); + unAckedMessageTrackerPtr_->stop(); } void ConsumerImpl::processPossibleToDLQ(const MessageId& messageId, ProcessDLQCallBack cb) { diff --git a/lib/ConsumerImpl.h b/lib/ConsumerImpl.h index 5e9ef04e..0f801f63 100644 --- a/lib/ConsumerImpl.h +++ b/lib/ConsumerImpl.h @@ -223,7 +223,7 @@ class ConsumerImpl : public ConsumerImplBase { CompressionCodecProvider compressionCodecProvider_; UnAckedMessageTrackerPtr unAckedMessageTrackerPtr_; BrokerConsumerStatsImpl brokerConsumerStats_; - NegativeAcksTracker negativeAcksTracker_; + std::shared_ptr negativeAcksTracker_; AckGroupingTrackerPtr ackGroupingTrackerPtr_; MessageCryptoPtr msgCrypto_; diff --git a/lib/MultiTopicsConsumerImpl.cc b/lib/MultiTopicsConsumerImpl.cc index abc54c80..15f9d9b8 100644 --- a/lib/MultiTopicsConsumerImpl.cc +++ b/lib/MultiTopicsConsumerImpl.cc @@ -86,6 +86,7 @@ MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, const std } else { unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerDisabled()); } + unAckedMessageTrackerPtr_->start(); auto partitionsUpdateInterval = static_cast(client->conf().getPartitionsUpdateInterval()); if (partitionsUpdateInterval > 0) { partitionsUpdateTimer_ = listenerExecutor_->createDeadlineTimer(); diff --git a/lib/NegativeAcksTracker.cc b/lib/NegativeAcksTracker.cc index 5c3ef3f8..0dd73589 100644 --- a/lib/NegativeAcksTracker.cc +++ b/lib/NegativeAcksTracker.cc @@ -49,8 +49,13 @@ void NegativeAcksTracker::scheduleTimer() { if (closed_) { return; } + std::weak_ptr weakSelf{shared_from_this()}; timer_->expires_from_now(timerInterval_); - timer_->async_wait(std::bind(&NegativeAcksTracker::handleTimer, this, std::placeholders::_1)); + timer_->async_wait([weakSelf](const boost::system::error_code &ec) { + if (auto self = weakSelf.lock()) { + self->handleTimer(ec); + } + }); } void NegativeAcksTracker::handleTimer(const boost::system::error_code &ec) { diff --git a/lib/NegativeAcksTracker.h b/lib/NegativeAcksTracker.h index 029f7d24..4b489844 100644 --- a/lib/NegativeAcksTracker.h +++ b/lib/NegativeAcksTracker.h @@ -40,7 +40,7 @@ using DeadlineTimerPtr = std::shared_ptr; class ExecutorService; using ExecutorServicePtr = std::shared_ptr; -class NegativeAcksTracker { +class NegativeAcksTracker : public std::enable_shared_from_this { public: NegativeAcksTracker(ClientImplPtr client, ConsumerImpl &consumer, const ConsumerConfiguration &conf); diff --git a/lib/PatternMultiTopicsConsumerImpl.cc b/lib/PatternMultiTopicsConsumerImpl.cc index e100a1c3..23e445ee 100644 --- a/lib/PatternMultiTopicsConsumerImpl.cc +++ b/lib/PatternMultiTopicsConsumerImpl.cc @@ -47,8 +47,13 @@ const PULSAR_REGEX_NAMESPACE::regex PatternMultiTopicsConsumerImpl::getPattern() void PatternMultiTopicsConsumerImpl::resetAutoDiscoveryTimer() { autoDiscoveryRunning_ = false; autoDiscoveryTimer_->expires_from_now(seconds(conf_.getPatternAutoDiscoveryPeriod())); - autoDiscoveryTimer_->async_wait( - std::bind(&PatternMultiTopicsConsumerImpl::autoDiscoveryTimerTask, this, std::placeholders::_1)); + + auto weakSelf = weak_from_this(); + autoDiscoveryTimer_->async_wait([weakSelf](const boost::system::error_code& err) { + if (auto self = weakSelf.lock()) { + self->autoDiscoveryTimerTask(err); + } + }); } void PatternMultiTopicsConsumerImpl::autoDiscoveryTimerTask(const boost::system::error_code& err) { @@ -222,8 +227,12 @@ void PatternMultiTopicsConsumerImpl::start() { if (conf_.getPatternAutoDiscoveryPeriod() > 0) { autoDiscoveryTimer_->expires_from_now(seconds(conf_.getPatternAutoDiscoveryPeriod())); - autoDiscoveryTimer_->async_wait( - std::bind(&PatternMultiTopicsConsumerImpl::autoDiscoveryTimerTask, this, std::placeholders::_1)); + auto weakSelf = weak_from_this(); + autoDiscoveryTimer_->async_wait([weakSelf](const boost::system::error_code& err) { + if (auto self = weakSelf.lock()) { + self->autoDiscoveryTimerTask(err); + } + }); } } diff --git a/lib/PatternMultiTopicsConsumerImpl.h b/lib/PatternMultiTopicsConsumerImpl.h index f13750a9..5d3ba9ec 100644 --- a/lib/PatternMultiTopicsConsumerImpl.h +++ b/lib/PatternMultiTopicsConsumerImpl.h @@ -86,6 +86,10 @@ class PatternMultiTopicsConsumerImpl : public MultiTopicsConsumerImpl { void onTopicsRemoved(NamespaceTopicsPtr removedTopics, ResultCallback callback); void handleOneTopicAdded(const Result result, const std::string& topic, std::shared_ptr> topicsNeedCreate, ResultCallback callback); + + std::weak_ptr weak_from_this() noexcept { + return std::static_pointer_cast(shared_from_this()); + } }; } // namespace pulsar diff --git a/lib/UnAckedMessageTrackerEnabled.cc b/lib/UnAckedMessageTrackerEnabled.cc index ff1b928f..061a1409 100644 --- a/lib/UnAckedMessageTrackerEnabled.cc +++ b/lib/UnAckedMessageTrackerEnabled.cc @@ -35,11 +35,11 @@ void UnAckedMessageTrackerEnabled::timeoutHandler() { ExecutorServicePtr executorService = client_->getIOExecutorProvider()->get(); timer_ = executorService->createDeadlineTimer(); timer_->expires_from_now(boost::posix_time::milliseconds(tickDurationInMs_)); - timer_->async_wait([&](const boost::system::error_code& ec) { - if (ec) { - LOG_DEBUG("Ignoring timer cancelled event, code[" << ec << "]"); - } else { - timeoutHandler(); + std::weak_ptr weakSelf{shared_from_this()}; + timer_->async_wait([weakSelf](const boost::system::error_code& ec) { + auto self = weakSelf.lock(); + if (self && !ec) { + self->timeoutHandler(); } }); } @@ -91,10 +91,10 @@ UnAckedMessageTrackerEnabled::UnAckedMessageTrackerEnabled(long timeoutMs, long std::set msgIds; timePartitions.push_back(msgIds); } - - timeoutHandler(); } +void UnAckedMessageTrackerEnabled::start() { timeoutHandler(); } + bool UnAckedMessageTrackerEnabled::add(const MessageId& msgId) { std::lock_guard acquire(lock_); auto id = discardBatch(msgId); @@ -172,9 +172,10 @@ void UnAckedMessageTrackerEnabled::clear() { } } -UnAckedMessageTrackerEnabled::~UnAckedMessageTrackerEnabled() { +void UnAckedMessageTrackerEnabled::stop() { + boost::system::error_code ec; if (timer_) { - timer_->cancel(); + timer_->cancel(ec); } } } /* namespace pulsar */ diff --git a/lib/UnAckedMessageTrackerEnabled.h b/lib/UnAckedMessageTrackerEnabled.h index 1453460c..6181a8a3 100644 --- a/lib/UnAckedMessageTrackerEnabled.h +++ b/lib/UnAckedMessageTrackerEnabled.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -34,19 +35,21 @@ class ConsumerImplBase; using ClientImplPtr = std::shared_ptr; using DeadlineTimerPtr = std::shared_ptr; -class UnAckedMessageTrackerEnabled : public UnAckedMessageTrackerInterface { +class UnAckedMessageTrackerEnabled : public std::enable_shared_from_this, + public UnAckedMessageTrackerInterface { public: - ~UnAckedMessageTrackerEnabled(); UnAckedMessageTrackerEnabled(long timeoutMs, ClientImplPtr, ConsumerImplBase&); UnAckedMessageTrackerEnabled(long timeoutMs, long tickDuration, ClientImplPtr, ConsumerImplBase&); - bool add(const MessageId& msgId); - bool remove(const MessageId& msgId); - void remove(const MessageIdList& msgIds); - void removeMessagesTill(const MessageId& msgId); - void removeTopicMessage(const std::string& topic); + void start() override; + void stop() override; + bool add(const MessageId& msgId) override; + bool remove(const MessageId& msgId) override; + void remove(const MessageIdList& msgIds) override; + void removeMessagesTill(const MessageId& msgId) override; + void removeTopicMessage(const std::string& topic) override; void timeoutHandler(); - void clear(); + void clear() override; protected: void timeoutHandlerHelper(); diff --git a/lib/UnAckedMessageTrackerInterface.h b/lib/UnAckedMessageTrackerInterface.h index d1fe7893..4df8819e 100644 --- a/lib/UnAckedMessageTrackerInterface.h +++ b/lib/UnAckedMessageTrackerInterface.h @@ -28,6 +28,8 @@ class UnAckedMessageTrackerInterface { public: virtual ~UnAckedMessageTrackerInterface() {} UnAckedMessageTrackerInterface() {} + virtual void start() {} + virtual void stop() {} virtual bool add(const MessageId& m) = 0; virtual bool remove(const MessageId& m) = 0; virtual void remove(const MessageIdList& msgIds) = 0; diff --git a/tests/BasicEndToEndTest.cc b/tests/BasicEndToEndTest.cc index e2c6697f..5dbccbfc 100644 --- a/tests/BasicEndToEndTest.cc +++ b/tests/BasicEndToEndTest.cc @@ -3973,6 +3973,7 @@ TEST(BasicEndToEndTest, testUnAckedMessageTrackerEnabledIndividualAck) { auto tracker0 = std::make_shared(unAckedMessagesTimeoutMs, clientImplPtr, consumerImpl0); + tracker0->start(); ASSERT_EQ(tracker0->getUnAckedMessagesTimeoutMs(), unAckedMessagesTimeoutMs); ASSERT_EQ(tracker0->getTickDurationInMs(), unAckedMessagesTimeoutMs); @@ -4048,6 +4049,7 @@ TEST(BasicEndToEndTest, testUnAckedMessageTrackerEnabledCumulativeAck) { } auto tracker = std::make_shared(unAckedMessagesTimeoutMs, clientImplPtr, consumerImpl0); + tracker->start(); for (auto idx = 0; idx < numMsg; ++idx) { ASSERT_TRUE(tracker->add(recvMsgId[idx])); } diff --git a/tests/ConsumerTest.cc b/tests/ConsumerTest.cc index 3ca48cea..f4ae88dc 100644 --- a/tests/ConsumerTest.cc +++ b/tests/ConsumerTest.cc @@ -993,6 +993,7 @@ TEST(ConsumerTest, testRedeliveryOfDecryptionFailedMessages) { auto consumer2ImplPtr = PulsarFriend::getConsumerImplPtr(consumer2); consumer2ImplPtr->unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerEnabled( 100, 100, PulsarFriend::getClientImplPtr(client), static_cast(*consumer2ImplPtr))); + consumer2ImplPtr->unAckedMessageTrackerPtr_->start(); ConsumerConfiguration consConfig3; consConfig3.setConsumerType(pulsar::ConsumerShared); @@ -1003,6 +1004,7 @@ TEST(ConsumerTest, testRedeliveryOfDecryptionFailedMessages) { auto consumer3ImplPtr = PulsarFriend::getConsumerImplPtr(consumer3); consumer3ImplPtr->unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerEnabled( 100, 100, PulsarFriend::getClientImplPtr(client), static_cast(*consumer3ImplPtr))); + consumer3ImplPtr->unAckedMessageTrackerPtr_->start(); int numberOfMessages = 20; std::string msgContent = "msg-content"; @@ -1222,7 +1224,7 @@ TEST(ConsumerTest, testNegativeAcksTrackerClose) { consumer.close(); auto consumerImplPtr = PulsarFriend::getConsumerImplPtr(consumer); - ASSERT_TRUE(consumerImplPtr->negativeAcksTracker_.nackedMessages_.empty()); + ASSERT_TRUE(consumerImplPtr->negativeAcksTracker_->nackedMessages_.empty()); client.close(); } From 27cba3e7d154f97e01911cd5de2cc0d4eaf2ef50 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 6 Dec 2023 10:41:04 +0800 Subject: [PATCH 16/54] Fix tlsTrustCertsFilePath config is not applied for OAuth2 (#364) ### Motivation https://github.com/apache/pulsar-client-cpp/pull/313 has reverted the fix of https://github.com/apache/pulsar-client-cpp/pull/190, which applies the `tlsTrustCertsFilePath` config for OAuth2 authentication. The macOS pre-built libraries are affected most because the bundled CA path is empty. ### Modification Apply the `tlsTrustCertsFilePath` for OAuth2. --- lib/ClientConfiguration.cc | 1 + lib/ClientConnection.cc | 12 ++++++++++-- lib/auth/AuthOauth2.cc | 9 +++++++-- run-unit-tests.sh | 9 ++++++++- tests/oauth2/Oauth2Test.cc | 21 +++++++++++++++++++++ 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/lib/ClientConfiguration.cc b/lib/ClientConfiguration.cc index 977a8807..63c0bf89 100644 --- a/lib/ClientConfiguration.cc +++ b/lib/ClientConfiguration.cc @@ -19,6 +19,7 @@ #include #include "ClientConfigurationImpl.h" +#include "auth/AuthOauth2.h" namespace pulsar { diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index 4086bdde..97d88476 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -33,6 +33,7 @@ #include "PulsarApi.pb.h" #include "ResultUtils.h" #include "Url.h" +#include "auth/AuthOauth2.h" #include "auth/InitialAuthData.h" #include "checksum/ChecksumProvider.h" @@ -193,6 +194,14 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: return; } + auto oauth2Auth = std::dynamic_pointer_cast(authentication_); + if (oauth2Auth) { + // Configure the TLS trust certs file for Oauth2 + auto authData = std::dynamic_pointer_cast( + std::make_shared(clientConfiguration.getTlsTrustCertsFilePath())); + oauth2Auth->getAuthData(authData); + } + if (clientConfiguration.isUseTls()) { #if BOOST_VERSION >= 105400 boost::asio::ssl::context ctx(boost::asio::ssl::context::tlsv12_client); @@ -223,8 +232,7 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: std::string tlsCertificates = clientConfiguration.getTlsCertificateFilePath(); std::string tlsPrivateKey = clientConfiguration.getTlsPrivateKeyFilePath(); - auto authData = std::dynamic_pointer_cast( - std::make_shared(clientConfiguration.getTlsTrustCertsFilePath())); + AuthenticationDataPtr authData; if (authentication_->getAuthData(authData) == ResultOk && authData->hasDataForTls()) { tlsCertificates = authData->getTlsCertificates(); tlsPrivateKey = authData->getTlsPrivateKey(); diff --git a/lib/auth/AuthOauth2.cc b/lib/auth/AuthOauth2.cc index 799843bc..c0745570 100644 --- a/lib/auth/AuthOauth2.cc +++ b/lib/auth/AuthOauth2.cc @@ -342,8 +342,13 @@ Oauth2TokenResultPtr ClientCredentialFlow::authenticate() { CurlWrapper::Options options; options.postFields = std::move(postData); - auto result = - curl.get(tokenEndPoint_, "Content-Type: application/x-www-form-urlencoded", options, nullptr); + std::unique_ptr tlsContext; + if (!tlsTrustCertsFilePath_.empty()) { + tlsContext.reset(new CurlWrapper::TlsContext); + tlsContext->trustCertsFilePath = tlsTrustCertsFilePath_; + } + auto result = curl.get(tokenEndPoint_, "Content-Type: application/x-www-form-urlencoded", options, + tlsContext.get()); if (!result.error.empty()) { LOG_ERROR("Failed to get the well-known configuration " << issuerUrl_ << ": " << result.error); return resultPtr; diff --git a/run-unit-tests.sh b/run-unit-tests.sh index 693267ff..898cfed0 100755 --- a/run-unit-tests.sh +++ b/run-unit-tests.sh @@ -35,7 +35,14 @@ docker compose -f tests/oauth2/docker-compose.yml up -d # Wait until the namespace is created, currently there is no good way to check it # because it's hard to configure OAuth2 authentication via CLI. sleep 15 -$CMAKE_BUILD_DIRECTORY/tests/Oauth2Test +$CMAKE_BUILD_DIRECTORY/tests/Oauth2Test --gtest_filter='-*testTlsTrustFilePath' +if [[ -f /etc/ssl/certs/ca-certificates.crt ]]; then + sudo mv /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/my-cert.crt +fi +$CMAKE_BUILD_DIRECTORY/tests/Oauth2Test --gtest_filter='*testTlsTrustFilePath' +if [[ -f /etc/ssl/certs/my-cert.crt ]]; then + sudo mv /etc/ssl/certs/my-cert.crt /etc/ssl/certs/ca-certificates.crt +fi docker compose -f tests/oauth2/docker-compose.yml down # Run BrokerMetadata tests diff --git a/tests/oauth2/Oauth2Test.cc b/tests/oauth2/Oauth2Test.cc index 719ae690..158065eb 100644 --- a/tests/oauth2/Oauth2Test.cc +++ b/tests/oauth2/Oauth2Test.cc @@ -22,6 +22,7 @@ #include #include +#include #include "lib/Base64Utils.h" @@ -64,6 +65,26 @@ TEST(Oauth2Test, testWrongUrl) { ASSERT_EQ(ResultAuthenticationError, testCreateProducer("my-protocol:" + gKeyPath)); } +TEST(Oauth2Test, testTlsTrustFilePath) { + const auto caPath = "/etc/ssl/certs/my-cert.crt"; + std::ifstream fin{caPath}; + if (!fin) { // Skip this test if the CA cert is not prepared + return; + } + fin.close(); + + ClientConfiguration conf; + conf.setTlsTrustCertsFilePath(caPath); + auto params = gCommonParams; + params["private_key"] = "file://" + gKeyPath; + conf.setAuth(AuthOauth2::create(params)); + + Client client{"pulsar://localhost:6650", conf}; + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer("oauth2-test", producer)); + client.close(); +} + int main(int argc, char* argv[]) { std::cout << "Load Oauth2 configs from " << gKeyPath << "..." << std::endl; boost::property_tree::ptree root; From 7087a8ef730a1422980f5ee01399dc3528eebe10 Mon Sep 17 00:00:00 2001 From: Rory Schadler <48921090+roryschadler@users.noreply.github.com> Date: Sun, 10 Dec 2023 22:33:58 -0500 Subject: [PATCH 17/54] Expose Get/Set Listener Name in C API (#370) --- include/pulsar/c/client_configuration.h | 5 +++++ lib/c/c_ClientConfiguration.cc | 9 +++++++++ tests/c/c_ClientConfigurationTest.cc | 3 +++ 3 files changed, 17 insertions(+) diff --git a/include/pulsar/c/client_configuration.h b/include/pulsar/c/client_configuration.h index 90a29a61..7ff5dcb7 100644 --- a/include/pulsar/c/client_configuration.h +++ b/include/pulsar/c/client_configuration.h @@ -187,6 +187,11 @@ PULSAR_PUBLIC int pulsar_client_configuration_is_validate_hostname(pulsar_client PULSAR_PUBLIC void pulsar_client_configuration_set_validate_hostname(pulsar_client_configuration_t *conf, int validateHostName); +PULSAR_PUBLIC void pulsar_client_configuration_set_listener_name(pulsar_client_configuration_t *conf, + const char *listenerName); + +PULSAR_PUBLIC const char *pulsar_client_configuration_get_listener_name(pulsar_client_configuration_t *conf); + /* * Get the stats interval set in the client. */ diff --git a/lib/c/c_ClientConfiguration.cc b/lib/c/c_ClientConfiguration.cc index ef19f3ac..96a1bf7b 100644 --- a/lib/c/c_ClientConfiguration.cc +++ b/lib/c/c_ClientConfiguration.cc @@ -189,3 +189,12 @@ void pulsar_client_configuration_set_memory_limit(pulsar_client_configuration_t unsigned long long pulsar_client_configuration_get_memory_limit(pulsar_client_configuration_t *conf) { return conf->conf.getMemoryLimit(); } + +void pulsar_client_configuration_set_listener_name(pulsar_client_configuration_t *conf, + const char *listenerName) { + conf->conf.setListenerName(listenerName); +} + +const char *pulsar_client_configuration_get_listener_name(pulsar_client_configuration_t *conf) { + return conf->conf.getListenerName().c_str(); +} diff --git a/tests/c/c_ClientConfigurationTest.cc b/tests/c/c_ClientConfigurationTest.cc index f22f28a4..3fec51a5 100644 --- a/tests/c/c_ClientConfigurationTest.cc +++ b/tests/c/c_ClientConfigurationTest.cc @@ -28,4 +28,7 @@ TEST(C_ClientConfigurationTest, testCApiConfig) { ASSERT_STREQ(pulsar_client_configuration_get_tls_private_key_file_path(conf), "private.key"); ASSERT_STREQ(pulsar_client_configuration_get_tls_certificate_file_path(conf), "certificate.pem"); + + pulsar_client_configuration_set_listener_name(conf, "listenerName"); + ASSERT_STREQ(pulsar_client_configuration_get_listener_name(conf), "listenerName"); } From 7baa312de02131273fa377a7407b6ad00bde4848 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 15 Dec 2023 16:44:13 +0800 Subject: [PATCH 18/54] Integrate vcpkg to manage dependencies for all platforms (#371) ### Motivation Currently we manage dependencies by ourselves: - When running unit tests in CI, dependencies are installed from the Debian package management tool `apt` - When verifying macOS build, dependencies are installed from the Homebrew package manager - When verifying Windows build, dependencies are installed via vcpkg but no versions are specified - When building pre-built binaries to release, dependencies are installed by scripts under `pkg/` directories. These scripts download the source code according to `dependencies.yaml` and build It makes the pre-built binaries never tested except for the simple manual tests when voting for a new release. As a result, regression like https://github.com/apache/pulsar-client-cpp/issues/363 could happen. This patch aims at integrating vcpkg to manage dependencies for all platforms, not only for Windows. ### Modifications Integrate the CMakeLists.txt with vcpkg by: 1. Introduce vcpkg as a submodule of this project 2. Update `vcpkg.json` to specify dependency versions. 3. Set `CMAKE_TOOLCHAIN_FILE` with the `vcpkg.cmake` in the submodule. Then, we can simply use `find_package` to find all dependencies and depend on them via a target like `CURL::libcurl` to have all include directores and link libraries. Update the `unit-tests` workflow to verify building the binaries via vcpkg to test. The README is also updated to show how much simpler to build with vcpkg now. ### TODO There are still some tasks that cannot be done by vcpkg: 1. The static library (e.g. `libpulsarwithdeps.a`) that bundles all 3rd-party dependencies. 2. The pre-built binaries are still built by scripts under `./pkg` directory. Specifically, vcpkg does not work with GCC 4.8 so on CentOS 7 we still need to build dependencies manually. So the previous CMakeLists.txt are retained and will be used if `INTEGRATE_VCPKG` is OFF (by default). They will be removed until the task above can be done by vcpkg in future. We also need to update `dependencies.yaml` to be consistent with `vcpkg.json` if all tasks above cannot be replaced by vcpkg. --- .github/workflows/ci-pr-validation.yaml | 46 ++-- .gitmodules | 3 + CMakeLists.txt | 341 ++++-------------------- LEGACY_BUILD.md | 250 +++++++++++++++++ LegacyFindPackages.cmake | 312 ++++++++++++++++++++++ README.md | 242 +++-------------- examples/CMakeLists.txt | 5 + lib/CMakeLists.txt | 47 +++- perf/BuildPerf.cmake | 24 ++ perf/CMakeLists.txt | 23 +- perf/LegacyBuildPerf.cmake | 37 +++ tests/BuildTests.cmake | 57 ++++ tests/CMakeLists.txt | 60 +---- tests/LegacyBuildTests.cmake | 77 ++++++ vcpkg | 1 + vcpkg.json | 115 ++++++-- 16 files changed, 1020 insertions(+), 620 deletions(-) create mode 100644 .gitmodules create mode 100644 LEGACY_BUILD.md create mode 100644 LegacyFindPackages.cmake create mode 100644 perf/BuildPerf.cmake create mode 100644 perf/LegacyBuildPerf.cmake create mode 100644 tests/BuildTests.cmake create mode 100644 tests/LegacyBuildTests.cmake create mode 160000 vcpkg diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 80e1a1d3..20d1e14e 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -65,40 +65,40 @@ jobs: steps: - name: checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive - - name: Install deps - run: | - sudo apt-get update -y && \ - sudo apt-get install -y \ - libcurl4-openssl-dev \ - protobuf-compiler \ - libprotobuf-dev \ - libboost-dev \ - libboost-program-options-dev \ - libzstd-dev \ - libsnappy-dev \ - libgmock-dev \ - libgtest-dev - - - name: Install gtest-parallel + - name: Build core libraries run: | - sudo curl -o /gtest-parallel https://raw.githubusercontent.com/google/gtest-parallel/master/gtest_parallel.py - - - name: CMake - run: cmake . -DCMAKE_BUILD_TYPE=Debug -DBUILD_PERF_TOOLS=ON + cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=OFF + cmake --build . -j8 - name: Check formatting - run: make check-format + run: | + ./vcpkg/vcpkg format-manifest vcpkg.json + if [[ $(git diff | wc -l) -gt 0 ]]; then + echo "Please run `./vcpkg/vcpkg format-manifest vcpkg.json` to reformat vcpkg.json" + fi + make check-format - - name: Build + - name: Build tests run: | - # Build the libraries first to avoid possible link failures - cmake --build . -j8 --target pulsarShared pulsarStatic + cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON cmake --build . -j8 + - name: Install gtest-parallel + run: | + sudo curl -o /gtest-parallel https://raw.githubusercontent.com/google/gtest-parallel/master/gtest_parallel.py + - name: Run unit tests run: RETRY_FAILED=3 ./run-unit-tests.sh + - name: Build perf tools + run: | + cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON -DBUILD_PERF_TOOLS=ON + cmake --build . -j8 + cpp20-build: name: Build with the C++20 standard runs-on: ubuntu-22.04 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..a0a57f3d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vcpkg"] + path = vcpkg + url = https://github.com/microsoft/vcpkg.git diff --git a/CMakeLists.txt b/CMakeLists.txt index fb4f1b12..5bde2b77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,23 @@ cmake_minimum_required(VERSION 3.13) +option(INTEGRATE_VCPKG "Integrate with Vcpkg" OFF) +if (INTEGRATE_VCPKG) + set(CMAKE_TOOLCHAIN_FILE "${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake") +endif () + +option(BUILD_TESTS "Build tests" ON) +message(STATUS "BUILD_TESTS: " ${BUILD_TESTS}) +if (BUILD_TESTS) + list(APPEND VCPKG_MANIFEST_FEATURES "tests") +endif () + +option(BUILD_PERF_TOOLS "Build Pulsar CLI perf producer/consumer" OFF) +message(STATUS "BUILD_PERF_TOOLS: " ${BUILD_PERF_TOOLS}) +if (BUILD_PERF_TOOLS) + list(APPEND VCPKG_MANIFEST_FEATURES "perf") +endif () + project (pulsar-cpp) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake_modules") @@ -33,30 +50,6 @@ message(STATUS "Pulsar Client version macro: ${PULSAR_CLIENT_VERSION_MACRO}") set(PVM_COMMENT "This is generated from Version.h.in by CMAKE. DO NOT EDIT DIRECTLY") configure_file(templates/Version.h.in include/pulsar/Version.h @ONLY) -option(LINK_STATIC "Link against static libraries" OFF) -if (VCPKG_TRIPLET) - message(STATUS "Use vcpkg, triplet is ${VCPKG_TRIPLET}") - set(CMAKE_PREFIX_PATH "${PROJECT_SOURCE_DIR}/vcpkg_installed/${VCPKG_TRIPLET}") - message(STATUS "Use CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH}") - set(PROTOC_PATH "${CMAKE_PREFIX_PATH}/tools/protobuf/protoc") - message(STATUS "Use protoc: ${PROTOC_PATH}") - set(VCPKG_ROOT "${PROJECT_SOURCE_DIR}/vcpkg_installed/${VCPKG_TRIPLET}") - set(VCPKG_DEBUG_ROOT "${VCPKG_ROOT}/debug") - if (CMAKE_BUILD_TYPE STREQUAL "Debug") - set(ZLIB_ROOT ${VCPKG_DEBUG_ROOT}) - set(OPENSSL_ROOT_DIR ${VCPKG_ROOT} ${VCPKG_DEBUG_ROOT}) - set(CMAKE_PREFIX_PATH ${VCPKG_DEBUG_ROOT} ${CMAKE_PREFIX_PATH}) - else () - set(OPENSSL_ROOT_DIR ${VCPKG_ROOT}) - endif () - if (VCPKG_TRIPLET MATCHES ".*-static") - set(LINK_STATIC ON) - else () - set(LINK_STATIC OFF) - endif () -endif() -MESSAGE(STATUS "LINK_STATIC: " ${LINK_STATIC}) - find_program(CCACHE_PROGRAM ccache) if(CCACHE_PROGRAM) set(CMAKE_CXX_COMPILER_LAUNCHER "ccache") @@ -71,12 +64,6 @@ MESSAGE(STATUS "BUILD_DYNAMIC_LIB: " ${BUILD_DYNAMIC_LIB}) option(BUILD_STATIC_LIB "Build static lib" ON) MESSAGE(STATUS "BUILD_STATIC_LIB: " ${BUILD_STATIC_LIB}) -option(BUILD_TESTS "Build tests" ON) -MESSAGE(STATUS "BUILD_TESTS: " ${BUILD_TESTS}) - -option(BUILD_PERF_TOOLS "Build Pulsar CLI perf producer/consumer" OFF) -MESSAGE(STATUS "BUILD_PERF_TOOLS: " ${BUILD_PERF_TOOLS}) - IF (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo) ENDIF () @@ -87,25 +74,6 @@ set(THREADS_PREFER_PTHREAD_FLAG TRUE) find_package(Threads REQUIRED) MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT}) -set(Boost_NO_BOOST_CMAKE ON) - -if (APPLE AND NOT LINK_STATIC) - # The latest Protobuf dependency on macOS requires the C++17 support and - # it could only be found by the CONFIG mode - set(LATEST_PROTOBUF TRUE) -else () - set(LATEST_PROTOBUF FALSE) -endif () - -if (NOT CMAKE_CXX_STANDARD) - if (LATEST_PROTOBUF) - set(CMAKE_CXX_STANDARD 17) - else () - set(CMAKE_CXX_STANDARD 11) - endif () -endif () -set(CMAKE_C_STANDARD 11) - # Compiler specific configuration: # https://stackoverflow.com/questions/10046114/in-cmake-how-can-i-test-if-the-compiler-is-clang if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") @@ -133,255 +101,44 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) add_definitions(-DBUILDING_PULSAR -DBOOST_ALL_NO_LIB -DBOOST_ALLOW_DEPRECATED_HEADERS) -# For dependencies other than OpenSSL, dynamic libraries are forbidden to link when LINK_STATIC is ON -if (LINK_STATIC) - if (NOT MSVC) - set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - endif() -endif () - -find_package(Boost REQUIRED) -message("Boost_INCLUDE_DIRS: " ${Boost_INCLUDE_DIRS}) - -set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/lib64/) -if (APPLE) - set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/local/opt/openssl/ /opt/homebrew/opt/openssl) -endif () -find_package(OpenSSL REQUIRED) -message("OPENSSL_INCLUDE_DIR: " ${OPENSSL_INCLUDE_DIR}) -message("OPENSSL_LIBRARIES: " ${OPENSSL_LIBRARIES}) - -if (LATEST_PROTOBUF) - # See https://github.com/apache/arrow/issues/35987 - add_definitions(-DPROTOBUF_USE_DLLS) - # Use Config mode to avoid FindProtobuf.cmake does not find the Abseil library - find_package(Protobuf REQUIRED CONFIG) -else () - find_package(Protobuf REQUIRED) -endif () -message("Protobuf_INCLUDE_DIRS: " ${Protobuf_INCLUDE_DIRS}) -message("Protobuf_LIBRARIES: " ${Protobuf_LIBRARIES}) - -# NOTE: CMake might not find curl and zlib on some platforms like Ubuntu, in this case, find them manually -set(CURL_NO_CURL_CMAKE ON) -find_package(curl QUIET) -if (NOT CURL_FOUND) - find_path(CURL_INCLUDE_DIRS NAMES curl/curl.h) - find_library(CURL_LIBRARIES NAMES curl curllib libcurl_imp curllib_static libcurl) -endif () -message("CURL_INCLUDE_DIRS: " ${CURL_INCLUDE_DIRS}) -message("CURL_LIBRARIES: " ${CURL_LIBRARIES}) -if (NOT CURL_INCLUDE_DIRS OR NOT CURL_LIBRARIES) - message(FATAL_ERROR "Could not find libcurl") -endif () - -find_package(zlib QUIET) -if (NOT ZLIB_FOUND) - find_path(ZLIB_INCLUDE_DIRS NAMES zlib.h) - find_library(ZLIB_LIBRARIES NAMES z zlib zdll zlib1 zlibstatic) -endif () -message("ZLIB_INCLUDE_DIRS: " ${ZLIB_INCLUDE_DIRS}) -message("ZLIB_LIBRARIES: " ${ZLIB_LIBRARIES}) -if (NOT ZLIB_INCLUDE_DIRS OR NOT ZLIB_LIBRARIES) - message(FATAL_ERROR "Could not find zlib") -endif () - -if (LINK_STATIC AND NOT VCPKG_TRIPLET) - find_library(LIB_ZSTD NAMES libzstd.a) - message(STATUS "ZStd: ${LIB_ZSTD}") - find_library(LIB_SNAPPY NAMES libsnappy.a) - message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}") +set(AUTOGEN_DIR ${PROJECT_BINARY_DIR}/generated) +file(MAKE_DIRECTORY ${AUTOGEN_DIR}) - if (MSVC) - add_definitions(-DCURL_STATICLIB) - endif() -elseif (LINK_STATIC AND VCPKG_TRIPLET) - find_package(Protobuf REQUIRED) - message(STATUS "Found protobuf static library: " ${Protobuf_LIBRARIES}) - if (MSVC AND (${CMAKE_BUILD_TYPE} STREQUAL Debug)) - find_library(ZLIB_LIBRARIES NAMES zlibd) - else () - find_library(ZLIB_LIBRARIES NAMES zlib z) - endif () - if (ZLIB_LIBRARIES) - message(STATUS "Found zlib static library: " ${ZLIB_LIBRARIES}) - else () - message(FATAL_ERROR "Failed to find zlib static library") - endif () - if (MSVC AND (${CMAKE_BUILD_TYPE} STREQUAL Debug)) - find_library(CURL_LIBRARIES NAMES libcurl-d) - else () - find_library(CURL_LIBRARIES NAMES libcurl) - endif () - if (CURL_LIBRARIES) - message(STATUS "Found libcurl: ${CURL_LIBRARIES}") - else () - message(FATAL_ERROR "Cannot find libcurl") - endif () - find_library(LIB_ZSTD zstd) - if (LIB_ZSTD) - message(STATUS "Found ZSTD library: ${LIB_ZSTD}") - endif () - find_library(LIB_SNAPPY NAMES snappy) - if (LIB_SNAPPY) - message(STATUS "Found Snappy library: ${LIB_SNAPPY}") +if (INTEGRATE_VCPKG) + if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 11) endif () -else() - if (MSVC AND (${CMAKE_BUILD_TYPE} STREQUAL Debug)) - find_library(LIB_ZSTD zstdd HINTS "${VCPKG_DEBUG_ROOT}/lib") - else () - find_library(LIB_ZSTD zstd) + set(CMAKE_C_STANDARD 11) + set(Boost_NO_BOOST_CMAKE ON) + find_package(Boost REQUIRED) + include_directories(${Boost_INCLUDE_DIRS}) + message("Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}") + + set(CURL_NO_CURL_CMAKE ON) + find_package(CURL REQUIRED) + find_package(ZLIB REQUIRED) + find_package(OpenSSL REQUIRED) + find_package(protobuf CONFIG REQUIRED) + find_package(zstd CONFIG REQUIRED) + find_package(Snappy CONFIG REQUIRED) + set(COMMON_LIBS CURL::libcurl + ZLIB::ZLIB + OpenSSL::SSL + OpenSSL::Crypto + protobuf::libprotobuf + $,zstd::libzstd_shared,zstd::libzstd_static> + Snappy::snappy + ) + add_definitions(-DHAS_ZSTD -DHAS_SNAPPY) + if (MSVC) + find_package(dlfcn-win32 CONFIG REQUIRED) endif () - if (MSVC AND (${CMAKE_BUILD_TYPE} STREQUAL Debug)) - find_library(LIB_SNAPPY NAMES snappyd HINTS "${VCPKG_DEBUG_ROOT}/lib") - else () - find_library(LIB_SNAPPY NAMES snappy libsnappy) + if (BUILD_PERF_TOOLS) + find_package(Boost COMPONENTS program_options REQUIRED) endif () -endif () - -if (Boost_MAJOR_VERSION EQUAL 1 AND Boost_MINOR_VERSION LESS 69) - # Boost System does not require linking since 1.69 - set(BOOST_COMPONENTS ${BOOST_COMPONENTS} system) - MESSAGE(STATUS "Linking with Boost:System") -endif() - -if (MSVC) - set(BOOST_COMPONENTS ${BOOST_COMPONENTS} date_time) -endif() - -if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) - # GCC 4.8.2 implementation of std::regex is buggy - set(BOOST_COMPONENTS ${BOOST_COMPONENTS} regex) - set(CMAKE_CXX_FLAGS " -DPULSAR_USE_BOOST_REGEX") - MESSAGE(STATUS "Using Boost::Regex") -elseif (CMAKE_COMPILER_IS_GNUCC) - MESSAGE(STATUS "Using std::regex") - # Turn on color error messages and show additional help with errors (only available in GCC v4.9+): - add_compile_options(-fdiagnostics-show-option -fdiagnostics-color) -endif() - -if(BUILD_PERF_TOOLS) - set(BOOST_COMPONENTS ${BOOST_COMPONENTS} program_options) -endif() - -find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS}) - -if (BUILD_TESTS) - find_path(GTEST_INCLUDE_PATH gtest/gtest.h) - find_path(GMOCK_INCLUDE_PATH gmock/gmock.h) -endif () - -if (NOT APPLE AND NOT MSVC) - # Hide all non-exported symbols to avoid conflicts - add_compile_options(-fvisibility=hidden) - if (CMAKE_COMPILER_IS_GNUCC) - add_link_options(-Wl,--exclude-libs=ALL) - endif () -endif () - -if (LIB_ZSTD) - set(HAS_ZSTD 1) -else () - set(HAS_ZSTD 0) -endif () -MESSAGE(STATUS "HAS_ZSTD: ${HAS_ZSTD}") - -if (LIB_SNAPPY) - set(HAS_SNAPPY 1) else () - set(HAS_SNAPPY 0) + include(./LegacyFindPackages.cmake) endif () -MESSAGE(STATUS "HAS_SNAPPY: ${HAS_SNAPPY}") - -set(ADDITIONAL_LIBRARIES $ENV{PULSAR_ADDITIONAL_LIBRARIES}) -link_directories( $ENV{PULSAR_ADDITIONAL_LIBRARY_PATH} ) - -set(AUTOGEN_DIR ${PROJECT_BINARY_DIR}/generated) -file(MAKE_DIRECTORY ${AUTOGEN_DIR}) - -include_directories( - ${PROJECT_SOURCE_DIR} - ${PROJECT_SOURCE_DIR}/include - ${PROJECT_BINARY_DIR}/include - ${AUTOGEN_DIR} - ${Boost_INCLUDE_DIRS} - ${OPENSSL_INCLUDE_DIR} - ${ZLIB_INCLUDE_DIRS} - ${CURL_INCLUDE_DIRS} - ${Protobuf_INCLUDE_DIRS} - ${GTEST_INCLUDE_PATH} - ${GMOCK_INCLUDE_PATH} -) - -set(COMMON_LIBS - ${COMMON_LIBS} - ${CMAKE_THREAD_LIBS_INIT} - ${Boost_REGEX_LIBRARY} - ${Boost_SYSTEM_LIBRARY} - ${Boost_DATE_TIME_LIBRARY} - ${CURL_LIBRARIES} - ${OPENSSL_LIBRARIES} - ${ZLIB_LIBRARIES} - ${ADDITIONAL_LIBRARIES} - ${CMAKE_DL_LIBS} -) - -if (LATEST_PROTOBUF) - # Protobuf_LIBRARIES is empty when finding Protobuf in Config mode - set(COMMON_LIBS ${COMMON_LIBS} protobuf::libprotobuf) -else () - set(COMMON_LIBS ${COMMON_LIBS} ${Protobuf_LIBRARIES}) -endif () - -if (MSVC) - set(COMMON_LIBS - ${COMMON_LIBS} - ${Boost_DATE_TIME_LIBRARY} - wldap32.lib - Normaliz.lib) - if (LINK_STATIC) - # add external dependencies of libcurl - set(COMMON_LIBS ${COMMON_LIBS} ws2_32.lib crypt32.lib) - # the default compile options have /MD, which cannot be used to build DLLs that link static libraries - string(REGEX REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) - string(REGEX REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) - string(REGEX REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}) - message(STATUS "CMAKE_CXX_FLAGS_DEBUG: " ${CMAKE_CXX_FLAGS_DEBUG}) - message(STATUS "CMAKE_CXX_FLAGS_RELEASE: " ${CMAKE_CXX_FLAGS_RELEASE}) - message(STATUS "CMAKE_CXX_FLAGS_RELWITHDEBINFO: " ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}) - endif () -else() - set(COMMON_LIBS ${COMMON_LIBS} m) -endif() - -if (USE_LOG4CXX) - set(COMMON_LIBS - ${COMMON_LIBS} - ${LOG4CXX_LIBRARY_PATH} - ${APR_LIBRARY_PATH} - ${APR_UTIL_LIBRARY_PATH} - ${EXPAT_LIBRARY_PATH} - ${ICONV_LIBRARY_PATH} - ) -endif () - -if (HAS_ZSTD) - set(COMMON_LIBS ${COMMON_LIBS} ${LIB_ZSTD} ) -endif () - -add_definitions(-DHAS_ZSTD=${HAS_ZSTD}) - -if (HAS_SNAPPY) - set(COMMON_LIBS ${COMMON_LIBS} ${LIB_SNAPPY} ) -endif () - -add_definitions(-DHAS_SNAPPY=${HAS_SNAPPY}) - -if(NOT APPLE AND NOT MSVC) - set(COMMON_LIBS ${COMMON_LIBS} rt) -endif () - -link_directories(${PROJECT_BINARY_DIR}/lib) set(LIB_NAME $ENV{PULSAR_LIBRARY_NAME}) if (NOT LIB_NAME) diff --git a/LEGACY_BUILD.md b/LEGACY_BUILD.md new file mode 100644 index 00000000..0920e2b5 --- /dev/null +++ b/LEGACY_BUILD.md @@ -0,0 +1,250 @@ + + +# Build without vcpkg + +## Requirements + +- A C++ compiler that supports C++11, like GCC >= 4.8 +- CMake >= 3.13 +- [Boost](http://www.boost.org/) +- [Protocol Buffer](https://developers.google.com/protocol-buffers/) >= 3 +- [libcurl](https://curl.se/libcurl/) +- [openssl](https://github.com/openssl/openssl) + +The default supported [compression types](include/pulsar/CompressionType.h) are: + +- `CompressionNone` +- `CompressionLZ4` + +If you want to enable other compression types, you need to install: + +- `CompressionZLib`: [zlib](https://zlib.net/) +- `CompressionZSTD`: [zstd](https://github.com/facebook/zstd) +- `CompressionSNAPPY`: [snappy](https://github.com/google/snappy) + +If you want to build and run the tests, you need to install [GTest](https://github.com/google/googletest). Otherwise, you need to add CMake option `-DBUILD_TESTS=OFF`. + +The [dependencies.yaml](./dependencies.yaml) file provides the recommended dependency versions, while you can still build from source with other dependency versions. If a dependency requires a higher C++ standard, e.g. C++14, you can specify the standard like: + +```bash +cmake . -DCMAKE_CXX_STANDARD=14 +``` + +> **Note**: +> +> On macOS, the default C++ standard is 17 because the latest Protobuf from Homebrew requires the C++17 support. + +## Compilation + +### Clone + +First of all, clone the source code: + +```shell +git clone https://github.com/apache/pulsar-client-cpp +cd pulsar-client-cpp +``` + +### Compile on Ubuntu + +#### Install all dependencies: + +```shell +sudo apt-get update -y && sudo apt-get install -y g++ cmake libssl-dev libcurl4-openssl-dev \ + libprotobuf-dev libboost-all-dev libgtest-dev libgmock-dev \ + protobuf-compiler +``` + +#### Compile Pulsar client library: + +```shell +cmake . +make +``` + +If you want to build performance tools, you need to run: + +```shell +cmake . -DBUILD_PERF_TOOLS=ON +make +``` + +#### Checks + +Client library will be placed in: + +``` +lib/libpulsar.so +lib/libpulsar.a +``` + +Examples will be placed in: + +``` +examples/ +``` + +Tools will be placed in: + +``` +perf/perfProducer +perf/perfConsumer +``` + +### Compile on Mac OS X + +#### Install all dependencies: + +```shell +brew install cmake openssl protobuf boost googletest zstd snappy +``` + +#### Compile Pulsar client library: + +```shell +cmake . +make +``` + +If you want to build performance tools, you need to run: + +```shell +cmake . -DBUILD_PERF_TOOLS=ON +make +``` + +#### Checks + +Client library will be placed in: + +``` +lib/libpulsar.dylib +lib/libpulsar.a +``` + +Examples will be placed in: + +``` +examples/ +``` + +Tools will be placed in: + +``` +perf/perfProducer +perf/perfConsumer +``` + +### Compile on Windows + +#### Install with [vcpkg](https://github.com/microsoft/vcpkg) + +It's highly recommended to use `vcpkg` for C++ package management on Windows. It's easy to install and well supported by Visual Studio (2015/2017/2019) and CMake. See [here](https://github.com/microsoft/vcpkg#quick-start-windows) for quick start. + +Take Windows 64-bit library as an example, you only need to run + +```bash +vcpkg install --feature-flags=manifests --triplet x64-windows +``` + +> **NOTE**: +> +> For Windows 32-bit library, change `x64-windows` to `x86-windows`, see [here](https://github.com/microsoft/vcpkg/blob/master/docs/users/triplets.md) for more details about the triplet concept in Vcpkg. + +The all dependencies, which are specified by [vcpkg.json](vcpkg.json), will be installed in `vcpkg_installed/` subdirectory, + +With `vcpkg`, you only need to run two commands: + +```bash +cmake \ + -B ./build \ + -A x64 \ + -DBUILD_TESTS=OFF \ + -DVCPKG_TRIPLET=x64-windows \ + -DCMAKE_BUILD_TYPE=Release \ + -S . +cmake --build ./build --config Release +``` + +Then all artifacts will be built into `build` subdirectory. + +> **NOTE**: +> +> 1. For Windows 32-bit, you need to use `-A Win32` and `-DVCPKG_TRIPLET=x86-windows`. +> 2. For MSVC Debug mode, you need to replace `Release` with `Debug` for both `CMAKE_BUILD_TYPE` variable and `--config` option. + +#### Install dependencies manually + +You need to install [dlfcn-win32](https://github.com/dlfcn-win32/dlfcn-win32) in addition. + +If you installed the dependencies manually, you need to run + +```shell +#If all dependencies are in your path, all that is necessary is +cmake . + +#if all dependencies are not in your path, then passing in a PROTOC_PATH and CMAKE_PREFIX_PATH is necessary +cmake -DPROTOC_PATH=C:/protobuf/bin/protoc -DCMAKE_PREFIX_PATH="C:/boost;C:/openssl;C:/zlib;C:/curl;C:/protobuf;C:/googletest;C:/dlfcn-win32" . + +#This will generate pulsar-cpp.sln. Open this in Visual Studio and build the desired configurations. +``` + +#### Checks + +Client library will be placed in: + +``` +build/lib/Release/pulsar.lib +build/lib/Release/pulsar.dll +``` + +#### Examples + +Add Windows environment paths: + +``` +build/lib/Release +vcpkg_installed +``` + +Examples will be available in: + +``` +build/examples/Release +``` + +## Tests + +```shell +# Execution +# Start standalone broker +./pulsar-test-service-start.sh + +# Run the tests +cd tests +./pulsar-tests + +# When no longer needed, stop standalone broker +./pulsar-test-service-stop.sh +``` + + diff --git a/LegacyFindPackages.cmake b/LegacyFindPackages.cmake new file mode 100644 index 00000000..10c9fa7c --- /dev/null +++ b/LegacyFindPackages.cmake @@ -0,0 +1,312 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +option(LINK_STATIC "Link against static libraries" OFF) +if (VCPKG_TRIPLET) + message(STATUS "Use vcpkg, triplet is ${VCPKG_TRIPLET}") + set(CMAKE_PREFIX_PATH "${PROJECT_SOURCE_DIR}/vcpkg_installed/${VCPKG_TRIPLET}") + message(STATUS "Use CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH}") + set(PROTOC_PATH "${CMAKE_PREFIX_PATH}/tools/protobuf/protoc") + message(STATUS "Use protoc: ${PROTOC_PATH}") + set(VCPKG_ROOT "${PROJECT_SOURCE_DIR}/vcpkg_installed/${VCPKG_TRIPLET}") + set(VCPKG_DEBUG_ROOT "${VCPKG_ROOT}/debug") + if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(ZLIB_ROOT ${VCPKG_DEBUG_ROOT}) + set(OPENSSL_ROOT_DIR ${VCPKG_ROOT} ${VCPKG_DEBUG_ROOT}) + set(CMAKE_PREFIX_PATH ${VCPKG_DEBUG_ROOT} ${CMAKE_PREFIX_PATH}) + else () + set(OPENSSL_ROOT_DIR ${VCPKG_ROOT}) + endif () + if (VCPKG_TRIPLET MATCHES ".*-static") + set(LINK_STATIC ON) + else () + set(LINK_STATIC OFF) + endif () +endif() +MESSAGE(STATUS "LINK_STATIC: " ${LINK_STATIC}) + +if (MSVC) + find_package(dlfcn-win32 REQUIRED) +endif () + +set(Boost_NO_BOOST_CMAKE ON) + +if (APPLE AND NOT LINK_STATIC) + # The latest Protobuf dependency on macOS requires the C++17 support and + # it could only be found by the CONFIG mode + set(LATEST_PROTOBUF TRUE) +else () + set(LATEST_PROTOBUF FALSE) +endif () + +if (NOT CMAKE_CXX_STANDARD) + if (LATEST_PROTOBUF) + set(CMAKE_CXX_STANDARD 17) + else () + set(CMAKE_CXX_STANDARD 11) + endif () +endif () +set(CMAKE_C_STANDARD 11) + +# For dependencies other than OpenSSL, dynamic libraries are forbidden to link when LINK_STATIC is ON +if (LINK_STATIC) + if (NOT MSVC) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + endif() +endif () + +find_package(Boost REQUIRED) +message("Boost_INCLUDE_DIRS: " ${Boost_INCLUDE_DIRS}) + +set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/lib64/) +if (APPLE) + set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/local/opt/openssl/ /opt/homebrew/opt/openssl) +endif () +find_package(OpenSSL REQUIRED) +message("OPENSSL_INCLUDE_DIR: " ${OPENSSL_INCLUDE_DIR}) +message("OPENSSL_LIBRARIES: " ${OPENSSL_LIBRARIES}) + +if (LATEST_PROTOBUF) + # See https://github.com/apache/arrow/issues/35987 + add_definitions(-DPROTOBUF_USE_DLLS) + # Use Config mode to avoid FindProtobuf.cmake does not find the Abseil library + find_package(Protobuf REQUIRED CONFIG) +else () + find_package(Protobuf REQUIRED) +endif () +message("Protobuf_INCLUDE_DIRS: " ${Protobuf_INCLUDE_DIRS}) +message("Protobuf_LIBRARIES: " ${Protobuf_LIBRARIES}) + +# NOTE: CMake might not find curl and zlib on some platforms like Ubuntu, in this case, find them manually +set(CURL_NO_CURL_CMAKE ON) +find_package(curl QUIET) +if (NOT CURL_FOUND) + find_path(CURL_INCLUDE_DIRS NAMES curl/curl.h) + find_library(CURL_LIBRARIES NAMES curl curllib libcurl_imp curllib_static libcurl) +endif () +message("CURL_INCLUDE_DIRS: " ${CURL_INCLUDE_DIRS}) +message("CURL_LIBRARIES: " ${CURL_LIBRARIES}) +if (NOT CURL_INCLUDE_DIRS OR NOT CURL_LIBRARIES) + message(FATAL_ERROR "Could not find libcurl") +endif () + +find_package(zlib QUIET) +if (NOT ZLIB_FOUND) + find_path(ZLIB_INCLUDE_DIRS NAMES zlib.h) + find_library(ZLIB_LIBRARIES NAMES z zlib zdll zlib1 zlibstatic) +endif () +message("ZLIB_INCLUDE_DIRS: " ${ZLIB_INCLUDE_DIRS}) +message("ZLIB_LIBRARIES: " ${ZLIB_LIBRARIES}) +if (NOT ZLIB_INCLUDE_DIRS OR NOT ZLIB_LIBRARIES) + message(FATAL_ERROR "Could not find zlib") +endif () + +if (LINK_STATIC AND NOT VCPKG_TRIPLET) + find_library(LIB_ZSTD NAMES libzstd.a) + message(STATUS "ZStd: ${LIB_ZSTD}") + find_library(LIB_SNAPPY NAMES libsnappy.a) + message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}") + + if (MSVC) + add_definitions(-DCURL_STATICLIB) + endif() +elseif (LINK_STATIC AND VCPKG_TRIPLET) + find_package(Protobuf REQUIRED) + message(STATUS "Found protobuf static library: " ${Protobuf_LIBRARIES}) + if (MSVC AND (${CMAKE_BUILD_TYPE} STREQUAL Debug)) + find_library(ZLIB_LIBRARIES NAMES zlibd) + else () + find_library(ZLIB_LIBRARIES NAMES zlib z) + endif () + if (ZLIB_LIBRARIES) + message(STATUS "Found zlib static library: " ${ZLIB_LIBRARIES}) + else () + message(FATAL_ERROR "Failed to find zlib static library") + endif () + if (MSVC AND (${CMAKE_BUILD_TYPE} STREQUAL Debug)) + find_library(CURL_LIBRARIES NAMES libcurl-d) + else () + find_library(CURL_LIBRARIES NAMES libcurl) + endif () + if (CURL_LIBRARIES) + message(STATUS "Found libcurl: ${CURL_LIBRARIES}") + else () + message(FATAL_ERROR "Cannot find libcurl") + endif () + find_library(LIB_ZSTD zstd) + if (LIB_ZSTD) + message(STATUS "Found ZSTD library: ${LIB_ZSTD}") + endif () + find_library(LIB_SNAPPY NAMES snappy) + if (LIB_SNAPPY) + message(STATUS "Found Snappy library: ${LIB_SNAPPY}") + endif () +else() + if (MSVC AND (${CMAKE_BUILD_TYPE} STREQUAL Debug)) + find_library(LIB_ZSTD zstdd HINTS "${VCPKG_DEBUG_ROOT}/lib") + else () + find_library(LIB_ZSTD zstd) + endif () + if (MSVC AND (${CMAKE_BUILD_TYPE} STREQUAL Debug)) + find_library(LIB_SNAPPY NAMES snappyd HINTS "${VCPKG_DEBUG_ROOT}/lib") + else () + find_library(LIB_SNAPPY NAMES snappy libsnappy) + endif () +endif () + +if (Boost_MAJOR_VERSION EQUAL 1 AND Boost_MINOR_VERSION LESS 69) + # Boost System does not require linking since 1.69 + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} system) + MESSAGE(STATUS "Linking with Boost:System") +endif() + +if (MSVC) + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} date_time) +endif() + +if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) + # GCC 4.8.2 implementation of std::regex is buggy + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} regex) + set(CMAKE_CXX_FLAGS " -DPULSAR_USE_BOOST_REGEX") + MESSAGE(STATUS "Using Boost::Regex") +elseif (CMAKE_COMPILER_IS_GNUCC) + MESSAGE(STATUS "Using std::regex") + # Turn on color error messages and show additional help with errors (only available in GCC v4.9+): + add_compile_options(-fdiagnostics-show-option -fdiagnostics-color) +endif() + +if(BUILD_PERF_TOOLS) + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} program_options) +endif() + +find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS}) + +if (BUILD_TESTS) + find_path(GTEST_INCLUDE_PATH gtest/gtest.h) + find_path(GMOCK_INCLUDE_PATH gmock/gmock.h) +endif () + +if (NOT APPLE AND NOT MSVC) + # Hide all non-exported symbols to avoid conflicts + add_compile_options(-fvisibility=hidden) + if (CMAKE_COMPILER_IS_GNUCC) + add_link_options(-Wl,--exclude-libs=ALL) + endif () +endif () + +if (LIB_ZSTD) + set(HAS_ZSTD 1) +else () + set(HAS_ZSTD 0) +endif () +MESSAGE(STATUS "HAS_ZSTD: ${HAS_ZSTD}") + +if (LIB_SNAPPY) + set(HAS_SNAPPY 1) +else () + set(HAS_SNAPPY 0) +endif () +MESSAGE(STATUS "HAS_SNAPPY: ${HAS_SNAPPY}") + +set(ADDITIONAL_LIBRARIES $ENV{PULSAR_ADDITIONAL_LIBRARIES}) +link_directories( $ENV{PULSAR_ADDITIONAL_LIBRARY_PATH} ) + +include_directories( + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_BINARY_DIR}/include + ${AUTOGEN_DIR} + ${Boost_INCLUDE_DIRS} + ${OPENSSL_INCLUDE_DIR} + ${ZLIB_INCLUDE_DIRS} + ${CURL_INCLUDE_DIRS} + ${Protobuf_INCLUDE_DIRS} + ${GTEST_INCLUDE_PATH} + ${GMOCK_INCLUDE_PATH} +) + +set(COMMON_LIBS + ${COMMON_LIBS} + ${CMAKE_THREAD_LIBS_INIT} + ${Boost_REGEX_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_DATE_TIME_LIBRARY} + ${CURL_LIBRARIES} + ${OPENSSL_LIBRARIES} + ${ZLIB_LIBRARIES} + ${ADDITIONAL_LIBRARIES} + ${CMAKE_DL_LIBS} +) + +if (LATEST_PROTOBUF) + # Protobuf_LIBRARIES is empty when finding Protobuf in Config mode + set(COMMON_LIBS ${COMMON_LIBS} protobuf::libprotobuf) +else () + set(COMMON_LIBS ${COMMON_LIBS} ${Protobuf_LIBRARIES}) +endif () + +if (MSVC) + set(COMMON_LIBS + ${COMMON_LIBS} + ${Boost_DATE_TIME_LIBRARY} + wldap32.lib + Normaliz.lib) + if (LINK_STATIC) + # add external dependencies of libcurl + set(COMMON_LIBS ${COMMON_LIBS} ws2_32.lib crypt32.lib) + # the default compile options have /MD, which cannot be used to build DLLs that link static libraries + string(REGEX REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) + string(REGEX REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) + string(REGEX REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}) + message(STATUS "CMAKE_CXX_FLAGS_DEBUG: " ${CMAKE_CXX_FLAGS_DEBUG}) + message(STATUS "CMAKE_CXX_FLAGS_RELEASE: " ${CMAKE_CXX_FLAGS_RELEASE}) + message(STATUS "CMAKE_CXX_FLAGS_RELWITHDEBINFO: " ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}) + endif () +else() + set(COMMON_LIBS ${COMMON_LIBS} m) +endif() + +if (USE_LOG4CXX) + set(COMMON_LIBS + ${COMMON_LIBS} + ${LOG4CXX_LIBRARY_PATH} + ${APR_LIBRARY_PATH} + ${APR_UTIL_LIBRARY_PATH} + ${EXPAT_LIBRARY_PATH} + ${ICONV_LIBRARY_PATH} + ) +endif () + +if (HAS_ZSTD) + set(COMMON_LIBS ${COMMON_LIBS} ${LIB_ZSTD} ) +endif () + +add_definitions(-DHAS_ZSTD=${HAS_ZSTD}) + +if (HAS_SNAPPY) + set(COMMON_LIBS ${COMMON_LIBS} ${LIB_SNAPPY} ) +endif () + +add_definitions(-DHAS_SNAPPY=${HAS_SNAPPY}) + +if(NOT APPLE AND NOT MSVC) + set(COMMON_LIBS ${COMMON_LIBS} rt) +endif () + +link_directories(${PROJECT_BINARY_DIR}/lib) diff --git a/README.md b/README.md index 055fb8fd..03957ce0 100644 --- a/README.md +++ b/README.md @@ -31,239 +31,83 @@ For how to use APIs to publish and consume messages, see [examples](https://gith Pulsar C++ client uses [doxygen](https://www.doxygen.nl) to build API documents. After installing `doxygen`, you only need to run `doxygen` to generate the API documents whose main page is under the `doxygen/html/index.html` path. -## Requirements +## Build with vcpkg -- A C++ compiler that supports C++11, like GCC >= 4.8 -- CMake >= 3.13 -- [Boost](http://www.boost.org/) -- [Protocol Buffer](https://developers.google.com/protocol-buffers/) >= 3 -- [libcurl](https://curl.se/libcurl/) -- [openssl](https://github.com/openssl/openssl) +Since it's integrated with vcpkg, see [vcpkg#README](https://github.com/microsoft/vcpkg#readme) for the requirements. See [LEGACY_BUILD](./LEGACY_BUILD.md) if you want to manage dependencies by yourself or you cannot install vcpkg in your own environment. -The default supported [compression types](include/pulsar/CompressionType.h) are: - -- `CompressionNone` -- `CompressionLZ4` - -If you want to enable other compression types, you need to install: - -- `CompressionZLib`: [zlib](https://zlib.net/) -- `CompressionZSTD`: [zstd](https://github.com/facebook/zstd) -- `CompressionSNAPPY`: [snappy](https://github.com/google/snappy) - -If you want to build and run the tests, you need to install [GTest](https://github.com/google/googletest). Otherwise, you need to add CMake option `-DBUILD_TESTS=OFF`. - -The [dependencies.yaml](./dependencies.yaml) file provides the recommended dependency versions, while you can still build from source with other dependency versions. If a dependency requires a higher C++ standard, e.g. C++14, you can specify the standard like: +### How to build from source ```bash -cmake . -DCMAKE_CXX_STANDARD=14 -``` - -> **Note**: -> -> On macOS, the default C++ standard is 17 because the latest Protobuf from Homebrew requires the C++17 support. - -## Platforms - -Pulsar C++ Client Library has been tested on: - -- Linux -- Mac OS X -- Windows x64 - -## Compilation - -### Clone - -First of all, clone the source code: - -```shell -git clone https://github.com/apache/pulsar-client-cpp +git clone https://github.com/apache/pulsar-client-cpp.git cd pulsar-client-cpp +git submodule update --init --recursive +cmake -B build -DINTEGRATE_VCPKG=ON +cmake --build build -j8 ``` -### Compile on Ubuntu - -#### Install all dependencies: - -```shell -sudo apt-get update -y && sudo apt-get install -y g++ cmake libssl-dev libcurl4-openssl-dev \ - libprotobuf-dev libboost-all-dev libgtest-dev libgmock-dev \ - protobuf-compiler -``` - -#### Compile Pulsar client library: - -```shell -cmake . -make -``` - -If you want to build performance tools, you need to run: - -```shell -cmake . -DBUILD_PERF_TOOLS=ON -make -``` - -#### Checks - -Client library will be placed in: - -``` -lib/libpulsar.so -lib/libpulsar.a -``` - -Examples will be placed in: - -``` -examples/ -``` - -Tools will be placed in: - -``` -perf/perfProducer -perf/perfConsumer -``` - -### Compile on Mac OS X +The 1st step will download vcpkg and then install all dependencies according to the version description in [vcpkg.json](./vcpkg.json). The 2nd step will build the Pulsar C++ libraries under `./build/lib/`, where `./build` is the CMake build directory. -#### Install all dependencies: +After the build, the hierarchy of the `build` directory will be: -```shell -brew install cmake openssl protobuf boost googletest zstd snappy ``` - -#### Compile Pulsar client library: - -```shell -cmake . -make -``` - -If you want to build performance tools, you need to run: - -```shell -cmake . -DBUILD_PERF_TOOLS=ON -make +build/ + include/ -- extra C++ headers + lib/ -- libraries + tests/ -- test executables + examples/ -- example executables + generated/ + lib/ -- protobuf source files for PulsarApi.proto + tests/ -- protobuf source files for *.proto used in tests ``` -#### Checks +### How to install -Client library will be placed in: +To install the C++ headers and libraries into a specific path, e.g. `/tmp/pulsar`, run the following commands: -``` -lib/libpulsar.dylib -lib/libpulsar.a -``` - -Examples will be placed in: - -``` -examples/ +```bash +cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_INSTALL_PREFIX=/tmp/pulsar +cmake --build build -j8 --target install ``` -Tools will be placed in: +For example, on macOS you will see: ``` -perf/perfProducer -perf/perfConsumer +/tmp/pulsar/ + include/pulsar -- C/C++ headers + lib/ + libpulsar.a -- Static library + libpulsar.dylib -- Dynamic library ``` -### Compile on Windows +### Tests -#### Install with [vcpkg](https://github.com/microsoft/vcpkg) +Tests are built by default. You should execute [run-unit-tests.sh](./run-unit-tests.sh) to run tests locally. -It's highly recommended to use `vcpkg` for C++ package management on Windows. It's easy to install and well supported by Visual Studio (2015/2017/2019) and CMake. See [here](https://github.com/microsoft/vcpkg#quick-start-windows) for quick start. - -Take Windows 64-bit library as an example, you only need to run +If you don't want to build the tests, disable the `BUILD_TESTS` option: ```bash -vcpkg install --feature-flags=manifests --triplet x64-windows +cmake -B build -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=OFF +cmake --build build -j8 ``` -> **NOTE**: -> -> For Windows 32-bit library, change `x64-windows` to `x86-windows`, see [here](https://github.com/microsoft/vcpkg/blob/master/docs/users/triplets.md) for more details about the triplet concept in Vcpkg. - -The all dependencies, which are specified by [vcpkg.json](vcpkg.json), will be installed in `vcpkg_installed/` subdirectory, +### Build perf tools -With `vcpkg`, you only need to run two commands: +If you want to build the perf tools, enable the `BUILD_PERF_TOOLS` option: ```bash -cmake \ - -B ./build \ - -A x64 \ - -DBUILD_TESTS=OFF \ - -DVCPKG_TRIPLET=x64-windows \ - -DCMAKE_BUILD_TYPE=Release \ - -S . -cmake --build ./build --config Release -``` - -Then all artifacts will be built into `build` subdirectory. - -> **NOTE**: -> -> 1. For Windows 32-bit, you need to use `-A Win32` and `-DVCPKG_TRIPLET=x86-windows`. -> 2. For MSVC Debug mode, you need to replace `Release` with `Debug` for both `CMAKE_BUILD_TYPE` variable and `--config` option. - -#### Install dependencies manually - -You need to install [dlfcn-win32](https://github.com/dlfcn-win32/dlfcn-win32) in addition. - -If you installed the dependencies manually, you need to run - -```shell -#If all dependencies are in your path, all that is necessary is -cmake . - -#if all dependencies are not in your path, then passing in a PROTOC_PATH and CMAKE_PREFIX_PATH is necessary -cmake -DPROTOC_PATH=C:/protobuf/bin/protoc -DCMAKE_PREFIX_PATH="C:/boost;C:/openssl;C:/zlib;C:/curl;C:/protobuf;C:/googletest;C:/dlfcn-win32" . - -#This will generate pulsar-cpp.sln. Open this in Visual Studio and build the desired configurations. -``` - -#### Checks - -Client library will be placed in: - -``` -build/lib/Release/pulsar.lib -build/lib/Release/pulsar.dll -``` - -#### Examples - -Add Windows environment paths: - -``` -build/lib/Release -vcpkg_installed +cmake -B build -DINTEGRATE_VCPKG=ON -DBUILD_PERF_TOOLS=ON +cmake --build build -j8 ``` -Examples will be available in: +Then the perf tools will be built under `./build/perf/`. -``` -build/examples/Release -``` - -## Tests - -```shell -# Execution -# Start standalone broker -./pulsar-test-service-start.sh +## Platforms -# Run the tests -cd tests -./pulsar-tests +Pulsar C++ Client Library has been tested on: -# When no longer needed, stop standalone broker -./pulsar-test-service-stop.sh -``` +- Linux +- Mac OS X +- Windows x64 ## Wireshark Dissector diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index af1c91aa..d54c2284 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -86,6 +86,11 @@ add_executable(SampleKeyValueSchemaConsumer ${SAMPLE_KEY_VALUE_SCHEMA_ add_executable(SampleKeyValueSchemaProducer ${SAMPLE_KEY_VALUE_SCHEMA_PRODUCER}) add_executable(SampleCustomLoggerCApi ${SAMPLE_CUSTOM_LOGGER_CAPI}) +if (INTEGRATE_VCPKG) + # pulsarShared already carries INCLUDE_DIRECTORIES and LINK_DIRECTORIES properties + set(CLIENT_LIBS "") +endif () + target_link_libraries(SampleAsyncProducer ${CLIENT_LIBS} pulsarShared) target_link_libraries(SampleConsumer ${CLIENT_LIBS} pulsarShared) target_link_libraries(SampleConsumerListener ${CLIENT_LIBS} pulsarShared) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 8bd9749c..6edc05e2 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -27,16 +27,25 @@ set(LIB_AUTOGEN_DIR ${AUTOGEN_DIR}/lib) file(MAKE_DIRECTORY ${LIB_AUTOGEN_DIR}) include_directories(${LIB_AUTOGEN_DIR}) -# Protobuf generation is only supported natively starting from CMake 3.8 -# Using custom command for now -set(PROTO_SOURCES ${LIB_AUTOGEN_DIR}/PulsarApi.pb.cc ${LIB_AUTOGEN_DIR}/PulsarApi.pb.h) -set(PULSAR_SOURCES ${PULSAR_SOURCES} ${PROTO_SOURCES}) -ADD_CUSTOM_COMMAND( - OUTPUT ${PROTO_SOURCES} - COMMAND ${PROTOC_PATH} -I ../proto ../proto/PulsarApi.proto --cpp_out=${LIB_AUTOGEN_DIR} - DEPENDS - ../proto/PulsarApi.proto - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +if (INTEGRATE_VCPKG) + add_library(PROTO_OBJECTS OBJECT "${CMAKE_SOURCE_DIR}/proto/PulsarApi.proto") + target_link_libraries(PROTO_OBJECTS PUBLIC protobuf::libprotobuf) + target_include_directories(PROTO_OBJECTS PUBLIC ${LIB_AUTOGEN_DIR}) + protobuf_generate( + TARGET PROTO_OBJECTS + IMPORT_DIRS ${CMAKE_SOURCE_DIR}/proto + PROTOC_OUT_DIR ${LIB_AUTOGEN_DIR}) + set(COMMON_LIBS ${COMMON_LIBS} PROTO_OBJECTS) +else () + set(PROTO_SOURCES ${LIB_AUTOGEN_DIR}/PulsarApi.pb.cc ${LIB_AUTOGEN_DIR}/PulsarApi.pb.h) + set(PULSAR_SOURCES ${PULSAR_SOURCES} ${PROTO_SOURCES}) + ADD_CUSTOM_COMMAND( + OUTPUT ${PROTO_SOURCES} + COMMAND ${PROTOC_PATH} -I ../proto ../proto/PulsarApi.proto --cpp_out=${LIB_AUTOGEN_DIR} + DEPENDS + ../proto/PulsarApi.proto + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +endif () set(LIBRARY_VERSION $ENV{PULSAR_LIBRARY_VERSION}) if (NOT LIBRARY_VERSION) @@ -44,7 +53,6 @@ if (NOT LIBRARY_VERSION) endif(NOT LIBRARY_VERSION) if (MSVC) - find_package(dlfcn-win32 REQUIRED) set(CMAKE_DL_LIBS dlfcn-win32::dl psapi.lib) if (CMAKE_BUILD_TYPE STREQUAL "Debug") get_target_property(dlfcn-win32_LIBRARY dlfcn-win32::dl IMPORTED_LOCATION_DEBUG) @@ -60,12 +68,23 @@ set(LIB_NAME_SHARED ${LIB_NAME}) # this is the "object library" target: compiles the sources only once add_library(PULSAR_OBJECT_LIB OBJECT ${PULSAR_SOURCES}) set_property(TARGET PULSAR_OBJECT_LIB PROPERTY POSITION_INDEPENDENT_CODE 1) +if (INTEGRATE_VCPKG) + target_link_libraries(PULSAR_OBJECT_LIB PROTO_OBJECTS) +endif () +target_include_directories(PULSAR_OBJECT_LIB PUBLIC + "${CMAKE_SOURCE_DIR}" + "${CMAKE_SOURCE_DIR}/include" + "${CMAKE_BINARY_DIR}/include") if (BUILD_DYNAMIC_LIB) add_library(pulsarShared SHARED $) set_property(TARGET pulsarShared PROPERTY OUTPUT_NAME ${LIB_NAME_SHARED}) set_property(TARGET pulsarShared PROPERTY VERSION ${LIBRARY_VERSION}) - target_link_libraries(pulsarShared ${COMMON_LIBS} ${CMAKE_DL_LIBS}) + target_link_libraries(pulsarShared PRIVATE ${COMMON_LIBS} ${CMAKE_DL_LIBS}) + target_include_directories(pulsarShared PUBLIC + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/include) + target_link_directories(pulsarShared PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) if (MSVC) target_include_directories(pulsarShared PRIVATE ${dlfcn-win32_INCLUDE_DIRS}) target_link_options(pulsarShared PRIVATE $<$:/NODEFAULTLIB:MSVCRT>) @@ -80,6 +99,10 @@ endif() if (BUILD_STATIC_LIB) add_library(pulsarStatic STATIC $) + target_include_directories(pulsarStatic PUBLIC + ${CMAKE_BINARY_DIR}/include + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/include) if (MSVC) set_property(TARGET pulsarStatic PROPERTY OUTPUT_NAME "${LIB_NAME}-static") target_include_directories(pulsarStatic PRIVATE ${dlfcn-win32_INCLUDE_DIRS}) diff --git a/perf/BuildPerf.cmake b/perf/BuildPerf.cmake new file mode 100644 index 00000000..8b5bc6f7 --- /dev/null +++ b/perf/BuildPerf.cmake @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +add_executable(perfProducer PerfProducer.cc) +target_link_libraries(perfProducer pulsarShared Boost::program_options) + +add_executable(perfConsumer PerfConsumer.cc) +target_link_libraries(perfConsumer pulsarShared Boost::program_options) diff --git a/perf/CMakeLists.txt b/perf/CMakeLists.txt index 586f9b0f..74215c7d 100644 --- a/perf/CMakeLists.txt +++ b/perf/CMakeLists.txt @@ -17,21 +17,8 @@ # under the License. # -# Test tools -add_definitions(-D_GLIBCXX_USE_NANOSLEEP) - -set(PERF_PRODUCER_SOURCES - PerfProducer.cc -) - -set(PERF_CONSUMER_SOURCES - PerfConsumer.cc -) - -add_executable(perfProducer ${PERF_PRODUCER_SOURCES}) -add_executable(perfConsumer ${PERF_CONSUMER_SOURCES}) - -set(TOOL_LIBS ${CLIENT_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_THREAD_LIBRARY}) - -target_link_libraries(perfProducer pulsarShared ${TOOL_LIBS}) -target_link_libraries(perfConsumer pulsarShared ${TOOL_LIBS}) +if (INTEGRATE_VCPKG) + include(BuildPerf.cmake) +else () + include(LegacyBuildPerf.cmake) +endif () diff --git a/perf/LegacyBuildPerf.cmake b/perf/LegacyBuildPerf.cmake new file mode 100644 index 00000000..586f9b0f --- /dev/null +++ b/perf/LegacyBuildPerf.cmake @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Test tools +add_definitions(-D_GLIBCXX_USE_NANOSLEEP) + +set(PERF_PRODUCER_SOURCES + PerfProducer.cc +) + +set(PERF_CONSUMER_SOURCES + PerfConsumer.cc +) + +add_executable(perfProducer ${PERF_PRODUCER_SOURCES}) +add_executable(perfConsumer ${PERF_CONSUMER_SOURCES}) + +set(TOOL_LIBS ${CLIENT_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_THREAD_LIBRARY}) + +target_link_libraries(perfProducer pulsarShared ${TOOL_LIBS}) +target_link_libraries(perfConsumer pulsarShared ${TOOL_LIBS}) diff --git a/tests/BuildTests.cmake b/tests/BuildTests.cmake new file mode 100644 index 00000000..c468f51f --- /dev/null +++ b/tests/BuildTests.cmake @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set(LIB_AUTOGEN_DIR ${AUTOGEN_DIR}/tests) +file(MAKE_DIRECTORY ${LIB_AUTOGEN_DIR}) +include_directories(${LIB_AUTOGEN_DIR}) + +find_package(GTest CONFIG REQUIRED) +set(GTEST_TARGETS GTest::gtest GTest::gtest_main GTest::gmock GTest::gmock_main) + +add_library(TESTS_PROTO_OBJECTS OBJECT + ${CMAKE_CURRENT_SOURCE_DIR}/PaddingDemo.proto + ${CMAKE_CURRENT_SOURCE_DIR}/proto/ExternalTest.proto + ${CMAKE_CURRENT_SOURCE_DIR}/proto/Test.proto + ) +target_link_libraries(TESTS_PROTO_OBJECTS PUBLIC protobuf::libprotobuf) +protobuf_generate( + TARGET TESTS_PROTO_OBJECTS + IMPORT_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/proto" "${CMAKE_CURRENT_SOURCE_DIR}" + PROTOC_OUT_DIR ${LIB_AUTOGEN_DIR}) + +file(GLOB TEST_SOURCES *.cc c/*.cc) +add_executable(pulsar-tests ${TEST_SOURCES}) +target_include_directories(pulsar-tests PRIVATE ${AUTOGEN_DIR}/lib) +target_compile_options(pulsar-tests PRIVATE -DTOKEN_PATH="${TOKEN_PATH}" -DTEST_CONF_DIR="${TEST_CONF_DIR}") +target_link_libraries(pulsar-tests PRIVATE pulsarStatic TESTS_PROTO_OBJECTS ${GTEST_TARGETS}) + +if (UNIX) + add_executable(ConnectionFailTest unix/ConnectionFailTest.cc HttpHelper.cc) + target_link_libraries(ConnectionFailTest pulsarStatic ${GTEST_TARGETS}) +endif () + +add_executable(BrokerMetadataTest brokermetadata/BrokerMetadataTest.cc) +target_link_libraries(BrokerMetadataTest pulsarStatic ${GTEST_TARGETS}) + +add_executable(Oauth2Test oauth2/Oauth2Test.cc) +target_compile_options(Oauth2Test PRIVATE -DTEST_CONF_DIR="${TEST_CONF_DIR}") +target_link_libraries(Oauth2Test pulsarStatic ${GTEST_TARGETS}) + +add_executable(ChunkDedupTest chunkdedup/ChunkDedupTest.cc HttpHelper.cc) +target_link_libraries(ChunkDedupTest pulsarStatic ${GTEST_TARGETS}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3bc5a156..f895d34e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,60 +17,8 @@ # under the License. # -if (NOT PROTOC_PATH) - set(PROTOC_PATH protoc) -endif() - -set(LIB_AUTOGEN_DIR ${AUTOGEN_DIR}/tests) -file(MAKE_DIRECTORY ${LIB_AUTOGEN_DIR}) -include_directories(${LIB_AUTOGEN_DIR}) - -set(PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/proto) -set(PROTO_SOURCES ${LIB_AUTOGEN_DIR}/Test.pb.cc ${LIB_AUTOGEN_DIR}/ExternalTest.pb.cc) -add_custom_command( - OUTPUT ${PROTO_SOURCES} - COMMAND ${PROTOC_PATH} -I ${PROTO_DIR} ${PROTO_DIR}/Test.proto ${PROTO_DIR}/ExternalTest.proto --cpp_out=${LIB_AUTOGEN_DIR}) - -set(PROTO_SOURCE_PADDING ${LIB_AUTOGEN_DIR}/PaddingDemo.pb.cc) -add_custom_command( - OUTPUT ${PROTO_SOURCE_PADDING} - COMMAND ${PROTOC_PATH} -I . ./PaddingDemo.proto --cpp_out=${LIB_AUTOGEN_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - -set(PROTO_SOURCES ${PROTO_SOURCES} ${PROTO_SOURCE_PADDING}) - -include_directories(${LIB_AUTOGEN_DIR}) - -find_library(GMOCK_LIBRARY_PATH gmock) -find_library(GTEST_LIBRARY_PATH gtest) -find_library(GMOCKD_LIBRARY_PATH gmockd) -find_library(GTESTD_LIBRARY_PATH gtestd) -if (NOT GMOCKD_LIBRARY_PATH) - set(GMOCKD_LIBRARY_PATH ${GMOCK_LIBRARY_PATH}) -endif() -if (NOT GTESTD_LIBRARY_PATH) - set(GTESTD_LIBRARY_PATH ${GTEST_LIBRARY_PATH}) -endif() - -file(GLOB TEST_SOURCES *.cc c/*.cc) - -add_executable(pulsar-tests ${TEST_SOURCES} ${PROTO_SOURCES}) - -target_include_directories(pulsar-tests PRIVATE ${AUTOGEN_DIR}/lib) -target_compile_options(pulsar-tests PRIVATE -DTOKEN_PATH="${TOKEN_PATH}" -DTEST_CONF_DIR="${TEST_CONF_DIR}") -target_link_libraries(pulsar-tests ${CLIENT_LIBS} pulsarStatic $<$:${GMOCKD_LIBRARY_PATH}> $<$:${GTESTD_LIBRARY_PATH}> $<$>:${GMOCK_LIBRARY_PATH}> $<$>:${GTEST_LIBRARY_PATH}>) - -if (UNIX) - add_executable(ConnectionFailTest unix/ConnectionFailTest.cc HttpHelper.cc) - target_link_libraries(ConnectionFailTest ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) +if (INTEGRATE_VCPKG) + include(BuildTests.cmake) +else () + include(LegacyBuildTests.cmake) endif () - -add_executable(BrokerMetadataTest brokermetadata/BrokerMetadataTest.cc) -target_link_libraries(BrokerMetadataTest ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) - -add_executable(Oauth2Test oauth2/Oauth2Test.cc) -target_compile_options(Oauth2Test PRIVATE -DTEST_CONF_DIR="${TEST_CONF_DIR}") -target_link_libraries(Oauth2Test ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) - -add_executable(ChunkDedupTest chunkdedup/ChunkDedupTest.cc HttpHelper.cc) -target_link_libraries(ChunkDedupTest ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) diff --git a/tests/LegacyBuildTests.cmake b/tests/LegacyBuildTests.cmake new file mode 100644 index 00000000..1e5335f7 --- /dev/null +++ b/tests/LegacyBuildTests.cmake @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +if (NOT PROTOC_PATH) + set(PROTOC_PATH protoc) +endif() + +set(LIB_AUTOGEN_DIR ${AUTOGEN_DIR}/tests) +file(MAKE_DIRECTORY ${LIB_AUTOGEN_DIR}) +include_directories(${LIB_AUTOGEN_DIR}) + +set(PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/proto) +set(PROTO_SOURCES ${LIB_AUTOGEN_DIR}/Test.pb.cc ${LIB_AUTOGEN_DIR}/ExternalTest.pb.cc) +add_custom_command( + OUTPUT ${PROTO_SOURCES} + COMMAND ${PROTOC_PATH} -I ${PROTO_DIR} ${PROTO_DIR}/Test.proto ${PROTO_DIR}/ExternalTest.proto --cpp_out=${LIB_AUTOGEN_DIR}) + +set(PROTO_SOURCE_PADDING ${LIB_AUTOGEN_DIR}/PaddingDemo.pb.cc) +add_custom_command( + OUTPUT ${PROTO_SOURCE_PADDING} + COMMAND ${PROTOC_PATH} -I . ./PaddingDemo.proto --cpp_out=${LIB_AUTOGEN_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + +set(PROTO_SOURCES ${PROTO_SOURCES} ${PROTO_SOURCE_PADDING}) + +include_directories(${LIB_AUTOGEN_DIR}) + +find_library(GMOCK_LIBRARY_PATH gmock) +find_library(GTEST_LIBRARY_PATH gtest) +find_library(GMOCKD_LIBRARY_PATH gmockd) +find_library(GTESTD_LIBRARY_PATH gtestd) +if (NOT GMOCKD_LIBRARY_PATH) + set(GMOCKD_LIBRARY_PATH ${GMOCK_LIBRARY_PATH}) +endif() +if (NOT GTESTD_LIBRARY_PATH) + set(GTESTD_LIBRARY_PATH ${GTEST_LIBRARY_PATH}) +endif() + +file(GLOB TEST_SOURCES *.cc c/*.cc) + +add_executable(pulsar-tests ${TEST_SOURCES} ${PROTO_SOURCES}) + +target_include_directories(pulsar-tests PRIVATE ${AUTOGEN_DIR}/lib) +target_compile_options(pulsar-tests PRIVATE -DTOKEN_PATH="${TOKEN_PATH}" -DTEST_CONF_DIR="${TEST_CONF_DIR}") +target_link_libraries(pulsar-tests ${CLIENT_LIBS} pulsarStatic $<$:${GMOCKD_LIBRARY_PATH}> $<$:${GTESTD_LIBRARY_PATH}> $<$>:${GMOCK_LIBRARY_PATH}> $<$>:${GTEST_LIBRARY_PATH}>) + +if (UNIX) + add_executable(ConnectionFailTest unix/ConnectionFailTest.cc HttpHelper.cc) + target_link_libraries(ConnectionFailTest ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) +endif () + +add_executable(BrokerMetadataTest brokermetadata/BrokerMetadataTest.cc) +target_link_libraries(BrokerMetadataTest ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) + +add_executable(Oauth2Test oauth2/Oauth2Test.cc) +target_compile_options(Oauth2Test PRIVATE -DTEST_CONF_DIR="${TEST_CONF_DIR}") +target_link_libraries(Oauth2Test ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) + +add_executable(ChunkDedupTest chunkdedup/ChunkDedupTest.cc HttpHelper.cc) +target_link_libraries(ChunkDedupTest ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) + diff --git a/vcpkg b/vcpkg new file mode 160000 index 00000000..b051745c --- /dev/null +++ b/vcpkg @@ -0,0 +1 @@ +Subproject commit b051745c68faa6f65c493371d564c4eb8af34dad diff --git a/vcpkg.json b/vcpkg.json index 5cc38635..1f9ed97f 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,29 +1,104 @@ { "name": "pulsar-cpp", - "version": "2.8.0", + "version": "3.5.0", "description": "Pulsar C++ SDK", + "builtin-baseline": "b051745c68faa6f65c493371d564c4eb8af34dad", "dependencies": [ - "boost-accumulators", - "boost-algorithm", - "boost-any", - "boost-circular-buffer", - "boost-asio", - "boost-date-time", - "boost-predef", - "boost-program-options", - "boost-property-tree", - "boost-random", - "boost-serialization", - "boost-xpressive", - "curl", - "openssl", - "protobuf", - "snappy", - "zlib", - "zstd", + { + "name": "boost-accumulators", + "version>=": "1.83.0" + }, + { + "name": "boost-algorithm", + "version>=": "1.83.0" + }, + { + "name": "boost-any", + "version>=": "1.83.0" + }, + { + "name": "boost-asio", + "version>=": "1.83.0" + }, + { + "name": "boost-circular-buffer", + "version>=": "1.83.0" + }, + { + "name": "boost-date-time", + "version>=": "1.83.0" + }, + { + "name": "boost-predef", + "version>=": "1.83.0" + }, + { + "name": "boost-property-tree", + "version>=": "1.83.0" + }, + { + "name": "boost-random", + "version>=": "1.83.0" + }, + { + "name": "boost-serialization", + "version>=": "1.83.0" + }, + { + "name": "boost-xpressive", + "version>=": "1.83.0" + }, + { + "name": "curl", + "default-features": false, + "features": [ + "openssl" + ], + "version>=": "8.4.0" + }, { "name": "dlfcn-win32", "platform": "windows" + }, + { + "name": "openssl", + "version>=": "3.1.4#1" + }, + { + "name": "protobuf", + "version>=": "3.21.12" + }, + { + "name": "snappy", + "version>=": "1.1.10" + }, + { + "name": "zlib", + "version>=": "1.3" + }, + { + "name": "zstd", + "version>=": "1.5.5" + } + ], + "features": { + "perf": { + "description": "Build Performance Tool", + "dependencies": [ + { + "name": "boost-program-options", + "version>=": "1.83.0" + } + ] + }, + "tests": { + "description": "Build Tests", + "dependencies": [ + { + "name": "gtest", + "version>=": "1.14.0" + } + ] } - ] + } } From e04686d6cd595c7801aabe37f1ca4b7dfdb8dc95 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 19 Dec 2023 10:41:22 +0800 Subject: [PATCH 19/54] Fix the unstable wireshark workflow on macOS (#374) See https://github.com/Homebrew/homebrew-core/issues/157142 --- .github/workflows/ci-pr-validation.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 20d1e14e..50a5c807 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -50,6 +50,8 @@ jobs: - name: Install deps (macOS) if: ${{ startsWith(matrix.os, 'macos') }} run: | + # See https://github.com/Homebrew/homebrew-core/issues/157142 + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 brew update brew install pkg-config wireshark protobuf - name: Build wireshark plugin From 25ea451eb3a79d48966689ec64460ae03d5d57da Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Mon, 18 Dec 2023 19:36:12 -0800 Subject: [PATCH 20/54] [PIP-60] [Proxy-Client] Support SNI routing for Pulsar CPP client (#373) * [PIP-60] [Proxy-Server] Support SNI routing for Pulsar CPP client * fix format * fix const def * Fix format check --------- Co-authored-by: Yunze Xu --- include/pulsar/ClientConfiguration.h | 31 ++++++++++++++++++++++++++++ lib/ClientConfiguration.cc | 16 ++++++++++++++ lib/ClientConfigurationImpl.h | 2 ++ lib/ClientConnection.cc | 20 ++++++++++++++---- lib/ClientConnection.h | 5 +++++ tests/ConsumerTest.cc | 12 +++++++++++ 6 files changed, 82 insertions(+), 4 deletions(-) diff --git a/include/pulsar/ClientConfiguration.h b/include/pulsar/ClientConfiguration.h index 3d651e9c..cc3c3ed3 100644 --- a/include/pulsar/ClientConfiguration.h +++ b/include/pulsar/ClientConfiguration.h @@ -32,6 +32,10 @@ class PULSAR_PUBLIC ClientConfiguration { ~ClientConfiguration(); ClientConfiguration(const ClientConfiguration&); ClientConfiguration& operator=(const ClientConfiguration&); + enum ProxyProtocol + { + SNI = 0 + }; /** * Configure a limit on the amount of memory that will be allocated by this client instance. @@ -320,6 +324,33 @@ class PULSAR_PUBLIC ClientConfiguration { */ ClientConfiguration& setConnectionTimeout(int timeoutMs); + /** + * Set proxy-service url when client would like to connect to broker via proxy. Client must configure both + * proxyServiceUrl and appropriate proxyProtocol. + * + * Example: pulsar+ssl://ats-proxy.example.com:4443 + * + * @param proxyServiceUrl proxy url to connect with broker + * @return + */ + ClientConfiguration& setProxyServiceUrl(const std::string& proxyServiceUrl); + + const std::string& getProxyServiceUrl() const; + + /** + * Set appropriate proxy-protocol along with proxy-service url. Currently Pulsar supports SNI proxy + * routing. + * + * SNI routing: + * https://docs.trafficserver.apache.org/en/latest/admin-guide/layer-4-routing.en.html#sni-routing. + * + * @param proxyProtocol possible options (SNI) + * @return + */ + ClientConfiguration& setProxyProtocol(ProxyProtocol proxyProtocol); + + ProxyProtocol getProxyProtocol() const; + /** * The getter associated with setConnectionTimeout(). */ diff --git a/lib/ClientConfiguration.cc b/lib/ClientConfiguration.cc index 63c0bf89..6e7c7456 100644 --- a/lib/ClientConfiguration.cc +++ b/lib/ClientConfiguration.cc @@ -134,6 +134,22 @@ ClientConfiguration& ClientConfiguration::setConcurrentLookupRequest(int concurr return *this; } +ClientConfiguration& ClientConfiguration::setProxyServiceUrl(const std::string& proxyServiceUrl) { + impl_->proxyServiceUrl = proxyServiceUrl; + return *this; +} + +const std::string& ClientConfiguration::getProxyServiceUrl() const { return impl_->proxyServiceUrl; } + +ClientConfiguration& ClientConfiguration::setProxyProtocol(ClientConfiguration::ProxyProtocol proxyProtocol) { + impl_->proxyProtocol = proxyProtocol; + return *this; +} + +ClientConfiguration::ProxyProtocol ClientConfiguration::getProxyProtocol() const { + return impl_->proxyProtocol; +} + int ClientConfiguration::getConcurrentLookupRequest() const { return impl_->concurrentLookupRequest; } ClientConfiguration& ClientConfiguration::setMaxLookupRedirects(int maxLookupRedirects) { diff --git a/lib/ClientConfigurationImpl.h b/lib/ClientConfigurationImpl.h index 3458a052..b62b97c6 100644 --- a/lib/ClientConfigurationImpl.h +++ b/lib/ClientConfigurationImpl.h @@ -46,6 +46,8 @@ struct ClientConfigurationImpl { std::string listenerName; int connectionTimeoutMs{10000}; // 10 seconds std::string description; + std::string proxyServiceUrl; + ClientConfiguration::ProxyProtocol proxyProtocol; std::unique_ptr takeLogger() { return std::move(loggerFactory); } }; diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index 97d88476..61aa7f73 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -209,7 +209,15 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: boost::asio::ssl::context ctx(executor_->getIOService(), boost::asio::ssl::context::tlsv1_client); #endif Url serviceUrl; + Url proxyUrl; Url::parse(physicalAddress, serviceUrl); + proxyServiceUrl_ = clientConfiguration.getProxyServiceUrl(); + proxyProtocol_ = clientConfiguration.getProxyProtocol(); + if (proxyProtocol_ == ClientConfiguration::SNI && !proxyServiceUrl_.empty()) { + Url::parse(proxyServiceUrl_, proxyUrl); + isSniProxy_ = true; + LOG_INFO("Configuring SNI Proxy-url=" << proxyServiceUrl_); + } if (clientConfiguration.isTlsAllowInsecureConnection()) { ctx.set_verify_mode(boost::asio::ssl::context::verify_none); isTlsAllowInsecureConnection_ = true; @@ -257,7 +265,8 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: if (!clientConfiguration.isTlsAllowInsecureConnection() && clientConfiguration.isValidateHostName()) { LOG_DEBUG("Validating hostname for " << serviceUrl.host() << ":" << serviceUrl.port()); - tlsSocket_->set_verify_callback(boost::asio::ssl::rfc2818_verification(serviceUrl.host())); + std::string urlHost = isSniProxy_ ? proxyUrl.host() : serviceUrl.host(); + tlsSocket_->set_verify_callback(boost::asio::ssl::rfc2818_verification(urlHost)); } LOG_DEBUG("TLS SNI Host: " << serviceUrl.host()); @@ -403,7 +412,8 @@ void ClientConnection::handleTcpConnected(const boost::system::error_code& err, if (logicalAddress_ == physicalAddress_) { LOG_INFO(cnxString_ << "Connected to broker"); } else { - LOG_INFO(cnxString_ << "Connected to broker through proxy. Logical broker: " << logicalAddress_); + LOG_INFO(cnxString_ << "Connected to broker through proxy. Logical broker: " << logicalAddress_ + << ", proxy: " << proxyServiceUrl_); } Lock lock(mutex_); @@ -572,7 +582,8 @@ void ClientConnection::tcpConnectAsync() { boost::system::error_code err; Url service_url; - if (!Url::parse(physicalAddress_, service_url)) { + std::string hostUrl = isSniProxy_ ? proxyServiceUrl_ : physicalAddress_; + if (!Url::parse(hostUrl, service_url)) { LOG_ERROR(cnxString_ << "Invalid Url, unable to parse: " << err << " " << err.message()); close(); return; @@ -600,7 +611,8 @@ void ClientConnection::tcpConnectAsync() { void ClientConnection::handleResolve(const boost::system::error_code& err, tcp::resolver::iterator endpointIterator) { if (err) { - LOG_ERROR(cnxString_ << "Resolve error: " << err << " : " << err.message()); + std::string hostUrl = isSniProxy_ ? cnxString_ : proxyServiceUrl_; + LOG_ERROR(hostUrl << "Resolve error: " << err << " : " << err.message()); close(); return; } diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index ce2bd885..1bc1bd85 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -333,6 +333,10 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this server:6650] std::string cnxString_; @@ -384,6 +388,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this Date: Wed, 27 Dec 2023 00:52:19 +0900 Subject: [PATCH 21/54] Fix issue where custom logger setting is ignored (#377) ### Motivation I noticed that there are cases where the specified logger factory is not used. For example, if we compile and run the following code, it expects `FileLoggerFactory` to be used, but it actually uses the default `ConsoleLoggerFactory`. ```cpp ClientConfiguration config = ClientConfiguration(); config.setLogger(new FileLoggerFactory(Logger::LEVEL_INFO, "pulsar-client-cpp.log")); ParamMap params; params["providerDomain"] = "provider.foo.bar"; params["tenantDomain"] = "tenant.foo.bar"; params["tenantService"] = "test-service"; params["privateKey"] = "file:///path/to/private.key"; params["keyId"] = "0"; params["ztsUrl"] = "https://zts.athenz.example.com:443"; AuthenticationPtr auth = AuthAthenz::create(params); config.setAuth(auth); Client client("pulsar://localhost:6650", config); ``` This happens if logging is performed before `LogUtils::setLoggerFactory` is executed in the constructor of `ClientImpl`. https://github.com/apache/pulsar-client-cpp/blob/25ea451eb3a79d48966689ec64460ae03d5d57da/lib/ClientImpl.cc#L94-L99 When logging is performed, `LogUtils::getLoggerFactory` is called. 1. https://github.com/apache/pulsar-client-cpp/blob/25ea451eb3a79d48966689ec64460ae03d5d57da/lib/LogUtils.h#L60 1. https://github.com/apache/pulsar-client-cpp/blob/25ea451eb3a79d48966689ec64460ae03d5d57da/lib/LogUtils.h#L43 If the constructor of `ClientImpl` has not been executed at this point, the default `ConsoleLoggerFactory` is instantiated, set to the static variable `s_loggerFactory`, and returned to the caller. https://github.com/apache/pulsar-client-cpp/blob/25ea451eb3a79d48966689ec64460ae03d5d57da/lib/LogUtils.cc#L38-L44 Loggers generated from the returned `LoggerFactory` will be set to the static and thread_local variable `threadSpecificLogPtr`, and reused from now on. https://github.com/apache/pulsar-client-cpp/blob/25ea451eb3a79d48966689ec64460ae03d5d57da/lib/LogUtils.h#L38-L47 After that, `LogUtils::setLoggerFactory` is executed again in the constructor of `ClientImpl`, but the value can only be set to `s_loggerFactory` once, and it will be ignored from the second time. This is clearly stated in [the comments](https://github.com/apache/pulsar-client-cpp/blob/25ea451eb3a79d48966689ec64460ae03d5d57da/include/pulsar/ClientConfiguration.h#L183-L186). https://github.com/apache/pulsar-client-cpp/blob/25ea451eb3a79d48966689ec64460ae03d5d57da/lib/LogUtils.cc#L30-L36 Even if we can set it again, if a logger is already set to `threadSpecificLogPtr`, a new logger will not be generated from the new `LoggerFactory`. ### Modifications Keep an instance of the default `ConsoleLoggerFactory` in `s_defaultLoggerFactory`, a variable separate from `s_loggerFactory`, and return it if `LoggerFactory` is not set yet. In addition, keep the pointer value of the `LoggerFactory` that generated the logger cached in `threadSpecificLogPtr`, and regenerate the logger if the `LoggerFactory` pointer changes. --- lib/ClientImpl.cc | 6 ++---- lib/LogUtils.cc | 7 ++++--- lib/LogUtils.h | 23 +++++++++++++---------- tests/CustomLoggerTest.cc | 10 ++++++++++ 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/lib/ClientImpl.cc b/lib/ClientImpl.cc index 1032b79e..2cfbd387 100644 --- a/lib/ClientImpl.cc +++ b/lib/ClientImpl.cc @@ -92,11 +92,9 @@ ClientImpl::ClientImpl(const std::string& serviceUrl, const ClientConfiguration& consumerIdGenerator_(0), closingError(ResultOk) { std::unique_ptr loggerFactory = clientConfiguration_.impl_->takeLogger(); - if (!loggerFactory) { - // Use default simple console logger - loggerFactory.reset(new ConsoleLoggerFactory); + if (loggerFactory) { + LogUtils::setLoggerFactory(std::move(loggerFactory)); } - LogUtils::setLoggerFactory(std::move(loggerFactory)); LookupServicePtr underlyingLookupServicePtr; if (serviceNameResolver_.useHttp()) { diff --git a/lib/LogUtils.cc b/lib/LogUtils.cc index b210f1bc..8739bb82 100644 --- a/lib/LogUtils.cc +++ b/lib/LogUtils.cc @@ -25,6 +25,7 @@ namespace pulsar { +static std::atomic s_defaultLoggerFactory(new ConsoleLoggerFactory()); static std::atomic s_loggerFactory(nullptr); void LogUtils::setLoggerFactory(std::unique_ptr loggerFactory) { @@ -37,10 +38,10 @@ void LogUtils::setLoggerFactory(std::unique_ptr loggerFactory) { LoggerFactory* LogUtils::getLoggerFactory() { if (s_loggerFactory.load() == nullptr) { - std::unique_ptr newFactory(new ConsoleLoggerFactory()); - setLoggerFactory(std::move(newFactory)); + return s_defaultLoggerFactory.load(); + } else { + return s_loggerFactory.load(); } - return s_loggerFactory.load(); } std::string LogUtils::getLoggerName(const std::string& path) { diff --git a/lib/LogUtils.h b/lib/LogUtils.h index c8abd138..3df1a7ab 100644 --- a/lib/LogUtils.h +++ b/lib/LogUtils.h @@ -34,16 +34,19 @@ namespace pulsar { #define PULSAR_UNLIKELY(expr) (expr) #endif -#define DECLARE_LOG_OBJECT() \ - static pulsar::Logger* logger() { \ - static thread_local std::unique_ptr threadSpecificLogPtr; \ - pulsar::Logger* ptr = threadSpecificLogPtr.get(); \ - if (PULSAR_UNLIKELY(!ptr)) { \ - std::string logger = pulsar::LogUtils::getLoggerName(__FILE__); \ - threadSpecificLogPtr.reset(pulsar::LogUtils::getLoggerFactory()->getLogger(logger)); \ - ptr = threadSpecificLogPtr.get(); \ - } \ - return ptr; \ +#define DECLARE_LOG_OBJECT() \ + static pulsar::Logger* logger() { \ + static thread_local uintptr_t loggerFactoryPtr = 0; \ + static thread_local std::unique_ptr threadSpecificLogPtr; \ + pulsar::Logger* ptr = threadSpecificLogPtr.get(); \ + if (PULSAR_UNLIKELY(loggerFactoryPtr != (uintptr_t)pulsar::LogUtils::getLoggerFactory()) || \ + PULSAR_UNLIKELY(!ptr)) { \ + std::string logger = pulsar::LogUtils::getLoggerName(__FILE__); \ + threadSpecificLogPtr.reset(pulsar::LogUtils::getLoggerFactory()->getLogger(logger)); \ + ptr = threadSpecificLogPtr.get(); \ + loggerFactoryPtr = (uintptr_t)pulsar::LogUtils::getLoggerFactory(); \ + } \ + return ptr; \ } #define LOG_DEBUG(message) \ diff --git a/tests/CustomLoggerTest.cc b/tests/CustomLoggerTest.cc index dde4056d..22fbb4e6 100644 --- a/tests/CustomLoggerTest.cc +++ b/tests/CustomLoggerTest.cc @@ -107,3 +107,13 @@ TEST(CustomLoggerTest, testConsoleLoggerFactory) { ASSERT_FALSE(logger->isEnabled(Logger::LEVEL_WARN)); ASSERT_TRUE(logger->isEnabled(Logger::LEVEL_ERROR)); } + +TEST(CustomLoggerTest, testSetAndGetLoggerFactory) { + LoggerFactory *oldFactory = LogUtils::getLoggerFactory(); + LoggerFactory *newFactory = new ConsoleLoggerFactory(Logger::LEVEL_ERROR); + std::unique_ptr newFactoryPtr(newFactory); + LogUtils::setLoggerFactory(std::move(newFactoryPtr)); + ASSERT_NE(oldFactory, LogUtils::getLoggerFactory()); + ASSERT_EQ(newFactory, LogUtils::getLoggerFactory()); + LogUtils::resetLoggerFactory(); +} From a4b7f120aa0a7f74d8e1d837629a48caab1403bd Mon Sep 17 00:00:00 2001 From: erobot Date: Thu, 4 Jan 2024 10:28:41 +0800 Subject: [PATCH 22/54] Retry on new partition producer creation failure (#378) Fixes #319 ### Motivation Already created producer should not fail after new partition producers creation failure. ### Modifications `ProducerImpl`: Add an option retryOnCreationError to control whether to retry on creation error `PartitionedProducerImpl`: Use retryOnCreationError=true to create new partition producers --- lib/PartitionedProducerImpl.cc | 12 +++++++----- lib/PartitionedProducerImpl.h | 2 +- lib/ProducerImpl.cc | 7 ++++--- lib/ProducerImpl.h | 5 ++++- tests/ProducerTest.cc | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 48 insertions(+), 10 deletions(-) diff --git a/lib/PartitionedProducerImpl.cc b/lib/PartitionedProducerImpl.cc index a799d5c8..54e96c84 100644 --- a/lib/PartitionedProducerImpl.cc +++ b/lib/PartitionedProducerImpl.cc @@ -92,10 +92,12 @@ unsigned int PartitionedProducerImpl::getNumPartitionsWithLock() const { return getNumPartitions(); } -ProducerImplPtr PartitionedProducerImpl::newInternalProducer(unsigned int partition, bool lazy) { +ProducerImplPtr PartitionedProducerImpl::newInternalProducer(unsigned int partition, bool lazy, + bool retryOnCreationError) { using namespace std::placeholders; auto client = client_.lock(); - auto producer = std::make_shared(client, *topicName_, conf_, interceptors_, partition); + auto producer = std::make_shared(client, *topicName_, conf_, interceptors_, partition, + retryOnCreationError); if (!client) { return producer; } @@ -127,13 +129,13 @@ void PartitionedProducerImpl::start() { for (unsigned int i = 0; i < getNumPartitions(); i++) { bool lazy = (short)i != partition; - producers_.push_back(newInternalProducer(i, lazy)); + producers_.push_back(newInternalProducer(i, lazy, false)); } producers_[partition]->start(); } else { for (unsigned int i = 0; i < getNumPartitions(); i++) { - producers_.push_back(newInternalProducer(i, false)); + producers_.push_back(newInternalProducer(i, false, false)); } for (ProducerList::const_iterator prod = producers_.begin(); prod != producers_.end(); prod++) { @@ -461,7 +463,7 @@ void PartitionedProducerImpl::handleGetPartitions(Result result, for (unsigned int i = currentNumPartitions; i < newNumPartitions; i++) { ProducerImplPtr producer; try { - producer = newInternalProducer(i, lazy); + producer = newInternalProducer(i, lazy, true); } catch (const std::runtime_error& e) { LOG_ERROR("Failed to create producer for partition " << i << ": " << e.what()); producers.clear(); diff --git a/lib/PartitionedProducerImpl.h b/lib/PartitionedProducerImpl.h index 25ba9c33..2d07a81a 100644 --- a/lib/PartitionedProducerImpl.h +++ b/lib/PartitionedProducerImpl.h @@ -135,7 +135,7 @@ class PartitionedProducerImpl : public ProducerImplBase, unsigned int getNumPartitions() const; unsigned int getNumPartitionsWithLock() const; - ProducerImplPtr newInternalProducer(unsigned int partition, bool lazy); + ProducerImplPtr newInternalProducer(unsigned int partition, bool lazy, bool retryOnCreationError); MessageRoutingPolicyPtr getMessageRouter(); void runPartitionUpdateTask(); void getPartitionMetadata(); diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index 61b95bf0..0a129259 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -48,7 +48,7 @@ DECLARE_LOG_OBJECT() ProducerImpl::ProducerImpl(ClientImplPtr client, const TopicName& topicName, const ProducerConfiguration& conf, const ProducerInterceptorsPtr& interceptors, - int32_t partition) + int32_t partition, bool retryOnCreationError) : HandlerBase(client, (partition < 0) ? topicName.toString() : topicName.getTopicPartitionName(partition), Backoff(milliseconds(client->getClientConfig().getInitialBackoffIntervalMs()), milliseconds(client->getClientConfig().getMaxBackoffIntervalMs()), @@ -67,7 +67,8 @@ ProducerImpl::ProducerImpl(ClientImplPtr client, const TopicName& topicName, dataKeyRefreshTask_(*executor_, 4 * 60 * 60 * 1000), memoryLimitController_(client->getMemoryLimitController()), chunkingEnabled_(conf_.isChunkingEnabled() && topicName.isPersistent() && !conf_.getBatchingEnabled()), - interceptors_(interceptors) { + interceptors_(interceptors), + retryOnCreationError_(retryOnCreationError) { LOG_DEBUG("ProducerName - " << producerName_ << " Created producer on topic " << topic() << " id: " << producerId_); if (!producerName_.empty()) { @@ -273,7 +274,7 @@ Result ProducerImpl::handleCreateProducer(const ClientConnectionPtr& cnx, Result lock.unlock(); producerCreatedPromise_.setFailed(result); handleResult = result; - } else if (producerCreatedPromise_.isComplete()) { + } else if (producerCreatedPromise_.isComplete() || retryOnCreationError_) { if (result == ResultProducerBlockedQuotaExceededException) { LOG_WARN(getName() << "Backlog is exceeded on topic. Sending exception to producer"); failPendingMessages(ResultProducerBlockedQuotaExceededException, false); diff --git a/lib/ProducerImpl.h b/lib/ProducerImpl.h index 2fb0b886..b467458d 100644 --- a/lib/ProducerImpl.h +++ b/lib/ProducerImpl.h @@ -66,7 +66,8 @@ class ProducerImpl : public HandlerBase, public ProducerImplBase { public: ProducerImpl(ClientImplPtr client, const TopicName& topic, const ProducerConfiguration& producerConfiguration, - const ProducerInterceptorsPtr& interceptors, int32_t partition = -1); + const ProducerInterceptorsPtr& interceptors, int32_t partition = -1, + bool retryOnCreationError = false); ~ProducerImpl(); // overrided methods from ProducerImplBase @@ -202,6 +203,8 @@ class ProducerImpl : public HandlerBase, public ProducerImplBase { boost::optional topicEpoch; ProducerInterceptorsPtr interceptors_; + + bool retryOnCreationError_; }; struct ProducerImplCmp { diff --git a/tests/ProducerTest.cc b/tests/ProducerTest.cc index 45bb3aae..bb58a4ef 100644 --- a/tests/ProducerTest.cc +++ b/tests/ProducerTest.cc @@ -649,4 +649,36 @@ TEST(ProducerTest, testReconnectMultiConnectionsPerBroker) { client.close(); } +TEST(ProducerTest, testFailedToCreateNewPartitionProducer) { + const std::string topic = + "public/default/testFailedToCreateNewPartitionProducer" + std::to_string(time(nullptr)); + std::string topicOperateUrl = adminUrl + "admin/v2/persistent/" + topic + "/partitions"; + + int res = makePutRequest(topicOperateUrl, "2"); + ASSERT_TRUE(res == 204 || res == 409) << "res: " << res; + + ClientConfiguration clientConf; + clientConf.setPartititionsUpdateInterval(1); + Client client(serviceUrl, clientConf); + ProducerConfiguration conf; + Producer producer; + client.createProducer(topic, conf, producer); + ASSERT_TRUE(waitUntil(std::chrono::seconds(1), [&producer]() -> bool { return producer.isConnected(); })); + + PartitionedProducerImpl& partitionedProducer = PulsarFriend::getPartitionedProducerImpl(producer); + PulsarFriend::updatePartitions(partitionedProducer, 3); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + auto& newProducer = PulsarFriend::getInternalProducerImpl(producer, 2); + ASSERT_FALSE(newProducer.isConnected()); // should fail with topic not found + + res = makePostRequest(topicOperateUrl, "3"); + ASSERT_TRUE(res == 204 || res == 409) << "res: " << res; + + ASSERT_TRUE( + waitUntil(std::chrono::seconds(5), [&newProducer]() -> bool { return newProducer.isConnected(); })); + + producer.close(); + client.close(); +} + INSTANTIATE_TEST_CASE_P(Pulsar, ProducerTest, ::testing::Values(true, false)); From f7e493b593223d0e961f0babbbb47809a4123126 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 5 Jan 2024 14:57:50 +0800 Subject: [PATCH 23/54] Remove the Boost.Random dependency (#380) Master issue: https://github.com/apache/pulsar-client-cpp/issues/367 ### Motivation We still depends on the Boost.Random dependency on some places, which could be replaced by the C++ standard random library. ### Modifications - Replace Boost.Random usages with the standard `` header - Remove the `boost-random` dependency from `vcpkg.json` --- lib/Backoff.cc | 3 +-- lib/Backoff.h | 4 ++-- lib/RoundRobinMessageRouter.cc | 7 +++---- vcpkg.json | 4 ---- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/Backoff.cc b/lib/Backoff.cc index 4d954220..fdb13592 100644 --- a/lib/Backoff.cc +++ b/lib/Backoff.cc @@ -21,7 +21,6 @@ #include /* time */ #include -#include namespace pulsar { @@ -47,7 +46,7 @@ TimeDuration Backoff::next() { } } // Add Randomness - boost::random::uniform_int_distribution dist; + std::uniform_int_distribution dist; int randomNumber = dist(rng_); current = current - (current * (randomNumber % 10) / 100); diff --git a/lib/Backoff.h b/lib/Backoff.h index 4bcebc75..a59c00f5 100644 --- a/lib/Backoff.h +++ b/lib/Backoff.h @@ -21,7 +21,7 @@ #include #include -#include +#include namespace pulsar { @@ -39,7 +39,7 @@ class PULSAR_PUBLIC Backoff { TimeDuration next_; TimeDuration mandatoryStop_; boost::posix_time::ptime firstBackoffTime_; - boost::random::mt19937 rng_; + std::mt19937 rng_; bool mandatoryStopMade_ = false; friend class PulsarFriend; diff --git a/lib/RoundRobinMessageRouter.cc b/lib/RoundRobinMessageRouter.cc index e33f8d95..9693cc25 100644 --- a/lib/RoundRobinMessageRouter.cc +++ b/lib/RoundRobinMessageRouter.cc @@ -18,8 +18,7 @@ */ #include "RoundRobinMessageRouter.h" -#include -#include +#include #include "Hash.h" #include "TimeUtils.h" @@ -37,8 +36,8 @@ RoundRobinMessageRouter::RoundRobinMessageRouter(ProducerConfiguration::HashingS lastPartitionChange_(TimeUtils::currentTimeMillis()), msgCounter_(0), cumulativeBatchSize_(0) { - boost::random::mt19937 rng(time(nullptr)); - boost::random::uniform_int_distribution dist; + std::mt19937 rng(time(nullptr)); + std::uniform_int_distribution dist; currentPartitionCursor_ = dist(rng); } diff --git a/vcpkg.json b/vcpkg.json index 1f9ed97f..d023a406 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -36,10 +36,6 @@ "name": "boost-property-tree", "version>=": "1.83.0" }, - { - "name": "boost-random", - "version>=": "1.83.0" - }, { "name": "boost-serialization", "version>=": "1.83.0" From 90ea3695b80660f837785d98e96047d90de3f64f Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 8 Jan 2024 02:13:49 +0800 Subject: [PATCH 24/54] Depend on the independent Asio instead of Boost.Asio by default (#382) Fixes https://github.com/apache/pulsar-client-cpp/issues/367 ### Motivation See the difference of Asio and Boost.Asio here: https://think-async.com/Asio/AsioAndBoostAsio.html Asio is updated more frequently than Boost.Asio and its release does not need to be synchronous with other Boost components. Depending on the independent Asio could make it easier for a newer Asio release. ### Modifications Import `asio` 1.28.2 as the dependency and remove the `boost-asio` dependency from the vcpkg.json. Since the latest Asio already removed the `deadline_timer`, this patch replaces all `deadline_timer` with `steady_timer`, which uses `std::chrono` rather than Boost.Date_Time component to compute the timeout. Add a `USE_ASIO` CMake option to determine whether Asio or Boost.Asio is depended. For vcpkg users, the option is always enabled. Finally, simplify the vcpkg.json by removing some `boost-*` dependencies depended indirectly by the rest two major dependencies: - boost-accumulators: latency percentiles computation - boost-property-tree: JSON operations These two dependencies are hard to remove for now unless introducing other dependencies so they will be kept from some time. --- .../workflows/ci-build-binary-artifacts.yaml | 2 + .github/workflows/ci-pr-validation.yaml | 2 + CMakeLists.txt | 11 + LegacyFindPackages.cmake | 4 - lib/AckGroupingTrackerEnabled.cc | 6 +- lib/AckGroupingTrackerEnabled.h | 3 +- lib/AsioDefines.h | 32 +++ lib/{TimeUtils.cc => AsioTimer.h} | 19 +- lib/Backoff.cc | 7 +- lib/Backoff.h | 8 +- lib/ClientConnection.cc | 254 ++++++++---------- lib/ClientConnection.h | 73 +++-- lib/CompressionCodec.cc | 1 - lib/ConnectionPool.cc | 9 +- lib/ConsumerImpl.cc | 19 +- lib/ConsumerImplBase.cc | 4 +- lib/ExecutorService.cc | 24 +- lib/ExecutorService.h | 20 +- lib/HandlerBase.cc | 9 +- lib/HandlerBase.h | 10 +- lib/Int64SerDes.h | 7 +- lib/MultiTopicsConsumerImpl.cc | 10 +- lib/MultiTopicsConsumerImpl.h | 3 +- lib/NegativeAcksTracker.cc | 12 +- lib/NegativeAcksTracker.h | 8 +- lib/OpSendMsg.h | 6 +- lib/PartitionedProducerImpl.cc | 8 +- lib/PartitionedProducerImpl.h | 6 +- lib/PatternMultiTopicsConsumerImpl.cc | 12 +- lib/PatternMultiTopicsConsumerImpl.h | 5 +- lib/PeriodicTask.cc | 8 +- lib/PeriodicTask.h | 2 +- lib/ProducerImpl.cc | 25 +- lib/ProducerImpl.h | 15 +- lib/RetryableOperation.h | 23 +- lib/RoundRobinMessageRouter.cc | 5 +- lib/RoundRobinMessageRouter.h | 6 +- lib/SharedBuffer.h | 27 +- lib/TimeUtils.h | 16 +- lib/UnAckedMessageTrackerEnabled.cc | 6 +- lib/UnAckedMessageTrackerEnabled.h | 3 +- lib/auth/athenz/ZTSClient.cc | 2 - lib/stats/ConsumerStatsImpl.cc | 6 +- lib/stats/ConsumerStatsImpl.h | 5 +- lib/stats/ProducerStatsBase.h | 4 +- lib/stats/ProducerStatsDisabled.h | 2 +- lib/stats/ProducerStatsImpl.cc | 15 +- lib/stats/ProducerStatsImpl.h | 8 +- tests/AuthPluginTest.cc | 12 +- tests/AuthTokenTest.cc | 1 - tests/BackoffTest.cc | 26 +- tests/ConsumerTest.cc | 2 +- tests/PulsarFriend.h | 2 +- tests/RoundRobinMessageRouterTest.cc | 16 +- vcpkg.json | 37 +-- 55 files changed, 441 insertions(+), 427 deletions(-) create mode 100644 lib/AsioDefines.h rename lib/{TimeUtils.cc => AsioTimer.h} (71%) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index a2b7be84..63644e5a 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -148,6 +148,7 @@ jobs: mkdir -p $BUILD_DIR cmake -B $BUILD_DIR \ -G "${{ matrix.generator }}" ${{ matrix.arch }} \ + -DUSE_ASIO=ON \ -DBUILD_TESTS=OFF \ -DVCPKG_TRIPLET=${{ matrix.triplet }} \ -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} \ @@ -174,6 +175,7 @@ jobs: mkdir -p $BUILD_DIR cmake -B $BUILD_DIR \ -G "${{ matrix.generator }}" ${{ matrix.arch }} \ + -DUSE_ASIO=ON \ -DBUILD_TESTS=OFF \ -DVCPKG_TRIPLET=${{ matrix.triplet }} \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR_DEBUG \ diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 50a5c807..56309e9a 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -191,6 +191,7 @@ jobs: cmake \ -B ./build-1 \ -G "${{ matrix.generator }}" ${{ matrix.arch }} \ + -DUSE_ASIO=ON \ -DBUILD_TESTS=OFF \ -DVCPKG_TRIPLET="${{ matrix.triplet }}" \ -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \ @@ -232,6 +233,7 @@ jobs: cmake \ -B ./build-2 \ -G "${{ matrix.generator }}" ${{ matrix.arch }} \ + -DUSE_ASIO=ON \ -DBUILD_TESTS=OFF \ -DVCPKG_TRIPLET="${{ matrix.triplet }}" \ -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 5bde2b77..662e84af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,8 +19,11 @@ cmake_minimum_required(VERSION 3.13) +option(USE_ASIO "Use Asio instead of Boost.Asio" OFF) + option(INTEGRATE_VCPKG "Integrate with Vcpkg" OFF) if (INTEGRATE_VCPKG) + set(USE_ASIO ON) set(CMAKE_TOOLCHAIN_FILE "${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake") endif () @@ -129,6 +132,10 @@ if (INTEGRATE_VCPKG) $,zstd::libzstd_shared,zstd::libzstd_static> Snappy::snappy ) + if (USE_ASIO) + find_package(asio CONFIG REQUIRED) + set(COMMON_LIBS ${COMMON_LIBS} asio::asio) + endif () add_definitions(-DHAS_ZSTD -DHAS_SNAPPY) if (MSVC) find_package(dlfcn-win32 CONFIG REQUIRED) @@ -140,6 +147,10 @@ else () include(./LegacyFindPackages.cmake) endif () +if (USE_ASIO) + add_definitions(-DUSE_ASIO) +endif () + set(LIB_NAME $ENV{PULSAR_LIBRARY_NAME}) if (NOT LIB_NAME) set(LIB_NAME pulsar) diff --git a/LegacyFindPackages.cmake b/LegacyFindPackages.cmake index 10c9fa7c..5004545b 100644 --- a/LegacyFindPackages.cmake +++ b/LegacyFindPackages.cmake @@ -176,10 +176,6 @@ if (Boost_MAJOR_VERSION EQUAL 1 AND Boost_MINOR_VERSION LESS 69) MESSAGE(STATUS "Linking with Boost:System") endif() -if (MSVC) - set(BOOST_COMPONENTS ${BOOST_COMPONENTS} date_time) -endif() - if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) # GCC 4.8.2 implementation of std::regex is buggy set(BOOST_COMPONENTS ${BOOST_COMPONENTS} regex) diff --git a/lib/AckGroupingTrackerEnabled.cc b/lib/AckGroupingTrackerEnabled.cc index d90cc874..7233b2c9 100644 --- a/lib/AckGroupingTrackerEnabled.cc +++ b/lib/AckGroupingTrackerEnabled.cc @@ -117,7 +117,7 @@ void AckGroupingTrackerEnabled::close() { this->flush(); std::lock_guard lock(this->mutexTimer_); if (this->timer_) { - boost::system::error_code ec; + ASIO_ERROR ec; this->timer_->cancel(ec); } } @@ -168,9 +168,9 @@ void AckGroupingTrackerEnabled::scheduleTimer() { std::lock_guard lock(this->mutexTimer_); this->timer_ = this->executor_->createDeadlineTimer(); - this->timer_->expires_from_now(boost::posix_time::milliseconds(std::max(1L, this->ackGroupingTimeMs_))); + this->timer_->expires_from_now(std::chrono::milliseconds(std::max(1L, this->ackGroupingTimeMs_))); auto self = shared_from_this(); - this->timer_->async_wait([this, self](const boost::system::error_code& ec) -> void { + this->timer_->async_wait([this, self](const ASIO_ERROR& ec) -> void { if (!ec) { this->flush(); this->scheduleTimer(); diff --git a/lib/AckGroupingTrackerEnabled.h b/lib/AckGroupingTrackerEnabled.h index ec1d66be..b04f4059 100644 --- a/lib/AckGroupingTrackerEnabled.h +++ b/lib/AckGroupingTrackerEnabled.h @@ -22,18 +22,17 @@ #include #include -#include #include #include #include #include "AckGroupingTracker.h" +#include "AsioTimer.h" namespace pulsar { class ClientImpl; using ClientImplPtr = std::shared_ptr; -using DeadlineTimerPtr = std::shared_ptr; class ExecutorService; using ExecutorServicePtr = std::shared_ptr; class HandlerBase; diff --git a/lib/AsioDefines.h b/lib/AsioDefines.h new file mode 100644 index 00000000..2e89812c --- /dev/null +++ b/lib/AsioDefines.h @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +// This header defines common macros to use Asio or Boost.Asio. +#pragma once + +#ifdef USE_ASIO +#define ASIO ::asio +#define ASIO_ERROR asio::error_code +#define ASIO_SUCCESS (ASIO_ERROR{}) +#define ASIO_SYSTEM_ERROR asio::system_error +#else +#define ASIO boost::asio +#define ASIO_ERROR boost::system::error_code +#define ASIO_SUCCESS boost::system::errc::make_error_code(boost::system::errc::success) +#define ASIO_SYSTEM_ERROR boost::system::system_error +#endif diff --git a/lib/TimeUtils.cc b/lib/AsioTimer.h similarity index 71% rename from lib/TimeUtils.cc rename to lib/AsioTimer.h index 7eecb86b..d0c3de58 100644 --- a/lib/TimeUtils.cc +++ b/lib/AsioTimer.h @@ -16,17 +16,16 @@ * specific language governing permissions and limitations * under the License. */ +#pragma once -#include "TimeUtils.h" +#ifdef USE_ASIO +#include +#else +#include +#endif -namespace pulsar { +#include -ptime TimeUtils::now() { return microsec_clock::universal_time(); } +#include "AsioDefines.h" -int64_t TimeUtils::currentTimeMillis() { - static ptime time_t_epoch(boost::gregorian::date(1970, 1, 1)); - - time_duration diff = now() - time_t_epoch; - return diff.total_milliseconds(); -} -} // namespace pulsar \ No newline at end of file +using DeadlineTimerPtr = std::shared_ptr; diff --git a/lib/Backoff.cc b/lib/Backoff.cc index fdb13592..e2c43d1c 100644 --- a/lib/Backoff.cc +++ b/lib/Backoff.cc @@ -21,6 +21,9 @@ #include /* time */ #include +#include + +#include "TimeUtils.h" namespace pulsar { @@ -33,8 +36,8 @@ TimeDuration Backoff::next() { // Check for mandatory stop if (!mandatoryStopMade_) { - const boost::posix_time::ptime& now = boost::posix_time::microsec_clock::universal_time(); - TimeDuration timeElapsedSinceFirstBackoff = boost::posix_time::milliseconds(0); + auto now = TimeUtils::now(); + TimeDuration timeElapsedSinceFirstBackoff = std::chrono::nanoseconds(0); if (initial_ == current) { firstBackoffTime_ = now; } else { diff --git a/lib/Backoff.h b/lib/Backoff.h index a59c00f5..d9d7fae3 100644 --- a/lib/Backoff.h +++ b/lib/Backoff.h @@ -20,12 +20,12 @@ #define _PULSAR_BACKOFF_HEADER_ #include -#include +#include #include -namespace pulsar { +#include "TimeUtils.h" -using TimeDuration = boost::posix_time::time_duration; +namespace pulsar { class PULSAR_PUBLIC Backoff { public: @@ -38,7 +38,7 @@ class PULSAR_PUBLIC Backoff { const TimeDuration max_; TimeDuration next_; TimeDuration mandatoryStop_; - boost::posix_time::ptime firstBackoffTime_; + decltype(std::chrono::high_resolution_clock::now()) firstBackoffTime_; std::mt19937 rng_; bool mandatoryStopMade_ = false; diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index 61aa7f73..82ab4928 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -23,6 +23,7 @@ #include #include +#include "AsioDefines.h" #include "Commands.h" #include "ConnectionPool.h" #include "ConsumerImpl.h" @@ -39,7 +40,7 @@ DECLARE_LOG_OBJECT() -using namespace boost::asio::ip; +using namespace ASIO::ip; namespace pulsar { @@ -162,19 +163,13 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: const ClientConfiguration& clientConfiguration, const AuthenticationPtr& authentication, const std::string& clientVersion, ConnectionPool& pool, size_t poolIndex) - : operationsTimeout_(seconds(clientConfiguration.getOperationTimeoutSeconds())), + : operationsTimeout_(std::chrono::seconds(clientConfiguration.getOperationTimeoutSeconds())), authentication_(authentication), serverProtocolVersion_(proto::ProtocolVersion_MIN), executor_(executor), resolver_(executor_->createTcpResolver()), socket_(executor_->createSocket()), -#if BOOST_VERSION >= 107000 - strand_(boost::asio::make_strand(executor_->getIOService().get_executor())), -#elif BOOST_VERSION >= 106600 - strand_(executor_->getIOService().get_executor()), -#else - strand_(executor_->getIOService()), -#endif + strand_(ASIO::make_strand(executor_->getIOService().get_executor())), logicalAddress_(logicalAddress), physicalAddress_(physicalAddress), cnxString_("[ -> " + physicalAddress + "] "), @@ -203,11 +198,7 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: } if (clientConfiguration.isUseTls()) { -#if BOOST_VERSION >= 105400 - boost::asio::ssl::context ctx(boost::asio::ssl::context::tlsv12_client); -#else - boost::asio::ssl::context ctx(executor_->getIOService(), boost::asio::ssl::context::tlsv1_client); -#endif + ASIO::ssl::context ctx(ASIO::ssl::context::tlsv12_client); Url serviceUrl; Url proxyUrl; Url::parse(physicalAddress, serviceUrl); @@ -219,10 +210,10 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: LOG_INFO("Configuring SNI Proxy-url=" << proxyServiceUrl_); } if (clientConfiguration.isTlsAllowInsecureConnection()) { - ctx.set_verify_mode(boost::asio::ssl::context::verify_none); + ctx.set_verify_mode(ASIO::ssl::context::verify_none); isTlsAllowInsecureConnection_ = true; } else { - ctx.set_verify_mode(boost::asio::ssl::context::verify_peer); + ctx.set_verify_mode(ASIO::ssl::context::verify_peer); std::string trustCertFilePath = clientConfiguration.getTlsTrustCertsFilePath(); if (!trustCertFilePath.empty()) { @@ -252,12 +243,12 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: LOG_ERROR(tlsCertificates << ": No such tlsCertificates"); throw ResultAuthenticationError; } - ctx.use_private_key_file(tlsPrivateKey, boost::asio::ssl::context::pem); - ctx.use_certificate_file(tlsCertificates, boost::asio::ssl::context::pem); + ctx.use_private_key_file(tlsPrivateKey, ASIO::ssl::context::pem); + ctx.use_certificate_file(tlsCertificates, ASIO::ssl::context::pem); } else { if (file_exists(tlsPrivateKey) && file_exists(tlsCertificates)) { - ctx.use_private_key_file(tlsPrivateKey, boost::asio::ssl::context::pem); - ctx.use_certificate_file(tlsCertificates, boost::asio::ssl::context::pem); + ctx.use_private_key_file(tlsPrivateKey, ASIO::ssl::context::pem); + ctx.use_certificate_file(tlsCertificates, ASIO::ssl::context::pem); } } @@ -266,14 +257,13 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: if (!clientConfiguration.isTlsAllowInsecureConnection() && clientConfiguration.isValidateHostName()) { LOG_DEBUG("Validating hostname for " << serviceUrl.host() << ":" << serviceUrl.port()); std::string urlHost = isSniProxy_ ? proxyUrl.host() : serviceUrl.host(); - tlsSocket_->set_verify_callback(boost::asio::ssl::rfc2818_verification(urlHost)); + tlsSocket_->set_verify_callback(ASIO::ssl::rfc2818_verification(urlHost)); } LOG_DEBUG("TLS SNI Host: " << serviceUrl.host()); if (!SSL_set_tlsext_host_name(tlsSocket_->native_handle(), serviceUrl.host().c_str())) { - boost::system::error_code ec{static_cast(::ERR_get_error()), - boost::asio::error::get_ssl_category()}; - LOG_ERROR(boost::system::system_error{ec}.what() << ": Error while setting TLS SNI"); + ASIO_ERROR ec{static_cast(::ERR_get_error()), ASIO::error::get_ssl_category()}; + LOG_ERROR(ec.message() << ": Error while setting TLS SNI"); return; } } @@ -310,9 +300,9 @@ void ClientConnection::handlePulsarConnected(const proto::CommandConnected& cmdC // Only send keep-alive probes if the broker supports it keepAliveTimer_ = executor_->createDeadlineTimer(); if (keepAliveTimer_) { - keepAliveTimer_->expires_from_now(boost::posix_time::seconds(KeepAliveIntervalInSeconds)); + keepAliveTimer_->expires_from_now(std::chrono::seconds(KeepAliveIntervalInSeconds)); auto weakSelf = weak_from_this(); - keepAliveTimer_->async_wait([weakSelf](const boost::system::error_code&) { + keepAliveTimer_->async_wait([weakSelf](const ASIO_ERROR&) { auto self = weakSelf.lock(); if (self) { self->handleKeepAliveTimeout(); @@ -357,13 +347,12 @@ void ClientConnection::startConsumerStatsTimer(std::vector consumerSta if (consumerStatsRequestTimer_) { consumerStatsRequestTimer_->expires_from_now(operationsTimeout_); auto weakSelf = weak_from_this(); - consumerStatsRequestTimer_->async_wait( - [weakSelf, consumerStatsRequests](const boost::system::error_code& err) { - auto self = weakSelf.lock(); - if (self) { - self->handleConsumerStatsTimeout(err, consumerStatsRequests); - } - }); + consumerStatsRequestTimer_->async_wait([weakSelf, consumerStatsRequests](const ASIO_ERROR& err) { + auto self = weakSelf.lock(); + if (self) { + self->handleConsumerStatsTimeout(err, consumerStatsRequests); + } + }); } lock.unlock(); // Complex logic since promises need to be fulfilled outside the lock @@ -375,19 +364,19 @@ void ClientConnection::startConsumerStatsTimer(std::vector consumerSta /// The number of unacknowledged probes to send before considering the connection dead and notifying the /// application layer -typedef boost::asio::detail::socket_option::integer tcp_keep_alive_count; +typedef ASIO::detail::socket_option::integer tcp_keep_alive_count; /// The interval between subsequential keepalive probes, regardless of what the connection has exchanged in /// the meantime -typedef boost::asio::detail::socket_option::integer tcp_keep_alive_interval; +typedef ASIO::detail::socket_option::integer tcp_keep_alive_interval; /// The interval between the last data packet sent (simple ACKs are not considered data) and the first /// keepalive /// probe; after the connection is marked to need keepalive, this counter is not used any further #ifdef __APPLE__ -typedef boost::asio::detail::socket_option::integer tcp_keep_alive_idle; +typedef ASIO::detail::socket_option::integer tcp_keep_alive_idle; #else -typedef boost::asio::detail::socket_option::integer tcp_keep_alive_idle; +typedef ASIO::detail::socket_option::integer tcp_keep_alive_idle; #endif /* @@ -396,15 +385,14 @@ typedef boost::asio::detail::socket_option::integer t * if async_connect without any error, connected_ would be set to true * at this point the connection is deemed valid to be used by clients of this class */ -void ClientConnection::handleTcpConnected(const boost::system::error_code& err, - tcp::resolver::iterator endpointIterator) { +void ClientConnection::handleTcpConnected(const ASIO_ERROR& err, tcp::resolver::iterator endpointIterator) { if (!err) { std::stringstream cnxStringStream; try { cnxStringStream << "[" << socket_->local_endpoint() << " -> " << socket_->remote_endpoint() << "] "; cnxString_ = cnxStringStream.str(); - } catch (const boost::system::system_error& e) { + } catch (const ASIO_SYSTEM_ERROR& e) { LOG_ERROR("Failed to get endpoint: " << e.what()); close(ResultRetryable); return; @@ -424,7 +412,7 @@ void ClientConnection::handleTcpConnected(const boost::system::error_code& err, state_ = TcpConnected; lock.unlock(); - boost::system::error_code error; + ASIO_ERROR error; socket_->set_option(tcp::no_delay(true), error); if (error) { LOG_WARN(cnxString_ << "Socket failed to set tcp::no_delay: " << error.message()); @@ -457,7 +445,7 @@ void ClientConnection::handleTcpConnected(const boost::system::error_code& err, if (tlsSocket_) { if (!isTlsAllowInsecureConnection_) { - boost::system::error_code err; + ASIO_ERROR err; Url service_url; if (!Url::parse(physicalAddress_, service_url)) { LOG_ERROR(cnxString_ << "Invalid Url, unable to parse: " << err << " " << err.message()); @@ -466,26 +454,21 @@ void ClientConnection::handleTcpConnected(const boost::system::error_code& err, } } auto weakSelf = weak_from_this(); - auto callback = [weakSelf](const boost::system::error_code& err) { + auto callback = [weakSelf](const ASIO_ERROR& err) { auto self = weakSelf.lock(); if (self) { self->handleHandshake(err); } }; -#if BOOST_VERSION >= 106600 - tlsSocket_->async_handshake(boost::asio::ssl::stream::client, - boost::asio::bind_executor(strand_, callback)); -#else - tlsSocket_->async_handshake(boost::asio::ssl::stream::client, - strand_.wrap(callback)); -#endif + tlsSocket_->async_handshake(ASIO::ssl::stream::client, + ASIO::bind_executor(strand_, callback)); } else { - handleHandshake(boost::system::errc::make_error_code(boost::system::errc::success)); + handleHandshake(ASIO_SUCCESS); } } else if (endpointIterator != tcp::resolver::iterator()) { LOG_WARN(cnxString_ << "Failed to establish connection: " << err.message()); // The connection failed. Try the next endpoint in the list. - boost::system::error_code closeError; + ASIO_ERROR closeError; socket_->close(closeError); // ignore the error of close if (closeError) { LOG_WARN(cnxString_ << "Failed to close socket: " << err.message()); @@ -497,15 +480,14 @@ void ClientConnection::handleTcpConnected(const boost::system::error_code& err, connectTimeoutTask_->start(); tcp::endpoint endpoint = *endpointIterator; auto weakSelf = weak_from_this(); - socket_->async_connect(endpoint, - [weakSelf, endpointIterator](const boost::system::error_code& err) { - auto self = weakSelf.lock(); - if (self) { - self->handleTcpConnected(err, endpointIterator); - } - }); + socket_->async_connect(endpoint, [weakSelf, endpointIterator](const ASIO_ERROR& err) { + auto self = weakSelf.lock(); + if (self) { + self->handleTcpConnected(err, endpointIterator); + } + }); } else { - if (err == boost::asio::error::operation_aborted) { + if (err == ASIO::error::operation_aborted) { // TCP connect timeout, which is not retryable close(); } else { @@ -518,7 +500,7 @@ void ClientConnection::handleTcpConnected(const boost::system::error_code& err, } } -void ClientConnection::handleHandshake(const boost::system::error_code& err) { +void ClientConnection::handleHandshake(const ASIO_ERROR& err) { if (err) { LOG_ERROR(cnxString_ << "Handshake failed: " << err.message()); close(); @@ -537,13 +519,12 @@ void ClientConnection::handleHandshake(const boost::system::error_code& err) { // Send CONNECT command to broker auto self = shared_from_this(); asyncWrite(buffer.const_asio_buffer(), - customAllocWriteHandler([this, self, buffer](const boost::system::error_code& err, size_t) { + customAllocWriteHandler([this, self, buffer](const ASIO_ERROR& err, size_t) { handleSentPulsarConnect(err, buffer); })); } -void ClientConnection::handleSentPulsarConnect(const boost::system::error_code& err, - const SharedBuffer& buffer) { +void ClientConnection::handleSentPulsarConnect(const ASIO_ERROR& err, const SharedBuffer& buffer) { if (isClosed()) { return; } @@ -557,8 +538,7 @@ void ClientConnection::handleSentPulsarConnect(const boost::system::error_code& readNextCommand(); } -void ClientConnection::handleSentAuthResponse(const boost::system::error_code& err, - const SharedBuffer& buffer) { +void ClientConnection::handleSentAuthResponse(const ASIO_ERROR& err, const SharedBuffer& buffer) { if (isClosed()) { return; } @@ -580,7 +560,7 @@ void ClientConnection::tcpConnectAsync() { return; } - boost::system::error_code err; + ASIO_ERROR err; Url service_url; std::string hostUrl = isSniProxy_ ? proxyServiceUrl_ : physicalAddress_; if (!Url::parse(hostUrl, service_url)) { @@ -599,17 +579,15 @@ void ClientConnection::tcpConnectAsync() { LOG_DEBUG(cnxString_ << "Resolving " << service_url.host() << ":" << service_url.port()); tcp::resolver::query query(service_url.host(), std::to_string(service_url.port())); auto weakSelf = weak_from_this(); - resolver_->async_resolve( - query, [weakSelf](const boost::system::error_code& err, tcp::resolver::iterator iterator) { - auto self = weakSelf.lock(); - if (self) { - self->handleResolve(err, iterator); - } - }); + resolver_->async_resolve(query, [weakSelf](const ASIO_ERROR& err, tcp::resolver::iterator iterator) { + auto self = weakSelf.lock(); + if (self) { + self->handleResolve(err, iterator); + } + }); } -void ClientConnection::handleResolve(const boost::system::error_code& err, - tcp::resolver::iterator endpointIterator) { +void ClientConnection::handleResolve(const ASIO_ERROR& err, tcp::resolver::iterator endpointIterator) { if (err) { std::string hostUrl = isSniProxy_ ? cnxString_ : proxyServiceUrl_; LOG_ERROR(hostUrl << "Resolve error: " << err << " : " << err.message()); @@ -642,13 +620,12 @@ void ClientConnection::handleResolve(const boost::system::error_code& err, if (endpointIterator != tcp::resolver::iterator()) { LOG_DEBUG(cnxString_ << "Resolved hostname " << endpointIterator->host_name() // << " to " << endpointIterator->endpoint()); - socket_->async_connect(*endpointIterator, - [weakSelf, endpointIterator](const boost::system::error_code& err) { - auto self = weakSelf.lock(); - if (self) { - self->handleTcpConnected(err, endpointIterator); - } - }); + socket_->async_connect(*endpointIterator, [weakSelf, endpointIterator](const ASIO_ERROR& err) { + auto self = weakSelf.lock(); + if (self) { + self->handleTcpConnected(err, endpointIterator); + } + }); } else { LOG_WARN(cnxString_ << "No IP address found"); close(); @@ -659,15 +636,13 @@ void ClientConnection::handleResolve(const boost::system::error_code& err, void ClientConnection::readNextCommand() { const static uint32_t minReadSize = sizeof(uint32_t); auto self = shared_from_this(); - asyncReceive( - incomingBuffer_.asio_buffer(), - customAllocReadHandler([this, self](const boost::system::error_code& err, size_t bytesTransferred) { - handleRead(err, bytesTransferred, minReadSize); - })); + asyncReceive(incomingBuffer_.asio_buffer(), + customAllocReadHandler([this, self](const ASIO_ERROR& err, size_t bytesTransferred) { + handleRead(err, bytesTransferred, minReadSize); + })); } -void ClientConnection::handleRead(const boost::system::error_code& err, size_t bytesTransferred, - uint32_t minReadSize) { +void ClientConnection::handleRead(const ASIO_ERROR& err, size_t bytesTransferred, uint32_t minReadSize) { if (isClosed()) { return; } @@ -675,9 +650,9 @@ void ClientConnection::handleRead(const boost::system::error_code& err, size_t b incomingBuffer_.bytesWritten(bytesTransferred); if (err || bytesTransferred == 0) { - if (err == boost::asio::error::operation_aborted) { + if (err == ASIO::error::operation_aborted) { LOG_DEBUG(cnxString_ << "Read operation was canceled: " << err.message()); - } else if (bytesTransferred == 0 || err == boost::asio::error::eof) { + } else if (bytesTransferred == 0 || err == ASIO::error::eof) { LOG_DEBUG(cnxString_ << "Server closed the connection: " << err.message()); } else { LOG_ERROR(cnxString_ << "Read operation failed: " << err.message()); @@ -689,11 +664,11 @@ void ClientConnection::handleRead(const boost::system::error_code& err, size_t b SharedBuffer buffer = incomingBuffer_.slice(bytesTransferred); auto self = shared_from_this(); auto nextMinReadSize = minReadSize - bytesTransferred; - asyncReceive(buffer.asio_buffer(), customAllocReadHandler([this, self, nextMinReadSize]( - const boost::system::error_code& err, - size_t bytesTransferred) { - handleRead(err, bytesTransferred, nextMinReadSize); - })); + asyncReceive(buffer.asio_buffer(), + customAllocReadHandler( + [this, self, nextMinReadSize](const ASIO_ERROR& err, size_t bytesTransferred) { + handleRead(err, bytesTransferred, nextMinReadSize); + })); } else { processIncomingBuffer(); } @@ -720,12 +695,11 @@ void ClientConnection::processIncomingBuffer() { incomingBuffer_ = SharedBuffer::copyFrom(incomingBuffer_, newBufferSize); } auto self = shared_from_this(); - asyncReceive( - incomingBuffer_.asio_buffer(), - customAllocReadHandler([this, self, bytesToReceive](const boost::system::error_code& err, - size_t bytesTransferred) { - handleRead(err, bytesTransferred, bytesToReceive); - })); + asyncReceive(incomingBuffer_.asio_buffer(), + customAllocReadHandler( + [this, self, bytesToReceive](const ASIO_ERROR& err, size_t bytesTransferred) { + handleRead(err, bytesTransferred, bytesToReceive); + })); return; } @@ -803,11 +777,11 @@ void ClientConnection::processIncomingBuffer() { uint32_t minReadSize = sizeof(uint32_t) - incomingBuffer_.readableBytes(); auto self = shared_from_this(); - asyncReceive(incomingBuffer_.asio_buffer(), - customAllocReadHandler([this, self, minReadSize](const boost::system::error_code& err, - size_t bytesTransferred) { - handleRead(err, bytesTransferred, minReadSize); - })); + asyncReceive( + incomingBuffer_.asio_buffer(), + customAllocReadHandler([this, self, minReadSize](const ASIO_ERROR& err, size_t bytesTransferred) { + handleRead(err, bytesTransferred, minReadSize); + })); return; } @@ -1056,7 +1030,7 @@ void ClientConnection::newLookup(const SharedBuffer& cmd, const uint64_t request requestData.timer = executor_->createDeadlineTimer(); requestData.timer->expires_from_now(operationsTimeout_); auto weakSelf = weak_from_this(); - requestData.timer->async_wait([weakSelf, requestData](const boost::system::error_code& ec) { + requestData.timer->async_wait([weakSelf, requestData](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (self) { self->handleLookupTimeout(ec, requestData); @@ -1082,11 +1056,7 @@ void ClientConnection::sendCommand(const SharedBuffer& cmd) { self->sendCommandInternal(cmd); } }; -#if BOOST_VERSION >= 106600 - boost::asio::post(strand_, callback); -#else - strand_.post(callback); -#endif + ASIO::post(strand_, callback); } else { sendCommandInternal(cmd); } @@ -1099,8 +1069,9 @@ void ClientConnection::sendCommand(const SharedBuffer& cmd) { void ClientConnection::sendCommandInternal(const SharedBuffer& cmd) { auto self = shared_from_this(); asyncWrite(cmd.const_asio_buffer(), - customAllocWriteHandler([this, self, cmd](const boost::system::error_code& err, - size_t bytesTransferred) { handleSend(err, cmd); })); + customAllocWriteHandler([this, self, cmd](const ASIO_ERROR& err, size_t bytesTransferred) { + handleSend(err, cmd); + })); } void ClientConnection::sendMessage(const std::shared_ptr& args) { @@ -1116,21 +1087,18 @@ void ClientConnection::sendMessage(const std::shared_ptr& args) { // Capture the buffer because asio does not copy the buffer, if the buffer is destroyed before the // callback is called, an invalid buffer range might be passed to the underlying socket send. asyncWrite(buffer, customAllocWriteHandler( - [this, self, buffer](const boost::system::error_code& err, - size_t bytesTransferred) { handleSendPair(err); })); + [this, self, buffer](const ASIO_ERROR& err, size_t bytesTransferred) { + handleSendPair(err); + })); }; if (tlsSocket_) { -#if BOOST_VERSION >= 106600 - boost::asio::post(strand_, sendMessageInternal); -#else - strand_.post(sendMessageInternal); -#endif + ASIO::post(strand_, sendMessageInternal); } else { sendMessageInternal(); } } -void ClientConnection::handleSend(const boost::system::error_code& err, const SharedBuffer&) { +void ClientConnection::handleSend(const ASIO_ERROR& err, const SharedBuffer&) { if (isClosed()) { return; } @@ -1142,7 +1110,7 @@ void ClientConnection::handleSend(const boost::system::error_code& err, const Sh } } -void ClientConnection::handleSendPair(const boost::system::error_code& err) { +void ClientConnection::handleSendPair(const ASIO_ERROR& err) { if (isClosed()) { return; } @@ -1166,8 +1134,8 @@ void ClientConnection::sendPendingCommands() { if (any.type() == typeid(SharedBuffer)) { SharedBuffer buffer = boost::any_cast(any); asyncWrite(buffer.const_asio_buffer(), - customAllocWriteHandler([this, self, buffer](const boost::system::error_code& err, - size_t) { handleSend(err, buffer); })); + customAllocWriteHandler( + [this, self, buffer](const ASIO_ERROR& err, size_t) { handleSend(err, buffer); })); } else { assert(any.type() == typeid(std::shared_ptr)); @@ -1178,9 +1146,9 @@ void ClientConnection::sendPendingCommands() { // Capture the buffer because asio does not copy the buffer, if the buffer is destroyed before the // callback is called, an invalid buffer range might be passed to the underlying socket send. - asyncWrite(buffer, - customAllocWriteHandler([this, self, buffer](const boost::system::error_code& err, - size_t) { handleSendPair(err); })); + asyncWrite(buffer, customAllocWriteHandler([this, self, buffer](const ASIO_ERROR& err, size_t) { + handleSendPair(err); + })); } } else { // No more pending writes @@ -1202,7 +1170,7 @@ Future ClientConnection::sendRequestWithId(SharedBuffer cm requestData.timer = executor_->createDeadlineTimer(); requestData.timer->expires_from_now(operationsTimeout_); auto weakSelf = weak_from_this(); - requestData.timer->async_wait([weakSelf, requestData](const boost::system::error_code& ec) { + requestData.timer->async_wait([weakSelf, requestData](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (self) { self->handleRequestTimeout(ec, requestData); @@ -1216,21 +1184,19 @@ Future ClientConnection::sendRequestWithId(SharedBuffer cm return requestData.promise.getFuture(); } -void ClientConnection::handleRequestTimeout(const boost::system::error_code& ec, - PendingRequestData pendingRequestData) { +void ClientConnection::handleRequestTimeout(const ASIO_ERROR& ec, PendingRequestData pendingRequestData) { if (!ec && !pendingRequestData.hasGotResponse->load()) { pendingRequestData.promise.setFailed(ResultTimeout); } } -void ClientConnection::handleLookupTimeout(const boost::system::error_code& ec, - LookupRequestData pendingRequestData) { +void ClientConnection::handleLookupTimeout(const ASIO_ERROR& ec, LookupRequestData pendingRequestData) { if (!ec) { pendingRequestData.promise->setFailed(ResultTimeout); } } -void ClientConnection::handleGetLastMessageIdTimeout(const boost::system::error_code& ec, +void ClientConnection::handleGetLastMessageIdTimeout(const ASIO_ERROR& ec, ClientConnection::LastMessageIdRequestData data) { if (!ec) { data.promise->setFailed(ResultTimeout); @@ -1255,9 +1221,9 @@ void ClientConnection::handleKeepAliveTimeout() { // be zero And we do not attempt to dereference the pointer. Lock lock(mutex_); if (keepAliveTimer_) { - keepAliveTimer_->expires_from_now(boost::posix_time::seconds(KeepAliveIntervalInSeconds)); + keepAliveTimer_->expires_from_now(std::chrono::seconds(KeepAliveIntervalInSeconds)); auto weakSelf = weak_from_this(); - keepAliveTimer_->async_wait([weakSelf](const boost::system::error_code&) { + keepAliveTimer_->async_wait([weakSelf](const ASIO_ERROR&) { auto self = weakSelf.lock(); if (self) { self->handleKeepAliveTimeout(); @@ -1268,7 +1234,7 @@ void ClientConnection::handleKeepAliveTimeout() { } } -void ClientConnection::handleConsumerStatsTimeout(const boost::system::error_code& ec, +void ClientConnection::handleConsumerStatsTimeout(const ASIO_ERROR& ec, std::vector consumerStatsRequests) { if (ec) { LOG_DEBUG(cnxString_ << " Ignoring timer cancelled event, code[" << ec << "]"); @@ -1285,15 +1251,15 @@ void ClientConnection::close(Result result, bool detach) { state_ = Disconnected; if (socket_) { - boost::system::error_code err; - socket_->shutdown(boost::asio::socket_base::shutdown_both, err); + ASIO_ERROR err; + socket_->shutdown(ASIO::socket_base::shutdown_both, err); socket_->close(err); if (err) { LOG_WARN(cnxString_ << "Failed to close socket: " << err.message()); } } if (tlsSocket_) { - boost::system::error_code err; + ASIO_ERROR err; tlsSocket_->lowest_layer().close(err); if (err) { LOG_WARN(cnxString_ << "Failed to close TLS socket: " << err.message()); @@ -1432,7 +1398,7 @@ Future ClientConnection::newGetLastMessageId(u requestData.timer = executor_->createDeadlineTimer(); requestData.timer->expires_from_now(operationsTimeout_); auto weakSelf = weak_from_this(); - requestData.timer->async_wait([weakSelf, requestData](const boost::system::error_code& ec) { + requestData.timer->async_wait([weakSelf, requestData](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (self) { self->handleGetLastMessageIdTimeout(ec, requestData); @@ -1823,7 +1789,7 @@ void ClientConnection::handleAuthChallenge() { } auto self = shared_from_this(); asyncWrite(buffer.const_asio_buffer(), - customAllocWriteHandler([this, self, buffer](const boost::system::error_code& err, size_t) { + customAllocWriteHandler([this, self, buffer](const ASIO_ERROR& err, size_t) { handleSentAuthResponse(err, buffer); })); } diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index 1bc1bd85..69155fdd 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -23,13 +23,20 @@ #include #include -#include +#ifdef USE_ASIO +#include +#include +#include +#include +#include +#else #include -#include #include #include #include #include +#endif +#include #include #include #include @@ -37,19 +44,18 @@ #include #include +#include "AsioTimer.h" #include "Commands.h" #include "GetLastMessageIdResponse.h" #include "LookupDataResult.h" #include "SharedBuffer.h" +#include "TimeUtils.h" #include "UtilAllocator.h" - namespace pulsar { class PulsarFriend; -using DeadlineTimerPtr = std::shared_ptr; -using TimeDuration = boost::posix_time::time_duration; -using TcpResolverPtr = std::shared_ptr; +using TcpResolverPtr = std::shared_ptr; class ExecutorService; using ExecutorServicePtr = std::shared_ptr; @@ -114,10 +120,10 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this SocketPtr; - typedef std::shared_ptr> TlsSocketPtr; + typedef std::shared_ptr SocketPtr; + typedef std::shared_ptr> TlsSocketPtr; typedef std::shared_ptr ConnectionPtr; - typedef std::function ConnectionListener; + typedef std::function ConnectionListener; typedef std::vector::iterator ListenerIterator; /* @@ -224,17 +230,16 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this= 106600 - boost::asio::async_write(*tlsSocket_, buffers, boost::asio::bind_executor(strand_, handler)); -#else - boost::asio::async_write(*tlsSocket_, buffers, strand_.wrap(handler)); -#endif + ASIO::async_write(*tlsSocket_, buffers, ASIO::bind_executor(strand_, handler)); } else { - boost::asio::async_write(*socket_, buffers, handler); + ASIO::async_write(*socket_, buffers, handler); } } @@ -296,11 +296,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this= 106600 - tlsSocket_->async_read_some(buffers, boost::asio::bind_executor(strand_, handler)); -#else - tlsSocket_->async_read_some(buffers, strand_.wrap(handler)); -#endif + tlsSocket_->async_read_some(buffers, ASIO::bind_executor(strand_, handler)); } else { socket_->async_receive(buffers, handler); } @@ -321,11 +317,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this= 106600 - boost::asio::strand strand_; -#else - boost::asio::io_service::strand strand_; -#endif + ASIO::strand strand_; const std::string logicalAddress_; /* @@ -343,7 +335,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this consumerStatsRequests); + void handleConsumerStatsTimeout(const ASIO_ERROR& ec, std::vector consumerStatsRequests); void startConsumerStatsTimer(std::vector consumerStatsRequests); uint32_t maxPendingLookupRequest_; diff --git a/lib/CompressionCodec.cc b/lib/CompressionCodec.cc index 991d52c0..6105887a 100644 --- a/lib/CompressionCodec.cc +++ b/lib/CompressionCodec.cc @@ -45,7 +45,6 @@ CompressionCodec& CompressionCodecProvider::getCodec(CompressionType compression default: return compressionCodecNone_; } - BOOST_THROW_EXCEPTION(std::logic_error("Invalid CompressionType enumeration value")); } SharedBuffer CompressionCodecNone::encode(const SharedBuffer& raw) { return raw; } diff --git a/lib/ConnectionPool.cc b/lib/ConnectionPool.cc index 95170a94..4cc8883a 100644 --- a/lib/ConnectionPool.cc +++ b/lib/ConnectionPool.cc @@ -18,15 +18,20 @@ */ #include "ConnectionPool.h" +#ifdef USE_ASIO +#include +#include +#else #include #include +#endif #include "ClientConnection.h" #include "ExecutorService.h" #include "LogUtils.h" -using boost::asio::ip::tcp; -namespace ssl = boost::asio::ssl; +using ASIO::ip::tcp; +namespace ssl = ASIO::ssl; typedef ssl::stream ssl_socket; DECLARE_LOG_OBJECT() diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index f9770d80..52162189 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -26,6 +26,7 @@ #include "AckGroupingTracker.h" #include "AckGroupingTrackerDisabled.h" #include "AckGroupingTrackerEnabled.h" +#include "AsioDefines.h" #include "BatchMessageAcker.h" #include "BatchedMessageIdImpl.h" #include "BitSet.h" @@ -55,6 +56,9 @@ namespace pulsar { DECLARE_LOG_OBJECT() +using std::chrono::milliseconds; +using std::chrono::seconds; + ConsumerImpl::ConsumerImpl(const ClientImplPtr client, const std::string& topic, const std::string& subscriptionName, const ConsumerConfiguration& conf, bool isPersistent, const ConsumerInterceptorsPtr& interceptors, @@ -402,10 +406,9 @@ void ConsumerImpl::discardChunkMessages(std::string uuid, MessageId messageId, b } void ConsumerImpl::triggerCheckExpiredChunkedTimer() { - checkExpiredChunkedTimer_->expires_from_now( - boost::posix_time::milliseconds(expireTimeOfIncompleteChunkedMessageMs_)); + checkExpiredChunkedTimer_->expires_from_now(milliseconds(expireTimeOfIncompleteChunkedMessageMs_)); std::weak_ptr weakSelf{shared_from_this()}; - checkExpiredChunkedTimer_->async_wait([this, weakSelf](const boost::system::error_code& ec) -> void { + checkExpiredChunkedTimer_->async_wait([this, weakSelf](const ASIO_ERROR& ec) -> void { auto self = weakSelf.lock(); if (!self) { return; @@ -1581,7 +1584,7 @@ void ConsumerImpl::internalGetLastMessageIdAsync(const BackoffPtr& backoff, Time } } else { TimeDuration next = std::min(remainTime, backoff->next()); - if (next.total_milliseconds() <= 0) { + if (toMillis(next) <= 0) { LOG_ERROR(getName() << " Client Connection not ready for Consumer"); callback(ResultNotConnected, MessageId()); return; @@ -1592,8 +1595,8 @@ void ConsumerImpl::internalGetLastMessageIdAsync(const BackoffPtr& backoff, Time auto self = shared_from_this(); timer->async_wait([this, backoff, remainTime, timer, next, callback, - self](const boost::system::error_code& ec) -> void { - if (ec == boost::asio::error::operation_aborted) { + self](const ASIO_ERROR& ec) -> void { + if (ec == ASIO::error::operation_aborted) { LOG_DEBUG(getName() << " Get last message id operation was cancelled, code[" << ec << "]."); return; } @@ -1602,7 +1605,7 @@ void ConsumerImpl::internalGetLastMessageIdAsync(const BackoffPtr& backoff, Time return; } LOG_WARN(getName() << " Could not get connection while getLastMessageId -- Will try again in " - << next.total_milliseconds() << " ms") + << toMillis(next) << " ms") this->internalGetLastMessageIdAsync(backoff, remainTime, timer, callback); }); } @@ -1693,7 +1696,7 @@ std::shared_ptr ConsumerImpl::get_shared_this_ptr() { } void ConsumerImpl::cancelTimers() noexcept { - boost::system::error_code ec; + ASIO_ERROR ec; batchReceiveTimer_->cancel(ec); checkExpiredChunkedTimer_->cancel(ec); unAckedMessageTrackerPtr_->stop(); diff --git a/lib/ConsumerImplBase.cc b/lib/ConsumerImplBase.cc index 39212371..851d41e8 100644 --- a/lib/ConsumerImplBase.cc +++ b/lib/ConsumerImplBase.cc @@ -51,9 +51,9 @@ ConsumerImplBase::ConsumerImplBase(ClientImplPtr client, const std::string& topi void ConsumerImplBase::triggerBatchReceiveTimerTask(long timeoutMs) { if (timeoutMs > 0) { - batchReceiveTimer_->expires_from_now(boost::posix_time::milliseconds(timeoutMs)); + batchReceiveTimer_->expires_from_now(std::chrono::milliseconds(timeoutMs)); std::weak_ptr weakSelf{shared_from_this()}; - batchReceiveTimer_->async_wait([weakSelf](const boost::system::error_code& ec) { + batchReceiveTimer_->async_wait([weakSelf](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (self && !ec) { self->doBatchReceiveTimeTask(); diff --git a/lib/ExecutorService.cc b/lib/ExecutorService.cc index 53be8eaf..794e3619 100644 --- a/lib/ExecutorService.cc +++ b/lib/ExecutorService.cc @@ -32,7 +32,7 @@ void ExecutorService::start() { auto self = shared_from_this(); std::thread t{[this, self] { LOG_DEBUG("Run io_service in a single thread"); - boost::system::error_code ec; + ASIO_ERROR ec; while (!closed_) { io_service_.restart(); IOService::work work{getIOService()}; @@ -63,22 +63,22 @@ ExecutorServicePtr ExecutorService::create() { } /* - * factory method of boost::asio::ip::tcp::socket associated with io_service_ instance + * factory method of ASIO::ip::tcp::socket associated with io_service_ instance * @ returns shared_ptr to this socket */ SocketPtr ExecutorService::createSocket() { try { - return SocketPtr(new boost::asio::ip::tcp::socket(io_service_)); - } catch (const boost::system::system_error &e) { + return SocketPtr(new ASIO::ip::tcp::socket(io_service_)); + } catch (const ASIO_SYSTEM_ERROR &e) { restart(); auto error = std::string("Failed to create socket: ") + e.what(); throw std::runtime_error(error); } } -TlsSocketPtr ExecutorService::createTlsSocket(SocketPtr &socket, boost::asio::ssl::context &ctx) { - return std::shared_ptr>( - new boost::asio::ssl::stream(*socket, ctx)); +TlsSocketPtr ExecutorService::createTlsSocket(SocketPtr &socket, ASIO::ssl::context &ctx) { + return std::shared_ptr>( + new ASIO::ssl::stream(*socket, ctx)); } /* @@ -87,8 +87,8 @@ TlsSocketPtr ExecutorService::createTlsSocket(SocketPtr &socket, boost::asio::ss */ TcpResolverPtr ExecutorService::createTcpResolver() { try { - return TcpResolverPtr(new boost::asio::ip::tcp::resolver(io_service_)); - } catch (const boost::system::system_error &e) { + return TcpResolverPtr(new ASIO::ip::tcp::resolver(io_service_)); + } catch (const ASIO_SYSTEM_ERROR &e) { restart(); auto error = std::string("Failed to create resolver: ") + e.what(); throw std::runtime_error(error); @@ -97,10 +97,10 @@ TcpResolverPtr ExecutorService::createTcpResolver() { DeadlineTimerPtr ExecutorService::createDeadlineTimer() { try { - return DeadlineTimerPtr(new boost::asio::deadline_timer(io_service_)); - } catch (const boost::system::system_error &e) { + return DeadlineTimerPtr(new ASIO::steady_timer(io_service_)); + } catch (const ASIO_SYSTEM_ERROR &e) { restart(); - auto error = std::string("Failed to create deadline_timer: ") + e.what(); + auto error = std::string("Failed to create steady_timer: ") + e.what(); throw std::runtime_error(error); } } diff --git a/lib/ExecutorService.h b/lib/ExecutorService.h index a373c0af..89d06d30 100644 --- a/lib/ExecutorService.h +++ b/lib/ExecutorService.h @@ -22,10 +22,15 @@ #include #include -#include +#ifdef USE_ASIO +#include +#include +#include +#else #include #include #include +#endif #include #include #include @@ -33,14 +38,15 @@ #include #include +#include "AsioTimer.h" + namespace pulsar { -typedef std::shared_ptr SocketPtr; -typedef std::shared_ptr > TlsSocketPtr; -typedef std::shared_ptr TcpResolverPtr; -typedef std::shared_ptr DeadlineTimerPtr; +typedef std::shared_ptr SocketPtr; +typedef std::shared_ptr > TlsSocketPtr; +typedef std::shared_ptr TcpResolverPtr; class PULSAR_PUBLIC ExecutorService : public std::enable_shared_from_this { public: - using IOService = boost::asio::io_service; + using IOService = ASIO::io_service; using SharedPtr = std::shared_ptr; static SharedPtr create(); @@ -51,7 +57,7 @@ class PULSAR_PUBLIC ExecutorService : public std::enable_shared_from_thisgetIOExecutorProvider()->get()), mutex_(), creationTimestamp_(TimeUtils::now()), - operationTimeut_(seconds(client->conf().getOperationTimeoutSeconds())), + operationTimeut_(std::chrono::seconds(client->conf().getOperationTimeoutSeconds())), state_(NotStarted), backoff_(backoff), epoch_(0), @@ -147,13 +148,13 @@ void HandlerBase::scheduleReconnection() { if (state == Pending || state == Ready) { TimeDuration delay = backoff_.next(); - LOG_INFO(getName() << "Schedule reconnection in " << (delay.total_milliseconds() / 1000.0) << " s"); + LOG_INFO(getName() << "Schedule reconnection in " << (toMillis(delay) / 1000.0) << " s"); timer_->expires_from_now(delay); // passing shared_ptr here since time_ will get destroyed, so tasks will be cancelled // so we will not run into the case where grabCnx is invoked on out of scope handler auto name = getName(); std::weak_ptr weakSelf{shared_from_this()}; - timer_->async_wait([name, weakSelf](const boost::system::error_code& ec) { + timer_->async_wait([name, weakSelf](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (self) { self->handleTimeout(ec); @@ -164,7 +165,7 @@ void HandlerBase::scheduleReconnection() { } } -void HandlerBase::handleTimeout(const boost::system::error_code& ec) { +void HandlerBase::handleTimeout(const ASIO_ERROR& ec) { if (ec) { LOG_DEBUG(getName() << "Ignoring timer cancelled event, code[" << ec << "]"); return; diff --git a/lib/HandlerBase.h b/lib/HandlerBase.h index f62c4df0..68c0b6a6 100644 --- a/lib/HandlerBase.h +++ b/lib/HandlerBase.h @@ -20,20 +20,17 @@ #define _PULSAR_HANDLER_BASE_HEADER_ #include -#include #include #include #include +#include "AsioTimer.h" #include "Backoff.h" #include "Future.h" +#include "TimeUtils.h" namespace pulsar { -using namespace boost::posix_time; -using boost::posix_time::milliseconds; -using boost::posix_time::seconds; - class ClientImpl; using ClientImplPtr = std::shared_ptr; using ClientImplWeakPtr = std::weak_ptr; @@ -42,7 +39,6 @@ using ClientConnectionPtr = std::shared_ptr; using ClientConnectionWeakPtr = std::weak_ptr; class ExecutorService; using ExecutorServicePtr = std::shared_ptr; -using DeadlineTimerPtr = std::shared_ptr; class HandlerBase : public std::enable_shared_from_this { public: @@ -95,7 +91,7 @@ class HandlerBase : public std::enable_shared_from_this { void handleDisconnection(Result result, const ClientConnectionPtr& cnx); - void handleTimeout(const boost::system::error_code& ec); + void handleTimeout(const ASIO_ERROR& ec); protected: ClientImplWeakPtr client_; diff --git a/lib/Int64SerDes.h b/lib/Int64SerDes.h index dbc5d8a7..f1f5eef3 100644 --- a/lib/Int64SerDes.h +++ b/lib/Int64SerDes.h @@ -20,7 +20,12 @@ #include -#include // for ntohl +// for ntohl +#ifdef USE_ASIO +#include +#else +#include +#endif namespace pulsar { diff --git a/lib/MultiTopicsConsumerImpl.cc b/lib/MultiTopicsConsumerImpl.cc index 15f9d9b8..af70623d 100644 --- a/lib/MultiTopicsConsumerImpl.cc +++ b/lib/MultiTopicsConsumerImpl.cc @@ -18,6 +18,7 @@ */ #include "MultiTopicsConsumerImpl.h" +#include #include #include "ClientImpl.h" @@ -37,6 +38,9 @@ DECLARE_LOG_OBJECT() using namespace pulsar; +using std::chrono::milliseconds; +using std::chrono::seconds; + MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, TopicNamePtr topicName, int numPartitions, const std::string& subscriptionName, const ConsumerConfiguration& conf, @@ -90,7 +94,7 @@ MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, const std auto partitionsUpdateInterval = static_cast(client->conf().getPartitionsUpdateInterval()); if (partitionsUpdateInterval > 0) { partitionsUpdateTimer_ = listenerExecutor_->createDeadlineTimer(); - partitionsUpdateInterval_ = boost::posix_time::seconds(partitionsUpdateInterval); + partitionsUpdateInterval_ = seconds(partitionsUpdateInterval); lookupServicePtr_ = client->getLookup(); } @@ -936,7 +940,7 @@ uint64_t MultiTopicsConsumerImpl::getNumberOfConnectedConsumer() { void MultiTopicsConsumerImpl::runPartitionUpdateTask() { partitionsUpdateTimer_->expires_from_now(partitionsUpdateInterval_); auto weakSelf = weak_from_this(); - partitionsUpdateTimer_->async_wait([weakSelf](const boost::system::error_code& ec) { + partitionsUpdateTimer_->async_wait([weakSelf](const ASIO_ERROR& ec) { // If two requests call runPartitionUpdateTask at the same time, the timer will fail, and it // cannot continue at this time, and the request needs to be ignored. auto self = weakSelf.lock(); @@ -1087,7 +1091,7 @@ void MultiTopicsConsumerImpl::beforeConnectionChange(ClientConnection& cnx) { void MultiTopicsConsumerImpl::cancelTimers() noexcept { if (partitionsUpdateTimer_) { - boost::system::error_code ec; + ASIO_ERROR ec; partitionsUpdateTimer_->cancel(ec); } } diff --git a/lib/MultiTopicsConsumerImpl.h b/lib/MultiTopicsConsumerImpl.h index d4127f63..c5834eaa 100644 --- a/lib/MultiTopicsConsumerImpl.h +++ b/lib/MultiTopicsConsumerImpl.h @@ -32,6 +32,7 @@ #include "LookupDataResult.h" #include "SynchronizedHashMap.h" #include "TestUtil.h" +#include "TimeUtils.h" #include "UnboundedBlockingQueue.h" namespace pulsar { @@ -119,7 +120,7 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase { std::atomic_int incomingMessagesSize_ = {0}; MessageListener messageListener_; DeadlineTimerPtr partitionsUpdateTimer_; - boost::posix_time::time_duration partitionsUpdateInterval_; + TimeDuration partitionsUpdateInterval_; LookupServicePtr lookupServicePtr_; std::shared_ptr> numberTopicPartitions_; std::atomic failedResult{ResultOk}; diff --git a/lib/NegativeAcksTracker.cc b/lib/NegativeAcksTracker.cc index 0dd73589..e443496d 100644 --- a/lib/NegativeAcksTracker.cc +++ b/lib/NegativeAcksTracker.cc @@ -40,9 +40,9 @@ NegativeAcksTracker::NegativeAcksTracker(ClientImplPtr client, ConsumerImpl &con nackDelay_ = std::chrono::milliseconds(std::max(conf.getNegativeAckRedeliveryDelayMs(), MIN_NACK_DELAY_MILLIS)); - timerInterval_ = boost::posix_time::milliseconds((long)(nackDelay_.count() / 3)); - LOG_DEBUG("Created negative ack tracker with delay: " << nackDelay_.count() - << " ms - Timer interval: " << timerInterval_); + timerInterval_ = std::chrono::milliseconds((long)(nackDelay_.count() / 3)); + LOG_DEBUG("Created negative ack tracker with delay: " << nackDelay_.count() << " ms - Timer interval: " + << timerInterval_.count()); } void NegativeAcksTracker::scheduleTimer() { @@ -51,14 +51,14 @@ void NegativeAcksTracker::scheduleTimer() { } std::weak_ptr weakSelf{shared_from_this()}; timer_->expires_from_now(timerInterval_); - timer_->async_wait([weakSelf](const boost::system::error_code &ec) { + timer_->async_wait([weakSelf](const ASIO_ERROR &ec) { if (auto self = weakSelf.lock()) { self->handleTimer(ec); } }); } -void NegativeAcksTracker::handleTimer(const boost::system::error_code &ec) { +void NegativeAcksTracker::handleTimer(const ASIO_ERROR &ec) { if (ec) { // Ignore cancelled events return; @@ -107,7 +107,7 @@ void NegativeAcksTracker::add(const MessageId &m) { void NegativeAcksTracker::close() { closed_ = true; - boost::system::error_code ec; + ASIO_ERROR ec; timer_->cancel(ec); std::lock_guard lock(mutex_); nackedMessages_.clear(); diff --git a/lib/NegativeAcksTracker.h b/lib/NegativeAcksTracker.h index 4b489844..472e9763 100644 --- a/lib/NegativeAcksTracker.h +++ b/lib/NegativeAcksTracker.h @@ -23,12 +23,13 @@ #include #include -#include #include #include #include #include +#include "AsioDefines.h" +#include "AsioTimer.h" #include "TestUtil.h" namespace pulsar { @@ -36,7 +37,6 @@ namespace pulsar { class ConsumerImpl; class ClientImpl; using ClientImplPtr = std::shared_ptr; -using DeadlineTimerPtr = std::shared_ptr; class ExecutorService; using ExecutorServicePtr = std::shared_ptr; @@ -56,13 +56,13 @@ class NegativeAcksTracker : public std::enable_shared_from_this nackedMessages_; diff --git a/lib/OpSendMsg.h b/lib/OpSendMsg.h index a1319e10..46fd9c1c 100644 --- a/lib/OpSendMsg.h +++ b/lib/OpSendMsg.h @@ -23,8 +23,6 @@ #include #include -#include - #include "ChunkMessageIdImpl.h" #include "PulsarApi.pb.h" #include "SharedBuffer.h" @@ -53,7 +51,7 @@ struct OpSendMsg { const int32_t numChunks; const uint32_t messagesCount; const uint64_t messagesSize; - const boost::posix_time::ptime timeout; + const ptime timeout; const SendCallback sendCallback; std::vector> trackerCallbacks; ChunkMessageIdListPtr chunkMessageIdList; @@ -98,7 +96,7 @@ struct OpSendMsg { numChunks(metadata.num_chunks_from_msg()), messagesCount(messagesCount), messagesSize(messagesSize), - timeout(TimeUtils::now() + boost::posix_time::milliseconds(sendTimeoutMs)), + timeout(TimeUtils::now() + std::chrono::milliseconds(sendTimeoutMs)), sendCallback(std::move(callback)), chunkMessageIdList(std::move(chunkMessageIdList)), sendArgs(new SendArguments(producerId, metadata.sequence_id(), metadata, payload)) {} diff --git a/lib/PartitionedProducerImpl.cc b/lib/PartitionedProducerImpl.cc index 54e96c84..4178096c 100644 --- a/lib/PartitionedProducerImpl.cc +++ b/lib/PartitionedProducerImpl.cc @@ -58,7 +58,7 @@ PartitionedProducerImpl::PartitionedProducerImpl(ClientImplPtr client, const Top if (partitionsUpdateInterval > 0) { listenerExecutor_ = client->getListenerExecutorProvider()->get(); partitionsUpdateTimer_ = listenerExecutor_->createDeadlineTimer(); - partitionsUpdateInterval_ = boost::posix_time::seconds(partitionsUpdateInterval); + partitionsUpdateInterval_ = std::chrono::seconds(partitionsUpdateInterval); lookupServicePtr_ = client->getLookup(); } } @@ -69,7 +69,7 @@ MessageRoutingPolicyPtr PartitionedProducerImpl::getMessageRouter() { return std::make_shared( conf_.getHashingScheme(), conf_.getBatchingEnabled(), conf_.getBatchingMaxMessages(), conf_.getBatchingMaxAllowedSizeInBytes(), - boost::posix_time::milliseconds(conf_.getBatchingMaxPublishDelayMs())); + std::chrono::milliseconds(conf_.getBatchingMaxPublishDelayMs())); case ProducerConfiguration::CustomPartition: return conf_.getMessageRouterPtr(); case ProducerConfiguration::UseSinglePartition: @@ -422,7 +422,7 @@ void PartitionedProducerImpl::flushAsync(FlushCallback callback) { void PartitionedProducerImpl::runPartitionUpdateTask() { auto weakSelf = weak_from_this(); partitionsUpdateTimer_->expires_from_now(partitionsUpdateInterval_); - partitionsUpdateTimer_->async_wait([weakSelf](const boost::system::error_code& ec) { + partitionsUpdateTimer_->async_wait([weakSelf](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (self) { self->getPartitionMetadata(); @@ -524,7 +524,7 @@ uint64_t PartitionedProducerImpl::getNumberOfConnectedProducer() { void PartitionedProducerImpl::cancelTimers() noexcept { if (partitionsUpdateTimer_) { - boost::system::error_code ec; + ASIO_ERROR ec; partitionsUpdateTimer_->cancel(ec); } } diff --git a/lib/PartitionedProducerImpl.h b/lib/PartitionedProducerImpl.h index 2d07a81a..610c74ed 100644 --- a/lib/PartitionedProducerImpl.h +++ b/lib/PartitionedProducerImpl.h @@ -20,21 +20,21 @@ #include #include -#include #include #include #include +#include "AsioTimer.h" #include "LookupDataResult.h" #include "ProducerImplBase.h" #include "ProducerInterceptors.h" +#include "TimeUtils.h" namespace pulsar { class ClientImpl; using ClientImplPtr = std::shared_ptr; using ClientImplWeakPtr = std::weak_ptr; -using DeadlineTimerPtr = std::shared_ptr; class ExecutorService; using ExecutorServicePtr = std::shared_ptr; class LookupService; @@ -128,7 +128,7 @@ class PartitionedProducerImpl : public ProducerImplBase, ExecutorServicePtr listenerExecutor_; DeadlineTimerPtr partitionsUpdateTimer_; - boost::posix_time::time_duration partitionsUpdateInterval_; + TimeDuration partitionsUpdateInterval_; LookupServicePtr lookupServicePtr_; ProducerInterceptorsPtr interceptors_; diff --git a/lib/PatternMultiTopicsConsumerImpl.cc b/lib/PatternMultiTopicsConsumerImpl.cc index 23e445ee..4fc7bb61 100644 --- a/lib/PatternMultiTopicsConsumerImpl.cc +++ b/lib/PatternMultiTopicsConsumerImpl.cc @@ -27,6 +27,8 @@ DECLARE_LOG_OBJECT() using namespace pulsar; +using std::chrono::seconds; + PatternMultiTopicsConsumerImpl::PatternMultiTopicsConsumerImpl( ClientImplPtr client, const std::string pattern, CommandGetTopicsOfNamespace_Mode getTopicsMode, const std::vector& topics, const std::string& subscriptionName, @@ -49,15 +51,15 @@ void PatternMultiTopicsConsumerImpl::resetAutoDiscoveryTimer() { autoDiscoveryTimer_->expires_from_now(seconds(conf_.getPatternAutoDiscoveryPeriod())); auto weakSelf = weak_from_this(); - autoDiscoveryTimer_->async_wait([weakSelf](const boost::system::error_code& err) { + autoDiscoveryTimer_->async_wait([weakSelf](const ASIO_ERROR& err) { if (auto self = weakSelf.lock()) { self->autoDiscoveryTimerTask(err); } }); } -void PatternMultiTopicsConsumerImpl::autoDiscoveryTimerTask(const boost::system::error_code& err) { - if (err == boost::asio::error::operation_aborted) { +void PatternMultiTopicsConsumerImpl::autoDiscoveryTimerTask(const ASIO_ERROR& err) { + if (err == ASIO::error::operation_aborted) { LOG_DEBUG(getName() << "Timer cancelled: " << err.message()); return; } else if (err) { @@ -228,7 +230,7 @@ void PatternMultiTopicsConsumerImpl::start() { if (conf_.getPatternAutoDiscoveryPeriod() > 0) { autoDiscoveryTimer_->expires_from_now(seconds(conf_.getPatternAutoDiscoveryPeriod())); auto weakSelf = weak_from_this(); - autoDiscoveryTimer_->async_wait([weakSelf](const boost::system::error_code& err) { + autoDiscoveryTimer_->async_wait([weakSelf](const ASIO_ERROR& err) { if (auto self = weakSelf.lock()) { self->autoDiscoveryTimerTask(err); } @@ -247,6 +249,6 @@ void PatternMultiTopicsConsumerImpl::closeAsync(ResultCallback callback) { } void PatternMultiTopicsConsumerImpl::cancelTimers() noexcept { - boost::system::error_code ec; + ASIO_ERROR ec; autoDiscoveryTimer_->cancel(ec); } diff --git a/lib/PatternMultiTopicsConsumerImpl.h b/lib/PatternMultiTopicsConsumerImpl.h index 5d3ba9ec..f272df22 100644 --- a/lib/PatternMultiTopicsConsumerImpl.h +++ b/lib/PatternMultiTopicsConsumerImpl.h @@ -22,6 +22,7 @@ #include #include +#include "AsioTimer.h" #include "LookupDataResult.h" #include "MultiTopicsConsumerImpl.h" #include "NamespaceName.h" @@ -56,7 +57,7 @@ class PatternMultiTopicsConsumerImpl : public MultiTopicsConsumerImpl { const PULSAR_REGEX_NAMESPACE::regex getPattern(); - void autoDiscoveryTimerTask(const boost::system::error_code& err); + void autoDiscoveryTimerTask(const ASIO_ERROR& err); // filter input `topics` with given `pattern`, return matched topics. Do not match topic domain. static NamespaceTopicsPtr topicsPatternFilter(const std::vector& topics, @@ -74,7 +75,7 @@ class PatternMultiTopicsConsumerImpl : public MultiTopicsConsumerImpl { const std::string patternString_; const PULSAR_REGEX_NAMESPACE::regex pattern_; const CommandGetTopicsOfNamespace_Mode getTopicsMode_; - typedef std::shared_ptr TimerPtr; + typedef std::shared_ptr TimerPtr; TimerPtr autoDiscoveryTimer_; bool autoDiscoveryRunning_; NamespaceNamePtr namespaceName_; diff --git a/lib/PeriodicTask.cc b/lib/PeriodicTask.cc index 6046eae2..9fde012a 100644 --- a/lib/PeriodicTask.cc +++ b/lib/PeriodicTask.cc @@ -18,6 +18,8 @@ */ #include "PeriodicTask.h" +#include + namespace pulsar { void PeriodicTask::start() { @@ -27,7 +29,7 @@ void PeriodicTask::start() { state_ = Ready; if (periodMs_ >= 0) { std::weak_ptr weakSelf{shared_from_this()}; - timer_->expires_from_now(boost::posix_time::millisec(periodMs_)); + timer_->expires_from_now(std::chrono::milliseconds(periodMs_)); timer_->async_wait([weakSelf](const ErrorCode& ec) { auto self = weakSelf.lock(); if (self) { @@ -48,7 +50,7 @@ void PeriodicTask::stop() noexcept { } void PeriodicTask::handleTimeout(const ErrorCode& ec) { - if (state_ != Ready || ec.value() == boost::system::errc::operation_canceled) { + if (state_ != Ready || ec == ASIO::error::operation_aborted) { return; } @@ -57,7 +59,7 @@ void PeriodicTask::handleTimeout(const ErrorCode& ec) { // state_ may be changed in handleTimeout, so we check state_ again if (state_ == Ready) { auto self = shared_from_this(); - timer_->expires_from_now(boost::posix_time::millisec(periodMs_)); + timer_->expires_from_now(std::chrono::milliseconds(periodMs_)); timer_->async_wait([this, self](const ErrorCode& ec) { handleTimeout(ec); }); } } diff --git a/lib/PeriodicTask.h b/lib/PeriodicTask.h index 5de81ae4..bc186348 100644 --- a/lib/PeriodicTask.h +++ b/lib/PeriodicTask.h @@ -36,7 +36,7 @@ namespace pulsar { */ class PeriodicTask : public std::enable_shared_from_this { public: - using ErrorCode = boost::system::error_code; + using ErrorCode = ASIO_ERROR; using CallbackType = std::function; enum State : std::uint8_t diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index 0a129259..fc39b231 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -20,7 +20,7 @@ #include -#include +#include #include "BatchMessageContainer.h" #include "BatchMessageKeyBasedContainer.h" @@ -46,6 +46,8 @@ namespace pulsar { DECLARE_LOG_OBJECT() +using std::chrono::milliseconds; + ProducerImpl::ProducerImpl(ClientImplPtr client, const TopicName& topicName, const ProducerConfiguration& conf, const ProducerInterceptorsPtr& interceptors, int32_t partition, bool retryOnCreationError) @@ -465,7 +467,7 @@ void ProducerImpl::sendAsync(const Message& msg, SendCallback callback) { Producer producer = Producer(shared_from_this()); auto interceptorMessage = interceptors_->beforeSend(producer, msg); - const auto now = boost::posix_time::microsec_clock::universal_time(); + const auto now = TimeUtils::now(); auto self = shared_from_this(); sendAsyncWithStatsUpdate(interceptorMessage, [this, self, now, callback, producer, interceptorMessage]( Result result, const MessageId& messageId) { @@ -564,10 +566,9 @@ void ProducerImpl::sendAsyncWithStatsUpdate(const Message& msg, SendCallback&& c bool isFirstMessage = batchMessageContainer_->isFirstMessageToAdd(msg); bool isFull = batchMessageContainer_->add(msg, callback); if (isFirstMessage) { - batchTimer_->expires_from_now( - boost::posix_time::milliseconds(conf_.getBatchingMaxPublishDelayMs())); + batchTimer_->expires_from_now(milliseconds(conf_.getBatchingMaxPublishDelayMs())); auto weakSelf = weak_from_this(); - batchTimer_->async_wait([this, weakSelf](const boost::system::error_code& ec) { + batchTimer_->async_wait([this, weakSelf](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (!self) { return; @@ -824,14 +825,14 @@ Future ProducerImpl::getProducerCreatedFuture() uint64_t ProducerImpl::getProducerId() const { return producerId_; } -void ProducerImpl::handleSendTimeout(const boost::system::error_code& err) { +void ProducerImpl::handleSendTimeout(const ASIO_ERROR& err) { const auto state = state_.load(); if (state != Pending && state != Ready) { return; } Lock lock(mutex_); - if (err == boost::asio::error::operation_aborted) { + if (err == ASIO::error::operation_aborted) { LOG_DEBUG(getName() << "Timer cancelled: " << err.message()); return; } else if (err) { @@ -847,8 +848,8 @@ void ProducerImpl::handleSendTimeout(const boost::system::error_code& err) { } else { // If there is at least one message, calculate the diff between the message timeout and // the current time. - time_duration diff = pendingMessagesQueue_.front()->timeout - TimeUtils::now(); - if (diff.total_milliseconds() <= 0) { + auto diff = pendingMessagesQueue_.front()->timeout - TimeUtils::now(); + if (toMillis(diff) <= 0) { // The diff is less than or equal to zero, meaning that the message has been expired. LOG_DEBUG(getName() << "Timer expired. Calling timeout callbacks."); pendingMessages = getPendingCallbacksWhenFailed(); @@ -856,7 +857,7 @@ void ProducerImpl::handleSendTimeout(const boost::system::error_code& err) { asyncWaitSendTimeout(milliseconds(conf_.getSendTimeout())); } else { // The diff is greater than zero, set the timeout to the diff value - LOG_DEBUG(getName() << "Timer hasn't expired yet, setting new timeout " << diff); + LOG_DEBUG(getName() << "Timer hasn't expired yet, setting new timeout " << diff.count()); asyncWaitSendTimeout(diff); } } @@ -1000,7 +1001,7 @@ void ProducerImpl::shutdown() { void ProducerImpl::cancelTimers() noexcept { dataKeyRefreshTask_.stop(); - boost::system::error_code ec; + ASIO_ERROR ec; batchTimer_->cancel(ec); sendTimer_->cancel(ec); } @@ -1026,7 +1027,7 @@ void ProducerImpl::asyncWaitSendTimeout(DurationType expiryTime) { sendTimer_->expires_from_now(expiryTime); auto weakSelf = weak_from_this(); - sendTimer_->async_wait([weakSelf](const boost::system::error_code& err) { + sendTimer_->async_wait([weakSelf](const ASIO_ERROR& err) { auto self = weakSelf.lock(); if (self) { std::static_pointer_cast(self)->handleSendTimeout(err); diff --git a/lib/ProducerImpl.h b/lib/ProducerImpl.h index b467458d..97816050 100644 --- a/lib/ProducerImpl.h +++ b/lib/ProducerImpl.h @@ -19,6 +19,12 @@ #ifndef LIB_PRODUCERIMPL_H_ #define LIB_PRODUCERIMPL_H_ +#include "TimeUtils.h" +#ifdef USE_ASIO +#include +#else +#include +#endif #include #include #include @@ -30,6 +36,7 @@ #if defined(_MSC_VER) || defined(__APPLE__) #include "OpSendMsg.h" #endif +#include "AsioDefines.h" #include "PendingFailures.h" #include "PeriodicTask.h" #include "ProducerImplBase.h" @@ -39,7 +46,7 @@ namespace pulsar { class BatchMessageContainerBase; class ClientImpl; using ClientImplPtr = std::shared_ptr; -using DeadlineTimerPtr = std::shared_ptr; +using DeadlineTimerPtr = std::shared_ptr; class MessageCrypto; using MessageCryptoPtr = std::shared_ptr; class ProducerImpl; @@ -137,7 +144,7 @@ class ProducerImpl : public HandlerBase, public ProducerImplBase { void resendMessages(ClientConnectionPtr cnx); - void refreshEncryptionKey(const boost::system::error_code& ec); + void refreshEncryptionKey(const ASIO_ERROR& ec); bool encryptMessage(proto::MessageMetadata& metadata, SharedBuffer& payload, SharedBuffer& encryptedPayload); @@ -183,8 +190,8 @@ class ProducerImpl : public HandlerBase, public ProducerImplBase { std::string schemaVersion_; DeadlineTimerPtr sendTimer_; - void handleSendTimeout(const boost::system::error_code& err); - using DurationType = typename boost::asio::deadline_timer::duration_type; + void handleSendTimeout(const ASIO_ERROR& err); + using DurationType = TimeDuration; void asyncWaitSendTimeout(DurationType expiryTime); Promise producerCreatedPromise_; diff --git a/lib/RetryableOperation.h b/lib/RetryableOperation.h index d026e424..9c920da1 100644 --- a/lib/RetryableOperation.h +++ b/lib/RetryableOperation.h @@ -30,6 +30,7 @@ #include "Future.h" #include "LogUtils.h" #include "ResultUtils.h" +#include "TimeUtils.h" namespace pulsar { @@ -43,9 +44,8 @@ class RetryableOperation : public std::enable_shared_from_thiscancel(ec); } @@ -100,7 +100,7 @@ class RetryableOperation : public std::enable_shared_from_thisexpires_from_now(delay); auto nextRemainingTime = remainingTime - delay; - LOG_INFO("Reschedule " << name_ << " for " << delay.total_milliseconds() - << " ms, remaining time: " << nextRemainingTime.total_milliseconds() - << " ms"); - timer_->async_wait([this, weakSelf, nextRemainingTime](const boost::system::error_code& ec) { + LOG_INFO("Reschedule " << name_ << " for " << toMillis(delay) + << " ms, remaining time: " << toMillis(nextRemainingTime) << " ms"); + timer_->async_wait([this, weakSelf, nextRemainingTime](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (!self) { return; } if (ec) { - if (ec == boost::asio::error::operation_aborted) { + if (ec == ASIO::error::operation_aborted) { LOG_DEBUG("Timer for " << name_ << " is cancelled"); promise_.setFailed(ResultTimeout); } else { LOG_WARN("Timer for " << name_ << " failed: " << ec.message()); } } else { - LOG_DEBUG("Run operation " << name_ << ", remaining time: " - << nextRemainingTime.total_milliseconds() << " ms"); + LOG_DEBUG("Run operation " << name_ << ", remaining time: " << toMillis(nextRemainingTime) + << " ms"); runImpl(nextRemainingTime); } }); diff --git a/lib/RoundRobinMessageRouter.cc b/lib/RoundRobinMessageRouter.cc index 9693cc25..1bc0b306 100644 --- a/lib/RoundRobinMessageRouter.cc +++ b/lib/RoundRobinMessageRouter.cc @@ -26,8 +26,7 @@ namespace pulsar { RoundRobinMessageRouter::RoundRobinMessageRouter(ProducerConfiguration::HashingScheme hashingScheme, bool batchingEnabled, uint32_t maxBatchingMessages, - uint32_t maxBatchingSize, - boost::posix_time::time_duration maxBatchingDelay) + uint32_t maxBatchingSize, TimeDuration maxBatchingDelay) : MessageRouterBase(hashingScheme), batchingEnabled_(batchingEnabled), maxBatchingMessages_(maxBatchingMessages), @@ -74,7 +73,7 @@ int RoundRobinMessageRouter::getPartition(const Message& msg, const TopicMetadat int64_t now = TimeUtils::currentTimeMillis(); if (messageCount >= maxBatchingMessages_ || (messageSize >= maxBatchingSize_ - batchSize) || - (now - lastPartitionChange >= maxBatchingDelay_.total_milliseconds())) { + (now - lastPartitionChange >= toMillis(maxBatchingDelay_))) { uint32_t currentPartitionCursor = ++currentPartitionCursor_; lastPartitionChange_ = now; cumulativeBatchSize_ = messageSize; diff --git a/lib/RoundRobinMessageRouter.h b/lib/RoundRobinMessageRouter.h index 753573a1..03cfac80 100644 --- a/lib/RoundRobinMessageRouter.h +++ b/lib/RoundRobinMessageRouter.h @@ -23,16 +23,16 @@ #include #include -#include #include "MessageRouterBase.h" +#include "TimeUtils.h" namespace pulsar { class PULSAR_PUBLIC RoundRobinMessageRouter : public MessageRouterBase { public: RoundRobinMessageRouter(ProducerConfiguration::HashingScheme hashingScheme, bool batchingEnabled, uint32_t maxBatchingMessages, uint32_t maxBatchingSize, - boost::posix_time::time_duration maxBatchingDelay); + TimeDuration maxBatchingDelay); virtual ~RoundRobinMessageRouter(); virtual int getPartition(const Message& msg, const TopicMetadata& topicMetadata); @@ -40,7 +40,7 @@ class PULSAR_PUBLIC RoundRobinMessageRouter : public MessageRouterBase { const bool batchingEnabled_; const uint32_t maxBatchingMessages_; const uint32_t maxBatchingSize_; - const boost::posix_time::time_duration maxBatchingDelay_; + const TimeDuration maxBatchingDelay_; std::atomic currentPartitionCursor_; std::atomic lastPartitionChange_; diff --git a/lib/SharedBuffer.h b/lib/SharedBuffer.h index 7ee26184..26fc59ed 100644 --- a/lib/SharedBuffer.h +++ b/lib/SharedBuffer.h @@ -22,12 +22,19 @@ #include #include +#ifdef USE_ASIO +#include +#include +#else #include #include +#endif #include #include #include +#include "AsioDefines.h" + namespace pulsar { class SharedBuffer { @@ -144,13 +151,13 @@ class SharedBuffer { inline bool writable() const { return writableBytes() > 0; } - boost::asio::const_buffers_1 const_asio_buffer() const { - return boost::asio::const_buffers_1(ptr_ + readIdx_, readableBytes()); + ASIO::const_buffers_1 const_asio_buffer() const { + return ASIO::const_buffers_1(ptr_ + readIdx_, readableBytes()); } - boost::asio::mutable_buffers_1 asio_buffer() { + ASIO::mutable_buffers_1 asio_buffer() { assert(data_); - return boost::asio::buffer(ptr_ + writeIdx_, writableBytes()); + return ASIO::buffer(ptr_ + writeIdx_, writableBytes()); } void write(const char* data, uint32_t size) { @@ -239,17 +246,17 @@ class CompositeSharedBuffer { } // Implement the ConstBufferSequence requirements. - typedef boost::asio::const_buffer value_type; - typedef boost::asio::const_buffer* iterator; - typedef const boost::asio::const_buffer* const_iterator; + typedef ASIO::const_buffer value_type; + typedef ASIO::const_buffer* iterator; + typedef const ASIO::const_buffer* const_iterator; - const boost::asio::const_buffer* begin() const { return &(asioBuffers_.at(0)); } + const ASIO::const_buffer* begin() const { return &(asioBuffers_.at(0)); } - const boost::asio::const_buffer* end() const { return begin() + Size; } + const ASIO::const_buffer* end() const { return begin() + Size; } private: std::array sharedBuffers_; - std::array asioBuffers_; + std::array asioBuffers_; }; typedef CompositeSharedBuffer<2> PairSharedBuffer; diff --git a/lib/TimeUtils.h b/lib/TimeUtils.h index a55773d9..03f563c3 100644 --- a/lib/TimeUtils.h +++ b/lib/TimeUtils.h @@ -21,19 +21,23 @@ #include #include -#include #include namespace pulsar { -using namespace boost::posix_time; -using boost::posix_time::milliseconds; -using boost::posix_time::seconds; +using ptime = decltype(std::chrono::high_resolution_clock::now()); +using TimeDuration = std::chrono::nanoseconds; + +inline decltype(std::chrono::milliseconds(0).count()) toMillis(TimeDuration duration) { + return std::chrono::duration_cast(duration).count(); +} class PULSAR_PUBLIC TimeUtils { public: - static ptime now(); - static int64_t currentTimeMillis(); + static ptime now() { return std::chrono::high_resolution_clock::now(); } + static int64_t currentTimeMillis() { + return std::chrono::duration_cast(now().time_since_epoch()).count(); + } }; // This class processes a timeout with the following semantics: diff --git a/lib/UnAckedMessageTrackerEnabled.cc b/lib/UnAckedMessageTrackerEnabled.cc index 061a1409..e371af99 100644 --- a/lib/UnAckedMessageTrackerEnabled.cc +++ b/lib/UnAckedMessageTrackerEnabled.cc @@ -34,9 +34,9 @@ void UnAckedMessageTrackerEnabled::timeoutHandler() { timeoutHandlerHelper(); ExecutorServicePtr executorService = client_->getIOExecutorProvider()->get(); timer_ = executorService->createDeadlineTimer(); - timer_->expires_from_now(boost::posix_time::milliseconds(tickDurationInMs_)); + timer_->expires_from_now(std::chrono::milliseconds(tickDurationInMs_)); std::weak_ptr weakSelf{shared_from_this()}; - timer_->async_wait([weakSelf](const boost::system::error_code& ec) { + timer_->async_wait([weakSelf](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (self && !ec) { self->timeoutHandler(); @@ -173,7 +173,7 @@ void UnAckedMessageTrackerEnabled::clear() { } void UnAckedMessageTrackerEnabled::stop() { - boost::system::error_code ec; + ASIO_ERROR ec; if (timer_) { timer_->cancel(ec); } diff --git a/lib/UnAckedMessageTrackerEnabled.h b/lib/UnAckedMessageTrackerEnabled.h index 6181a8a3..83edc4cb 100644 --- a/lib/UnAckedMessageTrackerEnabled.h +++ b/lib/UnAckedMessageTrackerEnabled.h @@ -18,13 +18,13 @@ */ #ifndef LIB_UNACKEDMESSAGETRACKERENABLED_H_ #define LIB_UNACKEDMESSAGETRACKERENABLED_H_ -#include #include #include #include #include #include +#include "AsioTimer.h" #include "TestUtil.h" #include "UnAckedMessageTrackerInterface.h" @@ -33,7 +33,6 @@ namespace pulsar { class ClientImpl; class ConsumerImplBase; using ClientImplPtr = std::shared_ptr; -using DeadlineTimerPtr = std::shared_ptr; class UnAckedMessageTrackerEnabled : public std::enable_shared_from_this, public UnAckedMessageTrackerInterface { diff --git a/lib/auth/athenz/ZTSClient.cc b/lib/auth/athenz/ZTSClient.cc index 230713ea..35387d96 100644 --- a/lib/auth/athenz/ZTSClient.cc +++ b/lib/auth/athenz/ZTSClient.cc @@ -44,8 +44,6 @@ namespace ptree = boost::property_tree; #pragma clang diagnostic ignored "-Wunknown-warning-option" #endif -#include - #if defined(__clang__) #pragma clang diagnostic pop #endif diff --git a/lib/stats/ConsumerStatsImpl.cc b/lib/stats/ConsumerStatsImpl.cc index 056dbf6e..0eefabdc 100644 --- a/lib/stats/ConsumerStatsImpl.cc +++ b/lib/stats/ConsumerStatsImpl.cc @@ -46,7 +46,7 @@ ConsumerStatsImpl::ConsumerStatsImpl(const ConsumerStatsImpl& stats) totalAckedMsgMap_(stats.totalAckedMsgMap_), statsIntervalInSeconds_(stats.statsIntervalInSeconds_) {} -void ConsumerStatsImpl::flushAndReset(const boost::system::error_code& ec) { +void ConsumerStatsImpl::flushAndReset(const ASIO_ERROR& ec) { if (ec) { LOG_DEBUG("Ignoring timer cancelled event, code[" << ec << "]"); return; @@ -85,9 +85,9 @@ void ConsumerStatsImpl::messageAcknowledged(Result res, CommandAck_AckType ackTy } void ConsumerStatsImpl::scheduleTimer() { - timer_->expires_from_now(boost::posix_time::seconds(statsIntervalInSeconds_)); + timer_->expires_from_now(std::chrono::seconds(statsIntervalInSeconds_)); std::weak_ptr weakSelf{shared_from_this()}; - timer_->async_wait([this, weakSelf](const boost::system::error_code& ec) { + timer_->async_wait([this, weakSelf](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (!self) { return; diff --git a/lib/stats/ConsumerStatsImpl.h b/lib/stats/ConsumerStatsImpl.h index 44f927f5..03f3a474 100644 --- a/lib/stats/ConsumerStatsImpl.h +++ b/lib/stats/ConsumerStatsImpl.h @@ -20,17 +20,16 @@ #ifndef PULSAR_CONSUMER_STATS_IMPL_H_ #define PULSAR_CONSUMER_STATS_IMPL_H_ -#include #include #include #include #include #include "ConsumerStatsBase.h" +#include "lib/AsioTimer.h" #include "lib/ExecutorService.h" namespace pulsar { -using DeadlineTimerPtr = std::shared_ptr; class ExecutorService; using ExecutorServicePtr = std::shared_ptr; @@ -58,7 +57,7 @@ class ConsumerStatsImpl : public std::enable_shared_from_this public: ConsumerStatsImpl(std::string, ExecutorServicePtr, unsigned int); ConsumerStatsImpl(const ConsumerStatsImpl& stats); - void flushAndReset(const boost::system::error_code&); + void flushAndReset(const ASIO_ERROR&); void start() override; void receivedMessage(Message&, Result) override; void messageAcknowledged(Result, CommandAck_AckType, uint32_t ackNums) override; diff --git a/lib/stats/ProducerStatsBase.h b/lib/stats/ProducerStatsBase.h index fe0ba0a5..b24266e8 100644 --- a/lib/stats/ProducerStatsBase.h +++ b/lib/stats/ProducerStatsBase.h @@ -22,14 +22,14 @@ #include #include -#include +#include "lib/TimeUtils.h" namespace pulsar { class ProducerStatsBase { public: virtual void start() {} virtual void messageSent(const Message& msg) = 0; - virtual void messageReceived(Result, const boost::posix_time::ptime&) = 0; + virtual void messageReceived(Result, const ptime&) = 0; virtual ~ProducerStatsBase(){}; }; diff --git a/lib/stats/ProducerStatsDisabled.h b/lib/stats/ProducerStatsDisabled.h index df1df0f8..df1da783 100644 --- a/lib/stats/ProducerStatsDisabled.h +++ b/lib/stats/ProducerStatsDisabled.h @@ -25,7 +25,7 @@ namespace pulsar { class ProducerStatsDisabled : public ProducerStatsBase { public: virtual void messageSent(const Message& msg){}; - virtual void messageReceived(Result, const boost::posix_time::ptime&){}; + virtual void messageReceived(Result, const ptime&){}; }; } // namespace pulsar #endif // PULSAR_PRODUCER_STATS_DISABLED_HEADER diff --git a/lib/stats/ProducerStatsImpl.cc b/lib/stats/ProducerStatsImpl.cc index 3d3629db..15e9e67e 100644 --- a/lib/stats/ProducerStatsImpl.cc +++ b/lib/stats/ProducerStatsImpl.cc @@ -20,9 +20,11 @@ #include "ProducerStatsImpl.h" #include +#include #include "lib/ExecutorService.h" #include "lib/LogUtils.h" +#include "lib/TimeUtils.h" #include "lib/Utils.h" namespace pulsar { @@ -65,7 +67,7 @@ ProducerStatsImpl::ProducerStatsImpl(const ProducerStatsImpl& stats) void ProducerStatsImpl::start() { scheduleTimer(); } -void ProducerStatsImpl::flushAndReset(const boost::system::error_code& ec) { +void ProducerStatsImpl::flushAndReset(const ASIO_ERROR& ec) { if (ec) { LOG_DEBUG("Ignoring timer cancelled event, code[" << ec << "]"); return; @@ -93,9 +95,10 @@ void ProducerStatsImpl::messageSent(const Message& msg) { totalBytesSent_ += msg.getLength(); } -void ProducerStatsImpl::messageReceived(Result res, const boost::posix_time::ptime& publishTime) { - boost::posix_time::ptime currentTime = boost::posix_time::microsec_clock::universal_time(); - double diffInMicros = (currentTime - publishTime).total_microseconds(); +void ProducerStatsImpl::messageReceived(Result res, const ptime& publishTime) { + auto currentTime = TimeUtils::now(); + double diffInMicros = + std::chrono::duration_cast(currentTime - publishTime).count(); std::lock_guard lock(mutex_); totalLatencyAccumulator_(diffInMicros); latencyAccumulator_(diffInMicros); @@ -106,9 +109,9 @@ void ProducerStatsImpl::messageReceived(Result res, const boost::posix_time::pti ProducerStatsImpl::~ProducerStatsImpl() { timer_->cancel(); } void ProducerStatsImpl::scheduleTimer() { - timer_->expires_from_now(boost::posix_time::seconds(statsIntervalInSeconds_)); + timer_->expires_from_now(std::chrono::seconds(statsIntervalInSeconds_)); std::weak_ptr weakSelf{shared_from_this()}; - timer_->async_wait([this, weakSelf](const boost::system::error_code& ec) { + timer_->async_wait([this, weakSelf](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (!self) { return; diff --git a/lib/stats/ProducerStatsImpl.h b/lib/stats/ProducerStatsImpl.h index 8cd10992..5d445c60 100644 --- a/lib/stats/ProducerStatsImpl.h +++ b/lib/stats/ProducerStatsImpl.h @@ -30,20 +30,18 @@ #include #include #include -#include -#include #include #include #include #include #include "ProducerStatsBase.h" +#include "lib/AsioTimer.h" namespace pulsar { class ExecutorService; using ExecutorServicePtr = std::shared_ptr; -using DeadlineTimerPtr = std::shared_ptr; typedef boost::accumulators::accumulator_set< double, @@ -83,11 +81,11 @@ class ProducerStatsImpl : public std::enable_shared_from_this void start() override; - void flushAndReset(const boost::system::error_code&); + void flushAndReset(const ASIO_ERROR&); void messageSent(const Message&) override; - void messageReceived(Result, const boost::posix_time::ptime&) override; + void messageReceived(Result, const ptime&) override; ~ProducerStatsImpl(); diff --git a/tests/AuthPluginTest.cc b/tests/AuthPluginTest.cc index 9fd048b7..b091f973 100644 --- a/tests/AuthPluginTest.cc +++ b/tests/AuthPluginTest.cc @@ -21,9 +21,14 @@ #include #include +#ifdef USE_ASIO +#include +#else #include +#endif #include +#include "lib/AsioDefines.h" #include "lib/Future.h" #include "lib/Latch.h" #include "lib/LogUtils.h" @@ -287,10 +292,9 @@ namespace testAthenz { std::string principalToken; void mockZTS(Latch& latch, int port) { LOG_INFO("-- MockZTS started"); - boost::asio::io_service io; - boost::asio::ip::tcp::iostream stream; - boost::asio::ip::tcp::acceptor acceptor(io, - boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), port)); + ASIO::io_service io; + ASIO::ip::tcp::iostream stream; + ASIO::ip::tcp::acceptor acceptor(io, ASIO::ip::tcp::endpoint(ASIO::ip::tcp::v4(), port)); LOG_INFO("-- MockZTS waiting for connnection"); latch.countdown(); diff --git a/tests/AuthTokenTest.cc b/tests/AuthTokenTest.cc index a04da085..7595f44e 100644 --- a/tests/AuthTokenTest.cc +++ b/tests/AuthTokenTest.cc @@ -22,7 +22,6 @@ #include #include -#include #include #include #include diff --git a/tests/BackoffTest.cc b/tests/BackoffTest.cc index d066b944..5fe4f71a 100644 --- a/tests/BackoffTest.cc +++ b/tests/BackoffTest.cc @@ -26,42 +26,42 @@ #include "lib/stats/ProducerStatsImpl.h" using namespace pulsar; -using boost::posix_time::milliseconds; -using boost::posix_time::seconds; +using std::chrono::milliseconds; +using std::chrono::seconds; static bool checkExactAndDecrementTimer(Backoff& backoff, const unsigned int& t2) { - const unsigned int& t1 = backoff.next().total_milliseconds(); - boost::posix_time::ptime& firstBackOffTime = PulsarFriend::getFirstBackoffTime(backoff); + auto t1 = toMillis(backoff.next()); + auto& firstBackOffTime = PulsarFriend::getFirstBackoffTime(backoff); firstBackOffTime -= milliseconds(t2); return t1 == t2; } static bool withinTenPercentAndDecrementTimer(Backoff& backoff, const unsigned int& t2) { - const unsigned int& t1 = backoff.next().total_milliseconds(); - boost::posix_time::ptime& firstBackOffTime = PulsarFriend::getFirstBackoffTime(backoff); + auto t1 = toMillis(backoff.next()); + auto& firstBackOffTime = PulsarFriend::getFirstBackoffTime(backoff); firstBackOffTime -= milliseconds(t2); return (t1 >= t2 * 0.9 && t1 <= t2); } TEST(BackoffTest, mandatoryStopTestNegativeTest) { Backoff backoff(milliseconds(100), seconds(60), milliseconds(1900)); - ASSERT_EQ(backoff.next().total_milliseconds(), 100); - backoff.next().total_milliseconds(); // 200 - backoff.next().total_milliseconds(); // 400 - backoff.next().total_milliseconds(); // 800 + ASSERT_EQ(toMillis(backoff.next()), 100); + backoff.next(); // 200 + backoff.next(); // 400 + backoff.next(); // 800 ASSERT_FALSE(withinTenPercentAndDecrementTimer(backoff, 400)); } TEST(BackoffTest, firstBackoffTimerTest) { Backoff backoff(milliseconds(100), seconds(60), milliseconds(1900)); - ASSERT_EQ(backoff.next().total_milliseconds(), 100); - boost::posix_time::ptime firstBackOffTime = PulsarFriend::getFirstBackoffTime(backoff); + ASSERT_EQ(toMillis(backoff.next()), 100); + auto firstBackOffTime = PulsarFriend::getFirstBackoffTime(backoff); std::this_thread::sleep_for(std::chrono::milliseconds(300)); TimeDuration diffBackOffTime = PulsarFriend::getFirstBackoffTime(backoff) - firstBackOffTime; ASSERT_EQ(diffBackOffTime, milliseconds(0)); // no change since reset not called backoff.reset(); - ASSERT_EQ(backoff.next().total_milliseconds(), 100); + ASSERT_EQ(toMillis(backoff.next()), 100); diffBackOffTime = PulsarFriend::getFirstBackoffTime(backoff) - firstBackOffTime; ASSERT_TRUE(diffBackOffTime >= milliseconds(300) && diffBackOffTime < seconds(1)); } diff --git a/tests/ConsumerTest.cc b/tests/ConsumerTest.cc index c7e98bce..f97457fe 100644 --- a/tests/ConsumerTest.cc +++ b/tests/ConsumerTest.cc @@ -955,7 +955,7 @@ TEST(ConsumerTest, testGetLastMessageIdBlockWhenConnectionDisconnected) { auto elapsed = TimeUtils::now() - start; // getLastMessageIdAsync should be blocked until operationTimeout when the connection is disconnected. - ASSERT_GE(elapsed.seconds(), operationTimeout); + ASSERT_GE(std::chrono::duration_cast(elapsed).count(), operationTimeout); } TEST(ConsumerTest, testRedeliveryOfDecryptionFailedMessages) { diff --git a/tests/PulsarFriend.h b/tests/PulsarFriend.h index c2863e8c..73778842 100644 --- a/tests/PulsarFriend.h +++ b/tests/PulsarFriend.h @@ -170,7 +170,7 @@ class PulsarFriend { handler.connection_ = conn; } - static boost::posix_time::ptime& getFirstBackoffTime(Backoff& backoff) { + static auto getFirstBackoffTime(Backoff& backoff) -> decltype(backoff.firstBackoffTime_)& { return backoff.firstBackoffTime_; } diff --git a/tests/RoundRobinMessageRouterTest.cc b/tests/RoundRobinMessageRouterTest.cc index 56a76050..145c45ae 100644 --- a/tests/RoundRobinMessageRouterTest.cc +++ b/tests/RoundRobinMessageRouterTest.cc @@ -31,7 +31,7 @@ TEST(RoundRobinMessageRouterTest, onePartition) { const int numPartitions = 1; RoundRobinMessageRouter router(ProducerConfiguration::BoostHash, false, 1, 1, - boost::posix_time::milliseconds(0)); + std::chrono::milliseconds(0)); Message msg1 = MessageBuilder().setPartitionKey("my-key-1").setContent("one").build(); Message msg2 = MessageBuilder().setPartitionKey("my-key-2").setContent("two").build(); @@ -49,7 +49,7 @@ TEST(RoundRobinMessageRouterTest, sameKey) { const int numPartitions = 13; RoundRobinMessageRouter router(ProducerConfiguration::BoostHash, false, 1, 1, - boost::posix_time::milliseconds(0)); + std::chrono::milliseconds(0)); Message msg1 = MessageBuilder().setPartitionKey("my-key").setContent("one").build(); Message msg2 = MessageBuilder().setPartitionKey("my-key").setContent("two").build(); @@ -63,7 +63,7 @@ TEST(RoundRobinMessageRouterTest, batchingDisabled) { const int numPartitions = 13; RoundRobinMessageRouter router(ProducerConfiguration::BoostHash, false, 1, 1, - boost::posix_time::milliseconds(0)); + std::chrono::milliseconds(0)); Message msg1 = MessageBuilder().setContent("one").build(); Message msg2 = MessageBuilder().setContent("two").build(); @@ -77,7 +77,7 @@ TEST(RoundRobinMessageRouterTest, batchingEnabled) { const int numPartitions = 13; RoundRobinMessageRouter router(ProducerConfiguration::BoostHash, true, 1000, 100000, - boost::posix_time::seconds(1)); + std::chrono::seconds(1)); int p = -1; for (int i = 0; i < 100; i++) { @@ -96,7 +96,7 @@ TEST(RoundRobinMessageRouterTest, maxDelay) { const int numPartitions = 13; RoundRobinMessageRouter router(ProducerConfiguration::BoostHash, true, 1000, 100000, - boost::posix_time::seconds(1)); + std::chrono::seconds(1)); int p1 = -1; for (int i = 0; i < 100; i++) { @@ -132,8 +132,7 @@ TEST(RoundRobinMessageRouterTest, maxDelay) { TEST(RoundRobinMessageRouterTest, maxNumberOfMessages) { const int numPartitions = 13; - RoundRobinMessageRouter router(ProducerConfiguration::BoostHash, true, 2, 1000, - boost::posix_time::seconds(1)); + RoundRobinMessageRouter router(ProducerConfiguration::BoostHash, true, 2, 1000, std::chrono::seconds(1)); Message msg1 = MessageBuilder().setContent("one").build(); Message msg2 = MessageBuilder().setContent("two").build(); @@ -150,8 +149,7 @@ TEST(RoundRobinMessageRouterTest, maxNumberOfMessages) { TEST(RoundRobinMessageRouterTest, maxBatchSize) { const int numPartitions = 13; - RoundRobinMessageRouter router(ProducerConfiguration::BoostHash, true, 10, 8, - boost::posix_time::seconds(1)); + RoundRobinMessageRouter router(ProducerConfiguration::BoostHash, true, 10, 8, std::chrono::seconds(1)); Message msg1 = MessageBuilder().setContent("one").build(); Message msg2 = MessageBuilder().setContent("two").build(); diff --git a/vcpkg.json b/vcpkg.json index d023a406..5ff44100 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -5,45 +5,20 @@ "builtin-baseline": "b051745c68faa6f65c493371d564c4eb8af34dad", "dependencies": [ { - "name": "boost-accumulators", - "version>=": "1.83.0" - }, - { - "name": "boost-algorithm", - "version>=": "1.83.0" - }, - { - "name": "boost-any", - "version>=": "1.83.0" - }, - { - "name": "boost-asio", - "version>=": "1.83.0" - }, - { - "name": "boost-circular-buffer", - "version>=": "1.83.0" - }, - { - "name": "boost-date-time", - "version>=": "1.83.0" + "name": "asio", + "features": [ + "openssl" + ], + "version>=": "1.28.2" }, { - "name": "boost-predef", + "name": "boost-accumulators", "version>=": "1.83.0" }, { "name": "boost-property-tree", "version>=": "1.83.0" }, - { - "name": "boost-serialization", - "version>=": "1.83.0" - }, - { - "name": "boost-xpressive", - "version>=": "1.83.0" - }, { "name": "curl", "default-features": false, From 9ffd2ef45f87e52745d3baad454aae081f035507 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 9 Jan 2024 13:53:44 +0800 Subject: [PATCH 25/54] Fix Boost source code download failure (#384) * Fix Boost source code download failure ### Motivation The main branch is broken because Boost is downloaded from JFrog, which now requires log in. ### Modifications Switch to Github to download Boost. Since GitHub only has the Boost release since 1.81.0, upgrade the version to 1.83.0, which is also consistent with vcpkg.json. * Remove unused option --- dependencies.yaml | 2 +- pkg/apk/Dockerfile | 9 ++++----- pkg/deb/Dockerfile | 9 ++++----- pkg/mac/build-static-library.sh | 11 +++++++---- pkg/rpm/Dockerfile | 9 ++++----- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 18be4383..a02ebdca 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -17,7 +17,7 @@ # under the License. # -boost : 1.80.0 +boost : 1.83.0 cmake: 3.24.2 protobuf: 3.20.0 zlib: 1.2.12 diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index 182c7d78..3cfe6583 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -43,13 +43,12 @@ ADD .build/dep-version.py /usr/local/bin # Download and compile boost RUN BOOST_VERSION=$(dep-version.py boost) && \ - BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ - curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ - tar xfz boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ - cd boost_${BOOST_VERSION_UNDESRSCORE} && \ + curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \ + tar zxf boost-${BOOST_VERSION}.tar.gz && \ + cd boost-${BOOST_VERSION} && \ ./bootstrap.sh --with-libraries=regex && \ ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ - rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} + rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION} # Download and compile protobuf RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ diff --git a/pkg/deb/Dockerfile b/pkg/deb/Dockerfile index aa22a805..7873cdf5 100644 --- a/pkg/deb/Dockerfile +++ b/pkg/deb/Dockerfile @@ -41,13 +41,12 @@ ADD .build/dep-version.py /usr/local/bin # Download and compile boost RUN BOOST_VERSION=$(dep-version.py boost) && \ - BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ - curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ - tar xfz boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ - cd boost_${BOOST_VERSION_UNDESRSCORE} && \ + curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \ + tar zxf boost-${BOOST_VERSION}.tar.gz && \ + cd boost-${BOOST_VERSION} && \ ./bootstrap.sh --with-libraries=regex && \ ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ - rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} + rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION} RUN CMAKE_VERSION=$(dep-version.py cmake) && \ curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \ diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index f0253fba..d49e199e 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -45,13 +45,16 @@ ZSTD_VERSION=$(./dep-version.py zstd) SNAPPY_VERSION=$(./dep-version.py snappy) CURL_VERSION=$(./dep-version.py curl) -BOOST_VERSION_=${BOOST_VERSION//./_} if [ ! -f boost/.done ]; then echo "Building Boost $BOOST_VERSION" - curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz - tar zxf boost_${BOOST_VERSION_}.tar.gz + curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz + tar zxf boost-${BOOST_VERSION}.tar.gz mkdir -p $PREFIX/include - cp -rf boost_${BOOST_VERSION_}/boost $PREFIX/include/ + pushd boost-${BOOST_VERSION} + ./bootstrap.sh + ./b2 headers + cp -rf boost $PREFIX/include/ + popd mkdir -p boost touch boost/.done else diff --git a/pkg/rpm/Dockerfile b/pkg/rpm/Dockerfile index 25e56ec8..b82840fa 100644 --- a/pkg/rpm/Dockerfile +++ b/pkg/rpm/Dockerfile @@ -42,13 +42,12 @@ ADD .build/dep-version.py /usr/local/bin # GCC 4.8.2 implementation of std::regex is buggy, so we install boost::regex here RUN BOOST_VERSION=$(dep-version.py boost) && \ echo "BOOST VERSION: '${BOOST_VERSION}'" && \ - BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ - curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ - tar xfz boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ - cd boost_${BOOST_VERSION_UNDESRSCORE} && \ + curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \ + tar zxf boost-${BOOST_VERSION}.tar.gz && \ + cd boost-${BOOST_VERSION} && \ ./bootstrap.sh --with-libraries=regex && \ ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ - rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} + rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION} RUN CMAKE_VERSION=$(dep-version.py cmake) && \ curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \ From a1cf401ec0dd423871537559050396a2b001065d Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Mon, 15 Jan 2024 15:11:23 +0800 Subject: [PATCH 26/54] Fix `StartMessageIdInclusive` not work when reader reads from latest msg id (#386) Fixes #385 ### Motivation The reader with `StartMessageIdInclusive` enabled should be able to reads messages from the latest message ID. ### Modifications - If `StartMessageIdInclusive` is enabled, the reader will seek and read the latest message in the topic. --- lib/ConsumerImpl.cc | 30 +++++++++++++++++++++++------- tests/ReaderTest.cc | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index 52162189..dbd3b659 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -1508,18 +1508,34 @@ void ConsumerImpl::hasMessageAvailableAsync(HasMessageAvailableCallback callback if (messageId == MessageId::latest()) { lock.unlock(); - getLastMessageIdAsync([callback](Result result, const GetLastMessageIdResponse& response) { + auto self = get_shared_this_ptr(); + getLastMessageIdAsync([self, callback](Result result, const GetLastMessageIdResponse& response) { if (result != ResultOk) { callback(result, {}); return; } - if (response.hasMarkDeletePosition() && response.getLastMessageId().entryId() >= 0) { - // We only care about comparing ledger ids and entry ids as mark delete position doesn't have - // other ids such as batch index - callback(ResultOk, compareLedgerAndEntryId(response.getMarkDeletePosition(), - response.getLastMessageId()) < 0); + auto handleResponse = [self, response, callback] { + if (response.hasMarkDeletePosition() && response.getLastMessageId().entryId() >= 0) { + // We only care about comparing ledger ids and entry ids as mark delete position doesn't + // have other ids such as batch index + auto compareResult = compareLedgerAndEntryId(response.getMarkDeletePosition(), + response.getLastMessageId()); + callback(ResultOk, self->config_.isStartMessageIdInclusive() ? compareResult <= 0 + : compareResult < 0); + } else { + callback(ResultOk, false); + } + }; + if (self->config_.isStartMessageIdInclusive()) { + self->seekAsync(response.getLastMessageId(), [callback, handleResponse](Result result) { + if (result != ResultOk) { + callback(result, {}); + return; + } + handleResponse(); + }); } else { - callback(ResultOk, false); + handleResponse(); } }); } else { diff --git a/tests/ReaderTest.cc b/tests/ReaderTest.cc index ac2fa234..723972d4 100644 --- a/tests/ReaderTest.cc +++ b/tests/ReaderTest.cc @@ -752,4 +752,36 @@ TEST(ReaderSeekTest, testSeekForMessageId) { producer.close(); } +TEST(ReaderSeekTest, testStartAtLatestMessageId) { + Client client(serviceUrl); + + const std::string topic = "test-seek-latest-message-id-" + std::to_string(time(nullptr)); + + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer(topic, producer)); + + MessageId id; + ASSERT_EQ(ResultOk, producer.send(MessageBuilder().setContent("msg").build(), id)); + + Reader readerExclusive; + ASSERT_EQ(ResultOk, + client.createReader(topic, MessageId::latest(), ReaderConfiguration(), readerExclusive)); + + Reader readerInclusive; + ASSERT_EQ(ResultOk, + client.createReader(topic, MessageId::latest(), + ReaderConfiguration().setStartMessageIdInclusive(true), readerInclusive)); + + Message msg; + bool hasMsgAvaliable = false; + readerInclusive.hasMessageAvailable(hasMsgAvaliable); + ASSERT_TRUE(hasMsgAvaliable); + ASSERT_EQ(ResultOk, readerInclusive.readNext(msg, 3000)); + ASSERT_EQ(ResultTimeout, readerExclusive.readNext(msg, 3000)); + + readerExclusive.close(); + readerInclusive.close(); + producer.close(); +} + INSTANTIATE_TEST_SUITE_P(Pulsar, ReaderTest, ::testing::Values(true, false)); From 6eb228e117b52a95c144c95d5694860ac98446b9 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 19 Jan 2024 09:57:25 +0800 Subject: [PATCH 27/54] Fix wrong TimeUtils::currentTimeMillis() implementation (#387) ### Motivation `TimeUtils::currentTimeMillis()` is used to get the timestamp since 1970 but `high_resolution_clock` does not necessarily represent the system clock. For example, given the following code: ```c++ template inline int64_t getTimestamp() { using namespace std::chrono; return duration_cast(Clock::now().time_since_epoch()).count(); } int main() { std::cout << time(nullptr) << std::endl; std::cout << getTimestamp() << std::endl; std::cout << getTimestamp() << std::endl; } ``` The outputs could be: ``` 1705584279 1705584279351 38566832 ``` ### Modifications Use `std::system_clock` for `currentTimeMillis()`. Add `BackoffTest.testCurrentTimeMillis` to verify the result is similar to 1000 times of `time(nullptr)` (the timestamp in seconds from the C API). --- lib/TimeUtils.h | 3 ++- tests/BackoffTest.cc | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/TimeUtils.h b/lib/TimeUtils.h index 03f563c3..53f537a2 100644 --- a/lib/TimeUtils.h +++ b/lib/TimeUtils.h @@ -35,8 +35,9 @@ inline decltype(std::chrono::milliseconds(0).count()) toMillis(TimeDuration dura class PULSAR_PUBLIC TimeUtils { public: static ptime now() { return std::chrono::high_resolution_clock::now(); } + static int64_t currentTimeMillis() { - return std::chrono::duration_cast(now().time_since_epoch()).count(); + return toMillis(std::chrono::system_clock::now().time_since_epoch()); } }; diff --git a/tests/BackoffTest.cc b/tests/BackoffTest.cc index 5fe4f71a..9c30a639 100644 --- a/tests/BackoffTest.cc +++ b/tests/BackoffTest.cc @@ -17,12 +17,14 @@ * under the License. */ #include +#include #include #include "PulsarFriend.h" #include "lib/Backoff.h" #include "lib/ClientConnection.h" +#include "lib/TimeUtils.h" #include "lib/stats/ProducerStatsImpl.h" using namespace pulsar; @@ -43,6 +45,12 @@ static bool withinTenPercentAndDecrementTimer(Backoff& backoff, const unsigned i return (t1 >= t2 * 0.9 && t1 <= t2); } +TEST(BackoffTest, testCurrentTimeMillis) { + auto t1 = TimeUtils::currentTimeMillis(); + auto t2 = 1000L * time(nullptr); + ASSERT_TRUE(t1 - t2 < 1000L) << "t1: " << t1 << ", t2: " << t2; +} + TEST(BackoffTest, mandatoryStopTestNegativeTest) { Backoff backoff(milliseconds(100), seconds(60), milliseconds(1900)); ASSERT_EQ(toMillis(backoff.next()), 100); From 72b7311aeef32e28a28e926da686aaf948e8f948 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 19 Jan 2024 21:31:19 +0800 Subject: [PATCH 28/54] Fix the operation timeout is not honored for GetSchema requests (#383) --- lib/ClientConfiguration.cc | 7 +++-- lib/ClientConfigurationImpl.h | 4 ++- lib/ClientConnection.cc | 31 ++++++++++++++++++--- lib/ClientConnection.h | 8 +++++- lib/ClientImpl.cc | 7 ++++- lib/ClientImpl.h | 2 ++ lib/Future.h | 2 +- lib/RetryableLookupService.h | 12 +++++---- lib/RetryableOperation.h | 7 ++--- lib/RetryableOperationCache.h | 9 ++++--- tests/LookupServiceTest.cc | 40 +++++++++++++++++++++++++--- tests/PulsarWrapper.h | 31 +++++++++++++++++++++ tests/RetryableOperationCacheTest.cc | 9 ++++--- 13 files changed, 140 insertions(+), 29 deletions(-) create mode 100644 tests/PulsarWrapper.h diff --git a/lib/ClientConfiguration.cc b/lib/ClientConfiguration.cc index 6e7c7456..6a91ec1d 100644 --- a/lib/ClientConfiguration.cc +++ b/lib/ClientConfiguration.cc @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +#include #include #include "ClientConfigurationImpl.h" @@ -61,11 +62,13 @@ Authentication& ClientConfiguration::getAuth() const { return *impl_->authentica const AuthenticationPtr& ClientConfiguration::getAuthPtr() const { return impl_->authenticationPtr; } ClientConfiguration& ClientConfiguration::setOperationTimeoutSeconds(int timeout) { - impl_->operationTimeoutSeconds = timeout; + impl_->operationTimeout = std::chrono::seconds(timeout); return *this; } -int ClientConfiguration::getOperationTimeoutSeconds() const { return impl_->operationTimeoutSeconds; } +int ClientConfiguration::getOperationTimeoutSeconds() const { + return std::chrono::duration_cast(impl_->operationTimeout).count(); +} ClientConfiguration& ClientConfiguration::setIOThreads(int threads) { impl_->ioThreads = threads; diff --git a/lib/ClientConfigurationImpl.h b/lib/ClientConfigurationImpl.h index b62b97c6..08d4b6e4 100644 --- a/lib/ClientConfigurationImpl.h +++ b/lib/ClientConfigurationImpl.h @@ -21,6 +21,8 @@ #include +#include + namespace pulsar { struct ClientConfigurationImpl { @@ -28,7 +30,7 @@ struct ClientConfigurationImpl { uint64_t memoryLimit{0ull}; int ioThreads{1}; int connectionsPerBroker{1}; - int operationTimeoutSeconds{30}; + std::chrono::nanoseconds operationTimeout{30L * 1000 * 1000 * 1000}; int messageListenerThreads{1}; int concurrentLookupRequest{50000}; int maxLookupRedirects{20}; diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index 82ab4928..b2916bdb 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -24,6 +24,7 @@ #include #include "AsioDefines.h" +#include "ClientImpl.h" #include "Commands.h" #include "ConnectionPool.h" #include "ConsumerImpl.h" @@ -163,7 +164,7 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: const ClientConfiguration& clientConfiguration, const AuthenticationPtr& authentication, const std::string& clientVersion, ConnectionPool& pool, size_t poolIndex) - : operationsTimeout_(std::chrono::seconds(clientConfiguration.getOperationTimeoutSeconds())), + : operationsTimeout_(ClientImpl::getOperationTimeout(clientConfiguration)), authentication_(authentication), serverProtocolVersion_(proto::ProtocolVersion_MIN), executor_(executor), @@ -1278,6 +1279,7 @@ void ClientConnection::close(Result result, bool detach) { auto pendingConsumerStatsMap = std::move(pendingConsumerStatsMap_); auto pendingGetLastMessageIdRequests = std::move(pendingGetLastMessageIdRequests_); auto pendingGetNamespaceTopicsRequests = std::move(pendingGetNamespaceTopicsRequests_); + auto pendingGetSchemaRequests = std::move(pendingGetSchemaRequests_); numOfPendingLookupRequest_ = 0; @@ -1342,6 +1344,9 @@ void ClientConnection::close(Result result, bool detach) { for (auto& kv : pendingGetNamespaceTopicsRequests) { kv.second.setFailed(result); } + for (auto& kv : pendingGetSchemaRequests) { + kv.second.promise.setFailed(result); + } } bool ClientConnection::isClosed() const { return state_ == Disconnected; } @@ -1430,6 +1435,7 @@ Future ClientConnection::newGetTopicsOfNamespace( Future ClientConnection::newGetSchema(const std::string& topicName, const std::string& version, uint64_t requestId) { Lock lock(mutex_); + Promise promise; if (isClosed()) { lock.unlock(); @@ -1438,8 +1444,27 @@ Future ClientConnection::newGetSchema(const std::string& top return promise.getFuture(); } - pendingGetSchemaRequests_.insert(std::make_pair(requestId, promise)); + auto timer = executor_->createDeadlineTimer(); + pendingGetSchemaRequests_.emplace(requestId, GetSchemaRequest{promise, timer}); lock.unlock(); + + auto weakSelf = weak_from_this(); + timer->expires_from_now(operationsTimeout_); + timer->async_wait([this, weakSelf, requestId](const ASIO_ERROR& ec) { + auto self = weakSelf.lock(); + if (!self) { + return; + } + Lock lock(mutex_); + auto it = pendingGetSchemaRequests_.find(requestId); + if (it != pendingGetSchemaRequests_.end()) { + auto promise = std::move(it->second.promise); + pendingGetSchemaRequests_.erase(it); + lock.unlock(); + promise.setFailed(ResultTimeout); + } + }); + sendCommand(Commands::newGetSchema(topicName, version, requestId)); return promise.getFuture(); } @@ -1867,7 +1892,7 @@ void ClientConnection::handleGetSchemaResponse(const proto::CommandGetSchemaResp Lock lock(mutex_); auto it = pendingGetSchemaRequests_.find(response.request_id()); if (it != pendingGetSchemaRequests_.end()) { - Promise getSchemaPromise = it->second; + Promise getSchemaPromise = it->second.promise; pendingGetSchemaRequests_.erase(it); lock.unlock(); diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index 69155fdd..1d44f058 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -42,6 +42,7 @@ #include #include #include +#include #include #include "AsioTimer.h" @@ -224,6 +225,11 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this promise; + DeadlineTimerPtr timer; + }; + /* * handler for connectAsync * creates a ConnectionPtr which has a valid ClientConnection object @@ -363,7 +369,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this> PendingGetNamespaceTopicsMap; PendingGetNamespaceTopicsMap pendingGetNamespaceTopicsRequests_; - typedef std::map> PendingGetSchemaMap; + typedef std::unordered_map PendingGetSchemaMap; PendingGetSchemaMap pendingGetSchemaRequests_; mutable std::mutex mutex_; diff --git a/lib/ClientImpl.cc b/lib/ClientImpl.cc index 2cfbd387..76d4389e 100644 --- a/lib/ClientImpl.cc +++ b/lib/ClientImpl.cc @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -109,7 +110,7 @@ ClientImpl::ClientImpl(const std::string& serviceUrl, const ClientConfiguration& } lookupServicePtr_ = RetryableLookupService::create( - underlyingLookupServicePtr, clientConfiguration_.getOperationTimeoutSeconds(), ioExecutorProvider_); + underlyingLookupServicePtr, clientConfiguration_.impl_->operationTimeout, ioExecutorProvider_); } ClientImpl::~ClientImpl() { shutdown(); } @@ -768,4 +769,8 @@ std::string ClientImpl::getClientVersion(const ClientConfiguration& clientConfig return oss.str(); } +std::chrono::nanoseconds ClientImpl::getOperationTimeout(const ClientConfiguration& clientConfiguration) { + return clientConfiguration.impl_->operationTimeout; +} + } /* namespace pulsar */ diff --git a/lib/ClientImpl.h b/lib/ClientImpl.h index f7b5a891..762aa60f 100644 --- a/lib/ClientImpl.h +++ b/lib/ClientImpl.h @@ -125,6 +125,8 @@ class ClientImpl : public std::enable_shared_from_this { ConnectionPool& getConnectionPool() noexcept { return pool_; } + static std::chrono::nanoseconds getOperationTimeout(const ClientConfiguration& clientConfiguration); + friend class PulsarFriend; private: diff --git a/lib/Future.h b/lib/Future.h index 290ebc6f..69db74a3 100644 --- a/lib/Future.h +++ b/lib/Future.h @@ -141,7 +141,7 @@ class Promise { Future getFuture() const { return Future{state_}; } private: - const InternalStatePtr state_; + InternalStatePtr state_; }; } // namespace pulsar diff --git a/lib/RetryableLookupService.h b/lib/RetryableLookupService.h index b8e3e0d7..561855f9 100644 --- a/lib/RetryableLookupService.h +++ b/lib/RetryableLookupService.h @@ -18,6 +18,8 @@ */ #pragma once +#include + #include "LookupDataResult.h" #include "LookupService.h" #include "NamespaceName.h" @@ -81,15 +83,15 @@ class RetryableLookupService : public LookupService { RetryableOperationCachePtr namespaceLookupCache_; RetryableOperationCachePtr getSchemaCache_; - RetryableLookupService(std::shared_ptr lookupService, int timeoutSeconds, + RetryableLookupService(std::shared_ptr lookupService, TimeDuration timeout, ExecutorServiceProviderPtr executorProvider) : lookupService_(lookupService), - lookupCache_(RetryableOperationCache::create(executorProvider, timeoutSeconds)), + lookupCache_(RetryableOperationCache::create(executorProvider, timeout)), partitionLookupCache_( - RetryableOperationCache::create(executorProvider, timeoutSeconds)), + RetryableOperationCache::create(executorProvider, timeout)), namespaceLookupCache_( - RetryableOperationCache::create(executorProvider, timeoutSeconds)), - getSchemaCache_(RetryableOperationCache::create(executorProvider, timeoutSeconds)) {} + RetryableOperationCache::create(executorProvider, timeout)), + getSchemaCache_(RetryableOperationCache::create(executorProvider, timeout)) {} }; } // namespace pulsar diff --git a/lib/RetryableOperation.h b/lib/RetryableOperation.h index 9c920da1..dba190f4 100644 --- a/lib/RetryableOperation.h +++ b/lib/RetryableOperation.h @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -40,11 +41,11 @@ class RetryableOperation : public std::enable_shared_from_this()>&& func, int timeoutSeconds, - DeadlineTimerPtr timer) + RetryableOperation(const std::string& name, std::function()>&& func, + TimeDuration timeout, DeadlineTimerPtr timer) : name_(name), func_(std::move(func)), - timeout_(std::chrono::seconds(timeoutSeconds)), + timeout_(timeout), backoff_(std::chrono::milliseconds(100), timeout_ + timeout_, std::chrono::milliseconds(0)), timer_(timer) {} diff --git a/lib/RetryableOperationCache.h b/lib/RetryableOperationCache.h index 70fa9140..e42460dd 100644 --- a/lib/RetryableOperationCache.h +++ b/lib/RetryableOperationCache.h @@ -18,6 +18,7 @@ */ #pragma once +#include #include #include @@ -40,8 +41,8 @@ class RetryableOperationCache : public std::enable_shared_from_this; @@ -69,7 +70,7 @@ class RetryableOperationCache : public std::enable_shared_from_this::create(key, std::move(func), timeoutSeconds_, timer); + auto operation = RetryableOperation::create(key, std::move(func), timeout_, timer); auto future = operation->run(); operations_[key] = operation; lock.unlock(); @@ -106,7 +107,7 @@ class RetryableOperationCache : public std::enable_shared_from_this>> operations_; mutable std::mutex mutex_; diff --git a/tests/LookupServiceTest.cc b/tests/LookupServiceTest.cc index 3457bd4d..0fe13851 100644 --- a/tests/LookupServiceTest.cc +++ b/tests/LookupServiceTest.cc @@ -22,11 +22,14 @@ #include #include +#include #include +#include #include #include "HttpHelper.h" #include "PulsarFriend.h" +#include "PulsarWrapper.h" #include "lib/BinaryProtoLookupService.h" #include "lib/ClientConnection.h" #include "lib/ConnectionPool.h" @@ -48,7 +51,10 @@ namespace pulsar { class LookupServiceTest : public ::testing::TestWithParam { public: - void SetUp() override { client_ = Client{GetParam()}; } + void SetUp() override { + serviceUrl_ = GetParam(); + client_ = Client{serviceUrl_}; + } void TearDown() override { client_.close(); } template @@ -63,6 +69,7 @@ class LookupServiceTest : public ::testing::TestWithParam { } protected: + std::string serviceUrl_; Client client_{httpLookupUrl}; }; @@ -159,7 +166,7 @@ TEST(LookupServiceTest, testRetry) { ClientConfiguration conf; auto lookupService = RetryableLookupService::create( - std::make_shared(serviceNameResolver, pool, conf), 30 /* seconds */, + std::make_shared(serviceNameResolver, pool, conf), std::chrono::seconds(30), executorProvider); PulsarFriend::setServiceUrlIndex(serviceNameResolver, 0); @@ -194,8 +201,8 @@ TEST(LookupServiceTest, testTimeout) { constexpr int timeoutInSeconds = 2; auto lookupService = RetryableLookupService::create( - std::make_shared(serviceNameResolver, pool, conf), timeoutInSeconds, - executorProvider); + std::make_shared(serviceNameResolver, pool, conf), + std::chrono::seconds(timeoutInSeconds), executorProvider); auto topicNamePtr = TopicName::get("lookup-service-test-retry"); decltype(std::chrono::high_resolution_clock::now()) startTime; @@ -431,6 +438,31 @@ TEST_P(LookupServiceTest, testGetSchemaByVersion) { producer2.close(); } +TEST_P(LookupServiceTest, testGetSchemaTimeout) { + if (serviceUrl_.find("pulsar://") == std::string::npos) { + // HTTP request timeout can only be configured with seconds + return; + } + const auto topic = "lookup-service-test-get-schema-timeout"; + Producer producer; + ProducerConfiguration producerConf; + producerConf.setSchema(SchemaInfo(STRING, "", "")); + ASSERT_EQ(ResultOk, client_.createProducer(topic, producerConf, producer)); + ASSERT_EQ(ResultOk, producer.send(MessageBuilder().setContent("msg").build())); + client_.close(); + + ClientConfiguration conf; + PulsarWrapper::deref(conf).operationTimeout = std::chrono::nanoseconds(1); + client_ = Client{serviceUrl_, conf}; + auto promise = std::make_shared>(); + client_.getSchemaInfoAsync(topic, 0L, + [promise](Result result, const SchemaInfo&) { promise->set_value(result); }); + auto future = promise->get_future(); + ASSERT_EQ(future.wait_for(std::chrono::milliseconds(100)), std::future_status::ready); + ASSERT_EQ(future.get(), ResultTimeout); + client_.close(); +} + INSTANTIATE_TEST_SUITE_P(Pulsar, LookupServiceTest, ::testing::Values(binaryLookupUrl, httpLookupUrl)); class BinaryProtoLookupServiceRedirectTestHelper : public BinaryProtoLookupService { diff --git a/tests/PulsarWrapper.h b/tests/PulsarWrapper.h new file mode 100644 index 00000000..87626e1c --- /dev/null +++ b/tests/PulsarWrapper.h @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include "lib/ClientConfigurationImpl.h" +#include "pulsar/ClientConfiguration.h" + +namespace pulsar { + +class PulsarWrapper { + public: + static ClientConfigurationImpl& deref(ClientConfiguration conf) { return *conf.impl_; } +}; + +} // namespace pulsar diff --git a/tests/RetryableOperationCacheTest.cc b/tests/RetryableOperationCacheTest.cc index ea1eb695..2a6948e3 100644 --- a/tests/RetryableOperationCacheTest.cc +++ b/tests/RetryableOperationCacheTest.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include "lib/RetryableOperationCache.h" @@ -82,7 +83,7 @@ class RetryableOperationCacheTest : public ::testing::Test { using namespace pulsar; TEST_F(RetryableOperationCacheTest, testRetry) { - auto cache = RetryableOperationCache::create(provider_, 30 /* seconds */); + auto cache = RetryableOperationCache::create(provider_, std::chrono::seconds(30)); for (int i = 0; i < 10; i++) { futures_.emplace_back(cache->run("key-" + std::to_string(i), CountdownFunc{i * 100})); } @@ -94,7 +95,7 @@ TEST_F(RetryableOperationCacheTest, testRetry) { } TEST_F(RetryableOperationCacheTest, testCache) { - auto cache = RetryableOperationCache::create(provider_, 30 /* seconds */); + auto cache = RetryableOperationCache::create(provider_, std::chrono::seconds(30)); constexpr int numKeys = 5; for (int i = 0; i < 100; i++) { futures_.emplace_back(cache->run("key-" + std::to_string(i % numKeys), CountdownFunc{i * 100})); @@ -107,7 +108,7 @@ TEST_F(RetryableOperationCacheTest, testCache) { } TEST_F(RetryableOperationCacheTest, testTimeout) { - auto cache = RetryableOperationCache::create(provider_, 1 /* seconds */); + auto cache = RetryableOperationCache::create(provider_, std::chrono::seconds(1)); auto future = cache->run("key", CountdownFunc{0, 1000 /* retry count */}); try { wait(future); @@ -118,7 +119,7 @@ TEST_F(RetryableOperationCacheTest, testTimeout) { } TEST_F(RetryableOperationCacheTest, testClear) { - auto cache = RetryableOperationCache::create(provider_, 30 /* seconds */); + auto cache = RetryableOperationCache::create(provider_, std::chrono::seconds(30)); for (int i = 0; i < 10; i++) { futures_.emplace_back(cache->run("key-" + std::to_string(i), CountdownFunc{100})); } From e13bd6df7c3796400188900d96a833c1eefb7e41 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 24 Jan 2024 13:07:42 +0800 Subject: [PATCH 29/54] [docs] Add example for how to import the library via vcpkg (#379) --- README.md | 10 +++++++ vcpkg | 2 +- vcpkg-example/CMakeLists.txt | 18 +++++++++++++ vcpkg-example/README.md | 52 ++++++++++++++++++++++++++++++++++++ vcpkg-example/main.cc | 8 ++++++ vcpkg-example/vcpkg.json | 11 ++++++++ 6 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 vcpkg-example/CMakeLists.txt create mode 100644 vcpkg-example/README.md create mode 100644 vcpkg-example/main.cc create mode 100644 vcpkg-example/vcpkg.json diff --git a/README.md b/README.md index 03957ce0..f0c13f07 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,16 @@ For the supported Pulsar features, see [Client Feature Matrix](https://pulsar.ap For how to use APIs to publish and consume messages, see [examples](https://github.com/apache/pulsar-client-cpp/tree/main/examples). +## Import the library into your project + +### CMake with vcpkg integration + +Navigate to [vcpkg-example](./vcpkg-example) for how to import the `pulsar-client-cpp` into your project via vcpkg. + +### Download pre-built binaries + +For non-vcpkg projects, you can download pre-built binaries from the [official release page](https://pulsar.apache.org/download/#pulsar-c-client). + ## Generate the API documents Pulsar C++ client uses [doxygen](https://www.doxygen.nl) to build API documents. After installing `doxygen`, you only need to run `doxygen` to generate the API documents whose main page is under the `doxygen/html/index.html` path. diff --git a/vcpkg b/vcpkg index b051745c..97dd2672 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit b051745c68faa6f65c493371d564c4eb8af34dad +Subproject commit 97dd26728e3856ed1ab62ee74ee3a391d9c81d19 diff --git a/vcpkg-example/CMakeLists.txt b/vcpkg-example/CMakeLists.txt new file mode 100644 index 00000000..40eff936 --- /dev/null +++ b/vcpkg-example/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.15) + +if (NOT CMAKE_TOOLCHAIN_FILE) + set(CMAKE_TOOLCHAIN_FILE "${CMAKE_SOURCE_DIR}/../vcpkg/scripts/buildsystems/vcpkg.cmake") +endif () +if (VCPKG_TARGET_TRIPLET MATCHES ".*windows-static") + cmake_policy(SET CMP0091 NEW) + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") +endif () +project(PulsarDemo CXX) + +if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 11) +endif () + +find_package(unofficial-pulsar CONFIG) +add_executable(main main.cc) +target_link_libraries(main PRIVATE unofficial::pulsar::pulsar) diff --git a/vcpkg-example/README.md b/vcpkg-example/README.md new file mode 100644 index 00000000..15266d69 --- /dev/null +++ b/vcpkg-example/README.md @@ -0,0 +1,52 @@ +# vcpkg-example + +A simple example vcpkg project that imported pulsar-client-cpp 3.4.2 as the dependency. + +## How to build + +Before running the commands below, ensure the vcpkg has been installed. If you have already downloaded the submodule of this project, just run the following commands. If you want to specify an existing vcpkg installation directory (assuming it's `VCPKG_ROOT`), add the `-DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake` option to the 1st command. + +```bash +cmake -B build +cmake --build build +``` + +Then the `main` executable will be generated under `./build` for single-configuration generators or `build/Debug` for multi-configuration generators. + +## How to link release libraries + +By default, debug libraries are linked so that the executable can be debugged by debuggers. However, when used for production, release libraries should be linked for better performance. + +See [cmake-generators](https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html) for the concept of CMake Generator. + +### Single-configuration generator + +With single-configuration generators like Unix Makefiles on Linux and macOS, you need to specify the `CMAKE_BUILD_TYPE`. + +```bash +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build +``` + +The `main` executable will still be generated under the `./build` directory but release libraries are linked now. + +### Multi-configuration generator + +With multi-configuration generators like Visual Studio on Windows, you just need to specify the `--config` option. + +```bash +cmake -B build +cmake --build build --config Release +``` + +The `main` executable that links release libraries will be generated under the `./build/Release` directory. + +## Link static libraries on Windows + +The default vcpkg triplet is `x64-windows` on Windows. If you changed the triplet for static libraries, like `x64-windows-static`, you need to add a line before the `project(PulsarDemo CXX)` in CMakeLists.txt. + +```cmake +set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") +``` + +See [CMAKE_MSVC_RUNTIME_LIBRARY](https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html) for details. diff --git a/vcpkg-example/main.cc b/vcpkg-example/main.cc new file mode 100644 index 00000000..f4784faa --- /dev/null +++ b/vcpkg-example/main.cc @@ -0,0 +1,8 @@ +#include +using namespace pulsar; + +int main(int argc, char *argv[]) { + Client client{"pulsar://localhost:6650"}; + client.close(); + return 0; +} diff --git a/vcpkg-example/vcpkg.json b/vcpkg-example/vcpkg.json new file mode 100644 index 00000000..e48c7217 --- /dev/null +++ b/vcpkg-example/vcpkg.json @@ -0,0 +1,11 @@ +{ + "name": "vcpkg-pulsar-demo", + "version-string": "0.1.0", + "builtin-baseline": "38d1652f152d36481f2f4e8a85c0f1e14f3769f7", + "dependencies": [ + { + "name": "pulsar-client-cpp", + "version>=": "3.4.2#1" + } + ] +} From d1dd08ba3d9c964506ead1a9e3cde0cccad4621b Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 24 Jan 2024 06:39:17 -0800 Subject: [PATCH 30/54] [feat] PIP-307 added assigned broker urls for CloseProudcer and CloseConsumer commands and handler logic (#389) Master Issue: #[pip-307](https://github.com/apache/pulsar/blob/master/pip/pip-307.md) ### Motivation As part of PIP-307, I am adding the client logic to this c++ client. ### Modifications - Added assigned broker urls to CloseProudcer and CloseConsumer commands. - Updated the client reconnection logic to directly connect to the assigned broker urls ### Verifying this change - Added ExtensibleLoadManagerTest to cover this logic --- lib/BinaryProtoLookupService.cc | 4 +- lib/ClientConnection.cc | 30 ++- lib/ClientConnection.h | 4 + lib/ClientImpl.cc | 35 +++- lib/ClientImpl.h | 7 + lib/ConsumerImpl.cc | 9 +- lib/ConsumerImpl.h | 1 + lib/HandlerBase.cc | 29 ++- lib/HandlerBase.h | 17 +- lib/LookupService.h | 1 + lib/ProducerImpl.cc | 9 +- lib/ProducerImpl.h | 1 + proto/PulsarApi.proto | 4 + run-unit-tests.sh | 8 + tests/BuildTests.cmake | 3 + tests/LegacyBuildTests.cmake | 3 + .../extensibleLM/ExtensibleLoadManagerTest.cc | 198 ++++++++++++++++++ tests/extensibleLM/docker-compose.yml | 150 +++++++++++++ 18 files changed, 492 insertions(+), 21 deletions(-) create mode 100644 tests/extensibleLM/ExtensibleLoadManagerTest.cc create mode 100644 tests/extensibleLM/docker-compose.yml diff --git a/lib/BinaryProtoLookupService.cc b/lib/BinaryProtoLookupService.cc index 87f02ea4..2d9ffc42 100644 --- a/lib/BinaryProtoLookupService.cc +++ b/lib/BinaryProtoLookupService.cc @@ -87,9 +87,9 @@ auto BinaryProtoLookupService::findBroker(const std::string& address, bool autho << ", from " << cnx->cnxString()); if (data->shouldProxyThroughServiceUrl()) { // logicalAddress is the proxy's address, we should still connect through proxy - promise->setValue({responseBrokerAddress, address}); + promise->setValue({responseBrokerAddress, address, true}); } else { - promise->setValue({responseBrokerAddress, responseBrokerAddress}); + promise->setValue({responseBrokerAddress, responseBrokerAddress, false}); } } }); diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index b2916bdb..0641809e 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -1762,6 +1762,30 @@ void ClientConnection::handleError(const proto::CommandError& error) { } } +boost::optional ClientConnection::getAssignedBrokerServiceUrl( + const proto::CommandCloseProducer& closeProducer) { + if (tlsSocket_) { + if (closeProducer.has_assignedbrokerserviceurltls()) { + return closeProducer.assignedbrokerserviceurltls(); + } + } else if (closeProducer.has_assignedbrokerserviceurl()) { + return closeProducer.assignedbrokerserviceurl(); + } + return boost::none; +} + +boost::optional ClientConnection::getAssignedBrokerServiceUrl( + const proto::CommandCloseConsumer& closeConsumer) { + if (tlsSocket_) { + if (closeConsumer.has_assignedbrokerserviceurltls()) { + return closeConsumer.assignedbrokerserviceurltls(); + } + } else if (closeConsumer.has_assignedbrokerserviceurl()) { + return closeConsumer.assignedbrokerserviceurl(); + } + return boost::none; +} + void ClientConnection::handleCloseProducer(const proto::CommandCloseProducer& closeProducer) { int producerId = closeProducer.producer_id(); @@ -1775,7 +1799,8 @@ void ClientConnection::handleCloseProducer(const proto::CommandCloseProducer& cl lock.unlock(); if (producer) { - producer->disconnectProducer(); + auto assignedBrokerServiceUrl = getAssignedBrokerServiceUrl(closeProducer); + producer->disconnectProducer(assignedBrokerServiceUrl); } } else { LOG_ERROR(cnxString_ << "Got invalid producer Id in closeProducer command: " << producerId); @@ -1795,7 +1820,8 @@ void ClientConnection::handleCloseConsumer(const proto::CommandCloseConsumer& cl lock.unlock(); if (consumer) { - consumer->disconnectConsumer(); + auto assignedBrokerServiceUrl = getAssignedBrokerServiceUrl(closeconsumer); + consumer->disconnectConsumer(assignedBrokerServiceUrl); } } else { LOG_ERROR(cnxString_ << "Got invalid consumer Id in closeConsumer command: " << consumerId); diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index 1d44f058..851ec0c3 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -421,6 +421,10 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this getAssignedBrokerServiceUrl( + const proto::CommandCloseProducer& closeProducer); + boost::optional getAssignedBrokerServiceUrl( + const proto::CommandCloseConsumer& closeConsumer); }; } // namespace pulsar diff --git a/lib/ClientImpl.cc b/lib/ClientImpl.cc index 76d4389e..63c85b0f 100644 --- a/lib/ClientImpl.cc +++ b/lib/ClientImpl.cc @@ -91,7 +91,9 @@ ClientImpl::ClientImpl(const std::string& serviceUrl, const ClientConfiguration& ClientImpl::getClientVersion(clientConfiguration)), producerIdGenerator_(0), consumerIdGenerator_(0), - closingError(ResultOk) { + closingError(ResultOk), + useProxy_(false), + lookupCount_(0L) { std::unique_ptr loggerFactory = clientConfiguration_.impl_->takeLogger(); if (loggerFactory) { LogUtils::setLoggerFactory(std::move(loggerFactory)); @@ -532,6 +534,8 @@ Future ClientImpl::getConnection(const std::string& promise.setFailed(result); return; } + useProxy_ = data.proxyThroughServiceUrl; + lookupCount_++; pool_.getConnectionAsync(data.logicalAddress, data.physicalAddress, key) .addListener([promise](Result result, const ClientConnectionWeakPtr& weakCnx) { if (result == ResultOk) { @@ -550,6 +554,33 @@ Future ClientImpl::getConnection(const std::string& return promise.getFuture(); } +const std::string& ClientImpl::getPhysicalAddress(const std::string& logicalAddress) { + if (useProxy_) { + return serviceNameResolver_.resolveHost(); + } else { + return logicalAddress; + } +} + +Future ClientImpl::connect(const std::string& logicalAddress, size_t key) { + const auto& physicalAddress = getPhysicalAddress(logicalAddress); + Promise promise; + pool_.getConnectionAsync(logicalAddress, physicalAddress, key) + .addListener([promise](Result result, const ClientConnectionWeakPtr& weakCnx) { + if (result == ResultOk) { + auto cnx = weakCnx.lock(); + if (cnx) { + promise.setValue(cnx); + } else { + promise.setFailed(ResultConnectError); + } + } else { + promise.setFailed(result); + } + }); + return promise.getFuture(); +} + void ClientImpl::handleGetPartitions(const Result result, const LookupDataResultPtr partitionMetadata, TopicNamePtr topicName, GetPartitionsCallback callback) { if (result != ResultOk) { @@ -635,6 +666,7 @@ void ClientImpl::closeAsync(CloseCallback callback) { if (*numberOfOpenHandlers == 0 && callback) { handleClose(ResultOk, numberOfOpenHandlers, callback); } + lookupCount_ = 0; } void ClientImpl::handleClose(Result result, SharedInt numberOfOpenHandlers, ResultCallback callback) { @@ -722,6 +754,7 @@ void ClientImpl::shutdown() { partitionListenerExecutorProvider_->close(timeoutProcessor.getLeftTimeout()); timeoutProcessor.tok(); LOG_DEBUG("partitionListenerExecutorProvider_ is closed"); + lookupCount_ = 0; } uint64_t ClientImpl::newProducerId() { diff --git a/lib/ClientImpl.h b/lib/ClientImpl.h index 762aa60f..a2649a6a 100644 --- a/lib/ClientImpl.h +++ b/lib/ClientImpl.h @@ -97,6 +97,8 @@ class ClientImpl : public std::enable_shared_from_this { Future getConnection(const std::string& topic, size_t key); + Future connect(const std::string& logicalAddress, size_t key); + void closeAsync(CloseCallback callback); void shutdown(); @@ -124,6 +126,7 @@ class ClientImpl : public std::enable_shared_from_this { std::shared_ptr> getRequestIdGenerator() const { return requestIdGenerator_; } ConnectionPool& getConnectionPool() noexcept { return pool_; } + uint64_t getLookupCount() { return lookupCount_; } static std::chrono::nanoseconds getOperationTimeout(const ClientConfiguration& clientConfiguration); @@ -160,6 +163,8 @@ class ClientImpl : public std::enable_shared_from_this { const std::string& consumerName, const ConsumerConfiguration& conf, SubscribeCallback callback); + const std::string& getPhysicalAddress(const std::string& logicalAddress); + static std::string getClientVersion(const ClientConfiguration& clientConfiguration); enum State @@ -191,6 +196,8 @@ class ClientImpl : public std::enable_shared_from_this { SynchronizedHashMap consumers_; std::atomic closingError; + std::atomic useProxy_; + std::atomic lookupCount_; friend class Client; }; diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index dbd3b659..86dddb0d 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -1234,10 +1234,13 @@ void ConsumerImpl::negativeAcknowledge(const MessageId& messageId) { negativeAcksTracker_->add(messageId); } -void ConsumerImpl::disconnectConsumer() { - LOG_INFO("Broker notification of Closed consumer: " << consumerId_); +void ConsumerImpl::disconnectConsumer() { disconnectConsumer(boost::none); } + +void ConsumerImpl::disconnectConsumer(const boost::optional& assignedBrokerUrl) { + LOG_INFO("Broker notification of Closed consumer: " + << consumerId_ << (assignedBrokerUrl ? (" assignedBrokerUrl: " + assignedBrokerUrl.get()) : "")); resetCnx(); - scheduleReconnection(); + scheduleReconnection(assignedBrokerUrl); } void ConsumerImpl::closeAsync(ResultCallback originalCallback) { diff --git a/lib/ConsumerImpl.h b/lib/ConsumerImpl.h index 0f801f63..5612091e 100644 --- a/lib/ConsumerImpl.h +++ b/lib/ConsumerImpl.h @@ -131,6 +131,7 @@ class ConsumerImpl : public ConsumerImplBase { void hasMessageAvailableAsync(HasMessageAvailableCallback callback) override; virtual void disconnectConsumer(); + virtual void disconnectConsumer(const boost::optional& assignedBrokerUrl); Result fetchSingleMessageFromBroker(Message& msg); virtual bool isCumulativeAcknowledgementAllowed(ConsumerType consumerType); diff --git a/lib/HandlerBase.cc b/lib/HandlerBase.cc index fa21a57d..c5327fcf 100644 --- a/lib/HandlerBase.cc +++ b/lib/HandlerBase.cc @@ -68,7 +68,18 @@ void HandlerBase::setCnx(const ClientConnectionPtr& cnx) { connection_ = cnx; } -void HandlerBase::grabCnx() { +void HandlerBase::grabCnx() { grabCnx(boost::none); } + +Future HandlerBase::getConnection( + const ClientImplPtr& client, const boost::optional& assignedBrokerUrl) { + if (assignedBrokerUrl && client->getLookupCount() > 0) { + return client->connect(assignedBrokerUrl.get(), connectionKeySuffix_); + } else { + return client->getConnection(topic(), connectionKeySuffix_); + } +} + +void HandlerBase::grabCnx(const boost::optional& assignedBrokerUrl) { bool expectedState = false; if (!reconnectionPending_.compare_exchange_strong(expectedState, true)) { LOG_INFO(getName() << "Ignoring reconnection attempt since there's already a pending reconnection"); @@ -90,7 +101,7 @@ void HandlerBase::grabCnx() { return; } auto self = shared_from_this(); - auto cnxFuture = client->getConnection(topic(), connectionKeySuffix_); + auto cnxFuture = getConnection(client, assignedBrokerUrl); cnxFuture.addListener([this, self](Result result, const ClientConnectionPtr& cnx) { if (result == ResultOk) { LOG_DEBUG(getName() << "Connected to broker: " << cnx->cnxString()); @@ -141,12 +152,12 @@ void HandlerBase::handleDisconnection(Result result, const ClientConnectionPtr& break; } } - -void HandlerBase::scheduleReconnection() { +void HandlerBase::scheduleReconnection() { scheduleReconnection(boost::none); } +void HandlerBase::scheduleReconnection(const boost::optional& assignedBrokerUrl) { const auto state = state_.load(); if (state == Pending || state == Ready) { - TimeDuration delay = backoff_.next(); + TimeDuration delay = assignedBrokerUrl ? std::chrono::milliseconds(0) : backoff_.next(); LOG_INFO(getName() << "Schedule reconnection in " << (toMillis(delay) / 1000.0) << " s"); timer_->expires_from_now(delay); @@ -154,10 +165,10 @@ void HandlerBase::scheduleReconnection() { // so we will not run into the case where grabCnx is invoked on out of scope handler auto name = getName(); std::weak_ptr weakSelf{shared_from_this()}; - timer_->async_wait([name, weakSelf](const ASIO_ERROR& ec) { + timer_->async_wait([name, weakSelf, assignedBrokerUrl](const ASIO_ERROR& ec) { auto self = weakSelf.lock(); if (self) { - self->handleTimeout(ec); + self->handleTimeout(ec, assignedBrokerUrl); } else { LOG_WARN(name << "Cancel the reconnection since the handler is destroyed"); } @@ -165,13 +176,13 @@ void HandlerBase::scheduleReconnection() { } } -void HandlerBase::handleTimeout(const ASIO_ERROR& ec) { +void HandlerBase::handleTimeout(const ASIO_ERROR& ec, const boost::optional& assignedBrokerUrl) { if (ec) { LOG_DEBUG(getName() << "Ignoring timer cancelled event, code[" << ec << "]"); return; } else { epoch_++; - grabCnx(); + grabCnx(assignedBrokerUrl); } } diff --git a/lib/HandlerBase.h b/lib/HandlerBase.h index 68c0b6a6..b9d98553 100644 --- a/lib/HandlerBase.h +++ b/lib/HandlerBase.h @@ -20,6 +20,7 @@ #define _PULSAR_HANDLER_BASE_HEADER_ #include +#include #include #include #include @@ -53,11 +54,22 @@ class HandlerBase : public std::enable_shared_from_this { void resetCnx() { setCnx(nullptr); } protected: + /* + * tries reconnection and sets connection_ to valid object + * @param assignedBrokerUrl assigned broker url to directly connect to without lookup + */ + void grabCnx(const boost::optional& assignedBrokerUrl); + /* * tries reconnection and sets connection_ to valid object */ void grabCnx(); + /* + * Schedule reconnection after backoff time + * @param assignedBrokerUrl assigned broker url to directly connect to without lookup + */ + void scheduleReconnection(const boost::optional& assignedBrokerUrl); /* * Schedule reconnection after backoff time */ @@ -89,9 +101,12 @@ class HandlerBase : public std::enable_shared_from_this { private: const std::shared_ptr topic_; + Future getConnection(const ClientImplPtr& client, + const boost::optional& assignedBrokerUrl); + void handleDisconnection(Result result, const ClientConnectionPtr& cnx); - void handleTimeout(const ASIO_ERROR& ec); + void handleTimeout(const ASIO_ERROR& ec, const boost::optional& assignedBrokerUrl); protected: ClientImplWeakPtr client_; diff --git a/lib/LookupService.h b/lib/LookupService.h index 35f47308..b50d1f82 100644 --- a/lib/LookupService.h +++ b/lib/LookupService.h @@ -42,6 +42,7 @@ class LookupService { struct LookupResult { std::string logicalAddress; std::string physicalAddress; + bool proxyThroughServiceUrl; friend std::ostream& operator<<(std::ostream& os, const LookupResult& lookupResult) { return os << "logical address: " << lookupResult.logicalAddress diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index fc39b231..2e5cd444 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -971,12 +971,15 @@ bool ProducerImpl::encryptMessage(proto::MessageMetadata& metadata, SharedBuffer encryptedPayload); } -void ProducerImpl::disconnectProducer() { - LOG_INFO("Broker notification of Closed producer: " << producerId_); +void ProducerImpl::disconnectProducer(const boost::optional& assignedBrokerUrl) { + LOG_INFO("Broker notification of Closed producer: " + << producerId_ << (assignedBrokerUrl ? (" assignedBrokerUrl: " + assignedBrokerUrl.get()) : "")); resetCnx(); - scheduleReconnection(); + scheduleReconnection(assignedBrokerUrl); } +void ProducerImpl::disconnectProducer() { disconnectProducer(boost::none); } + void ProducerImpl::start() { HandlerBase::start(); diff --git a/lib/ProducerImpl.h b/lib/ProducerImpl.h index 97816050..f650b1f9 100644 --- a/lib/ProducerImpl.h +++ b/lib/ProducerImpl.h @@ -98,6 +98,7 @@ class ProducerImpl : public HandlerBase, public ProducerImplBase { bool ackReceived(uint64_t sequenceId, MessageId& messageId); + virtual void disconnectProducer(const boost::optional& assignedBrokerUrl); virtual void disconnectProducer(); uint64_t getProducerId() const; diff --git a/proto/PulsarApi.proto b/proto/PulsarApi.proto index 7be65224..a2548f3a 100644 --- a/proto/PulsarApi.proto +++ b/proto/PulsarApi.proto @@ -623,11 +623,15 @@ message CommandReachedEndOfTopic { message CommandCloseProducer { required uint64 producer_id = 1; required uint64 request_id = 2; + optional string assignedBrokerServiceUrl = 3; + optional string assignedBrokerServiceUrlTls = 4; } message CommandCloseConsumer { required uint64 consumer_id = 1; required uint64 request_id = 2; + optional string assignedBrokerServiceUrl = 3; + optional string assignedBrokerServiceUrlTls = 4; } message CommandRedeliverUnacknowledgedMessages { diff --git a/run-unit-tests.sh b/run-unit-tests.sh index 898cfed0..226789c5 100755 --- a/run-unit-tests.sh +++ b/run-unit-tests.sh @@ -30,6 +30,14 @@ fi export http_proxy= export https_proxy= + +# Run ExtensibleLoadManager tests +docker compose -f tests/extensibleLM/docker-compose.yml up -d +until curl http://localhost:8080/metrics > /dev/null 2>&1 ; do sleep 1; done +sleep 5 +$CMAKE_BUILD_DIRECTORY/tests/ExtensibleLoadManagerTest +docker compose -f tests/extensibleLM/docker-compose.yml down + # Run OAuth2 tests docker compose -f tests/oauth2/docker-compose.yml up -d # Wait until the namespace is created, currently there is no good way to check it diff --git a/tests/BuildTests.cmake b/tests/BuildTests.cmake index c468f51f..0fe74300 100644 --- a/tests/BuildTests.cmake +++ b/tests/BuildTests.cmake @@ -55,3 +55,6 @@ target_link_libraries(Oauth2Test pulsarStatic ${GTEST_TARGETS}) add_executable(ChunkDedupTest chunkdedup/ChunkDedupTest.cc HttpHelper.cc) target_link_libraries(ChunkDedupTest pulsarStatic ${GTEST_TARGETS}) + +add_executable(ExtensibleLoadManagerTest extensibleLM/ExtensibleLoadManagerTest.cc HttpHelper.cc) +target_link_libraries(ExtensibleLoadManagerTest PRIVATE pulsarStatic ${GTEST_TARGETS}) diff --git a/tests/LegacyBuildTests.cmake b/tests/LegacyBuildTests.cmake index 1e5335f7..c6f4e96c 100644 --- a/tests/LegacyBuildTests.cmake +++ b/tests/LegacyBuildTests.cmake @@ -75,3 +75,6 @@ target_link_libraries(Oauth2Test ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PAT add_executable(ChunkDedupTest chunkdedup/ChunkDedupTest.cc HttpHelper.cc) target_link_libraries(ChunkDedupTest ${CLIENT_LIBS} pulsarStatic ${GTEST_LIBRARY_PATH}) +add_executable(ExtensibleLoadManagerTest extensibleLM/ExtensibleLoadManagerTest.cc HttpHelper.cc) +target_include_directories(ExtensibleLoadManagerTest PRIVATE ${AUTOGEN_DIR}/lib) +target_link_libraries(ExtensibleLoadManagerTest PRIVATE pulsarStatic ${GTEST_LIBRARY_PATH}) diff --git a/tests/extensibleLM/ExtensibleLoadManagerTest.cc b/tests/extensibleLM/ExtensibleLoadManagerTest.cc new file mode 100644 index 00000000..4472d9d9 --- /dev/null +++ b/tests/extensibleLM/ExtensibleLoadManagerTest.cc @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +// Run `docker-compose up -d` to set up the test environment for this test. +#include + +#include + +#include "include/pulsar/Client.h" +#include "lib/LogUtils.h" +#include "lib/Semaphore.h" +#include "tests/HttpHelper.h" +#include "tests/PulsarFriend.h" + +DECLARE_LOG_OBJECT() + +using namespace pulsar; + +bool checkTime() { + const static auto start = std::chrono::high_resolution_clock::now(); + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start).count(); + return duration < 180 * 1000; +} + +TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { + const static std::string adminUrl = "http://localhost:8080/"; + const static std::string topicName = + "persistent://public/unload-test/topic-1" + std::to_string(time(NULL)); + + ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { + std::string url = adminUrl + "admin/v2/namespaces/public/unload-test?bundles=1"; + int res = makePutRequest(url, ""); + return res == 204 || res == 409; + })); + + Client client{"pulsar://localhost:6650"}; + Producer producer; + ProducerConfiguration producerConfiguration; + Result producerResult = client.createProducer(topicName, producerConfiguration, producer); + ASSERT_EQ(producerResult, ResultOk); + Consumer consumer; + Result consumerResult = client.subscribe(topicName, "sub", consumer); + ASSERT_EQ(consumerResult, ResultOk); + + Semaphore firstUnloadSemaphore(0); + Semaphore secondUnloadSemaphore(0); + Semaphore halfPubWaitSemaphore(0); + const int msgCount = 10; + int produced = 0; + auto produce = [&]() { + int i = 0; + while (i < msgCount && checkTime()) { + if (i == 3) { + firstUnloadSemaphore.acquire(); + } + + if (i == 5) { + halfPubWaitSemaphore.release(); + } + + if (i == 8) { + secondUnloadSemaphore.acquire(); + } + + std::string content = std::to_string(i); + const auto msg = MessageBuilder().setContent(content).build(); + + ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { + Result sendResult = producer.send(msg); + return sendResult == ResultOk; + })); + + LOG_INFO("produced index:" << i); + produced++; + i++; + } + LOG_INFO("producer finished"); + }; + + int consumed = 0; + auto consume = [&]() { + Message receivedMsg; + int i = 0; + while (i < msgCount && checkTime()) { + ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { + Result receiveResult = + consumer.receive(receivedMsg, 1000); // Assumed that we wait 1000 ms for each message + return receiveResult == ResultOk; + })); + LOG_INFO("received index:" << i); + + int id = std::stoi(receivedMsg.getDataAsString()); + if (id < i) { + continue; + } + ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { + Result ackResult = consumer.acknowledge(receivedMsg); + return ackResult == ResultOk; + })); + LOG_INFO("acked index:" << i); + + consumed++; + i++; + } + LOG_INFO("consumer finished"); + }; + + std::thread produceThread(produce); + std::thread consumeThread(consume); + + auto unload = [&] { + auto clientImplPtr = PulsarFriend::getClientImplPtr(client); + auto &consumerImpl = PulsarFriend::getConsumerImpl(consumer); + auto &producerImpl = PulsarFriend::getProducerImpl(producer); + uint64_t lookupCountBeforeUnload; + std::string destinationBroker; + while (checkTime()) { + // make sure producers and consumers are ready + ASSERT_TRUE(waitUntil(std::chrono::seconds(30), + [&] { return consumerImpl.isConnected() && producerImpl.isConnected(); })); + + std::string url = adminUrl + "lookup/v2/topic/persistent/public/unload-test/topic-1"; + std::string responseDataBeforeUnload; + int res = makeGetRequest(url, responseDataBeforeUnload); + if (res != 200) { + continue; + } + destinationBroker = responseDataBeforeUnload.find("broker-2") == std::string::npos + ? "broker-2:8080" + : "broker-1:8080"; + lookupCountBeforeUnload = clientImplPtr->getLookupCount(); + ASSERT_TRUE(lookupCountBeforeUnload > 0); + + url = adminUrl + + "admin/v2/namespaces/public/unload-test/0x00000000_0xffffffff/unload?destinationBroker=" + + destinationBroker; + LOG_INFO("before lookup responseData:" << responseDataBeforeUnload << ",unload url:" << url + << ",lookupCountBeforeUnload:" << lookupCountBeforeUnload); + res = makePutRequest(url, ""); + LOG_INFO("unload res:" << res); + if (res != 204) { + continue; + } + + // make sure producers and consumers are ready + ASSERT_TRUE(waitUntil(std::chrono::seconds(30), + [&] { return consumerImpl.isConnected() && producerImpl.isConnected(); })); + std::string responseDataAfterUnload; + ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { + url = adminUrl + "lookup/v2/topic/persistent/public/unload-test/topic-1"; + res = makeGetRequest(url, responseDataAfterUnload); + return res == 200 && responseDataAfterUnload.find(destinationBroker) != std::string::npos; + })); + LOG_INFO("after lookup responseData:" << responseDataAfterUnload << ",res:" << res); + + // TODO: check lookup counter after pip-307 is released + auto lookupCountAfterUnload = clientImplPtr->getLookupCount(); + ASSERT_TRUE(lookupCountBeforeUnload < lookupCountAfterUnload); + break; + } + }; + LOG_INFO("starting first unload"); + unload(); + firstUnloadSemaphore.release(); + halfPubWaitSemaphore.acquire(); + LOG_INFO("starting second unload"); + unload(); + secondUnloadSemaphore.release(); + + produceThread.join(); + consumeThread.join(); + ASSERT_EQ(consumed, msgCount); + ASSERT_EQ(produced, msgCount); + ASSERT_TRUE(checkTime()) << "timed out"; + client.close(); +} + +int main(int argc, char *argv[]) { + ::testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} diff --git a/tests/extensibleLM/docker-compose.yml b/tests/extensibleLM/docker-compose.yml new file mode 100644 index 00000000..8d3c33a3 --- /dev/null +++ b/tests/extensibleLM/docker-compose.yml @@ -0,0 +1,150 @@ +version: '3' +networks: + pulsar: + driver: bridge +services: + # Start ZooKeeper + zookeeper: + image: apachepulsar/pulsar:latest + container_name: zookeeper + restart: on-failure + networks: + - pulsar + environment: + - metadataStoreUrl=zk:zookeeper:2181 + - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m + command: > + bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \ + bin/generate-zookeeper-config.sh conf/zookeeper.conf && \ + exec bin/pulsar zookeeper" + healthcheck: + test: ["CMD", "bin/pulsar-zookeeper-ruok.sh"] + interval: 10s + timeout: 5s + retries: 30 + + # Initialize cluster metadata + pulsar-init: + container_name: pulsar-init + hostname: pulsar-init + image: apachepulsar/pulsar:latest + networks: + - pulsar + environment: + - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m + command: > + bin/pulsar initialize-cluster-metadata \ + --cluster cluster-a \ + --zookeeper zookeeper:2181 \ + --configuration-store zookeeper:2181 \ + --web-service-url http://broker-1:8080 \ + --broker-service-url pulsar://broker-1:6650 + depends_on: + zookeeper: + condition: service_healthy + + # Start bookie + bookie: + image: apachepulsar/pulsar:latest + container_name: bookie + restart: on-failure + networks: + - pulsar + environment: + - clusterName=cluster-a + - zkServers=zookeeper:2181 + - metadataServiceUri=metadata-store:zk:zookeeper:2181 + - advertisedAddress=bookie + - BOOKIE_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m + depends_on: + zookeeper: + condition: service_healthy + pulsar-init: + condition: service_completed_successfully + command: bash -c "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + + proxy: + image: apachepulsar/pulsar:latest + container_name: proxy + hostname: proxy + restart: on-failure + networks: + - pulsar + environment: + - metadataStoreUrl=zk:zookeeper:2181 + - zookeeperServers=zookeeper:2181 + - clusterName=cluster-a + - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m + ports: + - "8080:8080" + - "6650:6650" + depends_on: + broker-1: + condition: service_started + broker-2: + condition: service_started + command: bash -c "bin/apply-config-from-env.py conf/proxy.conf && exec bin/pulsar proxy" + + # Start broker 1 + broker-1: + image: apachepulsar/pulsar:latest + container_name: broker-1 + hostname: broker-1 + restart: on-failure + networks: + - pulsar + environment: + - metadataStoreUrl=zk:zookeeper:2181 + - zookeeperServers=zookeeper:2181 + - clusterName=cluster-a + - managedLedgerDefaultEnsembleSize=1 + - managedLedgerDefaultWriteQuorum=1 + - managedLedgerDefaultAckQuorum=1 + - advertisedAddress=broker-1 + - internalListenerName=internal + - advertisedListeners=internal:pulsar://broker-1:6650 + - PULSAR_MEM=-Xms256m -Xmx256m -XX:MaxDirectMemorySize=56m + # Load Manager. Here uses the extensible load balancer, sets the unloading strategy to TransferShedder, and enables debug mode. + - loadManagerClassName=org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl + - loadBalancerLoadSheddingStrategy=org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder + - loadBalancerSheddingEnabled=false + - loadBalancerDebugModeEnabled=true + - PULSAR_PREFIX_defaultNumberOfNamespaceBundles=1 + depends_on: + zookeeper: + condition: service_healthy + bookie: + condition: service_started + command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" + + # Start broker 2 + broker-2: + image: apachepulsar/pulsar:latest + container_name: broker-2 + hostname: broker-2 + restart: on-failure + networks: + - pulsar + environment: + - metadataStoreUrl=zk:zookeeper:2181 + - zookeeperServers=zookeeper:2181 + - clusterName=cluster-a + - managedLedgerDefaultEnsembleSize=1 + - managedLedgerDefaultWriteQuorum=1 + - managedLedgerDefaultAckQuorum=1 + - advertisedAddress=broker-2 + - internalListenerName=internal + - advertisedListeners=internal:pulsar://broker-2:6650 + - PULSAR_MEM=-Xms256m -Xmx256m -XX:MaxDirectMemorySize=56m + # Load Manager. Here uses the extensible load balancer, sets the unloading strategy to TransferShedder, and enables debug mode. + - loadManagerClassName=org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl + - loadBalancerLoadSheddingStrategy=org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder + - loadBalancerSheddingEnabled=false + - loadBalancerDebugModeEnabled=true + - PULSAR_PREFIX_defaultNumberOfNamespaceBundles=1 + depends_on: + zookeeper: + condition: service_healthy + bookie: + condition: service_started + command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" \ No newline at end of file From d107d32a9c1b4bf54ca7b81ab97c96e6ff037579 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 2 Feb 2024 19:35:48 +0800 Subject: [PATCH 31/54] Pin the Pulsar version to 3.1.1 temporarily for the regression in Pulsar (#395) ### Motivation Pulsar 3.1.2 introduces a regression that makes two tests fail: - AuthPluginTest.testTlsDetectPulsarSslWithInvalidBroker (try #1) - AuthPluginTest.testTlsDetectHttpsWithInvalidBroker (try #1) ### Modifications Pin the Pulsar version to 3.1.1 until there is a new Pulsar release that fixes the regression. --- pulsar-test-service-start.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-test-service-start.sh b/pulsar-test-service-start.sh index 23f70102..9c6209e7 100755 --- a/pulsar-test-service-start.sh +++ b/pulsar-test-service-start.sh @@ -25,12 +25,12 @@ SRC_DIR=$PWD ./pulsar-test-service-stop.sh -CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600) +CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:3.1.1 sleep 3600) build-support/setup-test-service-container.sh $CONTAINER_ID start-test-service-inside-container.sh docker cp $CONTAINER_ID:/pulsar/data/tokens/token.txt .test-token.txt -CONTAINER_ID=$(docker run -i --user $(id -u) -p 8081:8081 -p 6652:6652 -p 8444:8444 -p 6653:6653 --rm --detach apachepulsar/pulsar:latest sleep 3600) +CONTAINER_ID=$(docker run -i --user $(id -u) -p 8081:8081 -p 6652:6652 -p 8444:8444 -p 6653:6653 --rm --detach apachepulsar/pulsar:3.1.1 sleep 3600) build-support/setup-test-service-container.sh $CONTAINER_ID start-mim-test-service-inside-container.sh echo "-- Ready to start tests" From a1e2b4a2be5bcf9424768da7d2161a42e16f57bf Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 2 Feb 2024 19:36:47 +0800 Subject: [PATCH 32/54] Fix multi-topics consumer could receive old messages after seek (#388) ### Motivation See https://github.com/apache/pulsar/pull/21945 ### Modifications In C++ client, the multi-topics consumer receives messages by configuring internal consumers with a message listener that adds messages to `incomingMessages_`. So this patch pauses the listeners before seek and resumes them after seek. Add `MultiTopicsConsumerTest.testSeekToNewerPosition` for test. --- lib/ConsumerImpl.h | 2 - lib/MultiResultCallback.h | 51 --------------- lib/MultiTopicsConsumerImpl.cc | 38 +++++++++-- lib/MultiTopicsConsumerImpl.h | 1 + tests/MultiTopicsConsumerTest.cc | 105 +++++++++++++++++++++++++++++++ tests/ThreadSafeMessages.h | 75 ++++++++++++++++++++++ 6 files changed, 215 insertions(+), 57 deletions(-) delete mode 100644 lib/MultiResultCallback.h create mode 100644 tests/MultiTopicsConsumerTest.cc create mode 100644 tests/ThreadSafeMessages.h diff --git a/lib/ConsumerImpl.h b/lib/ConsumerImpl.h index 5612091e..524acb8e 100644 --- a/lib/ConsumerImpl.h +++ b/lib/ConsumerImpl.h @@ -333,8 +333,6 @@ class ConsumerImpl : public ConsumerImplBase { const ClientConnectionPtr& cnx, MessageId& messageId); friend class PulsarFriend; - - // these two declared friend to access setNegativeAcknowledgeEnabledForTesting friend class MultiTopicsConsumerImpl; FRIEND_TEST(ConsumerTest, testRedeliveryOfDecryptionFailedMessages); diff --git a/lib/MultiResultCallback.h b/lib/MultiResultCallback.h deleted file mode 100644 index 739bc4a6..00000000 --- a/lib/MultiResultCallback.h +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#pragma once - -#include // for ResultCallback - -#include -#include - -namespace pulsar { - -class MultiResultCallback { - public: - MultiResultCallback(ResultCallback callback, int numToComplete) - : callback_(callback), - numToComplete_(numToComplete), - numCompletedPtr_(std::make_shared(0)) {} - - void operator()(Result result) { - if (result == ResultOk) { - if (++(*numCompletedPtr_) == numToComplete_) { - callback_(result); - } - } else { - callback_(result); - } - } - - private: - ResultCallback callback_; - const int numToComplete_; - const std::shared_ptr numCompletedPtr_; -}; - -} // namespace pulsar diff --git a/lib/MultiTopicsConsumerImpl.cc b/lib/MultiTopicsConsumerImpl.cc index af70623d..a0854cf8 100644 --- a/lib/MultiTopicsConsumerImpl.cc +++ b/lib/MultiTopicsConsumerImpl.cc @@ -28,7 +28,6 @@ #include "LookupService.h" #include "MessageImpl.h" #include "MessagesImpl.h" -#include "MultiResultCallback.h" #include "MultiTopicsBrokerConsumerStatsImpl.h" #include "TopicName.h" #include "UnAckedMessageTrackerDisabled.h" @@ -521,6 +520,9 @@ void MultiTopicsConsumerImpl::closeAsync(ResultCallback originalCallback) { } void MultiTopicsConsumerImpl::messageReceived(Consumer consumer, const Message& msg) { + if (PULSAR_UNLIKELY(duringSeek_.load(std::memory_order_acquire))) { + return; + } LOG_DEBUG("Received Message from one of the topic - " << consumer.getTopic() << " message:" << msg.getDataAsString()); msg.impl_->setTopicName(consumer.impl_->getTopicPtr()); @@ -907,9 +909,37 @@ void MultiTopicsConsumerImpl::seekAsync(uint64_t timestamp, ResultCallback callb return; } - MultiResultCallback multiResultCallback(callback, consumers_.size()); - consumers_.forEachValue([×tamp, &multiResultCallback](ConsumerImplPtr consumer) { - consumer->seekAsync(timestamp, multiResultCallback); + duringSeek_.store(true, std::memory_order_release); + consumers_.forEachValue([](const ConsumerImplPtr& consumer) { consumer->pauseMessageListener(); }); + unAckedMessageTrackerPtr_->clear(); + incomingMessages_.clear(); + incomingMessagesSize_ = 0L; + + auto weakSelf = weak_from_this(); + auto numConsumersLeft = std::make_shared>(consumers_.size()); + auto wrappedCallback = [this, weakSelf, callback, numConsumersLeft](Result result) { + auto self = weakSelf.lock(); + if (PULSAR_UNLIKELY(!self)) { + callback(result); + return; + } + if (result != ResultOk) { + *numConsumersLeft = 0; // skip the following callbacks + callback(result); + return; + } + if (--*numConsumersLeft > 0) { + return; + } + duringSeek_.store(false, std::memory_order_release); + listenerExecutor_->postWork([this, self] { + consumers_.forEachValue( + [](const ConsumerImplPtr& consumer) { consumer->resumeMessageListener(); }); + }); + callback(ResultOk); + }; + consumers_.forEachValue([timestamp, &wrappedCallback](const ConsumerImplPtr& consumer) { + consumer->seekAsync(timestamp, wrappedCallback); }); } diff --git a/lib/MultiTopicsConsumerImpl.h b/lib/MultiTopicsConsumerImpl.h index c5834eaa..9d71a044 100644 --- a/lib/MultiTopicsConsumerImpl.h +++ b/lib/MultiTopicsConsumerImpl.h @@ -131,6 +131,7 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase { const Commands::SubscriptionMode subscriptionMode_; boost::optional startMessageId_; ConsumerInterceptorsPtr interceptors_; + std::atomic_bool duringSeek_{false}; /* methods */ void handleSinglePartitionConsumerCreated(Result result, ConsumerImplBaseWeakPtr consumerImplBaseWeakPtr, diff --git a/tests/MultiTopicsConsumerTest.cc b/tests/MultiTopicsConsumerTest.cc new file mode 100644 index 00000000..5aae1eb9 --- /dev/null +++ b/tests/MultiTopicsConsumerTest.cc @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include + +#include + +#include "ThreadSafeMessages.h" +#include "lib/LogUtils.h" + +static const std::string lookupUrl = "pulsar://localhost:6650"; + +DECLARE_LOG_OBJECT() + +using namespace pulsar; + +extern std::string unique_str(); + +TEST(MultiTopicsConsumerTest, testSeekToNewerPosition) { + const std::string topicPrefix = "multi-topics-consumer-seek-to-newer-position"; + Client client{lookupUrl}; + std::vector topics{topicPrefix + unique_str(), topicPrefix + unique_str()}; + Producer producer1; + ASSERT_EQ(ResultOk, client.createProducer(topics[0], producer1)); + Producer producer2; + ASSERT_EQ(ResultOk, client.createProducer(topics[1], producer2)); + producer1.send(MessageBuilder().setContent("1-0").build()); + producer2.send(MessageBuilder().setContent("2-0").build()); + producer1.send(MessageBuilder().setContent("1-1").build()); + producer2.send(MessageBuilder().setContent("2-1").build()); + + Consumer consumer; + ConsumerConfiguration conf; + conf.setSubscriptionInitialPosition(InitialPositionEarliest); + ASSERT_EQ(ResultOk, client.subscribe(topics, "sub", conf, consumer)); + std::vector timestamps; + Message msg; + for (int i = 0; i < 4; i++) { + ASSERT_EQ(ResultOk, consumer.receive(msg, 3000)); + timestamps.emplace_back(msg.getPublishTimestamp()); + } + std::sort(timestamps.begin(), timestamps.end()); + const auto timestamp = timestamps[2]; + consumer.close(); + + ThreadSafeMessages messages{2}; + + // Test synchronous receive after seek + ASSERT_EQ(ResultOk, client.subscribe(topics, "sub-2", conf, consumer)); + consumer.seek(timestamp); + for (int i = 0; i < 2; i++) { + ASSERT_EQ(ResultOk, consumer.receive(msg, 3000)); + messages.add(msg); + } + ASSERT_EQ(messages.getSortedValues(), (std::vector{"1-1", "2-1"})); + consumer.close(); + + // Test asynchronous receive after seek + ASSERT_EQ(ResultOk, client.subscribe(topics, "sub-3", conf, consumer)); + messages.clear(); + consumer.seek(timestamp); + for (int i = 0; i < 2; i++) { + consumer.receiveAsync([&messages](Result result, const Message& msg) { + if (result == ResultOk) { + messages.add(msg); + } else { + LOG_ERROR("Failed to receive: " << result); + } + }); + } + ASSERT_TRUE(messages.wait(std::chrono::seconds(3))); + ASSERT_EQ(messages.getSortedValues(), (std::vector{"1-1", "2-1"})); + consumer.close(); + + // Test message listener + conf.setMessageListener([&messages](Consumer consumer, Message msg) { messages.add(msg); }); + messages.clear(); + messages.setMinNumMsgs(4); + ASSERT_EQ(ResultOk, client.subscribe(topics, "sub-4", conf, consumer)); + ASSERT_TRUE(messages.wait(std::chrono::seconds(3))); + ASSERT_EQ(messages.getSortedValues(), (std::vector{"1-0", "1-1", "2-0", "2-1"})); + messages.clear(); + messages.setMinNumMsgs(2); + consumer.seek(timestamp); + ASSERT_TRUE(messages.wait(std::chrono::seconds(3))); + ASSERT_EQ(messages.getSortedValues(), (std::vector{"1-1", "2-1"})); + + client.close(); +} diff --git a/tests/ThreadSafeMessages.h b/tests/ThreadSafeMessages.h new file mode 100644 index 00000000..f30dbf02 --- /dev/null +++ b/tests/ThreadSafeMessages.h @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace pulsar { + +// When we receive messages in the message listener or the callback of receiveAsync(), we need to verify the +// received messages in the test thread. This class is a helper class for thread-safe access to the messages. +class ThreadSafeMessages { + public: + ThreadSafeMessages(size_t minNumMsgs) : minNumMsgs_(minNumMsgs) {} + + template + bool wait(Duration duration) { + std::unique_lock lock{mutex_}; + return cond_.wait_for(lock, duration, [this] { return msgs_.size() >= minNumMsgs_; }); + } + + void add(const Message& msg) { + std::lock_guard lock{mutex_}; + msgs_.emplace_back(msg); + if (msgs_.size() >= minNumMsgs_) { + cond_.notify_all(); + } + } + + void clear() { + std::lock_guard lock{mutex_}; + msgs_.clear(); + } + + std::vector getSortedValues() const { + std::unique_lock lock{mutex_}; + std::vector values(msgs_.size()); + std::transform(msgs_.cbegin(), msgs_.cend(), values.begin(), + [](const Message& msg) { return msg.getDataAsString(); }); + lock.unlock(); + std::sort(values.begin(), values.end()); + return values; + } + + void setMinNumMsgs(size_t minNumMsgs) noexcept { minNumMsgs_ = minNumMsgs; } + + private: + std::atomic_size_t minNumMsgs_; + std::vector msgs_; + mutable std::mutex mutex_; + mutable std::condition_variable cond_; +}; + +} // namespace pulsar From c7e53acfc7c31c19ba623fed011caee3c6571a3f Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 2 Feb 2024 19:37:02 +0800 Subject: [PATCH 33/54] Do not close the socket when the broker failed due to MetadataStoreException (#390) ### Motivation When the broker failed to acquire the ownership of a namespace bundle by `LockBusyException`. It means there is another broker that has acquired the metadata store path and didn't release that path. For example: Broker 1: ``` 2024-01-24T23:35:36,626+0000 [metadata-store-10-1] WARN org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup for topic persistent://// with error org.apache.pulsar.broker.PulsarServerException: Failed to acquire ownership for namespace bundle //0x50000000_0x51000000 Caused by: java.util.concurrent.CompletionException: org.apache.pulsar.metadata.api.MetadataStoreException$LockBusyException: Resource at /namespace///0x50000000_0x51000000 is already locked ``` Broker 2: ``` 2024-01-24T23:35:36,650+0000 [broker-topic-workers-OrderedExecutor-3-0] INFO org.apache.pulsar.broker.PulsarService - Loaded 1 topics on //0x50000000_0x51000000 -- time taken: 0.044 seconds ``` After broker 2 released the lock at 23:35:36,650, the lookup request to broker 1 should tell the client that namespace bundle 0x50000000_0x51000000 is currently being unloaded and in the next retry the client will connect to the new owner broker. Here is another typical error: ``` 2024-01-24T23:57:57,264+0000 [pulsar-io-4-5] INFO org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup for topic persistent://// with error Namespace bundle //0x0d000000_0x0e000000 is being unloaded ``` Though after https://github.com/apache/pulsar/pull/21211, the server error becomes `MetadataError` rather than `ServiceNotReady`. However, since the `ServerError` is `ServiceNotReady`, the client will close the connection. If there are many other producers or consumers on the same connection, they will all reestablish connection to the broker, which is unnecessary and brings much pressure to broker side. ### Modifications In `checkServerError`, when the error code is `ServiceNotReady`, check the error message as well, if it hit the case in `handleLookupError`, do not close the connection. Add `ConnectionTest` on a mocked `ClientConnection` object to verify `close()` will not be called. --- lib/ClientConnection.cc | 20 +++--------- lib/ClientConnection.h | 2 +- lib/ClientConnectionAdaptor.h | 61 +++++++++++++++++++++++++++++++++++ tests/ConnectionTest.cc | 48 +++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 16 deletions(-) create mode 100644 lib/ClientConnectionAdaptor.h create mode 100644 tests/ConnectionTest.cc diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index 0641809e..844d58f2 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -24,6 +24,7 @@ #include #include "AsioDefines.h" +#include "ClientConnectionAdaptor.h" #include "ClientImpl.h" #include "Commands.h" #include "ConnectionPool.h" @@ -1469,19 +1470,8 @@ Future ClientConnection::newGetSchema(const std::string& top return promise.getFuture(); } -void ClientConnection::checkServerError(ServerError error) { - switch (error) { - case proto::ServerError::ServiceNotReady: - close(ResultDisconnected); - break; - case proto::ServerError::TooManyRequests: - // TODO: Implement maxNumberOfRejectedRequestPerConnection like - // https://github.com/apache/pulsar/pull/274 - close(ResultDisconnected); - break; - default: - break; - } +void ClientConnection::checkServerError(ServerError error, const std::string& message) { + pulsar::adaptor::checkServerError(*this, error, message); } void ClientConnection::handleSendReceipt(const proto::CommandSendReceipt& sendReceipt) { @@ -1573,7 +1563,7 @@ void ClientConnection::handlePartitionedMetadataResponse( << partitionMetadataResponse.request_id() << " error: " << partitionMetadataResponse.error() << " msg: " << partitionMetadataResponse.message()); - checkServerError(partitionMetadataResponse.error()); + checkServerError(partitionMetadataResponse.error(), partitionMetadataResponse.message()); lookupDataPromise->setFailed( getResult(partitionMetadataResponse.error(), partitionMetadataResponse.message())); } else { @@ -1650,7 +1640,7 @@ void ClientConnection::handleLookupTopicRespose( LOG_ERROR(cnxString_ << "Failed lookup req_id: " << lookupTopicResponse.request_id() << " error: " << lookupTopicResponse.error() << " msg: " << lookupTopicResponse.message()); - checkServerError(lookupTopicResponse.error()); + checkServerError(lookupTopicResponse.error(), lookupTopicResponse.message()); lookupDataPromise->setFailed( getResult(lookupTopicResponse.error(), lookupTopicResponse.message())); } else { diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index 851ec0c3..b16fc694 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -404,7 +404,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this + +#include "ProtoApiEnums.h" +#include "PulsarApi.pb.h" + +namespace pulsar { + +namespace adaptor { + +template +inline void checkServerError(Connection& connection, ServerError error, const std::string& message) { + switch (error) { + case proto::ServerError::ServiceNotReady: + // There are some typical error messages that should not trigger the + // close() of the connection. + // "Namespace bundle ... is being unloaded" + // "KeeperException$..." + // "Failed to acquire ownership for namespace bundle ..." + // Before https://github.com/apache/pulsar/pull/21211, the error of the 1st and 2nd messages + // is ServiceNotReady. Before https://github.com/apache/pulsar/pull/21993, the error of the 3rd + // message is ServiceNotReady. + if (message.find("Failed to acquire ownership") == std::string::npos && + message.find("KeeperException") == std::string::npos && + message.find("is being unloaded") == std::string::npos) { + connection.close(ResultDisconnected); + } + break; + case proto::ServerError::TooManyRequests: + // TODO: Implement maxNumberOfRejectedRequestPerConnection like + // https://github.com/apache/pulsar/pull/274 + connection.close(ResultDisconnected); + break; + default: + break; + } +} + +} // namespace adaptor + +} // namespace pulsar diff --git a/tests/ConnectionTest.cc b/tests/ConnectionTest.cc new file mode 100644 index 00000000..e0d063e9 --- /dev/null +++ b/tests/ConnectionTest.cc @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + */ +#include +#include + +#include + +#include "lib/ClientConnection.h" +#include "lib/ClientConnectionAdaptor.h" + +using namespace pulsar; + +class MockClientConnection { + public: + MOCK_METHOD(void, close, (Result)); + + void checkServerError(ServerError error, const std::string& message) { + ::pulsar::adaptor::checkServerError(*this, error, message); + } +}; + +// These error messages come from +// https://github.com/apache/pulsar/blob/a702e5a582eaa8292720f9e25fc2dcf858078813/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java#L334-L351 +static const std::vector retryableErrorMessages{ + "Namespace bundle public/default/0x00000000_0xffffffff is being unloaded", + "org.apache.zookeeper.KeeperException$OperationTimeoutException: KeeperErrorCode = OperationTimeout for " + "/namespace/public/default/0x00000000_0xffffffff", + "Failed to acquire ownership for namespace bundle public/default/0x00000000_0xffffffff"}; + +TEST(ConnectionTest, testCheckServerError) { + MockClientConnection conn; + EXPECT_CALL(conn, close(ResultDisconnected)).Times(0); + for (auto&& msg : retryableErrorMessages) { + conn.checkServerError(pulsar::proto::ServiceNotReady, msg); + } +} From 77e0c22690cde3bb340f353fd6d6e7dd0e1cd45c Mon Sep 17 00:00:00 2001 From: erobot Date: Mon, 5 Feb 2024 10:40:52 +0800 Subject: [PATCH 34/54] Fix an unbalanced release of the producer's pending semaphore (#392) ### Motivation Current code releases the producer's pending semaphore twice when batch is off and message is too big. The unbalanced release overflows the semaphore, and subsequent sends will fail with ProducerQueueIsFull. ### Modifications Remove the redundant semaphore release as the necessary release will be done in `handleFailedResult`. --- lib/ProducerImpl.cc | 1 - tests/ProducerTest.cc | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index 2e5cd444..14a74c63 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -630,7 +630,6 @@ void ProducerImpl::sendAsyncWithStatsUpdate(const Message& msg, SendCallback&& c const uint32_t msgHeadersAndPayloadSize = msgMetadataSize + payloadSize; if (msgHeadersAndPayloadSize > maxMessageSize) { lock.unlock(); - releaseSemaphoreForSendOp(*op); LOG_WARN(getName() << " - compressed Message size " << msgHeadersAndPayloadSize << " cannot exceed " << maxMessageSize << " bytes unless chunking is enabled"); diff --git a/tests/ProducerTest.cc b/tests/ProducerTest.cc index bb58a4ef..21e491de 100644 --- a/tests/ProducerTest.cc +++ b/tests/ProducerTest.cc @@ -243,6 +243,8 @@ TEST_P(ProducerTest, testMaxMessageSize) { ASSERT_EQ(ResultMessageTooBig, producer.send(MessageBuilder().setContent(std::string(maxMessageSize, 'b')).build())); + ASSERT_EQ(ResultOk, producer.send(MessageBuilder().setContent(msg).build())); + client.close(); } From 1f94dd92de579444cf1706a941eac6f117a67781 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 5 Feb 2024 18:18:35 +0800 Subject: [PATCH 35/54] Disable topic level policies to make tests work (#397) --- pulsar-test-service-start.sh | 4 ++-- test-conf/standalone-ssl-mim.conf | 5 +++++ test-conf/standalone-ssl.conf | 9 +++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pulsar-test-service-start.sh b/pulsar-test-service-start.sh index 9c6209e7..23f70102 100755 --- a/pulsar-test-service-start.sh +++ b/pulsar-test-service-start.sh @@ -25,12 +25,12 @@ SRC_DIR=$PWD ./pulsar-test-service-stop.sh -CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:3.1.1 sleep 3600) +CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600) build-support/setup-test-service-container.sh $CONTAINER_ID start-test-service-inside-container.sh docker cp $CONTAINER_ID:/pulsar/data/tokens/token.txt .test-token.txt -CONTAINER_ID=$(docker run -i --user $(id -u) -p 8081:8081 -p 6652:6652 -p 8444:8444 -p 6653:6653 --rm --detach apachepulsar/pulsar:3.1.1 sleep 3600) +CONTAINER_ID=$(docker run -i --user $(id -u) -p 8081:8081 -p 6652:6652 -p 8444:8444 -p 6653:6653 --rm --detach apachepulsar/pulsar:latest sleep 3600) build-support/setup-test-service-container.sh $CONTAINER_ID start-mim-test-service-inside-container.sh echo "-- Ready to start tests" diff --git a/test-conf/standalone-ssl-mim.conf b/test-conf/standalone-ssl-mim.conf index 359645ea..198a9002 100644 --- a/test-conf/standalone-ssl-mim.conf +++ b/test-conf/standalone-ssl-mim.conf @@ -307,3 +307,8 @@ maxMessageSize=1024000 # Disable consistent hashing to fix flaky `KeySharedConsumerTest#testMultiTopics`. subscriptionKeySharedUseConsistentHashing=false + +# It's true by default since 2.11. After https://github.com/apache/pulsar/pull/21445, we must configure +# brokerClientAuthenticationPlugin and brokerClientAuthenticationParameters correctly when enabling topic +# level policies. Otherwise, no topic could be loaded. +topicLevelPoliciesEnabled=false diff --git a/test-conf/standalone-ssl.conf b/test-conf/standalone-ssl.conf index 4b150076..36ad9b56 100644 --- a/test-conf/standalone-ssl.conf +++ b/test-conf/standalone-ssl.conf @@ -19,10 +19,6 @@ ### --- General broker settings --- ### -# Disable system topic -systemTopicEnabled=false -topicLevelPoliciesEnabled=false - # Zookeeper quorum connection string zookeeperServers= @@ -317,3 +313,8 @@ subscriptionKeySharedUseConsistentHashing=false # Enable batch index ACK acknowledgmentAtBatchIndexLevelEnabled=true + +# It's true by default since 2.11. After https://github.com/apache/pulsar/pull/21445, we must configure +# brokerClientAuthenticationPlugin and brokerClientAuthenticationParameters correctly when enabling topic +# level policies. Otherwise, no topic could be loaded. +topicLevelPoliciesEnabled=false From 289c9301918aa66d56322c794895a40010d8885a Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 6 Feb 2024 22:12:58 +0800 Subject: [PATCH 36/54] Fix broken main branch after Pulsar image was upgraded to 3.2.0 (#400) The `apachepulsar/pulsar:latest` image was upgraded to 3.2.0 recently, which results the following failed tests: - ExtensibleLoadManagerTest: there are no more lookup requests after namespace unload - ClientTest.testWrongListener: the lookup error message from broker does not contain the exception type --- lib/ClientConnection.cc | 6 +++--- tests/extensibleLM/ExtensibleLoadManagerTest.cc | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index 844d58f2..e006f0ec 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -81,9 +81,9 @@ static Result getResult(ServerError serverError, const std::string& message) { return ResultConsumerBusy; case ServiceNotReady: - // If the error is not caused by a PulsarServerException, treat it as retryable. - return (message.find("PulsarServerException") == std::string::npos) ? ResultRetryable - : ResultServiceUnitNotReady; + return (message.find("the broker do not have test listener") == std::string::npos) + ? ResultRetryable + : ResultServiceUnitNotReady; case ProducerBlockedQuotaExceededError: return ResultProducerBlockedQuotaExceededError; diff --git a/tests/extensibleLM/ExtensibleLoadManagerTest.cc b/tests/extensibleLM/ExtensibleLoadManagerTest.cc index 4472d9d9..a6379f6e 100644 --- a/tests/extensibleLM/ExtensibleLoadManagerTest.cc +++ b/tests/extensibleLM/ExtensibleLoadManagerTest.cc @@ -169,9 +169,8 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { })); LOG_INFO("after lookup responseData:" << responseDataAfterUnload << ",res:" << res); - // TODO: check lookup counter after pip-307 is released auto lookupCountAfterUnload = clientImplPtr->getLookupCount(); - ASSERT_TRUE(lookupCountBeforeUnload < lookupCountAfterUnload); + ASSERT_EQ(lookupCountBeforeUnload, lookupCountAfterUnload); break; } }; From 71077b866f31cc6c3847192285e1d3f00683bf22 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 7 Feb 2024 04:40:19 +0800 Subject: [PATCH 37/54] Fix segmenatation fault if connection is closed during handshake (#399) Fixes https://github.com/apache/pulsar-client-cpp/issues/398 ### Motivation There is an implicit requirement for the `async_handshake` API that before it's done, the `socket` and `ssl::stream` objects must exist. Otherwise segmenatation fault might happen. ### Modifications Capture `socket_` and `tlsSocket_` in the handshake callback. It can be verified by modifying the code according to #398 and rerun the test. --- lib/ClientConnection.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index e006f0ec..23e60225 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -456,7 +456,11 @@ void ClientConnection::handleTcpConnected(const ASIO_ERROR& err, tcp::resolver:: } } auto weakSelf = weak_from_this(); - auto callback = [weakSelf](const ASIO_ERROR& err) { + auto socket = socket_; + auto tlsSocket = tlsSocket_; + // socket and ssl::stream objects must exist until async_handshake is done, otherwise segmentation + // fault might happen + auto callback = [weakSelf, socket, tlsSocket](const ASIO_ERROR& err) { auto self = weakSelf.lock(); if (self) { self->handleHandshake(err); From 398350075341527399305dc92a0847ed3eb4c726 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 6 Feb 2024 18:25:59 -0800 Subject: [PATCH 38/54] Add codeql code scanning configuration (#10) --- .github/workflows/codeql-analysis.yml | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..c877c642 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '16 21 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-22.04 + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'cpp' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + + - name: Install deps + run: | + sudo apt-get update -y + sudo apt-get install -y libcurl4-openssl-dev libssl-dev \ + protobuf-compiler libprotobuf-dev libboost-dev \ + libboost-dev libboost-program-options-dev \ + libzstd-dev libsnappy-dev + + - name: Build + run: | + cmake . -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DBUILD_PERF_TOOLS=ON + cmake --build . -j8 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From 68b4244f321345928e0943387c1dca7b5f448d2c Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 7 Feb 2024 10:26:36 +0800 Subject: [PATCH 39/54] Fix creating producer or consumer is not retried for connection failure (#396) Fixes https://github.com/apache/pulsar-client-cpp/issues/391 ### Motivation When `connectionFailed` is called, no matter if the result is retryable the creation of producer or consumer will fail without retry. ### Modifications Check if the result is retryable in `connectionFailed` for `ProducerImpl` and `ConsumerImpl` and only fail for non-retryable errors or the timeout error. Register another timer in `HandlerBase` to propagate the timeout error to `connectionFailed`. Add `testRetryUntilSucceed`, `testRetryTimeout`, `testNoRetry` to verify client could retry according to the result returned by `ClientImpl::getConnection`. --- lib/ClientConnection.cc | 11 +++-- lib/ClientConnectionAdaptor.h | 4 +- lib/ClientImpl.cc | 4 +- lib/ClientImpl.h | 8 +-- lib/ConsumerImpl.cc | 2 +- lib/Future.h | 9 ++++ lib/HandlerBase.cc | 21 ++++++-- lib/HandlerBase.h | 1 + lib/ProducerImpl.cc | 2 +- lib/ResultUtils.h | 29 ++++++++++- tests/ClientTest.cc | 75 ++++++++++++++++++++++++++-- tests/MockClientImpl.h | 92 +++++++++++++++++++++++++++++++++++ 12 files changed, 239 insertions(+), 19 deletions(-) create mode 100644 tests/MockClientImpl.h diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index 23e60225..00041b2a 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -83,7 +83,7 @@ static Result getResult(ServerError serverError, const std::string& message) { case ServiceNotReady: return (message.find("the broker do not have test listener") == std::string::npos) ? ResultRetryable - : ResultServiceUnitNotReady; + : ResultConnectError; case ProducerBlockedQuotaExceededError: return ResultProducerBlockedQuotaExceededError; @@ -508,8 +508,13 @@ void ClientConnection::handleTcpConnected(const ASIO_ERROR& err, tcp::resolver:: void ClientConnection::handleHandshake(const ASIO_ERROR& err) { if (err) { - LOG_ERROR(cnxString_ << "Handshake failed: " << err.message()); - close(); + if (err.value() == ASIO::ssl::error::stream_truncated) { + LOG_WARN(cnxString_ << "Handshake failed: " << err.message()); + close(ResultRetryable); + } else { + LOG_ERROR(cnxString_ << "Handshake failed: " << err.message()); + close(); + } return; } diff --git a/lib/ClientConnectionAdaptor.h b/lib/ClientConnectionAdaptor.h index 8f5599fb..2c299373 100644 --- a/lib/ClientConnectionAdaptor.h +++ b/lib/ClientConnectionAdaptor.h @@ -37,12 +37,14 @@ inline void checkServerError(Connection& connection, ServerError error, const st // "Namespace bundle ... is being unloaded" // "KeeperException$..." // "Failed to acquire ownership for namespace bundle ..." + // "the broker do not have test listener" // Before https://github.com/apache/pulsar/pull/21211, the error of the 1st and 2nd messages // is ServiceNotReady. Before https://github.com/apache/pulsar/pull/21993, the error of the 3rd // message is ServiceNotReady. if (message.find("Failed to acquire ownership") == std::string::npos && message.find("KeeperException") == std::string::npos && - message.find("is being unloaded") == std::string::npos) { + message.find("is being unloaded") == std::string::npos && + message.find("the broker do not have test listener") == std::string::npos) { connection.close(ResultDisconnected); } break; diff --git a/lib/ClientImpl.cc b/lib/ClientImpl.cc index 63c85b0f..ae339731 100644 --- a/lib/ClientImpl.cc +++ b/lib/ClientImpl.cc @@ -517,7 +517,7 @@ void ClientImpl::handleConsumerCreated(Result result, ConsumerImplBaseWeakPtr co } } -Future ClientImpl::getConnection(const std::string& topic, size_t key) { +GetConnectionFuture ClientImpl::getConnection(const std::string& topic, size_t key) { Promise promise; const auto topicNamePtr = TopicName::get(topic); @@ -562,7 +562,7 @@ const std::string& ClientImpl::getPhysicalAddress(const std::string& logicalAddr } } -Future ClientImpl::connect(const std::string& logicalAddress, size_t key) { +GetConnectionFuture ClientImpl::connect(const std::string& logicalAddress, size_t key) { const auto& physicalAddress = getPhysicalAddress(logicalAddress); Promise promise; pool_.getConnectionAsync(logicalAddress, physicalAddress, key) diff --git a/lib/ClientImpl.h b/lib/ClientImpl.h index a2649a6a..7126542b 100644 --- a/lib/ClientImpl.h +++ b/lib/ClientImpl.h @@ -63,13 +63,14 @@ class TopicName; using TopicNamePtr = std::shared_ptr; using NamespaceTopicsPtr = std::shared_ptr>; +using GetConnectionFuture = Future; std::string generateRandomName(); class ClientImpl : public std::enable_shared_from_this { public: ClientImpl(const std::string& serviceUrl, const ClientConfiguration& clientConfiguration); - ~ClientImpl(); + virtual ~ClientImpl(); /** * @param autoDownloadSchema When it is true, Before creating a producer, it will try to get the schema @@ -95,9 +96,10 @@ class ClientImpl : public std::enable_shared_from_this { void getPartitionsForTopicAsync(const std::string& topic, GetPartitionsCallback callback); - Future getConnection(const std::string& topic, size_t key); + // Use virtual method to test + virtual GetConnectionFuture getConnection(const std::string& topic, size_t key); - Future connect(const std::string& logicalAddress, size_t key); + GetConnectionFuture connect(const std::string& logicalAddress, size_t key); void closeAsync(CloseCallback callback); void shutdown(); diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index 86dddb0d..0051ed6d 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -277,7 +277,7 @@ void ConsumerImpl::connectionFailed(Result result) { // Keep a reference to ensure object is kept alive auto ptr = get_shared_this_ptr(); - if (consumerCreatedPromise_.setFailed(result)) { + if (!isResultRetryable(result) && consumerCreatedPromise_.setFailed(result)) { state_ = Failed; } } diff --git a/lib/Future.h b/lib/Future.h index 69db74a3..22d43cbe 100644 --- a/lib/Future.h +++ b/lib/Future.h @@ -116,6 +116,8 @@ class Future { Result get(Type &result) { return state_->get(result); } + static Future failed(Result result); + private: InternalStatePtr state_; @@ -144,6 +146,13 @@ class Promise { InternalStatePtr state_; }; +template +inline Future Future::failed(Result result) { + Promise promise; + promise.setFailed(result); + return promise.getFuture(); +} + } // namespace pulsar #endif diff --git a/lib/HandlerBase.cc b/lib/HandlerBase.cc index c5327fcf..52e20d2d 100644 --- a/lib/HandlerBase.cc +++ b/lib/HandlerBase.cc @@ -42,9 +42,14 @@ HandlerBase::HandlerBase(const ClientImplPtr& client, const std::string& topic, backoff_(backoff), epoch_(0), timer_(executor_->createDeadlineTimer()), + creationTimer_(executor_->createDeadlineTimer()), reconnectionPending_(false) {} -HandlerBase::~HandlerBase() { timer_->cancel(); } +HandlerBase::~HandlerBase() { + ASIO_ERROR ignored; + timer_->cancel(ignored); + creationTimer_->cancel(ignored); +} void HandlerBase::start() { // guard against concurrent state changes such as closing @@ -52,6 +57,16 @@ void HandlerBase::start() { if (state_.compare_exchange_strong(state, Pending)) { grabCnx(); } + creationTimer_->expires_from_now(operationTimeut_); + std::weak_ptr weakSelf{shared_from_this()}; + creationTimer_->async_wait([this, weakSelf](const ASIO_ERROR& error) { + auto self = weakSelf.lock(); + if (self && !error) { + connectionFailed(ResultTimeout); + ASIO_ERROR ignored; + timer_->cancel(ignored); + } + }); } ClientConnectionWeakPtr HandlerBase::getCnx() const { @@ -96,7 +111,7 @@ void HandlerBase::grabCnx(const boost::optional& assignedBrokerUrl) ClientImplPtr client = client_.lock(); if (!client) { LOG_WARN(getName() << "Client is invalid when calling grabCnx()"); - connectionFailed(ResultConnectError); + connectionFailed(ResultAlreadyClosed); reconnectionPending_ = false; return; } @@ -108,7 +123,7 @@ void HandlerBase::grabCnx(const boost::optional& assignedBrokerUrl) connectionOpened(cnx).addListener([this, self](Result result, bool) { // Do not use bool, only Result. reconnectionPending_ = false; - if (isResultRetryable(result)) { + if (result != ResultOk && isResultRetryable(result)) { scheduleReconnection(); } }); diff --git a/lib/HandlerBase.h b/lib/HandlerBase.h index b9d98553..32e124a2 100644 --- a/lib/HandlerBase.h +++ b/lib/HandlerBase.h @@ -140,6 +140,7 @@ class HandlerBase : public std::enable_shared_from_this { private: DeadlineTimerPtr timer_; + DeadlineTimerPtr creationTimer_; mutable std::mutex connectionMutex_; std::atomic reconnectionPending_; diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index 14a74c63..890d476c 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -176,7 +176,7 @@ void ProducerImpl::connectionFailed(Result result) { // if producers are lazy, then they should always try to restart // so don't change the state and allow reconnections return; - } else if (producerCreatedPromise_.setFailed(result)) { + } else if (!isResultRetryable(result) && producerCreatedPromise_.setFailed(result)) { state_ = Failed; } } diff --git a/lib/ResultUtils.h b/lib/ResultUtils.h index b5ec6cdb..dfba7eb0 100644 --- a/lib/ResultUtils.h +++ b/lib/ResultUtils.h @@ -18,12 +18,39 @@ */ #pragma once +#include #include +#include + namespace pulsar { inline bool isResultRetryable(Result result) { - return result == ResultRetryable || result == ResultDisconnected; + assert(result != ResultOk); + if (result == ResultRetryable || result == ResultDisconnected) { + return true; + } + + static const std::unordered_set fatalResults{ResultConnectError, + ResultTimeout, + ResultAuthenticationError, + ResultAuthorizationError, + ResultInvalidUrl, + ResultInvalidConfiguration, + ResultIncompatibleSchema, + ResultTopicNotFound, + ResultOperationNotSupported, + ResultNotAllowedError, + ResultChecksumError, + ResultCryptoError, + ResultConsumerAssignError, + ResultProducerBusy, + ResultConsumerBusy, + ResultLookupError, + ResultTooManyLookupRequestException, + ResultProducerBlockedQuotaExceededException, + ResultProducerBlockedQuotaExceededError}; + return fatalResults.find(static_cast(result)) == fatalResults.cend(); } } // namespace pulsar diff --git a/tests/ClientTest.cc b/tests/ClientTest.cc index 8c5a32ab..1e8e67d5 100644 --- a/tests/ClientTest.cc +++ b/tests/ClientTest.cc @@ -25,6 +25,7 @@ #include #include +#include "MockClientImpl.h" #include "PulsarAdminHelper.h" #include "PulsarFriend.h" #include "WaitUtils.h" @@ -36,6 +37,7 @@ DECLARE_LOG_OBJECT() using namespace pulsar; +using testing::AtLeast; static std::string lookupUrl = "pulsar://localhost:6650"; static std::string adminUrl = "http://localhost:8080/"; @@ -248,7 +250,7 @@ TEST(ClientTest, testWrongListener) { Client client(lookupUrl, ClientConfiguration().setListenerName("test")); Producer producer; - ASSERT_EQ(ResultServiceUnitNotReady, client.createProducer(topic, producer)); + ASSERT_EQ(ResultConnectError, client.createProducer(topic, producer)); ASSERT_EQ(ResultProducerNotInitialized, producer.close()); ASSERT_EQ(PulsarFriend::getProducers(client).size(), 0); ASSERT_EQ(ResultOk, client.close()); @@ -257,7 +259,7 @@ TEST(ClientTest, testWrongListener) { // creation of Consumer or Reader could fail with ResultConnectError. client = Client(lookupUrl, ClientConfiguration().setListenerName("test")); Consumer consumer; - ASSERT_EQ(ResultServiceUnitNotReady, client.subscribe(topic, "sub", consumer)); + ASSERT_EQ(ResultConnectError, client.subscribe(topic, "sub", consumer)); ASSERT_EQ(ResultConsumerNotInitialized, consumer.close()); ASSERT_EQ(PulsarFriend::getConsumers(client).size(), 0); @@ -266,7 +268,7 @@ TEST(ClientTest, testWrongListener) { client = Client(lookupUrl, ClientConfiguration().setListenerName("test")); Consumer multiTopicsConsumer; - ASSERT_EQ(ResultServiceUnitNotReady, + ASSERT_EQ(ResultConnectError, client.subscribe({topic + "-partition-0", topic + "-partition-1", topic + "-partition-2"}, "sub", multiTopicsConsumer)); @@ -278,7 +280,7 @@ TEST(ClientTest, testWrongListener) { // Currently Reader can only read a non-partitioned topic in C++ client Reader reader; - ASSERT_EQ(ResultServiceUnitNotReady, + ASSERT_EQ(ResultConnectError, client.createReader(topic + "-partition-0", MessageId::earliest(), {}, reader)); ASSERT_EQ(ResultConsumerNotInitialized, reader.close()); ASSERT_EQ(PulsarFriend::getConsumers(client).size(), 0); @@ -434,3 +436,68 @@ TEST(ClientTest, testConnectionClose) { client.close(); } } + +TEST(ClientTest, testRetryUntilSucceed) { + auto clientImpl = std::make_shared(lookupUrl); + constexpr int kFailCount = 3; + EXPECT_CALL(*clientImpl, getConnection).Times((kFailCount + 1) * 2); + std::atomic_int count{0}; + ON_CALL(*clientImpl, getConnection) + .WillByDefault([&clientImpl, &count](const std::string &topic, size_t index) { + if (count++ < kFailCount) { + return GetConnectionFuture::failed(ResultRetryable); + } + return clientImpl->getConnectionReal(topic, index); + }); + + auto topic = "client-test-retry-until-succeed"; + ASSERT_EQ(ResultOk, clientImpl->createProducer(topic).result); + count = 0; + ASSERT_EQ(ResultOk, clientImpl->subscribe(topic).result); + ASSERT_EQ(ResultOk, clientImpl->close()); +} + +TEST(ClientTest, testRetryTimeout) { + auto clientImpl = + std::make_shared(lookupUrl, ClientConfiguration().setOperationTimeoutSeconds(2)); + EXPECT_CALL(*clientImpl, getConnection).Times(AtLeast(2 * 2)); + ON_CALL(*clientImpl, getConnection).WillByDefault([](const std::string &topic, size_t index) { + return GetConnectionFuture::failed(ResultRetryable); + }); + + auto topic = "client-test-retry-timeout"; + { + MockClientImpl::SyncOpResult result = clientImpl->createProducer(topic); + ASSERT_EQ(ResultTimeout, result.result); + ASSERT_TRUE(result.timeMs >= 2000 && result.timeMs < 2100) << "producer: " << result.timeMs << " ms"; + } + { + MockClientImpl::SyncOpResult result = clientImpl->subscribe(topic); + ASSERT_EQ(ResultTimeout, result.result); + ASSERT_TRUE(result.timeMs >= 2000 && result.timeMs < 2100) << "consumer: " << result.timeMs << " ms"; + } + + ASSERT_EQ(ResultOk, clientImpl->close()); +} + +TEST(ClientTest, testNoRetry) { + auto clientImpl = + std::make_shared(lookupUrl, ClientConfiguration().setOperationTimeoutSeconds(100)); + EXPECT_CALL(*clientImpl, getConnection).Times(2); + ON_CALL(*clientImpl, getConnection).WillByDefault([](const std::string &, size_t) { + return GetConnectionFuture::failed(ResultAuthenticationError); + }); + + auto topic = "client-test-no-retry"; + { + MockClientImpl::SyncOpResult result = clientImpl->createProducer(topic); + ASSERT_EQ(ResultAuthenticationError, result.result); + ASSERT_TRUE(result.timeMs < 1000) << "producer: " << result.timeMs << " ms"; + } + { + MockClientImpl::SyncOpResult result = clientImpl->subscribe(topic); + LOG_INFO("It takes " << result.timeMs << " ms to subscribe"); + ASSERT_EQ(ResultAuthenticationError, result.result); + ASSERT_TRUE(result.timeMs < 1000) << "consumer: " << result.timeMs << " ms"; + } +} diff --git a/tests/MockClientImpl.h b/tests/MockClientImpl.h new file mode 100644 index 00000000..aa4208a9 --- /dev/null +++ b/tests/MockClientImpl.h @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include + +#include +#include +#include + +#include "lib/ClientImpl.h" + +namespace pulsar { + +class MockClientImpl : public ClientImpl { + public: + struct SyncOpResult { + Result result; + long timeMs; + }; + using PromisePtr = std::shared_ptr>; + MockClientImpl(const std::string& serviceUrl, ClientConfiguration conf = {}) + : ClientImpl(serviceUrl, conf) {} + + MOCK_METHOD((Future), getConnection, (const std::string&, size_t), + (override)); + + SyncOpResult createProducer(const std::string& topic) { + using namespace std::chrono; + auto start = high_resolution_clock::now(); + auto promise = createPromise(); + createProducerAsync(topic, {}, [&start, promise](Result result, Producer) { + auto timeMs = duration_cast(high_resolution_clock::now() - start).count(); + promise->set_value({result, timeMs}); + }); + return wait(promise); + } + + SyncOpResult subscribe(const std::string& topic) { + using namespace std::chrono; + auto start = std::chrono::high_resolution_clock::now(); + auto promise = createPromise(); + subscribeAsync(topic, "sub", {}, [&start, &promise](Result result, Consumer) { + auto timeMs = duration_cast(high_resolution_clock::now() - start).count(); + promise->set_value({result, timeMs}); + }); + return wait(promise); + } + + GetConnectionFuture getConnectionReal(const std::string& topic, size_t key) { + return ClientImpl::getConnection(topic, key); + } + + Result close() { + auto promise = createPromise(); + closeAsync([promise](Result result) { promise->set_value({result, 0L}); }); + return wait(promise).result; + } + + private: + static PromisePtr createPromise() { return std::make_shared>(); } + + static SyncOpResult wait(const PromisePtr& promise) { + using namespace std::chrono; + auto future = promise->get_future(); + auto status = future.wait_for(std::chrono::seconds(10)); + if (status == std::future_status::ready) { + return future.get(); + } else { + return {ResultUnknownError, -1L}; + } + } +}; + +} // namespace pulsar From 7891ac569eeca8e8cfeedf4ee005a26f98a0ca71 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Mon, 19 Feb 2024 10:18:11 +0800 Subject: [PATCH 40/54] Disable batch send for dlq producer. (#403) --- lib/ConsumerImpl.cc | 1 + tests/DeadLetterQueueTest.cc | 2 ++ 2 files changed, 3 insertions(+) diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index 0051ed6d..4181f05c 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -1736,6 +1736,7 @@ void ConsumerImpl::processPossibleToDLQ(const MessageId& messageId, ProcessDLQCa ProducerConfiguration producerConfiguration; producerConfiguration.setSchema(config_.getSchema()); producerConfiguration.setBlockIfQueueFull(false); + producerConfiguration.setBatchingEnabled(false); producerConfiguration.impl_->initialSubscriptionName = deadLetterPolicy_.getInitialSubscriptionName(); ClientImplPtr client = client_.lock(); diff --git a/tests/DeadLetterQueueTest.cc b/tests/DeadLetterQueueTest.cc index 1a747cc3..06cd1964 100644 --- a/tests/DeadLetterQueueTest.cc +++ b/tests/DeadLetterQueueTest.cc @@ -270,6 +270,8 @@ TEST_P(DeadLetterQueueTest, testSendDLQTriggerByRedeliverUnacknowledgedMessages) ASSERT_EQ(msg.getPartitionKey(), "p-key"); ASSERT_EQ(msg.getOrderingKey(), "o-key"); ASSERT_EQ(msg.getProperty("pk-1"), "pv-1"); + ASSERT_EQ(msg.getMessageId().batchSize(), 0); + ASSERT_EQ(msg.getMessageId().batchIndex(), -1); ASSERT_TRUE(msg.getProperty(SYSTEM_PROPERTY_REAL_TOPIC).find(topic_)); ASSERT_FALSE(msg.getProperty(PROPERTY_ORIGIN_MESSAGE_ID).empty()); } From 543e51c7ecd842056f93859defd23b851bfe842e Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Tue, 20 Feb 2024 19:02:31 -0800 Subject: [PATCH 41/54] [feat] PIP-188 Support blue-green migration (#402) ### Motivation Support blue-green migration pip-188 for cpp client ### Modifications - added blue-green client logic - register the producer instance in the producers map before sending produce creation command. This is required since broker could send topic migration command in the middle of creating the producer. --- lib/BinaryProtoLookupService.cc | 1 - lib/BinaryProtoLookupService.h | 8 +- lib/ClientConnection.cc | 57 +++++- lib/ClientConnection.h | 9 +- lib/ClientImpl.cc | 56 ++++-- lib/ClientImpl.h | 15 +- lib/HTTPLookupService.cc | 4 +- lib/HTTPLookupService.h | 6 +- lib/HandlerBase.cc | 16 +- lib/HandlerBase.h | 4 + lib/LookupService.h | 3 + lib/ProducerImpl.cc | 5 +- lib/RetryableLookupService.h | 4 + lib/ServiceNameResolver.h | 15 +- proto/PulsarApi.proto | 16 ++ run-unit-tests.sh | 3 + tests/ClientTest.cc | 17 +- tests/LookupServiceTest.cc | 32 ++-- tests/MockClientImpl.h | 6 +- tests/PulsarFriend.h | 12 +- tests/blue-green/docker-compose.yml | 152 ++++++++++++++++ .../extensibleLM/ExtensibleLoadManagerTest.cc | 169 ++++++++++++------ tests/extensibleLM/docker-compose.yml | 4 + 23 files changed, 487 insertions(+), 127 deletions(-) create mode 100644 tests/blue-green/docker-compose.yml diff --git a/lib/BinaryProtoLookupService.cc b/lib/BinaryProtoLookupService.cc index 2d9ffc42..489d8a27 100644 --- a/lib/BinaryProtoLookupService.cc +++ b/lib/BinaryProtoLookupService.cc @@ -22,7 +22,6 @@ #include "ConnectionPool.h" #include "LogUtils.h" #include "NamespaceName.h" -#include "ServiceNameResolver.h" #include "TopicName.h" DECLARE_LOG_OBJECT() diff --git a/lib/BinaryProtoLookupService.h b/lib/BinaryProtoLookupService.h index a3c059e4..6132825d 100644 --- a/lib/BinaryProtoLookupService.h +++ b/lib/BinaryProtoLookupService.h @@ -38,9 +38,9 @@ using GetSchemaPromisePtr = std::shared_ptr>; class PULSAR_PUBLIC BinaryProtoLookupService : public LookupService { public: - BinaryProtoLookupService(ServiceNameResolver& serviceNameResolver, ConnectionPool& pool, + BinaryProtoLookupService(const std::string& serviceUrl, ConnectionPool& pool, const ClientConfiguration& clientConfiguration) - : serviceNameResolver_(serviceNameResolver), + : serviceNameResolver_(serviceUrl), cnxPool_(pool), listenerName_(clientConfiguration.getListenerName()), maxLookupRedirects_(clientConfiguration.getMaxLookupRedirects()) {} @@ -54,6 +54,8 @@ class PULSAR_PUBLIC BinaryProtoLookupService : public LookupService { Future getSchema(const TopicNamePtr& topicName, const std::string& version) override; + ServiceNameResolver& getServiceNameResolver() override { return serviceNameResolver_; } + protected: // Mark findBroker as protected to make it accessible from test. LookupResultFuture findBroker(const std::string& address, bool authoritative, const std::string& topic, @@ -63,7 +65,7 @@ class PULSAR_PUBLIC BinaryProtoLookupService : public LookupService { std::mutex mutex_; uint64_t requestIdGenerator_ = 0; - ServiceNameResolver& serviceNameResolver_; + ServiceNameResolver serviceNameResolver_; ConnectionPool& cnxPool_; std::string listenerName_; const int32_t maxLookupRedirects_; diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index 00041b2a..0beb739c 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -403,7 +403,8 @@ void ClientConnection::handleTcpConnected(const ASIO_ERROR& err, tcp::resolver:: LOG_INFO(cnxString_ << "Connected to broker"); } else { LOG_INFO(cnxString_ << "Connected to broker through proxy. Logical broker: " << logicalAddress_ - << ", proxy: " << proxyServiceUrl_); + << ", proxy: " << proxyServiceUrl_ + << ", physical address:" << physicalAddress_); } Lock lock(mutex_); @@ -945,6 +946,10 @@ void ClientConnection::handleIncomingCommand(BaseCommand& incomingCmd) { handleError(incomingCmd.error()); break; + case BaseCommand::TOPIC_MIGRATED: + handleTopicMigrated(incomingCmd.topicmigrated()); + break; + case BaseCommand::CLOSE_PRODUCER: handleCloseProducer(incomingCmd.close_producer()); break; @@ -1761,6 +1766,56 @@ void ClientConnection::handleError(const proto::CommandError& error) { } } +std::string ClientConnection::getMigratedBrokerServiceUrl( + const proto::CommandTopicMigrated& commandTopicMigrated) { + if (tlsSocket_) { + if (commandTopicMigrated.has_brokerserviceurltls()) { + return commandTopicMigrated.brokerserviceurltls(); + } + } else if (commandTopicMigrated.has_brokerserviceurl()) { + return commandTopicMigrated.brokerserviceurl(); + } + return ""; +} + +void ClientConnection::handleTopicMigrated(const proto::CommandTopicMigrated& commandTopicMigrated) { + const long resourceId = commandTopicMigrated.resource_id(); + const std::string migratedBrokerServiceUrl = getMigratedBrokerServiceUrl(commandTopicMigrated); + + if (migratedBrokerServiceUrl.empty()) { + LOG_WARN("Failed to find the migrated broker url for resource:" + << resourceId + << (commandTopicMigrated.has_brokerserviceurl() + ? ", migratedBrokerUrl: " + commandTopicMigrated.brokerserviceurl() + : "") + << (commandTopicMigrated.has_brokerserviceurltls() + ? ", migratedBrokerUrlTls: " + commandTopicMigrated.brokerserviceurltls() + : "")); + return; + } + + Lock lock(mutex_); + if (commandTopicMigrated.resource_type() == proto::CommandTopicMigrated_ResourceType_Producer) { + auto it = producers_.find(resourceId); + if (it != producers_.end()) { + auto producer = it->second.lock(); + producer->setRedirectedClusterURI(migratedBrokerServiceUrl); + LOG_INFO("Producer id:" << resourceId << " is migrated to " << migratedBrokerServiceUrl); + } else { + LOG_WARN("Got invalid producer Id in topicMigrated command: " << resourceId); + } + } else { + auto it = consumers_.find(resourceId); + if (it != consumers_.end()) { + auto consumer = it->second.lock(); + consumer->setRedirectedClusterURI(migratedBrokerServiceUrl); + LOG_INFO("Consumer id:" << resourceId << " is migrated to " << migratedBrokerServiceUrl); + } else { + LOG_WARN("Got invalid consumer Id in topicMigrated command: " << resourceId); + } + } +} + boost::optional ClientConnection::getAssignedBrokerServiceUrl( const proto::CommandCloseProducer& closeProducer) { if (tlsSocket_) { diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index b16fc694..3c83b4db 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -85,6 +85,7 @@ class BrokerEntryMetadata; class CommandActiveConsumerChange; class CommandAckResponse; class CommandMessage; +class CommandTopicMigrated; class CommandCloseConsumer; class CommandCloseProducer; class CommandConnected; @@ -414,6 +415,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this getAssignedBrokerServiceUrl( - const proto::CommandCloseProducer& closeProducer); - boost::optional getAssignedBrokerServiceUrl( - const proto::CommandCloseConsumer& closeConsumer); + boost::optional getAssignedBrokerServiceUrl(const proto::CommandCloseProducer&); + boost::optional getAssignedBrokerServiceUrl(const proto::CommandCloseConsumer&); + std::string getMigratedBrokerServiceUrl(const proto::CommandTopicMigrated&); }; } // namespace pulsar diff --git a/lib/ClientImpl.cc b/lib/ClientImpl.cc index ae339731..3d19c426 100644 --- a/lib/ClientImpl.cc +++ b/lib/ClientImpl.cc @@ -79,8 +79,8 @@ typedef std::vector StringList; ClientImpl::ClientImpl(const std::string& serviceUrl, const ClientConfiguration& clientConfiguration) : mutex_(), state_(Open), - serviceNameResolver_(serviceUrl), - clientConfiguration_(ClientConfiguration(clientConfiguration).setUseTls(serviceNameResolver_.useTls())), + clientConfiguration_(ClientConfiguration(clientConfiguration) + .setUseTls(ServiceNameResolver::useTls(ServiceURI(serviceUrl)))), memoryLimitController_(clientConfiguration.getMemoryLimit()), ioExecutorProvider_(std::make_shared(clientConfiguration_.getIOThreads())), listenerExecutorProvider_( @@ -98,25 +98,28 @@ ClientImpl::ClientImpl(const std::string& serviceUrl, const ClientConfiguration& if (loggerFactory) { LogUtils::setLoggerFactory(std::move(loggerFactory)); } + lookupServicePtr_ = createLookup(serviceUrl); +} + +ClientImpl::~ClientImpl() { shutdown(); } +LookupServicePtr ClientImpl::createLookup(const std::string& serviceUrl) { LookupServicePtr underlyingLookupServicePtr; - if (serviceNameResolver_.useHttp()) { + if (ServiceNameResolver::useHttp(ServiceURI(serviceUrl))) { LOG_DEBUG("Using HTTP Lookup"); underlyingLookupServicePtr = std::make_shared( - std::ref(serviceNameResolver_), std::cref(clientConfiguration_), - std::cref(clientConfiguration_.getAuthPtr())); + serviceUrl, std::cref(clientConfiguration_), std::cref(clientConfiguration_.getAuthPtr())); } else { LOG_DEBUG("Using Binary Lookup"); underlyingLookupServicePtr = std::make_shared( - std::ref(serviceNameResolver_), std::ref(pool_), std::cref(clientConfiguration_)); + serviceUrl, std::ref(pool_), std::cref(clientConfiguration_)); } - lookupServicePtr_ = RetryableLookupService::create( + auto lookupServicePtr = RetryableLookupService::create( underlyingLookupServicePtr, clientConfiguration_.impl_->operationTimeout, ioExecutorProvider_); + return lookupServicePtr; } -ClientImpl::~ClientImpl() { shutdown(); } - const ClientConfiguration& ClientImpl::conf() const { return clientConfiguration_; } MemoryLimitController& ClientImpl::getMemoryLimitController() { return memoryLimitController_; } @@ -129,7 +132,21 @@ ExecutorServiceProviderPtr ClientImpl::getPartitionListenerExecutorProvider() { return partitionListenerExecutorProvider_; } -LookupServicePtr ClientImpl::getLookup() { return lookupServicePtr_; } +LookupServicePtr ClientImpl::getLookup(const std::string& redirectedClusterURI) { + if (redirectedClusterURI.empty()) { + return lookupServicePtr_; + } + + Lock lock(mutex_); + auto it = redirectedClusterLookupServicePtrs_.find(redirectedClusterURI); + if (it == redirectedClusterLookupServicePtrs_.end()) { + auto lookup = createLookup(redirectedClusterURI); + redirectedClusterLookupServicePtrs_.emplace(redirectedClusterURI, lookup); + return lookup; + } + + return it->second; +} void ClientImpl::createProducerAsync(const std::string& topic, ProducerConfiguration conf, CreateProducerCallback callback, bool autoDownloadSchema) { @@ -517,7 +534,8 @@ void ClientImpl::handleConsumerCreated(Result result, ConsumerImplBaseWeakPtr co } } -GetConnectionFuture ClientImpl::getConnection(const std::string& topic, size_t key) { +GetConnectionFuture ClientImpl::getConnection(const std::string& redirectedClusterURI, + const std::string& topic, size_t key) { Promise promise; const auto topicNamePtr = TopicName::get(topic); @@ -528,7 +546,8 @@ GetConnectionFuture ClientImpl::getConnection(const std::string& topic, size_t k } auto self = shared_from_this(); - lookupServicePtr_->getBroker(*topicNamePtr) + getLookup(redirectedClusterURI) + ->getBroker(*topicNamePtr) .addListener([this, self, promise, key](Result result, const LookupService::LookupResult& data) { if (result != ResultOk) { promise.setFailed(result); @@ -554,16 +573,18 @@ GetConnectionFuture ClientImpl::getConnection(const std::string& topic, size_t k return promise.getFuture(); } -const std::string& ClientImpl::getPhysicalAddress(const std::string& logicalAddress) { +const std::string& ClientImpl::getPhysicalAddress(const std::string& redirectedClusterURI, + const std::string& logicalAddress) { if (useProxy_) { - return serviceNameResolver_.resolveHost(); + return getLookup(redirectedClusterURI)->getServiceNameResolver().resolveHost(); } else { return logicalAddress; } } -GetConnectionFuture ClientImpl::connect(const std::string& logicalAddress, size_t key) { - const auto& physicalAddress = getPhysicalAddress(logicalAddress); +GetConnectionFuture ClientImpl::connect(const std::string& redirectedClusterURI, + const std::string& logicalAddress, size_t key) { + const auto& physicalAddress = getPhysicalAddress(redirectedClusterURI, logicalAddress); Promise promise; pool_.getConnectionAsync(logicalAddress, physicalAddress, key) .addListener([promise](Result result, const ClientConnectionWeakPtr& weakCnx) { @@ -633,6 +654,9 @@ void ClientImpl::closeAsync(CloseCallback callback) { memoryLimitController_.close(); lookupServicePtr_->close(); + for (const auto& it : redirectedClusterLookupServicePtrs_) { + it.second->close(); + } auto producers = producers_.move(); auto consumers = consumers_.move(); diff --git a/lib/ClientImpl.h b/lib/ClientImpl.h index 7126542b..27cde3a1 100644 --- a/lib/ClientImpl.h +++ b/lib/ClientImpl.h @@ -97,9 +97,11 @@ class ClientImpl : public std::enable_shared_from_this { void getPartitionsForTopicAsync(const std::string& topic, GetPartitionsCallback callback); // Use virtual method to test - virtual GetConnectionFuture getConnection(const std::string& topic, size_t key); + virtual GetConnectionFuture getConnection(const std::string& redirectedClusterURI, + const std::string& topic, size_t key); - GetConnectionFuture connect(const std::string& logicalAddress, size_t key); + GetConnectionFuture connect(const std::string& redirectedClusterURI, const std::string& logicalAddress, + size_t key); void closeAsync(CloseCallback callback); void shutdown(); @@ -119,7 +121,7 @@ class ClientImpl : public std::enable_shared_from_this { ExecutorServiceProviderPtr getIOExecutorProvider(); ExecutorServiceProviderPtr getListenerExecutorProvider(); ExecutorServiceProviderPtr getPartitionListenerExecutorProvider(); - LookupServicePtr getLookup(); + LookupServicePtr getLookup(const std::string& redirectedClusterURI = ""); void cleanupProducer(ProducerImplBase* address) { producers_.remove(address); } @@ -165,7 +167,10 @@ class ClientImpl : public std::enable_shared_from_this { const std::string& consumerName, const ConsumerConfiguration& conf, SubscribeCallback callback); - const std::string& getPhysicalAddress(const std::string& logicalAddress); + const std::string& getPhysicalAddress(const std::string& redirectedClusterURI, + const std::string& logicalAddress); + + LookupServicePtr createLookup(const std::string& serviceUrl); static std::string getClientVersion(const ClientConfiguration& clientConfiguration); @@ -179,7 +184,6 @@ class ClientImpl : public std::enable_shared_from_this { std::mutex mutex_; State state_; - ServiceNameResolver serviceNameResolver_; ClientConfiguration clientConfiguration_; MemoryLimitController memoryLimitController_; @@ -188,6 +192,7 @@ class ClientImpl : public std::enable_shared_from_this { ExecutorServiceProviderPtr partitionListenerExecutorProvider_; LookupServicePtr lookupServicePtr_; + std::unordered_map redirectedClusterLookupServicePtrs_; ConnectionPool pool_; uint64_t producerIdGenerator_; diff --git a/lib/HTTPLookupService.cc b/lib/HTTPLookupService.cc index 0959af2a..93b9db44 100644 --- a/lib/HTTPLookupService.cc +++ b/lib/HTTPLookupService.cc @@ -46,11 +46,11 @@ const static std::string ADMIN_PATH_V2 = "/admin/v2/"; const static std::string PARTITION_METHOD_NAME = "partitions"; const static int NUMBER_OF_LOOKUP_THREADS = 1; -HTTPLookupService::HTTPLookupService(ServiceNameResolver &serviceNameResolver, +HTTPLookupService::HTTPLookupService(const std::string &serviceUrl, const ClientConfiguration &clientConfiguration, const AuthenticationPtr &authData) : executorProvider_(std::make_shared(NUMBER_OF_LOOKUP_THREADS)), - serviceNameResolver_(serviceNameResolver), + serviceNameResolver_(serviceUrl), authenticationPtr_(authData), lookupTimeoutInSeconds_(clientConfiguration.getOperationTimeoutSeconds()), maxLookupRedirects_(clientConfiguration.getMaxLookupRedirects()), diff --git a/lib/HTTPLookupService.h b/lib/HTTPLookupService.h index cf0b0ad1..17dd110e 100644 --- a/lib/HTTPLookupService.h +++ b/lib/HTTPLookupService.h @@ -40,7 +40,7 @@ class HTTPLookupService : public LookupService, public std::enable_shared_from_t typedef Promise LookupPromise; ExecutorServiceProviderPtr executorProvider_; - ServiceNameResolver& serviceNameResolver_; + ServiceNameResolver serviceNameResolver_; AuthenticationPtr authenticationPtr_; int lookupTimeoutInSeconds_; const int maxLookupRedirects_; @@ -64,7 +64,7 @@ class HTTPLookupService : public LookupService, public std::enable_shared_from_t Result sendHTTPRequest(std::string completeUrl, std::string& responseData, long& responseCode); public: - HTTPLookupService(ServiceNameResolver&, const ClientConfiguration&, const AuthenticationPtr&); + HTTPLookupService(const std::string&, const ClientConfiguration&, const AuthenticationPtr&); LookupResultFuture getBroker(const TopicName& topicName) override; @@ -74,6 +74,8 @@ class HTTPLookupService : public LookupService, public std::enable_shared_from_t Future getTopicsOfNamespaceAsync( const NamespaceNamePtr& nsName, CommandGetTopicsOfNamespace_Mode mode) override; + + ServiceNameResolver& getServiceNameResolver() override { return serviceNameResolver_; } }; } // namespace pulsar diff --git a/lib/HandlerBase.cc b/lib/HandlerBase.cc index 52e20d2d..46b918f6 100644 --- a/lib/HandlerBase.cc +++ b/lib/HandlerBase.cc @@ -43,7 +43,8 @@ HandlerBase::HandlerBase(const ClientImplPtr& client, const std::string& topic, epoch_(0), timer_(executor_->createDeadlineTimer()), creationTimer_(executor_->createDeadlineTimer()), - reconnectionPending_(false) {} + reconnectionPending_(false), + redirectedClusterURI_("") {} HandlerBase::~HandlerBase() { ASIO_ERROR ignored; @@ -88,9 +89,9 @@ void HandlerBase::grabCnx() { grabCnx(boost::none); } Future HandlerBase::getConnection( const ClientImplPtr& client, const boost::optional& assignedBrokerUrl) { if (assignedBrokerUrl && client->getLookupCount() > 0) { - return client->connect(assignedBrokerUrl.get(), connectionKeySuffix_); + return client->connect(getRedirectedClusterURI(), assignedBrokerUrl.get(), connectionKeySuffix_); } else { - return client->getConnection(topic(), connectionKeySuffix_); + return client->getConnection(getRedirectedClusterURI(), topic(), connectionKeySuffix_); } } @@ -209,4 +210,13 @@ Result HandlerBase::convertToTimeoutIfNecessary(Result result, ptime startTimest } } +void HandlerBase::setRedirectedClusterURI(const std::string& serviceUrl) { + Lock lock(mutex_); + redirectedClusterURI_ = serviceUrl; +} +const std::string& HandlerBase::getRedirectedClusterURI() { + Lock lock(mutex_); + return redirectedClusterURI_; +} + } // namespace pulsar diff --git a/lib/HandlerBase.h b/lib/HandlerBase.h index 32e124a2..415e234c 100644 --- a/lib/HandlerBase.h +++ b/lib/HandlerBase.h @@ -52,6 +52,8 @@ class HandlerBase : public std::enable_shared_from_this { ClientConnectionWeakPtr getCnx() const; void setCnx(const ClientConnectionPtr& cnx); void resetCnx() { setCnx(nullptr); } + void setRedirectedClusterURI(const std::string& serviceUrl); + const std::string& getRedirectedClusterURI(); protected: /* @@ -145,6 +147,8 @@ class HandlerBase : public std::enable_shared_from_this { mutable std::mutex connectionMutex_; std::atomic reconnectionPending_; ClientConnectionWeakPtr connection_; + std::string redirectedClusterURI_; + friend class ClientConnection; friend class PulsarFriend; }; diff --git a/lib/LookupService.h b/lib/LookupService.h index b50d1f82..684984fc 100644 --- a/lib/LookupService.h +++ b/lib/LookupService.h @@ -29,6 +29,7 @@ #include "Future.h" #include "LookupDataResult.h" #include "ProtoApiEnums.h" +#include "ServiceNameResolver.h" namespace pulsar { using NamespaceTopicsPtr = std::shared_ptr>; @@ -86,6 +87,8 @@ class LookupService { virtual Future getSchema(const TopicNamePtr& topicName, const std::string& version = "") = 0; + virtual ServiceNameResolver& getServiceNameResolver() = 0; + virtual ~LookupService() {} virtual void close() {} diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index 890d476c..f84c255e 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -144,7 +144,11 @@ Future ProducerImpl::connectionOpened(const ClientConnectionPtr& c return promise.getFuture(); } + LOG_INFO("Creating producer for topic:" << topic() << ", producerName:" << producerName_ << " on " + << cnx->cnxString()); ClientImplPtr client = client_.lock(); + cnx->registerProducer(producerId_, shared_from_this()); + int requestId = client->newRequestId(); SharedBuffer cmd = Commands::newProducer(topic(), producerId_, producerName_, requestId, @@ -214,7 +218,6 @@ Result ProducerImpl::handleCreateProducer(const ClientConnectionPtr& cnx, Result // set the cnx pointer so that new messages will be sent immediately LOG_INFO(getName() << "Created producer on broker " << cnx->cnxString()); - cnx->registerProducer(producerId_, shared_from_this()); producerName_ = responseData.producerName; schemaVersion_ = responseData.schemaVersion; producerStr_ = "[" + topic() + ", " + producerName_ + "] "; diff --git a/lib/RetryableLookupService.h b/lib/RetryableLookupService.h index 561855f9..8bc40bf3 100644 --- a/lib/RetryableLookupService.h +++ b/lib/RetryableLookupService.h @@ -76,6 +76,10 @@ class RetryableLookupService : public LookupService { }); } + ServiceNameResolver& getServiceNameResolver() override { + return lookupService_->getServiceNameResolver(); + } + private: const std::shared_ptr lookupService_; RetryableOperationCachePtr lookupCache_; diff --git a/lib/ServiceNameResolver.h b/lib/ServiceNameResolver.h index 8457d0e1..e6b2523b 100644 --- a/lib/ServiceNameResolver.h +++ b/lib/ServiceNameResolver.h @@ -36,14 +36,17 @@ class ServiceNameResolver { ServiceNameResolver(const ServiceNameResolver&) = delete; ServiceNameResolver& operator=(const ServiceNameResolver&) = delete; - bool useTls() const noexcept { - return serviceUri_.getScheme() == PulsarScheme::PULSAR_SSL || - serviceUri_.getScheme() == PulsarScheme::HTTPS; + bool useTls() const noexcept { return useTls(serviceUri_); } + + static bool useTls(const ServiceURI& serviceUri) noexcept { + return serviceUri.getScheme() == PulsarScheme::PULSAR_SSL || + serviceUri.getScheme() == PulsarScheme::HTTPS; } - bool useHttp() const noexcept { - return serviceUri_.getScheme() == PulsarScheme::HTTP || - serviceUri_.getScheme() == PulsarScheme::HTTPS; + bool useHttp() const noexcept { return useTls(serviceUri_); } + + static bool useHttp(const ServiceURI& serviceUri) noexcept { + return serviceUri.getScheme() == PulsarScheme::HTTP || serviceUri.getScheme() == PulsarScheme::HTTPS; } const std::string& resolveHost() { diff --git a/proto/PulsarApi.proto b/proto/PulsarApi.proto index a2548f3a..4e207913 100644 --- a/proto/PulsarApi.proto +++ b/proto/PulsarApi.proto @@ -262,6 +262,7 @@ enum ProtocolVersion { v17 = 17; // Added support ack receipt v18 = 18; // Add client support for broker entry metadata v19 = 19; // Add CommandTcClientConnectRequest and CommandTcClientConnectResponse + v20 = 20; // Add client support for topic migration redirection CommandTopicMigrated } message CommandConnect { @@ -620,6 +621,18 @@ message CommandReachedEndOfTopic { required uint64 consumer_id = 1; } +message CommandTopicMigrated { + enum ResourceType { + Producer = 0; + Consumer = 1; + } + required uint64 resource_id = 1; + required ResourceType resource_type = 2; + optional string brokerServiceUrl = 3; + optional string brokerServiceUrlTls = 4; + +} + message CommandCloseProducer { required uint64 producer_id = 1; required uint64 request_id = 2; @@ -1029,6 +1042,7 @@ message BaseCommand { WATCH_TOPIC_UPDATE = 66; WATCH_TOPIC_LIST_CLOSE = 67; + TOPIC_MIGRATED = 68; } @@ -1110,4 +1124,6 @@ message BaseCommand { optional CommandWatchTopicListSuccess watchTopicListSuccess = 65; optional CommandWatchTopicUpdate watchTopicUpdate = 66; optional CommandWatchTopicListClose watchTopicListClose = 67; + + optional CommandTopicMigrated topicMigrated = 68; } diff --git a/run-unit-tests.sh b/run-unit-tests.sh index 226789c5..698ca62c 100755 --- a/run-unit-tests.sh +++ b/run-unit-tests.sh @@ -33,9 +33,12 @@ export https_proxy= # Run ExtensibleLoadManager tests docker compose -f tests/extensibleLM/docker-compose.yml up -d +docker compose -f tests/blue-green/docker-compose.yml up -d until curl http://localhost:8080/metrics > /dev/null 2>&1 ; do sleep 1; done +until curl http://localhost:8081/metrics > /dev/null 2>&1 ; do sleep 1; done sleep 5 $CMAKE_BUILD_DIRECTORY/tests/ExtensibleLoadManagerTest +docker compose -f tests/blue-green/docker-compose.yml down docker compose -f tests/extensibleLM/docker-compose.yml down # Run OAuth2 tests diff --git a/tests/ClientTest.cc b/tests/ClientTest.cc index 1e8e67d5..b5142860 100644 --- a/tests/ClientTest.cc +++ b/tests/ClientTest.cc @@ -443,7 +443,8 @@ TEST(ClientTest, testRetryUntilSucceed) { EXPECT_CALL(*clientImpl, getConnection).Times((kFailCount + 1) * 2); std::atomic_int count{0}; ON_CALL(*clientImpl, getConnection) - .WillByDefault([&clientImpl, &count](const std::string &topic, size_t index) { + .WillByDefault([&clientImpl, &count](const std::string &redirectedClusterURI, + const std::string &topic, size_t index) { if (count++ < kFailCount) { return GetConnectionFuture::failed(ResultRetryable); } @@ -461,9 +462,10 @@ TEST(ClientTest, testRetryTimeout) { auto clientImpl = std::make_shared(lookupUrl, ClientConfiguration().setOperationTimeoutSeconds(2)); EXPECT_CALL(*clientImpl, getConnection).Times(AtLeast(2 * 2)); - ON_CALL(*clientImpl, getConnection).WillByDefault([](const std::string &topic, size_t index) { - return GetConnectionFuture::failed(ResultRetryable); - }); + ON_CALL(*clientImpl, getConnection) + .WillByDefault([](const std::string &redirectedClusterURI, const std::string &topic, size_t index) { + return GetConnectionFuture::failed(ResultRetryable); + }); auto topic = "client-test-retry-timeout"; { @@ -484,9 +486,10 @@ TEST(ClientTest, testNoRetry) { auto clientImpl = std::make_shared(lookupUrl, ClientConfiguration().setOperationTimeoutSeconds(100)); EXPECT_CALL(*clientImpl, getConnection).Times(2); - ON_CALL(*clientImpl, getConnection).WillByDefault([](const std::string &, size_t) { - return GetConnectionFuture::failed(ResultAuthenticationError); - }); + ON_CALL(*clientImpl, getConnection) + .WillByDefault([](const std::string &redirectedClusterURI, const std::string &, size_t) { + return GetConnectionFuture::failed(ResultAuthenticationError); + }); auto topic = "client-test-no-retry"; { diff --git a/tests/LookupServiceTest.cc b/tests/LookupServiceTest.cc index 0fe13851..924acd43 100644 --- a/tests/LookupServiceTest.cc +++ b/tests/LookupServiceTest.cc @@ -84,8 +84,7 @@ TEST(LookupServiceTest, basicLookup) { ClientConfiguration conf; ExecutorServiceProviderPtr ioExecutorProvider_(std::make_shared(1)); ConnectionPool pool_(conf, ioExecutorProvider_, authData, ""); - ServiceNameResolver serviceNameResolver(url); - BinaryProtoLookupService lookupService(serviceNameResolver, pool_, conf); + BinaryProtoLookupService lookupService(url, pool_, conf); TopicNamePtr topicName = TopicName::get("topic"); @@ -148,26 +147,24 @@ static void testMultiAddresses(LookupService& lookupService) { TEST(LookupServiceTest, testMultiAddresses) { ConnectionPool pool({}, std::make_shared(1), AuthFactory::Disabled(), ""); - ServiceNameResolver serviceNameResolver("pulsar://localhost,localhost:9999"); ClientConfiguration conf; - BinaryProtoLookupService binaryLookupService(serviceNameResolver, pool, conf); + BinaryProtoLookupService binaryLookupService("pulsar://localhost,localhost:9999", pool, conf); testMultiAddresses(binaryLookupService); // HTTPLookupService calls shared_from_this() internally, we must create a shared pointer to test - ServiceNameResolver serviceNameResolverForHttp("http://localhost,localhost:9999"); auto httpLookupServicePtr = std::make_shared( - std::ref(serviceNameResolverForHttp), ClientConfiguration{}, AuthFactory::Disabled()); + "http://localhost,localhost:9999", ClientConfiguration{}, AuthFactory::Disabled()); testMultiAddresses(*httpLookupServicePtr); } TEST(LookupServiceTest, testRetry) { auto executorProvider = std::make_shared(1); ConnectionPool pool({}, executorProvider, AuthFactory::Disabled(), ""); - ServiceNameResolver serviceNameResolver("pulsar://localhost:9999,localhost"); ClientConfiguration conf; auto lookupService = RetryableLookupService::create( - std::make_shared(serviceNameResolver, pool, conf), std::chrono::seconds(30), - executorProvider); + std::make_shared("pulsar://localhost:9999,localhost", pool, conf), + std::chrono::seconds(30), executorProvider); + ServiceNameResolver& serviceNameResolver = lookupService->getServiceNameResolver(); PulsarFriend::setServiceUrlIndex(serviceNameResolver, 0); auto topicNamePtr = TopicName::get("lookup-service-test-retry"); @@ -196,12 +193,12 @@ TEST(LookupServiceTest, testRetry) { TEST(LookupServiceTest, testTimeout) { auto executorProvider = std::make_shared(1); ConnectionPool pool({}, executorProvider, AuthFactory::Disabled(), ""); - ServiceNameResolver serviceNameResolver("pulsar://localhost:9990,localhost:9902,localhost:9904"); ClientConfiguration conf; constexpr int timeoutInSeconds = 2; auto lookupService = RetryableLookupService::create( - std::make_shared(serviceNameResolver, pool, conf), + std::make_shared("pulsar://localhost:9990,localhost:9902,localhost:9904", + pool, conf), std::chrono::seconds(timeoutInSeconds), executorProvider); auto topicNamePtr = TopicName::get("lookup-service-test-retry"); @@ -467,9 +464,9 @@ INSTANTIATE_TEST_SUITE_P(Pulsar, LookupServiceTest, ::testing::Values(binaryLook class BinaryProtoLookupServiceRedirectTestHelper : public BinaryProtoLookupService { public: - BinaryProtoLookupServiceRedirectTestHelper(ServiceNameResolver& serviceNameResolver, ConnectionPool& pool, + BinaryProtoLookupServiceRedirectTestHelper(const std::string& serviceUrl, ConnectionPool& pool, const ClientConfiguration& clientConfiguration) - : BinaryProtoLookupService(serviceNameResolver, pool, clientConfiguration) {} + : BinaryProtoLookupService(serviceUrl, pool, clientConfiguration) {} LookupResultFuture findBroker(const std::string& address, bool authoritative, const std::string& topic, size_t redirectCount) { @@ -484,14 +481,13 @@ TEST(LookupServiceTest, testRedirectionLimit) { conf.setMaxLookupRedirects(redirect_limit); ExecutorServiceProviderPtr ioExecutorProvider_(std::make_shared(1)); ConnectionPool pool_(conf, ioExecutorProvider_, authData, ""); - std::string url = "pulsar://localhost:6650"; - ServiceNameResolver serviceNameResolver(url); - BinaryProtoLookupServiceRedirectTestHelper lookupService(serviceNameResolver, pool_, conf); + string url = "pulsar://localhost:6650"; + BinaryProtoLookupServiceRedirectTestHelper lookupService(url, pool_, conf); const auto topicNamePtr = TopicName::get("topic"); for (auto idx = 0; idx < redirect_limit + 5; ++idx) { - auto future = - lookupService.findBroker(serviceNameResolver.resolveHost(), false, topicNamePtr->toString(), idx); + auto future = lookupService.findBroker(lookupService.getServiceNameResolver().resolveHost(), false, + topicNamePtr->toString(), idx); LookupService::LookupResult lookupResult; auto result = future.get(lookupResult); diff --git a/tests/MockClientImpl.h b/tests/MockClientImpl.h index aa4208a9..074808fc 100644 --- a/tests/MockClientImpl.h +++ b/tests/MockClientImpl.h @@ -39,8 +39,8 @@ class MockClientImpl : public ClientImpl { MockClientImpl(const std::string& serviceUrl, ClientConfiguration conf = {}) : ClientImpl(serviceUrl, conf) {} - MOCK_METHOD((Future), getConnection, (const std::string&, size_t), - (override)); + MOCK_METHOD((Future), getConnection, + (const std::string&, const std::string&, size_t), (override)); SyncOpResult createProducer(const std::string& topic) { using namespace std::chrono; @@ -65,7 +65,7 @@ class MockClientImpl : public ClientImpl { } GetConnectionFuture getConnectionReal(const std::string& topic, size_t key) { - return ClientImpl::getConnection(topic, key); + return ClientImpl::getConnection("", topic, key); } Result close() { diff --git a/tests/PulsarFriend.h b/tests/PulsarFriend.h index 73778842..bfa11ef1 100644 --- a/tests/PulsarFriend.h +++ b/tests/PulsarFriend.h @@ -26,6 +26,7 @@ #include "lib/ClientImpl.h" #include "lib/ConsumerConfigurationImpl.h" #include "lib/ConsumerImpl.h" +#include "lib/LookupService.h" #include "lib/MessageImpl.h" #include "lib/MultiTopicsConsumerImpl.h" #include "lib/NamespaceName.h" @@ -38,6 +39,7 @@ using std::string; namespace pulsar { +using ClientConnectionWeakPtr = std::weak_ptr; class PulsarFriend { public: static ProducerStatsImplPtr getProducerStatsPtr(Producer producer) { @@ -166,6 +168,14 @@ class PulsarFriend { static ClientConnectionWeakPtr getClientConnection(HandlerBase& handler) { return handler.connection_; } + static std::string getConnectionPhysicalAddress(HandlerBase& handler) { + auto cnx = handler.connection_.lock(); + if (cnx) { + return cnx->physicalAddress_; + } + return ""; + } + static void setClientConnection(HandlerBase& handler, ClientConnectionWeakPtr conn) { handler.connection_ = conn; } @@ -177,7 +187,7 @@ class PulsarFriend { static void setServiceUrlIndex(ServiceNameResolver& resolver, size_t index) { resolver.index_ = index; } static void setServiceUrlIndex(const Client& client, size_t index) { - setServiceUrlIndex(client.impl_->serviceNameResolver_, index); + setServiceUrlIndex(client.impl_->lookupServicePtr_->getServiceNameResolver(), index); } static proto::MessageMetadata& getMessageMetadata(Message& message) { return message.impl_->metadata; } diff --git a/tests/blue-green/docker-compose.yml b/tests/blue-green/docker-compose.yml new file mode 100644 index 00000000..b2d22ebf --- /dev/null +++ b/tests/blue-green/docker-compose.yml @@ -0,0 +1,152 @@ +version: '3' +networks: + green-pulsar: + driver: bridge +services: + # Start ZooKeeper + zookeeper: + image: apachepulsar/pulsar:latest + container_name: green-zookeeper + restart: on-failure + networks: + - green-pulsar + environment: + - metadataStoreUrl=zk:zookeeper:2181 + - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m + command: > + bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \ + bin/generate-zookeeper-config.sh conf/zookeeper.conf && \ + exec bin/pulsar zookeeper" + healthcheck: + test: ["CMD", "bin/pulsar-zookeeper-ruok.sh"] + interval: 10s + timeout: 5s + retries: 30 + + # Initialize cluster metadata + pulsar-init: + container_name: green-pulsar-init + hostname: pulsar-init + image: apachepulsar/pulsar:latest + networks: + - green-pulsar + environment: + - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m + command: > + bin/pulsar initialize-cluster-metadata \ + --cluster cluster-a \ + --zookeeper zookeeper:2181 \ + --configuration-store zookeeper:2181 \ + --web-service-url http://broker-1:8080 \ + --broker-service-url pulsar://broker-1:6650 + depends_on: + zookeeper: + condition: service_healthy + + # Start bookie + bookie: + image: apachepulsar/pulsar:latest + container_name: green-bookie + restart: on-failure + networks: + - green-pulsar + environment: + - clusterName=cluster-a + - zkServers=zookeeper:2181 + - metadataServiceUri=metadata-store:zk:zookeeper:2181 + - advertisedAddress=bookie + - BOOKIE_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m + depends_on: + zookeeper: + condition: service_healthy + pulsar-init: + condition: service_completed_successfully + command: bash -c "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + + proxy: + image: apachepulsar/pulsar:latest + container_name: green-proxy + hostname: proxy + restart: on-failure + networks: + - green-pulsar + environment: + - metadataStoreUrl=zk:zookeeper:2181 + - zookeeperServers=zookeeper:2181 + - clusterName=cluster-a + - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m + ports: + - "8081:8080" + - "6651:6650" + depends_on: + broker-1: + condition: service_started + broker-2: + condition: service_started + command: bash -c "bin/apply-config-from-env.py conf/proxy.conf && exec bin/pulsar proxy" + + # Start broker 1 + broker-1: + image: apachepulsar/pulsar:latest + container_name: green-broker-1 + hostname: broker-1 + restart: on-failure + networks: + - green-pulsar + environment: + - metadataStoreUrl=zk:zookeeper:2181 + - zookeeperServers=zookeeper:2181 + - clusterName=cluster-a + - managedLedgerDefaultEnsembleSize=1 + - managedLedgerDefaultWriteQuorum=1 + - managedLedgerDefaultAckQuorum=1 + - advertisedAddress=green-broker-1 + - internalListenerName=internal + - advertisedListeners=internal:pulsar://green-broker-1:6650 + - PULSAR_MEM=-Xms256m -Xmx256m -XX:MaxDirectMemorySize=56m + # Load Manager. Here uses the extensible load balancer, sets the unloading strategy to TransferShedder, and enables debug mode. + - loadManagerClassName=org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl + - loadBalancerLoadSheddingStrategy=org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder + - loadBalancerSheddingEnabled=false + - loadBalancerDebugModeEnabled=true + - brokerServiceCompactionThresholdInBytes=1000000 + - PULSAR_PREFIX_defaultNumberOfNamespaceBundles=1 + depends_on: + zookeeper: + condition: service_healthy + bookie: + condition: service_started + command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" + + # Start broker 2 + broker-2: + image: apachepulsar/pulsar:latest + container_name: green-broker-2 + hostname: broker-2 + restart: on-failure + networks: + - green-pulsar + environment: + - metadataStoreUrl=zk:zookeeper:2181 + - zookeeperServers=zookeeper:2181 + - clusterName=cluster-a + - managedLedgerDefaultEnsembleSize=1 + - managedLedgerDefaultWriteQuorum=1 + - managedLedgerDefaultAckQuorum=1 + - advertisedAddress=green-broker-2 + - internalListenerName=internal + - advertisedListeners=internal:pulsar://green-broker-2:6650 + - PULSAR_MEM=-Xms256m -Xmx256m -XX:MaxDirectMemorySize=56m + # Load Manager. Here uses the extensible load balancer, sets the unloading strategy to TransferShedder, and enables debug mode. + - loadManagerClassName=org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl + - loadBalancerLoadSheddingStrategy=org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder + - loadBalancerSheddingEnabled=false + - loadBalancerDebugModeEnabled=true + - brokerServiceCompactionThresholdInBytes=1000000 + - PULSAR_PREFIX_defaultNumberOfNamespaceBundles=1 + depends_on: + zookeeper: + condition: service_healthy + bookie: + condition: service_started + command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" \ No newline at end of file diff --git a/tests/extensibleLM/ExtensibleLoadManagerTest.cc b/tests/extensibleLM/ExtensibleLoadManagerTest.cc index a6379f6e..f4e2c81c 100644 --- a/tests/extensibleLM/ExtensibleLoadManagerTest.cc +++ b/tests/extensibleLM/ExtensibleLoadManagerTest.cc @@ -20,6 +20,7 @@ #include #include +#include #include "include/pulsar/Client.h" #include "lib/LogUtils.h" @@ -35,21 +36,43 @@ bool checkTime() { const static auto start = std::chrono::high_resolution_clock::now(); auto end = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(end - start).count(); - return duration < 180 * 1000; + return duration < 300 * 1000; } TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { - const static std::string adminUrl = "http://localhost:8080/"; - const static std::string topicName = - "persistent://public/unload-test/topic-1" + std::to_string(time(NULL)); + const static std::string blueAdminUrl = "http://localhost:8080/"; + const static std::string greenAdminUrl = "http://localhost:8081/"; + const static std::string topicNameSuffix = std::to_string(time(NULL)); + const static std::string topicName = "persistent://public/unload-test/topic-" + topicNameSuffix; ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { - std::string url = adminUrl + "admin/v2/namespaces/public/unload-test?bundles=1"; + std::string url = blueAdminUrl + "admin/v2/clusters/cluster-a/migrate?migrated=false"; + int res = makePostRequest(url, R"( + { + "serviceUrl": "http://localhost:8081", + "serviceUrlTls":"https://localhost:8085", + "brokerServiceUrl": "pulsar://localhost:6651", + "brokerServiceUrlTls": "pulsar+ssl://localhost:6655" + })"); + LOG_INFO("res:" << res); + return res == 200; + })); + + ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { + std::string url = blueAdminUrl + "admin/v2/namespaces/public/unload-test?bundles=1"; int res = makePutRequest(url, ""); return res == 204 || res == 409; })); - Client client{"pulsar://localhost:6650"}; + ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { + std::string url = greenAdminUrl + "admin/v2/namespaces/public/unload-test?bundles=1"; + int res = makePutRequest(url, ""); + return res == 204 || res == 409; + })); + + ClientConfiguration conf; + conf.setIOThreads(8); + Client client{"pulsar://localhost:6650", conf}; Producer producer; ProducerConfiguration producerConfiguration; Result producerResult = client.createProducer(topicName, producerConfiguration, producer); @@ -58,24 +81,25 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { Result consumerResult = client.subscribe(topicName, "sub", consumer); ASSERT_EQ(consumerResult, ResultOk); - Semaphore firstUnloadSemaphore(0); - Semaphore secondUnloadSemaphore(0); - Semaphore halfPubWaitSemaphore(0); - const int msgCount = 10; - int produced = 0; + Semaphore unloadSemaphore(0); + Semaphore pubWaitSemaphore(0); + Semaphore migrationSemaphore(0); + + const int msgCount = 20; + SynchronizedHashMap producedMsgs; auto produce = [&]() { int i = 0; while (i < msgCount && checkTime()) { - if (i == 3) { - firstUnloadSemaphore.acquire(); + if (i == 3 || i == 8 || i == 17) { + unloadSemaphore.acquire(); } - if (i == 5) { - halfPubWaitSemaphore.release(); + if (i == 5 || i == 15) { + pubWaitSemaphore.release(); } - if (i == 8) { - secondUnloadSemaphore.acquire(); + if (i == 12) { + migrationSemaphore.acquire(); } std::string content = std::to_string(i); @@ -86,37 +110,33 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { return sendResult == ResultOk; })); - LOG_INFO("produced index:" << i); - produced++; + LOG_INFO("produced i:" << i); + producedMsgs.emplace(i, i); i++; } LOG_INFO("producer finished"); }; - - int consumed = 0; + std::atomic stopConsumer(false); + SynchronizedHashMap consumedMsgs; auto consume = [&]() { Message receivedMsg; - int i = 0; - while (i < msgCount && checkTime()) { - ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { - Result receiveResult = - consumer.receive(receivedMsg, 1000); // Assumed that we wait 1000 ms for each message - return receiveResult == ResultOk; - })); - LOG_INFO("received index:" << i); - - int id = std::stoi(receivedMsg.getDataAsString()); - if (id < i) { + while (checkTime()) { + if (stopConsumer && producedMsgs.size() == msgCount && consumedMsgs.size() == msgCount) { + break; + } + Result receiveResult = + consumer.receive(receivedMsg, 1000); // Assumed that we wait 1000 ms for each message + if (receiveResult != ResultOk) { continue; } + int i = std::stoi(receivedMsg.getDataAsString()); + LOG_INFO("received i:" << i); ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { Result ackResult = consumer.acknowledge(receivedMsg); return ackResult == ResultOk; })); - LOG_INFO("acked index:" << i); - - consumed++; - i++; + LOG_INFO("acked i:" << i); + consumedMsgs.emplace(i, i); } LOG_INFO("consumer finished"); }; @@ -124,7 +144,8 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { std::thread produceThread(produce); std::thread consumeThread(consume); - auto unload = [&] { + auto unload = [&](bool migrated) { + const std::string &adminUrl = migrated ? greenAdminUrl : blueAdminUrl; auto clientImplPtr = PulsarFriend::getClientImplPtr(client); auto &consumerImpl = PulsarFriend::getConsumerImpl(consumer); auto &producerImpl = PulsarFriend::getProducerImpl(producer); @@ -135,15 +156,17 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { ASSERT_TRUE(waitUntil(std::chrono::seconds(30), [&] { return consumerImpl.isConnected() && producerImpl.isConnected(); })); - std::string url = adminUrl + "lookup/v2/topic/persistent/public/unload-test/topic-1"; + std::string url = + adminUrl + "lookup/v2/topic/persistent/public/unload-test/topic" + topicNameSuffix; std::string responseDataBeforeUnload; int res = makeGetRequest(url, responseDataBeforeUnload); if (res != 200) { continue; } - destinationBroker = responseDataBeforeUnload.find("broker-2") == std::string::npos - ? "broker-2:8080" - : "broker-1:8080"; + std::string prefix = migrated ? "green-" : ""; + destinationBroker = + prefix + (responseDataBeforeUnload.find("broker-2") == std::string::npos ? "broker-2:8080" + : "broker-1:8080"); lookupCountBeforeUnload = clientImplPtr->getLookupCount(); ASSERT_TRUE(lookupCountBeforeUnload > 0); @@ -163,31 +186,69 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { [&] { return consumerImpl.isConnected() && producerImpl.isConnected(); })); std::string responseDataAfterUnload; ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { - url = adminUrl + "lookup/v2/topic/persistent/public/unload-test/topic-1"; + url = adminUrl + "lookup/v2/topic/persistent/public/unload-test/topic" + topicNameSuffix; res = makeGetRequest(url, responseDataAfterUnload); return res == 200 && responseDataAfterUnload.find(destinationBroker) != std::string::npos; })); LOG_INFO("after lookup responseData:" << responseDataAfterUnload << ",res:" << res); auto lookupCountAfterUnload = clientImplPtr->getLookupCount(); - ASSERT_EQ(lookupCountBeforeUnload, lookupCountAfterUnload); + if (lookupCountBeforeUnload != lookupCountAfterUnload) { + continue; + } break; } }; - LOG_INFO("starting first unload"); - unload(); - firstUnloadSemaphore.release(); - halfPubWaitSemaphore.acquire(); - LOG_INFO("starting second unload"); - unload(); - secondUnloadSemaphore.release(); + LOG_INFO("#### starting first unload ####"); + unload(false); + unloadSemaphore.release(); + pubWaitSemaphore.acquire(); + LOG_INFO("#### starting second unload ####"); + unload(false); + unloadSemaphore.release(); - produceThread.join(); + LOG_INFO("#### migrating the cluster ####"); + migrationSemaphore.release(); + ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { + std::string url = blueAdminUrl + "admin/v2/clusters/cluster-a/migrate?migrated=true"; + int res = makePostRequest(url, R"({ + "serviceUrl": "http://localhost:8081", + "serviceUrlTls":"https://localhost:8085", + "brokerServiceUrl": "pulsar://localhost:6651", + "brokerServiceUrlTls": "pulsar+ssl://localhost:6655" + })"); + LOG_INFO("res:" << res); + return res == 200; + })); + ASSERT_TRUE(waitUntil(std::chrono::seconds(130), [&] { + auto &consumerImpl = PulsarFriend::getConsumerImpl(consumer); + auto &producerImpl = PulsarFriend::getProducerImpl(producer); + auto consumerConnAddress = PulsarFriend::getConnectionPhysicalAddress(consumerImpl); + auto producerConnAddress = PulsarFriend::getConnectionPhysicalAddress(producerImpl); + return consumerImpl.isConnected() && producerImpl.isConnected() && + consumerConnAddress.find("6651") != std::string::npos && + producerConnAddress.find("6651") != std::string::npos; + })); + pubWaitSemaphore.acquire(); + LOG_INFO("#### starting third unload after migration ####"); + unload(true); + unloadSemaphore.release(); + + stopConsumer = true; consumeThread.join(); - ASSERT_EQ(consumed, msgCount); - ASSERT_EQ(produced, msgCount); - ASSERT_TRUE(checkTime()) << "timed out"; + produceThread.join(); + ASSERT_EQ(producedMsgs.size(), msgCount); + ASSERT_EQ(consumedMsgs.size(), msgCount); + for (int i = 0; i < msgCount; i++) { + producedMsgs.remove(i); + consumedMsgs.remove(i); + } + ASSERT_EQ(producedMsgs.size(), 0); + ASSERT_EQ(consumedMsgs.size(), 0); + client.close(); + + ASSERT_TRUE(checkTime()) << "timed out"; } int main(int argc, char *argv[]) { diff --git a/tests/extensibleLM/docker-compose.yml b/tests/extensibleLM/docker-compose.yml index 8d3c33a3..14876395 100644 --- a/tests/extensibleLM/docker-compose.yml +++ b/tests/extensibleLM/docker-compose.yml @@ -109,6 +109,8 @@ services: - loadBalancerLoadSheddingStrategy=org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder - loadBalancerSheddingEnabled=false - loadBalancerDebugModeEnabled=true + - clusterMigrationCheckDurationSeconds=1 + - brokerServiceCompactionThresholdInBytes=1000000 - PULSAR_PREFIX_defaultNumberOfNamespaceBundles=1 depends_on: zookeeper: @@ -141,6 +143,8 @@ services: - loadBalancerLoadSheddingStrategy=org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder - loadBalancerSheddingEnabled=false - loadBalancerDebugModeEnabled=true + - clusterMigrationCheckDurationSeconds=1 + - brokerServiceCompactionThresholdInBytes=1000000 - PULSAR_PREFIX_defaultNumberOfNamespaceBundles=1 depends_on: zookeeper: From 9f96eb98ff52cebc8b4360de5e6d44b42b89c6e9 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 29 Feb 2024 10:56:20 +0800 Subject: [PATCH 42/54] Fix blue-green migration might be stuck due to an existing reconnection (#406) Fixes https://github.com/apache/pulsar-client-cpp/issues/405 ### Motivation After triggering a blue-green migration, the socket will be disconnected and then schedule a reconnection to the blue cluster. However, the blue cluster could never respond with a response for Producer or Subscribe commands. Take producer as example, it means `connectionOpened` will not complete and `reconnectionPending_` will not become false. Then, after receiving a `CommandProducerClose` command from the blue cluster, a new reconnection will be scheduled to the green cluster but it will be skipped because `reconnectionPending_` is true, which means the previous `connectionOpened` future is not completed until the 30s timeout is reached. ``` 2024-02-26 06:09:30.251 INFO [139737465607744] HandlerBase:101 | [persistent://public/unload-test/topic-1708927732, sub, 0] Ignoring reconnection attempt since there's already a pending reconnection 2024-02-26 06:10:00.035 WARN [139737859880512] ProducerImpl:291 | [persistent://public/unload-test/topic-1708927732, cluster-a-0-0] Failed to reconnect producer: TimeOut ``` ### Modifications When receiving the `TOPIC_MIGRATED` command, cancel the pending `Producer` and `Subscribe` commands so that `connectionOpened` will fail with a retryable error. In the next time of reconnection, the green cluster will be connected. Fix the `ExtensibleLoadManagerTest` with a more strict timeout check. After this change, it will pass in about 3 seconds locally, while in CI even if it passed, it takes about 70 seconds before. Besides, fix the possible crash on macOS when closing the client, see https://github.com/apache/pulsar-client-cpp/issues/405#issuecomment-1963969215 --- lib/ClientConnection.cc | 12 +++++ lib/ClientConnection.h | 2 + lib/ConsumerImpl.cc | 4 +- lib/HandlerBase.h | 9 ++++ lib/ProducerImpl.cc | 3 +- lib/stats/ConsumerStatsBase.h | 1 + lib/stats/ConsumerStatsImpl.h | 4 ++ tests/PulsarFriend.h | 2 +- .../extensibleLM/ExtensibleLoadManagerTest.cc | 45 +++++++++++-------- 9 files changed, 60 insertions(+), 22 deletions(-) diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index 0beb739c..abd38b46 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -1800,6 +1800,7 @@ void ClientConnection::handleTopicMigrated(const proto::CommandTopicMigrated& co if (it != producers_.end()) { auto producer = it->second.lock(); producer->setRedirectedClusterURI(migratedBrokerServiceUrl); + unsafeRemovePendingRequest(producer->firstRequestIdAfterConnect()); LOG_INFO("Producer id:" << resourceId << " is migrated to " << migratedBrokerServiceUrl); } else { LOG_WARN("Got invalid producer Id in topicMigrated command: " << resourceId); @@ -1809,6 +1810,7 @@ void ClientConnection::handleTopicMigrated(const proto::CommandTopicMigrated& co if (it != consumers_.end()) { auto consumer = it->second.lock(); consumer->setRedirectedClusterURI(migratedBrokerServiceUrl); + unsafeRemovePendingRequest(consumer->firstRequestIdAfterConnect()); LOG_INFO("Consumer id:" << resourceId << " is migrated to " << migratedBrokerServiceUrl); } else { LOG_WARN("Got invalid consumer Id in topicMigrated command: " << resourceId); @@ -2027,4 +2029,14 @@ void ClientConnection::handleAckResponse(const proto::CommandAckResponse& respon } } +void ClientConnection::unsafeRemovePendingRequest(long requestId) { + auto it = pendingRequests_.find(requestId); + if (it != pendingRequests_.end()) { + it->second.promise.setFailed(ResultDisconnected); + ASIO_ERROR ec; + it->second.timer->cancel(ec); + pendingRequests_.erase(it); + } +} + } // namespace pulsar diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index 3c83b4db..418cb2f9 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -426,6 +426,8 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this getAssignedBrokerServiceUrl(const proto::CommandCloseProducer&); boost::optional getAssignedBrokerServiceUrl(const proto::CommandCloseConsumer&); std::string getMigratedBrokerServiceUrl(const proto::CommandTopicMigrated&); + // This method must be called when `mutex_` is held + void unsafeRemovePendingRequest(long requestId); }; } // namespace pulsar diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index 4181f05c..fa33e280 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -251,7 +251,7 @@ Future ConsumerImpl::connectionOpened(const ClientConnectionPtr& c unAckedMessageTrackerPtr_->clear(); ClientImplPtr client = client_.lock(); - uint64_t requestId = client->newRequestId(); + long requestId = client->newRequestId(); SharedBuffer cmd = Commands::newSubscribe( topic(), subscription_, consumerId_, requestId, getSubType(), getConsumerName(), subscriptionMode_, subscribeMessageId, readCompacted_, config_.getProperties(), config_.getSubscriptionProperties(), @@ -260,6 +260,7 @@ Future ConsumerImpl::connectionOpened(const ClientConnectionPtr& c // Keep a reference to ensure object is kept alive. auto self = get_shared_this_ptr(); + setFirstRequestIdAfterConnect(requestId); cnx->sendRequestWithId(cmd, requestId) .addListener([this, self, cnx, promise](Result result, const ResponseData& responseData) { Result handleResult = handleCreateConsumer(cnx, result); @@ -1719,6 +1720,7 @@ void ConsumerImpl::cancelTimers() noexcept { batchReceiveTimer_->cancel(ec); checkExpiredChunkedTimer_->cancel(ec); unAckedMessageTrackerPtr_->stop(); + consumerStatsBasePtr_->stop(); } void ConsumerImpl::processPossibleToDLQ(const MessageId& messageId, ProcessDLQCallBack cb) { diff --git a/lib/HandlerBase.h b/lib/HandlerBase.h index 415e234c..b68dce32 100644 --- a/lib/HandlerBase.h +++ b/lib/HandlerBase.h @@ -100,6 +100,10 @@ class HandlerBase : public std::enable_shared_from_this { const std::string& topic() const { return *topic_; } const std::shared_ptr& getTopicPtr() const { return topic_; } + long firstRequestIdAfterConnect() const { + return firstRequestIdAfterConnect_.load(std::memory_order_acquire); + } + private: const std::shared_ptr topic_; @@ -140,6 +144,10 @@ class HandlerBase : public std::enable_shared_from_this { Result convertToTimeoutIfNecessary(Result result, ptime startTimestamp) const; + void setFirstRequestIdAfterConnect(long requestId) { + firstRequestIdAfterConnect_.store(requestId, std::memory_order_release); + } + private: DeadlineTimerPtr timer_; DeadlineTimerPtr creationTimer_; @@ -148,6 +156,7 @@ class HandlerBase : public std::enable_shared_from_this { std::atomic reconnectionPending_; ClientConnectionWeakPtr connection_; std::string redirectedClusterURI_; + std::atomic firstRequestIdAfterConnect_{-1L}; friend class ClientConnection; friend class PulsarFriend; diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index f84c255e..4399ce5f 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -149,7 +149,7 @@ Future ProducerImpl::connectionOpened(const ClientConnectionPtr& c ClientImplPtr client = client_.lock(); cnx->registerProducer(producerId_, shared_from_this()); - int requestId = client->newRequestId(); + long requestId = client->newRequestId(); SharedBuffer cmd = Commands::newProducer(topic(), producerId_, producerName_, requestId, conf_.getProperties(), conf_.getSchema(), epoch_, @@ -159,6 +159,7 @@ Future ProducerImpl::connectionOpened(const ClientConnectionPtr& c // Keep a reference to ensure object is kept alive. auto self = shared_from_this(); + setFirstRequestIdAfterConnect(requestId); cnx->sendRequestWithId(cmd, requestId) .addListener([this, self, cnx, promise](Result result, const ResponseData& responseData) { Result handleResult = handleCreateProducer(cnx, result, responseData); diff --git a/lib/stats/ConsumerStatsBase.h b/lib/stats/ConsumerStatsBase.h index 6e2e71b8..e8e17c78 100644 --- a/lib/stats/ConsumerStatsBase.h +++ b/lib/stats/ConsumerStatsBase.h @@ -28,6 +28,7 @@ namespace pulsar { class ConsumerStatsBase { public: virtual void start() {} + virtual void stop() {} virtual void receivedMessage(Message&, Result) = 0; virtual void messageAcknowledged(Result, CommandAck_AckType, uint32_t ackNums = 1) = 0; virtual ~ConsumerStatsBase() {} diff --git a/lib/stats/ConsumerStatsImpl.h b/lib/stats/ConsumerStatsImpl.h index 03f3a474..3333ea85 100644 --- a/lib/stats/ConsumerStatsImpl.h +++ b/lib/stats/ConsumerStatsImpl.h @@ -59,6 +59,10 @@ class ConsumerStatsImpl : public std::enable_shared_from_this ConsumerStatsImpl(const ConsumerStatsImpl& stats); void flushAndReset(const ASIO_ERROR&); void start() override; + void stop() override { + ASIO_ERROR error; + timer_->cancel(error); + } void receivedMessage(Message&, Result) override; void messageAcknowledged(Result, CommandAck_AckType, uint32_t ackNums) override; virtual ~ConsumerStatsImpl(); diff --git a/tests/PulsarFriend.h b/tests/PulsarFriend.h index bfa11ef1..e7084050 100644 --- a/tests/PulsarFriend.h +++ b/tests/PulsarFriend.h @@ -169,7 +169,7 @@ class PulsarFriend { static ClientConnectionWeakPtr getClientConnection(HandlerBase& handler) { return handler.connection_; } static std::string getConnectionPhysicalAddress(HandlerBase& handler) { - auto cnx = handler.connection_.lock(); + auto cnx = handler.getCnx().lock(); if (cnx) { return cnx->physicalAddress_; } diff --git a/tests/extensibleLM/ExtensibleLoadManagerTest.cc b/tests/extensibleLM/ExtensibleLoadManagerTest.cc index f4e2c81c..c7e5aa0f 100644 --- a/tests/extensibleLM/ExtensibleLoadManagerTest.cc +++ b/tests/extensibleLM/ExtensibleLoadManagerTest.cc @@ -25,6 +25,7 @@ #include "include/pulsar/Client.h" #include "lib/LogUtils.h" #include "lib/Semaphore.h" +#include "lib/TimeUtils.h" #include "tests/HttpHelper.h" #include "tests/PulsarFriend.h" @@ -40,6 +41,9 @@ bool checkTime() { } TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { + constexpr auto maxWaitTime = std::chrono::seconds(5); + constexpr long maxWaitTimeMs = maxWaitTime.count() * 1000L; + const static std::string blueAdminUrl = "http://localhost:8080/"; const static std::string greenAdminUrl = "http://localhost:8081/"; const static std::string topicNameSuffix = std::to_string(time(NULL)); @@ -105,12 +109,13 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { std::string content = std::to_string(i); const auto msg = MessageBuilder().setContent(content).build(); - ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { - Result sendResult = producer.send(msg); - return sendResult == ResultOk; - })); + auto start = TimeUtils::currentTimeMillis(); + Result sendResult = producer.send(msg); + auto elapsed = TimeUtils::currentTimeMillis() - start; + LOG_INFO("produce i: " << i << " " << elapsed << " ms"); + ASSERT_EQ(sendResult, ResultOk); + ASSERT_TRUE(elapsed < maxWaitTimeMs); - LOG_INFO("produced i:" << i); producedMsgs.emplace(i, i); i++; } @@ -124,18 +129,20 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { if (stopConsumer && producedMsgs.size() == msgCount && consumedMsgs.size() == msgCount) { break; } - Result receiveResult = - consumer.receive(receivedMsg, 1000); // Assumed that we wait 1000 ms for each message - if (receiveResult != ResultOk) { - continue; - } + auto start = TimeUtils::currentTimeMillis(); + Result receiveResult = consumer.receive(receivedMsg, maxWaitTimeMs); + auto elapsed = TimeUtils::currentTimeMillis() - start; int i = std::stoi(receivedMsg.getDataAsString()); - LOG_INFO("received i:" << i); - ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { - Result ackResult = consumer.acknowledge(receivedMsg); - return ackResult == ResultOk; - })); - LOG_INFO("acked i:" << i); + LOG_INFO("receive i: " << i << " " << elapsed << " ms"); + ASSERT_EQ(receiveResult, ResultOk); + ASSERT_TRUE(elapsed < maxWaitTimeMs); + + start = TimeUtils::currentTimeMillis(); + Result ackResult = consumer.acknowledge(receivedMsg); + elapsed = TimeUtils::currentTimeMillis() - start; + LOG_INFO("acked i:" << i << " " << elapsed << " ms"); + ASSERT_TRUE(elapsed < maxWaitTimeMs); + ASSERT_EQ(ackResult, ResultOk); consumedMsgs.emplace(i, i); } LOG_INFO("consumer finished"); @@ -153,7 +160,7 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { std::string destinationBroker; while (checkTime()) { // make sure producers and consumers are ready - ASSERT_TRUE(waitUntil(std::chrono::seconds(30), + ASSERT_TRUE(waitUntil(maxWaitTime, [&] { return consumerImpl.isConnected() && producerImpl.isConnected(); })); std::string url = @@ -182,7 +189,7 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { } // make sure producers and consumers are ready - ASSERT_TRUE(waitUntil(std::chrono::seconds(30), + ASSERT_TRUE(waitUntil(maxWaitTime, [&] { return consumerImpl.isConnected() && producerImpl.isConnected(); })); std::string responseDataAfterUnload; ASSERT_TRUE(waitUntil(std::chrono::seconds(60), [&] { @@ -220,7 +227,7 @@ TEST(ExtensibleLoadManagerTest, testPubSubWhileUnloading) { LOG_INFO("res:" << res); return res == 200; })); - ASSERT_TRUE(waitUntil(std::chrono::seconds(130), [&] { + ASSERT_TRUE(waitUntil(maxWaitTime, [&] { auto &consumerImpl = PulsarFriend::getConsumerImpl(consumer); auto &producerImpl = PulsarFriend::getProducerImpl(producer); auto consumerConnAddress = PulsarFriend::getConnectionPhysicalAddress(consumerImpl); From 6e1ad1742ac98149c56faa32dc39d421dfe8ebd8 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 1 Mar 2024 01:34:21 +0800 Subject: [PATCH 43/54] Update dependencies to latest versions (#404) * Update dependencies to latest versions Restore protobuf version * Fix build script * Fix Linux build Dockerfiles * Downgrade Boost to 1.83 --- dependencies.yaml | 15 ++++++++------- pkg/apk/Dockerfile | 2 +- pkg/deb/Dockerfile | 2 +- pkg/mac/build-static-library.sh | 4 ++++ pkg/rpm/Dockerfile | 4 ++-- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index a02ebdca..8d338e4d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -17,11 +17,12 @@ # under the License. # -boost : 1.83.0 -cmake: 3.24.2 +# Note: GCC 4.8 is incompatible with Boost >= 1.84 for the missed std::align +boost: 1.83.0 +cmake: 3.28.3 protobuf: 3.20.0 -zlib: 1.2.12 -zstd: 1.5.2 -snappy: 1.1.9 -openssl: 1.1.1v -curl: 8.4.0 +zlib: 1.3.1 +zstd: 1.5.5 +snappy: 1.1.10 +openssl: 1.1.1w +curl: 8.6.0 diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index 3cfe6583..d7d87181 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -104,7 +104,7 @@ RUN CURL_VERSION=$(dep-version.py curl) && \ curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ tar xfz curl-${CURL_VERSION}.tar.gz && \ cd curl-${CURL_VERSION} && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \ make -j8 && make install && \ rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} diff --git a/pkg/deb/Dockerfile b/pkg/deb/Dockerfile index 7873cdf5..502b0934 100644 --- a/pkg/deb/Dockerfile +++ b/pkg/deb/Dockerfile @@ -109,7 +109,7 @@ RUN CURL_VERSION=$(dep-version.py curl) && \ curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ tar xfz curl-${CURL_VERSION}.tar.gz && \ cd curl-${CURL_VERSION} && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \ make -j8 && make install && \ rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh index d49e199e..4b97ac73 100755 --- a/pkg/mac/build-static-library.sh +++ b/pkg/mac/build-static-library.sh @@ -140,6 +140,9 @@ if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz tar zxf ${SNAPPY_VERSION}.tar.gz pushd snappy-${SNAPPY_VERSION} + # Without this patch, snappy 1.10 will report a sign-compare error, which cannot be suppressed with the -Wno-sign-compare option in CI + curl -O -L https://raw.githubusercontent.com/microsoft/vcpkg/2024.02.14/ports/snappy/no-werror.patch + patch Date: Fri, 1 Mar 2024 10:39:12 +0800 Subject: [PATCH 44/54] Release 3.5.0 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index b10ed391..1545d966 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.5.0-pre +3.5.0 From e2cacb7dfb57b6d059b49fead2e1611548ff89b0 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 4 Mar 2024 01:24:09 +0800 Subject: [PATCH 45/54] Fix the incompatibility with Clang and C++20 (#408) ### Motivation When I built with Clang and C++20, there were the following errors: > ISO C++20 considers use of overloaded operator '==' (with operand types 'pulsar::NamespaceName' and 'pulsar::NamespaceName') to be ambiguous despite there being a unique best viable function [-Werror,-Wambiguous-reversed-operator] See the detailed answer here: https://stackoverflow.com/questions/60386792/c20-comparison-warning-about-ambiguous-reversed-operator ### Modifications Make the member functions of `operator=` const in `TopicName` and `NamespaceName` and add a workflow to cover this case. --- .github/workflows/ci-pr-validation.yaml | 7 ++++++- lib/NamespaceName.cc | 2 +- lib/NamespaceName.h | 2 +- lib/TopicName.cc | 2 +- lib/TopicName.h | 2 +- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 56309e9a..61134aae 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -289,7 +289,6 @@ jobs: timeout-minutes: 120 name: Build CPP Client on macOS runs-on: macos-12 - needs: unit-tests steps: - name: checkout uses: actions/checkout@v3 @@ -306,6 +305,12 @@ jobs: run: | cmake --build ./build-macos --parallel --config Release + - name: Build with C++20 + shell: bash + run: | + cmake -B build-macos-cpp20 -DCMAKE_CXX_STANDARD=20 + cmake --build build-macos-cpp20 -j8 + cpp-build-macos-static: timeout-minutes: 120 name: Build CPP Client on macOS with static dependencies diff --git a/lib/NamespaceName.cc b/lib/NamespaceName.cc index f493db25..d635480c 100644 --- a/lib/NamespaceName.cc +++ b/lib/NamespaceName.cc @@ -93,7 +93,7 @@ std::shared_ptr NamespaceName::getNamespaceObject() { return std::shared_ptr(this); } -bool NamespaceName::operator==(const NamespaceName& namespaceName) { +bool NamespaceName::operator==(const NamespaceName& namespaceName) const { return this->namespace_.compare(namespaceName.namespace_) == 0; } diff --git a/lib/NamespaceName.h b/lib/NamespaceName.h index ce451a20..d1f5765b 100644 --- a/lib/NamespaceName.h +++ b/lib/NamespaceName.h @@ -37,7 +37,7 @@ class PULSAR_PUBLIC NamespaceName : public ServiceUnitId { static std::shared_ptr get(const std::string& property, const std::string& cluster, const std::string& namespaceName); static std::shared_ptr get(const std::string& property, const std::string& namespaceName); - bool operator==(const NamespaceName& namespaceName); + bool operator==(const NamespaceName& namespaceName) const; bool isV2(); std::string toString(); diff --git a/lib/TopicName.cc b/lib/TopicName.cc index 5b892fc8..487eee53 100644 --- a/lib/TopicName.cc +++ b/lib/TopicName.cc @@ -164,7 +164,7 @@ std::string TopicName::getLocalName() { return localName_; } std::string TopicName::getEncodedLocalName() const { return getEncodedName(localName_); } -bool TopicName::operator==(const TopicName& other) { +bool TopicName::operator==(const TopicName& other) const { return (this->topicName_.compare(other.topicName_) == 0); } diff --git a/lib/TopicName.h b/lib/TopicName.h index 8cc9cb53..bee8138a 100644 --- a/lib/TopicName.h +++ b/lib/TopicName.h @@ -65,7 +65,7 @@ class PULSAR_PUBLIC TopicName : public ServiceUnitId { NamespaceNamePtr getNamespaceName(); int getPartitionIndex() const noexcept { return partition_; } static std::shared_ptr get(const std::string& topicName); - bool operator==(const TopicName& other); + bool operator==(const TopicName& other) const; static std::string getEncodedName(const std::string& nameBeforeEncoding); static std::string removeDomain(const std::string& topicName); static bool containsDomain(const std::string& topicName); From d0a322704cea8dba5925d5e3760061bdc0d84282 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Mon, 4 Mar 2024 18:28:08 -0800 Subject: [PATCH 46/54] [improve] add physicalAddress as part of connection pool key (#411) ### Motivation Context: https://github.com/apache/pulsar/pull/22085/files#r1497008116 Currently, the connection pool key does not include physicalAddress (currently logicalAddress + keySuffix). This can be a problem when the same logicalAddresses are in the migrated(green) cluster. (the connection pool will return the connection to the old(blue) cluster) ### Modifications Add physicalAddress as part of the connection pool key --- lib/ClientConnection.cc | 2 +- lib/ConnectionPool.cc | 15 +++++++++++---- lib/ConnectionPool.h | 3 ++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index abd38b46..04202d31 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -1321,7 +1321,7 @@ void ClientConnection::close(Result result, bool detach) { } // Remove the connection from the pool before completing any promise if (detach) { - pool_.remove(logicalAddress_ + "-" + std::to_string(poolIndex_), this); + pool_.remove(logicalAddress_, physicalAddress_, poolIndex_, this); } auto self = shared_from_this(); diff --git a/lib/ConnectionPool.cc b/lib/ConnectionPool.cc index 4cc8883a..9a614ed1 100644 --- a/lib/ConnectionPool.cc +++ b/lib/ConnectionPool.cc @@ -66,6 +66,13 @@ bool ConnectionPool::close() { return true; } +static const std::string getKey(const std::string& logicalAddress, const std::string& physicalAddress, + size_t keySuffix) { + std::stringstream ss; + ss << logicalAddress << '-' << physicalAddress << '-' << keySuffix; + return ss.str(); +} + Future ConnectionPool::getConnectionAsync(const std::string& logicalAddress, const std::string& physicalAddress, size_t keySuffix) { @@ -77,9 +84,7 @@ Future ConnectionPool::getConnectionAsync(const std::unique_lock lock(mutex_); - std::stringstream ss; - ss << logicalAddress << '-' << keySuffix; - const std::string key = ss.str(); + auto key = getKey(logicalAddress, physicalAddress, keySuffix); PoolMap::iterator cnxIt = pool_.find(key); if (cnxIt != pool_.end()) { @@ -127,7 +132,9 @@ Future ConnectionPool::getConnectionAsync(const return future; } -void ConnectionPool::remove(const std::string& key, ClientConnection* value) { +void ConnectionPool::remove(const std::string& logicalAddress, const std::string& physicalAddress, + size_t keySuffix, ClientConnection* value) { + auto key = getKey(logicalAddress, physicalAddress, keySuffix); std::lock_guard lock(mutex_); auto it = pool_.find(key); if (it != pool_.end() && it->second.get() == value) { diff --git a/lib/ConnectionPool.h b/lib/ConnectionPool.h index a51205b6..e044d62c 100644 --- a/lib/ConnectionPool.h +++ b/lib/ConnectionPool.h @@ -51,7 +51,8 @@ class PULSAR_PUBLIC ConnectionPool { */ bool close(); - void remove(const std::string& key, ClientConnection* value); + void remove(const std::string& logicalAddress, const std::string& physicalAddress, size_t keySuffix, + ClientConnection* value); /** * Get a connection from the pool. From 747c1863018cb345b4e2bbbaaf7a6549c9c6b429 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 5 Mar 2024 12:35:52 +0800 Subject: [PATCH 47/54] [CI] Pin the clang-format version to 11 (#412) ### Motivation The `clang-format` tool on `ubuntu:22.04` runner was upgraded to 14 recently, whose rule is slightly different from 11. ### Modifications Use clang-format-action for code format check and specify the version to 11. --- .github/workflows/ci-pr-validation.yaml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 61134aae..d9dc2ca5 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -29,8 +29,20 @@ concurrency: jobs: + formatting-check: + name: Formatting Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run clang-format style check for C/C++/Protobuf programs. + uses: jidicula/clang-format-action@v4.11.0 + with: + clang-format-version: '11' + exclude-regex: '.*\.(proto|hpp)' + wireshark-dissector-build: name: Build the Wireshark dissector + needs: formatting-check runs-on: ${{ matrix.os }} timeout-minutes: 60 strategy: @@ -61,6 +73,7 @@ jobs: unit-tests: name: Run unit tests + needs: formatting-check runs-on: ubuntu-22.04 timeout-minutes: 120 @@ -81,8 +94,8 @@ jobs: ./vcpkg/vcpkg format-manifest vcpkg.json if [[ $(git diff | wc -l) -gt 0 ]]; then echo "Please run `./vcpkg/vcpkg format-manifest vcpkg.json` to reformat vcpkg.json" + exit 1 fi - make check-format - name: Build tests run: | @@ -103,6 +116,7 @@ jobs: cpp20-build: name: Build with the C++20 standard + needs: formatting-check runs-on: ubuntu-22.04 timeout-minutes: 60 @@ -288,6 +302,7 @@ jobs: cpp-build-macos: timeout-minutes: 120 name: Build CPP Client on macOS + needs: formatting-check runs-on: macos-12 steps: - name: checkout @@ -337,7 +352,7 @@ jobs: check-completion: name: Check Completion runs-on: ubuntu-latest - needs: [wireshark-dissector-build, unit-tests, cpp20-build, cpp-build-windows, package, cpp-build-macos] + needs: [formatting-check, wireshark-dissector-build, unit-tests, cpp20-build, cpp-build-windows, package, cpp-build-macos] steps: - run: true From 436050092930b3eca73883cc7623f8cebf08f636 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Mon, 11 Mar 2024 15:46:13 +0900 Subject: [PATCH 48/54] Change return code of MultiTopicsConsumerImpl::closeAsync after unsubscribe or close (#413) Fixes #88 ### Motivation https://github.com/apache/pulsar-client-cpp/pull/338 fixed `ConsumerImpl::closeAsync` so that closing after it was already closed or unsubscribed returns `ResultOk` instead of `ResultAlreadyClosed`. However, `MultiTopicsConsumerImpl::closeAsync` still returns `ResultAlreadyClosed`. This seems to be a change omission. --- lib/MultiTopicsConsumerImpl.cc | 4 ++-- tests/BasicEndToEndTest.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/MultiTopicsConsumerImpl.cc b/lib/MultiTopicsConsumerImpl.cc index a0854cf8..14847851 100644 --- a/lib/MultiTopicsConsumerImpl.cc +++ b/lib/MultiTopicsConsumerImpl.cc @@ -475,7 +475,7 @@ void MultiTopicsConsumerImpl::closeAsync(ResultCallback originalCallback) { }; const auto state = state_.load(); if (state == Closing || state == Closed) { - callback(ResultAlreadyClosed); + callback(ResultOk); return; } @@ -488,7 +488,7 @@ void MultiTopicsConsumerImpl::closeAsync(ResultCallback originalCallback) { if (consumers.empty()) { LOG_DEBUG("TopicsConsumer have no consumers to close " << " topic" << topic() << " subscription - " << subscriptionName_); - callback(ResultAlreadyClosed); + callback(ResultOk); return; } diff --git a/tests/BasicEndToEndTest.cc b/tests/BasicEndToEndTest.cc index 5dbccbfc..42c072ad 100644 --- a/tests/BasicEndToEndTest.cc +++ b/tests/BasicEndToEndTest.cc @@ -1694,7 +1694,7 @@ TEST(BasicEndToEndTest, testSeekOnPartitionedTopic) { ASSERT_EQ(expected.str(), msgReceived.getDataAsString()); ASSERT_EQ(ResultOk, consumer.acknowledge(msgReceived)); ASSERT_EQ(ResultOk, consumer.unsubscribe()); - ASSERT_EQ(ResultAlreadyClosed, consumer.close()); + ASSERT_EQ(ResultOk, consumer.close()); ASSERT_EQ(ResultOk, producer.close()); ASSERT_EQ(ResultOk, client.close()); } From ee1d7b9c5b1eed6fe0a279203648a06c30a3feb0 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 11 Mar 2024 19:09:18 +0800 Subject: [PATCH 49/54] Fix hasMessageAvailable might return true after seeking to latest (#409) * ### Motivation After a seek operation is done, the `startMessageId` will be updated until the reconnection due to the seek is done in `connectionOpened`. So before it's updated, `hasMessageAvailable` could compare with an outdated `startMessageId` and return a wrong value. ### Modifications Replace `duringSeek` with a `SeekStatus` field: - `NOT_STARTED`: initial, or a seek operation is done. `seek` could only succeed in this status. - `IN_PROGRESS`: A seek operation has started but the client does not receive the response from broker. - `COMPLETED`: The client has received the seek response but the seek future is not done. After the status becomes `COMPLETED`, if the connection is not ready, next time the connection is established, the status will change from `COMPLETED` to `NOT_STARTED` and then seek future will be completed in the internal executor. Add `testHasMessageAvailableAfterSeekToEnd` and `testSeekInProgress`. --- lib/ConsumerImpl.cc | 87 ++++++++++++++++++++++++++------------------- lib/ConsumerImpl.h | 31 ++++++++++++++-- lib/Synchronized.h | 5 +++ tests/ReaderTest.cc | 55 ++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 39 deletions(-) diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index fa33e280..ebc85188 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -236,16 +236,14 @@ Future ConsumerImpl::connectionOpened(const ClientConnectionPtr& c // sending the subscribe request. cnx->registerConsumer(consumerId_, get_shared_this_ptr()); - if (duringSeek_) { + if (duringSeek()) { ackGroupingTrackerPtr_->flushAndClean(); } Lock lockForMessageId(mutexForMessageId_); - // Update startMessageId so that we can discard messages after delivery restarts - const auto startMessageId = clearReceiveQueue(); + clearReceiveQueue(); const auto subscribeMessageId = - (subscriptionMode_ == Commands::SubscriptionModeNonDurable) ? startMessageId : boost::none; - startMessageId_ = startMessageId; + (subscriptionMode_ == Commands::SubscriptionModeNonDurable) ? startMessageId_.get() : boost::none; lockForMessageId.unlock(); unAckedMessageTrackerPtr_->clear(); @@ -1048,14 +1046,21 @@ void ConsumerImpl::messageProcessed(Message& msg, bool track) { * Clear the internal receiver queue and returns the message id of what was the 1st message in the queue that * was * not seen by the application + * `startMessageId_` is updated so that we can discard messages after delivery restarts. */ -boost::optional ConsumerImpl::clearReceiveQueue() { - bool expectedDuringSeek = true; - if (duringSeek_.compare_exchange_strong(expectedDuringSeek, false)) { - return seekMessageId_.get(); +void ConsumerImpl::clearReceiveQueue() { + if (duringSeek()) { + startMessageId_ = seekMessageId_.get(); + SeekStatus expected = SeekStatus::COMPLETED; + if (seekStatus_.compare_exchange_strong(expected, SeekStatus::NOT_STARTED)) { + auto seekCallback = seekCallback_.release(); + executor_->postWork([seekCallback] { seekCallback(ResultOk); }); + } + return; } else if (subscriptionMode_ == Commands::SubscriptionModeDurable) { - return startMessageId_.get(); + return; } + Message nextMessageInQueue; if (incomingMessages_.peekAndClear(nextMessageInQueue)) { // There was at least one message pending in the queue @@ -1071,16 +1076,12 @@ boost::optional ConsumerImpl::clearReceiveQueue() { .ledgerId(nextMessageId.ledgerId()) .entryId(nextMessageId.entryId() - 1) .build(); - return previousMessageId; + startMessageId_ = previousMessageId; } else if (lastDequedMessageId_ != MessageId::earliest()) { // If the queue was empty we need to restart from the message just after the last one that has been // dequeued // in the past - return lastDequedMessageId_; - } else { - // No message was received or dequeued by this consumer. Next message would still be the - // startMessageId - return startMessageId_.get(); + startMessageId_ = lastDequedMessageId_; } } @@ -1500,18 +1501,15 @@ void ConsumerImpl::seekAsync(uint64_t timestamp, ResultCallback callback) { bool ConsumerImpl::isReadCompacted() { return readCompacted_; } -inline bool hasMoreMessages(const MessageId& lastMessageIdInBroker, const MessageId& messageId) { - return lastMessageIdInBroker > messageId && lastMessageIdInBroker.entryId() != -1; -} - void ConsumerImpl::hasMessageAvailableAsync(HasMessageAvailableCallback callback) { - const auto startMessageId = startMessageId_.get(); - Lock lock(mutexForMessageId_); - const auto messageId = - (lastDequedMessageId_ == MessageId::earliest()) ? startMessageId.value() : lastDequedMessageId_; - - if (messageId == MessageId::latest()) { - lock.unlock(); + bool compareMarkDeletePosition; + { + std::lock_guard lock{mutexForMessageId_}; + compareMarkDeletePosition = + (lastDequedMessageId_ == MessageId::earliest()) && + (startMessageId_.get().value_or(MessageId::earliest()) == MessageId::latest()); + } + if (compareMarkDeletePosition) { auto self = get_shared_this_ptr(); getLastMessageIdAsync([self, callback](Result result, const GetLastMessageIdResponse& response) { if (result != ResultOk) { @@ -1543,16 +1541,15 @@ void ConsumerImpl::hasMessageAvailableAsync(HasMessageAvailableCallback callback } }); } else { - if (hasMoreMessages(lastMessageIdInBroker_, messageId)) { - lock.unlock(); + if (hasMoreMessages()) { callback(ResultOk, true); return; } - lock.unlock(); - - getLastMessageIdAsync([callback, messageId](Result result, const GetLastMessageIdResponse& response) { - callback(result, (result == ResultOk) && hasMoreMessages(response.getLastMessageId(), messageId)); - }); + auto self = get_shared_this_ptr(); + getLastMessageIdAsync( + [this, self, callback](Result result, const GetLastMessageIdResponse& response) { + callback(result, (result == ResultOk) && hasMoreMessages()); + }); } } @@ -1656,9 +1653,18 @@ void ConsumerImpl::seekAsyncInternal(long requestId, SharedBuffer seek, const Me return; } + auto expected = SeekStatus::NOT_STARTED; + if (!seekStatus_.compare_exchange_strong(expected, SeekStatus::IN_PROGRESS)) { + LOG_ERROR(getName() << " attempted to seek (" << seekId << ", " << timestamp << " when the status is " + << static_cast(expected)); + callback(ResultNotAllowedError); + return; + } + const auto originalSeekMessageId = seekMessageId_.get(); seekMessageId_ = seekId; - duringSeek_ = true; + seekStatus_ = SeekStatus::IN_PROGRESS; + seekCallback_ = std::move(callback); if (timestamp > 0) { LOG_INFO(getName() << " Seeking subscription to " << timestamp); } else { @@ -1682,12 +1688,19 @@ void ConsumerImpl::seekAsyncInternal(long requestId, SharedBuffer seek, const Me Lock lock(mutexForMessageId_); lastDequedMessageId_ = MessageId::earliest(); lock.unlock(); + if (getCnx().expired()) { + // It's during reconnection, complete the seek future after connection is established + seekStatus_ = SeekStatus::COMPLETED; + } else { + startMessageId_ = seekMessageId_.get(); + seekCallback_.release()(result); + } } else { LOG_ERROR(getName() << "Failed to seek: " << result); seekMessageId_ = originalSeekMessageId; - duringSeek_ = false; + seekStatus_ = SeekStatus::NOT_STARTED; + seekCallback_.release()(result); } - callback(result); }); } diff --git a/lib/ConsumerImpl.h b/lib/ConsumerImpl.h index 524acb8e..82e323b6 100644 --- a/lib/ConsumerImpl.h +++ b/lib/ConsumerImpl.h @@ -75,6 +75,13 @@ const static std::string SYSTEM_PROPERTY_REAL_TOPIC = "REAL_TOPIC"; const static std::string PROPERTY_ORIGIN_MESSAGE_ID = "ORIGIN_MESSAGE_ID"; const static std::string DLQ_GROUP_TOPIC_SUFFIX = "-DLQ"; +enum class SeekStatus : std::uint8_t +{ + NOT_STARTED, + IN_PROGRESS, + COMPLETED +}; + class ConsumerImpl : public ConsumerImplBase { public: ConsumerImpl(const ClientImplPtr client, const std::string& topic, const std::string& subscriptionName, @@ -193,7 +200,7 @@ class ConsumerImpl : public ConsumerImplBase { const DeadlineTimerPtr& timer, BrokerGetLastMessageIdCallback callback); - boost::optional clearReceiveQueue(); + void clearReceiveQueue(); void seekAsyncInternal(long requestId, SharedBuffer seek, const MessageId& seekId, long timestamp, ResultCallback callback); void processPossibleToDLQ(const MessageId& messageId, ProcessDLQCallBack cb); @@ -239,10 +246,13 @@ class ConsumerImpl : public ConsumerImplBase { MessageId lastDequedMessageId_{MessageId::earliest()}; MessageId lastMessageIdInBroker_{MessageId::earliest()}; - std::atomic_bool duringSeek_{false}; + std::atomic seekStatus_{SeekStatus::NOT_STARTED}; + Synchronized seekCallback_{[](Result) {}}; Synchronized> startMessageId_; Synchronized seekMessageId_{MessageId::earliest()}; + bool duringSeek() const { return seekStatus_ != SeekStatus::NOT_STARTED; } + class ChunkedMessageCtx { public: ChunkedMessageCtx() : totalChunks_(0) {} @@ -332,6 +342,23 @@ class ConsumerImpl : public ConsumerImplBase { const proto::MessageIdData& messageIdData, const ClientConnectionPtr& cnx, MessageId& messageId); + bool hasMoreMessages() const { + std::lock_guard lock{mutexForMessageId_}; + if (lastMessageIdInBroker_.entryId() == -1L) { + return false; + } + + const auto inclusive = config_.isStartMessageIdInclusive(); + if (lastDequedMessageId_ == MessageId::earliest()) { + // If startMessageId_ is none, use latest so that this method will return false + const auto startMessageId = startMessageId_.get().value_or(MessageId::latest()); + return inclusive ? (lastMessageIdInBroker_ >= startMessageId) + : (lastMessageIdInBroker_ > startMessageId); + } else { + return lastMessageIdInBroker_ > lastDequedMessageId_; + } + } + friend class PulsarFriend; friend class MultiTopicsConsumerImpl; diff --git a/lib/Synchronized.h b/lib/Synchronized.h index a98c08da..5449a9fe 100644 --- a/lib/Synchronized.h +++ b/lib/Synchronized.h @@ -30,6 +30,11 @@ class Synchronized { return value_; } + T&& release() { + std::lock_guard lock(mutex_); + return std::move(value_); + } + Synchronized& operator=(const T& value) { std::lock_guard lock(mutex_); value_ = value; diff --git a/tests/ReaderTest.cc b/tests/ReaderTest.cc index 723972d4..9d2fe5f7 100644 --- a/tests/ReaderTest.cc +++ b/tests/ReaderTest.cc @@ -752,6 +752,8 @@ TEST(ReaderSeekTest, testSeekForMessageId) { producer.close(); } +class ReaderSeekTest : public ::testing::TestWithParam {}; + TEST(ReaderSeekTest, testStartAtLatestMessageId) { Client client(serviceUrl); @@ -784,4 +786,57 @@ TEST(ReaderSeekTest, testStartAtLatestMessageId) { producer.close(); } +TEST(ReaderTest, testSeekInProgress) { + Client client(serviceUrl); + const auto topic = "test-seek-in-progress-" + std::to_string(time(nullptr)); + Reader reader; + ASSERT_EQ(ResultOk, client.createReader(topic, MessageId::earliest(), {}, reader)); + + reader.seekAsync(MessageId::earliest(), [](Result) {}); + Promise promise; + reader.seekAsync(MessageId::earliest(), [promise](Result result) { promise.setValue(result); }); + Result result; + promise.getFuture().get(result); + ASSERT_EQ(result, ResultNotAllowedError); + client.close(); +} + +TEST_P(ReaderSeekTest, testHasMessageAvailableAfterSeekToEnd) { + Client client(serviceUrl); + const auto topic = "test-has-message-available-after-seek-to-end-" + std::to_string(time(nullptr)); + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer(topic, producer)); + Reader reader; + ASSERT_EQ(ResultOk, client.createReader(topic, MessageId::earliest(), {}, reader)); + + producer.send(MessageBuilder().setContent("msg-0").build()); + producer.send(MessageBuilder().setContent("msg-1").build()); + + bool hasMessageAvailable; + if (GetParam()) { + // Test the case when `ConsumerImpl.lastMessageIdInBroker_` has been initialized + ASSERT_EQ(ResultOk, reader.hasMessageAvailable(hasMessageAvailable)); + } + + ASSERT_EQ(ResultOk, reader.seek(MessageId::latest())); + ASSERT_EQ(ResultOk, reader.hasMessageAvailable(hasMessageAvailable)); + ASSERT_FALSE(hasMessageAvailable); + + producer.send(MessageBuilder().setContent("msg-2").build()); + ASSERT_EQ(ResultOk, reader.hasMessageAvailable(hasMessageAvailable)); + ASSERT_TRUE(hasMessageAvailable); + + Message msg; + ASSERT_EQ(ResultOk, reader.readNext(msg, 1000)); + ASSERT_EQ("msg-2", msg.getDataAsString()); + + // Test the 2nd seek + ASSERT_EQ(ResultOk, reader.seek(MessageId::latest())); + ASSERT_EQ(ResultOk, reader.hasMessageAvailable(hasMessageAvailable)); + ASSERT_FALSE(hasMessageAvailable); + + client.close(); +} + INSTANTIATE_TEST_SUITE_P(Pulsar, ReaderTest, ::testing::Values(true, false)); +INSTANTIATE_TEST_SUITE_P(Pulsar, ReaderSeekTest, ::testing::Values(true, false)); From e7793d6752a6bbfd5f526ecf207bd3738179f29e Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 14 Mar 2024 20:11:26 +0800 Subject: [PATCH 50/54] Support customize vcpkg directory when INTEGRATE_VCPKG is ON (#417) ### Motivation Currently when INTEGRATE_VCPKG is ON, the CMAKE_TOOLCHAIN_FILE variable is always a subdirectory of `${CMAKE_SOURCE_DIR}/vcpkg`. We can only customize the vcpkg directory when INTEGRATE_VCPKG is OFF, while the legacy CMake logic is incompatible with this way. ### Modifications When INTEGRATE_VCPKG is ON, only set CMAKE_TOOLCHAIN_FILE if it's not defined. The workflow and README are updated for it. (cherry picked from commit 821871777e247e1ccbfa323ea0d5136cf0e18711) --- .github/workflows/ci-pr-validation.yaml | 5 +++++ CMakeLists.txt | 4 +++- README.md | 13 +++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index d9dc2ca5..5010411b 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -114,6 +114,11 @@ jobs: cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON -DBUILD_PERF_TOOLS=ON cmake --build . -j8 + - name: Verify custom vcpkg installation + run: | + mv vcpkg /tmp/ + cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_TOOLCHAIN_FILE="/tmp/vcpkg/scripts/buildsystems/vcpkg.cmake" + cpp20-build: name: Build with the C++20 standard needs: formatting-check diff --git a/CMakeLists.txt b/CMakeLists.txt index 662e84af..b0046534 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,9 @@ option(USE_ASIO "Use Asio instead of Boost.Asio" OFF) option(INTEGRATE_VCPKG "Integrate with Vcpkg" OFF) if (INTEGRATE_VCPKG) set(USE_ASIO ON) - set(CMAKE_TOOLCHAIN_FILE "${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake") + if (NOT CMAKE_TOOLCHAIN_FILE) + set(CMAKE_TOOLCHAIN_FILE "${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake") + endif () endif () option(BUILD_TESTS "Build tests" ON) diff --git a/README.md b/README.md index f0c13f07..4c86b631 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ Since it's integrated with vcpkg, see [vcpkg#README](https://github.com/microsof ### How to build from source +The simplest way is to clone this project with the vcpkg submodule. + ```bash git clone https://github.com/apache/pulsar-client-cpp.git cd pulsar-client-cpp @@ -57,6 +59,17 @@ cmake --build build -j8 The 1st step will download vcpkg and then install all dependencies according to the version description in [vcpkg.json](./vcpkg.json). The 2nd step will build the Pulsar C++ libraries under `./build/lib/`, where `./build` is the CMake build directory. +> You can also add the CMAKE_TOOLCHAIN_FILE option if your system already have vcpkg installed. +> +> ```bash +> git clone https://github.com/apache/pulsar-client-cpp.git +> cd pulsar-client-cpp +> # For example, you can install vcpkg in /tmp/vcpkg +> cd /tmp && git clone https://github.com/microsoft/vcpkg.git && cd - +> cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_TOOLCHAIN_FILE="/tmp/vcpkg/scripts/buildsystems/vcpkg.cmake" +> cmake --build build -j8 +> ``` + After the build, the hierarchy of the `build` directory will be: ``` From 0e0ca8da3e7cabf76adb5d5ae30283f909e0aaa3 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Sun, 17 Mar 2024 23:10:15 +0800 Subject: [PATCH 51/54] Fix broken wireshark build workflow on macOS (#414) ### Motivation See https://github.com/apache/pulsar-client-cpp/actions/runs/8276076995/job/22644077705 ``` Error: The `brew link` step did not complete successfully The formula built, but is not symlinked into /usr/local Could not symlink bin/2to3 ``` `brew install` failed because the wireshark dependency depends on python@3.12 and it failed at `brew link`. ### Modifications Remove the existing binaries that might conflict. (cherry picked from commit c513f29fadce86bacaf4f878d8d25064f7560083) --- .github/workflows/ci-pr-validation.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 5010411b..9f113b03 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -64,6 +64,11 @@ jobs: run: | # See https://github.com/Homebrew/homebrew-core/issues/157142 export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + cd /usr/local/bin + rm -f 2to3* idle3* pydoc* python3* + rm -f /usr/local/share/man/man1/python3.1 /usr/local/lib/pkgconfig/python3* + cd /usr/local/Frameworks/Python.framework + rm -rf Headers Python Resources Versions/Current brew update brew install pkg-config wireshark protobuf - name: Build wireshark plugin From 7cfd77506c83f259c69b440082d3b558b60695e4 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 27 Mar 2024 02:47:42 -0700 Subject: [PATCH 52/54] Fix minor issues reported by CodeQL (#421) (cherry picked from commit 763b85c6c4b9bb648b9f7cf62f9ed09f04f3decb) --- lib/lz4/lz4.cc | 6 +++--- perf/PerfProducer.cc | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/lz4/lz4.cc b/lib/lz4/lz4.cc index d63b977a..2f98fb38 100644 --- a/lib/lz4/lz4.cc +++ b/lib/lz4/lz4.cc @@ -1175,9 +1175,9 @@ FORCE_INLINE int LZ4_decompress_generic( s = *ip++; length += s; } while (likely((endOnInput) ? ip < iend - RUN_MASK : 1) && (s == 255)); - if ((safeDecode) && unlikely((size_t)(op + length) < (size_t)(op))) + if ((safeDecode) && unlikely(length >= (size_t)(oend - op))) goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((size_t)(ip + length) < (size_t)(ip))) + if ((safeDecode) && unlikely(length >= (size_t)(iend - ip))) goto _output_error; /* overflow detection */ } @@ -1220,7 +1220,7 @@ FORCE_INLINE int LZ4_decompress_generic( s = *ip++; length += s; } while (s == 255); - if ((safeDecode) && unlikely((size_t)(op + length) < (size_t)op)) + if ((safeDecode) && unlikely(length >= (size_t)(oend - op))) goto _output_error; /* overflow detection */ } length += MINMATCH; diff --git a/perf/PerfProducer.cc b/perf/PerfProducer.cc index aeda8e84..cbfef688 100644 --- a/perf/PerfProducer.cc +++ b/perf/PerfProducer.cc @@ -160,7 +160,7 @@ void startPerfProducer(const Arguments& args, pulsar::ProducerConfiguration& pro limiter = std::make_shared(args.rate); } - producerList.resize(args.numTopics * args.numProducers); + producerList.resize((size_t)args.numTopics * args.numProducers); for (int i = 0; i < args.numTopics; i++) { std::string topic = (args.numTopics == 1) ? args.topic : args.topic + "-" + std::to_string(i); LOG_INFO("Adding " << args.numProducers << " producers on topic " << topic); From b2ad3525eb762de5b92cc2af92cbe2e02c3e90cd Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 28 Mar 2024 10:36:12 +0800 Subject: [PATCH 53/54] Fix wrong results of hasMessageAvailable and readNext after seeking by timestamp (#422) Fixes https://github.com/apache/pulsar-client-cpp/issues/420 It's a catch-up for https://github.com/apache/pulsar/pull/22363 (cherry picked from commit 27d8cc01d5b7cfce94a7d71259dca5cea83ae01e) --- lib/ConsumerImpl.cc | 41 ++++++++++++---------- lib/ConsumerImpl.h | 15 +++++++- tests/ReaderTest.cc | 84 ++++++++++++++++++++++++++++++++++++--------- 3 files changed, 104 insertions(+), 36 deletions(-) diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index ebc85188..1a0b0cb9 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -1050,7 +1050,9 @@ void ConsumerImpl::messageProcessed(Message& msg, bool track) { */ void ConsumerImpl::clearReceiveQueue() { if (duringSeek()) { - startMessageId_ = seekMessageId_.get(); + if (!hasSoughtByTimestamp_.load(std::memory_order_acquire)) { + startMessageId_ = seekMessageId_.get(); + } SeekStatus expected = SeekStatus::COMPLETED; if (seekStatus_.compare_exchange_strong(expected, SeekStatus::NOT_STARTED)) { auto seekCallback = seekCallback_.release(); @@ -1476,7 +1478,7 @@ void ConsumerImpl::seekAsync(const MessageId& msgId, ResultCallback callback) { return; } const auto requestId = client->newRequestId(); - seekAsyncInternal(requestId, Commands::newSeek(consumerId_, requestId, msgId), msgId, 0L, callback); + seekAsyncInternal(requestId, Commands::newSeek(consumerId_, requestId, msgId), SeekArg{msgId}, callback); } void ConsumerImpl::seekAsync(uint64_t timestamp, ResultCallback callback) { @@ -1495,8 +1497,8 @@ void ConsumerImpl::seekAsync(uint64_t timestamp, ResultCallback callback) { return; } const auto requestId = client->newRequestId(); - seekAsyncInternal(requestId, Commands::newSeek(consumerId_, requestId, timestamp), MessageId::earliest(), - timestamp, callback); + seekAsyncInternal(requestId, Commands::newSeek(consumerId_, requestId, timestamp), SeekArg{timestamp}, + callback); } bool ConsumerImpl::isReadCompacted() { return readCompacted_; } @@ -1509,7 +1511,7 @@ void ConsumerImpl::hasMessageAvailableAsync(HasMessageAvailableCallback callback (lastDequedMessageId_ == MessageId::earliest()) && (startMessageId_.get().value_or(MessageId::earliest()) == MessageId::latest()); } - if (compareMarkDeletePosition) { + if (compareMarkDeletePosition || hasSoughtByTimestamp_.load(std::memory_order_acquire)) { auto self = get_shared_this_ptr(); getLastMessageIdAsync([self, callback](Result result, const GetLastMessageIdResponse& response) { if (result != ResultOk) { @@ -1518,8 +1520,8 @@ void ConsumerImpl::hasMessageAvailableAsync(HasMessageAvailableCallback callback } auto handleResponse = [self, response, callback] { if (response.hasMarkDeletePosition() && response.getLastMessageId().entryId() >= 0) { - // We only care about comparing ledger ids and entry ids as mark delete position doesn't - // have other ids such as batch index + // We only care about comparing ledger ids and entry ids as mark delete position + // doesn't have other ids such as batch index auto compareResult = compareLedgerAndEntryId(response.getMarkDeletePosition(), response.getLastMessageId()); callback(ResultOk, self->config_.isStartMessageIdInclusive() ? compareResult <= 0 @@ -1528,7 +1530,8 @@ void ConsumerImpl::hasMessageAvailableAsync(HasMessageAvailableCallback callback callback(ResultOk, false); } }; - if (self->config_.isStartMessageIdInclusive()) { + if (self->config_.isStartMessageIdInclusive() && + !self->hasSoughtByTimestamp_.load(std::memory_order_acquire)) { self->seekAsync(response.getLastMessageId(), [callback, handleResponse](Result result) { if (result != ResultOk) { callback(result, {}); @@ -1644,8 +1647,8 @@ bool ConsumerImpl::isConnected() const { return !getCnx().expired() && state_ == uint64_t ConsumerImpl::getNumberOfConnectedConsumer() { return isConnected() ? 1 : 0; } -void ConsumerImpl::seekAsyncInternal(long requestId, SharedBuffer seek, const MessageId& seekId, - long timestamp, ResultCallback callback) { +void ConsumerImpl::seekAsyncInternal(long requestId, SharedBuffer seek, const SeekArg& seekArg, + ResultCallback callback) { ClientConnectionPtr cnx = getCnx().lock(); if (!cnx) { LOG_ERROR(getName() << " Client Connection not ready for Consumer"); @@ -1655,21 +1658,21 @@ void ConsumerImpl::seekAsyncInternal(long requestId, SharedBuffer seek, const Me auto expected = SeekStatus::NOT_STARTED; if (!seekStatus_.compare_exchange_strong(expected, SeekStatus::IN_PROGRESS)) { - LOG_ERROR(getName() << " attempted to seek (" << seekId << ", " << timestamp << " when the status is " + LOG_ERROR(getName() << " attempted to seek " << seekArg << " when the status is " << static_cast(expected)); callback(ResultNotAllowedError); return; } const auto originalSeekMessageId = seekMessageId_.get(); - seekMessageId_ = seekId; - seekStatus_ = SeekStatus::IN_PROGRESS; - seekCallback_ = std::move(callback); - if (timestamp > 0) { - LOG_INFO(getName() << " Seeking subscription to " << timestamp); + if (boost::get(&seekArg)) { + hasSoughtByTimestamp_.store(true, std::memory_order_release); } else { - LOG_INFO(getName() << " Seeking subscription to " << seekId); + seekMessageId_ = *boost::get(&seekArg); } + seekStatus_ = SeekStatus::IN_PROGRESS; + seekCallback_ = std::move(callback); + LOG_INFO(getName() << " Seeking subscription to " << seekArg); std::weak_ptr weakSelf{get_shared_this_ptr()}; @@ -1692,7 +1695,9 @@ void ConsumerImpl::seekAsyncInternal(long requestId, SharedBuffer seek, const Me // It's during reconnection, complete the seek future after connection is established seekStatus_ = SeekStatus::COMPLETED; } else { - startMessageId_ = seekMessageId_.get(); + if (!hasSoughtByTimestamp_.load(std::memory_order_acquire)) { + startMessageId_ = seekMessageId_.get(); + } seekCallback_.release()(result); } } else { diff --git a/lib/ConsumerImpl.h b/lib/ConsumerImpl.h index 82e323b6..35636ad0 100644 --- a/lib/ConsumerImpl.h +++ b/lib/ConsumerImpl.h @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -201,7 +202,18 @@ class ConsumerImpl : public ConsumerImplBase { BrokerGetLastMessageIdCallback callback); void clearReceiveQueue(); - void seekAsyncInternal(long requestId, SharedBuffer seek, const MessageId& seekId, long timestamp, + using SeekArg = boost::variant; + friend std::ostream& operator<<(std::ostream& os, const SeekArg& seekArg) { + auto ptr = boost::get(&seekArg); + if (ptr) { + os << *ptr; + } else { + os << *boost::get(&seekArg); + } + return os; + } + + void seekAsyncInternal(long requestId, SharedBuffer seek, const SeekArg& seekArg, ResultCallback callback); void processPossibleToDLQ(const MessageId& messageId, ProcessDLQCallBack cb); @@ -250,6 +262,7 @@ class ConsumerImpl : public ConsumerImplBase { Synchronized seekCallback_{[](Result) {}}; Synchronized> startMessageId_; Synchronized seekMessageId_{MessageId::earliest()}; + std::atomic hasSoughtByTimestamp_{false}; bool duringSeek() const { return seekStatus_ != SeekStatus::NOT_STARTED; } diff --git a/tests/ReaderTest.cc b/tests/ReaderTest.cc index 9d2fe5f7..92fdf624 100644 --- a/tests/ReaderTest.cc +++ b/tests/ReaderTest.cc @@ -700,9 +700,16 @@ TEST_P(ReaderTest, testReceiveAfterSeek) { client.close(); } -TEST(ReaderSeekTest, testSeekForMessageId) { - Client client(serviceUrl); +class ReaderSeekTest : public ::testing::TestWithParam { + public: + void SetUp() override { client = Client{serviceUrl}; } + void TearDown() override { client.close(); } + + Client client{serviceUrl}; +}; + +TEST_F(ReaderSeekTest, testSeekForMessageId) { const std::string topic = "test-seek-for-message-id-" + std::to_string(time(nullptr)); Producer producer; @@ -752,18 +759,24 @@ TEST(ReaderSeekTest, testSeekForMessageId) { producer.close(); } -class ReaderSeekTest : public ::testing::TestWithParam {}; - -TEST(ReaderSeekTest, testStartAtLatestMessageId) { - Client client(serviceUrl); +#define EXPECT_HAS_MESSAGE_AVAILABLE(reader, expected) \ + { \ + bool actual; \ + ASSERT_EQ(ResultOk, reader.hasMessageAvailable(actual)); \ + EXPECT_EQ(actual, (expected)); \ + } +TEST_F(ReaderSeekTest, testStartAtLatestMessageId) { const std::string topic = "test-seek-latest-message-id-" + std::to_string(time(nullptr)); Producer producer; ASSERT_EQ(ResultOk, client.createProducer(topic, producer)); MessageId id; - ASSERT_EQ(ResultOk, producer.send(MessageBuilder().setContent("msg").build(), id)); + for (int i = 0; i < 10; i++) { + ASSERT_EQ(ResultOk, + producer.send(MessageBuilder().setContent("msg-" + std::to_string(i)).build(), id)); + } Reader readerExclusive; ASSERT_EQ(ResultOk, @@ -774,20 +787,24 @@ TEST(ReaderSeekTest, testStartAtLatestMessageId) { client.createReader(topic, MessageId::latest(), ReaderConfiguration().setStartMessageIdInclusive(true), readerInclusive)); + EXPECT_HAS_MESSAGE_AVAILABLE(readerExclusive, false); + EXPECT_HAS_MESSAGE_AVAILABLE(readerInclusive, true); + Message msg; - bool hasMsgAvaliable = false; - readerInclusive.hasMessageAvailable(hasMsgAvaliable); - ASSERT_TRUE(hasMsgAvaliable); ASSERT_EQ(ResultOk, readerInclusive.readNext(msg, 3000)); - ASSERT_EQ(ResultTimeout, readerExclusive.readNext(msg, 3000)); + ASSERT_EQ(msg.getDataAsString(), "msg-9"); + + readerInclusive.seek(0L); + EXPECT_HAS_MESSAGE_AVAILABLE(readerInclusive, true); + ASSERT_EQ(ResultOk, readerInclusive.readNext(msg, 3000)); + ASSERT_EQ(msg.getDataAsString(), "msg-0"); readerExclusive.close(); readerInclusive.close(); producer.close(); } -TEST(ReaderTest, testSeekInProgress) { - Client client(serviceUrl); +TEST_F(ReaderSeekTest, testSeekInProgress) { const auto topic = "test-seek-in-progress-" + std::to_string(time(nullptr)); Reader reader; ASSERT_EQ(ResultOk, client.createReader(topic, MessageId::earliest(), {}, reader)); @@ -798,11 +815,9 @@ TEST(ReaderTest, testSeekInProgress) { Result result; promise.getFuture().get(result); ASSERT_EQ(result, ResultNotAllowedError); - client.close(); } TEST_P(ReaderSeekTest, testHasMessageAvailableAfterSeekToEnd) { - Client client(serviceUrl); const auto topic = "test-has-message-available-after-seek-to-end-" + std::to_string(time(nullptr)); Producer producer; ASSERT_EQ(ResultOk, client.createProducer(topic, producer)); @@ -814,7 +829,6 @@ TEST_P(ReaderSeekTest, testHasMessageAvailableAfterSeekToEnd) { bool hasMessageAvailable; if (GetParam()) { - // Test the case when `ConsumerImpl.lastMessageIdInBroker_` has been initialized ASSERT_EQ(ResultOk, reader.hasMessageAvailable(hasMessageAvailable)); } @@ -834,8 +848,44 @@ TEST_P(ReaderSeekTest, testHasMessageAvailableAfterSeekToEnd) { ASSERT_EQ(ResultOk, reader.seek(MessageId::latest())); ASSERT_EQ(ResultOk, reader.hasMessageAvailable(hasMessageAvailable)); ASSERT_FALSE(hasMessageAvailable); +} - client.close(); +TEST_P(ReaderSeekTest, testHasMessageAvailableAfterSeekTimestamp) { + using namespace std::chrono; + const auto topic = "test-has-message-available-after-seek-timestamp-" + std::to_string(time(nullptr)); + Producer producer; + ASSERT_EQ(ResultOk, client.createProducer(topic, producer)); + MessageId sentMsgId; + const auto timestampBeforeSend = + duration_cast(system_clock::now().time_since_epoch()).count(); + ASSERT_EQ(ResultOk, producer.send(MessageBuilder().setContent("msg").build(), sentMsgId)); + + auto createReader = [this, &topic](Reader& reader, const MessageId& msgId) { + ASSERT_EQ(ResultOk, client.createReader(topic, msgId, {}, reader)); + if (GetParam()) { + if (msgId == MessageId::earliest()) { + EXPECT_HAS_MESSAGE_AVAILABLE(reader, true); + } else { + EXPECT_HAS_MESSAGE_AVAILABLE(reader, false); + } + } + }; + + std::vector msgIds{MessageId::earliest(), sentMsgId, MessageId::latest()}; + + for (auto&& msgId : msgIds) { + Reader reader; + createReader(reader, msgId); + ASSERT_EQ(ResultOk, + reader.seek(duration_cast(system_clock::now().time_since_epoch()).count())); + EXPECT_HAS_MESSAGE_AVAILABLE(reader, false); + } + for (auto&& msgId : msgIds) { + Reader reader; + createReader(reader, msgId); + ASSERT_EQ(ResultOk, reader.seek(timestampBeforeSend)); + EXPECT_HAS_MESSAGE_AVAILABLE(reader, true); + } } INSTANTIATE_TEST_SUITE_P(Pulsar, ReaderTest, ::testing::Values(true, false)); From 0ba3d7d8e9f7d3fb2f2cc3cdf918bb63c4791afc Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 28 Mar 2024 10:45:07 +0800 Subject: [PATCH 54/54] Release 3.5.1 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 1545d966..d5c0c991 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.5.0 +3.5.1