From dbececa68f274259dc6b6068861646f5b02ebc5d Mon Sep 17 00:00:00 2001 From: Barthelemy von Haller Date: Mon, 10 Sep 2018 17:16:22 +0200 Subject: [PATCH 01/10] CCDB api import from QC (O2-347) --- CCDB/CMakeLists.txt | 2 + CCDB/README.md | 19 ++ CCDB/include/CCDB/CcdbApi.h | 175 +++++++++++++++++ CCDB/src/CcdbApi.cxx | 370 ++++++++++++++++++++++++++++++++++++ CCDB/test/testCcdbApi.cxx | 176 +++++++++++++++++ cmake/O2Dependencies.cmake | 3 + 6 files changed, 745 insertions(+) create mode 100644 CCDB/include/CCDB/CcdbApi.h create mode 100644 CCDB/src/CcdbApi.cxx create mode 100644 CCDB/test/testCcdbApi.cxx diff --git a/CCDB/CMakeLists.txt b/CCDB/CMakeLists.txt index f48781e589151..2a2fa6ec8df98 100644 --- a/CCDB/CMakeLists.txt +++ b/CCDB/CMakeLists.txt @@ -21,6 +21,7 @@ set(SRCS src/ObjectHandler.cxx src/Storage.cxx src/XmlHandler.cxx + src/CcdbApi.cxx ) set(HEADERS @@ -91,6 +92,7 @@ install( set(TEST_SRCS test/testWriteReadAny.cxx + test/testCcdbApi.cxx ) O2_GENERATE_TESTS( diff --git a/CCDB/README.md b/CCDB/README.md index 3102b69c5a549..585e7a8fb4c53 100644 --- a/CCDB/README.md +++ b/CCDB/README.md @@ -1,3 +1,22 @@ +## CCDB API + +The CCDB API class (`CcdbApi`) is implemented using libcurl and gives +access to the CCDB via its REST api. + +Usage : +``` +// init +CcdbApi api; +map metadata; // can be empty +api.init("http://ccdb-test.cern.ch:8080"); +// store +auto h1 = new TH1F("object1", "object1", 100, 0, 99); +api.store(h1, "Test/Detector", metadata); +// retrieve +auto h1back = api.retrieve("Test/Detector", metadata); + +``` + ## Conditions MQ Conditions MQ is a client/server CCDB implementation for O2. Currently the implementation supports two backends, an OCDB and a Riak one. diff --git a/CCDB/include/CCDB/CcdbApi.h b/CCDB/include/CCDB/CcdbApi.h new file mode 100644 index 0000000000000..699930178dd6c --- /dev/null +++ b/CCDB/include/CCDB/CcdbApi.h @@ -0,0 +1,175 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file CcdbApi.h +/// \author Barthelemy von Haller +/// + +#ifndef PROJECT_CCDBAPI_H +#define PROJECT_CCDBAPI_H + +#include +#include +#include +#include +#include +#include + +namespace o2 { +namespace ccdb { + +/** + * Interface to the CCDB. + * It uses Curl to talk to the REST api. + * + * @todo use smart pointers ? + * @todo handle errors and exceptions + * @todo extend code coverage + */ +class CcdbApi //: public DatabaseInterface +{ + public: + /// \brief Default constructor + CcdbApi(); + /// \brief Default destructor + virtual ~CcdbApi(); + + /** + * Initialize connection to CCDB + * + * @param host The URL to the CCDB (e.g. "ccdb-test.cern.ch:8080") + */ + void init(std::string host); + + /** + * Stores an object in the CCDB + * + * @param rootObject Shared pointer to the object to store. + * @param path The path where the object is going to be stored. + * @param metadata Key-values representing the metadata for this object. + * @param startValidityTimestamp Start of validity. If omitted, current timestamp is used. + * @param endValidityTimestamp End of validity. If omitted, current timestamp + 1 year is used. + */ + void store(TObject *rootObject, std::string path, std::map metadata, + long startValidityTimestamp = -1, long endValidityTimestamp = -1); + + /** + * Retrieve object at the given path for the given timestamp. + * + * @param path The path where the object is to be found. + * @param metadata Key-values representing the metadata to filter out objects. + * @param timestamp Timestamp of the object to retrieve. If omitted, current timestamp is used. + * @return the object, or nullptr if none were found. + */ + TObject *retrieve(std::string path, std::map metadata, + long timestamp = -1); + +// std::vector getListOfTasksWithPublications(); +// std::vector getPublishedObjectNames(std::string taskName); + + /** + * Delete all versions of the object at this path. + * + * @todo Raise an exception if no such object exist. + * @param path + */ + void truncate(std::string path); + + /** + * Delete the matching version of this object. + * + * @todo Raise an exception if no such object exist. + * @param path Path to the object to delete + * @param timestamp Timestamp of the object to delete. + */ + void deleteObject(std::string path, long timestamp = -1); + + /** + * Return the listing of objects, and in some cases subfolders, matching this path. + * The path can contain sql patterns (correctly encoded) or regexps. + * + * In the case where there is no pattern, the list of subfolders is returned along with all the objects at + * this path. It does not work recursively and objects from the subfolders are not returned. + * + * In the case where there is a pattern, subfolders are not returned and any object matching the pattern will + * be returned, including those in subfolders if they match. + * + * Example : Task/Detector will return objects and subfolders in /Task/Detector but not the object(s) in + * /Task/Detector/Sub. + * Example : Task/Detector/.* will return any object below Detector recursively. + * Example : Te*e* will return any object matching this pattern, including Test/detector and TestSecond/A/B. + * + * @todo accept should use an enum class. + * @param path The path to the folder we want to list the children of (default : top dir). + * @param latestOnly In case there are several versions of the same object, list only the latest one. + * @param returnFormat The format of the returned string -> one of "text/plain (default)", "application/json", "text/xml" + * @return The listing of folder and/or objects in the format requested + */ + std::string list(std::string path = "", bool latestOnly = false, std::string returnFormat = "text/plain"); + + private: + /** + * Get the current timestamp. + * + * @return the current timestamp as a long + */ + long getCurrentTimestamp(); + /** + * Transform and return a string representation of the given timestamp. + * + * @param timestamp + * @return a string representation of the given timestamp. + */ + std::string getTimestampString(long timestamp); + /** + * Compute and return a timestamp X seconds in the future. + * + * @param secondsInFuture The number of seconds in the future. + * @return the future timestamp + */ + long getFutureTimestamp(int secondsInFuture); + + /** + * Build the full url to store an object. + * + * @param path The path where the object is going to be stored. + * @param metadata Key-values representing the metadata for this object. + * @param startValidityTimestamp Start of validity. If omitted or negative, the current timestamp is used. + * @param endValidityTimestamp End of validity. If omitted or negative, current timestamp + 1 year is used. + * @return The full url to store an object (url / startValidity / endValidity / [metadata &]* ) + */ + std::string getFullUrlForStorage(const std::string &path, const std::map &metadata, + long startValidityTimestamp = -1, long endValidityTimestamp = -1); + + /** + * Build the full url to store an object. + * @param path The path where the object is going to be found. + * @param metadata Key-values representing the metadata for this object. + * @param timestamp When the object we retrieve must be valid. If omitted or negative, the current timestamp is used. + * @return The full url to store an object (url / startValidity / endValidity / [metadata &]* ) + */ + std::string getFullUrlForRetrieval(const std::string &path, const std::map &metadata, + long timestamp = -1); + + /** + * Initialization of CURL + */ + void curlInit(); + + /// Base URL of the CCDB (with port) + std::string mUrl; + +}; + +} +} + +#endif //PROJECT_CCDBAPI_H diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx new file mode 100644 index 0000000000000..3bba5aea92374 --- /dev/null +++ b/CCDB/src/CcdbApi.cxx @@ -0,0 +1,370 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file CcdbApi.cxx +/// \author Barthelemy von Haller +/// + +#include "CCDB/CcdbApi.h" +#include +#include +#include + +namespace o2 { +namespace ccdb { + +using namespace std; + +CcdbApi::CcdbApi() : mUrl("") +{} + +CcdbApi::~CcdbApi() +{ + curl_global_cleanup(); +} + +void CcdbApi::curlInit() +{ + // todo : are there other things to initialize globally for curl ? + curl_global_init(CURL_GLOBAL_DEFAULT); +} + +void CcdbApi::init(std::string host) +{ + mUrl = host; + curlInit(); +} + +void CcdbApi::store(TObject *rootObject, std::string path, std::map metadata, + long startValidityTimestamp, long endValidityTimestamp) +{ + // Serialize the object + TMessage message(kMESS_OBJECT); + message.Reset(); + message.WriteObjectAny(rootObject, rootObject->IsA()); + + string fullUrl = getFullUrlForStorage(path, metadata, startValidityTimestamp, endValidityTimestamp); + + // Curl preparation + CURL *curl; + struct curl_httppost *formpost = nullptr; + struct curl_httppost *lastptr = nullptr; + struct curl_slist *headerlist = nullptr; + static const char buf[] = "Expect:"; + // todo : what is the correct file name ? + string tmpFileName = string(rootObject->GetName()) + "_" + getTimestampString(getCurrentTimestamp()) + ".root"; + curl_formadd(&formpost, + &lastptr, + CURLFORM_COPYNAME, "send", + CURLFORM_BUFFER, tmpFileName.c_str(), + CURLFORM_BUFFERPTR, message.Buffer(), + CURLFORM_BUFFERLENGTH, message.Length(), + CURLFORM_END); + + curl = curl_easy_init(); + headerlist = curl_slist_append(headerlist, buf); + if (curl != nullptr) { + /* what URL that receives this POST */ + curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerlist); + curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); + + /* Perform the request, res will get the return code */ + CURLcode res = curl_easy_perform(curl); + /* Check for errors */ + if (res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + } + + /* always cleanup */ + curl_easy_cleanup(curl); + + /* then cleanup the formpost chain */ + curl_formfree(formpost); + /* free slist */ + curl_slist_free_all(headerlist); + } +} + +string CcdbApi::getFullUrlForStorage(const string &path, const map &metadata, + long startValidityTimestamp, long endValidityTimestamp) +{ + // Prepare timestamps + string startValidityString = getTimestampString(startValidityTimestamp < 0 ? + getCurrentTimestamp() : + startValidityTimestamp); + string endValidityString = getTimestampString(endValidityTimestamp < 0 ? + getFutureTimestamp(60 * 60 * 24 * 365) : + endValidityTimestamp); + // Build URL + string fullUrl = mUrl + "/" + path + "/" + startValidityString + "/" + endValidityString + "/"; + // Add metadata + for (auto &kv : metadata) { + fullUrl += kv.first + "=" + kv.second + "&"; + } + return fullUrl; +} + +// todo make a single method of the one above and below +string CcdbApi::getFullUrlForRetrieval(const string &path, const map &metadata, long timestamp) +{ + // Prepare timestamps + string validityString = getTimestampString(timestamp < 0 ? + getCurrentTimestamp() : + timestamp); + // Build URL + string fullUrl = mUrl + "/" + path + "/" + validityString + "/"; + // Add metadata + for (auto &kv : metadata) { + fullUrl += kv.first + "=" + kv.second + "/"; + } + return fullUrl; +} + +/** + * Struct to store the data we will receive from the CCDB with CURL. + */ +struct MemoryStruct +{ + char *memory; + unsigned int size; +}; + +/** + * Callback used by CURL to store the data received from the CCDB. + * See https://curl.haxx.se/libcurl/c/getinmemory.html + * @param contents + * @param size + * @param nmemb + * @param userp a MemoryStruct where data is stored. + * @return the size of the data we received and stored at userp. + */ +static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) +{ + size_t realsize = size * nmemb; + auto *mem = (struct MemoryStruct *) userp; + + mem->memory = (char *) realloc(mem->memory, mem->size + realsize + 1); + if (mem->memory == nullptr) { + printf("not enough memory (realloc returned NULL)\n"); + return 0; + } + + memcpy(&(mem->memory[mem->size]), contents, realsize); + mem->size += realsize; + mem->memory[mem->size] = 0; + + return realsize; +} + +TObject *CcdbApi::retrieve(std::string path, std::map metadata, + long timestamp) +{ + // Note : based on https://curl.haxx.se/libcurl/c/getinmemory.html + // Thus it does not comply to our coding guidelines as it is a copy paste. + + string fullUrl = getFullUrlForRetrieval(path, metadata, timestamp); + + // Prepare CURL + CURL *curl_handle; + CURLcode res; + struct MemoryStruct chunk{(char *) malloc(1)/*memory*/, 0/*size*/}; + TObject *result = nullptr; + + /* init the curl session */ + curl_handle = curl_easy_init(); + + /* specify URL to get */ + curl_easy_setopt(curl_handle, CURLOPT_URL, fullUrl.c_str()); + + /* send all data to this function */ + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); + + /* we pass our 'chunk' struct to the callback function */ + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *) &chunk); + + /* some servers don't like requests that are made without a user-agent + field, so we provide one */ + curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0"); + + /* if redirected , we tell libcurl to follow redirection */ + curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L); + + /* get it! */ + res = curl_easy_perform(curl_handle); + + /* check for errors */ + if (res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + } else { + /* + * Now, our chunk.memory points to a memory block that is chunk.size + * bytes big and contains the remote file. + */ + +// printf("%lu bytes retrieved\n", (long) chunk.size); + + long response_code; + res = curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &response_code); + if ((res == CURLE_OK) && (response_code != 404)) { + TMessage mess(kMESS_OBJECT); + mess.SetBuffer(chunk.memory, chunk.size, kFALSE); + mess.SetReadMode(); + mess.Reset(); + result = (TObject *) (mess.ReadObjectAny(mess.GetClass())); + if (result == nullptr) { + cerr << "couldn't retrieve the object " << path << endl; + } + } else { + cerr << "invalid URL : " << fullUrl << endl; + } + + // Print data +// cout << "size : " << chunk.size << endl; +// cout << "data : " << endl; +// char* mem = (char*)chunk.memory; +// for (int i = 0 ; i < chunk.size/4 ; i++) { +// cout << mem; +// mem += 4; +// } + } + + /* cleanup curl stuff */ + curl_easy_cleanup(curl_handle); + + free(chunk.memory); + + return result; +} + +size_t CurlWrite_CallbackFunc_StdString2(void *contents, size_t size, size_t nmemb, std::string *s) +{ + size_t newLength = size * nmemb; + size_t oldLength = s->size(); + try { + s->resize(oldLength + newLength); + } + catch (std::bad_alloc &e) { + cerr << "memory error when getting data from CCDB" << endl; + return 0; + } + + std::copy((char *) contents, (char *) contents + newLength, s->begin() + oldLength); + return size * nmemb; +} + +std::string CcdbApi::list(std::string path, bool latestOnly, std::string returnFormat) +{ + CURL *curl; + CURLcode res; + string fullUrl = mUrl; + fullUrl += latestOnly ? "/latest/" : "/browse/"; + fullUrl += path; + std::string result; + + curl = curl_easy_init(); + if (curl != nullptr) { + + curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString2); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result); + + struct curl_slist *headers = NULL; + headers = curl_slist_append(headers, (string("Accept: ") + returnFormat).c_str()); + headers = curl_slist_append(headers, (string("Content-Type: ") + returnFormat).c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + // Perform the request, res will get the return code + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); + } + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + } + + return result; +} + +long CcdbApi::getFutureTimestamp(int secondsInFuture) +{ + std::chrono::seconds sec(secondsInFuture); + auto future = std::chrono::system_clock::now() + sec; + auto future_ms = std::chrono::time_point_cast(future); + auto epoch = future_ms.time_since_epoch(); + auto value = std::chrono::duration_cast(epoch); + return value.count(); +} + +long CcdbApi::getCurrentTimestamp() +{ + auto now = std::chrono::system_clock::now(); + auto now_ms = std::chrono::time_point_cast(now); + auto epoch = now_ms.time_since_epoch(); + auto value = std::chrono::duration_cast(epoch); + return value.count(); +} + +std::string CcdbApi::getTimestampString(long timestamp) +{ + stringstream ss; + ss << timestamp; + return ss.str(); +} + +void CcdbApi::deleteObject(std::string path, long timestamp) +{ + CURL *curl; + CURLcode res; + stringstream fullUrl; + long timestampLocal = timestamp == -1 ? getCurrentTimestamp() : timestamp; + + fullUrl << mUrl << "/" << path << "/" << timestampLocal; + + curl = curl_easy_init(); + if (curl != nullptr) { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str()); + + // Perform the request, res will get the return code + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); + } + curl_easy_cleanup(curl); + } +} + +void CcdbApi::truncate(std::string path) +{ + CURL *curl; + CURLcode res; + stringstream fullUrl; + fullUrl << mUrl << "/truncate/" << path; + + curl = curl_easy_init(); + if (curl != nullptr) { + curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str()); + + // Perform the request, res will get the return code + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); + } + curl_easy_cleanup(curl); + } +} + +} +} + diff --git a/CCDB/test/testCcdbApi.cxx b/CCDB/test/testCcdbApi.cxx new file mode 100644 index 0000000000000..f8430dc14ff74 --- /dev/null +++ b/CCDB/test/testCcdbApi.cxx @@ -0,0 +1,176 @@ +/// +/// \file testCcdbApi.cxx +/// \author Barthelemy von Haller +/// + +#include "CCDB/CcdbApi.h" + +#define BOOST_TEST_MODULE Quality test +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include +#include +#include +#include "curl/curl.h" + +#include +#include +#include +#include +#include + +using namespace std; +using namespace o2::ccdb; + +struct test_fixture +{ + test_fixture() + { + api.init("http://ccdb-test.cern.ch:8080"); + } + + ~test_fixture() + { + } + + CcdbApi api; + map metadata; +}; + +BOOST_AUTO_TEST_CASE(store_test) +{ + test_fixture f; + + auto h1 = new TH1F("object1", "object1", 100, 0, 99); + f.api.store(h1, "Test/Detector", f.metadata); +} + +BOOST_AUTO_TEST_CASE(retrieve_test) +{ + test_fixture f; + + auto h1 = f.api.retrieve("Test/Detector", f.metadata); + BOOST_CHECK(h1 != nullptr); + BOOST_CHECK_EQUAL(h1->GetName(), "object1"); + + auto h2 = f.api.retrieve("asdf/asdf", f.metadata); + BOOST_CHECK_EQUAL(h2, nullptr); +} + +BOOST_AUTO_TEST_CASE(truncate_test) +{ + test_fixture f; + + auto h1 = f.api.retrieve("Test/Detector", f.metadata); + BOOST_CHECK(h1 != nullptr); + f.api.truncate("Test/Detector"); + h1 = f.api.retrieve("Test/Detector", f.metadata); + BOOST_CHECK(h1 == nullptr); +} + +BOOST_AUTO_TEST_CASE(delete_test) +{ + test_fixture f; + + auto h1 = new TH1F("object1", "object1", 100, 0, 99); + f.api.store(h1, "Test/Detector", f.metadata); + auto h2 = f.api.retrieve("Test/Detector", f.metadata); + BOOST_CHECK(h2 != nullptr); + f.api.deleteObject("Test/Detector"); + h2 = f.api.retrieve("Test/Detector", f.metadata); + BOOST_CHECK(h2 == nullptr); +} + + +/// trim from start (in place) +/// https://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +static inline void ltrim(std::string &s) +{ + s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { + return !std::isspace(ch); + })); +} + +/// trim from end (in place) +/// https://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +static inline void rtrim(std::string &s) +{ + s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { + return !std::isspace(ch); + }).base(), s.end()); +} + +void countItems(const string &s, int &countObjects, int &countSubfolders) +{ + countObjects = 0; + countSubfolders = 0; + std::stringstream ss(s); + std::string line; + bool subfolderMode = false; + while (std::getline(ss, line, '\n')) { + ltrim(line); + rtrim(line); + if (line.length() == 0) { + continue; + } + + if (line.find("subfolders") != std::string::npos) { + subfolderMode = true; + continue; + } + + if (subfolderMode) { + if (line.find(']') == 0) { + break; + } else { + countSubfolders++; + } + } + + if (line.find("\"path\"") == 0) { + countObjects++; + } + } +} + +BOOST_AUTO_TEST_CASE(list_test) +{ + test_fixture f; + + // test non-empty top dir + string s = f.api.list("", "application/json"); // top dir + long nbLines = std::count(s.begin(), s.end(), '\n') + 1; + BOOST_CHECK(nbLines > 5); + + // test empty dir + f.api.truncate("Test/Detector*"); + s = f.api.list("Test/Detector", false, "application/json"); + int countObjects = 0; + int countSubfolders = 0; + countItems(s, countObjects, countSubfolders); + BOOST_CHECK_EQUAL(countObjects, 0); + + // more complex tree + auto h1 = new TH1F("object1", "object1", 100, 0, 99); + f.api.store(h1, "Test", f.metadata); + f.api.store(h1, "Test/Detector", f.metadata); + f.api.store(h1, "Test/Detector", f.metadata); + f.api.store(h1, "Test/Detector", f.metadata); + f.api.store(h1, "Test/Detector/Sub/abc", f.metadata); + + s = f.api.list("Test/Detector", false, "application/json"); + countItems(s, countObjects, countSubfolders); + BOOST_CHECK_EQUAL(countObjects, 3); +// BOOST_CHECK_EQUAL(countSubfolders, 1); + + s = f.api.list("Test/Detector*", false, "application/json"); + countItems(s, countObjects, countSubfolders); + cout << "s : " << s << endl; + BOOST_CHECK_EQUAL(countObjects, 4); +// BOOST_CHECK_EQUAL(countSubfolders, 0); + + s = f.api.list("Test/Detector", true, "application/json"); + countItems(s, countObjects, countSubfolders); + BOOST_CHECK_EQUAL(countObjects, 1); +} diff --git a/cmake/O2Dependencies.cmake b/cmake/O2Dependencies.cmake index 20ba3a01f5229..c54acf38efa78 100644 --- a/cmake/O2Dependencies.cmake +++ b/cmake/O2Dependencies.cmake @@ -72,6 +72,7 @@ find_package(RapidJSON REQUIRED) find_package(GLFW) find_package(benchmark QUIET) find_package(Arrow) +find_package(CURL REQUIRED) if (DDS_FOUND) add_definitions(-DENABLE_DDS) @@ -447,6 +448,7 @@ o2_define_bucket( ParMQ fairmq_bucket pthread Core Tree XMLParser Hist Net RIO z + ${CURL_LIBRARIES} INCLUDE_DIRECTORIES ${FAIRROOT_INCLUDE_DIR} @@ -454,6 +456,7 @@ o2_define_bucket( SYSTEMINCLUDE_DIRECTORIES ${PROTOBUF_INCLUDE_DIR} + ${CURL_INCLUDE_DIRS} ) o2_define_bucket( From 83bd4ab02f0979ab4a07cce025b61cb450073247 Mon Sep 17 00:00:00 2001 From: Barthelemy von Haller Date: Mon, 10 Sep 2018 17:18:31 +0200 Subject: [PATCH 02/10] formatting --- CCDB/include/CCDB/CcdbApi.h | 20 ++++---- CCDB/src/CcdbApi.cxx | 97 ++++++++++++++++++------------------- CCDB/test/testCcdbApi.cxx | 41 ++++++++-------- 3 files changed, 76 insertions(+), 82 deletions(-) diff --git a/CCDB/include/CCDB/CcdbApi.h b/CCDB/include/CCDB/CcdbApi.h index 699930178dd6c..7d672e2b93ed7 100644 --- a/CCDB/include/CCDB/CcdbApi.h +++ b/CCDB/include/CCDB/CcdbApi.h @@ -23,8 +23,10 @@ #include #include -namespace o2 { -namespace ccdb { +namespace o2 +{ +namespace ccdb +{ /** * Interface to the CCDB. @@ -58,7 +60,7 @@ class CcdbApi //: public DatabaseInterface * @param startValidityTimestamp Start of validity. If omitted, current timestamp is used. * @param endValidityTimestamp End of validity. If omitted, current timestamp + 1 year is used. */ - void store(TObject *rootObject, std::string path, std::map metadata, + void store(TObject* rootObject, std::string path, std::map metadata, long startValidityTimestamp = -1, long endValidityTimestamp = -1); /** @@ -69,11 +71,11 @@ class CcdbApi //: public DatabaseInterface * @param timestamp Timestamp of the object to retrieve. If omitted, current timestamp is used. * @return the object, or nullptr if none were found. */ - TObject *retrieve(std::string path, std::map metadata, + TObject* retrieve(std::string path, std::map metadata, long timestamp = -1); -// std::vector getListOfTasksWithPublications(); -// std::vector getPublishedObjectNames(std::string taskName); + // std::vector getListOfTasksWithPublications(); + // std::vector getPublishedObjectNames(std::string taskName); /** * Delete all versions of the object at this path. @@ -146,7 +148,7 @@ class CcdbApi //: public DatabaseInterface * @param endValidityTimestamp End of validity. If omitted or negative, current timestamp + 1 year is used. * @return The full url to store an object (url / startValidity / endValidity / [metadata &]* ) */ - std::string getFullUrlForStorage(const std::string &path, const std::map &metadata, + std::string getFullUrlForStorage(const std::string& path, const std::map& metadata, long startValidityTimestamp = -1, long endValidityTimestamp = -1); /** @@ -156,7 +158,7 @@ class CcdbApi //: public DatabaseInterface * @param timestamp When the object we retrieve must be valid. If omitted or negative, the current timestamp is used. * @return The full url to store an object (url / startValidity / endValidity / [metadata &]* ) */ - std::string getFullUrlForRetrieval(const std::string &path, const std::map &metadata, + std::string getFullUrlForRetrieval(const std::string& path, const std::map& metadata, long timestamp = -1); /** @@ -166,9 +168,7 @@ class CcdbApi //: public DatabaseInterface /// Base URL of the CCDB (with port) std::string mUrl; - }; - } } diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index 3bba5aea92374..96c509f7a2f91 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -18,13 +18,16 @@ #include #include -namespace o2 { -namespace ccdb { +namespace o2 +{ +namespace ccdb +{ using namespace std; CcdbApi::CcdbApi() : mUrl("") -{} +{ +} CcdbApi::~CcdbApi() { @@ -43,7 +46,7 @@ void CcdbApi::init(std::string host) curlInit(); } -void CcdbApi::store(TObject *rootObject, std::string path, std::map metadata, +void CcdbApi::store(TObject* rootObject, std::string path, std::map metadata, long startValidityTimestamp, long endValidityTimestamp) { // Serialize the object @@ -54,10 +57,10 @@ void CcdbApi::store(TObject *rootObject, std::string path, std::mapGetName()) + "_" + getTimestampString(getCurrentTimestamp()) + ".root"; @@ -95,36 +98,30 @@ void CcdbApi::store(TObject *rootObject, std::string path, std::map &metadata, +string CcdbApi::getFullUrlForStorage(const string& path, const map& metadata, long startValidityTimestamp, long endValidityTimestamp) { // Prepare timestamps - string startValidityString = getTimestampString(startValidityTimestamp < 0 ? - getCurrentTimestamp() : - startValidityTimestamp); - string endValidityString = getTimestampString(endValidityTimestamp < 0 ? - getFutureTimestamp(60 * 60 * 24 * 365) : - endValidityTimestamp); + string startValidityString = getTimestampString(startValidityTimestamp < 0 ? getCurrentTimestamp() : startValidityTimestamp); + string endValidityString = getTimestampString(endValidityTimestamp < 0 ? getFutureTimestamp(60 * 60 * 24 * 365) : endValidityTimestamp); // Build URL string fullUrl = mUrl + "/" + path + "/" + startValidityString + "/" + endValidityString + "/"; // Add metadata - for (auto &kv : metadata) { + for (auto& kv : metadata) { fullUrl += kv.first + "=" + kv.second + "&"; } return fullUrl; } // todo make a single method of the one above and below -string CcdbApi::getFullUrlForRetrieval(const string &path, const map &metadata, long timestamp) +string CcdbApi::getFullUrlForRetrieval(const string& path, const map& metadata, long timestamp) { // Prepare timestamps - string validityString = getTimestampString(timestamp < 0 ? - getCurrentTimestamp() : - timestamp); + string validityString = getTimestampString(timestamp < 0 ? getCurrentTimestamp() : timestamp); // Build URL string fullUrl = mUrl + "/" + path + "/" + validityString + "/"; // Add metadata - for (auto &kv : metadata) { + for (auto& kv : metadata) { fullUrl += kv.first + "=" + kv.second + "/"; } return fullUrl; @@ -133,9 +130,8 @@ string CcdbApi::getFullUrlForRetrieval(const string &path, const mapmemory = (char *) realloc(mem->memory, mem->size + realsize + 1); + mem->memory = (char*)realloc(mem->memory, mem->size + realsize + 1); if (mem->memory == nullptr) { printf("not enough memory (realloc returned NULL)\n"); return 0; @@ -166,7 +162,7 @@ static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, voi return realsize; } -TObject *CcdbApi::retrieve(std::string path, std::map metadata, +TObject* CcdbApi::retrieve(std::string path, std::map metadata, long timestamp) { // Note : based on https://curl.haxx.se/libcurl/c/getinmemory.html @@ -175,10 +171,12 @@ TObject *CcdbApi::retrieve(std::string path, std::map string fullUrl = getFullUrlForRetrieval(path, metadata, timestamp); // Prepare CURL - CURL *curl_handle; + CURL* curl_handle; CURLcode res; - struct MemoryStruct chunk{(char *) malloc(1)/*memory*/, 0/*size*/}; - TObject *result = nullptr; + struct MemoryStruct chunk { + (char*)malloc(1) /*memory*/, 0 /*size*/ + }; + TObject* result = nullptr; /* init the curl session */ curl_handle = curl_easy_init(); @@ -190,7 +188,7 @@ TObject *CcdbApi::retrieve(std::string path, std::map curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); /* we pass our 'chunk' struct to the callback function */ - curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *) &chunk); + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void*)&chunk); /* some servers don't like requests that are made without a user-agent field, so we provide one */ @@ -212,7 +210,7 @@ TObject *CcdbApi::retrieve(std::string path, std::map * bytes big and contains the remote file. */ -// printf("%lu bytes retrieved\n", (long) chunk.size); + // printf("%lu bytes retrieved\n", (long) chunk.size); long response_code; res = curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &response_code); @@ -221,7 +219,7 @@ TObject *CcdbApi::retrieve(std::string path, std::map mess.SetBuffer(chunk.memory, chunk.size, kFALSE); mess.SetReadMode(); mess.Reset(); - result = (TObject *) (mess.ReadObjectAny(mess.GetClass())); + result = (TObject*)(mess.ReadObjectAny(mess.GetClass())); if (result == nullptr) { cerr << "couldn't retrieve the object " << path << endl; } @@ -230,13 +228,13 @@ TObject *CcdbApi::retrieve(std::string path, std::map } // Print data -// cout << "size : " << chunk.size << endl; -// cout << "data : " << endl; -// char* mem = (char*)chunk.memory; -// for (int i = 0 ; i < chunk.size/4 ; i++) { -// cout << mem; -// mem += 4; -// } + // cout << "size : " << chunk.size << endl; + // cout << "data : " << endl; + // char* mem = (char*)chunk.memory; + // for (int i = 0 ; i < chunk.size/4 ; i++) { + // cout << mem; + // mem += 4; + // } } /* cleanup curl stuff */ @@ -247,25 +245,24 @@ TObject *CcdbApi::retrieve(std::string path, std::map return result; } -size_t CurlWrite_CallbackFunc_StdString2(void *contents, size_t size, size_t nmemb, std::string *s) +size_t CurlWrite_CallbackFunc_StdString2(void* contents, size_t size, size_t nmemb, std::string* s) { size_t newLength = size * nmemb; size_t oldLength = s->size(); try { s->resize(oldLength + newLength); - } - catch (std::bad_alloc &e) { + } catch (std::bad_alloc& e) { cerr << "memory error when getting data from CCDB" << endl; return 0; } - std::copy((char *) contents, (char *) contents + newLength, s->begin() + oldLength); + std::copy((char*)contents, (char*)contents + newLength, s->begin() + oldLength); return size * nmemb; } std::string CcdbApi::list(std::string path, bool latestOnly, std::string returnFormat) { - CURL *curl; + CURL* curl; CURLcode res; string fullUrl = mUrl; fullUrl += latestOnly ? "/latest/" : "/browse/"; @@ -279,7 +276,7 @@ std::string CcdbApi::list(std::string path, bool latestOnly, std::string returnF curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString2); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result); - struct curl_slist *headers = NULL; + struct curl_slist* headers = NULL; headers = curl_slist_append(headers, (string("Accept: ") + returnFormat).c_str()); headers = curl_slist_append(headers, (string("Content-Type: ") + returnFormat).c_str()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); @@ -324,7 +321,7 @@ std::string CcdbApi::getTimestampString(long timestamp) void CcdbApi::deleteObject(std::string path, long timestamp) { - CURL *curl; + CURL* curl; CURLcode res; stringstream fullUrl; long timestampLocal = timestamp == -1 ? getCurrentTimestamp() : timestamp; @@ -347,7 +344,7 @@ void CcdbApi::deleteObject(std::string path, long timestamp) void CcdbApi::truncate(std::string path) { - CURL *curl; + CURL* curl; CURLcode res; stringstream fullUrl; fullUrl << mUrl << "/truncate/" << path; @@ -364,7 +361,5 @@ void CcdbApi::truncate(std::string path) curl_easy_cleanup(curl); } } - } } - diff --git a/CCDB/test/testCcdbApi.cxx b/CCDB/test/testCcdbApi.cxx index f8430dc14ff74..8e0f0458e606d 100644 --- a/CCDB/test/testCcdbApi.cxx +++ b/CCDB/test/testCcdbApi.cxx @@ -23,19 +23,18 @@ using namespace std; using namespace o2::ccdb; -struct test_fixture -{ - test_fixture() - { - api.init("http://ccdb-test.cern.ch:8080"); - } +struct test_fixture { + test_fixture() + { + api.init("http://ccdb-test.cern.ch:8080"); + } - ~test_fixture() - { - } + ~test_fixture() + { + } - CcdbApi api; - map metadata; + CcdbApi api; + map metadata; }; BOOST_AUTO_TEST_CASE(store_test) @@ -82,26 +81,26 @@ BOOST_AUTO_TEST_CASE(delete_test) BOOST_CHECK(h2 == nullptr); } - /// trim from start (in place) /// https://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring -static inline void ltrim(std::string &s) +static inline void ltrim(std::string& s) { s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { - return !std::isspace(ch); - })); + return !std::isspace(ch); + })); } /// trim from end (in place) /// https://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring -static inline void rtrim(std::string &s) +static inline void rtrim(std::string& s) { s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { - return !std::isspace(ch); - }).base(), s.end()); + return !std::isspace(ch); + }).base(), + s.end()); } -void countItems(const string &s, int &countObjects, int &countSubfolders) +void countItems(const string& s, int& countObjects, int& countSubfolders) { countObjects = 0; countSubfolders = 0; @@ -162,13 +161,13 @@ BOOST_AUTO_TEST_CASE(list_test) s = f.api.list("Test/Detector", false, "application/json"); countItems(s, countObjects, countSubfolders); BOOST_CHECK_EQUAL(countObjects, 3); -// BOOST_CHECK_EQUAL(countSubfolders, 1); + // BOOST_CHECK_EQUAL(countSubfolders, 1); s = f.api.list("Test/Detector*", false, "application/json"); countItems(s, countObjects, countSubfolders); cout << "s : " << s << endl; BOOST_CHECK_EQUAL(countObjects, 4); -// BOOST_CHECK_EQUAL(countSubfolders, 0); + // BOOST_CHECK_EQUAL(countSubfolders, 0); s = f.api.list("Test/Detector", true, "application/json"); countItems(s, countObjects, countSubfolders); From 808e258974b922e44cc781671a118b2fcc5c531c Mon Sep 17 00:00:00 2001 From: Barthelemy von Haller Date: Mon, 10 Sep 2018 17:27:20 +0200 Subject: [PATCH 03/10] copyright and some other warnings fixes --- CCDB/src/CcdbApi.cxx | 2 +- CCDB/test/testCcdbApi.cxx | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index 96c509f7a2f91..16c75dff85d55 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -276,7 +276,7 @@ std::string CcdbApi::list(std::string path, bool latestOnly, std::string returnF curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString2); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result); - struct curl_slist* headers = NULL; + struct curl_slist* headers = nullptr; headers = curl_slist_append(headers, (string("Accept: ") + returnFormat).c_str()); headers = curl_slist_append(headers, (string("Content-Type: ") + returnFormat).c_str()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); diff --git a/CCDB/test/testCcdbApi.cxx b/CCDB/test/testCcdbApi.cxx index 8e0f0458e606d..fbcef1f097e20 100644 --- a/CCDB/test/testCcdbApi.cxx +++ b/CCDB/test/testCcdbApi.cxx @@ -1,3 +1,13 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + /// /// \file testCcdbApi.cxx /// \author Barthelemy von Haller @@ -14,7 +24,7 @@ #include #include "curl/curl.h" -#include +#include #include #include #include From 904130939fade37f8b234dd20c7fa54cc022ee03 Mon Sep 17 00:00:00 2001 From: Barthelemy von Haller Date: Mon, 10 Sep 2018 17:34:37 +0200 Subject: [PATCH 04/10] fix warning raw string --- CCDB/test/testCcdbApi.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CCDB/test/testCcdbApi.cxx b/CCDB/test/testCcdbApi.cxx index fbcef1f097e20..b5571dc655d0d 100644 --- a/CCDB/test/testCcdbApi.cxx +++ b/CCDB/test/testCcdbApi.cxx @@ -137,7 +137,7 @@ void countItems(const string& s, int& countObjects, int& countSubfolders) } } - if (line.find("\"path\"") == 0) { + if (line.find(R"("path")") == 0) { countObjects++; } } From 9c308a9459f99cfc9eb7e5eaad4f19573f016df6 Mon Sep 17 00:00:00 2001 From: bvonhall Date: Tue, 11 Sep 2018 15:57:37 +0200 Subject: [PATCH 05/10] Fixes for mac --- CCDB/src/CcdbApi.cxx | 1 + CCDB/test/testCcdbApi.cxx | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index 16c75dff85d55..fb63739964961 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -17,6 +17,7 @@ #include #include #include +#include namespace o2 { diff --git a/CCDB/test/testCcdbApi.cxx b/CCDB/test/testCcdbApi.cxx index b5571dc655d0d..c5088fe24232b 100644 --- a/CCDB/test/testCcdbApi.cxx +++ b/CCDB/test/testCcdbApi.cxx @@ -13,17 +13,14 @@ /// \author Barthelemy von Haller /// -#include "CCDB/CcdbApi.h" - -#define BOOST_TEST_MODULE Quality test +#define BOOST_TEST_MODULE CCDB #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK +#include "CCDB/CcdbApi.h" #include #include #include -#include "curl/curl.h" - #include #include #include @@ -64,7 +61,7 @@ BOOST_AUTO_TEST_CASE(retrieve_test) BOOST_CHECK_EQUAL(h1->GetName(), "object1"); auto h2 = f.api.retrieve("asdf/asdf", f.metadata); - BOOST_CHECK_EQUAL(h2, nullptr); + BOOST_CHECK(h2 == nullptr); } BOOST_AUTO_TEST_CASE(truncate_test) From b95cb635add1af20496e7675ff50a9305e435f5b Mon Sep 17 00:00:00 2001 From: bvonhall Date: Wed, 12 Sep 2018 17:06:12 +0200 Subject: [PATCH 06/10] a few more output to understand the failure on macos CI --- CCDB/CMakeLists.txt | 2 +- CCDB/test/testCcdbApi.cxx | 39 ++++++++++++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/CCDB/CMakeLists.txt b/CCDB/CMakeLists.txt index 2a2fa6ec8df98..3459e7b44d882 100644 --- a/CCDB/CMakeLists.txt +++ b/CCDB/CMakeLists.txt @@ -93,7 +93,7 @@ install( set(TEST_SRCS test/testWriteReadAny.cxx test/testCcdbApi.cxx -) +) O2_GENERATE_TESTS( BUCKET_NAME ${BUCKET_NAME} diff --git a/CCDB/test/testCcdbApi.cxx b/CCDB/test/testCcdbApi.cxx index c5088fe24232b..59006adf56cad 100644 --- a/CCDB/test/testCcdbApi.cxx +++ b/CCDB/test/testCcdbApi.cxx @@ -44,6 +44,26 @@ struct test_fixture { map metadata; }; + +long getFutureTimestamp(int secondsInFuture) +{ + std::chrono::seconds sec(secondsInFuture); + auto future = std::chrono::system_clock::now() + sec; + auto future_ms = std::chrono::time_point_cast(future); + auto epoch = future_ms.time_since_epoch(); + auto value = std::chrono::duration_cast(epoch); + return value.count(); +} + +long getCurrentTimestamp() +{ + auto now = std::chrono::system_clock::now(); + auto now_ms = std::chrono::time_point_cast(now); + auto epoch = now_ms.time_since_epoch(); + auto value = std::chrono::duration_cast(epoch); + return value.count(); +} + BOOST_AUTO_TEST_CASE(store_test) { test_fixture f; @@ -80,7 +100,9 @@ BOOST_AUTO_TEST_CASE(delete_test) test_fixture f; auto h1 = new TH1F("object1", "object1", 100, 0, 99); - f.api.store(h1, "Test/Detector", f.metadata); + long from = getCurrentTimestamp(); + long to = getFutureTimestamp(60 * 60 * 24 * 365 * 10); + f.api.store(h1, "Test/Detector", f.metadata, from, to); // test with explicit dates auto h2 = f.api.retrieve("Test/Detector", f.metadata); BOOST_CHECK(h2 != nullptr); f.api.deleteObject("Test/Detector"); @@ -127,10 +149,9 @@ void countItems(const string& s, int& countObjects, int& countSubfolders) } if (subfolderMode) { - if (line.find(']') == 0) { + countSubfolders++; + if (line.find(']') != std::string::npos) { break; - } else { - countSubfolders++; } } @@ -159,22 +180,26 @@ BOOST_AUTO_TEST_CASE(list_test) // more complex tree auto h1 = new TH1F("object1", "object1", 100, 0, 99); + cout << "storing object 1 in Test" << endl; f.api.store(h1, "Test", f.metadata); + cout << "storing object 2 in Test/Detector" << endl; f.api.store(h1, "Test/Detector", f.metadata); + cout << "storing object 3 in Test/Detector" << endl; f.api.store(h1, "Test/Detector", f.metadata); + cout << "storing object 4 in Test/Detector" << endl; f.api.store(h1, "Test/Detector", f.metadata); + cout << "storing object 5 in Test/Detector/Sub/abc" << endl; f.api.store(h1, "Test/Detector/Sub/abc", f.metadata); s = f.api.list("Test/Detector", false, "application/json"); countItems(s, countObjects, countSubfolders); BOOST_CHECK_EQUAL(countObjects, 3); - // BOOST_CHECK_EQUAL(countSubfolders, 1); + BOOST_CHECK_EQUAL(countSubfolders, 1); s = f.api.list("Test/Detector*", false, "application/json"); countItems(s, countObjects, countSubfolders); - cout << "s : " << s << endl; BOOST_CHECK_EQUAL(countObjects, 4); - // BOOST_CHECK_EQUAL(countSubfolders, 0); + BOOST_CHECK_EQUAL(countSubfolders, 0); s = f.api.list("Test/Detector", true, "application/json"); countItems(s, countObjects, countSubfolders); From de8556461dbaf41d89126f0d7e573a5aa10e50ce Mon Sep 17 00:00:00 2001 From: bvonhall Date: Wed, 12 Sep 2018 17:14:50 +0200 Subject: [PATCH 07/10] default values for validity were missing --- CCDB/src/CcdbApi.cxx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index fb63739964961..0dc0e5369fa17 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -55,7 +55,18 @@ void CcdbApi::store(TObject* rootObject, std::string path, std::mapIsA()); - string fullUrl = getFullUrlForStorage(path, metadata, startValidityTimestamp, endValidityTimestamp); + // Prepare + long sanitizedStartValidityTimestamp = startValidityTimestamp; + if(startValidityTimestamp == -1) { + cout << "Start of Validity not set, current timestamp used." << endl; + sanitizedStartValidityTimestamp = getCurrentTimestamp(); + } + long sanitizedEndValidityTimestamp = endValidityTimestamp; + if(endValidityTimestamp == -1) { + cout << "End of Validity not set, start of validity plus 1 year used." << endl; + sanitizedEndValidityTimestamp = getFutureTimestamp(60 * 60 * 24 * 365); + } + string fullUrl = getFullUrlForStorage(path, metadata, sanitizedStartValidityTimestamp, sanitizedEndValidityTimestamp); // Curl preparation CURL* curl; From ffcd2005eae89abf745cc998a8e7929eb500e578 Mon Sep 17 00:00:00 2001 From: bvonhall Date: Wed, 12 Sep 2018 17:23:51 +0200 Subject: [PATCH 08/10] fix for CC7 : missing header --- CCDB/test/testCcdbApi.cxx | 1 + 1 file changed, 1 insertion(+) diff --git a/CCDB/test/testCcdbApi.cxx b/CCDB/test/testCcdbApi.cxx index 59006adf56cad..4fa2acd974875 100644 --- a/CCDB/test/testCcdbApi.cxx +++ b/CCDB/test/testCcdbApi.cxx @@ -26,6 +26,7 @@ #include #include #include +#include using namespace std; using namespace o2::ccdb; From 20489157922c5190b6f9d0fcb83f0f001f411a78 Mon Sep 17 00:00:00 2001 From: bvonhall Date: Wed, 12 Sep 2018 17:32:32 +0200 Subject: [PATCH 09/10] formatting --- CCDB/include/CCDB/CcdbApi.h | 4 ++-- CCDB/src/CcdbApi.cxx | 8 ++++---- CCDB/test/testCcdbApi.cxx | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CCDB/include/CCDB/CcdbApi.h b/CCDB/include/CCDB/CcdbApi.h index 7d672e2b93ed7..59a11b139ff90 100644 --- a/CCDB/include/CCDB/CcdbApi.h +++ b/CCDB/include/CCDB/CcdbApi.h @@ -169,7 +169,7 @@ class CcdbApi //: public DatabaseInterface /// Base URL of the CCDB (with port) std::string mUrl; }; -} -} +} // namespace ccdb +} // namespace o2 #endif //PROJECT_CCDBAPI_H diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index 0dc0e5369fa17..7355a24f4ca7e 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -57,12 +57,12 @@ void CcdbApi::store(TObject* rootObject, std::string path, std::map metadata; }; - long getFutureTimestamp(int secondsInFuture) { std::chrono::seconds sec(secondsInFuture); @@ -126,7 +125,8 @@ static inline void rtrim(std::string& s) { s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); - }).base(), + }) + .base(), s.end()); } From 87eba5ad643846de584883e252b64d7d7bfd7bde Mon Sep 17 00:00:00 2001 From: bvonhall Date: Wed, 12 Sep 2018 17:42:16 +0200 Subject: [PATCH 10/10] formatting --- CCDB/test/testCcdbApi.cxx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CCDB/test/testCcdbApi.cxx b/CCDB/test/testCcdbApi.cxx index 15fe170575839..02be1a50946cc 100644 --- a/CCDB/test/testCcdbApi.cxx +++ b/CCDB/test/testCcdbApi.cxx @@ -125,8 +125,7 @@ static inline void rtrim(std::string& s) { s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); - }) - .base(), + }).base(), s.end()); }