Skip to content

Commit b0dac81

Browse files
sawenzelsawenzel
committed
CCDB API: generalization of content retrival (part 1)
This commit achieves the following: a) support for cases in which the CCDB server does not serve content directly but redirects to other resources b) support for objects on ALIEN resource Changes are offered only for retrieval of objects via the strongly typed interfaces (no new features for interfaces based on TObject*). A new test is added - querying from a server with ALIEN backend. This concerns: https://alice.its.cern.ch/jira/browse/O2-1809 The PR is potentially incomplete. Further changes in other places or tools likely.
1 parent 8f41f35 commit b0dac81

4 files changed

Lines changed: 292 additions & 83 deletions

File tree

‎CCDB/CMakeLists.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,12 @@ o2_add_test(CcdbApi
5151
PUBLIC_LINK_LIBRARIES O2::CCDB
5252
LABELS ccdb)
5353

54+
o2_add_test(CcdbApi-Alien
55+
SOURCES test/testCcdbApi_alien.cxx
56+
COMPONENT_NAME ccdb
57+
PUBLIC_LINK_LIBRARIES O2::CCDB
58+
LABELS ccdb)
59+
5460
o2_add_test(BasicCCDBManager
5561
SOURCES test/testBasicCCDBManager.cxx
5662
COMPONENT_NAME ccdb

‎CCDB/include/CCDB/CcdbApi.h‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "CCDB/CcdbObjectInfo.h"
2626

2727
class TFile;
28+
class TGrid;
2829

2930
namespace o2
3031
{
@@ -377,6 +378,26 @@ class CcdbApi //: public DatabaseInterface
377378
*/
378379
void* extractFromLocalFile(std::string const& filename, TClass const* cl) const;
379380

381+
/**
382+
* Helper function to download binary content from alien:// storage
383+
* @param fullUrl The alien URL
384+
* @param tcl The TClass object describing the serialized type
385+
* @return raw pointer to created object
386+
*/
387+
void* downloadAlienContent(std::string const& fullUrl, TClass* tcl) const;
388+
389+
// initialize the TGrid (Alien connection)
390+
bool initTGrid() const;
391+
// checks if an alien token is available, required to make a TGrid connection
392+
bool checkAlienToken() const;
393+
394+
/// Queries the CCDB server and navigates through possible redirects until binary content is found; Retrieves content as instance
395+
/// given by TClass if that is possible. Returns nullptr if something fails...
396+
void* navigateURLsAndRetrieveContent(CURL*, std::string const& url, TClass* cl, std::map<std::string, std::string>* headers) const;
397+
398+
// helper that interprets a content chunk as TMemFile and extracts the object therefrom
399+
void* interpretAsTMemFileAndExtract(char* contentptr, size_t contentsize, TClass* cl) const;
400+
380401
/**
381402
* Initialization of CURL
382403
*/
@@ -386,6 +407,10 @@ class CcdbApi //: public DatabaseInterface
386407
std::string mUrl{};
387408
std::string mSnapshotTopPath{};
388409
bool mInSnapshotMode = false;
410+
mutable std::multimap<std::string, std::string> mHeaderData; //! a "global" internal data structure that can be filled with HTTP header information
411+
// (without need to recreate this structure locally each time)
412+
mutable TGrid* mAlienInstance = nullptr; // a cached connection to TGrid (needed for Alien locations)
413+
bool mHaveAlienToken = false; // stores if an alien token is available
389414

390415
ClassDefNV(CcdbApi, 1);
391416
};

‎CCDB/src/CcdbApi.cxx‎

Lines changed: 159 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <TMessage.h>
2323
#include <sstream>
2424
#include <TFile.h>
25+
#include <TGrid.h>
2526
#include <TSystem.h>
2627
#include <TStreamerInfo.h>
2728
#include <TMemFile.h>
@@ -70,6 +71,12 @@ void CcdbApi::init(std::string const& host)
7071
} else {
7172
curlInit();
7273
}
74+
75+
// find out if we can can in principle connect to Alien
76+
mHaveAlienToken = checkAlienToken();
77+
if (!mHaveAlienToken) {
78+
LOG(WARN) << "CCDB: Did not find an alien token; Cannot serve objects located on alien://";
79+
}
7380
}
7481

7582
/**
@@ -391,15 +398,16 @@ std::string CcdbApi::generateFileName(const std::string& inp)
391398

392399
namespace
393400
{
401+
template <typename MapType = std::map<std::string, std::string>>
394402
size_t header_map_callback(char* buffer, size_t size, size_t nitems, void* userdata)
395403
{
396-
auto* headers = static_cast<std::map<std::string, std::string>*>(userdata);
404+
auto* headers = static_cast<MapType*>(userdata);
397405
auto header = std::string(buffer, size * nitems);
398406
std::string::size_type index = header.find(':', 0);
399407
if (index != std::string::npos) {
400-
headers->insert(std::make_pair(
401-
boost::algorithm::trim_copy(header.substr(0, index)),
402-
boost::algorithm::trim_copy(header.substr(index + 1))));
408+
const auto key = boost::algorithm::trim_copy(header.substr(0, index));
409+
const auto value = boost::algorithm::trim_copy(header.substr(index + 1));
410+
headers->insert(std::make_pair(key, value));
403411
}
404412
return size * nitems;
405413
}
@@ -458,7 +466,7 @@ TObject* CcdbApi::retrieveFromTFile(std::string const& path, std::map<std::strin
458466
// setup curl for headers handling
459467
if (headers != nullptr) {
460468
list = curl_slist_append(list, ("If-None-Match: " + to_string(timestamp)).c_str());
461-
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback);
469+
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback<>);
462470
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, headers);
463471
}
464472

@@ -643,6 +651,145 @@ void* CcdbApi::extractFromLocalFile(std::string const& filename, TClass const* t
643651
return extractFromTFile(f, tcl);
644652
}
645653

654+
bool CcdbApi::checkAlienToken() const
655+
{
656+
// a somewhat weird construction to programmatically find out if we
657+
// have a GRID token; Can be replaced with something more elegant once
658+
// alien-token-info does not ask for passwords interactively
659+
auto returncode = system("timeout 1s timeout 1s alien-token-info &> /dev/null");
660+
return returncode == 0;
661+
}
662+
663+
bool CcdbApi::initTGrid() const
664+
{
665+
if (!mAlienInstance) {
666+
if (mHaveAlienToken) {
667+
mAlienInstance = TGrid::Connect("alien");
668+
}
669+
}
670+
return mAlienInstance != nullptr;
671+
}
672+
673+
void* CcdbApi::downloadAlienContent(std::string const& url, TClass* cl) const
674+
{
675+
if (!initTGrid()) {
676+
return nullptr;
677+
}
678+
auto memfile = TMemFile::Open(url.c_str(), "OPEN");
679+
if (memfile) {
680+
auto content = extractFromTFile(*memfile, cl);
681+
delete memfile;
682+
return content;
683+
}
684+
return nullptr;
685+
}
686+
687+
void* CcdbApi::interpretAsTMemFileAndExtract(char* contentptr, size_t contentsize, TClass* tcl) const
688+
{
689+
void* result = nullptr;
690+
Int_t previousErrorLevel = gErrorIgnoreLevel;
691+
gErrorIgnoreLevel = kFatal;
692+
TMemFile memFile("name", contentptr, contentsize, "READ");
693+
gErrorIgnoreLevel = previousErrorLevel;
694+
if (!memFile.IsZombie()) {
695+
result = extractFromTFile(memFile, tcl);
696+
if (!result) {
697+
LOG(ERROR) << o2::utils::concat_string("Couldn't retrieve object corresponding to ", tcl->GetName(), " from TFile");
698+
}
699+
memFile.Close();
700+
}
701+
return result;
702+
}
703+
704+
// navigate sequence of URLs until TFile content is found; object is extracted and returned
705+
void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string const& url, TClass* cl, std::map<string, string>* headers) const
706+
{
707+
// let's see first of all if the url is something specific that curl cannot handle
708+
if (url.find("alien:/", 0) != std::string::npos) {
709+
return downloadAlienContent(url, cl);
710+
}
711+
// add other final cases here
712+
// example root://
713+
714+
// otherwise make an HTTP/CURL request
715+
// specify URL to get
716+
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
717+
// some servers don't like requests that are made without a user-agent
718+
// field, so we provide one
719+
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
720+
// if redirected , we tell libcurl NOT to follow redirection
721+
curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 0L);
722+
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback<decltype(mHeaderData)>);
723+
mHeaderData.clear();
724+
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, (void*)&mHeaderData);
725+
726+
MemoryStruct chunk{(char*)malloc(1), 0};
727+
728+
// send all data to this function
729+
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
730+
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void*)&chunk);
731+
732+
auto res = curl_easy_perform(curl_handle);
733+
long response_code = -1;
734+
void* content = nullptr;
735+
bool errorflag = false;
736+
bool cachingflag = false;
737+
if (res == CURLE_OK && curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &response_code) == CURLE_OK) {
738+
if (headers) {
739+
for (auto& p : mHeaderData) {
740+
(*headers)[p.first] = p.second;
741+
}
742+
}
743+
if (200 <= response_code && response_code < 300) {
744+
// good response and the content is directly provided and should have been dumped into "chunk"
745+
content = interpretAsTMemFileAndExtract(chunk.memory, chunk.size, cl);
746+
} else if (response_code == 304) {
747+
// this means the object exist but I am not serving
748+
// it since it's already in your possession
749+
750+
// there is nothing to be done here
751+
cachingflag = true;
752+
}
753+
// this is a more general redirection
754+
else if (300 <= response_code && response_code < 400) {
755+
// we try content locations in order of appearance until one succeeds
756+
// 1st: The "Location" field
757+
// 2nd: Possible "Content-Location" fields
758+
auto tryLocations = [this, &curl_handle, &content, cl](auto range) {
759+
for (auto it = range.first; it != range.second; ++it) {
760+
auto nextlocation = it->second;
761+
LOG(DEBUG) << "Trying content location " << nextlocation;
762+
content = navigateURLsAndRetrieveContent(curl_handle, nextlocation, cl, nullptr);
763+
if (content /* or other success marker in future */) {
764+
break;
765+
}
766+
}
767+
};
768+
tryLocations(mHeaderData.equal_range("Location"));
769+
if (content == nullptr) {
770+
tryLocations(mHeaderData.equal_range("Content-Location"));
771+
}
772+
} else if (response_code == 404) {
773+
LOG(ERROR) << "Requested resource does not exist";
774+
errorflag = true;
775+
} else {
776+
errorflag = true;
777+
}
778+
// cleanup
779+
if (chunk.memory != nullptr) {
780+
free(chunk.memory);
781+
}
782+
} else {
783+
LOG(ERROR) << "Curl request to " << url << " failed ";
784+
errorflag = true;
785+
}
786+
// indicate that an error occurred ---> used by caching layers (such as CCDBManager)
787+
if (errorflag && headers) {
788+
(*headers)["Error"] = "An error occurred during retrieval";
789+
}
790+
return content;
791+
}
792+
646793
void* CcdbApi::retrieveFromTFile(std::type_info const& tinfo, std::string const& path,
647794
std::map<std::string, std::string> const& metadata, long timestamp,
648795
std::map<std::string, std::string>* headers, std::string const& etag,
@@ -655,103 +802,32 @@ void* CcdbApi::retrieveFromTFile(std::type_info const& tinfo, std::string const&
655802
return nullptr;
656803
}
657804

658-
// Note : based on https://curl.haxx.se/libcurl/c/getinmemory.html
659-
// Thus it does not comply to our coding guidelines as it is a copy paste.
660-
661-
// Prepare CURL
662-
CURL* curl_handle;
663-
CURLcode res;
664-
struct MemoryStruct chunk {
665-
(char*)malloc(1) /*memory*/, 0 /*size*/
666-
};
667-
668-
/* init the curl session */
669-
curl_handle = curl_easy_init();
670-
805+
CURL* curl_handle = curl_easy_init();
671806
string fullUrl = getFullUrlForRetrieval(curl_handle, path, metadata, timestamp);
672807
// if we are in snapshot mode we can simply open the file; extract the object and return
673808
if (mInSnapshotMode) {
674809
return extractFromLocalFile(fullUrl, tcl);
675810
}
676811

677-
/* specify URL to get */
678-
curl_easy_setopt(curl_handle, CURLOPT_URL, fullUrl.c_str());
679-
680-
/* send all data to this function */
681-
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
682-
683-
/* we pass our 'chunk' struct to the callback function */
684-
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void*)&chunk);
685-
686-
/* some servers don't like requests that are made without a user-agent
687-
field, so we provide one */
688-
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
689-
690-
/* if redirected , we tell libcurl to follow redirection */
691-
curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L);
692-
812+
// add some global options to the curl query
693813
struct curl_slist* list = nullptr;
694814
if (!etag.empty()) {
695815
list = curl_slist_append(list, ("If-None-Match: " + etag).c_str());
696816
}
697-
698817
if (!createdNotAfter.empty()) {
699818
list = curl_slist_append(list, ("If-Not-After: " + createdNotAfter).c_str());
700819
}
701-
702820
if (!createdNotBefore.empty()) {
703821
list = curl_slist_append(list, ("If-Not-Before: " + createdNotBefore).c_str());
704822
}
705-
706-
// setup curl for headers handling
707-
if (headers != nullptr) {
823+
if (headers) {
708824
list = curl_slist_append(list, ("If-None-Match: " + to_string(timestamp)).c_str());
709-
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback);
710-
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, headers);
711-
}
712-
713-
if (list) {
714-
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
715-
}
716-
717-
/* get it! */
718-
res = curl_easy_perform(curl_handle);
719-
std::string errStr;
720-
void* result = nullptr;
721-
if (res == CURLE_OK) {
722-
long response_code;
723-
res = curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &response_code);
724-
if ((res == CURLE_OK) && (response_code != 404)) {
725-
Int_t previousErrorLevel = gErrorIgnoreLevel;
726-
gErrorIgnoreLevel = kFatal;
727-
TMemFile memFile("name", chunk.memory, chunk.size, "READ");
728-
gErrorIgnoreLevel = previousErrorLevel;
729-
if (!memFile.IsZombie()) {
730-
result = extractFromTFile(memFile, tcl);
731-
if (!result) {
732-
errStr = o2::utils::concat_string("Couldn't retrieve the object ", path);
733-
LOG(ERROR) << errStr;
734-
}
735-
memFile.Close();
736-
} else {
737-
LOG(DEBUG) << "Object " << path << " is stored in a TMemFile";
738-
}
739-
} else {
740-
errStr = o2::utils::concat_string("Invalid URL : ", fullUrl);
741-
LOG(ERROR) << errStr;
742-
}
743-
} else {
744-
errStr = o2::utils::concat_string("curl_easy_perform() failed: ", curl_easy_strerror(res));
745-
fprintf(stderr, "%s", errStr.c_str());
746-
}
747-
748-
if (!errStr.empty() && headers) {
749-
(*headers)["Error"] = errStr;
750825
}
826+
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
751827

828+
auto content = navigateURLsAndRetrieveContent(curl_handle, fullUrl, tcl, headers);
752829
curl_easy_cleanup(curl_handle);
753-
free(chunk.memory);
754-
return result;
830+
return content;
755831
}
756832

757833
size_t CurlWrite_CallbackFunc_StdString2(void* contents, size_t size, size_t nmemb, std::string* s)
@@ -930,7 +1006,7 @@ std::map<std::string, std::string> CcdbApi::retrieveHeaders(std::string const& p
9301006
/* get us the resource without a body! */
9311007
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
9321008
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
933-
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_map_callback);
1009+
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_map_callback<>);
9341010
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers);
9351011

9361012
// Perform the request, res will get the return code

0 commit comments

Comments
 (0)