Skip to content

Commit 0979945

Browse files
shahor02shahor02
committed
Utility to adjust previous EOV to SOV of overriding CCDB object, use in Populator
If the CcdbObjectInfo of the object sent to the populator was created with adjustableEOV flag (default), then, after uploading new object populator will override the EOV of the previous overlapping object (if any) to the SOV of the new object. This is done only provided the object to override has metadata key \"adjustableEOV\" defined (value is irrelevant) and has no \"default\" key. Newly uploaded object will have this \"adjustableEOV\" assigned in its metadata.
1 parent 262ceb1 commit 0979945

6 files changed

Lines changed: 76 additions & 17 deletions

File tree

‎CCDB/include/CCDB/CCDBTimeStampUtils.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@
1616
#ifndef O2_CCDBTIMESTAMPUTILS_H
1717
#define O2_CCDBTIMESTAMPUTILS_H
1818

19-
/// a couple of static helper functions to create timestamp values for CCDB queries
19+
/// a couple of static helper functions to create timestamp values for CCDB queries or override obsolete objects
2020

2121
namespace o2
2222
{
2323
namespace ccdb
2424
{
25+
class CcdbApi;
26+
class CcdbObjectInfo;
2527

2628
/// returns the timestamp in long corresponding to "now + secondsInFuture"
2729
long getFutureTimestamp(int secondsInFuture);
@@ -32,6 +34,9 @@ long getCurrentTimestamp();
3234
/// \brief Converting time into numerical time stamp representation
3335
long createTimestamp(int year, int month, int day, int hour, int minutes, int seconds);
3436

37+
/// \brief set EOV of overriden objects to SOV-1 of overriding one if it is allowed
38+
int adjustOverriddenEOV(CcdbApi& api, const CcdbObjectInfo& infoNew);
39+
3540
} // namespace ccdb
3641
} // namespace o2
3742

‎CCDB/include/CCDB/CcdbApi.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ class CcdbApi //: public DatabaseInterface
204204
* @param timestamp The timestamp to select the object
205205
* @param id The id, if any, to select the object
206206
*/
207-
void updateMetadata(std::string const& path, std::map<std::string, std::string> const& metadata, long timestamp, std::string const& id = "");
207+
void updateMetadata(std::string const& path, std::map<std::string, std::string> const& metadata, long timestamp, std::string const& id = "", long newEOV = 0);
208208

209209
/**
210210
* Return the listing of objects, and in some cases subfolders, matching this path.

‎CCDB/include/CCDB/CcdbObjectInfo.h‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,19 @@ class CcdbObjectInfo
3333
static constexpr long YEAR = 364 * DAY;
3434
static constexpr long INFINITE_TIMESTAMP = 9999999999999;
3535
static constexpr long INFINITE_TIMESTAMP_SECONDS = 2000000000; // not really inifinity, but close to std::numeric_limits<int>::max() till 18.05.2033
36+
static constexpr const char* AdjustableEOV = "adjustableEOV";
37+
static constexpr const char* DefaultObj = "default";
3638

3739
CcdbObjectInfo() = default;
3840
CcdbObjectInfo(std::string path, std::string objType, std::string flName,
3941
std::map<std::string, std::string> metadata,
40-
long startValidityTimestamp, long endValidityTimestamp)
41-
: mObjType(std::move(objType)), mFileName(std::move(flName)), mPath(std::move(path)), mMD(std::move(metadata)), mStart(startValidityTimestamp), mEnd(endValidityTimestamp) {}
42+
long startValidityTimestamp, long endValidityTimestamp, bool adjustableEOV = true)
43+
: mObjType(std::move(objType)), mFileName(std::move(flName)), mPath(std::move(path)), mMD(std::move(metadata)), mStart(startValidityTimestamp), mEnd(endValidityTimestamp)
44+
{
45+
if (adjustableEOV) {
46+
setAdjustableEOV();
47+
}
48+
}
4249
~CcdbObjectInfo() = default;
4350

4451
[[nodiscard]] const std::string& getObjectType() const { return mObjType; }
@@ -51,7 +58,26 @@ class CcdbObjectInfo
5158
void setPath(const std::string& path) { mPath = path; }
5259

5360
[[nodiscard]] const std::map<std::string, std::string>& getMetaData() const { return mMD; }
54-
void setMetaData(const std::map<std::string, std::string>& md) { mMD = md; }
61+
void setMetaData(const std::map<std::string, std::string>& md)
62+
{
63+
mMD = md;
64+
if (mAdjustableEOV) {
65+
setAdjustableEOV();
66+
}
67+
}
68+
69+
void setAdjustableEOV()
70+
{
71+
mAdjustableEOV = true;
72+
if (mMD.find(DefaultObj) != mMD.end()) {
73+
LOGP(fatal, "default object cannot have adjustable EOV, {}", mPath);
74+
}
75+
if (mMD.find(AdjustableEOV) == mMD.end()) {
76+
mMD[AdjustableEOV] = "true";
77+
}
78+
}
79+
80+
bool isAdjustableEOV() const { return mAdjustableEOV; }
5581

5682
[[nodiscard]] long getStartValidityTimestamp() const { return mStart; }
5783
void setStartValidityTimestamp(long start) { mStart = start; }
@@ -66,8 +92,8 @@ class CcdbObjectInfo
6692
std::map<std::string, std::string> mMD; // metadata
6793
long mStart = 0; // start of the validity of the object
6894
long mEnd = 0; // end of the validity of the object
69-
70-
ClassDefNV(CcdbObjectInfo, 1);
95+
bool mAdjustableEOV = false; // each new object may override EOV of object it overrides to its own SOV
96+
ClassDefNV(CcdbObjectInfo, 2);
7197
};
7298

7399
} // namespace o2::ccdb

‎CCDB/src/CCDBTimeStampUtils.cxx‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
//
1313
// Created by Sandro Wenzel on 2019-08-20.
1414
//
15-
#include <CCDB/CCDBTimeStampUtils.h>
15+
#include "CCDB/CCDBTimeStampUtils.h"
16+
#include "CCDB/CcdbApi.h"
17+
#include "CCDB/CcdbObjectInfo.h"
1618
#include <chrono>
1719
#include <ctime>
1820

@@ -54,4 +56,25 @@ long createTimestamp(int year, int month, int day, int hour, int minutes, int se
5456
return static_cast<long>(timeformat);
5557
}
5658

57-
} // namespace o2::ccdb
59+
int adjustOverriddenEOV(CcdbApi& api, const CcdbObjectInfo& infoNew)
60+
{
61+
int res = 0;
62+
if (infoNew.getStartValidityTimestamp() > 0) {
63+
std::map<std::string, std::string> dummyMD, prevHeader;
64+
dummyMD[o2::ccdb::CcdbObjectInfo::AdjustableEOV] = "true";
65+
prevHeader = api.retrieveHeaders(infoNew.getPath(), dummyMD, infoNew.getStartValidityTimestamp() - 1); // is there an adjustable object to override?
66+
const auto itETag = prevHeader.find("ETag");
67+
if (itETag != prevHeader.end() &&
68+
prevHeader.find(o2::ccdb::CcdbObjectInfo::AdjustableEOV) != prevHeader.end() &&
69+
prevHeader.find(o2::ccdb::CcdbObjectInfo::DefaultObj) == prevHeader.end()) {
70+
std::string etag = itETag->second;
71+
etag.erase(remove(etag.begin(), etag.end(), '\"'), etag.end());
72+
LOGP(info, "Adjusting EOV of previous {}/{}/{} to {} (id:{})", infoNew.getPath(), prevHeader["Valid-From"], prevHeader["Valid-Until"], infoNew.getStartValidityTimestamp() - 1, etag);
73+
// equivalent of std::string cmd = fmt::format("sh -c \"curl -X PUT {}{}{}/{}/{}\"", api.getURL(), api.getURL().back() == '/' ? "" : "/", infoNew.getPath(), infoNew.getStartValidityTimestamp() - 1, infoNew.getStartValidityTimestamp());
74+
api.updateMetadata(infoNew.getPath(), {}, infoNew.getStartValidityTimestamp() - 1, etag, infoNew.getStartValidityTimestamp());
75+
}
76+
}
77+
return res;
78+
}
79+
80+
} // namespace o2::ccdb

‎CCDB/src/CcdbApi.cxx‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1280,14 +1280,17 @@ TClass* CcdbApi::tinfo2TClass(std::type_info const& tinfo)
12801280
return cl;
12811281
}
12821282

1283-
void CcdbApi::updateMetadata(std::string const& path, std::map<std::string, std::string> const& metadata, long timestamp, std::string const& id)
1283+
void CcdbApi::updateMetadata(std::string const& path, std::map<std::string, std::string> const& metadata, long timestamp, std::string const& id, long newEOV)
12841284
{
12851285
CURL* curl = curl_easy_init();
12861286
if (curl != nullptr) {
12871287
CURLcode res;
12881288
stringstream fullUrl;
12891289
for (size_t hostIndex = 0; hostIndex < hostsPool.size(); hostIndex++) {
12901290
fullUrl << getHostUrl(hostIndex) << "/" << path << "/" << timestamp;
1291+
if (newEOV > 0) {
1292+
fullUrl << "/" << newEOV;
1293+
}
12911294
if (!id.empty()) {
12921295
fullUrl << "/" << id;
12931296
}
@@ -1305,9 +1308,11 @@ void CcdbApi::updateMetadata(std::string const& path, std::map<std::string, std:
13051308
}
13061309

13071310
if (curl != nullptr) {
1311+
LOG(debug) << "passing to curl: " << fullUrl.str();
13081312
curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str());
13091313
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT"); // make sure we use PUT
13101314
curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1315+
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
13111316
curlSetSSLOptions(curl);
13121317

13131318
// Perform the request, res will get the return code

‎Detectors/Calibration/workflow/CCDBPopulatorSpec.h‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,11 @@
2525
#include "Framework/DataDescriptorQueryBuilder.h"
2626
#include "Headers/DataHeader.h"
2727
#include "DetectorsCalibration/Utils.h"
28-
#include "CCDB/BasicCCDBManager.h"
2928
#include "CCDB/CcdbApi.h"
3029
#include "CCDB/CcdbObjectInfo.h"
30+
#include "CCDB/CCDBTimeStampUtils.h"
3131
#include "CommonUtils/NameConf.h"
3232

33-
using CcdbManager = o2::ccdb::BasicCCDBManager;
34-
3533
namespace o2
3634
{
3735
namespace calibration
@@ -49,9 +47,7 @@ class CCDBPopulator : public o2::framework::Task
4947
mSSpecMin = ic.options().get<std::int64_t>("sspec-min");
5048
mSSpecMax = ic.options().get<std::int64_t>("sspec-max");
5149
mFatalOnFailure = ic.options().get<bool>("no-fatal-on-failure");
52-
auto& mgr = CcdbManager::instance();
53-
mgr.setURL(mCCDBpath);
54-
mAPI.init(mgr.getURL());
50+
mAPI.init(mCCDBpath);
5551
}
5652

5753
void run(o2::framework::ProcessingContext& pc) final
@@ -63,7 +59,6 @@ class CCDBPopulator : public o2::framework::Task
6359
if (runNoFromDH > 0) {
6460
runNoStr = std::to_string(runNoFromDH);
6561
}
66-
6762
std::map<std::string, std::string> metadata;
6863
for (int isl = 0; isl < nSlots; isl++) {
6964
auto refWrp = pc.inputs().get("clbWrapper", isl);
@@ -90,6 +85,11 @@ class CCDBPopulator : public o2::framework::Task
9085
if (res && mFatalOnFailure) {
9186
LOGP(fatal, "failed on uploading to {} / {}", mAPI.getURL(), wrp->getPath());
9287
}
88+
89+
// do we need to override previous object?
90+
if (wrp->isAdjustableEOV()) {
91+
o2::ccdb::adjustOverriddenEOV(mAPI, *wrp.get());
92+
}
9393
}
9494
}
9595

0 commit comments

Comments
 (0)