Skip to content

Commit 6814170

Browse files
wiechulashahor02
wiechula
authored and
shahor02
committed
TPC: IDC FLP processing improvements
* Skip empty pages * Don't rely on FEEid in subspec (might be masked), but process all HBFs * Add raw data type definitions
1 parent ba5e7a3 commit 6814170

3 files changed

Lines changed: 153 additions & 52 deletions

File tree

‎DataFormats/Detectors/TPC/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ o2_target_root_dictionary(
5151
include/DataFormatsTPC/TrackTPC.h
5252
include/DataFormatsTPC/LaserTrack.h
5353
include/DataFormatsTPC/Constants.h
54+
include/DataFormatsTPC/RawDataTypes.h
5455
include/DataFormatsTPC/Defs.h
5556
include/DataFormatsTPC/dEdxInfo.h
5657
include/DataFormatsTPC/CompressedClusters.h

‎DataFormats/Detectors/TPC/include/DataFormatsTPC/RawDataTypes.h‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
///
13+
/// @file RawDataTypes.h
14+
/// @author Jens Wiechula
15+
///
16+
17+
#ifndef AliceO2_TPC_RawDataTypes_H
18+
#define AliceO2_TPC_RawDataTypes_H
19+
20+
#include <unordered_map>
21+
#include <string_view>
22+
23+
namespace o2::tpc::raw_data_types
24+
{
25+
enum class Type {
26+
RAWDATA = 0, ///< GBT raw data
27+
LinkZS = 1, ///< Link-based zero suppression
28+
ZS = 2, ///< final zero suppression
29+
IDC = 3, ///< integrated digitial current, with priority bit to end up in separate buffer
30+
IAC = 4, ///< Analogue currents from the current monitor
31+
};
32+
33+
const std::unordered_map<Type, std::string_view> TypeNameMap{
34+
{Type::RAWDATA, "RAWDATA"},
35+
{Type::LinkZS, "LinkZS"},
36+
{Type::ZS, "ZS"},
37+
{Type::IDC, "IDC"},
38+
{Type::IAC, "IAC"},
39+
};
40+
41+
} // namespace o2::tpc::raw_data_types
42+
43+
#endif

‎Detectors/TPC/workflow/src/IDCToVectorSpec.cxx‎

Lines changed: 109 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@
99
// granted to it by virtue of its status as an Intergovernmental Organization
1010
// or submit itself to any jurisdiction.
1111

12+
#include <iterator>
13+
#include <limits>
1214
#include <memory>
15+
#include <stdexcept>
1316
#include <vector>
1417
#include <string>
1518
#include <algorithm>
@@ -29,11 +32,14 @@
2932
#include "DPLUtils/RawParser.h"
3033
#include "Headers/DataHeader.h"
3134
#include "CommonUtils/TreeStreamRedirector.h"
35+
#include "CommonUtils/NameConf.h"
3236
#include "DataFormatsTPC/Constants.h"
3337
#include "CommonConstants/LHCConstants.h"
38+
#include "CCDB/BasicCCDBManager.h"
3439

3540
#include "DataFormatsTPC/Defs.h"
3641
#include "DataFormatsTPC/IDC.h"
42+
#include "DataFormatsTPC/RawDataTypes.h"
3743
#include "TPCBase/Utils.h"
3844
#include "TPCBase/RDHUtils.h"
3945
#include "TPCBase/Mapper.h"
@@ -43,6 +49,7 @@ using o2::constants::lhc::LHCMaxBunches;
4349
using o2::header::gDataOriginTPC;
4450
using o2::tpc::constants::LHCBCPERTIMEBIN;
4551
using RDHUtils = o2::raw::RDHUtils;
52+
using RawDataType = o2::tpc::raw_data_types::Type;
4653

4754
namespace o2::tpc
4855
{
@@ -59,17 +66,39 @@ class IDCToVectorDevice : public o2::framework::Task
5966
if (ic.options().get<bool>("write-debug")) {
6067
mDebugStream = std::make_unique<o2::utils::TreeStreamRedirector>("idc_vector_debug.root", "recreate");
6168
}
62-
const auto pedestalFile = ic.options().get<std::string>("pedestal-file");
69+
auto pedestalFile = ic.options().get<std::string>("pedestal-url");
6370
if (pedestalFile.length()) {
64-
LOGP(info, "Setting pedestal file: {}", pedestalFile);
65-
auto calPads = utils::readCalPads(pedestalFile, "Pedestals");
66-
if (calPads.size() != 1) {
67-
LOGP(error, "Pedestal could not be loaded from file {}", pedestalFile);
71+
if (pedestalFile.find("ccdb") != std::string::npos) {
72+
if (pedestalFile.find("-default") != std::string::npos) {
73+
pedestalFile = o2::base::NameConf::getCCDBServer();
74+
}
75+
LOGP(info, "Loading pedestals from ccdb: {}", pedestalFile);
76+
auto& cdb = o2::ccdb::BasicCCDBManager::instance();
77+
cdb.setURL(pedestalFile);
78+
if (cdb.isHostReachable()) {
79+
auto pedestalNoise = cdb.get<std::unordered_map<std::string, CalPad>>("TPC/Calib/PedestalNoise");
80+
try {
81+
if (!pedestalNoise) {
82+
throw std::runtime_error("Couldn't retrieve PedestaNoise map");
83+
}
84+
mPedestal = std::make_unique<CalPad>(pedestalNoise->at("Pedestals"));
85+
} catch (const std::exception& e) {
86+
LOGP(fatal, "could not load pedestals from {} ({}), required for IDC processing", pedestalFile, e.what());
87+
}
88+
} else {
89+
LOGP(fatal, "ccdb access to {} requested, but host is not reachable. Cannot load pedestals, required for IDC processing", pedestalFile);
90+
}
6891
} else {
69-
for (auto p : calPads) {
70-
mNoisePedestal.emplace_back(p);
92+
LOGP(info, "Loading pedestals from file: {}", pedestalFile);
93+
auto calPads = utils::readCalPads(pedestalFile, "Pedestals");
94+
if (calPads.size() != 1) {
95+
LOGP(fatal, "Pedestal could not be loaded from file {}, required for IDC processing", pedestalFile);
96+
} else {
97+
mPedestal.reset(calPads[0]);
7198
}
7299
}
100+
} else {
101+
LOGP(error, "No pedestal file set, IDCs will be without pedestal subtraction!");
73102
}
74103

75104
initIDC();
@@ -85,48 +114,57 @@ class IDCToVectorDevice : public o2::framework::Task
85114
uint32_t tfCounter = 0;
86115
bool first = true;
87116

88-
CalPad* pedestals = nullptr;
89-
if (mNoisePedestal.size() && mNoisePedestal[0]) {
90-
pedestals = mNoisePedestal[0].get();
91-
}
117+
CalPad* pedestals = mPedestal.get();
92118

93119
for (auto const& ref : InputRecordWalker(pc.inputs(), filter)) {
94120
const auto* dh = DataRefUtils::getHeader<o2::header::DataHeader*>(ref);
95-
// ---| extract hardware information to do the processing |---
96-
const auto feeId = (FEEIDType)dh->subSpecification;
97-
const auto link = rdh_utils::getLink(feeId);
98-
const uint32_t cruID = rdh_utils::getCRU(feeId);
99-
const auto endPoint = rdh_utils::getEndPoint(feeId);
100121
tfCounter = dh->tfCounter;
101122

102-
// only select IDCs
103-
// ToDo: cleanup once IDCs will be propagated not as RAWDATA, but IDC.
104-
if (link != rdh_utils::IDCLinkID) {
105-
continue;
106-
}
107-
LOGP(info, "IDC Processing firstTForbit {:9}, tfCounter {:5}, run {:6}, feeId {:6} ({:3}/{}/{:2})", dh->firstTForbit, dh->tfCounter, dh->runNumber, feeId, cruID, endPoint, link);
108-
109-
if (std::find(mCRUs.begin(), mCRUs.end(), cruID) == mCRUs.end()) {
110-
LOGP(error, "IDC CRU {:3} not configured in CRUs, skipping", cruID);
111-
continue;
112-
}
113-
114-
const CRU cru(cruID);
115-
const int sector = cru.sector();
116-
const auto& partInfo = mapper.getPartitionInfo(cru.partition());
117-
const int fecLinkOffsetCRU = (partInfo.getNumberOfFECs() + 1) / 2;
118-
const int fecSectorOffset = partInfo.getSectorFECOffset();
119-
const GlobalPadNumber regionPadOffset = Mapper::GLOBALPADOFFSET[cru.region()];
120-
const GlobalPadNumber numberPads = Mapper::PADSPERREGION[cru.region()];
121-
int sampaOnFEC{}, channelOnSAMPA{};
122-
auto& idcVec = mIDCvectors[cruID];
123-
auto& infoVec = mIDCInfos[cruID];
124-
125123
// ---| data loop |---
126124
const gsl::span<const char> raw = pc.inputs().get<gsl::span<char>>(ref);
127125
o2::framework::RawParser parser(raw.data(), raw.size());
128126
for (auto it = parser.begin(), end = parser.end(); it != end; ++it) {
129127
const auto size = it.size();
128+
// skip empty packages (HBF open)
129+
if (size == 0) {
130+
continue;
131+
}
132+
133+
auto* rdhPtr = it.get_if<o2::header::RAWDataHeaderV6>();
134+
if (!rdhPtr) {
135+
throw std::runtime_error("could not get RDH from packet");
136+
}
137+
138+
// ---| extract hardware information to do the processing |---
139+
const auto feeId = (FEEIDType)RDHUtils::getFEEID(*rdhPtr);
140+
const auto link = rdh_utils::getLink(feeId);
141+
const uint32_t cruID = rdh_utils::getCRU(feeId);
142+
const auto endPoint = rdh_utils::getEndPoint(feeId);
143+
const auto detField = RDHUtils::getDetectorField(*rdhPtr);
144+
145+
// only select IDCs
146+
// ToDo: cleanup once IDCs will be propagated not as RAWDATA, but IDC.
147+
if ((detField != (decltype(detField))RawDataType::IDC) || (link != rdh_utils::IDCLinkID)) {
148+
continue;
149+
}
150+
LOGP(info, "IDC Processing firstTForbit {:9}, tfCounter {:5}, run {:6}, feeId {:6} ({:3}/{}/{:2})", dh->firstTForbit, dh->tfCounter, dh->runNumber, feeId, cruID, endPoint, link);
151+
152+
if (std::find(mCRUs.begin(), mCRUs.end(), cruID) == mCRUs.end()) {
153+
LOGP(error, "IDC CRU {:3} not configured in CRUs, skipping", cruID);
154+
continue;
155+
}
156+
157+
const CRU cru(cruID);
158+
const int sector = cru.sector();
159+
const auto& partInfo = mapper.getPartitionInfo(cru.partition());
160+
const int fecLinkOffsetCRU = (partInfo.getNumberOfFECs() + 1) / 2;
161+
const int fecSectorOffset = partInfo.getSectorFECOffset();
162+
const GlobalPadNumber regionPadOffset = Mapper::GLOBALPADOFFSET[cru.region()];
163+
const GlobalPadNumber numberPads = Mapper::PADSPERREGION[cru.region()];
164+
int sampaOnFEC{}, channelOnSAMPA{};
165+
auto& idcVec = mIDCvectors[cruID];
166+
auto& infoVec = mIDCInfos[cruID];
167+
130168
assert(size == sizeof(idc::Container));
131169
auto data = it.data();
132170
auto& idcs = *((idc::Container*)(data));
@@ -138,7 +176,6 @@ class IDCToVectorDevice : public o2::framework::Task
138176
if (!infoVec.size()) {
139177
infoVec.emplace_back(orbit, bc);
140178
infoIt = infoVec.end() - 1;
141-
//} else if (!infoVec.back().matches(orbit, bc)) {
142179
} else if (infoIt == infoVec.end()) {
143180
auto& lastInfo = infoVec.back();
144181
if ((orbit - lastInfo.heartbeatOrbit) != mNOrbitsIDC) {
@@ -160,15 +197,7 @@ class IDCToVectorDevice : public o2::framework::Task
160197
const size_t idcOffset = std::distance(infoVec.begin(), infoIt);
161198

162199
// TODO: for debugging, remove later
163-
/*
164-
auto* rdhPtr = it.get_if<o2::header::RAWDataHeaderV6>();
165-
const auto feeId2 = (FEEIDType)RDHUtils::getFEEID(*rdhPtr);
166-
const auto link2 = rdh_utils::getLink(feeId2);
167-
const uint32_t cruID2 = rdh_utils::getCRU(feeId2);
168-
const auto endPoint2 = rdh_utils::getEndPoint(feeId2);
169-
const auto detField = RDHUtils::getDetectorField(*rdhPtr);
170-
LOGP(info, "processing IDCs for CRU {}, ep {}, feeId {:6} ({:3}/{}/{:2}), detField: {}, orbit {}, bc {}, idcOffset {}, idcVec size {}, epSeen {:02b}", cruID, endPoint, feeId2, cruID2, endPoint2, link2, detField, orbit, bc, idcOffset, idcVec.size(), lastInfo.epSeen);
171-
*/
200+
// LOGP(info, "processing IDCs for CRU {}, ep {}, feeId {:6} ({:3}/{}/{:2}), detField: {}, orbit {}, bc {}, idcOffset {}, idcVec size {}, epSeen {:02b}", cruID, endPoint, feeId, cruID, endPoint, link, detField, orbit, bc, idcOffset, idcVec.size(), lastInfo.epSeen);
172201

173202
const float norm = 1. / float(mTimeStampsPerIntegrationInterval);
174203
for (uint32_t iLink = 0; iLink < idc::Links; ++iLink) {
@@ -208,6 +237,10 @@ class IDCToVectorDevice : public o2::framework::Task
208237
LOGP(info, "endOfStream");
209238
ec.services().get<ControlService>().readyToQuit(QuitRequest::Me);
210239
if (mDebugStream) {
240+
// set some default aliases
241+
auto& stream = (*mDebugStream) << "idcs";
242+
auto& tree = stream.getTree();
243+
tree.SetAlias("sector", "int(cru/10)");
211244
mDebugStream->Close();
212245
}
213246
}
@@ -238,7 +271,7 @@ class IDCToVectorDevice : public o2::framework::Task
238271
std::unordered_map<uint32_t, std::vector<float>> mIDCvectors; ///< decoded IDCs per cru for each pad in the region over all IDC packets in the TF
239272
std::unordered_map<uint32_t, std::vector<IDCInfo>> mIDCInfos; ///< IDC packet information within the TF
240273
std::unique_ptr<o2::utils::TreeStreamRedirector> mDebugStream; ///< debug output streamer
241-
std::vector<std::unique_ptr<CalPad>> mNoisePedestal{}; ///< noise and pedestal values
274+
std::unique_ptr<CalPad> mPedestal{}; ///< noise and pedestal values
242275

243276
//____________________________________________________________________________
244277
void snapshotIDCs(DataAllocator& output)
@@ -310,6 +343,8 @@ class IDCToVectorDevice : public o2::framework::Task
310343
mDebugStream->GetFile()->cd();
311344
auto& stream = (*mDebugStream) << "idcs";
312345
uint32_t seen = 0;
346+
static uint32_t firstOrbit = std::numeric_limits<uint32_t>::max();
347+
313348
for (auto cru : mCRUs) {
314349
if (mIDCInfos.find(cru) == mIDCInfos.end()) {
315350
continue;
@@ -320,6 +355,9 @@ class IDCToVectorDevice : public o2::framework::Task
320355
for (int i = 0; i < infos.size(); ++i) {
321356
auto& info = infos[i];
322357

358+
if (firstOrbit == std::numeric_limits<uint32_t>::max()) {
359+
firstOrbit = info.heartbeatOrbit;
360+
}
323361
auto idcFirst = idcVec.begin() + i * Mapper::PADSPERREGION[cru % Mapper::NREGIONS];
324362
auto idcLast = idcFirst + Mapper::PADSPERREGION[cru % Mapper::NREGIONS];
325363
std::vector<float> idcs(idcFirst, idcLast);
@@ -331,17 +369,36 @@ class IDCToVectorDevice : public o2::framework::Task
331369
const short pads = (short)mapper.getNumberOfPadsInRowSector(row[ipad]);
332370
cpad[ipad] = (short)padPos.getPad() - pads / 2;
333371
}
334-
float mean = std::accumulate(idcs.begin(), idcs.end(), 0.f) / float(idcs.size());
372+
auto idcSort = idcs;
373+
std::sort(idcSort.begin(), idcSort.end());
374+
const auto idcSize = idcSort.size();
375+
float median = idcSize % 2 ? idcSort[idcSize / 2] : (idcSort[idcSize / 2] + idcSort[idcSize / 2 - 1]) / 2.f;
376+
// outlier removal
377+
auto itEnd = idcSort.end();
378+
while (std::abs(*(itEnd - 1) - median) > 40) {
379+
--itEnd;
380+
}
381+
382+
float mean = 0;
383+
const auto nForMean = std::distance(idcSort.begin(), itEnd);
384+
if (nForMean > 0) {
385+
mean = std::accumulate(idcSort.begin(), itEnd, 0.f) / float(nForMean);
386+
}
387+
uint32_t outliers = uint32_t(idcSort.size() - nForMean);
335388

336389
stream << "cru=" << cru
390+
<< "entry=" << i
337391
<< "epSeen=" << info.epSeen
338392
<< "tfCounter=" << tfCounter
393+
<< "firstOrbit=" << firstOrbit
339394
<< "orbit=" << info.heartbeatOrbit
340395
<< "bc=" << info.heartbeatBC
341396
<< "idcs=" << idcs
342397
<< "cpad=" << cpad
343398
<< "row=" << row
399+
<< "outliers=" << outliers
344400
<< "idc_mean=" << mean
401+
<< "idc_median=" << median
345402
<< "\n";
346403
}
347404
}
@@ -366,7 +423,7 @@ o2::framework::DataProcessorSpec getIDCToVectorSpec(const std::string inputSpec,
366423
AlgorithmSpec{adaptFromTask<device>(crus)},
367424
Options{
368425
{"write-debug", VariantType::Bool, false, {"write a debug output tree."}},
369-
{"pedestal-file", VariantType::String, "", {"file with pedestals and noise for zero suppression"}},
426+
{"pedestal-url", VariantType::String, "ccdb-default", {"ccdb-default: load from NameConf::getCCDBServer() OR ccdb url (must contain 'ccdb' OR pedestal file name"}},
370427
} // end Options
371428
}; // end DataProcessorSpec
372429
}

0 commit comments

Comments
 (0)