From 69d6252069c0a22edaa13e4ceef27c69bceb20c6 Mon Sep 17 00:00:00 2001 From: Felix Schlepper Date: Wed, 12 Aug 2026 15:07:36 +0200 Subject: [PATCH] ITS: slab allocator idea Signed-off-by: Felix Schlepper --- Detectors/ITSMFT/ITS/tracking/CMakeLists.txt | 1 + .../include/ITStracking/CapacityEstimator.h | 110 +++ .../include/ITStracking/SlabBumpAllocator.h | 397 +++++++++++ .../tracking/include/ITStracking/TimeFrame.h | 13 +- .../include/ITStracking/TrackerTraits.h | 22 +- .../include/ITStracking/TrackingTopology.h | 29 +- .../tracking/include/ITStracking/Vertexer.h | 2 + .../ITS/tracking/src/CapacityEstimator.cxx | 141 ++++ .../ITSMFT/ITS/tracking/src/TimeFrame.cxx | 6 + Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx | 1 + .../ITSMFT/ITS/tracking/src/TrackerTraits.cxx | 655 ++++++++++-------- .../ITS/tracking/src/TrackingInterface.cxx | 1 + .../ITSMFT/ITS/tracking/src/Vertexer.cxx | 3 +- .../ITSMFT/ITS/tracking/test/CMakeLists.txt | 6 + .../tracking/test/testSlabBumpAllocator.cxx | 526 ++++++++++++++ .../tracking/test/testTrackingTopology.cxx | 59 ++ 16 files changed, 1670 insertions(+), 302 deletions(-) create mode 100644 Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h create mode 100644 Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h create mode 100644 Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx create mode 100644 Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx diff --git a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt index 1dd64b6f1874b..17420d47a2732 100644 --- a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt @@ -13,6 +13,7 @@ o2_add_library(ITStracking TARGETVARNAME targetName SOURCES src/ClusterLines.cxx src/Cluster.cxx + src/CapacityEstimator.cxx src/Configuration.cxx src/FastMultEstConfig.cxx src/FastMultEst.cxx diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h new file mode 100644 index 0000000000000..aa37de186b910 --- /dev/null +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h @@ -0,0 +1,110 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file CapacityEstimator.h +/// \brief Cross-timeframe output-size prediction. +/// + +#ifndef TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ +#define TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ + +#include +#include +#include + +namespace o2::its +{ + +enum SlabSite : uint8_t { + Tracklets = 0, + Cells, + Neighbours, + Roads, + NSlabSite, +}; +constexpr const char* const SlabSiteNames[SlabSite::NSlabSite]{"Tracklets", "Cells", "Neighbours", "Roads"}; + +class CapacityEstimator +{ + public: + struct Config { + float alpha{0.2f}; + float marginInit{1.30f}; + float marginMin{1.10f}; + float marginMax{4.00f}; + float marginUp{1.50f}; + float marginOverflowSlack{1.05f}; + float marginDown{0.98f}; + float lowWatermark{0.60f}; + uint32_t decayAfter{2}; + size_t floorSlots{1024}; + }; + + using KeyType = uint64_t; + + struct Decoded { + SlabSite site; + int iteration; + int variant; + int slot; + }; + + static constexpr KeyType makeKey(SlabSite site, int iteration, int variant, int slot) noexcept + { + return (static_cast(site) << 56) | + (static_cast(iteration & 0xFF) << 48) | + (static_cast(variant & 0xFFFF) << 32) | + static_cast(static_cast(slot)); + } + + static constexpr Decoded decodeKey(KeyType key) noexcept + { + return { + .site = static_cast((key >> 56) & 0xFF), + .iteration = static_cast((key >> 48) & 0xFF), + .variant = static_cast((key >> 32) & 0xFFFF), + .slot = static_cast(static_cast(key & 0xFFFFFFFF))}; + } + + static constexpr int makeVariant(int high, int low) noexcept + { + return ((high & 0xFF) << 8) | (low & 0xFF); + } + + static constexpr int getVariantHigh(int variant) noexcept + { + return (variant >> 8) & 0xFF; + } + + static constexpr int getVariantLow(int variant) noexcept + { + return variant & 0xFF; + } + + CapacityEstimator(); + explicit CapacityEstimator(Config cfg); + ~CapacityEstimator(); + CapacityEstimator(const CapacityEstimator&) = delete; + CapacityEstimator& operator=(const CapacityEstimator&) = delete; + + void reset(); + size_t capacity(uint64_t key, double scale) const; + void update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited); + void print() const; + + private: + struct Impl; + std::unique_ptr mImpl; +}; + +} // namespace o2::its + +#endif /* TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h new file mode 100644 index 0000000000000..e32516ea1e0e0 --- /dev/null +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h @@ -0,0 +1,397 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file SlabBumpAllocator.h +/// \brief Lock-free slot allocator and single-pass sink. +/// + +#ifndef TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ +#define TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ITStracking/BoundedAllocator.h" + +namespace o2::its +{ + +class SlabBumpAllocator +{ + public: + struct Range { + size_t base{0}; + size_t n{0}; + bool valid() const noexcept { return n != 0; } + }; + + SlabBumpAllocator(size_t capacity, size_t slab) noexcept + : mCapacity{capacity}, mSlab{slab ? slab : size_t{1}} {} + + Range grab() noexcept + { + if (mExhausted.load(std::memory_order_relaxed)) { + return {}; + } + const size_t base = mCursor.fetch_add(mSlab, std::memory_order_relaxed); + if (base >= mCapacity) { + mExhausted.store(true, std::memory_order_relaxed); + return {}; + } + return {.base = base, .n = std::min(mSlab, mCapacity - base)}; + } + + [[nodiscard]] size_t capacity() const noexcept { return mCapacity; } + [[nodiscard]] size_t slab() const noexcept { return mSlab; } + [[nodiscard]] size_t watermark() const noexcept + { + return std::min(mCursor.load(std::memory_order_relaxed), mCapacity); + } + + static size_t suggestSlab(size_t capacity, int nThreads, size_t minSlab = 256, size_t maxSlab = 4096) noexcept + { + const size_t t = static_cast(std::max(1, nThreads)); + const size_t fairShare = std::max(1, capacity / t); + return std::clamp(std::max(1, capacity / (8 * t)), + std::min(minSlab, fairShare), + std::min(maxSlab, fairShare)); + } + + void resetCapacity(size_t capacity) noexcept + { + assert(mCursor.load(std::memory_order_relaxed) == 0); + mCapacity = capacity; + mExhausted.store(capacity == 0, std::memory_order_relaxed); + } + + private: + std::atomic mCursor{0}; + std::atomic mExhausted{false}; + size_t mCapacity; + size_t mSlab; +}; + +enum class SlabMode : uint8_t { + Unordered, + GroupedByProducer +}; + +struct SlabSinkStats { + size_t requested{0}; ///< slots the caller predicted it would need + size_t capacity{0}; ///< slots the memory pool actually granted + size_t emitted{0}; + size_t spilled{0}; + bool overflowed{false}; ///< something did not fit into the staging area + bool memoryLimited{false}; ///< the pool granted less than was requested +}; + +template +class SlabSink +{ + static constexpr int32_t NoProducer = -1; + + public: + struct Config { + size_t capacity{0}; ///< predicted number of slots + int nThreads{1}; ///< workers that will feed this sink + int nConcurrentSinks{1}; ///< sinks that may be alive on the same pool at the same time + size_t slabOverride{0}; ///< 0: derive the slab size from the granted capacity + }; + + static constexpr size_t BytesPerSlot = Mode == SlabMode::GroupedByProducer ? (2 * sizeof(T)) + sizeof(int32_t) : sizeof(T); + + struct Run { + size_t begin{0}; + size_t end{0}; + }; + + class Handle + { + public: + explicit Handle(SlabSink* sink) + : mSink{sink}, mRuns{sink->memoryResource()}, mSpill{sink->memoryResource()}, mSpillProducer{sink->memoryResource()} {} + + void beginProducer(int32_t p) noexcept { mProducer = p; } + + template + void emplace(Args&&... args) + { + if constexpr (Mode == SlabMode::GroupedByProducer) { + assert(mProducer != NoProducer); + } + ++mEmitted; + if (mSlot == mSlotEnd && !refill()) { + mSpill.emplace_back(std::forward(args)...); + if constexpr (Mode == SlabMode::GroupedByProducer) { + mSpillProducer.push_back(mProducer); + } + return; + } + mSink->store(mSlot++, mProducer, std::forward(args)...); + } + + [[nodiscard]] size_t emitted() const noexcept { return mEmitted; } + [[nodiscard]] size_t spilled() const noexcept { return mSpill.size(); } + + private: + friend class SlabSink; + + bool refill() + { + if (mDrained) { // the arena is gone, do not touch the shared cursor again + return false; + } + closeRun(); + const auto r = mSink->mAlloc.grab(); + if (!r.valid()) { + mDrained = true; + return false; + } + mRunBegin = r.base; + mSlot = r.base; + mSlotEnd = r.base + r.n; + return true; + } + + void closeRun() + { + if constexpr (Mode == SlabMode::Unordered) { + if (mSlot > mRunBegin) { + mRuns.push_back(Run{.begin = mRunBegin, .end = mSlot}); + mRunBegin = mSlot; // only advanced once push_back succeeded, so a throw can be retried + } + } + } + + SlabSink* mSink{nullptr}; + size_t mSlot{0}; + size_t mSlotEnd{0}; + size_t mRunBegin{0}; + int32_t mProducer{NoProducer}; + bool mDrained{false}; + size_t mEmitted{0}; + bounded_vector mRuns; + bounded_vector mSpill; + bounded_vector mSpillProducer; + }; + + SlabSink(const Config& cfg, std::pmr::memory_resource* mr) + : SlabSink{cfg, grantedCapacity(cfg.capacity, cfg.nConcurrentSinks, mr), mr} {} + + SlabSink(SlabSink&&) = delete; + SlabSink(const SlabSink&) = delete; + SlabSink& operator=(SlabSink&&) = delete; + SlabSink& operator=(const SlabSink&) = delete; + ~SlabSink() = default; + + Handle& local() { return mHandles.local(); } + + [[nodiscard]] std::pmr::memory_resource* memoryResource() const noexcept { return mMR; } + + [[nodiscard]] SlabSinkStats stats() const + { + SlabSinkStats s; + s.requested = mRequested; + s.capacity = mAlloc.capacity(); + s.memoryLimited = s.capacity < s.requested; + for (const auto& h : mHandles) { + s.emitted += h.emitted(); + s.spilled += h.spilled(); + } + s.overflowed = s.spilled != 0; + return s; + } + + void finalizeUnordered(bounded_vector& dest) + { + static_assert(Mode == SlabMode::Unordered); + assert(!mFinalized); + assert(dest.get_allocator().resource()->is_equal(*mMR)); + mFinalized = true; + + bounded_vector runs{mMR}; + size_t nRuns{0}; + for (auto& h : mHandles) { + h.closeRun(); + nRuns += h.mRuns.size(); + } + runs.reserve(nRuns); + for (const auto& h : mHandles) { + runs.insert(runs.end(), h.mRuns.begin(), h.mRuns.end()); + } + std::sort(runs.begin(), runs.end(), [](const Run& a, const Run& b) { return a.begin < b.begin; }); + + // Runs are disjoint and now ordered, so the compaction target never runs ahead of the source. + size_t outputSize{0}; + for (const auto& run : runs) { + for (size_t slot{run.begin}; slot < run.end; ++slot) { + if (outputSize != slot) { + mStaging[outputSize] = std::move(mStaging[slot]); + } + ++outputSize; + } + } + deepVectorClear(runs, mMR); + mStaging.resize(outputSize); + dest.swap(mStaging); + + for (auto& h : mHandles) { + dest.insert(dest.end(), std::make_move_iterator(h.mSpill.begin()), std::make_move_iterator(h.mSpill.end())); + deepVectorClear(h.mSpill, mMR); + } + shrinkIfWasteful(dest); + deepVectorClear(mStaging, mMR); + } + + void finalizeGrouped(size_t nProducers, bounded_vector& lut, bounded_vector& dest) + { + static_assert(Mode == SlabMode::GroupedByProducer); + assert(!mFinalized); + mFinalized = true; + const size_t wm = mAlloc.watermark(); + + lut.assign(nProducers + 1, 0); + + for (size_t s = 0; s < wm; ++s) { + const int32_t p = mProducerOf[s]; + if (p != NoProducer) { + ++lut[p + 1]; + } + } + for (const auto& h : mHandles) { + for (const int32_t p : h.mSpillProducer) { + ++lut[p + 1]; + } + } + std::inclusive_scan(lut.begin(), lut.end(), lut.begin()); + + bounded_vector cursor(lut.begin(), lut.begin() + static_cast(nProducers), mMR); + for (size_t s = 0; s < wm; ++s) { + const int32_t p = mProducerOf[s]; + mProducerOf[s] = (p != NoProducer) ? cursor[p]++ : -1; + } + + const auto total = static_cast(lut.back()); + dest.resize(total); + for (auto& h : mHandles) { + for (size_t i = 0; i < h.mSpill.size(); ++i) { + dest[cursor[h.mSpillProducer[i]]++] = std::move(h.mSpill[i]); + } + deepVectorClear(h.mSpill, mMR); + deepVectorClear(h.mSpillProducer, mMR); + } + deepVectorClear(cursor, mMR); + + T* const staging = mStaging.data(); + tbb::parallel_for(tbb::blocked_range(0, wm, 4096), [&](const tbb::blocked_range& r) { + for (size_t s = r.begin(); s != r.end(); ++s) { + const int d = mProducerOf[s]; + if (d < 0) { + continue; + } + dest[d] = std::move(staging[s]); + } + }); + + deepVectorClear(mStaging, mMR); + deepVectorClear(mProducerOf, mMR); + } + + private: + SlabSink(const Config& cfg, size_t granted, std::pmr::memory_resource* mr) + : mMR{mr}, + mRequested{cfg.capacity}, + mAlloc{granted, cfg.slabOverride ? cfg.slabOverride : SlabBumpAllocator::suggestSlab(granted, cfg.nThreads)}, + mStaging{mr}, + mProducerOf{mr}, + mHandles{[this]() { return Handle{this}; }} + { + try { + mStaging.resize(granted); + if constexpr (Mode == SlabMode::GroupedByProducer) { + mProducerOf.assign(granted, NoProducer); + } + } catch (const std::bad_alloc&) { + discardPreallocation(); + } catch (const std::length_error&) { + discardPreallocation(); + } + } + + static size_t grantedCapacity(size_t requested, int nConcurrentSinks, const std::pmr::memory_resource* mr) noexcept + { + const auto* bounded = dynamic_cast(mr); + if (bounded == nullptr) { + return requested; + } + const size_t used = bounded->getUsedMemory(); + const size_t limit = bounded->getMaxMemory(); + const size_t remaining = used < limit ? limit - used : 0; + // Keep half of what is left for the spill vectors and whatever else is still live, then + // split the rest between the sinks that may be running on this pool at the same time. + const size_t budget = (remaining / 2) / static_cast(std::max(1, nConcurrentSinks)); + return std::min(requested, budget / BytesPerSlot); + } + + static void shrinkIfWasteful(bounded_vector& v) + { + if (v.capacity() > v.size() + (v.size() / 4)) { + v.shrink_to_fit(); + } + } + + void discardPreallocation() + { + // Capacity prediction is only an optimization; spilling preserves the output. + deepVectorClear(mStaging, mMR); + deepVectorClear(mProducerOf, mMR); + mAlloc.resetCapacity(0); + } + + template + void store(size_t slot, [[maybe_unused]] int32_t producer, Args&&... args) + { + mStaging[slot] = T(std::forward(args)...); + if constexpr (Mode == SlabMode::GroupedByProducer) { + mProducerOf[slot] = producer; + } + } + + std::pmr::memory_resource* mMR{nullptr}; + size_t mRequested{0}; + SlabBumpAllocator mAlloc; + bounded_vector mStaging; + bounded_vector mProducerOf; + tbb::enumerable_thread_specific mHandles; + bool mFinalized{false}; +}; + +template +using UnorderedSlabSink = SlabSink; + +template +using GroupedSlabSink = SlabSink; + +} // namespace o2::its + +#endif /* TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h index ae466a32bfc89..11246fa0ee3b0 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h @@ -23,6 +23,7 @@ #include "DataFormatsITS/TrackITS.h" #include "DataFormatsITS/Vertex.h" +#include "ITStracking/CapacityEstimator.h" #include "ITStracking/Cell.h" #include "ITStracking/Cluster.h" #include "ITStracking/Configuration.h" @@ -55,6 +56,7 @@ class ROFRecord; namespace its { + namespace gpu { template @@ -71,8 +73,10 @@ struct TimeFrame { using TrackSeedN = TrackSeed; friend class gpu::TimeFrameGPU; - TimeFrame() = default; - virtual ~TimeFrame() = default; + TimeFrame(); + virtual ~TimeFrame(); + TimeFrame(const TimeFrame&) = delete; + TimeFrame& operator=(const TimeFrame&) = delete; const Vertex& getPrimaryVertex(const int ivtx) const { return mPrimaryVertices[ivtx]; } auto& getPrimaryVertices() { return mPrimaryVertices; }; @@ -227,6 +231,9 @@ struct TimeFrame { /// staggering void setIsStaggered(bool b) noexcept { mIsStaggered = b; } + CapacityEstimator& getCapacityEstimator() noexcept { return mCapacityEstimator; } + const CapacityEstimator& getCapacityEstimator() const noexcept { return mCapacityEstimator; } + // Vertexer void computeTrackletsPerROFScans(); void computeTracletsPerClusterScans(); @@ -318,6 +325,8 @@ struct TimeFrame { std::vector> mCellsNeighboursLUT; bounded_vector mBogusClusters; /// keep track of clusters with wild coordinates + CapacityEstimator mCapacityEstimator; + // Vertexer bounded_vector mPrimaryVertices; bounded_vector mPrimaryVerticesLabels; diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h index 4d6378aded0e8..276e06dc94e6f 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h @@ -17,15 +17,18 @@ #define TRACKINGITSU_INCLUDE_TRACKERTRAITS_H_ #include +#include #include #include "DetectorsBase/Propagator.h" #include "ITStracking/Configuration.h" #include "ITStracking/IndexTableUtils.h" +#include "ITStracking/CapacityEstimator.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Cell.h" #include "ITStracking/BoundedAllocator.h" #include "ITStracking/TrackExtensionHypothesis.h" +#include "ITStracking/TrackFollower.h" #include "ITStracking/TrackITSInternal.h" // #define OPTIMISATION_OUTPUT @@ -40,12 +43,24 @@ namespace its { class TrackITSExt; +template +struct RoadSeed { + TrackSeed seed; + int cellId{constants::UnusedIndex}; + int cellTopologyId{constants::UnusedIndex}; + + RoadSeed() = default; + RoadSeed(TrackSeed&& inputSeed, int inputCellId, int inputCellTopologyId) + : seed{std::move(inputSeed)}, cellId{inputCellId}, cellTopologyId{inputCellTopologyId} {} +}; + template class TrackerTraits { public: using IndexTableUtilsN = IndexTableUtils; using TrackSeedN = TrackSeed; + using RoadSeedN = RoadSeed; virtual ~TrackerTraits() = default; virtual void adoptTimeFrame(TimeFrame* tf) { mTimeFrame = tf; } @@ -57,7 +72,7 @@ class TrackerTraits virtual void findRoads(const int iteration); template - void processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, const bounded_vector& currentCellSeed, const bounded_vector& currentCellId, const bounded_vector& currentCellTopologyId, bounded_vector& updatedCellSeed, bounded_vector& updatedCellId, bounded_vector& updatedCellTopologyId); + void processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const bounded_vector& currentSeeds, bounded_vector& updatedSeeds); void acceptTracks(int iteration, bounded_vector& tracks, const bounded_vector& trackIndices, bounded_vector>& firstClusters); void markTracks(int iteration); @@ -66,7 +81,6 @@ class TrackerTraits { mTrkParams = trkPars; } - TimeFrame* getTimeFrame() { return mTimeFrame; } virtual void setBz(float bz); float getBz() const { return mBz; } @@ -104,7 +118,9 @@ class TrackerTraits const int iteration, const TrackingFrameInfo* const* tfInfos, const Cluster* const* unsortedClusters, - const o2::base::Propagator* propagator); + const o2::base::Propagator* propagator, + const TrackFollowContext& followCtx, + TrackFollowerScratch& scratch); o2::gpu::GPUChainITS* mChain = nullptr; TimeFrame* mTimeFrame; diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingTopology.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingTopology.h index 16f5e6f01e873..80432ebc4151c 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingTopology.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingTopology.h @@ -70,6 +70,7 @@ class TrackingTopology const CellTopology* cells{nullptr}; const Range* cellsByFirstLinkIndex{nullptr}; const Id* cellsByFirstLink{nullptr}; + const Id* maxCellLevel{nullptr}; ///< host only, see getDeviceView Mask seedingLayerMask{0}; Id nLinks{0}; Id nCells{0}; @@ -78,6 +79,7 @@ class TrackingTopology GPUhdi() const LayerLink& getLink(Id id) const { return links[id]; } GPUhdi() const CellTopology& getCell(Id id) const { return cells[id]; } GPUhdi() Range getCellsStartingWithLink(Id linkId) const { return cellsByFirstLinkIndex[linkId]; } + GPUhdi() Id getMaxCellLevel(Id id) const { return maxCellLevel[id]; } #ifndef GPUCA_GPUCODE std::string asString() const @@ -93,7 +95,7 @@ class TrackingTopology const auto& c = cells[cellId]; const auto& first = links[c.firstLink]; const auto& second = links[c.secondLink]; - out += fmt::format("\n {}: {} -> {} -> {} hitMask={} links=({}, {})", cellId, first.fromLayer, first.toLayer, second.toLayer, c.hitLayerMask.asString(), c.firstLink, c.secondLink); + out += fmt::format("\n {}: {} -> {} -> {} hitMask={} links=({}, {}) maxLevel={}", cellId, first.fromLayer, first.toLayer, second.toLayer, c.hitLayerMask.asString(), c.firstLink, c.secondLink, maxCellLevel != nullptr ? int(maxCellLevel[cellId]) : -1); } return out; } @@ -143,6 +145,7 @@ class TrackingTopology } fillCellsByLink(); + fillMaxCellLevels(); } View getView() const @@ -151,6 +154,7 @@ class TrackingTopology mCells.data(), mCellsByFirstLinkIndex.data(), mCellsByFirstLink.data(), + mMaxCellLevel.data(), mSeedingLayerMask, mNLinks, mNCells, @@ -166,6 +170,7 @@ class TrackingTopology deviceCells, deviceCellsByFirstLinkIndex, deviceCellsByFirstLink, + nullptr, mSeedingLayerMask, mNLinks, mNCells, @@ -176,6 +181,7 @@ class TrackingTopology const auto& getCells() const noexcept { return mCells; } const auto& getCellsByFirstLinkIndex() const noexcept { return mCellsByFirstLinkIndex; } const auto& getCellsByFirstLink() const noexcept { return mCellsByFirstLink; } + const auto& getMaxCellLevels() const noexcept { return mMaxCellLevel; } Id getNLinks() const noexcept { return mNLinks; } Id getNCells() const noexcept { return mNCells; } Id getNCellsByFirstLink() const noexcept { return mNCellsByFirstLink; } @@ -190,6 +196,26 @@ class TrackingTopology mCells.fill({}); mCellsByFirstLinkIndex.fill(Range{0, 0}); mCellsByFirstLink.fill(0); + mMaxCellLevel.fill(0); + } + + void fillMaxCellLevels() + { + for (Id cellId = 0; cellId < mNCells; ++cellId) { + mMaxCellLevel[cellId] = 1; + } + for (int outerLayer = 0; outerLayer < mMaxLayers; ++outerLayer) { + for (Id cellId = 0; cellId < mNCells; ++cellId) { + if (mCells[cellId].hitLayerMask.last() != outerLayer) { + continue; + } + const auto& successors = mCellsByFirstLinkIndex[mCells[cellId].secondLink]; + for (Id i = 0; i < successors.getEntries(); ++i) { + const Id next = mCellsByFirstLink[successors.getFirstEntry() + i]; + mMaxCellLevel[next] = o2::gpu::CAMath::Max(mMaxCellLevel[next], static_cast(mMaxCellLevel[cellId] + 1)); + } + } + } } void fillCellsByLink() @@ -230,6 +256,7 @@ class TrackingTopology std::array mCells{}; std::array mCellsByFirstLinkIndex{}; std::array mCellsByFirstLink{}; + std::array mMaxCellLevel{}; }; } // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h index c899dde24ed44..59c22b505bc94 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h @@ -99,6 +99,7 @@ class Vertexer private: std::uint32_t mTimeFrameCounter = 0; + double mTotalTime{0}; VertexerTraitsN* mTraits = nullptr; /// Observer pointer, not owned by this class TimeFrameN* mTimeFrame = nullptr; /// Observer pointer, not owned by this class @@ -164,6 +165,7 @@ float Vertexer::evaluateTask(void (Vertexer::*task)(T...), std LOGP(info, "iter:{}:{}: {}", iteration, StateNames[mCurStep], mMemoryPool->asString()); } + mTotalTime += diff; return diff; } diff --git a/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx b/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx new file mode 100644 index 0000000000000..6405e2aa0fb74 --- /dev/null +++ b/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx @@ -0,0 +1,141 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "ITStracking/CapacityEstimator.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "Framework/Logger.h" + +namespace o2::its +{ + +struct CapacityEstimator::Impl { + struct Entry { + float ratio{0.f}; + float margin{0.f}; + uint32_t nSamples{0}; + uint32_t nLowStreak{0}; + }; + + explicit Impl(Config config) : cfg{config} {} + + Config cfg; + mutable std::mutex mutex; + std::unordered_map entries; +}; + +CapacityEstimator::CapacityEstimator() : CapacityEstimator{Config{}} {} + +CapacityEstimator::CapacityEstimator(Config cfg) : mImpl{std::make_unique(cfg)} {} + +CapacityEstimator::~CapacityEstimator() = default; + +void CapacityEstimator::reset() +{ + std::lock_guard lock{mImpl->mutex}; + mImpl->entries.clear(); +} + +size_t CapacityEstimator::capacity(uint64_t key, double scale) const +{ + if (!(scale > 0.)) { + return 0; + } + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end() || it->second.nSamples == 0) { + return mImpl->cfg.floorSlots; + } + const auto& e = it->second; + const double raw = double(e.ratio) * scale * double(e.margin); + if (!std::isfinite(raw) || raw < 0.) { + return mImpl->cfg.floorSlots; + } + if (raw >= static_cast(std::numeric_limits::max())) { + return std::numeric_limits::max(); + } + return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); +} + +void CapacityEstimator::update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited) +{ + if (!(scale > 0.)) { + return; + } + std::lock_guard lock{mImpl->mutex}; + auto& e = mImpl->entries[key]; + const auto& cfg = mImpl->cfg; + + const bool firstSample = e.nSamples == 0; + if (firstSample) { + e.margin = cfg.marginInit; + } + const auto sample = static_cast(double(emitted) / scale); + e.ratio = firstSample ? sample : (cfg.alpha * sample) + ((1.f - cfg.alpha) * e.ratio); + ++e.nSamples; + + if (memoryLimited) { + e.nLowStreak = 0; + e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); + return; + } + if (overflowed) { + e.nLowStreak = 0; + if (!firstSample) { + const float shortfall = capacityUsed ? static_cast(double(emitted) / double(capacityUsed)) : cfg.marginUp; + e.margin = std::min(cfg.marginMax, e.margin * std::clamp(shortfall * cfg.marginOverflowSlack, 1.02f, cfg.marginUp)); + } + return; + } + const float util = capacityUsed ? float(double(emitted) / double(capacityUsed)) : 1.f; + if (util < cfg.lowWatermark) { + if (++e.nLowStreak >= cfg.decayAfter) { + e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); + e.nLowStreak = 0; + } + } else if (e.nLowStreak > 0) { + --e.nLowStreak; + } +} + +void CapacityEstimator::print() const +{ + std::lock_guard lock{mImpl->mutex}; + std::vector keys; + keys.reserve(mImpl->entries.size()); + for (const auto& [key, _] : mImpl->entries) { + keys.push_back(key); + } + std::sort(keys.begin(), keys.end(), [](KeyType a, KeyType b) { + const auto da = decodeKey(a); + const auto db = decodeKey(b); + return std::tie(da.site, da.iteration, da.variant, da.slot) < + std::tie(db.site, db.iteration, db.variant, db.slot); + }); + if (keys.empty()) { + return; + } + LOGP(info, "Printing CapacityEstimators:"); + for (const auto key : keys) { + const auto& value = mImpl->entries.at(key); + const auto decoded = decodeKey(key); + LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | sam:{} | low:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, value.nSamples, value.nLowStreak); + } +} + +} // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx index 2ef2f7b724337..497857295a269 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx @@ -43,6 +43,12 @@ constexpr float DefClusErrorCol = o2::itsmft::SegmentationAlpide::PitchCol * 0.5 constexpr float DefClusError2Row = DefClusErrorRow * DefClusErrorRow; constexpr float DefClusError2Col = DefClusErrorCol * DefClusErrorCol; +template +TimeFrame::TimeFrame() = default; + +template +TimeFrame::~TimeFrame() = default; + template void TimeFrame::addPrimaryVertex(const Vertex& vert) { diff --git a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx index 4862c3add5893..28e967386e984 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx @@ -59,6 +59,7 @@ float Tracker::clustersToTracks(const LogFunc& logger, const LogFunc& e if (mTrkParams[iteration].DropTFUponFailure) { mMemoryPool->print(); mTimeFrame->wipe(); + mTimeFrame->getCapacityEstimator().reset(); ++mNumberOfDroppedTFs; error(std::format("...Dropping TimeSlice {} (out of {} dropped {})...", mTimeSlice, mTimeFrameCounter, mNumberOfDroppedTFs)); } else { diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx index 7489e334996a0..589702e735118 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx @@ -25,6 +25,7 @@ #include #include #include +#include #include "DetectorsBase/Propagator.h" #include "GPUCommonMath.h" @@ -34,6 +35,7 @@ #include "ITStracking/IndexTableUtils.h" #include "ITStracking/LayerMask.h" #include "ITStracking/ROFLookupTables.h" +#include "ITStracking/SlabBumpAllocator.h" #include "ITStracking/TrackerTraits.h" #include "ITStracking/TrackFollower.h" #include "ITStracking/TrackHelpers.h" @@ -42,12 +44,6 @@ namespace o2::its { -struct PassMode { - using OnePass = std::integral_constant; - using TwoPassCount = std::integral_constant; - using TwoPassInsert = std::integral_constant; -}; - template void TrackerTraits::computeLayerTracklets(const int iteration, int iVertex) { @@ -62,31 +58,29 @@ void TrackerTraits::computeLayerTracklets(const int iteration, int iVer gsl::span diamondSpan(&diamondVert, 1); mTaskArena->execute([&] { - auto forTracklets = [&](auto Tag, int linkId, int pivotROF, int base, int& offset) -> int { + auto forTracklets = [&](int linkId, int pivotROF, auto&& emit) { const auto& link = topology.getLink(linkId); if (!mTimeFrame->getROFMaskView().isROFEnabled(link.fromLayer, pivotROF)) { - return 0; + return; } gsl::span primaryVertices = mTrkParams[iteration].UseDiamond ? diamondSpan : mTimeFrame->getPrimaryVertices(link.fromLayer, pivotROF); if (primaryVertices.empty()) { - return 0; + return; } const int startVtx = iVertex >= 0 ? iVertex : 0; const int endVtx = iVertex >= 0 ? o2::gpu::CAMath::Min(iVertex + 1, int(primaryVertices.size())) : int(primaryVertices.size()); if (endVtx <= startVtx || (iVertex + 1) > primaryVertices.size()) { - return 0; + return; } const auto& rofOverlap = mTimeFrame->getROFOverlapTableView().getOverlap(link.fromLayer, link.toLayer, pivotROF); if (!rofOverlap.getEntries()) { - return 0; + return; } - int localCount = 0; - auto& tracklets = mTimeFrame->getTracklets()[linkId]; auto layer0 = mTimeFrame->getClustersOnLayer(pivotROF, link.fromLayer); if (layer0.empty()) { - return 0; + return; } const float meanDeltaR = mTrkParams[iteration].LayerRadii[link.toLayer] - mTrkParams[iteration].LayerRadii[link.fromLayer]; @@ -160,64 +154,58 @@ void TrackerTraits::computeLayerTracklets(const int iteration, int iVer math_utils::isPhiDifferenceBelow(currentCluster.phi, nextCluster.phi, phiCut)) { const float phi{o2::gpu::CAMath::ATan2(currentCluster.yCoordinate - nextCluster.yCoordinate, currentCluster.xCoordinate - nextCluster.xCoordinate)}; const float tanL = (currentCluster.zCoordinate - nextCluster.zCoordinate) / (currentCluster.radius - nextCluster.radius); - if constexpr (decltype(Tag)::value == PassMode::OnePass::value) { - tracklets.emplace_back(currentSortedIndex, mTimeFrame->getSortedIndex(targetROF, link.toLayer, iNext), tanL, phi, ts); - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassCount::value) { - ++localCount; - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassInsert::value) { - const int idx = base + offset++; - tracklets[idx] = Tracklet(currentSortedIndex, mTimeFrame->getSortedIndex(targetROF, link.toLayer, iNext), tanL, phi, ts); - } + emit(currentSortedIndex, mTimeFrame->getSortedIndex(targetROF, link.toLayer, iNext), tanL, phi, ts); } } } } } } - return localCount; }; - int dummy{0}; if (mTaskArena->max_concurrency() <= 1) { for (int linkId{0}; linkId < topology.nLinks; ++linkId) { const int fromLayer = topology.getLink(linkId).fromLayer; - const int startROF = 0, endROF = mTimeFrame->getROFOverlapTableView().getLayer(fromLayer).mNROFsTF; - for (int pivotROF{startROF}; pivotROF < endROF; ++pivotROF) { - forTracklets(PassMode::OnePass{}, linkId, pivotROF, 0, dummy); + const int endROF = mTimeFrame->getROFOverlapTableView().getLayer(fromLayer).mNROFsTF; + auto& tracklets = mTimeFrame->getTracklets()[linkId]; + for (int pivotROF{0}; pivotROF < endROF; ++pivotROF) { + forTracklets(linkId, pivotROF, [&tracklets](auto&&... args) { tracklets.emplace_back(std::forward(args)...); }); } } } else { + const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); + const int nConcurrentSinks = std::min(static_cast(topology.nLinks), maxConcurrency); tbb::parallel_for(0, static_cast(topology.nLinks), [&](const int linkId) { const int fromLayer = topology.getLink(linkId).fromLayer; const int startROF = 0, endROF = mTimeFrame->getROFOverlapTableView().getLayer(fromLayer).mNROFsTF; - bounded_vector perROFCount((endROF - startROF) + 1, mMemoryPool.get()); - tbb::parallel_for(startROF, endROF, [&](const int pivotROF) { - perROFCount[pivotROF - startROF] = forTracklets(PassMode::TwoPassCount{}, linkId, pivotROF, 0, dummy); - }); - std::exclusive_scan(perROFCount.begin(), perROFCount.end(), perROFCount.begin(), 0); - const int nTracklets = perROFCount.back(); - mTimeFrame->getTracklets()[linkId].resize(nTracklets); - if (nTracklets == 0) { - return; - } + auto& tracklets = mTimeFrame->getTracklets()[linkId]; + const auto key = CapacityEstimator::makeKey(SlabSite::Tracklets, iteration, iVertex + 1, linkId); + const auto scale = static_cast(mTimeFrame->getClusters()[fromLayer].size()); + const size_t capacity = mTimeFrame->getCapacityEstimator().capacity(key, scale); + + UnorderedSlabSink sink{{.capacity = capacity, .nThreads = maxConcurrency, .nConcurrentSinks = nConcurrentSinks}, mMemoryPool.get()}; tbb::parallel_for(startROF, endROF, [&](const int pivotROF) { - int baseIdx = perROFCount[pivotROF - startROF]; - if (baseIdx == perROFCount[pivotROF + 1 - startROF]) { - return; - } - int localIdx = 0; - forTracklets(PassMode::TwoPassInsert{}, linkId, pivotROF, baseIdx, localIdx); + auto& handle = sink.local(); + forTracklets(linkId, pivotROF, [&handle](auto&&... args) { handle.emplace(std::forward(args)...); }); }); + const auto st = sink.stats(); + sink.finalizeUnordered(tracklets); + mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); }); } tbb::parallel_for(0, static_cast(topology.nLinks), [&](const int linkId) { /// Sort tracklets & remove duplicates - // duplicates can exist simply since we evaluate per vertex auto& trkl{mTimeFrame->getTracklets()[linkId]}; - std::sort(trkl.begin(), trkl.end()); - trkl.erase(std::unique(trkl.begin(), trkl.end()), trkl.end()); - trkl.shrink_to_fit(); + if (mTaskArena->max_concurrency() > 1) { + tbb::parallel_sort(trkl.begin(), trkl.end()); + } else { + std::sort(trkl.begin(), trkl.end()); + } + if (iVertex < 0) { // duplicates can exist simply since we evaluate for all vertices if we do perVertex duplicates cannot exist + trkl.erase(std::unique(trkl.begin(), trkl.end()), trkl.end()); + trkl.shrink_to_fit(); + } auto& lut{mTimeFrame->getTrackletsLookupTable()[linkId]}; if (!trkl.empty()) { for (const auto& tkl : trkl) { @@ -257,16 +245,26 @@ template void TrackerTraits::computeLayerCells(const int iteration) { const auto topology = mTimeFrame->getTrackingTopologyView(); - for (int cellTopologyId = 0; cellTopologyId < topology.nCells; ++cellTopologyId) { - deepVectorClear(mTimeFrame->getCells()[cellTopologyId]); - deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); - if (mTimeFrame->hasMCinformation() && mTrkParams[iteration].CreateArtefactLabels) { - deepVectorClear(mTimeFrame->getCellsLabel(cellTopologyId)); - } - } + const bool createLabels = mTimeFrame->hasMCinformation() && mTrkParams[iteration].CreateArtefactLabels; mTaskArena->execute([&] { - auto forTrackletCells = [&](auto Tag, int cellTopologyId, bounded_vector& layerCells, int iTracklet, int offset = 0) -> int { + const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); + auto clearTopology = [&](const int cellTopologyId) { + deepVectorClear(mTimeFrame->getCells()[cellTopologyId]); + deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); + if (createLabels) { + deepVectorClear(mTimeFrame->getCellsLabel(cellTopologyId)); + } + }; + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(topology.nCells), clearTopology); + } else { + for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { + clearTopology(cellTopologyId); + } + } + + auto forTrackletCells = [&](int cellTopologyId, int iTracklet, auto&& emit) { const auto& cellTopology = topology.getCell(cellTopologyId); const auto& firstLink = topology.getLink(cellTopology.firstLink); const auto& secondLink = topology.getLink(cellTopology.secondLink); @@ -274,7 +272,6 @@ void TrackerTraits::computeLayerCells(const int iteration) const int nextLayerClusterIndex{currentTracklet.secondClusterIndex}; const int nextLayerFirstTrackletIndex{mTimeFrame->getTrackletsLookupTable()[cellTopology.secondLink][nextLayerClusterIndex]}; const int nextLayerLastTrackletIndex{mTimeFrame->getTrackletsLookupTable()[cellTopology.secondLink][nextLayerClusterIndex + 1]}; - int foundCells{0}; for (int iNextTracklet{nextLayerFirstTrackletIndex}; iNextTracklet < nextLayerLastTrackletIndex; ++iNextTracklet) { const Tracklet& nextTracklet{mTimeFrame->getTracklets()[cellTopology.secondLink][iNextTracklet]}; if (nextTracklet.firstClusterIndex != nextLayerClusterIndex) { @@ -293,10 +290,10 @@ void TrackerTraits::computeLayerCells(const int iteration) mTimeFrame->getClusters()[firstLink.toLayer][nextTracklet.firstClusterIndex].clusterId, mTimeFrame->getClusters()[secondLink.toLayer][nextTracklet.secondClusterIndex].clusterId}; const int hitLayers[3]{firstLink.fromLayer, firstLink.toLayer, secondLink.toLayer}; - const auto& cluster1_glo = mTimeFrame->getUnsortedClusters()[firstLink.fromLayer][clusId[0]]; - const auto& cluster2_glo = mTimeFrame->getUnsortedClusters()[firstLink.toLayer][clusId[1]]; - const auto& cluster3_tf = mTimeFrame->getTrackingFrameInfoOnLayer(secondLink.toLayer)[clusId[2]]; - auto track{o2::its::track::buildTrackSeed(cluster1_glo, cluster2_glo, cluster3_tf, mBz)}; + const auto& cluster1Glo = mTimeFrame->getUnsortedClusters()[firstLink.fromLayer][clusId[0]]; + const auto& cluster2Glo = mTimeFrame->getUnsortedClusters()[firstLink.toLayer][clusId[1]]; + const auto& cluster3Tf = mTimeFrame->getTrackingFrameInfoOnLayer(secondLink.toLayer)[clusId[2]]; + auto track{o2::its::track::buildTrackSeed(cluster1Glo, cluster2Glo, cluster3Tf, mBz)}; float chi2{0.f}; bool good{false}; @@ -331,67 +328,56 @@ void TrackerTraits::computeLayerCells(const int iteration) if (good) { TimeEstBC ts = currentTracklet.getTimeStamp(); ts += nextTracklet.getTimeStamp(); - if constexpr (decltype(Tag)::value == PassMode::OnePass::value) { - layerCells.emplace_back(cellTopology.hitLayerMask, clusId[0], clusId[1], clusId[2], iTracklet, iNextTracklet, track, chi2, ts); - ++foundCells; - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassCount::value) { - ++foundCells; - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassInsert::value) { - layerCells[offset++] = CellSeed(cellTopology.hitLayerMask, clusId[0], clusId[1], clusId[2], iTracklet, iNextTracklet, track, chi2, ts); - ++foundCells; - } else { - static_assert(false, "Unknown mode!"); - } + emit(cellTopology.hitLayerMask, clusId[0], clusId[1], clusId[2], iTracklet, iNextTracklet, track, chi2, ts); } } } - return foundCells; }; + bounded_vector activeTopologies(mMemoryPool.get()); + activeTopologies.reserve(topology.nCells); for (int cellTopologyId = 0; cellTopologyId < topology.nCells; ++cellTopologyId) { const auto& cellTopology = topology.getCell(cellTopologyId); - if (mTimeFrame->getTracklets()[cellTopology.firstLink].empty() || - mTimeFrame->getTracklets()[cellTopology.secondLink].empty()) { - continue; + if (!mTimeFrame->getTracklets()[cellTopology.firstLink].empty() && + !mTimeFrame->getTracklets()[cellTopology.secondLink].empty()) { + activeTopologies.push_back(cellTopologyId); } + } + + const int nConcurrentSinks = std::min(maxConcurrency, static_cast(activeTopologies.size())); + auto processTopology = [&](const int cellTopologyId) { + const auto& cellTopology = topology.getCell(cellTopologyId); auto& layerCells = mTimeFrame->getCells()[cellTopologyId]; + auto& lut = mTimeFrame->getCellsLookupTable()[cellTopologyId]; const int currentLayerTrackletsNum{static_cast(mTimeFrame->getTracklets()[cellTopology.firstLink].size())}; - bounded_vector perTrackletCount(currentLayerTrackletsNum + 1, 0, mMemoryPool.get()); - if (mTaskArena->max_concurrency() <= 1) { - for (int iTracklet{0}; iTracklet < currentLayerTrackletsNum; ++iTracklet) { - perTrackletCount[iTracklet] = forTrackletCells(PassMode::OnePass{}, cellTopologyId, layerCells, iTracklet); - } - std::exclusive_scan(perTrackletCount.begin(), perTrackletCount.end(), perTrackletCount.begin(), 0); - } else { - tbb::parallel_for(0, currentLayerTrackletsNum, [&](const int iTracklet) { - perTrackletCount[iTracklet] = forTrackletCells(PassMode::TwoPassCount{}, cellTopologyId, layerCells, iTracklet); - }); - std::exclusive_scan(perTrackletCount.begin(), perTrackletCount.end(), perTrackletCount.begin(), 0); - auto totalCells{perTrackletCount.back()}; - if (totalCells == 0) { - auto& lut = mTimeFrame->getCellsLookupTable()[cellTopologyId]; - lut.resize(currentLayerTrackletsNum + 1); - std::fill(lut.begin(), lut.end(), 0); - continue; - } - layerCells.resize(totalCells); + const auto key = CapacityEstimator::makeKey(SlabSite::Cells, iteration, 0, cellTopologyId); + const auto scale = static_cast(currentLayerTrackletsNum); + if (maxConcurrency > 1) { + const size_t capacity = mTimeFrame->getCapacityEstimator().capacity(key, scale); + GroupedSlabSink sink{{.capacity = capacity, .nThreads = maxConcurrency, .nConcurrentSinks = nConcurrentSinks}, mMemoryPool.get()}; tbb::parallel_for(0, currentLayerTrackletsNum, [&](const int iTracklet) { - int offset = perTrackletCount[iTracklet]; - if (offset == perTrackletCount[iTracklet + 1]) { - return; - } - forTrackletCells(PassMode::TwoPassInsert{}, cellTopologyId, layerCells, iTracklet, offset); + auto& handle = sink.local(); + handle.beginProducer(iTracklet); + forTrackletCells(cellTopologyId, iTracklet, [&handle](auto&&... args) { handle.emplace(std::forward(args)...); }); }); + const auto st = sink.stats(); + sink.finalizeGrouped(size_t(currentLayerTrackletsNum), lut, layerCells); + mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + } else { + lut.resize(currentLayerTrackletsNum + 1); + for (int iTracklet{0}; iTracklet < currentLayerTrackletsNum; ++iTracklet) { + lut[iTracklet] = static_cast(layerCells.size()); + forTrackletCells(cellTopologyId, iTracklet, [&](auto&&... args) { + layerCells.emplace_back(std::forward(args)...); + }); + } + lut.back() = static_cast(layerCells.size()); } - auto& lut = mTimeFrame->getCellsLookupTable()[cellTopologyId]; - lut.resize(currentLayerTrackletsNum + 1); - std::copy_n(perTrackletCount.begin(), currentLayerTrackletsNum + 1, lut.begin()); - - if (mTimeFrame->hasMCinformation() && mTrkParams[iteration].CreateArtefactLabels) { + if (createLabels) { auto& labels = mTimeFrame->getCellsLabel(cellTopologyId); labels.reserve(layerCells.size()); for (const auto& cell : layerCells) { @@ -400,13 +386,30 @@ void TrackerTraits::computeLayerCells(const int iteration) labels.emplace_back(currentLab == nextLab ? currentLab : MCCompLabel()); } } + }; + + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(activeTopologies.size()), [&](const int i) { + processTopology(activeTopologies[i]); + }); + } else { + for (const int cellTopologyId : activeTopologies) { + processTopology(cellTopologyId); + } } - }); - for (int linkId = 0; linkId < topology.nLinks; ++linkId) { - deepVectorClear(mTimeFrame->getTracklets()[linkId]); - deepVectorClear(mTimeFrame->getTrackletsLabel(linkId)); - } + auto clearTracklets = [&](const int linkId) { + deepVectorClear(mTimeFrame->getTracklets()[linkId]); + deepVectorClear(mTimeFrame->getTrackletsLabel(linkId)); + }; + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(topology.nLinks), clearTracklets); + } else { + for (int linkId{0}; linkId < topology.nLinks; ++linkId) { + clearTracklets(linkId); + } + } + }); } template @@ -414,16 +417,29 @@ void TrackerTraits::findCellsNeighbours(const int iteration) { const auto topology = mTimeFrame->getTrackingTopologyView(); mTaskArena->execute([&] { - std::vector> cellsNeighboursByTarget; - cellsNeighboursByTarget.reserve(topology.nCells); - for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { + const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); + auto clearNeighbours = [&](const int cellTopologyId) { deepVectorClear(mTimeFrame->getCellsNeighbours()[cellTopologyId]); deepVectorClear(mTimeFrame->getCellsNeighboursTopology()[cellTopologyId]); deepVectorClear(mTimeFrame->getCellsNeighboursLUT()[cellTopologyId]); - cellsNeighboursByTarget.emplace_back(mMemoryPool.get()); + }; + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(topology.nCells), clearNeighbours); + } else { + for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { + clearNeighbours(cellTopologyId); + } } + auto neighbourLess = [](const CellNeighbour& a, const CellNeighbour& b) { + return std::tie(a.nextCellTopology, a.nextCell, a.cellTopology, a.cell) < + std::tie(b.nextCellTopology, b.nextCell, b.cellTopology, b.cell); + }; + for (int outerLayer{0}; outerLayer < NLayers; ++outerLayer) { + bounded_vector activeTopologies(mMemoryPool.get()); + activeTopologies.reserve(topology.nCells); + size_t sourceCellCount{0}; for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { const auto& cellTopology = topology.getCell(cellTopologyId); if (cellTopology.hitLayerMask.last() != outerLayer || @@ -434,129 +450,197 @@ void TrackerTraits::findCellsNeighbours(const int iteration) if (!successors.getEntries()) { continue; } + activeTopologies.push_back(cellTopologyId); + sourceCellCount += mTimeFrame->getCells()[cellTopologyId].size(); + } - tbb::enumerable_thread_specific> sourceNeighbours([&]() { return bounded_vector{mMemoryPool.get()}; }); - tbb::parallel_for(0, static_cast(mTimeFrame->getCells()[cellTopologyId].size()), [&](const int iCell) { - auto& localNeighbours = sourceNeighbours.local(); - const auto& currentCellSeed{mTimeFrame->getCells()[cellTopologyId][iCell]}; - const int nextLayerTrackletIndex{currentCellSeed.getSecondTrackletIndex()}; - for (int iSuccessor{0}; iSuccessor < successors.getEntries(); ++iSuccessor) { - const int nextCellTopologyId = topology.cellsByFirstLink[successors.getFirstEntry() + iSuccessor]; - if (mTimeFrame->getCells()[nextCellTopologyId].empty() || - mTimeFrame->getCellsLookupTable()[nextCellTopologyId].empty()) { - continue; + if (activeTopologies.empty()) { + continue; + } + + auto forSourceCell = [&](const int cellTopologyId, const int iCell, auto&& emit) { + const auto& cellTopology = topology.getCell(cellTopologyId); + const auto successors = topology.getCellsStartingWithLink(cellTopology.secondLink); + const auto& currentCellSeed{mTimeFrame->getCells()[cellTopologyId][iCell]}; + const int nextLayerTrackletIndex{currentCellSeed.getSecondTrackletIndex()}; + for (int iSuccessor{0}; iSuccessor < successors.getEntries(); ++iSuccessor) { + const int nextCellTopologyId = topology.cellsByFirstLink[successors.getFirstEntry() + iSuccessor]; + if (mTimeFrame->getCells()[nextCellTopologyId].empty() || + mTimeFrame->getCellsLookupTable()[nextCellTopologyId].empty()) { + continue; + } + const auto& nextCellLUT = mTimeFrame->getCellsLookupTable()[nextCellTopologyId]; + if (nextLayerTrackletIndex + 1 >= static_cast(nextCellLUT.size())) { + continue; + } + const int nextLayerFirstCellIndex{nextCellLUT[nextLayerTrackletIndex]}; + const int nextLayerLastCellIndex{nextCellLUT[nextLayerTrackletIndex + 1]}; + for (int iNextCell{nextLayerFirstCellIndex}; iNextCell < nextLayerLastCellIndex; ++iNextCell) { + const auto& nextCellSeedRef{mTimeFrame->getCells()[nextCellTopologyId][iNextCell]}; + if (nextCellSeedRef.getFirstTrackletIndex() != nextLayerTrackletIndex || !currentCellSeed.getTimeStamp().isCompatible(nextCellSeedRef.getTimeStamp())) { + break; } - const auto& nextCellLUT = mTimeFrame->getCellsLookupTable()[nextCellTopologyId]; - if (nextLayerTrackletIndex + 1 >= static_cast(nextCellLUT.size())) { + + auto nextCellSeed{mTimeFrame->getCells()[nextCellTopologyId][iNextCell]}; /// copy + if (!nextCellSeed.rotate(currentCellSeed.getAlpha()) || + !nextCellSeed.propagateTo(currentCellSeed.getX(), getBz())) { continue; } - const int nextLayerFirstCellIndex{nextCellLUT[nextLayerTrackletIndex]}; - const int nextLayerLastCellIndex{nextCellLUT[nextLayerTrackletIndex + 1]}; - for (int iNextCell{nextLayerFirstCellIndex}; iNextCell < nextLayerLastCellIndex; ++iNextCell) { - const auto& nextCellSeedRef{mTimeFrame->getCells()[nextCellTopologyId][iNextCell]}; - if (nextCellSeedRef.getFirstTrackletIndex() != nextLayerTrackletIndex || !currentCellSeed.getTimeStamp().isCompatible(nextCellSeedRef.getTimeStamp())) { - break; - } - auto nextCellSeed{mTimeFrame->getCells()[nextCellTopologyId][iNextCell]}; /// copy - if (!nextCellSeed.rotate(currentCellSeed.getAlpha()) || - !nextCellSeed.propagateTo(currentCellSeed.getX(), getBz())) { - continue; - } - - float chi2 = currentCellSeed.getPredictedChi2(nextCellSeed); - if (chi2 > mTrkParams[iteration].MaxChi2ClusterAttachment) { - continue; - } - - const int nextLevel = currentCellSeed.getLevel() + 1; - localNeighbours.emplace_back(cellTopologyId, iCell, nextCellTopologyId, iNextCell, nextLevel); + float chi2 = currentCellSeed.getPredictedChi2(nextCellSeed); + if (chi2 > mTrkParams[iteration].MaxChi2ClusterAttachment) { + continue; } - } - }); - bounded_vector count(topology.nCells, 0, mMemoryPool.get()); - for (const auto& localNeighbours : sourceNeighbours) { - for (const auto& neigh : localNeighbours) { - ++count[neigh.nextCellTopology]; + const int nextLevel = currentCellSeed.getLevel() + 1; + emit(cellTopologyId, iCell, nextCellTopologyId, iNextCell, nextLevel); } } - for (size_t i{0}; i < topology.nCells; ++i) { - cellsNeighboursByTarget[i].reserve(count[i]); - } - for (const auto& localNeighbours : sourceNeighbours) { - for (const auto& neigh : localNeighbours) { - cellsNeighboursByTarget[neigh.nextCellTopology].emplace_back(neigh); - if (neigh.level > mTimeFrame->getCells()[neigh.nextCellTopology][neigh.nextCell].getLevel()) { - mTimeFrame->getCells()[neigh.nextCellTopology][neigh.nextCell].setLevel(neigh.level); - } + }; + + bounded_vector waveNeighbours{mMemoryPool.get()}; + const auto key = CapacityEstimator::makeKey(SlabSite::Neighbours, iteration, 0, outerLayer); + const auto scale = static_cast(sourceCellCount); + if (maxConcurrency > 1) { + const size_t capacity = mTimeFrame->getCapacityEstimator().capacity(key, scale); + UnorderedSlabSink sink{{.capacity = capacity, .nThreads = maxConcurrency}, mMemoryPool.get()}; + tbb::parallel_for(0, static_cast(activeTopologies.size()), [&](const int i) { + const int cellTopologyId = activeTopologies[i]; + tbb::parallel_for(0, static_cast(mTimeFrame->getCells()[cellTopologyId].size()), [&](const int iCell) { + auto& handle = sink.local(); + forSourceCell(cellTopologyId, iCell, [&handle](auto&&... args) { + handle.emplace(std::forward(args)...); + }); + }); + }); + const auto st = sink.stats(); + sink.finalizeUnordered(waveNeighbours); + mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + tbb::parallel_sort(waveNeighbours.begin(), waveNeighbours.end(), neighbourLess); + } else { + for (const int cellTopologyId : activeTopologies) { + for (int iCell{0}; iCell < static_cast(mTimeFrame->getCells()[cellTopologyId].size()); ++iCell) { + forSourceCell(cellTopologyId, iCell, [&](auto&&... args) { + waveNeighbours.emplace_back(std::forward(args)...); + }); } } + std::sort(waveNeighbours.begin(), waveNeighbours.end(), neighbourLess); } - } - for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { - auto& cellsNeighbours = cellsNeighboursByTarget[cellTopologyId]; - if (cellsNeighbours.empty()) { - continue; + struct TargetSpan { + int topologyId; + size_t begin; + size_t end; + }; + bounded_vector targetSpans{mMemoryPool.get()}; + targetSpans.reserve(topology.nCells); + for (int targetTopologyId{0}; targetTopologyId < topology.nCells; ++targetTopologyId) { + const auto first = std::lower_bound(waveNeighbours.begin(), waveNeighbours.end(), targetTopologyId, + [](const CellNeighbour& neighbour, int id) { return neighbour.nextCellTopology < id; }); + const auto last = std::upper_bound(first, waveNeighbours.end(), targetTopologyId, + [](int id, const CellNeighbour& neighbour) { return id < neighbour.nextCellTopology; }); + if (first != last) { + targetSpans.push_back({targetTopologyId, static_cast(first - waveNeighbours.begin()), static_cast(last - waveNeighbours.begin())}); + } } - std::sort(cellsNeighbours.begin(), cellsNeighbours.end(), [](const auto& a, const auto& b) { - return a.nextCell < b.nextCell; - }); - - auto& cellsNeighbourLUT = mTimeFrame->getCellsNeighboursLUT()[cellTopologyId]; - cellsNeighbourLUT.assign(mTimeFrame->getCells()[cellTopologyId].size(), 0); - for (const auto& neigh : cellsNeighbours) { - ++cellsNeighbourLUT[neigh.nextCell]; + auto finalizeTarget = [&](const int i) { + const auto [targetTopologyId, begin, end] = targetSpans[i]; + auto& cellsNeighbourLUT = mTimeFrame->getCellsNeighboursLUT()[targetTopologyId]; + cellsNeighbourLUT.assign(mTimeFrame->getCells()[targetTopologyId].size(), 0); + for (size_t j{begin}; j < end; ++j) { + const auto& neighbour = waveNeighbours[j]; + ++cellsNeighbourLUT[neighbour.nextCell]; + auto& targetCell = mTimeFrame->getCells()[targetTopologyId][neighbour.nextCell]; + if (neighbour.level > targetCell.getLevel()) { + targetCell.setLevel(neighbour.level); + } + } + std::inclusive_scan(cellsNeighbourLUT.begin(), cellsNeighbourLUT.end(), cellsNeighbourLUT.begin()); + + auto& cellsNeighbours = mTimeFrame->getCellsNeighbours()[targetTopologyId]; + auto& cellsNeighboursTopology = mTimeFrame->getCellsNeighboursTopology()[targetTopologyId]; + cellsNeighbours.resize(end - begin); + cellsNeighboursTopology.resize(end - begin); + for (size_t j{begin}; j < end; ++j) { + cellsNeighbours[j - begin] = waveNeighbours[j].cell; + cellsNeighboursTopology[j - begin] = waveNeighbours[j].cellTopology; + } + }; + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(targetSpans.size()), finalizeTarget); + } else { + for (int i{0}; i < static_cast(targetSpans.size()); ++i) { + finalizeTarget(i); + } } - std::inclusive_scan(cellsNeighbourLUT.begin(), cellsNeighbourLUT.end(), cellsNeighbourLUT.begin()); - - mTimeFrame->getCellsNeighbours()[cellTopologyId].reserve(cellsNeighbours.size()); - mTimeFrame->getCellsNeighboursTopology()[cellTopologyId].reserve(cellsNeighbours.size()); - std::ranges::transform(cellsNeighbours, std::back_inserter(mTimeFrame->getCellsNeighbours()[cellTopologyId]), [](const auto& neigh) { return neigh.cell; }); - std::ranges::transform(cellsNeighbours, std::back_inserter(mTimeFrame->getCellsNeighboursTopology()[cellTopologyId]), [](const auto& neigh) { return neigh.cellTopology; }); } // clean up LUTs - for (auto& cellLUT : mTimeFrame->getCellsLookupTable()) { - deepVectorClear(cellLUT); + auto clearCellLUT = [&](const int cellTopologyId) { + deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); + }; + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(topology.nCells), clearCellLUT); + } else { + for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { + clearCellLUT(cellTopologyId); + } } }); } template template -void TrackerTraits::processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, const bounded_vector& currentCellSeed, const bounded_vector& currentCellId, const bounded_vector& currentCellTopologyId, bounded_vector& updatedCellSeeds, bounded_vector& updatedCellsIds, bounded_vector& updatedCellsTopologyIds) +void TrackerTraits::processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const bounded_vector& currentSeeds, bounded_vector& updatedSeeds) { + constexpr bool IsInitial = std::is_same_v; + static_assert(IsInitial || std::is_same_v); auto propagator = o2::base::Propagator::Instance(); mTaskArena->execute([&] { - auto forCellNeighbours = [&](auto Tag, int iCell, int offset = 0) -> int { - const auto& currentCell{currentCellSeed[iCell]}; - const int cellTopologyId = currentCellTopologyId.empty() ? defaultCellTopologyId : currentCellTopologyId[iCell]; - - if constexpr (decltype(Tag)::value != PassMode::TwoPassInsert::value) { - if (currentCell.getLevel() != iLevel) { - return 0; + auto forCellNeighbours = [&](int iCell, auto&& emit) { + const auto& inputSeed = currentSeeds[iCell]; + const auto& currentCell = [&]() -> const auto& { + if constexpr (IsInitial) { + return inputSeed; + } else { + return inputSeed.seed; } - if (currentCellId.empty()) { - for (int layer = 0; layer < NLayers; ++layer) { - const int clusterIndex = currentCell.getCluster(layer); - if (clusterIndex != constants::UnusedIndex && mTimeFrame->isClusterUsed(layer, clusterIndex)) { - return 0; /// this we do only on the first iteration, hence the check on currentCellId - } + }(); + const int cellTopologyId = [&]() { + if constexpr (IsInitial) { + return defaultCellTopologyId; + } else { + return inputSeed.cellTopologyId; + } + }(); + const int cellId = [&]() { + if constexpr (IsInitial) { + return iCell; + } else { + return inputSeed.cellId; + } + }(); + + if (currentCell.getLevel() != iLevel) { + return; + } + if constexpr (IsInitial) { + for (int layer = 0; layer < NLayers; ++layer) { + const int clusterIndex = currentCell.getCluster(layer); + if (clusterIndex != constants::UnusedIndex && mTimeFrame->isClusterUsed(layer, clusterIndex)) { + return; } } } - const int cellId = currentCellId.empty() ? iCell : currentCellId[iCell]; if (cellTopologyId < 0 || mTimeFrame->getCellsNeighboursLUT()[cellTopologyId].empty()) { - return 0; + return; } const int startNeighbourId{cellId ? mTimeFrame->getCellsNeighboursLUT()[cellTopologyId][cellId - 1] : 0}; const int endNeighbourId{mTimeFrame->getCellsNeighboursLUT()[cellTopologyId][cellId]}; - int foundSeeds{0}; for (int iNeighbourCell{startNeighbourId}; iNeighbourCell < endNeighbourId; ++iNeighbourCell) { const int neighbourCellTopologyId = mTimeFrame->getCellsNeighboursTopology()[cellTopologyId][iNeighbourCell]; const int neighbourCellId = mTimeFrame->getCellsNeighbours()[cellTopologyId][iNeighbourCell]; @@ -605,60 +689,34 @@ void TrackerTraits::processNeighbours(int iteration, int defaultCellTop continue; } - if constexpr (decltype(Tag)::value != PassMode::TwoPassCount::value) { - seed.getClusters()[neighbourLayer] = neighbourCluster; - auto mask = seed.getHitLayerMask(); - mask.set(neighbourLayer); - seed.setHitLayerMask(mask); - seed.setLevel(neighbourCell.getLevel()); - seed.setFirstTrackletIndex(neighbourCell.getFirstTrackletIndex()); - seed.setSecondTrackletIndex(neighbourCell.getSecondTrackletIndex()); - } - - if constexpr (decltype(Tag)::value == PassMode::OnePass::value) { - updatedCellSeeds.push_back(seed); - updatedCellsIds.push_back(neighbourCellId); - updatedCellsTopologyIds.push_back(neighbourCellTopologyId); - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassCount::value) { - ++foundSeeds; - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassInsert::value) { - updatedCellSeeds[offset] = seed; - updatedCellsIds[offset] = neighbourCellId; - updatedCellsTopologyIds[offset++] = neighbourCellTopologyId; - } else { - static_assert(false, "Unknown mode!"); - } + seed.getClusters()[neighbourLayer] = neighbourCluster; + auto mask = seed.getHitLayerMask(); + mask.set(neighbourLayer); + seed.setHitLayerMask(mask); + seed.setLevel(neighbourCell.getLevel()); + seed.setFirstTrackletIndex(neighbourCell.getFirstTrackletIndex()); + seed.setSecondTrackletIndex(neighbourCell.getSecondTrackletIndex()); + emit(std::move(seed), neighbourCellId, neighbourCellTopologyId); } - return foundSeeds; }; - const int nCells = static_cast(currentCellSeed.size()); + const int nCells = static_cast(currentSeeds.size()); if (mTaskArena->max_concurrency() <= 1) { for (int iCell{0}; iCell < nCells; ++iCell) { - forCellNeighbours(PassMode::OnePass{}, iCell); + forCellNeighbours(iCell, [&](auto&&... args) { updatedSeeds.emplace_back(std::forward(args)...); }); } } else { - bounded_vector perCellCount(nCells + 1, 0, mMemoryPool.get()); - tbb::parallel_for(0, nCells, [&](const int iCell) { - perCellCount[iCell] = forCellNeighbours(PassMode::TwoPassCount{}, iCell); - }); - - std::exclusive_scan(perCellCount.begin(), perCellCount.end(), perCellCount.begin(), 0); - auto totalNeighbours{perCellCount.back()}; - if (totalNeighbours == 0) { - return; - } - updatedCellSeeds.resize(totalNeighbours); - updatedCellsIds.resize(totalNeighbours); - updatedCellsTopologyIds.resize(totalNeighbours); + const auto scale = static_cast(nCells); + const size_t capacity = mTimeFrame->getCapacityEstimator().capacity(capacityKey, scale); + UnorderedSlabSink sink{{.capacity = capacity, .nThreads = mTaskArena->max_concurrency()}, mMemoryPool.get()}; tbb::parallel_for(0, nCells, [&](const int iCell) { - int offset = perCellCount[iCell]; - if (offset == perCellCount[iCell + 1]) { - return; - } - forCellNeighbours(PassMode::TwoPassInsert{}, iCell, offset); + auto& handle = sink.local(); + forCellNeighbours(iCell, [&](auto&&... args) { handle.emplace(std::forward(args)...); }); }); + const auto st = sink.stats(); + sink.finalizeUnordered(updatedSeeds); + mTimeFrame->getCapacityEstimator().update(capacityKey, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); } }); } @@ -669,7 +727,9 @@ bool TrackerTraits::finaliseTrackSeed(const TrackSeedN& seed, const int iteration, const TrackingFrameInfo* const* tfInfos, const Cluster* const* unsortedClusters, - const o2::base::Propagator* propagator) + const o2::base::Propagator* propagator, + const TrackFollowContext& followCtx, + TrackFollowerScratch& scratch) { const auto& trkParams = mTrkParams[iteration]; const track::TrackFitContext fitCtx{ @@ -703,32 +763,12 @@ bool TrackerTraits::finaliseTrackSeed(const TrackSeedN& seed, return passesFinalLengthCut(track); } - const int maxHypotheses = std::max(1, trkParams.TrackFollowerMaxHypotheses); - TrackFollowerScratch scratch{mMemoryPool.get()}; - if (static_cast(scratch.activeHypotheses.size()) < maxHypotheses) { - scratch.activeHypotheses.resize(maxHypotheses); - } - if (static_cast(scratch.nextHypotheses.size()) < maxHypotheses) { - scratch.nextHypotheses.resize(maxHypotheses); + if (static_cast(scratch.activeHypotheses.size()) < followCtx.maxHypotheses) { + scratch.activeHypotheses.resize(followCtx.maxHypotheses); } - - const Cluster* clustersPtrs[NLayers]{}; - const unsigned char* usedClustersPtrs[NLayers]{}; - const int* clustersIndexTablesPtrs[NLayers]{}; - const int* rofClustersPtrs[NLayers]{}; - for (int iLayer{0}; iLayer < NLayers; ++iLayer) { - clustersPtrs[iLayer] = mTimeFrame->getClusters()[iLayer].data(); - usedClustersPtrs[iLayer] = mTimeFrame->getUsedClusters(iLayer).data(); - clustersIndexTablesPtrs[iLayer] = mTimeFrame->getIndexTable(0, iLayer).data(); - rofClustersPtrs[iLayer] = mTimeFrame->getROFrameClusters(iLayer).data(); + if (static_cast(scratch.nextHypotheses.size()) < followCtx.maxHypotheses) { + scratch.nextHypotheses.resize(followCtx.maxHypotheses); } - const TrackFollowContext followCtx{ - &mTimeFrame->getIndexTableUtils(), - mTimeFrame->getROFMaskView(), - mTimeFrame->getROFOverlapTableView(), - clustersPtrs, usedClustersPtrs, clustersIndexTablesPtrs, rofClustersPtrs, - trkParams.LayerRadii.data(), trkParams.PhiBins, maxHypotheses, - trkParams.TrackFollowerNSigmaCutPhi, trkParams.TrackFollowerNSigmaCutZ}; const auto backup = internalTrack; auto best = internalTrack; @@ -768,6 +808,8 @@ void TrackerTraits::findRoads(const int iteration) unsortedClusters[iLayer] = mTimeFrame->getUnsortedClusters()[iLayer].data(); } const auto topology = mTimeFrame->getTrackingTopologyView(); + tbb::enumerable_thread_specific followerScratch{ + [mr = mMemoryPool.get()]() { return TrackFollowerScratch{mr}; }}; for (int startLevel{mTrkParams[iteration].CellsPerRoad()}; startLevel >= mTrkParams[iteration].CellMinimumLevel(); --startLevel) { const track::TrackSeedSelector seedFilter{constants::MaxTrackSeedQ2Pt, mTrkParams[iteration].MaxChi2NDF, startLevel, mTrkParams[iteration].MaxHoles, mTrkParams[iteration].getMinSeedingClusters(), mTrkParams[iteration].HoleLayerMask, mTrkParams[iteration].getNonSeedingLayerMask()}; @@ -775,33 +817,36 @@ void TrackerTraits::findRoads(const int iteration) bounded_vector trackSeeds(mMemoryPool.get()); for (int startCellTopologyId{0}; startCellTopologyId < topology.nCells; ++startCellTopologyId) { const int startLayer = topology.getCell(startCellTopologyId).hitLayerMask.last(); - if (!(mTrkParams[iteration].StartLayerMask.has(startLayer)) || mTimeFrame->getCells()[startCellTopologyId].empty()) { + if (!(mTrkParams[iteration].StartLayerMask.has(startLayer)) || + mTimeFrame->getCells()[startCellTopologyId].empty() || + topology.getMaxCellLevel(startCellTopologyId) < startLevel) { continue; } - bounded_vector lastCellId(mMemoryPool.get()), updatedCellId(mMemoryPool.get()); - bounded_vector lastCellTopologyId(mMemoryPool.get()), updatedCellTopologyId(mMemoryPool.get()); - bounded_vector lastCellSeed(mMemoryPool.get()), updatedCellSeed(mMemoryPool.get()); + bounded_vector lastSeeds(mMemoryPool.get()), updatedSeeds(mMemoryPool.get()); + + auto roadKey = [&](int level) { + return CapacityEstimator::makeKey(SlabSite::Roads, iteration, CapacityEstimator::makeVariant(startLevel, level), startCellTopologyId); + }; - processNeighbours(iteration, startCellTopologyId, startLevel, mTimeFrame->getCells()[startCellTopologyId], lastCellId, lastCellTopologyId, updatedCellSeed, updatedCellId, updatedCellTopologyId); + processNeighbours(iteration, startCellTopologyId, startLevel, roadKey(startLevel), mTimeFrame->getCells()[startCellTopologyId], updatedSeeds); int level = startLevel; - while (level > 2 && !updatedCellSeed.empty()) { - lastCellSeed.swap(updatedCellSeed); - lastCellId.swap(updatedCellId); - lastCellTopologyId.swap(updatedCellTopologyId); - deepVectorClear(updatedCellSeed); /// tame the memory peaks - deepVectorClear(updatedCellId); /// tame the memory peaks - deepVectorClear(updatedCellTopologyId); - processNeighbours(iteration, constants::UnusedIndex, --level, lastCellSeed, lastCellId, lastCellTopologyId, updatedCellSeed, updatedCellId, updatedCellTopologyId); + while (level > 2 && !updatedSeeds.empty()) { + lastSeeds.swap(updatedSeeds); + deepVectorClear(updatedSeeds); + --level; + processNeighbours(iteration, constants::UnusedIndex, level, roadKey(level), lastSeeds, updatedSeeds); } - deepVectorClear(lastCellId); /// tame the memory peaks - deepVectorClear(lastCellTopologyId); /// tame the memory peaks - deepVectorClear(lastCellSeed); /// tame the memory peaks + deepVectorClear(lastSeeds); - if (!updatedCellSeed.empty()) { - trackSeeds.reserve(trackSeeds.size() + std::count_if(updatedCellSeed.begin(), updatedCellSeed.end(), seedFilter)); - std::copy_if(updatedCellSeed.begin(), updatedCellSeed.end(), std::back_inserter(trackSeeds), seedFilter); + if (!updatedSeeds.empty()) { + trackSeeds.reserve(trackSeeds.size() + std::count_if(updatedSeeds.begin(), updatedSeeds.end(), [&](const auto& road) { return seedFilter(road.seed); })); + for (auto& road : updatedSeeds) { + if (seedFilter(road.seed)) { + trackSeeds.emplace_back(std::move(road.seed)); + } + } } } @@ -809,6 +854,25 @@ void TrackerTraits::findRoads(const int iteration) continue; } + const Cluster* clustersPtrs[NLayers]{}; + const unsigned char* usedClustersPtrs[NLayers]{}; + const int* clustersIndexTablesPtrs[NLayers]{}; + const int* rofClustersPtrs[NLayers]{}; + for (int iLayer{0}; iLayer < NLayers; ++iLayer) { + clustersPtrs[iLayer] = mTimeFrame->getClusters()[iLayer].data(); + usedClustersPtrs[iLayer] = mTimeFrame->getUsedClusters(iLayer).data(); + clustersIndexTablesPtrs[iLayer] = mTimeFrame->getIndexTable(0, iLayer).data(); + rofClustersPtrs[iLayer] = mTimeFrame->getROFrameClusters(iLayer).data(); + } + const TrackFollowContext followCtx{ + &mTimeFrame->getIndexTableUtils(), + mTimeFrame->getROFMaskView(), + mTimeFrame->getROFOverlapTableView(), + clustersPtrs, usedClustersPtrs, clustersIndexTablesPtrs, rofClustersPtrs, + mTrkParams[iteration].LayerRadii.data(), mTrkParams[iteration].PhiBins, + std::max(1, mTrkParams[iteration].TrackFollowerMaxHypotheses), + mTrkParams[iteration].TrackFollowerNSigmaCutPhi, mTrkParams[iteration].TrackFollowerNSigmaCutZ}; + bounded_vector tracks(mMemoryPool.get()); mTaskArena->execute([&] { const int nSeeds = static_cast(trackSeeds.size()); @@ -830,10 +894,11 @@ void TrackerTraits::findRoads(const int iteration) tbb::parallel_for(tbb::blocked_range(0, nSeeds, chunkSize), [&](const auto& range) { bounded_vector localTracks(mMemoryPool.get()); localTracks.reserve(std::min(chunkSize, static_cast(range.size()))); + auto& scratch = followerScratch.local(); for (int iSeed{range.begin()}; iSeed < range.end(); ++iSeed) { - TrackITSExt temporaryTrack; - if (finaliseTrackSeed(trackSeeds[iSeed], temporaryTrack, iteration, tfInfos, unsortedClusters, propagator)) { - localTracks.push_back(temporaryTrack); + localTracks.emplace_back(); + if (!finaliseTrackSeed(trackSeeds[iSeed], localTracks.back(), iteration, tfInfos, unsortedClusters, propagator, followCtx, scratch)) { + localTracks.pop_back(); } if (static_cast(localTracks.size()) == chunkSize) { flushTracks(localTracks); @@ -1010,16 +1075,16 @@ void TrackerTraits::setNThreads(int n, std::shared_ptr } template class TrackerTraits<7>; -template void TrackerTraits<7>::processNeighbours(int, int, int, const bounded_vector&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); -template void TrackerTraits<7>::processNeighbours>(int, int, int, const bounded_vector>&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); +template void TrackerTraits<7>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); +template void TrackerTraits<7>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); // ALICE3 upgrade #ifdef ENABLE_UPGRADES template class TrackerTraits<11>; -template void TrackerTraits<11>::processNeighbours(int, int, int, const bounded_vector&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); -template void TrackerTraits<11>::processNeighbours>(int, int, int, const bounded_vector>&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); +template void TrackerTraits<11>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); +template void TrackerTraits<11>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); template class TrackerTraits<13>; -template void TrackerTraits<13>::processNeighbours(int, int, int, const bounded_vector&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); -template void TrackerTraits<13>::processNeighbours>(int, int, int, const bounded_vector>&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); +template void TrackerTraits<13>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); +template void TrackerTraits<13>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); #endif } // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx index 3f98e146996cf..83a1086ec5263 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx @@ -472,6 +472,7 @@ void ITSTrackingInterface::printSummary() const { mVertexer->printSummary(); mTracker->printSummary(); + mTimeFrame->getCapacityEstimator().print(); } void ITSTrackingInterface::setTraitsFromProvider(VertexerTraitsN* vertexerTraits, diff --git a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx index ba37275f87688..d25d5efbec262 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx @@ -158,7 +158,8 @@ void Vertexer::addTimingStatCurStep(int iteration, double timeMs) template void Vertexer::printSummary() const { - LOGP(info, "Vertexer summary: Processed {} TFs", mTimeFrameCounter); + auto avgTF = mTotalTime * 1.e-3 / ((mTimeFrameCounter > 0) ? (double)mTimeFrameCounter : -1.0); + LOGP(info, "Vertexer summary: Processed {} TFs in TOT={:.2f} s, AVG/TF={:.2f} s", mTimeFrameCounter, mTotalTime * 1.e-3, avgTF); for (size_t iteration = 0; iteration < mTimingStats.size(); ++iteration) { for (size_t state = 0; state < NSteps; ++state) { const auto& stats = mTimingStats[iteration][state]; diff --git a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt index f8fce10b78602..c7c4d6dc101a2 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt @@ -15,6 +15,12 @@ o2_add_test(boundedmemoryresource LABELS "its;tracking" PUBLIC_LINK_LIBRARIES O2::ITStracking) +o2_add_test(slabbumpallocator + SOURCES testSlabBumpAllocator.cxx + COMPONENT_NAME its-tracking + LABELS "its;tracking" + PUBLIC_LINK_LIBRARIES O2::ITStracking TBB::tbb) + o2_add_test(roflookuptables SOURCES testROFLookupTables.cxx COMPONENT_NAME its-tracking diff --git a/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx b/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx new file mode 100644 index 0000000000000..9cae6fd11a132 --- /dev/null +++ b/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx @@ -0,0 +1,526 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test SlabBumpAllocator +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ITStracking/BoundedAllocator.h" +#include "ITStracking/CapacityEstimator.h" +#include "ITStracking/SlabBumpAllocator.h" + +using namespace o2::its; + +namespace +{ + +struct Rec { + int a{-1}; + int b{-1}; + float payload{0.f}; + Rec() = default; + Rec(int aa, int bb, float p) : a{aa}, b{bb}, payload{p} {} + bool operator<(const Rec& o) const + { + if ((a < 0) != (o.a < 0)) { + return o.a < 0; + } + return a != o.a ? a < o.a : b < o.b; + } + bool operator==(const Rec& o) const { return a == o.a && b == o.b; } +}; + +std::ostream& operator<<(std::ostream& os, const Rec& r) +{ + return os << "Rec{" << r.a << ',' << r.b << ',' << r.payload << '}'; +} + +class StingyResource final : public std::pmr::memory_resource +{ + public: + explicit StingyResource(size_t maxBytes) : mMax{maxBytes} {} + + private: + void* do_allocate(size_t bytes, size_t alignment) final + { + if (bytes > mMax) { + throw std::bad_alloc{}; + } + return std::pmr::new_delete_resource()->allocate(bytes, alignment); + } + void do_deallocate(void* p, size_t bytes, size_t alignment) final + { + std::pmr::new_delete_resource()->deallocate(p, bytes, alignment); + } + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final { return this == &other; } + + size_t mMax; +}; + +template +void runConcurrently(F&& f) +{ + tbb::task_arena arena{4}; + arena.execute(std::forward(f)); +} + +template +void produce(int i, uint32_t seed, Emit&& emit) +{ + std::mt19937 rng(seed + (uint32_t(i) * 2654435761u)); + const int n = int(rng() % 12); + for (int k = 0; k < n; ++k) { + emit(i, k, float((i * 100) + k)); + } +} + +std::vector> reference(int nProducers, uint32_t seed) +{ + std::vector> out(nProducers); + for (int i = 0; i < nProducers; ++i) { + produce(i, seed, [&](int a, int b, float p) { out[i].emplace_back(a, b, p); }); + } + return out; +} + +void checkGrouped(int nProducers, size_t capacity, size_t slab, size_t maxMemory = std::numeric_limits::max()) +{ + constexpr uint32_t seed = 7u; + BoundedMemoryResource mr{maxMemory}; + + const auto ref = reference(nProducers, seed); + std::vector flat; + std::vector refLut(nProducers + 1, 0); + for (int i = 0; i < nProducers; ++i) { + refLut[i + 1] = refLut[i] + int(ref[i].size()); + flat.insert(flat.end(), ref[i].begin(), ref[i].end()); + } + + GroupedSlabSink sink{{.capacity = capacity, .nThreads = 4, .slabOverride = slab}, &mr}; + runConcurrently([&] { + tbb::parallel_for(0, nProducers, [&](int i) { + auto& h = sink.local(); + h.beginProducer(i); + produce(i, seed, [&](int a, int b, float p) { h.emplace(a, b, p); }); + }); + }); + + const auto st = sink.stats(); + BOOST_TEST(st.emitted == flat.size()); + + bounded_vector lut{&mr}; + bounded_vector dest{&mr}; + sink.finalizeGrouped(size_t(nProducers), lut, dest); + + BOOST_REQUIRE(lut.size() == size_t(nProducers) + 1); + BOOST_TEST(std::equal(lut.begin(), lut.end(), refLut.begin())); + BOOST_REQUIRE(dest.size() == flat.size()); + for (size_t i = 0; i < flat.size(); ++i) { + BOOST_TEST(dest[i] == flat[i]); + BOOST_TEST(dest[i].payload == flat[i].payload); + } +} + +void checkUnordered(int nProducers, size_t capacity, size_t slab, size_t maxMemory = std::numeric_limits::max()) +{ + constexpr uint32_t seed = 11u; + BoundedMemoryResource mr{maxMemory}; + + const auto ref = reference(nProducers, seed); + std::vector flat; + for (const auto& v : ref) { + flat.insert(flat.end(), v.begin(), v.end()); + } + std::sort(flat.begin(), flat.end()); + flat.erase(std::unique(flat.begin(), flat.end()), flat.end()); + + UnorderedSlabSink sink{{.capacity = capacity, .nThreads = 4, .slabOverride = slab}, &mr}; + runConcurrently([&] { + tbb::parallel_for(0, nProducers, [&](int i) { + auto& h = sink.local(); + produce(i, seed, [&](int a, int b, float p) { h.emplace(a, b, p); }); + }); + }); + + const auto st = sink.stats(); + BOOST_TEST(st.emitted == flat.size()); + + bounded_vector dest{&mr}; + sink.finalizeUnordered(dest); + + std::sort(dest.begin(), dest.end()); + + BOOST_REQUIRE(dest.size() == flat.size()); + for (size_t i = 0; i < flat.size(); ++i) { + BOOST_TEST(dest[i] == flat[i]); + BOOST_TEST(dest[i].payload == flat[i].payload); + } +} + +} // namespace + +BOOST_AUTO_TEST_CASE(slab_hands_out_disjoint_ranges) +{ + SlabBumpAllocator alloc{1000, 256}; + std::vector seen(1000, 0); + size_t got{0}; + while (true) { + const auto r = alloc.grab(); + if (!r.valid()) { + break; + } + BOOST_REQUIRE(r.base + r.n <= 1000); + for (size_t s = r.base; s < r.base + r.n; ++s) { + BOOST_REQUIRE(seen[s] == 0); + seen[s] = 1; + } + got += r.n; + } + BOOST_TEST(got == 1000u); + BOOST_TEST(alloc.watermark() <= 1000u); +} + +BOOST_AUTO_TEST_CASE(slab_never_exceeds_a_threads_fair_share) +{ + BOOST_TEST(SlabBumpAllocator::suggestSlab(64, 8) <= 8u); + BOOST_TEST(SlabBumpAllocator::suggestSlab(0, 8) >= 1u); + BOOST_TEST(SlabBumpAllocator::suggestSlab(1u << 20, 8) == 4096u); +} + +BOOST_AUTO_TEST_CASE(grouped_reproduces_two_pass_layout) +{ + checkGrouped(2000, 40000, 512); + checkGrouped(300, 20000, 4096); +} + +BOOST_AUTO_TEST_CASE(grouped_survives_capacity_underestimate) +{ + checkGrouped(2000, 3000, 256); + checkGrouped(500, 0, 1, 1u << 20); +} + +BOOST_AUTO_TEST_CASE(grouped_survives_capacity_overestimate) +{ + checkGrouped(20, 1u << 20, 256, 1u << 16); +} + +BOOST_AUTO_TEST_CASE(grouped_keeps_order_across_slab_and_spill_boundaries) +{ + BoundedMemoryResource mr; + const std::vector counts{3, 5, 6, 0, 2}; + GroupedSlabSink sink{{.capacity = 10, .nThreads = 1, .slabOverride = 4}, &mr}; + + auto& h = sink.local(); + for (size_t p = 0; p < counts.size(); ++p) { + h.beginProducer(int(p)); + for (int k = 0; k < counts[p]; ++k) { + h.emplace(int(p), k, float(k)); + } + } + const auto st = sink.stats(); + BOOST_TEST(st.emitted == 16u); + BOOST_TEST(st.spilled == 6u); // capacity 10 of 16 + BOOST_TEST(st.overflowed); + + bounded_vector lut{&mr}; + bounded_vector dest{&mr}; + sink.finalizeGrouped(counts.size(), lut, dest); + + BOOST_REQUIRE(lut.size() == counts.size() + 1); + BOOST_REQUIRE(dest.size() == 16u); + int expected{0}; + for (size_t p = 0; p < counts.size(); ++p) { + BOOST_TEST(lut[p] == expected); + for (int k = 0; k < counts[p]; ++k) { + BOOST_TEST(dest[expected + k] == Rec(int(p), k, 0.f)); + } + expected += counts[p]; + } + BOOST_TEST(lut.back() == expected); +} + +BOOST_AUTO_TEST_CASE(unordered_reproduces_emitted_records) +{ + checkUnordered(2000, 40000, 512); + checkUnordered(300, 20000, 4096); +} + +BOOST_AUTO_TEST_CASE(unordered_survives_capacity_underestimate) +{ + checkUnordered(2000, 3000, 256); + checkUnordered(500, 0, 1, 1u << 20); +} + +BOOST_AUTO_TEST_CASE(unordered_keeps_records_across_slab_and_spill_boundaries) +{ + BoundedMemoryResource mr; + UnorderedSlabSink sink{{.capacity = 10, .nThreads = 1, .slabOverride = 4}, &mr}; + + auto& h = sink.local(); + for (int i = 0; i < 14; ++i) { + h.emplace(i, i + 1, float(i)); + } + const auto st = sink.stats(); + BOOST_TEST(st.emitted == 14u); + BOOST_TEST(st.spilled == 4u); + + bounded_vector dest{&mr}; + sink.finalizeUnordered(dest); + + BOOST_REQUIRE(dest.size() == 14u); + for (int i = 0; i < 14; ++i) { + BOOST_TEST(dest[i] == Rec(i, i + 1, float(i))); + } +} + +BOOST_AUTO_TEST_CASE(unordered_removes_unused_slots) +{ + BoundedMemoryResource mr; + UnorderedSlabSink sink{{.capacity = 10, .nThreads = 1, .slabOverride = 4}, &mr}; + sink.local().emplace(1, 2, 3.f); + sink.local().emplace(); + + bounded_vector dest{&mr}; + sink.finalizeUnordered(dest); + + BOOST_REQUIRE(dest.size() == 2u); + BOOST_TEST(dest.front() == Rec(1, 2, 3.f)); + BOOST_TEST(dest.front().payload == 3.f); + BOOST_TEST(dest.back() == Rec{}); +} + +BOOST_AUTO_TEST_CASE(unordered_does_not_hand_back_an_oversized_buffer) +{ + BoundedMemoryResource mr; + UnorderedSlabSink sink{{.capacity = 100000, .nThreads = 1, .slabOverride = 256}, &mr}; + + auto& h = sink.local(); + for (int i = 0; i < 100; ++i) { + h.emplace(i, i + 1, float(i)); + } + bounded_vector dest{&mr}; + sink.finalizeUnordered(dest); + + BOOST_REQUIRE(dest.size() == 100u); + BOOST_TEST(dest.capacity() < 1000u); +} + +BOOST_AUTO_TEST_CASE(capacity_is_clamped_to_what_the_pool_can_spare) +{ + constexpr size_t maxMemory = 1u << 16; + BoundedMemoryResource mr{maxMemory}; + UnorderedSlabSink sink{{.capacity = 1u << 20, .nThreads = 4}, &mr}; + + const auto st = sink.stats(); + BOOST_TEST(st.requested == size_t{1u << 20}); + BOOST_TEST(st.capacity > 0u); + BOOST_TEST(st.capacity < st.requested); + BOOST_TEST(st.memoryLimited); + BOOST_TEST(st.capacity * sizeof(Rec) <= maxMemory / 2); +} + +BOOST_AUTO_TEST_CASE(capacity_is_split_between_concurrent_sinks) +{ + size_t alone{0}, shared{0}; + { + BoundedMemoryResource mr{1u << 16}; + UnorderedSlabSink sink{{.capacity = 1u << 20, .nThreads = 4, .nConcurrentSinks = 1}, &mr}; + alone = sink.stats().capacity; + } + { + BoundedMemoryResource mr{1u << 16}; + UnorderedSlabSink sink{{.capacity = 1u << 20, .nThreads = 4, .nConcurrentSinks = 4}, &mr}; + shared = sink.stats().capacity; + } + BOOST_TEST(shared > 0u); + BOOST_TEST(shared < alone); + BOOST_TEST(shared * 4 <= alone + 8); // integer division slack +} + +BOOST_AUTO_TEST_CASE(unordered_survives_a_failed_preallocation) +{ + StingyResource mr{1u << 12}; + UnorderedSlabSink sink{{.capacity = 1u << 20, .nThreads = 1}, &mr}; + + const auto st = sink.stats(); + BOOST_TEST(st.capacity == 0u); + BOOST_TEST(st.memoryLimited); + + auto& handle = sink.local(); + for (int i = 0; i < 10; ++i) { + handle.emplace(i, i + 1, float(i)); + } + + bounded_vector dest{&mr}; + sink.finalizeUnordered(dest); + BOOST_REQUIRE(dest.size() == 10u); + for (int i = 0; i < 10; ++i) { + BOOST_TEST(dest[i] == Rec(i, i + 1, float(i))); + } +} + +BOOST_AUTO_TEST_CASE(estimator_cold_start_has_capacity) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Cells, 0, 0, 3); + BOOST_TEST(est.capacity(key, 1000.) == 1024u); + + est.update(key, 0., 0, 0, false, false); + BOOST_TEST(est.capacity(key, 0.) == 0u); + BOOST_TEST(est.capacity(key, 1000.) == 1024u); + + est.update(key, 1000., 0, 1024, false, false); + BOOST_TEST(est.capacity(key, 1000.) == 1024u); +} + +BOOST_AUTO_TEST_CASE(estimator_converges_and_reacts_to_overflow) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Tracklets, 1, 0, 0); + constexpr double scale = 1000.; + constexpr double rate = 5.; + + for (int tf = 0; tf < 12; ++tf) { + const size_t cap = est.capacity(key, scale); + const auto emitted = size_t(scale * rate); + est.update(key, scale, emitted, cap != 0 ? cap : emitted, cap != 0 && emitted > cap, false); + } + + const size_t cap = est.capacity(key, scale); + BOOST_TEST(cap >= size_t(scale * rate)); + BOOST_TEST(cap <= size_t(scale * rate * 1.35)); + + const size_t bigger = est.capacity(key, 2. * scale); + BOOST_TEST(bigger > size_t(2. * scale * rate)); + BOOST_TEST(bigger <= size_t(2. * scale * rate * 1.35)); + + est.update(key, scale, size_t(scale * rate * 4.), size_t(scale * rate), true, false); + BOOST_TEST(est.capacity(key, scale) > cap); +} + +BOOST_AUTO_TEST_CASE(estimator_backs_off_when_the_pool_refuses) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Roads, 2, 0, 0); + constexpr double scale = 1000.; + constexpr double rate = 5.; + const auto emitted = size_t(scale * rate); + + for (int tf = 0; tf < 12; ++tf) { + const size_t cap = est.capacity(key, scale); + est.update(key, scale, emitted, cap, emitted > cap, false); + } + const size_t settled = est.capacity(key, scale); + + for (int tf = 0; tf < 12; ++tf) { + est.update(key, scale, emitted, 100, true, true); + } + BOOST_TEST(est.capacity(key, scale) < settled); +} + +BOOST_AUTO_TEST_CASE(estimator_grows_in_proportion_to_the_miss) +{ + CapacityEstimator est; + constexpr double scale = 1000.; + const auto nearMiss = CapacityEstimator::makeKey(SlabSite::Cells, 3, 0, 0); + const auto wayOff = CapacityEstimator::makeKey(SlabSite::Cells, 3, 0, 1); + + for (const auto key : {nearMiss, wayOff}) { + est.update(key, scale, 2000, 2000, false, false); + } + const size_t settled = est.capacity(nearMiss, scale); + + est.update(nearMiss, scale, 2000, 1900, true, false); // overran by 5% + est.update(wayOff, scale, 2000, 500, true, false); // overran by 4x + + const size_t afterNearMiss = est.capacity(nearMiss, scale); + const size_t afterWayOff = est.capacity(wayOff, scale); + BOOST_TEST(afterNearMiss > settled); + BOOST_TEST(afterNearMiss < afterWayOff); + BOOST_TEST(afterNearMiss < size_t(1.25 * double(settled))); + BOOST_TEST(afterWayOff > size_t(1.4 * double(settled))); +} + +BOOST_AUTO_TEST_CASE(estimator_recovers_from_a_single_overflow) +{ + CapacityEstimator::Config cfg; + cfg.decayAfter = 1; + CapacityEstimator est{cfg}; + const auto key = CapacityEstimator::makeKey(SlabSite::Tracklets, 4, 0, 0); + constexpr double scale = 1000.; + + est.update(key, scale, 2000, 2000, false, false); + est.update(key, scale, 2000, 500, true, false); + const size_t inflated = est.capacity(key, scale); + + for (int tf = 0; tf < 30; ++tf) { + est.update(key, scale, 2000, 20000, false, false); // 10% utilisation + } + const size_t recovered = est.capacity(key, scale); + BOOST_TEST(recovered < inflated); + BOOST_TEST(recovered <= size_t(2. * scale * double(cfg.marginMin)) + 2); +} + +BOOST_AUTO_TEST_CASE(estimator_decay_survives_interleaved_busy_timeframes) +{ + CapacityEstimator::Config cfg; + cfg.decayAfter = 4; + CapacityEstimator est{cfg}; + const auto key = CapacityEstimator::makeKey(SlabSite::Roads, 5, 0, 0); + constexpr double scale = 1000.; + + est.update(key, scale, 2000, 2000, false, false); + est.update(key, scale, 2000, 500, true, false); + const size_t inflated = est.capacity(key, scale); + + for (int tf = 0; tf < 80; ++tf) { + const bool quiet = (tf % 4) != 3; + est.update(key, scale, 2000, quiet ? 20000 : 2000, false, false); + } + BOOST_TEST(est.capacity(key, scale) < inflated); +} + +BOOST_AUTO_TEST_CASE(estimator_reset_forgets_inflated_margins) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Cells, 0, 0, 0); + constexpr double scale = 1000.; + + for (int tf = 0; tf < 6; ++tf) { + est.update(key, scale, size_t(scale * 5.), 10, true, false); + } + BOOST_TEST(est.capacity(key, scale) > 5000u); + + est.reset(); + BOOST_TEST(est.capacity(key, scale) == 1024u); +} + +BOOST_AUTO_TEST_CASE(estimator_keys_separate_the_road_walk_steps) +{ + const auto a = CapacityEstimator::makeKey(SlabSite::Roads, 0, CapacityEstimator::makeVariant(6, 4), 1); + const auto b = CapacityEstimator::makeKey(SlabSite::Roads, 0, CapacityEstimator::makeVariant(5, 4), 1); + const auto c = CapacityEstimator::makeKey(SlabSite::Roads, 0, CapacityEstimator::makeVariant(6, 4), 2); + BOOST_TEST(a != b); + BOOST_TEST(a != c); + BOOST_TEST(b != c); +} diff --git a/Detectors/ITSMFT/ITS/tracking/test/testTrackingTopology.cxx b/Detectors/ITSMFT/ITS/tracking/test/testTrackingTopology.cxx index d3f249650a287..6c76bcd193ec8 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/testTrackingTopology.cxx +++ b/Detectors/ITSMFT/ITS/tracking/test/testTrackingTopology.cxx @@ -72,6 +72,65 @@ BOOST_AUTO_TEST_CASE(trackingtopology_basic) } } +/// Without holes the cell graph is a single chain, so cell i - spanning layers i, i+1, i+2 - +/// can only ever be reached by the i cells below it. +BOOST_AUTO_TEST_CASE(trackingtopology_max_cell_level_is_the_chain_depth) +{ + o2::its::TrackingTopology<7> topo; + topo.init(7, 0, 0); + const auto view = topo.getView(); + view.print(); + + BOOST_REQUIRE_EQUAL(view.nLinks, 6); + BOOST_REQUIRE_EQUAL(view.nCells, 5); + for (int i{0}; i < view.nCells; ++i) { + BOOST_CHECK_EQUAL(int(view.getMaxCellLevel(i)), i + 1); + } +} + +/// With a hole allowed the graph branches, and the depth is the longest path ending on a cell +/// rather than its index. Every cell must still be reachable by at least one chain, and no cell +/// may claim a level deeper than the number of cells that could precede it. +BOOST_AUTO_TEST_CASE(trackingtopology_max_cell_level_follows_the_longest_path) +{ + o2::its::TrackingTopology<5> topo; + topo.init(5, 1, 1 << 2); + const auto view = topo.getView(); + view.print(); + + bool sawBranching = false; + for (int i{0}; i < view.nCells; ++i) { + const auto level = int(view.getMaxCellLevel(i)); + BOOST_CHECK_GE(level, 1); + BOOST_CHECK_LE(level, int(view.nCells)); + // A cell reached by a chain of n predecessors needs n+2 layers below its outer one. + BOOST_CHECK_LE(level, view.getCell(i).hitLayerMask.last() - 1); + sawBranching |= level != i + 1; + } + BOOST_CHECK(sawBranching); // otherwise this is just the chain case again +} + +/// Neighbour construction can finalize a target after one source-layer wave: every predecessor +/// of a target ends on the destination layer of the target's first link. +BOOST_AUTO_TEST_CASE(trackingtopology_predecessors_belong_to_one_layer_wave) +{ + o2::its::TrackingTopology<7> topo; + topo.init(7, 2, (1 << 2) | (1 << 4)); + const auto view = topo.getView(); + + for (int sourceId{0}; sourceId < view.nCells; ++sourceId) { + const auto& source = view.getCell(sourceId); + const int sourceWave = source.hitLayerMask.last(); + const auto successors = view.getCellsStartingWithLink(source.secondLink); + for (int i{0}; i < successors.getEntries(); ++i) { + const int targetId = view.cellsByFirstLink[successors.getFirstEntry() + i]; + const auto& target = view.getCell(targetId); + BOOST_CHECK_EQUAL(target.firstLink, source.secondLink); + BOOST_CHECK_EQUAL(sourceWave, view.getLink(target.firstLink).toLayer); + } + } +} + BOOST_AUTO_TEST_CASE(trackingtopology_single_allowed_hole) { o2::its::TrackingTopology<5> topo;