GeezerVison is showing github.com — Add a multithreaded TPC digitizer device (#16090) · AliceO2Group/AliceO2@4c92364
File treeExpand file treeCollapse file tree Open diff view settings Expand file treeCollapse file tree Open diff view settings Collapse file Copy file name to clipboardExpand all lines: Detectors/TPC/simulation/include/TPCSimulation/DigitContainer.h +2-1Lines changed: 2 additions & 1 deletion | Original file line number | Diff line number | Diff line change |
|---|
@@ -104,7 +104,8 @@ class DigitContainer | 104 | 104 | std::unique_ptr<DigitTime::PrevDigitInfoArray> mPrevDigArr; ///< Keep track of ToT and ion tail cumul from last time bin | 105 | 105 | o2::utils::DebugStreamer mStreamer; ///< Debug streamer | 106 | 106 | const CalPad* mPadParams[3] = {nullptr, nullptr, nullptr}; //!< Ion tail fraction, ion tail slope and common mode k-values per pad | 107 | | - const CalDet<bool>* mDeadMap = nullptr; //!< Dead channel map | | 107 | + const CalDet<bool>* mDeadMap = nullptr; //!< Dead channel map, points to mDeadMapCopy if set | | 108 | + std::unique_ptr<CalDet<bool>> mDeadMapCopy; //!< Own copy of the dead channel map, which the CDBInterface rebuilds on each access | 108 | 109 | bool mCMCEnabled = false; //!< Common mode correction enabled in the FEE config (DigiMode Auto) | 109 | 110 | bool mCalibrationsUpdated = false; //!< Whether updateCalibrations was called | 110 | 111 | SAMPAProcessing* mSAMPAProcessing = nullptr; //!< SAMPAProcessing used for the output, the singleton if not set | |
Collapse file Copy file name to clipboardExpand all lines: Detectors/TPC/simulation/include/TPCSimulation/Digitizer.h +15-1Lines changed: 15 additions & 1 deletion | Original file line number | Diff line number | Diff line change |
|---|
@@ -143,15 +143,28 @@ class Digitizer | 143 | 143 | /// in case of scaled distortions, the distortions can be recalculated to ensure consistent distortions and corrections | 144 | 144 | void recalculateDistortions(); | 145 | 145 |
| | 146 | + /// Use the distortion maps and scaling of another digitizer, which keeps ownership of the maps | | 147 | + void setSCDistortionsFrom(const Digitizer& other); | | 148 | + | 146 | 149 | /// Use digitizer-owned random rings filled from seed and positioned per sector and timeframe in init(), | 147 | | - /// so that the digits of a sector do not depend on which sectors were processed before | | 150 | + /// so that the digits of a sector do not depend on which sectors were processed before. | | 151 | + /// All sectors share the ring contents and differ in the start positions, as with the singleton rings. | 148 | 152 | /// \param seed base seed, typically DigiParams.seed | 149 | 153 | void setUseSectorSeeding(ULong64_t seed); | 150 | 154 |
| 151 | 155 | /// Set the timeframe identifier that enters the sector seeding | 152 | 156 | void setSeedingTimeframe(ULong64_t timeframe) { mSeedingTimeframe = timeframe; } | 153 | 157 |
| | 158 | + /// Use copies of the sector-seeding random rings of another digitizer, which fills them if needed | | 159 | + void setSectorSeedingFrom(Digitizer& other); | | 160 | + | | 161 | + /// Whether flush() also flushes the process-wide debug streamer | | 162 | + void setFlushDebugStreamer(bool flush) { mFlushDebugStreamer = flush; } | | 163 | + | 154 | 164 | private: | | 165 | + /// Fill the random rings of the sector seeding, if not done yet | | 166 | + void initSectorSeeding(); | | 167 | + | 155 | 168 | ElectronTransport& electronTransport(); | 156 | 169 | GEMAmplification& gemAmplification(); | 157 | 170 | SAMPAProcessing& sampaProcessing(); | @@ -162,6 +175,7 @@ class Digitizer | 162 | 175 | ULong64_t mSeed = 0; //!< Base seed for sector seeding | 163 | 176 | ULong64_t mSeedingTimeframe = 0; //!< Timeframe identifier for sector seeding | 164 | 177 | bool mUseSectorSeeding = false; //!< Whether sector seeding is used | | 178 | + bool mFlushDebugStreamer = true; //!< Whether flush() flushes the debug streamer | 165 | 179 |
| 166 | 180 | DigitContainer mDigitContainer; ///< Container for the Digits | 167 | 181 | std::vector<float> mSignalArray; //!< Workspace for the shaped signal of one electron | |
Collapse file Copy file name to clipboardExpand all lines: Detectors/TPC/simulation/src/DigitContainer.cxx +15-10Lines changed: 15 additions & 10 deletions | Original file line number | Diff line number | Diff line change |
|---|
| 14 | 14 | | 15 | 15 |
| 16 | 16 | #include "TPCSimulation/DigitContainer.h" | | 17 | +#include <atomic> | 17 | 18 | #include <memory> | 18 | 19 | #include <type_traits> | 19 | 20 | #include <fairlogger/Logger.h> | @@ -60,15 +61,6 @@ void DigitContainer::fillOutputContainer(std::vector<Digit>& output, | 60 | 61 | mPrevDigArr = std::make_unique<DigitTime::PrevDigitInfoArray>(); | 61 | 62 | } | 62 | 63 |
| 63 | | - static bool reportedSettings = false; | 64 | | - if (!reportedSettings) { | 65 | | - reportSettings(); | 66 | | - if (mDeadMap) { | 67 | | - LOGP(info, "Using dead map with {} masked pads", mDeadMap->getSum<int>()); | 68 | | - } | 69 | | - reportedSettings = true; | 70 | | - } | 71 | | - | 72 | 64 | for (auto& time : mTimeBins) { | 73 | 65 | /// the time bins between the last event and the timing of this event are uncorrelated and can be written out | 74 | 66 | /// OR the readout is triggered (i.e. not continuous) and we can dump everything in any case, as long it is within one drift time interval | @@ -161,12 +153,25 @@ void DigitContainer::updateCalibrations() | 161 | 153 | } | 162 | 154 |
| 163 | 155 | // dead channel map | 164 | | - mDeadMap = eleParam.applyDeadMap ? &cdb.getDeadChannelMap() : nullptr; | | 156 | + mDeadMap = nullptr; | | 157 | + mDeadMapCopy.reset(); | | 158 | + if (eleParam.applyDeadMap) { | | 159 | + mDeadMapCopy = std::make_unique<CalDet<bool>>(cdb.getDeadChannelMap()); | | 160 | + mDeadMap = mDeadMapCopy.get(); | | 161 | + } | 165 | 162 |
| 166 | 163 | // common mode correction in the FEE, needed to resolve DigiMode Auto | 167 | 164 | mCMCEnabled = (eleParam.DigiMode == DigitzationMode::Auto) && cdb.getFEEConfig().isCMCEnabled(); | 168 | 165 |
| 169 | 166 | mCalibrationsUpdated = true; | | 167 | + | | 168 | + static std::atomic<bool> reportedSettings{false}; | | 169 | + if (!reportedSettings.exchange(true)) { | | 170 | + reportSettings(); | | 171 | + if (mDeadMap) { | | 172 | + LOGP(info, "Using dead map with {} masked pads", mDeadMap->getSum<int>()); | | 173 | + } | | 174 | + } | 170 | 175 | } | 171 | 176 |
| 172 | 177 | void DigitContainer::reportSettings() | |
Collapse file Copy file name to clipboardExpand all lines: Detectors/TPC/simulation/src/Digitizer.cxx +50-10Lines changed: 50 additions & 10 deletions | Original file line number | Diff line number | Diff line change |
|---|
| 31 | 31 |
| 32 | 32 | #include <fairlogger/Logger.h> | 33 | 33 | #include <TRandom3.h> | | 34 | +#include <mutex> | 34 | 35 |
| 35 | 36 | ClassImp(o2::tpc::Digitizer); | 36 | 37 |
| @@ -42,6 +43,13 @@ Digitizer::Digitizer() = default; | 42 | 43 |
| 43 | 44 | namespace | 44 | 45 | { | | 46 | +/// serialises the calls that read calibrations through the CDBInterface, which is not thread safe | | 47 | +std::mutex& calibrationMutex() | | 48 | +{ | | 49 | + static std::mutex mutex; | | 50 | + return mutex; | | 51 | +} | | 52 | + | 45 | 53 | // splitmix64 finaliser, to derive well separated seeds from the base seed | 46 | 54 | ULong64_t mixSeed(ULong64_t x) | 47 | 55 | { | @@ -80,17 +88,36 @@ void Digitizer::setUseSectorSeeding(ULong64_t seed) | 80 | 88 | mSeed = seed; | 81 | 89 | } | 82 | 90 |
| | 91 | +void Digitizer::initSectorSeeding() | | 92 | +{ | | 93 | + if (mUseSectorSeeding && !mGEMAmplification) { | | 94 | + // the rings are filled once, in the construction order of the singletons | | 95 | + TRandom3 rng(toTRandom3Seed(mixSeed(mSeed))); | | 96 | + mGEMAmplification = std::make_unique<GEMAmplification>(&rng); | | 97 | + mElectronTransport = std::make_unique<ElectronTransport>(&rng); | | 98 | + mSAMPAProcessing = std::make_unique<SAMPAProcessing>(&rng); | | 99 | + mDigitContainer.setSAMPAProcessing(mSAMPAProcessing.get()); | | 100 | + } | | 101 | +} | | 102 | + | | 103 | +void Digitizer::setSectorSeedingFrom(Digitizer& other) | | 104 | +{ | | 105 | + other.initSectorSeeding(); | | 106 | + mUseSectorSeeding = other.mUseSectorSeeding; | | 107 | + mSeed = other.mSeed; | | 108 | + if (other.mGEMAmplification) { | | 109 | + mGEMAmplification = std::make_unique<GEMAmplification>(*other.mGEMAmplification); | | 110 | + mElectronTransport = std::make_unique<ElectronTransport>(*other.mElectronTransport); | | 111 | + mSAMPAProcessing = std::make_unique<SAMPAProcessing>(*other.mSAMPAProcessing); | | 112 | + mDigitContainer.setSAMPAProcessing(mSAMPAProcessing.get()); | | 113 | + } | | 114 | +} | | 115 | + | 83 | 116 | void Digitizer::init() | 84 | 117 | { | | 118 | + std::lock_guard<std::mutex> lock(calibrationMutex()); | 85 | 119 | if (mUseSectorSeeding) { | 86 | | - if (!mGEMAmplification) { | 87 | | - // the rings are filled once, in the construction order of the singletons | 88 | | - TRandom3 rng(toTRandom3Seed(mixSeed(mSeed))); | 89 | | - mGEMAmplification = std::make_unique<GEMAmplification>(&rng); | 90 | | - mElectronTransport = std::make_unique<ElectronTransport>(&rng); | 91 | | - mSAMPAProcessing = std::make_unique<SAMPAProcessing>(&rng); | 92 | | - mDigitContainer.setSAMPAProcessing(mSAMPAProcessing.get()); | 93 | | - } | | 120 | + initSectorSeeding(); | 94 | 121 | const ULong64_t sectorSeed = mixSeed(mSeed ^ mixSeed(static_cast<ULong64_t>(mSector) + 1)); | 95 | 122 | TRandom3 rng(toTRandom3Seed(mixSeed(sectorSeed ^ mixSeed(mSeedingTimeframe)))); | 96 | 123 | mGEMAmplification->setRandomRingPositions(rng); | @@ -230,7 +257,7 @@ void Digitizer::flush(std::vector<o2::tpc::Digit>& digits, | 230 | 257 | SAMPAProcessing& sampaProcessing = this->sampaProcessing(); | 231 | 258 | mDigitContainer.fillOutputContainer(digits, labels, commonModeOutput, mSector, sampaProcessing.getTimeBinFromTime(mEventTime - mOutputDigitTimeOffset), mIsContinuous, finalFlush); | 232 | 259 | // flushing debug output to file | 233 | | - if (((finalFlush && mIsContinuous) || (!mIsContinuous)) && mSpaceCharge) { | | 260 | + if (mFlushDebugStreamer && ((finalFlush && mIsContinuous) || (!mIsContinuous)) && mSpaceCharge) { | 234 | 261 | o2::utils::DebugStreamer::instance()->flush(); | 235 | 262 | } | 236 | 263 | } | @@ -289,12 +316,25 @@ void Digitizer::setStartTime(double time) | 289 | 316 | // note that negative times w.r.t start of timeframe/data-taking == mOutputDigitTimeOffset | 290 | 317 | // will yield the 0-th bin (due to casting logic in sampaProcessing) | 291 | 318 | SAMPAProcessing& sampaProcessing = this->sampaProcessing(); | 292 | | - sampaProcessing.updateParameters(mVDrift); | | 319 | + { | | 320 | + std::lock_guard<std::mutex> lock(calibrationMutex()); | | 321 | + sampaProcessing.updateParameters(mVDrift); | | 322 | + } | 293 | 323 | const auto timediff = time - mOutputDigitTimeOffset; | 294 | 324 | const auto starttimebin = sampaProcessing.getTimeBinFromTime(timediff); | 295 | 325 | mDigitContainer.setStartTime(starttimebin); | 296 | 326 | } | 297 | 327 |
| | 328 | +void Digitizer::setSCDistortionsFrom(const Digitizer& other) | | 329 | +{ | | 330 | + mSpaceChargeOwned.reset(); | | 331 | + mSpaceCharge = other.mSpaceCharge; | | 332 | + mSpaceChargeDer = other.mSpaceChargeDer; | | 333 | + mUseSCDistortions = other.mUseSCDistortions; | | 334 | + mLumiScaleFactor = other.mLumiScaleFactor; | | 335 | + mUseScaledDistortions = other.mUseScaledDistortions; | | 336 | +} | | 337 | + | 298 | 338 | void Digitizer::setLumiScaleFactor() | 299 | 339 | { | 300 | 340 | mLumiScaleFactor = (CorrMapParam::Instance().lumiInst - mSpaceCharge->getMeanLumi()) / mSpaceChargeDer->getMeanLumi(); | |
Collapse file Copy file name to clipboardExpand all lines: Steer/DigitizerWorkflow/CMakeLists.txt +2Lines changed: 2 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -25,6 +25,8 @@ o2_add_executable(digitizer-workflow | 25 | 25 | src/SimReaderSpec.cxx | 26 | 26 | src/SimpleDigitizerWorkflow.cxx | 27 | 27 | src/TPCDigitizerSpec.cxx | | 28 | + src/TPCSectorDigitization.cxx | | 29 | + src/TPCDigitizerMTSpec.cxx | 28 | 30 | src/ZDCDigitizerSpec.cxx | 29 | 31 | src/TOFDigitizerSpec.cxx | 30 | 32 | $<$<BOOL:${ENABLE_UPGRADES}>:src/ITS3DigitizerSpec.cxx> | |
Collapse file Copy file name to clipboardExpand all lines: Steer/DigitizerWorkflow/src/SimpleDigitizerWorkflow.cxx +11-2Lines changed: 11 additions & 2 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -186,6 +186,7 @@ void customize(std::vector<o2::framework::ConfigParamSpec>& workflowOptions) | 186 | 186 | // Option to write TPC digits internaly, without forwarding to a special writer instance. | 187 | 187 | // This is useful in GRID productions with small available memory. | 188 | 188 | workflowOptions.push_back(ConfigParamSpec{"tpc-chunked-writer", o2::framework::VariantType::Bool, false, {"Write independent TPC digit chunks as soon as they can be flushed."}}); | | 189 | + workflowOptions.push_back(ConfigParamSpec{"tpc-digi-threads", o2::framework::VariantType::Int, 0, {"Digitize TPC in a single device with this many threads instead of lanes (0 = use lanes); requires --tpc-chunked-writer"}}); | 189 | 190 | workflowOptions.push_back(ConfigParamSpec{"tpc-distortion-type", o2::framework::VariantType::Int, 0, {"Simulate distortions in the TPC (0=no distortions, 1=distortions without scaling, 2=distortions with CTP scaling)"}}); | 190 | 191 |
| 191 | 192 | std::string simhelp("Comma separated list of simulation prefixes (for background, signal productions)"); | @@ -624,8 +625,16 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext) | 624 | 625 |
| 625 | 626 | auto internalwrite = configcontext.options().get<bool>("tpc-chunked-writer"); | 626 | 627 | auto distortionType = configcontext.options().get<int>("tpc-distortion-type"); | 627 | | - WorkflowSpec tpcPipelines = o2::tpc::getTPCDigitizerSpec(lanes, tpcsectors, mctruth, internalwrite, distortionType); | 628 | | - specs.insert(specs.end(), tpcPipelines.begin(), tpcPipelines.end()); | | 628 | + auto digiThreads = configcontext.options().get<int>("tpc-digi-threads"); | | 629 | + if (digiThreads > 0) { | | 630 | + if (!internalwrite) { | | 631 | + LOG(fatal) << "--tpc-digi-threads requires --tpc-chunked-writer"; | | 632 | + } | | 633 | + specs.emplace_back(o2::tpc::getTPCDigitizerMTSpec(digiThreads, tpcsectors, distortionType)); | | 634 | + } else { | | 635 | + WorkflowSpec tpcPipelines = o2::tpc::getTPCDigitizerSpec(lanes, tpcsectors, mctruth, internalwrite, distortionType); | | 636 | + specs.insert(specs.end(), tpcPipelines.begin(), tpcPipelines.end()); | | 637 | + } | 629 | 638 |
| 630 | 639 | if (configcontext.options().get<std::string>("tpc-reco-type").empty() == false) { | 631 | 640 | throw std::runtime_error("option 'tpc-reco-type' is deprecated, please connect workflows on the command line by pipe"); | |
|