Skip to content

DPL: add DataTakingSupport plugin #10019

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ktf merged 1 commit into from
Oct 7, 2022
Merged

DPL: add DataTakingSupport plugin #10019

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Framework/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ add_subdirectory(Utils)

add_subdirectory(AnalysisSupport)
add_subdirectory(PhysicsSupport)
add_subdirectory(DataTakingSupport)

add_subdirectory(RootAnalysisHelpers)

Expand Down
8 changes: 6 additions & 2 deletions Framework/Core/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,6 @@ o2_add_library(Framework
test/TestClasses.cxx
PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
PUBLIC_LINK_LIBRARIES AliceO2::Configuration
AliceO2::InfoLogger
AliceO2::Monitoring
CURL::libcurl
FairMQ::FairMQ
Expand Down Expand Up @@ -205,7 +204,6 @@ foreach(t
HistogramRegistry
HTTPParser
IndexBuilder
InfoLogger
InputRecord
InputRecordWalker
InputSpan
Expand Down Expand Up @@ -248,6 +246,12 @@ o2_add_test(Root2ArrowTable NAME test_Framework_test_Root2ArrowTable
LABELS framework
PUBLIC_LINK_LIBRARIES O2::Framework ROOT::ROOTDataFrame)

o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
SOURCES test/test_InfoLogger.cxx
COMPONENT_NAME Framework
LABELS framework
PUBLIC_LINK_LIBRARIES O2::Framework AliceO2::InfoLogger)

o2_add_executable(dpl-null-sink
SOURCES src/o2NullSink.cxx
PUBLIC_LINK_LIBRARIES O2::Framework
Expand Down
2 changes: 0 additions & 2 deletions Framework/Core/include/Framework/CommonServices.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,6 @@ struct CommonServices {
static ServiceSpec driverClientSpec();
static ServiceSpec monitoringSpec();
static ServiceSpec datatakingContextSpec();
static ServiceSpec infologgerContextSpec();
static ServiceSpec infologgerSpec();
static ServiceSpec configurationSpec();
static ServiceSpec controlSpec();
static ServiceSpec rootFileSpec();
Expand Down
163 changes: 5 additions & 158 deletions Framework/Core/src/CommonServices.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
#include <Configuration/ConfigurationInterface.h>
#include <Configuration/ConfigurationFactory.h>
#include <Monitoring/MonitoringFactory.h>
#include <InfoLogger/InfoLogger.hxx>

#include <fairmq/Device.h>
#include <fairmq/shmem/Monitor.h>
Expand All @@ -59,8 +58,6 @@
#include <cstdlib>
#include <cstring>

using AliceO2::InfoLogger::InfoLogger;
using AliceO2::InfoLogger::InfoLoggerContext;
using o2::configuration::ConfigurationFactory;
using o2::configuration::ConfigurationInterface;
using o2::monitoring::Monitoring;
Expand All @@ -76,12 +73,6 @@ using Value = o2::monitoring::tags::Value;
namespace o2::framework
{

/// This is a global service because read only
template <>
struct ServiceKindExtractor<InfoLoggerContext> {
constexpr static ServiceKind kind = ServiceKind::Global;
};

#define MONITORING_QUEUE_SIZE 100
o2::framework::ServiceSpec CommonServices::monitoringSpec()
{
Expand Down Expand Up @@ -200,154 +191,9 @@ o2::framework::ServiceSpec CommonServices::datatakingContextSpec()
.kind = ServiceKind::Serial};
}

o2::framework::ServiceSpec CommonServices::infologgerContextSpec()
{
return ServiceSpec{
.name = "infologger-contex",
.init = simpleServiceInit<InfoLoggerContext, InfoLoggerContext>(),
.configure = noConfiguration(),
.start = [](ServiceRegistry& services, void* service) {
auto& infoLoggerContext = services.get<InfoLoggerContext>();
auto run = services.get<RawDeviceService>().device()->fConfig->GetProperty<std::string>("runNumber", "unspecified");
infoLoggerContext.setField(InfoLoggerContext::FieldName::Run, run);
auto partition = services.get<RawDeviceService>().device()->fConfig->GetProperty<std::string>("environment_id", "unspecified");
infoLoggerContext.setField(InfoLoggerContext::FieldName::Partition, partition);
},
.kind = ServiceKind::Serial};
}

// Creates the sink for FairLogger / InfoLogger integration
auto createInfoLoggerSinkHelper(InfoLogger* logger, InfoLoggerContext* ctx)
{
return [logger,
ctx](const std::string& content, const fair::LogMetaData& metadata) {
// translate FMQ metadata
InfoLogger::InfoLogger::Severity severity = InfoLogger::Severity::Undefined;
int level = InfoLogger::undefinedMessageOption.level;

if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::nolog)) {
// discard
return;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::fatal)) {
severity = InfoLogger::Severity::Fatal;
level = 1;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::error)) {
severity = InfoLogger::Severity::Error;
level = 3;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::alarm)) {
severity = InfoLogger::Severity::Warning;
level = 4;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::important)) {
severity = InfoLogger::Severity::Info;
level = 5;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::warn)) {
severity = InfoLogger::Severity::Warning;
level = 6;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::state)) {
severity = InfoLogger::Severity::Info;
level = 8;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::info)) {
severity = InfoLogger::Severity::Info;
level = 10;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::debug)) {
severity = InfoLogger::Severity::Debug;
level = 11;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::debug1)) {
severity = InfoLogger::Severity::Debug;
level = 12;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::debug2)) {
severity = InfoLogger::Severity::Debug;
level = 13;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::debug3)) {
severity = InfoLogger::Severity::Debug;
level = 14;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::debug4)) {
severity = InfoLogger::Severity::Debug;
level = 15;
} else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::trace)) {
severity = InfoLogger::Severity::Debug;
level = 50;
}

InfoLogger::InfoLoggerMessageOption opt = {
severity,
level,
InfoLogger::undefinedMessageOption.errorCode,
metadata.file.c_str(),
atoi(metadata.line.c_str())};

if (logger) {
logger->log(opt, *ctx, "%s", content.c_str());
}
};
};

struct MissingService {
};

o2::framework::ServiceSpec CommonServices::infologgerSpec()
{
return ServiceSpec{
.name = "infologger",
.init = [](ServiceRegistry& services, DeviceState&, fair::mq::ProgOptions& options) -> ServiceHandle {
auto infoLoggerMode = options.GetPropertyAsString("infologger-mode");
auto infoLoggerSeverity = options.GetPropertyAsString("infologger-severity");
if (infoLoggerSeverity.empty() == false && options.GetPropertyAsString("infologger-mode") == "") {
LOGP(info, "Using O2_INFOLOGGER_MODE=infoLoggerD since infologger-severity is set");
infoLoggerMode = "infoLoggerD";
}
if (infoLoggerMode != "") {
setenv("O2_INFOLOGGER_MODE", infoLoggerMode.c_str(), 1);
}
char const* infoLoggerEnv = getenv("O2_INFOLOGGER_MODE");
if (infoLoggerEnv == nullptr || strcmp(infoLoggerEnv, "none") == 0) {
return ServiceHandle{.hash = TypeIdHelpers::uniqueId<MissingService>(),
.instance = nullptr,
.kind = ServiceKind::Serial,
.name = "infologger"};
}
InfoLogger* infoLoggerService = nullptr;
try {
infoLoggerService = new InfoLogger;
} catch (...) {
LOGP(error, "Unable to initialise InfoLogger with O2_INFOLOGGER_MODE={}.", infoLoggerMode);
return ServiceHandle{.hash = TypeIdHelpers::uniqueId<MissingService>(),
.instance = nullptr,
.kind = ServiceKind::Serial,
.name = "infologger"};
}
auto infoLoggerContext = &services.get<InfoLoggerContext>();
// Only print the first 10 characters and the last 18 if the
// string length is greater than 32 bytes.
auto truncate = [](std::string in) -> std::string {
if (in.size() < 32) {
return in;
}
char name[32];
memcpy(name, in.data(), 10);
name[10] = '.';
name[11] = '.';
name[12] = '.';
memcpy(name + 13, in.data() + in.size() - 18, 18);
name[31] = 0;
return name;
};
infoLoggerContext->setField(InfoLoggerContext::FieldName::Facility, truncate(services.get<DeviceSpec const>().name));
infoLoggerContext->setField(InfoLoggerContext::FieldName::System, std::string("DPL"));
infoLoggerService->setContext(*infoLoggerContext);

if (infoLoggerSeverity != "") {
fair::Logger::AddCustomSink("infologger", infoLoggerSeverity, createInfoLoggerSinkHelper(infoLoggerService, infoLoggerContext));
}
return ServiceHandle{.hash = TypeIdHelpers::uniqueId<InfoLogger>(),
.instance = infoLoggerService,
.kind = ServiceKind::Serial,
.name = "infologger"};
},
.configure = noConfiguration(),
.kind = ServiceKind::Serial};
}

o2::framework::ServiceSpec CommonServices::configurationSpec()
{
return ServiceSpec{
Expand Down Expand Up @@ -917,8 +763,6 @@ std::vector<ServiceSpec> CommonServices::defaultServices(int numThreads)
driverClientSpec(),
datatakingContextSpec(),
monitoringSpec(),
infologgerContextSpec(),
infologgerSpec(),
configurationSpec(),
controlSpec(),
rootFileSpec(),
Expand All @@ -936,16 +780,19 @@ std::vector<ServiceSpec> CommonServices::defaultServices(int numThreads)
decongestionSpec(),
CommonMessageBackends::rawBufferBackendSpec()};

std::string loadableServicesStr = "O2FrameworkDataTakingSupport:InfoLoggerContext,O2FrameworkDataTakingSupport:InfoLogger";
// Load plugins depending on the environment
std::vector<LoadableService> loadableServices = {};
char* loadableServicesEnv = getenv("DPL_LOAD_SERVICES");
// String to define the services to load is:
//
// library1:name1,library2:name2,...
if (loadableServicesEnv) {
loadableServices = ServiceHelpers::parseServiceSpecString(loadableServicesEnv);
ServiceHelpers::loadFromPlugin(loadableServices, specs);
loadableServicesStr += ",";
loadableServicesStr += loadableServicesEnv;
}
loadableServices = ServiceHelpers::parseServiceSpecString(loadableServicesStr.c_str());
ServiceHelpers::loadFromPlugin(loadableServices, specs);
// I should make it optional depending wether the GUI is there or not...
specs.push_back(CommonServices::guiMetricsSpec());
if (numThreads) {
Expand Down
2 changes: 0 additions & 2 deletions Framework/Core/src/runDataProcessing.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@
#include <Configuration/ConfigurationInterface.h>
#include <Configuration/ConfigurationFactory.h>
#include <Monitoring/MonitoringFactory.h>
#include <InfoLogger/InfoLogger.hxx>
#include "ResourcesMonitoringHelper.h"

#include <fairmq/Device.h>
Expand Down Expand Up @@ -146,7 +145,6 @@

using namespace o2::monitoring;
using namespace o2::configuration;
using namespace AliceO2::InfoLogger;

using namespace o2::framework;
namespace bpo = boost::program_options;
Expand Down
1 change: 0 additions & 1 deletion Framework/Core/test/test_SlowConsumer.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
#include "Framework/RawDeviceService.h"
#include "Framework/ControlService.h"
#include <fairmq/Device.h>
#include <InfoLogger/InfoLogger.hxx>

#include <chrono>
#include <thread>
Expand Down
14 changes: 14 additions & 0 deletions Framework/DataTakingSupport/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
# All rights not expressly granted are reserved.
#
# This software is distributed under the terms of the GNU General Public
# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
#
# In applying this license CERN does not waive the privileges and immunities
# granted to it by virtue of its status as an Intergovernmental Organization
# or submit itself to any jurisdiction.
o2_add_library(FrameworkDataTakingSupport
SOURCES src/Plugin.cxx
PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
PUBLIC_LINK_LIBRARIES O2::Framework AliceO2::InfoLogger)
Loading