Skip to content

Commit 97e484f

Browse files
authored
Add mermaid output support (#8525)
* add mermaid output support * Addition of a test and a fix * format
1 parent 33b3108 commit 97e484f

5 files changed

Lines changed: 255 additions & 2 deletions

File tree

Framework/Core/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ o2_add_library(Framework
7676
src/ConfigurationOptionsRetriever.cxx
7777
src/FreePortFinder.cxx
7878
src/GraphvizHelpers.cxx
79+
src/MermaidHelpers.cxx
7980
src/HTTPParser.cxx
8081
src/InputRecord.cxx
8182
src/InputRouteHelpers.cxx
@@ -208,6 +209,7 @@ foreach(t
208209
InputSpec
209210
Kernels
210211
LogParsingHelpers
212+
Mermaid
211213
OptionsHelpers
212214
OverrideLabels
213215
PtrHelpers
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
#include "MermaidHelpers.h"
13+
#include <map>
14+
#include <iostream>
15+
#include <string>
16+
17+
namespace o2
18+
{
19+
namespace framework
20+
{
21+
22+
namespace
23+
{
24+
std::string quote(std::string const& s) { return R"(")" + s + R"(")"; }
25+
} // namespace
26+
27+
/// Helper to dump a set of devices as a mermaid file
28+
void MermaidHelpers::dumpDeviceSpec2Mermaid(std::ostream& out, const std::vector<DeviceSpec>& specs)
29+
{
30+
out << "flowchart TD\n";
31+
std::map<std::string, std::string> outputChannel2Device;
32+
std::map<std::string, unsigned int> outputChannel2Port;
33+
34+
for (auto& spec : specs) {
35+
auto id = spec.id;
36+
out << " " << id << "\n";
37+
for (auto&& output : spec.outputChannels) {
38+
outputChannel2Device.insert(std::make_pair(output.name, id));
39+
outputChannel2Port.insert(std::make_pair(output.name, output.port));
40+
}
41+
}
42+
for (auto& spec : specs) {
43+
for (auto& input : spec.inputChannels) {
44+
auto outputName = input.name;
45+
out << " " << outputChannel2Device[outputName] << "-- " << input.port << ":" << outputName << " -->" << spec.id << "\n";
46+
}
47+
}
48+
}
49+
50+
} // namespace framework
51+
} // namespace o2
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
#ifndef FRAMEWORK_MERMAIDHELPERS_H
12+
#define FRAMEWORK_MERMAIDHELPERS_H
13+
14+
#include "Framework/WorkflowSpec.h"
15+
#include "Framework/DeviceSpec.h"
16+
#include <vector>
17+
#include <iosfwd>
18+
19+
namespace o2
20+
{
21+
namespace framework
22+
{
23+
24+
struct MermaidHelpers {
25+
using Devices = std::vector<DeviceSpec>;
26+
static void dumpDataProcessorSpec2Mermaid(std::ostream&, const WorkflowSpec& specs);
27+
static void dumpDeviceSpec2Mermaid(std::ostream&, const Devices& specs);
28+
};
29+
30+
} // namespace framework
31+
} // namespace o2
32+
#endif // FRAMEWORK_MERMAIDHELPERS_H

Framework/Core/src/runDataProcessing.cxx

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
#include "O2ControlHelpers.h"
6565
#include "DeviceSpecHelpers.h"
6666
#include "GraphvizHelpers.h"
67+
#include "MermaidHelpers.h"
6768
#include "PropertyTreeHelpers.h"
6869
#include "SimpleResourceManager.h"
6970
#include "WorkflowSerializationHelpers.h"
@@ -2174,7 +2175,23 @@ void initialiseDriverControl(bpo::variables_map const& varmap,
21742175
control.state = DriverControlState::PLAY;
21752176
}
21762177

2177-
if (varmap["graphviz"].as<bool>()) {
2178+
if (varmap["mermaid"].as<bool>()) {
2179+
// Dump a mermaid representation of what I will do.
2180+
control.callbacks = {[](WorkflowSpec const&,
2181+
DeviceSpecs const& specs,
2182+
DeviceExecutions const&,
2183+
DataProcessorInfos&,
2184+
CommandInfo const&) {
2185+
MermaidHelpers::dumpDeviceSpec2Mermaid(std::cout, specs);
2186+
}};
2187+
control.forcedTransitions = {
2188+
DriverState::EXIT, //
2189+
DriverState::PERFORM_CALLBACKS, //
2190+
DriverState::MERGE_CONFIGS, //
2191+
DriverState::IMPORT_CURRENT_WORKFLOW, //
2192+
DriverState::MATERIALISE_WORKFLOW //
2193+
};
2194+
} else if (varmap["graphviz"].as<bool>()) {
21782195
// Dump a graphviz representation of what I will do.
21792196
control.callbacks = {[](WorkflowSpec const&,
21802197
DeviceSpecs const& specs,
@@ -2389,7 +2406,8 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
23892406
"what to do when a device has an error: quit, wait") // //
23902407
("min-failure-level", bpo::value<LogParsingHelpers::LogLevel>(&minFailureLevel)->default_value(LogParsingHelpers::LogLevel::Fatal), // //
23912408
"minimum message level which will be considered as fatal and exit with 1") // //
2392-
("graphviz,g", bpo::value<bool>()->zero_tokens()->default_value(false), "produce graph output") // //
2409+
("graphviz,g", bpo::value<bool>()->zero_tokens()->default_value(false), "produce graphviz output") // //
2410+
("mermaid", bpo::value<bool>()->zero_tokens()->default_value(false), "produce graph output in mermaid format") // //
23932411
("timeout,t", bpo::value<uint64_t>()->default_value(0), "forced exit timeout (in seconds)") // //
23942412
("dds,D", bpo::value<bool>()->zero_tokens()->default_value(false), "create DDS configuration") // //
23952413
("dds-workflow-suffix,D", bpo::value<std::string>()->default_value(""), "suffix for DDS names") // //
@@ -2603,12 +2621,16 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
26032621
conflicting_options(varmap, "dds", "dump-workflow");
26042622
conflicting_options(varmap, "dds", "run");
26052623
conflicting_options(varmap, "dds", "graphviz");
2624+
conflicting_options(varmap, "dds", "mermaid");
26062625
conflicting_options(varmap, "o2-control", "dump-workflow");
26072626
conflicting_options(varmap, "o2-control", "run");
26082627
conflicting_options(varmap, "o2-control", "graphviz");
2628+
conflicting_options(varmap, "o2-control", "mermaid");
26092629
conflicting_options(varmap, "run", "dump-workflow");
26102630
conflicting_options(varmap, "run", "graphviz");
2631+
conflicting_options(varmap, "run", "mermaid");
26112632
conflicting_options(varmap, "dump-workflow", "graphviz");
2633+
conflicting_options(varmap, "dump-workflow", "mermaid");
26122634
conflicting_options(varmap, "no-batch", "batch");
26132635

26142636
if (varmap.count("help")) {
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
#define BOOST_TEST_MODULE Test Framework MermaidHelpers
12+
#define BOOST_TEST_MAIN
13+
#define BOOST_TEST_DYN_LINK
14+
15+
#include "Mocking.h"
16+
#include "../src/ComputingResourceHelpers.h"
17+
#include "../src/DeviceSpecHelpers.h"
18+
#include "../src/MermaidHelpers.h"
19+
#include "../src/SimpleResourceManager.h"
20+
#include "Framework/DeviceSpec.h"
21+
#include "Framework/WorkflowSpec.h"
22+
#include "Headers/DataHeader.h"
23+
24+
#include <boost/test/unit_test.hpp>
25+
#include <sstream>
26+
#include <iostream>
27+
28+
using namespace o2::framework;
29+
30+
// because comparing the whole thing is a pain.
31+
void lineByLineComparison(const std::string& as, const std::string& bs)
32+
{
33+
std::istringstream a(as);
34+
std::istringstream b(bs);
35+
36+
char bufferA[1024];
37+
char bufferB[1024];
38+
while (a.good() && b.good()) {
39+
a.getline(bufferA, 1024);
40+
b.getline(bufferB, 1024);
41+
BOOST_CHECK_EQUAL(std::string(bufferA), std::string(bufferB));
42+
}
43+
BOOST_CHECK(a.eof());
44+
BOOST_CHECK(b.eof());
45+
}
46+
47+
// This is how you can define your processing in a declarative way
48+
WorkflowSpec defineDataProcessing()
49+
{
50+
return {{"A", Inputs{},
51+
Outputs{OutputSpec{"TST", "A1"},
52+
OutputSpec{"TST", "A2"}}},
53+
{"B",
54+
{InputSpec{"x", "TST", "A1"}},
55+
Outputs{OutputSpec{"TST", "B1"}}},
56+
{"C", Inputs{InputSpec{"x", "TST", "A2"}},
57+
Outputs{OutputSpec{"TST", "C1"}}},
58+
{"D",
59+
Inputs{InputSpec{"i1", "TST", "B1"},
60+
InputSpec{"i2", "TST", "C1"}},
61+
Outputs{}}};
62+
}
63+
64+
WorkflowSpec defineDataProcessing2()
65+
{
66+
return {
67+
{"A",
68+
{},
69+
{
70+
OutputSpec{"TST", "A"},
71+
}},
72+
timePipeline({"B",
73+
{InputSpec{"a", "TST", "A"}},
74+
{OutputSpec{"TST", "B"}}},
75+
3),
76+
timePipeline({"C",
77+
{InputSpec{"b", "TST", "B"}},
78+
{OutputSpec{"TST", "C"}}},
79+
2),
80+
};
81+
}
82+
83+
BOOST_AUTO_TEST_CASE(TestMermaid)
84+
{
85+
auto workflow = defineDataProcessing();
86+
std::ostringstream str;
87+
std::vector<DeviceSpec> devices;
88+
for (auto& device : devices) {
89+
BOOST_CHECK(device.id != "");
90+
}
91+
auto configContext = makeEmptyConfigContext();
92+
auto channelPolicies = ChannelConfigurationPolicy::createDefaultPolicies(*configContext);
93+
auto completionPolicies = CompletionPolicy::createDefaultPolicies();
94+
auto callbacksPolicies = CallbacksPolicy::createDefaultPolicies();
95+
std::vector<ComputingResource> resources = {ComputingResourceHelpers::getLocalhostResource()};
96+
SimpleResourceManager rm(resources);
97+
DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(workflow, channelPolicies, completionPolicies, callbacksPolicies, devices, rm, "workflow-id", *configContext);
98+
str.str("");
99+
MermaidHelpers::dumpDeviceSpec2Mermaid(str, devices);
100+
lineByLineComparison(str.str(), R"EXPECTED(flowchart TD
101+
A
102+
B
103+
C
104+
D
105+
A-- 22000:from_A_to_B -->B
106+
A-- 22001:from_A_to_C -->C
107+
B-- 22002:from_B_to_D -->D
108+
C-- 22003:from_C_to_D -->D
109+
)EXPECTED");
110+
}
111+
112+
BOOST_AUTO_TEST_CASE(TestMermaidWithPipeline)
113+
{
114+
auto workflow = defineDataProcessing2();
115+
std::ostringstream str;
116+
std::vector<DeviceSpec> devices;
117+
for (auto& device : devices) {
118+
BOOST_CHECK(device.id != "");
119+
}
120+
auto configContext = makeEmptyConfigContext();
121+
auto channelPolicies = ChannelConfigurationPolicy::createDefaultPolicies(*configContext);
122+
auto completionPolicies = CompletionPolicy::createDefaultPolicies();
123+
auto callbacksPolicies = CallbacksPolicy::createDefaultPolicies();
124+
std::vector<ComputingResource> resources = {ComputingResourceHelpers::getLocalhostResource()};
125+
SimpleResourceManager rm(resources);
126+
DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(workflow, channelPolicies, completionPolicies, callbacksPolicies, devices, rm, "workflow-id", *configContext);
127+
str.str("");
128+
MermaidHelpers::dumpDeviceSpec2Mermaid(str, devices);
129+
lineByLineComparison(str.str(), R"EXPECTED(flowchart TD
130+
A
131+
B_t0
132+
B_t1
133+
B_t2
134+
C_t0
135+
C_t1
136+
A-- 22000:from_A_to_B_t0 -->B_t0
137+
A-- 22001:from_A_to_B_t1 -->B_t1
138+
A-- 22002:from_A_to_B_t2 -->B_t2
139+
B_t0-- 22003:from_B_t0_to_C_t0 -->C_t0
140+
B_t1-- 22005:from_B_t1_to_C_t0 -->C_t0
141+
B_t2-- 22007:from_B_t2_to_C_t0 -->C_t0
142+
B_t0-- 22004:from_B_t0_to_C_t1 -->C_t1
143+
B_t1-- 22006:from_B_t1_to_C_t1 -->C_t1
144+
B_t2-- 22008:from_B_t2_to_C_t1 -->C_t1
145+
)EXPECTED");
146+
}

0 commit comments

Comments
 (0)