Skip to content

Commit 0977f19

Browse files
knopers8sawenzel
knopers8
authored and
sawenzel
committed
[QC-892] Simplify and update Data Sampling benchmark
1 parent f5bfa53 commit 0977f19

3 files changed

Lines changed: 68 additions & 221 deletions

File tree

‎Utilities/DataSampling/scripts/o2-datasampling-benchmark.sh‎

Lines changed: 43 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ RESULTS_FILE='data-sampling-benchmark-'$(date +"%y-%m-%d_%H%M")
2121
# \param 1 : fractions array name
2222
# \param 2 : payload sizes array name
2323
# \param 3 : number of producers array name
24-
# \param 4 : number of dispatchers array name
24+
# \param 4 : number of dispatchers array name (not supported atm)
2525
# \param 5 : repetitions
2626
# \param 6 : test duration
2727
# \param 7 : warm up cycles - how many first metrics should be ignored (they are sent each 10s)
@@ -60,20 +60,19 @@ function benchmark() {
6060
local repo_branch=$(git rev-parse --abbrev-ref HEAD)
6161
local results_filename='data-sampling-benchmark-'$(date +"%y-%m-%d_%H%M")'-'$test_name
6262
local test_date=$(date +"%y-%m-%d %H:%M:%S")
63-
local memory_soft_limit_mbytes=$((available_memory_bytes / 5000000)) # we stop creating new messages at 4/5 of the maximum allowed usage
64-
65-
printf "DATA SAMPLING BENCHMARK RESULTS\n" > $results_filename
66-
printf "Test date: %s\n" "$test_date" >> $results_filename
67-
printf "Latest commit: %s\n" "$repo_latest_commit" >> $results_filename
68-
printf "Branch: %s\n" "$repo_branch" >> $results_filename
69-
printf "Repetitions: %s\n" "$repetitions" >> $results_filename
70-
printf "Test duration [s]: %s\n" "$test_duration" >> $results_filename
71-
printf "Warm up cycles: %s\n" "$warm_up_cycles" >> $results_filename
72-
printf "Available memory [B]: %s\n" "$available_memory_bytes" >> $results_filename
73-
printf "Memory soft limit [MB]: %s\n" "$memory_soft_limit_mbytes" >> $results_filename
74-
echo "fraction , payload size , nb producers , nb dispatchers , messages per second" >> $results_filename
75-
76-
local common_args="--run -b --infologger-severity info --shm-segment-size "$available_memory_bytes" --test-duration "$test_duration" --throttling "$memory_soft_limit_mbytes
63+
local run_log='run_log'
64+
65+
printf "DATA SAMPLING BENCHMARK RESULTS\n" > "$results_filename"
66+
printf "Test date: %s\n" "$test_date" >> "$results_filename"
67+
printf "Latest commit: %s\n" "$repo_latest_commit" >> "$results_filename"
68+
printf "Branch: %s\n" "$repo_branch" >> "$results_filename"
69+
printf "Repetitions: %s\n" "$repetitions" >> "$results_filename"
70+
printf "Test duration [s]: %s\n" "$test_duration" >> "$results_filename"
71+
printf "Warm up cycles: %s\n" "$warm_up_cycles" >> "$results_filename"
72+
printf "Available memory [B]: %s\n" "$available_memory_bytes" >> "$results_filename"
73+
echo "fraction , payload size , nb producers , nb dispatchers , messages per second" >> "$results_filename"
74+
75+
local common_args="--run -b --shm-throw-bad-alloc false --monitoring-backend infologger:///prod?metric --shm-segment-size "$available_memory_bytes
7776
if [[ $fill == "yes" ]]; then
7877
common_args=$common_args' --fill'
7978
fi
@@ -84,53 +83,44 @@ function benchmark() {
8483
for nb_dispatchers in ${number_of_dispatchers[@]}; do
8584
for ((rep=0;rep<repetitions;rep++)); do
8685
echo "************************************************************"
87-
echo "Launching test for payload size $payload_size bytes, $nb_producers producers, $nb_dispatchers dispatchers, sampling fraction $fraction"
86+
echo "Launching test for payload size $payload_size bytes, $nb_producers producers, 1 dispatchers, sampling fraction $fraction"
8887

8988
printf "%15s," "$fraction" >> $results_filename
9089
printf "%16s," "$payload_size" >> $results_filename
9190
printf "%16s," "$nb_producers" >> $results_filename
92-
printf "%16s," "$nb_dispatchers" >> $results_filename
91+
printf "%16s," "1" >> $results_filename
9392

9493
messages_per_second=
9594
while [ "$messages_per_second" == 'error' ] || [ -z "$messages_per_second" ]; do
9695
if [ "$messages_per_second" == 'error' ]; then
9796
echo "Retrying the test because of an error"
9897
fi
9998

100-
# running the benchmark, extracting an array of metrics, ignoring the first warm_up_cycles-1
101-
# fixme: we assume that the metrics are produced in even (10s) time intervals and all are printed,
102-
# we should at least be able notice when something doesn't seem right
99+
rm -f $run_log
100+
echo "Starting the DPL workflow..."
101+
timeout -k 60s $test_duration_timeout o2-datasampling-datasampling-benchmark $common_args --payload-size $payload_size --producers $nb_producers --dispatchers 1 --sampling-fraction $fraction > "$run_log"
102+
echo "...done, performing cleanups."
103+
pkill -f o2-testworkflows-datasampling-benchmark
104+
sleep 5
105+
pkill -9 -f o2-testworkflows-datasampling-benchmark
103106

104-
metrics=
105-
mapfile -t metrics < \
106-
<( timeout -k 60s $test_duration_timeout o2-testworkflows-datasampling-benchmark $common_args --payload-size $payload_size --producers $nb_producers --dispatchers $nb_dispatchers --sampling-fraction $fraction \
107-
| grep -o 'Dispatcher_messages_evaluated,[0-9] [0-9]\{1,\}' \
107+
mapfile -t metrics_messages_evaluated < \
108+
<( grep -o 'Dispatcher_messages_evaluated,[0-9] [0-9]\{1,\}' "$run_log" \
108109
| sed -e 's/Dispatcher_messages_evaluated,[0-9]\{1,\} //' \
109-
| tail -n +$((warm_up_cycles * nb_dispatchers + 1)) )
110+
| tail -n +$((warm_up_cycles + 1)) )
111+
mapfile -t metrics_test_duration < \
112+
<( grep -a 'Dispatcher_messages_evaluated' "$run_log" \
113+
| grep -o -e '[0-9]\{1,\} pipeline_id' \
114+
| sed -e 's/ pipeline_id//' \
115+
| tail -n +$((warm_up_cycles + 1)) )
110116

111-
pkill -9 -f o2-testworkflows-datasampling-benchmark
112117

113-
if [ ${#metrics[@]} -ge $(( 2 * nb_dispatchers )) ]; then
114-
115-
total_start=0
116-
for ((i = 0; i < nb_dispatchers; i++))
117-
do
118-
(( total_start+=metrics[i] ))
119-
done
120-
121-
total_end=0
122-
for ((i = 1; i < $((1 + nb_dispatchers)); i++))
123-
do
124-
(( total_end+=metrics[-i] ))
125-
done
126-
127-
(( messages_per_second = (total_end - total_start) / (${#metrics[@]} / nb_dispatchers - 1 ) ))
128-
# divide by 10, keeping the last digit
129-
if [ $messages_per_second -gt 9 ]; then
130-
messages_per_second=${messages_per_second:0:-1}.${messages_per_second: -1}
131-
else
132-
messages_per_second=0.$messages_per_second
133-
fi
118+
if [ ${#metrics_messages_evaluated[@]} -ge 2 ] && [ ${#metrics_test_duration[@]} -ge 2 ]; then
119+
120+
(( total_metrics_messages_evaluated = metrics_messages_evaluated[-1] - metrics_messages_evaluated[0] ))
121+
(( total_test_duration_ms = metrics_test_duration[-1] - metrics_test_duration[0] ))
122+
123+
messages_per_second=`echo "scale=3; $total_metrics_messages_evaluated*1000/$total_test_duration_ms" | bc -l`
134124
else
135125
messages_per_second='error'
136126
fi
@@ -140,10 +130,10 @@ function benchmark() {
140130
printf "\n" >> $results_filename
141131

142132
echo "Dispatcher_messages_evaluated metrics:"
143-
if [ ${#metrics[@]} -gt 0 ]; then
144-
printf '%s\n' "${metrics[@]}"
133+
if [ ${#metrics_messages_evaluated[@]} -gt 0 ]; then
134+
printf '%s\n' "${metrics_messages_evaluated[@]}"
145135
else
146-
echo $metrics
136+
echo $metrics_messages_evaluated
147137
fi
148138
printf 'Messages per second: %s\n' "${messages_per_second}"
149139
done
@@ -201,12 +191,13 @@ while getopts 'hfm:t:' option; do
201191
esac
202192
done
203193

194+
REPETITIONS=1;
195+
TEST_DURATION=300;
196+
204197
FRACTIONS=(1.00);
205198
PAYLOAD_SIZE=(16777216 67108864 268435456 1073741824);
206199
NB_PRODUCERS=(8);
207200
NB_DISPATCHERS=(1);
208-
REPETITIONS=1;
209-
TEST_DURATION=300;
210201
WARM_UP_CYCLES=2;
211202
TEST_NAME='memory'
212203

@@ -216,8 +207,6 @@ FRACTIONS=(0.00 1.00);
216207
PAYLOAD_SIZE=(1 256 1024 4096 16384 65536 262144 1048576 4194304 16777216 67108864 268435456 1073741824);
217208
NB_PRODUCERS=(8);
218209
NB_DISPATCHERS=(1);
219-
REPETITIONS=1;
220-
TEST_DURATION=300;
221210
WARM_UP_CYCLES=6;
222211
TEST_NAME='payloads'
223212

@@ -227,8 +216,6 @@ FRACTIONS=(0.00 1.00);
227216
PAYLOAD_SIZE=(2097152);
228217
NB_PRODUCERS=(1 2 4 8 16 32);
229218
NB_DISPATCHERS=(1);
230-
REPETITIONS=1;
231-
TEST_DURATION=300;
232219
WARM_UP_CYCLES=6;
233220
TEST_NAME='producers-2MiB'
234221

@@ -238,20 +225,7 @@ FRACTIONS=(0.0000 0.0001 0.0010 0.0100 0.1000 0.5000 1.0000);
238225
PAYLOAD_SIZE=(2097152);
239226
NB_PRODUCERS=(8);
240227
NB_DISPATCHERS=(1);
241-
REPETITIONS=1;
242-
TEST_DURATION=300;
243228
WARM_UP_CYCLES=6;
244229
TEST_NAME='fractions'
245230

246231
benchmark FRACTIONS PAYLOAD_SIZE NB_PRODUCERS NB_DISPATCHERS $REPETITIONS $TEST_DURATION $WARM_UP_CYCLES $TEST_NAME $MEMORY_USAGE $FILL
247-
248-
FRACTIONS=(0.0000 1.0000);
249-
PAYLOAD_SIZE=(256 2097152);
250-
NB_PRODUCERS=(8);
251-
NB_DISPATCHERS=(1 2 4 8);
252-
REPETITIONS=1;
253-
TEST_DURATION=300;
254-
WARM_UP_CYCLES=6;
255-
TEST_NAME='dispatchers'
256-
257-
benchmark FRACTIONS PAYLOAD_SIZE NB_PRODUCERS NB_DISPATCHERS $REPETITIONS $TEST_DURATION $WARM_UP_CYCLES $TEST_NAME $MEMORY_USAGE $FILL

‎Utilities/DataSampling/src/Dispatcher.cxx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,8 @@ void Dispatcher::reportStats(Monitoring& monitoring) const
142142
dispatcherTotalAcceptedMessages += policy->getTotalAcceptedMessages();
143143
}
144144

145-
monitoring.send(Metric{dispatcherTotalEvaluatedMessages, "Dispatcher_messages_evaluated"}.addTag(tags::Key::Subsystem, tags::Value::DataSampling));
146-
monitoring.send(Metric{dispatcherTotalAcceptedMessages, "Dispatcher_messages_passed"}.addTag(tags::Key::Subsystem, tags::Value::DataSampling));
145+
monitoring.send(Metric{dispatcherTotalEvaluatedMessages, "Dispatcher_messages_evaluated", Verbosity::Prod}.addTag(tags::Key::Subsystem, tags::Value::DataSampling));
146+
monitoring.send(Metric{dispatcherTotalAcceptedMessages, "Dispatcher_messages_passed", Verbosity::Prod}.addTag(tags::Key::Subsystem, tags::Value::DataSampling));
147147
}
148148

149149
DataSamplingHeader Dispatcher::prepareDataSamplingHeader(const DataSamplingPolicy& policy)

0 commit comments

Comments
 (0)