Skip to content

Commit a069094

Browse files
aehmttwgz
aehmttw
authored and
gz
committed
Add Flink hashmap test
Signed-off-by: Matei <matei@feldera.com>
1 parent 2f4539b commit a069094

File tree

4 files changed

+72
-3
lines changed

4 files changed

+72
-3
lines changed

‎Earthfile‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -539,12 +539,20 @@ flink-benchmark:
539539
RUN apt-get -y install maven bc
540540
RUN benchmark/flink/setup-flink.sh
541541

542-
RUN echo "when,runner,mode,language,name,num_cores,num_events,elapsed,peak_memory_kb,cpu_secs" >> flink_results.csv
542+
# Run with rocksdb
543+
COPY benchmark/flink/flink-conf-rocksdb.yaml benchmark/flink/flink-conf.yaml
544+
WITH DOCKER
545+
RUN cd benchmark && ./run-nexmark.sh -r flink --query all --events 100M && mv nexmark.csv flink_results.csv
546+
END
547+
RUN cat benchmark/flink/flink-conf.yaml
548+
SAVE ARTIFACT benchmark/flink_results.csv AS LOCAL flink_results_rocksdb.csv
543549

550+
# Run with hashmap
551+
COPY benchmark/flink/flink-conf-hashmap.yaml benchmark/flink/flink-conf.yaml
544552
WITH DOCKER
545553
RUN cd benchmark && ./run-nexmark.sh -r flink --query all --events 100M && mv nexmark.csv flink_results.csv
546554
END
547-
SAVE ARTIFACT benchmark/flink_results.csv AS LOCAL .
555+
SAVE ARTIFACT benchmark/flink_results.csv AS LOCAL flink_results_hashmap.csv
548556

549557
ci-tests:
550558
BUILD +formatting-check

‎benchmark/flink/flink-conf-hashmap.yaml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
taskmanager.memory.process.size: 8G
20+
jobmanager.rpc.address: nexmark-jobmanager-1
21+
jobmanager.rpc.port: 6123
22+
jobmanager.memory.process.size: 8G
23+
taskmanager.numberOfTaskSlots: 1
24+
parallelism.default: 8
25+
io.tmp.dirs: /tmp
26+
27+
#==============================================================================
28+
# JVM
29+
#==============================================================================
30+
31+
# JVM options for GC
32+
env.java.opts: -verbose:gc -XX:NewRatio=3 -XX:+PrintGCDetails -XX:ParallelGCThreads=4
33+
env.java.opts.jobmanager: -Xloggc:/opt/flink/log/jobmanager-gc.log
34+
env.java.opts.taskmanager: -Xloggc:/opt/flink/log/taskmanager-gc.log
35+
36+
#==============================================================================
37+
# State & Checkpoint
38+
#==============================================================================
39+
40+
state.backend: hashmap
41+
# for example, hdfs://benchmark01/checkpoint
42+
state.checkpoints.dir: file:///tmp/checkpoints
43+
state.backend.rocksdb.localdir: /tmp/rocksdb
44+
state.backend.incremental: true
45+
execution.checkpointing.interval: 180000
46+
execution.checkpointing.mode: EXACTLY_ONCE
47+
state.backend.local-recovery: true
48+
49+
#==============================================================================
50+
# Runtime Others
51+
#==============================================================================
52+
53+
# configuration options for adjusting and tuning table programs.
54+
table.exec.mini-batch.enabled: true
55+
table.exec.mini-batch.allow-latency: 2s
56+
table.exec.mini-batch.size: 50000
57+
table.optimizer.distinct-agg.split.enabled: true
58+
59+
# disable final checkpoint to avoid test waiting for the last checkpoint complete
60+
execution.checkpointing.checkpoints-after-tasks-finish.enabled: false
61+
## EOF ##

benchmark/flink/flink-conf.yaml renamed to benchmark/flink/flink-conf-rocksdb.yaml

File renamed without changes.

‎benchmark/flink/refresh-flink-benchmark.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ mkdir $flink_dir
88
cd ../..
99
earthly --verbose -P +flink-benchmark
1010
cd benchmark/flink
11-
mv ../../flink_results.csv $flink_dir
11+
mv ../../flink_results_*.csv $flink_dir

0 commit comments

Comments
 (0)