Skip to content

Commit 03881b7

Browse files
aehmttwaehmttw
authored
Added SQL benchmark tests to CI (#1857)
Signed-off-by: Matei <matei@feldera.com>
1 parent a8a1f46 commit 03881b7

File tree

3 files changed

+48
-11
lines changed

3 files changed

+48
-11
lines changed

‎Earthfile‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,13 @@ install-deps:
3434
RUN sudo apt-get install nodejs -y
3535
RUN npm install --global yarn
3636
RUN npm install --global openapi-typescript-codegen
37+
RUN apt install unzip -y
38+
RUN apt install python3-requests -y
39+
RUN arch=`dpkg --print-architecture`; \
40+
curl -LO https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-$arch.zip \
41+
&& unzip rpk-linux-$arch.zip -d /bin/ \
42+
&& rpk version \
43+
&& rm rpk-linux-$arch.zip
3744

3845
install-rust:
3946
FROM +install-deps
@@ -320,12 +327,6 @@ pipeline-manager-container-cors-all:
320327
build-demo-container:
321328
FROM +install-rust
322329
WORKDIR /
323-
RUN apt install unzip -y
324-
RUN arch=`dpkg --print-architecture`; \
325-
curl -LO https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-$arch.zip \
326-
&& unzip rpk-linux-$arch.zip -d /bin/ \
327-
&& rpk version \
328-
&& rm rpk-linux-$arch.zip
329330
# Install snowsql
330331
RUN curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.28-linux_x86_64.bash \
331332
&& SNOWSQL_DEST=/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.28-linux_x86_64.bash \
@@ -513,11 +514,30 @@ ui-playwright-tests:
513514
END
514515

515516
benchmark:
516-
FROM +build-nexmark
517+
FROM +build-manager
517518
COPY scripts/bench.bash scripts/bench.bash
518-
519-
RUN bash scripts/bench.bash
519+
COPY benchmark/feldera-sql/run.py benchmark/feldera-sql/run.py
520+
COPY +build-manager/pipeline-manager .
521+
COPY +build-sql/sql-to-dbsp-compiler sql-to-dbsp-compiler
522+
RUN mkdir -p /working-dir/cargo_workspace
523+
COPY Cargo.lock /working-dir/cargo_workspace/Cargo.lock
524+
ENV PGHOST=localhost
525+
ENV PGUSER=postgres
526+
ENV PGCLIENTENCODING=UTF8
527+
ENV PGPORT=5432
528+
ENV RUST_LOG=error
529+
ENV WITH_POSTGRES=1
530+
ENV IN_CI=1
531+
WITH DOCKER --pull postgres
532+
RUN docker run --shm-size=512MB -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -e PGDATA=/dev/shm -d postgres && \
533+
sleep 10 && \
534+
(./pipeline-manager --bind-address=0.0.0.0 --api-server-working-directory=/working-dir --compiler-working-directory=/working-dir --runner-working-directory=/working-dir --sql-compiler-home=/dbsp/sql-to-dbsp-compiler --dbsp-override-path=/dbsp --db-connection-string=postgresql://postgres:postgres@localhost:5432 --compilation-profile=optimized &) && \
535+
sleep 5 && \
536+
docker run --name redpanda -p 9092:9092 --rm -itd docker.redpanda.com/vectorized/redpanda:v23.2.3 redpanda start --smp 2 \
537+
&& bash scripts/bench.bash
538+
END
520539
SAVE ARTIFACT crates/nexmark/nexmark_results.csv AS LOCAL .
540+
SAVE ARTIFACT crates/nexmark/sql_nexmark_results.csv AS LOCAL .
521541
SAVE ARTIFACT crates/nexmark/dram_nexmark_results.csv AS LOCAL .
522542
SAVE ARTIFACT crates/dbsp/galen_results.csv AS LOCAL .
523543
#SAVE ARTIFACT crates/dbsp/ldbc_results.csv AS LOCAL .

‎scripts/bench-publish.bash‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ fi
1717

1818
NEXMARK_CSV_FILE='nexmark_results.csv'
1919
NEXMARK_DRAM_CSV_FILE='dram_nexmark_results.csv'
20+
NEXMARK_SQL_CSV_FILE='sql_nexmark_results.csv'
2021
NEXMARK_PERSISTENCE_CSV_FILE='persistence_nexmark_results.csv'
2122
GALEN_CSV_FILE='galen_results.csv'
2223
LDBC_CSV_FILE='ldbc_results.csv'
@@ -45,10 +46,11 @@ fi
4546

4647
# Copy nexmark results
4748
mkdir -p ${DEPLOY_DIR}
48-
mv ${NEXMARK_CSV_FILE} ${NEXMARK_DRAM_CSV_FILE} ${DEPLOY_DIR}
49+
mv ${NEXMARK_CSV_FILE} ${NEXMARK_DRAM_CSV_FILE} ${NEXMARK_SQL_CSV_FILE} ${DEPLOY_DIR}
4950
gzip -f ${DEPLOY_DIR}/${NEXMARK_CSV_FILE}
5051
#gzip -f ${DEPLOY_DIR}/${NEXMARK_PERSISTENCE_CSV_FILE}
5152
gzip -f ${DEPLOY_DIR}/${NEXMARK_DRAM_CSV_FILE}
53+
gzip -f ${DEPLOY_DIR}/${NEXMARK_SQL_CSV_FILE}
5254

5355
# Add galen results to repo
5456
DEPLOY_DIR="gh-pages/galen/${CI_MACHINE_TYPE}/${PR_COMMIT_SHA}/"

‎scripts/bench.bash‎

100644100755
Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@ rm -rf gh-pages
1212
fi
1313
NEXMARK_CSV_FILE='nexmark_results.csv'
1414
NEXMARK_DRAM_CSV_FILE='dram_nexmark_results.csv'
15+
NEXMARK_SQL_CSV_FILE='sql_nexmark_results.csv'
1516
NEXMARK_PERSISTENCE_CSV_FILE='persistence_nexmark_results.csv'
1617
GALEN_CSV_FILE='galen_results.csv'
1718
LDBC_CSV_FILE='ldbc_results.csv'
18-
rm -f crates/nexmark/${NEXMARK_CSV_FILE} crates/dbsp/${GALEN_CSV_FILE} crates/dbsp/${LDBC_CSV_FILE} crates/nexmark/${NEXMARK_DRAM_CSV_FILE} crates/nexmark/${NEXMARK_PERSISTENCE_CSV_FILE}
19+
rm -f crates/nexmark/${NEXMARK_CSV_FILE} crates/nexmark/${NEXMARK_SQL_CSV_FILE} crates/dbsp/${GALEN_CSV_FILE} crates/dbsp/${LDBC_CSV_FILE} crates/nexmark/${NEXMARK_DRAM_CSV_FILE} crates/nexmark/${NEXMARK_PERSISTENCE_CSV_FILE}
1920

2021
# Run nexmark benchmark
2122
EVENT_RATE=10000000
@@ -28,6 +29,20 @@ if [ "$SMOKE" != "" ]; then
2829
fi
2930
cargo bench --bench nexmark -- --first-event-rate=${EVENT_RATE} --max-events=${MAX_EVENTS} --cpu-cores ${CORES} --num-event-generators ${GENERATORS} --source-buffer-size 10000 --input-batch-size 40000 --csv ${NEXMARK_CSV_FILE}
3031

32+
# Run nexmark SQL benchmark
33+
# This test requires a running instance of redpanda and pipeline-manager.
34+
# The Earthfile should run those.
35+
# 100M events causes out of memory problems with SQL tests
36+
MAX_EVENTS=10000000
37+
if [ "$SMOKE" != "" ]; then
38+
MAX_EVENTS=1000000
39+
fi
40+
KAFKA_BROKER=localhost:9092
41+
rpk topic -X brokers=$KAFKA_BROKER delete bid auction person
42+
cargo run -p dbsp_nexmark --example generate --features with-kafka -- --max-events ${MAX_EVENTS} -O bootstrap.servers=$KAFKA_BROKER
43+
FELDERA_API=http://localhost:8080
44+
python3 benchmark/feldera-sql/run.py --api-url $FELDERA_API --kafka-broker $KAFKA_BROKER --csv crates/nexmark/${NEXMARK_SQL_CSV_FILE}
45+
3146
# Run galen benchmark
3247
cargo bench --bench galen --features="with-csv" -- --workers 10 --csv ${GALEN_CSV_FILE}
3348

0 commit comments

Comments
 (0)