Skip to content

Commit 1c061bf

Browse files
mihaibudiuryzhyk
mihaibudiu
authored and
ryzhyk
committed
Remove the JIT backend.
Remove the JIT backend and all the code that depends on it throughout the project. Signed-off-by: Leonid Ryzhyk <leonid@feldera.com> Signed-off-by: Mihai Budiu <mbudiu@feldera.com>
1 parent 1dfbc51 commit 1c061bf

File tree

311 files changed

+554
-64554
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

311 files changed

+554
-64554
lines changed

‎Cargo.lock‎

Lines changed: 182 additions & 711 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Earthfile‎

Lines changed: 11 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,6 @@ prepare-cache:
116116
# https://github.com/earthly/earthly/issues/786 is fixed this can go away
117117
# and crates can be copied instead:
118118
RUN mkdir -p .cargo
119-
RUN mkdir -p crates/dataflow-jit
120119
RUN mkdir -p crates/nexmark
121120
RUN mkdir -p crates/dbsp
122121
RUN mkdir -p crates/adapters
@@ -131,7 +130,6 @@ prepare-cache:
131130

132131
COPY --keep-ts Cargo.toml .
133132
COPY --keep-ts Cargo.lock .
134-
COPY --keep-ts crates/dataflow-jit/Cargo.toml crates/dataflow-jit/
135133
COPY --keep-ts crates/nexmark/Cargo.toml crates/nexmark/
136134
COPY --keep-ts crates/dbsp/Cargo.toml crates/dbsp/
137135
COPY --keep-ts crates/pipeline-types/Cargo.toml crates/pipeline-types/
@@ -145,14 +143,12 @@ prepare-cache:
145143
COPY --keep-ts sql-to-dbsp-compiler/lib/tuple/Cargo.toml sql-to-dbsp-compiler/lib/tuple/
146144
COPY --keep-ts sql-to-dbsp-compiler/temp/Cargo.toml sql-to-dbsp-compiler/temp/
147145

148-
RUN mkdir -p crates/dataflow-jit/src && touch crates/dataflow-jit/src/lib.rs
149146
RUN mkdir -p crates/nexmark/src && touch crates/nexmark/src/lib.rs
150147
RUN mkdir -p crates/dbsp/src && touch crates/dbsp/src/lib.rs
151148
RUN mkdir -p crates/dbsp/examples && touch crates/dbsp/examples/degrees.rs && touch crates/dbsp/examples/orgchart.rs
152149
RUN mkdir -p crates/pipeline-types/src && touch crates/pipeline-types/src/lib.rs
153150
RUN mkdir -p crates/adapters/src && touch crates/adapters/src/lib.rs
154151
RUN mkdir -p crates/adapters/examples && touch crates/adapters/examples/server.rs
155-
RUN mkdir -p crates/dataflow-jit/src && touch crates/dataflow-jit/src/main.rs
156152
RUN mkdir -p crates/nexmark/benches/nexmark-gen && touch crates/nexmark/benches/nexmark-gen/main.rs
157153
RUN mkdir -p crates/nexmark/benches/nexmark && touch crates/nexmark/benches/nexmark/main.rs
158154
RUN mkdir -p crates/dbsp/benches/gdelt && touch crates/dbsp/benches/gdelt/main.rs
@@ -213,9 +209,6 @@ build-cache:
213209
RUN cargo +$RUST_TOOLCHAIN build $RUST_BUILD_PROFILE --package pipeline-manager
214210
RUN cargo +$RUST_TOOLCHAIN test $RUST_BUILD_PROFILE --package pipeline-manager --no-run
215211
RUN cargo +$RUST_TOOLCHAIN clippy $RUST_BUILD_PROFILE --package pipeline-manager
216-
RUN cargo +$RUST_TOOLCHAIN build $RUST_BUILD_PROFILE --package dataflow-jit
217-
RUN cargo +$RUST_TOOLCHAIN test $RUST_BUILD_PROFILE --package dataflow-jit --no-run
218-
RUN cargo +$RUST_TOOLCHAIN clippy $RUST_BUILD_PROFILE --package dataflow-jit
219212
RUN cargo +$RUST_TOOLCHAIN build $RUST_BUILD_PROFILE --package dbsp_nexmark
220213
RUN cargo +$RUST_TOOLCHAIN test $RUST_BUILD_PROFILE --package dbsp_nexmark --no-run
221214
RUN cargo +$RUST_TOOLCHAIN clippy $RUST_BUILD_PROFILE --package dbsp_nexmark
@@ -262,26 +255,11 @@ build-dbsp:
262255
RUN cargo +$RUST_TOOLCHAIN clippy $RUST_BUILD_PROFILE --package pipeline_types -- -D warnings
263256
RUN cargo +$RUST_TOOLCHAIN test $RUST_BUILD_PROFILE --package pipeline_types --no-run
264257

265-
266-
build-dataflow-jit:
267-
ARG RUST_TOOLCHAIN=$RUST_VERSION
268-
ARG RUST_BUILD_PROFILE=$RUST_BUILD_MODE
269-
270-
FROM +build-dbsp --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
271-
272-
RUN rm -rf crates/dataflow-jit
273-
COPY --keep-ts --dir crates/dataflow-jit crates/dataflow-jit
274-
275-
RUN cargo +$RUST_TOOLCHAIN build $RUST_BUILD_PROFILE --package dataflow-jit
276-
RUN cd crates/dataflow-jit && cargo +$RUST_TOOLCHAIN machete
277-
RUN cargo +$RUST_TOOLCHAIN test $RUST_BUILD_PROFILE --package dataflow-jit --no-run
278-
RUN cargo +$RUST_TOOLCHAIN clippy $RUST_BUILD_PROFILE --package dataflow-jit -- -D warnings
279-
280258
build-sql:
281259
ARG RUST_TOOLCHAIN=$RUST_VERSION
282260
ARG RUST_BUILD_PROFILE=$RUST_BUILD_MODE
283261

284-
FROM +build-dataflow-jit --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
262+
FROM +build-dbsp --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
285263
COPY --keep-ts sql-to-dbsp-compiler sql-to-dbsp-compiler
286264

287265
COPY demo/hello-world/combiner.sql demo/hello-world/combiner.sql
@@ -307,7 +285,7 @@ build-adapters:
307285
RUN rm -rf crates/adapters
308286
COPY --keep-ts --dir crates/adapters crates/adapters
309287

310-
RUN cargo +$RUST_TOOLCHAIN build $RUST_BUILD_PROFILE --package dbsp_adapters --features="with-jit"
288+
RUN cargo +$RUST_TOOLCHAIN build $RUST_BUILD_PROFILE --package dbsp_adapters
311289
RUN cd crates/adapters && cargo +$RUST_TOOLCHAIN machete
312290
RUN cargo +$RUST_TOOLCHAIN clippy $RUST_BUILD_PROFILE --package dbsp_adapters -- -D warnings
313291
ENV RUST_BACKTRACE=1
@@ -331,11 +309,9 @@ build-manager:
331309

332310
IF [ -f ./target/debug/pipeline-manager ]
333311
SAVE ARTIFACT --keep-ts ./target/debug/pipeline-manager pipeline-manager
334-
SAVE ARTIFACT --keep-ts ./target/debug/pipeline pipeline
335312
END
336313
IF [ -f ./target/release/pipeline-manager ]
337314
SAVE ARTIFACT --keep-ts ./target/release/pipeline-manager pipeline-manager
338-
SAVE ARTIFACT --keep-ts ./target/release/pipeline pipeline
339315
END
340316

341317
test-sql:
@@ -395,15 +371,6 @@ test-nexmark:
395371
FROM +build-nexmark --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
396372
RUN cargo +$RUST_TOOLCHAIN test $RUST_BUILD_PROFILE --package dbsp_nexmark
397373

398-
test-dataflow-jit:
399-
ARG RUST_TOOLCHAIN=$RUST_VERSION
400-
ARG RUST_BUILD_PROFILE=$RUST_BUILD_MODE
401-
402-
FROM +build-dataflow-jit --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
403-
# Tests use this demo directory
404-
COPY --keep-ts demo/project_demo01-TimeSeriesEnrich demo/project_demo01-TimeSeriesEnrich
405-
RUN cargo +$RUST_TOOLCHAIN test $RUST_BUILD_PROFILE --package dataflow-jit
406-
407374
test-adapters:
408375
ARG RUST_TOOLCHAIN=$RUST_VERSION
409376
ARG RUST_BUILD_PROFILE=$RUST_BUILD_MODE
@@ -413,7 +380,7 @@ test-adapters:
413380
RUN docker run -p 9092:9092 --rm -itd docker.redpanda.com/vectorized/redpanda:v23.2.3 \
414381
redpanda start --smp 2 && \
415382
sleep 5 && \
416-
cargo +$RUST_TOOLCHAIN test $RUST_BUILD_PROFILE --package dbsp_adapters --features="with-jit" --package sqllib
383+
cargo +$RUST_TOOLCHAIN test $RUST_BUILD_PROFILE --package dbsp_adapters --package sqllib
417384
END
418385

419386
test-manager:
@@ -468,13 +435,13 @@ test-python:
468435

469436
FROM +build-manager --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
470437
COPY +build-manager/pipeline-manager .
471-
COPY +build-manager/pipeline .
472438
RUN mkdir -p /root/.local/lib/python3.10
473439
RUN mkdir -p /root/.local/bin
474440

475441
COPY +install-python/python3.10 /root/.local/lib/python3.10
476442
COPY +install-python/bin /root/.local/bin
477443

444+
COPY +build-manager/pipeline-manager .
478445
COPY +build-sql/sql-to-dbsp-compiler sql-to-dbsp-compiler
479446

480447
COPY demo/demo_notebooks demo/demo_notebooks
@@ -490,7 +457,7 @@ test-python:
490457
WITH DOCKER --pull postgres
491458
RUN docker run --shm-size=512MB -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -e PGDATA=/dev/shm -d postgres && \
492459
sleep 10 && \
493-
(./pipeline-manager --bind-address=0.0.0.0 --api-server-working-directory=/working-dir --compiler-working-directory=/working-dir --runner-working-directory=/working-dir --sql-compiler-home=/dbsp/sql-to-dbsp-compiler --dbsp-override-path=/dbsp --jit-pipeline-runner-path=/dbsp/pipeline --db-connection-string=postgresql://postgres:postgres@localhost:5432 &) && \
460+
(./pipeline-manager --bind-address=0.0.0.0 --api-server-working-directory=/working-dir --compiler-working-directory=/working-dir --runner-working-directory=/working-dir --sql-compiler-home=/dbsp/sql-to-dbsp-compiler --dbsp-override-path=/dbsp --db-connection-string=postgresql://postgres:postgres@localhost:5432 &) && \
494461
sleep 5 && \
495462
python3 python/test.py && \
496463
cd demo/demo_notebooks && jupyter execute fraud_detection.ipynb --JupyterApp.log_level='DEBUG'
@@ -503,7 +470,6 @@ test-rust:
503470
BUILD +test-dbsp --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
504471
BUILD +test-nexmark --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
505472
BUILD +test-adapters --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
506-
BUILD +test-dataflow-jit --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
507473
BUILD +test-manager --RUST_TOOLCHAIN=$RUST_TOOLCHAIN --RUST_BUILD_PROFILE=$RUST_BUILD_PROFILE
508474

509475
# TODO: the following two container tasks duplicate work that we otherwise do in the Dockerfile,
@@ -515,29 +481,20 @@ build-pipeline-manager-container:
515481
# First, copy over the artifacts built from previous stages
516482
RUN mkdir -p database-stream-processor/sql-to-dbsp-compiler/SQL-compiler/target
517483
COPY +build-manager/pipeline-manager .
518-
COPY +build-manager/pipeline .
519484
COPY +build-sql/sql2dbsp-jar-with-dependencies.jar database-stream-processor/sql-to-dbsp-compiler/SQL-compiler/target/
520485

521486
# Then copy over the crates needed by the sql compiler
522487
COPY crates/dbsp database-stream-processor/crates/dbsp
523488
COPY crates/pipeline-types database-stream-processor/crates/pipeline-types
524489
COPY crates/adapters database-stream-processor/crates/adapters
525-
COPY crates/dataflow-jit database-stream-processor/crates/dataflow-jit
526490
COPY README.md database-stream-processor/README.md
527491

528492
# Then copy over the required SQL compiler files
529493
COPY sql-to-dbsp-compiler/SQL-compiler/sql-to-dbsp /database-stream-processor/sql-to-dbsp-compiler/SQL-compiler/sql-to-dbsp
530494
COPY sql-to-dbsp-compiler/lib /database-stream-processor/sql-to-dbsp-compiler/lib
531495
COPY sql-to-dbsp-compiler/temp /database-stream-processor/sql-to-dbsp-compiler/temp
532496
RUN ./pipeline-manager --bind-address=0.0.0.0 --api-server-working-directory=/working-dir --compiler-working-directory=/working-dir --runner-working-directory=/working-dir --sql-compiler-home=/database-stream-processor/sql-to-dbsp-compiler --dbsp-override-path=/database-stream-processor --precompile
533-
ENTRYPOINT ["./pipeline-manager", \
534-
"--bind-address=0.0.0.0", \
535-
"--api-server-working-directory=/working-dir", \
536-
"--compiler-working-directory=/working-dir", \
537-
"--runner-working-directory=/working-dir", \
538-
"--sql-compiler-home=/database-stream-processor/sql-to-dbsp-compiler", \
539-
"--dbsp-override-path=/database-stream-processor", \
540-
"--jit-pipeline-runner-path=/pipeline"]
497+
ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--api-server-working-directory=/working-dir", "--compiler-working-directory=/working-dir", "--runner-working-directory=/working-dir", "--sql-compiler-home=/database-stream-processor/sql-to-dbsp-compiler", "--dbsp-override-path=/database-stream-processor"]
541498

542499
# TODO: mirrors the Dockerfile. See note above.
543500
build-demo-container:
@@ -626,13 +583,15 @@ integration-test-container:
626583
integration-tests:
627584
FROM earthly/dind:alpine
628585
COPY deploy/docker-compose.yml .
629-
COPY deploy/docker-compose-test.yml .
630586
COPY deploy/.env .
631587
ENV FELDERA_VERSION=latest
632588
WITH DOCKER --pull postgres \
633589
--load ghcr.io/feldera/pipeline-manager:latest=+build-pipeline-manager-container \
634-
--load test:latest=+integration-test-container
635-
RUN COMPOSE_HTTP_TIMEOUT=120 RUST_LOG=debug,tokio_postgres=info docker-compose --env-file .env -f docker-compose.yml -f docker-compose-test.yml up --force-recreate --exit-code-from test db pipeline-manager test
590+
--compose docker-compose.yml \
591+
--service db \
592+
--service pipeline-manager \
593+
--load itest:latest=+integration-test-container
594+
RUN sleep 5 && docker run --env-file .env --network default_default itest:latest
636595
END
637596

638597
benchmark:

‎crates/adapters/Cargo.toml‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ publish = false
1414
default = ["with-kafka"]
1515
with-kafka = ["rdkafka"]
1616
test-utils = ["size-of", "proptest", "proptest-derive"]
17-
with-jit = ["dataflow-jit"]
1817

1918
[dependencies]
2019
pipeline_types = { path = "../pipeline-types/" }
@@ -26,7 +25,6 @@ anyhow = { version = "1.0.57", features = ["backtrace"] }
2625
circular-queue = { version = "0.2.6", features = ["serde_support"] }
2726
crossbeam = "0.8.2"
2827
dbsp = { path = "../dbsp" }
29-
dataflow-jit = { path = "../dataflow-jit", optional = true}
3028
serde = { version = "1.0", features = ["derive", "rc"] }
3129
erased-serde = "0.3.23"
3230
once_cell = "1.9.0"
@@ -83,14 +81,3 @@ test_bin = "0.4.0"
8381
reqwest = { version = "0.11.20", features = ["blocking"] }
8482
serial_test = "2.0.0"
8583
rust_decimal_macros = "1.32"
86-
87-
[[bin]]
88-
name = "pipeline"
89-
path = "src/jit/pipeline.rs"
90-
required-features = ["with-kafka", "with-jit"]
91-
92-
[[test]]
93-
name = "jit_integration_test"
94-
path = "tests/jit_integration_test.rs"
95-
# These tests run the `pipeline` binary built above, which requires these features:
96-
required-features = ["with-kafka", "with-jit"]

‎crates/adapters/src/catalog.rs‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,7 @@ pub trait SerCursor {
179179
/// Serialize the `(key, weight)` tuple.
180180
///
181181
/// FIXME: This only exists to support the CSV serializer, which outputs
182-
/// key and weight in the same CSV record. This should be eliminated once
183-
/// we have a better idea about the common denominator between JIT and
184-
/// serde serializers.
182+
/// key and weight in the same CSV record.
185183
fn serialize_key_weight(&mut self, dst: &mut Vec<u8>) -> AnyResult<()>;
186184

187185
/// Serialize current value. Panics if invalid.
@@ -357,9 +355,6 @@ pub trait CircuitCatalog: Send {
357355
}
358356

359357
/// Circuit catalog implementation.
360-
// For now, we use a common `CircuitCatalog` implementation for JIT and
361-
// statically compiled circuits (with each module adding its own impls
362-
// to this type).
363358
pub struct Catalog {
364359
pub(crate) input_collection_handles: BTreeMap<String, Box<dyn DeCollectionHandle>>,
365360
pub(crate) output_batch_handles: BTreeMap<String, OutputCollectionHandles>,

‎crates/adapters/src/circuit_handle.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use crate::ControllerError;
22
use dbsp::DBSPHandle;
33
use std::path::PathBuf;
44

5-
/// Trait that captures common behavior of static and JIT-compiled circuits.
5+
/// Trait for DBSP circuit handle objects.
66
pub trait DbspCircuitHandle {
77
fn step(&mut self) -> Result<(), ControllerError>;
88

‎crates/adapters/src/controller/error.rs‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -375,10 +375,6 @@ pub enum ControllerError {
375375
/// Error evaluating the DBSP circuit.
376376
DbspError { error: DBSPError },
377377

378-
/// JIT compiler errors.
379-
// TODO: use actual JIT error type.
380-
JitError { error: String },
381-
382378
/// Error inside the Prometheus module.
383379
PrometheusError { error: String },
384380

@@ -474,7 +470,6 @@ impl DetailedError for ControllerError {
474470
Self::OutputTransportError { .. } => Cow::from("OutputTransportError"),
475471
Self::PrometheusError { .. } => Cow::from("PrometheusError"),
476472
Self::DbspError { error } => error.error_code(),
477-
Self::JitError { .. } => Cow::from("JitCompilerError"),
478473
Self::DbspPanic => Cow::from("DbspPanic"),
479474
Self::ControllerPanic => Cow::from("ControllerPanic"),
480475
}
@@ -552,9 +547,6 @@ impl Display for ControllerError {
552547
Self::DbspError { error } => {
553548
write!(f, "DBSP error: {error}")
554549
}
555-
Self::JitError { error } => {
556-
write!(f, "JIT compiler error: {error}")
557-
}
558550
Self::DbspPanic => {
559551
write!(f, "Panic inside the DBSP runtime")
560552
}
@@ -727,12 +719,6 @@ impl ControllerError {
727719
}
728720
}
729721

730-
pub fn jit_error(error: &str) -> Self {
731-
Self::JitError {
732-
error: error.to_string(),
733-
}
734-
}
735-
736722
pub fn dbsp_error(error: DBSPError) -> Self {
737723
Self::DbspError { error }
738724
}

‎crates/adapters/src/controller/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ impl Controller {
176176
)
177177
});
178178
// If `recv` fails, it indicates that the circuit thread panicked
179-
// during initialization, mostly likely in JIT compilation.
179+
// during initialization.
180180
init_status_receiver
181181
.recv()
182182
.map_err(|_| ControllerError::dbsp_panic())??;

‎crates/adapters/src/jit/catalog.rs‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎crates/adapters/src/jit/csv.rs‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

0 commit comments

Comments
 (0)