pipeline-manager: copy overriding Cargo.lock for each Rust compilation

snkas · snkas · commit e1068f843a79 · 2025-02-27T17:29:49.000Z
The `Cargo.lock` at the root of the repository is copied overriding to
`.feldera/compiler/rust-compilation/Cargo.lock` at the start of each
Rust compilation. This is such that the crate versions are generally the
same for both main repository and pipeline compilation, with the
exception of differences between that will be decided ad-hoc by the
latter each Rust compilation anew.

Signed-off-by: Simon Kassing &lt;simon.kassing@feldera.com&gt;
diff --git a/Earthfile b/Earthfile
@@ -324,9 +324,8 @@ build-pipeline-manager-container:
     COPY +build-manager/pipeline-manager .
     COPY +build-sql/sql2dbsp-jar-with-dependencies.jar database-stream-processor/sql-to-dbsp-compiler/SQL-compiler/target/
 
-    # Reuse `Cargo.lock` to ensure consistent crate versions.
-    RUN mkdir -p .feldera/compiler/rust-compilation
-    COPY --chown=feldera Cargo.lock .feldera/compiler/rust-compilation/Cargo.lock
+    # Copy over `Cargo.lock` into home directory (copied overriding at each pipeline Rust compilation)
+    COPY --chown=feldera Cargo.lock Cargo.lock
 
     # Copy over demos
     RUN mkdir -p demos
@@ -347,8 +346,8 @@ build-pipeline-manager-container:
     RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
     ENV PATH="$PATH:/home/feldera/.cargo/bin"
 
-    RUN ./pipeline-manager --bind-address=0.0.0.0 --sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler --compilation-profile=unoptimized --dbsp-override-path=/home/feldera/database-stream-processor --precompile
-    ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler", "--dbsp-override-path=/home/feldera/database-stream-processor", "--compilation-profile=unoptimized", "--demos-dir", "/home/feldera/demos"]
+    RUN ./pipeline-manager --bind-address=0.0.0.0 --sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler --compilation-cargo-lock-path=/home/feldera/Cargo.lock --compilation-profile=unoptimized --dbsp-override-path=/home/feldera/database-stream-processor --precompile
+    ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler", "--compilation-cargo-lock-path=/home/feldera/Cargo.lock", "--dbsp-override-path=/home/feldera/database-stream-processor", "--compilation-profile=unoptimized", "--demos-dir", "/home/feldera/demos"]
 
 # Same as the above, but with a permissive CORS setting, else playwright doesn't work
 pipeline-manager-container-cors-all:
diff --git a/crates/pipeline-manager/src/compiler/main.rs b/crates/pipeline-manager/src/compiler/main.rs
@@ -308,6 +308,7 @@ mod test {
             compiler_working_directory: existing_path.to_string_lossy().to_string(),
             compilation_profile: CompilationProfile::Optimized,
             sql_compiler_home: "".to_string(),
+            compilation_cargo_lock_path: "".to_string(),
             dbsp_override_path: "".to_string(),
             precompile: false,
             binary_ref_host: "".to_string(),
@@ -324,6 +325,7 @@ mod test {
             compiler_working_directory: non_existing_path.to_string_lossy().to_string(),
             compilation_profile: CompilationProfile::Optimized,
             sql_compiler_home: "".to_string(),
+            compilation_cargo_lock_path: "".to_string(),
             dbsp_override_path: "".to_string(),
             precompile: false,
             binary_ref_host: "".to_string(),
diff --git a/crates/pipeline-manager/src/compiler/rust_compiler.rs b/crates/pipeline-manager/src/compiler/rust_compiler.rs
@@ -309,6 +309,10 @@ fn calculate_source_checksum(
             "config.sql_compiler_home",
             config.sql_compiler_home.to_string().as_bytes(),
         ),
+        (
+            "config.compilation_cargo_lock_path",
+            config.compilation_cargo_lock_path.to_string().as_bytes(),
+        ),
         (
             "config.dbsp_override_path",
             config.dbsp_override_path.to_string().as_bytes(),
@@ -709,6 +713,14 @@ async fn prepare_workspace(
         .join("Cargo.toml");
     recreate_file_with_content(&cargo_toml_file_path, &cargo_toml).await?;
 
+    // Copy over the Cargo.lock such that it is the same for each compilation
+    let cargo_lock_source_path = Path::new(&config.compilation_cargo_lock_path);
+    let cargo_lock_target_path = config
+        .working_dir()
+        .join("rust-compilation")
+        .join("Cargo.lock");
+    copy_file(cargo_lock_source_path, &cargo_lock_target_path).await?;
+
     Ok(())
 }
 
@@ -1249,6 +1261,7 @@ mod test {
             compiler_working_directory: "".to_string(),
             compilation_profile: CompilationProfile::Dev,
             sql_compiler_home: "".to_string(),
+            compilation_cargo_lock_path: "".to_string(),
             dbsp_override_path: "".to_string(),
             precompile: false,
             binary_ref_host: "".to_string(),
@@ -1258,6 +1271,7 @@ mod test {
             compiler_working_directory: "a".to_string(),
             compilation_profile: CompilationProfile::Dev,
             sql_compiler_home: "".to_string(),
+            compilation_cargo_lock_path: "".to_string(),
             dbsp_override_path: "".to_string(),
             precompile: false,
             binary_ref_host: "".to_string(),
@@ -1267,6 +1281,7 @@ mod test {
             compiler_working_directory: "".to_string(),
             compilation_profile: CompilationProfile::Dev,
             sql_compiler_home: "b".to_string(),
+            compilation_cargo_lock_path: "".to_string(),
             dbsp_override_path: "".to_string(),
             precompile: false,
             binary_ref_host: "".to_string(),
@@ -1276,6 +1291,7 @@ mod test {
             compiler_working_directory: "".to_string(),
             compilation_profile: CompilationProfile::Dev,
             sql_compiler_home: "".to_string(),
+            compilation_cargo_lock_path: "".to_string(),
             dbsp_override_path: "c".to_string(),
             precompile: false,
             binary_ref_host: "".to_string(),
diff --git a/crates/pipeline-manager/src/compiler/test.rs b/crates/pipeline-manager/src/compiler/test.rs
@@ -38,6 +38,7 @@ impl CompilerTest {
         };
         let compiler_config = CompilerConfig {
             sql_compiler_home: "../../sql-to-dbsp-compiler".to_owned(),
+            compilation_cargo_lock_path: "../../Cargo.lock".to_owned(),
             dbsp_override_path: "not-used".to_owned(),
             compilation_profile: CompilationProfile::Optimized,
             precompile: false,
diff --git a/crates/pipeline-manager/src/config.rs b/crates/pipeline-manager/src/config.rs
@@ -106,6 +106,11 @@ fn default_sql_compiler_home() -> String {
     "sql-to-dbsp-compiler".to_string()
 }
 
+/// Default location of the Rust compilation `Cargo.lock`.
+fn default_compilation_cargo_lock_path() -> String {
+    "Cargo.lock".to_string()
+}
+
 /// The default Rust compilation profile.
 fn default_compilation_profile() -> CompilationProfile {
     CompilationProfile::Optimized
@@ -353,6 +358,12 @@ pub struct CompilerConfig {
     #[arg(long, default_value_t = default_sql_compiler_home())]
     pub sql_compiler_home: String,
 
+    /// Location of the `Cargo.lock` file which will be copied overriding
+    /// at each pipeline Rust compilation.
+    #[serde(default = "default_compilation_cargo_lock_path")]
+    #[arg(long, default_value_t = default_compilation_cargo_lock_path())]
+    pub compilation_cargo_lock_path: String,
+
     /// Override DBSP dependencies in generated Rust crates.
     ///
     /// By default, the Rust crates generated by the SQL compiler depend on local
@@ -396,6 +407,8 @@ impl CompilerConfig {
         help_create_dir(&self.compiler_working_directory)?;
         self.compiler_working_directory = help_canonicalize_dir(&self.compiler_working_directory)?;
         self.sql_compiler_home = help_canonicalize_dir(&self.sql_compiler_home)?;
+        self.compilation_cargo_lock_path =
+            help_canonicalize_dir(&self.compilation_cargo_lock_path)?;
         self.dbsp_override_path = help_canonicalize_dir(&self.dbsp_override_path)?;
         Ok(self)
     }
diff --git a/crates/pipeline-manager/src/integration_test.rs b/crates/pipeline-manager/src/integration_test.rs
@@ -118,6 +118,7 @@ async fn initialize_local_pipeline_manager_instance() -> TempDir {
     let compiler_config = CompilerConfig {
         compiler_working_directory: workdir.join("compiler").to_string_lossy().to_string(),
         sql_compiler_home: "../../sql-to-dbsp-compiler".to_owned(),
+        compilation_cargo_lock_path: "../../Cargo.lock".to_owned(),
         dbsp_override_path: "../../".to_owned(),
         compilation_profile: CompilationProfile::Unoptimized,
         precompile: true,
diff --git a/deploy/Dockerfile b/deploy/Dockerfile
@@ -111,9 +111,8 @@ COPY --from=builder /app/target/release/pipeline-manager pipeline-manager
 RUN mkdir -p lib/sql-to-dbsp-compiler/SQL-compiler/target
 COPY --from=javabuild /sql/sql-to-dbsp-compiler/SQL-compiler/target/sql2dbsp-jar-with-dependencies.jar lib/sql-to-dbsp-compiler/SQL-compiler/target/sql2dbsp-jar-with-dependencies.jar
 
-# Reuse `Cargo.lock` to ensure consistent crate versions.
-RUN mkdir -p .feldera/compiler/rust-compilation
-COPY --chown=feldera Cargo.lock .feldera/compiler/rust-compilation/Cargo.lock
+# Copy over `Cargo.lock` into home directory (copied overriding at each pipeline Rust compilation)
+COPY --chown=feldera Cargo.lock Cargo.lock
 
 # Copy over demos
 RUN mkdir -p demos
@@ -140,6 +139,6 @@ RUN arch=`dpkg --print-architecture | sed "s/arm64/aarch64/g" | sed "s/amd64/x86
 ENV PATH="$PATH:/home/feldera/.cargo/bin:/home/feldera/mold/bin"
 ENV RUSTFLAGS="-C link-arg=-fuse-ld=mold"
 # Run the precompile phase to speed up Rust compilations during deployment
-RUN ./pipeline-manager --bind-address=0.0.0.0 --sql-compiler-home=/home/feldera/lib/sql-to-dbsp-compiler --dbsp-override-path=/home/feldera/lib --precompile
+RUN ./pipeline-manager --bind-address=0.0.0.0 --sql-compiler-home=/home/feldera/lib/sql-to-dbsp-compiler --compilation-cargo-lock-path=/home/feldera/Cargo.lock --dbsp-override-path=/home/feldera/lib --precompile
 ENV BANNER_ADDR=localhost
-ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/feldera/lib/sql-to-dbsp-compiler", "--dbsp-override-path=/home/feldera/lib", "--demos-dir", "/home/feldera/demos"]
+ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/feldera/lib/sql-to-dbsp-compiler", "--compilation-cargo-lock-path=/home/feldera/Cargo.lock", "--dbsp-override-path=/home/feldera/lib", "--demos-dir", "/home/feldera/demos"]
