From 5188288d9095172bf76624f24852ee86a28d5d4c Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Sun, 5 Jul 2026 12:45:56 +0200 Subject: [PATCH 1/5] Implement and document thread-safe use of the iterate API 1. It exposes machinery to manually construct recursive circuits using the lower-level `iterate` API instead of the higher-level `recursive` API, that is, it publicly exports `Consensus` to make such circuits thread-safe. 2. The four sibling methods `iterate_with_condition`, `iterate_with_conditions`, `iterate_with_condition_and_scheduler`, and `iterate_with_conditions_and_scheduler` are now thread-safe by wrapping their condition(s) in a `Consensus`. 3. It updates `tutorial10.rs` and `tutorial11.rs` to show semantically equivalent circuits using the `iterate` API next to the `recursive` API and does a very basic runtime comparison among the two to demonstrate the extra `distinct` happening behind the scenes and its costs. 4. It contributes the new tutorial `tutorial13.rs` which does the same thing for mutually recursive circuits by reusing the graph coloring example from the documentation of `recursive_dynamic`. Signed-off-by: Leo Stewen --- .gitignore | 8 +- crates/dbsp/Cargo.toml | 4 + .../tutorial/transitive_closure/mod.rs | 191 ++++++++++ crates/dbsp/examples/tutorial/tutorial10.rs | 311 ++++++++++------ crates/dbsp/examples/tutorial/tutorial11.rs | 298 ++++++++++------ crates/dbsp/examples/tutorial/tutorial13.rs | 333 ++++++++++++++++++ crates/dbsp/src/circuit.rs | 4 +- crates/dbsp/src/circuit/circuit_builder.rs | 16 + crates/dbsp/src/circuit/runtime.rs | 9 +- crates/dbsp/src/lib.rs | 4 +- crates/dbsp/src/operator/condition.rs | 115 +++++- crates/dbsp/src/operator/recursive.rs | 4 +- crates/dbsp/src/tutorial.rs | 21 +- 13 files changed, 1071 insertions(+), 247 deletions(-) create mode 100644 crates/dbsp/examples/tutorial/transitive_closure/mod.rs create mode 100644 crates/dbsp/examples/tutorial/tutorial13.rs diff --git a/.gitignore b/.gitignore index 3dfb331f474..bec1df1d12a 100644 --- a/.gitignore +++ b/.gitignore @@ -101,4 +101,10 @@ playwright/.cache/ # Devcontainers - the lock file should be committed when more people use them .devcontainer/devcontainer-lock.json js-packages/support-bundle-triage/ -.roc-check/ \ No newline at end of file +.roc-check/ + +# Nix +flake.nix +flake.lock +.envrc +.direnv diff --git a/crates/dbsp/Cargo.toml b/crates/dbsp/Cargo.toml index 8af54c76821..a84ab21d337 100644 --- a/crates/dbsp/Cargo.toml +++ b/crates/dbsp/Cargo.toml @@ -221,6 +221,10 @@ path = "examples/tutorial/tutorial11.rs" name = "tutorial12" path = "examples/tutorial/tutorial12/tutorial12.rs" +[[example]] +name = "tutorial13" +path = "examples/tutorial/tutorial13.rs" + [[example]] name = "coord" path = "examples/dist/coord.rs" diff --git a/crates/dbsp/examples/tutorial/transitive_closure/mod.rs b/crates/dbsp/examples/tutorial/transitive_closure/mod.rs new file mode 100644 index 00000000000..0ba2d04811b --- /dev/null +++ b/crates/dbsp/examples/tutorial/transitive_closure/mod.rs @@ -0,0 +1,191 @@ +#![allow(unused)] +use anyhow::Result; +use dbsp::{ + Circuit, Consensus, DBSPHandle, IndexedZSetReader, NestedCircuit, OrdIndexedZSet, OutputHandle, + SchedulerError, Stream, ZSetHandle, ZWeight, indexed_zset, + typed_batch::SpineSnapshot, + utils::{Tup2, Tup3, Tup4}, +}; +use std::cell::Cell; + +pub type NodeId = u64; +pub type Weight = u64; +pub type CumWeight = u64; +pub type Hopcnt = u64; +pub type WeightedEdge = Tup3; +pub type Path = Tup2; +pub type TransClosurePath = Tup4; +pub type Accumulator = Stream>; + +pub struct TransClosureCircuit { + pub label: &'static str, + pub handle: DBSPHandle, + pub edges_input: ZSetHandle, + pub closure_output: OutputHandle>>, +} + +pub fn edges_data(steps: usize) -> impl Iterator>> { + vec![ + // The first step adds a graph of four nodes: + // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| + vec![ + Tup2(Tup3(0, 1, 1), 1), + Tup2(Tup3(1, 2, 1), 1), + Tup2(Tup3(2, 3, 2), 1), + Tup2(Tup3(3, 4, 2), 1), + ], + // The second step removes the edge |1| -1-> |2|. + vec![Tup2(Tup3(1, 2, 1), -1)], + // The third step reinserts the edge |1| -1-> |2| and introduces + // a cycle through the edge |4| -3-> |0|. In total, the graph now looks + // like this: + // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| + // ^ | + // | | + // ------------------3------------------ + vec![Tup2(Tup3(1, 2, 1), 1), Tup2(Tup3(4, 0, 3), 1)], + ] + .into_iter() + .take(steps) +} + +pub fn expected_closure( + steps: usize, +) -> impl Iterator> { + vec![ + // We expect the full transitive closure in the first step. + indexed_zset! { Path => TransClosurePath: + Tup2(0, 1) => { Tup4(0, 1, 1, 1) => 1 }, + Tup2(0, 2) => { Tup4(0, 2, 2, 2) => 1 }, + Tup2(0, 3) => { Tup4(0, 3, 4, 3) => 1 }, + Tup2(0, 4) => { Tup4(0, 4, 6, 4) => 1 }, + Tup2(1, 2) => { Tup4(1, 2, 1, 1) => 1 }, + Tup2(1, 3) => { Tup4(1, 3, 3, 2) => 1 }, + Tup2(1, 4) => { Tup4(1, 4, 5, 3) => 1 }, + Tup2(2, 3) => { Tup4(2, 3, 2, 1) => 1 }, + Tup2(2, 4) => { Tup4(2, 4, 4, 2) => 1 }, + Tup2(3, 4) => { Tup4(3, 4, 2, 1) => 1 }, + }, + // These paths are removed in the second step. + indexed_zset! { Path => TransClosurePath: + Tup2(0, 2) => { Tup4(0, 2, 2, 2) => -1 }, + Tup2(0, 3) => { Tup4(0, 3, 4, 3) => -1 }, + Tup2(0, 4) => { Tup4(0, 4, 6, 4) => -1 }, + Tup2(1, 2) => { Tup4(1, 2, 1, 1) => -1 }, + Tup2(1, 3) => { Tup4(1, 3, 3, 2) => -1 }, + Tup2(1, 4) => { Tup4(1, 4, 5, 3) => -1 }, + }, + // Third step restores the deletions of the second step and includes + // the newly discovered paths from the cycle. + indexed_zset! { Path => TransClosurePath: + Tup2(0, 2) => { Tup4(0, 2, 2, 2) => 1 }, + Tup2(0, 3) => { Tup4(0, 3, 4, 3) => 1 }, + Tup2(0, 4) => { Tup4(0, 4, 6, 4) => 1 }, + Tup2(1, 2) => { Tup4(1, 2, 1, 1) => 1 }, + Tup2(1, 3) => { Tup4(1, 3, 3, 2) => 1 }, + Tup2(1, 4) => { Tup4(1, 4, 5, 3) => 1 }, + + Tup2(0, 0) => { Tup4(0, 0, 9, 5) => 1 }, + Tup2(1, 0) => { Tup4(1, 0, 8, 4) => 1 }, + Tup2(1, 1) => { Tup4(1, 1, 9, 5) => 1 }, + Tup2(2, 0) => { Tup4(2, 0, 7, 3) => 1 }, + Tup2(2, 1) => { Tup4(2, 1, 8, 4) => 1 }, + Tup2(2, 2) => { Tup4(2, 2, 9, 5) => 1 }, + Tup2(3, 0) => { Tup4(3, 0, 5, 2) => 1 }, + Tup2(3, 1) => { Tup4(3, 1, 6, 3) => 1 }, + Tup2(3, 2) => { Tup4(3, 2, 7, 4) => 1 }, + Tup2(3, 3) => { Tup4(3, 3, 9, 5) => 1 }, + Tup2(4, 0) => { Tup4(4, 0, 3, 1) => 1 }, + Tup2(4, 1) => { Tup4(4, 1, 4, 2) => 1 }, + Tup2(4, 2) => { Tup4(4, 2, 5, 3) => 1 }, + Tup2(4, 3) => { Tup4(4, 3, 7, 4) => 1 }, + Tup2(4, 4) => { Tup4(4, 4, 9, 5) => 1 }, + }, + ] + .into_iter() + .take(steps) +} + +pub fn threads(force_threads: Option) -> usize { + let threads = force_threads.unwrap_or_else(|| { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) + }); + println!("Running with {threads} threads"); + threads +} + +pub fn drive_circuit(circuit: TransClosureCircuit, steps: usize) -> Result { + let label = circuit.label; + let mut handle = circuit.handle; + let edges_input = circuit.edges_input; + let closure_output = circuit.closure_output; + + let mut edges_data = edges_data(steps); + let mut expected_closure_output = expected_closure(steps); + + println!("=== {label} ==="); + + let start = std::time::Instant::now(); + + for i in 1..=steps { + println!("Iteration {} starts...", i); + edges_input.append(&mut edges_data.next().unwrap()); + handle.transaction()?; + let output = closure_output.concat(); + output.iter().for_each( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt), z_weight)| { + println!( + "{start} -> {end} (cum weight: {cum_weight}, hops: {hopcnt}) => {z_weight}" + ); + }, + ); + assert_eq!( + output.consolidate(), + expected_closure_output.next().unwrap() + ); + println!("Iteration {} finished.", i); + } + + Ok(start.elapsed().as_micros()) +} + +pub const MAX_ITERATIONS: usize = 4096; + +/// Hybrid termination for the `iterate`-based variants: stop once the subcircuit +/// reaches a fixed point, or after `max_iterations` steps if it never does. +/// +/// Testing the frontier for emptiness is not a reliable convergence signal with +/// multiple workers. The incremental join buffers results for future nested +/// timestamps, so the frontier delta can be transiently empty for a step while +/// data is still in flight, and the exact gaps vary with cross-worker +/// scheduling. `is_fixedpoint` accounts for that buffered state, so it never +/// stops early. It is a per-worker check, hence we combine the votes across all +/// workers with `Consensus`. +pub fn make_termination_check( + child_circuit: &NestedCircuit, + label: &'static str, + max_iterations: usize, +) -> impl AsyncFn() -> Result + use<> { + let child_circuit = child_circuit.clone(); + let consensus = Consensus::new(label); + let step = Cell::new(0usize); + + async move || { + let step_count = step.get() + 1; + step.set(step_count); + + // The step bound is data-independent, hence identical on every worker; + // the fixed-point check is per-worker. + let fixedpoint = child_circuit.is_fixedpoint(0); + let iteration_ceiling = step_count >= max_iterations; + let done = consensus.check(fixedpoint || iteration_ceiling).await?; + if done { + // Reset the step counter for the next transaction. + step.set(0); + } + + Ok(done) + } +} diff --git a/crates/dbsp/examples/tutorial/tutorial10.rs b/crates/dbsp/examples/tutorial/tutorial10.rs index 38090af4e62..aad939957c7 100644 --- a/crates/dbsp/examples/tutorial/tutorial10.rs +++ b/crates/dbsp/examples/tutorial/tutorial10.rs @@ -1,122 +1,215 @@ +mod transitive_closure; + use anyhow::Result; -use dbsp::typed_batch::IndexedZSetReader; use dbsp::{ - Circuit, OrdZSet, Runtime, Stream, - operator::Generator, - utils::{Tup3, Tup4}, - zset, zset_set, + Circuit, OrdIndexedZSet, Runtime, Stream, ZWeight, + operator::Z1, + trace::BatchReaderFactories, + utils::{Tup2, Tup3, Tup4}, +}; +use transitive_closure::{ + Accumulator, MAX_ITERATIONS, Path, TransClosureCircuit, TransClosurePath, WeightedEdge, + drive_circuit, make_termination_check, }; +/// If we set STEPS to 3 in this tutorial, the computation will not terminate +/// due to the cycle introduced in the third step in the input data. Tutorial 11 +/// lifts this restriction. We recommend to view tutorial 10 and 11 with your +/// favorite diff tool side-by-side to see the differences in both the +/// [`recursive_variant`] and [`iterative_variant`] of each tutorial. +const STEPS: usize = 2; + fn main() -> Result<()> { - // Set this value to 3 and uncomment the data in the arrays the Rust compiler - // points out, to see a fixed-point computation which does _not_ terminate. - const STEPS: usize = 2; - - let threads = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4); - - let (mut circuit_handle, output_handle) = Runtime::init_circuit( - threads, - move |root_circuit| { - let mut edges_data = ([ - // The first step adds a graph of four nodes: - // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| - zset_set! { Tup3(0_usize, 1_usize, 1_usize), Tup3(1, 2, 1), Tup3(2, 3, 2), Tup3(3, 4, 2) }, - // The second step removes the edge |1| -1-> |2|. - zset! { Tup3(1, 2, 1) => -1 }, - // The third step would introduce a cycle but that would - // cause the fixed-point computation to never terminate because we keep - // on finding new paths with a higher cumulative weight and hopcount. - // In total, we have the following graph: - // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| - // ^ | - // | | - // ------------------3------------------ - // zset_set! { Tup3(1,2,1), Tup3(4, 0, 3)} - ] as [OrdZSet>; STEPS]) - .into_iter(); - - let edges = root_circuit.add_source(Generator::new(move || edges_data.next().unwrap())); + // Play around with different thread setups on your machine. With this little + // toy data more than two threads usually mean more unproductive coordination + // overhead but the sweet spot may vary depending on your setup. You might + // want to run this in release mode via: + // `cargo run --profile release --example tutorial10` + let threads = transitive_closure::threads(Some(2)); + + // Uncomment to run with one round each to ignore for the measurements to warm up CPU caches. + // let _elapsed_rec = drive_circuit(recursive_variant(threads)?, STEPS)?; + let elapsed_rec = drive_circuit(recursive_variant(threads)?, STEPS)?; + // let _elapsed_iter = drive_circuit(iterative_variant(threads)?, STEPS)?; + let elapsed_iter = drive_circuit(iterative_variant(threads)?, STEPS)?; + + // As the recursive variant performs an unnecessary distinct, you should + // observe a speedup > 1 with the iterative variant. + println!("=== Stats ==="); + println!("Recursive variant: {elapsed_rec} μs"); + println!("Iterative variant: {elapsed_iter} μs"); + let speedup = elapsed_rec as f64 / elapsed_iter as f64; + println!("Speedup factor (recursive (old) relative to iterative (new)): {speedup:.02}"); + + Ok(()) +} + +/// Computes the transitive closure of a graph using DBSP's `recursive` API. The +/// resulting transitive closure consists of (FromNode, ToNode, CumulativeWeight, +/// Hopcount) tuples. +/// +/// # Note +/// +/// This variant only works on acyclic graphs, whereas tutorial 11 fixes this +/// at the expense of some extra aggregation step. +fn recursive_variant(threads: usize) -> Result { + let (handle, (edges_input, closure_output)) = + Runtime::init_circuit(threads, move |root_circuit| { + let (edges, edges_input) = root_circuit.add_input_zset::(); // Create a base stream with all paths of length 1. - let len_1 = edges.map(|Tup3(from, to, weight)| Tup4(*from, *to, *weight, 1)); - - let closure = root_circuit.recursive( - |child_circuit, len_n_minus_1: Stream<_, OrdZSet>>| { - // Import the `edges` and `len_1` stream from the parent circuit. - let edges = edges.delta0(child_circuit); - let len_1 = len_1.delta0(child_circuit); - - // Perform an iterative step (n-1 to n) through joining the - // paths of length n-1 with the edges. - let len_n = len_n_minus_1 - .map_index(|Tup4(start, end, cum_weight, hopcnt)| { - (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) - }) - .join( - &edges - .map_index(|Tup3(from, to, weight)| (*from, Tup3(*from, *to, *weight))), - |_end_from, - Tup4(start, _end, cum_weight, hopcnt), - Tup3(_from, to, weight)| { - Tup4(*start, *to, cum_weight + weight, hopcnt + 1) + let len_1 = edges.map_index(|Tup3(from, to, weight)| { + (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) + }); + + let closure = root_circuit.recursive(|child_circuit, closure: Accumulator| { + // Import the edges and len_1 streams from the parent circuit into + // the child circuit. A nested circuit has its own nested clock for + // each tick of the parent clock. For each _first tick_ of the + // _nested clock_ delta0 emits the data from the parent circuit + // and on subsequent ticks (of the nested clock) it is empty. + let edges = edges.delta0(child_circuit); + let len_1 = len_1.delta0(child_circuit); + + // Extend the current closure by joining the previously identified + // paths of length n-1 with the edges to also find paths of length n. + let closure = len_1.plus( + &closure + .map_index( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { + (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) }, ) - .plus(&len_1); - - Ok(len_n) - }, - )?; - - Ok(closure.output()) - }, - )?; - - let mut expected_outputs = ([ - // We expect the full transitive closure in the first step. - zset! { - Tup4(0, 1, 1, 1) => 1, - Tup4(0, 2, 2, 2) => 1, - Tup4(0, 3, 4, 3) => 1, - Tup4(0, 4, 6, 4) => 1, - Tup4(1, 2, 1, 1) => 1, - Tup4(1, 3, 3, 2) => 1, - Tup4(1, 4, 5, 3) => 1, - Tup4(2, 3, 2, 1) => 1, - Tup4(2, 4, 4, 2) => 1, - Tup4(3, 4, 2, 1) => 1, - }, - // These paths are removed in the second step. - zset! { - Tup4(0, 2, 2, 2) => -1, - Tup4(0, 3, 4, 3) => -1, - Tup4(0, 4, 6, 4) => -1, - Tup4(1, 2, 1, 1) => -1, - Tup4(1, 3, 3, 2) => -1, - Tup4(1, 4, 5, 3) => -1, - }, - // This does not matter, as the computation does not terminate - // anymore due to the cycle. - // zset! {}, - ] as [OrdZSet>; STEPS]) - .into_iter(); - - for i in 0..STEPS { - let iteration = i + 1; - println!("Iteration {} starts...", iteration); - circuit_handle.transaction()?; - let output = output_handle.consolidate(); - assert_eq!(output, expected_outputs.next().unwrap()); - output - .iter() - .for_each(|(Tup4(start, end, cum_weight, hopcnt), _, z_weight)| { - println!( - "{start} -> {end} (cum weight: {cum_weight}, hops: {hopcnt}) => {z_weight}" + .join_index( + &edges.map_index(|Tup3(from, to, weight)| { + (*from, Tup3(*from, *to, *weight)) + }), + |_end_from, + Tup4(start, _end, cum_weight, hopcnt), + Tup3(_from, to, weight)| { + Some(( + Tup2(*start, *to), + Tup4(*start, *to, cum_weight + weight, hopcnt + 1), + )) + }, + ), ); + // Note: Unlike in tutorial 11, there is no aggregation happening + // here, causing the circuit to rediscover paths which live + // on a cycle over and over again but with higher cumulative + // weights and hopcounts if operating on cyclic graphs. + // The implicit distinct of the `ChildCircuit::recursive` API does + // not fix the termination either, as the value domain carries the + // changing values (cumulative weight and hopcount). + + Ok(closure) + })?; + + Ok((edges_input, closure.accumulate_output())) + })?; + + Ok(TransClosureCircuit { + label: "Recursive Variant", + handle, + edges_input, + closure_output, + }) +} + +/// Computes the transitive closure of a graph using DBSP's lower level `iterate` +/// API. Again, the resulting transitive closure consists of (FromNode, ToNode, +/// CumulativeWeight, Hopcount) tuples. +/// +/// # Note +/// +/// This variant only works on acyclic graphs, whereas tutorial 11 fixes this +/// at the expense of some extra aggregation step. This iterative variant is faster +/// than [`recursive_variant`] because you don't have to pay for deduplication +/// (via distinct) in case you know your data is acyclic anyways. This is similar +/// to the difference between UNION vs UNION ALL in recursive CTEs in SQL, if that +/// is familiar to you. +fn iterative_variant(threads: usize) -> Result { + let (handle, (edges_input, closure_output)) = + Runtime::init_circuit(threads, move |root_circuit| { + let (edges, edges_input) = root_circuit.add_input_zset::(); + + let len_1 = edges.map_index(|Tup3(from, to, weight)| { + (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) }); - println!("Iteration {} finished.", iteration); - } - Ok(()) + let closure = root_circuit.iterate(|child_circuit| { + let edges = edges.delta0(child_circuit); + let len_1 = len_1.delta0(child_circuit); + + // The closure frontier carries just the frontier, that is, + // the delta from the previous iteration. + let (closure_frontier, closure_frontier_feedback) = child_circuit + .add_feedback(Z1::new(OrdIndexedZSet::::default())); + + let new_paths = len_1.plus( + &closure_frontier + .map_index( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { + (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) + }, + ) + .join_index( + &edges.map_index(|Tup3(from, to, weight)| { + (*from, Tup3(*from, *to, *weight)) + }), + |_end_from, + Tup4(start, _end, cum_weight, hopcnt), + Tup3(_from, to, weight)| { + Some(( + Tup2(*start, *to), + Tup4(*start, *to, cum_weight + weight, hopcnt + 1), + )) + }, + ), + ); + + // Hook up the Z1 operator with the current frontier (new_paths) + // such that it becomes the previuos frontier in the next iteration. + closure_frontier_feedback.connect(&new_paths); + + // Accumulate every frontier across the inner iterations and export + // the result to the parent. We use `integrate_trace` (from the dynamic + // API, mirroring the behavior of `ChildCircuit::recursive`) rather than + // `integrate`. This is because `integrate` holds its running sum in + // a `Z1`, whose conservative fixed-point check (input and output + // both empty) never holds once paths accumulate, which would keep + // the subcircuit from ever reaching a fixed point. The trace-append + // operators behind `integrate_trace` always report a fixed point, + // so convergence is detected as soon as the frontier drains. + let closure = Stream::dyn_integrate_trace( + &new_paths.inner(), + &BatchReaderFactories::new::(), + ); + + let termination_check = make_termination_check( + child_circuit, + "tutorial 10 (iterative)", + MAX_ITERATIONS, + ); + + Ok((termination_check, closure.export())) + })?; + + // Consolidate the exported trace `closure` into an OrdIndexedZSet. + let closure = Stream::dyn_consolidate( + &closure, + &BatchReaderFactories::new::(), + ) + .typed::>() + .accumulate_output(); + + Ok((edges_input, closure)) + })?; + + Ok(TransClosureCircuit { + label: "Iterative Variant", + handle, + edges_input, + closure_output, + }) } diff --git a/crates/dbsp/examples/tutorial/tutorial11.rs b/crates/dbsp/examples/tutorial/tutorial11.rs index 09580d4b1ea..acb537bbd65 100644 --- a/crates/dbsp/examples/tutorial/tutorial11.rs +++ b/crates/dbsp/examples/tutorial/tutorial11.rs @@ -1,134 +1,212 @@ +mod transitive_closure; + use anyhow::Result; -use dbsp::OrdZSet; -use dbsp::typed_batch::IndexedZSetReader; use dbsp::{ - Circuit, NestedCircuit, OrdIndexedZSet, Runtime, Stream, indexed_zset, - operator::{Generator, Min}, + Circuit, OrdIndexedZSet, Runtime, Stream, ZWeight, + operator::{Min, Z1}, + trace::BatchReaderFactories, utils::{Tup2, Tup3, Tup4}, - zset_set, +}; +use transitive_closure::{ + Accumulator, MAX_ITERATIONS, Path, TransClosureCircuit, TransClosurePath, WeightedEdge, + drive_circuit, make_termination_check, }; -type Accumulator = - Stream, Tup4>>; +// Now we allow for the cyclic data, too, by allowing to consume the third step. +const STEPS: usize = 3; fn main() -> Result<()> { - const STEPS: usize = 2; - - let threads = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4); - - let (mut circuit_handle, output_handle) = Runtime::init_circuit( - threads, - move |root_circuit| { - let mut edges_data = ([ - // The first step adds a graph of four nodes, just like before: - // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| - zset_set! { Tup3(0_usize, 1_usize, 1_usize), Tup3(1, 2, 1), Tup3(2, 3, 2), Tup3(3, 4, 2) }, - // The second step introduces a cycle. Due to the code changes below, - // the query does terminate though. In total, the graph now looks - // like this: - // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| - // ^ | - // | | - // ------------------3------------------ - zset_set! { Tup3(4, 0, 3)} - ] as [OrdZSet>; STEPS]) - .into_iter(); - - let edges = root_circuit.add_source(Generator::new(move || edges_data.next().unwrap())); + // Play around with different thread setups on your machine. With this little + // toy data more than two threads usually mean more unproductive coordination + // overhead but the sweet spot may vary depending on your setup. You might + // want to run this in release mode via: + // `cargo run --profile release --example tutorial11` + let threads = transitive_closure::threads(Some(2)); + + // Uncomment to run with one round each to ignore for the measurements to warm up CPU caches. + // let _elapsed_rec = drive_circuit(recursive_variant(threads)?, STEPS)?; + let elapsed_rec = drive_circuit(recursive_variant(threads)?, STEPS)?; + // let _elapsed_iter = drive_circuit(iterative_variant(threads)?, STEPS)?; + let elapsed_iter = drive_circuit(iterative_variant(threads)?, STEPS)?; + + // As the recursive variant performs a duplicated distinct, you should + // observe a (small) speedup > 1 with the iterative variant. + println!("=== Stats ==="); + println!("Recursive variant: {elapsed_rec} μs"); + println!("Iterative variant: {elapsed_iter} μs"); + let speedup = elapsed_rec as f64 / elapsed_iter as f64; + println!("Speedup factor (recursive (old) relative to iterative (new)): {speedup:.02}"); + + Ok(()) +} + +/// Computes the transitive closure of a graph using DBSP's `recursive` API. The +/// resulting transitive closure consists of (FromNode, ToNode, CumulativeWeight, +/// Hopcount) tuples. +/// +/// # Note +/// +/// While this works on both cyclic and acyclic graphs (unlike tutorial 10), +/// it deduplicates twice: First, implicitly due to the recursive API and, +/// second, due to the aggregation which is required for termination upon +/// cyclic graphs. The [`iterative_variant`] below fixes this and only relies +/// on the aggregation to ensure termination and is therefore a bit faster. +fn recursive_variant(threads: usize) -> Result { + let (handle, (edges_input, closure_output)) = + Runtime::init_circuit(threads, move |root_circuit| { + let (edges, edges_input) = root_circuit.add_input_zset::(); // Create a base stream with all paths of length 1. let len_1 = edges.map_index(|Tup3(from, to, weight)| { (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) }); - let closure = root_circuit.recursive(|child_circuit, len_n_minus_1: Accumulator| { - // Import the `edges` and `len_1` stream from the parent circuit. + let closure = root_circuit.recursive(|child_circuit, closure: Accumulator| { + // Import the edges and len_1 streams from the parent circuit into + // the child circuit. A nested circuit has its own nested clock for + // each tick of the parent clock. For each _first tick_ of the + // _nested clock_ delta0 emits the data from the parent circuit + // and on subsequent ticks (of the nested clock) it is empty. let edges = edges.delta0(child_circuit); let len_1 = len_1.delta0(child_circuit); - // Perform an iterative step (n-1 to n) through joining the - // paths of length n-1 with the edges. - let len_n = len_n_minus_1 - .map_index( - |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { - (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) - }, + // Extend the current closure by joining the previously identified + // paths of length n-1 with the edges to also find paths of length n. + let closure = len_1 + .plus( + &closure + .map_index( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { + (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) + }, + ) + .join_index( + &edges.map_index(|Tup3(from, to, weight)| { + (*from, Tup3(*from, *to, *weight)) + }), + |_end_from, + Tup4(start, _end, cum_weight, hopcnt), + Tup3(_from, to, weight)| { + Some(( + Tup2(*start, *to), + Tup4(*start, *to, cum_weight + weight, hopcnt + 1), + )) + }, + ), ) - .join_index( - &edges - .map_index(|Tup3(from, to, weight)| (*from, Tup3(*from, *to, *weight))), - |_end_from, - Tup4(start, _end, cum_weight, hopcnt), - Tup3(_from, to, weight)| { - Some(( - Tup2(*start, *to), - Tup4(*start, *to, cum_weight + weight, hopcnt + 1), - )) - }, - ) - .plus(&len_1) + // The distinct is hidden but still happening due to the way + // the `ChildCircuit::recursive` API works. .aggregate(Min); - Ok(len_n) + Ok(closure) })?; - Ok(closure.output()) - }, - )?; - - let mut expected_outputs = ([ - // The transitive closure in the first step remains the same as in - // `tutorial10.rs`. - indexed_zset! { Tup2 => Tup4: - Tup2(0, 1) => { Tup4(0, 1, 1, 1) => 1 }, - Tup2(0, 2) => { Tup4(0, 2, 2, 2) => 1 }, - Tup2(0, 3) => { Tup4(0, 3, 4, 3) => 1 }, - Tup2(0, 4) => { Tup4(0, 4, 6, 4) => 1 }, - Tup2(1, 2) => { Tup4(1, 2, 1, 1) => 1 }, - Tup2(1, 3) => { Tup4(1, 3, 3, 2) => 1 }, - Tup2(1, 4) => { Tup4(1, 4, 5, 3) => 1 }, - Tup2(2, 3) => { Tup4(2, 3, 2, 1) => 1 }, - Tup2(2, 4) => { Tup4(2, 4, 4, 2) => 1 }, - Tup2(3, 4) => { Tup4(3, 4, 2, 1) => 1 }, - }, - // The second step's introduction of a cycle yields these new paths. - indexed_zset! { Tup2 => Tup4: - Tup2(0, 0) => { Tup4(0, 0, 9, 5) => 1 }, - Tup2(1, 0) => { Tup4(1, 0, 8, 4) => 1 }, - Tup2(1, 1) => { Tup4(1, 1, 9, 5) => 1 }, - Tup2(2, 0) => { Tup4(2, 0, 7, 3) => 1 }, - Tup2(2, 1) => { Tup4(2, 1, 8, 4) => 1 }, - Tup2(2, 2) => { Tup4(2, 2, 9, 5) => 1 }, - Tup2(3, 0) => { Tup4(3, 0, 5, 2) => 1 }, - Tup2(3, 1) => { Tup4(3, 1, 6, 3) => 1 }, - Tup2(3, 2) => { Tup4(3, 2, 7, 4) => 1 }, - Tup2(3, 3) => { Tup4(3, 3, 9, 5) => 1 }, - Tup2(4, 0) => { Tup4(4, 0, 3, 1) => 1 }, - Tup2(4, 1) => { Tup4(4, 1, 4, 2) => 1 }, - Tup2(4, 2) => { Tup4(4, 2, 5, 3) => 1 }, - Tup2(4, 3) => { Tup4(4, 3, 7, 4) => 1 }, - Tup2(4, 4) => { Tup4(4, 4, 9, 5) => 1 }, - }, - ] as [_; STEPS]) - .into_iter(); - - for i in 0..STEPS { - let iteration = i + 1; - println!("Iteration {} starts...", iteration); - circuit_handle.transaction()?; - let output = output_handle.consolidate(); - assert_eq!(output, expected_outputs.next().unwrap()); - output.iter().for_each( - |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt), z_weight)| { - println!( - "{start} -> {end} (cum weight: {cum_weight}, hops: {hopcnt}) => {z_weight}" + Ok((edges_input, closure.accumulate_output())) + })?; + + Ok(TransClosureCircuit { + label: "Recursive Variant", + handle, + edges_input, + closure_output, + }) +} + +/// Computes the transitive closure of a graph using DBSP's lower level `iterate` +/// API. Again, the resulting transitive closure consists of (FromNode, ToNode, +/// CumulativeWeight, Hopcount) tuples. +/// +/// # Note +/// +/// This variant also works on cyclic graphs. The iterative variant is faster +/// than the [`recursive_variant`] because you don't have to pay for deduplication +/// (via distinct) despite already being accounted for by the min-aggregation. +fn iterative_variant(threads: usize) -> Result { + let (handle, (edges_input, closure_output)) = + Runtime::init_circuit(threads, move |root_circuit| { + let (edges, edges_input) = root_circuit.add_input_zset::(); + + let len_1 = edges.map_index(|Tup3(from, to, weight)| { + (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) + }); + + let closure = root_circuit.iterate(|child_circuit| { + let edges = edges.delta0(child_circuit); + let len_1 = len_1.delta0(child_circuit); + + // The closure frontier carries just the frontier, that is, + // the delta from the previous iteration. + let (closure_frontier, closure_frontier_feedback) = child_circuit + .add_feedback(Z1::new(OrdIndexedZSet::::default())); + + let new_paths = len_1 + .plus( + &closure_frontier + .map_index( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { + (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) + }, + ) + .join_index( + &edges.map_index(|Tup3(from, to, weight)| { + (*from, Tup3(*from, *to, *weight)) + }), + |_end_from, + Tup4(start, _end, cum_weight, hopcnt), + Tup3(_from, to, weight)| { + Some(( + Tup2(*start, *to), + Tup4(*start, *to, cum_weight + weight, hopcnt + 1), + )) + }, + ), + ) + // No distinct here (and not even behind the scenes), + // just aggregation with Min to stop iteration. + .aggregate(Min); + + // Hook up the Z1 operator with the current frontier (new_paths) + // such that it becomes the previous frontier in the next iteration. + closure_frontier_feedback.connect(&new_paths); + + // Accumulate every frontier across the inner iterations and export + // the result to the parent. We use `integrate_trace` (from the dynamic + // API, mirroring the behavior of `ChildCircuit::recursive`) rather than + // `integrate`. This is because `integrate` holds its running sum in + // a `Z1`, whose conservative fixed-point check (input and output + // both empty) never holds once paths accumulate, which would keep + // the subcircuit from ever reaching a fixed point. The trace-append + // operators behind `integrate_trace` always report a fixed point, + // so convergence is detected as soon as the frontier drains. + let closure = Stream::dyn_integrate_trace( + &new_paths.inner(), + &BatchReaderFactories::new::(), ); - }, - ); - println!("Iteration {} finished.", iteration); - } - Ok(()) + let termination_check = make_termination_check( + child_circuit, + "tutorial 11 (iterative)", + MAX_ITERATIONS, + ); + + Ok((termination_check, closure.export())) + })?; + + // Consolidate the exported trace `closure` into an OrdIndexedZSet. + let closure = Stream::dyn_consolidate( + &closure, + &BatchReaderFactories::new::(), + ) + .typed::>() + .accumulate_output(); + + Ok((edges_input, closure)) + })?; + + Ok(TransClosureCircuit { + label: "Iterative Variant", + handle, + edges_input, + closure_output, + }) } diff --git a/crates/dbsp/examples/tutorial/tutorial13.rs b/crates/dbsp/examples/tutorial/tutorial13.rs new file mode 100644 index 00000000000..4bd9c8dd42b --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial13.rs @@ -0,0 +1,333 @@ +mod transitive_closure; + +use anyhow::Result; +use dbsp::{ + Circuit, DBSPHandle, NestedCircuit, OrdIndexedZSet, OrdZSet, OutputHandle, Runtime, Stream, + ZSetHandle, ZWeight, + operator::Z1, + trace::BatchReaderFactories, + typed_batch::{IndexedZSetReader, SpineSnapshot}, + utils::Tup2, + zset, +}; +use transitive_closure::{MAX_ITERATIONS, make_termination_check, threads}; + +fn main() -> Result<()> { + // Play around with different thread setups on your machine. With this little + // toy data more than two threads usually mean more unproductive coordination + // overhead but the sweet spot may vary depending on your setup. You might + // want to run this in release mode via: + // `cargo run --profile release --example tutorial13` + let threads = threads(Some(2)); + + // Uncomment to run with one round each to ignore for the measurements to warm up CPU caches. + // let _elapsed_rec = drive_circuit(recursive_variant(threads)?)?; + let elapsed_rec = drive_circuit(recursive_variant(threads)?)?; + // let _elapsed_iter = drive_circuit(iterative_variant(threads)?)?; + let elapsed_iter = drive_circuit(iterative_variant(threads)?)?; + + // As the work performed by both variants in this tutorial is roughly the + // same, you should observe comparable numbers. + println!("=== Stats ==="); + println!("Recursive variant: {elapsed_rec} μs"); + println!("Iterative variant: {elapsed_iter} μs"); + let speedup = elapsed_rec as f64 / elapsed_iter as f64; + println!("Speedup factor (recursive (old) relative to iterative (new)): {speedup:.02}"); + + Ok(()) +} + +type Node = usize; +type Edge = Tup2; + +struct BipartiteGraphCircuit { + label: &'static str, + handle: DBSPHandle, + init_input: ZSetHandle, + edges_input: ZSetHandle, + red_output: OutputHandle>>, + blue_output: OutputHandle>>, +} + +const STEPS: usize = 4; + +fn init_data() -> impl Iterator>> { + // We start the discovery by marking node `|0|` red. + let data = [vec![Tup2(0, 1)], vec![], vec![], vec![]]; + data.into_iter() +} + +fn edges_data() -> impl Iterator>> { + let data = [ + // The first step adds a graph of four nodes: + // |0| --> |1| --> |2| --> |3| --> |4| + vec![ + Tup2(Tup2(0, 1), 1), + Tup2(Tup2(1, 2), 1), + Tup2(Tup2(2, 3), 1), + Tup2(Tup2(3, 4), 1), + ], + // Now, we have the following graph in total: + // |0| --> |1| --> |2| --> |3| --> |4| + // ^ | + // | | + // ------ |5| <----- + vec![Tup2(Tup2(2, 5), 1), Tup2(Tup2(5, 0), 1)], + // And we introduce an odd-length cycle, rendering the graph + // non-bipartite anymore (all nodes are red _and_ blue): + // |0| --> |1| --> |2| --> |3| --> |4| + // ^ | | + // | | | + // ------ |5| <----- | + // | | + // --------------------------------- + vec![Tup2(Tup2(4, 0), 1)], + vec![Tup2(Tup2(0, 1), -1)], + ]; + data.into_iter() +} + +fn expected_red() -> impl Iterator> { + let data = [ + zset! { + 0 => 1, + 2 => 1, + 4 => 1, + }, + zset! {}, + zset! { + 1 => 1, + 3 => 1, + 5 => 1, + }, + zset! { + 1 => -1, + 2 => -1, + 3 => -1, + 4 => -1, + 5 => -1, + }, + ]; + data.into_iter() +} + +fn expected_blue() -> impl Iterator> { + let data = [ + zset! { + 1 => 1, + 3 => 1, + }, + zset! { + 5 => 1, + }, + zset! { + 0 => 1, + 2 => 1, + 4 => 1, + }, + zset! { + 0 => -1, + 1 => -1, + 2 => -1, + 3 => -1, + 4 => -1, + 5 => -1, + }, + ]; + data.into_iter() +} + +fn drive_circuit(circuit: BipartiteGraphCircuit) -> Result { + let label = circuit.label; + let mut handle = circuit.handle; + let init_input = circuit.init_input; + let edges_input = circuit.edges_input; + let red_output = circuit.red_output; + let blue_output = circuit.blue_output; + + let mut init_data = init_data(); + let mut edges_data = edges_data(); + let mut expected_blue_output = expected_blue(); + let mut expected_red_output = expected_red(); + + println!("=== {label} ==="); + + let start = std::time::Instant::now(); + + for i in 1..=STEPS { + println!("Iteration {i} starts..."); + init_input.append(&mut init_data.next().unwrap()); + edges_input.append(&mut edges_data.next().unwrap()); + handle.transaction().unwrap(); + let red_output = red_output.concat(); + red_output + .iter() + .for_each(|(node, _, zweight)| println!("Red {node} => {zweight}")); + let blue_output = blue_output.concat(); + blue_output + .iter() + .for_each(|(node, _, zweight)| println!("Blue {node} => {zweight}")); + assert_eq!( + red_output.consolidate(), + expected_red_output.next().unwrap() + ); + assert_eq!( + blue_output.consolidate(), + expected_blue_output.next().unwrap() + ); + println!("Iteration {i} finished."); + } + + Ok(start.elapsed().as_micros()) +} + +/// We compute the graph-coloring (by painting nodes red and blue) which is +/// one way to figure out if a graph is bipartite. The computation is +/// mutually recursive and happens to use the +/// [`ChildCircuit::recursive_dynamic`](`dbsp::circuit::ChildCircuit::recursive_dynamic`) +/// API but if you know your recursive streams at compile time you could also use +/// [`ChildCircuit::recursive`](`dbsp::circuit::ChildCircuit::recursive`) but +/// with a tuple carrying the recursive red and blue streams. +fn recursive_variant(threads: usize) -> Result { + let (handle, ((init_input, edges_input), (red_output, blue_output))) = + Runtime::init_circuit(threads, move |root_circuit| { + let (edges, edges_input) = root_circuit.add_input_zset::(); + let (init, init_input) = root_circuit.add_input_zset::(); + + let recursive_streams = root_circuit.recursive_dynamic( + 2, + |child_circuit, + mut recursive_streams: Vec>>| { + let edges = edges.delta0(child_circuit); + let init = init.delta0(child_circuit); + + let red = &recursive_streams[0]; + let blue = &recursive_streams[1]; + + let new_red = blue + .map_index(|blue_node| (*blue_node, *blue_node)) + .join( + &edges.map_index(|Tup2(from, to)| (*from, *to)), + |_blue_node, _, new_red_node| *new_red_node, + ) + .plus(&init); + + let new_blue = red.map_index(|red_node| (*red_node, *red_node)).join( + &edges.map_index(|Tup2(from, to)| (*from, *to)), + |_red_node, _, new_blue_node| *new_blue_node, + ); + + recursive_streams[0] = new_red; + recursive_streams[1] = new_blue; + Ok(recursive_streams) + }, + )?; + + let red_output = recursive_streams[0].accumulate_output(); + let blue_output = recursive_streams[1].accumulate_output(); + + Ok(((init_input, edges_input), (red_output, blue_output))) + })?; + + Ok(BipartiteGraphCircuit { + label: "Recursive Variant", + handle, + init_input, + edges_input, + red_output, + blue_output, + }) +} + +/// This iterative variant is a hand-rolled expansion of [`recursive_variant`], +/// built on the lower-level [`Circuit::iterate`] primitive. DBSP's recursive API +/// splices a `distinct` onto every output stream for us (see +/// [`ChildCircuit::recursive`](`dbsp::circuit::ChildCircuit::recursive`)); +/// with `iterate` we are responsible for deduplication ourselves. +fn iterative_variant(threads: usize) -> Result { + let (handle, (init_input, edges_input, red_output, blue_output)) = + Runtime::init_circuit(threads, move |root_circuit| { + let (init, init_input) = root_circuit.add_input_zset::(); + let (edges, edges_input) = root_circuit.add_input_zset::(); + + let recursive_streams = root_circuit.iterate(|child_circuit| { + let edges = edges.delta0(child_circuit); + let init = init.delta0(child_circuit); + let edges_indexed = edges.map_index(|Tup2(from, to)| (*from, *to)); + + // Feedback carries only the frontier (the delta from the last step). + let (red, red_feedback) = + child_circuit.add_feedback(Z1::new(OrdZSet::::default())); + let (blue, blue_feedback) = + child_circuit.add_feedback(Z1::new(OrdZSet::::default())); + + /// Convert a stream of Nodes into the indexed form the join expects and + /// extend it by one hop along `edges`. Deduplication is left to the caller, + /// because it must wrap the whole per-step expression (base case included) to + /// produce the correct cross-transaction delta. + fn hop( + frontier: &Stream>, + edges: &Stream>, + ) -> Stream> { + frontier + .map_index(|node| (*node, *node)) + .join(edges, |_from, _node, to| *to) + } + + // Extend each frontier by one hop. At clock tick 0 the frontier is + // empty, so `new_red` is just the base case `init`; from clock tick 1 + // on `init` is empty, so it is purely `frontier ⋈ edges`. + // `distinct` caps the weight of every node at 1, which both + // removes within-step duplicates and stops nodes discovered in + // an earlier step (or transaction) from being re-emitted. It + // must wrap the entire expression, base case included. + let new_red = init.plus(&hop(&blue, &edges_indexed)).distinct(); + let new_blue = hop(&red, &edges_indexed).distinct(); + + red_feedback.connect(&new_red); + blue_feedback.connect(&new_blue); + + // Accumulate every frontier across the nested clock cycles into the + // full colorings and export them to the parent. + let red = Stream::dyn_integrate_trace( + &new_red.inner(), + &BatchReaderFactories::new::(), + ); + let blue = Stream::dyn_integrate_trace( + &new_blue.inner(), + &BatchReaderFactories::new::(), + ); + + let termination_check = make_termination_check( + child_circuit, + "tutorial 13 (iterative)", + MAX_ITERATIONS, + ); + + Ok((termination_check, vec![red.export(), blue.export()])) + })?; + + // Consolidate each exported trace (red and blue) into a single ZSet. + let mut recursive_streams = recursive_streams + .into_iter() + .map(|s| { + Stream::dyn_consolidate(&s, &BatchReaderFactories::new::()) + .typed::>() + .accumulate_output() + }) + .collect::>(); + let blue_output = recursive_streams.pop().unwrap(); + let red_output = recursive_streams.pop().unwrap(); + + Ok((init_input, edges_input, red_output, blue_output)) + })?; + + Ok(BipartiteGraphCircuit { + label: "Iterative Variant", + handle, + init_input, + edges_input, + red_output, + blue_output, + }) +} diff --git a/crates/dbsp/src/circuit.rs b/crates/dbsp/src/circuit.rs index dc5c53ac0a0..0abde9ad185 100644 --- a/crates/dbsp/src/circuit.rs +++ b/crates/dbsp/src/circuit.rs @@ -47,8 +47,8 @@ pub use dbsp_handle::{ splitter_output_first_chunk_size, }; pub use runtime::{ - Error as RuntimeError, LocalStore, LocalStoreMarker, Runtime, RuntimeHandle, WeakRuntime, - WorkerLocation, WorkerLocations, + Consensus, Error as RuntimeError, LocalStore, LocalStoreMarker, Runtime, RuntimeHandle, + WeakRuntime, WorkerLocation, WorkerLocations, }; pub use schedule::Error as SchedulerError; diff --git a/crates/dbsp/src/circuit/circuit_builder.rs b/crates/dbsp/src/circuit/circuit_builder.rs index 8129d910204..b17ea080051 100644 --- a/crates/dbsp/src/circuit/circuit_builder.rs +++ b/crates/dbsp/src/circuit/circuit_builder.rs @@ -2762,6 +2762,22 @@ pub trait Circuit: CircuitBase + Clone + WithClock { F: FnOnce(&mut IterativeCircuit) -> Result, S: Scheduler + 'static; + /// Returns `true` if every operator in scope `scope` has reached a fixed + /// point, i.e., a state in which its output stays constant as long as its + /// input does. + /// + /// This is the same condition [`fixedpoint`](Self::fixedpoint) uses to stop + /// iterating. Unlike an emptiness test on an output stream, it accounts for + /// state still buffered inside operators (for example, results a join has + /// precomputed for future nested timestamps). Custom iteration built on + /// [`iterate`](Self::iterate) can call this from its termination check to + /// detect convergence, typically combining the per-worker results across + /// workers via [`Consensus`]. See the documentation of [`Scope`] for the + /// meaning of the parameter but passing `0` refers to the current circuit. + fn is_fixedpoint(&self, scope: Scope) -> bool { + self.check_fixedpoint(scope) + } + /// Make the contents of `parent_stream` available in the nested circuit /// via an [`ImportOperator`]. /// diff --git a/crates/dbsp/src/circuit/runtime.rs b/crates/dbsp/src/circuit/runtime.rs index aa520c315b4..3b86700c705 100644 --- a/crates/dbsp/src/circuit/runtime.rs +++ b/crates/dbsp/src/circuit/runtime.rs @@ -1356,7 +1356,14 @@ impl Runtime { /// A synchronization primitive that allows multiple threads within a runtime to agree /// when a condition is satisfied. -pub(crate) struct Consensus(Broadcast); +/// +/// Each worker submits a local vote via [`check`](Self::check); the call +/// returns `true` only when every worker in the runtime votes `true`. It is +/// the building block for combining per-worker termination decisions (for +/// example, per-worker fixed-point status) into a single, runtime-wide +/// decision when driving a nested circuit built on +/// [`Circuit::iterate`](super::circuit_builder::Circuit::iterate). +pub struct Consensus(Broadcast); impl Consensus { pub fn new(name: impl Display) -> Self { diff --git a/crates/dbsp/src/lib.rs b/crates/dbsp/src/lib.rs index c541fe65bc3..24d1bc06411 100644 --- a/crates/dbsp/src/lib.rs +++ b/crates/dbsp/src/lib.rs @@ -100,8 +100,8 @@ pub use crate::time::Timestamp; pub use algebra::{DynZWeight, ZWeight}; pub use circuit::{ - ChildCircuit, Circuit, CircuitHandle, DBSPHandle, NestedCircuit, RootCircuit, Runtime, - RuntimeError, SchedulerError, Stream, WeakRuntime, + ChildCircuit, Circuit, CircuitHandle, Consensus, DBSPHandle, NestedCircuit, RootCircuit, + Runtime, RuntimeError, SchedulerError, Stream, WeakRuntime, }; #[cfg(not(feature = "backend-mode"))] pub use operator::FilterMap; diff --git a/crates/dbsp/src/operator/condition.rs b/crates/dbsp/src/operator/condition.rs index 90495158504..1232fb264a2 100644 --- a/crates/dbsp/src/operator/condition.rs +++ b/crates/dbsp/src/operator/condition.rs @@ -6,10 +6,11 @@ use crate::{ circuit::{ ChildCircuit, Circuit, Stream, circuit_builder::IterativeCircuit, + runtime::Consensus, schedule::{Error as SchedulerError, Scheduler}, }, }; -use std::{cell::Cell, marker::PhantomData, rc::Rc}; +use std::{cell::Cell, fmt::Display, marker::PhantomData, rc::Rc}; impl Stream where @@ -60,7 +61,8 @@ where { self.iterate(|child| { let (condition, res) = constructor(child)?; - Ok((async move || Ok(condition.check()), res)) + let check = with_consensus("iterate with condition", move || condition.check()); + Ok((check, res)) }) } @@ -78,7 +80,10 @@ where { self.iterate_with_scheduler::<_, _, _, S>(|child| { let (condition, res) = constructor(child)?; - Ok((async move || Ok(condition.check()), res)) + let check = with_consensus("iterate with condition and scheduler", move || { + condition.check() + }); + Ok((check, res)) }) } @@ -96,10 +101,10 @@ where { self.iterate(|child| { let (conditions, res) = constructor(child)?; - Ok(( - async move || Ok(conditions.iter().all(Condition::check)), - res, - )) + let check = with_consensus("iterate with conditions", move || { + conditions.iter().all(Condition::check) + }); + Ok((check, res)) }) } @@ -117,14 +122,31 @@ where { self.iterate_with_scheduler::<_, _, _, S>(|child| { let (conditions, res) = constructor(child)?; - Ok(( - async move || Ok(conditions.iter().all(Condition::check)), - res, - )) + let check = with_consensus("iterate with conditions and scheduler", move || { + conditions.iter().all(Condition::check) + }); + Ok((check, res)) }) } } +/// Wrap a per-worker termination decision in a runtime-wide [`Consensus`] so +/// that a nested loop only stops once _every_ worker agrees. +/// +/// A worker whose share of the data converges first must keep iterating until +/// its peers catch up; otherwise it would leave the loop while they still have +/// work to do. The [`Consensus`] object is created once, here, and reused +/// across every clock cycle, mirroring +/// [`Circuit::fixedpoint`](crate::Circuit::fixedpoint). +#[inline] +fn with_consensus( + name: impl Display, + decide: impl Fn() -> bool + 'static, +) -> impl AsyncFn() -> Result { + let consensus = Consensus::new(name); + async move || consensus.check(decide()).await +} + /// A condition attached to a stream that can be used /// to terminate the execution of a subcircuit /// (see [`ChildCircuit::iterate_with_condition`] and @@ -152,7 +174,7 @@ impl Condition { #[cfg(test)] mod test { use crate::{ - Circuit, RootCircuit, Stream, + Circuit, RootCircuit, Runtime, Stream, circuit::{ circuit_builder::IterativeCircuit, schedule::{DynamicScheduler, Scheduler}, @@ -258,4 +280,73 @@ mod test { circuit.transaction().unwrap(); } } + + /// Regression test for the cross-worker [`Consensus`](crate::Consensus) in + /// [`super::ChildCircuit::iterate_with_condition`]. + /// + /// With more than one worker each worker holds only a shard of the data and + /// reaches its local termination condition at a different nested clock tick. + /// Without consensus a worker could leave the loop while its peers still had + /// work to do, producing a wrong or nondeterministic result. We therefore + /// require that a multi-worker run computes the same reachable set as a + /// single-worker run, across repeated builds to shake out scheduling races. + #[test] + fn iterate_with_condition_multi_worker() { + // Nodes reachable from {1, 2, 3} in the graph below. + let reachable_from_init = |workers: usize| { + let (mut circuit, (edges_input, init_input, reachable_output)) = + Runtime::init_circuit(workers, |root| { + let (edges, edges_input) = root.add_input_zset::>(); + let (init, init_input) = root.add_input_zset::(); + + let reachable = root.iterate_with_condition(|child| { + let edges = edges.delta0(child).integrate(); + let init = init.delta0(child).integrate(); + + let edges_indexed: Stream<_, OrdIndexedZSet> = + edges.map_index(|Tup2(k, v)| (*k, *v)); + + let feedback = >>::new(child); + let feedback_pairs: Stream<_, OrdZSet<(u64, ())>> = + feedback.stream().map(|&node| (node, ())); + let feedback_indexed: Stream<_, OrdIndexedZSet> = + feedback_pairs.map_index(|(k, v)| (*k, *v)); + + let suc = + feedback_indexed.stream_join(&edges_indexed, |_node, &(), &to| to); + let reachable = init.plus(&suc).stream_distinct(); + feedback.connect(&reachable); + + let condition = reachable.differentiate().condition(|z| z.is_empty()); + Ok((condition, reachable.export())) + })?; + + Ok((edges_input, init_input, reachable.output())) + }) + .unwrap(); + + edges_input.append(&mut vec![ + Tup2(Tup2(0, 3), 1), + Tup2(Tup2(1, 2), 1), + Tup2(Tup2(2, 1), 1), + Tup2(Tup2(3, 1), 1), + Tup2(Tup2(3, 4), 1), + Tup2(Tup2(4, 5), 1), + Tup2(Tup2(4, 6), 1), + Tup2(Tup2(5, 6), 1), + Tup2(Tup2(5, 1), 1), + ]); + init_input.append(&mut vec![Tup2(1, 1), Tup2(2, 1), Tup2(3, 1)]); + circuit.transaction().unwrap(); + reachable_output.consolidate() + }; + + let expected = zset! { 1 => 1, 2 => 1, 3 => 1, 4 => 1, 5 => 1, 6 => 1 }; + // Single worker run. + assert_eq!(reachable_from_init(1), expected); + // Repeat the multi-worker run to give scheduling races a chance to surface. + for _ in 0..5 { + assert_eq!(reachable_from_init(4), expected); + } + } } diff --git a/crates/dbsp/src/operator/recursive.rs b/crates/dbsp/src/operator/recursive.rs index 1538a738682..b416e823577 100644 --- a/crates/dbsp/src/operator/recursive.rs +++ b/crates/dbsp/src/operator/recursive.rs @@ -267,7 +267,9 @@ where /// no node is both red and blue the graph happens to be bipartite. In the /// first two computation steps the graph is bipartite but the added edge /// in the third step adds an odd-length cycle which destroys the bipartite - /// property and all nodes are colored red and blue. + /// property and all nodes are colored red and blue. See also `tutorial13.rs` + /// for a variant of this computation using the lower-level [`Circuit::iterate`] + /// API. /// /// ``` /// use dbsp::{ diff --git a/crates/dbsp/src/tutorial.rs b/crates/dbsp/src/tutorial.rs index a4caafd4f24..89ca47b0837 100644 --- a/crates/dbsp/src/tutorial.rs +++ b/crates/dbsp/src/tutorial.rs @@ -2021,8 +2021,8 @@ //! We have two execution contexts: a [root circuit](`RootCircuit`) //! and a [child circuit](`crate::NestedCircuit`). //! The root circuit is the one that is built by the parameter to the -//! [`Runtime::init_circuit`] function. The child circuit is defined by the parameter to -//! the [`ChildCircuit<()>::recursive`](`crate::ChildCircuit::recursive`) function. +//! [`Runtime::init_circuit`] function. The child circuit is defined by the +//! parameter to the [`ChildCircuit::recursive`] function. //! We also make use of the [`delta0`](`crate::operator::Delta0`) operator to //! import streams from a parent circuit into a child circuit. //! Finally, we pick up the [incremental computation](#incremental-computation) @@ -2169,9 +2169,8 @@ //! //! To fix this issue, we have to change the code to stop iterating once the //! shortest path for each pair of nodes has been discovered. One approach to -//! achieve this is to group by each pair and use the -//! [`Min`](`crate::operator::dynamic::aggregate::Min`) aggregation operator -//! to only retain the shortest path for each pair. +//! achieve this is to group by each pair and use the [`aggregate::Min`] +//! aggregation operator to only retain the shortest path for each pair. //! Aggregation requires to index the stream, so there are more code changes //! required than shown here. You can find the full code in `tutorial11.rs` but //! the important changes take place within the child circuit: @@ -2292,8 +2291,12 @@ //! The examples on the transitive closure above demonstrate how to express //! self-recursive queries. DBSP also supports _mutually_-recursive queries. //! As this is a little bit more involved than what would fit in here, we defer -//! the interested reader to the example given in `tutorial12` which +//! the interested reader to the example given in `tutorial12.rs` which //! demonstrates mutual recursion in the domain of static program analysis. +//! `tutorial13.rs` provides another example on graph colouring which additionally +//! shows how to manually construct recursive circuits using the lower-level +//! [`Circuit::iterate`] API as opposed to the higher-level +//! [`ChildCircuit::recursive`] API. //! //! # Next steps //! @@ -2315,8 +2318,8 @@ //! let (mut circuit, (/*handles*/)) = Runtime::init_circuit(4, build_circuit)?; //! ``` use crate::{ - CircuitHandle, IndexedZSet, OrdPartitionedIndexedZSet, OutputHandle, RootCircuit, Runtime, - Stream, ZSet, ZSetHandle, - operator::{Aggregator, Max}, + ChildCircuit, Circuit, CircuitHandle, IndexedZSet, OrdPartitionedIndexedZSet, OutputHandle, + RootCircuit, Runtime, Stream, ZSet, ZSetHandle, + operator::{Aggregator, Max, dynamic::aggregate}, utils::{Tup0, Tup1, Tup10}, }; From 7c4d6d7491ab2cb3e9f7bae749f15d1a581d26a5 Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Fri, 17 Jul 2026 17:38:27 +0200 Subject: [PATCH 2/5] Address PR feedback Signed-off-by: Leo Stewen --- .gitignore | 4 +- .../examples/tutorial/shared_helper/mod.rs | 62 +++++++++++++++++++ .../tutorial/transitive_closure/mod.rs | 58 +++-------------- crates/dbsp/examples/tutorial/tutorial10.rs | 57 ++++++++--------- crates/dbsp/examples/tutorial/tutorial11.rs | 52 ++++++++-------- crates/dbsp/examples/tutorial/tutorial13.rs | 35 +++++------ crates/dbsp/src/circuit/circuit_builder.rs | 25 ++++---- crates/dbsp/src/circuit/runtime.rs | 12 ++++ crates/dbsp/src/operator.rs | 2 +- crates/dbsp/src/operator/condition.rs | 40 +++++++----- 10 files changed, 196 insertions(+), 151 deletions(-) create mode 100644 crates/dbsp/examples/tutorial/shared_helper/mod.rs diff --git a/.gitignore b/.gitignore index bec1df1d12a..3d7ecc32d44 100644 --- a/.gitignore +++ b/.gitignore @@ -103,8 +103,6 @@ playwright/.cache/ js-packages/support-bundle-triage/ .roc-check/ -# Nix -flake.nix -flake.lock +# Local dev configs .envrc .direnv diff --git a/crates/dbsp/examples/tutorial/shared_helper/mod.rs b/crates/dbsp/examples/tutorial/shared_helper/mod.rs new file mode 100644 index 00000000000..2c3e47531e2 --- /dev/null +++ b/crates/dbsp/examples/tutorial/shared_helper/mod.rs @@ -0,0 +1,62 @@ +use dbsp::{Circuit, Consensus, NestedCircuit, SchedulerError}; +use std::cell::Cell; + +pub fn threads(force_threads: Option) -> usize { + let threads = force_threads.unwrap_or_else(|| { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) + }); + println!("Running with {threads} threads"); + threads +} + +pub const MAX_ITERATIONS: usize = 4096; + +/// Hybrid termination for the `iterate`-based variants: stop once the subcircuit +/// reaches a fixed point, or after `max_iterations` steps if it never does. +/// +/// Testing the frontier for emptiness is not a reliable convergence signal with +/// multiple workers. The incremental join buffers results for future nested +/// timestamps, so the frontier delta can be transiently empty for a step while +/// data is still in flight, and the exact gaps vary with cross-worker +/// scheduling. `is_fixedpoint` accounts for that buffered state, so it never +/// stops early. It is a per-worker check, hence we combine the votes across all +/// workers with `Consensus`. +/// +/// The returned closure is run once for every tick of the nested circuit's clock +/// on each worker of DBSP's runtime. Hence, the closure is stateful across nested +/// clock ticks _within_ a transaction and requires resetting between transactions. +/// +/// As the broadcast between all workers happens once per every nested clock tick, +/// a straggler (that is, a worker with more data to process than others) still +/// induces a cross-thread round trip for every worker, even though some of them +/// might have already reached a fixedpoint for their sharded data. Possibly, +/// batching the vote (e.g. every N ticks) is a common trade-off for heavier +/// workloads. +pub fn make_termination_check( + child_circuit: &NestedCircuit, + label: &'static str, + max_iterations: usize, +) -> impl AsyncFn() -> Result + use<> { + let child_circuit = child_circuit.clone(); + let consensus = Consensus::new(label); + let step = Cell::new(0usize); + + async move || { + let step_count = step.get() + 1; + step.set(step_count); + + // The step bound is data-independent, hence identical on every worker; + // the fixed-point check is per-worker. + let fixedpoint = child_circuit.is_fixedpoint(0); + let iteration_ceiling = step_count >= max_iterations; + let done = consensus.check(fixedpoint || iteration_ceiling).await?; + if done { + // Reset the step counter for the next transaction. + step.set(0); + } + + Ok(done) + } +} diff --git a/crates/dbsp/examples/tutorial/transitive_closure/mod.rs b/crates/dbsp/examples/tutorial/transitive_closure/mod.rs index 0ba2d04811b..97e7712c659 100644 --- a/crates/dbsp/examples/tutorial/transitive_closure/mod.rs +++ b/crates/dbsp/examples/tutorial/transitive_closure/mod.rs @@ -1,7 +1,7 @@ #![allow(unused)] use anyhow::Result; use dbsp::{ - Circuit, Consensus, DBSPHandle, IndexedZSetReader, NestedCircuit, OrdIndexedZSet, OutputHandle, + Circuit, DBSPHandle, IndexedZSetReader, NestedCircuit, OrdIndexedZSet, OutputHandle, SchedulerError, Stream, ZSetHandle, ZWeight, indexed_zset, typed_batch::SpineSnapshot, utils::{Tup2, Tup3, Tup4}, @@ -24,6 +24,10 @@ pub struct TransClosureCircuit { pub closure_output: OutputHandle>>, } +/// Input data to form a graph. +/// +/// The outer vector elements are in order of insertion/deletion and steps is +/// the "transaction" number. pub fn edges_data(steps: usize) -> impl Iterator>> { vec![ // The first step adds a graph of four nodes: @@ -49,6 +53,9 @@ pub fn edges_data(steps: usize) -> impl Iterator impl Iterator> { @@ -106,16 +113,6 @@ pub fn expected_closure( .take(steps) } -pub fn threads(force_threads: Option) -> usize { - let threads = force_threads.unwrap_or_else(|| { - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4) - }); - println!("Running with {threads} threads"); - threads -} - pub fn drive_circuit(circuit: TransClosureCircuit, steps: usize) -> Result { let label = circuit.label; let mut handle = circuit.handle; @@ -150,42 +147,3 @@ pub fn drive_circuit(circuit: TransClosureCircuit, steps: usize) -> Result Ok(start.elapsed().as_micros()) } - -pub const MAX_ITERATIONS: usize = 4096; - -/// Hybrid termination for the `iterate`-based variants: stop once the subcircuit -/// reaches a fixed point, or after `max_iterations` steps if it never does. -/// -/// Testing the frontier for emptiness is not a reliable convergence signal with -/// multiple workers. The incremental join buffers results for future nested -/// timestamps, so the frontier delta can be transiently empty for a step while -/// data is still in flight, and the exact gaps vary with cross-worker -/// scheduling. `is_fixedpoint` accounts for that buffered state, so it never -/// stops early. It is a per-worker check, hence we combine the votes across all -/// workers with `Consensus`. -pub fn make_termination_check( - child_circuit: &NestedCircuit, - label: &'static str, - max_iterations: usize, -) -> impl AsyncFn() -> Result + use<> { - let child_circuit = child_circuit.clone(); - let consensus = Consensus::new(label); - let step = Cell::new(0usize); - - async move || { - let step_count = step.get() + 1; - step.set(step_count); - - // The step bound is data-independent, hence identical on every worker; - // the fixed-point check is per-worker. - let fixedpoint = child_circuit.is_fixedpoint(0); - let iteration_ceiling = step_count >= max_iterations; - let done = consensus.check(fixedpoint || iteration_ceiling).await?; - if done { - // Reset the step counter for the next transaction. - step.set(0); - } - - Ok(done) - } -} diff --git a/crates/dbsp/examples/tutorial/tutorial10.rs b/crates/dbsp/examples/tutorial/tutorial10.rs index aad939957c7..0123a58026b 100644 --- a/crates/dbsp/examples/tutorial/tutorial10.rs +++ b/crates/dbsp/examples/tutorial/tutorial10.rs @@ -1,3 +1,4 @@ +mod shared_helper; mod transitive_closure; use anyhow::Result; @@ -7,16 +8,18 @@ use dbsp::{ trace::BatchReaderFactories, utils::{Tup2, Tup3, Tup4}, }; +use shared_helper::{MAX_ITERATIONS, make_termination_check}; use transitive_closure::{ - Accumulator, MAX_ITERATIONS, Path, TransClosureCircuit, TransClosurePath, WeightedEdge, - drive_circuit, make_termination_check, + Accumulator, Path, TransClosureCircuit, TransClosurePath, WeightedEdge, drive_circuit, }; /// If we set STEPS to 3 in this tutorial, the computation will not terminate -/// due to the cycle introduced in the third step in the input data. Tutorial 11 -/// lifts this restriction. We recommend to view tutorial 10 and 11 with your +/// due to the cycle introduced in the third transaction of the input data. +/// Tutorial 11 fixes the circuit such that the computation terminates again +/// even for cyclic graphs. We recommend to view tutorial 10 and 11 with your /// favorite diff tool side-by-side to see the differences in both the -/// [`recursive_variant`] and [`iterative_variant`] of each tutorial. +/// [`transitive_closure_acyclic_recursive`] and +/// [`transitive_closure_acyclic_iterate`] of each tutorial. const STEPS: usize = 2; fn main() -> Result<()> { @@ -25,21 +28,18 @@ fn main() -> Result<()> { // overhead but the sweet spot may vary depending on your setup. You might // want to run this in release mode via: // `cargo run --profile release --example tutorial10` - let threads = transitive_closure::threads(Some(2)); + let threads = shared_helper::threads(Some(2)); - // Uncomment to run with one round each to ignore for the measurements to warm up CPU caches. - // let _elapsed_rec = drive_circuit(recursive_variant(threads)?, STEPS)?; - let elapsed_rec = drive_circuit(recursive_variant(threads)?, STEPS)?; - // let _elapsed_iter = drive_circuit(iterative_variant(threads)?, STEPS)?; - let elapsed_iter = drive_circuit(iterative_variant(threads)?, STEPS)?; + let elapsed_rec = drive_circuit(transitive_closure_acyclic_recursive(threads)?, STEPS)?; + let elapsed_iter = drive_circuit(transitive_closure_acyclic_iterate(threads)?, STEPS)?; // As the recursive variant performs an unnecessary distinct, you should - // observe a speedup > 1 with the iterative variant. + // observe a speedup > 1 with the iterate variant. println!("=== Stats ==="); println!("Recursive variant: {elapsed_rec} μs"); - println!("Iterative variant: {elapsed_iter} μs"); + println!("Iterate variant: {elapsed_iter} μs"); let speedup = elapsed_rec as f64 / elapsed_iter as f64; - println!("Speedup factor (recursive (old) relative to iterative (new)): {speedup:.02}"); + println!("Speedup factor (recursive (old) relative to iterate (new)): {speedup:.02}"); Ok(()) } @@ -52,7 +52,7 @@ fn main() -> Result<()> { /// /// This variant only works on acyclic graphs, whereas tutorial 11 fixes this /// at the expense of some extra aggregation step. -fn recursive_variant(threads: usize) -> Result { +fn transitive_closure_acyclic_recursive(threads: usize) -> Result { let (handle, (edges_input, closure_output)) = Runtime::init_circuit(threads, move |root_circuit| { let (edges, edges_input) = root_circuit.add_input_zset::(); @@ -123,12 +123,12 @@ fn recursive_variant(threads: usize) -> Result { /// # Note /// /// This variant only works on acyclic graphs, whereas tutorial 11 fixes this -/// at the expense of some extra aggregation step. This iterative variant is faster -/// than [`recursive_variant`] because you don't have to pay for deduplication -/// (via distinct) in case you know your data is acyclic anyways. This is similar -/// to the difference between UNION vs UNION ALL in recursive CTEs in SQL, if that -/// is familiar to you. -fn iterative_variant(threads: usize) -> Result { +/// at the expense of some extra aggregation step. This iterate variant is faster +/// than [`transitive_closure_acyclic_recursive`] because you don't have to pay +/// for deduplication (via distinct) in case you know your data is acyclic anyways. +/// This is similar to the difference between UNION vs UNION ALL in recursive +/// CTEs in SQL, if that is familiar to you. +fn transitive_closure_acyclic_iterate(threads: usize) -> Result { let (handle, (edges_input, closure_output)) = Runtime::init_circuit(threads, move |root_circuit| { let (edges, edges_input) = root_circuit.add_input_zset::(); @@ -169,7 +169,7 @@ fn iterative_variant(threads: usize) -> Result { ); // Hook up the Z1 operator with the current frontier (new_paths) - // such that it becomes the previuos frontier in the next iteration. + // such that it becomes the previous frontier in the next iteration. closure_frontier_feedback.connect(&new_paths); // Accumulate every frontier across the inner iterations and export @@ -186,16 +186,17 @@ fn iterative_variant(threads: usize) -> Result { &BatchReaderFactories::new::(), ); - let termination_check = make_termination_check( - child_circuit, - "tutorial 10 (iterative)", - MAX_ITERATIONS, - ); + let termination_check = + make_termination_check(child_circuit, "tutorial 10 (iterate)", MAX_ITERATIONS); Ok((termination_check, closure.export())) })?; // Consolidate the exported trace `closure` into an OrdIndexedZSet. + // After the nested circuit, each element in the input streams is a + // trace, consisting of multiple batches of updates. This consolidates + // the trace into a single batch, which uses less memory and can be + // handled more efficiently by most operators than the trace. let closure = Stream::dyn_consolidate( &closure, &BatchReaderFactories::new::(), @@ -207,7 +208,7 @@ fn iterative_variant(threads: usize) -> Result { })?; Ok(TransClosureCircuit { - label: "Iterative Variant", + label: "Iterate Variant", handle, edges_input, closure_output, diff --git a/crates/dbsp/examples/tutorial/tutorial11.rs b/crates/dbsp/examples/tutorial/tutorial11.rs index acb537bbd65..97afd538238 100644 --- a/crates/dbsp/examples/tutorial/tutorial11.rs +++ b/crates/dbsp/examples/tutorial/tutorial11.rs @@ -1,3 +1,4 @@ +mod shared_helper; mod transitive_closure; use anyhow::Result; @@ -7,12 +8,13 @@ use dbsp::{ trace::BatchReaderFactories, utils::{Tup2, Tup3, Tup4}, }; +use shared_helper::{MAX_ITERATIONS, make_termination_check}; use transitive_closure::{ - Accumulator, MAX_ITERATIONS, Path, TransClosureCircuit, TransClosurePath, WeightedEdge, - drive_circuit, make_termination_check, + Accumulator, Path, TransClosureCircuit, TransClosurePath, WeightedEdge, drive_circuit, }; -// Now we allow for the cyclic data, too, by allowing to consume the third step. +// Now we allow for a cyclic graph by including the data from the third +// transaction which introduces a cycle to the graph. const STEPS: usize = 3; fn main() -> Result<()> { @@ -21,21 +23,18 @@ fn main() -> Result<()> { // overhead but the sweet spot may vary depending on your setup. You might // want to run this in release mode via: // `cargo run --profile release --example tutorial11` - let threads = transitive_closure::threads(Some(2)); + let threads = shared_helper::threads(Some(2)); - // Uncomment to run with one round each to ignore for the measurements to warm up CPU caches. - // let _elapsed_rec = drive_circuit(recursive_variant(threads)?, STEPS)?; - let elapsed_rec = drive_circuit(recursive_variant(threads)?, STEPS)?; - // let _elapsed_iter = drive_circuit(iterative_variant(threads)?, STEPS)?; - let elapsed_iter = drive_circuit(iterative_variant(threads)?, STEPS)?; + let elapsed_rec = drive_circuit(transitive_closure_cyclic_recursive(threads)?, STEPS)?; + let elapsed_iter = drive_circuit(transitive_closure_cyclic_iterate(threads)?, STEPS)?; // As the recursive variant performs a duplicated distinct, you should - // observe a (small) speedup > 1 with the iterative variant. + // observe a (small) speedup > 1 with the iterate variant. println!("=== Stats ==="); println!("Recursive variant: {elapsed_rec} μs"); - println!("Iterative variant: {elapsed_iter} μs"); + println!("Iterate variant: {elapsed_iter} μs"); let speedup = elapsed_rec as f64 / elapsed_iter as f64; - println!("Speedup factor (recursive (old) relative to iterative (new)): {speedup:.02}"); + println!("Speedup factor (recursive (old) relative to iterate (new)): {speedup:.02}"); Ok(()) } @@ -49,9 +48,10 @@ fn main() -> Result<()> { /// While this works on both cyclic and acyclic graphs (unlike tutorial 10), /// it deduplicates twice: First, implicitly due to the recursive API and, /// second, due to the aggregation which is required for termination upon -/// cyclic graphs. The [`iterative_variant`] below fixes this and only relies -/// on the aggregation to ensure termination and is therefore a bit faster. -fn recursive_variant(threads: usize) -> Result { +/// cyclic graphs. The [`transitive_closure_cyclic_iterate`] below fixes this +/// and only relies on the aggregation to ensure termination and is therefore +/// a bit faster. +fn transitive_closure_cyclic_recursive(threads: usize) -> Result { let (handle, (edges_input, closure_output)) = Runtime::init_circuit(threads, move |root_circuit| { let (edges, edges_input) = root_circuit.add_input_zset::(); @@ -118,10 +118,11 @@ fn recursive_variant(threads: usize) -> Result { /// /// # Note /// -/// This variant also works on cyclic graphs. The iterative variant is faster -/// than the [`recursive_variant`] because you don't have to pay for deduplication -/// (via distinct) despite already being accounted for by the min-aggregation. -fn iterative_variant(threads: usize) -> Result { +/// This variant also works on cyclic graphs. The iterate variant is faster +/// than the [`transitive_closure_cyclic_recursive`] because you don't have to +/// pay for deduplication (via distinct) despite already being accounted for by +/// the min-aggregation. +fn transitive_closure_cyclic_iterate(threads: usize) -> Result { let (handle, (edges_input, closure_output)) = Runtime::init_circuit(threads, move |root_circuit| { let (edges, edges_input) = root_circuit.add_input_zset::(); @@ -183,16 +184,17 @@ fn iterative_variant(threads: usize) -> Result { &BatchReaderFactories::new::(), ); - let termination_check = make_termination_check( - child_circuit, - "tutorial 11 (iterative)", - MAX_ITERATIONS, - ); + let termination_check = + make_termination_check(child_circuit, "tutorial 11 (iterate)", MAX_ITERATIONS); Ok((termination_check, closure.export())) })?; // Consolidate the exported trace `closure` into an OrdIndexedZSet. + // After the nested circuit, each element in the input streams is a + // trace, consisting of multiple batches of updates. This consolidates + // the trace into a single batch, which uses less memory and can be + // handled more efficiently by most operators than the trace. let closure = Stream::dyn_consolidate( &closure, &BatchReaderFactories::new::(), @@ -204,7 +206,7 @@ fn iterative_variant(threads: usize) -> Result { })?; Ok(TransClosureCircuit { - label: "Iterative Variant", + label: "Iterate Variant", handle, edges_input, closure_output, diff --git a/crates/dbsp/examples/tutorial/tutorial13.rs b/crates/dbsp/examples/tutorial/tutorial13.rs index 4bd9c8dd42b..f237b2e98b8 100644 --- a/crates/dbsp/examples/tutorial/tutorial13.rs +++ b/crates/dbsp/examples/tutorial/tutorial13.rs @@ -1,3 +1,4 @@ +mod shared_helper; mod transitive_closure; use anyhow::Result; @@ -10,7 +11,7 @@ use dbsp::{ utils::Tup2, zset, }; -use transitive_closure::{MAX_ITERATIONS, make_termination_check, threads}; +use shared_helper::{MAX_ITERATIONS, make_termination_check}; fn main() -> Result<()> { // Play around with different thread setups on your machine. With this little @@ -18,21 +19,18 @@ fn main() -> Result<()> { // overhead but the sweet spot may vary depending on your setup. You might // want to run this in release mode via: // `cargo run --profile release --example tutorial13` - let threads = threads(Some(2)); + let threads = shared_helper::threads(Some(2)); - // Uncomment to run with one round each to ignore for the measurements to warm up CPU caches. - // let _elapsed_rec = drive_circuit(recursive_variant(threads)?)?; - let elapsed_rec = drive_circuit(recursive_variant(threads)?)?; - // let _elapsed_iter = drive_circuit(iterative_variant(threads)?)?; - let elapsed_iter = drive_circuit(iterative_variant(threads)?)?; + let elapsed_rec = drive_circuit(graph_coloring_recursive(threads)?)?; + let elapsed_iter = drive_circuit(graph_coloring_iterate(threads)?)?; // As the work performed by both variants in this tutorial is roughly the // same, you should observe comparable numbers. println!("=== Stats ==="); println!("Recursive variant: {elapsed_rec} μs"); - println!("Iterative variant: {elapsed_iter} μs"); + println!("Iterate variant: {elapsed_iter} μs"); let speedup = elapsed_rec as f64 / elapsed_iter as f64; - println!("Speedup factor (recursive (old) relative to iterative (new)): {speedup:.02}"); + println!("Speedup factor (recursive (old) relative to iterate (new)): {speedup:.02}"); Ok(()) } @@ -188,7 +186,7 @@ fn drive_circuit(circuit: BipartiteGraphCircuit) -> Result { /// API but if you know your recursive streams at compile time you could also use /// [`ChildCircuit::recursive`](`dbsp::circuit::ChildCircuit::recursive`) but /// with a tuple carrying the recursive red and blue streams. -fn recursive_variant(threads: usize) -> Result { +fn graph_coloring_recursive(threads: usize) -> Result { let (handle, ((init_input, edges_input), (red_output, blue_output))) = Runtime::init_circuit(threads, move |root_circuit| { let (edges, edges_input) = root_circuit.add_input_zset::(); @@ -239,12 +237,12 @@ fn recursive_variant(threads: usize) -> Result { }) } -/// This iterative variant is a hand-rolled expansion of [`recursive_variant`], +/// This iterate-based variant is a hand-rolled expansion of [`graph_coloring_recursive`], /// built on the lower-level [`Circuit::iterate`] primitive. DBSP's recursive API /// splices a `distinct` onto every output stream for us (see /// [`ChildCircuit::recursive`](`dbsp::circuit::ChildCircuit::recursive`)); /// with `iterate` we are responsible for deduplication ourselves. -fn iterative_variant(threads: usize) -> Result { +fn graph_coloring_iterate(threads: usize) -> Result { let (handle, (init_input, edges_input, red_output, blue_output)) = Runtime::init_circuit(threads, move |root_circuit| { let (init, init_input) = root_circuit.add_input_zset::(); @@ -298,16 +296,17 @@ fn iterative_variant(threads: usize) -> Result { &BatchReaderFactories::new::(), ); - let termination_check = make_termination_check( - child_circuit, - "tutorial 13 (iterative)", - MAX_ITERATIONS, - ); + let termination_check = + make_termination_check(child_circuit, "tutorial 13 (iterate)", MAX_ITERATIONS); Ok((termination_check, vec![red.export(), blue.export()])) })?; // Consolidate each exported trace (red and blue) into a single ZSet. + // After the nested circuit, each element in the input streams is a + // trace, consisting of multiple batches of updates. This consolidates + // the trace into a single batch, which uses less memory and can be + // handled more efficiently by most operators than the trace. let mut recursive_streams = recursive_streams .into_iter() .map(|s| { @@ -323,7 +322,7 @@ fn iterative_variant(threads: usize) -> Result { })?; Ok(BipartiteGraphCircuit { - label: "Iterative Variant", + label: "Iterate Variant", handle, init_input, edges_input, diff --git a/crates/dbsp/src/circuit/circuit_builder.rs b/crates/dbsp/src/circuit/circuit_builder.rs index b17ea080051..f2446689231 100644 --- a/crates/dbsp/src/circuit/circuit_builder.rs +++ b/crates/dbsp/src/circuit/circuit_builder.rs @@ -34,7 +34,6 @@ use crate::{ QuaternaryOperator, SinkOperator, SourceOperator, StrictUnaryOperator, TernaryOperator, TernarySinkOperator, UnaryOperator, }, - runtime::Consensus, schedule::{ CommitProgress, DynamicScheduler, Error as SchedulerError, Executor, IterativeExecutor, OnceExecutor, Scheduler, @@ -43,9 +42,12 @@ use crate::{ }, circuit_cache_key, ir::LABEL_MIR_NODE_ID, - operator::dynamic::{ - balance::{Balancer, BalancerError, BalancerHint, PartitioningPolicy}, - recorder::{Recorder, RecorderControl, RecorderControlId, RecorderId}, + operator::{ + condition::with_consensus, + dynamic::{ + balance::{Balancer, BalancerError, BalancerHint, PartitioningPolicy}, + recorder::{Recorder, RecorderControl, RecorderControlId, RecorderId}, + }, }, time::{Timestamp, UnitTimestamp}, trace::Batch, @@ -2772,8 +2774,9 @@ pub trait Circuit: CircuitBase + Clone + WithClock { /// precomputed for future nested timestamps). Custom iteration built on /// [`iterate`](Self::iterate) can call this from its termination check to /// detect convergence, typically combining the per-worker results across - /// workers via [`Consensus`]. See the documentation of [`Scope`] for the - /// meaning of the parameter but passing `0` refers to the current circuit. + /// workers via [`Consensus`](crate::circuit::runtime::Consensus). + /// See the documentation of [`Scope`] for the meaning of the parameter + /// but passing `0` refers to the current circuit. fn is_fixedpoint(&self, scope: Scope) -> bool { self.check_fixedpoint(scope) } @@ -4678,13 +4681,11 @@ where let res = constructor(child)?; let child_clone = child.clone(); - let consensus = Consensus::new("fixed point"); + let termination_check = + with_consensus(self.global_node_id(), "fixed point", move || { + child_clone.inner().check_fixedpoint(0) + }); - let termination_check = async move || { - // Send local fixed point status to all peers. - let local_fixedpoint = child_clone.inner().check_fixedpoint(0); - consensus.check(local_fixedpoint).await - }; let mut executor = >::new(termination_check); executor.prepare(child, None)?; Ok((res, executor)) diff --git a/crates/dbsp/src/circuit/runtime.rs b/crates/dbsp/src/circuit/runtime.rs index 3b86700c705..1e200aaf0a5 100644 --- a/crates/dbsp/src/circuit/runtime.rs +++ b/crates/dbsp/src/circuit/runtime.rs @@ -1363,6 +1363,18 @@ impl Runtime { /// example, per-worker fixed-point status) into a single, runtime-wide /// decision when driving a nested circuit built on /// [`Circuit::iterate`](super::circuit_builder::Circuit::iterate). +/// +/// # Safety +/// +/// Callers must respect the following invariants: +/// +/// 1. Every worker must call [`check`](Self::check) the same number of times +/// and in the same order relative to other ongoing `Consensus` in the same +/// subcircuit. A worker that skips a call to [`check`](Self::check) will +/// stop its peers from making progress. +/// 2. [`Consensus`] must be constructed once per logical decision and reused +/// across clock ticks. See the +/// [`with_consensus`](crate::operator::condition::with_consensus) helper. pub struct Consensus(Broadcast); impl Consensus { diff --git a/crates/dbsp/src/operator.rs b/crates/dbsp/src/operator.rs index 19ff96d359a..15c037b2183 100644 --- a/crates/dbsp/src/operator.rs +++ b/crates/dbsp/src/operator.rs @@ -16,7 +16,7 @@ pub mod communication; pub(crate) mod inspect; mod accumulator; -mod condition; +pub(crate) mod condition; mod count; mod csv; mod delta0; diff --git a/crates/dbsp/src/operator/condition.rs b/crates/dbsp/src/operator/condition.rs index 1232fb264a2..e5e31526731 100644 --- a/crates/dbsp/src/operator/condition.rs +++ b/crates/dbsp/src/operator/condition.rs @@ -4,8 +4,8 @@ use crate::{ Timestamp, circuit::{ - ChildCircuit, Circuit, Stream, - circuit_builder::IterativeCircuit, + ChildCircuit, Circuit, GlobalNodeId, Stream, + circuit_builder::{CircuitBase, IterativeCircuit}, runtime::Consensus, schedule::{Error as SchedulerError, Scheduler}, }, @@ -61,7 +61,10 @@ where { self.iterate(|child| { let (condition, res) = constructor(child)?; - let check = with_consensus("iterate with condition", move || condition.check()); + let check = + with_consensus(self.global_node_id(), "iterate with condition", move || { + condition.check() + }); Ok((check, res)) }) } @@ -80,9 +83,11 @@ where { self.iterate_with_scheduler::<_, _, _, S>(|child| { let (condition, res) = constructor(child)?; - let check = with_consensus("iterate with condition and scheduler", move || { - condition.check() - }); + let check = with_consensus( + self.global_node_id(), + "iterate with condition and scheduler", + move || condition.check(), + ); Ok((check, res)) }) } @@ -101,9 +106,11 @@ where { self.iterate(|child| { let (conditions, res) = constructor(child)?; - let check = with_consensus("iterate with conditions", move || { - conditions.iter().all(Condition::check) - }); + let check = with_consensus( + self.global_node_id(), + "iterate with conditions", + move || conditions.iter().all(Condition::check), + ); Ok((check, res)) }) } @@ -122,9 +129,11 @@ where { self.iterate_with_scheduler::<_, _, _, S>(|child| { let (conditions, res) = constructor(child)?; - let check = with_consensus("iterate with conditions and scheduler", move || { - conditions.iter().all(Condition::check) - }); + let check = with_consensus( + self.global_node_id(), + "iterate with conditions and scheduler", + move || conditions.iter().all(Condition::check), + ); Ok((check, res)) }) } @@ -139,11 +148,14 @@ where /// across every clock cycle, mirroring /// [`Circuit::fixedpoint`](crate::Circuit::fixedpoint). #[inline] -fn with_consensus( +pub(crate) fn with_consensus( + global_node_id: GlobalNodeId, name: impl Display, decide: impl Fn() -> bool + 'static, ) -> impl AsyncFn() -> Result { - let consensus = Consensus::new(name); + let consensus = Consensus::new(format!("{} {}", global_node_id, name)); + // The ordering is important here: The worker's local decision (`decide()`) + // must be captured _before_ announcing it to the broadcast. async move || consensus.check(decide()).await } From d762396207aa4cdbbf539445617168aa8b21e6b8 Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Mon, 20 Jul 2026 16:07:33 +0200 Subject: [PATCH 3/5] Provide criterion benchmarks comparing iterative and recursive variants Signed-off-by: Leo Stewen --- crates/dbsp/Cargo.toml | 12 + crates/dbsp/benches/graph_coloring.rs | 362 +++++++++++++++++ .../benches/transitive_closure_acyclic.rs | 348 ++++++++++++++++ .../dbsp/benches/transitive_closure_cyclic.rs | 373 ++++++++++++++++++ crates/dbsp/examples/tutorial/tutorial13.rs | 34 +- 5 files changed, 1111 insertions(+), 18 deletions(-) create mode 100644 crates/dbsp/benches/graph_coloring.rs create mode 100644 crates/dbsp/benches/transitive_closure_acyclic.rs create mode 100644 crates/dbsp/benches/transitive_closure_cyclic.rs diff --git a/crates/dbsp/Cargo.toml b/crates/dbsp/Cargo.toml index a84ab21d337..fdd6e5e52be 100644 --- a/crates/dbsp/Cargo.toml +++ b/crates/dbsp/Cargo.toml @@ -167,6 +167,18 @@ harness = false name = "filter_predictor" harness = false +[[bench]] +name = "transitive_closure_acyclic" +harness = false + +[[bench]] +name = "transitive_closure_cyclic" +harness = false + +[[bench]] +name = "graph_coloring" +harness = false + [[example]] name = "orgchart" diff --git a/crates/dbsp/benches/graph_coloring.rs b/crates/dbsp/benches/graph_coloring.rs new file mode 100644 index 00000000000..f724bdf801b --- /dev/null +++ b/crates/dbsp/benches/graph_coloring.rs @@ -0,0 +1,362 @@ +//! Benchmark comparing two graph 2-coloring (bipartiteness) approaches: +//! +//! - **recursive**: `Circuit::recursive_dynamic`, which splices an implicit +//! `distinct` onto every recursive output stream. +//! See [`build_recursive_variant`]. +//! - **iterate**: `Circuit::iterate` with a manual feedback loop that carries +//! only the per-iteration frontier and deduplicates with an explicit +//! `distinct`. See [`build_iterate_variant`]. +//! +//! Unlike the transitive-closure benchmarks, both variants perform the same +//! deduplication work (implicit vs. explicit `distinct`). Hence, we should see +//! comparable run times. The coloring propagates from a single seed node +//! along directed edges: a red node paints its out-neighbors blue and vice +//! versa. On a strongly connected graph with odd cycles every reachable node +//! ends up both red and blue, so both variants converge to the same colorings +//! (the benchmark asserts this before measuring). +//! +//! Run with, e.g.: +//! `cargo bench --bench graph_coloring` + +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use dbsp::{ + Circuit, Consensus, DBSPHandle, NestedCircuit, NumEntries, OrdIndexedZSet, OrdZSet, + OutputHandle, Runtime, Stream, ZSetHandle, ZWeight, mimalloc::MiMalloc, operator::Z1, + trace::BatchReaderFactories, typed_batch::SpineSnapshot, utils::Tup2, +}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use std::cell::Cell; + +#[global_allocator] +static ALLOC: MiMalloc = MiMalloc; + +type NodeId = u64; +type Edge = Tup2; + +struct BipartiteGraphCircuit { + handle: DBSPHandle, + init_input: ZSetHandle, + edges_input: ZSetHandle, + red_output: OutputHandle>>, + blue_output: OutputHandle>>, +} + +/// Number of DBSP worker threads used for every circuit in this benchmark. +const WORKERS: usize = 4; + +/// Deterministic seed so every run sees the same graphs. +const SEED: u64 = 0x0BADC0DE_DEADBEEF; + +/// Out-edges emitted per node by [`generate_graph`]. +const OUT_DEGREE: usize = 3; + +/// A single-transaction workload: the seed node(s) plus all edges of one graph. +#[derive(Clone)] +struct Workload { + init: Vec>, + edges: Vec>, +} + +/// Generates a random directed graph plus a single seed node (node `0`, painted +/// red) as one insertion batch. +/// +/// Each node emits `out_degree` edges to distinct random targets (excluding +/// itself). With `out_degree >= 2` and a few hundred nodes the graph is +/// strongly connected with overwhelming probability and contains odd cycles, so +/// the coloring reaches every node in both colors. That makes the two variants +/// produce identical red/blue sets regardless of the exact structure. +fn generate_graph(nodes: NodeId, out_degree: usize) -> Workload { + let mut rng = ChaCha8Rng::seed_from_u64(SEED); + // At most `nodes - 1` distinct targets exist per node (self is excluded). + let fanout = out_degree.min(nodes.saturating_sub(1) as usize); + let mut edges = Vec::with_capacity(nodes as usize * fanout); + + for from in 0..nodes { + let mut targets = Vec::with_capacity(fanout); + while targets.len() < fanout { + let to = rng.gen_range(0..nodes); + if to != from && !targets.contains(&to) { + targets.push(to); + } + } + for to in targets { + edges.push(Tup2(Tup2(from, to), 1)); + } + } + + // Seed the discovery by marking node `0` red. + let init = vec![Tup2(0, 1)]; + + Workload { init, edges } +} + +/// Feeds a whole graph into a freshly built circuit in one transaction and +/// returns the number of `(red, blue)` color marks discovered. +/// +/// The circuit (and its worker threads) is dropped on return, so the caller +/// must supply a fresh circuit for each measured iteration. +fn drive_once(mut circuit: BipartiteGraphCircuit, mut workload: Workload) -> (usize, usize) { + circuit.init_input.append(&mut workload.init); + circuit.edges_input.append(&mut workload.edges); + circuit + .handle + .transaction() + .expect("transaction should succeed"); + let red = circuit + .red_output + .concat() + .consolidate() + .num_entries_shallow(); + let blue = circuit + .blue_output + .concat() + .consolidate() + .num_entries_shallow(); + (red, blue) +} + +/// Cross-checks that both variants compute the same coloring on the given graph, +/// returning the number of `(red, blue)` color marks (used both as a sanity check +/// and to label the benchmark with the output cardinality). +fn assert_variants_agree(workload: &Workload) -> (usize, usize) { + let recursive = build_recursive_variant().expect("build recursive circuit"); + let iterate = build_iterate_variant().expect("build iterate circuit"); + + let mut recursive_circuit = recursive.handle; + let mut iterate_circuit = iterate.handle; + + recursive.init_input.append(&mut workload.init.clone()); + recursive.edges_input.append(&mut workload.edges.clone()); + iterate.init_input.append(&mut workload.init.clone()); + iterate.edges_input.append(&mut workload.edges.clone()); + recursive_circuit.transaction().unwrap(); + iterate_circuit.transaction().unwrap(); + + let recursive_red = recursive.red_output.concat().consolidate(); + let recursive_blue = recursive.blue_output.concat().consolidate(); + let iterate_red = iterate.red_output.concat().consolidate(); + let iterate_blue = iterate.blue_output.concat().consolidate(); + + assert_eq!( + recursive_red, iterate_red, + "recursive and iterate variants must agree on the red coloring" + ); + assert_eq!( + recursive_blue, iterate_blue, + "recursive and iterate variants must agree on the blue coloring" + ); + + ( + recursive_red.num_entries_shallow(), + recursive_blue.num_entries_shallow(), + ) +} + +/// The graph sizes (node counts) we benchmark. +const GRAPH_SIZES: &[NodeId] = &[500, 1000, 2000]; + +fn graph_coloring_benches(c: &mut Criterion) { + let mut group = c.benchmark_group("graph-coloring"); + // Building a circuit spins up worker threads, so keep the sample count + // modest to bound the wall-clock time of the whole benchmark. + group.sample_size(30); + + for &nodes in GRAPH_SIZES { + let workload = generate_graph(nodes, OUT_DEGREE); + let (red_marks, blue_marks) = assert_variants_agree(&workload); + let shape = format!("n{nodes}"); + let edges = workload.edges.len(); + println!("graph {shape}: {edges} edges, {red_marks} red / {blue_marks} blue color marks",); + + // Report throughput in edges processed per second, which scales with the + // graph size and the join work both variants perform. + group.throughput(Throughput::Elements(edges as u64)); + + group.bench_with_input( + BenchmarkId::new("recursive", &shape), + &workload, + |b, workload| { + b.iter_batched( + || { + ( + build_recursive_variant().expect("build recursive circuit"), + workload.clone(), + ) + }, + |(circuit, workload)| drive_once(circuit, workload), + BatchSize::PerIteration, + ); + }, + ); + + group.bench_with_input( + BenchmarkId::new("iterate", &shape), + &workload, + |b, workload| { + b.iter_batched( + || { + ( + build_iterate_variant().expect("build iterate circuit"), + workload.clone(), + ) + }, + |(circuit, workload)| drive_once(circuit, workload), + BatchSize::PerIteration, + ); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, graph_coloring_benches); +criterion_main!(benches); + +/// Helper function to advance the frontier of red and blue nodes by exploring +/// one more hop in the graph along `edges`. +fn hop( + frontier: &Stream>, + edges: &Stream>, +) -> Stream> { + frontier + .map_index(|node| (*node, *node)) + .join(edges, |_from, _node, to| *to) +} + +/// Computes the red/blue graph coloring using DBSP's +/// [`recursive` API](dbsp::circuit::ChildCircuit::recursive), +/// which deduplicates every recursive output stream implicitly. +fn build_recursive_variant() -> anyhow::Result { + let (handle, ((init_input, edges_input), (red_output, blue_output))) = + Runtime::init_circuit(WORKERS, move |root_circuit| { + let (edges, edges_input) = root_circuit.add_input_zset::(); + let (init, init_input) = root_circuit.add_input_zset::(); + + let (red_output, blue_output) = root_circuit.recursive( + |child_circuit, + (red, blue): ( + Stream>, + Stream>, + )| { + let edges = edges.delta0(child_circuit); + let init = init.delta0(child_circuit); + let edges_indexed = edges.map_index(|Tup2(from, to)| (*from, *to)); + + let new_red = init.plus(&hop(&blue, &edges_indexed)); + let new_blue = hop(&red, &edges_indexed); + + Ok((new_red, new_blue)) + }, + )?; + + let red_output = red_output.accumulate_output(); + let blue_output = blue_output.accumulate_output(); + + Ok(((init_input, edges_input), (red_output, blue_output))) + })?; + + Ok(BipartiteGraphCircuit { + handle, + init_input, + edges_input, + red_output, + blue_output, + }) +} + +/// Upper bound on nested iterations before the iterate variant gives up waiting +/// for a fixed point. +const MAX_ITERATIONS: usize = 4096; + +/// Computes the red/blue graph coloring using DBSP's lower level +/// [`iterate` API](dbsp::circuit::Circuit::iterate). +/// +/// It uses a hybrid termination check: Either stop once the subcircuit +/// reaches a fixed point, or after [`MAX_ITERATIONS`] steps if it never does. +fn build_iterate_variant() -> anyhow::Result { + let (handle, (init_input, edges_input, red_output, blue_output)) = + Runtime::init_circuit(WORKERS, move |root_circuit| { + let (init, init_input) = root_circuit.add_input_zset::(); + let (edges, edges_input) = root_circuit.add_input_zset::(); + + let (red_output, blue_output) = root_circuit.iterate(|child_circuit| { + let edges = edges.delta0(child_circuit); + let init = init.delta0(child_circuit); + let edges_indexed = edges.map_index(|Tup2(from, to)| (*from, *to)); + + // Feedback carries only the frontier (the delta from the last step). + let (red, red_feedback) = + child_circuit.add_feedback(Z1::new(OrdZSet::::default())); + let (blue, blue_feedback) = + child_circuit.add_feedback(Z1::new(OrdZSet::::default())); + + // Extend each frontier by one hop and deduplicate with `distinct`, + // which caps every node's weight at 1 and stops nodes discovered + // earlier from being re-emitted. + let new_red = init.plus(&hop(&blue, &edges_indexed)).distinct(); + let new_blue = hop(&red, &edges_indexed).distinct(); + + red_feedback.connect(&new_red); + blue_feedback.connect(&new_blue); + + // Accumulate every frontier across the nested clock cycles into the + // full colorings and export them to the parent. + let red = Stream::dyn_integrate_trace( + &new_red.inner(), + &BatchReaderFactories::new::(), + ); + let blue = Stream::dyn_integrate_trace( + &new_blue.inner(), + &BatchReaderFactories::new::(), + ); + + let child_circuit = child_circuit.clone(); + let consensus = Consensus::new("iterate"); + let step = Cell::new(0usize); + + let termination_check = async move || { + let step_count = step.get() + 1; + step.set(step_count); + + // The step bound is data-independent, hence identical on every worker; + // the fixed-point check is per-worker. + let fixedpoint = child_circuit.is_fixedpoint(0); + let iteration_ceiling = step_count >= MAX_ITERATIONS; + let done = consensus.check(fixedpoint || iteration_ceiling).await?; + if done { + // Reset the step counter for the next transaction. + step.set(0); + } + + Ok(done) + }; + + Ok((termination_check, (red.export(), blue.export()))) + })?; + + let red_output = Stream::dyn_consolidate( + &red_output, + &BatchReaderFactories::new::(), + ) + .typed::>() + .accumulate_output(); + let blue_output = Stream::dyn_consolidate( + &blue_output, + &BatchReaderFactories::new::(), + ) + .typed::>() + .accumulate_output(); + + Ok((init_input, edges_input, red_output, blue_output)) + })?; + + Ok(BipartiteGraphCircuit { + handle, + init_input, + edges_input, + red_output, + blue_output, + }) +} diff --git a/crates/dbsp/benches/transitive_closure_acyclic.rs b/crates/dbsp/benches/transitive_closure_acyclic.rs new file mode 100644 index 00000000000..807575a27af --- /dev/null +++ b/crates/dbsp/benches/transitive_closure_acyclic.rs @@ -0,0 +1,348 @@ +//! Benchmark comparing two transitive-closure computation approaches on acyclic +//! graphs similar to `examples/tutorial/tutorial10.rs`: +//! +//! - **recursive**: `Circuit::recursive`, which applies an implicit `distinct` +//! on every nested clock tick. See [`build_recursive_variant`]. +//! - **iterate**: `Circuit::iterate` with a manual feedback loop that only +//! carries the per-iteration frontier and therefore skips the `distinct`. +//! See [`build_iterate_variant`]. +//! +//! The two variants are only semantically equivalent when there is at most one +//! path between any pair of nodes (see [`generate_forest`] for why), so the +//! benchmark drives random forests (and no general DAGs) and asserts the +//! variants agree before measuring. In that regime the recursive variant's +//! `distinct` removes nothing yet still costs, so the iterate variant is +//! expected to win. This mirrors `UNION` vs `UNION ALL` in recursive SQL CTEs. +//! +//! Run with, e.g.: +//! `cargo bench --bench transitive_closure_acyclic` + +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use dbsp::{ + Circuit, Consensus, DBSPHandle, NestedCircuit, NumEntries, OrdIndexedZSet, OutputHandle, + Runtime, Stream, ZSetHandle, ZWeight, + mimalloc::MiMalloc, + operator::Z1, + trace::BatchReaderFactories, + typed_batch::SpineSnapshot, + utils::{Tup2, Tup3, Tup4}, +}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use std::cell::Cell; + +#[global_allocator] +static ALLOC: MiMalloc = MiMalloc; + +type NodeId = u64; +type Weight = u64; +type CumWeight = u64; +type Hopcnt = u64; +type WeightedEdge = Tup3; +type Path = Tup2; +type TransClosurePath = Tup4; +type Accumulator = Stream>; + +struct TransClosureCircuit { + handle: DBSPHandle, + edges_input: ZSetHandle, + closure_output: OutputHandle>>, +} + +/// Number of DBSP worker threads used for every circuit in this benchmark. +const WORKERS: usize = 4; + +/// Deterministic seed so every run sees the same graphs. +const SEED: u64 = 0x0BADC0DE_DEADBEEF; + +/// Largest edge weight generated by [`generate_forest`]. +const MAX_EDGE_WEIGHT: Weight = 10; + +/// A single-transaction workload: all edges of one acyclic graph. +type Workload = Vec>; + +/// Generates a random weighted *forest* (each non-root node has exactly one +/// parent) as a single insertion batch. +/// +/// Why a forest rather than an arbitrary DAG? The two variants are only +/// semantically equivalent when there is **at most one path between any pair of +/// nodes**. When several distinct paths connect the same `(start, end)` and +/// happen to share the same cumulative weight and hop count, the recursive +/// variant's `distinct` collapses them into a single tuple, whereas the iterate +/// variant (UNION ALL semantics) keeps the multiplicity, so the two disagree on +/// a general DAG. A forest guarantees a unique path per reachable pair: +/// the `distinct` becomes pure overhead that removes nothing, and both variants +/// produce identical transitive closures. +/// +/// Node `child` (for `child` in `1..nodes`) draws a uniformly random parent from +/// `0..child`, so edges always point from a lower id to a higher one and no cycle +/// can form. Node `0` is the single root. A bushy tree still exercises wide +/// fan-out joins because low-id nodes accumulate many children. +fn generate_forest(nodes: NodeId) -> Workload { + let mut rng = ChaCha8Rng::seed_from_u64(SEED); + let mut edges = Vec::with_capacity(nodes.saturating_sub(1) as usize); + + for child in 1..nodes { + let parent = rng.gen_range(0..child); + let weight = rng.gen_range(1..=MAX_EDGE_WEIGHT); + edges.push(Tup2(Tup3(parent, child, weight), 1)); + } + + edges +} + +/// Feeds a whole graph into a freshly built circuit in one transaction and +/// returns the number of paths in the resulting transitive closure. +/// +/// The circuit (and its worker threads) is dropped on return, so the caller +/// must supply a fresh circuit for each measured iteration. +fn drive_once(mut circuit: TransClosureCircuit, mut workload: Workload) -> usize { + circuit.edges_input.append(&mut workload); + circuit + .handle + .transaction() + .expect("transaction should succeed"); + circuit + .closure_output + .concat() + .consolidate() + .num_entries_shallow() +} + +/// Cross-checks that both variants compute the same closure on the given graph, +/// returning the shared closure size (used both as a sanity check and to label +/// the benchmark with the output cardinality). +fn assert_variants_agree(workload: &Workload) -> usize { + let recursive = build_recursive_variant().expect("build recursive circuit"); + let iterate = build_iterate_variant().expect("build iterate circuit"); + + let mut recursive_circuit = recursive.handle; + let mut iterate_circuit = iterate.handle; + + recursive.edges_input.append(&mut workload.clone()); + iterate.edges_input.append(&mut workload.clone()); + recursive_circuit.transaction().unwrap(); + iterate_circuit.transaction().unwrap(); + + let recursive_closure = recursive.closure_output.concat().consolidate(); + let iterate_closure = iterate.closure_output.concat().consolidate(); + + assert_eq!( + recursive_closure, iterate_closure, + "recursive and iterate variants must agree on acyclic input" + ); + + recursive_closure.num_entries_shallow() +} + +/// The forest sizes (in terms of node counts) we benchmark. +const GRAPH_SIZES: &[NodeId] = &[500, 1000, 2000]; + +fn transitive_closure_benches(c: &mut Criterion) { + let mut group = c.benchmark_group("transitive-closure-acyclic"); + // Building a circuit spins up worker threads, so keep the sample count + // modest to bound the wall-clock time of the whole benchmark. + group.sample_size(25); + + for &nodes in GRAPH_SIZES { + let workload = generate_forest(nodes); + let closure_size = assert_variants_agree(&workload); + let shape = format!("n{nodes}"); + println!( + "graph {shape}: {} edges, closure of {closure_size} paths", + workload.len() + ); + + // Report throughput in paths of the computed closure per second so the + // two variants are compared on the same unit of useful work. + group.throughput(Throughput::Elements(closure_size as u64)); + + group.bench_with_input( + BenchmarkId::new("recursive", &shape), + &workload, + |b, workload| { + b.iter_batched( + || { + ( + build_recursive_variant().expect("build recursive circuit"), + workload.clone(), + ) + }, + |(circuit, workload)| drive_once(circuit, workload), + BatchSize::PerIteration, + ); + }, + ); + + group.bench_with_input( + BenchmarkId::new("iterate", &shape), + &workload, + |b, workload| { + b.iter_batched( + || { + ( + build_iterate_variant().expect("build iterate circuit"), + workload.clone(), + ) + }, + |(circuit, workload)| drive_once(circuit, workload), + BatchSize::PerIteration, + ); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, transitive_closure_benches); +criterion_main!(benches); + +/// Computes the transitive closure of an acyclic graph using DBSP's +/// [`recursive` API](dbsp::circuit::ChildCircuit::recursive). +fn build_recursive_variant() -> anyhow::Result { + let (handle, (edges_input, closure_output)) = + Runtime::init_circuit(WORKERS, move |root_circuit| { + let (edges, edges_input) = root_circuit.add_input_zset::(); + + let len_1 = edges.map_index(|Tup3(from, to, weight)| { + (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) + }); + + let closure = root_circuit.recursive(|child_circuit, closure: Accumulator| { + let edges = edges.delta0(child_circuit); + let len_1 = len_1.delta0(child_circuit); + + let closure = len_1 + .plus( + &closure + .map_index( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { + (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) + }, + ) + .join_index( + &edges.map_index(|Tup3(from, to, weight)| { + (*from, Tup3(*from, *to, *weight)) + }), + |_end_from, + Tup4(start, _end, cum_weight, hopcnt), + Tup3(_from, to, weight)| { + Some(( + Tup2(*start, *to), + Tup4(*start, *to, cum_weight + weight, hopcnt + 1), + )) + }, + ), + // + ) + // We explicitly mark the stream as distinct to avoid the + // unnecessary implicit distinct. + .mark_distinct(); + + Ok(closure) + })?; + + Ok((edges_input, closure.accumulate_output())) + })?; + + Ok(TransClosureCircuit { + handle, + edges_input, + closure_output, + }) +} + +/// Upper bound on nested iterations before the iterate variant gives up waiting +/// for a fixed point. +const MAX_ITERATIONS: usize = 4096; + +/// Computes the transitive closure of an acyclic graph using DBSP's lower level +/// [`iterate` API](dbsp::circuit::Circuit::iterate). +/// +/// It uses a hybrid termination check: Either stop once the subcircuit +/// reaches a fixed point, or after [`MAX_ITERATIONS`] steps if it never does. +fn build_iterate_variant() -> anyhow::Result { + let (handle, (edges_input, closure_output)) = + Runtime::init_circuit(WORKERS, move |root_circuit| { + let (edges, edges_input) = root_circuit.add_input_zset::(); + + let len_1 = edges.map_index(|Tup3(from, to, weight)| { + (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) + }); + + let closure = root_circuit.iterate(|child_circuit| { + let edges = edges.delta0(child_circuit); + let len_1 = len_1.delta0(child_circuit); + + let (closure_frontier, closure_frontier_feedback) = child_circuit + .add_feedback(Z1::new(OrdIndexedZSet::::default())); + + let new_paths = len_1.plus( + &closure_frontier + .map_index( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { + (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) + }, + ) + .join_index( + &edges.map_index(|Tup3(from, to, weight)| { + (*from, Tup3(*from, *to, *weight)) + }), + |_end_from, + Tup4(start, _end, cum_weight, hopcnt), + Tup3(_from, to, weight)| { + Some(( + Tup2(*start, *to), + Tup4(*start, *to, cum_weight + weight, hopcnt + 1), + )) + }, + ), + ); + + closure_frontier_feedback.connect(&new_paths); + + let closure = Stream::dyn_integrate_trace( + &new_paths.inner(), + &BatchReaderFactories::new::(), + ); + + let child_circuit = child_circuit.clone(); + let consensus = Consensus::new("iterate"); + let step = Cell::new(0usize); + let termination_check = async move || { + let step_count = step.get() + 1; + step.set(step_count); + + // The step bound is data-independent, hence identical on every worker; + // the fixed-point check is per-worker. + let fixedpoint = child_circuit.is_fixedpoint(0); + let iteration_ceiling = step_count >= MAX_ITERATIONS; + let done = consensus.check(fixedpoint || iteration_ceiling).await?; + if done { + // Reset the step counter for the next transaction. + step.set(0); + } + + Ok(done) + }; + + Ok((termination_check, closure.export())) + })?; + + let closure = Stream::dyn_consolidate( + &closure, + &BatchReaderFactories::new::(), + ) + .typed::>() + .accumulate_output(); + + Ok((edges_input, closure)) + })?; + + Ok(TransClosureCircuit { + handle, + edges_input, + closure_output, + }) +} diff --git a/crates/dbsp/benches/transitive_closure_cyclic.rs b/crates/dbsp/benches/transitive_closure_cyclic.rs new file mode 100644 index 00000000000..73625e9f053 --- /dev/null +++ b/crates/dbsp/benches/transitive_closure_cyclic.rs @@ -0,0 +1,373 @@ +//! Benchmark comparing two transitive-closure computation approaches on cyclic +//! graphs similar to `examples/tutorial/tutorial11.rs`: +//! +//! - **recursive**: `Circuit::recursive` (which applies an implicit `distinct` +//! on every nested clock tick) followed by a `Min` aggregation that both keeps +//! the shortest path per pair and guarantees termination on cycles. +//! See [`build_recursive_variant`]. +//! - **iterate**: `Circuit::iterate` with a manual feedback loop that carries +//! only the per-iteration frontier and relies solely on the same `Min` +//! aggregation for termination and does not implicitly add a `distinct`. +//! See [`build_iterate_variant`]. +//! +//! On cyclic graphs the two approaches from the sister benchmark +//! `transitive_closure_acyclic` would not terminate, so both variants aggregate +//! with `Min` to collapse every `(start, end)` pair to its shortest path. +//! That aggregation makes the two variants produce identical closures +//! regardless of graph shape (the benchmark asserts this before measuring). +//! The recursive variant deduplicates twice, once via the hidden `distinct` and +//! once via `Min`, whereas the iterate variant deduplicates only via `Min`. +//! This benchmark asserts that the additional `distinct` of the recursive +//! variant is essentially a no-op because it operates on top of already +//! distinct data. +//! +//! Run with, e.g.: +//! `cargo bench --bench transitive_closure_cyclic` + +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use dbsp::{ + Circuit, Consensus, DBSPHandle, NestedCircuit, NumEntries, OrdIndexedZSet, OutputHandle, + Runtime, Stream, ZSetHandle, ZWeight, + mimalloc::MiMalloc, + operator::{Min, Z1}, + trace::BatchReaderFactories, + typed_batch::SpineSnapshot, + utils::{Tup2, Tup3, Tup4}, +}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use std::cell::Cell; + +#[global_allocator] +static ALLOC: MiMalloc = MiMalloc; + +type NodeId = u64; +type Weight = u64; +type CumWeight = u64; +type Hopcnt = u64; +type WeightedEdge = Tup3; +type Path = Tup2; +type TransClosurePath = Tup4; +type Accumulator = Stream>; + +struct TransClosureCircuit { + handle: DBSPHandle, + edges_input: ZSetHandle, + closure_output: OutputHandle>>, +} + +/// Number of DBSP worker threads used for every circuit in this benchmark. +const WORKERS: usize = 4; + +/// Deterministic seed so every run sees the same graphs. +const SEED: u64 = 0x0BADC0DE_DEADBEEF; + +/// Largest edge weight generated by [`generate_cyclic_graph`]. +const MAX_EDGE_WEIGHT: Weight = 10; + +/// Out-edges emitted per node by [`generate_cyclic_graph`]. +const OUT_DEGREE: usize = 3; + +/// A single-transaction workload: all edges of one cyclic graph. +type Workload = Vec>; + +/// Generates a random weighted *cyclic* graph as a single insertion batch. +/// +/// Each node emits `out_degree` edges to distinct random targets drawn from the +/// whole node range (excluding itself). Unlike the acyclic benchmark, targets +/// may point back to lower ids, so cycles form freely; with `out_degree >= 2` +/// and a few hundred nodes the graph is strongly connected with overwhelming +/// probability, yielding a near-complete transitive closure. +/// +/// Both variants rely on the `Min` aggregation for termination: Because `Min` +/// collapses every `(start, end)` pair to a single shortest path, the two +/// variants produce identical closures no matter how many distinct paths +/// connect a pair of nodes. +fn generate_cyclic_graph(nodes: NodeId, out_degree: usize) -> Workload { + let mut rng = ChaCha8Rng::seed_from_u64(SEED); + // At most `nodes - 1` distinct targets exist per node (self is excluded). + let fanout = out_degree.min(nodes.saturating_sub(1) as usize); + let mut edges = Vec::with_capacity(nodes as usize * fanout); + + for from in 0..nodes { + let mut targets = Vec::with_capacity(fanout); + while targets.len() < fanout { + let to = rng.gen_range(0..nodes); + if to != from && !targets.contains(&to) { + targets.push(to); + } + } + for to in targets { + let weight = rng.gen_range(1..=MAX_EDGE_WEIGHT); + edges.push(Tup2(Tup3(from, to, weight), 1)); + } + } + + edges +} + +/// Feeds a whole graph into a freshly built circuit in one transaction and +/// returns the number of paths in the resulting transitive closure. +/// +/// The circuit (and its worker threads) is dropped on return, so the caller +/// must supply a fresh circuit for each measured iteration. +fn drive_once(mut circuit: TransClosureCircuit, mut workload: Workload) -> usize { + circuit.edges_input.append(&mut workload); + circuit + .handle + .transaction() + .expect("transaction should succeed"); + circuit + .closure_output + .concat() + .consolidate() + .num_entries_shallow() +} + +/// Cross-checks that both variants compute the same closure on the given graph, +/// returning the shared closure size (used both as a sanity check and to label +/// the benchmark with the output cardinality). +fn assert_variants_agree(workload: &Workload) -> usize { + let recursive = build_recursive_variant().expect("build recursive circuit"); + let iterate = build_iterate_variant().expect("build iterate circuit"); + + let mut recursive_circuit = recursive.handle; + let mut iterate_circuit = iterate.handle; + + recursive.edges_input.append(&mut workload.clone()); + iterate.edges_input.append(&mut workload.clone()); + recursive_circuit.transaction().unwrap(); + iterate_circuit.transaction().unwrap(); + + let recursive_closure = recursive.closure_output.concat().consolidate(); + let iterate_closure = iterate.closure_output.concat().consolidate(); + + assert_eq!( + recursive_closure, iterate_closure, + "recursive and iterate variants must agree on cyclic input" + ); + + recursive_closure.num_entries_shallow() +} + +/// The forest sizes (in terms of node counts) we benchmark. The transitive +/// closure of a strongly connected graph is roughly `nodes^2`, +/// so these stay modest. +const GRAPH_SIZES: &[NodeId] = &[100, 200, 400]; + +fn transitive_closure_benches(c: &mut Criterion) { + let mut group = c.benchmark_group("transitive-closure-cyclic"); + // Building a circuit spins up worker threads, so keep the sample count + // modest to bound the wall-clock time of the whole benchmark. + group.sample_size(20); + + for &nodes in GRAPH_SIZES { + let workload = generate_cyclic_graph(nodes, OUT_DEGREE); + let closure_size = assert_variants_agree(&workload); + let shape = format!("n{nodes}"); + println!( + "graph {shape}: {} edges, closure of {closure_size} paths", + workload.len() + ); + + // Report throughput in paths of the computed closure per second so the + // two variants are compared on the same unit of useful work. + group.throughput(Throughput::Elements(closure_size as u64)); + + group.bench_with_input( + BenchmarkId::new("recursive", &shape), + &workload, + |b, workload| { + b.iter_batched( + || { + ( + build_recursive_variant().expect("build recursive circuit"), + workload.clone(), + ) + }, + |(circuit, workload)| drive_once(circuit, workload), + BatchSize::PerIteration, + ); + }, + ); + + group.bench_with_input( + BenchmarkId::new("iterate", &shape), + &workload, + |b, workload| { + b.iter_batched( + || { + ( + build_iterate_variant().expect("build iterate circuit"), + workload.clone(), + ) + }, + |(circuit, workload)| drive_once(circuit, workload), + BatchSize::PerIteration, + ); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, transitive_closure_benches); +criterion_main!(benches); + +/// Computes the transitive closure of a cyclic graph using DBSP's +/// [`recursive` API](dbsp::circuit::ChildCircuit::recursive) plus a `Min` +/// aggregation. +fn build_recursive_variant() -> anyhow::Result { + let (handle, (edges_input, closure_output)) = + Runtime::init_circuit(WORKERS, move |root_circuit| { + let (edges, edges_input) = root_circuit.add_input_zset::(); + + let len_1 = edges.map_index(|Tup3(from, to, weight)| { + (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) + }); + + let closure = root_circuit.recursive(|child_circuit, closure: Accumulator| { + let edges = edges.delta0(child_circuit); + let len_1 = len_1.delta0(child_circuit); + + let closure = len_1 + .plus( + &closure + .map_index( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { + (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) + }, + ) + .join_index( + &edges.map_index(|Tup3(from, to, weight)| { + (*from, Tup3(*from, *to, *weight)) + }), + |_end_from, + Tup4(start, _end, cum_weight, hopcnt), + Tup3(_from, to, weight)| { + Some(( + Tup2(*start, *to), + Tup4(*start, *to, cum_weight + weight, hopcnt + 1), + )) + }, + ), + ) + // The `Min` aggregation keeps the shortest path per pair and, + // crucially, guarantees termination on cyclic graphs. + .aggregate(Min) + // We explicitly mark the stream as distinct to avoid the + // implicit distinct because the aggregation step already + // ensures distinctness. + .mark_distinct(); + + Ok(closure) + })?; + + Ok((edges_input, closure.accumulate_output())) + })?; + + Ok(TransClosureCircuit { + handle, + edges_input, + closure_output, + }) +} + +/// Upper bound on nested iterations before the iterate variant gives up waiting +/// for a fixed point. +const MAX_ITERATIONS: usize = 4096; + +/// Computes the transitive closure of a cyclic graph using DBSP's lower level +/// [`iterate` API](dbsp::circuit::Circuit::iterate) plus a `Min` aggregation. +/// +/// It uses a hybrid termination check: Either stop once the subcircuit +/// reaches a fixed point, or after [`MAX_ITERATIONS`] steps if it never does. +fn build_iterate_variant() -> anyhow::Result { + let (handle, (edges_input, closure_output)) = + Runtime::init_circuit(WORKERS, move |root_circuit| { + let (edges, edges_input) = root_circuit.add_input_zset::(); + + let len_1 = edges.map_index(|Tup3(from, to, weight)| { + (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) + }); + + let closure = root_circuit.iterate(|child_circuit| { + let edges = edges.delta0(child_circuit); + let len_1 = len_1.delta0(child_circuit); + + let (closure_frontier, closure_frontier_feedback) = child_circuit + .add_feedback(Z1::new(OrdIndexedZSet::::default())); + + let new_paths = len_1 + .plus( + &closure_frontier + .map_index( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { + (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) + }, + ) + .join_index( + &edges.map_index(|Tup3(from, to, weight)| { + (*from, Tup3(*from, *to, *weight)) + }), + |_end_from, + Tup4(start, _end, cum_weight, hopcnt), + Tup3(_from, to, weight)| { + Some(( + Tup2(*start, *to), + Tup4(*start, *to, cum_weight + weight, hopcnt + 1), + )) + }, + ), + ) + // No `distinct` here (not even behind the scenes); the `Min` + // aggregation alone deduplicates and stops the iteration. + .aggregate(Min); + + closure_frontier_feedback.connect(&new_paths); + + let closure = Stream::dyn_integrate_trace( + &new_paths.inner(), + &BatchReaderFactories::new::(), + ); + + let child_circuit = child_circuit.clone(); + let consensus = Consensus::new("iterate"); + let step = Cell::new(0usize); + let termination_check = async move || { + let step_count = step.get() + 1; + step.set(step_count); + + // The step bound is data-independent, hence identical on every worker; + // the fixed-point check is per-worker. + let fixedpoint = child_circuit.is_fixedpoint(0); + let iteration_ceiling = step_count >= MAX_ITERATIONS; + let done = consensus.check(fixedpoint || iteration_ceiling).await?; + if done { + // Reset the step counter for the next transaction. + step.set(0); + } + + Ok(done) + }; + + Ok((termination_check, closure.export())) + })?; + + let closure = Stream::dyn_consolidate( + &closure, + &BatchReaderFactories::new::(), + ) + .typed::>() + .accumulate_output(); + + Ok((edges_input, closure)) + })?; + + Ok(TransClosureCircuit { + handle, + edges_input, + closure_output, + }) +} diff --git a/crates/dbsp/examples/tutorial/tutorial13.rs b/crates/dbsp/examples/tutorial/tutorial13.rs index f237b2e98b8..320e63f1620 100644 --- a/crates/dbsp/examples/tutorial/tutorial13.rs +++ b/crates/dbsp/examples/tutorial/tutorial13.rs @@ -192,39 +192,37 @@ fn graph_coloring_recursive(threads: usize) -> Result { let (edges, edges_input) = root_circuit.add_input_zset::(); let (init, init_input) = root_circuit.add_input_zset::(); - let recursive_streams = root_circuit.recursive_dynamic( - 2, + let (red_output, blue_output) = root_circuit.recursive( |child_circuit, - mut recursive_streams: Vec>>| { + (red, blue): ( + Stream>, + Stream>, + )| { let edges = edges.delta0(child_circuit); let init = init.delta0(child_circuit); - let red = &recursive_streams[0]; - let blue = &recursive_streams[1]; - - let new_red = blue - .map_index(|blue_node| (*blue_node, *blue_node)) - .join( + let new_red = + init.plus(&blue.map_index(|blue_node| (*blue_node, *blue_node)).join( &edges.map_index(|Tup2(from, to)| (*from, *to)), |_blue_node, _, new_red_node| *new_red_node, - ) - .plus(&init); + )); let new_blue = red.map_index(|red_node| (*red_node, *red_node)).join( &edges.map_index(|Tup2(from, to)| (*from, *to)), |_red_node, _, new_blue_node| *new_blue_node, ); - recursive_streams[0] = new_red; - recursive_streams[1] = new_blue; - Ok(recursive_streams) + Ok((new_red, new_blue)) }, )?; - let red_output = recursive_streams[0].accumulate_output(); - let blue_output = recursive_streams[1].accumulate_output(); - - Ok(((init_input, edges_input), (red_output, blue_output))) + Ok(( + (init_input, edges_input), + ( + red_output.accumulate_output(), + blue_output.accumulate_output(), + ), + )) })?; Ok(BipartiteGraphCircuit { From 16488f440285e4eb0c2d2b7081b7009f77f73bcf Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Mon, 20 Jul 2026 16:19:08 +0200 Subject: [PATCH 4/5] Reset tutorials Signed-off-by: Leo Stewen --- crates/dbsp/Cargo.toml | 4 - .../examples/tutorial/shared_helper/mod.rs | 62 ---- .../tutorial/transitive_closure/mod.rs | 149 -------- crates/dbsp/examples/tutorial/tutorial10.rs | 312 ++++++----------- crates/dbsp/examples/tutorial/tutorial11.rs | 300 ++++++---------- crates/dbsp/examples/tutorial/tutorial13.rs | 330 ------------------ crates/dbsp/src/tutorial.rs | 8 +- 7 files changed, 221 insertions(+), 944 deletions(-) delete mode 100644 crates/dbsp/examples/tutorial/shared_helper/mod.rs delete mode 100644 crates/dbsp/examples/tutorial/transitive_closure/mod.rs delete mode 100644 crates/dbsp/examples/tutorial/tutorial13.rs diff --git a/crates/dbsp/Cargo.toml b/crates/dbsp/Cargo.toml index fdd6e5e52be..283d0f18d41 100644 --- a/crates/dbsp/Cargo.toml +++ b/crates/dbsp/Cargo.toml @@ -233,10 +233,6 @@ path = "examples/tutorial/tutorial11.rs" name = "tutorial12" path = "examples/tutorial/tutorial12/tutorial12.rs" -[[example]] -name = "tutorial13" -path = "examples/tutorial/tutorial13.rs" - [[example]] name = "coord" path = "examples/dist/coord.rs" diff --git a/crates/dbsp/examples/tutorial/shared_helper/mod.rs b/crates/dbsp/examples/tutorial/shared_helper/mod.rs deleted file mode 100644 index 2c3e47531e2..00000000000 --- a/crates/dbsp/examples/tutorial/shared_helper/mod.rs +++ /dev/null @@ -1,62 +0,0 @@ -use dbsp::{Circuit, Consensus, NestedCircuit, SchedulerError}; -use std::cell::Cell; - -pub fn threads(force_threads: Option) -> usize { - let threads = force_threads.unwrap_or_else(|| { - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4) - }); - println!("Running with {threads} threads"); - threads -} - -pub const MAX_ITERATIONS: usize = 4096; - -/// Hybrid termination for the `iterate`-based variants: stop once the subcircuit -/// reaches a fixed point, or after `max_iterations` steps if it never does. -/// -/// Testing the frontier for emptiness is not a reliable convergence signal with -/// multiple workers. The incremental join buffers results for future nested -/// timestamps, so the frontier delta can be transiently empty for a step while -/// data is still in flight, and the exact gaps vary with cross-worker -/// scheduling. `is_fixedpoint` accounts for that buffered state, so it never -/// stops early. It is a per-worker check, hence we combine the votes across all -/// workers with `Consensus`. -/// -/// The returned closure is run once for every tick of the nested circuit's clock -/// on each worker of DBSP's runtime. Hence, the closure is stateful across nested -/// clock ticks _within_ a transaction and requires resetting between transactions. -/// -/// As the broadcast between all workers happens once per every nested clock tick, -/// a straggler (that is, a worker with more data to process than others) still -/// induces a cross-thread round trip for every worker, even though some of them -/// might have already reached a fixedpoint for their sharded data. Possibly, -/// batching the vote (e.g. every N ticks) is a common trade-off for heavier -/// workloads. -pub fn make_termination_check( - child_circuit: &NestedCircuit, - label: &'static str, - max_iterations: usize, -) -> impl AsyncFn() -> Result + use<> { - let child_circuit = child_circuit.clone(); - let consensus = Consensus::new(label); - let step = Cell::new(0usize); - - async move || { - let step_count = step.get() + 1; - step.set(step_count); - - // The step bound is data-independent, hence identical on every worker; - // the fixed-point check is per-worker. - let fixedpoint = child_circuit.is_fixedpoint(0); - let iteration_ceiling = step_count >= max_iterations; - let done = consensus.check(fixedpoint || iteration_ceiling).await?; - if done { - // Reset the step counter for the next transaction. - step.set(0); - } - - Ok(done) - } -} diff --git a/crates/dbsp/examples/tutorial/transitive_closure/mod.rs b/crates/dbsp/examples/tutorial/transitive_closure/mod.rs deleted file mode 100644 index 97e7712c659..00000000000 --- a/crates/dbsp/examples/tutorial/transitive_closure/mod.rs +++ /dev/null @@ -1,149 +0,0 @@ -#![allow(unused)] -use anyhow::Result; -use dbsp::{ - Circuit, DBSPHandle, IndexedZSetReader, NestedCircuit, OrdIndexedZSet, OutputHandle, - SchedulerError, Stream, ZSetHandle, ZWeight, indexed_zset, - typed_batch::SpineSnapshot, - utils::{Tup2, Tup3, Tup4}, -}; -use std::cell::Cell; - -pub type NodeId = u64; -pub type Weight = u64; -pub type CumWeight = u64; -pub type Hopcnt = u64; -pub type WeightedEdge = Tup3; -pub type Path = Tup2; -pub type TransClosurePath = Tup4; -pub type Accumulator = Stream>; - -pub struct TransClosureCircuit { - pub label: &'static str, - pub handle: DBSPHandle, - pub edges_input: ZSetHandle, - pub closure_output: OutputHandle>>, -} - -/// Input data to form a graph. -/// -/// The outer vector elements are in order of insertion/deletion and steps is -/// the "transaction" number. -pub fn edges_data(steps: usize) -> impl Iterator>> { - vec![ - // The first step adds a graph of four nodes: - // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| - vec![ - Tup2(Tup3(0, 1, 1), 1), - Tup2(Tup3(1, 2, 1), 1), - Tup2(Tup3(2, 3, 2), 1), - Tup2(Tup3(3, 4, 2), 1), - ], - // The second step removes the edge |1| -1-> |2|. - vec![Tup2(Tup3(1, 2, 1), -1)], - // The third step reinserts the edge |1| -1-> |2| and introduces - // a cycle through the edge |4| -3-> |0|. In total, the graph now looks - // like this: - // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| - // ^ | - // | | - // ------------------3------------------ - vec![Tup2(Tup3(1, 2, 1), 1), Tup2(Tup3(4, 0, 3), 1)], - ] - .into_iter() - .take(steps) -} - -/// The expected output of the transitive closure after each transaction/step. -/// -/// `steps` is the "transaction" number. -pub fn expected_closure( - steps: usize, -) -> impl Iterator> { - vec![ - // We expect the full transitive closure in the first step. - indexed_zset! { Path => TransClosurePath: - Tup2(0, 1) => { Tup4(0, 1, 1, 1) => 1 }, - Tup2(0, 2) => { Tup4(0, 2, 2, 2) => 1 }, - Tup2(0, 3) => { Tup4(0, 3, 4, 3) => 1 }, - Tup2(0, 4) => { Tup4(0, 4, 6, 4) => 1 }, - Tup2(1, 2) => { Tup4(1, 2, 1, 1) => 1 }, - Tup2(1, 3) => { Tup4(1, 3, 3, 2) => 1 }, - Tup2(1, 4) => { Tup4(1, 4, 5, 3) => 1 }, - Tup2(2, 3) => { Tup4(2, 3, 2, 1) => 1 }, - Tup2(2, 4) => { Tup4(2, 4, 4, 2) => 1 }, - Tup2(3, 4) => { Tup4(3, 4, 2, 1) => 1 }, - }, - // These paths are removed in the second step. - indexed_zset! { Path => TransClosurePath: - Tup2(0, 2) => { Tup4(0, 2, 2, 2) => -1 }, - Tup2(0, 3) => { Tup4(0, 3, 4, 3) => -1 }, - Tup2(0, 4) => { Tup4(0, 4, 6, 4) => -1 }, - Tup2(1, 2) => { Tup4(1, 2, 1, 1) => -1 }, - Tup2(1, 3) => { Tup4(1, 3, 3, 2) => -1 }, - Tup2(1, 4) => { Tup4(1, 4, 5, 3) => -1 }, - }, - // Third step restores the deletions of the second step and includes - // the newly discovered paths from the cycle. - indexed_zset! { Path => TransClosurePath: - Tup2(0, 2) => { Tup4(0, 2, 2, 2) => 1 }, - Tup2(0, 3) => { Tup4(0, 3, 4, 3) => 1 }, - Tup2(0, 4) => { Tup4(0, 4, 6, 4) => 1 }, - Tup2(1, 2) => { Tup4(1, 2, 1, 1) => 1 }, - Tup2(1, 3) => { Tup4(1, 3, 3, 2) => 1 }, - Tup2(1, 4) => { Tup4(1, 4, 5, 3) => 1 }, - - Tup2(0, 0) => { Tup4(0, 0, 9, 5) => 1 }, - Tup2(1, 0) => { Tup4(1, 0, 8, 4) => 1 }, - Tup2(1, 1) => { Tup4(1, 1, 9, 5) => 1 }, - Tup2(2, 0) => { Tup4(2, 0, 7, 3) => 1 }, - Tup2(2, 1) => { Tup4(2, 1, 8, 4) => 1 }, - Tup2(2, 2) => { Tup4(2, 2, 9, 5) => 1 }, - Tup2(3, 0) => { Tup4(3, 0, 5, 2) => 1 }, - Tup2(3, 1) => { Tup4(3, 1, 6, 3) => 1 }, - Tup2(3, 2) => { Tup4(3, 2, 7, 4) => 1 }, - Tup2(3, 3) => { Tup4(3, 3, 9, 5) => 1 }, - Tup2(4, 0) => { Tup4(4, 0, 3, 1) => 1 }, - Tup2(4, 1) => { Tup4(4, 1, 4, 2) => 1 }, - Tup2(4, 2) => { Tup4(4, 2, 5, 3) => 1 }, - Tup2(4, 3) => { Tup4(4, 3, 7, 4) => 1 }, - Tup2(4, 4) => { Tup4(4, 4, 9, 5) => 1 }, - }, - ] - .into_iter() - .take(steps) -} - -pub fn drive_circuit(circuit: TransClosureCircuit, steps: usize) -> Result { - let label = circuit.label; - let mut handle = circuit.handle; - let edges_input = circuit.edges_input; - let closure_output = circuit.closure_output; - - let mut edges_data = edges_data(steps); - let mut expected_closure_output = expected_closure(steps); - - println!("=== {label} ==="); - - let start = std::time::Instant::now(); - - for i in 1..=steps { - println!("Iteration {} starts...", i); - edges_input.append(&mut edges_data.next().unwrap()); - handle.transaction()?; - let output = closure_output.concat(); - output.iter().for_each( - |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt), z_weight)| { - println!( - "{start} -> {end} (cum weight: {cum_weight}, hops: {hopcnt}) => {z_weight}" - ); - }, - ); - assert_eq!( - output.consolidate(), - expected_closure_output.next().unwrap() - ); - println!("Iteration {} finished.", i); - } - - Ok(start.elapsed().as_micros()) -} diff --git a/crates/dbsp/examples/tutorial/tutorial10.rs b/crates/dbsp/examples/tutorial/tutorial10.rs index 0123a58026b..38090af4e62 100644 --- a/crates/dbsp/examples/tutorial/tutorial10.rs +++ b/crates/dbsp/examples/tutorial/tutorial10.rs @@ -1,216 +1,122 @@ -mod shared_helper; -mod transitive_closure; - use anyhow::Result; +use dbsp::typed_batch::IndexedZSetReader; use dbsp::{ - Circuit, OrdIndexedZSet, Runtime, Stream, ZWeight, - operator::Z1, - trace::BatchReaderFactories, - utils::{Tup2, Tup3, Tup4}, -}; -use shared_helper::{MAX_ITERATIONS, make_termination_check}; -use transitive_closure::{ - Accumulator, Path, TransClosureCircuit, TransClosurePath, WeightedEdge, drive_circuit, + Circuit, OrdZSet, Runtime, Stream, + operator::Generator, + utils::{Tup3, Tup4}, + zset, zset_set, }; -/// If we set STEPS to 3 in this tutorial, the computation will not terminate -/// due to the cycle introduced in the third transaction of the input data. -/// Tutorial 11 fixes the circuit such that the computation terminates again -/// even for cyclic graphs. We recommend to view tutorial 10 and 11 with your -/// favorite diff tool side-by-side to see the differences in both the -/// [`transitive_closure_acyclic_recursive`] and -/// [`transitive_closure_acyclic_iterate`] of each tutorial. -const STEPS: usize = 2; - fn main() -> Result<()> { - // Play around with different thread setups on your machine. With this little - // toy data more than two threads usually mean more unproductive coordination - // overhead but the sweet spot may vary depending on your setup. You might - // want to run this in release mode via: - // `cargo run --profile release --example tutorial10` - let threads = shared_helper::threads(Some(2)); - - let elapsed_rec = drive_circuit(transitive_closure_acyclic_recursive(threads)?, STEPS)?; - let elapsed_iter = drive_circuit(transitive_closure_acyclic_iterate(threads)?, STEPS)?; - - // As the recursive variant performs an unnecessary distinct, you should - // observe a speedup > 1 with the iterate variant. - println!("=== Stats ==="); - println!("Recursive variant: {elapsed_rec} μs"); - println!("Iterate variant: {elapsed_iter} μs"); - let speedup = elapsed_rec as f64 / elapsed_iter as f64; - println!("Speedup factor (recursive (old) relative to iterate (new)): {speedup:.02}"); - - Ok(()) -} - -/// Computes the transitive closure of a graph using DBSP's `recursive` API. The -/// resulting transitive closure consists of (FromNode, ToNode, CumulativeWeight, -/// Hopcount) tuples. -/// -/// # Note -/// -/// This variant only works on acyclic graphs, whereas tutorial 11 fixes this -/// at the expense of some extra aggregation step. -fn transitive_closure_acyclic_recursive(threads: usize) -> Result { - let (handle, (edges_input, closure_output)) = - Runtime::init_circuit(threads, move |root_circuit| { - let (edges, edges_input) = root_circuit.add_input_zset::(); + // Set this value to 3 and uncomment the data in the arrays the Rust compiler + // points out, to see a fixed-point computation which does _not_ terminate. + const STEPS: usize = 2; + + let threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + + let (mut circuit_handle, output_handle) = Runtime::init_circuit( + threads, + move |root_circuit| { + let mut edges_data = ([ + // The first step adds a graph of four nodes: + // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| + zset_set! { Tup3(0_usize, 1_usize, 1_usize), Tup3(1, 2, 1), Tup3(2, 3, 2), Tup3(3, 4, 2) }, + // The second step removes the edge |1| -1-> |2|. + zset! { Tup3(1, 2, 1) => -1 }, + // The third step would introduce a cycle but that would + // cause the fixed-point computation to never terminate because we keep + // on finding new paths with a higher cumulative weight and hopcount. + // In total, we have the following graph: + // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| + // ^ | + // | | + // ------------------3------------------ + // zset_set! { Tup3(1,2,1), Tup3(4, 0, 3)} + ] as [OrdZSet>; STEPS]) + .into_iter(); + + let edges = root_circuit.add_source(Generator::new(move || edges_data.next().unwrap())); // Create a base stream with all paths of length 1. - let len_1 = edges.map_index(|Tup3(from, to, weight)| { - (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) - }); - - let closure = root_circuit.recursive(|child_circuit, closure: Accumulator| { - // Import the edges and len_1 streams from the parent circuit into - // the child circuit. A nested circuit has its own nested clock for - // each tick of the parent clock. For each _first tick_ of the - // _nested clock_ delta0 emits the data from the parent circuit - // and on subsequent ticks (of the nested clock) it is empty. - let edges = edges.delta0(child_circuit); - let len_1 = len_1.delta0(child_circuit); - - // Extend the current closure by joining the previously identified - // paths of length n-1 with the edges to also find paths of length n. - let closure = len_1.plus( - &closure - .map_index( - |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { - (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) - }, - ) - .join_index( - &edges.map_index(|Tup3(from, to, weight)| { - (*from, Tup3(*from, *to, *weight)) - }), + let len_1 = edges.map(|Tup3(from, to, weight)| Tup4(*from, *to, *weight, 1)); + + let closure = root_circuit.recursive( + |child_circuit, len_n_minus_1: Stream<_, OrdZSet>>| { + // Import the `edges` and `len_1` stream from the parent circuit. + let edges = edges.delta0(child_circuit); + let len_1 = len_1.delta0(child_circuit); + + // Perform an iterative step (n-1 to n) through joining the + // paths of length n-1 with the edges. + let len_n = len_n_minus_1 + .map_index(|Tup4(start, end, cum_weight, hopcnt)| { + (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) + }) + .join( + &edges + .map_index(|Tup3(from, to, weight)| (*from, Tup3(*from, *to, *weight))), |_end_from, - Tup4(start, _end, cum_weight, hopcnt), - Tup3(_from, to, weight)| { - Some(( - Tup2(*start, *to), - Tup4(*start, *to, cum_weight + weight, hopcnt + 1), - )) - }, - ), - ); - // Note: Unlike in tutorial 11, there is no aggregation happening - // here, causing the circuit to rediscover paths which live - // on a cycle over and over again but with higher cumulative - // weights and hopcounts if operating on cyclic graphs. - // The implicit distinct of the `ChildCircuit::recursive` API does - // not fix the termination either, as the value domain carries the - // changing values (cumulative weight and hopcount). - - Ok(closure) - })?; - - Ok((edges_input, closure.accumulate_output())) - })?; - - Ok(TransClosureCircuit { - label: "Recursive Variant", - handle, - edges_input, - closure_output, - }) -} - -/// Computes the transitive closure of a graph using DBSP's lower level `iterate` -/// API. Again, the resulting transitive closure consists of (FromNode, ToNode, -/// CumulativeWeight, Hopcount) tuples. -/// -/// # Note -/// -/// This variant only works on acyclic graphs, whereas tutorial 11 fixes this -/// at the expense of some extra aggregation step. This iterate variant is faster -/// than [`transitive_closure_acyclic_recursive`] because you don't have to pay -/// for deduplication (via distinct) in case you know your data is acyclic anyways. -/// This is similar to the difference between UNION vs UNION ALL in recursive -/// CTEs in SQL, if that is familiar to you. -fn transitive_closure_acyclic_iterate(threads: usize) -> Result { - let (handle, (edges_input, closure_output)) = - Runtime::init_circuit(threads, move |root_circuit| { - let (edges, edges_input) = root_circuit.add_input_zset::(); - - let len_1 = edges.map_index(|Tup3(from, to, weight)| { - (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) - }); - - let closure = root_circuit.iterate(|child_circuit| { - let edges = edges.delta0(child_circuit); - let len_1 = len_1.delta0(child_circuit); - - // The closure frontier carries just the frontier, that is, - // the delta from the previous iteration. - let (closure_frontier, closure_frontier_feedback) = child_circuit - .add_feedback(Z1::new(OrdIndexedZSet::::default())); - - let new_paths = len_1.plus( - &closure_frontier - .map_index( - |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { - (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) + Tup4(start, _end, cum_weight, hopcnt), + Tup3(_from, to, weight)| { + Tup4(*start, *to, cum_weight + weight, hopcnt + 1) }, ) - .join_index( - &edges.map_index(|Tup3(from, to, weight)| { - (*from, Tup3(*from, *to, *weight)) - }), - |_end_from, - Tup4(start, _end, cum_weight, hopcnt), - Tup3(_from, to, weight)| { - Some(( - Tup2(*start, *to), - Tup4(*start, *to, cum_weight + weight, hopcnt + 1), - )) - }, - ), + .plus(&len_1); + + Ok(len_n) + }, + )?; + + Ok(closure.output()) + }, + )?; + + let mut expected_outputs = ([ + // We expect the full transitive closure in the first step. + zset! { + Tup4(0, 1, 1, 1) => 1, + Tup4(0, 2, 2, 2) => 1, + Tup4(0, 3, 4, 3) => 1, + Tup4(0, 4, 6, 4) => 1, + Tup4(1, 2, 1, 1) => 1, + Tup4(1, 3, 3, 2) => 1, + Tup4(1, 4, 5, 3) => 1, + Tup4(2, 3, 2, 1) => 1, + Tup4(2, 4, 4, 2) => 1, + Tup4(3, 4, 2, 1) => 1, + }, + // These paths are removed in the second step. + zset! { + Tup4(0, 2, 2, 2) => -1, + Tup4(0, 3, 4, 3) => -1, + Tup4(0, 4, 6, 4) => -1, + Tup4(1, 2, 1, 1) => -1, + Tup4(1, 3, 3, 2) => -1, + Tup4(1, 4, 5, 3) => -1, + }, + // This does not matter, as the computation does not terminate + // anymore due to the cycle. + // zset! {}, + ] as [OrdZSet>; STEPS]) + .into_iter(); + + for i in 0..STEPS { + let iteration = i + 1; + println!("Iteration {} starts...", iteration); + circuit_handle.transaction()?; + let output = output_handle.consolidate(); + assert_eq!(output, expected_outputs.next().unwrap()); + output + .iter() + .for_each(|(Tup4(start, end, cum_weight, hopcnt), _, z_weight)| { + println!( + "{start} -> {end} (cum weight: {cum_weight}, hops: {hopcnt}) => {z_weight}" ); + }); + println!("Iteration {} finished.", iteration); + } - // Hook up the Z1 operator with the current frontier (new_paths) - // such that it becomes the previous frontier in the next iteration. - closure_frontier_feedback.connect(&new_paths); - - // Accumulate every frontier across the inner iterations and export - // the result to the parent. We use `integrate_trace` (from the dynamic - // API, mirroring the behavior of `ChildCircuit::recursive`) rather than - // `integrate`. This is because `integrate` holds its running sum in - // a `Z1`, whose conservative fixed-point check (input and output - // both empty) never holds once paths accumulate, which would keep - // the subcircuit from ever reaching a fixed point. The trace-append - // operators behind `integrate_trace` always report a fixed point, - // so convergence is detected as soon as the frontier drains. - let closure = Stream::dyn_integrate_trace( - &new_paths.inner(), - &BatchReaderFactories::new::(), - ); - - let termination_check = - make_termination_check(child_circuit, "tutorial 10 (iterate)", MAX_ITERATIONS); - - Ok((termination_check, closure.export())) - })?; - - // Consolidate the exported trace `closure` into an OrdIndexedZSet. - // After the nested circuit, each element in the input streams is a - // trace, consisting of multiple batches of updates. This consolidates - // the trace into a single batch, which uses less memory and can be - // handled more efficiently by most operators than the trace. - let closure = Stream::dyn_consolidate( - &closure, - &BatchReaderFactories::new::(), - ) - .typed::>() - .accumulate_output(); - - Ok((edges_input, closure)) - })?; - - Ok(TransClosureCircuit { - label: "Iterate Variant", - handle, - edges_input, - closure_output, - }) + Ok(()) } diff --git a/crates/dbsp/examples/tutorial/tutorial11.rs b/crates/dbsp/examples/tutorial/tutorial11.rs index 97afd538238..09580d4b1ea 100644 --- a/crates/dbsp/examples/tutorial/tutorial11.rs +++ b/crates/dbsp/examples/tutorial/tutorial11.rs @@ -1,214 +1,134 @@ -mod shared_helper; -mod transitive_closure; - use anyhow::Result; +use dbsp::OrdZSet; +use dbsp::typed_batch::IndexedZSetReader; use dbsp::{ - Circuit, OrdIndexedZSet, Runtime, Stream, ZWeight, - operator::{Min, Z1}, - trace::BatchReaderFactories, + Circuit, NestedCircuit, OrdIndexedZSet, Runtime, Stream, indexed_zset, + operator::{Generator, Min}, utils::{Tup2, Tup3, Tup4}, -}; -use shared_helper::{MAX_ITERATIONS, make_termination_check}; -use transitive_closure::{ - Accumulator, Path, TransClosureCircuit, TransClosurePath, WeightedEdge, drive_circuit, + zset_set, }; -// Now we allow for a cyclic graph by including the data from the third -// transaction which introduces a cycle to the graph. -const STEPS: usize = 3; +type Accumulator = + Stream, Tup4>>; fn main() -> Result<()> { - // Play around with different thread setups on your machine. With this little - // toy data more than two threads usually mean more unproductive coordination - // overhead but the sweet spot may vary depending on your setup. You might - // want to run this in release mode via: - // `cargo run --profile release --example tutorial11` - let threads = shared_helper::threads(Some(2)); - - let elapsed_rec = drive_circuit(transitive_closure_cyclic_recursive(threads)?, STEPS)?; - let elapsed_iter = drive_circuit(transitive_closure_cyclic_iterate(threads)?, STEPS)?; - - // As the recursive variant performs a duplicated distinct, you should - // observe a (small) speedup > 1 with the iterate variant. - println!("=== Stats ==="); - println!("Recursive variant: {elapsed_rec} μs"); - println!("Iterate variant: {elapsed_iter} μs"); - let speedup = elapsed_rec as f64 / elapsed_iter as f64; - println!("Speedup factor (recursive (old) relative to iterate (new)): {speedup:.02}"); - - Ok(()) -} - -/// Computes the transitive closure of a graph using DBSP's `recursive` API. The -/// resulting transitive closure consists of (FromNode, ToNode, CumulativeWeight, -/// Hopcount) tuples. -/// -/// # Note -/// -/// While this works on both cyclic and acyclic graphs (unlike tutorial 10), -/// it deduplicates twice: First, implicitly due to the recursive API and, -/// second, due to the aggregation which is required for termination upon -/// cyclic graphs. The [`transitive_closure_cyclic_iterate`] below fixes this -/// and only relies on the aggregation to ensure termination and is therefore -/// a bit faster. -fn transitive_closure_cyclic_recursive(threads: usize) -> Result { - let (handle, (edges_input, closure_output)) = - Runtime::init_circuit(threads, move |root_circuit| { - let (edges, edges_input) = root_circuit.add_input_zset::(); + const STEPS: usize = 2; + + let threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + + let (mut circuit_handle, output_handle) = Runtime::init_circuit( + threads, + move |root_circuit| { + let mut edges_data = ([ + // The first step adds a graph of four nodes, just like before: + // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| + zset_set! { Tup3(0_usize, 1_usize, 1_usize), Tup3(1, 2, 1), Tup3(2, 3, 2), Tup3(3, 4, 2) }, + // The second step introduces a cycle. Due to the code changes below, + // the query does terminate though. In total, the graph now looks + // like this: + // |0| -1-> |1| -1-> |2| -2-> |3| -2-> |4| + // ^ | + // | | + // ------------------3------------------ + zset_set! { Tup3(4, 0, 3)} + ] as [OrdZSet>; STEPS]) + .into_iter(); + + let edges = root_circuit.add_source(Generator::new(move || edges_data.next().unwrap())); // Create a base stream with all paths of length 1. let len_1 = edges.map_index(|Tup3(from, to, weight)| { (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) }); - let closure = root_circuit.recursive(|child_circuit, closure: Accumulator| { - // Import the edges and len_1 streams from the parent circuit into - // the child circuit. A nested circuit has its own nested clock for - // each tick of the parent clock. For each _first tick_ of the - // _nested clock_ delta0 emits the data from the parent circuit - // and on subsequent ticks (of the nested clock) it is empty. + let closure = root_circuit.recursive(|child_circuit, len_n_minus_1: Accumulator| { + // Import the `edges` and `len_1` stream from the parent circuit. let edges = edges.delta0(child_circuit); let len_1 = len_1.delta0(child_circuit); - // Extend the current closure by joining the previously identified - // paths of length n-1 with the edges to also find paths of length n. - let closure = len_1 - .plus( - &closure - .map_index( - |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { - (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) - }, - ) - .join_index( - &edges.map_index(|Tup3(from, to, weight)| { - (*from, Tup3(*from, *to, *weight)) - }), - |_end_from, - Tup4(start, _end, cum_weight, hopcnt), - Tup3(_from, to, weight)| { - Some(( - Tup2(*start, *to), - Tup4(*start, *to, cum_weight + weight, hopcnt + 1), - )) - }, - ), + // Perform an iterative step (n-1 to n) through joining the + // paths of length n-1 with the edges. + let len_n = len_n_minus_1 + .map_index( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { + (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) + }, ) - // The distinct is hidden but still happening due to the way - // the `ChildCircuit::recursive` API works. - .aggregate(Min); - - Ok(closure) - })?; - - Ok((edges_input, closure.accumulate_output())) - })?; - - Ok(TransClosureCircuit { - label: "Recursive Variant", - handle, - edges_input, - closure_output, - }) -} - -/// Computes the transitive closure of a graph using DBSP's lower level `iterate` -/// API. Again, the resulting transitive closure consists of (FromNode, ToNode, -/// CumulativeWeight, Hopcount) tuples. -/// -/// # Note -/// -/// This variant also works on cyclic graphs. The iterate variant is faster -/// than the [`transitive_closure_cyclic_recursive`] because you don't have to -/// pay for deduplication (via distinct) despite already being accounted for by -/// the min-aggregation. -fn transitive_closure_cyclic_iterate(threads: usize) -> Result { - let (handle, (edges_input, closure_output)) = - Runtime::init_circuit(threads, move |root_circuit| { - let (edges, edges_input) = root_circuit.add_input_zset::(); - - let len_1 = edges.map_index(|Tup3(from, to, weight)| { - (Tup2(*from, *to), Tup4(*from, *to, *weight, 1)) - }); - - let closure = root_circuit.iterate(|child_circuit| { - let edges = edges.delta0(child_circuit); - let len_1 = len_1.delta0(child_circuit); - - // The closure frontier carries just the frontier, that is, - // the delta from the previous iteration. - let (closure_frontier, closure_frontier_feedback) = child_circuit - .add_feedback(Z1::new(OrdIndexedZSet::::default())); - - let new_paths = len_1 - .plus( - &closure_frontier - .map_index( - |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt))| { - (*end, Tup4(*start, *end, *cum_weight, *hopcnt)) - }, - ) - .join_index( - &edges.map_index(|Tup3(from, to, weight)| { - (*from, Tup3(*from, *to, *weight)) - }), - |_end_from, - Tup4(start, _end, cum_weight, hopcnt), - Tup3(_from, to, weight)| { - Some(( - Tup2(*start, *to), - Tup4(*start, *to, cum_weight + weight, hopcnt + 1), - )) - }, - ), + .join_index( + &edges + .map_index(|Tup3(from, to, weight)| (*from, Tup3(*from, *to, *weight))), + |_end_from, + Tup4(start, _end, cum_weight, hopcnt), + Tup3(_from, to, weight)| { + Some(( + Tup2(*start, *to), + Tup4(*start, *to, cum_weight + weight, hopcnt + 1), + )) + }, ) - // No distinct here (and not even behind the scenes), - // just aggregation with Min to stop iteration. + .plus(&len_1) .aggregate(Min); - // Hook up the Z1 operator with the current frontier (new_paths) - // such that it becomes the previous frontier in the next iteration. - closure_frontier_feedback.connect(&new_paths); - - // Accumulate every frontier across the inner iterations and export - // the result to the parent. We use `integrate_trace` (from the dynamic - // API, mirroring the behavior of `ChildCircuit::recursive`) rather than - // `integrate`. This is because `integrate` holds its running sum in - // a `Z1`, whose conservative fixed-point check (input and output - // both empty) never holds once paths accumulate, which would keep - // the subcircuit from ever reaching a fixed point. The trace-append - // operators behind `integrate_trace` always report a fixed point, - // so convergence is detected as soon as the frontier drains. - let closure = Stream::dyn_integrate_trace( - &new_paths.inner(), - &BatchReaderFactories::new::(), - ); - - let termination_check = - make_termination_check(child_circuit, "tutorial 11 (iterate)", MAX_ITERATIONS); - - Ok((termination_check, closure.export())) + Ok(len_n) })?; - // Consolidate the exported trace `closure` into an OrdIndexedZSet. - // After the nested circuit, each element in the input streams is a - // trace, consisting of multiple batches of updates. This consolidates - // the trace into a single batch, which uses less memory and can be - // handled more efficiently by most operators than the trace. - let closure = Stream::dyn_consolidate( - &closure, - &BatchReaderFactories::new::(), - ) - .typed::>() - .accumulate_output(); - - Ok((edges_input, closure)) - })?; + Ok(closure.output()) + }, + )?; + + let mut expected_outputs = ([ + // The transitive closure in the first step remains the same as in + // `tutorial10.rs`. + indexed_zset! { Tup2 => Tup4: + Tup2(0, 1) => { Tup4(0, 1, 1, 1) => 1 }, + Tup2(0, 2) => { Tup4(0, 2, 2, 2) => 1 }, + Tup2(0, 3) => { Tup4(0, 3, 4, 3) => 1 }, + Tup2(0, 4) => { Tup4(0, 4, 6, 4) => 1 }, + Tup2(1, 2) => { Tup4(1, 2, 1, 1) => 1 }, + Tup2(1, 3) => { Tup4(1, 3, 3, 2) => 1 }, + Tup2(1, 4) => { Tup4(1, 4, 5, 3) => 1 }, + Tup2(2, 3) => { Tup4(2, 3, 2, 1) => 1 }, + Tup2(2, 4) => { Tup4(2, 4, 4, 2) => 1 }, + Tup2(3, 4) => { Tup4(3, 4, 2, 1) => 1 }, + }, + // The second step's introduction of a cycle yields these new paths. + indexed_zset! { Tup2 => Tup4: + Tup2(0, 0) => { Tup4(0, 0, 9, 5) => 1 }, + Tup2(1, 0) => { Tup4(1, 0, 8, 4) => 1 }, + Tup2(1, 1) => { Tup4(1, 1, 9, 5) => 1 }, + Tup2(2, 0) => { Tup4(2, 0, 7, 3) => 1 }, + Tup2(2, 1) => { Tup4(2, 1, 8, 4) => 1 }, + Tup2(2, 2) => { Tup4(2, 2, 9, 5) => 1 }, + Tup2(3, 0) => { Tup4(3, 0, 5, 2) => 1 }, + Tup2(3, 1) => { Tup4(3, 1, 6, 3) => 1 }, + Tup2(3, 2) => { Tup4(3, 2, 7, 4) => 1 }, + Tup2(3, 3) => { Tup4(3, 3, 9, 5) => 1 }, + Tup2(4, 0) => { Tup4(4, 0, 3, 1) => 1 }, + Tup2(4, 1) => { Tup4(4, 1, 4, 2) => 1 }, + Tup2(4, 2) => { Tup4(4, 2, 5, 3) => 1 }, + Tup2(4, 3) => { Tup4(4, 3, 7, 4) => 1 }, + Tup2(4, 4) => { Tup4(4, 4, 9, 5) => 1 }, + }, + ] as [_; STEPS]) + .into_iter(); + + for i in 0..STEPS { + let iteration = i + 1; + println!("Iteration {} starts...", iteration); + circuit_handle.transaction()?; + let output = output_handle.consolidate(); + assert_eq!(output, expected_outputs.next().unwrap()); + output.iter().for_each( + |(Tup2(_start, _end), Tup4(start, end, cum_weight, hopcnt), z_weight)| { + println!( + "{start} -> {end} (cum weight: {cum_weight}, hops: {hopcnt}) => {z_weight}" + ); + }, + ); + println!("Iteration {} finished.", iteration); + } - Ok(TransClosureCircuit { - label: "Iterate Variant", - handle, - edges_input, - closure_output, - }) + Ok(()) } diff --git a/crates/dbsp/examples/tutorial/tutorial13.rs b/crates/dbsp/examples/tutorial/tutorial13.rs deleted file mode 100644 index 320e63f1620..00000000000 --- a/crates/dbsp/examples/tutorial/tutorial13.rs +++ /dev/null @@ -1,330 +0,0 @@ -mod shared_helper; -mod transitive_closure; - -use anyhow::Result; -use dbsp::{ - Circuit, DBSPHandle, NestedCircuit, OrdIndexedZSet, OrdZSet, OutputHandle, Runtime, Stream, - ZSetHandle, ZWeight, - operator::Z1, - trace::BatchReaderFactories, - typed_batch::{IndexedZSetReader, SpineSnapshot}, - utils::Tup2, - zset, -}; -use shared_helper::{MAX_ITERATIONS, make_termination_check}; - -fn main() -> Result<()> { - // Play around with different thread setups on your machine. With this little - // toy data more than two threads usually mean more unproductive coordination - // overhead but the sweet spot may vary depending on your setup. You might - // want to run this in release mode via: - // `cargo run --profile release --example tutorial13` - let threads = shared_helper::threads(Some(2)); - - let elapsed_rec = drive_circuit(graph_coloring_recursive(threads)?)?; - let elapsed_iter = drive_circuit(graph_coloring_iterate(threads)?)?; - - // As the work performed by both variants in this tutorial is roughly the - // same, you should observe comparable numbers. - println!("=== Stats ==="); - println!("Recursive variant: {elapsed_rec} μs"); - println!("Iterate variant: {elapsed_iter} μs"); - let speedup = elapsed_rec as f64 / elapsed_iter as f64; - println!("Speedup factor (recursive (old) relative to iterate (new)): {speedup:.02}"); - - Ok(()) -} - -type Node = usize; -type Edge = Tup2; - -struct BipartiteGraphCircuit { - label: &'static str, - handle: DBSPHandle, - init_input: ZSetHandle, - edges_input: ZSetHandle, - red_output: OutputHandle>>, - blue_output: OutputHandle>>, -} - -const STEPS: usize = 4; - -fn init_data() -> impl Iterator>> { - // We start the discovery by marking node `|0|` red. - let data = [vec![Tup2(0, 1)], vec![], vec![], vec![]]; - data.into_iter() -} - -fn edges_data() -> impl Iterator>> { - let data = [ - // The first step adds a graph of four nodes: - // |0| --> |1| --> |2| --> |3| --> |4| - vec![ - Tup2(Tup2(0, 1), 1), - Tup2(Tup2(1, 2), 1), - Tup2(Tup2(2, 3), 1), - Tup2(Tup2(3, 4), 1), - ], - // Now, we have the following graph in total: - // |0| --> |1| --> |2| --> |3| --> |4| - // ^ | - // | | - // ------ |5| <----- - vec![Tup2(Tup2(2, 5), 1), Tup2(Tup2(5, 0), 1)], - // And we introduce an odd-length cycle, rendering the graph - // non-bipartite anymore (all nodes are red _and_ blue): - // |0| --> |1| --> |2| --> |3| --> |4| - // ^ | | - // | | | - // ------ |5| <----- | - // | | - // --------------------------------- - vec![Tup2(Tup2(4, 0), 1)], - vec![Tup2(Tup2(0, 1), -1)], - ]; - data.into_iter() -} - -fn expected_red() -> impl Iterator> { - let data = [ - zset! { - 0 => 1, - 2 => 1, - 4 => 1, - }, - zset! {}, - zset! { - 1 => 1, - 3 => 1, - 5 => 1, - }, - zset! { - 1 => -1, - 2 => -1, - 3 => -1, - 4 => -1, - 5 => -1, - }, - ]; - data.into_iter() -} - -fn expected_blue() -> impl Iterator> { - let data = [ - zset! { - 1 => 1, - 3 => 1, - }, - zset! { - 5 => 1, - }, - zset! { - 0 => 1, - 2 => 1, - 4 => 1, - }, - zset! { - 0 => -1, - 1 => -1, - 2 => -1, - 3 => -1, - 4 => -1, - 5 => -1, - }, - ]; - data.into_iter() -} - -fn drive_circuit(circuit: BipartiteGraphCircuit) -> Result { - let label = circuit.label; - let mut handle = circuit.handle; - let init_input = circuit.init_input; - let edges_input = circuit.edges_input; - let red_output = circuit.red_output; - let blue_output = circuit.blue_output; - - let mut init_data = init_data(); - let mut edges_data = edges_data(); - let mut expected_blue_output = expected_blue(); - let mut expected_red_output = expected_red(); - - println!("=== {label} ==="); - - let start = std::time::Instant::now(); - - for i in 1..=STEPS { - println!("Iteration {i} starts..."); - init_input.append(&mut init_data.next().unwrap()); - edges_input.append(&mut edges_data.next().unwrap()); - handle.transaction().unwrap(); - let red_output = red_output.concat(); - red_output - .iter() - .for_each(|(node, _, zweight)| println!("Red {node} => {zweight}")); - let blue_output = blue_output.concat(); - blue_output - .iter() - .for_each(|(node, _, zweight)| println!("Blue {node} => {zweight}")); - assert_eq!( - red_output.consolidate(), - expected_red_output.next().unwrap() - ); - assert_eq!( - blue_output.consolidate(), - expected_blue_output.next().unwrap() - ); - println!("Iteration {i} finished."); - } - - Ok(start.elapsed().as_micros()) -} - -/// We compute the graph-coloring (by painting nodes red and blue) which is -/// one way to figure out if a graph is bipartite. The computation is -/// mutually recursive and happens to use the -/// [`ChildCircuit::recursive_dynamic`](`dbsp::circuit::ChildCircuit::recursive_dynamic`) -/// API but if you know your recursive streams at compile time you could also use -/// [`ChildCircuit::recursive`](`dbsp::circuit::ChildCircuit::recursive`) but -/// with a tuple carrying the recursive red and blue streams. -fn graph_coloring_recursive(threads: usize) -> Result { - let (handle, ((init_input, edges_input), (red_output, blue_output))) = - Runtime::init_circuit(threads, move |root_circuit| { - let (edges, edges_input) = root_circuit.add_input_zset::(); - let (init, init_input) = root_circuit.add_input_zset::(); - - let (red_output, blue_output) = root_circuit.recursive( - |child_circuit, - (red, blue): ( - Stream>, - Stream>, - )| { - let edges = edges.delta0(child_circuit); - let init = init.delta0(child_circuit); - - let new_red = - init.plus(&blue.map_index(|blue_node| (*blue_node, *blue_node)).join( - &edges.map_index(|Tup2(from, to)| (*from, *to)), - |_blue_node, _, new_red_node| *new_red_node, - )); - - let new_blue = red.map_index(|red_node| (*red_node, *red_node)).join( - &edges.map_index(|Tup2(from, to)| (*from, *to)), - |_red_node, _, new_blue_node| *new_blue_node, - ); - - Ok((new_red, new_blue)) - }, - )?; - - Ok(( - (init_input, edges_input), - ( - red_output.accumulate_output(), - blue_output.accumulate_output(), - ), - )) - })?; - - Ok(BipartiteGraphCircuit { - label: "Recursive Variant", - handle, - init_input, - edges_input, - red_output, - blue_output, - }) -} - -/// This iterate-based variant is a hand-rolled expansion of [`graph_coloring_recursive`], -/// built on the lower-level [`Circuit::iterate`] primitive. DBSP's recursive API -/// splices a `distinct` onto every output stream for us (see -/// [`ChildCircuit::recursive`](`dbsp::circuit::ChildCircuit::recursive`)); -/// with `iterate` we are responsible for deduplication ourselves. -fn graph_coloring_iterate(threads: usize) -> Result { - let (handle, (init_input, edges_input, red_output, blue_output)) = - Runtime::init_circuit(threads, move |root_circuit| { - let (init, init_input) = root_circuit.add_input_zset::(); - let (edges, edges_input) = root_circuit.add_input_zset::(); - - let recursive_streams = root_circuit.iterate(|child_circuit| { - let edges = edges.delta0(child_circuit); - let init = init.delta0(child_circuit); - let edges_indexed = edges.map_index(|Tup2(from, to)| (*from, *to)); - - // Feedback carries only the frontier (the delta from the last step). - let (red, red_feedback) = - child_circuit.add_feedback(Z1::new(OrdZSet::::default())); - let (blue, blue_feedback) = - child_circuit.add_feedback(Z1::new(OrdZSet::::default())); - - /// Convert a stream of Nodes into the indexed form the join expects and - /// extend it by one hop along `edges`. Deduplication is left to the caller, - /// because it must wrap the whole per-step expression (base case included) to - /// produce the correct cross-transaction delta. - fn hop( - frontier: &Stream>, - edges: &Stream>, - ) -> Stream> { - frontier - .map_index(|node| (*node, *node)) - .join(edges, |_from, _node, to| *to) - } - - // Extend each frontier by one hop. At clock tick 0 the frontier is - // empty, so `new_red` is just the base case `init`; from clock tick 1 - // on `init` is empty, so it is purely `frontier ⋈ edges`. - // `distinct` caps the weight of every node at 1, which both - // removes within-step duplicates and stops nodes discovered in - // an earlier step (or transaction) from being re-emitted. It - // must wrap the entire expression, base case included. - let new_red = init.plus(&hop(&blue, &edges_indexed)).distinct(); - let new_blue = hop(&red, &edges_indexed).distinct(); - - red_feedback.connect(&new_red); - blue_feedback.connect(&new_blue); - - // Accumulate every frontier across the nested clock cycles into the - // full colorings and export them to the parent. - let red = Stream::dyn_integrate_trace( - &new_red.inner(), - &BatchReaderFactories::new::(), - ); - let blue = Stream::dyn_integrate_trace( - &new_blue.inner(), - &BatchReaderFactories::new::(), - ); - - let termination_check = - make_termination_check(child_circuit, "tutorial 13 (iterate)", MAX_ITERATIONS); - - Ok((termination_check, vec![red.export(), blue.export()])) - })?; - - // Consolidate each exported trace (red and blue) into a single ZSet. - // After the nested circuit, each element in the input streams is a - // trace, consisting of multiple batches of updates. This consolidates - // the trace into a single batch, which uses less memory and can be - // handled more efficiently by most operators than the trace. - let mut recursive_streams = recursive_streams - .into_iter() - .map(|s| { - Stream::dyn_consolidate(&s, &BatchReaderFactories::new::()) - .typed::>() - .accumulate_output() - }) - .collect::>(); - let blue_output = recursive_streams.pop().unwrap(); - let red_output = recursive_streams.pop().unwrap(); - - Ok((init_input, edges_input, red_output, blue_output)) - })?; - - Ok(BipartiteGraphCircuit { - label: "Iterate Variant", - handle, - init_input, - edges_input, - red_output, - blue_output, - }) -} diff --git a/crates/dbsp/src/tutorial.rs b/crates/dbsp/src/tutorial.rs index 89ca47b0837..d87b81a0bfa 100644 --- a/crates/dbsp/src/tutorial.rs +++ b/crates/dbsp/src/tutorial.rs @@ -2293,10 +2293,6 @@ //! As this is a little bit more involved than what would fit in here, we defer //! the interested reader to the example given in `tutorial12.rs` which //! demonstrates mutual recursion in the domain of static program analysis. -//! `tutorial13.rs` provides another example on graph colouring which additionally -//! shows how to manually construct recursive circuits using the lower-level -//! [`Circuit::iterate`] API as opposed to the higher-level -//! [`ChildCircuit::recursive`] API. //! //! # Next steps //! @@ -2318,8 +2314,8 @@ //! let (mut circuit, (/*handles*/)) = Runtime::init_circuit(4, build_circuit)?; //! ``` use crate::{ - ChildCircuit, Circuit, CircuitHandle, IndexedZSet, OrdPartitionedIndexedZSet, OutputHandle, - RootCircuit, Runtime, Stream, ZSet, ZSetHandle, + ChildCircuit, CircuitHandle, IndexedZSet, OrdPartitionedIndexedZSet, OutputHandle, RootCircuit, + Runtime, Stream, ZSet, ZSetHandle, operator::{Aggregator, Max, dynamic::aggregate}, utils::{Tup0, Tup1, Tup10}, }; From c73daf126c45779370eb8ec12f9146e9604feb32 Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Mon, 20 Jul 2026 17:08:03 +0200 Subject: [PATCH 5/5] Reset iterate_* APIs to intentionally not use Consensus Signed-off-by: Leo Stewen --- crates/dbsp/src/circuit/circuit_builder.rs | 25 ++-- crates/dbsp/src/circuit/runtime.rs | 7 +- crates/dbsp/src/operator.rs | 2 +- crates/dbsp/src/operator/condition.rs | 131 +++------------------ crates/dbsp/src/operator/recursive.rs | 4 +- 5 files changed, 31 insertions(+), 138 deletions(-) diff --git a/crates/dbsp/src/circuit/circuit_builder.rs b/crates/dbsp/src/circuit/circuit_builder.rs index f2446689231..b17ea080051 100644 --- a/crates/dbsp/src/circuit/circuit_builder.rs +++ b/crates/dbsp/src/circuit/circuit_builder.rs @@ -34,6 +34,7 @@ use crate::{ QuaternaryOperator, SinkOperator, SourceOperator, StrictUnaryOperator, TernaryOperator, TernarySinkOperator, UnaryOperator, }, + runtime::Consensus, schedule::{ CommitProgress, DynamicScheduler, Error as SchedulerError, Executor, IterativeExecutor, OnceExecutor, Scheduler, @@ -42,12 +43,9 @@ use crate::{ }, circuit_cache_key, ir::LABEL_MIR_NODE_ID, - operator::{ - condition::with_consensus, - dynamic::{ - balance::{Balancer, BalancerError, BalancerHint, PartitioningPolicy}, - recorder::{Recorder, RecorderControl, RecorderControlId, RecorderId}, - }, + operator::dynamic::{ + balance::{Balancer, BalancerError, BalancerHint, PartitioningPolicy}, + recorder::{Recorder, RecorderControl, RecorderControlId, RecorderId}, }, time::{Timestamp, UnitTimestamp}, trace::Batch, @@ -2774,9 +2772,8 @@ pub trait Circuit: CircuitBase + Clone + WithClock { /// precomputed for future nested timestamps). Custom iteration built on /// [`iterate`](Self::iterate) can call this from its termination check to /// detect convergence, typically combining the per-worker results across - /// workers via [`Consensus`](crate::circuit::runtime::Consensus). - /// See the documentation of [`Scope`] for the meaning of the parameter - /// but passing `0` refers to the current circuit. + /// workers via [`Consensus`]. See the documentation of [`Scope`] for the + /// meaning of the parameter but passing `0` refers to the current circuit. fn is_fixedpoint(&self, scope: Scope) -> bool { self.check_fixedpoint(scope) } @@ -4681,11 +4678,13 @@ where let res = constructor(child)?; let child_clone = child.clone(); - let termination_check = - with_consensus(self.global_node_id(), "fixed point", move || { - child_clone.inner().check_fixedpoint(0) - }); + let consensus = Consensus::new("fixed point"); + let termination_check = async move || { + // Send local fixed point status to all peers. + let local_fixedpoint = child_clone.inner().check_fixedpoint(0); + consensus.check(local_fixedpoint).await + }; let mut executor = >::new(termination_check); executor.prepare(child, None)?; Ok((res, executor)) diff --git a/crates/dbsp/src/circuit/runtime.rs b/crates/dbsp/src/circuit/runtime.rs index 1e200aaf0a5..906c2369e85 100644 --- a/crates/dbsp/src/circuit/runtime.rs +++ b/crates/dbsp/src/circuit/runtime.rs @@ -1354,8 +1354,8 @@ impl Runtime { } } -/// A synchronization primitive that allows multiple threads within a runtime to agree -/// when a condition is satisfied. +/// A synchronization primitive that allows multiple threads within a runtime to +/// agree when a condition is satisfied. /// /// Each worker submits a local vote via [`check`](Self::check); the call /// returns `true` only when every worker in the runtime votes `true`. It is @@ -1373,8 +1373,7 @@ impl Runtime { /// subcircuit. A worker that skips a call to [`check`](Self::check) will /// stop its peers from making progress. /// 2. [`Consensus`] must be constructed once per logical decision and reused -/// across clock ticks. See the -/// [`with_consensus`](crate::operator::condition::with_consensus) helper. +/// across clock ticks. pub struct Consensus(Broadcast); impl Consensus { diff --git a/crates/dbsp/src/operator.rs b/crates/dbsp/src/operator.rs index 15c037b2183..19ff96d359a 100644 --- a/crates/dbsp/src/operator.rs +++ b/crates/dbsp/src/operator.rs @@ -16,7 +16,7 @@ pub mod communication; pub(crate) mod inspect; mod accumulator; -pub(crate) mod condition; +mod condition; mod count; mod csv; mod delta0; diff --git a/crates/dbsp/src/operator/condition.rs b/crates/dbsp/src/operator/condition.rs index e5e31526731..90495158504 100644 --- a/crates/dbsp/src/operator/condition.rs +++ b/crates/dbsp/src/operator/condition.rs @@ -4,13 +4,12 @@ use crate::{ Timestamp, circuit::{ - ChildCircuit, Circuit, GlobalNodeId, Stream, - circuit_builder::{CircuitBase, IterativeCircuit}, - runtime::Consensus, + ChildCircuit, Circuit, Stream, + circuit_builder::IterativeCircuit, schedule::{Error as SchedulerError, Scheduler}, }, }; -use std::{cell::Cell, fmt::Display, marker::PhantomData, rc::Rc}; +use std::{cell::Cell, marker::PhantomData, rc::Rc}; impl Stream where @@ -61,11 +60,7 @@ where { self.iterate(|child| { let (condition, res) = constructor(child)?; - let check = - with_consensus(self.global_node_id(), "iterate with condition", move || { - condition.check() - }); - Ok((check, res)) + Ok((async move || Ok(condition.check()), res)) }) } @@ -83,12 +78,7 @@ where { self.iterate_with_scheduler::<_, _, _, S>(|child| { let (condition, res) = constructor(child)?; - let check = with_consensus( - self.global_node_id(), - "iterate with condition and scheduler", - move || condition.check(), - ); - Ok((check, res)) + Ok((async move || Ok(condition.check()), res)) }) } @@ -106,12 +96,10 @@ where { self.iterate(|child| { let (conditions, res) = constructor(child)?; - let check = with_consensus( - self.global_node_id(), - "iterate with conditions", - move || conditions.iter().all(Condition::check), - ); - Ok((check, res)) + Ok(( + async move || Ok(conditions.iter().all(Condition::check)), + res, + )) }) } @@ -129,36 +117,14 @@ where { self.iterate_with_scheduler::<_, _, _, S>(|child| { let (conditions, res) = constructor(child)?; - let check = with_consensus( - self.global_node_id(), - "iterate with conditions and scheduler", - move || conditions.iter().all(Condition::check), - ); - Ok((check, res)) + Ok(( + async move || Ok(conditions.iter().all(Condition::check)), + res, + )) }) } } -/// Wrap a per-worker termination decision in a runtime-wide [`Consensus`] so -/// that a nested loop only stops once _every_ worker agrees. -/// -/// A worker whose share of the data converges first must keep iterating until -/// its peers catch up; otherwise it would leave the loop while they still have -/// work to do. The [`Consensus`] object is created once, here, and reused -/// across every clock cycle, mirroring -/// [`Circuit::fixedpoint`](crate::Circuit::fixedpoint). -#[inline] -pub(crate) fn with_consensus( - global_node_id: GlobalNodeId, - name: impl Display, - decide: impl Fn() -> bool + 'static, -) -> impl AsyncFn() -> Result { - let consensus = Consensus::new(format!("{} {}", global_node_id, name)); - // The ordering is important here: The worker's local decision (`decide()`) - // must be captured _before_ announcing it to the broadcast. - async move || consensus.check(decide()).await -} - /// A condition attached to a stream that can be used /// to terminate the execution of a subcircuit /// (see [`ChildCircuit::iterate_with_condition`] and @@ -186,7 +152,7 @@ impl Condition { #[cfg(test)] mod test { use crate::{ - Circuit, RootCircuit, Runtime, Stream, + Circuit, RootCircuit, Stream, circuit::{ circuit_builder::IterativeCircuit, schedule::{DynamicScheduler, Scheduler}, @@ -292,73 +258,4 @@ mod test { circuit.transaction().unwrap(); } } - - /// Regression test for the cross-worker [`Consensus`](crate::Consensus) in - /// [`super::ChildCircuit::iterate_with_condition`]. - /// - /// With more than one worker each worker holds only a shard of the data and - /// reaches its local termination condition at a different nested clock tick. - /// Without consensus a worker could leave the loop while its peers still had - /// work to do, producing a wrong or nondeterministic result. We therefore - /// require that a multi-worker run computes the same reachable set as a - /// single-worker run, across repeated builds to shake out scheduling races. - #[test] - fn iterate_with_condition_multi_worker() { - // Nodes reachable from {1, 2, 3} in the graph below. - let reachable_from_init = |workers: usize| { - let (mut circuit, (edges_input, init_input, reachable_output)) = - Runtime::init_circuit(workers, |root| { - let (edges, edges_input) = root.add_input_zset::>(); - let (init, init_input) = root.add_input_zset::(); - - let reachable = root.iterate_with_condition(|child| { - let edges = edges.delta0(child).integrate(); - let init = init.delta0(child).integrate(); - - let edges_indexed: Stream<_, OrdIndexedZSet> = - edges.map_index(|Tup2(k, v)| (*k, *v)); - - let feedback = >>::new(child); - let feedback_pairs: Stream<_, OrdZSet<(u64, ())>> = - feedback.stream().map(|&node| (node, ())); - let feedback_indexed: Stream<_, OrdIndexedZSet> = - feedback_pairs.map_index(|(k, v)| (*k, *v)); - - let suc = - feedback_indexed.stream_join(&edges_indexed, |_node, &(), &to| to); - let reachable = init.plus(&suc).stream_distinct(); - feedback.connect(&reachable); - - let condition = reachable.differentiate().condition(|z| z.is_empty()); - Ok((condition, reachable.export())) - })?; - - Ok((edges_input, init_input, reachable.output())) - }) - .unwrap(); - - edges_input.append(&mut vec![ - Tup2(Tup2(0, 3), 1), - Tup2(Tup2(1, 2), 1), - Tup2(Tup2(2, 1), 1), - Tup2(Tup2(3, 1), 1), - Tup2(Tup2(3, 4), 1), - Tup2(Tup2(4, 5), 1), - Tup2(Tup2(4, 6), 1), - Tup2(Tup2(5, 6), 1), - Tup2(Tup2(5, 1), 1), - ]); - init_input.append(&mut vec![Tup2(1, 1), Tup2(2, 1), Tup2(3, 1)]); - circuit.transaction().unwrap(); - reachable_output.consolidate() - }; - - let expected = zset! { 1 => 1, 2 => 1, 3 => 1, 4 => 1, 5 => 1, 6 => 1 }; - // Single worker run. - assert_eq!(reachable_from_init(1), expected); - // Repeat the multi-worker run to give scheduling races a chance to surface. - for _ in 0..5 { - assert_eq!(reachable_from_init(4), expected); - } - } } diff --git a/crates/dbsp/src/operator/recursive.rs b/crates/dbsp/src/operator/recursive.rs index b416e823577..1538a738682 100644 --- a/crates/dbsp/src/operator/recursive.rs +++ b/crates/dbsp/src/operator/recursive.rs @@ -267,9 +267,7 @@ where /// no node is both red and blue the graph happens to be bipartite. In the /// first two computation steps the graph is bipartite but the added edge /// in the third step adds an odd-length cycle which destroys the bipartite - /// property and all nodes are colored red and blue. See also `tutorial13.rs` - /// for a variant of this computation using the lower-level [`Circuit::iterate`] - /// API. + /// property and all nodes are colored red and blue. /// /// ``` /// use dbsp::{