From 998fc38ac35948697a6c97ac76a266dc4daea67e Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 4 Apr 2026 13:49:46 -0700 Subject: [PATCH 1/3] dbsp: add roaring-based file filters --- Cargo.toml | 1 + crates/dbsp/src/circuit/metadata.rs | 28 +- crates/dbsp/src/dynamic/data.rs | 12 +- crates/dbsp/src/storage.rs | 1 - crates/dbsp/src/storage/file.rs | 11 +- crates/dbsp/src/storage/file/filter.rs | 266 ++++++++++ crates/dbsp/src/storage/file/filter/bloom.rs | 28 + .../dbsp/src/storage/file/filter/roaring.rs | 387 ++++++++++++++ .../{filter_stats.rs => file/filter/stats.rs} | 4 + crates/dbsp/src/storage/file/format.rs | 90 ++-- crates/dbsp/src/storage/file/reader.rs | 215 +++++--- crates/dbsp/src/storage/file/test.rs | 487 +++++++++++++++++- crates/dbsp/src/storage/file/writer.rs | 161 +++--- .../dbsp/src/storage/tracking_bloom_filter.rs | 10 +- crates/dbsp/src/trace.rs | 24 +- crates/dbsp/src/trace/filter.rs | 5 + .../{ord/batch_filter.rs => filter/batch.rs} | 54 +- .../src/trace/{ord => filter}/key_range.rs | 0 crates/dbsp/src/trace/ord.rs | 4 - .../src/trace/ord/fallback/indexed_wset.rs | 29 +- .../dbsp/src/trace/ord/fallback/key_batch.rs | 20 +- .../dbsp/src/trace/ord/fallback/val_batch.rs | 20 +- crates/dbsp/src/trace/ord/fallback/wset.rs | 27 +- .../src/trace/ord/file/indexed_wset_batch.rs | 47 +- crates/dbsp/src/trace/ord/file/key_batch.rs | 45 +- crates/dbsp/src/trace/ord/file/val_batch.rs | 44 +- crates/dbsp/src/trace/ord/file/wset_batch.rs | 46 +- .../src/trace/ord/vec/indexed_wset_batch.rs | 7 +- crates/dbsp/src/trace/ord/vec/key_batch.rs | 7 +- crates/dbsp/src/trace/ord/vec/val_batch.rs | 7 +- crates/dbsp/src/trace/ord/vec/wset_batch.rs | 6 +- crates/dbsp/src/trace/sampling.rs | 61 +++ crates/dbsp/src/trace/spine_async.rs | 144 +++--- crates/dbsp/src/trace/spine_async/snapshot.rs | 26 +- crates/dbsp/src/trace/test.rs | 157 +++++- crates/dbsp/src/trace/test/test_batch.rs | 2 +- crates/dbsp/src/utils.rs | 2 + crates/dbsp/src/utils/supports_roaring.rs | 262 ++++++++++ crates/feldera-macros/src/lib.rs | 4 +- crates/feldera-macros/src/tuples.rs | 9 + crates/feldera-types/src/config/dev_tweaks.rs | 8 + crates/fxp/src/dbsp_impl.rs | 4 +- crates/nexmark/src/queries/q9.rs | 1 + crates/sqllib/tests/tuple_proptest.rs | 11 +- .../src/bin/golden-writer.rs | 6 +- crates/storage/src/error.rs | 10 +- 46 files changed, 2408 insertions(+), 392 deletions(-) create mode 100644 crates/dbsp/src/storage/file/filter.rs create mode 100644 crates/dbsp/src/storage/file/filter/bloom.rs create mode 100644 crates/dbsp/src/storage/file/filter/roaring.rs rename crates/dbsp/src/storage/{filter_stats.rs => file/filter/stats.rs} (94%) rename crates/dbsp/src/trace/{ord/batch_filter.rs => filter/batch.rs} (85%) rename crates/dbsp/src/trace/{ord => filter}/key_range.rs (100%) create mode 100644 crates/dbsp/src/trace/sampling.rs create mode 100644 crates/dbsp/src/utils/supports_roaring.rs diff --git a/Cargo.toml b/Cargo.toml index 10d170e2164..96f6919ca05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -217,6 +217,7 @@ reqwest-websocket = "0.5.0" rkyv = { version = "0.7.45", default-features = false } rmp-serde = "1.3.0" rmpv = "1.3.0" +roaring = "0.11.3" rstest = "0.15" # Make sure this is the same rustls version used by the `tonic` crate. # See the `ensure_default_crypto_provider` function. diff --git a/crates/dbsp/src/circuit/metadata.rs b/crates/dbsp/src/circuit/metadata.rs index 5522bd42e96..cbf6319abca 100644 --- a/crates/dbsp/src/circuit/metadata.rs +++ b/crates/dbsp/src/circuit/metadata.rs @@ -167,7 +167,7 @@ pub const PREFIX_BATCHES_STATS: MetricId = MetricId(Cow::Borrowed("prefix_batche pub const INPUT_INTEGRAL_RECORDS_COUNT: MetricId = MetricId(Cow::Borrowed("input_integral_records_count")); -pub const CIRCUIT_METRICS: [CircuitMetric; 70] = [ +pub const CIRCUIT_METRICS: [CircuitMetric; 74] = [ // State CircuitMetric { name: USED_MEMORY_BYTES, @@ -269,7 +269,7 @@ pub const CIRCUIT_METRICS: [CircuitMetric; 70] = [ name: BLOOM_FILTER_BITS_PER_KEY, category: CircuitMetricCategory::State, advanced: false, - description: "Average number of bits per key in the Bloom filter.", + description: "Average number of bits per key across batches that use a Bloom filter.", }, CircuitMetric { name: BLOOM_FILTER_SIZE_BYTES, @@ -295,6 +295,30 @@ pub const CIRCUIT_METRICS: [CircuitMetric; 70] = [ advanced: false, description: "Hit rate of the Bloom filter.", }, + CircuitMetric { + name: ROARING_FILTER_SIZE_BYTES, + category: CircuitMetricCategory::State, + advanced: false, + description: "Size of the bitmap filter in bytes.", + }, + CircuitMetric { + name: ROARING_FILTER_HITS_COUNT, + category: CircuitMetricCategory::State, + advanced: false, + description: "The number of hits across all bitmap filters. The hits are summed across the bitmap filters for all batches in the spine.", + }, + CircuitMetric { + name: ROARING_FILTER_MISSES_COUNT, + category: CircuitMetricCategory::State, + advanced: false, + description: "The number of misses across all bitmap filters. The misses are summed across the bitmap filters for all batches in the spine.", + }, + CircuitMetric { + name: ROARING_FILTER_HIT_RATE_PERCENT, + category: CircuitMetricCategory::State, + advanced: false, + description: "Hit rate of the bitmap filter.", + }, CircuitMetric { name: RANGE_FILTER_SIZE_BYTES, category: CircuitMetricCategory::State, diff --git a/crates/dbsp/src/dynamic/data.rs b/crates/dbsp/src/dynamic/data.rs index ce1645a1c27..f328ad90106 100644 --- a/crates/dbsp/src/dynamic/data.rs +++ b/crates/dbsp/src/dynamic/data.rs @@ -12,6 +12,7 @@ use crate::{ rkyv::SerializeDyn, }, hash::default_hash, + utils::SupportsRoaring, }; /// Defines the minimal set of operations that must be supported by @@ -19,7 +20,16 @@ use crate::{ /// /// This trait is object safe and can be invoked via dynamic dispatch. pub trait Data: - Comparable + Clonable + SerializeDyn + DeserializableDyn + Send + Sync + Debug + AsAny + SizeOf + Comparable + + Clonable + + SerializeDyn + + DeserializableDyn + + Send + + Sync + + Debug + + AsAny + + SizeOf + + SupportsRoaring { /// Compute a hash of the object using default hasher and seed. fn default_hash(&self) -> u64; diff --git a/crates/dbsp/src/storage.rs b/crates/dbsp/src/storage.rs index 25c5557567c..cb2fa45d11a 100644 --- a/crates/dbsp/src/storage.rs +++ b/crates/dbsp/src/storage.rs @@ -7,7 +7,6 @@ pub mod backend; pub mod buffer_cache; pub mod dirlock; pub mod file; -pub mod filter_stats; pub mod tracking_bloom_filter; use fdlimit::{Outcome::LimitRaised, raise_fd_limit}; diff --git a/crates/dbsp/src/storage/file.rs b/crates/dbsp/src/storage/file.rs index ec32a6b3729..08c67285609 100644 --- a/crates/dbsp/src/storage/file.rs +++ b/crates/dbsp/src/storage/file.rs @@ -36,8 +36,10 @@ //! value and for sequential reads. It should be possible to disable indexing //! by data value for workloads that don't require it. //! -//! Layer files support approximate set membership query in `~O(1)` time using -//! [a filter block](format::FilterBlock). +//! Layer files support cheap key-membership tests using a per-batch filter +//! block. The default filter is Bloom-based; key types whose per-batch span +//! fits in `u32` can alternatively use an exact roaring bitmap filter by +//! storing keys relative to the batch minimum. //! //! Layer files should support 1 TB data size. //! @@ -98,6 +100,7 @@ use std::{ use std::{any::Any, sync::Arc}; use std::{fmt::Debug, ptr::NonNull}; +mod filter; pub mod format; mod item; pub mod reader; @@ -108,6 +111,10 @@ use crate::{ dynamic::{DataTrait, Erase, Factory, WithFactory}, storage::file::item::RefTup2Factory, }; +pub use filter::BatchKeyFilter; +pub use filter::FilterPlan; +pub use filter::TrackingRoaringBitmap; +pub use filter::{FilterKind, FilterStats, TrackingFilterStats}; pub use item::{ArchivedItem, Item, ItemFactory, WithItemFactory}; const BLOOM_FILTER_SEED: u128 = 42; diff --git a/crates/dbsp/src/storage/file/filter.rs b/crates/dbsp/src/storage/file/filter.rs new file mode 100644 index 00000000000..2e3d71866e4 --- /dev/null +++ b/crates/dbsp/src/storage/file/filter.rs @@ -0,0 +1,266 @@ +mod bloom; +mod roaring; +mod stats; + +use crate::{ + Runtime, + dynamic::{DataTrait, DynVec}, + storage::tracking_bloom_filter::TrackingBloomFilter, + trace::{BatchReader, BatchReaderFactories, sample_keys_from_batches}, +}; +use dyn_clone::clone_box; +use rand::thread_rng; +use std::io; + +pub use roaring::TrackingRoaringBitmap; +pub(crate) use roaring::{ + FILTER_PLAN_MIN_SAMPLE_SIZE, FILTER_PLAN_SAMPLE_PERCENT, RoaringLookupSampleStats, +}; +pub use stats::{FilterKind, FilterStats, TrackingFilterStats}; + +/// In-memory representation of the per-batch key filter. +#[derive(Debug)] +pub enum BatchKeyFilter { + /// Probabilistic Bloom filter over key hashes. + Bloom(TrackingBloomFilter), + + /// Exact roaring bitmap for key types whose batch's range fits in `u32`. + RoaringU32(TrackingRoaringBitmap), +} + +impl BatchKeyFilter { + pub(crate) fn new_bloom(estimated_keys: usize, bloom_false_positive_rate: f64) -> Self { + Self::Bloom(bloom::new_bloom_filter( + estimated_keys, + bloom_false_positive_rate, + )) + } + + pub(crate) fn new_roaring_u32(min: &K) -> Self + where + K: DataTrait + ?Sized, + { + Self::RoaringU32(TrackingRoaringBitmap::with_min(min)) + } + + pub(crate) fn deserialize_bloom(num_hashes: u32, data: Vec) -> Self { + Self::Bloom(bloom::deserialize_bloom_filter(num_hashes, data)) + } + + pub(crate) fn deserialize_roaring_u32(data: &[u8], min: &K) -> io::Result + where + K: DataTrait + ?Sized, + { + TrackingRoaringBitmap::deserialize_from(data, min).map(Self::RoaringU32) + } + + pub(crate) fn insert_key(&mut self, key: &K) + where + K: DataTrait + ?Sized, + { + match self { + Self::Bloom(filter) => { + filter.insert_hash(key.default_hash()); + } + Self::RoaringU32(filter) => { + filter.insert_key(key); + } + } + } + pub(crate) fn finalize(&mut self) { + match self { + Self::Bloom(_) => {} + Self::RoaringU32(filter) => filter.finalize(), + } + } +} + +/// Merge-time input used to choose the batch membership filter before writing. +/// +/// The writer must know upfront whether it is building Bloom or bitmap state, +/// because it cannot switch filters after the first key is written. The plan +/// therefore bundles: +/// - the merged batch bounds, which tell us whether min-offset roaring fits; +/// - a sampled subset of input keys, which lets us predict lookup behavior +/// when Bloom and roaring are both enabled. +pub struct FilterPlan +where + K: DataTrait + ?Sized, +{ + min: Box, + max: Box, + sampled_keys: Option>>, +} + +impl FilterPlan +where + K: DataTrait + ?Sized, +{ + fn sample_count_for_filter_plan(num_keys: usize) -> usize { + let scaled = ((num_keys as f64) * (FILTER_PLAN_SAMPLE_PERCENT / 100.0)).ceil() as usize; + scaled.max(FILTER_PLAN_MIN_SAMPLE_SIZE).min(num_keys) + } + + /// Builds a filter plan from the known minimum and maximum batch keys. + pub fn from_bounds(min: &K, max: &K) -> Self { + Self { + min: clone_box(min), + max: clone_box(max), + sampled_keys: None, + } + } + + #[cfg(test)] + pub(crate) fn with_sampled_keys(mut self, sampled_keys: Box>) -> Self { + self.sampled_keys = Some(sampled_keys); + self + } + + pub(crate) fn from_batches<'a, B, I>(batches: I) -> Option + where + B: BatchReader, + I: IntoIterator, + { + let batches: Vec<&'a B> = batches.into_iter().collect(); + let mut bounds: Option<(Box, Box)> = None; + for batch in &batches { + let (batch_min, batch_max) = batch.key_bounds()?; + match bounds.as_mut() { + Some((min, max)) => { + if batch_min < min.as_ref() { + *min = clone_box(batch_min); + } + if batch_max > max.as_ref() { + *max = clone_box(batch_max); + } + } + None => bounds = Some((clone_box(batch_min), clone_box(batch_max))), + } + } + + bounds.map(|(min, max)| { + let mut plan = Self { + min, + max, + sampled_keys: None, + }; + if plan.roaring_range_fits() { + plan.sampled_keys = Self::collect_sampled_keys_from_batches(&batches); + } + plan + }) + } + + fn collect_sampled_keys_from_batches(batches: &[&B]) -> Option>> + where + B: BatchReader, + { + let first_batch = batches.first()?; + let mut sampled_keys = first_batch.factories().keys_factory().default_box(); + let total_sample_size = batches + .iter() + .map(|batch| Self::sample_count_for_filter_plan(batch.key_count())) + .sum::(); + sampled_keys.reserve(total_sample_size); + + let mut rng = thread_rng(); + sample_keys_from_batches( + &first_batch.factories(), + batches, + &mut rng, + |batch| Self::sample_count_for_filter_plan(batch.key_count()), + sampled_keys.as_mut(), + ); + + (!sampled_keys.is_empty()).then_some(sampled_keys) + } + + fn roaring_range_fits(&self) -> bool { + self.min.supports_roaring32() && self.max.into_roaring_u32(self.min.as_data()).is_some() + } + + fn can_use_roaring(&self, enable_roaring: bool) -> bool { + enable_roaring && self.roaring_range_fits() + } + + fn predict_lookup_prefers_roaring(&self, estimated_keys: usize) -> bool { + let sampled_keys = match self.sampled_keys.as_ref() { + Some(sampled_keys) => sampled_keys, + None => return false, + }; + + let mut roaring_keys = Vec::with_capacity(sampled_keys.len()); + for index in 0..sampled_keys.len() { + let roaring_key = match sampled_keys + .index(index) + .into_roaring_u32(self.min.as_data()) + { + Some(roaring_key) => roaring_key, + None => return false, + }; + roaring_keys.push(roaring_key); + } + roaring_keys.sort_unstable(); + roaring_keys.dedup(); + + RoaringLookupSampleStats::from_sample(estimated_keys, &roaring_keys) + .map(|stats| stats.lookup_prefers_roaring()) + .unwrap_or(false) + } + + fn preferred_filter( + &self, + estimated_keys: usize, + enable_roaring: bool, + bloom_false_positive_rate: f64, + ) -> BatchKeyFilter { + if self.can_use_roaring(enable_roaring) + && self.predict_lookup_prefers_roaring(estimated_keys) + { + BatchKeyFilter::new_roaring_u32(self.min.as_ref()) + } else { + BatchKeyFilter::new_bloom(estimated_keys, bloom_false_positive_rate) + } + } + + /// Chooses the membership filter to build for a batch with `estimated_keys` + /// rows, using the enabled Bloom/roaring settings and an optional batch + /// bounds plan. + pub fn decide_filter( + filter_plan: Option<&Self>, + estimated_keys: usize, + ) -> Option { + // Choose between Bloom, roaring, or no membership filter using the + // following rules: + // + // - If Bloom and roaring are both enabled, prefer roaring when the + // plan proves the batch range fits in `u32` and the sampled-key + // lookup predictor says roaring should beat Bloom. If sampling is + // unavailable or the predictor cannot run, fall back to Bloom. + // - If only Bloom is enabled, always build Bloom. + // - If only roaring is enabled, build roaring only when the plan + // proves the batch range fits in `u32`; otherwise build no + // membership filter. + // - If both are disabled, build no membership filter. + // + // The "no plan => no roaring" rule is intentional: without known + // batch bounds we cannot safely decide that min-offset roaring + // encoding will fit, and we do not allow switching filters after + // writing has started. + let enable_roaring = Runtime::with_dev_tweaks(|dev_tweaks| dev_tweaks.enable_roaring()); + let bloom_false_positive_rate = Runtime::with_dev_tweaks(|dev_tweaks| { + let rate = dev_tweaks.bloom_false_positive_rate(); + (rate > 0.0 && rate < 1.0).then_some(rate) + }); + match (bloom_false_positive_rate, filter_plan) { + (Some(rate), Some(filter_plan)) => { + Some(filter_plan.preferred_filter(estimated_keys, enable_roaring, rate)) + } + (Some(rate), None) => Some(BatchKeyFilter::new_bloom(estimated_keys, rate)), + (None, Some(filter_plan)) if filter_plan.can_use_roaring(enable_roaring) => { + Some(BatchKeyFilter::new_roaring_u32(filter_plan.min.as_ref())) + } + (None, _) => None, + } + } +} diff --git a/crates/dbsp/src/storage/file/filter/bloom.rs b/crates/dbsp/src/storage/file/filter/bloom.rs new file mode 100644 index 00000000000..698500a8784 --- /dev/null +++ b/crates/dbsp/src/storage/file/filter/bloom.rs @@ -0,0 +1,28 @@ +use crate::storage::tracking_bloom_filter::TrackingBloomFilter; +use fastbloom::BloomFilter; + +use super::super::BLOOM_FILTER_SEED; + +pub(super) fn new_bloom_filter( + estimated_keys: usize, + bloom_false_positive_rate: f64, +) -> TrackingBloomFilter { + TrackingBloomFilter::new( + BloomFilter::with_false_pos(bloom_false_positive_rate) + .seed(&BLOOM_FILTER_SEED) + .expected_items({ + // `.max(64)` works around a fastbloom bug that hangs when the + // expected number of items is zero (see + // ). + estimated_keys.max(64) + }), + ) +} + +pub(super) fn deserialize_bloom_filter(num_hashes: u32, data: Vec) -> TrackingBloomFilter { + TrackingBloomFilter::new( + BloomFilter::from_vec(data) + .seed(&BLOOM_FILTER_SEED) + .hashes(num_hashes), + ) +} diff --git a/crates/dbsp/src/storage/file/filter/roaring.rs b/crates/dbsp/src/storage/file/filter/roaring.rs new file mode 100644 index 00000000000..2c8ed0f65f3 --- /dev/null +++ b/crates/dbsp/src/storage/file/filter/roaring.rs @@ -0,0 +1,387 @@ +use crate::{ + dynamic::{DataTrait, DynData}, + storage::file::{FilterStats, TrackingFilterStats}, +}; +use dyn_clone::clone_box; +use roaring::RoaringBitmap; +use size_of::SizeOf; +use std::{collections::HashMap, io, mem::size_of_val}; + +/// Sample 0.1% of keys per batch when building a merge-time filter plan. +pub(crate) const FILTER_PLAN_SAMPLE_PERCENT: f64 = 0.1; +/// Never sample fewer than this many keys from a batch for the filter plan. +pub(crate) const FILTER_PLAN_MIN_SAMPLE_SIZE: usize = 1_024; + +/// Roaring bitmap wrapper that tracks hit/miss counts during membership probes. +#[derive(Debug)] +pub struct TrackingRoaringBitmap { + bitmap: RoaringBitmap, + min: Box, + tracking: TrackingFilterStats, +} + +impl TrackingRoaringBitmap { + pub(crate) fn new(bitmap: RoaringBitmap, min: &K) -> Self + where + K: DataTrait + ?Sized, + { + let mut filter = Self { + bitmap, + min: clone_box(min.as_data()), + tracking: TrackingFilterStats::new(0), + }; + filter.refresh_stats_size(); + filter + } + + pub(crate) fn with_min(min: &K) -> Self + where + K: DataTrait + ?Sized, + { + Self::new(RoaringBitmap::new(), min) + } + + pub(crate) fn insert(&mut self, value: u32) { + self.bitmap.insert(value); + } + + pub(crate) fn insert_key(&mut self, key: &K) + where + K: DataTrait + ?Sized, + { + self.insert(self.roaring_u32(key)); + } + + pub(crate) fn finalize(&mut self) { + self.bitmap.optimize(); + self.refresh_stats_size(); + } + + // Bloom filters allocate their backing bitset up front, so their tracked + // size is stable after construction. Roaring bitmaps grow as keys are + // inserted and can shrink again after `optimize()`, so refresh the tracked + // size once the batch is finalized instead of trying to maintain it on + // every insert. + fn refresh_stats_size(&mut self) { + let min_size = self.min.size_of().total_bytes(); + self.tracking + .set_size_byte(size_of_val(&self.bitmap) + self.bitmap.serialized_size() + min_size); + } + + pub(crate) fn contains(&self, value: u32) -> bool { + let is_hit = self.bitmap.contains(value); + self.tracking.record(is_hit); + is_hit + } + + fn roaring_u32(&self, key: &K) -> u32 + where + K: DataTrait + ?Sized, + { + key.into_roaring_u32_checked(self.min.as_ref()) + } + + pub(crate) fn maybe_contains_key(&self, key: &K) -> bool + where + K: DataTrait + ?Sized, + { + self.contains(self.roaring_u32(key)) + } + + pub(crate) fn stats(&self) -> FilterStats { + self.tracking.stats() + } + + pub(crate) fn serialized_size(&self) -> usize { + self.bitmap.serialized_size() + } + + pub(crate) fn serialize_into(&self, writer: W) -> io::Result<()> { + self.bitmap.serialize_into(writer) + } + + pub(crate) fn deserialize_from(reader: R, min: &K) -> io::Result + where + R: io::Read, + K: DataTrait + ?Sized, + { + Ok(Self::new(RoaringBitmap::deserialize_from(reader)?, min)) + } +} + +/// Sample-derived summary of how a batch's key distribution maps onto +/// Roaring's container layout for lookup prediction. +/// +/// This exists because Roaring is not uniformly "better than Bloom": +/// - keys are first partitioned by their high 16 bits, so a `u32` domain is +/// split into `2^16` containers; +/// - within each touched container, roaring-rs keeps values in an array until +/// the container reaches about 4096 entries, then upgrades it to a bitmap; +/// - sparse batches therefore tend to pay binary-search costs in many small +/// array containers, while dense batches benefit from cheap bitmap probes. +/// +/// The predictor estimates those two things from a sample: +/// - how many 16-bit containers the batch likely touches +/// - how many keys each touched container likely holds +#[derive(Debug, Clone, Copy)] +pub(crate) struct RoaringLookupSampleStats { + // Estimated number of real keys per touched 16-bit window after rescaling + // the sampled keys/window by the sample fraction. + estimated_keys_per_window: f64, + // Estimated number of distinct 16-bit windows touched by the full batch. + estimated_touched_windows: f64, +} + +impl RoaringLookupSampleStats { + const ROARING_WINDOW_CAPACITY: f64 = 65_536.0; + const ROARING_BITMAP_CONTAINER_THRESHOLD: f64 = 4_096.0; + const LOOKUP_ROARING_WINDOW_PROBABILITY_THRESHOLD: f64 = 0.1; + const LOOKUP_ROARING_BITMAP_WINDOW_PROBABILITY_PENALTY: f64 = 0.1; + const LOOKUP_ROARING_ARRAY_WINDOW_PROBABILITY_PENALTY_BASE: f64 = 0.25; + const LOOKUP_ROARING_ARRAY_WINDOW_PROBABILITY_PENALTY_PER_LOG2_KEY: f64 = 0.15; + const TOUCHED_WINDOWS_CHAO1_DAMPING: f64 = 0.25; + const U32_WINDOW_COUNT: usize = 1 << 16; + + /// Estimate roaring friendliness for lookups from a small sample of keys. + /// + /// The estimator works is based on `crates/dbsp/benches/filter_predictor.rs`: + /// 1. Bucket sampled keys by their high 16 bits, which matches Roaring's + /// top-level `u32` container layout. + /// 2. Rescale sampled keys/window by the sample fraction so large & dense + /// batches do not look artificially sparse. + /// 3. Estimate the full-batch touched-window count by combining a uniform + /// occupancy model with a Chao1 unseen-window correction. + /// + /// Example: + /// - If the batch has `1_000_000` keys and the sample contains `1_000`, + /// the sample fraction is `0.001` (`0.1%`). + /// - If those `1_000` sampled keys touch `50` windows, then the sampled + /// average is `20` keys/window and the rescaled estimate is + /// `20 / 0.001 = 20_000` real keys/window. + /// - If many sampled windows are singletons, the Chao1 correction pushes + /// the touched-window estimate upward because the sample likely missed + /// many windows entirely. + pub(crate) fn from_sample(batch_keys: usize, sampled_keys: &[u32]) -> Option { + if batch_keys == 0 || sampled_keys.is_empty() { + return None; + } + + let sampled_key_count = sampled_keys.len(); + let mut per_window: HashMap = HashMap::new(); + for &key in sampled_keys { + let window = (key >> 16) as u16; + *per_window.entry(window).or_insert(0) += 1; + } + + let distinct_windows = per_window.len(); + if distinct_windows == 0 { + return None; + } + + let sample_fraction = sampled_key_count as f64 / batch_keys as f64; + if sample_fraction <= 0.0 { + return None; + } + + let avg_sample_keys_per_window = sampled_key_count as f64 / distinct_windows as f64; + // Without this rescaling, large but dense batches look artificially + // sparse and the predictor drifts toward Bloom. + let estimated_keys_per_window = + (avg_sample_keys_per_window / sample_fraction).min(Self::ROARING_WINDOW_CAPACITY); + // Sparse, wide samples often show up as many singleton windows and very + // few doubletons. Those are exactly the signals the Chao1 correction + // uses to estimate how many windows the sample likely missed entirely. + let sample_singleton_windows = per_window.values().filter(|&&count| count == 1).count(); + let sample_doubleton_windows = per_window.values().filter(|&&count| count == 2).count(); + let estimated_touched_windows = estimate_touched_windows( + batch_keys, + sampled_key_count, + distinct_windows, + sample_singleton_windows, + sample_doubleton_windows, + ); + + Some(Self { + estimated_keys_per_window, + estimated_touched_windows, + }) + } + + /// Predict whether lookup-heavy workloads should prefer Roaring. + /// + /// Random probes only pay container cost when they land in a touched + /// 16-bit window, so touched-window count is normalized into a + /// probability. Array containers get a size-dependent penalty because + /// `ArrayStore::contains()` gets slower as they grow, while bitmap + /// containers are treated as near-constant-time once the estimated + /// keys/window crosses Roaring's array-to-bitmap threshold. + pub(crate) fn lookup_prefers_roaring(&self) -> bool { + let lookup_window_probability = + (self.estimated_touched_windows / Self::U32_WINDOW_COUNT as f64).clamp(0.0, 1.0); + // roaring-rs switches between array and bitmap containers around 4096 + // elements. Bitmap containers are close to a constant-time bit test, + // but array containers use binary search and get meaningfully slower + // as they grow. + let lookup_container_penalty = + if self.estimated_keys_per_window >= Self::ROARING_BITMAP_CONTAINER_THRESHOLD { + Self::LOOKUP_ROARING_BITMAP_WINDOW_PROBABILITY_PENALTY + } else { + Self::LOOKUP_ROARING_ARRAY_WINDOW_PROBABILITY_PENALTY_BASE + + Self::LOOKUP_ROARING_ARRAY_WINDOW_PROBABILITY_PENALTY_PER_LOG2_KEY + * (self.estimated_keys_per_window + 1.0).log2() + }; + let lookup_cost_proxy = lookup_window_probability * lookup_container_penalty; + let lookup_score = Self::LOOKUP_ROARING_WINDOW_PROBABILITY_THRESHOLD + / lookup_cost_proxy.max(f64::MIN_POSITIVE); + lookup_score >= 1.0 + } +} + +/// Estimate how many distinct 16-bit Roaring windows the full batch touches. +/// +/// This combines: +/// 1. A uniform occupancy estimate that works well when windows are populated +/// fairly evenly. +/// 2. A Chao1-style unseen-window estimate that reacts when the sample is full +/// of singleton windows and is therefore likely missing many windows. +/// +/// The blend exists because just doing a uniform-only estimate under-counts +/// touched windows on sparse, wide distributions and makes random Roaring +/// lookups look cheaper than they are. +fn estimate_touched_windows( + batch_keys: usize, + sampled_keys: usize, + distinct_windows: usize, + sample_singleton_windows: usize, + sample_doubleton_windows: usize, +) -> f64 { + if batch_keys == 0 || sampled_keys == 0 || distinct_windows == 0 { + return 0.0; + } + if sampled_keys >= batch_keys { + return distinct_windows as f64; + } + + let uniform_estimate = + estimate_uniform_touched_windows(batch_keys, sampled_keys, distinct_windows); + let chao1_estimate = estimate_chao1_touched_windows( + distinct_windows, + sample_singleton_windows, + sample_doubleton_windows, + ); + blend_touched_window_estimates( + uniform_estimate, + chao1_estimate, + RoaringLookupSampleStats::TOUCHED_WINDOWS_CHAO1_DAMPING, + ) +} + +/// Estimate touched windows under a "roughly uniform occupancy" assumption. +/// +/// Intuition: +/// - assume the full batch touches `W` windows and spreads keys across them +/// fairly evenly; +/// - given the sample fraction, solve for the `W` that would yield the +/// observed sampled distinct-window count. +/// +/// This is the baseline estimate because it behaves sensibly on compact or +/// moderately regular distributions. It falls apart on sparse wide batches, +/// where many windows are touched so rarely that the sample never sees them. +fn estimate_uniform_touched_windows( + batch_keys: usize, + sampled_keys: usize, + distinct_windows: usize, +) -> f64 { + if batch_keys == 0 || sampled_keys == 0 || distinct_windows == 0 { + return 0.0; + } + if sampled_keys >= batch_keys { + return distinct_windows as f64; + } + + let sample_fraction = sampled_keys as f64 / batch_keys as f64; + let mut low = distinct_windows as f64; + let mut high = batch_keys.min(RoaringLookupSampleStats::U32_WINDOW_COUNT) as f64; + + if low >= high { + return low; + } + + let log_unseen = (-sample_fraction).ln_1p(); + for _ in 0..100 { + let mid = (low + high) * 0.5; + let avg_keys_per_window = batch_keys as f64 / mid; + let observed_windows = mid * (1.0 - (avg_keys_per_window * log_unseen).exp()); + + if observed_windows < distinct_windows as f64 { + low = mid; + } else { + high = mid; + } + } + + high +} + +/// Estimate touched windows with a Chao1-style unseen-species correction. +/// +/// Here the "species" are touched 16-bit windows: +/// - `distinct_windows` is how many windows the sample observed +/// - `sample_singleton_windows` counts windows seen exactly once +/// - `sample_doubleton_windows` counts windows seen exactly twice +/// +/// Raw Chao1 is intentionally not used directly in the final predictor because +/// it can overreact when `f2` is tiny. We still compute it because it pushes +/// the estimate upward in the cases we care about here: batches that touch +/// many 16-bit windows, but only a few sampled keys land in each window. +fn estimate_chao1_touched_windows( + distinct_windows: usize, + sample_singleton_windows: usize, + sample_doubleton_windows: usize, +) -> f64 { + let chao1_estimate = if sample_doubleton_windows > 0 { + distinct_windows as f64 + + (sample_singleton_windows * sample_singleton_windows) as f64 + / (2.0 * sample_doubleton_windows as f64) + } else { + distinct_windows as f64 + + (sample_singleton_windows.saturating_mul(sample_singleton_windows.saturating_sub(1)) + / 2) as f64 + }; + + chao1_estimate + .max(distinct_windows as f64) + .min(RoaringLookupSampleStats::U32_WINDOW_COUNT as f64) +} + +fn blend_touched_window_estimates(uniform_estimate: f64, chao1_estimate: f64, alpha: f64) -> f64 { + // Raw Chao1 reacts strongly to singleton-heavy samples, which is useful + // for sparse wide batches but too aggressive to use directly. Blend it + // toward the uniform estimate so the unseen-window correction only nudges + // the final estimate in the right direction. + uniform_estimate + alpha * (chao1_estimate - uniform_estimate) +} + +#[cfg(test)] +mod tests { + use super::TrackingRoaringBitmap; + use crate::storage::file::FilterStats; + + #[test] + fn tracking_roaring_bitmap_stats() { + let mut filter = TrackingRoaringBitmap::with_min((&0u32) as &crate::dynamic::DynData); + filter.insert(1); + filter.insert(3); + + assert!(filter.contains(1)); + assert!(!filter.contains(2)); + assert_eq!( + filter.stats(), + FilterStats { + size_byte: filter.stats().size_byte, + hits: 1, + misses: 1, + } + ); + } +} diff --git a/crates/dbsp/src/storage/filter_stats.rs b/crates/dbsp/src/storage/file/filter/stats.rs similarity index 94% rename from crates/dbsp/src/storage/filter_stats.rs rename to crates/dbsp/src/storage/file/filter/stats.rs index 54167363333..69ac9993ecc 100644 --- a/crates/dbsp/src/storage/filter_stats.rs +++ b/crates/dbsp/src/storage/file/filter/stats.rs @@ -59,6 +59,10 @@ impl TrackingFilterStats { } } + pub(crate) fn set_size_byte(&mut self, size_byte: usize) { + self.size_byte = size_byte; + } + /// Records the result of one filter probe. pub fn record(&self, is_hit: bool) { if is_hit { diff --git a/crates/dbsp/src/storage/file/format.rs b/crates/dbsp/src/storage/file/format.rs index b0b9cc66657..8c9c3310dee 100644 --- a/crates/dbsp/src/storage/file/format.rs +++ b/crates/dbsp/src/storage/file/format.rs @@ -75,12 +75,10 @@ //! //! Decompressing a compressed block yields the regular index or data block //! format starting with a [`BlockHeader`]. -use crate::storage::tracking_bloom_filter::TrackingBloomFilter; -use crate::storage::{buffer_cache::FBuf, file::BLOOM_FILTER_SEED}; +use crate::storage::buffer_cache::FBuf; use binrw::{BinRead, BinResult, BinWrite, Error as BinError, binrw, binwrite}; #[cfg(doc)] use crc32c; -use fastbloom::BloomFilter; use num_derive::FromPrimitive; use num_traits::FromPrimitive; use size_of::SizeOf; @@ -107,8 +105,11 @@ pub const INDEX_BLOCK_MAGIC: [u8; 4] = *b"LFIB"; /// Magic number for the file trailer block. pub const FILE_TRAILER_BLOCK_MAGIC: [u8; 4] = *b"LFFT"; -/// Magic number for filter blocks. -pub const FILTER_BLOCK_MAGIC: [u8; 4] = *b"LFFB"; +/// Magic number for Bloom filter blocks. +pub const BLOOM_FILTER_BLOCK_MAGIC: [u8; 4] = *b"LFFB"; + +/// Magic number for roaring bitmap filter blocks. +pub const ROARING_BITMAP_FILTER_BLOCK_MAGIC: [u8; 4] = *b"LFFR"; /// 8-byte header at the beginning of each block. /// @@ -171,13 +172,13 @@ pub struct FileTrailer { #[br(count = n_columns)] pub columns: Vec, - /// File offset in bytes of the [FilterBlock]. + /// File offset in bytes of the filter block. /// /// This is 0 if there is no filter block, or if the filter block size is /// bigger than `i32::MAX`. pub filter_offset: u64, - /// Size in bytes of the [FilterBlock]. + /// Size in bytes of the filter block. /// /// This is 0 if there is no filter block, or if the filter block size is /// bigger than `i32::MAX`. @@ -205,7 +206,7 @@ pub struct FileTrailer { /// future expansion. pub incompatible_features: u64, - /// File offset in bytes of the [FilterBlock]. + /// File offset in bytes of the filter block. /// /// This is 0 if there is no filter block, or if the filter block size is /// less than `i32::MAX`. If this is nonzero, then @@ -213,7 +214,7 @@ pub struct FileTrailer { /// [FileTrailer::compatible_features]. pub filter_offset64: u64, - /// Size in bytes of the [FilterBlock]. + /// Size in bytes of the filter block. /// /// This is 0 if there is no filter block, or if the filter block size is /// less than `i32::MAX`. If this is nonzero, then @@ -555,12 +556,15 @@ impl Compression { /// /// The Bloom filter contains a member for each key in column 0. #[binrw] -pub struct FilterBlock { +pub struct BloomFilterBlock { /// Block header with "LFFB" magic. - #[brw(assert(header.magic == FILTER_BLOCK_MAGIC, "filter block has bad magic"))] + #[brw(assert( + header.magic == BLOOM_FILTER_BLOCK_MAGIC, + "bloom filter block has bad magic" + ))] pub header: BlockHeader, - /// [BloomFilter::num_hashes]. + /// Number of hashes used by the Bloom filter. pub num_hashes: u32, /// Number of elements in `data`. @@ -572,24 +576,17 @@ pub struct FilterBlock { pub data: Vec, } -impl From for TrackingBloomFilter { - fn from(block: FilterBlock) -> Self { - TrackingBloomFilter::new( - BloomFilter::from_vec(block.data) - .seed(&BLOOM_FILTER_SEED) - .hashes(block.num_hashes), - ) - } -} - /// A block representing a Bloom filter (with data by reference). #[binwrite] -pub struct FilterBlockRef<'a> { +pub struct BloomFilterBlockRef<'a> { /// Block header with "LFFB" magic. - #[bw(assert(header.magic == FILTER_BLOCK_MAGIC, "filter block has bad magic"))] + #[bw(assert( + header.magic == BLOOM_FILTER_BLOCK_MAGIC, + "bloom filter block has bad magic" + ))] pub header: BlockHeader, - /// [BloomFilter::num_hashes]. + /// Number of hashes used by the Bloom filter. pub num_hashes: u32, /// Number of elements in `data`. @@ -600,12 +597,39 @@ pub struct FilterBlockRef<'a> { pub data: &'a [u64], } -impl<'a> From<&'a TrackingBloomFilter> for FilterBlockRef<'a> { - fn from(value: &'a TrackingBloomFilter) -> Self { - FilterBlockRef { - header: BlockHeader::new(&FILTER_BLOCK_MAGIC), - num_hashes: value.num_hashes(), - data: value.as_slice(), - } - } +/// A block representing a roaring bitmap filter. +#[binrw] +pub struct RoaringBitmapFilterBlock { + /// Block header with "LFFR" magic. + #[brw(assert( + header.magic == ROARING_BITMAP_FILTER_BLOCK_MAGIC, + "roaring filter block has bad magic" + ))] + pub header: BlockHeader, + + /// Number of bytes in `data`. + #[bw(try_calc(u64::try_from(data.len())))] + pub len: u64, + + /// Serialized roaring bitmap contents. + #[br(count = len)] + pub data: Vec, +} + +/// A block representing a roaring bitmap filter (with data by reference). +#[binwrite] +pub struct RoaringBitmapFilterBlockRef<'a> { + /// Block header with "LFFR" magic. + #[bw(assert( + header.magic == ROARING_BITMAP_FILTER_BLOCK_MAGIC, + "roaring filter block has bad magic" + ))] + pub header: BlockHeader, + + /// Number of bytes in `data`. + #[bw(try_calc(u64::try_from(data.len())))] + pub len: u64, + + /// Serialized roaring bitmap contents. + pub data: &'a [u8], } diff --git a/crates/dbsp/src/storage/file/reader.rs b/crates/dbsp/src/storage/file/reader.rs index edaba67ee38..65746c1ded7 100644 --- a/crates/dbsp/src/storage/file/reader.rs +++ b/crates/dbsp/src/storage/file/reader.rs @@ -2,22 +2,22 @@ //! //! [`Reader`] is the top-level interface for reading layer files. -use super::format::{Compression, FileTrailer}; -use super::{AnyFactories, Deserializer, Factories}; +use super::format::{BloomFilterBlock, Compression, FileTrailer, RoaringBitmapFilterBlock}; +use super::{AnyFactories, BatchKeyFilter, Deserializer, Factories}; use crate::dynamic::{DynVec, WeightTrait}; use crate::storage::buffer_cache::CacheAccess; -use crate::storage::file::format::{BatchMetadata, FilterBlock}; -use crate::storage::tracking_bloom_filter::TrackingBloomFilter; use crate::storage::{ backend::StorageError, buffer_cache::{BufferCache, FBuf}, file::format::{ - DataBlockHeader, FileTrailerColumn, IndexBlockHeader, NodeType, VERSION_NUMBER, Varint, + BLOOM_FILTER_BLOCK_MAGIC, BatchMetadata, DataBlockHeader, FileTrailerColumn, + IndexBlockHeader, MIN_SUPPORTED_VERSION, NodeType, ROARING_BITMAP_FILTER_BLOCK_MAGIC, + Varint, }, file::item::ArchivedItem, }; use crate::{ - dynamic::{DataTrait, DeserializeDyn, Factory}, + dynamic::{DataTrait, DeserializeDyn, DynData, Factory}, storage::{ backend::{BlockLocation, FileReader, InvalidBlockLocation, StorageBackend}, buffer_cache::{AtomicCacheStats, CacheStats}, @@ -112,12 +112,14 @@ pub enum CorruptionError { }, /// Invalid version number in file trailer. - #[error("File has invalid version {version} (expected {expected_version})")] + #[error( + "File has unsupported storage format version {version}; supported versions are {min_supported_version} and newer" + )] InvalidVersion { /// Version in file. version: u32, - /// Expected version ([`VERSION_NUMBER`]). - expected_version: u32, + /// Minimum supported version. + min_supported_version: u32, }, /// Invalid version number in file trailer. @@ -327,6 +329,35 @@ pub enum CorruptionError { /// Invalid filter block location. #[error("Invalid file block location ({0}).")] InvalidFilterLocation(InvalidBlockLocation), + + /// Filter block payload could not be decoded. + #[error("Invalid {kind} filter encoding in block ({location}): {inner}")] + InvalidFilterEncoding { + /// Block location. + location: BlockLocation, + /// Filter kind. + kind: &'static str, + /// Underlying parse error. + inner: String, + }, + + /// Roaring bitmap filter block payload could not be decoded. + #[error("Invalid roaring bitmap filter encoding in block ({location}): {inner}")] + InvalidRoaringBitmapFilterEncoding { + /// Block location. + location: BlockLocation, + /// Underlying parse error. + inner: String, + }, + + /// Filter block magic is unknown. + #[error("Unknown filter block magic {magic:?} in block ({location}).")] + UnknownFilterBlockMagic { + /// Block location. + location: BlockLocation, + /// Unknown magic. + magic: [u8; 4], + }, } /// Reader for an array of [Varint]s in a storage file. @@ -1347,19 +1378,6 @@ struct Column { n_rows: u64, } -impl FilterBlock { - fn new(file_handle: &dyn FileReader, location: BlockLocation) -> Result { - let block = file_handle.read_block(location)?; - Self::read_le(&mut io::Cursor::new(block.as_slice())).map_err(|e| { - Error::Corruption(CorruptionError::Binrw { - location, - block_type: "filter", - inner: e.to_string(), - }) - }) - } -} - impl Column { fn new(factories: &AnyFactories, info: &FileTrailerColumn) -> Result { let FileTrailerColumn { @@ -1509,6 +1527,67 @@ fn decompress( Ok(raw) } +fn parse_filter_block BinRead = ()>>( + block: &FBuf, + location: BlockLocation, + block_type: &'static str, +) -> Result { + T::read_le(&mut io::Cursor::new(block.as_slice())).map_err(|e| { + Error::Corruption(CorruptionError::Binrw { + location, + block_type, + inner: e.to_string(), + }) + }) +} + +fn read_filter_block( + file_handle: &dyn FileReader, + location: BlockLocation, + roaring_min: Option<&DynData>, +) -> Result { + let block = file_handle.read_block(location)?; + if block.len() < 8 { + return Err(Error::Corruption(CorruptionError::InvalidFilterEncoding { + location, + kind: "unknown", + inner: format!("block too short: {} bytes", block.len()), + })); + } + + let mut magic = [0u8; 4]; + magic.copy_from_slice(&block[4..8]); + + match magic { + BLOOM_FILTER_BLOCK_MAGIC => { + let block: BloomFilterBlock = parse_filter_block(&block, location, "bloom filter")?; + Ok(BatchKeyFilter::deserialize_bloom( + block.num_hashes, + block.data, + )) + } + ROARING_BITMAP_FILTER_BLOCK_MAGIC => { + let block: RoaringBitmapFilterBlock = + parse_filter_block(&block, location, "roaring bitmap filter")?; + let roaring_min = roaring_min.ok_or_else(|| { + Error::Corruption(CorruptionError::InvalidRoaringBitmapFilterEncoding { + location, + inner: "roaring bitmap filter requires the batch minimum".to_string(), + }) + })?; + BatchKeyFilter::deserialize_roaring_u32(&block.data, roaring_min).map_err(|e| { + Error::Corruption(CorruptionError::InvalidRoaringBitmapFilterEncoding { + location, + inner: e.to_string(), + }) + }) + } + magic => Err(Error::Corruption( + CorruptionError::UnknownFilterBlockMagic { location, magic }, + )), + } +} + /// Layer file column specification. /// /// A column specification must take the form `K0, A0, N0`, where `(K0, A0)` is @@ -1554,6 +1633,7 @@ where pub struct Reader { file: ImmutableFileRef, columns: Vec, + membership_filter_location: Option, /// Additional metadata added to the file by the writer. pub(crate) metadata: BatchMetadata, @@ -1591,8 +1671,8 @@ where factories: &[&AnyFactories], cache: fn() -> Option>, file: Arc, - membership_filter: Option, - ) -> Result<(Self, Option), Error> { + membership_filter: Option, + ) -> Result<(Self, Option), Error> { let file_size = file.get_size()?; if file_size < 512 || (file_size % 512) != 0 { return Err(CorruptionError::InvalidFileSize(file_size).into()); @@ -1606,12 +1686,10 @@ where &stats, )?; - // v4/v5 isn't backwards compatible. do not attempt to support - // older formats. - if file_trailer.version < VERSION_NUMBER { + if file_trailer.version < MIN_SUPPORTED_VERSION { return Err(CorruptionError::InvalidVersion { version: file_trailer.version, - expected_version: VERSION_NUMBER, + min_supported_version: MIN_SUPPORTED_VERSION, } .into()); } @@ -1623,11 +1701,8 @@ where ); } - if file_trailer.incompatible_features != 0 { - return Err(CorruptionError::UnsupportedIncompatibleFeatures( - file_trailer.incompatible_features, - ) - .into()); + if let Some(features) = file_trailer.unknown_incompatible_features() { + return Err(CorruptionError::UnsupportedIncompatibleFeatures(features).into()); } assert_eq!(factories.len(), file_trailer.columns.len()); @@ -1659,34 +1734,26 @@ where .into()); } } - - fn read_filter_block( - file_handle: &dyn FileReader, - offset: u64, - size: usize, - ) -> Result { - Ok(FilterBlock::new( - file_handle, - BlockLocation::new(offset, size).map_err(|error: InvalidBlockLocation| { + let membership_filter_location = if file_trailer.has_filter64() { + Some( + BlockLocation::new( + file_trailer.filter_offset64, + file_trailer.filter_size64 as usize, + ) + .map_err(|error: InvalidBlockLocation| { Error::Corruption(CorruptionError::InvalidFilterLocation(error)) })?, - )? - .into()) - } - let membership_filter = if let Some(membership_filter) = membership_filter { - Some(membership_filter) - } else if file_trailer.has_filter64() { - Some(read_filter_block( - &*file, - file_trailer.filter_offset64, - file_trailer.filter_size64 as usize, - )?) + ) } else if file_trailer.filter_offset != 0 { - Some(read_filter_block( - &*file, - file_trailer.filter_offset, - file_trailer.filter_size as usize, - )?) + Some( + BlockLocation::new( + file_trailer.filter_offset, + file_trailer.filter_size as usize, + ) + .map_err(|error: InvalidBlockLocation| { + Error::Corruption(CorruptionError::InvalidFilterLocation(error)) + })?, + ) } else { None }; @@ -1701,6 +1768,7 @@ where file_trailer.version, ), columns, + membership_filter_location, metadata: file_trailer.metadata.clone(), _phantom: PhantomData, }, @@ -1723,15 +1791,6 @@ where Self::new(factories, cache, storage_backend.open(path)?) } - pub(crate) fn open_with_filter( - factories: &[&AnyFactories], - cache: fn() -> Option>, - storage_backend: &dyn StorageBackend, - path: &StoragePath, - ) -> Result<(Self, Option), Error> { - Self::new_with_filter(factories, cache, storage_backend.open(path)?, None) - } - /// The number of columns in the layer file. /// /// This is a fixed value for any given `Reader`. @@ -1780,6 +1839,15 @@ where pub fn metadata(&self) -> &BatchMetadata { &self.metadata } + + fn read_membership_filter( + &self, + roaring_min: Option<&DynData>, + ) -> Result, Error> { + self.membership_filter_location + .map(|location| read_filter_block(&*self.file.file_handle, location, roaring_min)) + .transpose() + } } impl Reader<(&'static K, &'static A, N)> @@ -1788,6 +1856,19 @@ where A: DataTrait + ?Sized, (&'static K, &'static A, N): ColumnSpec, { + pub(crate) fn open_with_filter( + factories: &[&AnyFactories], + cache: fn() -> Option>, + storage_backend: &dyn StorageBackend, + path: &StoragePath, + ) -> Result<(Self, Option), Error> { + let reader = Self::open(factories, cache, storage_backend, path)?; + let key_range = reader.key_range()?; + let roaring_min = key_range.as_ref().map(|(min, _)| min.as_ref().as_data()); + let membership_filter = reader.read_membership_filter(roaring_min)?; + Ok((reader, membership_filter)) + } + /// Returns the min and max keys stored in column 0. /// /// The bounds are loaded from the root node when first requested and can diff --git a/crates/dbsp/src/storage/file/test.rs b/crates/dbsp/src/storage/file/test.rs index ba77ef7ded9..062c5cd3ead 100644 --- a/crates/dbsp/src/storage/file/test.rs +++ b/crates/dbsp/src/storage/file/test.rs @@ -1,28 +1,29 @@ -use std::{marker::PhantomData, sync::Arc}; +use std::{io::Cursor, marker::PhantomData, sync::Arc}; use crate::{ DBWeight, dynamic::{DataTrait, DowncastTrait, DynWeight, Factory, LeanVec, Vector, WithFactory}, storage::{ - backend::StorageBackend, + backend::{BlockLocation, StorageBackend}, buffer_cache::BufferCache, file::{ - format::{BatchMetadata, Compression}, + format::{ + BLOOM_FILTER_BLOCK_MAGIC, BatchMetadata, Compression, FileTrailer, + ROARING_BITMAP_FILTER_BLOCK_MAGIC, + }, reader::{BulkRows, FilteredKeys, Reader}, }, }, trace::{ BatchReaderFactories, Builder, VecIndexedWSetFactories, VecWSetFactories, - ord::{ - batch_filter::BatchFilters, - vec::{indexed_wset_batch::VecIndexedWSetBuilder, wset_batch::VecWSetBuilder}, - }, + filter::BatchFilters, + ord::vec::{indexed_wset_batch::VecIndexedWSetBuilder, wset_batch::VecWSetBuilder}, }, - utils::test::init_test_logger, + utils::{Tup1, test::init_test_logger}, }; use super::{ - Factories, + Factories, FilterPlan, reader::{ColumnSpec, RowGroup}, writer::{Parameters, Writer1, Writer2}, }; @@ -31,6 +32,7 @@ use crate::{ DBData, dynamic::{DynData, Erase}, }; +use binrw::BinRead; use feldera_types::config::{StorageConfig, StorageOptions}; use rand::{Rng, seq::SliceRandom, thread_rng}; use tempfile::tempdir; @@ -712,6 +714,73 @@ fn test_key_range( assert_eq!(max.downcast_checked::(), &expected_max); } +fn filter_block_magic(reader: &Reader<(&'static K, &'static A, N)>) -> Option<[u8; 4]> +where + K: DataTrait + ?Sized, + A: DataTrait + ?Sized, + (&'static K, &'static A, N): ColumnSpec, +{ + let file_size = reader.byte_size().unwrap() as usize; + let trailer_block = reader + .file_handle() + .read_block(BlockLocation::new((file_size - 512) as u64, 512).unwrap()) + .unwrap(); + let trailer = FileTrailer::read_le(&mut Cursor::new(trailer_block.as_slice())).unwrap(); + let offset = if trailer.has_filter64() { + trailer.filter_offset64 + } else { + trailer.filter_offset + }; + let size = if trailer.has_filter64() { + trailer.filter_size64 as usize + } else { + trailer.filter_size as usize + }; + if offset == 0 { + return None; + } + + let filter_block = reader + .file_handle() + .read_block(BlockLocation::new(offset, size).unwrap()) + .unwrap(); + let mut magic = [0u8; 4]; + magic.copy_from_slice(&filter_block[4..8]); + Some(magic) +} + +fn incompatible_features(reader: &Reader<(&'static K, &'static A, N)>) -> u64 +where + K: DataTrait + ?Sized, + A: DataTrait + ?Sized, + (&'static K, &'static A, N): ColumnSpec, +{ + let file_size = reader.byte_size().unwrap() as usize; + let trailer_block = reader + .file_handle() + .read_block(BlockLocation::new((file_size - 512) as u64, 512).unwrap()) + .unwrap(); + let trailer = FileTrailer::read_le(&mut Cursor::new(trailer_block.as_slice())).unwrap(); + trailer.incompatible_features +} + +fn sampled_filter_plan( + factories: &Factories, + keys: &[K], +) -> FilterPlan +where + K: DBData + Erase, +{ + let mut sampled_keys = factories.keys_factory.default_box(); + sampled_keys.reserve(keys.len()); + for key in keys { + sampled_keys.push_ref(key.erase()); + } + + FilterPlan::from_bounds(keys.first().unwrap().erase(), keys.last().unwrap().erase()) + .with_sampled_keys(sampled_keys) +} + fn test_two_columns(parameters: Parameters) where T: TwoColumns, @@ -734,7 +803,7 @@ where test_buffer_cache, &*storage_backend, parameters, - T::n0(), + FilterPlan::::decide_filter(None, T::n0()), ) .unwrap(); let n0 = T::n0(); @@ -800,7 +869,7 @@ where test_buffer_cache, &*storage_backend, parameters, - T::n0(), + FilterPlan::::decide_filter(None, T::n0()), ) .unwrap(); let n0 = T::n0(); @@ -945,7 +1014,7 @@ where test_buffer_cache, &*storage_backend, parameters.clone(), - n, + FilterPlan::::decide_filter(None, n), ) .unwrap(); for row in 0..n { @@ -956,7 +1025,7 @@ where let (reader, filters) = if reopen { println!("closing writer and reopening as reader"); let path = writer.path().clone(); - let (_file_handle, _bloom_filter, _key_bounds) = + let (_file_handle, _key_filter, _key_bounds) = writer.close(BatchMetadata::default()).unwrap(); let (reader, membership_filter) = Reader::open_with_filter( &[&factories.any_factories()], @@ -1006,7 +1075,7 @@ fn test_one_column_zset( test_buffer_cache, &*storage_backend, parameters.clone(), - n, + FilterPlan::::decide_filter(None, n), ) .unwrap(); for row in 0..n { @@ -1017,7 +1086,7 @@ fn test_one_column_zset( let reader = if reopen { println!("closing writer and reopening as reader"); let path = writer.path().clone(); - let (_file_handle, _bloom_filter, _key_bounds) = + let (_file_handle, _key_filter, _key_bounds) = writer.close(BatchMetadata::default()).unwrap(); Reader::open( &[&factories.any_factories()], @@ -1063,7 +1132,7 @@ fn one_column_key_range() { test_buffer_cache, &*storage_backend, Parameters::default(), - keys.len(), + FilterPlan::::decide_filter(None, keys.len()), ) .unwrap(); for key in keys { @@ -1072,7 +1141,7 @@ fn one_column_key_range() { let reader = if reopen { let path = writer.path().clone(); - let (_file_handle, _bloom_filter, _key_bounds) = + let (_file_handle, _key_filter, _key_bounds) = writer.close(BatchMetadata::default()).unwrap(); Reader::open( &[&factories.any_factories()], @@ -1099,6 +1168,390 @@ fn one_column_key_range() { } } +#[test] +fn test_bloom_filter_roundtrip_and_block_kind() { + init_test_logger(); + + for reopen in [false, true] { + let factories = Factories::::new::(); + let tempdir = tempdir().unwrap(); + let storage_backend = ::new( + &StorageConfig { + path: tempdir.path().to_string_lossy().to_string(), + cache: Default::default(), + }, + &StorageOptions::default(), + ) + .unwrap(); + + let mut writer = Writer1::new( + &factories, + test_buffer_cache, + &*storage_backend, + Parameters::default(), + FilterPlan::::decide_filter(None, 3), + ) + .unwrap(); + for key in [1i64, 3, 7] { + writer.write0((&key, &())).unwrap(); + } + + let (reader, filters) = if reopen { + let path = writer.path().clone(); + let (_file_handle, _key_filter, _key_bounds) = + writer.close(BatchMetadata::default()).unwrap(); + let (reader, membership_filter) = Reader::open_with_filter( + &[&factories.any_factories()], + test_buffer_cache, + &*storage_backend, + &path, + ) + .unwrap(); + let key_range = reader.key_range().unwrap().map(Into::into); + let filters = BatchFilters::from_file(key_range, membership_filter); + (reader, filters) + } else { + writer.into_reader(BatchMetadata::default()).unwrap() + }; + + for key in [1i64, 3, 7] { + assert!(filters.maybe_contains_key(key.erase(), None)); + } + assert_eq!(filter_block_magic(&reader), Some(BLOOM_FILTER_BLOCK_MAGIC)); + assert_eq!(incompatible_features(&reader), 0); + } +} + +#[test] +fn test_roaring_u32_filter_roundtrip_exact_and_block_kind() { + init_test_logger(); + + for reopen in [false, true] { + let factories = Factories::::new::(); + let tempdir = tempdir().unwrap(); + let storage_backend = ::new( + &StorageConfig { + path: tempdir.path().to_string_lossy().to_string(), + cache: Default::default(), + }, + &StorageOptions::default(), + ) + .unwrap(); + + let filter_plan = sampled_filter_plan(&factories, &[1u32, 3, 7]); + let mut writer = Writer1::new( + &factories, + test_buffer_cache, + &*storage_backend, + Parameters::default(), + FilterPlan::decide_filter(Some(&filter_plan), 3), + ) + .unwrap(); + for key in [1u32, 3, 7] { + writer.write0((&key, &())).unwrap(); + } + + let (reader, filters) = if reopen { + let path = writer.path().clone(); + let (_file_handle, _key_filter, _key_bounds) = + writer.close(BatchMetadata::default()).unwrap(); + let (reader, membership_filter) = Reader::open_with_filter( + &[&factories.any_factories()], + test_buffer_cache, + &*storage_backend, + &path, + ) + .unwrap(); + let key_range = reader.key_range().unwrap().map(Into::into); + let filters = BatchFilters::from_file(key_range, membership_filter); + (reader, filters) + } else { + writer.into_reader(BatchMetadata::default()).unwrap() + }; + + for key in [1u32, 3, 7] { + assert!(filters.maybe_contains_key(key.erase(), None)); + } + for key in [0u32, 2, 9] { + assert!(!filters.maybe_contains_key(key.erase(), None)); + } + assert_eq!( + filter_block_magic(&reader), + Some(ROARING_BITMAP_FILTER_BLOCK_MAGIC) + ); + assert_ne!(incompatible_features(&reader), 0); + } +} + +#[test] +fn test_roaring_tup1_i32_filter_roundtrip_exact_and_block_kind() { + init_test_logger(); + + for reopen in [false, true] { + let factories = Factories::::new::, ()>(); + let tempdir = tempdir().unwrap(); + let storage_backend = ::new( + &StorageConfig { + path: tempdir.path().to_string_lossy().to_string(), + cache: Default::default(), + }, + &StorageOptions::default(), + ) + .unwrap(); + + let filter_plan = sampled_filter_plan(&factories, &[Tup1(-7i32), Tup1(1), Tup1(3)]); + let mut writer = Writer1::new( + &factories, + test_buffer_cache, + &*storage_backend, + Parameters::default(), + FilterPlan::decide_filter(Some(&filter_plan), 3), + ) + .unwrap(); + for key in [Tup1(-7i32), Tup1(1), Tup1(3)] { + writer.write0((&key, &())).unwrap(); + } + + let (reader, filters) = if reopen { + let path = writer.path().clone(); + let (_file_handle, _key_filter, _key_bounds) = + writer.close(BatchMetadata::default()).unwrap(); + let (reader, membership_filter) = Reader::open_with_filter( + &[&factories.any_factories()], + test_buffer_cache, + &*storage_backend, + &path, + ) + .unwrap(); + let key_range = reader.key_range().unwrap().map(Into::into); + let filters = BatchFilters::from_file(key_range, membership_filter); + (reader, filters) + } else { + writer.into_reader(BatchMetadata::default()).unwrap() + }; + + for key in [Tup1(-7i32), Tup1(1), Tup1(3)] { + assert!(filters.maybe_contains_key(key.erase(), None)); + } + for key in [Tup1(-8i32), Tup1(0), Tup1(9)] { + assert!(!filters.maybe_contains_key(key.erase(), None)); + } + assert_eq!( + filter_block_magic(&reader), + Some(ROARING_BITMAP_FILTER_BLOCK_MAGIC) + ); + assert_ne!(incompatible_features(&reader), 0); + } +} + +#[test] +fn test_writer_without_filter_plan_uses_bloom_filter() { + init_test_logger(); + + let factories = Factories::::new::(); + let tempdir = tempdir().unwrap(); + let storage_backend = ::new( + &StorageConfig { + path: tempdir.path().to_string_lossy().to_string(), + cache: Default::default(), + }, + &StorageOptions::default(), + ) + .unwrap(); + + let mut writer = Writer1::new( + &factories, + test_buffer_cache, + &*storage_backend, + Parameters::default(), + FilterPlan::::decide_filter(None, 2), + ) + .unwrap(); + for key in [5u32, 8] { + writer.write0((&key, &())).unwrap(); + } + + let (reader, _filters) = writer.into_reader(BatchMetadata::default()).unwrap(); + assert_eq!(filter_block_magic(&reader), Some(BLOOM_FILTER_BLOCK_MAGIC)); +} + +#[test] +fn test_filter_plan_without_sample_falls_back_to_bloom() { + init_test_logger(); + + let filter_plan = FilterPlan::from_bounds((&1u32) as &DynData, (&7u32) as &DynData); + assert!(matches!( + FilterPlan::decide_filter(Some(&filter_plan), 3), + Some(super::BatchKeyFilter::Bloom(_)) + )); +} + +#[test] +fn test_filter_plan_predictor_prefers_roaring_for_dense_sample() { + init_test_logger(); + + let factories = Factories::::new::(); + let keys: Vec = (0..50_000).collect(); + let filter_plan = sampled_filter_plan(&factories, keys.as_slice()); + + assert!(matches!( + FilterPlan::decide_filter(Some(&filter_plan), keys.len()), + Some(super::BatchKeyFilter::RoaringU32(_)) + )); +} + +#[test] +fn test_filter_plan_predictor_prefers_bloom_for_sparse_wide_sample() { + init_test_logger(); + + let factories = Factories::::new::(); + let keys: Vec = (0..50_000).map(|index| index << 16).collect(); + let filter_plan = sampled_filter_plan(&factories, keys.as_slice()); + + assert!(matches!( + FilterPlan::decide_filter(Some(&filter_plan), keys.len()), + Some(super::BatchKeyFilter::Bloom(_)) + )); +} + +#[test] +fn test_roaring_i64_filter_roundtrip_uses_batch_min_offset() { + init_test_logger(); + + for reopen in [false, true] { + let factories = Factories::::new::(); + let tempdir = tempdir().unwrap(); + let storage_backend = ::new( + &StorageConfig { + path: tempdir.path().to_string_lossy().to_string(), + cache: Default::default(), + }, + &StorageOptions::default(), + ) + .unwrap(); + + let min = (i64::from(u32::MAX) * 4) + 10; + let keys = [min, min + 3, min + 7]; + let filter_plan = sampled_filter_plan(&factories, &keys); + let mut writer = Writer1::new( + &factories, + test_buffer_cache, + &*storage_backend, + Parameters::default(), + FilterPlan::decide_filter(Some(&filter_plan), keys.len()), + ) + .unwrap(); + for key in keys { + writer.write0((&key, &())).unwrap(); + } + + let (reader, filters) = if reopen { + let path = writer.path().clone(); + let (_file_handle, _key_filter, _key_bounds) = + writer.close(BatchMetadata::default()).unwrap(); + let (reader, membership_filter) = Reader::open_with_filter( + &[&factories.any_factories()], + test_buffer_cache, + &*storage_backend, + &path, + ) + .unwrap(); + let key_range = reader.key_range().unwrap().map(Into::into); + let filters = BatchFilters::from_file(key_range, membership_filter); + (reader, filters) + } else { + writer.into_reader(BatchMetadata::default()).unwrap() + }; + + for key in keys { + assert!(filters.maybe_contains_key((&key) as &DynData, None)); + } + for key in [min - 1, min + 4, min + 9] { + assert!(!filters.maybe_contains_key((&key) as &DynData, None)); + } + assert_eq!( + filter_block_magic(&reader), + Some(ROARING_BITMAP_FILTER_BLOCK_MAGIC) + ); + } +} + +#[test] +fn test_roaring_u64_filter_roundtrip_uses_batch_min_offset() { + init_test_logger(); + + let factories = Factories::::new::(); + let tempdir = tempdir().unwrap(); + let storage_backend = ::new( + &StorageConfig { + path: tempdir.path().to_string_lossy().to_string(), + cache: Default::default(), + }, + &StorageOptions::default(), + ) + .unwrap(); + + let base = (u64::from(u32::MAX) << 8) + 11; + let keys = [base, base + 2, base + 9]; + let filter_plan = sampled_filter_plan(&factories, &keys); + let mut writer = Writer1::new( + &factories, + test_buffer_cache, + &*storage_backend, + Parameters::default(), + FilterPlan::decide_filter(Some(&filter_plan), keys.len()), + ) + .unwrap(); + for key in keys { + writer.write0((&key, &())).unwrap(); + } + + let (reader, filters) = writer.into_reader(BatchMetadata::default()).unwrap(); + for key in keys { + assert!(filters.maybe_contains_key((&key) as &DynData, None)); + } + for key in [base - 1, base + 3, base + 20] { + assert!(!filters.maybe_contains_key((&key) as &DynData, None)); + } + assert_eq!( + filter_block_magic(&reader), + Some(ROARING_BITMAP_FILTER_BLOCK_MAGIC) + ); +} + +#[test] +fn test_i64_keys_fallback_to_bloom_when_span_exceeds_u32() { + init_test_logger(); + + let factories = Factories::::new::(); + let tempdir = tempdir().unwrap(); + let storage_backend = ::new( + &StorageConfig { + path: tempdir.path().to_string_lossy().to_string(), + cache: Default::default(), + }, + &StorageOptions::default(), + ) + .unwrap(); + + let max = i64::from(u32::MAX) + 1; + let filter_plan = FilterPlan::from_bounds((&0i64) as &DynData, (&max) as &DynData); + let mut writer = Writer1::new( + &factories, + test_buffer_cache, + &*storage_backend, + Parameters::default(), + FilterPlan::decide_filter(Some(&filter_plan), 2), + ) + .unwrap(); + for key in [0i64, max] { + writer.write0((&key, &())).unwrap(); + } + + let (reader, _filters) = writer.into_reader(BatchMetadata::default()).unwrap(); + assert_eq!(filter_block_magic(&reader), Some(BLOOM_FILTER_BLOCK_MAGIC)); +} + fn test_i64_helper(parameters: Parameters) { init_test_logger(); test_one_column( diff --git a/crates/dbsp/src/storage/file/writer.rs b/crates/dbsp/src/storage/file/writer.rs index 189603f5a57..816c727a077 100644 --- a/crates/dbsp/src/storage/file/writer.rs +++ b/crates/dbsp/src/storage/file/writer.rs @@ -4,51 +4,44 @@ //! 2-column layer file. To write more columns, either add another `Writer` //! struct, which is easily done, or mark the currently private `Writer` as //! `pub`. +use super::format::Compression; +use super::{AnyFactories, BatchKeyFilter, Factories, reader::Reader}; use crate::storage::{ backend::{BlockLocation, FileReader, FileWriter, StorageBackend, StorageError}, buffer_cache::{BufferCache, FBuf, FBufSerializer, LimitExceeded}, file::{ - BLOOM_FILTER_SEED, SerializerInner, + SerializerInner, format::{ - BatchMetadata, BlockHeader, COMPATIBLE_FEATURE_FILTER64, + BatchMetadata, BlockHeader, BloomFilterBlockRef, COMPATIBLE_FEATURE_FILTER64, COMPATIBLE_FEATURE_NEGATIVE_WEIGHT_COUNT, DATA_BLOCK_MAGIC, DataBlockHeader, - FILE_TRAILER_BLOCK_MAGIC, FileTrailer, FileTrailerColumn, FilterBlockRef, FixedLen, - INDEX_BLOCK_MAGIC, IndexBlockHeader, NodeType, VERSION_NUMBER, Varint, + FILE_TRAILER_BLOCK_MAGIC, FileTrailer, FileTrailerColumn, FixedLen, + INCOMPATIBLE_FEATURE_ROARING_FILTERS, INDEX_BLOCK_MAGIC, IndexBlockHeader, NodeType, + ROARING_BITMAP_FILTER_BLOCK_MAGIC, RoaringBitmapFilterBlockRef, VERSION_NUMBER, Varint, }, reader::TreeNode, }, }; +use crate::{ + Runtime, + dynamic::{DataTrait, DeserializeDyn, SerializeDyn}, + storage::file::ItemFactory, + trace::filter::{BatchFilters, key_range::KeyRange}, +}; use binrw::{ BinWrite, io::{Cursor, NoSeek}, }; use crc32c::crc32c; -#[cfg(debug_assertions)] use dyn_clone::clone_box; -use fastbloom::BloomFilter; use feldera_buffer_cache::CacheEntry; use feldera_storage::StoragePath; use snap::raw::{Encoder, max_compress_len}; -use std::{ - cell::RefCell, - sync::{Arc, Once}, -}; +use std::{cell::RefCell, sync::Arc}; use std::{ marker::PhantomData, mem::{replace, take}, ops::Range, }; -use tracing::info; - -use super::format::Compression; -use super::{AnyFactories, Factories, reader::Reader}; -use crate::storage::tracking_bloom_filter::TrackingBloomFilter; -use crate::{ - Runtime, - dynamic::{DataTrait, DeserializeDyn, SerializeDyn}, - storage::file::ItemFactory, - trace::ord::{BatchFilters, key_range::KeyRange}, -}; struct VarintWriter { varint: Varint, @@ -1140,50 +1133,23 @@ impl BlockWriter { struct Writer { cache: fn() -> Option>, writer: BlockWriter, - bloom_filter: Option, + key_filter: Option, cws: Vec, finished_columns: Vec, serializer: SerializerInner, } impl Writer { - fn bloom_false_positive_rate() -> Option { - let rate = Runtime::with_dev_tweaks(|dev_tweaks| dev_tweaks.bloom_false_positive_rate()); - let rate = (rate > 0.0 && rate < 1.0).then_some(rate); - - static ONCE: Once = Once::new(); - ONCE.call_once(|| { - if let Some(rate) = rate { - info!("Using Bloom filter false positive rate {rate}"); - } else { - info!("Bloom filters disabled"); - } - }); - rate - } - pub fn new( factories: &[&AnyFactories], cache: fn() -> Option>, storage_backend: &dyn StorageBackend, parameters: Parameters, n_columns: usize, - estimated_keys: usize, + key_filter: Option, ) -> Result { assert_eq!(factories.len(), n_columns); - let bloom_filter = Self::bloom_false_positive_rate().map(|bloom_false_positive_rate| { - TrackingBloomFilter::new( - BloomFilter::with_false_pos(bloom_false_positive_rate) - .seed(&BLOOM_FILTER_SEED) - .expected_items({ - // `.max(64)` works around a fastbloom bug that hangs when the - // expected number of items is zero (see - // https://github.com/tomtomwombat/fastbloom/issues/17). - estimated_keys.max(64) - }), - ) - }); let parameters = Arc::new(parameters); let cws = factories .iter() @@ -1197,7 +1163,7 @@ impl Writer { cache().expect("Should have a buffer cache"), storage_backend.create_with_prefix(&worker.into())?, ), - bloom_filter, + key_filter, cws, finished_columns, serializer: SerializerInner::new(), @@ -1218,11 +1184,10 @@ impl Writer { None }; - if column == 0 { - // Add `key` to bloom filter. - if let Some(bloom_filter) = &mut self.bloom_filter { - bloom_filter.insert_hash(item.0.default_hash()); - } + if column == 0 + && let Some(key_filter) = &mut self.key_filter + { + key_filter.insert_key(item.0); } // Add `value` to row group for column. @@ -1252,22 +1217,50 @@ impl Writer { pub fn close( mut self, metadata: BatchMetadata, - ) -> Result<(Arc, Option), StorageError> { + ) -> Result<(Arc, Option), StorageError> { debug_assert_eq!(self.cws.len(), self.finished_columns.len()); - // Write the Bloom filter. - let filter_location = if let Some(bloom_filter) = &self.bloom_filter { - let filter_block = FilterBlockRef::from(bloom_filter); - // std::mem::size_of::() should be an - // upper bound: in-memory struct size + bloom payload bytes. - let estimated_block_size = (std::mem::size_of::() - + std::mem::size_of_val(filter_block.data)) - // our binrw min block size is 512 so we round it up to avoid another - // reallocation - .next_multiple_of(512); - self.writer - .write_block(filter_block.into_block(estimated_block_size), None)? - .1 + if let Some(key_filter) = &mut self.key_filter { + key_filter.finalize(); + } + + // Write the batch key filter. + let mut incompatible_features = 0; + let filter_location = if let Some(key_filter) = &self.key_filter { + match key_filter { + BatchKeyFilter::Bloom(filter) => { + let filter_block = BloomFilterBlockRef { + header: BlockHeader::new( + &crate::storage::file::format::BLOOM_FILTER_BLOCK_MAGIC, + ), + num_hashes: filter.num_hashes(), + data: filter.as_slice(), + }; + let estimated_block_size = (std::mem::size_of::() + + std::mem::size_of_val(filter_block.data)) + .next_multiple_of(512); + self.writer + .write_block(filter_block.into_block(estimated_block_size), None)? + .1 + } + BatchKeyFilter::RoaringU32(filter) => { + incompatible_features |= INCOMPATIBLE_FEATURE_ROARING_FILTERS; + let mut data = Vec::with_capacity(filter.serialized_size()); + filter + .serialize_into(&mut data) + .map_err(|_| StorageError::RoaringBitmapFilter)?; + let filter_block = RoaringBitmapFilterBlockRef { + header: BlockHeader::new(&ROARING_BITMAP_FILTER_BLOCK_MAGIC), + data: &data, + }; + let estimated_block_size = (std::mem::size_of::() + + data.len()) + .next_multiple_of(512); + self.writer + .write_block(filter_block.into_block(estimated_block_size), None)? + .1 + } + } } else { BlockLocation { offset: 0, size: 0 } }; @@ -1282,7 +1275,7 @@ impl Writer { filter_offset: 0, filter_size: 0, compatible_features: COMPATIBLE_FEATURE_NEGATIVE_WEIGHT_COUNT, - incompatible_features: 0, + incompatible_features, filter_offset64: 0, filter_size64: 0, metadata, @@ -1305,7 +1298,7 @@ impl Writer { self.writer .insert_cache_entry(location, Arc::new(file_trailer)); - Ok((self.writer.complete()?, self.bloom_filter)) + Ok((self.writer.complete()?, self.key_filter)) } pub fn n_columns(&self) -> usize { @@ -1354,7 +1347,7 @@ impl Writer { /// }, &StorageOptions::default()).unwrap(); /// let parameters = Parameters::default(); /// let mut file = -/// Writer1::new(&factories, || Some(Arc::new(BufferCache::new(1024 * 1024))), &*storage_backend, parameters, 1_000_000).unwrap(); +/// Writer1::new(&factories, || Some(Arc::new(BufferCache::new(1024 * 1024))), &*storage_backend, parameters, None).unwrap(); /// for i in 0..1000_u32 { /// file.write0((i.erase(), ().erase())).unwrap(); /// } @@ -1383,7 +1376,7 @@ where cache: fn() -> Option>, storage_backend: &dyn StorageBackend, parameters: Parameters, - estimated_keys: usize, + key_filter: Option, ) -> Result { Ok(Self { factories: factories.clone(), @@ -1393,7 +1386,7 @@ where storage_backend, parameters, 1, - estimated_keys, + key_filter, )?, _phantom: PhantomData, #[cfg(debug_assertions)] @@ -1434,7 +1427,7 @@ where ) -> Result< ( Arc, - Option, + Option, Option<(Box, Box)>, ), StorageError, @@ -1462,12 +1455,12 @@ where let any_factories = self.factories.any_factories(); let cache = self.inner.cache; - let (file_handle, bloom_filter, key_bounds) = self.close(metadata)?; + let (file_handle, key_filter, key_bounds) = self.close(metadata)?; let key_range = key_bounds .as_ref() .map(|(min, max)| KeyRange::from_refs(min.as_ref(), max.as_ref())); let (reader, membership_filter) = - Reader::new_with_filter(&[&any_factories], cache, file_handle, bloom_filter)?; + Reader::new_with_filter(&[&any_factories], cache, file_handle, key_filter)?; let filters = BatchFilters::from_file(key_range, membership_filter); Ok((reader, filters)) } @@ -1518,7 +1511,7 @@ where /// }, &StorageOptions::default()).unwrap(); /// let parameters = Parameters::default(); /// let mut file = -/// Writer2::new(&factories, &factories, || Some(Arc::new(BufferCache::new(1024 * 1024))), &*storage_backend, parameters, 1_000_000).unwrap(); +/// Writer2::new(&factories, &factories, || Some(Arc::new(BufferCache::new(1024 * 1024))), &*storage_backend, parameters, None).unwrap(); /// for i in 0..1000_u32 { /// for j in 0..10_u32 { /// file.write1((&j, &())).unwrap(); @@ -1558,7 +1551,7 @@ where cache: fn() -> Option>, storage_backend: &dyn StorageBackend, parameters: Parameters, - estimated_keys: usize, + key_filter: Option, ) -> Result { Ok(Self { factories0: factories0.clone(), @@ -1569,7 +1562,7 @@ where storage_backend, parameters, 2, - estimated_keys, + key_filter, )?, #[cfg(debug_assertions)] prev0: None, @@ -1640,7 +1633,7 @@ where ) -> Result< ( Arc, - Option, + Option, Option<(Box, Box)>, ), StorageError, @@ -1674,7 +1667,7 @@ where let any_factories0 = self.factories0.any_factories(); let any_factories1 = self.factories1.any_factories(); let cache = self.inner.cache; - let (file_handle, bloom_filter, key_bounds) = self.close(metadata)?; + let (file_handle, key_filter, key_bounds) = self.close(metadata)?; let key_range = key_bounds .as_ref() .map(|(min, max)| KeyRange::from_refs(min.as_ref(), max.as_ref())); @@ -1682,7 +1675,7 @@ where &[&any_factories0, &any_factories1], cache, file_handle, - bloom_filter, + key_filter, )?; let filters = BatchFilters::from_file(key_range, membership_filter); Ok((reader, filters)) diff --git a/crates/dbsp/src/storage/tracking_bloom_filter.rs b/crates/dbsp/src/storage/tracking_bloom_filter.rs index 7f226789112..a50cd65e4ad 100644 --- a/crates/dbsp/src/storage/tracking_bloom_filter.rs +++ b/crates/dbsp/src/storage/tracking_bloom_filter.rs @@ -1,4 +1,4 @@ -use crate::storage::filter_stats::{FilterStats, TrackingFilterStats}; +use crate::storage::file::{FilterStats, TrackingFilterStats}; use fastbloom::BloomFilter; /// Bloom filter which tracks the number of hits and misses when lookups are performed. @@ -53,7 +53,7 @@ impl TrackingBloomFilter { #[cfg(test)] mod tests { use super::TrackingBloomFilter; - use crate::storage::filter_stats::FilterStats; + use crate::storage::file::FilterStats; use fastbloom::BloomFilter; #[test] @@ -67,7 +67,7 @@ mod tests { FilterStats { size_byte: 96 + 8192 / 8, hits: 0, - misses: 0, + misses: 0 } ); filter.insert_hash(123); @@ -79,7 +79,7 @@ mod tests { FilterStats { size_byte: 96 + 8192 / 8, hits: 1, - misses: 2, + misses: 2 } ); } @@ -91,7 +91,7 @@ mod tests { FilterStats { size_byte: 0, hits: 0, - misses: 0, + misses: 0 } ); } diff --git a/crates/dbsp/src/trace.rs b/crates/dbsp/src/trace.rs index 573331b9aec..ce5cd191960 100644 --- a/crates/dbsp/src/trace.rs +++ b/crates/dbsp/src/trace.rs @@ -31,11 +31,12 @@ use crate::dynamic::{ClonableTrait, DynDataTyped, DynUnit, Weight}; use crate::storage::buffer_cache::CacheStats; use crate::storage::file::SerializerInner; pub use crate::storage::file::{DbspSerializer, Deserializable, Deserializer, Rkyv}; +use crate::storage::file::{FilterKind, FilterStats}; use crate::trace::cursor::{ DefaultPushCursor, FilteredMergeCursor, FilteredMergeCursorWithSnapshot, PushCursor, UnfilteredMergeCursor, }; -use crate::utils::IsNone; +use crate::utils::{IsNone, SupportsRoaring}; use crate::{dynamic::ArchivedDBData, storage::buffer_cache::FBuf}; use cursor::CursorFactory; use enum_map::Enum; @@ -52,7 +53,9 @@ pub mod cursor; pub mod filter; pub mod layers; pub mod ord; +mod sampling; pub mod spine_async; +pub(crate) use sampling::sample_keys_from_batches; pub use spine_async::{BatchReaderWithSnapshot, ListMerger, Spine, SpineSnapshot, WithSnapshot}; #[cfg(test)] @@ -102,6 +105,7 @@ pub trait DBData: + Debug + ArchivedDBData + IsNone + + SupportsRoaring + 'static { } @@ -119,6 +123,7 @@ impl DBData for T where + Debug + ArchivedDBData + IsNone + + SupportsRoaring + 'static { } @@ -484,6 +489,16 @@ where FilterStats::default() } + /// Cached minimum and maximum keys for this batch, when available. + /// + /// File-backed batches materialize these bounds at write time. In-memory + /// batches can compute them from their ordered key storage. Merge builders + /// use these bounds to decide upfront whether a batch span can be encoded + /// into a roaring bitmap. + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + None + } + /// Where the batch's data is stored. fn location(&self) -> BatchLocation { BatchLocation::Memory @@ -534,7 +549,6 @@ where /// * The output sample contains keys sorted in ascending order. fn sample_keys(&self, rng: &mut RG, sample_size: usize, sample: &mut DynVec) where - Self::Time: PartialEq<()>, RG: Rng; /// Returns num_partitions-1 keys from the batch that partition the batch into num_partitions @@ -663,6 +677,9 @@ where fn range_filter_stats(&self) -> FilterStats { (**self).range_filter_stats() } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + (**self).key_bounds() + } fn location(&self) -> BatchLocation { (**self).location() } @@ -674,7 +691,6 @@ where } fn sample_keys(&self, rng: &mut RG, sample_size: usize, sample: &mut DynVec) where - Self::Time: PartialEq<()>, RG: Rng, { (**self).sample_keys(rng, sample_size, sample) @@ -998,7 +1014,7 @@ where location: Option, ) -> Self where - B: BatchReader, + B: BatchReader, I: IntoIterator + Clone, { let _ = location; diff --git a/crates/dbsp/src/trace/filter.rs b/crates/dbsp/src/trace/filter.rs index 3c1ed2da1eb..5a6f3ae5e41 100644 --- a/crates/dbsp/src/trace/filter.rs +++ b/crates/dbsp/src/trace/filter.rs @@ -3,10 +3,15 @@ //! Filters are used by the garbage collector to discard unused records. //! We support different several types of filters for keys and values. +pub(crate) mod batch; +pub(crate) mod key_range; + use dyn_clone::DynClone; use crate::{circuit::metadata::MetaItem, dynamic::Factory}; +pub(crate) use batch::BatchFilters; + pub trait FilterFunc: Fn(&V) -> bool + DynClone + Send + Sync {} impl FilterFunc for F where F: Fn(&V) -> bool + Clone + Send + Sync + 'static {} diff --git a/crates/dbsp/src/trace/ord/batch_filter.rs b/crates/dbsp/src/trace/filter/batch.rs similarity index 85% rename from crates/dbsp/src/trace/ord/batch_filter.rs rename to crates/dbsp/src/trace/filter/batch.rs index 6a47f708398..8253e805229 100644 --- a/crates/dbsp/src/trace/ord/batch_filter.rs +++ b/crates/dbsp/src/trace/filter/batch.rs @@ -6,14 +6,15 @@ use crate::{ dynamic::{DataTrait, DynVec}, storage::{ - file::reader::FilteredKeys, - filter_stats::{FilterStats, TrackingFilterStats}, + file::{ + BatchKeyFilter, FilterKind, FilterStats, TrackingFilterStats, TrackingRoaringBitmap, + reader::FilteredKeys, + }, tracking_bloom_filter::TrackingBloomFilter, }, - trace::ord::key_range::KeyRange, + trace::filter::key_range::KeyRange, }; use size_of::SizeOf; -use smallvec::SmallVec; use std::sync::Arc; /// A cheap, in-memory precheck used by `seek_key_exact`. @@ -109,14 +110,10 @@ where /// pay the hash or bloom lookup cost. pub(crate) fn from_file( key_range: Option>, - membership_filter: Option, + membership_filter: Option, ) -> Self { - Self::new( - key_range, - membership_filter - .map(Arc::new) - .map(|filter| filter as Arc>), - ) + let membership_filter = membership_filter.map(Arc::>::from); + Self::new(key_range, membership_filter) } /// Returns cumulative statistics for the range and membership filters. @@ -141,9 +138,7 @@ where pub(crate) fn filtered_keys<'a>(&self, keys: &'a DynVec) -> FilteredKeys<'a, K> { debug_assert!(keys.is_sorted_by(&|a, b| a.cmp(b))); - // Preserve the old `FilteredKeys` heuristic: if too many keys pass, - // avoid allocating the index vector and just keep the original slice. - let mut filter_pass_keys = SmallVec::<[_; 50]>::new(); + let mut filter_pass_keys = Vec::with_capacity(keys.len().min(50)); for (index, key) in keys.dyn_iter().enumerate() { if self.maybe_contains_key(key, None) { filter_pass_keys.push(index); @@ -153,7 +148,7 @@ where } } - FilteredKeys::with_filter_pass_keys(keys, Some(filter_pass_keys.into_vec())) + FilteredKeys::with_filter_pass_keys(keys, Some(filter_pass_keys)) } /// Returns `false` only when `key` is definitely not present. @@ -227,12 +222,35 @@ where } } +impl BatchFilter for TrackingRoaringBitmap +where + K: DataTrait + ?Sized, +{ + fn maybe_contains_key(&self, key: &K, _hash: &mut Option) -> bool { + self.maybe_contains_key(key) + } + + fn stats(&self) -> FilterStats { + TrackingRoaringBitmap::stats(self) + } +} + +impl From for Arc> +where + K: DataTrait + ?Sized, +{ + fn from(filter: BatchKeyFilter) -> Self { + match filter { + BatchKeyFilter::Bloom(filter) => Arc::new(filter), + BatchKeyFilter::RoaringU32(filter) => Arc::new(filter), + } + } +} + #[cfg(test)] mod tests { use super::{BatchFilter, TrackedRangeFilter}; - use crate::{ - dynamic::DynData, storage::filter_stats::FilterStats, trace::ord::key_range::KeyRange, - }; + use crate::{dynamic::DynData, storage::file::FilterStats, trace::filter::key_range::KeyRange}; use std::sync::Arc; #[test] diff --git a/crates/dbsp/src/trace/ord/key_range.rs b/crates/dbsp/src/trace/filter/key_range.rs similarity index 100% rename from crates/dbsp/src/trace/ord/key_range.rs rename to crates/dbsp/src/trace/filter/key_range.rs diff --git a/crates/dbsp/src/trace/ord.rs b/crates/dbsp/src/trace/ord.rs index e0bf2b0c689..1be1646b897 100644 --- a/crates/dbsp/src/trace/ord.rs +++ b/crates/dbsp/src/trace/ord.rs @@ -1,11 +1,7 @@ -pub(crate) mod batch_filter; pub mod fallback; pub mod file; -pub(crate) mod key_range; pub mod merge_batcher; pub mod vec; - -pub use batch_filter::{BatchFilterStats, BatchFilters}; pub use fallback::{ indexed_wset::{ FallbackIndexedWSet, FallbackIndexedWSet as OrdIndexedWSet, FallbackIndexedWSetBuilder, diff --git a/crates/dbsp/src/trace/ord/fallback/indexed_wset.rs b/crates/dbsp/src/trace/ord/fallback/indexed_wset.rs index 54dbbc0b4ac..74e03e28af4 100644 --- a/crates/dbsp/src/trace/ord/fallback/indexed_wset.rs +++ b/crates/dbsp/src/trace/ord/fallback/indexed_wset.rs @@ -1,6 +1,6 @@ use super::utils::{copy_to_builder, pick_merge_destination}; use crate::storage::file::SerializerInner; -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::{FilterKind, FilterStats}; use crate::{ DBWeight, Error, NumEntries, algebra::{AddAssignByRef, AddByRef, NegByRef, ZRingValue}, @@ -290,6 +290,13 @@ where } } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + match &self.inner { + Inner::File(file) => file.key_bounds(), + Inner::Vec(vec) => vec.key_bounds(), + } + } + #[inline] fn location(&self) -> BatchLocation { match &self.inner { @@ -514,17 +521,23 @@ where location: Option, ) -> Self where - B: BatchReader, + B: BatchReader, I: IntoIterator + Clone, { + let key_capacity = batches.clone().into_iter().map(|b| b.key_count()).sum(); + let value_capacity = batches.clone().into_iter().map(|b| b.len()).sum(); Self { factories: factories.clone(), - inner: BuilderInner::new( - factories, - batches.clone().into_iter().map(|b| b.key_count()).sum(), - batches.clone().into_iter().map(|b| b.len()).sum(), - pick_merge_destination(batches, location).into(), - ), + inner: match pick_merge_destination(batches.clone(), location) { + BatchLocation::Memory => BuilderInner::Vec(VecIndexedWSetBuilder::with_capacity( + &factories.vec_indexed_wset_factory, + key_capacity, + value_capacity, + )), + BatchLocation::Storage => BuilderInner::File(FileIndexedWSetBuilder::for_merge( + factories, batches, location, + )), + }, } } diff --git a/crates/dbsp/src/trace/ord/fallback/key_batch.rs b/crates/dbsp/src/trace/ord/fallback/key_batch.rs index e6725e7dcd2..3a8e676385b 100644 --- a/crates/dbsp/src/trace/ord/fallback/key_batch.rs +++ b/crates/dbsp/src/trace/ord/fallback/key_batch.rs @@ -1,5 +1,5 @@ use super::utils::{copy_to_builder, pick_merge_destination}; -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::{FilterKind, FilterStats}; use crate::{ DBData, DBWeight, NumEntries, Timestamp, dynamic::{ @@ -282,6 +282,13 @@ where } } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + match &self.inner { + Inner::File(file) => file.key_bounds(), + Inner::Vec(vec) => vec.key_bounds(), + } + } + #[inline] fn location(&self) -> BatchLocation { match &self.inner { @@ -299,7 +306,6 @@ where fn sample_keys(&self, rng: &mut RG, sample_size: usize, output: &mut DynVec) where - Self::Time: PartialEq<()>, RG: Rng, { match &self.inner { @@ -408,23 +414,23 @@ where location: Option, ) -> Self where - B: BatchReader, + B: BatchReader, I: IntoIterator + Clone, { let key_capacity = batches.clone().into_iter().map(|b| b.key_count()).sum(); let value_capacity = batches.clone().into_iter().map(|b| b.len()).sum(); Self { factories: factories.clone(), - inner: match pick_merge_destination(batches, location) { + inner: match pick_merge_destination(batches.clone(), location) { BatchLocation::Memory => BuilderInner::Vec(VecKeyBuilder::with_capacity( &factories.vec, key_capacity, value_capacity, )), - BatchLocation::Storage => BuilderInner::File(FileKeyBuilder::with_capacity( + BatchLocation::Storage => BuilderInner::File(FileKeyBuilder::for_merge( &factories.file, - key_capacity, - value_capacity, + batches, + location, )), }, } diff --git a/crates/dbsp/src/trace/ord/fallback/val_batch.rs b/crates/dbsp/src/trace/ord/fallback/val_batch.rs index f376a626681..bb5cd2ac69a 100644 --- a/crates/dbsp/src/trace/ord/fallback/val_batch.rs +++ b/crates/dbsp/src/trace/ord/fallback/val_batch.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use super::utils::{copy_to_builder, pick_merge_destination}; use crate::storage::buffer_cache::CacheStats; -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::{FilterKind, FilterStats}; use crate::trace::cursor::{DelegatingCursor, PushCursor}; use crate::trace::ord::file::val_batch::FileValBuilder; use crate::trace::ord::vec::val_batch::VecValBuilder; @@ -289,6 +289,13 @@ where } } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + match &self.inner { + Inner::File(file) => file.key_bounds(), + Inner::Vec(vec) => vec.key_bounds(), + } + } + #[inline] fn location(&self) -> BatchLocation { match &self.inner { @@ -307,7 +314,6 @@ where fn sample_keys(&self, rng: &mut RG, sample_size: usize, output: &mut DynVec) where RG: Rng, - T: PartialEq<()>, { match &self.inner { Inner::Vec(vec) => vec.sample_keys(rng, sample_size, output), @@ -425,23 +431,23 @@ where location: Option, ) -> Self where - B: BatchReader, + B: BatchReader, I: IntoIterator + Clone, { let key_capacity = batches.clone().into_iter().map(|b| b.key_count()).sum(); let value_capacity = batches.clone().into_iter().map(|b| b.len()).sum(); Self { factories: factories.clone(), - inner: match pick_merge_destination(batches, location) { + inner: match pick_merge_destination(batches.clone(), location) { BatchLocation::Memory => BuilderInner::Vec(VecValBuilder::with_capacity( &factories.vec, key_capacity, value_capacity, )), - BatchLocation::Storage => BuilderInner::File(FileValBuilder::with_capacity( + BatchLocation::Storage => BuilderInner::File(FileValBuilder::for_merge( &factories.file, - key_capacity, - value_capacity, + batches, + location, )), }, } diff --git a/crates/dbsp/src/trace/ord/fallback/wset.rs b/crates/dbsp/src/trace/ord/fallback/wset.rs index c1de66b50e4..9e6f2f21bfc 100644 --- a/crates/dbsp/src/trace/ord/fallback/wset.rs +++ b/crates/dbsp/src/trace/ord/fallback/wset.rs @@ -1,5 +1,5 @@ use super::utils::{copy_to_builder, pick_merge_destination}; -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::{FilterKind, FilterStats}; use crate::{ DBWeight, NumEntries, algebra::{AddAssignByRef, AddByRef, NegByRef, ZRingValue}, @@ -288,6 +288,13 @@ where } } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + match &self.inner { + Inner::File(file) => file.key_bounds(), + Inner::Vec(vec) => vec.key_bounds(), + } + } + #[inline] fn location(&self) -> BatchLocation { match &self.inner { @@ -495,16 +502,22 @@ where location: Option, ) -> Self where - B: BatchReader, + B: BatchReader, I: IntoIterator + Clone, { + let key_capacity = batches.clone().into_iter().map(|b| b.key_count()).sum(); Self { factories: factories.clone(), - inner: BuilderInner::new( - factories, - batches.clone().into_iter().map(|b| b.key_count()).sum(), - pick_merge_destination(batches, location).into(), - ), + inner: match pick_merge_destination(batches.clone(), location) { + BatchLocation::Memory => BuilderInner::Vec(VecWSetBuilder::with_capacity( + &factories.vec_wset_factory, + key_capacity, + key_capacity, + )), + BatchLocation::Storage => { + BuilderInner::File(FileWSetBuilder::for_merge(factories, batches, location)) + } + }, } } diff --git a/crates/dbsp/src/trace/ord/file/indexed_wset_batch.rs b/crates/dbsp/src/trace/ord/file/indexed_wset_batch.rs index 4148e0e134c..c7498a1dc42 100644 --- a/crates/dbsp/src/trace/ord/file/indexed_wset_batch.rs +++ b/crates/dbsp/src/trace/ord/file/indexed_wset_batch.rs @@ -1,5 +1,5 @@ use crate::storage::file::format::BatchMetadata; -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::{FilterKind, FilterStats}; use crate::{ DBData, DBWeight, NumEntries, Runtime, algebra::{AddAssignByRef, AddByRef, NegByRef}, @@ -10,7 +10,7 @@ use crate::{ storage::{ buffer_cache::CacheStats, file::{ - Factories as FileFactories, + Factories as FileFactories, FilterPlan, reader::{BulkRows, Cursor as FileCursor, Error as ReaderError, Reader}, writer::Writer2, }, @@ -19,8 +19,9 @@ use crate::{ Batch, BatchFactories, BatchLocation, BatchReader, BatchReaderFactories, Builder, Cursor, FileValBatch, VecIndexedWSetFactories, WeightedItem, cursor::{CursorFactory, CursorFactoryWrapper, Pending, Position, PushCursor}, + filter::BatchFilters, merge_batches_by_reference, - ord::{batch_filter::BatchFilters, file::UnwrapStorage, merge_batcher::MergeBatcher}, + ord::{file::UnwrapStorage, merge_batcher::MergeBatcher}, }, }; use crate::{DynZWeight, ZWeight}; @@ -284,7 +285,7 @@ where Runtime::buffer_cache, &*Runtime::storage_backend().unwrap_storage(), Runtime::file_writer_parameters(), - self.key_count(), + FilterPlan::::decide_filter(None, self.key_count()), ) .unwrap_storage(); @@ -402,6 +403,10 @@ where self.filters.stats().range_filter } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + self.filters.key_bounds() + } + #[inline] fn location(&self) -> BatchLocation { BatchLocation::Storage @@ -913,7 +918,39 @@ where Runtime::buffer_cache, &*Runtime::storage_backend().unwrap_storage(), Runtime::file_writer_parameters(), - key_capacity, + FilterPlan::::decide_filter(None, key_capacity), + ) + .unwrap_storage(), + weight: factories.weight_factory().default_box(), + num_tuples: 0, + stats: BatchMetadata::default(), + } + } + + fn for_merge<'a, B, I>( + factories: &FileIndexedWSetFactories, + batches: I, + _location: Option, + ) -> Self + where + B: BatchReader, + I: IntoIterator + Clone, + { + let key_capacity = batches.clone().into_iter().map(|b| b.key_count()).sum(); + let filter_plan = FilterPlan::from_batches(batches.clone()); + let key_filter = filter_plan.map_or_else( + || FilterPlan::::decide_filter(None, key_capacity), + |filter_plan| FilterPlan::decide_filter(Some(&filter_plan), key_capacity), + ); + Self { + factories: factories.clone(), + writer: Writer2::new( + &factories.factories0, + &factories.factories1, + Runtime::buffer_cache, + &*Runtime::storage_backend().unwrap_storage(), + Runtime::file_writer_parameters(), + key_filter, ) .unwrap_storage(), weight: factories.weight_factory().default_box(), diff --git a/crates/dbsp/src/trace/ord/file/key_batch.rs b/crates/dbsp/src/trace/ord/file/key_batch.rs index 4c02b61b065..f01e051902f 100644 --- a/crates/dbsp/src/trace/ord/file/key_batch.rs +++ b/crates/dbsp/src/trace/ord/file/key_batch.rs @@ -1,5 +1,5 @@ use crate::storage::file::format::BatchMetadata; -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::{FilterKind, FilterStats}; use crate::trace::cursor::Position; use crate::{ DBData, DBWeight, NumEntries, Runtime, Timestamp, @@ -10,7 +10,7 @@ use crate::{ storage::{ buffer_cache::CacheStats, file::{ - Factories as FileFactories, + Factories as FileFactories, FilterPlan, reader::{Cursor as FileCursor, Error as ReaderError, Reader}, writer::Writer2, }, @@ -18,7 +18,8 @@ use crate::{ trace::{ Batch, BatchFactories, BatchLocation, BatchReader, BatchReaderFactories, Builder, Cursor, WeightedItem, - ord::{batch_filter::BatchFilters, file::UnwrapStorage, merge_batcher::MergeBatcher}, + filter::BatchFilters, + ord::{file::UnwrapStorage, merge_batcher::MergeBatcher}, }, utils::Tup2, }; @@ -307,6 +308,10 @@ where self.filters.stats().range_filter } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + self.filters.key_bounds() + } + #[inline] fn location(&self) -> BatchLocation { BatchLocation::Storage @@ -673,7 +678,39 @@ where Runtime::buffer_cache, &*Runtime::storage_backend().unwrap_storage(), Runtime::file_writer_parameters(), - key_capacity, + FilterPlan::::decide_filter(None, key_capacity), + ) + .unwrap_storage(), + key: factories.opt_key_factory.default_box(), + num_tuples: 0, + stats: BatchMetadata::default(), + } + } + + fn for_merge<'a, B, I>( + factories: &FileKeyBatchFactories, + batches: I, + _location: Option, + ) -> Self + where + B: BatchReader, + I: IntoIterator + Clone, + { + let key_capacity = batches.clone().into_iter().map(|b| b.key_count()).sum(); + let filter_plan = FilterPlan::from_batches(batches.clone()); + let key_filter = filter_plan.map_or_else( + || FilterPlan::::decide_filter(None, key_capacity), + |filter_plan| FilterPlan::decide_filter(Some(&filter_plan), key_capacity), + ); + Self { + factories: factories.clone(), + writer: Writer2::new( + &factories.factories0, + &factories.factories1, + Runtime::buffer_cache, + &*Runtime::storage_backend().unwrap_storage(), + Runtime::file_writer_parameters(), + key_filter, ) .unwrap_storage(), key: factories.opt_key_factory.default_box(), diff --git a/crates/dbsp/src/trace/ord/file/val_batch.rs b/crates/dbsp/src/trace/ord/file/val_batch.rs index 6c3824a302f..f1ef3905703 100644 --- a/crates/dbsp/src/trace/ord/file/val_batch.rs +++ b/crates/dbsp/src/trace/ord/file/val_batch.rs @@ -1,5 +1,5 @@ use crate::storage::buffer_cache::CacheStats; -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::{FilterKind, FilterStats}; use crate::trace::BatchLocation; use crate::trace::cursor::Position; use crate::trace::ord::file::UnwrapStorage; @@ -10,14 +10,14 @@ use crate::{ Factory, LeanVec, WeightTrait, WithFactory, }, storage::file::{ - Factories as FileFactories, + Factories as FileFactories, FilterPlan, format::BatchMetadata, reader::{Cursor as FileCursor, Error as ReaderError, Reader}, writer::Writer2, }, trace::{ Batch, BatchFactories, BatchReader, BatchReaderFactories, Builder, Cursor, WeightedItem, - ord::{batch_filter::BatchFilters, merge_batcher::MergeBatcher}, + filter::BatchFilters, ord::merge_batcher::MergeBatcher, }, utils::Tup2, }; @@ -329,6 +329,10 @@ where self.filters.stats().range_filter } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + self.filters.key_bounds() + } + #[inline] fn location(&self) -> BatchLocation { BatchLocation::Storage @@ -716,7 +720,39 @@ where Runtime::buffer_cache, &*Runtime::storage_backend().unwrap_storage(), Runtime::file_writer_parameters(), - key_capacity, + FilterPlan::::decide_filter(None, key_capacity), + ) + .unwrap_storage(), + time_diffs: factories.timediff_factory.default_box(), + num_tuples: 0, + stats: BatchMetadata::default(), + } + } + + fn for_merge<'a, B, I>( + factories: &FileValBatchFactories, + batches: I, + _location: Option, + ) -> Self + where + B: BatchReader, + I: IntoIterator + Clone, + { + let key_capacity = batches.clone().into_iter().map(|b| b.key_count()).sum(); + let filter_plan = FilterPlan::from_batches(batches.clone()); + let key_filter = filter_plan.map_or_else( + || FilterPlan::::decide_filter(None, key_capacity), + |filter_plan| FilterPlan::decide_filter(Some(&filter_plan), key_capacity), + ); + Self { + factories: factories.clone(), + writer: Writer2::new( + &factories.factories0, + &factories.factories1, + Runtime::buffer_cache, + &*Runtime::storage_backend().unwrap_storage(), + Runtime::file_writer_parameters(), + key_filter, ) .unwrap_storage(), time_diffs: factories.timediff_factory.default_box(), diff --git a/crates/dbsp/src/trace/ord/file/wset_batch.rs b/crates/dbsp/src/trace/ord/file/wset_batch.rs index 33a005f98d6..9d06f3d0601 100644 --- a/crates/dbsp/src/trace/ord/file/wset_batch.rs +++ b/crates/dbsp/src/trace/ord/file/wset_batch.rs @@ -1,5 +1,5 @@ use crate::storage::file::format::BatchMetadata; -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::{FilterKind, FilterStats}; use crate::{ DBData, DBWeight, NumEntries, Runtime, algebra::{AddAssignByRef, AddByRef, NegByRef}, @@ -10,7 +10,7 @@ use crate::{ storage::{ buffer_cache::CacheStats, file::{ - Factories as FileFactories, + Factories as FileFactories, FilterPlan, reader::{BulkRows, Cursor as FileCursor, Error as ReaderError, Reader}, writer::Writer1, }, @@ -19,8 +19,9 @@ use crate::{ Batch, BatchFactories, BatchLocation, BatchReader, BatchReaderFactories, Builder, Cursor, DbspSerializer, Deserializer, FileKeyBatch, VecWSetFactories, WeightedItem, cursor::{CursorFactoryWrapper, Pending, Position, PushCursor}, + filter::BatchFilters, merge_batches_by_reference, - ord::{batch_filter::BatchFilters, file::UnwrapStorage, merge_batcher::MergeBatcher}, + ord::{file::UnwrapStorage, merge_batcher::MergeBatcher}, }, }; use crate::{DynZWeight, ZWeight}; @@ -258,7 +259,7 @@ where Runtime::buffer_cache, &*Runtime::storage_backend().unwrap(), Runtime::file_writer_parameters(), - self.key_count(), + FilterPlan::::decide_filter(None, self.key_count()), ) .unwrap_storage(); @@ -391,6 +392,10 @@ where self.filters.stats().range_filter } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + self.filters.key_bounds() + } + #[inline] fn location(&self) -> BatchLocation { BatchLocation::Storage @@ -826,7 +831,38 @@ where Runtime::buffer_cache, &*Runtime::storage_backend().unwrap_storage(), Runtime::file_writer_parameters(), - key_capacity, + FilterPlan::::decide_filter(None, key_capacity), + ) + .unwrap_storage(), + weight: factories.weight_factory().default_box(), + num_tuples: 0, + stats: BatchMetadata::default(), + } + } + + fn for_merge<'a, B, I>( + factories: & as BatchReader>::Factories, + batches: I, + _location: Option, + ) -> Self + where + B: BatchReader, + I: IntoIterator + Clone, + { + let key_capacity = batches.clone().into_iter().map(|b| b.key_count()).sum(); + let filter_plan = FilterPlan::from_batches(batches.clone()); + let key_filter = filter_plan.map_or_else( + || FilterPlan::::decide_filter(None, key_capacity), + |filter_plan| FilterPlan::decide_filter(Some(&filter_plan), key_capacity), + ); + Self { + factories: factories.clone(), + writer: Writer1::new( + &factories.file_factories, + Runtime::buffer_cache, + &*Runtime::storage_backend().unwrap_storage(), + Runtime::file_writer_parameters(), + key_filter, ) .unwrap_storage(), weight: factories.weight_factory().default_box(), diff --git a/crates/dbsp/src/trace/ord/vec/indexed_wset_batch.rs b/crates/dbsp/src/trace/ord/vec/indexed_wset_batch.rs index 092e87d6717..5d47c5e129f 100644 --- a/crates/dbsp/src/trace/ord/vec/indexed_wset_batch.rs +++ b/crates/dbsp/src/trace/ord/vec/indexed_wset_batch.rs @@ -1,5 +1,5 @@ +use crate::storage::file::FilterStats; use crate::storage::file::SerializerInner; -use crate::storage::filter_stats::FilterStats; use crate::trace::ord::merge_batcher::MergeBatcher; use crate::{ DBData, DBWeight, Error, NumEntries, @@ -461,9 +461,12 @@ where FilterStats::default() } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + Some((self.layer.keys.first()?, self.layer.keys.last()?)) + } + fn sample_keys(&self, rng: &mut RG, sample_size: usize, sample: &mut DynVec) where - Self::Time: PartialEq<()>, RG: Rng, { self.layer.sample_keys(rng, sample_size, sample); diff --git a/crates/dbsp/src/trace/ord/vec/key_batch.rs b/crates/dbsp/src/trace/ord/vec/key_batch.rs index a1598db2d41..c7c1f24a8ca 100644 --- a/crates/dbsp/src/trace/ord/vec/key_batch.rs +++ b/crates/dbsp/src/trace/ord/vec/key_batch.rs @@ -1,4 +1,4 @@ -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::FilterStats; use crate::trace::ord::merge_batcher::MergeBatcher; use crate::{ DBData, DBWeight, NumEntries, Timestamp, @@ -320,9 +320,12 @@ where FilterStats::default() } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + Some((self.layer.keys.first()?, self.layer.keys.last()?)) + } + fn sample_keys(&self, rng: &mut RG, sample_size: usize, sample: &mut DynVec) where - Self::Time: PartialEq<()>, RG: Rng, { self.layer.sample_keys(rng, sample_size, sample); diff --git a/crates/dbsp/src/trace/ord/vec/val_batch.rs b/crates/dbsp/src/trace/ord/vec/val_batch.rs index e19217ba530..b58383a63c7 100644 --- a/crates/dbsp/src/trace/ord/vec/val_batch.rs +++ b/crates/dbsp/src/trace/ord/vec/val_batch.rs @@ -1,5 +1,5 @@ use crate::ZWeight; -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::FilterStats; use crate::trace::cursor::Position; use crate::trace::ord::merge_batcher::MergeBatcher; use crate::{ @@ -381,9 +381,12 @@ where FilterStats::default() } + fn key_bounds(&self) -> Option<(&Self::Key, &Self::Key)> { + Some((self.layer.keys.first()?, self.layer.keys.last()?)) + } + fn sample_keys(&self, rng: &mut RG, sample_size: usize, sample: &mut DynVec) where - Self::Time: PartialEq<()>, RG: Rng, { self.layer.sample_keys(rng, sample_size, sample); diff --git a/crates/dbsp/src/trace/ord/vec/wset_batch.rs b/crates/dbsp/src/trace/ord/vec/wset_batch.rs index 2e262d21ede..5c21bce1b9f 100644 --- a/crates/dbsp/src/trace/ord/vec/wset_batch.rs +++ b/crates/dbsp/src/trace/ord/vec/wset_batch.rs @@ -1,4 +1,4 @@ -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::FilterStats; use crate::{ DBData, DBWeight, NumEntries, algebra::{NegByRef, ZRingValue}, @@ -363,6 +363,10 @@ impl BatchReader for VecWSet Option<(&Self::Key, &Self::Key)> { + Some((self.layer.keys.first()?, self.layer.keys.last()?)) + } + fn sample_keys(&self, rng: &mut RG, sample_size: usize, sample: &mut DynVec) where RG: Rng, diff --git a/crates/dbsp/src/trace/sampling.rs b/crates/dbsp/src/trace/sampling.rs new file mode 100644 index 00000000000..1798c4bfc8d --- /dev/null +++ b/crates/dbsp/src/trace/sampling.rs @@ -0,0 +1,61 @@ +use crate::{ + dynamic::DynVec, + trace::{BatchReader, BatchReaderFactories, Cursor, cursor::CursorList}, +}; +use rand::Rng; + +/// Samples keys from a set of batches by invoking each batch's +/// [`BatchReader::sample_keys`] implementation and merging the results. +/// +/// `sample_size_for` decides how many keys to request from each batch. The +/// helper deduplicates keys across batches before appending them to `sample`, +/// which keeps it usable for overlapping inputs such as merge planning. +pub(crate) fn sample_keys_from_batches( + factories: &B::Factories, + batches: &[&B], + rng: &mut RG, + mut sample_size_for: F, + sample: &mut DynVec, +) where + B: BatchReader, + RG: Rng, + F: FnMut(&B) -> usize, +{ + if batches.is_empty() { + return; + } + + let total_sample_size = batches + .iter() + .map(|batch| sample_size_for(*batch)) + .sum::(); + if total_sample_size == 0 { + return; + } + + let mut intermediate = factories.keys_factory().default_box(); + let mut merged_cursor = CursorList::new( + factories.weight_factory(), + batches.iter().map(|batch| batch.cursor()).collect(), + ); + intermediate.reserve(total_sample_size); + + for batch in batches { + let sample_size = sample_size_for(*batch); + if sample_size == 0 { + continue; + } + batch.sample_keys(rng, sample_size, intermediate.as_mut()); + } + + intermediate.as_mut().sort_unstable(); + intermediate.dedup(); + for key in intermediate.dyn_iter_mut() { + merged_cursor.seek_key(key); + if let Some(current_key) = merged_cursor.get_key() + && current_key == key + { + sample.push_ref(key); + } + } +} diff --git a/crates/dbsp/src/trace/spine_async.rs b/crates/dbsp/src/trace/spine_async.rs index 11defddadf1..ff3f14eafe6 100644 --- a/crates/dbsp/src/trace/spine_async.rs +++ b/crates/dbsp/src/trace/spine_async.rs @@ -18,17 +18,19 @@ use crate::{ MERGING_MEMORY_RECORDS_COUNT, MERGING_SIZE_BYTES, MERGING_STORAGE_RECORDS_COUNT, MetaItem, MetricId, MetricReading, NEGATIVE_WEIGHT_COUNT, OperatorMeta, RANGE_FILTER_HIT_RATE_PERCENT, RANGE_FILTER_HITS_COUNT, RANGE_FILTER_MISSES_COUNT, - RANGE_FILTER_SIZE_BYTES, SPINE_BATCHES_COUNT, SPINE_STORAGE_SIZE_BYTES, + RANGE_FILTER_SIZE_BYTES, ROARING_FILTER_HIT_RATE_PERCENT, ROARING_FILTER_HITS_COUNT, + ROARING_FILTER_MISSES_COUNT, ROARING_FILTER_SIZE_BYTES, SPINE_BATCHES_COUNT, + SPINE_STORAGE_SIZE_BYTES, }, metrics::COMPACTION_STALL_TIME_NANOSECONDS, negative_weight_multiplier, runtime::{TOKIO_BUFFER_CACHE, TOKIO_WORKER_INDEX}, }, - dynamic::{DynVec, Factory, Weight}, + dynamic::{DynVec, Factory}, samply::SamplySpan, storage::{ buffer_cache::{BufferCache, CacheStats}, - filter_stats::FilterStats, + file::{FilterKind, FilterStats}, }, time::Timestamp, trace::{ @@ -36,6 +38,7 @@ use crate::{ cursor::{CursorList, Position}, merge_batches, ord::fallback::pick_insert_destination, + sample_keys_from_batches, spine_async::{ list_merger::ArcListMerger, push_merger::ArcPushMerger, snapshot::FetchList, }, @@ -68,7 +71,6 @@ use std::{ use std::{collections::VecDeque, sync::atomic::Ordering}; use std::{ fmt::{self, Debug, Display, Formatter}, - ops::DerefMut, sync::Condvar, }; use std::{ops::RangeInclusive, sync::Mutex}; @@ -780,19 +782,25 @@ where let mut cache_stats = spine_stats.cache_stats; let mut storage_size = 0; let mut merging_size = 0; - let mut membership_filter_stats = FilterStats::default(); + let mut membership_filter_stats = BTreeMap::::new(); let mut range_filter_stats = FilterStats::default(); - let mut storage_records = 0; + let mut bloom_filter_records = 0; for (batch, merging) in batches { cache_stats += batch.cache_stats(); - membership_filter_stats += batch.membership_filter_stats(); + let kind = batch.membership_filter_kind(); + if kind != FilterKind::None { + *membership_filter_stats.entry(kind).or_default() += + batch.membership_filter_stats(); + } + if kind == FilterKind::Bloom { + bloom_filter_records += batch.key_count(); + } range_filter_stats += batch.range_filter_stats(); let on_storage = batch.location() == BatchLocation::Storage; if on_storage || merging { let size = batch.approximate_byte_size(); if on_storage { storage_size += size; - storage_records += batch.key_count(); } if merging { merging_size += size; @@ -800,9 +808,16 @@ where } } - if storage_records > 0 { + let bloom_filter_stats = membership_filter_stats + .remove(&FilterKind::Bloom) + .unwrap_or_default(); + let roaring_filter_stats = membership_filter_stats + .remove(&FilterKind::Roaring) + .unwrap_or_default(); + + if bloom_filter_records > 0 { let bits_per_key = - membership_filter_stats.size_byte as f64 * 8.0 / storage_records as f64; + bloom_filter_stats.size_byte as f64 * 8.0 / bloom_filter_records as f64; let bits_per_key = bits_per_key as usize; meta.extend(metadata! { BLOOM_FILTER_BITS_PER_KEY => MetaItem::Int(bits_per_key) @@ -839,25 +854,48 @@ where MetricReading::new( BLOOM_FILTER_SIZE_BYTES, Vec::new(), - MetaItem::bytes(membership_filter_stats.size_byte), + MetaItem::bytes(bloom_filter_stats.size_byte), ), MetricReading::new( BLOOM_FILTER_HITS_COUNT, Vec::new(), - MetaItem::Count(membership_filter_stats.hits), + MetaItem::Count(bloom_filter_stats.hits), ), MetricReading::new( BLOOM_FILTER_MISSES_COUNT, Vec::new(), - MetaItem::Count(membership_filter_stats.misses), + MetaItem::Count(bloom_filter_stats.misses), ), MetricReading::new( BLOOM_FILTER_HIT_RATE_PERCENT, Vec::new(), MetaItem::Percent { - numerator: membership_filter_stats.hits as u64, - denominator: membership_filter_stats.hits as u64 - + membership_filter_stats.misses as u64, + numerator: bloom_filter_stats.hits as u64, + denominator: bloom_filter_stats.hits as u64 + bloom_filter_stats.misses as u64, + }, + ), + MetricReading::new( + ROARING_FILTER_SIZE_BYTES, + Vec::new(), + MetaItem::bytes(roaring_filter_stats.size_byte), + ), + MetricReading::new( + ROARING_FILTER_HITS_COUNT, + Vec::new(), + MetaItem::Count(roaring_filter_stats.hits), + ), + MetricReading::new( + ROARING_FILTER_MISSES_COUNT, + Vec::new(), + MetaItem::Count(roaring_filter_stats.misses), + ), + MetricReading::new( + ROARING_FILTER_HIT_RATE_PERCENT, + Vec::new(), + MetaItem::Percent { + numerator: roaring_filter_stats.hits as u64, + denominator: roaring_filter_stats.hits as u64 + + roaring_filter_stats.misses as u64, }, ), MetricReading::new( @@ -1291,57 +1329,6 @@ where } } -/// Samples `sample_size` keys from a set of batches. -/// -/// See [`BatchReader::sample_keys`](`crate::trace::BatchReader::sample_keys`) for more details. -pub(crate) fn sample_keys_from_batches( - factories: &B::Factories, - batches: &[Arc], - rng: &mut RG, - sample_size: usize, - sample: &mut DynVec, -) where - B: Batch, - B::Time: PartialEq<()>, - RG: Rng, -{ - let total_keys = batches.iter().map(|batch| batch.key_count()).sum::(); - - if sample_size == 0 || total_keys == 0 { - // Avoid division by zero. - return; - } - - // Sample each batch, picking the number of keys proportional to - // batch size. - let mut intermediate = factories.keys_factory().default_box(); - intermediate.reserve(sample_size); - - for batch in batches { - batch.sample_keys( - rng, - ((batch.key_count() as u128) * (sample_size as u128) / (total_keys as u128)) as usize, - intermediate.as_mut(), - ); - } - - // Drop duplicate keys and keys that appear with 0 weight, i.e., - // get canceled out across multiple batches. - intermediate.deref_mut().sort_unstable(); - intermediate.dedup(); - - let mut cursor = SpineCursor::new_cursor(factories, batches.to_vec()); - for key in intermediate.dyn_iter_mut() { - cursor.seek_key(key); - if let Some(current_key) = cursor.get_key() - && current_key == key - { - debug_assert!(cursor.val_valid() && !cursor.weight().is_zero()); - sample.push_ref(key); - } - } -} - impl BatchReader for Spine where B: Batch, @@ -1382,14 +1369,6 @@ where .sum() } - fn membership_filter_stats(&self) -> FilterStats { - self.merger - .get_batches() - .iter() - .map(|batch| batch.membership_filter_stats()) - .sum() - } - fn range_filter_stats(&self) -> FilterStats { self.merger .get_batches() @@ -1404,14 +1383,23 @@ where fn sample_keys(&self, rng: &mut RG, sample_size: usize, sample: &mut DynVec) where - Self::Time: PartialEq<()>, RG: Rng, { + let batches = self.merger.get_batches(); + let total_keys = batches.iter().map(|batch| batch.key_count()).sum::(); + let batch_refs: Vec<_> = batches.iter().map(Arc::as_ref).collect(); sample_keys_from_batches( &self.factories, - &self.merger.get_batches(), + &batch_refs, rng, - sample_size, + |batch| { + if sample_size == 0 || total_keys == 0 { + 0 + } else { + ((batch.key_count() as u128) * (sample_size as u128) / (total_keys as u128)) + as usize + } + }, sample, ); } diff --git a/crates/dbsp/src/trace/spine_async/snapshot.rs b/crates/dbsp/src/trace/spine_async/snapshot.rs index 8f7e080779d..a2bf046d3d7 100644 --- a/crates/dbsp/src/trace/spine_async/snapshot.rs +++ b/crates/dbsp/src/trace/spine_async/snapshot.rs @@ -13,10 +13,12 @@ use size_of::SizeOf; use super::SpineCursor; use crate::NumEntries; use crate::dynamic::{DynVec, Factory}; -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::FilterStats; use crate::trace::cursor::{CursorFactory, CursorList}; -use crate::trace::spine_async::sample_keys_from_batches; -use crate::trace::{Batch, BatchReader, BatchReaderFactories, Cursor, Spine, merge_batches}; +use crate::trace::{ + Batch, BatchReader, BatchReaderFactories, Cursor, Spine, merge_batches, + sample_keys_from_batches, +}; pub trait WithSnapshot: Sized { type Batch: Batch; @@ -241,14 +243,26 @@ where fn sample_keys(&self, rng: &mut RG, sample_size: usize, sample: &mut DynVec) where - Self::Time: PartialEq<()>, RG: Rng, { + let total_keys = self + .batches + .iter() + .map(|batch| batch.key_count()) + .sum::(); + let batch_refs: Vec<_> = self.batches.iter().map(Arc::as_ref).collect(); sample_keys_from_batches( &self.factories, - self.batches.as_slice(), + &batch_refs, rng, - sample_size, + |batch| { + if sample_size == 0 || total_keys == 0 { + 0 + } else { + ((batch.key_count() as u128) * (sample_size as u128) / (total_keys as u128)) + as usize + } + }, sample, ); } diff --git a/crates/dbsp/src/trace/test.rs b/crates/dbsp/src/trace/test.rs index b1849209bd9..94dc4a07068 100644 --- a/crates/dbsp/src/trace/test.rs +++ b/crates/dbsp/src/trace/test.rs @@ -12,14 +12,15 @@ use size_of::SizeOf; use crate::{ DynZWeight, Runtime, ZWeight, algebra::{ - IndexedZSet, NegByRef, OrdIndexedZSet, OrdIndexedZSetFactories, OrdZSet, OrdZSetFactories, - ZBatch, ZSet, + AddByRef, IndexedZSet, NegByRef, OrdIndexedZSet, OrdIndexedZSetFactories, OrdZSet, + OrdZSetFactories, ZBatch, ZSet, }, circuit::{CircuitConfig, mkconfig}, dynamic::{DowncastTrait, DynData, DynUnit, DynWeightedPairs, Erase, LeanVec, pair::DynPair}, + storage::{buffer_cache::CacheStats, file::FilterKind}, trace::{ - Batch, BatchReader, BatchReaderFactories, Builder, FileIndexedWSetFactories, - FileWSetFactories, GroupFilter, Spine, Trace, + Batch, BatchLocation, BatchReader, BatchReaderFactories, Builder, FileIndexedWSetFactories, + FileWSetFactories, GroupFilter, ListMerger, Spine, Trace, cursor::{Cursor, CursorPair}, ord::{ FileIndexedWSet, FileKeyBatch, FileKeyBatchFactories, FileValBatch, @@ -31,7 +32,7 @@ use crate::{ assert_trace_eq, test_batch_sampling, test_trace_sampling, }, }, - utils::{Tup2, Tup3, Tup4}, + utils::{Tup1, Tup2, Tup3, Tup4}, }; use super::Filter; @@ -828,6 +829,13 @@ where F: FnOnce() + Clone + Send + 'static, { let (_temp_dir, config) = mkconfig(); + run_in_circuit_with_storage_config(config, f); +} + +fn run_in_circuit_with_storage_config(config: CircuitConfig, f: F) +where + F: FnOnce() + Clone + Send + 'static, +{ let count = Arc::new(AtomicUsize::new(0)); Runtime::init_circuit(config, { let count = count.clone(); @@ -843,6 +851,145 @@ where assert_eq!(count.load(Ordering::Relaxed), 1); } +fn total_cache_accesses(stats: CacheStats) -> u64 { + stats + .0 + .iter() + .map(|(_, accesses)| accesses.iter().map(|(_, counts)| counts.count).sum::()) + .sum() +} + +fn build_file_wset_u32(keys: &[u32]) -> FileWSet { + let factories = >::new::(); + let mut builder = + as Batch>::Builder::with_capacity(&factories, keys.len(), 0); + + for key in keys { + let weight: ZWeight = 1; + builder.push_time_diff(&(), weight.erase()); + builder.push_key(key.erase()); + } + + builder.done() +} + +fn build_file_wset_tup1_i32(keys: &[i32]) -> FileWSet { + let factories = >::new::, (), ZWeight>(); + let mut builder = + as Batch>::Builder::with_capacity(&factories, keys.len(), 0); + + for key in keys { + let weight: ZWeight = 1; + builder.push_time_diff(&(), weight.erase()); + builder.push_key(Tup1(*key).erase()); + } + + builder.done() +} + +fn build_fallback_wset_i32(keys: &[i32]) -> crate::trace::FallbackWSet { + let factories = + >::new::(); + let mut erased_tuples = zset_tuples(keys.iter().copied().map(|key| Tup2(key, 1)).collect()); + crate::trace::FallbackWSet::::dyn_from_tuples( + &factories, + (), + &mut erased_tuples, + ) +} + +#[test] +fn test_file_wset_roaring_u32_seek_key_exact_skips_absent_reads() { + let (_temp_dir, mut config) = mkconfig(); + config.dev_tweaks.enable_roaring = Some(true); + + run_in_circuit_with_storage_config(config, move || { + let batch = build_file_wset_u32(&[1, 3, 7]); + let mut cursor = batch.cursor(); + let before = total_cache_accesses(batch.cache_stats()); + + let missing = 2u32; + assert!(!cursor.seek_key_exact(missing.erase(), None)); + assert_eq!(total_cache_accesses(batch.cache_stats()), before); + + let present = 3u32; + assert!(cursor.seek_key_exact(present.erase(), None)); + }); +} + +#[test] +fn test_file_wset_tup1_i32_roaring_seek_key_exact_skips_absent_reads() { + let (_temp_dir, mut config) = mkconfig(); + config.dev_tweaks.enable_roaring = Some(true); + + run_in_circuit_with_storage_config(config, move || { + let batch = build_file_wset_tup1_i32(&[-7, 1, 3]); + let mut cursor = batch.cursor(); + let before = total_cache_accesses(batch.cache_stats()); + + let missing = Tup1(2i32); + assert!(!cursor.seek_key_exact(missing.erase(), None)); + assert_eq!(total_cache_accesses(batch.cache_stats()), before); + + let present = Tup1(3i32); + assert!(cursor.seek_key_exact(present.erase(), None)); + }); +} + +#[test] +fn test_file_wset_roaring_filter_rebuilt_after_merge() { + let (_temp_dir, mut config) = mkconfig(); + config.dev_tweaks.enable_roaring = Some(true); + + run_in_circuit_with_storage_config(config, move || { + let lhs = build_file_wset_u32(&[1, 5]); + let rhs = build_file_wset_u32(&[3, 7]); + let merged = lhs.add_by_ref(&rhs); + + let mut cursor = merged.cursor(); + let before = total_cache_accesses(merged.cache_stats()); + + let missing = 4u32; + assert!(!cursor.seek_key_exact(missing.erase(), None)); + assert_eq!(total_cache_accesses(merged.cache_stats()), before); + + let present = 7u32; + assert!(cursor.seek_key_exact(present.erase(), None)); + }); +} + +#[test] +fn test_fallback_wset_roaring_filter_rebuilt_after_storage_merge() { + let (_temp_dir, mut config) = mkconfig(); + config.dev_tweaks.enable_roaring = Some(true); + config.storage.as_mut().unwrap().options.min_storage_bytes = Some(0); + + run_in_circuit_with_storage_config(config, move || { + let lhs = build_fallback_wset_i32(&[1, 5]); + let rhs = build_fallback_wset_i32(&[3, 7]); + let factories = + >::new::(); + let merged: crate::trace::FallbackWSet = ListMerger::merge( + &factories, + as Batch>::Builder::for_merge( + &factories, + [&lhs, &rhs], + Some(BatchLocation::Storage), + ), + vec![lhs.merge_cursor(None, None), rhs.merge_cursor(None, None)], + ); + + assert_eq!(merged.membership_filter_kind(), FilterKind::Roaring); + + let mut cursor = merged.cursor(); + let before = total_cache_accesses(merged.cache_stats()); + + let missing = 4i32; + assert!(!cursor.seek_key_exact(missing.erase(), None)); + assert_eq!(total_cache_accesses(merged.cache_stats()), before); + }); +} + proptest! { #![proptest_config(ProptestConfig::with_cases(1000))] diff --git a/crates/dbsp/src/trace/test/test_batch.rs b/crates/dbsp/src/trace/test/test_batch.rs index 99d64417552..34bc1728ca8 100644 --- a/crates/dbsp/src/trace/test/test_batch.rs +++ b/crates/dbsp/src/trace/test/test_batch.rs @@ -3,7 +3,7 @@ //! So far, only methods/traits used in tests have been implemented. #![allow(clippy::type_complexity)] -use crate::storage::filter_stats::FilterStats; +use crate::storage::file::FilterStats; use crate::{ DBData, DBWeight, NumEntries, Timestamp, dynamic::{ diff --git a/crates/dbsp/src/utils.rs b/crates/dbsp/src/utils.rs index 6849a75f3b2..a8528842407 100644 --- a/crates/dbsp/src/utils.rs +++ b/crates/dbsp/src/utils.rs @@ -6,6 +6,7 @@ mod consolidation; mod graph; mod is_none; mod sort; +mod supports_roaring; pub mod tuple; #[cfg(test)] @@ -31,6 +32,7 @@ pub use consolidation::{ pub use graph::components; pub use is_none::IsNone; +pub use supports_roaring::SupportsRoaring; #[allow(unused_imports)] pub use dot::{DotEdgeAttributes, DotNodeAttributes}; diff --git a/crates/dbsp/src/utils/supports_roaring.rs b/crates/dbsp/src/utils/supports_roaring.rs new file mode 100644 index 00000000000..0fa1388b807 --- /dev/null +++ b/crates/dbsp/src/utils/supports_roaring.rs @@ -0,0 +1,262 @@ +//! Trait for key types that can be mapped into a roaring bitmap domain. + +use crate::dynamic::{BSet, DowncastTrait, DynData, LeanVec}; +use crate::time::UnitTimestamp; +use std::collections::BTreeMap; +use std::rc::Rc; +use std::sync::Arc; +use uuid::Uuid; + +pub trait SupportsRoaring { + #[inline] + fn supports_roaring32(&self) -> bool { + false + } + + #[inline] + fn roaring_u32_offset(&self, _min: &Self) -> Option + where + Self: Sized, + { + None + } + + #[inline] + fn into_roaring_u32(&self, _min: &DynData) -> Option { + None + } + + #[inline] + fn into_roaring_u32_checked(&self, min: &DynData) -> u32 { + self.into_roaring_u32(min) + .expect("roaring-u32 filter was selected for a key outside the planned batch range") + } +} + +#[macro_export] +macro_rules! never_roaring_filter { + ($($ty:ty),* $(,)?) => { + $( + impl $crate::utils::SupportsRoaring for $ty {} + )* + }; +} + +never_roaring_filter!( + (), + bool, + char, + i8, + i16, + i128, + u8, + u16, + u128, + f32, + f64, + usize, + isize, + String, + UnitTimestamp, + Uuid +); + +impl SupportsRoaring for u32 { + #[inline] + fn supports_roaring32(&self) -> bool { + true + } + + #[inline] + fn roaring_u32_offset(&self, min: &Self) -> Option { + self.checked_sub(*min) + } + + #[inline] + fn into_roaring_u32(&self, min: &DynData) -> Option { + self.roaring_u32_offset(min.downcast_checked::()) + } +} + +impl SupportsRoaring for i32 { + #[inline] + fn supports_roaring32(&self) -> bool { + true + } + + #[inline] + fn roaring_u32_offset(&self, min: &Self) -> Option { + let diff = i64::from(*self) - i64::from(*min); + (0..=i64::from(u32::MAX)) + .contains(&diff) + .then_some(diff as u32) + } + + #[inline] + fn into_roaring_u32(&self, min: &DynData) -> Option { + self.roaring_u32_offset(min.downcast_checked::()) + } +} + +impl SupportsRoaring for u64 { + #[inline] + fn supports_roaring32(&self) -> bool { + true + } + + #[inline] + fn roaring_u32_offset(&self, min: &Self) -> Option { + self.checked_sub(*min) + .filter(|diff| *diff <= u64::from(u32::MAX)) + .map(|diff| diff as u32) + } + + #[inline] + fn into_roaring_u32(&self, min: &DynData) -> Option { + self.roaring_u32_offset(min.downcast_checked::()) + } +} + +impl SupportsRoaring for i64 { + #[inline] + fn supports_roaring32(&self) -> bool { + true + } + + #[inline] + fn roaring_u32_offset(&self, min: &Self) -> Option { + let diff = i128::from(*self) - i128::from(*min); + (0..=i128::from(u32::MAX)) + .contains(&diff) + .then_some(diff as u32) + } + + #[inline] + fn into_roaring_u32(&self, min: &DynData) -> Option { + self.roaring_u32_offset(min.downcast_checked::()) + } +} + +impl SupportsRoaring for Option {} + +#[macro_export] +macro_rules! never_roaring_filter_1 { + ($($wrapper:ident),* $(,)?) => { + $( + impl $crate::utils::SupportsRoaring for $wrapper {} + )* + }; +} + +never_roaring_filter_1!(Vec, LeanVec, BSet); + +#[macro_export] +macro_rules! delegate_supports_roaring { + ($($wrapper:ident),* $(,)?) => { + $( + impl $crate::utils::SupportsRoaring for $wrapper { + #[inline] + fn supports_roaring32(&self) -> bool { + self.as_ref().supports_roaring32() + } + + #[inline] + fn roaring_u32_offset(&self, min: &Self) -> Option { + self.as_ref().roaring_u32_offset(min.as_ref()) + } + + #[inline] + fn into_roaring_u32(&self, min: &$crate::dynamic::DynData) -> Option { + self.as_ref().into_roaring_u32(min) + } + + #[inline] + fn into_roaring_u32_checked(&self, min: &$crate::dynamic::DynData) -> u32 { + self.as_ref().into_roaring_u32_checked(min) + } + } + )* + }; +} + +delegate_supports_roaring!(Box, Rc, Arc); + +#[macro_export] +macro_rules! never_roaring_filter_tuples { + ($($name:ident),+) => { + impl<$($name),+> SupportsRoaring for ($($name,)+) {} + }; +} + +never_roaring_filter_tuples!(A); +never_roaring_filter_tuples!(A, B); +never_roaring_filter_tuples!(A, B, C); +never_roaring_filter_tuples!(A, B, C, D); +never_roaring_filter_tuples!(A, B, C, D, E); +never_roaring_filter_tuples!(A, B, C, D, E, F); + +impl SupportsRoaring for BTreeMap {} + +impl SupportsRoaring for crate::utils::Tup1 { + #[inline] + fn supports_roaring32(&self) -> bool { + self.0.supports_roaring32() + } + + #[inline] + fn roaring_u32_offset(&self, min: &Self) -> Option { + self.0.roaring_u32_offset(&min.0) + } + + #[inline] + fn into_roaring_u32(&self, min: &DynData) -> Option { + self.roaring_u32_offset(min.downcast_checked::()) + } + + #[inline] + fn into_roaring_u32_checked(&self, min: &DynData) -> u32 { + self.roaring_u32_offset(min.downcast_checked::()) + .expect("roaring-u32 filter was selected for a key outside the planned batch range") + } +} + +#[cfg(test)] +mod test { + use super::SupportsRoaring; + use crate::{dynamic::DynData, utils::Tup1}; + + #[test] + fn supported_roaring_keys() { + assert!(7u32.supports_roaring32()); + assert_eq!(7u32.into_roaring_u32((&0u32) as &DynData), Some(7)); + + assert!((-7i32).supports_roaring32()); + assert_eq!((-7i32).into_roaring_u32((&-10i32) as &DynData), Some(3)); + + assert!(Tup1(-7i32).supports_roaring32()); + assert_eq!( + Tup1(-7i32).into_roaring_u32((&Tup1(-10i32)) as &DynData), + Some(3) + ); + + assert!(11u64.supports_roaring32()); + assert_eq!(11u64.into_roaring_u32((&9u64) as &DynData), Some(2)); + + assert!((-2i64).supports_roaring32()); + assert_eq!((-2i64).into_roaring_u32((&-5i64) as &DynData), Some(3)); + } + + #[test] + fn unsupported_roaring_keys() { + assert!(!"feldera".to_string().supports_roaring32()); + assert_eq!( + "feldera" + .to_string() + .into_roaring_u32((&String::new()) as &DynData), + None + ); + + assert_eq!(11u64.into_roaring_u32((&(u64::MAX - 1)) as &DynData), None); + assert_eq!(5i64.into_roaring_u32((&10i64) as &DynData), None); + } +} diff --git a/crates/feldera-macros/src/lib.rs b/crates/feldera-macros/src/lib.rs index 0461482f405..c4b3d9086bd 100644 --- a/crates/feldera-macros/src/lib.rs +++ b/crates/feldera-macros/src/lib.rs @@ -1,4 +1,4 @@ -//! Procedural macros for Feldera tuple types and `IsNone`. +//! Procedural macros for Feldera tuple types and utility traits. //! //! The `declare_tuple!` macro decides which layout to use based on tuple size //! and the active storage format rules. @@ -51,6 +51,8 @@ pub fn derive_not_none(item: TokenStream) -> TokenStream { inner } } + + impl #impl_generics ::dbsp::utils::SupportsRoaring for #ident #ty_generics #where_clause {} }; TokenStream::from(expanded) diff --git a/crates/feldera-macros/src/tuples.rs b/crates/feldera-macros/src/tuples.rs index 6de257791cb..256f7b5e7a9 100644 --- a/crates/feldera-macros/src/tuples.rs +++ b/crates/feldera-macros/src/tuples.rs @@ -247,6 +247,14 @@ pub(super) fn declare_tuple_impl(tuple: TupleDef) -> TokenStream2 { } }; + let roaring_u32_key_impl = if num_elements == 1 { + quote! {} + } else { + quote! { + impl<#(#generics),*> ::dbsp::utils::SupportsRoaring for #name<#(#generics),*> {} + } + }; + let sparse_get_methods = fields .iter() .enumerate() @@ -969,6 +977,7 @@ pub(super) fn declare_tuple_impl(tuple: TupleDef) -> TokenStream2 { #copy_impl #checkpoint_impl #not_an_option + #roaring_u32_key_impl }); expanded diff --git a/crates/feldera-types/src/config/dev_tweaks.rs b/crates/feldera-types/src/config/dev_tweaks.rs index 057daa4c3cb..91679b15cb2 100644 --- a/crates/feldera-types/src/config/dev_tweaks.rs +++ b/crates/feldera-types/src/config/dev_tweaks.rs @@ -170,6 +170,11 @@ pub struct DevTweaks { #[serde(skip_serializing_if = "Option::is_none")] pub bloom_false_positive_rate: Option, + /// Whether file-backed batches may use roaring membership filters when the + /// key type supports them. + #[serde(skip_serializing_if = "Option::is_none")] + pub enable_roaring: Option, + /// Maximum batch size in records for level 0 merges. #[serde(skip_serializing_if = "Option::is_none")] pub max_level0_batch_size_records: Option, @@ -240,6 +245,9 @@ impl DevTweaks { pub fn bloom_false_positive_rate(&self) -> f64 { self.bloom_false_positive_rate.unwrap_or(0.0001) } + pub fn enable_roaring(&self) -> bool { + self.enable_roaring.unwrap_or(true) + } pub fn negative_weight_multiplier(&self) -> u16 { self.negative_weight_multiplier.unwrap_or(0) } diff --git a/crates/fxp/src/dbsp_impl.rs b/crates/fxp/src/dbsp_impl.rs index 37f8ec9160c..d7e999435c6 100644 --- a/crates/fxp/src/dbsp_impl.rs +++ b/crates/fxp/src/dbsp_impl.rs @@ -1,6 +1,6 @@ use dbsp::NumEntries; use dbsp::algebra::{HasOne, HasZero, MulByRef, OptionWeightType}; -use dbsp::utils::IsNone; +use dbsp::utils::{IsNone, SupportsRoaring}; use feldera_types::serde_with_context::{ DeserializeWithContext, SerializeWithContext, SqlSerdeConfig, serde_config::DecimalFormat, }; @@ -38,6 +38,8 @@ impl IsNone for Fixed { } } +impl SupportsRoaring for Fixed {} + impl OptionWeightType for Fixed {} impl OptionWeightType for &Fixed {} diff --git a/crates/nexmark/src/queries/q9.rs b/crates/nexmark/src/queries/q9.rs index 9aa8b12abfc..2e3040c1d22 100644 --- a/crates/nexmark/src/queries/q9.rs +++ b/crates/nexmark/src/queries/q9.rs @@ -43,6 +43,7 @@ pub struct Q9Output( ); dbsp::never_none!(Q9Output); +dbsp::never_roaring_filter!(Q9Output); type Q9Stream = Stream>; diff --git a/crates/sqllib/tests/tuple_proptest.rs b/crates/sqllib/tests/tuple_proptest.rs index a42fa01dc2e..cb206c97d95 100644 --- a/crates/sqllib/tests/tuple_proptest.rs +++ b/crates/sqllib/tests/tuple_proptest.rs @@ -8,6 +8,7 @@ use dbsp::storage::backend::memory_impl::MemoryBackend; use dbsp::storage::buffer_cache::BufferCache; use dbsp::storage::file::Factories; use dbsp::storage::file::{ + FilterPlan, format::BatchMetadata, writer::{Parameters, Writer1}, }; @@ -308,8 +309,14 @@ where let backend = MemoryBackend::new(); let factories = Factories::::new::(); let parameters = Parameters::default(); - let mut writer = Writer1::new(&factories, buffer_cache, &backend, parameters, values.len()) - .map_err(|err| TestCaseError::fail(format!("writer init failed: {err:?}")))?; + let mut writer = Writer1::new( + &factories, + buffer_cache, + &backend, + parameters, + FilterPlan::::decide_filter(None, values.len()), + ) + .map_err(|err| TestCaseError::fail(format!("writer init failed: {err:?}")))?; let aux = (); for value in &values { diff --git a/crates/storage-test-compat/src/bin/golden-writer.rs b/crates/storage-test-compat/src/bin/golden-writer.rs index ba8f2a2a8d5..41f6f2cc81e 100644 --- a/crates/storage-test-compat/src/bin/golden-writer.rs +++ b/crates/storage-test-compat/src/bin/golden-writer.rs @@ -13,7 +13,7 @@ use dbsp::storage::file::format::BatchMetadata; use dbsp::storage::file::format::Compression; use dbsp::storage::file::format::VERSION_NUMBER; use dbsp::storage::file::writer::{Parameters, Writer1}; -use dbsp::storage::file::Factories; +use dbsp::storage::file::{Factories, FilterPlan}; use feldera_types::config::{StorageConfig, StorageOptions}; use storage_test_compat::{ @@ -102,7 +102,7 @@ where buffer_cache, &*storage_backend, parameters, - rows, + FilterPlan::::decide_filter(None, rows), )?; for row in 0..rows { @@ -112,7 +112,7 @@ where } let tmp_path = writer.path().clone(); - let (_file_handle, _bloom_filter, _key_bounds) = writer.close(BatchMetadata::default())?; + let (_file_handle, _key_filter, _key_bounds) = writer.close(BatchMetadata::default())?; let content = storage_backend.read(&tmp_path)?; storage_backend.write(&output_storage_path, (*content).clone())?; storage_backend.delete(&tmp_path)?; diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index f45874795fe..b4e3dab6070 100644 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -37,10 +37,14 @@ pub enum StorageError { /// Cannot perform operation because storage is not enabled. #[error("Cannot perform operation because storage is not enabled.")] StorageDisabled, - /// Error while creating a bloom filter. - #[error("Failed to serialize/deserialize bloom filter.")] + /// Error while creating a batch key filter. + #[error("Failed to serialize/deserialize batch key filter.")] BloomFilter, + /// Error while serializing a roaring bitmap batch key filter. + #[error("Failed to serialize roaring bitmap batch key filter.")] + RoaringBitmapFilter, + /// Path is not valid in storage. /// /// Storage paths may not be absolute, may not start with a drive letter (on @@ -147,7 +151,7 @@ impl StorageError { StorageError::NoPersistentId(_) => ErrorKind::Other, StorageError::CheckpointNotFound(_) => ErrorKind::NotFound, StorageError::StorageDisabled => ErrorKind::Other, - StorageError::BloomFilter => ErrorKind::Other, + StorageError::BloomFilter | StorageError::RoaringBitmapFilter => ErrorKind::Other, StorageError::InvalidPath(_) => ErrorKind::Other, StorageError::InvalidURL(_) => ErrorKind::Other, StorageError::ObjectStore { kind, .. } => *kind, From d18c80fd00afba229385576ec3821f30dae6355e Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 4 Apr 2026 13:49:51 -0700 Subject: [PATCH 2/3] bench: add filter benchmark programs and plot scripts --- crates/dbsp/Cargo.toml | 9 + crates/dbsp/benches/filter_bitmap.rs | 1431 +++++++++++++++++ crates/dbsp/benches/filter_predictor.rs | 1932 +++++++++++++++++++++++ scripts/plot_filter_bitmap.py | 855 ++++++++++ 4 files changed, 4227 insertions(+) create mode 100644 crates/dbsp/benches/filter_bitmap.rs create mode 100644 crates/dbsp/benches/filter_predictor.rs create mode 100644 scripts/plot_filter_bitmap.py diff --git a/crates/dbsp/Cargo.toml b/crates/dbsp/Cargo.toml index c6da2493f09..636e755efb0 100644 --- a/crates/dbsp/Cargo.toml +++ b/crates/dbsp/Cargo.toml @@ -83,6 +83,7 @@ tracing = { workspace = true } snap = { workspace = true } enum-map = { workspace = true } fastbloom = { workspace = true } +roaring = { workspace = true } core_affinity = { workspace = true } indexmap = { workspace = true } feldera-storage = { workspace = true } @@ -165,6 +166,14 @@ harness = false name = "window_min" harness = false +[[bench]] +name = "filter_bitmap" +harness = false + +[[bench]] +name = "filter_predictor" +harness = false + [[example]] name = "orgchart" diff --git a/crates/dbsp/benches/filter_bitmap.rs b/crates/dbsp/benches/filter_bitmap.rs new file mode 100644 index 00000000000..3c6d6762c2f --- /dev/null +++ b/crates/dbsp/benches/filter_bitmap.rs @@ -0,0 +1,1431 @@ +//! Membership benchmark for `fastbloom` vs `roaring`. +//! +//! Examples: +//! `cargo bench -p dbsp --bench filter_bitmap -- --csv-output filter_bitmap.csv` +//! `cargo bench -p dbsp --bench filter_bitmap -- --key-types u32,u64 --key-spaces consecutive,full_range` + +use clap::{Parser, ValueEnum}; +use csv::Writer; +use dbsp::storage::file::BLOOM_FILTER_FALSE_POSITIVE_RATE; +use fastbloom::BloomFilter; +use rand::{RngCore, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use rand_distr::{Distribution, Normal}; +use roaring::{RoaringBitmap, RoaringTreemap}; +use serde::Serialize; +use std::{ + fmt::{Display, Formatter}, + fs::File, + mem::size_of_val, + path::PathBuf, + time::Instant, +}; + +const DEFAULT_BLOOM_SEED: u128 = 42; +const MIN_BLOOM_EXPECTED_ITEMS: u64 = 64; +const U32_KEY_SPACE_SIZE: u64 = u32::MAX as u64 + 1; +const DEFAULT_LOOKUP_LIMIT: u64 = 50_000_000; +const DEFAULT_KEY_EPS_VALUES: [f64; 6] = [1e-6, 1e-4, 1e-3, 1e-2, 1e-1, 5e-1]; + +// Mirror the spine_async size bands and include the near-full u32 domain case. +const DEFAULT_SPINE_LEVEL_SIZES: [u64; 6] = + [14_999, 99_999, 999_999, 9_999_999, 99_999_999, 999_999_999]; + +fn main() { + let args = Args::parse(); + let key_types = args.key_types(); + let key_spaces = args.key_spaces(); + let num_elements_list = args.num_elements(); + args.validate(&key_types, &key_spaces, &num_elements_list); + + let csv_file = File::create(&args.csv_output) + .unwrap_or_else(|error| panic!("failed to create {}: {error}", args.csv_output.display())); + let mut csv_writer = Writer::from_writer(csv_file); + + println!("benchmark=filter_bitmap"); + println!( + "num_elements={}", + num_elements_list + .iter() + .map(u64::to_string) + .collect::>() + .join(",") + ); + println!("repetitions={}", args.repetitions); + println!("insert_order={}", args.insert_order); + println!("lookup_order={}", args.lookup_order); + println!("insert_seed={}", args.insert_seed); + println!("lookup_seed={}", args.lookup_seed); + println!("key_space_seed={}", args.key_space_seed); + println!( + "key_types={}", + key_types + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") + ); + println!( + "key_spaces={}", + key_spaces + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") + ); + println!( + "key_eps={}", + args.key_eps() + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") + ); + println!( + "structures={}", + args.structures + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") + ); + println!( + "bloom_false_positive_rate={}", + args.bloom_false_positive_rate + ); + println!("bloom_seed={}", args.bloom_seed); + println!("csv_output={}", args.csv_output.display()); + println!(); + + for &key_type in &key_types { + for &key_space in &key_spaces { + for key_eps in args.key_eps_for(key_space) { + let config = BenchmarkConfig { + key_type, + key_space, + key_eps, + }; + + for &num_elements in &num_elements_list { + let lookup_count = args.lookup_count_for(num_elements); + let false_positive_lookup_count = + args.false_positive_lookup_count_for(config, num_elements, lookup_count); + let bloom_expected_items = args + .bloom_expected_items + .unwrap_or(num_elements) + .max(MIN_BLOOM_EXPECTED_ITEMS); + + for structure in &args.structures { + let result = match structure { + Structure::Bloom => benchmark_bloom( + &args, + config, + num_elements, + lookup_count, + false_positive_lookup_count, + bloom_expected_items, + ), + Structure::Roaring => { + benchmark_roaring(&args, config, num_elements, lookup_count) + } + }; + + print_report( + *structure, + config, + &result, + num_elements, + lookup_count, + false_positive_lookup_count, + ); + + csv_writer + .serialize(CsvRow::from_result( + CsvRowContext { + structure: *structure, + config, + args: &args, + num_elements, + lookup_count, + false_positive_lookup_count, + bloom_expected_items, + }, + &result, + )) + .expect("failed to write CSV row"); + csv_writer.flush().expect("failed to flush CSV writer"); + } + } + } + } + } +} + +#[derive(Parser, Debug, Clone)] +#[command(name = "filter_bitmap")] +#[command(about = "Benchmark fastbloom against roaring bitmap or treemap membership queries")] +struct Args { + /// Comma-separated input sizes. Underscores and `u32::MAX` are accepted. + #[arg(long, value_name = "CSV")] + num_elements: Option, + + /// Number of successful lookups to benchmark for each input size. + /// Defaults to min(num_elements, 50_000_000). + #[arg(long)] + lookup_count: Option, + + /// Number of negative lookups used to measure bloom false positives for each input size. + #[arg(long)] + false_positive_lookup_count: Option, + + /// Number of repeated benchmark runs used to compute min/avg/max/std. + #[arg(long, default_value_t = 3)] + repetitions: usize, + + /// Structures to benchmark. + #[arg(long, value_delimiter = ',', default_value = "bloom,roaring")] + structures: Vec, + + /// Key types to benchmark. + #[arg(long, value_delimiter = ',', default_value = "u32")] + key_types: Vec, + + /// Key-space models to benchmark. + /// + /// `consecutive` inserts keys from `0..n`. + /// `full_range` samples `n` distinct keys from the full type domain. + /// `half_normal` spreads `n` unique keys across `0..u32::MAX` with + /// a half-normal offset distribution controlled by `--key-eps`. + #[arg(long, value_delimiter = ',', default_value = "consecutive")] + key_spaces: Vec, + + /// Seed used by the full-range sampler and half-normal quantile phase. + #[arg(long, default_value_t = 2)] + key_space_seed: u64, + + /// Comma-separated epsilon values used by `--key-spaces half-normal`. + #[arg(long, value_name = "CSV")] + key_eps: Option, + + /// Insert order over the chosen keyset. + #[arg(long, default_value_t = Order::Sequential)] + insert_order: Order, + + /// Lookup order over the chosen keyset or sampled subset. + #[arg(long, default_value_t = Order::Random)] + lookup_order: Order, + + /// Seed used when `insert-order=random`. + #[arg(long, default_value_t = 0)] + insert_seed: u64, + + /// Seed used when `lookup-order=random`. + #[arg(long, default_value_t = 1)] + lookup_seed: u64, + + /// Bloom filter false-positive rate. Defaults to DBSP storage default. + #[arg(long, default_value_t = BLOOM_FILTER_FALSE_POSITIVE_RATE)] + bloom_false_positive_rate: f64, + + /// Bloom filter seed. Defaults to DBSP storage seed. + #[arg(long, default_value_t = DEFAULT_BLOOM_SEED)] + bloom_seed: u128, + + /// Backward-compatible alias for `--key-types u64`. + #[doc(hidden)] + #[arg(long, hide = true, default_value_t = false)] + u64_keys: bool, + + /// Expected number of items passed to the bloom filter builder for each input size. + #[arg(long)] + bloom_expected_items: Option, + + /// Output CSV path. + #[arg(long, default_value = "filter_bitmap.csv")] + csv_output: PathBuf, + + // When running with `cargo bench` the binary gets the `--bench` flag, so we + // have to parse and ignore it so clap doesn't reject it. + #[doc(hidden)] + #[arg(long = "bench", hide = true)] + __bench: bool, +} + +impl Args { + fn key_types(&self) -> Vec { + let raw = if self.u64_keys { + vec![KeyType::U64] + } else { + self.key_types.clone() + }; + dedup(raw) + } + + fn key_spaces(&self) -> Vec { + dedup(self.key_spaces.clone()) + } + + fn key_eps(&self) -> Vec { + match &self.key_eps { + Some(csv) => parse_f64_csv(csv, "--key-eps"), + None => DEFAULT_KEY_EPS_VALUES.to_vec(), + } + } + + fn key_eps_for(&self, key_space: KeySpace) -> Vec> { + match key_space { + KeySpace::HalfNormal => self.key_eps().into_iter().map(Some).collect(), + _ => vec![None], + } + } + + fn num_elements(&self) -> Vec { + match &self.num_elements { + Some(csv) => parse_u64_csv(csv), + None => DEFAULT_SPINE_LEVEL_SIZES.to_vec(), + } + } + + fn lookup_count_for(&self, num_elements: u64) -> u64 { + self.lookup_count + .map(|lookup_count| lookup_count.min(num_elements)) + .unwrap_or(num_elements.min(DEFAULT_LOOKUP_LIMIT)) + } + + fn false_positive_lookup_count_for( + &self, + config: BenchmarkConfig, + num_elements: u64, + _lookup_count: u64, + ) -> u64 { + self.false_positive_lookup_count + .map(|count| { + let max_false_positive_lookup_count = + config.max_false_positive_lookup_count(num_elements); + count.min(max_false_positive_lookup_count) + }) + .unwrap_or(0) + } + + fn validate(&self, key_types: &[KeyType], key_spaces: &[KeySpace], num_elements_list: &[u64]) { + let key_eps = self.key_eps(); + + assert!( + !num_elements_list.is_empty(), + "--num-elements must select at least one size" + ); + assert!( + self.repetitions > 0, + "--repetitions must be greater than zero" + ); + assert!( + !self.structures.is_empty(), + "--structures must select at least one structure" + ); + assert!( + !key_types.is_empty(), + "--key-types must select at least one key type" + ); + assert!( + !key_spaces.is_empty(), + "--key-spaces must select at least one key-space mode" + ); + if key_spaces.contains(&KeySpace::HalfNormal) { + assert!( + !key_eps.is_empty(), + "--key-eps must select at least one epsilon for key-space half_normal" + ); + for eps in &key_eps { + assert!( + eps.is_finite() && *eps > 0.0, + "--key-eps values must be finite and greater than zero" + ); + } + } + assert!( + self.bloom_false_positive_rate > 0.0 && self.bloom_false_positive_rate < 1.0, + "--bloom-false-positive-rate must be between 0 and 1" + ); + + for &num_elements in num_elements_list { + assert!( + num_elements > 0, + "--num-elements values must be greater than zero" + ); + + for &key_type in key_types { + for &key_space in key_spaces { + let config = BenchmarkConfig { + key_type, + key_space, + key_eps: None, + }; + config.validate_num_elements(num_elements); + } + } + } + } +} + +fn dedup(values: Vec) -> Vec +where + T: PartialEq, +{ + let mut out = Vec::with_capacity(values.len()); + for value in values { + if !out.contains(&value) { + out.push(value); + } + } + out +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum Structure { + #[value(name = "bloom")] + Bloom, + #[value(name = "roaring")] + Roaring, +} + +impl Display for Structure { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Bloom => f.write_str("bloom"), + Self::Roaring => f.write_str("roaring"), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum KeyType { + #[value(name = "u32")] + U32, + #[value(name = "u64")] + U64, +} + +impl Display for KeyType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::U32 => f.write_str("u32"), + Self::U64 => f.write_str("u64"), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum KeySpace { + #[value(name = "consecutive")] + Consecutive, + #[value(name = "full_range", alias = "full-range")] + FullRange, + #[value(name = "half_normal", alias = "half-normal")] + HalfNormal, +} + +impl Display for KeySpace { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Consecutive => f.write_str("consecutive"), + Self::FullRange => f.write_str("full_range"), + Self::HalfNormal => f.write_str("half_normal"), + } + } +} + +#[derive(Clone, Copy, Debug)] +struct BenchmarkConfig { + key_type: KeyType, + key_space: KeySpace, + key_eps: Option, +} + +impl BenchmarkConfig { + fn validate_num_elements(self, num_elements: u64) { + match (self.key_type, self.key_space) { + (KeyType::U32, _) | (_, KeySpace::HalfNormal) => assert!( + num_elements <= U32_KEY_SPACE_SIZE, + "--num-elements values must be <= {} for this key type/key space", + U32_KEY_SPACE_SIZE + ), + (KeyType::U64, _) => {} + } + } + + fn max_false_positive_lookup_count(self, num_elements: u64) -> u64 { + match (self.key_type, self.key_space) { + (_, KeySpace::HalfNormal) => U32_KEY_SPACE_SIZE - num_elements, + (KeyType::U32, _) => U32_KEY_SPACE_SIZE - num_elements, + (KeyType::U64, _) => u64::MAX - num_elements + 1, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum Order { + #[value(name = "sequential")] + Sequential, + #[value(name = "random")] + Random, +} + +impl Display for Order { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Sequential => f.write_str("sequential"), + Self::Random => f.write_str("random"), + } + } +} + +#[derive(Clone, Copy, Debug)] +struct AffinePermutation { + len: u64, + multiplier: u64, + offset: u64, +} + +impl AffinePermutation { + fn sequential(len: u64) -> Self { + Self { + len, + multiplier: 1, + offset: 0, + } + } + + fn random(len: u64, seed: u64) -> Self { + if len <= 1 { + return Self::sequential(len); + } + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let mut multiplier = (rng.next_u64() % len) | 1; + while gcd(multiplier, len) != 1 { + multiplier = (multiplier + 2) % len; + if multiplier == 0 { + multiplier = 1; + } + } + let offset = rng.next_u64() % len; + Self { + len, + multiplier, + offset, + } + } + + fn for_order(len: u64, order: Order, seed: u64) -> Self { + match order { + Order::Sequential => Self::sequential(len), + Order::Random => Self::random(len, seed), + } + } + + fn index_at(&self, position: u64) -> u64 { + debug_assert!(position < self.len); + (self + .multiplier + .wrapping_mul(position) + .wrapping_add(self.offset)) + % self.len + } +} + +#[derive(Clone, Copy, Debug)] +struct WrappingPermutation64 { + multiplier: u64, + offset: u64, +} + +impl WrappingPermutation64 { + fn sequential() -> Self { + Self { + multiplier: 1, + offset: 0, + } + } + + fn random(seed: u64) -> Self { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + Self { + multiplier: rng.next_u64() | 1, + offset: rng.next_u64(), + } + } + + fn for_order(order: Order, seed: u64) -> Self { + match order { + Order::Sequential => Self::sequential(), + Order::Random => Self::random(seed), + } + } + + fn index_at(&self, position: u64) -> u64 { + position + .wrapping_mul(self.multiplier) + .wrapping_add(self.offset) + } +} + +#[derive(Clone, Copy, Debug)] +struct HalfNormalKeySampler { + eps: f64, + seed: u64, +} + +impl HalfNormalKeySampler { + fn new(eps: f64, seed: u64) -> Self { + Self { eps, seed } + } + + fn present_keys_u32(&self, num_elements: u64) -> Vec { + let len = usize::try_from(num_elements).expect("num_elements must fit in usize"); + let mut rng = ChaCha8Rng::seed_from_u64(self.seed); + let sigma = self.eps * u32::MAX as f64; + let distribution = Normal::new(0.0, sigma) + .expect("half-normal epsilon must produce a positive standard deviation"); + let mut keys = Vec::with_capacity(len); + + for _ in 0..num_elements { + let sampled = distribution.sample(&mut rng).abs().round(); + keys.push(sampled.clamp(0.0, u32::MAX as f64) as u32); + } + + keys.sort_unstable(); + project_sorted_unique_u32_domain(&mut keys); + keys + } + + fn present_keys_u64(&self, num_elements: u64) -> Vec { + self.present_keys_u32(num_elements) + .into_iter() + .map(u64::from) + .collect() + } +} + +#[derive(Clone, Copy, Debug)] +enum U32KeySampler { + Consecutive, + FullRange(AffinePermutation), + HalfNormal(HalfNormalKeySampler), +} + +impl U32KeySampler { + fn new(key_space: KeySpace, _num_elements: u64, key_eps: Option, seed: u64) -> Self { + match key_space { + KeySpace::Consecutive => Self::Consecutive, + KeySpace::FullRange => Self::FullRange(AffinePermutation::for_order( + U32_KEY_SPACE_SIZE, + Order::Random, + seed, + )), + KeySpace::HalfNormal => Self::HalfNormal(HalfNormalKeySampler::new( + key_eps.expect("half_normal key space requires key_eps"), + seed, + )), + } + } + + fn present_key(&self, set_index: u64) -> u32 { + match self { + Self::Consecutive => set_index as u32, + Self::FullRange(permutation) => permutation.index_at(set_index) as u32, + Self::HalfNormal(_) => { + panic!("half_normal key space requires pre-generated keys") + } + } + } + + fn absent_key(&self, num_elements: u64, absent_index: u64) -> u32 { + let domain_index = num_elements + .checked_add(absent_index) + .expect("u32 absent-key generation overflowed"); + match self { + Self::Consecutive => domain_index as u32, + Self::FullRange(permutation) => permutation.index_at(domain_index) as u32, + Self::HalfNormal(_) => { + panic!("half_normal key space requires prepared absent keys") + } + } + } +} + +#[derive(Clone, Copy, Debug)] +enum U64KeySampler { + Consecutive, + FullRange(WrappingPermutation64), + HalfNormal(HalfNormalKeySampler), +} + +impl U64KeySampler { + fn new(key_space: KeySpace, _num_elements: u64, key_eps: Option, seed: u64) -> Self { + match key_space { + KeySpace::Consecutive => Self::Consecutive, + KeySpace::FullRange => { + Self::FullRange(WrappingPermutation64::for_order(Order::Random, seed)) + } + KeySpace::HalfNormal => Self::HalfNormal(HalfNormalKeySampler::new( + key_eps.expect("half_normal key space requires key_eps"), + seed, + )), + } + } + + fn present_key(&self, set_index: u64) -> u64 { + match self { + Self::Consecutive => set_index, + Self::FullRange(permutation) => permutation.index_at(set_index), + Self::HalfNormal(_) => { + panic!("half_normal key space requires pre-generated keys") + } + } + } + + fn absent_key(&self, num_elements: u64, absent_index: u64) -> u64 { + let domain_index = num_elements + .checked_add(absent_index) + .expect("u64 absent-key generation overflowed"); + match self { + Self::Consecutive => domain_index, + Self::FullRange(permutation) => permutation.index_at(domain_index), + Self::HalfNormal(_) => { + panic!("half_normal key space requires prepared absent keys") + } + } + } +} + +fn gcd(mut lhs: u64, mut rhs: u64) -> u64 { + while rhs != 0 { + let next = lhs % rhs; + lhs = rhs; + rhs = next; + } + lhs +} + +fn parse_u64_csv(csv: &str) -> Vec { + let mut out: Vec = csv + .split(',') + .filter(|entry| !entry.trim().is_empty()) + .map(|entry| { + parse_u64_token(entry.trim()) + .unwrap_or_else(|error| panic!("invalid u64 in --num-elements: {entry} ({error})")) + }) + .collect(); + out.sort_unstable(); + out.dedup(); + out +} + +fn parse_f64_csv(csv: &str, flag_name: &str) -> Vec { + let mut out: Vec = csv + .split(',') + .filter(|entry| !entry.trim().is_empty()) + .map(|entry| { + entry + .trim() + .parse::() + .unwrap_or_else(|error| panic!("invalid f64 in {flag_name}: {entry} ({error})")) + }) + .collect(); + out.sort_by(|lhs, rhs| lhs.partial_cmp(rhs).expect("NaN was already rejected")); + out.dedup(); + out +} + +fn parse_u64_token(token: &str) -> Result { + match token { + "u32::MAX" | "u32_max" | "max_u32" => Ok(u32::MAX as u64), + _ => token + .replace('_', "") + .parse::() + .map_err(|error| error.to_string()), + } +} + +fn project_sorted_unique_u32_domain(keys: &mut [u32]) { + if keys.is_empty() { + return; + } + + for (index, key) in keys.iter_mut().enumerate() { + let min_key = u32::try_from(index).expect("key count exceeded u32 domain"); + if *key < min_key { + *key = min_key; + } + } + + for index in (0..keys.len()).rev() { + let tail = keys.len() - 1 - index; + let max_key = u32::MAX + .checked_sub(u32::try_from(tail).expect("key count exceeded u32 domain")) + .expect("tail adjustment underflowed"); + if keys[index] > max_key { + keys[index] = max_key; + } + if index + 1 < keys.len() && keys[index] >= keys[index + 1] { + keys[index] = keys[index + 1] - 1; + } + } + + debug_assert!(keys.windows(2).all(|window| window[0] < window[1])); +} + +fn absent_keys_from_sorted_present_u32(present_keys: &[u32], count: u64) -> Vec { + let target_len = usize::try_from(count).expect("false-positive lookup count must fit in usize"); + let mut absent_keys = Vec::with_capacity(target_len); + let mut candidate = 0u64; + + for &present_key in present_keys { + let present_key = present_key as u64; + while candidate < present_key && absent_keys.len() < target_len { + absent_keys.push(candidate as u32); + candidate += 1; + } + if absent_keys.len() == target_len { + return absent_keys; + } + candidate = present_key + .checked_add(1) + .expect("half-normal absent-key generation overflowed"); + } + + while absent_keys.len() < target_len { + absent_keys.push(candidate as u32); + candidate = candidate + .checked_add(1) + .expect("half-normal absent-key generation overflowed"); + } + + absent_keys +} + +fn absent_keys_from_sorted_present_u64(present_keys: &[u64], count: u64) -> Vec { + let target_len = usize::try_from(count).expect("false-positive lookup count must fit in usize"); + let mut absent_keys = Vec::with_capacity(target_len); + let mut candidate = 0u64; + + for &present_key in present_keys { + while candidate < present_key && absent_keys.len() < target_len { + absent_keys.push(candidate); + candidate += 1; + } + if absent_keys.len() == target_len { + return absent_keys; + } + candidate = present_key + .checked_add(1) + .expect("half-normal absent-key generation overflowed"); + } + + while absent_keys.len() < target_len { + absent_keys.push(candidate); + candidate = candidate + .checked_add(1) + .expect("half-normal absent-key generation overflowed"); + } + + absent_keys +} + +#[derive(Debug, Clone, Copy)] +struct SummaryStats { + min: f64, + avg: f64, + max: f64, + stddev: f64, +} + +impl SummaryStats { + fn from_samples(samples: &[f64]) -> Self { + let min = samples.iter().copied().fold(f64::INFINITY, f64::min); + let max = samples.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let avg = samples.iter().sum::() / samples.len() as f64; + let variance = samples + .iter() + .map(|sample| { + let delta = *sample - avg; + delta * delta + }) + .sum::() + / samples.len() as f64; + Self { + min, + avg, + max, + stddev: variance.sqrt(), + } + } +} + +#[derive(Debug, Clone, Copy)] +struct BenchmarkResult { + insert_ns_per_element: SummaryStats, + lookup_ns_per_element: SummaryStats, + bytes_used: usize, + false_positive_rate_percent: Option, +} + +#[derive(Debug, Serialize)] +struct CsvRow { + structure: &'static str, + key_type: &'static str, + key_space: &'static str, + key_eps: Option, + num_elements: u64, + lookup_count: u64, + false_positive_lookup_count: u64, + repetitions: usize, + insert_order: &'static str, + lookup_order: &'static str, + insert_seed: u64, + lookup_seed: u64, + key_space_seed: u64, + bloom_false_positive_rate_target_percent: f64, + bloom_seed: u128, + bloom_expected_items: u64, + bytes_used: usize, + bytes_per_element: f64, + bits_per_element: Option, + insert_ns_per_element_min: f64, + insert_ns_per_element_avg: f64, + insert_ns_per_element_max: f64, + insert_ns_per_element_stddev: f64, + lookup_ns_per_element_min: f64, + lookup_ns_per_element_avg: f64, + lookup_ns_per_element_max: f64, + lookup_ns_per_element_stddev: f64, + false_positive_rate_percent_min: Option, + false_positive_rate_percent_avg: Option, + false_positive_rate_percent_max: Option, + false_positive_rate_percent_stddev: Option, +} + +#[derive(Clone, Copy)] +struct CsvRowContext<'a> { + structure: Structure, + config: BenchmarkConfig, + args: &'a Args, + num_elements: u64, + lookup_count: u64, + false_positive_lookup_count: u64, + bloom_expected_items: u64, +} + +impl CsvRow { + fn from_result(context: CsvRowContext<'_>, result: &BenchmarkResult) -> Self { + let bits_per_element = (context.structure == Structure::Bloom) + .then_some((result.bytes_used as f64 * 8.0) / context.num_elements as f64); + let false_positive_stats = result.false_positive_rate_percent; + + Self { + structure: context.structure.as_str(), + key_type: context.config.key_type.as_str(), + key_space: context.config.key_space.as_str(), + key_eps: context.config.key_eps, + num_elements: context.num_elements, + lookup_count: context.lookup_count, + false_positive_lookup_count: context.false_positive_lookup_count, + repetitions: context.args.repetitions, + insert_order: context.args.insert_order.as_str(), + lookup_order: context.args.lookup_order.as_str(), + insert_seed: context.args.insert_seed, + lookup_seed: context.args.lookup_seed, + key_space_seed: context.args.key_space_seed, + bloom_false_positive_rate_target_percent: context.args.bloom_false_positive_rate + * 100.0, + bloom_seed: context.args.bloom_seed, + bloom_expected_items: context.bloom_expected_items, + bytes_used: result.bytes_used, + bytes_per_element: result.bytes_used as f64 / context.num_elements as f64, + bits_per_element, + insert_ns_per_element_min: result.insert_ns_per_element.min, + insert_ns_per_element_avg: result.insert_ns_per_element.avg, + insert_ns_per_element_max: result.insert_ns_per_element.max, + insert_ns_per_element_stddev: result.insert_ns_per_element.stddev, + lookup_ns_per_element_min: result.lookup_ns_per_element.min, + lookup_ns_per_element_avg: result.lookup_ns_per_element.avg, + lookup_ns_per_element_max: result.lookup_ns_per_element.max, + lookup_ns_per_element_stddev: result.lookup_ns_per_element.stddev, + false_positive_rate_percent_min: false_positive_stats.map(|stats| stats.min), + false_positive_rate_percent_avg: false_positive_stats.map(|stats| stats.avg), + false_positive_rate_percent_max: false_positive_stats.map(|stats| stats.max), + false_positive_rate_percent_stddev: false_positive_stats.map(|stats| stats.stddev), + } + } +} + +impl Structure { + fn as_str(self) -> &'static str { + match self { + Self::Bloom => "bloom", + Self::Roaring => "roaring", + } + } +} + +impl KeyType { + fn as_str(self) -> &'static str { + match self { + Self::U32 => "u32", + Self::U64 => "u64", + } + } +} + +impl KeySpace { + fn as_str(self) -> &'static str { + match self { + Self::Consecutive => "consecutive", + Self::FullRange => "full_range", + Self::HalfNormal => "half_normal", + } + } +} + +impl Order { + fn as_str(self) -> &'static str { + match self { + Self::Sequential => "sequential", + Self::Random => "random", + } + } +} + +fn benchmark_bloom( + args: &Args, + config: BenchmarkConfig, + num_elements: u64, + lookup_count: u64, + false_positive_lookup_count: u64, + bloom_expected_items: u64, +) -> BenchmarkResult { + match config.key_type { + KeyType::U32 => benchmark_bloom_u32( + args, + config, + num_elements, + lookup_count, + false_positive_lookup_count, + bloom_expected_items, + ), + KeyType::U64 => benchmark_bloom_u64( + args, + config, + num_elements, + lookup_count, + false_positive_lookup_count, + bloom_expected_items, + ), + } +} + +fn benchmark_bloom_u32( + args: &Args, + config: BenchmarkConfig, + num_elements: u64, + lookup_count: u64, + false_positive_lookup_count: u64, + bloom_expected_items: u64, +) -> BenchmarkResult { + let mut insert_samples = Vec::with_capacity(args.repetitions); + let mut lookup_samples = Vec::with_capacity(args.repetitions); + let mut false_positive_rate_percent_samples = Vec::with_capacity(args.repetitions); + let mut bytes_used = 0; + let expected_items = + usize::try_from(bloom_expected_items).expect("bloom expected items must fit in usize"); + + for repetition in 0..args.repetitions { + let sampler = U32KeySampler::new( + config.key_space, + num_elements, + config.key_eps, + args.key_space_seed.wrapping_add(repetition as u64), + ); + let present_keys = sorted_present_keys_u32(sampler, num_elements); + let insert_permutation = AffinePermutation::for_order( + num_elements, + args.insert_order, + args.insert_seed.wrapping_add(repetition as u64), + ); + let lookup_permutation = AffinePermutation::for_order( + num_elements, + args.lookup_order, + args.lookup_seed.wrapping_add(repetition as u64), + ); + let false_positive_permutation = (false_positive_lookup_count > 0).then(|| { + AffinePermutation::for_order( + false_positive_lookup_count, + args.lookup_order, + args.lookup_seed.wrapping_add(repetition as u64), + ) + }); + let absent_keys = matches!(sampler, U32KeySampler::HalfNormal(_)).then(|| { + absent_keys_from_sorted_present_u32(&present_keys, false_positive_lookup_count) + }); + + let mut bloom = BloomFilter::with_false_pos(args.bloom_false_positive_rate) + .seed(&args.bloom_seed) + .expected_items(expected_items.max(MIN_BLOOM_EXPECTED_ITEMS as usize)); + + let insert_started = Instant::now(); + for index in 0..num_elements { + let key = present_keys[insert_permutation.index_at(index) as usize]; + bloom.insert(&key); + } + let insert_elapsed = insert_started.elapsed(); + + let lookup_started = Instant::now(); + let mut hits = 0u64; + for index in 0..lookup_count { + let key = present_keys[lookup_permutation.index_at(index) as usize]; + hits += u64::from(bloom.contains(&key)); + } + let lookup_elapsed = lookup_started.elapsed(); + + assert_eq!(hits, lookup_count, "expected all lookup keys to be present"); + + if let Some(false_positive_permutation) = false_positive_permutation { + let mut false_positives = 0u64; + for index in 0..false_positive_lookup_count { + let absent_index = false_positive_permutation.index_at(index); + let key = absent_keys.as_ref().map_or_else( + || sampler.absent_key(num_elements, absent_index), + |keys| keys[absent_index as usize], + ); + false_positives += u64::from(bloom.contains(&key)); + } + false_positive_rate_percent_samples + .push((false_positives as f64 / false_positive_lookup_count as f64) * 100.0); + } + + bytes_used = size_of_val(bloom.as_slice()); + insert_samples.push(insert_elapsed.as_nanos() as f64 / num_elements as f64); + lookup_samples.push(lookup_elapsed.as_nanos() as f64 / lookup_count as f64); + } + + BenchmarkResult { + insert_ns_per_element: SummaryStats::from_samples(&insert_samples), + lookup_ns_per_element: SummaryStats::from_samples(&lookup_samples), + bytes_used, + false_positive_rate_percent: (!false_positive_rate_percent_samples.is_empty()) + .then(|| SummaryStats::from_samples(&false_positive_rate_percent_samples)), + } +} + +fn benchmark_bloom_u64( + args: &Args, + config: BenchmarkConfig, + num_elements: u64, + lookup_count: u64, + false_positive_lookup_count: u64, + bloom_expected_items: u64, +) -> BenchmarkResult { + let mut insert_samples = Vec::with_capacity(args.repetitions); + let mut lookup_samples = Vec::with_capacity(args.repetitions); + let mut false_positive_rate_percent_samples = Vec::with_capacity(args.repetitions); + let mut bytes_used = 0; + let expected_items = + usize::try_from(bloom_expected_items).expect("bloom expected items must fit in usize"); + + for repetition in 0..args.repetitions { + let sampler = U64KeySampler::new( + config.key_space, + num_elements, + config.key_eps, + args.key_space_seed.wrapping_add(repetition as u64), + ); + let present_keys = sorted_present_keys_u64(sampler, num_elements); + let insert_permutation = AffinePermutation::for_order( + num_elements, + args.insert_order, + args.insert_seed.wrapping_add(repetition as u64), + ); + let lookup_permutation = AffinePermutation::for_order( + num_elements, + args.lookup_order, + args.lookup_seed.wrapping_add(repetition as u64), + ); + let false_positive_permutation = (false_positive_lookup_count > 0).then(|| { + AffinePermutation::for_order( + false_positive_lookup_count, + args.lookup_order, + args.lookup_seed.wrapping_add(repetition as u64), + ) + }); + let absent_keys = matches!(sampler, U64KeySampler::HalfNormal(_)).then(|| { + absent_keys_from_sorted_present_u64(&present_keys, false_positive_lookup_count) + }); + + let mut bloom = BloomFilter::with_false_pos(args.bloom_false_positive_rate) + .seed(&args.bloom_seed) + .expected_items(expected_items.max(MIN_BLOOM_EXPECTED_ITEMS as usize)); + + let insert_started = Instant::now(); + for index in 0..num_elements { + let key = present_keys[insert_permutation.index_at(index) as usize]; + bloom.insert(&key); + } + let insert_elapsed = insert_started.elapsed(); + + let lookup_started = Instant::now(); + let mut hits = 0u64; + for index in 0..lookup_count { + let key = present_keys[lookup_permutation.index_at(index) as usize]; + hits += u64::from(bloom.contains(&key)); + } + let lookup_elapsed = lookup_started.elapsed(); + + assert_eq!(hits, lookup_count, "expected all lookup keys to be present"); + + if let Some(false_positive_permutation) = false_positive_permutation { + let mut false_positives = 0u64; + for index in 0..false_positive_lookup_count { + let absent_index = false_positive_permutation.index_at(index); + let key = absent_keys.as_ref().map_or_else( + || sampler.absent_key(num_elements, absent_index), + |keys| keys[absent_index as usize], + ); + false_positives += u64::from(bloom.contains(&key)); + } + false_positive_rate_percent_samples + .push((false_positives as f64 / false_positive_lookup_count as f64) * 100.0); + } + + bytes_used = size_of_val(bloom.as_slice()); + insert_samples.push(insert_elapsed.as_nanos() as f64 / num_elements as f64); + lookup_samples.push(lookup_elapsed.as_nanos() as f64 / lookup_count as f64); + } + + BenchmarkResult { + insert_ns_per_element: SummaryStats::from_samples(&insert_samples), + lookup_ns_per_element: SummaryStats::from_samples(&lookup_samples), + bytes_used, + false_positive_rate_percent: (!false_positive_rate_percent_samples.is_empty()) + .then(|| SummaryStats::from_samples(&false_positive_rate_percent_samples)), + } +} + +fn benchmark_roaring( + args: &Args, + config: BenchmarkConfig, + num_elements: u64, + lookup_count: u64, +) -> BenchmarkResult { + match config.key_type { + KeyType::U32 => benchmark_roaring_u32(args, config, num_elements, lookup_count), + KeyType::U64 => benchmark_roaring_u64(args, config, num_elements, lookup_count), + } +} + +fn benchmark_roaring_u32( + args: &Args, + config: BenchmarkConfig, + num_elements: u64, + lookup_count: u64, +) -> BenchmarkResult { + let mut insert_samples = Vec::with_capacity(args.repetitions); + let mut lookup_samples = Vec::with_capacity(args.repetitions); + let mut bytes_used = 0; + + for repetition in 0..args.repetitions { + let sampler = U32KeySampler::new( + config.key_space, + num_elements, + config.key_eps, + args.key_space_seed.wrapping_add(repetition as u64), + ); + let present_keys = sorted_present_keys_u32(sampler, num_elements); + let insert_permutation = AffinePermutation::for_order( + num_elements, + args.insert_order, + args.insert_seed.wrapping_add(repetition as u64), + ); + let lookup_permutation = AffinePermutation::for_order( + num_elements, + args.lookup_order, + args.lookup_seed.wrapping_add(repetition as u64), + ); + + let insert_started = Instant::now(); + let mut bitmap = RoaringBitmap::new(); + for index in 0..num_elements { + bitmap.insert(present_keys[insert_permutation.index_at(index) as usize]); + } + let insert_elapsed = insert_started.elapsed(); + let _ = bitmap.optimize(); + + let lookup_started = Instant::now(); + let mut hits = 0u64; + for index in 0..lookup_count { + let key = present_keys[lookup_permutation.index_at(index) as usize]; + hits += u64::from(bitmap.contains(key)); + } + let lookup_elapsed = lookup_started.elapsed(); + + assert_eq!(hits, lookup_count, "expected all lookup keys to be present"); + bytes_used = bitmap.serialized_size(); + insert_samples.push(insert_elapsed.as_nanos() as f64 / num_elements as f64); + lookup_samples.push(lookup_elapsed.as_nanos() as f64 / lookup_count as f64); + } + + BenchmarkResult { + insert_ns_per_element: SummaryStats::from_samples(&insert_samples), + lookup_ns_per_element: SummaryStats::from_samples(&lookup_samples), + bytes_used, + false_positive_rate_percent: None, + } +} + +fn benchmark_roaring_u64( + args: &Args, + config: BenchmarkConfig, + num_elements: u64, + lookup_count: u64, +) -> BenchmarkResult { + let mut insert_samples = Vec::with_capacity(args.repetitions); + let mut lookup_samples = Vec::with_capacity(args.repetitions); + let mut bytes_used = 0; + + for repetition in 0..args.repetitions { + let sampler = U64KeySampler::new( + config.key_space, + num_elements, + config.key_eps, + args.key_space_seed.wrapping_add(repetition as u64), + ); + let present_keys = sorted_present_keys_u64(sampler, num_elements); + let insert_permutation = AffinePermutation::for_order( + num_elements, + args.insert_order, + args.insert_seed.wrapping_add(repetition as u64), + ); + let lookup_permutation = AffinePermutation::for_order( + num_elements, + args.lookup_order, + args.lookup_seed.wrapping_add(repetition as u64), + ); + + let insert_started = Instant::now(); + let mut treemap = RoaringTreemap::new(); + for index in 0..num_elements { + treemap.insert(present_keys[insert_permutation.index_at(index) as usize]); + } + let insert_elapsed = insert_started.elapsed(); + + let lookup_started = Instant::now(); + let mut hits = 0u64; + for index in 0..lookup_count { + let key = present_keys[lookup_permutation.index_at(index) as usize]; + hits += u64::from(treemap.contains(key)); + } + let lookup_elapsed = lookup_started.elapsed(); + + assert_eq!(hits, lookup_count, "expected all lookup keys to be present"); + bytes_used = treemap.serialized_size(); + insert_samples.push(insert_elapsed.as_nanos() as f64 / num_elements as f64); + lookup_samples.push(lookup_elapsed.as_nanos() as f64 / lookup_count as f64); + } + + BenchmarkResult { + insert_ns_per_element: SummaryStats::from_samples(&insert_samples), + lookup_ns_per_element: SummaryStats::from_samples(&lookup_samples), + bytes_used, + false_positive_rate_percent: None, + } +} + +fn sorted_present_keys_u32(sampler: U32KeySampler, num_elements: u64) -> Vec { + match sampler { + U32KeySampler::HalfNormal(sampler) => sampler.present_keys_u32(num_elements), + _ => { + let mut present_keys = Vec::with_capacity( + usize::try_from(num_elements).expect("num_elements must fit in usize"), + ); + for set_index in 0..num_elements { + present_keys.push(sampler.present_key(set_index)); + } + present_keys.sort_unstable(); + present_keys + } + } +} + +fn sorted_present_keys_u64(sampler: U64KeySampler, num_elements: u64) -> Vec { + match sampler { + U64KeySampler::HalfNormal(sampler) => sampler.present_keys_u64(num_elements), + _ => { + let mut present_keys = Vec::with_capacity( + usize::try_from(num_elements).expect("num_elements must fit in usize"), + ); + for set_index in 0..num_elements { + present_keys.push(sampler.present_key(set_index)); + } + present_keys.sort_unstable(); + present_keys + } + } +} + +fn print_report( + structure: Structure, + config: BenchmarkConfig, + result: &BenchmarkResult, + num_elements: u64, + lookup_count: u64, + false_positive_lookup_count: u64, +) { + println!("structure={structure}"); + println!("key_type={}", config.key_type.as_str()); + println!("key_space={}", config.key_space.as_str()); + if let Some(key_eps) = config.key_eps { + println!("key_eps={key_eps}"); + } + println!("num_elements={num_elements}"); + println!("bytes_used={}", result.bytes_used); + println!( + "bytes_used_human={}", + format_bytes(result.bytes_used as f64) + ); + println!( + "bytes_per_element={}", + format_bytes(result.bytes_used as f64 / num_elements as f64) + ); + if structure == Structure::Bloom { + println!( + "bits_per_element={:.6}", + (result.bytes_used as f64 * 8.0) / num_elements as f64 + ); + } + print_stats("insert_ns_per_element", result.insert_ns_per_element); + print_stats("lookup_ns_per_element", result.lookup_ns_per_element); + println!("lookup_count={lookup_count}"); + if let Some(stats) = result.false_positive_rate_percent { + println!("false_positive_lookup_count={false_positive_lookup_count}"); + print_stats("false_positive_rate_percent", stats); + } + println!(); +} + +fn print_stats(label: &str, stats: SummaryStats) { + println!("{label}.min={:.6}", stats.min); + println!("{label}.avg={:.6}", stats.avg); + println!("{label}.max={:.6}", stats.max); + println!("{label}.stddev={:.6}", stats.stddev); +} + +fn format_bytes(bytes: f64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + + let mut value = bytes; + let mut unit_index = 0; + while value >= 1024.0 && unit_index + 1 < UNITS.len() { + value /= 1024.0; + unit_index += 1; + } + + format!("{value:.6} {}", UNITS[unit_index]) +} diff --git a/crates/dbsp/benches/filter_predictor.rs b/crates/dbsp/benches/filter_predictor.rs new file mode 100644 index 00000000000..040202d09b3 --- /dev/null +++ b/crates/dbsp/benches/filter_predictor.rs @@ -0,0 +1,1932 @@ +//! Predictor benchmark for deciding between `fastbloom` and `roaring` on u32 keys. +//! +//! Examples: +//! `cargo bench -p dbsp --bench filter_predictor -- --csv-output filter_predictor.csv` +//! `cargo bench -p dbsp --bench filter_predictor -- --num-keys 99_999,999_999 --distributions gaussian,bimodal,exponential --gaussian-means 0.1,0.5,0.9 --gaussian-stddevs 1e-6,1e-4,1e-2` + +use clap::{Parser, ValueEnum}; +use csv::Writer; +use dbsp::storage::file::BLOOM_FILTER_FALSE_POSITIVE_RATE; +use fastbloom::BloomFilter; +use rand::{RngCore, SeedableRng, seq::index::sample}; +use rand_chacha::ChaCha8Rng; +use rand_distr::{Distribution, Exp, Normal}; +use roaring::RoaringBitmap; +use serde::Serialize; +use std::{ + collections::HashMap, + fmt::{Display, Formatter}, + fs::File, + mem::size_of_val, + path::PathBuf, + sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, + }, + thread, + time::Instant, +}; + +const DEFAULT_BLOOM_SEED: u128 = 42; +const DEFAULT_GAUSSIAN_MEAN_FRACTIONS: [f64; 1] = [0.5]; +const DEFAULT_GAUSSIAN_STDDEV_FRACTIONS: [f64; 10] = + [1e-6, 1e-5, 5e-5, 1e-4, 5e-4, 1e-3, 1e-2, 5e-2, 1e-1, 5e-1]; +const DEFAULT_LOOKUP_LIMIT: u64 = 5_000_000; +const DEFAULT_SAMPLE_PERCENT: f64 = 0.1; +const DEFAULT_MIN_SAMPLE_SIZE: usize = 1_024; +const BIMODAL_LEFT_PEAK_FRAC: f64 = 0.25; +const BIMODAL_RIGHT_PEAK_FRAC: f64 = 0.75; +const MIN_BLOOM_EXPECTED_ITEMS: u64 = 64; +const U32_KEY_SPACE_SIZE: u64 = u32::MAX as u64 + 1; +const DEFAULT_NUM_KEYS: [u64; 10] = [ + 14_999, + 49_999, + 99_999, + 499_999, + 999_999, + 4_999_999, + 9_999_999, + 49_999_999, + 99_999_999, + 999_999_999, +]; + +// Build and memory mostly care about how much work or storage Roaring pays per +// touched 16-bit container, so these predictors stay intentionally simple and +// depend primarily on estimated keys per touched window. +const BUILD_ROARING_ESTIMATED_KEYS_PER_WINDOW_THRESHOLD: f64 = 4.0; +const MEMORY_ROARING_ESTIMATED_KEYS_PER_WINDOW_THRESHOLD: f64 = 32.0; + +// roaring-rs switches array containers to bitmap containers around 4096 keys. +// That transition materially changes lookup behavior, so the lookup predictor +// treats it as a first-class boundary. +const ROARING_BITMAP_CONTAINER_THRESHOLD: f64 = 4_096.0; + +// Lookup prediction is framed as a coarse cost proxy. If the estimated cost of +// reaching and searching a Roaring container stays below this budget, predict +// Roaring; otherwise predict Bloom. +const LOOKUP_ROARING_WINDOW_PROBABILITY_THRESHOLD: f64 = 0.1; +const LOOKUP_ROARING_BITMAP_WINDOW_PROBABILITY_PENALTY: f64 = 0.1; +const LOOKUP_ROARING_ARRAY_WINDOW_PROBABILITY_PENALTY_BASE: f64 = 0.25; +const LOOKUP_ROARING_ARRAY_WINDOW_PROBABILITY_PENALTY_PER_LOG2_KEY: f64 = 0.15; + +// Raw Chao1 fixes a real failure mode in sparse, very wide distributions, where +// the old uniform estimator badly under-counted touched windows and therefore +// over-predicted Roaring for random u32 lookups. Damping keeps that correction +// from overreacting on samples with only a small amount of singleton noise. +const TOUCHED_WINDOWS_CHAO1_DAMPING: f64 = 0.25; +const U32_WINDOW_COUNT: usize = 1 << 16; + +fn main() { + let args = Args::parse(); + let distributions = args.distributions(); + let num_keys_list = args.num_keys(); + let gaussian_means = args.gaussian_means(); + let gaussian_stddevs = args.gaussian_stddevs(); + args.validate( + &distributions, + &num_keys_list, + &gaussian_means, + &gaussian_stddevs, + ); + let run_configs = build_run_configs( + &args, + &distributions, + &num_keys_list, + &gaussian_means, + &gaussian_stddevs, + ); + let worker_threads = args.worker_threads(run_configs.len()); + + println!("benchmark=filter_predictor"); + println!( + "distributions={}", + distributions + .iter() + .map(|distribution| distribution.as_str()) + .collect::>() + .join(",") + ); + println!( + "num_keys={}", + num_keys_list + .iter() + .map(u64::to_string) + .collect::>() + .join(",") + ); + println!( + "gaussian_means={}", + gaussian_means + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") + ); + println!( + "gaussian_stddevs={}", + gaussian_stddevs + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") + ); + println!("repetitions={}", args.repetitions); + println!("distribution_seed={}", args.distribution_seed); + println!("sample_seed={}", args.sample_seed); + println!("lookup_seed={}", args.lookup_seed); + println!("threads={}", worker_threads); + println!("lookup_space={}", args.lookup_space.as_str()); + println!( + "sample_size_override_percent={}", + option_f64(args.sample_size) + ); + println!("lookup_count_override={}", option_u64(args.lookup_count)); + println!( + "bloom_false_positive_rate={}", + args.bloom_false_positive_rate + ); + println!("bloom_seed={}", args.bloom_seed); + println!( + "bloom_expected_items_override={}", + option_u64(args.bloom_expected_items) + ); + println!("csv_output={}", args.csv_output.display()); + println!(); + + let rows = execute_runs(&args, &run_configs, worker_threads); + + let csv_file = File::create(&args.csv_output) + .unwrap_or_else(|error| panic!("failed to create {}: {error}", args.csv_output.display())); + let mut csv_writer = Writer::from_writer(csv_file); + for row in &rows { + print_run_report(row); + csv_writer + .serialize(row) + .expect("failed to write filter predictor CSV row"); + } + csv_writer + .flush() + .expect("failed to flush filter predictor CSV"); + + let accuracy = summarize_accuracy(&rows); + print_summary(&rows, &accuracy); +} + +#[derive(Parser, Debug, Clone)] +#[command(name = "filter_predictor")] +#[command(about = "Benchmark a simple roaring-vs-bloom predictor on gaussian u32 keysets")] +struct Args { + /// Comma-separated key counts. Underscores and `u32::MAX` are accepted. + #[arg(long, value_name = "CSV")] + num_keys: Option, + + /// Comma-separated distribution families to run. + /// Supported values: `gaussian`, `consecutive`, `round_robin_window`, + /// `bimodal`, `exponential`. + #[arg(long, value_name = "CSV")] + distributions: Option, + + /// Gaussian mean values expressed as fractions of `u32::MAX`. + /// Only used by the `gaussian` distribution family. + #[arg(long, value_name = "CSV")] + gaussian_means: Option, + + /// Spread parameters expressed as fractions of `u32::MAX`. + /// Used as: + /// - gaussian standard deviation for `gaussian` + /// - per-peak standard deviation for `bimodal` + /// - exponential scale for `exponential` + #[arg(long, value_name = "CSV")] + gaussian_stddevs: Option, + + /// Number of repeated runs per `(num_keys, mean, stddev)` configuration. + #[arg(long, default_value_t = 3)] + repetitions: usize, + + /// Number of benchmark configurations to run concurrently. + /// `1` keeps runs sequential. + #[arg(long, default_value_t = 1)] + threads: usize, + + /// Lookup workload. + /// `present` samples only keys from the batch. + /// `full_u32` samples random u32 keys from the full domain. + #[arg(long, value_enum, default_value_t = LookupSpace::FullU32)] + lookup_space: LookupSpace, + + /// Number of lookups to benchmark per run. + /// Defaults to `min(num_keys, 5_000_000)` for `present` and `5_000_000` + /// for `full_u32`. + #[arg(long)] + lookup_count: Option, + + /// Predictor sample size as a percentage of the batch. + /// For example, `0.1` samples 0.1% of the keys. + #[arg(long)] + sample_size: Option, + + /// Seed for gaussian key generation. + #[arg(long, default_value_t = 0)] + distribution_seed: u64, + + /// Seed for the predictor's internal sampling pass. + #[arg(long, default_value_t = 1)] + sample_seed: u64, + + /// Seed for randomized successful lookups. + #[arg(long, default_value_t = 2)] + lookup_seed: u64, + + /// Bloom filter false-positive rate. + #[arg(long, default_value_t = BLOOM_FILTER_FALSE_POSITIVE_RATE)] + bloom_false_positive_rate: f64, + + /// Bloom filter seed. + #[arg(long, default_value_t = DEFAULT_BLOOM_SEED)] + bloom_seed: u128, + + /// Expected items passed to the bloom filter builder. + #[arg(long)] + bloom_expected_items: Option, + + /// Output CSV path. + #[arg(long, default_value = "filter_predictor.csv")] + csv_output: PathBuf, + + #[doc(hidden)] + #[arg(long = "bench", hide = true)] + __bench: bool, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, ValueEnum)] +enum LookupSpace { + Present, + FullU32, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, ValueEnum)] +enum DistributionKind { + Gaussian, + Consecutive, + RoundRobinWindow, + Bimodal, + Exponential, +} + +impl DistributionKind { + fn as_str(self) -> &'static str { + match self { + Self::Gaussian => "gaussian", + Self::Consecutive => "consecutive", + Self::RoundRobinWindow => "round_robin_window", + Self::Bimodal => "bimodal", + Self::Exponential => "exponential", + } + } + + fn uses_gaussian_mean(self) -> bool { + matches!(self, Self::Gaussian) + } + + fn uses_spread_param(self) -> bool { + matches!(self, Self::Gaussian | Self::Bimodal | Self::Exponential) + } +} + +const DEFAULT_DISTRIBUTIONS: [DistributionKind; 5] = [ + DistributionKind::Gaussian, + DistributionKind::Consecutive, + DistributionKind::RoundRobinWindow, + DistributionKind::Bimodal, + DistributionKind::Exponential, +]; + +impl LookupSpace { + fn as_str(self) -> &'static str { + match self { + Self::Present => "present", + Self::FullU32 => "full_u32", + } + } +} + +impl Args { + fn distributions(&self) -> Vec { + match &self.distributions { + Some(csv) => parse_distribution_csv(csv), + None => DEFAULT_DISTRIBUTIONS.to_vec(), + } + } + + fn num_keys(&self) -> Vec { + match &self.num_keys { + Some(csv) => parse_u64_csv(csv), + None => DEFAULT_NUM_KEYS.to_vec(), + } + } + + fn gaussian_means(&self) -> Vec { + match &self.gaussian_means { + Some(csv) => parse_f64_csv(csv, "--gaussian-means"), + None => DEFAULT_GAUSSIAN_MEAN_FRACTIONS.to_vec(), + } + } + + fn gaussian_stddevs(&self) -> Vec { + match &self.gaussian_stddevs { + Some(csv) => parse_f64_csv(csv, "--gaussian-stddevs"), + None => DEFAULT_GAUSSIAN_STDDEV_FRACTIONS.to_vec(), + } + } + + fn lookup_count_for(&self, num_keys: u64) -> u64 { + self.lookup_count + .map(|lookup_count| match self.lookup_space { + LookupSpace::Present => lookup_count.min(num_keys), + LookupSpace::FullU32 => lookup_count, + }) + .unwrap_or(match self.lookup_space { + LookupSpace::Present => num_keys.min(DEFAULT_LOOKUP_LIMIT), + LookupSpace::FullU32 => DEFAULT_LOOKUP_LIMIT, + }) + } + + fn sample_size_for(&self, num_keys: u64) -> usize { + match self.sample_size { + Some(sample_percent) => sample_count_from_percent(num_keys, sample_percent, 1), + None => default_sample_size(num_keys), + } + } + + fn worker_threads(&self, run_count: usize) -> usize { + self.threads.max(1).min(run_count.max(1)) + } + + fn validate( + &self, + distributions: &[DistributionKind], + num_keys_list: &[u64], + gaussian_means: &[f64], + gaussian_stddevs: &[f64], + ) { + assert!( + !distributions.is_empty(), + "--distributions must select at least one family" + ); + assert!( + !num_keys_list.is_empty(), + "--num-keys must select at least one size" + ); + if distributions + .iter() + .copied() + .any(DistributionKind::uses_gaussian_mean) + { + assert!( + !gaussian_means.is_empty(), + "--gaussian-means must select at least one value when gaussian is enabled" + ); + } + if distributions + .iter() + .copied() + .any(DistributionKind::uses_spread_param) + { + assert!( + !gaussian_stddevs.is_empty(), + "--gaussian-stddevs must select at least one value when gaussian, bimodal, or exponential is enabled" + ); + } + assert!( + self.repetitions > 0, + "--repetitions must be greater than zero" + ); + assert!(self.threads > 0, "--threads must be greater than zero"); + assert!( + self.bloom_false_positive_rate > 0.0 && self.bloom_false_positive_rate < 1.0, + "--bloom-false-positive-rate must be between 0 and 1" + ); + + for &num_keys in num_keys_list { + assert!(num_keys > 0, "--num-keys values must be greater than zero"); + assert!( + num_keys <= U32_KEY_SPACE_SIZE, + "--num-keys values must be <= {}", + U32_KEY_SPACE_SIZE + ); + } + for &gaussian_mean in gaussian_means { + assert!( + gaussian_mean.is_finite() && (0.0..=1.0).contains(&gaussian_mean), + "--gaussian-means values must be finite fractions in [0, 1]" + ); + } + for &gaussian_stddev in gaussian_stddevs { + assert!( + gaussian_stddev.is_finite() && gaussian_stddev > 0.0, + "--gaussian-stddevs values must be finite and greater than zero" + ); + } + if let Some(sample_percent) = self.sample_size { + assert!( + sample_percent.is_finite() && sample_percent > 0.0 && sample_percent <= 100.0, + "--sample-size must be a finite percentage in (0, 100]" + ); + } + if let Some(lookup_count) = self.lookup_count { + assert!(lookup_count > 0, "--lookup-count must be greater than zero"); + } + if let Some(bloom_expected_items) = self.bloom_expected_items { + assert!( + bloom_expected_items > 0, + "--bloom-expected-items must be greater than zero" + ); + } + } +} + +#[derive(Debug, Clone, Copy)] +struct GaussianDistribution { + mean_frac: f64, + stddev_frac: f64, +} + +impl GaussianDistribution { + fn mean_value(self) -> f64 { + self.mean_frac * u32::MAX as f64 + } + + fn stddev_value(self) -> f64 { + self.stddev_frac * u32::MAX as f64 + } +} + +#[derive(Debug, Clone, Copy)] +enum DistributionSpec { + Gaussian(GaussianDistribution), + Consecutive, + RoundRobinWindow, + Bimodal { stddev_frac: f64 }, + Exponential { scale_frac: f64 }, +} + +impl DistributionSpec { + fn as_str(self) -> &'static str { + match self { + Self::Gaussian(_) => "gaussian", + Self::Consecutive => "consecutive", + Self::RoundRobinWindow => "round_robin_window", + Self::Bimodal { .. } => "bimodal", + Self::Exponential { .. } => "exponential", + } + } + + fn parameter_name(self) -> &'static str { + match self { + Self::Gaussian(_) => "stddev_frac", + Self::Bimodal { .. } => "stddev_frac", + Self::Exponential { .. } => "scale_frac", + Self::Consecutive | Self::RoundRobinWindow => "none", + } + } + + fn parameter_frac(self) -> Option { + match self { + Self::Gaussian(distribution) => Some(distribution.stddev_frac), + Self::Bimodal { stddev_frac } => Some(stddev_frac), + Self::Exponential { scale_frac } => Some(scale_frac), + Self::Consecutive | Self::RoundRobinWindow => None, + } + } + + fn parameter_value(self) -> Option { + match self { + Self::Gaussian(distribution) => Some(distribution.stddev_value()), + Self::Bimodal { stddev_frac } => Some(stddev_frac * u32::MAX as f64), + Self::Exponential { scale_frac } => Some(scale_frac * u32::MAX as f64), + Self::Consecutive | Self::RoundRobinWindow => None, + } + } + + fn gaussian_mean_frac(self) -> Option { + match self { + Self::Gaussian(distribution) => Some(distribution.mean_frac), + Self::Consecutive + | Self::RoundRobinWindow + | Self::Bimodal { .. } + | Self::Exponential { .. } => None, + } + } + + fn gaussian_mean_value(self) -> Option { + match self { + Self::Gaussian(distribution) => Some(distribution.mean_value()), + Self::Consecutive + | Self::RoundRobinWindow + | Self::Bimodal { .. } + | Self::Exponential { .. } => None, + } + } + + fn gaussian_stddev_frac(self) -> Option { + match self { + Self::Gaussian(distribution) => Some(distribution.stddev_frac), + Self::Consecutive + | Self::RoundRobinWindow + | Self::Bimodal { .. } + | Self::Exponential { .. } => None, + } + } + + fn gaussian_stddev_value(self) -> Option { + match self { + Self::Gaussian(distribution) => Some(distribution.stddev_value()), + Self::Consecutive + | Self::RoundRobinWindow + | Self::Bimodal { .. } + | Self::Exponential { .. } => None, + } + } +} + +#[derive(Debug, Clone, Copy)] +struct RunConfig { + run_index: usize, + num_keys: u64, + distribution: DistributionSpec, + repetition: usize, + distribution_seed: u64, + sample_seed: u64, + lookup_seed: u64, +} + +fn build_run_configs( + args: &Args, + distributions: &[DistributionKind], + num_keys_list: &[u64], + gaussian_means: &[f64], + gaussian_stddevs: &[f64], +) -> Vec { + let mut run_configs = Vec::new(); + + for &num_keys in num_keys_list { + for &distribution_kind in distributions { + match distribution_kind { + DistributionKind::Gaussian => { + for &gaussian_mean_frac in gaussian_means { + for &gaussian_stddev_frac in gaussian_stddevs { + let distribution = DistributionSpec::Gaussian(GaussianDistribution { + mean_frac: gaussian_mean_frac, + stddev_frac: gaussian_stddev_frac, + }); + push_run_configs(&mut run_configs, args, num_keys, distribution); + } + } + } + DistributionKind::Consecutive => { + push_run_configs( + &mut run_configs, + args, + num_keys, + DistributionSpec::Consecutive, + ); + } + DistributionKind::RoundRobinWindow => { + push_run_configs( + &mut run_configs, + args, + num_keys, + DistributionSpec::RoundRobinWindow, + ); + } + DistributionKind::Bimodal => { + for &stddev_frac in gaussian_stddevs { + push_run_configs( + &mut run_configs, + args, + num_keys, + DistributionSpec::Bimodal { stddev_frac }, + ); + } + } + DistributionKind::Exponential => { + for &scale_frac in gaussian_stddevs { + push_run_configs( + &mut run_configs, + args, + num_keys, + DistributionSpec::Exponential { scale_frac }, + ); + } + } + } + } + } + + run_configs +} + +fn push_run_configs( + run_configs: &mut Vec, + args: &Args, + num_keys: u64, + distribution: DistributionSpec, +) { + for repetition in 0..args.repetitions { + run_configs.push(RunConfig { + run_index: run_configs.len(), + num_keys, + distribution, + repetition, + distribution_seed: args.distribution_seed.wrapping_add(repetition as u64), + sample_seed: args.sample_seed.wrapping_add(repetition as u64), + lookup_seed: args.lookup_seed.wrapping_add(repetition as u64), + }); + } +} + +fn execute_runs(args: &Args, run_configs: &[RunConfig], worker_threads: usize) -> Vec { + if worker_threads <= 1 { + return run_configs + .iter() + .copied() + .map(|run_config| run_single_config(args, run_config)) + .collect(); + } + + let next_index = AtomicUsize::new(0); + let (tx, rx) = mpsc::channel::<(usize, CsvRow)>(); + + thread::scope(|scope| { + for _ in 0..worker_threads { + let tx = tx.clone(); + let next_index = &next_index; + let run_configs = run_configs; + let args = args; + scope.spawn(move || { + loop { + let task_index = next_index.fetch_add(1, Ordering::Relaxed); + if task_index >= run_configs.len() { + break; + } + + let run_config = run_configs[task_index]; + let row = run_single_config(args, run_config); + tx.send((run_config.run_index, row)) + .expect("result receiver dropped unexpectedly"); + } + }); + } + + drop(tx); + + let mut rows_by_index: Vec> = std::iter::repeat_with(|| None) + .take(run_configs.len()) + .collect(); + for (run_index, row) in rx { + rows_by_index[run_index] = Some(row); + } + + rows_by_index + .into_iter() + .map(|row| row.expect("missing benchmark row")) + .collect() + }) +} + +fn run_single_config(args: &Args, run_config: RunConfig) -> CsvRow { + let generated_keys = generate_keys( + run_config.num_keys, + run_config.distribution, + run_config.distribution_seed, + ); + let batch = GeneratedBatch::new(generated_keys, run_config.sample_seed); + let lookup_count = args.lookup_count_for(run_config.num_keys); + let sample_size = args.sample_size_for(run_config.num_keys); + let sample_percent_of_batch = sample_size as f64 / run_config.num_keys as f64 * 100.0; + let bloom_expected_items = args + .bloom_expected_items + .unwrap_or(run_config.num_keys) + .max(MIN_BLOOM_EXPECTED_ITEMS); + + let predictor_stats = estimate_roaring_sample_stats(&batch, sample_size) + .expect("predictor sample should not be empty"); + let prediction = predict_filter_winner(&predictor_stats); + + let bloom = benchmark_bloom( + batch.keys(), + lookup_count, + run_config.lookup_seed, + args.lookup_space, + bloom_expected_items, + args.bloom_false_positive_rate, + args.bloom_seed, + ); + let roaring = benchmark_roaring( + batch.keys(), + lookup_count, + run_config.lookup_seed, + args.lookup_space, + ); + + let build_actual = actual_winner(bloom.build_ns_per_element, roaring.build_ns_per_element); + let lookup_actual = actual_winner(bloom.lookup_ns_per_element, roaring.lookup_ns_per_element); + let memory_actual = actual_winner(bloom.bytes_used as f64, roaring.bytes_used as f64); + + let build_prediction_correct = prediction.build_winner == build_actual; + let lookup_prediction_correct = prediction.lookup_winner == lookup_actual; + let memory_prediction_correct = prediction.memory_winner == memory_actual; + + CsvRow { + num_keys: run_config.num_keys, + distribution: run_config.distribution.as_str(), + distribution_param_name: run_config.distribution.parameter_name(), + distribution_param_frac: run_config.distribution.parameter_frac(), + distribution_param_value: run_config.distribution.parameter_value(), + gaussian_mean_frac: run_config.distribution.gaussian_mean_frac(), + gaussian_mean: run_config.distribution.gaussian_mean_value(), + gaussian_stddev_frac: run_config.distribution.gaussian_stddev_frac(), + gaussian_stddev: run_config.distribution.gaussian_stddev_value(), + repetition: run_config.repetition, + distribution_seed: run_config.distribution_seed, + sample_seed: run_config.sample_seed, + lookup_seed: run_config.lookup_seed, + lookup_space: args.lookup_space.as_str(), + lookup_count, + sample_size, + sample_percent_of_batch, + sample_fraction: sample_size as f64 / run_config.num_keys as f64, + bloom_false_positive_rate_target_percent: args.bloom_false_positive_rate * 100.0, + bloom_seed: args.bloom_seed, + bloom_expected_items, + predictor_sampled_keys: predictor_stats.sampled_keys, + predictor_distinct_windows: predictor_stats.distinct_windows, + predictor_avg_sample_keys_per_window: predictor_stats.avg_sample_keys_per_window, + predictor_same_window_rate: predictor_stats.same_window_rate, + predictor_estimated_keys_per_window: predictor_stats.estimated_keys_per_window, + predictor_estimated_touched_windows: predictor_stats.estimated_touched_windows, + predictor_estimated_window_fill_ratio: predictor_stats.estimated_window_fill_ratio, + predictor_density_score: prediction.density_score, + predictor_build_score: prediction.build_score, + predictor_lookup_score: prediction.lookup_score, + predictor_memory_score: prediction.memory_score, + predicted_build_winner: prediction.build_winner.as_str(), + predicted_lookup_winner: prediction.lookup_winner.as_str(), + predicted_memory_winner: prediction.memory_winner.as_str(), + bloom_build_ns_per_element: bloom.build_ns_per_element, + roaring_build_ns_per_element: roaring.build_ns_per_element, + build_ratio_bloom_over_roaring: bloom.build_ns_per_element / roaring.build_ns_per_element, + actual_build_winner: build_actual.as_str(), + build_prediction_correct, + bloom_lookup_ns_per_element: bloom.lookup_ns_per_element, + bloom_lookup_hits: bloom.lookup_hits, + bloom_lookup_hit_rate_percent: bloom.lookup_hits as f64 / lookup_count as f64 * 100.0, + roaring_lookup_ns_per_element: roaring.lookup_ns_per_element, + roaring_lookup_hits: roaring.lookup_hits, + roaring_lookup_hit_rate_percent: roaring.lookup_hits as f64 / lookup_count as f64 * 100.0, + lookup_ratio_bloom_over_roaring: bloom.lookup_ns_per_element + / roaring.lookup_ns_per_element, + actual_lookup_winner: lookup_actual.as_str(), + lookup_prediction_correct, + bloom_bytes_used: bloom.bytes_used, + roaring_bytes_used: roaring.bytes_used, + memory_ratio_bloom_over_roaring: bloom.bytes_used as f64 / roaring.bytes_used as f64, + actual_memory_winner: memory_actual.as_str(), + memory_prediction_correct, + } +} + +#[derive(Debug, Clone)] +struct GeneratedBatch { + keys: Vec, + sample_seed: u64, +} + +impl GeneratedBatch { + fn new(keys: Vec, sample_seed: u64) -> Self { + Self { keys, sample_seed } + } + + fn keys(&self) -> &[u32] { + &self.keys + } +} + +/// Minimal trait matching the predictor sketch. +pub trait SampleKeys { + fn sample_keys(&self, n: usize) -> Vec; + fn key_count(&self) -> usize; +} + +impl SampleKeys for GeneratedBatch { + fn sample_keys(&self, n: usize) -> Vec { + if self.keys.is_empty() { + return Vec::new(); + } + if n >= self.keys.len() { + return self.keys.clone(); + } + + let mut rng = ChaCha8Rng::seed_from_u64(self.sample_seed); + let mut indexes = sample(&mut rng, self.keys.len(), n).into_vec(); + indexes.sort_unstable(); + indexes.into_iter().map(|index| self.keys[index]).collect() + } + + fn key_count(&self) -> usize { + self.keys.len() + } +} + +#[derive(Debug, Clone)] +pub struct RoaringSampleStats { + /// Number of keys in the batch. + pub batch_keys: usize, + + /// Number of sampled keys actually returned. + pub sampled_keys: usize, + + /// Fraction of the batch included in the sample. + pub sample_fraction: f64, + + /// Number of distinct 16-bit windows (containers) touched by the sample. + pub distinct_windows: usize, + + /// Average number of sampled keys per touched window. + pub avg_sample_keys_per_window: f64, + + /// Fraction of adjacent sampled keys that stay in the same 2^16 window. + pub same_window_rate: f64, + + /// Estimated number of real keys per touched 16-bit window after + /// rescaling by the sample fraction. + pub estimated_keys_per_window: f64, + + /// Estimated number of distinct 16-bit windows touched by the full batch. + pub estimated_touched_windows: f64, + + /// Estimated occupancy of a touched window, normalized by 2^16. + pub estimated_window_fill_ratio: f64, +} + +/// Estimate Roaring-friendly batch structure from a small sample of keys. +/// +/// The estimator deliberately works in two layers: +/// 1. Sample `n` keys from the batch. +/// 2. Sort and dedup them so adjacency and per-window counts are stable. +/// 3. Bucket sampled keys by their high 16 bits, which matches Roaring's +/// top-level `u32` container layout. +/// 4. Compute sample-level statistics such as: +/// - sampled keys +/// - distinct touched windows +/// - average sampled keys per touched window +/// - adjacent-key same-window rate +/// 5. Rescale the sampled keys/window estimate by the sample fraction so large +/// dense batches do not look artificially sparse just because only a small +/// fraction of the batch was sampled. +/// 6. Estimate the full-batch touched-window count by combining: +/// - a uniform occupancy estimate, which works well when keys are spread +/// fairly evenly across windows +/// - a damped Chao1 correction, which helps when the sample is dominated by +/// singleton windows and the uniform estimate would under-count unseen +/// windows in sparse, wide distributions +/// 7. Derive the normalized window fill ratio from the estimated keys/window. +/// +/// Example: +/// - Suppose the batch contains `10_000` keys and we sample `1_000`. +/// - After sorting and deduping we still have `1_000` sampled keys, so the +/// sample fraction is `0.1`. +/// - If those sampled keys touch `50` distinct 16-bit windows, then the sample +/// average is `1_000 / 50 = 20` sampled keys per touched window. +/// - Rescaling by the sample fraction gives an estimated +/// `20 / 0.1 = 200` real keys per touched window. +/// - If most sampled windows are singletons, the Chao1-style correction will +/// push the touched-window estimate above the uniform estimate because the +/// sample is likely missing many windows entirely. +/// - If the sample instead shows repeated hits in the same windows, the uniform +/// estimate tends to dominate and the batch looks more Roaring-friendly. +pub fn estimate_roaring_sample_stats( + batch: &B, + n: usize, +) -> Option { + if n == 0 { + return None; + } + + let batch_keys = batch.key_count(); + if batch_keys == 0 { + return None; + } + + let mut keys = batch.sample_keys(n); + if keys.is_empty() { + return None; + } + + // Make adjacent-key and per-window statistics deterministic even if the + // caller samples in arbitrary order. + keys.sort_unstable(); + keys.dedup(); + + let sampled_keys = keys.len(); + if sampled_keys == 0 { + return None; + } + + let mut per_window: HashMap = HashMap::new(); + for &key in &keys { + let window = (key >> 16) as u16; + *per_window.entry(window).or_insert(0) += 1; + } + + let distinct_windows = per_window.len(); + let sample_fraction = sampled_keys as f64 / batch_keys as f64; + let avg_sample_keys_per_window = sampled_keys as f64 / distinct_windows as f64; + let same_window_rate = if sampled_keys > 1 { + (sampled_keys - distinct_windows) as f64 / (sampled_keys - 1) as f64 + } else { + 0.0 + }; + // The sampled average keys/window shrinks as batches get larger unless we + // scale it back up by the sample fraction. Without this rescaling, large + // but dense batches look artificially sparse and the predictor incorrectly + // drifts toward Bloom. + let estimated_keys_per_window = if sample_fraction > 0.0 { + (avg_sample_keys_per_window / sample_fraction).min(65_536.0) + } else { + 0.0 + }; + // Sparse, wide samples often show up as many singleton windows and very few + // doubletons. Those counts are exactly what the Chao1-style correction uses + // to estimate how many touched windows the sample likely missed entirely. + let sample_singleton_windows = per_window.values().filter(|&&count| count == 1).count(); + let sample_doubleton_windows = per_window.values().filter(|&&count| count == 2).count(); + let estimated_touched_windows = estimate_touched_windows( + batch_keys, + sampled_keys, + distinct_windows, + sample_singleton_windows, + sample_doubleton_windows, + ); + let estimated_window_fill_ratio = estimated_keys_per_window / 65_536.0; + + Some(RoaringSampleStats { + batch_keys, + sampled_keys, + sample_fraction, + distinct_windows, + avg_sample_keys_per_window, + same_window_rate, + estimated_keys_per_window, + estimated_touched_windows, + estimated_window_fill_ratio, + }) +} + +/// Estimate how many distinct 16-bit Roaring windows the full batch touches. +/// +/// This function combines two signals: +/// 1. A uniform occupancy estimate that works well when touched windows are +/// fairly evenly populated. +/// 2. A Chao1-style unseen-window estimate that reacts when the sample is full +/// of singleton windows and therefore likely missing many windows entirely. +/// +/// Example: +/// - Suppose a batch of `10_000` keys is sampled down to `1_000` keys. +/// - The sample touches `50` distinct windows. +/// - If many of those `50` windows only appear once in the sample, that is a +/// hint that the sample is only seeing the tip of a much wider distribution. +/// - The uniform estimate might still say "roughly 70 windows total", while +/// Chao1 might say "closer to 200 windows total". +/// - We blend the two so sparse wide batches move upward, but not so far that +/// a little singleton noise completely dominates the estimate. +/// +/// This blend exists because the original uniform-only estimator was the main +/// reason the predictor failed on wide Gaussians: it under-counted touched +/// windows, which made random full-u32 Roaring lookups appear cheaper than +/// they really were. +fn estimate_touched_windows( + batch_keys: usize, + sampled_keys: usize, + distinct_windows: usize, + sample_singleton_windows: usize, + sample_doubleton_windows: usize, +) -> f64 { + if batch_keys == 0 || sampled_keys == 0 || distinct_windows == 0 { + return 0.0; + } + if sampled_keys >= batch_keys { + return distinct_windows as f64; + } + + let uniform_estimate = + estimate_uniform_touched_windows(batch_keys, sampled_keys, distinct_windows); + let chao1_estimate = estimate_chao1_touched_windows( + distinct_windows, + sample_singleton_windows, + sample_doubleton_windows, + ); + + // The original uniform estimate works well when occupancy is reasonably + // even, but it collapses badly on sparse wide Gaussians: it can turn a + // singleton-heavy sample into only ~1k touched windows, which then makes + // random full-u32 lookups look far more Roaring-friendly than they are. + // Blend in a damped Chao1 correction so unseen windows move the estimate in + // the right direction without letting Chao1 dominate every noisy sample. + arithmetic_blend( + uniform_estimate, + chao1_estimate, + TOUCHED_WINDOWS_CHAO1_DAMPING, + ) +} + +/// Estimate touched windows under a "roughly uniform occupancy" assumption. +/// +/// Intuition: +/// - Assume the full batch touches `W` windows and spreads keys across them +/// fairly evenly. +/// - Given the sample fraction, solve for the `W` that would yield the observed +/// sampled distinct-window count. +/// +/// Example: +/// - If a `10_000`-key batch is sampled at `10%`, and the sample sees `50` +/// distinct windows, this function asks: +/// "For what total window count would a 10% sample be expected to see about +/// 50 windows?" +/// - It binary-searches that answer between the sampled distinct count and the +/// theoretical maximum number of windows. +/// +/// This is the baseline estimator because it behaves sensibly on compact or +/// moderately regular distributions. It falls apart on sparse wide batches, +/// where many windows are touched so rarely that the sample never sees them. +fn estimate_uniform_touched_windows( + batch_keys: usize, + sampled_keys: usize, + distinct_windows: usize, +) -> f64 { + if batch_keys == 0 || sampled_keys == 0 || distinct_windows == 0 { + return 0.0; + } + if sampled_keys >= batch_keys { + return distinct_windows as f64; + } + + // This model assumes touched windows are roughly uniform and solves for the + // total window count that would yield the observed sampled distinct window + // count. It is a good baseline, but it systematically underestimates very + // sparse wide batches because those batches have many unseen windows. + let sample_fraction = sampled_keys as f64 / batch_keys as f64; + let mut low = distinct_windows as f64; + let mut high = batch_keys.min(U32_WINDOW_COUNT) as f64; + + if low >= high { + return low; + } + + let log_unseen = (-sample_fraction).ln_1p(); + for _ in 0..100 { + let mid = (low + high) * 0.5; + let avg_keys_per_window = batch_keys as f64 / mid; + let observed_windows = mid * (1.0 - (avg_keys_per_window * log_unseen).exp()); + + if observed_windows < distinct_windows as f64 { + low = mid; + } else { + high = mid; + } + } + + high +} + +/// Estimate touched windows with a Chao1-style unseen-species correction. +/// +/// Here the "species" are touched 16-bit windows: +/// - `distinct_windows` is how many windows the sample observed +/// - `sample_singleton_windows` counts windows seen exactly once +/// - `sample_doubleton_windows` counts windows seen exactly twice +/// +/// Example: +/// - If a sample touches `50` windows, with `35` singletons and `2` +/// doubletons, that pattern is strong evidence that many windows were missed +/// entirely. +/// - Chao1 turns that singleton-heavy shape into a larger touched-window +/// estimate than the uniform model would produce. +/// +/// Raw Chao1 is intentionally not used directly in the final predictor because +/// it can overreact when `f2` is tiny. We still compute it here because it is +/// the right directional correction for the sparse-wide failure mode. +fn estimate_chao1_touched_windows( + distinct_windows: usize, + sample_singleton_windows: usize, + sample_doubleton_windows: usize, +) -> f64 { + // Chao1 is a classic unseen-species estimator. Here the "species" are + // touched 16-bit windows, and singleton-heavy samples are evidence that the + // sample missed many windows entirely. + let chao1_estimate = if sample_doubleton_windows > 0 { + distinct_windows as f64 + + (sample_singleton_windows * sample_singleton_windows) as f64 + / (2.0 * sample_doubleton_windows as f64) + } else { + distinct_windows as f64 + + (sample_singleton_windows.saturating_mul(sample_singleton_windows.saturating_sub(1)) + / 2) as f64 + }; + + chao1_estimate + .max(distinct_windows as f64) + .min(U32_WINDOW_COUNT as f64) +} + +fn arithmetic_blend(current: f64, chao1: f64, alpha: f64) -> f64 { + // Raw Chao1 reacts strongly to singleton-heavy samples, which is useful for + // sparse wide batches but too aggressive to use directly. Blend it toward + // the previous uniform estimate so the predictor only partially trusts the + // unseen-window correction. + current + alpha * (chao1 - current) +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum Winner { + Bloom, + Roaring, +} + +impl Winner { + fn as_str(self) -> &'static str { + match self { + Self::Bloom => "bloom", + Self::Roaring => "roaring", + } + } +} + +impl Display for Winner { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug, Clone, Copy)] +struct PredictorOutput { + density_score: f64, + build_score: f64, + lookup_score: f64, + memory_score: f64, + build_winner: Winner, + lookup_winner: Winner, + memory_winner: Winner, +} + +/// Convert sampled structural estimates into coarse Bloom-vs-Roaring winners. +/// +/// The predictor intentionally uses different signals for different metrics: +/// - build: mostly "how many keys end up in each touched window?" +/// - memory: same question, but with a higher density threshold +/// - lookup: "how often does a random probe reach a touched window, and how +/// expensive is that container likely to be once it does?" +/// +/// Example: +/// - Suppose a batch looks dense after sampling, with many keys per touched +/// window and only a small touched-window fraction. That usually pushes all +/// three metrics toward Roaring. +/// - Suppose instead the batch is spread across a large fraction of the 16-bit +/// windows and each window only has a modest number of keys. That is the +/// "many sparse array containers" regime where lookup can flip toward Bloom. +/// +/// The lookup path is where most of the iterations happened: +/// - using only keys/window missed sparse-wide cases +/// - using touched windows without normalizing them was the wrong shape +/// - using a flat array penalty missed that `ArrayStore::contains()` gets +/// slower as array containers grow +/// +/// The current formula keeps the model simple while preserving those learned +/// corrections from the benchmark runs. +fn predict_filter_winner(stats: &RoaringSampleStats) -> PredictorOutput { + let density_score = stats.estimated_window_fill_ratio; + // Build and memory stay as simple density rules: if touched windows are + // dense, Roaring tends to compress and build well; if they are sparse, + // Bloom tends to be cheaper. + let build_score = + stats.estimated_keys_per_window / BUILD_ROARING_ESTIMATED_KEYS_PER_WINDOW_THRESHOLD; + // For lookups we need more than density. Random u32 probes only pay inner + // container cost when they land in a touched 16-bit window, so the touched + // window estimate is normalized into a hit probability. If we omit this + // term, the predictor cannot distinguish dense-in-a-few-windows batches + // from equally dense batches spread across a large fraction of the domain. + let lookup_window_probability = + (stats.estimated_touched_windows / U32_WINDOW_COUNT as f64).clamp(0.0, 1.0); + // roaring-rs switches between array and bitmap containers around 4096 + // elements. Bitmap containers are close to a constant-time bit test, but + // array containers use binary search and get meaningfully slower as they + // grow. Without this size-dependent array penalty, medium-N wide Gaussians + // with many sparse array containers were still over-predicted as Roaring. + let lookup_container_penalty = + if stats.estimated_keys_per_window >= ROARING_BITMAP_CONTAINER_THRESHOLD { + LOOKUP_ROARING_BITMAP_WINDOW_PROBABILITY_PENALTY + } else { + LOOKUP_ROARING_ARRAY_WINDOW_PROBABILITY_PENALTY_BASE + + LOOKUP_ROARING_ARRAY_WINDOW_PROBABILITY_PENALTY_PER_LOG2_KEY + * (stats.estimated_keys_per_window + 1.0).log2() + }; + // lookup_score >= 1.0 means the estimated Roaring lookup cost stays under + // the current budget and we predict Roaring. The exact threshold is tuned + // empirically from benchmark output; the important part is the shape above. + let lookup_cost_proxy = lookup_window_probability * lookup_container_penalty; + let lookup_score = + LOOKUP_ROARING_WINDOW_PROBABILITY_THRESHOLD / lookup_cost_proxy.max(f64::MIN_POSITIVE); + let memory_score = + stats.estimated_keys_per_window / MEMORY_ROARING_ESTIMATED_KEYS_PER_WINDOW_THRESHOLD; + + PredictorOutput { + density_score, + build_score, + lookup_score, + memory_score, + build_winner: predicted_winner(build_score), + lookup_winner: predicted_winner(lookup_score), + memory_winner: predicted_winner(memory_score), + } +} + +fn predicted_winner(score: f64) -> Winner { + if score >= 1.0 { + Winner::Roaring + } else { + Winner::Bloom + } +} + +#[derive(Debug, Clone, Copy)] +struct Measurement { + build_ns_per_element: f64, + lookup_ns_per_element: f64, + lookup_hits: u64, + bytes_used: usize, +} + +fn benchmark_bloom( + keys: &[u32], + lookup_count: u64, + lookup_seed: u64, + lookup_space: LookupSpace, + bloom_expected_items: u64, + bloom_false_positive_rate: f64, + bloom_seed: u128, +) -> Measurement { + let expected_items = + usize::try_from(bloom_expected_items).expect("bloom expected items must fit in usize"); + let mut bloom = BloomFilter::with_false_pos(bloom_false_positive_rate) + .seed(&bloom_seed) + .expected_items(expected_items.max(MIN_BLOOM_EXPECTED_ITEMS as usize)); + + let build_started = Instant::now(); + for &key in keys { + bloom.insert(&key); + } + let build_elapsed = build_started.elapsed(); + + let (lookup_elapsed, hits) = benchmark_lookup(keys, lookup_count, lookup_seed, lookup_space, { + |key| bloom.contains(&key) + }); + + Measurement { + build_ns_per_element: build_elapsed.as_nanos() as f64 / keys.len() as f64, + lookup_ns_per_element: lookup_elapsed.as_nanos() as f64 / lookup_count as f64, + lookup_hits: hits, + bytes_used: size_of_val(bloom.as_slice()), + } +} + +fn benchmark_roaring( + keys: &[u32], + lookup_count: u64, + lookup_seed: u64, + lookup_space: LookupSpace, +) -> Measurement { + let build_started = Instant::now(); + let mut bitmap = RoaringBitmap::from_sorted_iter(keys.iter().copied()) + .expect("sorted gaussian keys should build a roaring bitmap"); + let build_elapsed = build_started.elapsed(); + let _ = bitmap.optimize(); + + let (lookup_elapsed, hits) = + benchmark_lookup(keys, lookup_count, lookup_seed, lookup_space, |key| { + bitmap.contains(key) + }); + + Measurement { + build_ns_per_element: build_elapsed.as_nanos() as f64 / keys.len() as f64, + lookup_ns_per_element: lookup_elapsed.as_nanos() as f64 / lookup_count as f64, + lookup_hits: hits, + bytes_used: bitmap.serialized_size(), + } +} + +fn benchmark_lookup( + keys: &[u32], + lookup_count: u64, + lookup_seed: u64, + lookup_space: LookupSpace, + mut contains: F, +) -> (std::time::Duration, u64) +where + F: FnMut(u32) -> bool, +{ + let lookup_started = Instant::now(); + let hits = match lookup_space { + LookupSpace::Present => { + let lookup_permutation = AffinePermutation::random(keys.len() as u64, lookup_seed); + let mut hits = 0u64; + for index in 0..lookup_count { + let key = keys[lookup_permutation.index_at(index) as usize]; + hits += u64::from(contains(key)); + } + assert_eq!( + hits, lookup_count, + "expected all present lookup keys to be present" + ); + hits + } + LookupSpace::FullU32 => { + let mut rng = ChaCha8Rng::seed_from_u64(lookup_seed); + let mut hits = 0u64; + for _ in 0..lookup_count { + hits += u64::from(contains(rng.next_u32())); + } + hits + } + }; + (lookup_started.elapsed(), hits) +} + +fn actual_winner(bloom_value: f64, roaring_value: f64) -> Winner { + if roaring_value < bloom_value { + Winner::Roaring + } else { + Winner::Bloom + } +} + +#[derive(Debug, Serialize)] +struct CsvRow { + num_keys: u64, + distribution: &'static str, + distribution_param_name: &'static str, + distribution_param_frac: Option, + distribution_param_value: Option, + gaussian_mean_frac: Option, + gaussian_mean: Option, + gaussian_stddev_frac: Option, + gaussian_stddev: Option, + repetition: usize, + distribution_seed: u64, + sample_seed: u64, + lookup_seed: u64, + lookup_space: &'static str, + lookup_count: u64, + sample_size: usize, + sample_percent_of_batch: f64, + sample_fraction: f64, + bloom_false_positive_rate_target_percent: f64, + bloom_seed: u128, + bloom_expected_items: u64, + predictor_sampled_keys: usize, + predictor_distinct_windows: usize, + predictor_avg_sample_keys_per_window: f64, + predictor_same_window_rate: f64, + predictor_estimated_keys_per_window: f64, + predictor_estimated_touched_windows: f64, + predictor_estimated_window_fill_ratio: f64, + predictor_density_score: f64, + predictor_build_score: f64, + predictor_lookup_score: f64, + predictor_memory_score: f64, + predicted_build_winner: &'static str, + predicted_lookup_winner: &'static str, + predicted_memory_winner: &'static str, + bloom_build_ns_per_element: f64, + roaring_build_ns_per_element: f64, + build_ratio_bloom_over_roaring: f64, + actual_build_winner: &'static str, + build_prediction_correct: bool, + bloom_lookup_ns_per_element: f64, + bloom_lookup_hits: u64, + bloom_lookup_hit_rate_percent: f64, + roaring_lookup_ns_per_element: f64, + roaring_lookup_hits: u64, + roaring_lookup_hit_rate_percent: f64, + lookup_ratio_bloom_over_roaring: f64, + actual_lookup_winner: &'static str, + lookup_prediction_correct: bool, + bloom_bytes_used: usize, + roaring_bytes_used: usize, + memory_ratio_bloom_over_roaring: f64, + actual_memory_winner: &'static str, + memory_prediction_correct: bool, +} + +#[derive(Debug, Default)] +struct AccuracySummary { + runs: usize, + build_correct: usize, + lookup_correct: usize, + memory_correct: usize, +} + +fn summarize_accuracy(rows: &[CsvRow]) -> AccuracySummary { + let mut accuracy = AccuracySummary::default(); + + for row in rows { + accuracy.runs += 1; + accuracy.build_correct += usize::from(row.build_prediction_correct); + accuracy.lookup_correct += usize::from(row.lookup_prediction_correct); + accuracy.memory_correct += usize::from(row.memory_prediction_correct); + } + + accuracy +} + +fn print_summary(rows: &[CsvRow], accuracy: &AccuracySummary) { + let wrong_rows: Vec<&CsvRow> = rows + .iter() + .filter(|row| { + !row.build_prediction_correct + || !row.lookup_prediction_correct + || !row.memory_prediction_correct + }) + .collect(); + let wrong_metric_predictions = wrong_rows + .iter() + .map(|row| { + usize::from(!row.build_prediction_correct) + + usize::from(!row.lookup_prediction_correct) + + usize::from(!row.memory_prediction_correct) + }) + .sum::(); + + println!("summary.runs={}", accuracy.runs); + println!( + "accuracy.build={}/{}", + accuracy.build_correct, accuracy.runs + ); + println!( + "accuracy.lookup={}/{}", + accuracy.lookup_correct, accuracy.runs + ); + println!( + "accuracy.memory={}/{}", + accuracy.memory_correct, accuracy.runs + ); + println!("wrong_predictions.run_count={}", wrong_rows.len()); + println!( + "wrong_predictions.metric_count={}", + wrong_metric_predictions + ); + + for row in wrong_rows { + println!( + "wrong_prediction {} num_keys={} repetition={} sample_size={} sample_percent_of_batch={:.6}", + distribution_summary_fields(row), + row.num_keys, + row.repetition, + row.sample_size, + row.sample_percent_of_batch + ); + println!( + "wrong_prediction.predictor avg_sample_keys_per_window={:.6} same_window_rate={:.6} estimated_keys_per_window={:.6} estimated_touched_windows={:.6} estimated_window_fill_ratio={:.6}", + row.predictor_avg_sample_keys_per_window, + row.predictor_same_window_rate, + row.predictor_estimated_keys_per_window, + row.predictor_estimated_touched_windows, + row.predictor_estimated_window_fill_ratio + ); + + if !row.build_prediction_correct { + println!( + "wrong_prediction.build predicted={} actual={} score={:.6} bloom_over_roaring={:.6}", + row.predicted_build_winner, + row.actual_build_winner, + row.predictor_build_score, + row.build_ratio_bloom_over_roaring + ); + } + if !row.lookup_prediction_correct { + println!( + "wrong_prediction.lookup predicted={} actual={} score={:.6} bloom_over_roaring={:.6}", + row.predicted_lookup_winner, + row.actual_lookup_winner, + row.predictor_lookup_score, + row.lookup_ratio_bloom_over_roaring + ); + } + if !row.memory_prediction_correct { + println!( + "wrong_prediction.memory predicted={} actual={} score={:.6} bloom_over_roaring={:.6}", + row.predicted_memory_winner, + row.actual_memory_winner, + row.predictor_memory_score, + row.memory_ratio_bloom_over_roaring + ); + } + } +} + +fn print_run_report(row: &CsvRow) { + println!("distribution={}", row.distribution); + println!("distribution_param_name={}", row.distribution_param_name); + println!( + "distribution_param_frac={}", + option_f64(row.distribution_param_frac) + ); + println!( + "distribution_param_value={}", + option_f64(row.distribution_param_value) + ); + println!("num_keys={}", row.num_keys); + println!("gaussian_mean_frac={}", option_f64(row.gaussian_mean_frac)); + println!("gaussian_mean={}", option_f64(row.gaussian_mean)); + println!( + "gaussian_stddev_frac={}", + option_f64(row.gaussian_stddev_frac) + ); + println!("gaussian_stddev={}", option_f64(row.gaussian_stddev)); + println!("repetition={}", row.repetition); + println!("lookup_space={}", row.lookup_space); + println!("sample_size={}", row.sample_size); + println!("sample_percent_of_batch={:.6}", row.sample_percent_of_batch); + println!("lookup_count={}", row.lookup_count); + println!("predictor.sampled_keys={}", row.predictor_sampled_keys); + println!( + "predictor.distinct_windows={}", + row.predictor_distinct_windows + ); + println!( + "predictor.avg_sample_keys_per_window={:.6}", + row.predictor_avg_sample_keys_per_window + ); + println!( + "predictor.same_window_rate={:.6}", + row.predictor_same_window_rate + ); + println!( + "predictor.estimated_keys_per_window={:.6}", + row.predictor_estimated_keys_per_window + ); + println!( + "predictor.estimated_touched_windows={:.6}", + row.predictor_estimated_touched_windows + ); + println!( + "predictor.estimated_window_fill_ratio={:.6}", + row.predictor_estimated_window_fill_ratio + ); + println!("predictor.build_score={:.6}", row.predictor_build_score); + println!("predictor.lookup_score={:.6}", row.predictor_lookup_score); + println!("predictor.memory_score={:.6}", row.predictor_memory_score); + println!("predicted.build_winner={}", row.predicted_build_winner); + println!("predicted.lookup_winner={}", row.predicted_lookup_winner); + println!("predicted.memory_winner={}", row.predicted_memory_winner); + println!( + "bloom.build_ns_per_element={:.6}", + row.bloom_build_ns_per_element + ); + println!( + "roaring.build_ns_per_element={:.6}", + row.roaring_build_ns_per_element + ); + println!( + "build_ratio_bloom_over_roaring={:.6}", + row.build_ratio_bloom_over_roaring + ); + println!("actual.build_winner={}", row.actual_build_winner); + println!("build_prediction_correct={}", row.build_prediction_correct); + println!( + "bloom.lookup_ns_per_element={:.6}", + row.bloom_lookup_ns_per_element + ); + println!("bloom.lookup_hits={}", row.bloom_lookup_hits); + println!( + "bloom.lookup_hit_rate_percent={:.6}", + row.bloom_lookup_hit_rate_percent + ); + println!( + "roaring.lookup_ns_per_element={:.6}", + row.roaring_lookup_ns_per_element + ); + println!("roaring.lookup_hits={}", row.roaring_lookup_hits); + println!( + "roaring.lookup_hit_rate_percent={:.6}", + row.roaring_lookup_hit_rate_percent + ); + println!( + "lookup_ratio_bloom_over_roaring={:.6}", + row.lookup_ratio_bloom_over_roaring + ); + println!("actual.lookup_winner={}", row.actual_lookup_winner); + println!( + "lookup_prediction_correct={}", + row.lookup_prediction_correct + ); + println!("bloom.bytes_used={}", row.bloom_bytes_used); + println!("roaring.bytes_used={}", row.roaring_bytes_used); + println!( + "memory_ratio_bloom_over_roaring={:.6}", + row.memory_ratio_bloom_over_roaring + ); + println!("actual.memory_winner={}", row.actual_memory_winner); + println!( + "memory_prediction_correct={}", + row.memory_prediction_correct + ); + println!(); +} + +#[derive(Clone, Copy, Debug)] +struct AffinePermutation { + len: u64, + multiplier: u64, + offset: u64, +} + +impl AffinePermutation { + fn sequential(len: u64) -> Self { + Self { + len, + multiplier: 1, + offset: 0, + } + } + + fn random(len: u64, seed: u64) -> Self { + if len <= 1 { + return Self::sequential(len); + } + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let mut multiplier = (rng.next_u64() % len) | 1; + while gcd(multiplier, len) != 1 { + multiplier = (multiplier + 2) % len; + if multiplier == 0 { + multiplier = 1; + } + } + let offset = rng.next_u64() % len; + Self { + len, + multiplier, + offset, + } + } + + fn index_at(&self, position: u64) -> u64 { + debug_assert!(position < self.len); + (self + .multiplier + .wrapping_mul(position) + .wrapping_add(self.offset)) + % self.len + } +} + +fn gcd(mut lhs: u64, mut rhs: u64) -> u64 { + while rhs != 0 { + let next = lhs % rhs; + lhs = rhs; + rhs = next; + } + lhs +} + +fn generate_keys(num_keys: u64, distribution: DistributionSpec, seed: u64) -> Vec { + match distribution { + DistributionSpec::Gaussian(distribution) => { + generate_gaussian_keys(num_keys, distribution, seed) + } + DistributionSpec::Consecutive => generate_consecutive_keys(num_keys), + DistributionSpec::RoundRobinWindow => generate_round_robin_window_keys(num_keys), + DistributionSpec::Bimodal { stddev_frac } => { + generate_bimodal_keys(num_keys, stddev_frac, seed) + } + DistributionSpec::Exponential { scale_frac } => { + generate_exponential_keys(num_keys, scale_frac, seed) + } + } +} + +fn generate_gaussian_keys( + num_keys: u64, + distribution: GaussianDistribution, + seed: u64, +) -> Vec { + let len = usize::try_from(num_keys).expect("num_keys must fit in usize"); + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let normal = Normal::new(distribution.mean_value(), distribution.stddev_value()) + .expect("gaussian distribution should have a positive standard deviation"); + let mut keys = Vec::with_capacity(len); + + for _ in 0..num_keys { + let sampled = normal.sample(&mut rng).round(); + keys.push(sampled.clamp(0.0, u32::MAX as f64) as u32); + } + + keys.sort_unstable(); + project_sorted_unique_u32_domain(&mut keys); + keys +} + +fn generate_consecutive_keys(num_keys: u64) -> Vec { + let len = usize::try_from(num_keys).expect("num_keys must fit in usize"); + (0..len) + .map(|index| u32::try_from(index).expect("consecutive key exceeded u32 domain")) + .collect() +} + +fn generate_round_robin_window_keys(num_keys: u64) -> Vec { + let len = usize::try_from(num_keys).expect("num_keys must fit in usize"); + let mut keys = Vec::with_capacity(len); + let full_layers = num_keys / U32_WINDOW_COUNT as u64; + let partial_windows = num_keys % U32_WINDOW_COUNT as u64; + + for window in 0..U32_WINDOW_COUNT as u64 { + let keys_in_window = full_layers + u64::from(window < partial_windows); + let window_base = window << 16; + for low in 0..keys_in_window { + keys.push( + u32::try_from(window_base + low).expect("round-robin key exceeded u32 domain"), + ); + } + } + + debug_assert_eq!(keys.len(), len); + keys +} + +fn generate_bimodal_keys(num_keys: u64, stddev_frac: f64, seed: u64) -> Vec { + let len = usize::try_from(num_keys).expect("num_keys must fit in usize"); + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let left = Normal::new( + BIMODAL_LEFT_PEAK_FRAC * u32::MAX as f64, + stddev_frac * u32::MAX as f64, + ) + .expect("bimodal distribution should have a positive standard deviation"); + let right = Normal::new( + BIMODAL_RIGHT_PEAK_FRAC * u32::MAX as f64, + stddev_frac * u32::MAX as f64, + ) + .expect("bimodal distribution should have a positive standard deviation"); + let mut keys = Vec::with_capacity(len); + + for _ in 0..num_keys { + let sampled = if rng.next_u32() & 1 == 0 { + left.sample(&mut rng) + } else { + right.sample(&mut rng) + } + .round(); + keys.push(sampled.clamp(0.0, u32::MAX as f64) as u32); + } + + keys.sort_unstable(); + project_sorted_unique_u32_domain(&mut keys); + keys +} + +fn generate_exponential_keys(num_keys: u64, scale_frac: f64, seed: u64) -> Vec { + let len = usize::try_from(num_keys).expect("num_keys must fit in usize"); + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let scale = (scale_frac * u32::MAX as f64).max(f64::MIN_POSITIVE); + let distribution = + Exp::new(1.0 / scale).expect("exponential distribution should have a positive scale"); + let mut keys = Vec::with_capacity(len); + + for _ in 0..num_keys { + let sampled = distribution.sample(&mut rng).round(); + keys.push(sampled.clamp(0.0, u32::MAX as f64) as u32); + } + + keys.sort_unstable(); + project_sorted_unique_u32_domain(&mut keys); + keys +} + +fn default_sample_size(num_keys: u64) -> usize { + sample_count_from_percent(num_keys, DEFAULT_SAMPLE_PERCENT, DEFAULT_MIN_SAMPLE_SIZE) +} + +fn parse_u64_csv(csv: &str) -> Vec { + let mut out: Vec = csv + .split(',') + .filter(|entry| !entry.trim().is_empty()) + .map(|entry| { + parse_u64_token(entry.trim()) + .unwrap_or_else(|error| panic!("invalid u64 in CSV: {entry} ({error})")) + }) + .collect(); + out.sort_unstable(); + out.dedup(); + out +} + +fn parse_f64_csv(csv: &str, flag_name: &str) -> Vec { + let mut out: Vec = csv + .split(',') + .filter(|entry| !entry.trim().is_empty()) + .map(|entry| { + entry + .trim() + .parse::() + .unwrap_or_else(|error| panic!("invalid f64 in {flag_name}: {entry} ({error})")) + }) + .collect(); + out.sort_by(|lhs, rhs| lhs.partial_cmp(rhs).expect("NaN was already rejected")); + out.dedup(); + out +} + +fn parse_distribution_csv(csv: &str) -> Vec { + let mut out = Vec::new(); + + for token in csv.split(',').filter(|entry| !entry.trim().is_empty()) { + let normalized = token.trim().replace('_', "-"); + let distribution = DistributionKind::from_str(&normalized, true).unwrap_or_else(|error| { + panic!("invalid distribution in --distributions: {token} ({error})") + }); + if !out.contains(&distribution) { + out.push(distribution); + } + } + + out +} + +fn parse_u64_token(token: &str) -> Result { + match token { + "u32::MAX" | "u32_max" | "max_u32" => Ok(u32::MAX as u64), + _ => token + .replace('_', "") + .parse::() + .map_err(|error| error.to_string()), + } +} + +fn project_sorted_unique_u32_domain(keys: &mut [u32]) { + if keys.is_empty() { + return; + } + + for (index, key) in keys.iter_mut().enumerate() { + let min_key = u32::try_from(index).expect("key count exceeded u32 domain"); + if *key < min_key { + *key = min_key; + } + } + + for index in (0..keys.len()).rev() { + let tail = keys.len() - 1 - index; + let max_key = u32::MAX + .checked_sub(u32::try_from(tail).expect("key count exceeded u32 domain")) + .expect("tail adjustment underflowed"); + if keys[index] > max_key { + keys[index] = max_key; + } + if index + 1 < keys.len() && keys[index] >= keys[index + 1] { + keys[index] = keys[index + 1] - 1; + } + } + + debug_assert!(keys.windows(2).all(|window| window[0] < window[1])); +} + +fn option_u64(value: Option) -> String { + value + .map(|value| value.to_string()) + .unwrap_or_else(|| "auto".to_string()) +} + +fn option_f64(value: Option) -> String { + value + .map(|value| value.to_string()) + .unwrap_or_else(|| "auto".to_string()) +} + +fn distribution_summary_fields(row: &CsvRow) -> String { + let mut fields = format!("distribution={}", row.distribution); + if let Some(gaussian_mean_frac) = row.gaussian_mean_frac { + fields.push_str(&format!(" gaussian_mean_frac={gaussian_mean_frac}")); + } + if let Some(distribution_param_frac) = row.distribution_param_frac { + fields.push_str(&format!( + " {}={distribution_param_frac}", + row.distribution_param_name + )); + } + fields +} + +fn sample_count_from_percent(num_keys: u64, sample_percent: f64, min_sample_size: usize) -> usize { + let scaled = ((num_keys as f64) * (sample_percent / 100.0)).ceil() as u64; + let sample_size = scaled.max(min_sample_size as u64).min(num_keys); + usize::try_from(sample_size).expect("sample size must fit in usize") +} diff --git a/scripts/plot_filter_bitmap.py b/scripts/plot_filter_bitmap.py new file mode 100644 index 00000000000..684dc31198d --- /dev/null +++ b/scripts/plot_filter_bitmap.py @@ -0,0 +1,855 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pandas>=2.2", +# "plotly>=5.24", +# "kaleido>=0.2.1", +# ] +# /// + +from __future__ import annotations + +import argparse +import math +from pathlib import Path + +import pandas as pd +import plotly.graph_objects as go +import plotly.io as pio +from plotly.subplots import make_subplots + + +KEY_TYPE_ORDER = ["u32", "u64"] +KEY_SPACE_ORDER = ["consecutive", "full_range", "half_normal"] +STRUCTURE_ORDER = ["bloom", "roaring"] +METRICS = [ + ( + "insert_ns_per_element_avg", + "insert_ns_per_element_min", + "insert_ns_per_element_max", + "Insert Time", + "Insert Time (ns/element)", + "ns", + ), + ( + "lookup_ns_per_element_avg", + "lookup_ns_per_element_min", + "lookup_ns_per_element_max", + "Lookup Time", + "Lookup Time (ns/element)", + "ns", + ), + ("bytes_used", None, None, "Memory Usage", "Memory Usage (bytes)", "bytes"), +] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Plot filter_bitmap.csv comparisons for bloom vs roaring." + ) + parser.add_argument( + "--input", + type=Path, + default=Path("crates/dbsp/filter_bitmap.csv"), + help="Input CSV produced by crates/dbsp/benches/filter_bitmap.rs", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("filter_bitmap_plots"), + help="Directory to write plots into", + ) + parser.add_argument( + "--write-png", + action="store_true", + help="Also export PNG images with Kaleido. Requires a working non-snap Chrome/Chromium.", + ) + return parser.parse_args() + + +def format_structure(name: str) -> str: + return { + "bloom": "fastbloom", + "roaring": "roaring", + }.get(name, name) + + +def format_key_type(name: str) -> str: + return { + "u32": "u32 Keys", + "u64": "u64 Keys", + }.get(name, name) + + +def format_key_space(name: str, key_type: str) -> str: + if name == "consecutive": + return "K={0..N}" + if name == "full_range": + max_label = "2^32" if key_type == "u32" else "2^64" + return f"K={{0..{max_label}}}" + if name == "half_normal": + return "Half-normal K={0..2^32}" + return name + + +def format_distribution_key_space(name: str) -> str: + if name == "half_normal": + return "Half-normal K={0..2^32}" + return name.replace("_", " ").title() + + +def format_num_elements(value: int) -> str: + return f"{value:,}" + + +def format_key_eps(value: float) -> str: + return f"{value:g}" + + +def format_bytes(value: float) -> str: + units = ["B", "KiB", "MiB", "GiB", "TiB"] + unit_index = 0 + while value >= 1024.0 and unit_index + 1 < len(units): + value /= 1024.0 + unit_index += 1 + return f"{value:.2f} {units[unit_index]}" + + +def format_ns_per_element(value: float) -> str: + return f"{value:.2f} ns" + + +def format_ratio(value: float) -> str: + return f"{value:.2f}x" + + +def metric_formatter(kind: str): + if kind == "bytes": + return format_bytes + return format_ns_per_element + + +def ordered_values(values: pd.Series, preferred_order: list[str]) -> list[str]: + present = {str(value) for value in values.dropna().unique()} + ordered = [value for value in preferred_order if value in present] + extras = sorted(present - set(preferred_order)) + return ordered + extras + + +def prepare_frame(frame: pd.DataFrame) -> pd.DataFrame: + frame = frame.copy() + + if "key_type" not in frame.columns: + frame["key_type"] = "u32" + if "key_space" not in frame.columns: + frame["key_space"] = "consecutive" + if "key_eps" not in frame.columns: + frame["key_eps"] = pd.NA + + numeric_columns = [ + "key_eps", + "num_elements", + "lookup_count", + "false_positive_lookup_count", + "repetitions", + "insert_seed", + "lookup_seed", + "key_space_seed", + "bloom_false_positive_rate_target_percent", + "bloom_seed", + "bloom_expected_items", + "bytes_used", + "bytes_per_element", + "bits_per_element", + "insert_ns_per_element_min", + "insert_ns_per_element_avg", + "insert_ns_per_element_max", + "insert_ns_per_element_stddev", + "lookup_ns_per_element_min", + "lookup_ns_per_element_avg", + "lookup_ns_per_element_max", + "lookup_ns_per_element_stddev", + "false_positive_rate_percent_min", + "false_positive_rate_percent_avg", + "false_positive_rate_percent_max", + "false_positive_rate_percent_stddev", + ] + for column in numeric_columns: + if column in frame.columns: + frame[column] = pd.to_numeric(frame[column], errors="coerce") + + group_columns = ["structure", "key_type", "key_space", "key_eps", "num_elements"] + agg_spec: dict[str, str] = {} + for column in frame.columns: + if column in group_columns: + continue + if not pd.api.types.is_numeric_dtype(frame[column]): + agg_spec[column] = "first" + elif column.endswith("_min"): + agg_spec[column] = "min" + elif column.endswith("_max"): + agg_spec[column] = "max" + elif column.endswith("_avg") or column.endswith("_stddev"): + agg_spec[column] = "mean" + elif column in { + "bytes_used", + "bytes_per_element", + "bits_per_element", + "bloom_false_positive_rate_target_percent", + }: + agg_spec[column] = "mean" + else: + agg_spec[column] = "first" + + frame = frame.groupby(group_columns, as_index=False, dropna=False).agg(agg_spec) + return frame.sort_values(group_columns) + + +def build_category_order( + frame: pd.DataFrame, + key_spaces: list[str], +) -> list[tuple[int, str]]: + ordered_sizes = sorted(int(value) for value in frame["num_elements"].unique()) + categories: list[tuple[int, str]] = [] + for size in ordered_sizes: + for key_space in key_spaces: + if ((frame["num_elements"] == size) & (frame["key_space"] == key_space)).any(): + categories.append((size, key_space)) + return categories + + +def category_axis(categories: list[tuple[int, str]], key_type: str) -> list[list[str]]: + return [ + [format_num_elements(size) for size, _ in categories], + [format_key_space(key_space, key_type) for _, key_space in categories], + ] + + +def build_metric_figure( + frame: pd.DataFrame, + y_column: str, + y_min_column: str | None, + y_max_column: str | None, + y_label: str, + title: str, + formatter, +) -> go.Figure: + key_types = ordered_values(frame["key_type"], KEY_TYPE_ORDER) + key_spaces = ordered_values(frame["key_space"], KEY_SPACE_ORDER) + colors = { + "bloom": "#0f766e", + "roaring": "#c2410c", + } + + fig = make_subplots( + rows=max(1, len(key_types)), + cols=1, + shared_xaxes=False, + vertical_spacing=0.18, + row_titles=[format_key_type(key_type) for key_type in key_types], + ) + + for row_index, key_type in enumerate(key_types, start=1): + row_frame = frame[frame["key_type"] == key_type] + categories = build_category_order(row_frame, key_spaces) + x_axis = category_axis(categories, key_type) + + for structure in STRUCTURE_ORDER: + structure_frame = ( + row_frame[row_frame["structure"] == structure] + .set_index(["num_elements", "key_space"]) + .sort_index() + ) + if structure_frame.empty: + continue + + y_values = [] + text_values = [] + error_plus = [] + error_minus = [] + for category in categories: + if category in structure_frame.index: + value = float(structure_frame.loc[category, y_column]) + y_values.append(value) + text_values.append(formatter(value)) + if y_min_column is not None and y_max_column is not None: + min_value = float(structure_frame.loc[category, y_min_column]) + max_value = float(structure_frame.loc[category, y_max_column]) + error_minus.append(max(0.0, value - min_value)) + error_plus.append(max(0.0, max_value - value)) + else: + error_minus.append(None) + error_plus.append(None) + else: + y_values.append(None) + text_values.append("") + error_minus.append(None) + error_plus.append(None) + + fig.add_trace( + go.Bar( + name=format_structure(structure), + x=x_axis, + y=y_values, + text=text_values, + textposition="outside", + cliponaxis=False, + marker_color=colors[structure], + showlegend=row_index == 1, + offsetgroup=structure, + legendgroup=structure, + error_y=( + dict( + type="data", + symmetric=False, + array=error_plus, + arrayminus=error_minus, + thickness=1.2, + width=3, + color="#334155", + ) + if y_min_column is not None and y_max_column is not None + else None + ), + ), + row=row_index, + col=1, + ) + + fig.update_xaxes(title_text="Input Size / Key Space", row=row_index, col=1) + fig.update_yaxes(title_text=y_label, type="log", row=row_index, col=1) + + fig.update_layout( + title=title, + barmode="group", + template="plotly_white", + width=max(1100, 160 * max(1, len(build_category_order(frame, key_spaces)))), + height=500 * max(1, len(key_types)), + legend_title_text="Structure", + margin=dict(t=90, r=30, b=80, l=80), + ) + return fig + + +def relative_frame(frame: pd.DataFrame, metric: str) -> pd.DataFrame: + if frame.empty: + return pd.DataFrame() + + pivot = ( + frame.pivot_table( + index=["key_type", "key_space", "key_eps", "num_elements"], + columns="structure", + values=metric, + aggfunc="first", + ) + .rename(columns={"bloom": "bloom_value", "roaring": "roaring_value"}) + .reset_index() + ) + if pivot.empty or {"bloom_value", "roaring_value"} - set(pivot.columns): + return pd.DataFrame() + + pivot = pivot.dropna(subset=["bloom_value", "roaring_value"]).copy() + if pivot.empty: + return pivot + + pivot["relative_factor"] = pivot["bloom_value"] / pivot["roaring_value"] + pivot["log2_relative_factor"] = pivot["relative_factor"].map(math.log2) + return pivot + + +def heatmap_tick_values(z_bound: float) -> list[float]: + step = 0.5 if z_bound <= 2.0 else 1.0 + tick_count = int(round((2 * z_bound) / step)) + return [(-z_bound + step * index) for index in range(tick_count + 1)] + + +def build_relative_heatmap_figure( + frame: pd.DataFrame, + metric: str, + title: str, + value_formatter, + colorbar_title: str, +) -> go.Figure | None: + ratio_frame = relative_frame(frame, metric) + if ratio_frame.empty: + return None + + key_types = ordered_values(ratio_frame["key_type"], KEY_TYPE_ORDER) + key_spaces = ordered_values(ratio_frame["key_space"], KEY_SPACE_ORDER) + max_abs_log2 = ratio_frame["log2_relative_factor"].abs().max() + z_bound = max(0.5, math.ceil(float(max_abs_log2) * 2.0) / 2.0) + tick_values = heatmap_tick_values(z_bound) + tick_text = [format_ratio(2**value) for value in tick_values] + + fig = make_subplots( + rows=max(1, len(key_types)), + cols=max(1, len(key_spaces)), + row_titles=[format_key_type(key_type) for key_type in key_types], + column_titles=[format_distribution_key_space(key_space) for key_space in key_spaces], + horizontal_spacing=0.08, + vertical_spacing=0.16, + ) + + max_num_values = 1 + max_eps_values = 1 + + for row_index, key_type in enumerate(key_types, start=1): + for col_index, key_space in enumerate(key_spaces, start=1): + subplot_frame = ratio_frame[ + (ratio_frame["key_type"] == key_type) + & (ratio_frame["key_space"] == key_space) + ] + if subplot_frame.empty: + continue + + eps_values = sorted(float(value) for value in subplot_frame["key_eps"].dropna().unique()) + num_values = sorted(int(value) for value in subplot_frame["num_elements"].unique()) + max_num_values = max(max_num_values, len(num_values)) + max_eps_values = max(max_eps_values, len(eps_values)) + + log2_table = ( + subplot_frame.pivot(index="key_eps", columns="num_elements", values="log2_relative_factor") + .reindex(index=eps_values, columns=num_values) + ) + ratio_table = ( + subplot_frame.pivot(index="key_eps", columns="num_elements", values="relative_factor") + .reindex(index=eps_values, columns=num_values) + ) + bloom_table = ( + subplot_frame.pivot(index="key_eps", columns="num_elements", values="bloom_value") + .reindex(index=eps_values, columns=num_values) + ) + roaring_table = ( + subplot_frame.pivot(index="key_eps", columns="num_elements", values="roaring_value") + .reindex(index=eps_values, columns=num_values) + ) + + text = [ + [ + format_ratio(value) if pd.notna(value) else "" + for value in row_values + ] + for row_values in ratio_table.values + ] + customdata = [ + [ + [ + value_formatter(bloom_value) if pd.notna(bloom_value) else "", + value_formatter(roaring_value) if pd.notna(roaring_value) else "", + ] + for bloom_value, roaring_value in zip(bloom_row, roaring_row) + ] + for bloom_row, roaring_row in zip(bloom_table.values, roaring_table.values) + ] + + fig.add_trace( + go.Heatmap( + x=[format_num_elements(value) for value in num_values], + y=[format_key_eps(value) for value in eps_values], + z=log2_table.values, + text=text, + customdata=customdata, + texttemplate="%{text}", + hoverongaps=False, + xgap=1, + ygap=1, + coloraxis="coloraxis", + hovertemplate=( + "num_elements=%{x}
" + "key_eps=%{y}
" + f"{colorbar_title}=%{{text}}
" + "fastbloom=%{customdata[0]}
" + "roaring=%{customdata[1]}" + "" + ), + ), + row=row_index, + col=col_index, + ) + + fig.update_xaxes(title_text="num_elements", row=row_index, col=col_index) + fig.update_yaxes(title_text="key_eps", row=row_index, col=col_index) + + fig.update_layout( + title=title, + template="plotly_white", + width=max(950, 280 * len(key_spaces) + 110 * max_num_values * len(key_spaces)), + height=max(480, 220 * len(key_types) + 70 * max_eps_values * len(key_types)), + margin=dict(t=110, r=40, b=80, l=90), + coloraxis=dict( + colorscale=[ + (0.0, "#b91c1c"), + (0.5, "#f8fafc"), + (1.0, "#15803d"), + ], + cmin=-z_bound, + cmax=z_bound, + colorbar=dict( + title=colorbar_title, + tickvals=tick_values, + ticktext=tick_text, + ), + ), + ) + return fig + + +def build_summary_figure(frame: pd.DataFrame) -> go.Figure: + key_types = ordered_values(frame["key_type"], KEY_TYPE_ORDER) + key_spaces = ordered_values(frame["key_space"], KEY_SPACE_ORDER) + colors = { + "bloom": "#0f766e", + "roaring": "#c2410c", + } + + fig = make_subplots( + rows=max(1, len(key_types)), + cols=3, + subplot_titles=[ + metric_title + for _ in key_types + for _, _, _, metric_title, _, _ in METRICS + ], + row_titles=[format_key_type(key_type) for key_type in key_types], + horizontal_spacing=0.06, + vertical_spacing=0.18, + ) + + for row_index, key_type in enumerate(key_types, start=1): + row_frame = frame[frame["key_type"] == key_type] + categories = build_category_order(row_frame, key_spaces) + x_axis = category_axis(categories, key_type) + + for col_index, ( + metric, + metric_min, + metric_max, + _metric_title, + y_label, + kind, + ) in enumerate(METRICS, start=1): + formatter = metric_formatter(kind) + for structure in STRUCTURE_ORDER: + structure_frame = ( + row_frame[row_frame["structure"] == structure] + .set_index(["num_elements", "key_space"]) + .sort_index() + ) + if structure_frame.empty: + continue + + y_values = [] + text_values = [] + error_plus = [] + error_minus = [] + for category in categories: + if category in structure_frame.index: + value = float(structure_frame.loc[category, metric]) + y_values.append(value) + text_values.append(formatter(value)) + if metric_min is not None and metric_max is not None: + min_value = float(structure_frame.loc[category, metric_min]) + max_value = float(structure_frame.loc[category, metric_max]) + error_minus.append(max(0.0, value - min_value)) + error_plus.append(max(0.0, max_value - value)) + else: + error_minus.append(None) + error_plus.append(None) + else: + y_values.append(None) + text_values.append("") + error_minus.append(None) + error_plus.append(None) + + fig.add_trace( + go.Bar( + name=format_structure(structure), + x=x_axis, + y=y_values, + text=text_values, + textposition="outside", + cliponaxis=False, + marker_color=colors[structure], + showlegend=row_index == 1 and col_index == 1, + offsetgroup=structure, + legendgroup=structure, + error_y=( + dict( + type="data", + symmetric=False, + array=error_plus, + arrayminus=error_minus, + thickness=1.2, + width=3, + color="#334155", + ) + if metric_min is not None and metric_max is not None + else None + ), + ), + row=row_index, + col=col_index, + ) + + fig.update_yaxes(title_text=y_label, type="log", row=row_index, col=col_index) + fig.update_xaxes( + title_text="Input Size / Key Space", + row=row_index, + col=col_index, + ) + + fig.update_layout( + title="filter_bitmap Summary", + barmode="group", + template="plotly_white", + width=max(1900, 260 * max(1, len(build_category_order(frame, key_spaces)))), + height=640 * max(1, len(key_types)), + legend_title_text="Structure", + margin=dict(t=110, r=30, b=90, l=70), + ) + return fig + + +def write_figure(fig: go.Figure, base_path: Path, write_png: bool) -> None: + fig.write_html(base_path.with_suffix(".html")) + if write_png: + try: + fig.write_image(base_path.with_suffix(".png"), scale=2) + except Exception as exc: # pragma: no cover - depends on local browser setup. + print(f"warning: failed to write {base_path.with_suffix('.png')}: {exc}") + + +def write_summary_dashboard( + sections: list[tuple[str, go.Figure]], + output_path: Path, +) -> None: + if not sections: + return + + grouped_summary = next( + ((title, figure) for title, figure in sections if title == "Grouped Summary"), + None, + ) + heatmap_sections = [ + (title, figure) for title, figure in sections if title != "Grouped Summary" + ] + + html_parts = [ + "", + "", + "", + " ", + " ", + " filter_bitmap Summary", + " ", + "", + "", + "
", + "

filter_bitmap Summary

", + ] + + next_plotlyjs_mode = "cdn" + + if grouped_summary is not None: + title, figure = grouped_summary + figure_html = pio.to_html( + figure, + full_html=False, + include_plotlyjs=next_plotlyjs_mode, + ) + next_plotlyjs_mode = False + html_parts.extend( + [ + "
", + f"

{title}

", + figure_html, + "
", + ] + ) + + if heatmap_sections: + html_parts.extend( + [ + "
", + "

Roaring Advantage Heatmaps

", + "
", + ] + ) + for title, figure in heatmap_sections: + figure_html = pio.to_html( + figure, + full_html=False, + include_plotlyjs=next_plotlyjs_mode, + ) + next_plotlyjs_mode = False + html_parts.extend( + [ + "
", + f"

{title}

", + figure_html, + "
", + ] + ) + html_parts.extend( + [ + "
", + "
", + ] + ) + + html_parts.extend(["
", "", ""]) + output_path.write_text("\n".join(html_parts), encoding="utf-8") + + +def main() -> None: + args = parse_args() + if not args.input.exists(): + raise SystemExit(f"input CSV not found: {args.input}") + + frame = pd.read_csv(args.input) + if frame.empty: + raise SystemExit(f"input CSV is empty: {args.input}") + + required_columns = { + "structure", + "num_elements", + "insert_ns_per_element_avg", + "lookup_ns_per_element_avg", + "bytes_used", + } + missing_columns = sorted(required_columns - set(frame.columns)) + if missing_columns: + raise SystemExit( + f"input CSV is missing required columns: {', '.join(missing_columns)}" + ) + + frame = prepare_frame(frame) + args.output_dir.mkdir(parents=True, exist_ok=True) + + standard_frame = frame[frame["key_eps"].isna()].copy() + distribution_frame = frame[frame["key_eps"].notna()].copy() + summary_sections: list[tuple[str, go.Figure]] = [] + + if not standard_frame.empty: + insert_figure = build_metric_figure( + standard_frame, + "insert_ns_per_element_avg", + "insert_ns_per_element_min", + "insert_ns_per_element_max", + "Insert Time (ns/element)", + "filter_bitmap: Insert Time", + format_ns_per_element, + ) + write_figure( + insert_figure, + args.output_dir / "filter_bitmap_insert_ns_per_element", + args.write_png, + ) + + lookup_figure = build_metric_figure( + standard_frame, + "lookup_ns_per_element_avg", + "lookup_ns_per_element_min", + "lookup_ns_per_element_max", + "Lookup Time (ns/element)", + "filter_bitmap: Lookup Time", + format_ns_per_element, + ) + write_figure( + lookup_figure, + args.output_dir / "filter_bitmap_lookup_ns_per_element", + args.write_png, + ) + + memory_figure = build_metric_figure( + standard_frame, + "bytes_used", + None, + None, + "Memory Usage (bytes)", + "filter_bitmap: Memory Usage", + format_bytes, + ) + write_figure( + memory_figure, + args.output_dir / "filter_bitmap_memory_bytes", + args.write_png, + ) + + summary_figure = build_summary_figure(standard_frame) + write_figure( + summary_figure, + args.output_dir / "filter_bitmap_summary", + args.write_png, + ) + summary_sections.append(("Grouped Summary", summary_figure)) + + if not distribution_frame.empty: + insert_heatmap = build_relative_heatmap_figure( + distribution_frame, + "insert_ns_per_element_avg", + "filter_bitmap: Roaring Insert Advantage Heatmap", + format_ns_per_element, + "fastbloom / roaring", + ) + if insert_heatmap is not None: + write_figure( + insert_heatmap, + args.output_dir / "filter_bitmap_insert_advantage_heatmap", + args.write_png, + ) + summary_sections.append(("Roaring Insert Advantage Heatmap", insert_heatmap)) + + lookup_heatmap = build_relative_heatmap_figure( + distribution_frame, + "lookup_ns_per_element_avg", + "filter_bitmap: Roaring Lookup Advantage Heatmap", + format_ns_per_element, + "fastbloom / roaring", + ) + if lookup_heatmap is not None: + write_figure( + lookup_heatmap, + args.output_dir / "filter_bitmap_lookup_advantage_heatmap", + args.write_png, + ) + summary_sections.append(("Roaring Lookup Advantage Heatmap", lookup_heatmap)) + + memory_heatmap = build_relative_heatmap_figure( + distribution_frame, + "bytes_used", + "filter_bitmap: Roaring Memory Advantage Heatmap", + format_bytes, + "fastbloom / roaring", + ) + if memory_heatmap is not None: + write_figure( + memory_heatmap, + args.output_dir / "filter_bitmap_memory_advantage_heatmap", + args.write_png, + ) + summary_sections.append(("Roaring Memory Advantage Heatmap", memory_heatmap)) + + write_summary_dashboard( + summary_sections, + args.output_dir / "filter_bitmap_summary.html", + ) + + print(f"wrote plots to {args.output_dir}") + + +if __name__ == "__main__": + main() From 60bfe96990f676df6297ee778def84f3f9a9e1bf Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 4 Apr 2026 13:55:51 -0700 Subject: [PATCH 3/3] storage: bump layer file format to v6 --- Cargo.lock | 1 + crates/dbsp/src/circuit/metadata.rs | 8 +++++++ crates/dbsp/src/storage/file.rs | 5 ++-- crates/dbsp/src/storage/file/filter/stats.rs | 9 +++++++ crates/dbsp/src/storage/file/format.rs | 21 ++++++++++++++++- crates/dbsp/src/trace.rs | 19 +++++++++++---- crates/dbsp/src/trace/filter/batch.rs | 22 ++++++++++++++++++ .../src/trace/ord/fallback/indexed_wset.rs | 7 ++++++ .../dbsp/src/trace/ord/fallback/key_batch.rs | 8 +++++++ .../dbsp/src/trace/ord/fallback/val_batch.rs | 8 +++++++ crates/dbsp/src/trace/ord/fallback/wset.rs | 7 ++++++ .../src/trace/ord/file/indexed_wset_batch.rs | 4 ++++ crates/dbsp/src/trace/ord/file/key_batch.rs | 4 ++++ crates/dbsp/src/trace/ord/file/val_batch.rs | 4 ++++ crates/dbsp/src/trace/ord/file/wset_batch.rs | 4 ++++ crates/dbsp/src/trace/spine_async/snapshot.rs | 7 ------ .../golden-batch-v6-large.feldera | Bin 0 -> 281600 bytes .../golden-batch-v6-small.feldera | Bin 0 -> 27648 bytes .../golden-batch-v6-snappy-large.feldera | Bin 0 -> 89088 bytes .../golden-batch-v6-snappy-small.feldera | Bin 0 -> 13312 bytes 20 files changed, 123 insertions(+), 15 deletions(-) create mode 100644 crates/storage-test-compat/golden-files/golden-batch-v6-large.feldera create mode 100644 crates/storage-test-compat/golden-files/golden-batch-v6-small.feldera create mode 100644 crates/storage-test-compat/golden-files/golden-batch-v6-snappy-large.feldera create mode 100644 crates/storage-test-compat/golden-files/golden-batch-v6-snappy-small.feldera diff --git a/Cargo.lock b/Cargo.lock index 179949b262e..ba3a3204287 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3822,6 +3822,7 @@ dependencies = [ "reqwest 0.12.24", "rkyv", "rmp-serde", + "roaring", "seq-macro", "serde", "serde_json", diff --git a/crates/dbsp/src/circuit/metadata.rs b/crates/dbsp/src/circuit/metadata.rs index cbf6319abca..d0c3001b97b 100644 --- a/crates/dbsp/src/circuit/metadata.rs +++ b/crates/dbsp/src/circuit/metadata.rs @@ -136,6 +136,14 @@ pub const BLOOM_FILTER_MISSES_COUNT: MetricId = pub const BLOOM_FILTER_HIT_RATE_PERCENT: MetricId = MetricId(Cow::Borrowed("bloom_filter_hit_rate_percent")); pub const BLOOM_FILTER_SIZE_BYTES: MetricId = MetricId(Cow::Borrowed("bloom_filter_size_bytes")); +pub const ROARING_FILTER_HITS_COUNT: MetricId = + MetricId(Cow::Borrowed("roaring_filter_hits_count")); +pub const ROARING_FILTER_MISSES_COUNT: MetricId = + MetricId(Cow::Borrowed("roaring_filter_misses_count")); +pub const ROARING_FILTER_HIT_RATE_PERCENT: MetricId = + MetricId(Cow::Borrowed("roaring_filter_hit_rate_percent")); +pub const ROARING_FILTER_SIZE_BYTES: MetricId = + MetricId(Cow::Borrowed("roaring_filter_size_bytes")); pub const RANGE_FILTER_HITS_COUNT: MetricId = MetricId(Cow::Borrowed("range_filter_hits_count")); pub const RANGE_FILTER_MISSES_COUNT: MetricId = MetricId(Cow::Borrowed("range_filter_misses_count")); diff --git a/crates/dbsp/src/storage/file.rs b/crates/dbsp/src/storage/file.rs index 08c67285609..e3abc88bfae 100644 --- a/crates/dbsp/src/storage/file.rs +++ b/crates/dbsp/src/storage/file.rs @@ -584,9 +584,8 @@ impl Deserializer { pub fn new(version: u32) -> Self { // Proper error is returned in reader.rs, this is a sanity check. assert!( - version >= format::VERSION_NUMBER, - "Unable to read old (pre-v{}) checkpoint data on this feldera version, pipeline needs to backfilled to start.", - format::VERSION_NUMBER + version >= format::MIN_SUPPORTED_VERSION, + "Unable to read checkpoint data with unsupported old storage format version {version} on this feldera version.", ); Self { version, diff --git a/crates/dbsp/src/storage/file/filter/stats.rs b/crates/dbsp/src/storage/file/filter/stats.rs index 69ac9993ecc..fbad894ee4c 100644 --- a/crates/dbsp/src/storage/file/filter/stats.rs +++ b/crates/dbsp/src/storage/file/filter/stats.rs @@ -1,6 +1,15 @@ use crossbeam::utils::CachePadded; use std::sync::atomic::{AtomicUsize, Ordering}; +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Ord, PartialOrd)] +pub enum FilterKind { + #[default] + None, + Bloom, + Roaring, + Range, +} + /// Statistics about an in-memory key filter. /// /// The statistics implement addition such that they can be summed across diff --git a/crates/dbsp/src/storage/file/format.rs b/crates/dbsp/src/storage/file/format.rs index 8c9c3310dee..da7994c5542 100644 --- a/crates/dbsp/src/storage/file/format.rs +++ b/crates/dbsp/src/storage/file/format.rs @@ -90,11 +90,15 @@ use size_of::SizeOf; /// - v3: Bloom filter format change. /// - v4: Tup None optimizations. /// - v5: Change in representation for Timestamp, ShortInterval +/// - v6: Roaring bitmap filter blocks. /// /// When a new version is created, make sure to generate new golden /// files for it in crate `storage-test-compat` to check for /// backwards compatibility. -pub const VERSION_NUMBER: u32 = 5; +pub const VERSION_NUMBER: u32 = 6; + +/// Oldest layer file format version this binary can read. +pub const MIN_SUPPORTED_VERSION: u32 = 5; /// Magic number for data blocks. pub const DATA_BLOCK_MAGIC: [u8; 4] = *b"LFDB"; @@ -244,6 +248,17 @@ impl FileTrailer { (self.compatible_features & feature) != 0 } + /// Returns the unknown incompatible features, if any. + pub fn unknown_incompatible_features(&self) -> Option { + let unknown_incompatible_features = + self.incompatible_features & !INCOMPATIBLE_FEATURE_ROARING_FILTERS; + if unknown_incompatible_features != 0 { + Some(unknown_incompatible_features) + } else { + None + } + } + /// Returns true if this file trailer has a 64-bit filter. pub fn has_filter64(&self) -> bool { self.has_compatible_feature(COMPATIBLE_FEATURE_FILTER64) @@ -260,6 +275,10 @@ pub const COMPATIBLE_FEATURE_FILTER64: u64 = 1 << 0; /// deserialized as if its value is 0. Conversely, old readers will simply ignore the field. pub const COMPATIBLE_FEATURE_NEGATIVE_WEIGHT_COUNT: u64 = 1 << 1; +/// Bit set to 1 in [FileTrailer::incompatible_features] if the file contains +/// roaring bitmap membership filter blocks. +pub const INCOMPATIBLE_FEATURE_ROARING_FILTERS: u64 = 1 << 0; + /// Information about a column. /// /// Embedded inside the [`FileTrailer`] block. diff --git a/crates/dbsp/src/trace.rs b/crates/dbsp/src/trace.rs index ce5cd191960..aad64648e9c 100644 --- a/crates/dbsp/src/trace.rs +++ b/crates/dbsp/src/trace.rs @@ -80,7 +80,6 @@ use crate::{ algebra::MonoidValue, dynamic::{DataTrait, DynPair, DynVec, DynWeightedPairs, Erase, Factory, WeightTrait}, storage::file::reader::Error as ReaderError, - storage::filter_stats::FilterStats, }; pub use cursor::{Cursor, MergeCursor}; pub use filter::{Filter, GroupFilter}; @@ -478,13 +477,22 @@ where /// [Cursor::seek_key_exact] after the range filter. /// /// Today this is usually a Bloom filter. Batches without such a filter - /// should return `FilterStats::default()`. - fn membership_filter_stats(&self) -> FilterStats; + /// should return zero/default stats. + fn membership_filter_stats(&self) -> FilterStats { + FilterStats::default() + } + + /// Filter kind for the secondary membership filter used by + /// [Cursor::seek_key_exact]. + fn membership_filter_kind(&self) -> FilterKind { + FilterKind::None + } /// Statistics of the in-memory range filter used by /// [Cursor::seek_key_exact]. /// - /// Batches without a range filter should return `FilterStats::default()`. + /// Returns range-filter stats. Batches without a range filter should + /// return zeroed range stats. fn range_filter_stats(&self) -> FilterStats { FilterStats::default() } @@ -674,6 +682,9 @@ where fn membership_filter_stats(&self) -> FilterStats { (**self).membership_filter_stats() } + fn membership_filter_kind(&self) -> FilterKind { + (**self).membership_filter_kind() + } fn range_filter_stats(&self) -> FilterStats { (**self).range_filter_stats() } diff --git a/crates/dbsp/src/trace/filter/batch.rs b/crates/dbsp/src/trace/filter/batch.rs index 8253e805229..91745d23783 100644 --- a/crates/dbsp/src/trace/filter/batch.rs +++ b/crates/dbsp/src/trace/filter/batch.rs @@ -32,6 +32,9 @@ where /// filters pays that cost at most once. fn maybe_contains_key(&self, key: &K, hash: &mut Option) -> bool; + /// Filter kind for observability. + fn kind(&self) -> FilterKind; + /// Statistics for this filter. fn stats(&self) -> FilterStats; } @@ -128,6 +131,13 @@ where } } + pub fn membership_filter_kind(&self) -> FilterKind { + self.membership_filter + .as_ref() + .map(|filter| filter.kind()) + .unwrap_or(FilterKind::None) + } + /// Returns the cached key bounds, when available. pub fn key_bounds(&self) -> Option<(&K, &K)> { self.range_filter.range.as_ref().map(|range| range.bounds()) @@ -203,6 +213,10 @@ where is_hit } + fn kind(&self) -> FilterKind { + FilterKind::Range + } + fn stats(&self) -> FilterStats { self.as_ref().stats() } @@ -217,6 +231,10 @@ where self.contains_hash(*hash) } + fn kind(&self) -> FilterKind { + FilterKind::Bloom + } + fn stats(&self) -> FilterStats { TrackingBloomFilter::stats(self) } @@ -230,6 +248,10 @@ where self.maybe_contains_key(key) } + fn kind(&self) -> FilterKind { + FilterKind::Roaring + } + fn stats(&self) -> FilterStats { TrackingRoaringBitmap::stats(self) } diff --git a/crates/dbsp/src/trace/ord/fallback/indexed_wset.rs b/crates/dbsp/src/trace/ord/fallback/indexed_wset.rs index 74e03e28af4..872de33b6e8 100644 --- a/crates/dbsp/src/trace/ord/fallback/indexed_wset.rs +++ b/crates/dbsp/src/trace/ord/fallback/indexed_wset.rs @@ -283,6 +283,13 @@ where } } + fn membership_filter_kind(&self) -> FilterKind { + match &self.inner { + Inner::File(file) => file.membership_filter_kind(), + Inner::Vec(vec) => vec.membership_filter_kind(), + } + } + fn range_filter_stats(&self) -> FilterStats { match &self.inner { Inner::File(file) => file.range_filter_stats(), diff --git a/crates/dbsp/src/trace/ord/fallback/key_batch.rs b/crates/dbsp/src/trace/ord/fallback/key_batch.rs index 3a8e676385b..d76d34fee3f 100644 --- a/crates/dbsp/src/trace/ord/fallback/key_batch.rs +++ b/crates/dbsp/src/trace/ord/fallback/key_batch.rs @@ -274,6 +274,14 @@ where } } + #[inline] + fn membership_filter_kind(&self) -> FilterKind { + match &self.inner { + Inner::File(file) => file.membership_filter_kind(), + Inner::Vec(vec) => vec.membership_filter_kind(), + } + } + #[inline] fn range_filter_stats(&self) -> FilterStats { match &self.inner { diff --git a/crates/dbsp/src/trace/ord/fallback/val_batch.rs b/crates/dbsp/src/trace/ord/fallback/val_batch.rs index bb5cd2ac69a..96e94c38919 100644 --- a/crates/dbsp/src/trace/ord/fallback/val_batch.rs +++ b/crates/dbsp/src/trace/ord/fallback/val_batch.rs @@ -281,6 +281,14 @@ where } } + #[inline] + fn membership_filter_kind(&self) -> FilterKind { + match &self.inner { + Inner::File(file) => file.membership_filter_kind(), + Inner::Vec(vec) => vec.membership_filter_kind(), + } + } + #[inline] fn range_filter_stats(&self) -> FilterStats { match &self.inner { diff --git a/crates/dbsp/src/trace/ord/fallback/wset.rs b/crates/dbsp/src/trace/ord/fallback/wset.rs index 9e6f2f21bfc..3757d52e7bd 100644 --- a/crates/dbsp/src/trace/ord/fallback/wset.rs +++ b/crates/dbsp/src/trace/ord/fallback/wset.rs @@ -281,6 +281,13 @@ where } } + fn membership_filter_kind(&self) -> FilterKind { + match &self.inner { + Inner::File(file) => file.membership_filter_kind(), + Inner::Vec(vec) => vec.membership_filter_kind(), + } + } + fn range_filter_stats(&self) -> FilterStats { match &self.inner { Inner::File(file) => file.range_filter_stats(), diff --git a/crates/dbsp/src/trace/ord/file/indexed_wset_batch.rs b/crates/dbsp/src/trace/ord/file/indexed_wset_batch.rs index c7498a1dc42..f4d0819cfb7 100644 --- a/crates/dbsp/src/trace/ord/file/indexed_wset_batch.rs +++ b/crates/dbsp/src/trace/ord/file/indexed_wset_batch.rs @@ -399,6 +399,10 @@ where self.filters.stats().membership_filter } + fn membership_filter_kind(&self) -> FilterKind { + self.filters.membership_filter_kind() + } + fn range_filter_stats(&self) -> FilterStats { self.filters.stats().range_filter } diff --git a/crates/dbsp/src/trace/ord/file/key_batch.rs b/crates/dbsp/src/trace/ord/file/key_batch.rs index f01e051902f..6ebbe942c77 100644 --- a/crates/dbsp/src/trace/ord/file/key_batch.rs +++ b/crates/dbsp/src/trace/ord/file/key_batch.rs @@ -304,6 +304,10 @@ where self.filters.stats().membership_filter } + fn membership_filter_kind(&self) -> FilterKind { + self.filters.membership_filter_kind() + } + fn range_filter_stats(&self) -> FilterStats { self.filters.stats().range_filter } diff --git a/crates/dbsp/src/trace/ord/file/val_batch.rs b/crates/dbsp/src/trace/ord/file/val_batch.rs index f1ef3905703..34b1e8cedb4 100644 --- a/crates/dbsp/src/trace/ord/file/val_batch.rs +++ b/crates/dbsp/src/trace/ord/file/val_batch.rs @@ -325,6 +325,10 @@ where self.filters.stats().membership_filter } + fn membership_filter_kind(&self) -> FilterKind { + self.filters.membership_filter_kind() + } + fn range_filter_stats(&self) -> FilterStats { self.filters.stats().range_filter } diff --git a/crates/dbsp/src/trace/ord/file/wset_batch.rs b/crates/dbsp/src/trace/ord/file/wset_batch.rs index 9d06f3d0601..10bf4bd7698 100644 --- a/crates/dbsp/src/trace/ord/file/wset_batch.rs +++ b/crates/dbsp/src/trace/ord/file/wset_batch.rs @@ -388,6 +388,10 @@ where self.filters.stats().membership_filter } + fn membership_filter_kind(&self) -> FilterKind { + self.filters.membership_filter_kind() + } + fn range_filter_stats(&self) -> FilterStats { self.filters.stats().range_filter } diff --git a/crates/dbsp/src/trace/spine_async/snapshot.rs b/crates/dbsp/src/trace/spine_async/snapshot.rs index a2bf046d3d7..92c0ea12a52 100644 --- a/crates/dbsp/src/trace/spine_async/snapshot.rs +++ b/crates/dbsp/src/trace/spine_async/snapshot.rs @@ -230,13 +230,6 @@ where .fold(0, |acc, batch| acc + batch.approximate_byte_size()) } - fn membership_filter_stats(&self) -> FilterStats { - self.batches - .iter() - .map(|b| b.membership_filter_stats()) - .sum() - } - fn range_filter_stats(&self) -> FilterStats { self.batches.iter().map(|b| b.range_filter_stats()).sum() } diff --git a/crates/storage-test-compat/golden-files/golden-batch-v6-large.feldera b/crates/storage-test-compat/golden-files/golden-batch-v6-large.feldera new file mode 100644 index 0000000000000000000000000000000000000000..c97031e27859bf91d8a207f2090860feec94357f GIT binary patch literal 281600 zcmdR%d3;k<+Q)ATNFh|pqN1YIfKrvEJ7p0Uq|8Ffa2jAk5Pb<}a{G7jU)xXj?Vz31NNIrpCXa865F{vn@FdzUA9e&;^t zx!>oWwCCQT>+90Tk1BR&YMK_ln*Cz0R9G0l>%5w_vTAi-|7FYi`d6=7ykb$`Z1y_% zE9_n~Z`I;?D^?r(jQ7u5wQAnFzVqk#ea2>c*w#M++YP_Li)-kETJ zhqqb3xu2yu);_kDx+ceJ6WeaPG?&%R@#MHt+WfJt+Wd>hEmBe%$&8kMmXhc4yl=jo`99Z~YAI zdk2^HcGkFz4-K!0v36dxk!?76e!9->)~o#~TG3eJ9h#PCHSen|P202Q`R5NE`rF^W z{`$Ay&|)l3#f=PH`cY4OkGQ_6n;8&ecaY>?Whe37tWPTDw-Wc1|MWlGb~fsj;WCQL zZrDGWq@Pllv+iu)6Ms5q!@%8{AM4vJ8&b5+J)QY1!A6q&2D_PGQx|D#12+iyodS8r z^XtwA%-jcRk&^-(D|sNxjVNSjEl{{bl2r2RSxvdo0lwIM&Mv z4ijyGV;o3@3^$Jb1c@&F*SNSje6M&Gb1q? z>g6D;m#Ol431Mf-v|esNJzJxMjqzt6SGXe;S4=zi!*vqrw(b%XVHK?GdZp|6&O*kQ(S$ExxA?MHTOZge z@TC>u%g4x?ns$(zK7}iQQzbX(f2)~t&cGMDC)+Ee(m?j zm)4vw9|(L&sdrVNzMU_5gfDsW>+zY$7w`61)B$`sA5W*K1Nbr-`O-?xmsUDoM!K;P z$FZnV@TJk~@d?P6OKbFY8hkk)`4Sxmd>M>_f1-|4Py`kg_n3Lm6eoOfRqO4&GzF2m8 z-qY0Lb_HL!BNbmvJNIMp#a7v2z<%i?;43Mu=rv$-Jwmrz{B+YsK1&wl<52RXd5C-KZ?3>qF?(1@}&*u%a;OQ zQtMqlbZqBKKH*EgoG zU$gSt`1*;A_Y2r~AC#rz{Uw-ZE4Ig~yFi}x#;{g*fjpavLT%)E2F}@$>no#^$hV?* z?bplKk^alhviFyg$Hrl6$MBeszx7@bU#YkuAMb-R_7(+aEj#za`N#DV>GF7=Y~%fz zFW>M$hrz8D)Ok;l)XlzZWbDmRrV8@kLjmI$NPmXt(zdK?oZ23`xDc|TyVRh zZgNMex@p?EAD%af^ixa?aRv6WdU-~cb0Fi%Si%#qTRd5&T^YDZ;K?b3C;vd6e8zc_ zc6Tw?i{J;S0TsNY>!o)0UXP6g2SpafMe4}!S*R~j-8^PwIyzB#BnTk z5q@>S&onyTpNaKyMvdN1qh4lVy{wuJ^>QfI%Twg_62i{3XuWJ|y#E2Im)8^Ct|0Z& zHPO!B*gx#`vbp1Zt>K%;{`yeGi*??%60Qt5_$%xdxH7q)9%Q(Upuv>gtu3Z(_F7Tx- z;mc>pmw$4;WC?sJ*zYRAMp%3?n(jy`yU@e*{itv9h3NYp)2@mz{V3|jiGJ--sBh=X0K%66^6T;S$d{|O$I8wIzFg=8hh=92U#25p+RFLTR_DtYH#XvU z7Js#0A921kI^OppU*^>4?KJpuA@Zec7Vu>l@};euFJ{tEzBD!7cb$y^!DHJ@__C7l zrF4y*FNqF6+pf=>JKldMJ$J^ZAHBw|Thb(aDLVKo>=yV^WVBex7ceKsm!g*P1w>iH z6nwGl^t`93#qA2da7QY>n0D^R;!7HpvTcKvUB8@pk#jKP%Xq>Uuv>gtpnZt=xvx}-=kS}R+zL-gqEMFcZe5odUsUmzSzsJs(hP)HQm%_r^ zzS%cv+P&?J=(75G*2T^tEWa-$`3-g}zgKF(zz!k5+mZY}jQRa9p5K*1eh=8s!zzsT z2kI>#Kq`x@-tnt1)-Y465zoNXV{#VZvkJowIOX}vl zgTKOVp>EDI;-Ty(Feg_x=e4wMf+%a4qCYJ=?N8%igkRjQsGHo8s&1Ng?uX}1BE7w! zVy(hnRxi))*FW)dEk7_PfTRv{f&SCOfoQ zeLPIxkNRf4B>KL6+QtaekD_jz=-2*)_0qxX<#?f9rq#QaqQ1Re&Lj15o_xH2GjiI5rbG)?UuB_By}DyRi|+vFP)j{Cd36@%|F5my2ukb{h4v57x_! zMNlu@V?n&Vyk0`s*;{D6Y-+r}tUCq-k8KO#?J80)gJ0PBt6gKSm(3mTAM9VYrM~m# zv^uXt!j<_4e}&xwSLPcnR&oW*$#G?VOSuA~tYHeSSoTzmH$UHT(_C=7f-BsSiYum_ z`{6o?bcd-STLCM(Ug_S~Ih65bBH;_zExuf)T@$!P;ERLs<#XiA7o0D<1-=a2?;4Md zu=rv$-H|f8uZQXTQQzbX(f2*6T@zvYQPhnS{o0?AFC92vW($02SMOSe`gXp|Cw!SN zzaGB}`Eu{}*wKrCFMXZh@aRRrms!Xchnz1CoiCTVu@T3y2r2l|=y-nx@@09A-cExr zeUUFmmjGWzAYUAEzL-fv`O?&QKSB7imGH$+_!8<|VH@9PJYwfdbI1GlWCwd*S@A<@ zowtL8FAEO-3cCfqEHHXQ$rmsu$Cm{y^M z&FSZ1`hL_m`9k!4Pifain0^#><3zvq7vxJi=gTsIFYW7H%TeFXmj#3`3*>ycANlg| z_E| zeuLf0Z@;!FaGQ|d=_J3u#Qgq>=l5YDzX$JkO~B)}^1DLcL&p0RY`hQ3(((SD?JmR3 zO2+&0c%N+J{ol4ccJ#Rc2Vbf4o+_!EzJtHQZlP}ajEE@v3CzjW zOxqons)BTs+*jArj)S)ds)5gk?kDLc;X>E0lUSM z)!KD|+XbGSN_g@W^5kpIleAsMTroCeziT3HSUfSB?9k?Bdzii-_04)o^nK50*F~6q z6m{c7zxFq*mmPV%+$Pk^4)v~;sBf>AK2k4z^6~yt$g%y~WBUq$W7$q{xUUd6=0%R3 zD(Bd#ZP?pO-PnlZSiC^^#sB`+=y-oU*2{G@dOMAJnT_>w-x{cwqp)6{DzBGjvhWo? z|E;O<{?DXd-b8r2hSba0W;=gf=l;@43`=vz`$Io_zo6^ek1VP4c9d{s;lW>Fx4@N! zhOCk+U`~!J3tP$+5M>Qh)CtQ@&v!9~`Z~8OxWXN&xMJG5AB!s;sg!LStn7LvGso#> ze3?x60(OfpmuuGt?hyFWk?`ef ztag2b=|@pFPV{TFdYE6E!TGXF;7fYFs|xk)d|62NvQU0KUWa_C-yYjJ2>6oY1cy5Z z0bk~hg6)oSzI4?2GRcjNIF3c;2lneD&X-2V`&S`fuB_49Y49Zn`Lc68@MSderK6lL zX3|i;G&SBYKNkan$F`mDB|!L+c-+pH(vfz)G+(HxLt_0yT>EN1yVmEKq`x@- ztiSI=NNBuIw(=x=KocPLq0&{Y8v!(O*`NfX1KM_{U zPWuyj*ZR%vin_@isp_U_=YDwJB+@%uBElN%W%aUWzH=nw$uzSSUL(gHXk|GSkj0DmjL9&dEKzXj{%4K;c@je41HgqeNUa6Qz^u~;uV%j=~nO6z4)^6Qc>Vx`hL_m`9k!4wc3^l(~qKV zoaooKM!t0AeEC4&OQ(7lcWmbinZKW-n7{uX^5w(rv7)iSmjXPUipBz879d}`$obMm z=gU;LtqU66*r(u2qvQRXkS{mZ=|LwT$wmsqy|1!k1eJ zUp5fFWQ?=(CG?SD>Z5x#)k;>&t%Yhaha zm#%~_-yvVV=X^OJ@Ws8~H5Jt@z8Fn+Xud)Z)AytPYTihq@2k_cMwosSb>l?8whi*- zG|rbV1-^8ycU_K-?R+8g_j45U_diCye6~H7IRW@m=mdwE6M!!j$d|5izI4_3GR@78 zuIXjh56w#nycjoleYcr}+V+cutF5}M&GJk9XM>gNRmoFy#3XOR2`yOrM?v~7WV zg#12@x5P&Z+Uy0V|ZoLt>(>HPg15LNf5WvBg# z-D@tmT~Rl=BURlr?c9%5H%~V;#1+`f>SgaD=NQJ5nS>``w|H`ec4OdPfhVUEp8SA3 z`48tw+CA3a_lx(trlYvU6QjuvZBdbj>HAUNtd~UJ_kwm~gy~07H%|0x+hV;ugV)Or z=_C2^Hgv6b1#rV&FUkD<9L4>62$4-}X>~x)9Wp3W0 zrk5WOe(`$Q=y?Awte1Dz=Wf9iPBey}loPhQ6ba}mmu(RgTdfC)?zx#O@5InTo z32(0=_0koy^EWolUN4(F-fw^E+i(5Ykba=fdxnH7Z2ms%7Ptaa)RkNTb8=j1>HPg1 z5M`lM)CtQ@&v%+x+^*mXcckKqY3F_{uAD)oY};UE*DF~AonskaW)Z%C-QvrY+D(BP zfiGtezWfLI@?XxEEP*d0_q)na-QtVUbVo|%Ko8URqrS-(qVIcAyD7r-qo^Av`n74u zmu{Rd-37j!R_|Ji`gXpM`TIGF`TPGvzWlm9_U;Ve%RoGx-kkw_S%iE!L(Z2obiT}R z+q$694cFhBaXo%7^5vcyy`2VM1|nbHy%YE{5&3e4oG%b|)-uYMrpEh0!k0S;U#=#6 zDgCFNFNrJdd};3W_}d#(&eq<0=Ww03n}jcH{yyv$_ySYZm3#qna(rp&{QVpdW!<3Q zi)E+hJxwZZSMY^9Qt`#Kb3Ybex=|_HHdxv9%Y}oS;}~CN6TX1m;>%Uq_CQ$ROEOUhH3QWxz8Fn+Xp0AVn7$wNO}-F)U%j?H!t|r48z=g;?T{~La=!Et z_;Px^YaQy_`9kLJ=P2gyr-WR_$6JMBu{pq(K~8WOn*)5QM80&B^QD{4mzi$(%<=qw zusM9WANg`$jowa!FN2UTv3r0orO20Va=t*=Sp$>h%cq1dI|*NcgfHc%F0_sB8y-9{ zd?`M6x2DAo>^aj2GOM2#4R(%a`8|i^H`uNGzFG?f?i2F+Op@P6F~5K0`CTdG_vrnu znb@gTeh<|5knz4ge;<_T_|y1!8_ct|q1f(ukY|H2&vws)JX?%;cBVYfz&UfK=i96# z@-3~Qv(W_p>rT>tRR#9`Qu5e1>=QNK2WPBJ3eH;gRBSlspY`{B2nmh%$u{0^_uQ*T z`usla{yOhjlDf&}@564PZo(9GWj}#Axw_fX`TIG1+PtbiEj#T`o|k;d?TWg|9jWT3 zY3F`;-Xzk`;u5wc>}B=x;vvpU8Be@~Ct$aD64Z7C?iYA+7U9W{$dmtZo>U7wDUm$S z24d&XmJabSeLw1(^^)lOV%m-f(~qKVoaom&uwI_c>*aW%UUsXOKhGwI%-?6*();tb zLykE@v5*fqHpB_x4Eca#OORt{$vJkG&ac^STNgCCv5fHRxW@Z?uwFh=qqozjmqV~# zh8}`?>A`w=mb_k?$wKR8Q{(-;)&qWwcR^*EWopQD(+ zpN@R#7>cc341Br72~l0U82GXj`Es_LFK6p~nd6rAgnVB@!Iwt={`NTX<*^#Qod#bn zLB6bg1o$!;`Es_LFJ{tEzBD!7&)^Ss!o#_n@a0;& zTQGq=|D(HvFKqrk>=yX4z-Y0OFJMlNFD;$Fp97+-VG6!jc6#2^Y~pqWU$`R`Uramq zWAUXsm9lMvm0iE|9qOFO_)<>z0(Ofp*J!r{9u)Y}o$%!+TF9!s^jFmiJ2x92a zmJjtXeLw1(d?EV2m$X|VOh1acaiU+Fj(jPbRp~#o= z$AB+WkT2ckd@++IS-z|$eAz|#vWf5|yvNR$%;6`7FUz{tytQS`?9e$z^jQ7uH_Tbe z@_Qc1Z?IeWeXVwD;2|Nu&msB!Gv@a%JiiYM`8`hZe489&ZaEq6>+|xav9{=FwC=(%OKB|W1gKO&ogk&oay;CH;H`PN%HL;(trE@Z0|26k6}vvF+Aqu zZ{s+HU#Yk*y+1$1P=AMlvmmD1xgX9yu9rxc$NOX(?^nH%QvK+d4bRtk&z00oHh&*> z3w6_HL{`~PU_Pdw^y@FHK}>0AGs!wCuD$`3L!u+ZA<_J5tq6)6V_y zyh)^=%Oz|}*vsl=f0uI-x&6R=x6*`(bTcv#@cxr8UbAWwefJW0FP`uqNP$@6R= zcI@tHE)Uc9qrO=$iN5a@?Y0QhkD_jz=+}0{df9{5%WXov?5=p8O%9pA&$gxG{WFnc zXNO|hmjlOKP6%iA<-oBO$gy+f96PrSdwaRt)&-4j{7m@8>t&;V-><828BS|!^mZEc z(uMUh`&p=$)39EiE3cPkve0_j)Oddge?TD~)xCtb*OPh~J8I{zYrDN(Hg~-Ldittw z*H8a?cb&I~gez?RKI|5_ve0O;k}F_Njw>ylzn=r5tYHeSSay28WBAPlw=1~99jUlt z+PNQ#D?LmN*$Pc%58CY!_JHgt>c)wFZ3glslk;Vlz?XCCT|7|s@jjWqpQD(+ ze-84cM<~{99q?ti6C%-V9q?u4DA?{H=SvTrFZ0|ydQC6OKkjUM6X#2#*W>lbmltdF zb{c#cj(q7>3w)W5eCZ+Qiam^pWhgAsmlyPR*PBL&|qJNLtN66u+O z%CW0pW!FzRZs%ln{Zv8t26l^Y*K2nKb_>@}nS^h@A>XveDtj4fB?8|jNS<#5O?POQ zxjjtZkNPIxh`#TS+8q(5A4T0b(XZ`nt zD0Vmqc{T#`>~IkB>@v)=^W=F3&Y7^DZwr#hx2#V(>+LmP_mTcPvfSQZN*)`BeS-eK z0Ai@ONx@kV)9tC)a?U^N`v!TuPqyzHc6W2uf7S8SK6PHFq;9hN2f%KjZo-ML>?bfM zS2tUF{{Vh76J>uwqh$Sw-D@s*e=F)HcciMDrk(rYd6P(Yni}E?>}B;buh==2@nj+4 z3D_;3RBLwz_6R(25}u?WPf|Hgss)~uN}i_!v2$qtVh_{zqrO=$iN5cEc4vg?M^QIU z^lQ6dz3j>BZ2Mjqr<)2O9mp;ULz_c#Ympqh1zcW+YyLdN~v8rBhxn@u-gdf4ru?Z#YEi z<^6=WTS&dsZnN_@criZG8N<@t?;Fw|eCw}$`}Vq_&f8PM6?Xps*e!4+&rnoy1Fh!lP?5P+pe!kM9o)N^*k+OQEhw1xK-{cF?_rwP%K&vd>M&nVze6g;zz#p zl=G#h&KI9s))Vr5M-+T%^!tX_kuQI$(c5Y8WhC+?dI0z`3;EJh&KEOjlD!_!z5oM) zhw}j8%T~gdj2G;D33=>%Y3}z8v-a)z`}xt?m)3cEN%+F<9{{@rzT_J%R`LbR$?>J7 z_YcSgQPwa8Uo1O4?`bx1yMiy=k%}*-o%^x)0>f^q$G5@Cu3rj9Ij1weR1&^`-Qvp@ z?e4%{fiJxXUs@qwT64Y}5cuMeJl_an=+G`7{r&-*Hdwa^#2PWhWa}coIRfV2gu`n zvW@rKK62v~ZKgf?YMu82N!?`k4}jf5-8AN#tG_1)^D+I@*#8&gw)Fl1%mKa>{b|{G zUhsOxKggHduBe;bk*aQ*cJ7DkB+@V761FAmW%Y7EiE{?y$r8d7uv9~4lPjPVfud5H|r(Q_Z`yii7@>r>c)wF?HO1vd-HnP z;na~_zoD1nc{;h|{sC-TI^G|M92*>pRox04D{+GNs#}3$YevEL1#*sEp!2KJ&0Ey; zvim>e^|I0N{(D$2|5~HB)2NpvSTC#If_mx2dU=7oUP9Pe^Ju+nYP>(YHwFX`?P0>( zn@GKM<=FWfd(~bqn>*g$chxheKk-J&7j@p=60WfO2f%KDE5eFo)N^*k+OD-hw1xK-{cF?_x(w`H^TIzs2eBxwcU^}S)4E3 z1-_hL?^=OJZy)cI`v>GI?jLXo@?}^kR<;xPG6qklvYo(}0P>}`oG-m~zASdzx}edG zpn@-rUXOo>eED0A-cExrV~{Up?*d=uB42vT`C=vwh6KZ?3>qF;L^^5sI#mp%etE>Jw*D3{zn zfNe|1`@@khBSJCPF5t^pCxp|r3;41Y`I05)OP0=;C2m_6G`jI=bNKQx^5vr%y`2VM z#v)%_9{^v!&O7ew;CTGr z!V3*&tbPs}=bX**dpXH(uv_`PT?+@E6!QB*lHX~V-|cvQR|@$(UGjXLTw`uK8Sm@& z4*+H9cz-13+2~L#D-3xy4#S!ihCEw`d3K>Z&%il5c0J#gCXsI?pLG_)uRTKguj*lY ze<{cN#$lhRzwd)H)-VNUEqf}qp7W1?zmQ0m$NOX(?~jQOeq_a_o7jE4FOt+vcK-m_ zE!53N&o2H%n;dzrtzsS@OS70xz zmxISU=P;hEAUpxP#gmYBU*IW$Cl?W(v_qb>=RBzvcv2>Lo({y$p=}uNVfud5H|r(Q z_a(IZB1}Jux^bdkdp6d~i+Q~qFVxE{#q)G>$^8S^wsgEd201n^6w7!JI5yr1(ad-d zIJO=+c9EQ87wP<3<`#RP@%6ogU;O*=M#uY~W4-)Wjowb9UXI6lnei#q%lTtL_ab?{ zgs`)>(0bX_c>haMFZU4M-c0Ic@K1LBY8Cc++1&B|Z7aU$e?jg$FV=Z4mT-mLKLByLQhHM3_ z?0RL$rA{y7%Syr*uv>iDq1_*NTHwpYgfH!pFAmO^-2z``NS`O@fk|7+ySS2cP&4Zd88d^!40;L8H! z%f)iOm`OwV($sjr^g;{>w)#=Rms<#5Lf_l@lCjm!m*!rNSAR3{i_n*g>*~CHBz$4_ z4}jeQUltfGR`LbR$?>J7_YcSgQPwa8Uo1O4?`=uLHn?5E7w$;K7t_xDSbXV2rEJ?^ zW!Ep4OmNO+e5oRQ0lUSQo3#f5&j@_!L-^uAzI5PxIUw+5rsVlX5JQJ{{bvja~+ zo?VG~)>ocq;G7BT`L;5Ne0!JV+hgf^^R3rDx4plVJT?ydM2+{s8Eb`tvz9#-o67lT zjrZm8KH0|mZ;y>_JNn(f{#)nmC#jq4{sFLCsGB|`BFcUOb8>aFrS}iW1yOZ>T6Wr> z*uCb0+ZA<_J5tq6)6V_yyh)_@Gd08&*vsnWuu|td#*u z`zW5LlS}R&z_z91{TaxyS)tgzr-5UoPB^-KPXouULXP#5bF5z*_Vy|_k4V$YA%{EL z-o&rR8y)Ze59{TRHF`UZdRdC~a^H7QFBf9H>?g06X0n`$>imJaO^x@jxCjG+pL?9} z_I6S)V~g$lb!n-s#RG2cc)xA7v)^~qzGm0s{Uuys_YZ*G0#_CqvP!OiIXSMh^!@?4 zAj%r1s1uf*p6_A|^>uDnaD_WkamBQAKNeT|Qz_dvSlRW8Ym#$5<3ztU6Zw+O z`LavkOW%6e8a#UYc%R%qAXjn!fH}yQxuMw3=YTJh@O0Yw9Ps7pQ6SJ?&X@i=UoLao zx}edGu!1j*j`x2>zWh?7x6|OuB;?D^{{mkYAz%8-`C=vw&YOKKU-H=eeb{aBC9m=M`)E#zFYGfdo4=2uEJ5Ua z5$yE5r%A=_a=w_36nwGl+>gZc%Z7mXE`cc%46aCurkS{r$FM9>P^iw?F zD38qFXWP>8{ygN%f>12@Jn+Tigh&RT2fhT6FWGXwWb1rc<>t|EdfED4oo#R8*DsCo zB_-_AKNQr$dOHoic#to_pMWow$d_z6U(BRQmM_H@V?eOg`v_ll623G%XXi`#gcHM; zp1-yI&zj8kt#gcau=?qq?5tq?bfMS2tTae?Jdlr0h@2PWuzP*IaPBqHc0Ws=8^~xgV=;a+4`# ztiWDYFGoyq`WR2v5T1bD;>k|!kwBfmlU%}+PRNtaoF~-+Ps%0F(}CDIv}>n$n7$wN z&3Z}neeY_IM3{aQb>l?8wkOuhJYFvk3H35t@jRV8GJl_KOUL_*LoOp|OGB~Jmw{tb zoDk2_mw{u~Ajfj$9Lv@Dwb~7zIiBB#2)}r}Y;?TeChXFE2U~~rb{h3^3ItLsZPlY& zulA>COR!$%%IhV*JNduwcj$uw!Gn8>@b<1wXyA)eme{@@5AL$p%jS;vUz_rm&ab4V zvHOtbNw~u1@563^D|tqXlw1LGa$ITY{QW!-Wero*3Co^}@#N<_4=-PGyMim+k%}v( zo%`WBiF9t_O7fFZhqF>t!`I681@`1pYoO;)KJbc@DKab4cXWP>2 z@#V;um7!SCtH773csdom3VgX1`I0B+OPjH0%{m!-&;JUL$=?94~9*W*=$FHaM`+)enB(bvwG(93qdGDoMD z@VoELsq^Mb_`>Gz!)}2u`FhBd|IeQX=H&R&()s%+%9^F%3p7f%)AL?G7V*XH3cheh zD!!O@?#JQ_H;HQ2I#}8D%gAZYMT{?N317f&@#QY<(ZCA=U-AiGx*%V=a=si8_%dJe zd?SdVL%VL8hw1xKe>HC*(f7TlJsM&9QPhnS{o3=9F9n=0UkZH5RXpD)kIdg^+fu$< zhJ5jdVwtZ2U#2;srJ1h*Up66M^5uNV*ZH!>&5y3><*m)(OS`a3_Z&+Ll7G7b5X z=>WbgL%!t8`2t~QK9c3jTZAvq5Wegpe94?*=S%qW6T_FWu4mf*X~Hu zRz8xj`_AKeY><#w*QeN*t3y$G2AAhrE z*gDJ%=VHc_^@Jy2w|KHkdn`~d@T8FN6|BN_gR16uaG=X2V&>Y zHkWyrz903?dP($sA83z7n0^#><3zuQX)XNoEFAL@M(o7awFPr-N{;s|l5N!2+!rOaEy>#7g=Wnd2)d~6g{^j5O zh&M|(W*JAp3)2wzS|zMR4Nk|prP zCwZO`#L$saJ;THF{itv9h3NbKrac~E`cc%46aCsO1A<5O9N|k1;Y(@U&X>eoJ71bR-tV4Q zlmFDDE-UK1MH0TS`TMY2;0sJqSMmkS$?>J7^Y`=kw0RX@EPE=(o1gc%X)d^3!58jG z#TV1g{cxQ`Iyae8wiU3l>z6Szol6;Ct{{8?yTzA#wI>2GfiFdbFJ~ZMx^cdg2z*&6 zdA`Q&m}dj!c?Ql|4(R!|A&GpeCizxN`mgGYrMAB>DR~Sn zIEKf3{H^Dw_)5hM`FP(v9L4z1vU5M2e_St-4)3AyKH0|mITv-juFL8+g>~LRlDf&} z@564PZo(9GWj}%Wn0_+H_s}o)@qSC^@8|Jp^QyXO*=c`@Iis(0yP|G#N2@%|0Sv8|z4 z=wsm6Y$t>@^f7R33vz6boMVG@eqG^~c7=53mxN!uUN$=3KP~Liodvsw^>!NdayHh> zkQ3_VDy)}-sUy6-pVZ6XP&=s|{)1C^vEbwJ8;mcXbm$Nxvb_;x2EP0*~ z#L$tlZH|ZO`%&NI3(@y|q&*d3`cc%46aCsg$d@6UFS7-{46JumVi4@(eKLPPPceW0 zM&!%(P;BjIz?V5raJcp};LBFz%V0TQ2J3ve(k<%=`M%PxI@=7KFO6Q0pAmNH9}1ox z*4t_DWe)OX?FGOWKk{X;oG)h5P`)%Z-VYJJJWu%Y0O3o>XXi`C&vw2vcf3C&`eydl zmv`J==N%&93!A?Wy9K_$6m=zEz?>XkS~`C}4@6m4D)?gA>3OgBzxd*I1z)%$6<$Wdb(1?%)lJjR{qVd= zqz~m1wk7Ok^>Sjl^D@ShYY0!kZt>(n?b*N|1)dBgJUIt>axUjd+Wo~`Ft$wcJR69e zL)%{NVfud5H|r(Q_kF588)5oU)QuDU+WuHChw*y3O{kYc6wkBCBlGv!wsgFICvxoW zP%Qgf;8?j6jxPIK;Mh&bv7vH~4Q<2T9(41FG`+ln@QYuMH#*)wC+yNa2fK&$b{h4v z9P4FvF4W64STBdl>!q12v|ctf-hUw*1A>QEPk8$XcS~`C}4@6nR6kM_FsTgm5zT>93;C2O9xFZ!; zOgs0(brR{rObyuzSlRVT={)Bu#+PdeU%+nh$)8p@ZZ#`~E$7!W+1-x0n<2wxH@%Ua=EjYa8BJ71c6Jzj9t zD|>J6w0c`r>yox%hn7*^wdjn{*++Vm_R8uVyttviE5F--s=TGysA6{}GlucqslprUysl&Ume1}V z0J|-|<-?R_72nXD6yMlqT6X^c6lI4a=bK>X$IQ7Sguqzkb~)cnM+&}KcJ7DkB+^|} z%C-$wcKtMIzSGb6wu$f!>=xf1)}9L-5cuXIeCvUH%jA435%{)3@_Zv`xbGZsb2ZEF z>qvfs-OBHX77fIO{2os7I}`K!Jf7c`LVmB5JYOe2M{j`r`-Xh|{sEvY{l4KL%(F-+ zcK8>_vjv!Ehkt=Q+ktsDT%Kp(oC)jsc5M>*wwvVJ?@9k1nP~4XC6A56K0)^nfEem= zRB#r=bbG4)F81R5v%YVT$NOaazTwH!`?qo2ams)?uUk?#+5H1xw@^3X#8>tcn3JoU zExmt0K8KO2KP@}$Pv~9iH@7S5CU>N&o2H%n;dzrtcUvOD8ti5Da&m?9a>kSE2~WUo z@npAF8#pNN#7%f|9`eM=c~UL#q)PHU9f+MnyQRXz^!=!B)=Q%AJFL}4n0^#><3ztU z59{R!UM~*`_0pwyo=!fwe*oK-e&6s&$Yq4?(NHXrvUrc)whAW%Gm-L8+ngubXg4Fr z+;Wb&b$)GvgMtq>@w>y3&bD{(dfDjr4d;hl`iFwO!g@Q6dRc*)kr)p3ay`~dx4d4O zEu{6bsqY)c=VCyx)h`j=?jiM33)uM^OmEW~KWZ$R`+dWV=g#PJ&t={F)OklpxWeuq z0J{aQ!HO!xwJi!YC8b%ECezKkGzaUx%Oa=z>q_;Q)#c}5UJN6M`}57YOf zzR4G&@B5cl7h(EQ)QuDU+I-|oG3Uz%0$+yLyQ(k|nzJNJ7zO?lI0r?=x8m8cjWvA!8aWKL!ZddSyJ5uq*v~xceU(7#O2x`_U zSlRW<)P>Ff<4ZN+3)n5b?9rYN91{3aO!(3h`O=H?<$%DKRg&i$K@1(*Z3{h2-;esM zc?*fY?{n?>2-A)f~QD7yj`yF%JbNw_t4@bJTZCb) zPKP|Z4fAZIJkP*6JN7p0XLAzyR`l=A*mJP>1L?o)FYW!MljbA5T1bD;>lj^#lY(VPeu`* zoR2)Yfb%5n0h7J4)spAwK*-;#k!QICeXRd6b-EqjY{%yGbWBvbl@! zs|&uj(eZxYuuFeiuuoWTr%^8}v0he9f_ixs*2_`ydWlDI?DykMjrTtw_41E|w~v#0 z>FTn)6&{kYh&^Jjm(3mTcYJL`Uf$uhi|V|iC0t?m4}jeQS71uBk}F_Njw>y_e?Y!D z&04_~%TCXCF{b)Dw=1~99jUlt+PNQ#E2F8DZ5yoYdZlc!b3NnBR>BvsTYPy;s}H;( z@MSdN%LT}n-kdL40$(ndJkJPX=t$YQ*u(VwsBiLx==;9X>LW})in?*4Ut5HHDdBwS zF7Rb!y^A}xkN3&_1M(I34|oyz^1D#1>@?uZVkd;N>@?uZ9e8Grmh)w_&X*h9wk~LV z<6Han5$8*z|+Nt5jLxGNt6f=Bf#;mZ?* zFQtR*d`Tqid};1@|B3e&eR}?rPbTWTB@(`{`v<^ofiEznS;-eLC&!nT-ajDUoMx@y zi)By6c=PifH_ZjNEBL}4srX{rxgV~RNG~xpWGi50*Do`cI5#l9Y$JRDyTzBswciC2 z0$)l9UwR{7vN&H#1iq}1Jl_an=+N$5;$ixJ)aMw`h`#S@?ROETA4T0b(XSndd>O;} z(nsLSD8=)Q^2z-J*tT@M|9j-i%b}R-OyJ8BJe^!;0$+9_UrOYBDbe||#VzXz`M!$g z@Fgeg(!B+DTlxKj77P4Y$nP;Ezq2sEFXZ`MDdcxR@_e0q zW8OL$@9Xyu0A=ZT{|}gFuZCh-=Rlq<#XQS82lDJr%(F4_JOk(K*!6tdnnb=GAo&(2 z{a00O?=K~fVY>b?Jm!CIH@Zdmm5S@q`2-L{{T&L<9?$&)EDIBCG5tFdx%Tjm;;>Z|VI5m;-z%`qQ%Wyx{eWe~>S^T~Rl=BURlr z?c5L7Nu(nnxezX4FRPccmN~CvJh_SR1nd@1_G!NlyeaTxEaAz8$dijWPpSo;td%@Z z2V&>Y?q23$`hL_m>m||m9npRtVfs3Bbm z9D6Mk%Qz1>w#*5E%s3A?b{BGNtej(Gb$)GglTK)4v(I;(ZB~9g-spJ0AnejV6wD9n z?KJA;GOU*w^PpZ{gXu6DN-+kPuTgZjXI?@e$-ercf5c1 z&1>#nKjA-P>b&D5Tw(VQfZYOD<{NrSu7Ei?uC(<20r?=x8m8ciWlzPR@$(%w%>}nB zxWXN&xMJG5AFh)~A7^UFR=~=xS7tAFUd8ybo$v+h7GIv!UJCq0;LA9|my3`u7jwSs z7WlGG@;oDmp(AD2au3t@qrS-(qVM~+_ELoDM^QIU^lOJ8U&eF3%og}Errxy{gJ2); zlluqcEAAigC*;c;q1e%0z?bDtaCo#A@a1mg%Q!h-#_4>y(Jku<`M!{XFO81(2ZUX^ zw_ssdZ>PbR<;a(#6~LEkks#ybd@++I+3WEH;md1;FHaM`gx9{{@rzAP|0K*<*{C&!nT-ajB8L|MZWe6j5Gyf+R;_{HrCzHmn> zzL<9I$KuO)LFL$0u(IoyIV+r3GroifU%+nh^-lvJ3e#Ue1^CI$v&bAKMSAJ>NHnF9X9a-CwXMthdwP%L?R6 zLnZKK6Y^!eoG)h5B+HllLJSBV&>_N?X9!=yzu5VbdFzSc%Tw=Hy<67yhksmZw1d@8 z?@DKo<@XMf-(a`$`)Tc!!2b#PeJRQBKA7Kqd43-j@_U2i`8xR(dJD*SU%!6GjPs?^?ci&M81`M-&t?3`TA43xoWKJ>$Rel zxJSujxqons)Ao=S?Dgf~g^{z+P4_ z=T*Y3~UXE8hPbZ(;KY(pZ$NPUpj=dj>?aKy^RXO44 z_GJUd?nRDGkaKK88}{~)n@6PStlPpZ0>k}SH;JFxSz0NyP!WDM^ z0N5>XWub9|O0Ix8Ij*$y{sH+Q${MDq6PBHx@7P`Dg4-2b;f_>XG40%s#g&PshHM3_ z?0TjAGUqjnFSiiBfZgKDv)Ug6?+ARENchqZ`O=^BB}?GTm6GQfK@1%!;mbTs-;eqx zUx>c%2knm$rXNM!IMJ_lAzw;4Uv>$6xwPIjAA?{Y@00rn8^uA!+JXnzFdZU*|{3{ zvKjd@QO*}LX((Ts`hWiN0T>WGwl@f0o+Erotg`c^v|ZcQ_)%lg-0^>zOc`%C-$wcKtGMmGfH0ms<&6z;5wnzxHb2U4bv9gfIP(FWH7xJPp4oJ@Fk3VDV6i3ROidhZd(^Ly0NJ_d>I~g=}v>Lu-;CCFRPF*!L`7b zYUE3)oG)h5B+Hjq2wxI}FHypmhCA(iDbGGJeA%$$f8UPWc<+mojOeoZIp6Qx#Pa(# zlHXvr^7}dMK;W-JeorF#osIdO!}GgR$nUEq&(|r)(OY02?-%Iv_d!`Y-v0;Y**`!g{{ll0?28Ci(Vf(tk%{_Wn{`@4;c8pz%IBV~3*P ztPx{oZfyR(Jl-eUc>jk(i>{rWar2HkuSZfh+5COjE!0go@s<4q=H%*TOXu$wAdHm# zY1vctUS%&{&tS{?&FzZ1$sMWcrfKJXxK1M7V~GfBu$R@#1*@IcF`nE`cmj5dCs8dP zcu(Mohwvl^d6LU{QZ4W#D0!X^#Ll5Tu-e1){itu&OQP>Ps>LHrKZ?3>qF*}#>*Zu# zFAoXzvQ+UrodPm{pKVLW`~M2LjL>}%ij}&6W2>DI%u*L{?0)2!N6s;i&aYeD@R{TJ zJ^nwPZExe($zA zgfC#X_)@DK47@M!WisJQF7hRh^JTZdmyMF=89@vkDGy%mVfud5H~B*JeLrdkBTPSv zx^bdkTa0{}!uj%nz?Vt&u7wx{+jzf#%-?6*((CcBkT2haVnxNkm&@^VDk=uPJb-+e zEa%H)oiDe!rClN2xlh5DM#uZ3!!F%fa8y`tr@@!YkuOD?fG=B-FO%hbF_VV!rK$1$ z5yF?h5WYN5_>$3arS0!Ip@-~zY3_Lcv>*T8aM52jwyX0_k?@7h--q1-U-FIkEBONE zT$YFK$=xg*#I5#k6xj7GI_aD#xyZm0iF1);KpazU(A? z0lUSQI_!t|r4 z8z=g;BatstIbXgM_~KDK->87h-)GxWzWf{c@?9vFSps}nuTx;mEhpoBef~ZuOUL^^V4nRq6iXWqc^1I1rj3U@ zdkFJvsyxrYIXiYe-|k2v-`0|R`#;iu*>mjurR1@3*e7bd56;-3C^&1`Q;q(%#)sD5 z_aP)S-Y465|7f>1*@y2sjXg(qnxt;B`TMY2sGBfFUD;1yPOfgYbpCz;pEj@RPs>jG z6T8=3aJ!;za!0DVY1+9Tt8Pv+HN+Lz%j)H#wayzDPwpZ-0lUSM7qmYGJ`{K|jqs!Z zc~Z!ElJ=1G_x(+h=jlM~99m?phw1xK->jEJ-}kfjrwG%JqHdh%*N(<|Ii1(b4xL8w z<87Ftc%DuHnZM7rrQ`h{kz+rFVn-$d$JRQ*`;m#jv4@dk)8rhRrt@p3o42UxQuN%Ol&NUfzWDa+DpxHZ|tlz+uz@sJKldbzUTdCJB|KJop-u~D{THg>=w8JQ`D7Q0dsO(Y3cm^0zPeC z#TCn*it*;>J8qf_ZdY)HJ5q7Qv~xdPCy_qg)R3)!m0hn?u5)f-e7T$O1?(1IUesO> z{9WM7bi$WH|W<#`hL_m`9k!4zi6*Vn0^#><3zu<1o=|N z`O;nB%hY<;5)6WUyiexu7bxcM|AKt^EfjlqGVo;`o=)#h2EIg)FVp3GnXdEYPPeQl zC4+STL^>!M3S%-Xi_ZHyGcI3-+IbY1Ap?qm-ydNZdd57@j zcZ4sckJ|Z?7;fiFbI1EHE$%w4-|eq_U*|28@P*Cahus2SV2ZkuFJMlNFD;$FU%;o$ ztN3Es>3MHU8n(gh3chehD!!O@?#JRw8I`hagOy#sEMD*2%J{O2@CED^U+T3t0u2IR z$_QTuAYY0&UrGeNTrYXP5ya4;J+j`z^!=!B@`dR8e%0QHF#Ra%#)*FI805`ct2%_%Lrbp9kJMS;LCa^gfliB__7=MQYPn1na-EH-0+#>`TbyX z_%bQ%(tQU@!+JXnzN|;S#O?sTgpe;~a=w^JlPq68C4708@Fhn0QvRl$FAbF^hA%z8 z{qI>LEAGBzh7n{|KbLH9Ze#g<56N$^TlxJvEfM%g$nO~>zl$)x2lD)`6!Lqs{jp^K2k?4(-t^JWSt@ z`ewZ(`aVtjbA;(fQ8!NXYsX=|oW<+qc%fdFDV}FjK<4kWZRvQw9dgXEBNmzq9J|5^ z(G1N6j_pB?&6IO&rp~WjZm|a%UtdP}#jnR39q&&KyL7L?DPg^xM!mcO>t(10>gCN? zFK5c@rI{?WUN$w}-#Zusf`|1U;q6PLUIsI(Y=3{#p0n4>=8pIO`1+X7oDH{sQ|Fx} z;R>6-54#1f%r{!B;!+M9t-1is87d>Mp%8O-^zTj0wLlIICQ3>_(Z zukzJx~G`I7OWoiEKD@4xxu;_>gboqDv+ zJ6pmRHh&*>3w(ho>Po(VIXS+xbpCz;h_X;B_+r^pG2Z;V$4zs=?FzndM=HLUcJ7Dk zB+_S_8nP9zvg?=SS2?#czJv*1z;5y7CG9VPPX)frCVUx;d>O*|azNnA7RmF4AchX@ zv8y~x-;eqxUx>ahRr^bX=|@pFPV{RpMZV18d|4*&Wv1f!LIq_0KHHX#_d6k9y6lLR zF9NbBH+tjj!8{{=o1c;Y2IxdeGKl=CF*;bN{A+a`IQ4aCl&JrVRUeLw1(^^)lOT5JCoVfs~Z9nSI#kS z8}|0Fn@6PS<7IkL!+JZ7dKtuenf(OR%iFPDdgb-fOcq)%n;P#A zxda1(hxQ@i?H@_KjJ;szuWPcsUN(2U-}~QNe|+lncYdhz&XsV5&EJRJ0#_CqEmm>` z%*k=3rSta-K$JC1Q70^WD#n|i@3?6$xLv^&?nuQI)6V^HokaRvQ$w}_R(8Eowb6Mq zJK$NOadet}~C{yE5(9y?;){J@uucsh0S17DsP z1p;&Be3`5B!M3*@%4U_B8P24&=*RIbY1Ap?qm- zydNff`8(mutAsC!PwadtU2o@0bFas@MH)UT>a^yks@5fK!wxN@ylc@Jp|g+lDD9Qi zJ9u$Je^-9D0ab$&!?KIhMwgF^P0XBfbXw`GeRDG^4li^q4X@Nzzq=-TL+EOGuULDZ z-?Q%z<3|;{Guf^%zMG$e`n&49<;U`^kljB3c3XTa%+}wosrZKGr1-`@)3W;qpeSpW zoNt1iA2a8UaWKL!ZkO}Tbfn;$W#@jlP9nWrP&sxLtnB*fvTK~TFupxV_y%^1Z-3O@ z2^<#qR!;af4Eg5bd@B+7c9Z1!M$mMJ_T)7lrte36lW#=dcZ&8-gy~07H%|0xCn4YF zalY*p_~unS->8t>KY(pZzi&7X`POqsObYrfON|tY}hw^~H=Nky$4iLUIw7;ws zyc2$x-*;m8cHVgxXrbG4=ffWmS^Zvht@Bov-w%=e2D_EtuWIiG{w3u1Jd)oo%t}mC{fPA6ksNz}DS2!h_6hpF0b;1fQNdXd)9tDHyV#5K&zeslkN3&; zeZ!OYED5ZAWX{Ao?|ez!WcLq%-9p`j6JOa+U{0=Xw)Fl1g&anz{ZLZr&fnlS_Ilae z?;B3LvOIiX_t!`2ybB~;VfPP!-2zwg^pL5q$HAN&S6X`ifI<*u%~I3}%bseq-l`M0 zX)d^3!4>XE#TC=e{a9S#e;PEUY%5@8*DI^9bKcJQ5+Qs6yTzBd_Fmu%fiDXPU);!- z5u7i(1-|T%JkJPX=tz0yIuFzLqrS-(qVH>`y%%BnQPhnS{n{zWmkQ384+Or`F%ve zmq!2ow!M3xeoafeF^w-H}YkHoG)h5P`)(veM9yz3=u3s*{-gyV(%WlFKuv>gN zsJ$QfQs7Gk;mZi*OEKrm0f8?!OP+58F?49pUhiT0e$-#h+eq|%?X~wKOh1acaiU*4 z75U=heECw~%Y4Q2jS9*A1K75dFFDAUydAOd^}v_w@pKAb4}5tB`BEY0ONGvthu!?> znqE%%rL*l#U2v;WzAO&A^tS~o!+JXnzFd!d3I7rJvJ3fAA?J&kG|BSiR>GH0311Eo zzGO!1d}}XjB#C^Bl6?CI>A&pP?fs?XF-++{hR1yTZFGz9D;3wJ&jW-Q>hDl+_IU0eAdmOS zHr_wuor=fij!pe@op+(6ZnFCaz;2;#78;RN_7j+o=_mdA%lHiROFMJBZ+3QJOYa}R z9N)h z_P4;-0#6nao{U7EjN&{=ixhLg*sYT1=|Jop+H=(&rte36vtAN?UkB}P5vCtS-8j*& zosRW#5wDjWI*;V~4Hb&#=@gRt2e57Fcz+;rZ19d))lI;$YA1NFx(PV8e-vyllyhvM z&ad5W-lC?L-G7zW%SOlh%fl|+b8uN$Z>Lc&tFc~Iy$<#AUaXf3<@FN6&V0~%+0=M{ zwhIHIH-No9BfNc`)JxY7cK*h;+UsR=$NPOIW@Y#4@9I?NT_oWOyMF-e7PvCNXs?nh zU`~!JExmt0A&9buDY#}KH04R|`0-3)wr4*9Z3&X+|x zUmkJG`$D!isNhSZ*W)Y0F5OpfMObgA!IvA5FJ*56Uuuvqi{yMUlZNu8squb{@a3O` zFK-aOly+a$N<8kwQ+B>Icf7yvNcfu@C;juxI&YqF*}$ z`LdYvrH{aug^K4J6_Wc0ux;she>n1G#EzKjcHqkvCxp{=JMblne5sW4rBdh19=ELv z8r}G`IefV+?9!bEtHOFa4ZdtazPR27zJ!r4m2$qANs}yJdJV^b;87hWd`S?#l#j6U zrQxq9hA(gT+dOFJxV57f8|`5AbNyE5E|%Yqk^Ba`mEUh@4S|0P`MsFrcM0bA7@psi zLVn*NdA?4eF}IzJ_x1Y+fU-J?$Ax?Df4N&o2H%n;dzrtUt(&AE3lW<%MII{_b{G3PIv-#izf-~qrkTU zPnHm#j6t4^jEJ-lS;U@%6og zUtRF+jgI#(54-fY1y_glb{h3^8*(P&L#UVcj|E3dQhaK*BxV$k^cj+^F!+Z9~lj#OMR?c5L7Nu)0|HDoJbW!EcL+~~ZQ@#TsCr|#_I zteU<*er77gZ5WC|7+1)1NHcl7M#xvleDUQcqoyt)#ErS0FG2`2BZSf&rMc;8dcJq6 z>87W-J>DKv2!qrh8k9;Ne*2uY);asE_MYj^^iT78>70Gmo%Pvg?RDPk>^t^3j4xn! z__E6Qw(xg}FZVIN{0I4RE%7BH@#S5``-~ulp0wGof3x_`u@X>Z_9*!6m=6! zKh_iZ(x3R!Q{u}#l|K5zl6$_-o*&Rz^ZbD8kS{kB$9I(iUtaS&kumZ5?|ndY0&Zo z%&GCEvCj|aZ2xZ-8ooI8bc{D$_t>;IWY_S89O?LC+sO~W=)}#A}FRi!qOds~^(aZ0*!Yb3Bf>_kFr6(A~#5 zcJf2~BfVsXI^GA`eE+Lud-hy)*f~=x@*hytP4WByI4spozZF^SIDvVuajJ8^-`M8| zbOuq;F-_fc?0h^iP3#TXHFc95>FTC!CqI-o$&3eV4S5HSihB7>k+)RvWD4U6*d3m% zHNG$WL*mH;j3?J2Pp&7Pw1{*iz4&m&`*a|79%F7%p3o1VzFjYwzQ48ceVNdYqHcoe z$L_>>`5@KHAyU2UuX&$NXZHL6(N@m)Z$pmVUK}r<030jw!r7Hi0FG56#~x5~?15(D zPQCYXM#7xHBQ@nyKgm-{Py^oJ$)e4jl(ptI)r0e2u@?ktWEn+$v@#>;8g zWZ=u(t02k`s`>Ju$(PBws*zA13v2jN=Y0Q}u+Q`sdOB?O)8b1p@@3esz?U-Q%Y$mZ z*h$0rQr~>Pn(<{DdPl|@4UMrf56^+$rkJP;jqJ( z?7G+Qqd6tMh~H`4`h64?L#gJAWarntIsWnw+0}fp9clRD*vXH>mjSZOUi)Ab_b<mFlXNuJOTpuWu)rtd$> z*it6+qo|u;`mx@~mxqWi<0Za4pn1PhHe0_h+RFKUZ{$m#;&^B}@TJ5HAq`CjzQmC) z1JryOVDe>3?q0*7J)T)y&&xU>Gr$eEz*-wivCCHc14&X}!`7%Jw7dvTE z1;`P`e{(IUU6 zv-}3Tli%x&n!?>uem}(Wy9eg?jg;U0r2HPKc)w2exn>XC^Zjgd{XQrw=lg!lv%8Dq z{bG=3Z(>;c#URh-VV*st&NFZ>gw1@LnnJ$yWBK+g8^78Q-Q%UZ--FZMPwV&5nK%^< zXRR0ub6xBA)$u;i=KKAh_~FP~%EJ>Y@*h^zO|gC-4oh_tE`04cfjPCh+1T~_*$5-; zcsllUb5z9-)iY>0|B_u(H_4H%ZrXP8LpsTfhaC}V4~~j@`TS7tM}j9a7*D|N@MMGW zL*btiPabAGxeozo=f;Ip2RF>@(el28GRjTJ>`1c_7r|Z>X0aW4(M>T`wW*;#YXRtZ%;G@=6Q{ z$Qi#e-hO^K8fde`&EHTjcfD-re80G3OwW(cM30U$*$oFInr^(wv!*yNoE8{Lb(O2xL+CcmUoQc%S^@> zuseMD%=oczkHnV%<4Yd$<-f$2h{Tst#rupPhMu$qZ{-R70P5R(Vfy}~jUUT|eiU^R zOh48a`7)6BvQgs8LzO=I!;)*hpUu|qi?;H9{QhE}6~qUN<2mzyFK^-Hlrs)R@5f&X`^;Yoy%;w8Y4PPP;NRuWr5+z2xRgL+{7?A9l{}7M_2vugD*$;EPzl4~Heb zbTUJx{d-b2m{a3RW7qGasOXl4FVHE|&aZp(T;w0JYxqKrbbPVxB-NdW?l{=L!7)>f4=W z`u<~#pUQ-O6m=6!KjufiJVJcgD)HrE&HIhA+4_CaR`TT`+i+kiTj5y5B>YZm8(6KpFd(nR@BcI-|BJyhQ@?>>bRr7756+ zg_viLsPhb*i*q;gZDtDjwu|N44mN%rFLRHV_I}Sg?fo_12WR3`G@Nzp>DG8V^F!zF z`w$Wy?*nbVU%7qz&egMzy|yC%QAOPp>-XWXR5xLXx^|qvoLb#%?E3v|S~jm6Psh&3 z6NlH{kX=(Z$&s#Z+II5e)Xhh24S5HSihB9dFz;BwlM2QYusb~2X#8CGkHnKl8BcCT zp4>t_X))IM`~GOf`*a|79%J#aJfR;zeVf-z-`~#oxlHIsQ8&T#V|Qb{ETnqbvelJz zzSRRY@6*X<>-R-lIp2Q_IrchJ)5?2 zeTZ6h&i7vn`^;Yo4GEk5wCd$Bte3Sdpt%iO{oz+(K(N<;Fy4N} z>ZR{4H-F=Qx$9*^=lkatY)_27qFJCKzfi#yv3?&8OI(2^>RPUVIW?{{cKv=fEt}VI z#j&Sjyy<$!roAD%hAZSq#}(U7en=;oQD|$ZcEBp`R|dc9{aEm27UK)p9lm^N{8IR@ z#Fs+Gms^l8|0BM1kofYE;(bOCLr+@OyLm!CfciFHn7;owdkEJnT*s`*lA@+Fq58VU8W zxP~uv&iCI4`%G`4*TZH%Exx>qeA$o*e3^uNDOB^tP8!aa`sVw_)ff;wot=y?Uo*b+ zc+Jh1Ri8oPc!o0iS%_~O{p zG2V3DW7FP{UBef0q~nWiCqJZ<%m~^VsvWS3`L!?e>>lLHW5ky; zCB8hWdB0IMTfZ;b%K84Y$d~7f&x;mw0R7OF8l-sOF2EG^z5XFXPKD#+PcwmtND{e5wA= z{^3jK)9!!ek)j>xk6B?8_4AeYz2ilG&tdrub|=5THfjsgr2Kx2<@c?a-?veI_mlFw zO!0o5Y-?>fo9~vjpbZW9mEu=fatpZ?jU! zx4|smcC+yt@P&K4v^=&>dwhU{0-WHg^4fHZ7aiji+PhL zNy?V8cHpR}m#+@@P7pk)WIO@8!;@;`*TQs(Cyz6p+=e{)KjO(?i6;@o`)nX~9%Je7 zJfR;zeY;*VeSenmYnjlGqHcoe$L_;=`2^L=?oz!BYTjp)&DQUWwsOAzGIH$I;&{~S^69yj?lJJ&r1b+xuKeo?)wbH4vp*k}3-4Go+9wCd$>td~XY zpk7YFdil7zUfRjR>t%iO{WH5^K=7pgWW4>B)yq)&Bd+hqjd$JkvZ3?+flqeY^wzxR z+E(O0q2P*GzYm8ct`t~3)^Y{Rsd1&T>-V!kRCG*JCmee^#+$BpY}y;LYq&y=bX>9R zluV3!6%Od6MzvcH~P>;!BmpmyZ?i7lIgi zjO8EX3H<=-69x>X@An$NmkIqS>L!?e?0)3SQ^c445?>zIyk97rt=|`I<$S*w`SNCQ zyw@h+%LiTvX|GMdmu1M8C)Ip;(&S5J?q0*7Jus~ye0ewQGaZJ8h0T6ieE9(R(yKl2 zWg7D3Ni|>Wq)C-8+ZkW}Vtm=e_!92n=1bO={lk~tcTf9%=ZNU4r>w||`uX}uZ@I|t zc`U!d?&SA(#`eNyQhq;Hs3eb?}IX*e_Gvr3-j!q z;&`_j$g`1{XWeQb&z57JJ*Cbwa4wvg`8GF&d~4gBDwx$1<8L;8<%RC?((>3k?fo_1 z2WO&38qPZQbm+GI|9?Aw-&e=`K%4JBx4!b#Qk;+<`pFC!VzUxGU+!$1C1v1F`cME5mt0KY;pny=40S_QsAfp&v!v z1k;Z_i1qRrs+U8gdikX0eKy%_{k~``=lky>$KEfFciajb3wz=0I&KAytw4@Ft>)O% z&BVv!xfGH5w@)&D5zXqH?~e%kOwXa=VY8oBy$oZ$?06>B%NbZNpH|mPJ6U+WtZ%+Q z`x*=gp4vZ*x0_kLjDO(fukV6Go8qrpn}*K!1JC?fd)MK|iTm+q6kHMO_u;U_mAfoG zEmy#t8dn;-em@&TMaMLC!m+1g(CB){roAD%hAZSq#}(U7en=;oQTHp%!7A=oib}mx z1YhPezJT4~%lF0~h0P_tJj3{sk9@h4_|ie*%LK*ygdm2Vv}9?X&<~)#%@?NcKiT-B zOz1~ZH^KB{1CTEv;>&P}FHcqa=nqTo`952}pRHNHKLYtOvN+zh7Wh(%ms8tX;LFOZ zU`U@)^W_HG3*~0jeTq8qxD-lzM(lKzKP#y z-17raRCG+uH_1-tOx%$|VC|A!%{SYThHs9Y{E$vEBgCa#ZLo^_r{WL2Qw84^FusA^ z;oD|oXJHG8Zz0CFJCSd_h;Q8_zD-oT-w3+yF;;z;C-eiTZ}W}m`%f`;mI?hR>L!?e z>>=dav&6UY65pQIyx%B?JwHISmESj%BHu<8$Bpg4w-3D-&h5arB=Rk!=3B_*+b6mB zJNvNd+Jf$w;KS6alW(PApZV!fIBfRQ;@gMFH=`5qEsA^#sreQ%f8YE`m2ZO>-)slq z&ZuF0tKRD7Td(K$58o<(exud0)3=X%*3uRAyJVDin#k{kEWg3-FTC!CqGWze9qR8ci^a~mv4^tP8U2`#CQUBhbJ}0?!rSPo;=5R(i?eF zKs*^N@uXbwJ{^di$5=BuPv{3w->#QT-`~O5T_*ISsGDH=u>jV~=c!&Ulj>zi^FEy% z_WS_RR({_w7CAPqIG)@C92@P0XeRdn$5tc9o>Oz|Ig?)ta^(@Id%xwuR3YJW)T;CQ zhL6HN^V6ZxVY8oBy&R31k-P}%Wd+vD=hXEQ!Y+IeE;CJs;-|jv8@gYM0fCPj>Er<0 zKeBphT;%3&XqCHOHuU?3D^6bgW^12kU`77(3a*If2f$&8E9YB1)N%#Psd1&T&kx7} zQPD9?op9{w7+<>Hv1xC}uHgzf(s9MMlONJaW;}0esCK|A?pKC><(Xk zF#as`NPKyo@udLy(uepGk@zxM@jfGnp(kzaM|na&fciFHn7;osJ1Y?S!&Y^ARNgW&qUA%{IbK(v+j;}eQ~)~`=0jz|9mzI@~bhtYq5FKdu5&#U?J zyvdh^x$ryt^6vl*U+TOcFAMuj-=Q&Kv!51UK0>}kF9W{JLcTn&=8K&)ga_hFecv~f zGrlw-2Y9kSF}`FLxcO4F%gvXDe&3M%tk-!xBi&xF$RDKOi+Fwj9G3Xf$?Bn&FJMlM zFO7YEKn{qCj%oPf*wZn-blqdq-jH3x7jmTIi)|-Aq?61TWNWB)z$)%v-WubbDfm*w z_yTr^FFzW43R_Bi8N~R~2l>*M_);bDWs2hcMi4`f@#&a6p&vl~fz(%~??2txQzrDI zsGDH=u}6?EFA!h0N_=@v^M0co_WS_RR`O*s@?~mqJly17>$QGkywKBdlQGTDoz%=& zi+mZR=F1?HFN<>N?CRgHZ3th+hJB{zP$X>j)8fk*?>bd zcgKL>sWl}B*xt(clJ&BiFX0moYld^1IPkF)eBI@VcW!|XB?*z+l zusiwvlkrz!hLqngu>9_e`R%9tu95P4s^a}RIo8^CHs3d&9{|eA`Tlgwvzf*5!ObDh z$}p^hn?s&`ih1^eI?uqlIQM4azr`u!Th1Y^tccM+&Di*LEO(EWmdCK9e=i=>-}}iw zZPRf_`F;S1q4^07XZPj#0qS@kX!HG=VLxtr>Go|+D)L`c)J^gH05~kw&CXV2wc`Zl zy~fGB|FV7u#>MqMz?{ZDKR`IZAI*3=cFGH?XY`A-CA+3>k|SN+wC&`Fbdnh_l7wpu zM@7ASC*q9>o-AQJ0lUMKt;XMlhe|wok@3WjJh_W_(qdd!5{yq%yiW&W=P}kt@`Qc> z_3e7e^!;ZVf0qgUDC#DdeykAd)rf~-|w!M4V~|Qu;Yn>J&*US zsmOmx!4>iR05~jhCEMzsmMdURjVq0Pen1Y0ijHZx;@J81Zk>$u57{+bAxAo{*mm;c zaOEXQW$#_Eiu;veW4#rEFH0F;!0zznXXBs3!z8}E#Q1U-^5t&gO9zQB(-rSCf*5+z zHjK>^`T^9p`NH)5XBq#L3H>PQCYXLKh53fH}yQ zxyA8Y4+Fl8^@77&4+FleN4~tI=F3YaUlO@{kAwDzhlm27pjMsp{Yhb;`RUNau-Q+G zFJqA}x84AJnTvdRNzE5KX$TL*m-^=WzUwd`&|RZBIl%Td#+M%dxcQPC>gG#B@5g_e zx#*rH>)wB@B7d-gFXH(Da9H9?j@3ggU%;FiUmE-TfE*AN9nAJ_Jy&=1X zFXTwa7u!yLNGF+LFZH*zfZ0$5WY+f`%JH)^03)Y zi!UD|Uwr=szQmC)gVlVolZNoHuY4KJ_|k$L;Az$}zVte|unC&1P4%q(!~`_A(L?!!E`hE=>q@X_evxtY#yrP@B8X_A87Oa+y9qWJS)BB zy%qVdDC(woegGVn>Sk{%RN8R@^Iqdr=X}4h&kx7}QFFZ1>oU2V&g8a~`*d>H^8-X% zIp0qp$CehyGmiz1jrT${Gmiz1eU2P^Ma{8SOnxoPl}DiN{WDwA9XNcBT6NC%r-yy! zr$f`iWSRR!1iyfUWNj0 z{u+m5y1pN8=zPC&(X=a%e&B`)75T3!xFVh(0EZ>66j(jfas|w(aiy`(56A&g(J>8I z9D6#(m#%kg+8eTKxI&I}T(Rxshjfw|ui6@_9k7b~mEjY-m4YuT8DGHe@a0z{J=jX( z%d3np_aa~JBfdl=zQh#oGlCd;(l$=W6Z!$vxB0^K{pT9#5uqPN-2~H*J&Al7LVW2d z@#W=8-@O*hxcpAimT$-%m2Wc*p^s-tUYr zMen)!l6kJ1FAbgVe>dm$jbHTbd0s{S5Cvbv^8?_p#Fsu+54C&&b838P?DGS1KvZ;0 z!xzWSuX_x)y&=1XFXTwa7u!yL9KH;(HB>ub756VACVJ-zz9bo6!0zznH={|gwZxYp zj4$^gU-}bYswBQtDBf=bG4vQ;PRtYf0o1qo!u0*;8BHQWKZ?2urXPC>`SKd^rN6|N zS2XW8%3;qB5N+jre--j&O>w;13w)XAg{W40fiD}8FGJLP8DjEfMXqWj)W^0rgfFqM z&-50GhRuFje3^)RsV)G%EI_^tQS-%48p6ZA@}<)a7!W+2mgE52+ZkWN^W1#Nx^@5X zrSz*KA}tey?Kr4R$BLe>a*2kCgKJHJ0D~F~9Gp{H~GmdzRw; zIyrsK9ue`=RURoYor@g=ae|~T#x})K&V^6nw?aUA9`-NnNI^GA`e1GGr>}@aC zem|ok|8+&(6weQU!&2S!TM^NY6PQ!0n~i;bKn{rN$J4R%@x(5Go3@?& zICb-NTSMM~qoQ7pEceb6JXy_n0(OTd+l^+yqa>cZ&UkV^^5g;HNsIAaNiRNI@je}h zoyYjPJWuEcP~WbXOyA$hXciIrQPfQ^{n#^DFW;bgIYg?LLp1Nx$zjhA5N+jr|1;#+ z7sc`N(}82<)P#N{rth|W&E&U6Es;H-xznjZ0LOd z#@hSeeWlNyk1O)uP;f;&KL8F(T)E5A*K!5Ssd1&T&kx7}QPD9?op9{@dY5Br?vq`^ z6>_BGift!94p-jbQm!^w#r;ZnviB3gmorf&Je3)8w&iChr zedeb_m0`1=7GEYKUxqyZd|8Zqc|*+?J81|H#FzT!`@MQ#K%l$EVdMbYI~iY+o!xxt zvE0pz7*}vmo8%cJ{)%V(xs!-LmgkxoDyHe?=)`xK8lKtsre$= z(=onu-DA_KG|=_c`|Qt^Hx=(@-FW=fvW51_uy7pCv;Y&4Gu{V3`tn11Y8cZCVi?(vU{|)lxyW)80JmAX|yqrSk0bi<-FGXs;6q$TkovRuN^|3(>;mf?R z&-50Ghs}Oke3^oL2|WyasY1RKsrh0j4dEdZyH_7y7BIeKk^}txU5qc)cewe|Yy1A; zON)8GTye{aoBI`8x}tu5IMusA(DZQ~KXl zkzb;yn_~Sw9G2=PT=?2?0&{A0v$5;c*5v9|B_u(H_4H%ZrXP8LwS?T zC~-ujJvb`r<)~@ig@PyR7*D|N@MM>9aPSz3Cnbz04F}|CY zC-eiTZ`Vtv@6Ry~jtKoI>L!?e?0Kx0Z&JNnCe_O#&HHq^u=V?*t(@=wfE@d&INswz z;Mg=T1hdD5z_D+UVim6-fmn=1`KJ-gixqdRc;Ahqh8?=KAd%uk0Fgw1|h^>W&I z@cTU;hkCgL>t%_$UP9Q#uMjS&Ue-6?A95oG1muh($N{$hJREmuGv3YLQ10PP$z*OD zI^Tcj-m#OA0Ww&ui@K}j2Z!*3-jC=_Y zUm_A;<|*E11TplaZJM4Z^aH4G^M&dAyBLQ=gnkrt6HGrg2>CLU__9&rOL3*|VGM$6 zzTbtd-xqD={rJztKI_-F6~}Wf1-?wj%PHql;LCT&mp9dXdDG;}r@7dPeb|(1_)_P5 ze{tAnx(+Q0oBg!-G9CGn^EB{fDe~n_HD4g?!Uy4j_)_2fcrD{gD{^3VNc`+!e92tn z=1b8%ZoV{hzTf(nv{@5>d*zaf{GkfIi1qt$SmH}3tAAR)fH^h3G?jPGaU3H<=-52T(legB1qCnEHtsGDH=u@{gpZxLU%N_;8N zyx*t`TfZ;bO1}JxeEGdNo|OxHnc;<=X5|83HX&bzs`)b1K18iL z`H~3x%wGysh0T6ie3^lK$$Ad>vJCk$RLvI%yYNAH*jK*Xl!pN^J0N~qlLKfPe=)vf z{psdQxM=_I<(VHk{rL9K@JVl3x}tuLndz+(`TZHoZ?HT0y~k)7JWk5*w^)7;#Qc7Q z^1DXL@A-=N>vXZ!mb3Z3xqcs%mGk`_m}fhS<1Ma&Je!GOZE+Rk+4q=dZ>jSPoQrce z^KE?!`4(jPb|f3Wjwe3m`umcW$JS}@ulYVW6Q`o#tYc5N#@o4%bLRUH5+3gZZN49B ze}4A-sa2;}&DZu^YO&twKrtf)J<}v ztDCl+{5W;3pk)YTl>Qg{|Kg zZRLD_H*#!GalG~#;8@fP-fOP`j%`Mcy{+ch+a|v@SYw`W$nvQFIQl_d|O>F?PMWbQoXEizTfV@7!W+QqsRfa|6%pg*Uin}_*d?F z+0glZr0?CA|9jq|T^0H7D7Yfl@55nU%>A0pNI z=lg8^eizO9{lAef{}#tLTnBuK;pMd9I^fF|#<3TrH zlFg2AeV^FS`Tjj!u3uDj{HIeY@`ow-BG&K2VTmuWL|w}lFsH_s#;)J*Ld)iLd~xjj zy2o(a8?tNoLXLENvF+r?;ma^vL$w1|asM*5!n;)PWh3JY*d4z7V;mODlK3)=@ud*? z5+uHKllZbo@qQzSp~v{4B2VZCP~YYY)AwI$92OD!QPfQ^{n*RMmv@OTXG(l|Tl0RS zE^Ph2Xe;OYX(c`@hD}Q1@f(3J6<&yH{6^qQ4f17}nlHmlzI>hwzq2p@j&2BFR)u}0 z?@%&q_S51^1@a|c417ruon*s?Xd%d^zvO zqlT<_H+}oNmW-&MAJ6hG6Z!om%Wtqd`Tehv89Y(S?{`^#2Qj}Nqx|kC<@aL6`*pfl zYtPwy-(0^B%6$H5b+;MjS@V*3*ZK=lg8^eizO9{S4&G zVI}e49l)14UT_$^1Nib2^5uOsU*0$Q@@1}SB-F=}8otyy-(Mg0nchO{!e&1$zRW?s z1V;d0)*xTrSM$YA8o~qdrM~%o=FJ!oJe_070k)ekz7%;Lcl|vl^FudZ8am$(HC^~s zi}|H{D)NUb_#)Qt!(oXputZ(U7ci&Bm&UH&?*gJCbQ-=m_H>LdUH90uH)PlFg&gVl zV%y0N=_E6T+Zw7Ju!{Sa36V#c52nhm(RjJ^V6XXVY8nW zUn-F=y-I;EYmqO*)qJs&hVZbjd>P32(vBQJ_l>5EFX8jte921eAHJmZ?bx;Vq7UC5 zVd;wcIdQHxDf0Upmfv7^^1F%AI(V{_-y>LlKZ*JM6yO2GI!kL+GU#F07!&tr@$HuSxHurdG zd2F5b{+jQDGjS>!&N}vVD}buM@2lf|pw0L1`s0Q6{YUf{@6Gx^Q8&f9+%M|akf!KMBZSg#zA3%M(UNU`uSL4Wt z(2t^Sg6YTJzg5orUJlp1&!!7ozc1R#`To(!v13c(9q$2-#l3KL9q$2-{em3( zK+Ulanu(8B=Tb!K-~Pq;MfI}I`Tmz-pXoWYF>LnJs+V!BmmSAKy7dz)r0?@5jFm`^-;=z6zWDwD>X)`Oe_Y9g75{iH_77;< zt$EmEWcF&E)28Uy+7o)T?{I49jOufIo!Vv(xF~s9$F41|?saYahOC=*-O}Us@;fv8 z*4*X0FZ`e}aKob=pD20;yRF?6{(MB+?pJlq%@UsomGPCYXM#82MI8d>b$E?E}sGje4`^2Z*-v`-T&dZzq++jfa45pLj8* z4*}n5k#Au&-@+!}zRkto*@w-7hVbp1u+MZIst%j|wD|T3^39kGeA|G03#<7SHh_{eD7M3-`}(R z2D_8r&5gFf)1>?^W%(V#{C<}5yPuTbD;4k8>3y!*1NZk0z0KzbfU@%YhLbVRPA!So z41_$Jk72DD2zmA^=2@va&%n75HuLSf6!NXZkyJs=ZX3plZ2W3>xW`M&W9zi{)AtP! zL$f;?&Vra}Pd7g$eu#h0dIEL4547(aHs7A#@AW~ehgIZ%sHmIb`2lcPs+(}(YsU%9 zsnyNKK0ly0!ALirj-8JujIQ%9*)?^O9O>$&Z6`mJH_40-9T8~{j*5CYb%FO&!IRC5 zCt!DY(!w}8c)G-s4;fFMMV>rIJQ*zUB&m3x4#du5{JtPh=m${Wu9r;T-_1BWBJ`uE zn_&8}H?dxhqI$VZs+VES`*eD<=Ld+k^81Fj!Gu?(Zh0T6i^>P7bMluTZ@^h@0AFAsmgkAU`TvENP z@B4;jtX_J_f!QJPbI1{}wf{Zti6-{G*a=N_*UN@}-*EXqdE@?j&z@El`J)tE5zh~R z!xC4{x4NO_3Yb&lN@JfN&>KWW$24`qv8Q7Y>3YYey&=1XE96MW728gJNGF*w%GOZr zfK}YDOk3z(C-}03@dfM-Uk)~o37#SGWfbGfbI6zHi7yd}FRK*qGlCd;(zY+m6Z!$v zxB0^K{nr@BM1+16brVcKHWc|Xn)tF&;!A0zuO9}%^?gHc_WS_RR^E@Fg?u@uBp!Vd z__ELo4x>*3Uw%ivj8gMul*yOxb5$duK31#YOP$|0Yzg~JZ=ua$v!51U79wAwvw<&P zAYVqQ`C=yx;eq&4-}en2Z^eM%>6}CkuBAzegGVn_|nPhp_VUTPK_^(eSSc15EUKM@WrvGV|?kl$ELj@yM{01NXHl3PJT!y znK9bdQ0;(K+`mj;`!#hhVrH3hgs## z2E8Bu$nqlU=ZwYP4I;mPVEGMpC%-*LyWm+;et*RBdl2UL3zXkAQhu*dykDocwYHtj z_s!=AfU*W}Ay@ap}AB0P)m-Wr}d)|ft zF*_iBP9X=-G!A9;(id~{H-3Y=UN&^T-{z6b1-rB7iTm*~1y{uL1K_a4m29hrTCRXO zHLf)F`2oE_RCG+k6~~^A@ulk>oA!q68m^Ed9an5S`5~QTMwzXl+5xM$Ux_BXp9{YH z#P|Yshc6k%@xgN>zLYV(yoh{xiTKh%;>)Ls_ZdM9J!!iVc|t#c`ZiyfzW+Mo_=wPt zqHcoe$A%$aBE*+=5??;5^!3Lexaa%q`2oE(&kwi+`EprF{MI4Bmjqr;w+;cm>_onl zsrgc7^5utI)kvt1g*1Gr^L~6=*k^hR{Sr3&Y4Ih2e7SWQ@a1ddOPQK4cG3_Yh%fcc z_v4H&r;-Cay~7w^dQ`jll6=(7mxkVtKX&kpZHbFtDX++nDEK0t9{`6XzT{Xv)ba() zsqv+;&kyJgqM~COzBqP%-D9}z4cRq(AxApC*mm;c@FikvsCK|A?q6a{yk7{uY-M}_ zyTg}5jT3?$CB8%$UtU7K3?{yGllZbu@qQzSp~u+0Bv0rEP~YYY)AwI*oDdQEQPfQ^ z{n)$6m$Af`GbO%^(Y)WNH+z17Xe;OYmm^=UD2e-ufG{K+P0vNqKt_77j~pWAQVXNwo#KGyOo>Sx7L??#c|KePM>yOZCC8Ck(|rTiYt z@_R7m_sf*u{iOU}uXw*sZ)9xkob4VjEstSI|6V+9*nHo4K7jIl@epIP!*%?9 zUmfoQZNC5W*h8zIDH!=vMgGT%x+$I?0EeZz+1naO?KpvXuW_>H_mBqe`F>-cAJ7{_ z&GAx?r(;hCef$5{z^1(+yQXfEBVFCJ?c~R)n;+X6@(vso^>WrS@0Wrnzc8MF-Qh{5 zaboa1i6Brv3 zdO42jWp}AwMl|o!>CK)WAll0L{x!(4YfIvp!+>MUyb#UIVZgCJkz*gLIrg#1udTWA z2-Lklp7D!lR_AL#?G4#ATp>p~uGn_+LpsTfakhqP2dv_LW%hFKSAs9w z7+=8d@a1sB8$4g)%Q(iDSCB8S5?>+`Up`a3&j@1ZN&9Pgp3o1VzRee=@4wOTMudJ8 zbrVcKHXQjfp7_#J;>*}d-vA7Pd%n+}AJAL#{DA9_FE^CLcfAjMS?&dgyWR)B>_NVa zQ}bn<$(NsVRU@H3mU%Q01?<$SbH2Yb>@zB))vEc)t@g`q>$yJSn`q;pR@MU+{XL<|m3Y-13__6}|QvEgXWfSscyqYg|(hwf@ zl`qp6U(O^4czUfFU&5o@e95|S|L~>j;la5#3|;WQ36`#?pOq`U)gr%tW%&(uC%;=6 z?SmIc`8|Q<_YlnQ*C@Yhr2PIu@qV4&ea#-Q`M&x508m!W_ix5L`(H`CY82$zO3bsW zQIKbUW1dY==NUK`!e+j0OCjGjuzWjlQS z{G;y|k{RlFA87OaBQ89}bI>s(n^xpcRMbuJ`~WyC)lI(@5A8UCIkmdk*yjiI22s&5 z&3HO?KAzUeNdJ&sQ#Z+xu5Q|P@B-czcHSG-Qh`V*i3x9HkTq& z|Mucz=nfn{ORYNR`+tXh=BGn{h0T6i^)iX|vV05F%gtCXC#vhEoh*lftU>j%zWM%> zJux8oduNjaY#+tyW&Bq+e|>}9^|GP!{ckRxGU2G@SNE&PpQPZ5czysJmbh}4)k7^; zz?>Ra8vFc!-XJPErs0ZXPsjMu^^Q$@Lv{^U$dQgKww?TtPBLSXt)bchtGHi@ukwB? z`0_jB3)mgL9BG^q%$E2viSgxip*|MY@TJcA{=Z?L=`Hk6*zBjpmsQA@VLt(1wjf_7srh0j4dH?K zQr~>Pn(^fva)77ThVdoY_NgXlvNk;?x%tx2zwamSKlGLs|99}Liv04u`BEU(@55n- zF9jVfeH~xWoDyHe?=)`xK8lKtsre$=`E@VH)Z8b#nlH8^4PP8P`EmGC&ZS&!u!{Sa zd8@tO3BGJ+d;z<|m!phRgE87C-xqD=e7|>z&kA0jl6Ytm@MX0Z!Wo(b zd@-(ucDb4_)DgM2Ah z^98~#d=MToO^4#AKE8Cl0|NpdH#(97Y`0~6slLq3mtO1k4_{9B;rZd$J~FM}WJ_1n z&rjBPH;Me-!SWmIPJXvBIt05&`8}ECcM;}yG39qZDZjr~ykDol`XAKY^Zf#I{XQrw z=lg!lv%5>;{iZ^mt--MNn+kcBhIuwwooC=&oV%HCzo(FI{aC)8%f_$vF86q8d2F5b zewyz?49$_$a2CW&d%F24@k9J`=KJb+A87OaIlb>1_t^Do0u}jF6m?Uq--p9e-GmEY zJ5FFut!_4U{eA($NIRa6osTDsuJbS1HFc95>FTC!CqI-o$&4wEh_nYsMZKKA*89ES z$sdd-V0U=Z);KMAp~RCZj3>p&lM>>|V2LNyiudV2>^w%3wRu86fckd5WcvPFjng7R zKZ?2urXL%H^>QlJ%Vko%EZ4kGr+}^B7j5Ny{~qMneI@Z8Gl64ky%5YEGl662$gwGE zj!iN7wLMoFfO>Y9F@90KtaHBK4C`gnQnQ~{yboBw!1!`LIl!|xmhmNXx|=UW*Sh)A(E0w{c^~yor1nN_?r3`0}md{YDT&kI`&hp3o1V{y^$0 z)A#?MaYjVwM^QJy^kW|(U#1gZwn}`NqIthj0b9Q>+Dg7WgnS8<#IxoCU)Fh{r&)7> zFHMmz)6{&KX7c5aTsph@w?iAkmxGZnElSOPT6|fDe98J3__7uGGEL1F2)pPW;bC9- zvX=3s6FIl9dP%h`P2T)z*>%K82ym}iA0@fPzT&(>p}wU`fi z)(rD(x;oFmxj1(--*%>uZ@XB&UBJe#W0R+w;80o{Esw3!-e2>5a3%yboOSH!*g4{# zGv9}h@OU3+^Zn_$|M$tYJLYt#$e*F8n_~Sw9G2=PEK%2v6PQ!0n~hz+UqH*|b>r#S z`FP5?XYP|-Q#Z+xu5Q|P@XnB?+=0}e=?qc-Qh_)I9@ zp1g%Td7F6BVzTr1{Y{GZ=|Jo}M)M7MLO+1|cD-c!{+`BJ5uqPN-2~H*m0`V{N%gX2 zn=9$WtEXw+r&GYz?~Ar_zW*3A?c-U!^qubJZ#?dJo>^qqaD^P{xMJJMkHeLjT*}o3tGHjO`po;I;L9Gy7qB~gInFpc zc$vhPnT#)QBVXPjzI2fI^1b4HMi4_!T8q!}gnj_^ZN4yl{~gBJ5uqPN-2~H*MUXF1 z;!8V;FVic11$g%E`952}U!YmP{}l4&nUeU1CBT=@yx?%d65va7`M8{K?KZgKOaq4WJ4#{OT|$T5#!RFNN5@I|cOhr<$IV2QexFJMlMFO6NlUqH*| zb$oH`=@?(S?s-y?UBef0q~nWiCqJZ<%!qO+R~xM2{w49b_b0)ZzZhS@?(pS!B-NdW?fV&lCCq)VKM<^!@q9IT4{BMco9`kBvpX#E36v zN_?52dB0HsTfZ;b%K84Y$d~6!;_(&0m(THXimw2^v_QT@)qIJXeEBoiH3W4w+8ye2#pH9}0Y_MZQGUe6f>;@UX9Z>C5<%Lk{pPvlw4`1>Jn9 zKD3SN?-dQ@%O4wuAO2>KktfD1T~R-meBs?H^80U=-(Yw0`vjw7@Ny}?V=TXiVSc|$ z`Q1;-?=6b=>l9dP&)Iz6T)z*>eEw;5_XW(emrCN1)sSajV4g)*L!KRsc@|UW88{ct z%zWFELcR@V`PPMv-++>6-v?*nR5bH_$DVElQ1$nH2nmn(fi~aYyy=&~ zeMf}OtH`fV)J?H|9}Y`(6PBoJ#|g}-)y>AP-!GtL^SbeL?0h_NcFTC! zCqGWztgtoY9XKlL<uI++eQ6lM2R@cabOW5l;q7JgHH<&jw=W zF+3adgnj_^?Rv@d{k@EHBSJrlx(TKq8;A9B7S+q{QoW37-e*(5*6)k9a=!mEa_rTT zc+ooG*hVizvuGV~>=5Kwg_>g(Ccpm5l}DiN{jH2&R4?nC?;nZvvURE1Ppe*T#ClnD z6x7S#uwGWE>!qD6giETI_09Ls?2Q4z)4GrxVB5>;WoVI`zs4QzdfCwVe!ulO)tBXN zOs~kFrQnKKzYm8ct`t~3)N%#Psd1&T>-P&lRCG*JCmee^#+R;lY}y;LYq&y=bX>9R zy) z7<$rLewioq1E_EFh3Wfy8|OuYeiU^ROg}ar`7)dM(o^C~tkOpdAKdeOwtl}rvwr_I zLwF#*)HmNRVtl!X9N_7l#Q0M5vzsrOgWP;+=>7N?UvJxV z-@z@Ot;nCP;EPzl4~Heb^s#!VepI|) z2x90lGQP?a`T^9p`NH)51;+Uip&v!v1k;aAK)%c&zVw&)QlWXjPyt)NFWSoaelhap z&60SpuYfOKc_C80z5>3qM83>c^JTWlmw$38y7h0THH0rmm-2m5`8|U2yGF|IpA_%6Dd=nVfX(;K_4}aA z=bu)0-@-h5rzGC(TgbDoG0(bv3wf4-c{WF#XW(2oGxP1=6!NWY7F95-C&ndg{K_wK zkC&Fm)@kpr`93%kJ<@R2v8O|~75_iKI^GA`eE-}_zMC@i*z1xN`IU;gDc0}9VX1EV zt%zvH3CyY0&Bm_ZF91>fcsh1Io;bYrhU}WUNse@N)3%czr*2l-8uAVt74>puwf9%S zlXS)tusb|yZ(I<(O5#Z+pwHSe=2VC(lqTRGo<7diHRNxb7`;8?X6&aUHT;Mk$a zu}U?^Dw~-fzk(uC|Mp47FQQqU^ZnznUbZVW`)Sq7YOI$XPl0;*2iD6Xsr zvcCEL>;eo3p4z440NbaqdKoWp^VhfAT`wCt-@o$1*Uxxs&t~yHzPSpni1qt$SmFxI z)U;dyb81{^?E3uznmy|3gk$H|JI^e#Yq&y=bX>9RF{q-B1SC-eiTZ}Wxe`~61eh|rIs zZi4B@%8@T|;>&P}FLNq=_u|>R=lg8^et~BF{s`pD$dY*5AAv95;N{fzN8rn0$d|cl zzRWfGl9mErsx^G6bH0B9^5ytav!51UzCpgUJstS66ZtY%%@;dq2oJ=U`sVvteJ~&x z=gY_ewohezNxtmnOOLF!P4T1FrlIrwBlA}t)@0O6e-CKdt$EmEWcF&E)28Uy+7o)T z?{I49jOufIo!Vv(xF~s9$F41|?saYahOC=*-O}Us@;fv8*4*X0FZ`e}aKob=pD21p z?u%;w@ozlmcfYD@Zk8~H@zZn-nxC)8kMGU5KH~WSaM~zk=9qVMIf5@)po9#%$H^)wXNGF*QmsIxN1*^D!TJ^2>cfq%&jBj9f z_;!ks9qcCYEzbBh68RP;zIBuM_KV{EM$mPSarn1+LO+1|Hs6@O|1KjtBJ`uEn_&8} z$;h{P#JBMh-zqilH|oQlA0XPw?;A>yZ=*`$#xKCPZ@n1PUx06!$hWwfZ*h}v=_&B7 z>xp#71Ul5JlW!*?-?B>0ep-C{7WrnJ4Sd^$e2c6324NSHgr|Mw+aSg_A31>T8yy(m zs>|Jc>vi@1;oF8$xB5>xW7f`jmaeGZtH1MZ7x~?czTqRxv$B$S&99JW-(jBB{0ez?IOf?rb)JE9 zaqecmHAx}gA}rr7XX979-aTGg9`DcR2S5zXk<@T@U!EVJj`xA~eZ%riN8D7h;~}x0 z;1fmN6weQU!&2RZ3tu}1ktZJ#PXm}3o_cOXggnkrt6HGrg73<}Es+Y^8dKuTePp1!iet>8zzi$|e92-{>PwoJY zZSq1WlRJQ8MY@iuV~o z3_WSBzt0o;0o1qo!u0+37#Bu_eiU^ROg}aa`LclcvQgs8yh`5#7zNk&4Sm@214LVS zKRyBZGN~jU-3@&C9xtcpZs1ERm*7XE45Go)T&T7WlVlwwo^v z{l4LvgCCFIbJFVDEAkg8_#&Pk0EZ>Mbh0|26Xa-`#nZ6`ktUlwpFR~xM2{^irn-kpLkEf`JH7<$>{V3`tn0{cgHN zAlgd4Oh&#;Es2N!2EJ@YzJ&hLcT0e^TkdY!o$AurL`Xef&gT#G-d!TU4`%rdb|=5jFfI;WC*}7-mfxc>zdxe-ot)ZPRh*K6~nJ9Y}xlT=Bwrh@p8v!`Xd#etUP=1Z3pDT3>BF8MAll0LehfJ_t0X?48E~w|3*HAb1CAYq99yL3*dmi(EmGju zaKZR{}H-F>j9^I72%iK0}zJFDZ_l^x56fUXAU##GYczysJmbj8_$!fU*=G3^-*yjiI z0a4L04ObjHzuv8rk^UjOhAZSq#}(U7ejKhWmQ?oM1*^DU+3L!?eEQ)-o zBEGbf__DCl_b`UQJ>O@~59p(Le!v{$%iNOqtp@{Nen7t5dNA;%&D9{VSk0HkCSMLt zfiFgTA`1AWR-Nw`0@kt<<@I}FaIK67OVLJVHc8w2jWY8^L^i47!c^L zaWy%>_Bo6%Jw9>sC3&lxFAbgVPwzjt^|IXmw5iCiQt(AQKL8F(e95tTsO1ZoQ{zix zpC8Z%L`BCmd~xjjy2o(a8?tNoLXLENvF+r?;Y*dRq1pkfxPSTVNADiNmzIn#V0ZX( zwsBdohs2jE#+NeWON98+P2$TRiuW5q3_ZrtKjsPj0P5R(Vfy|DjLRZIKZ?2urXPzT zUlPQZGbO$((!AfO4|{%qXe;OY^N=s|OX9u^;LDGAIr%bxFKv-8RcgLenS41U1-|rc z2w%=czH}@#`)TpzN92p|I^c_OEo4KLnlEx{(7+zK)D9y?$`>rTUru z!5BUK`A^=zM1E(m{06&|-{%;<;Ehs#Cs=+*Fu%uAe)p5|d#B?4 zI(@9Q?QFhpK0g4ImGk|Dm}iSi;vEi$Jo^drti$1uXGde6CDeHa&c(T#`Q}L>->O)? zUBkw2K<2Yea44;fmdCK9e=i<4Y`*V2A3*uOc!;stpXQu@q?gQ4$NNB=?{9r)>G4z6 zj47+gU!tg+;`sq^SgM=7>mDaC?=?o8(AW zH*Gulaq8w0NyXWPqoQ7ZvDN#x;K`wkCt!DY($Tm)m?!aM3FFCFCb}8Pc z1F`cM$8OCN`T^9p>m}3o4=^r|2>mGPCYXL~7S_w9R4=obM-) zV@pfonMVT0wt69!nMVT0jzNwsQFCmG$*-0v@axQz=?)xps8#2D{{pO+ol4DqTJ>@( z*2~PBp_6lH+vl-*8OnC^*I41Mmkpio5370Y z)GNPxcXLJlQUzDU^8?_p#FYZ8hg$xFIW?{{_W1#QKvZ;0!xhJ#j`5}I9h>%s>>93+ zBOO<4JNY4Kc4iAa3e zt$3di#L$!0?&mzAA3%MZFHGP6kdYe^`cc$PF#XtUHfiK4*UzV!*vee{DMhbi>((t9u`F=L?rE{s-Pm3=< zBVTsi3VdmDE%bY-nlBJ`@e#rU@uj}`evmI{xZ^*9U3pvv9 z#kP|lhcC-)4b=`<#r?~dzj*%@e92^d0lUMO^NcHkH%WY1#`rQ0`7)mPQYG=_PsRI< zAch{}xL@*wegO4tzA%0N!^Ra6p&v!v1k;brLB1>}zVw&)vPAQKqdx5U0ivy(@2^6> ztSO0Cw*$WXf_$lN2YhLVd|9UE%QBNMhhCu?3H7n<4dF|dQlIH9lv8T<)8fl7$d~G# zz?Y`Tmt|_c*hxcp*jK)E>W2Zr)9FqQu-%FACH#V$FInx5as9obp?sOplr;usivEzR@*!vy|Vbl{GLGhT_ff99>x20`t&t>z~=kr z^8-LxIp6;j^K5-dyebRwO!VIRsmg*pI}Y<~xjN6lx%dq;-wsP5-)=pH?vO$6IyQdg z<*{|z`|0@s5JR&=8qR{4Y3KZN=KJb+A87Oa?w!Uw{L1xbjjhOEp{SeU`2lcP zs+)dmAhqKJ=G5wDW1k<;2SoMb>Dc*rS|=m@Lv~HwBuBcsY1_$5&l@XyIMco9`kIlt;xsvMT5UE}+)4Wfo4|{%qXe;OYpCQM-D2bQ12aeTx;q1!W z1ILa>j;&C0Y(+Ei@yrzXHJkB^XjbQZ{}Qa17nhp-wCZIo*30s~P%jU{dbvVfFYRPG z6bFy$WqtGgzgWGzo*bZ`&a7U>*SPuXyVqSW8#>?b+x>#J|M)+-uOfe?f-Bfq1pkfxL>LM z)tfH((u(l~><(WpFs=&zPvXl;#+Ql6mr2Bz4iaDfR=m#$V(3ZB`ZZ7J2T*8_4_C)I;Q4}WarmChTGndUCkHUk%ljho%}d_N!l8!9k7b~ zmv4UaHW7Sj&G-U#hcBIttAn>nd`U9COhUet6JNSXeECQ5ek16*$2jq~JfR;zeVZ># z-~Xs_bwub#Q8&T#WAl(NtB5b-CBCfCyx*uVTfZ;b%K82`B|a;L-<8BeX98b-!^dq2(hA%^BiYB&pG zrk(T8neVIPeW1in=M*@55oKZo-AH9ValSRyP~Fe!nlmNIRa6 zosXxP1o98rHFc95>FTC!CqGWzTrJDk`*2j$%kQ>(n+cvA#drdChbKA4HNpRrc(R)D zWHRz(3h`vH#1mttk1pN7z=!X7;BK+D1!CthPTHO)^aH4G*Gs1F4;t4*gnkrt6HGrg zAM52Bs+Y^8dYRO`Pp26MULJz=a<#f%LfFNx5H6`+ z);HfDaxVr1Wr-HZQxZ)r_M{u%{W#QJ?W zEOF(0OIFJjFsH_q#;)J*3!oE?C9= z%BCIOg9KmNFus7@;Y%0eKf&82zN}$e(d`X+>qc0a7zF1xNq_y9XC-eiT zZ}Wxe`yVs@6A}7R)J-t`*aGCsTH?z_i7%@vee_MeYrfx?t=|`I<^A~2$d_#;@thps z%MQGpa&mw#Cm~5MS!MAFpM6`7b%Z)VqlBCG#COUy9Ch^QEEl{WCr}&A9uFOMkA&U#sAYSicX4 zCBAgBdZ^_Km{a3RW7qHZ1yRv44PP8PzwR;I_J-^lzK|mwUu--1arm;<)==$$RouUP z|A)7^;7eP^7qB~gxzM;a*i+)mTE>^D$d_rvmnw-b=`(#-VkaEFSY7uRC;yQr^aH3r zkowB>{f`^hMudJ8brVcKwh;O9De+~i#Fy2Y_Z#(P>-R-l$(LV|FTa<>vn~d{{DFMQ zx)}J<9{I9X&6l+%Uye$FFRf2&MWL{^)T)y&*C1cIm74vu`0@wxCF@DxOH1U-S~XuF z>_U<7u&;c%={^hy_WCArfbEMJU$QFQd*ac?mo1N`5^Yqk)x1xqFI&Ga+RFL#$ySFE#sV)yrL2FKeHNdU+_;%XR8{31Js~C0tUytZ%;G zu0I9@lk669fbC0Jz4Wzwz6qMFP5eW5y=>@w|FjP~R8Lv3C8r{Py@D%Z{XQI)xB^Sm zwOj#nYFugT`u)DNY+lC|$Ih>JIi}`5*)?1tM>?+9cJkwJWj&X2wZSUxS88^94-tGh zmhlDb4qq-Yt`FWR@nt>Z%M9epOyWxii7(A&`nsXI!xyXTp0p0T^Mrl?^=-Z|eg9L& z^%0>TMco9`k5wUGHV|LhNqqUV($^35-Sd66e!s6~{r=y`mw!v*8?FYv?8eJ!!_~l- zQ;{$0)qGiR^5y6h_~JXAhyop_R-O0bHy~fGFE#sV@ntviWy4Frm&1@R>(zXLunS4T z1M#K4`F@D;<$vS=<9-?AOONyazr8boj-tx`e3|{5g{l70>b}R z-Fs_!yh`XK|8stx|G&?ntKYj__xtMAt@#S0ivPIxaIq@aPe95%%rE$4>9W}z_i{5mXa_Na| z(D#zQ!52Z_bH6es0Qy1FjR^Xo2=V1B=1X@AU%qg7zEOd&ejm!x`F>ofsyDwuX*fIz z`Er8z5*~$oNh7{&k@IDX#+P<6@a3i2@Z|>LOO{`2r_PrX#Fy}2kS~eEmo0L>V7EhC z84u^mmpuYsZe|C1$0>>`@Fo9NJ71zz^~0B`|M`3SpN6lDe5LC`{@i~uwHd^BMwZwOma&;sfry1Y++H+yPudUxlWpVzg@7{>wtVwA& zFb3o7B*j@^493}|6lY(_;|!gHGcDfQ#}IE#yKx4?1Gz;we?<%I=gX1D`eE1Cd>@@b ziyY?rrd{NpIp4=F5&Qf1Hs9~%D(F9`GVp0d?p8_Og!TJaZONOsMBVW`p*gm^S=aUZ z1-xwD>3o`Y@qA)bn(u6P$eZlQDQ_Bf_G9MFt)__O6V^h${NYq;0`R1hz!S8aJW-V! ze3L9Z*(&fPL_Dcro-DTTq-nWIn6}LK^(MQNZl|(A-%I*Nz7+I5#mWr<&<~PsM9>fY zhw|muoG(XP^5qtX=h+kp>-V87o$sGV9CMY1OLCB7r&6&qOLCB7ml4Od$~m@G<5!0m z_;shiFQ!?w^ZlDBU*71~+NtNuQP%r<#Kjm}Jiz7Q;*S{k7YYA6i{XSM(xH3s^p(9t&92-~ax_-X^ zMWJC1dBU{EQ~&byPDNwGb_cGoBPXsHcJ{+Ml?h)P8nO>)h4IRP-%>9Cz9b8LLA%M9 zYn5!@WD8%u7Wh&@eEB!?C1BypdFAS8YJ@pn(VOmy>;79d=zB@u;ESN|S)gPGKtD*j z5kWt+nfS7e`7*)6m#@~Vcapw+zAvobFK}4DpFn(RUK;j|L%#f$iVl6_kT2bcFJH^~ z^0mg7jxq41(t$74&iBU=U&i{icItfjjrihw8u`+S`0}-!FGkQX9+)pR&G!@Uq7I>c z9?uS_JVfA2$wzj+Bo^BFQrr3d$AkBT*8Kd^Hx;?tBz%GO`&ez^%VfQUj(kCLY<#Kf z`uzeFg@!rs#k7mpJ$1LSVY>rg*pU-o3_JTV`LfN>kbOWaj9-2{o!T7uk|OX0?IvF` zlpJ4yg)iF#zWkf`@+R|TlZ7wmm#f!PBTT;NO?N3hPG^I@m-G$32>PA}l$-$Q2T3;~ z=!ZTezI?-cxzoaztq#u@DiGH1Ls>fCZ$W%%RT|D8k9;{zms9?DhLBgwRY-!IZb@Ye-8Q5n)tF!&KDzS7!T*lm)Qbe zCa?o+zA{wcOW9F7Us4{eAHK9MJ<)4(bG7L=x-R6;gJ)6`A-+2c@r`yfzK19``aG8S z{zi!JHz~f?aeVKy#J8(l{R17R8Q=O}tO@gdZT&tfi}O!?_X{b`+Lneh^DxfNP@HAv zVVw1#IQvE(XXqT9Y4MgEL%b~$;_Wu!{H^J@umK&SzH#I+{-NGkJm&eg)ra<(xh1_n ze*rp!uQ_nmw6h=PAL~^n$ou>EHs9}>rcQlw_ZtNjx!Waq6V~rzwIy$Q^saV1PiQ{t zJZa-EeII>q`~CR~>bia(9MH?*e42LgeDaRGWxGS(WJgYU)3CE2j+@Ga?JQy2VlCv$ zLyEfv@FZ2>3EEAb3{`IO6mjENz=7MSuotJTpdFNlP7wUT}m${2lTz9 zZ{$ls-?LD;DFFIG(v1lEp^qqE?%;g+h$UZcb9kOjfv|ob%F_A%MZ~f8rQ!6+$T7u@ z2b4Y;Io6Xnwq4G#?Tz5;DKYTtl)x{}m(|YqZ}Y2KtBNQ1wRY8tvtw)6eRr+)la#hBOk7d6am zQsz<;^IH#WThhKNIjc)r*Wzx`UTVLj0Yz6;4o)B1bVUBB@R*bvkB`k7za}rSU|*p+ zt!#!ed&`{kKbFk5wng^&Dt#7 zC%%z6CceQwb-RB6i9)mF<0gxpkD0lnAB^RV?Q*^ujvV-A+Sw25R3=0%DrbL0D~z8G z$Gckr-?|8VL%Yeh;mSDQR14pt0^c?e-@?qdObg#GC|7TwCK!Cfrn{8h@j0OHC4GZ$ zg1%>wGA;o6LDG!~`k{}BZ#$W9t1W!n?(lpgk8uA0C`*5D=uCWbmxh&T$hUann=%dg zmQH+&%J~-6_?8+2-!|5UZxe}c`F^dPI^W`nZ_3}0Z%M?rsGM)u?a)@n)4B4kVNojV zWxn#+0aaxRe2YGA=Ue`z^~1LXk4%{H+O*f)J9S;i-#;~Qw}$w>Scq@5oAEtDx!HHS zCBAnG@g1i4ev9LKswKXgm#f)yoMwFY)@tm(H+Z!B2cWX__lAop&bpR{_uYkY)_~${ z-(47Iy(!Li%Hs^3gRmBF?ik`NttV$tt=Yg^;rvyF?B~mo$NFK{)87|(&>1wxfwQJv zsA`BoP``~NF1x2g&gZc z9NQ)5*e;Ds zu3!f_@*mByaiy;J5Ae`QbIcQ_J)Sz0uXm$==>yvxxWbN{xMJAZ59?GW>=vbLWwgS0 z$2j?JP z8oBX+f^(2BeTgr-<$T$#@#W$nwo_1Tp~`_T)&AZv#jk2kiwpf)J9WM^q6i7DMZUBr zzU-Fs1-l)5FdmpMHT~X@J~LI5HeQq20o9BW_>$Q4!3M^c;X}y_cD~g1_lD6!Q!e_g z$Nr*<+&vP$!2JWT+QOH9dNUmPg67!xQrG(jcu*7?<}eO5?c#NB?NjYD+a37Aj-2>n z*x8TCmp!7Ct&CO}zx>kJ-4^(gCh!IACSOJ><9&Bp__9af%R9uEjm(!#7QVDBS8pP9 zlP`MHUCQN+b3orq`m?!_g1+ZrWqbhigQOb~^h2K$U-mLz4p{iI%i;M(9^w80P?qwg zH}R!!X}Ihj@nw&kFGkQX9?q37PY8S|U>0;`=eqE~hx#D~~gD4sB@!|GLHyZ$Tm6Ji_@) zA8S8fjy%>6yT0c8=nM|UfwQJvg&y&e%VSO5 z7XeQ$6L^AllP9B<+kAIfc=D~llXrAjF0Q27r+zEtnC^Ece8Q$wqR+Rpb!eDKb;M_Z)7 zUXlBqge!3W0IarfMPFy`%oQ}p#+AC>Kfr^c&>s$Y!nBLmyR{Fs&un+#3OjP*ieYCz zCRe@_rEF!i!g%HQdG2<=mu>=I&~EbOdL`F4!@`&E1irjSe0iVwl4jw{h2`qaq;B#> zZ@MdP;CVTq?PKd#Fu@{mkt)b>|L+=NZ&r+7w#Y6akziL z)x?*c_w3YF|e5q-^9~SsBg&k18j}iEi^^%=0l~>yNQrr3dKkmM)<<3y= z+beSSN%#Wy55Q^*Uk2(ebR1uzIX1r3_5J}K6orO4@Wr%?*FAN&v0=LdU)YfoUkp3@ zG5NC3(2#vVD~w-$J>T6P_|jeA3))S-{6Wd{6%`d-pE_#)_gmMD1v&<~PsM9>d?MSS_5`O@9Omv0@OZ{!i~9{^?PeE(YF z%aGEr`Y`h4eBz7xF!E&}@nxT!FZ(pUTowag_SA+iGl?%neyyE4U(P4Ks2?C-Qiw16 zqIrd>Rr`oUP<*zS-w*^yJ;H0rh+~P5Bgd{Jj_sFoY`?~@ z?lJIdwZJc?S+(>1yZx%x&Be3*T08Z8*^KgK;wP9dQz>8Wm*-0(SQwX_FKe3b9}x28 z9qd4B5WKR5d|7S=;&k(m#*?czpk(FIMFKAmIw!KLD#OT$!Xdz>zCx zj*TmIy?=lQMWJC1dBU{EQ}^-pPDNwGb_cGoBPXsHcJ{+Ml?gu>8nO>)h4IR%1a~Lk zOD};hXgB$CgEGnIweaN!fiE8rU;e{<30U}YQMsByjWGG5H{BICC?N;*y`*pOMbP&w zRVD>MKS;U}K|i#O_;P^xGQq-^@7Jr`QTF-1aQ^_0!~FxUC%%j+4Ih6J`I10E0cY*Eqpm3@Z~?mm(9$VO%}eilRV!D`_QFaeL)WBdr9Bm zi=gj$T$vmI{UGT^1pUxA#FrnLFLzq_vfttPMjqk*0Z^9C_irG+Uk=FmVgwE2 z;avIhPk}FYvI9E28wI|UO|kPO<#heq7oK-Q1l5@!ea9Z?v27 zoud@^?zY7Dk3xKJruhDl<9nYazS~Qluj47uS|H5#wfhI4vUI+G6UEuM((tC|G0vJ( zoNam@4j-lhxZZ_N_>`Eum3e%SRj-$!R~C=Q%8?IQoo z`M$ire{b{s#7&jYT=??qRTa4hC3zF>AAr@Cyy<~%t7aV>nq$kGb-jOp2SuIFr)d|@ zr+zS&H?}+EO?KpzHw`=cG4tj@i;DRXYaw5rNpyDxp7arTf_9T9H!2?A91Bkl3OxCc zc=BK7Nz>p^RttBKJWmJv&ZP`a%mIBb=^Ob{(Dyu{cmkjwB;AOhAKF3r@(}0CM=bgB zfWz~2Ji`40pe&v5-$EQ4Um9NHM~)>D$JY3fW7iPJ4$3)pun}}ddJOy;)R#x#*o$1M zcD{d~U)8={e6L??r=BkpDPOMHiTUyp%9jV_`O*lM3s6?!d|A_c|K7V(X%F1o#SW-; ztdK9m|FZK}9i80J>Y%ps{kAvk%=vy_-oGny4@tNJ_Yc5o3s(wtSx5e(IX14;_5J}K z6orO4Z|#r?$ueeJ|-7d=d0LPb!50&<~PsM9>dKi7!>m zm*p0|{J37_jstJ6{bAiXefL;+@{_=mkBKLr zFi#d+c#@Iv<`z?T66U(jyy)H+sq^K+zUXG)FUXf}#FwAte8FyqT^J9{mzu`oMFL;E?0}A9yug>ld3L^( zG*7Ypo><%Y{_u-lT6#~LPX<-w9+B_`*6(Arg)jZ|7CQ0;&9U*NuIu*;Q4|{HFm5sJ z;&o5mZEV=?Fm7Q-PJA)!?8h9p95FOxAJ7Wpmqu;eU4btH1-_u&=C3tv(t&o{z8bScByv)SG)Z#o64_aMN`dXKg9Yny$k*8%A;Vi#*QIIXKheZ9ok1c3giVAG<{C@88>efA3XSj2$%gz9AL4M%>Uo-B6*$;_MJ-^k(2}aUeY)6 zrJ(Owu1pVrevot{f_~^b%9qDDUp8w$jE^>Y#Nl~5g~IxMC`;%2^N3@AEDcwMkz+~3 zv8pg~Y&dc3sGMU*HGU0@fnO8)^9US|f=kuT_aF7E+P8}z@oVkW^JNm{%PJ)`smu9| zm7bI@kIM6<5iE>L&X+aK_m}%pv7z)nhaFHkSIC#@K|6oL3+?%`w)6dl3s)^XwyNN! zirixouE6?zthR6km#8~(1rLODu3whbR6IV>Tc)iix4OHK2AWuBccbJ+p)d!E3TtP2-6um)u1-|T#;?R@`aoVshwl)$u#+~X3y z!1{fxw(td)s5|ln&9U*NuIu*;dD*-ZUrf7r-CO%q`^{of5p>uGi#oJXe#M@#a-sTDCuV}LUd^z%1KkWLN@1rv~6o>h~X&3ot z&iAoP#Qy%h&G+};KIY?p9XhnLBKL$OZ^HV0thVG$T%zuHp3ody-mL5T{X!IVKA)yt zJfElL#$zlsn(j?Eb!M<}TqdMe(zL)fkd@1OA%9NP_&<~PsM9>fYK>6|{=gZNSe0kjA zc{YW@`h6%%=lg#qj{T)HTvCY~>p&bUsYH&AB#xbsbL@o1uR$^J>wv&7&X?8B_n-8u z+W9Sh!mqVc&zBu2UzQ|dzU)K!@`OBJ8o|Q26uJKrCDIcjW1Wnz?ZCn zOcZQ5m#Q6)KTUjj%CEIk=SxT8i?22Er7!X2q?|8C&@djDFE!2gO9a0BksVOKPZs!6 z@`jx+iJ5l3)ONn#BYU80ll#W|D{@as_yX(qvD(6y$$ASN`GV%y_)^#P`-Lb94Rhd& zX&0}1>TYAhb_c$&BPYHXcJ^cP<&>cz`+!y$za(^W_X57C0$@@10Z^)0mU<&?me zuZb_)m@k_we7Q{Wd?D;ZmvVil9MJcYzQGqk-?LKj20%YZx)DJ?^ds@*H|EQo7QUQt zc)n1fuznxP()s>U;!A01IDZTBrIQ=@8%73h|A0GrlJ)vwaU*;`=utzPC|)f5Y*;&l2C=B+s`g zEYMmY%=fkR`=~6=KlSE5NpbdcX*hEm##u7OS>`s3v+F6&ev`);ItOQ3ybX>a-jc51 z45qh4xlcHMYtGoumm`n$!>+ISK01RIIdImri~KX^`||$&z0LP8+_30k<=ZcMSLB|S zCsQ5<9R}JY-P&$)cJgxcJX{-RGRN>cgUOU$SH3ccJ^cD&C{lc z zeJ|-7`BKpLyrkS60R158Mg;xPAm$k0QtZKpZ3nB{JK}*7t^fT`Tp~MRqONO<$kT5dcI7de3_nt`LaLd z%hU3FX#@-7lJjLv^Zhsfn5sz|ulv~n)f5W(GJMIO8yH`P59&I5zO3zhKW_QlReN)L zZmq~YBjF0H-^XeTS8(OEBUjKI8&~SOe!q}cgFEF3(=J}`);`rfv)zF!?8u2LhMoPG zTsb33*~(~z@k(N6cOT%(5P>gfH~HdG=J+16@a2ram+i!t9n6@vOK0LsQWxaQ z0OHFTIbV#RVLUKjYMSq-+-v8{e0D(nK1JY5<@I*HWbLu@rMC0^e-2I?J~Qi)zC{f) zo0Pef#QfF++m^JiO3vz%*0s1>w3pg1X+Y6cm4nlVHXV^aDm*6T#^YnN#;?gsEZA45 zPAi+C%-%95{f{N{so9R3Hy5UUFnaiP8EL~5B@_PL4F8JYUsJfG;~T^&6}igUd`pM> z2Vk|yxAb&f!-;QXj)`xuPu=bxK%&qv`8e2O7r7&L7~AE1GaNba&9t*0lW(GutPJC) z7OAOlcapwC>1F?x$+tpf?%c)jGVRaUUOS!}r>FSn>Zx7y?_@q}j-Pt2J^Oyg*mRe2 zLuxk2dr98l8|oPPo>!E)t3W=eAXg)_BnWIU$oo`ydbR+OoK&45B`s$K(cFE#1?dLtzZsS1OaRhc8Z!d+kVejPgW zrkfDtjo_OZ+K%V=8v2DD2t0nOmTws=RNeixE5guDoo}gqNKoyh|9#}ZkyMuRjf}Cc zCs>$dbcpVwIKgnGf0)|BUXB(_vW=hePoA)qYss63@#(CjQ(7%!bBo}6Kxq`hGBGDaMhdh}< z2be79NwUV1i5c8`HE;8G$$3)kd~7}OWZep_ojOmhC7vXOkSF7aC&_Z27{S8B9Ud>$ zG#@MLLLEYfwnE^^Ljq4Klc(7Dk+t@Wtz6srSVeBi**xh3;}onmdD5r)aSEAZ;tA|i zH=d9vbfKIl7Q1+zixI_k`8dUJx(LV>^&w3|Fx zq&z)0U>TpJ2s}AMJW+W5m1*J0AJ+2ie-wi!*kqS7Gb0=Hy`*pOM9}xVr#!t1^n;`u z5%fbXh$o$yC#x+y>Ev*Kqdvm;1j0ABygv>GV1oo*LPe>GcP|g#JUA)fuyz!Rpa-JBD9C%{d z*^kMS!7O3hVl9kMvL>d^0-pRy;0fAIp1iEAo%_CpCxZo^bfVvLl9?xIE6wrA^tI|} zQa5;lO?JhVOw0y-FXUOlzk;KAA{7N$Q`h&v)h$PX^0*VgyU9JW1(I9YU?%An;_Bz>~_jJ8k2UtYr-w z(yscZ_Tv-vY@YOk@d;L&Jn2{c_=L1C{i@yQL7QoX>Fg#u5|Zt~<6rF`xt3r|#mC&|Q<6y`~$g(r8d zRj;R37(BryyOgDqvO(WV`UXz~eb2W_`6|#4l5RxM5A`OVT+2LJZQ;o^4(m(%3F8we zOL^iap1cU-lM~33NqqU7K%SHkPgFTiRE;MOWKbuZ!)EBATJWTEnci*1DP>wab)HNj zo+wu$Pkh7^Rn8NAf3113@?@dFld!;(fWVVznw=;48|sTEd1v$Fau}arwaJsqs~?|` zIVPUKK6T>>i9+|wd1A4P*SU7itS`39d15$n;E8ExKPFG|tTM)DtcCGO(Sp>+fhR8t zJVCq3lXsOb=6+}4NuI!yYltU-o8OI{jI^4_lh9=jkFyE4i9 zvi^JQ>%@~hIZyI5p7=9xpL2Q7KUoW&KN&eY9=?~)*tTuVlzxweBnPcJ!>{B$tgKtD*j z5kWt61My@c^JKM!C%F!PPugD?pFmm4lg`8w_tLO37kTmkUp{k@Cu@i&`Es7*Ydl$* zL7i|8n~k;M$;2{Mb6uPd-m%{rH5;G4TZUsT)s76naq36N^0_yIJAuoQlSV z?Q)(NjvRPm+Sw25R3^+fG^`)67RDzf{?vZ}Pu>xDf_9T9dzGVen^^uHX1>6aT;fR{ z^Ca!B=J@2bwd&0zZtw(~?24=OXM?_%^bMW}`ksrGqpLtaNV*Y0KU73KDQ2E5Z)J{8 z?pv?=NZ~$#()z zD&M`^#*?fG7u3`Er1)%}421CsR+~H-SpE2f%rWr<_Ng0BNEEtI&J&A0p8A=ub1E7e zw##{9IC9{LX=gvIQ<+d~XjngBEsRf=u1x(O;K@dTCulc$@~v`gZc__SiUpqJ5l`}& zCz%$W{C%x@3yB*%!6v(uk5^`czL)e3o(TG$OO#`)KtD*j5kWsRlX$X#d9vEVll#}J z^GM&ulYzqc1j$di>kKIw-%2@+3=zwK z#gi3h^W+K`pJ27glPjtppO85wp1?kJ;|YmEm&tizv5VI^b+fTyyPPM6BL|+CcJ^cP zWQCz&{eZPFK3Vft>LX%hhbAn8T~{ZI+Op8{6cel^89+WfDYcqlPiSr36!Ng z>G-&+@1Fd4IBEJF`tK)irJ}>6>96R&pBy5dtdR3$g~pSe8Mx26yyv%Og5bBgR4q?- zl&RXci?_r0WcnTY?^>!!4fM^%EnTMU~`pZc0lDOE$IVoDp%cO zAD>*)s-DItWoPqbB8*S4+T_W^>c=N!j)^C*Pu+MzqR>Tho>=Vh)Q^0f+k3aB|KBkj zIq<}^i#nAFWulZ`##$JkyzzGGr@)gv0#DFx@}#lS)|Y1CNtwWt`NWfA=1Hc7C;wfm z^2KfN1e@$q4!@lZ`d-pEcp~U~hAV9YpdTdNh@c-@N<8s1PgYxa@`A(jK_?316DUi0 z(wTVTemtzqM4r6Om(NV(Nfq&=OwN-sjVHS@;O!h=8*9Upon@-#zBmfwlbOhqw|RW> z2J&PJ@uW=7lQM08?Zr4q#G+Eu-?KFw#~p$@q_6|pzp22J=zr}z$@kP3Pd1#*lSwc> z!D^Eyld2z|kU1uvz&>^335i0N$$4V2$5X%7jwgmA2cDRAQKvFt11Z}#SPSEm%AKh{ z0Z$qT>xs~A^5kNrzi)(vCmRHwyg)oz!8}R(2M5c@f(t`tq-5ea^aOVy4~lKrao?cz?bo@hVvWG9bLF2nox zAEOhoLCzB+SQrQA%9E7IcAgAo2iSb2tH6`WKhL+1Pg0WVX?zksn`PbR19 zEpz&Ndostw6WFJ2JRwnNn4BjTyLg>bHyaza%jY|WBL|+CcJ{;Es!RwQ8rBb33*(cI zccuOeJZUKK1nnkIE>Q;fGA%p_3p`mtJSk(IWLkJ~aIH$q;!K`klU+)SUD=@TC4GY@ zf<8Qrct8O3gQOb~^h0kDPu^mlthVrEy~FcBCJW;eC`)LW%o`f}?#K*vsp`Do^up_*Yl!g^=7z!S8aJQ=2p^-ZzxWQV|$ z^~946%#*aW=J+H*az9pVvMVkz(8JQae{6PuddLcgT5S1PkNfTzOJ9g8_?rdF%j-$_Rlc zmD3hBAd|kyitkuY_O6g^+h4IPZcy}w{Nf&`9Xg7H>Tp8z^YT-##;K>H!Ntk(( zY2nEQlKZh@lU+*h_#DvplD@$cLEp1T85aQkAn8T~{m{q6lby_y)fS#?ceo#`M;M<# zS;~{n#1nUESeb@Ai6@>Y(~u|W#FMC;CsB{+@-M9~o}4+GCxtLR!D^Eyh1HKw z$Q%<-V4u42ghZjc2I-}HgSgDTaOVJV8LjODlutf7mlM--ypmQ$D`@M4R>_hJL!{w(AQ&{rm7czpO0I z`+MZ6CayrvKKGkX&TQCoL*vca@qc==%|lmTIq+!Sb7A%Nw;Q--w9edke6_k^=Wi(= zHDCVGuWu%F-SGTZ!e5J&$#2~*T= zJ^u6MP1b$edhfE#UZ*A6(Hs9NGV<3=U{@6R`O<8(OiJl%+qI}xw%gsM-FVDZ{waZ?*GrH)u@Fk5pE=do5 zu>bzTg$FHRnTJClB`){NA9ioW}YCB?Q*CQi; zYBI6uye4}O{Pp->CjT@3!)qoae*N*xv@Od&N@}p~c;(h7Zu(zw0sKGSh;{{4i{Sdd jv9Z<<8nqOK=5xcbjCG1KoT_2NV*&l!|NrfG-2?vz=A?6PKGhl8<)F!rTmySOmQ zGK7$|5NgcudtaYxzSs4=ANTdU=KkZJ`;Ytg?eX|tzRvYN&+~FlURRxSCVgWf8?$&_0h7yJPJi?LF|e`90(Qq!IPYBFVh+il&Y(ia&-=qKK>Wbg+c>o)gTixUw!d#1c$iJ?7DgtJ`s^XYH z3I9{D>x1dP&&{U*gDY|enw*<6KUwA`-In{_uNAhO5?kXYnQyD{`4oVy%CBK^Zoz!3 z%>R0U&D{)7r(%@wdzSh6G(MjK$fxpanw&$KpC*%&biX>f83C#NN0j%Wbd8&8GnDReo)gb1UZO%G{>(tev-iiabV%gYX?R zKA!?`Q2BLC&aIhm_7I;psRaXjWIVit`OU&NYkWQhU{?7-Cg(QH_x2E<@3m_$&Dztk z+gbzHEcuN+fcz@AuF1JAbNyxR$UY;Ef9bS)7bT7&x1%OEp8{}H`QMqG+c7^#=Dt`h zZ1!>YnPy6ygzu#B`4oVY%I|A(ZqNKsnSV_{T;V*1xsfL4j?9ggxxdcdnR}?t#j}(+i`>qd+^XP zId@`ytjzt*)4R1iZ*3byiHq=EG(MjKa8daKOwOH|A20LUb@;HuVR86ON?e8Ss`2?0 zfUClW!=kZA?#+1kt$}M6H)9XLP32<0cV(_c=I%G@anpXei`?#- z+t@5$oBbe{)Df+!xaf``x(z@NiHA@j=51@$3#eV;mx&AVD{zEC* zl~0*BQc_gpE~?4RrvQqoeC+pL%ny>e@3igel9_Yp3?)9o_tE%#3cyF@W54%ieyGep z+kSG7z*_C_>m^^|`)YhX1>mdlvETbJKTPHy;<@hT>@g7w4P3JnGxh+Asa)*$@0c4c zbC0-c6ThS7grk)3FN)dMGsQKz`4m8Lm5=@2m-(?WH!1l2yYb!MSSTrBBx8OFjnAh5 zN~nD7_kPTem-%13ST}R*vL30F_zB-n%%}Ywk3%|U^=TiXXRX+CnVCI_(iGF`||MKH4-3Jj${DtqY@%a>hzskpcAHsa^ zLZaW@6E+lZ+O_s?&(#Wl-R~7tF82FS=K9OrEicsHHf-8_{CcCJ$X!vBn@<5$RQcHN z!xd zDS)afAN&1#=Eut14gsaAzVG%1@0V)Aucq<&6hJkVkNrN1`SCLU^t(Uwh&a1{0wn>$ z577903Lrq?*T8-s&HO}}zw}p!k7N8C`O^!=EvT33#>D{DRW9~>40A0qci6VmueU63 zv0_Uq_E&lmrPsNaOP(#a1YYbeo)HU`1 z>Z)Aq_leB)m$|D~beZlk+O9Sw^+fJ^n%sN}pq|Rde*cmAK{EIC-sU~26LwFeq`vU$ zYkWQhP+#R^zfWR*sLUUnd_7>!nS^DO1Peb{@GJnLldmiiG#sjz(;1zi`@W**&mY zASDfj-%#W8DS(D5ANzeO^W$ayT0frsd70bjxs)^#ej|;~rvMr${F>PB)0m$q^Pidf z^mFL<%WfICW@&8f0W?;**zePsYmvG4dR4P?T5ey5k|xGASr0VPrRHG!uR^jnAh5nyGy3 z_ju-~$@~|`IsP(qjjx@7YZhL703j+D`+XL3GiB~5|A0AvyzJPSlI9|Jb4_kO1<+jO zW53U4ey+^za;a&J8kr7Jl(Z0j3ysgG09vSg?Dsj$Hy0NDzWYv8M%lf4uTc^z{7{Y0 zrvO4#KKA=u=6e?w{l0Q;K!oK){n96hLd0kNv)Y`JpoZ=9wQ}kLccH z4kc}b-$vu}DS$RAANzeF^TTBReRoHsg*kk9V&Iykt+5BtR^?*9Conf!=AN8g;)nE0 zVSOoSCvvybsI z%>ChV)YJ&qMfmz^Cy~38CO4k~=%n(o-`U#DkQ$<`)36y=`8%t8lO)AbXNJ; z@5`BQmH7+hJxH&3V{(3fQ=BQ?)W~MNO|DHCB!jo@2-|D60r?i@%gh&QXR$Ne_pv`| z@8=Nbkme9%PBiD5JLKP!-_>!1qt(&hDc&j5sZoLT1xy9|6ihDY?Hucz<{aR%&?VcY zgR8~W>=x~2b@OwNb5D0~k;Oe?CDx)M4@Yi0=(kAGQC0zTMC=J`*@%9_A3%w zB&|qL(Zr(JMLYQH@p1Ky_O<%@7mF{JUaV2^_55i?@y(Kqty-~Fk&8Up}bRW6=`E0~)pAKzSWyMWARB~&`sszIk=Mfxia^G3)}A=aBFdclJ3IquJQR4KzEgo=in;lo4v(3czU2` zx2ns|=1>wQ{4kBrrvSoKKAwZCneXi_&cV{}J{1j#T*befHf}+^^e`?4=%I4)99+X( zf0=v6yuM@0md(X;u&2n~Q*evr(4p0s>gDt%cNCE>yk*Z6!2AYA3+ zIk=Abp)&vVyn$`*ICRaWBtrNR8lO)AM5uf`2iG${Oy(c{>DoQFg<*gD{pGiR-S6M3 zTs#LiFgIG}K2+sm{;3&FKU306C_`S@vCn#| zvBBI}KRh?Ncv2`r}_RF{h^)k@77+|2v#eUz(Tz{E+@!rI|0oym?^QwbH?m?Q| zdp}}W*mZD^^@CR#rJ_RsX?kon0ncS1m>O~kXK11RCqS-+3c_JHGiPZuSX(O2&)a<2AYY6u@|ukNtj}`9U)G=pSlkEIl>Fo018_ zpP=#i6u<KapHu$Q%!`!#D02U($<3z#epLC`@28m`D|3&U^n3OI zw_jfT{E9UGlaLqE=*aMiXa0 z<|oVCP0OY%_H5)dnv$u)pQ`cs6u?xKkNuv)e5=fV?RKBQ2utXEN~Q^an#Si-0Mk@H z_Pdq&X)^!RFCm+ppS0O$;F@K+u?H|+h<<-}=={Kow^q%gBwqOO z8lO)A#H)Pl_sh)p_7VMFy`@)~p=%oc?ejCU{<`01sa)*$E6nwmxx>46N;nDy@crbo zMef;}+o=TiW4R6h3mRpy7v{Ev@?-@6+b zfv>mD75-d}&!+(9s(kGCYs?Rm`8V8mx0uJxd}`pDWuCDIFi+)TzfQ{@3)wrEOU<>v~6$I!Ov<@vRL?wH9nsLSgi7~ z-&2`ymH9p2*c{#UZgvAomI!}|#^+N2OH@Agdm8i8Wd2FdYra^vWc}YhKeP0&`+cd( z#eTod+)SA}u1t+OXZBkwQnF0sUZ%;-rvR3zeC+o-%+Hm%Es;@;zXqPi=V_M({=TiWQDj)m(9`n6@MZX8W@S00jK#qZH zmKDYxzzUU%{r($s{blaW9|ld|+{_o>cf3;MUa85=rvO%}eC+r8%ny>epKaY2xM^M% zK2NYp_^UKNp8{B=^0D6^Fh5l0kIQ%NzWK}sd>(kU@K@rKogkk6=pH34fi&=TiXdR6h3mW9G-p{5uxyBww6@`%|)B`0F)3p8{B~@V~=; z&tQI{%x{|H`!IBf53g_I7SziI<6?jfDi{0x33Dwn_tWrR(etNWPo!j{aZUF1`$kP} zJ_WE*T$qx@D+O;yo-r*aOjEXQ@2x6 z!g-4Se%)ql0c=ybcn-d1uD^VIUvG0xpVW69{&yzZMegmI+@w7 zn9ck!nZKD^?1}bE&UQC&&9c+j1K6o@@f>{1+-RBGy<@tlk~UW=j{FKe&4Hd@f`fXT#L+o?7IOw#R-(568?9^Ny1Ok_VIPTdzwqU4b94{3Zp1#n2^ z<2m@5`R3x{9BjX>XERID@%1T57JjnE=TiX5Dj)m(59WIp7ybV6-fFkS9oCgJaLsbq z*aJAMaY=8jGcN^jWY9#C>r z_(wH9p8`0l^0D8)GCx%2zxyC7V{m@+WJ-P!{x2GzPXYX*^0D7dA?Sz6`~zwaO!ciX zgFnq<+=6;JW?T$#Oyy#~+b}m;=C0o0@xCQKZSnnG$3^brn%sN};JC`ieh22q%G@Cv z=T}<#`6+(AeM0yrG(MjKIHB^f-))&6FY_1bb=Ado)6739IVt>;8lO)AoK*Pzu;24B zKT+mS+Z5KnT&1I*3|zCEGWGyYsa))LJLXzs?kgGYT^4;9i9K`LxF+lO)0*6T3gEQL z$9}hGezMG+bkwf#@mFu~`N=cFKcn&a6u=pkkNxhze5=ggGXMCK14~~Ir{t{g&uV-= z1#nj7W51i3pCau zoY&;$Qvl~xKK8pK^K)hH<{ei0jIv+9h>{fHr)YdW1(2fhvEQATZ!RJF{lJiT&&y3( z6{f^0e5=OiQvg<#kNsYN`Q9Z&zh78%&iXjVqoILomJ7xnzy+0y{a%o{{xbKg<(G=@ zyRl^}B^O2Ri<;bg3gDv3$9`u&Aj}jbb0=obDIv zTIOz3bIbR$+f~^{2@$!8CO4k~ASxgG-JSWdGIwN~THas!pB_xfRpDRN_;WH z_h5d!%>R;fO|9cNb_ylegnv!r^C^I93O^G2-IMu=GXJ$Z*0pCJ*u0 z=TiVTR6h25Vdh(9{z|7Sn4&6u^O};I!oR8U`4qrSm5=@I&HOZ(e>(LknOknPpMh(Z zUyVJ0UsW#ldlBYl%G|qh*4#{8(-B|)zhzvL_4_SNZaxKYOXXv~7iE5~%w1ulfBL?& z&w?mP6@IG5=TiWwDj)mZhxukd(eL9rdRz~7=#4#=Cj2yw&!+&=R6h2*FY~?qM88L0 z+U4`E`9@yf#x1Cq+s4HJw^c6odokwv%iPtRB5MavsSlbcTg+)?@1@5PxP zBy&go+J954Vl8V^a##3wH9nsLxU2H9-%Bt*ROY7*GY;>0zxg*x?g{^%#^+N2_f$Ui zyC3tzWPZ=ZzgN5L(#UM!n&mfR58yYIi~U}bxzRFrdH+VyOCRrzq~yNHeP5HCPXXLl z`PlEJm>(;1d(7zLyY$f08I(K_{sWEArvM(PeC+qq%#WA(7e&)kN80axK}ovs(=|Sy z0!UZ*{juN6Fh5b|pFB5W_|CyQOB=Xmd1&kbJXE>Z?`4^5k-0zFZ|Ya`<8FMP&LiWR ztY;o+a`P#GM=Brty&UtCW$w0TuC#x-sncpo9t;1m#^+N2k5xYQdwJ$tW&SqJ{EGZp z<;4g}GK8O@@%a=$hRVl&_h){Z%-=0|?7@^%_of-RW_e=l0X$K;*zXmXn<;ZwJ9jvt z0sRz0$y1U0sU|m{0(h$OvEM5)KUd~%aq-N(d2Ru_DS0ORXBwYR0X$Rr*zc8?Z!RhN zeP2vP_lJk852Peh_?a4?PXT1AeC+qi%=a!S`h8u#o^9qIIWXA3HOq5j58%1V#eT2C zTz{FnT0;L#d23t@@K)twzt>`}MdpsE-F~RU@MX^_c_(te)8yt;0Pj>j_IqvSC(GQ^Z4=4O zw{J&M@?QAwH9nsLc(3xY-|H~nD)TolvN>VA*L8e{`0a2V7~`3 zKTYNz*xhbfN}c6V2Ci8?7<&L8R4(>=UFK%W+)wJ)?H1f^VpB>!irgPHx%m{pN0pEL zUXS^?GIzddTc>^68rhW+{*)d&w?AooJ_Ybe-G$01BU z&h7rVa=h}@OREg#=PCuWp2J-t<(k}t+JS?({I+G+^$n$!9PnRV_^1B;hkxqldr;CvJT6@{j|-pr zKmU0Z|J3`DD6xpV|2N-X^H25XKh+2S)ZZue_bAO2U=weXZW9Cx@#m9Ru*XbT3$7+V zQ-I0U)sX2&WIMd6$0{w%ugw@bi?|@ zleaIf@~As)@3#+1)wCOWa99uLyU9rpQhY<4dY7_Yd1B@A1~Xp=H}Gj-|DZ}p%e1ij ztD@6KPg@zonBZl_gQ=re8ru82b3bvnZ&0jj< zR{o8S;hW|a{CQkN;`L`GAK5(E`f7IYsx#DbtVP7;4MPuAsdw?HTW+(Ka~ryz{M4X( zvx$of9&lY!euvGb>Cn@~KVou)?5tW*)#BQYn=)og_0Dx;&OiL#yvNnA?6_hTV-G#b zv9=9(liGbn##_tIq8B#JY#TJpsiTKl$NJg{Jt&&As*UJq_x_YKUN)O5_Y*fdA?hU)2ME7pxn;c7J2%CRIw!?AFPNw4JQa*ijFR^#dt>*D7TRAlE*KJtcjh|Mz zcd@KaZgRNt!?VG6UmV{&^l!fx`ycz4^8eu13_7md@5{i>+3z(|51E&&lBQ*ysDM?|T3L>+SWDcR%gfCntCI`kl4c zzR%8rI*b)RQblcP9`J7W&K=taGsbdfGcOe@U%tKA3%9E4F4^JL5(JR0!JS2Ji_~fY#Q<5hP_3~q7H9A!jjZP|lq*3{&s*=1y zylO~HQ&FdB%F0EDv1k?ZkC8@(M^t6%Qj^0q?R-^g?~xI{Wi=}N?}*4?{AZS{*~%Vi z>Q+(t2eYw}!Sg1;+^4IH|;Gd4Zs>lY6^{uF?p2jSe z#rUU6Fk2s?V%l#v_OGC-t}Wi59->mKSnTvL?~_F=vBMx14W z?`$?vRk4by;_Q&2yP3s?)0|+MDeMDZwZBUD+Ko4+Hd94uRp*v;R;jgYzfUJGb#=&B z{O}D!vP#2P*FkK;fDGo>GP9K0Fa1HiJxo$H^jE3dXB)hH!a8R=7)#&V9zFDWuks=I z|2w_I4|%b8&1C$KOj}HBUB>)tCj@lt*tc(5nyQY@3&*rqwb-Cy>W=5jN7QGv2kG#y zKEJQYWncQLBl8&Bs>zeuZB^w-_2;PadgTTif+~lUEyJQ>Rh_%z(4x}G*FzcGyZ80i z&z}9}o8Nzr+*FE{Q-x^p-*hUknkRfSRN5XYCe5$h%pZdw6W1=K`+HTYsL~dvrj)Me z8-OFt_x_2%8;AL3umoQ034@g?j zg84+6OKHRX>?X}LcBkI^6&<*)*tm3Ek8H=4(p}S8t%m(2-slyV&Q$%e>)f8XKnGX5_4N+rXupm7s_f8wtMe|PXFJ$&>?jXy2^ zbodLvUuFD-;4cb)9q|{Bzdra&z+V#n%=k;i-xU1q#9u!CPU7zl{!$-3dX({KTn5fl zy7bD5Z{|(0mXz{BBB~N>ihK zp6XVpW=3R`w-mwtiSpJ=N-I)T363sOshTxH6b2_odHZ)YMR`}mXYLh+PwZblQpGfZ z6ipRE^v&>%b#56iX@b|s>(bd&%aYCUtQ7tQ`)+wW`#vY0E#4fj4J$Bo@SVLr-s>oW zr+0`Ey(ai(-U@4($%2RGv8lEth#X%zaHQA%x*jH#K6gEgoqZ6dWrcgDzGUO_)4G1yjw0`@(=259X^A)RFIlO6rzKYEyD-*quS1fwU-k?0 zW|uySV&D3zKMAAJfj8)iAowYah14E~v0c>V*D%%+?^p_l*0bPiZjUb0G1#lqVDDa~ zp}((+&25?C#YW%~Bz811(@UM%^3Y{9tUl@{r2OUt*2L>bN`j=RQIH_9CJ)h?l$Ib>2@XFzu0#<%L4xQB5{b?M z7I8?`9;ydHbrw)t9|_gFv^10IX5DZ$&9Zc}nU%)BVBaq{vmbKIEN`<}9bRAv(rBh$ z@_IekN7bjc5xt=LYBN`T51VFN>Y@70)EQWe+=+>$!?M$p;=Nls?LcY_GF9Pz-^kWnBt<|I;>s!Z&jxGiFvgx*E9tz&V$Mhlvuc3RXLPKEP^3f%J ztSOzjDQT1f1=F$}xqgtM4i4sxKIkCPf0dwMXHxL*0nmH27m(w;;qZjL2>_&w6kKUMdL-h60|c%@(ah)%SVaR?WjL&2 z!@OtVpiA@y3yDx;h){-}Ae7|zDKgCaGPGpHe;oK~SLz#fled6Ta~Vd>5lIS+n!8{W zUE+)+Hx{i=VBXMDi6vIKp1vi-?1sU|V`(&-WywY?dDD>m8H4o~#8N15(yYJ`<2#p4 zSZ(GoN~mWr27s46V%gebq(U`w_W5B;}XB5d(hw?n4YqwSZET45cV=-CR~cDe6B%iK*SB zR8^_3Zv`p4;~lw_W7uqq6;i6{I)1@LUV)S`kRr0c(A6a+8&Kbfel98^F{(XsOi6|A!IC3DV3-Mf=S6j z!un_$2BVKbK9p05?WFTArk4!7NaoFWGpV?VAW~%H>Ob;(_1mIRy$d2$eOcccpE*t; z(pZ*h$pMixM5HTm#q3v+>q*TE!u#Xjw@}R`BDHWM(h?A9MYbcf7Krq6T)M;?1CZxw zIFT3&EUv-2fl2H%WqDcA4i!5^M4}Y$Ng^?pQe5Nl^zI=iQX#(jRC7k*L*t=?c2n7` zQxRZNvS3o+ry)F>A|xfh5R%HeA*oseeH+NoLqSpon`2oCB$Xv3{R$*qmD8xG0z)@9 z91Uz>L_Zgf^4T2QN{_Gp9%$i1iBxwaQtCp(NJQM&a`Wp|nNP4tCK{~-jn-s4bag?a z2A~mdB!Wg|iAH6SLv^Pds=-Bs9L3NGoN~BeQ3Et}!@#2Uc!#oXERsf5=P|T|Xf&Q^ zl=&Och|QI0bf$5_lEU(f)OM2(LMTB}r^qx)aZ8O-UVuhEZXv{~H`GT%$DSB=JcP!v zxt3QDLOwJYuVRq?CK4lUivmM;-??&Pl+@6OUakN-%;wr&@lf$T2D+94CDxlac)TN*aXg!6Sp^xjbRECp60eB_N!zl(5Jx0Z5~M+mjOgf6 z@(7z}TjinT1AJ0nQgRfOEL&(8EtDKWXC6XYZHJP(vmM2aphhDc%o|DQpr!w+g_1o; z$(w|t@%ZoKgo6CgG3e7#K2#(~O<&@}a2cI`B3`827En$KF5)n%UOM~o`ulg=uF}k= zv^uD<-hhwYOM%e@Hs7)u(D5Z0U5hJb*F}1yZB=0C;d_To=v!R=NFCx)svD2?f=36l z9j7C}qsGXPcw-QF>t~besGGY3Y3P3j5~g)QXPP?abv45D?I49w0n$Xaz_JEFDn~%N z4j}zbInt&&#Wl383k*G-GrPfIM)Y$5=@?sJTjMbR9swDC1f*pElD5z=76Skz8cH}C zN;o0q6IOu$%k@k80VXb*(NR!ABcq~I@+?gA?AD8 zj?8E%5&^{Xh8aC{xL&4%m_`EAd;$~lDQS+7F#^rZS?CiZA1ZRBQ*^~vyhv00RsE`{ zq~b5|PLg?)0F(Dhi~Z8V?*?hc5}5Q&^d0fB`zm0vu*H`30Fys~sU)tL-4I!lwrzpI z==(JXQ&My!LL2G%A@iKhth1FTmtkb3&H%(9zbHDovWz zs^DIgKPggGe4^OI>mHF8H2(6$tyWWgC&UF(AgLw3bNUO3B(bKvycE9>OhVBKox;2;P71o~ccUEz(y|&n zkXBNjG%f2m213YCy2EGMaS0Sj5hZ;!MMs1E?WPI>ORgHyDy4?RwVV?b`Aclo_-=?K z_nDjKGBR3GM_QlIYo!uOEToyfGsNt#gwj;D#Ig~gRDlNOpBSru$*EGi0z)4PCB00h z)MiF>bj8wZY>927hn^CKd;rA~%1CPeLW4h_wl!%9A~hkxUqmEb&UQq_A}X38l6YeX zj;}zGQ~{CHmm(=Y5mh7{z5=IPE`q5Unot($x_f(d6Wp=*AHUZ&$k(siRyvVMnG zQm|vy4uAYP$#Bwc3J^JxTtWhflnRmTM^^Nj zMntO9TpxpvKR_YUbhga08APf`M7kMQ%x=l4QilRVKd%hNP)tgSMylk-qpRT2@7WHu z5j<*+M2Rh06t>(a!ZHwu5TJy|#TZ8v&Upc5SB&>>Ml(hRoTvIR&ABqZGe zlK!SVX*1KbOaPUG>KM;vURZVD6nBOzBY`OEF-!1k%Z8udCSdyzr z6jX9CNv2Z2&@vQOtdyz@it3X}R9XRg&!SQ#x8P!-E%dQab)XVlGuaBuRs>fi3a-B~ zaBs_5Q>Ow$f^!;|-ol7pE-<~pR@ktVLz z!T`ia3xJ6?hT-^16kC-LTLUPz4BrxTw%}x&MNqXs^TK*`>M0*9rf;Y_WYQI{rI*=w zk>Y1qo zmk$+b(vNh>>*$4)RGZ7C(ao`VC(n|CI|gAn({Xxl6<$lK-%8&dAA69(q}j}B*#;(6 zAtv34D`t1)EU8O@A@LbZ@&c2*t&Y58FsT)CB;FVfCRK4VX&_DB)WwlSW!QR3k}{fc zDEU3Dm;@+2S>SeMNt)yDmo#`wz1?oA>QPK$$*uzT3qq-?8T7yHpF#;T_N*t;R4kZzXMu9^s5oUXVo;`v>tn?e^qx1?bmY( zZ6FHGB?@IG@dDCccw@S&UySSPcje~sckHHW0zh&B>6rkctVkxhCwi*6#SUxGM&AS4 z4aR8Wu``FQwCq6aRHHEnKCpWtEn;0`d5f*I?eNe}i*D7?EeYvR z6=+uv;qTPW7tHaqI;ITA*lz>o@Wx2=s7CQp4e>IF;-x4N$xvoBb`WuLK!i*iG^gxA zH(G$Bqn$+~UC>vP4(-qB)&~JEil7bQP=)^Jr)g4Ka9Q6ppV>(p86aCc<^o?;KpYC#Gxt?R$lNzt7=m2UMMF8w-dk_6|nX5C6kk^FLB^BYO9aO$4yp% zGmovZ>;iCJBH-MQD`pQw62!X27N<$!sl0aWmH<@*Y${nDG2;N6wtx+9j0S98a>8aX zB|z&a0oqKg$^YcPW=+P&s>S=dPJi8Q3ZkVnUMPAH_3yKvAQ?0bu=?NFpzj3%QaZxJ zQxo5Kyly^QZP^Xf1QBW;05uOO|JhQpyX5Guz-Fw$h+Zz(oMNkOyFI>hKfs^{V1p8o zx;lrA)yMg*R{?CQSsh`M02>2f!y97&n;?Qs5MW~_*xVOM5AMNsFy=LZ8v~kQuL3vy z@D4%lxM>~2akGVRvyhjE?htMa*)nd<`Huf{TL0hk?4}xs7%b&0)uCtQJI};T4JBf9 z$fLvBmESgv4apk3I)EZZy@0L3zwqzMYS0LLh>`gSV;}2hQd~i288{j3jPM@0V&+}8 z#s=e&n|nfse!=L635oip$AR+_qNaw`QF$tcemek%H^!nz4T_o?h?-=I8uRy*?a@ch1Y`BL0_n-g&6<8_z|Aq#iee`tyAKZ73`~M>Thxz|%PJ{nX6GYSsZ?EqS zYNZmjA~eC=1Gtc_wdA>Z03X2v=v7^PAU(uOEyaLeI!6yU0o)yc8iXM^09C4% znJi@Jdu*)@r#TVqvHAJb@&MMdI@nC%0c$FhetWA`4bLfZWz7T!0uM!DItW5jMb6-;)in-XIjl6&GMc9+TcC8{k_AZN&!o zhOhx9#A^C?41U4{*l80B)&!{;E9a|WI#``1_-6R}!3H2nwZr|fgpup3f#p$J?C%eY zrnb2Au+kdxYMQ@!^h_V1LGb~qLVCPfclK{rYkKvk;zC3ReII;5L%V+4cNI*4-v;zu z#MW8%;LcjJYkEZ$ZqHqdnMzLPd%{=XmC>lU)H7^=wQ-ZJYjs@7#7(vX*#N0A4ma7F z{3eS}#M))BA(YgI{t$^9?KWQd9E}_8<2FFK-0(lz2skt&7@tV?0+xlC@B(TIG{Om( zS6N?p-LG5&yGj4Fs;qDVPKOhaMZD08(dcn45mej=*rcPrFTS%hWzMmft+(t&BA};4 zK!rr0lw6yQC-Xw)SNpl0WzMug9&d0`Azgl-g`Cw=O)@2v`qU!OK=h z(R^eD9W~q3jJL)kEznb1pr7MlsgxEh<7t7174Qt+;scQ%bQD&=Q^*kDxIx!2Z{PEs zQTBij&?`L_rjphxl6X#s@ork0X7P>3|`r0)m0hv9xpd4n#CZLsV^hFpv8!liJ7E{)rN z9&Z1=utNLgX5qP+*vW`auEgO3w!yZ~L+t?E<3nhAe+kq3nuUfDj~f87P_1ihb(~m? zJfPF)B@%B;K!;kCJk&z+FqCF1C5gfSh*L=)iS(fpnkP=9+rWld{AiIrV2vrP4o|&y z(52H#x1>9X%t5jmDStqfWMKZ@Z)=}OinE(S#AL;30VKE0i(3E#Eq*3tA*oqseShee zj<@6bEnyoiSovWgbP=WTc_iqEnTpm(hKnrM3iqVWMs##(c!q7X?f1~IB0gws(va%6 zdhWQu2XV6iVkJ)3#Olah1~ED#0ppE{=nz60hCstKO2GR3h-nL};On4y%cnT$Cql^1 zXy%?pUs@EWE@IxYimrSoz0hK}6#N6;RLY$y<}KCiA>Q=@kN#Sl&s!q9=o9eahb!}z zrEHVs0O(qq<}Fej{D5NKqU{Zr1lVfueu@#W8MxZq>R4i-Pe%~Onkt~!bLp5-&V`t5t)ZmJ`=_1_x->&Vj=1d*zFjD7%g z7@;t08QW|*2xiqGW_f{GWoYuURlh7i=jcOD2bT}f5@SSH7r4%{&9;Lc!=MtVQWxNo zwB_mqSJH+Es{`QSvH`XLvszgl8LPmo7|djNV=|CchmchVGnsTCORD}8Fhteci#Xd? z0#q?*W_%7znSiM}?h~1$x;&b86QWiSq730Qk#W2)Ph|3E9Sc14vFbOwsjlFW(*}rW zVr7IU`9*Aib=`Rs)m5JeDMw=H@pxL!wpb1!p6b$oEQ5jSEv7NrzGR(n*#M_>HKLy@ zoIYY(Y==B_tc>9vO5sE`+uDVOF}&jFW&%VRMAz2pNLYuE=!$US4GTKdrEsc?a2ifX z%1jhUx{ULEBO<9Qnh6)sjh4jeXy=T^L6>c#7h2(#;u9#Cihjo{m7cC$KKsM&{v*!W zO`!rvPWzvm>E=c3f1z#?s#?VA2SUP8ctbAX3bxgf4+%r*GQ4qxYDgFZ5&Dtg!YLt} z8f!#9mxSloR$IP@gjMi4>yd<1w5=m77dWTO{)efZu7lNK*aR(NaWHR8L5EP1FccDw zAPIj&%~tw;EBtiQcS1pa@Fn!3b#OYEvq%aFk3L!fQZ~!A90ri;5s=hzumj4;UVpeG03^gh3sreM^-(N& zSOGhMNnNat&|EO78*)9~mN0zMvDZz8jNTb=MDq6k`-7S=lzr5iqGfF-Zd^m6h|T1adgIfHb|k5&c|1Dq`Dg zM?6MBH6Y_<%B08%7|IW;1(3Rgkr9?UZG*Uz>&)d7>8KuX?5JOAu7<8cw;&`yiBq5GGb{o z-Q9J+I6XJN`HSdKx*26P!dxJIp%X(CERU*eZivs zR>$c>U{O!xN4zl;ENbXv(OAllE>M1SfERb$6N;YhN`5Yi&c`~I-TCX++jdhUfg-uM z8+bR;=}CS83m}%Z8@Z9FZM;4iGK^P9w2JMr6o5pH@Gm6aO@YC9P?PebTw2^tBA0`U zMziCM=Qj}cw-i5)QD)* z2sFx|`U1XrS4 zyLXtr(O><3M;J90IFie}<+etR!1F5nfvp!Y0XBAvB4+5NPl1vXF!K1FeKp%{IgTi5 zOyjW}#;Kp291Vh5!WBiT%w9(Hb4Ae?Y`5*Whl;@%?O_x}sPt+Z78)iC8o3;RgAhf7 zt&Vla5fQx*MZ7T^9U4;2V$V4s+%-3Y&-yEY6{ag~BXExgj4+(4HbB2?I znEq)Sk;8(Fe|Y8%{)#4vcu9tYL#&R>*P(?G2lGZII)ssgVUTb<0bTmFA75gURJrAg zYp63>E`RQg_qd0yv;GElLTPi+IDw=z6AXTsR$oucf!DXeui)|=B zYTsL*icddDA<|lwXL$`o3MV4z;$Q@n^P|D!hv3F!uN1QXHNk3=SLOEn$t^f*bi=HU z^fv*d-bjvkV-A26?gY{V0#Z5w=`b&XT?UYzZU3wIhhqNIgowEsUPBZ1ynTK5+SuxL zQ^eD1O#j09=U>ffNAA!^KNJE?R#>!-?XlRwq6o^0{E-!vm$M=>IU49O(-f>~&+B7E zFHRw^pemWcLocvBHoM2SUkj*cBA1ucVTFdOj&@x~t^VBk)EbUXTtCq!=u2H_EGd(ncJ}4(Ode6~T(l+2< z{}JNQ25?Bt%8USkq|8lnTJ&L`&&r=FrN(Mq6Tu<5YV_pl_J210`B!tA{yO&6r$I-H z5=QIUUd!tUqb4*C%VUIAkn^HsSR{NG;rR&YsG8r`h+aI1gpOaay|&jqbPPdYG^Hq_ z!clmkVH!Xr*6er;jYbTOwK}5SM{7YEr*O+_~J-&y`#%Bec_ z(+`7kQ}AXSKpWUT%SkBLlrEtHt}Z~%gi^3k0JlrxnNb>q3;G$+(M`OIY@h9xM6(e;-p`m=adeQkk-4HYm@=dThf8DigF=x=%|Hy z{7@hCp*3+jROCXR(mLNo>xigYGKn8$Bw$U#&z?|03-Ujh0-_dpzTc$6+@Ss)luh`48Ka{B6YJu zL(qo0Z)LCvRpv*+3BXx26Tvjw>bP(ofiVCq;*CWbjEd$65!M{hG?k*sk%*iKjvHhb zQaKS&1JJw>h<>zOPDd*^LPE-QJWij0R>GzD4=AdN2IG~0RK3a8rbl{J7-Kg@0Z1i& z>i^d1=aNy?HBmnjGS0x;k&FRb*ddE!ah=@3qOwy7*H~F(Md~y>$R4K_IA>XjM)Y%& z@iIGPb9l%|&&IYOBB5fWj^spY?@&93;v7gg4<^5hkRlNh^2TBu&L1X?f`Zcs=ONcb zN`yPKS<(p!6VY^3MZXOBaKWTh67B@OkeNU-mymkJ3k8$9)SUk4<1Ljp%S?(Ls2_#T zJyT)QR+evh3ruQ3OsX6Q!=K28)WgJ6?2i)(@X;-T$L}(TG!R7MjVwIE-a;XgG`+p5 z>>8~0EG8tqMo7x2G;$62Lp)+7Ri~JuS16CLKko5!CsKHQzXcU0dPUkzEs03Vv!dSz zlt1Yg8aRV$|DiS;6i`4|G1^wB&}PLel)b0r2r|H9k#p;AhjeQ zRRNHyio{5rUSMc1oBft18PSKssAVQB=*8@??QM^*|7BpJl`#9UsKlZ)ek)QpN%k{h zaAko*OJVl=9vn&nhp2IVx5Qc!hFStcGbkgvhKy(>Larjv6Cfy*gr-#w3XR9=Fwv(3T@{_qG(ijRKADU@5wGt5euf2Y)+~;N82kFN^yxDR9 z>9(;WmUqCRRx}Q)VtiH;*$^A9c>R_QGNK0$9WLE>?1=3h59u19TWfSnLLSrtkE}N= zMflN|_&I}TIf5q}hQFT?1cMMfys@-usD`xyQCKTP&rFJ*`0JDmA&R9y5jvR5NmJ^e z7p=F^u?l^7jg`GlSG|Lx$2ydvM|U5u#GckCJ`G#?PR<&;skLB_Tp9`-5me2sV*tk3zYr0nZ$cx9-^ zV0{KY?;M3ZJJ>PHyC6>+B2V==IR8X8#74sK$C{LXJJ9oqht(j@S{VL*2YCjAJiM_S zxlkL0JZv`Arz`(X?&3FiQRwKii$af+hwns$?K7-n@qc#f*wFCZS5*}r%x-5lMZ02O7Wg)3-an;5C;fALG2!4HkYxvWj$wHYZ+KoUx&H8cBWUfM>o$OW1dmvCW5$Gil zD2P&`?LiZ&X_;eGL1!64*=8fUxghi-d)4-y$2e#VO0*RwK-OA;khq1f2ZA=j0{ABo zWCntGV+B&7Xkt(_7?ep2nn4W0EchjWr@CN}8O^W=FlaK)9_`MclVO}eg~XuUyfk!? z7-Sen3_=uL^|9q08s0d5l3xlTn1sh)<7JE;$MtbKx9oeqK_+#eqsf-_dVnbLpKAuZ32ke z&=c!lJr1JHh^4JC0{)GlNCt>_BOBe?QY^JaEX|=Hj4EACP6JG8Fb5VF2uvqMv|(n@<7vvTOj{&5m16yBPp$Y~q^$ z2Ezf!Hv!yE7Qitu0k~TLb5e{r5u(t|0{AmKZo`RA7+e;>aA5&#r&s{}6boQ9Oaafc z07~1en(VM0RtNh>H~>@hzAsh(NBIA&`>qWhzz4YR4j~U9HF9v@ZO8AsvIlS;RX_76<9~}+KYQZN`+tiCkh!)4JRI2A z%V%ww=5|y2rxj>1BaycQd_fanzh41H(0IG(g_L{o< zH10jS&{~PSpgrXU?co8OulZsnJgeFg-1K?)Ga&J?vu$OWLW}&J{rmxpyHb55KFqeg5FVJRi0uDG{sln!P)#1ME;8D|k z)^32$bOLrzvIkll=rdG58LBVB&6ej5dzjtwAyS17bPLww{#*-5Kpv8SaeRjfdCQws zLyhR=${&7ZcH4&@syD?wzN1pnZRgxvf>(<9UVsOXJ{*D-&$is?TUiYWb({Gr53O;!jpV5%hdLKJ!`3R8hL{plOIG6&(ltw#&oDyQs zn67KC^k~+=OdbXxP&GzxC>EH+2>W`*(gorcMm42O| z2)3WSVL1nabtZz<2EppkTxQ3sGjxuLWO&Gwx0qO)ZbUy9!>+M6Z09@%L37}x3$O0# zhSw_xKVU~#9#oq4!Ij+nfUjX%^E&*1{+QOJV_L%-Ycz)?)|vR#8S|ROG_Tp@a^N(? zc^e39rK6eA3(%ScXmtj(eBbMA^_hkF58oH?J%ZIiz)EeXFq+ny9DC%Mjo*jZ8|8}k zFuSQsEH>hD`T^TjQl8{yFX9L6;!dT6;reM%aw(4}_>&K?Ld(a9r!F)s>tMjv6*C&` zB-&j9#t~(AiFLz`=;(^4>#WfBv4@mV2!R-iC#vvv;=4QZFPtmd5B7@G3HdY?W4 zt$s`KEh(^yittJk#6uT{{aE|(;Uc>!24Fg&R!Y3=-);&R<0hnPzzF?x2)PVz$nk!V zy=f_ekTG-xbvM!8cA=1xe1VfacbZr~!ib(wQr>CeclM^O$U{kbY`v>o<<)jpcA9YW z1-=C--+?c%3IrJe#Ph~FNFPH|#z4v?M5a_CQ#ei*CPW;8reOg3&?-3{%J-NM=H%13 zZocKKWKJgeif?l+1#egumX!Bq>WZ$s(2_V(KLek7xl(93#2l7SK&7rkrO-Gy1Le%m z!nc=b1QlQiq@B3jcLsbHRC*tFz?VR!ktnzD#(GextCLDgDd#&)Io}5arHsYTuC_ea zg41n7hxjO#jW0h|5J@iexKXqzz{^kXhaZiY($4{6hYk(LCYzEtck*Z(;VlE6hSl^%{!^+Hg9=* zezXE2u_fS#bnUeMEm_Y~9O*{ml_cD`=+^a&{_U6AP2EL)O$ud=3e_(`CGj&(lE8kEY36N{DS<+2%1u^m9r0Cws^Cxrc;p@Hyj1 z!co9^EF2iQ^}9~Q4ktJ-_W^za1wVriFceaZ!Lhut3CDLQ1-nDR6{O%zVh{l>O323# zO-CPECZ|J1cGNVIkHNEadRhXPavzX-!K(y?j(nYQA;J6BPjSMRY#OW2#HY?tFm#li zvU~vy#Sw-Y#K96MCr8uyjuPZceu@+D3t;F=H~|{~Lt~K~@y2FgD9(wYY)Xz2Zc=je z5jdpI4;d3-uKMD8CO>qhMsrGP%qadaA;oU$AwVR10ohR3mJ;j*%zyw<(1s>X1J2+K z{TwK*Ml_0-_!C}RGmPlyLeWk3uI;?X z0B8$b^duC~)85^&)@ahT2hUwjz^{O!ui*p?1Bx<$BHq{n6!jn!^#F>jlog$Hd2rg` zl&uAfGSJM=0E`ym9eRL82qWy4j5px>0iGim74ULzH(u`jUQUWOrGEcyxLwoVZt6)% zk(&)r*(&)3Y=A-An>ejE=5hMDQ1BHDIvzo^<2-iUrKCI4pc+b(!0*$yAN7b&?PO7`GNM&q2# zXzi|9B3{0Ol0U!;7zH)P1LwSviw^OmWIU8yNidpEFrt$ggo5MI%$$Ngv_?*cinS*` zM=PSMrj>80cs-%W9401e({uj$v(ua{Zv+}5Xrh)fK|dcKdbNV0W=Zvv8+z;Puz5+kmH6=qBP~IBZUacG-x*b~ z`rWVPozAQoUxKWeADP{`>YdKm#tuv1&6G_6`8IZLzd_*+ry&#d3!u#!1x_c}2bPO~ zlab&Q4md@~i{P2iW>8~!(zauw5xrbwy2C!OUGx|Z9l(y>a^XoGF9J&#a`<5Q68$%sy_c)H8Z*uM4lZ;ga`AGo+u4T}d|=jxE}{vpuPGG3BDi|4j3d3y zm;ai?uE|Dpbm{jG`_OjDLqB?Ew(q&B4mE$4)QAlO8hV#2D$DIm@c{x!Hz49)@CL>} zjmbEkH+GWnBw`r@PFEZgBUI#vFGfFFK&Qh+j?||GAAq0Z`lXZpzEylx zQg9^n!&9dMP3=q9iRhA&`mv0rG>d*Q1X!=2>2-G2@(s|`m(bKS4rW1-C8_5@08RLS zq$hR)yag=XhDWeFuw+4o#2YBuv%XF&ttBi?AuN5tt4D)?rSbvgV)J`GwmB(Wg0(xU zREf{qck51Nudpi{D#DG`FY6R0kJ@I>+6)lCEkPml3^UGSNz$+>`YZ^sLE+Fz_L1c> zIMk0g)C?Soq$FuqP?G>1bIccMlGTT&Nwz6Qbalb#KKsaa*<&np0X_Qj3X*@&5FK^{ z>_ZR3KLQk-cc;4m(mgl?djd#P03>QiyE|)=SwA9DKM-jhWlEP^nNl3i+8v_-b@C}_ zmSlrSD?p@v?nJ8Ik`IXUM5IC@l3_kg+#FTa5t-E}s$TtI@7J&Tclpe2>My2kPnrZ# zLbCcGuA%@##FhIFrS1>8_jj9gu=J_=CD3!D5?ODsbCz!rS^X)pA~9y0BeJk8u3kV7 zx!0HXC(WB`L@!rZJz(c--+D+HgRTkaS{;+NzRE-HP8A;@pdP^=*c*W{6;$F48@lwT zfa;HcT2BFGxI^}wo`|L%BATY6dEpgw%EklkbgYv}FL(QcT0qypczelYR-%Zv#<%C*?@J@ZaNweEiV0=wp=+6*(AR&JEd-HBZFwfbp0}X?9B>@-eg6V?*O6!1fr-oI0WS!X%PpJoFhG6HSz+C zymK6R0|BGyWDlgq9>8dT6Gj_px~9HMCY`T%+2}4MNhQzP0`NKel7=o^o?@w8`wj|6 zi5`Wcz?BWTN$0WWC0>w3BWLKBLy0X4i5%<`%l9BrB9W*CNYqkfL~OCLDc8OkM)Yt2 zh_O#>-+O%X-N1)|lntR|#1hDm5EhK5!9K1rgbZ*vl3@cbf?b#R-8H)i8^nCI#{dD0BY(zWK=eN7ilD`}Fo3&oA@ zGG2)}pWYqb{i1i@J9bkNn1foiVhMzJ_~11P%o0z={{wSalACm@(X;f~kS-VR$EACl zeP;O)N+r=%w8ABlFA|9mTcYg#cVLzgJzUb2VxQT5^pGwN-3Fl>ibE`stbndD=MUW~ z#vA)_d=ivkNziQzak}Y0uHF9zK(GX%-7GZCHgu(hHtHd8Gm)cHo#0YB|N)P z6=KU2&>YG%qK^wS68pmTv&VPd6C@Zc8~>6#Ia39ksg~mi8w<{4f-}5v5S%d)XH4Ks zE^+2QIKwJ5w2#Ld4+3D41Zy&{hiStO)u>#~F`2Ewo0!~j)4B~0nahNm_jp0*Brgc{ zmT6P>#?re-0>8XvHw|_x2)WUQ2h4xA{tb2u8J00ezY+rOz*yrU^DaAY`2|CLFpa=y zjLfze`^a$Ea)mbebByTW3K=hU-u8=!eDUaJMz>9%%^*CW4%!I!F{jN-h?*KXj>;1- z^yeUIc;gVd45p|VjHua0QDeSG+09@CiAluE95gE*MW?m$u?lOrVV`|V7fefIQW4*H zCVmDoByw^@c$VU|KD!%|E<8Hzq|kM@LVHKgkW^@|CM21+Hj&1 zW|#l3m+=23EB-%!#s6pGCV=N{0hq(L0L*bP%kaJJ@!iZ7TL4;i@t*yV!a}Qv#@Bp5cJ%DA$ z$sWLMn*ZzptXZ=Tv#yCqNfsW!@$>Yn@I~!%c>v#M7c9Tx7K{D4;Q=(@9@|dj^xAB( z8N>7U0nBk+nTC7pJn{fi<1p^J$(cs_Jz&`cxPwyr(EGFz;CHy=s!PJhMf~4s1Kd2q zGG))8o_14;n0P!g}(gfrg?mr7W6z`BKvW5A=23Ux>$p$0~DN3?HHbDNHz+x}J?yJ-0hDZ>!DCAY`jx&x8{%viKJe0K@i0C;SH5xrb_goa(T{pKNdAKdX%X)y%F zOl`7rZ;2*!T5bSrh*Y9+j^o5!WCaUwFmD`12fl7Q1gXR>nz)oC3j5zM9FiuI34Zk3 z=!TVWX-JXL>LaoV+6=HQPs4toccC?INta0}xHYhB#LHf^(ajGdyL6A-^H+PjDOHq9 z{>$9}S*n|es!0p=Yat@-Sj7{uGwf^2HHer>*Uwq5g3vM>4%B_(l2ZHh2k;IvKY+}r^e4ker9IC2+4l7&dhc;gs4q*79r z3MF?_Qr72zYbrxVrF5a@LNs&VLswcArygP|vx-j7%O|wjEd|@4qme*!YHktH|gu&uO_izSxMa zE_i9#H@4qBMnQj|Wf(8umdA#G>Qq=AO8mRhkGX&s{AYn%09&nc92qNst;N6=ZxjGq zX@spb%x?04EvfoLplCcs<7k|5lz`P@G&9ZtRa=0nGzC?&u$+`+-~shBVd@-V%5aM? zHqte`vDGfizhZEOTPh1?!(CA@$?b9Z21H6G>isi zXN=P>Vq&9RO&d)pKg4+=%ZOgCQ1WG$ZGU)(m_WmxLWyd(LzR6d+}wX{5K3)x90@BC z4_OE$-Z+j9!zh%7A(ZwIlrj?ql*ZzGV?-Ebp_%Y0y3xuw9qpXqQ00;RLNBz^t&Xog zp4OC#jv$~&!TgfkKbH6UrHb8@E{OEsZU>m|rl4x-68%OfxF7Gx6+FkjwUj`?bh-{K zg|QeY7y}j7(1sIkJbG=35glD3mSf-AN<2iQGRSZe5hYx88h)hw=W5YdV~V0P^C!5m zbgk=<<1nm&8cT36Z=67fbP_Rrj=6l5;0^E#{P<8_R2qkaGK7r$`19yU%i?srSX2sW z$A{q+dJkIimdqU>pH%!6=TY$aMvGdm>o%p!aACZewp70fAO3*CqmNm!#Kw65Fk2hWekcK;fw2yB+A;mnRgzr~g z%N@aMxz1lJcWpiKTq3=4c-`H!z1y*2%Lun8y|eyQinz})|G|!2xvsDyBaa$^L7>!E zrZ3ZPh71Q49u=|gEPsMWBZx;`!J}9?k6K5YO-z#)wqIXnL@yVLbnH9ZpB@8XAZRgC zF6FAzu}Q*!s;Ef1x&OL?MBQ>6x-B5lGLVQj>>$wyBGCwtXg?)T`)MIb{6P66oN}UI z(K0l3m%*a#V9^LS7KyC@uM&wqB@$)k&@vJW^`WwE)cwFJA%%F@d*#Mqqg%g|sJEL& zBAl?}gi_f3EZCVOy@&;Hq+2whD6QWDB@bcb@o4&leQ)^-(KM3AV=P8$H<2W%*ONsc zz~x6ix!j13u4wXS-`oE35Hks5e-uR%%Dd{}LQI_0X~0=LIewR+YnG= zdd9B;D-V7bHGWvs*k05)^zzwz_d5GEK593OaVz$^(dY#nfMC%WHx|vd>bF6HBMOT? zXFpl~28+fJi{ikd9+Vs91CsG_bA~{Y70vwX zK+}Shu#L-w9jXf|*dm@fPadY2{UpH{u5zvuiL_b#?Rb)Tg zZhHu6##qmwI70bXJ=#elml+U&rW>5&Sa%rlkb^kljW^L@EXC1S#L;1jqufMcx|xmh z%@SdhgXX#$=thg-bhLsZu_NFgbnUdtEfw{lNQ(aruLPp5rR!$Z=pK33Zpsil0ybB6 z1Z++_0^$dN`ELO?e*qgH%WzXsHD{%M2UMgu<|>|NzgX@-#SFR!_oVti4sbfbz7)$8-Y0XJw z=JHy3{i$W6d!PIrkvU+BuJ(b&11eB#Si$L z9|ayY{nv{9@^cT`P2->R0zS6T{mh>P2A0OVVQJnf{VqsQpkV0&`_=Lfur!{q)C*WL z$|+JVSsh$El)kgdh<+|4Rc621{_z+CL%@s)a_7`sMXByJ> zC%S`a!5X~{I-XE;yvTmD+=q@6=`wob3dt0SG)cRiepLWqlGCJj*BH?cI%+If6o}5m z=1f)DZ?^j$QsUWJ^(4N<8OllO@k*-X)Dm+s-Gm%R@Vn4*4MgINQ?$R?L=tl%#4I4f zuOb5uO?np#75TAV=tOJibZE0o{Lm}Ujo3}Pa9S2u2Ung*J16HJ6m_I=kEVC})Ti)b zU`8y}kruAi=i!6DrU2<{cFpntK$=8A>H{G4mFq}5@Wg(*R~f@26HhFQPXdxGIS%y) zK+;;IO1$wdkTl7Oq~nAnzF*)iUPo$6bGDdr&-e+>=lO36Z{EE9>B(x{?54@Yq-Xg6 z#qX3q&wUdc>wH*Z(K`Jes9;x^bctQJJOq;_6O;OaN&P5Q+C#ep?xaWIXFID%@2@kW zCk6nU=v$Y6|JB)b+e445vlq@AlL34eSY-u@8yG+1o9hxOtpy?p8rOAY*+p+vfQfX6Xj;6l}O_zBw zDUBDC?#pNjPr3QQ&JJD^>?TXSYiwM7!$58Gr+NW1t8p*j^L#n(U|PIhzZZ(Wt_0LK z?03r}1eArwWj~D9{&K#wiym@!f$8*mBl@|(^b-5s_Q*rcbOgW@0#kJa)1)WBG!wxz zJI8UM2yw9D6*w&oNa{&r}b!F(4iYGuG7&9kx;$iBHpHJ zr*(5Fet9G!tEiUm!^~>sGpmIj|H)QZl)0k5-84l&>A!OXvMKIN%G#jc2MJH&4Jla) zxXk{ruq@gwa0*=rY(?A?N$JvVddA&Fr4Ke3(bFYj5c|W%vfO`fkc!b$Nkl3rO@4w% zNPKkj;0XL2YHWarym1H1cj(%S zNksE8h$t1uis@Tqz0pf1S#On^%BOEjHtP4|v%eu@Dd1aHVo_xcC}vZMO9^qX2#Q2W zYr|9Sd`tHysI&l7S_E6*1yE@tsKgr|fJ#%HRC=ALw3rg5yS$S0HLoN+?h0fh_4K%7 zpn9mU*v<6GCnz2|ALlaei$0mT4v*MT*4sD2|=8n~mt<3ZI(nPg`jZ=|-X340KCE^i1Usv);FBUp5#H6nI4qd6rQU1?p8dMFeU zdj{U4tNww)$4Wm+Ea@D0r5zVvZg)B5^Np8|*-bNmAnaHAl&46amlr9nWH!T1Jk|0o z`a=+p>gjyK_C5Q{B0;e2Y2#Kb z`VY>)xnCA8t^47kGj`J~w_?%bj7fVwVK6P5g|AwvTxaL%4?}^s6~g?;ZduBJFtdm- zgFqM)r9gYh8Mt3z%*VM#^l>pJgx#{0@%XOCf&sH-XW$G^#;gZpHo_Tr4UEYJV|e2n z7&D6)GYgDy5MyS5F*)aGGPW6D*(4Z~i)L5@Fs1tyxdPo7^J1>Rs;g%t%A3(J>o)xnNcWBsBtNmgE#4U1vuWs= zFeC?Kw1?tf98d^Tw9SYfF2dAif7`r0gv&s;Omy1>!psy&jvHbC3EftB0!uLFw*fJ{ z@iDs0rZ}06IC+cWq&QI|IanW)wun&KhGyk3bfXnEI$FVt6dcQydqfvaOJ`CM--aeW z2~mThP)V0{rqEyCC^*pee|>?lQ~i(rXO-;#YqV_ku&4X~GN1JSMg0%+|7E)Q|8lnL zk0QcOx%>ZqX16Wsti_9~1he%KDyAJQ{C{S}|F@U?e+RMV@9zKmWV;b3Lg2ai|LU;Y zHk{~0rpx~~PWb=kDE_|+@c&iDS6)1;&=9BYR9TuZv2O<&BH5KKP6S11!9W(!x?Qec;xWBK;za1VEyyEYx@x_n-UYk_f*T)M#{yVSSGd+K; zl^_2tyq;a*Z2g=2RQJOeLk(f&4*fBFJMX$of4{Ih77cEnbGYfxjQeM@oUR|DeHY$) z_K*MW!j04h)8B2}NOzFwj~bugMmmSzNM+C8JCwKQQQoe?om5>?VKTS;v4*kqV@&+H zXIA1{@_*{j|GMq{SFz~pr8&_CYy~~{{}a>ST(?{PD?9bC;(L2fx#eGBcP(Xc%b!cP z{AAqnQ{+l)J{cCK$)>;0b{f&kb<+=JcWq@o?p+gbcb+HLVYQishGgH&r<(ruAOqM3 z)89R006UQZ@W!Xe0OnE#Fc+r3Q&jZOYQzeWhwX0{-eado26hVDUn`^ocBBJym2|*q z`;)qN$@o%16 zHqBP;(icGR_i=aS+4QgMABzw2f_Ze$O~DO$2=4fKxZ@wD9VKMn-{-rG=;X>3>al-p zJ|22c#65k!T&mU1DKw-!{u{stkS-j8@9z=r`@4`X@Wy9&zj>4{%tN~HF5gbVx$z&? zi=^EmVc3P{iMHrS>)&*|NElGhr8Q+%nqFu{ThgT6C|y9Ew&*K!L{eVN3cG3k zGwZZ?bQ?-c!ZS;l%;70XdG7$Tga=HW^O5;(3aVD`)*pw0v?mo;@G84!(L%xbbQwc% zg{e>w^AYV4@>jTdY`)lSL`Rp3_1Qg})lLVW?Lh$5|aZlPg__+_r9bY|}PJG|T+ za1@@u(j&?v8{LhfZf@z_)chQs=97~1q2zm%c=4V8_TYqhLPQ)a<#vFGwERtniq)fl z?tENI^cJ+NEd^&mJ#}skzlf!p%P0iTd)R45xu3cpSs+%A*4XqX@aaELW+}h1`xakd zb^*;&QbF0FVwR#lipBh{I{vuiZ24C=oB7#InrYGehoxZg(;bKJ&l_yD0kXXD1t7b? zIa_(3W-C!;gY+mjGA~}T&F9k`*J$q+3jNxk9T zHDC{HB8 zirut`CNECopW82QijDsmOQ{_AoC?Vzx1eI{_UP>p@q6` zV*a9j)om|{3ww;{`DK}< zJ$h9SJf}cy_9b?dSVRG}2m$p00t!D}DHls15z4dDK@nFJVhKIbgO-XwkhmL3Qc3j8b z*&~Y%IxeP*7=|lMhZ-@UO97cJXiHLgUx}~w8qv|EWh3^;rt{E}yfInMhqxDFUkO2{ zC^|KF0KNsrzXJzg1*owXjOUFD^c9ehiy`D0A`{>F4d%KA=I)~Dgs+U^9T|Kis-2lhAS zyk1H>!fsk3c%=Nw%l4OE!2IX5A_GTD+;FsUzy1x#a8AKd2~$le4;(Ea91RDKM#$OJ z3Gz0$*0sOhZ$v*Ajl!5Jue`^XKOMAKO8FFR{kNDlk;tUP#q~SDA0UzX5az#XK+%4n zh&L_*MN0@pOMs%Ylt?wux~(3)ijyA07{ERf`_a@51dZOtdn|FIku<6YkD{`~qd$p9 znG>j*!piF8U*9Zz?^vfQV|pI2o0f_^NwNNIc**Itc@gX1QgPNI9f{MaRkQcNW>APNw3i^67t?Qmkt=w$rVH4tW;hF4;g2m=Q8v}(N>%F|FriW zU{PIN+wdMe=%Ap30wOAmg9SUHA{xtR?2(8=EU}EJV2K)goEQ@^F^T3;gJ3MNVTq2h zS9DYydspmTv3EsdZ~XV#`B#r zrcFRE9p~fdJ!Cm;zCdpbT}3b4PH$j9DUyTq6_g@5^}LkCxg{1DstAU&oOWL~1hKa! zjpZzawOg;Blu{BJ&EFwlcm$e16Tn!4-n?-ZjWG$9^h#_DKs-Po^5TE%@cXp_LEg6u zjJDC2#1Y&Bq^h>y{^v#5Jrad&QZ^HW#EuhL5r*{~M>_ zqnXQ0n}Nk)g^wOF51T*uXdLlTAMnxVlpZajJ?#x7YET*J!ZHgiRYt1HJna5%jxZdI z@iQ?J#D896@J#Z)92hB`lA{f-xa^E^2_YMI$*%1>Qi{_e2;EXYV!{`qUj zC>u2Z+0UAAVkyf_TL9w`C7d3!T(&|8r=KaDK8LsVML2~b95zBAq3*%zGL~Clr$$pX zmdjqq4a5YveJir|4YDi#=6fHuKG3Si zc%B{2UTNBfdp@R6(o?3l6$T}ZCrauU1?iuh9c_XT!lPeVB)B!Oy+o?N8XDE#RS?p3 zsQw~ANGp*Z@y10E(s&mk9ijB-2Bk+n#73DT!AAc|^>u=r&Y;HNi!zz(u@q30#y!T$BVZI!f75^Zc6ZXgdzM zP0-OQA^aN(LfQjDO41OLG^Qnwqyj`qFZjaJBFc{v>idM3^QTbi|Jj1&6?O*;VS33wnM=%2GZwjzk8WOB&?)1V@x0OAjlZl+fOxbz{qE2}NA zRO6`z%VRI%2I)xn{RE1qvxukhs7b-O$rVj^!A$p|{;Pw?SdD1njmzksOwp8#XgWsG zw5**-mB2vK4iQkR(Tw~CX0#zry%kmxbpQ+E+8OO}ONJv9Q;9wCNkIDE)2(0f@Tb}i z>jaT1{k0I_-|cR@RDTmR2PvU$W!51vA4YO4?fv#d8E^v^XDs7>62e|NK zSJzlzr{Y(W<+T@e!;hYq{e|$O#iS$@4G`R3H3CHdxl=v>f`35sS07NUL1*5$0)q*J z-~=FeoVeZBpi$WDM!#JGMsD&gY-w|xOvHlHCl>CHzI6Sx@hur&5Q-^@f|-Wjta@wW zgp`n9#B$QSRMRfp{wal;UNSFRF;LSlL`{RDAOe)LrLE#geNa+0Ub9`pOOL@zPa*wl z1YSx-qQn~@EB1?vmrf8b@j`%p;3a)n@h_=T^1sslC6uUAee23yWgXUu?_5a|Zt&i{ z?}-{}n!nbx8$g^^sOc5+wgrKjCK5Fb1~q*_snTLfmA3Kc-(B**>uW8r#J4M}Y-%xY zdytza{0JtQ#Fvu%N;&I#O%O%UTPl5}3qmF#id>t%)SKs~8Rf|y)u zN$NwS+1m+clQbd4?CVT>0Oc7aq+YXpwqS(RBy!0w;H*RBbZI+(wp|g@>TayFz)lUR z+ANG zro#d|6~{U(zuoAD<7nLJ6vDAQaGa<-+pgd!>5>XOv-+}}Nls@=z~Mk|-napa$+!hJ z8QgT5*qzq{#Dq^u7ZCEU<6)C7cNMwP6}sX8f{qo?wJxlaVkZJYDWkmLr0`q*V-8Ho z@o0)iFPg#qzTUJC_kB*`q%3Bz6$d9xAx^SIK>#SHOFQvg``dMY9-yS$Nltq!P||wR z{ZZp4C~1m|lFrcFO<#a(aSHRbq>b-f+=dqC?<$kdef-(?a~l+ma#(+LTTJ@rYMfs+ zjI?lrX+K~%uP_p0J~k5==~rT;FTqGdMY6?XG_{y*S|sj}X0 zGHG$t?AQ|=lsCzl_$fGO12~8`Zh?b-B@X%(9CVgAsJCmnb_~RGRB+G+G!v(QgAU{P zziK&X(I=dP0*Qli@YSO$#6j#UnS-v+UlBfM?0r0`HT9ixzkcQ3D?4UWHG#txZ8RMK zau<}q(Xj%y5CqOta>JqU%&$aFB<)_)D7uH|dc39l&PEGN)R?Kq3fM#3pc@0bX|U^n zkePyK*j@Di`B?4^>dBwvv~`5MLcPKr>ZkUbWp@b+w+J0WYbr!0_O#|f*2%#N+DZVtX>=K1;TlQwh_xfON0Iy8O|!s4#jZXp zXfNT0-B_4ShZ&ZP*smn_Q{B!7z!gk##&!k*X#kEl?x6oP0&W_BJ5Q8dwV);mIt9p1 z3iQ&@jGYBr+JGYyK{$ixVvEuTZLCQt;Xq6>?iO^TiwkSedB^OD)A{6V$tF`e?)Rd? zHaVHEtt8lHI{ zvL9X#uozz>`jivRpC!hqf$S-?#fAS=oBfSTS={Z(_a%By?~Og_8E;qp;Vlu9QLz{E zGVJ_lOR-}+RjHqWMe7+Fo>}TN9RdKC6rPcopRE*lW(M)h*Wj6NDDzoDs=s{-)7*Dj zV52fk1LkKh<@S~T3?`T zu5{-d_~opi8Yh}z3qUo;ae^6Is;SkQhf5GqO)jDuX97*c%!LZdvcFN2_IA(M^=U1K zb>`p5{@zi5y{#k)u(O$(U}4KPn+^lJ%WyItEFR3?RvJz|lU(l`xa4qnJywR;ens|| zx!D2>HCP%le|u>+u%%pIc ziEz0{;S%esW;j?alFo~0!K#KnV6m})@hb)f;x-D~h>C#0bltRJCZ+SrGtqSrIG~!$ z#tV}cA9$0uR{VdJf5;;LANXgBtp6L^uei;_bFVp{}X!L)}x0sF~)UIgt zf4Nv8Tc}3=H(cod5)}R464L)2SYvLe)&D)%V!?qBfg1f^BUZ?c1FcO_^?&1q{%;o7 z|LNHxKdu6pk!EgMow0_flJKd!`p=`Gw@zO?$?TCQEF{Uv`hUht|7{WKzy9-h>Ay_R zFQQq>ThxD@a~f!Na!H8)w)y@^{6}y8y~x#n83<%W{kL4G|4x$nFXbPp|D-ll^p6?2 z{!d>(=>HB3mGyt|1>dRvGb{Q(HoWM?CcTS{pXJe-QuY;FO-J$NTvg?Nxmkd%42IEJ zT>h7Uadd>p+VuxTwT16%u)GnmIlL-!Q3`ujW~ZOiL&r*DI4zyzyfFx)=vET{QR4wh zF3jS?sVx4xOv!twUrQ>xP@J!HKE{~(=lA}K&6kKQX?YLNC_I~K{c93$&Ul@bbp<~c z@pB$OXT7D&o;T2VgrA4_$;8hy{77%Ivh?`z!jAzzzW52iPYL`);->+AqVe+ye%j)v z1Ab!h6Neuge&*t58Gbh4=NW$Dva+)JWySY1;(LFSl`|vjcU>MJ*`q?JM*4THy|u@< zk%uQctg}aNWBn_>?PV(!WRBEwEDag{CafUd|2e2^in!D(kef(#$=?I%5?&tXlFf z$+>nIa)#~b%o~5;gmWlon1h_*D$PwY+hJ}3Z4MSpq-!Ey;Jr7%jJCk3x5yT#;$K^y zY=zKCXm49GG(e^xC7z~y0os~JD_4h>XYtz|*12Ly#Px_a)|TYCNw%bab2Sj^R_`#K z0EASJilhRFvLqLzr!_-1caEIcpa8&M=0g~9`CTgP(`sBD`GG2h9VwEmwr=L zYiA3A1@5gX-5{6#RYvAjCCQnR062Ca^Wu$1IOkkKaV}82Mw!Z8{EcTz)zo|^OePaVY7|PJVs7+{>G;&Yj&DW;`VPUQ7Ixt+6HO64`#F#PQ6{z7Rm8K8c+m@m;5KvP37x*Mv`qFdF{oS>GTIR6Y1ugyC zs(*G3gt~QmOlJV$9h{KUQa%=J3j>69x{M!jg(QJ&0$MumUOD3V9t-SL2tQ`Q_AoaH zr{F#pC=0dogvbKNjHCl|$zMG{_+gUMoCqxTpfhhg!zt|qp&bz3A_%Wg*$C`6Ntpsa z-ZdRIwEaz8#okegcH9F)>F~6*t&hz;3`K0DjOk+Kq|c@vn@{cB*?xi8JaX(coyC3M zRrn}BGukSHj}{Ogjf%=(qveI#6VOF?^kWQvd&?Vt8-kM>CpkkWfRpwj<>QU#;G_jE zPP$F`UXuVJv?;?^Yb(BUwe~GiWOW7Q?t*+7!@JD9?XdptwpPnJYhJb4`TeYfw@8_K z6~{wIq6q2weWr7O;-11v23Fh_4p#b|SZOp^=_fgrI!UsIZ(N&4FZWqsrxH>$D{c>W zbAhSgi-nX-k@C;ZWet?LsLH>l$f7<J6t%0;jCTVSUK zQcH%4CT=KBgYPe*K%(W_1!?Aff{XrC?=Q*uY$}3cKRWZqOBgJqKw5}Ex=X{mp|C2n z0aHm&L_BeeGcctMaI#SNNQ!RHL#iCz0qtx{iNhigScZara@~j=^Ww*zd>FFFVO=CB z$@S1SWlDLnuK--6K~lHzfawB|%v6vpz(QT?wm}prn{2XU0sxaR98&8?Runi2k|=NZun# z8bp-zoRAg};*e6t1=!FIICT{X(keP`dHV3hhmHGbM+0iQpnd^tYv|ON9JZ(I4v@Q6oo?zPkIO z?h-hF@+kMFbkjv(@lfF-Uslo<0X|wnd^8q(G>)>QWwf1rmV^ze)+Q_60!x*VK4B&8 z5pEtZ6Qr?}7>SgBi-fQNjHD|6J_RGSPjW7s4@OD{Bk{%?FwzoYq$OaaOiGYy1!~kb zuW`^oL!C2+}n@1y-YC?%Q51B3j%0J+KoRs`nDO+`f(o%BDad6h3 zb3Xr?3{L>w_zz#Ud)g)T8()LI<9B1KPmlGu|0wpcs z>Izc+b)gOIF71@Ue@V`SrNHAbI`ak#>|cW|BOI3j$A`r3QW16XCh2DR(Jk=(?TosBQHnh-_$m12chf!^ysF1?kuPmIV!8qZ9xDV@ zh=tl}fS{HWK_x~(5GdzM7a)ufI>0Wn4zP=6$0;5R)ji4CV}W2*_8T0t~59yFCCr%f3oaZpJr=TQqR zRg!AU%GhhV`NSNM$Vz1mYMGEYfTY6d=_BqFJwZ~vlAO`0AgQAuDc;Bdl3GC|wE`sd zh|;Egu1zQp98rQdAZk%Tyz*M}E6Wv!*fUQJ;J*EH)D=N|_Jl5`iN^gNYF*lBzOT^eQG$UW)0JLvTb}q{5 zN0(oPK4>#sGOVT`OZ*6*G%wz?`M3CDK6`UHtgA!@^)GdSt29vRwjVcL2b9lnKF(Hw ztemYDkmUJDGA?j}oIqWoR|>dNQRZ>n0y`DScC4JemK&1vH0^3a@(xIU1zsqS(5&eo zyig!kA|U$%$bm^t<0jy696fnMLhn@sx8fHp3<3SpqI)fMBN*OIF=KXz06SHP4=QoG{(M4L?`NsR%1?s||8mP2@Bo3X(uM zd%8@I@AJvrU-rTWgPDdTIrUq?Oec^#@dmUwY_*G-o)R-XBW9`wX3~e{{D#UU|0^Zn z>iX@reDruD%51H1TVVRXOua0D5lArym44B!AmFahUqhN6)%8e-b3!RMw>fL>%?g(f@mp- zuP&A4t4mJgPAoFe=7qYEp~rGg?_g>3^KgeXRr7G^Un}FVUKO0)I)8I}yzo6Ur*|ky zRT$eLuR7l?Grzn~p6VC(u~Hpp*D2F2VEPh1$b+dEt7xl>U`i#=oCuGdBvL5-ReIQq#UbTSVNX?vaeD|95WSPms$$f6x{DY_-a zRr+;&5+Su%(5`_!aOejP>)K|B(08Z--|?DGmrQxBmX3CxHr)n-uM`A>SS6bo2(Bdr zC*cAo%gNI<`Js1N3+Q>;0y`B!E30HTyCJv$cf5`e#6r4$&0i240R+E;5bywiI1L1O zLyzuj3Bk2M@C6Zji6Av|Vm>&e0U)xn`B16wH2%+-59XP&m7u0Lx^ljdWh4k=Z2(W7 zk*|t^_fg*F#U#Ew>^J?BL zq2^7}_elj(iuHzw$@K$Db+0-dJH1xFhtm!V0c%#shkyW0^u*4*-N^*dbUQwQO$CW4f;b^{skItf*=#MPvAr9`)vQhcYJcMcz8<~iW5_S+9v5xXg zuE&YUjJoyD=5ScoYjlDBzk4f>lx9nVEHBYIv_^h5`%a-Bj57k|M(=g_nT!jv}a$UH2{Dc=t`#IB1s^LRV8*y(FS^-x4=pTunVhVZ{P;tVwi1&8J3eIM_Ox$16AJ} z5BQQH4m=MG&I1hI@Pfexf^P%hdjt4HEx^L~dw{@=cMXCKZO>6x!8niTDr?aPZMf-E ze23Em?Z$;78|oI(^F+}qKm5Fr6V8DPrcB(ejuTF+3%;#FSXEm?FwRC|oM}s3@AQ2 zwUUP}Fuf8N2>w;Ype}D$u6=pMgd$Pt4r|)m1%lb6oHPyP9Lz9102VnE$|=FB*&2az z(ui`VgK}n2MzrGbFv)pGA)Wjg7TBny6UVCA8@YY&OF;>nY2X|dR5IWvh*5^g+JL}kLF|IJKD?DbD+bajD)=V@ z&9GA7pC>q3nwEbybm9C{hxn%yUrFl1SCXE~{L^suxN^U|yaoxtCXH5*_=n}<>qyz; zpG`^#kxC~0qUj-k%n9$~X;4WPVQY+N*+f1$1HL*FaS)1M+*SC;aM1z_HGJY(guSsF zjLTr=gxNGq%QlK9+VPOJyo3ZOv0Q7I3Blkkgu_KJ5N{yMVVfv~HX(#`6hay8ME(+az4_`Jt!q?>C8aPX{NLy13oSV52Fa;y$ zERm~w-4_KI-ud$1-!OpAhZOJu2GGkS1*C=#2GGrX0F|YHIVoSaQ@&o0FLj>42>NFs zpt9Qe|F9Bp|CIutBt7nr#f~j+TY;@_*Zr*yknb2hh;-jD+M*fck6bZ5#`l&>8RSD* z4O=q|@>^(-pM^nww!9LX$qPluTEK!=EU;3Cc|6%+Z{{{Wt;7(#Rau91as|U%wSWu8 z?($s-E#TwNu)cZ)V}AN$4E_Z$=5L`fe+$Hb66P6tFH=E}uY6D$(@hUp5@#tP#{4Tn z4_FPO{tJxyTa;1Xr3aLn$MNB-K8^il`2uYgjr|F~VC;XpAaK=sk9ui$?x8xzR<{*f zRz>qZsqB;j+^S6nj$SoA0pz(cg7S2r46AAT2&4a28d7IZ^G7x`2P3;3Bl`nhSVEQr z`d+obPE87WvYPgf+(2K2p?jOMSi9L(R)Vjfa!DYPZo^VY0-s}ezlyAYHwt2$-%44* zR%8Vploj-AhpYgXQ5wf9C9;C6Xs)dRGuj)c-XbeNC%%vKA+DX#X1Cr@DgkfakrFpV zh)GFP&!NtqA&1*LtlQjHX<1eEr7hV~1a8w{s5^Ge^b{CUnJSK9S@wagIWXKt*D(i| zI2Q=gKKDbpoP?qX^t)z(p$g(>>;rpqHxTJD_3iRHt=ASIyud;EYARO*E(eG!p$L2p zG_IjDZ}_6IEy054q_+XZT!bR81Xu>Y4;2vdzO`XQo8sFzgnRIf`^J-7N)7Un9Gh=> zOU5XGC#75#GnAdaMeIG>`|GxQ*eIJke%1TyD`cb<11=<#1e&E^du8B%+ny5sE2>ne% z;@Pl4d|l*Er2r#Ry@Tj6roWZoJYO*0yUU1e8H2XCKW4!@m z?NAu&8#ocK1=xs~sv=*roeHMXJ1Z9sUvEI79oi`R*CGRRr6f75I|VVhB!LBZv^0!y9mf#05T78QylFhRmQ=>1cJBF zG}H6*EU;&B-dC)h%%v-CLLao(Eydm>^fHDEV%nXweb?nj1?%qSODCsqn_l6jy%k=n z#LTvq9W$ietYv#Vu3?YLU<>37@Tse)#9L&6+mZh5ge=epS>J7}pYTRu(9&)fE#;-$ zZx`i$O~FZC{R|^=>e~9~SZ8a$lFDnPe9S(&@oV|Rm2Y7stkz}(<6j4q4|gW*4O($z z-h793kKiS_Cg4An2IBe3JsMs*bI0@=c;r)fDU8*#S-?wsh?f?CmwuNst7qvGsG;lXKX)mOKe&D1#;3VD%1SjnwPTB)b@}lje z@+CFfN!4)Bs)Cm8plN6YTFL=h+M}T*sdslCPR)pz!uj%Tb-sLi04WrO)8rdtOE=AX7z0`2&cW|m%qce7s~0< zbN9j$MekZ*r-oC1R^J}uhU5mg{XPmOEaQ6ZRtigCB@zej(i+LleuUFO$O8iq8g~&+ zyio)OdnuguBAmP_oD$oKWt8eTUWACKyJ$XZ3p3gcr``%DiNb&%(J3e~lM<~IQ-;U* zBnZ3hD1OSflkaMWb)R4)mo89~FLliZBkj{bs5^Jhlm!S4I3b5HoPB8f1Q70{%UFmj zTqNg9FWhTN6uW1EoeJRq_M!a~HwZW4KKB!ZSjhF-qtukp=mHM`!XwZH76lgf(3v-i z!eAdkxDOEKBM5m}z#8~_q`;4N?Eo9v1gEZ?kff?TxCb_;>!y@UO8FkZ>5WALU$tDJ{X%+8f#ik_Y03*(ngx87KowfRHjl zNW2jQLOMW%bO3~8pd@L7D@m%21J)93l!@lDIIvMJ9RGlZjYLhrkBN)uAq~lSgfHcO zCUa3-^J)Qa;T}G+8~Y&l0Pnir!JLK zq}NbOcz-L;+>8$_uu`LFFl%IQ;|8D;j-O6ZMC-ZxQDy=%8_%Dn!rDO0M4qnQPa z57A8M4IA1Fr>-JV;uFtkx^BwF^cls!rA?;TwIWkWKfPvQb9_UZcwV!k}s1r0Vi28ReRbf(IY4!zq~?*dF%V6fC-E89XUB^?L3F| zFzqn?Ya!ra4JBQEWbyzI{t6{UvZl6nprpe@Ny|Y=D@2N*W&eA&)FDQrXDH zn%djBefL|z7)Ru_Cw)4;`*DTpi>h03v^jedWON$}z^WjlM<65KFoBE?6B!)_8ToQ? zh}1k(lOHw2K^q80dL(3kgTP67z)6QSoFt9;jK`9NSg9tl66;4(w}f0p?0czL-AJKmAJrF zB1zKc0EERNl>%n0!rw{|0K}t`NEAd82kkT<1DHS48vJok@;%iD*DN*_dT0qR+ z(ktO#M|!2uC6A^bJu$r{-jZ{icu9UIuy#!;n_J(w=}i>4skz{$XJ}@A z2WHBLvmDnlQ_j}hEVqWDP6MvNf zwhqn3%h_e-)cIA^F|Gkb;_0fDb3zkU?8b9b9$;BSiK^P{V_PRg)d}*-HSpI|Ic4%l zH8-I_-jpa*fT7PVFjT|pYxc3dlN*w|5CJC-1R_x?inQsN@(es5d!&4r1aBtZK6p6jmn?R)s07%w<&_P756KV-Z#Kl+wu`VM_bvWZ@zwUKp?q-2iQd zOVL#*tkQSmlgR1ImIV*j%H0XG;FDqs_YNiC`;|B+H9+cazA)tlkVO?B>#!EKSO9sF zt|JwfxK_@Va;4%e1FAH+%nJ+bR2;uyE$p#wIPS)co+5fG4;YUt55Y^3^p-Q+wxmlI zg#nYD&W`}c3v}m=P?(%V2X+!j1`@^d!hkLD_hf}T z5q=q&f|h#hT~g#>quIMp@nq@NOOqGwx|qUBby>8nGg#>qvC`ToXaVJ9DR-(la0sM2 ze^DoUfRJ)0JMAq%NH0nJM~yNdq*E?JDngSteK{`3`IIQB-v~Z;bIJcS_1xc66tGg> zxQJTm_9G7KX}8rR|HCEzq5SW(hLCQ*GI;}qAcc_3tfj3B21D%ozksF$$RUb|NQggP3V8Vg{ro*(Sk^Hs+|eLOYV-a}Hj6x@y{S zlM)Y7;265$lZfX~-7D*s{W15P!+I9PqZS7AZ&wNraQzbr;8_iNy1Q>o`GFqkbvb(V zS&S_X=$)nOSdWV&AtW~2q&%sOVtHLfR^g2WCMtU0vKV`u8+uq-W9MLpWg~W)Y6Ggy z=L6^pCOc!>1A{k!jyKAq|5<|WET9V_)~;G!lMHb|hjmJantS|md3l_}dS0Xd`{$fNZ(y^GU7+Z>faobs(Gy!i&3QVZn^go4x0nx8+I%Amg?vPz zz)p1Cw3{ZS_d@*WqjMA!#gH=_C-peA+0kO@t^aEOAhh})`)91I|4Xx1+4Xk)U&dec zfARmz^nV!|{ojKeAq5dZCEWCXjaVC74~_nBlhFS;75(2DDgc--)!b4m0IZxN21i0* zX#{{jur_uaXl;fn06ZiFfEN`3U|~f7cpkg|e=7hio=gJ3kYp$8)r(gF%n`x`fW7AP zDu7`S05;EfivUpfoiFMCOtSuOTZOmi|2lL1Uo3)@>;J%Z9~u`51z^BOo`wb)*-!u$ zUn|T1^in@lcp=7Ox%{t*EdNUyDa-%L6nnq?Z^W1meHZsy9CA^}{~qQH@x`}Pl8>0A zn{Uk8+InKhyvXH$PK=qGMV{`JKhXVl>j^QmCbd?*#9g5{F_RE*sZ;&GuQ8@`(IuP@M*3&U8_D>ikGQg67vap z<#qQIKAm`<0PrtcLYFin{2x+?AHKCx$_U?teQNs*Bm5;A;Wv+smbNU$U^;4gw3i{3 zibcrkzbaA;%+xV{6#LZvncE0;1mp5$c?H%hgUb}&s{Sj35x-orGkpNYc?r`D-U!El ze~AYCOA!B+paFjjpJuoz|Kf3mI6m?-3C;8s82JNnv6s}5AI?q#zo-M)g@*lRd_ndi z4f_d0Fzmlw^%u2ue~a65EYlp;%gCbDHQF-8m4}d6b~JyNwRylJj}U(#U0TTlnzDAb zUdRJ3(|EcCL+w_K=XQ+eKB-hpLf$v3>JbAgH5C}m+Sz-#0euuB_Z4Mn_M%crq6!^V ztpS+qT>Aw^bPx39jVd_bWl9DvBN-@3vyjYon1x^&r8~NG6N!KaU*ul928OiXP5nhG zKve+y)8kazvx#n*MQUcmAYTm$D_;rO_{T$XSVgbZe1yHkxot+l|U(sNw zdz>qz5HO@FRXmq!#@gFx_P`etw&EJM0Ygkfyb7d>LK2GRuUf7c7;@tGE?p;q8RLIq z?d`qYfTV}eugWX5UYAHy@hNUbqx!3iw5m$7Gi4|M$%W3mQ59#sLP%Z#lBFoE`lOO( zQqmLsdI&hVi37H@^Gznakc4>MOdJ78JKB=55>6_mY!{P~*0qML*}3`b$g6x(@+5ai z0B*mGGAa3pb+GjT!(OFH$+jp60L7%lt6(Zsk@&OjkEDs!K&{o2ouj@1wdMx3@&d`O<7_z7FD&`^o!7@886(w0+(9+J~MtbXczm+Iq{vY&NR^Uehqv zQ+-HbfKXOptmdqv?Q<~JHDavon3L?FIY}ys^?Xy^N=H=J$G}o0tud^l{c|@LH~}8H z4$_jm0?Gc0t9ZnpN&qH1`+W<((u1#fBLaMNjri&sCMcocE2(IhS^}^S4%k}|l^)H0 zTR>DnAgXH$QPHYMhXG=@7aa95ag_OMS`={>43hPJjrVLlJJu&`sl$3*P}0BE`(4)r z6no|w5(qHMDFGG5thT-gsO#j9JK(1~#pJ}xFV(GTM5Jd7>{MDB%dGak8YuPaPNw-M z6p?OFJkiSRRae~z+FsTa0Zewb{Q)uIi8$d6=x5kDVox2;*(ZKRtzp%;NrTP~<^h-iX8*ZxDhvfM8idkkt zwe<%TQH4^rD=LHSmUF#AsZ=upRP@gJf630!abTmoU?bkB0XDknVxw}D+Kt$y!aa!bBpB;KDE{{uWl9#XkUpXC4=7!OeOsO?1*Qk5VVPw7==O)&M?9J1u-?|J&$2EW z;c<2Zez!F-#9n!a1OdfLN({ARU2FppL$}Er_rOi}itI>VIMtkQZdnpUD5WB5dB?z1 z4Wsd_i+!LQlxN}kcPNaoPOHDkp~O*DiC+{3_7=&`XTKmMuxKms#s}zso5JWe!l(ib z?1pevRWk(Lz7WyGEsnvIw!6tfAtNbz5OI1hx&_+CmJ-brP=*uuBsLoQMUvhB!Q4&` z>z!suq5iGP?~VpX-RpcI!N4(0!O_CH+6Do~J9H6yae@0phNKUKo&xnYhj46O3A3E|zEu8WW{q+t| ztSBmO@cMsWqK8cY$UC2c743FYXR%tliY~b~ebDZ;l+qorNyayVjoMfIef7;tt@GdI zZ1g67h!J;PSz)6X7H1m_Ho8k}v@Z%;KY6vb2)6B~7vV)B;&k8s`MK>@NRHYhJ9kY7 z9pwid@kVXX(OnlERifnRBqc|Eh>MnubLXP>BuCm^Ys0SeDSp2#&2(7rY1eC?M<{zl zc%e&nDkR_2a8Z^aq&RR0SGedC7H|6kTy&4PXg|2<0Odw&$A5?&>!PXVLmOp5O{K?Iz~b#+xPAQ>z!>+5jY!^ihkyB`>;p##dEcjCqxQ+pWwXFW2Cxxt)BzjaBR0AR zHVUJ(s8$t?p5|*DbC@6`1DeY&f{;q#6!$cQBn=x(G4wexQftbKoU17>GB+pbHfu$3F zz)ZT1gSf-82H4co0$dWM_}XAfkdGBHpOirzv~jBBClp zL_LUz`t!A~vLK?j)%jz+N>=jIDTV(0s@3n|;nNQ#jLQ_P(@!UoqcnIzhlmz^a`&Bh z(_wu$dRs*ZfV;_yOz$YE^>^?6eW+oioW3C?0Y`+wNNrh9+fXplLt>;uV5Gxxc4Q>Q z0tClFW?#N{{h@CR3{^^+%zD~~x_QH8aK;}80+CtnLp9P&p;-_jKkf}ZK}o%mozcs& zm*g7)y)3m@eQ?r4;-rV*q^gu5?Q`uVeUBrK6oll9X7mFPQdtnvLoFeB@nxd{L`dz3 zkeG!pCfz`WM0-fz=bU`t=z|CS9o9d@JWZDOVd==deniCeX78);_lG8^nB*5y3P{rQ zi3ioE>@(X}2&zBGF%QFMk03H={it}VxtXf{&^7Rjft?ywQ`l$ruiSvVf*^QAyml7W z4^T;h7x;5URUbrE-(+XxYQ%;gy7I<{=>7-A)gOqfY7|#t)kJp0js~_3q56J5qVEqP zw*2@?QsiSCf%d=2(8W+Zum;ji(5ALz_%Z@9mYCltzhSdc;ko07`X9!tyB>)t+rO0g zJ@=zKi>5@Y|2*{&AW;W=_sK#h(2gjTQa8LWKv3m;H92zJSOg}KB@a-o+0Hb z6cUO%uA$IUd)C|bHE8KE(bCZB#7obJmj?6Iqn+l3&2qjxY8 zz4K0%qW$?9h$k9C${i3=20+wQ2&n_>WBUe#^n?iM7zpY3_-MU;T?3y!24@L;_uk$@ zjz)T##sM*~RXJ%I>tp}M?fbtD+IUKGG|8(-P>d1UzDJ^TzlN8-1TTG+?96n6jRKG= z@kS%i(i5VkC!nQBN|ic9Xi}xIIPMt1OaWBskC~{_Uqx`!6D>DQvtb)p`XZmMQDNHV zre&mi#7@qU#7>}jY2^HoPv^(Aa#)`Vc9K_-{`nsFQ%yWEec_O@0QCcSp(xK`=*T{| z4M#*hCGR{A2S0(R2t`zsOf`S3VW%dAV_>O<)O7Z_eYhK>HxLHTh@FbU`mypvyNjK^ zMnnxycAngZ=qQY?ywMolpHf6UMMTx0h;p_QNz>2hGEM{)?=8WMcE+ibXDgucO zp-ZRTZhf@hD4GSQFT}42#g{tsr54@oc&_A`+cFZN2n7B?ovmjY6m_0~A?1K#Eu7NR zNHtuotgkHrAU>ljIf1J@2@sJtd6i1Vs{~YeU(>)C7^)!7V14ZgZXn*oZ9gXv%ftGK z@-1XTnC#T!jQP|sZi^#KvoC?cpY z`D#*E%?yrZdS_~l`rE&C^MKpnikGs|?-?om)UUPTbDQ4dH?TAO$afs zVj&fQU0ohRQ(_IB*#O&jh@h9`gQwx0XAnV1l2{p~&Q|kcF|bf$XBHb^|IQ7$J1~0% zvknNI=Xh}4CGz7V`dEa{&k*??h5JKyBk{&ZFnLL#^Ae#`n?ff#Qq|f_Lbr(`a=3*r zOld=oEEEnBb^gAh%cd0;DRDW4j-d%Y3A_rH$h^BkKKaIBeFX;6$ov4GmVw?;Y1H-p z-&Y#&blyQBl>nYu0j~=iXd4NvUeQ&Y!3C1k5jhbntJL_53W|Y+3f^os&_2=)yt^=a z4KwW0>0hLoEl~NXjt2@cBtzwQ5N6y9hKi&uo*D_~cLxO){ne>DYvpDaM*T?1f4 zdvDZLWJ8p^p(-3ch<2oulZ2SDuOOR`U+qsTe{fRM*PLwf1&4&;PU|UT)0GXfeGjsE zO=NR63Nk;D4Y6`4@h5c?OGTRb*)O1(NznG41$`?;gc8O@W^#o1%+gyi^sr1jg#Nl7xo|BfXjq? zvx9+nU|||)<~2Qs{zjJcahjQqP?-ry-xauharEVl7C2WHg-RAerQYvGPggO&ndmY@ zL<{d-0%o+cM!gk^k&MF-A#6BZHSL&5>AdDl^cDOT9Fy62Ro&1={iDZJ`LCSse@x5w zZ%#bQkFjSz=`tH#T6Q9$d84NFLmM*I1V3%@(+@w;PvyvwQ^y=u_E#Zb7}fyAufOO# zpl9dL2UXDdGN#j&=saLR=PxUC={zuAmrr7OJPm%q`aBs~sOyo#BiOT{M+r{@^VGHS zFnCsF`RatRI(RCmiPWodcp%f~_@c6BJ-j40cdu~oJf1rIe|Urue`YD3Gg+2r>B1-? zjyh#MnUs@%>eV*W`!wGiD1ucQU6hzE#0{BZ9BYuwQlz-+Fi$%4ZS^d3%Pz`{De z;U(hdu_5zuoV85V0zTOIWi?%-mrl_F=2f(SZ#J4c>pY|II^e=1rThaU5_0JC_jp}q z0h4qUe02KyKa&!$(a%oCdT=G6NxTo3&E-nKXwNT7Fs3L0OSbiG(4bYTZryaH3?BH- z>+`n(>l+*|5MGv*>}bHRj=u6tVJE!x5q8FAdfKIWGu1kP<{(}NFtVAhQFCvEi=TiBDa`1tuZ*t&9g5=BxDIP@qqdS)W&U9Ol(wvL$w^4x@SYfGrYd)1PtYEW zCqGaio|lW)RpZZ!wZ>y&P4PRL`@<+$9+wQ@*06fhkS~A8V9F)wfde-j6YP^ z*ncnl>g)lK3^cW&$e_znx8-k1uj=()t~Jj4J6ZW_>K5tzK@pbg?j`7 zv2{2^8(cS8mKj11!NBAzS(hfIl;BTu%%nm0M||khgS&^DG2vX3Mc79I`&i)a^wy7W z!a{4>byK35u@NpdT&&bbv+(WdU7`taTu*q|3p5Ag{{i-moC?^{1a`Cl8|IoMpmKdv z3_>6dXU0GlZTNvr8e%V-qG!gQC$Z$6g}Y&kgHrTR|9#Bg11#WX{p(v`r@(ED-@zu$ zw*zi(8{6%z_t8n~d|+A}U}LF1+AD~^2RNTS521~sF)4anDnVC#Y_uUH(aF9BNcH>I z<^SymBP0FM2$;J7X147BKDOY1n>n&r!Nvb?gh^Xh*9xT>tCOPt1|Uc!d0ER8+!JPb z_(eve2>Pzhy@{4^_PS-RRJ7&ssv;aVnpN4|#ebDoqi0?mLa6#u^zTEku8JUw?ra(W z1iw=d{Ee*{vJ+`kT{?5nH2+L57EB&HyHtwas}H?KbExdHiJdL5R585D*394OhG7`) z)65++E)X0+j+$|A4=@i- zDbU1IpAeKn{)xSkXnjCLZ~sFu&UyNRlfje8T*-y0a;#R5YWyK5|U{w_D{Dg(QEivQK&OK^l-RYI;RA&0t_hBnh*O#gI5U9zmVj$;e-Hi_Zi=;4j#?M;y_)E2lluZQ^cz{3JDrm*+>Q?^Ba}Pc zCgF6Q1Yp$^r>o`ytlIklW1Teq)^CI<36RZEAluH454jG=I?~rOlggs|MVpYTa6AdP&8` z_w3;tJl+{FuxEipKQ^LIU46j7->A71r|(4Ybpm{|2)@{t1Yai{*DBx}fo8x;z&AsQ)%tmAb!&ntVUH@-Q z$-r=~g5eH!V#p0(7)z(GOW&*PcH`vN;|T=W@^3A$QxVL?PRzgIhG0wFUT4MYtX%U< z=tYp%MUdCQlMEr_ooomo_!eGA|6PVN#u99?fNeHl6NJ~9z72uew`j6dbe$}B6{tyd zxYK3OW$z-?GV=7t>7>|M!s~_$(5fq5SKZ}x)w6-va1}9@ILovGh;5Y-^S#QR49PvZ zR9Fxu4>-A%zD)AF-^uS5)BB}Vze}HGfvt*KN%mxZZnvpVKipmd=edGp|20<%5^mKA zx9Wr&=Q;RXCP0`)JV%X#up6FWp{oFLo5|raatE1$JW(QcKt^FU6Mvr}5S&G17iKJA z_Q6xq(QG*278n|vanH-5+qg~`rf(mNQ|qHU~~9e?Ir!dK?D4Im*9kVXto8WDu|`n)A@Qg5c}h1 zN64K_CMb4TPaU|{L%|M#+f?R?zJKou)oyroQM|8< z3$!lT0Bw^D8eV?ldUHd=(Jg6Cf-G6QyCE9yZouhh&{s@m>a; ze$$9(N%x`m_gFZSGK@hA zK6-cr%#8uE_c?|4iPr;Q_R-Y^WY=r}>C||iQpF!<;V|+7{_s90!3eHqDKh=tL)Ct+ z1d-j(Rkfc_?u_wD#eQ|Ru^bp~mfg=Q4cqSC2{e9mB^0|7inECQs473ey9Do}{~B;g zCt>J>_vPU1S6mHX_^V_T;9VjotO$!|vW3ciB2^8=9Y`6=xC=HGE_gw}7N_`MoC~(N zY`}Iwg-v~YScXkJJ{*n5hjCSDxS~zueQSuoysazeAOb-+H0_{mU2x;^I-RBbpR>IjY5P}zgU~x`?6;uQ-K)gu*&VB7rAA?#3DBq zUz_ZaCdScYNs6 zqejEu^jC^YI-8H~yt9hEpXAgVykmWq`LV$0pcGSPDRa5RN*PtcA`@z*n~e>P%*p^VjHESj-6#%zp@!7t^1@b~}g{~YOmct`U_jq2n5zAQ>U`_U#l z2JCrNWcQr)&i=LLWi2%x-an?{kEfdcm|o%H!px1kE}l2dxVolH8?QQjUmkhh;+C|% z%>Cpo%X3&m*`aDL{EIYhaAaA>sBTA=H0pgKJ)zT*-4k=?wa45D+3-t$sp6(w`z$%W z+u;9W?V6EKI!$$&dv3{@x36!LZO;~&H(h*fJeYsl!RH&iOK+Nfs-aiePa=NFHO_zh zl);A={WNvn2VT`jmheh>ki+N*8?dy}mZjw*Dz$&Ht%-Tp_y$9Nn7z2l<&{~pN8YRU z(Sq_n0%yb04D%)+?mcF0;G3ZRgQ8~juEO(0)vV1$abXQ02 zz=uC2H!QSy(|}(V4VxNnY*fiOsPzI{_lft9CIwYpnwYfUr*^xWEw~anviqZi0XL-x_$`Jko9A;>9z{KCd#Etxdl%D6D6H^T2%vXXWqsvUcu$ zReHt!r(xd5+uHBz_|dPPF~h#~ZGUW9?v2-cuTE}K>DBZJUtjefG3MC1IV>($&hmG! zoa(ddzQ>oB@4UKIwSMKJ%?3RmmAHK0u!`I4_Sn>)w|aIT{}e*)qeK) zqSW*ktqL{1So%@3hq*iJzs!C0&hO9u({)vj{dGDR&K&9=dUD#qfSenjrJtVA>i@d4 z|M!G{LBKrC?-k38_2wcNm}bPn;G|9<}e0D=Dx0ZjUP literal 0 HcmV?d00001 diff --git a/crates/storage-test-compat/golden-files/golden-batch-v6-snappy-small.feldera b/crates/storage-test-compat/golden-files/golden-batch-v6-snappy-small.feldera new file mode 100644 index 0000000000000000000000000000000000000000..5e2a8d1349a54ee1663c0b8f3a08c1dd308d7f5b GIT binary patch literal 13312 zcmeI2c~s5&zyCkG+O;dq(@r}=*rp^TgbW#?%!#59$vkB|$edZm%#oou2t_!^aFprI z8%G%qA!ICsIyv0u>$5-a-|yb_{ob|iTKAuO*Y~!Tb=aRruh;XuKabbv^Lg*RPg+S* zavQ&e(R)4@PA4!tRsz_>;WTZAVSDA2Tu8~w~Gm|Fv=9)^9RP@taC7GzTb}CyV zBT4G3Rz*x2GkEmq29qX?Zs3g{$+`o6q$K-5y8zwc8}2ikr+PfIYcWVAnR{!Dj8q|E zQl&OERwo(kLWKs^lYCcT`15S*}@n7P?XQ zxM|o%&WzQQ@S5?+CIs@>MiQr6#Ya&^9#*6u` z*}AJs8$X-)e9~C;nUN|OsYqu^ z#e5~}AQUSDC>0bRyuT;<3}K)K*LQt!de6}9>VXu_Tn=YW;hIa<4KMT?fKdm`Ehbo{yWVA+XgS1)m9El0B! z(X6T1Fv+?XicJI9`sKr>j`z|=a%O{`Hln8uVZ$Zs-oO$NP2J!FfWr2pUx_KwXC_sgjhPyWfz`RZi!s z_%=}S&HO}8N&WMhU7S^x(^VGfDpOndHVEm+w+F#}ZHh`yT;$9RUEM@iH^SgsBrx)A zsp;JQK21X~(K=u{f$0c?Z&AR=x6zI}?k=4YvDy&LU5@6i_%_(!o9FnOs)PewW}e2v zIay*9s)%S+s2F@3f@0(wvzqv6M!yfKoK;nC#%onY*Qx}CXw1#L}gD@Bu1B{FdPv8HfW>!QnXP&@31?EW@j2i)r zjB`8XX;#Pi*lR%E{`AWOb<&bQ_Cgq}%$jZyVO~ z$-}j`9y-<&9qSPW-NpeU-4cqszg_(<_7rFJfz=mSeZrvIcwnSk?>?LB*w+cHFsxbw zdDR*yx=k?XroR`Iy}a!6>u;QS$1O6#-RpDc2lQ-+o()CMhJ-;j zx{r};3m^W{KjPA{nVfk8^A?zQ5M*l%*(QU7Y}y(>&Av|YuyBsFrMp5SIa?z|wm3z$ zu(S*B_ihZ*apogu^AXv6s3~Ndf?_1wwLW+3FBxetGmX)-v1r%ZCJrp@(Q+6#G9*#*J$q>mBZh2+H%%fKDn($ zwANG%;>|-b67P-WzSP694ShIkgPv_f&o+cXy!pULyr#1+{kp+!@^a1sfCUIFAPC|$ zfp`nRK|I~X*W)=tjWPGoCGO(EVQ6eIEa zpKz(aZGsM$!}e&}UNmh_7{pr)jKsUN;9y+b{zES~>j12Sz&a2H@e+VRJnhx#7Qc&bFF>j#zkf^ zoOMFSPNHKc!eHGpy2QDdzar@lKZq%;dF03~&N@T7Y^k$=I};4)meY041$WlX%e^B~ z&qc2^EL|6Q>AEP!tx$~Xy19K1KWP=J1<6YnBoYQuH5j)N3CXxEYo=_O5%RSQXI+u7 zt7zJlFc`NA7#X*F@t61G`?XufSvO$a1lEl(7`GZ28F%#I`1~-lFE0$yg5_wzigAgG zadS(m{!(x)YzSu|a7DU92IVq{#UpumJCpI=#V)*DTGi>AE^gK-;y!MKvcPUTMu>fV`au9xDa z0x3Y1tSVE58D$x1jmI0`GWIrEZBk^?S)HoZm`0gqn|heVo8_DNX_7Q$njYrq=C&5& zEpjdNmI;uJ{c*4{R&ZAxr<*rwWQ>_*t-+IiT=+ZWgeI3zig zIfOZ8Iodjnce>@|?VR9T*qa+U$hu zcqm3HO9~ZdGn7sf#+07dXiPJ&?0=YP*Xa&t{ctw>iL=>{FpTLI8dEKvL@vZShI(zd zRLWTxxG=$m5r+xgN)xJWI!#yO)3?fj(OX&>RO>IR)?b;>ZOVi$S}|m5&6;JH&;d}b zNE#r*4WMe6(Cr9E6MC6-d0D++LosLJXc{h>h7*Pf-2sdy^mch{w+Ci@%Q%Yw79p?* z!Z4x9z-U6peY^F@Zgto;L$rZ%w1LWm?o=l9M1!m5^9$Qoa5hMufGf&}!9c zNvk*b3^8OIEN2_6Oz0lNgnAy?Hgf8;w1M5Rxev#ccZi5KgnD8?_o5aJsMD;deS6Q?Q=WMq7g1yV9|uZx>R8K$kySD|HpM(YSu7h z8zyHPrYM(&YcyW~Y5j0`g3oDIjiX{F&J+;A!e-}a*z`BuNX-rKWb03JkR&@)E# zj3ErZ9U$LavDr#5NiJT|^OY-SBfyOi+z8^}+ppvsINd$3#VwM$PU~a{7b}O0ReU?B z__p>)Qu*-xyYO%}Qa-68MYxev4Za;hIPz`HmY6aA7Y9`5Y!sS~5=}=D2Hy??Bj2u9 zFHFpfGpfPaXkeoSHkvT_mJW=3b9@zHl=RW{h9TM*IocS-wcCjIz1d>p2ka)MAu2g!M785G1ooY)23m$#`0$fXOo?>rOAStOcZp>0!6w_ z_&GE1w&mFehG=nev^Yh#lZtL`l`XH#U|rif@$s2FrRg<_;zqameNcDGkg z;%q9`W2)#ml`!arxF(v)(Cx&FeP?gQudK`2G;q@dH;p*x_8aL2PU~A|%+k2kU%XPR zdg1arUG0oVr0I%oXB6Ff9ccYq{L*uJ&SuE#HbaD)LDit!S%f3qCjZi;aQ*ptTFz#o z=}ggdCSlO+95B+YUfah*?rwRM!Pzgsei7I&gh97#V5D1lOva0E+p@S}-Db(_HcQd% zyrSE@ytB({Ut4^Yv)OXA*&^C(DhAyypcv^kY0h6I!|mMBa}IjW5k2P+2Hh?KBi)h) z>0h5Yey|N^@xbB*79RxN{Gi(lk1;nX~z5I$t!MPZ(^=0hWtwx9&&zM5K0J$=L#6 z3k0@+FzA*GjC7m-BQVAKMYp4dXba_N3l-h2D7qCTy(wNCUTnW`2i>ldZs4?y zx;T4|+16%WvYoU<-q$ZtT)UyT7T&K{;%TEwc*}38oNTE`wv=mJ4h-VQ}phFmi3@WBXLi^u;d?i?%{u zv=xeLTygCYZ&tjd4z1YF*(&s0C3>zR46fY) zMy@rB9@*El@Hnv5z*Y-vbr4)@4%hC2gKJvN`48O*uJ}|hZo%6cMn+@xd>2$7_Pv?3?wEK!^@t%#FT|Aasm$MCWt_>pB z2C4+n9-tD5mKq(?_NUKfJo0Qr!;PZhM#3Q4LlR8~(PEp2T+XfX$5GCbz$FPTi8zS% zh(rUY^LgXEoNY2HHJs2*@}g~0T>DdTZTFXv3wL+W;hnP0a=6VR+-9l<*B&Drx%O(` zQJ<6*#kf~(LDMaw=@!D^+7n>pTD-~ck2M!};@M~`u&n~yN*G)#00!5zs%Y;HgD>yu zG7hKHd8Z35xZ4!do+_pd|GLm~^tREsAZ(ZSq}xTX?Nkb;JwqunEn(xen1bB?{+#VV z#~q^M4#HsCb6{lJfi(x&4@>`HoFxNG7Fco+OdAZ-3c-&ZFJZ&XH(4 zi1tDe?Pd6&5i1wmPU38ryk@&Zv|ZE_qP;{d60PlzUI%)|Ryc5$f}Sa&X9{5u?G-Q* zt@K!?zP>c7~@LRz@L~ zf3F;EuZXsniovxvC`PWi#MxddZ+0F%_o3%L(Q_YRaP2L*=AKzla(KG=@!~7~OVj^_ zc6|SF$Cza$@Bh+TIt~9P`#%k5scLf_T#-`$lkH#XKWzU7V@lu2pDei3_HWF&5$Esc z|?uqhU4LJ8VV%ry5r|`xV%)0{fLP3~LE646Ck#-L!Mv z*Invo5bvNY-a%ztKPuyD-=dbKY2_XGIP?(2E0PY0Xosj6#`P15(YQWsd-S05jZPyt zJB*%(MbE>8VO&20qj9}8#yV6pz#NbF>A=zjmL7z09fEN!1&49fRee2wYuC7p5r%9> zzTpB4@?iagXhJg@=zQV*^W|E%<=wTZyKDFmDg#)Cz%mHKyp{pOylOoUWfb4E9VcJoq)a(krZTSO%D85> z8*riPmos>(JT50YE|MLmS{T>AP>aU(594hSz51=1!`TV6JRw@1APnPLLE~B_Q?Kuy zHlRal!Yo9YkTadW>~h9a=4Rpj5CHNego)b;a5eD6iu^yqxa5BAi+g?na?~pEX}5;Iv+l(_IkhE>K$tXNq(r zT*hg2+q3UJ;Bo3Ax?U7rFA@gf%z%+_UCd`>$JV$V&eMFRa{;mjTt_WA2mZ*NR-3%W)U&cH4gQd$N*=4E);Ve*#gzMO2v)d%o z>8S06HTuX z2H|Xgk#NH=SG(^R>6ppcbzs*8cAYQ?XA6vcJ80amU}2MbE{13~=v+F0=pFi-J+qJBe*1FyY(P<`=w_lA%<*R&c+qnoD|#Qf3?1ozcFhsXSd~Sw?($w z)D*TkqZrv{ek3O6ug%wrIJ<+UcSO@Wguyl!U}Rgp3w5NJdad7cb{E)Pf!!qxwrPRE zHr+z*R;{eoRu4n8KjdhCD7v{Sx*aOrb~kTZPi+70$Z3q(k0ID3MgPejirgh99(z(}|? zBl!7K-4DOvtN>VnzzTvO+%O2|366xDyF6mt!BGd5_XVHI*`6xG)l`K0W|}gz$=Ac! zgg=wBJrmiUQBw$43&lvd;EOl9zfI}2m9yt)`dl=9P8fu%4UB~A*1^N|?*?zib5;nf zP+*0GK{!1y5-!w#>hYZOj}{n~?S&leg(6%XMYx8)pG<7UzXfvkQjYdgM0-iaAY5G( zBjGw-z4&N_o%dnRUZLkJ(eo8y5Uw5xR}I1)jjL<_^kk!0&WgYl39g7Z2v?tk1EWQX;*xM4?)x=` zv$tsaRy2J}7<6k0jC7kZWlgH{$K8>fy#w}6VDAWnZr;F1w{WlhnXjLP+8Cm}m!rK` zbZeyOW}AC!lhdBVC7gYbqkRz3K2R~}=7VCSn@i89yCCJ&Q%pV#1(XV_>A4 z)2>Yqrp82{;H(5#iNH#Npxbch)&v}M(>{K*a#t6-1I`=mX}k4N&h}B!t*N40gm3pT zX5%)z=IoQ4?UTs%iJC$;Ulb$V78)n9yB|MJ;OsM+eiltX69(Oy0VCZyy6#S#;d~n# z#!_IV0xKm9y7>Vk-D3NxH{>+i7-LwrFLJamif+vn-Cne89_k-Dt3795o<4k9`H?c{YD^!Ye~XYhj10DfM?xfQgH(*!yBq)f-55q z!ugYM;IyUnifen1O>&;9rIjm}@URAN`HxXe^E6E*BVhs zxNbip>uqV*?>1)@Sdt3Sw1O}Q*9I7bD>>Ytm!D;BjXtygl{SOVvGGba_H(jx`$79Z z6Z|tt_5Z&Asp0Iq+Fa#U>HELm{i*y^D)Uqrbo9z2D!;aJI$XZ$x!v@8@)GQBCRPy+)kL zz88bM9&q*(=kljGmp=)^gtiBU3H8rtqwhGO$MVMFRN{Z@|9<}23dG{PEHF(jyB`$X ztz(yV1xAuofRDOzN#}tg^+Y z?_q}T?e-1|!$!UJLhe>?|)`xV#g)0f)Sj%^DPnY&Nh2b3? zrN}-iG8k8ww>YeE$Kg~QP8Ek!>6dX`d55D~I$D9F72;?Gepx@9t^j0g=ju5A9ml_m zHC2v)NszBvuv>Km_r>eGkV|mT6`*V`a z%*^oj|8YNygUgG$b?n&A3dw?%bc=#>U5_=9jxHPDsjQB-N#M#H*Y!J-ld|t$-Qv)E z!I6Pasy9)OIX9!u@$3&)Ain`doS?>O4qcYFvCaP0s$Yj$YppVE*VR6t zN?B+$z(yZ2w@yiM)0l?w!PDnW+1sdh^SH}TCu-7dJ!(#OuRHC;v(nsP?+Iz!d@3RCZCt^A z>$r}a&S~-><{N(;^olo(c_EXZPNXo5Rvh zzSmam`S!=8krtLxm0CI9zS>tkvg}^Rgv~)_9fyXFZNBT<7W+P_TeI4otpD_q|HC(D zcaJ&yKW-TQ1p}3h4Ihm8?_-Mpca?mBs;{g9?f~>}anbTpUZy*O^4I_S&;R}v_&0Vt B+&BOL literal 0 HcmV?d00001