Run LDBC benchmarks during CI.

gz · gz · commit c725a7fc249d · 2022-11-01T19:25:47.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -29,4 +29,5 @@ benches/galen_data/
 benches/ldbc-graphalytics-data/
 nexmark_results.csv
 nexmark_comment.txt
-galen_results.csv
+galen_results.csv
+ldbc_results.csv
diff --git a/Cargo.toml b/Cargo.toml
@@ -102,6 +102,7 @@ harness = false
 
 [[bench]]
 name = "ldbc-graphalytics"
+required-features = ["with-csv"]
 harness = false
 
 [[bench]]
diff --git a/benches/ldbc-graphalytics/main.rs b/benches/ldbc-graphalytics/main.rs
@@ -30,10 +30,12 @@ use std::{
     borrow::Cow,
     cell::RefCell,
     fmt::Write as _,
+    fs::OpenOptions,
     io::{self, Write},
     mem::{replace, size_of, take},
     num::{NonZeroU8, NonZeroUsize},
     ops::{Add, Neg},
+    path::Path,
     rc::Rc,
     sync::{Arc, Mutex},
     thread,
@@ -43,6 +45,9 @@ use std::{
 #[global_allocator]
 static ALLOC: MiMalloc = MiMalloc;
 
+// If you change this, also adjust `scripts/ci.bash`:
+const LDBC_RESULT_CSV: &str = "ldbc_results.csv";
+
 enum OutputData {
     None,
     Bfs(DistanceSet<i8>),
@@ -238,6 +243,30 @@ fn main() {
             let kevps = evps / 1000.0;
             println!("achieved {kevps:.02} kEVPS ({evps:.02} EVPS)");
 
+            if config.output_csv {
+                let results_file_already_exists = Path::new(LDBC_RESULT_CSV).is_file();
+                // If the file exists, we append another row, this is different
+                // from other benchmarks that can run everything with a single
+                // invocation:
+                let file = OpenOptions::new()
+                    .write(true)
+                    .append(results_file_already_exists)
+                    .create(!results_file_already_exists)
+                    .open(LDBC_RESULT_CSV)
+                    .expect("failed to open results csv file for writing");
+                    let mut csv_writer = csv::Writer::from_writer(file);
+
+                if !results_file_already_exists {
+                    // Write a header row if the file is newly created
+                    csv_writer
+                        .write_record(&["name", "algorithm", "dataset", "threads", "elapsed", "elements", "evps"])
+                        .expect("failed to write csv header");
+                }
+                csv_writer
+                    .write_record(&["ldbc", args.algorithm(), config.dataset.name, threads.get().to_string().as_str(), elapsed.as_secs_f64().to_string().as_str(), elements.to_string().as_str(), evps.to_string().as_str()])
+                    .expect("failed to write csv record");
+            }
+
             let output = replace(&mut *output.borrow_mut(), OutputData::None);
             match output {
                 OutputData::None => println!("no output was produced"),
@@ -428,6 +457,14 @@ impl Args {
             | Self::ListDatasets { config } => config,
         }
     }
+
+    fn algorithm(&self) -> &str {
+        match Self::parse() {
+            Self::Bfs { .. } => "bfs",
+            Self::Pagerank { .. } => "pagerank",
+            _ => unreachable!(),
+        }
+    }
 }
 
 #[derive(Debug, Clone, Copy, Parser)]
@@ -449,6 +486,10 @@ struct Config {
     #[clap(long, default_value = "5")]
     iters: NonZeroU8,
 
+    /// Store results in a csv file in addition to printing on the command-line.
+    #[clap(long = "csv", env = "DBSP_RESULTS_AS_CSV")]
+    output_csv: bool,
+
     // When running with `cargo bench` the binary gets the `--bench` flag, so we
     // have to parse and ignore it so clap doesn't get angry
     #[doc(hidden)]
diff --git a/scripts/ci.bash b/scripts/ci.bash
@@ -16,7 +16,8 @@ rm -rf gh-pages
 fi
 NEXMARK_CSV_FILE='nexmark_results.csv'
 GALEN_CSV_FILE='galen_results.csv'
-rm -f ${NEXMARK_CSV_FILE} ${GALEN_CSV_FILE}
+LDBC_CSV_FILE='ldbc_results.csv'
+rm -f ${NEXMARK_CSV_FILE} ${GALEN_CSV_FILE} ${LDBC_CSV_FILE}
 rm -f nexmark_comment.txt
 
 # Run nexmark benchmark
@@ -31,6 +32,12 @@ cargo bench --bench nexmark --features with-nexmark -- --first-event-rate=${EVEN
 # Run galen benchmark
 cargo bench --bench galen --features="with-csv" -- --workers 10 --csv
 
+# Run ldbc benchmarks
+cargo bench --bench ldbc-graphalytics -- bfs graph500-22 --threads 1
+cargo bench --bench ldbc-graphalytics -- bfs datagen-9_1-fb --threads 6
+cargo bench --bench ldbc-graphalytics -- pagerank graph500-22 --threads 1
+cargo bench --bench ldbc-graphalytics -- pagerank datagen-9_1-fb --threads 6
+
 # Clone repo
 if [ ! -d "gh-pages" ]; then
     git clone --depth 1 -b main git@github.com:gz/dbsp-benchmarks.git gh-pages
@@ -68,6 +75,18 @@ mkdir -p ${DEPLOY_DIR}
 mv ${GALEN_CSV_FILE} ${DEPLOY_DIR}
 gzip -f ${DEPLOY_DIR}/${GALEN_CSV_FILE}
 
+# Add ldbc results to repo
+DEPLOY_DIR="gh-pages/ldbc/${CI_MACHINE_TYPE}/${GITHUB_SHA}/"
+if [ -d "${DEPLOY_DIR}" ]; then
+    # If we already have results for this SHA (the directory exists),
+    # we will add the new results in a subdir
+    DEPLOY_DIR=${DEPLOY_DIR}${DATE_PREFIX}
+fi
+# Copy ldbc results
+mkdir -p ${DEPLOY_DIR}
+mv ${LDBC_CSV_FILE} ${DEPLOY_DIR}
+gzip -f ${DEPLOY_DIR}/${LDBC_CSV_FILE}
+
 # Update CI history plots
 python3 gh-pages/_scripts/ci_history.py --append --machine $CI_MACHINE_TYPE
 
