Run benchmarks on object storage more times (#6920)

AdamGS · web-flow · commit 1e308979a2ec · 2026-03-12T14:57:48.000Z
Part of the overall attempt to reduce overall noise in benchmarks, just
running benchmarks on S3 twice as many times.

Signed-off-by: Adam Gutglick &lt;adam@spiraldb.com&gt;
diff --git a/.github/scripts/run-sql-bench.sh b/.github/scripts/run-sql-bench.sh
@@ -15,6 +15,7 @@
 #
 # Options:
 #   --scale-factor <sf>       Scale factor for the benchmark (e.g., 1.0, 10.0)
+#   --iterations <n>          Number of iterations to pass to each benchmark binary
 #   --remote-storage <url>    Remote storage URL (e.g., s3://bucket/path/)
 #                             If provided, runs in remote mode (no lance support).
 #   --benchmark-id <id>       Benchmark ID for error messages (e.g., tpch-s3)
@@ -26,6 +27,7 @@ targets="$2"
 shift 2
 
 scale_factor=""
+iterations=""
 remote_storage=""
 benchmark_id=""
 
@@ -35,6 +37,10 @@ while [[ $# -gt 0 ]]; do
             scale_factor="$2"
             shift 2
             ;;
+        --iterations)
+            iterations="$2"
+            shift 2
+            ;;
         --remote-storage)
             remote_storage="$2"
             shift 2
@@ -91,6 +97,9 @@ if [[ -n "$scale_factor" ]]; then
         opts="--opt scale-factor=$scale_factor"
     fi
 fi
+if [[ -n "$iterations" ]]; then
+    opts="-i $iterations $opts"
+fi
 
 touch results.json
 
diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml
@@ -37,7 +37,8 @@ on:
               "local_dir": "vortex-bench/data/tpch/1.0",
               "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
               "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-              "scale_factor": "1.0"
+              "scale_factor": "1.0",
+              "iterations": "10"
             },
             {
               "id": "tpch-nvme-10",
@@ -53,7 +54,8 @@ on:
               "local_dir": "vortex-bench/data/tpch/10.0",
               "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
               "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-              "scale_factor": "10.0"
+              "scale_factor": "10.0",
+              "iterations": "10"
             },
             {
               "id": "tpcds-nvme",
@@ -83,7 +85,8 @@ on:
               "local_dir": "vortex-bench/data/fineweb",
               "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/",
               "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-              "scale_factor": "100"
+              "scale_factor": "100",
+              "iterations": "10"
             },
             {
               "id": "polarsignals",
@@ -199,6 +202,7 @@ jobs:
           OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}"
         run: |
           bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ matrix.targets }}" \
+            ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \
             ${{ matrix.scale_factor && format('--scale-factor {0}', matrix.scale_factor) || '' }}
 
       - name: Run ${{ matrix.name }} benchmark (remote)
@@ -213,6 +217,7 @@ jobs:
           OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}"
         run: |
           bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ matrix.targets }}" \
+            ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \
             --remote-storage "${{ matrix.remote_storage }}" \
             --benchmark-id "${{ matrix.id }}" \
             ${{ matrix.scale_factor && format('--scale-factor {0}', matrix.scale_factor) || '' }}
