benchmark: Use CSV delimiter feature for TPC-H instead of rewriting input.

blp · blp · commit 489e092001e5 · 2024-12-18T20:57:04.000Z
Signed-off-by: Ben Pfaff &lt;blp@feldera.com&gt;
diff --git a/benchmark/feldera-sql/benchmarks/tpc-h/generate.bash b/benchmark/feldera-sql/benchmarks/tpc-h/generate.bash
@@ -1,28 +1,12 @@
-#!/bin/bash
-
-set -e
+#!/bin/sh -e
 
 echo "running tpc-h/generate.bash"
 THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-pushd ${THIS_DIR}
-
-rewrite_csv() {
-    for i in `ls *.csv`; do
-        csvformat -d"|" $i >x && mv x ${i}
-    done
-}
+cd "${THIS_DIR}"
 
-if [[ ! -d "data-large" ]]; then
+if test ! -d "data-large"; then
     git clone --quiet https://github.com/dbtoaster/dbtoaster-experiments-data.git
     mv dbtoaster-experiments-data/tpch/big data-large
-    cd data-large
-    rewrite_csv
-    cd ..
     mv dbtoaster-experiments-data/tpch/standard data-medium
-    cd data-medium
-    rewrite_csv
-    cd ..
     rm -rf dbtoaster-experiments-data
 fi
-
-popd
diff --git a/benchmark/feldera-sql/benchmarks/tpc-h/table.sql b/benchmark/feldera-sql/benchmarks/tpc-h/table.sql
@@ -23,7 +23,8 @@ CREATE TABLE LINEITEM (
         "config": {{ "path": "{folder}/data-large/lineitem.csv" }}
     }},
     "format": {{
-        "name": "csv"
+        "name": "csv",
+        "config": {{ "delimiter": "|" }}
     }}
 }}]');
 
@@ -43,7 +44,8 @@ CREATE TABLE ORDERS  (
         "config": {{ "path": "{folder}/data-large/orders.csv" }}
     }},
     "format": {{
-        "name": "csv"
+        "name": "csv",
+        "config": {{ "delimiter": "|" }}
     }}
 }}]');
 
@@ -63,7 +65,8 @@ CREATE TABLE PART (
         "config": {{ "path": "{folder}/data-large/part.csv" }}
     }},
     "format": {{
-        "name": "csv"
+        "name": "csv",
+        "config": {{ "delimiter": "|" }}
     }}
 }}]');
 
@@ -82,7 +85,8 @@ CREATE TABLE CUSTOMER (
         "config": {{ "path": "{folder}/data-large/customer.csv" }}
     }},
     "format": {{
-        "name": "csv"
+        "name": "csv",
+        "config": {{ "delimiter": "|" }}
     }}
 }}]');
 
@@ -100,7 +104,8 @@ CREATE TABLE SUPPLIER (
         "config": {{ "path": "{folder}/data-large/supplier.csv" }}
     }},
     "format": {{
-        "name": "csv"
+        "name": "csv",
+        "config": {{ "delimiter": "|" }}
     }}
 }}]');
 
@@ -116,7 +121,8 @@ CREATE TABLE PARTSUPP (
         "config": {{ "path": "{folder}/data-large/partsupp.csv" }}
     }},
     "format": {{
-        "name": "csv"
+        "name": "csv",
+        "config": {{ "delimiter": "|" }}
     }}
 }}]');
 
@@ -131,7 +137,8 @@ CREATE TABLE NATION  (
         "config": {{ "path": "{folder}/data-large/nation.csv" }}
     }},
     "format": {{
-        "name": "csv"
+        "name": "csv",
+        "config": {{ "delimiter": "|" }}
     }}
 }}]');
 
@@ -145,7 +152,8 @@ CREATE TABLE REGION  (
         "config": {{ "path": "{folder}/data-large/region.csv" }}
     }},
     "format": {{
-        "name": "csv"
+        "name": "csv",
+        "config": {{ "delimiter": "|" }}
     }}
 }}]');
 
