From b0b573b1e29e375eb3f7751d10c4f0e031083c44 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 15:16:34 -0500
Subject: [PATCH 01/20] introduce bindings for substrait and new substrait
 module

---
 Cargo.lock                         | 955 ++++++++++++++++++++---------
 Cargo.toml                         |  11 +-
 datafusion/substrait.py            |  23 +
 datafusion/tests/test_substrait.py |  60 ++
 src/context.rs                     |  15 +-
 src/errors.rs                      |  13 +
 src/lib.rs                         |   8 +
 src/logical.rs                     |  50 ++
 src/substrait.rs                   | 155 +++++
 9 files changed, 989 insertions(+), 301 deletions(-)
 create mode 100644 datafusion/substrait.py
 create mode 100644 datafusion/tests/test_substrait.py
 create mode 100644 src/logical.rs
 create mode 100644 src/substrait.rs

diff --git a/Cargo.lock b/Cargo.lock
index 0f3fbea4a..06b9fbc38 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -16,9 +16,9 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
 
 [[package]]
 name = "ahash"
-version = "0.8.0"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57e6e951cfbb2db8de1828d49073a113a29fd7117b1596caa781a258c7e38d72"
+checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107"
 dependencies = [
  "cfg-if",
  "const-random",
@@ -29,9 +29,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "0.7.19"
+version = "0.7.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
+checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
 dependencies = [
  "memchr",
 ]
@@ -60,6 +60,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
+
 [[package]]
 name = "apache-avro"
 version = "0.14.0"
@@ -83,7 +89,7 @@ dependencies = [
  "strum_macros",
  "thiserror",
  "typed-builder",
- "uuid 1.2.1",
+ "uuid 1.2.2",
  "zerocopy",
 ]
 
@@ -101,11 +107,12 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
 
 [[package]]
 name = "arrow"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fe17dc0113da7e2eaeaedbd304d347aa8ea64916d225b79a5c3f3b6b5d8da4c"
+checksum = "1b556d39f9d19e363833a0fe65d591cd0e2ecc0977589a78179b592bea8dc945"
 dependencies = [
  "ahash",
+ "arrow-arith",
  "arrow-array",
  "arrow-buffer",
  "arrow-cast",
@@ -114,25 +121,35 @@ dependencies = [
  "arrow-ipc",
  "arrow-json",
  "arrow-ord",
+ "arrow-row",
  "arrow-schema",
  "arrow-select",
  "arrow-string",
  "bitflags",
- "chrono",
  "comfy-table",
+ "pyo3",
+]
+
+[[package]]
+name = "arrow-arith"
+version = "31.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85c61b9235694b48f60d89e0e8d6cb478f39c65dd14b0fe1c3f04379b7d50068"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "chrono",
  "half",
- "hashbrown 0.13.1",
- "multiversion",
  "num",
- "pyo3",
- "regex",
 ]
 
 [[package]]
 name = "arrow-array"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9452131e027aec3276e43449162af084db611c42ef875e54d231e6580bc6254"
+checksum = "a1e6e839764618a911cc460a58ebee5ad3d42bc12d9a5e96a29b7cc296303aa1"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -140,15 +157,15 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "half",
- "hashbrown 0.13.1",
+ "hashbrown 0.13.2",
  "num",
 ]
 
 [[package]]
 name = "arrow-buffer"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a301001e8ed7da638a12fa579ac5f3f154c44c0655f2ca6ed0f8586b418a779"
+checksum = "03a21d232b1bc1190a3fdd2f9c1e39b7cd41235e95a0d44dd4f522bc5f495748"
 dependencies = [
  "half",
  "num",
@@ -156,9 +173,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "048c91d067f2eb8cc327f086773e5b0f0d7714780807fc4db09366584e23bac8"
+checksum = "83dcdb1436cac574f1c1b30fda91c53c467534337bef4064bbd4ea2d6fbc6e04"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -172,9 +189,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-csv"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed914cd0006a3bb9cac8136b3098ac7796ad26b82362f00d4f2e7c1a54684b86"
+checksum = "a01677ae9458f5af9e35e1aa6ba97502f539e621db0c6672566403f97edd0448"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -183,6 +200,7 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "csv",
+ "csv-core",
  "lazy_static",
  "lexical-core",
  "regex",
@@ -190,9 +208,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e59619d9d102e4e6b22087b2bd60c07df76fcb68683620841718f6bc8e8f02cb"
+checksum = "14e3e69c9fd98357eeeab4aa0f626ecf7ecf663e68e8fc04eac87c424a414477"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -202,9 +220,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb7ad6d2fa06a1cebdaa213c59fc953b9230e560d8374aba133b572b864ec55e"
+checksum = "64cac2706acbd796965b6eaf0da30204fe44aacf70273f8cb3c9b7d7f3d4c190"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -216,9 +234,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-json"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e22efab3ad70336057660c5e5f2b72e2417e3444c27cb42dc477d678ddd6979"
+checksum = "7790e8b7df2d8ef5ac802377ac256cf2fb80cbf7d44b82d6464e20ace6232a5a"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -234,9 +252,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ord"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e23b623332804a65ad11e7732c351896dcb132c19f8e25d99fdb13b00aae5206"
+checksum = "c7ee6e1b761dfffaaf7b5bbe68c113a576a3a802146c5c0b9fcec781e30d80a3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -246,17 +264,32 @@ dependencies = [
  "num",
 ]
 
+[[package]]
+name = "arrow-row"
+version = "31.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e65bfedf782fc92721e796fdd26ae7343c98ba9a9243d62def9e4e1c4c1cf0b"
+dependencies = [
+ "ahash",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "half",
+ "hashbrown 0.13.2",
+]
+
 [[package]]
 name = "arrow-schema"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69ef17c144f1253b9864f5a3e8f4c6f1e436bdd52394855d5942f132f776b64e"
+checksum = "73ca49d010b27e2d73f70c1d1f90c1b378550ed0f4ad379c4dea0c997d97d723"
 
 [[package]]
 name = "arrow-select"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2accaf218ff107e3df0ee8f1e09b092249a1cc741c4377858a1470fd27d7096"
+checksum = "976cbaeb1a85c09eea81f3f9c149c758630ff422ed0238624c5c3f4704b6a53c"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -267,9 +300,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-string"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a0954f9e1f45b04815ddacbde72899bf3c03a08fa6c0375f42178c4a01a510"
+checksum = "3d4882762f8f48a9218946c016553d38b04b4fe8202038dad4141b3b887b7da8"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -296,6 +329,17 @@ dependencies = [
  "xz2",
 ]
 
+[[package]]
+name = "async-recursion"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b015a331cc64ebd1774ba119538573603427eaace0a1950c423ab971f903796"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.61"
@@ -315,9 +359,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "base64"
-version = "0.13.0"
+version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "base64"
@@ -325,6 +369,12 @@ version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
 
+[[package]]
+name = "base64"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -333,18 +383,18 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "blake2"
-version = "0.10.4"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9cf849ee05b2ee5fba5e36f97ff8ec2533916700fc0758d40d92136a42f3388"
+checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
 dependencies = [
  "digest",
 ]
 
 [[package]]
 name = "blake3"
-version = "1.3.1"
+version = "1.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f"
+checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef"
 dependencies = [
  "arrayref",
  "arrayvec",
@@ -376,9 +426,9 @@ dependencies = [
 
 [[package]]
 name = "brotli-decompressor"
-version = "2.3.2"
+version = "2.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80"
+checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -398,9 +448,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.11.0"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
+checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
 
 [[package]]
 name = "byteorder"
@@ -410,9 +460,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "bytes"
-version = "1.2.1"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
+checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"
 
 [[package]]
 name = "bzip2"
@@ -437,9 +487,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.73"
+version = "1.0.78"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
 dependencies = [
  "jobserver",
 ]
@@ -475,9 +525,9 @@ dependencies = [
 
 [[package]]
 name = "comfy-table"
-version = "6.1.0"
+version = "6.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85914173c2f558d61613bfbbf1911f14e630895087a7ed2fafc0f5319e1536e7"
+checksum = "6e7b787b0dc42e8111badfdbe4c3059158ccb2db8780352fa1b01e8ccf45cc4d"
 dependencies = [
  "strum",
  "strum_macros",
@@ -486,9 +536,9 @@ dependencies = [
 
 [[package]]
 name = "const-random"
-version = "0.1.13"
+version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f590d95d011aa80b063ffe3253422ed5aa462af4e9867d43ce8337562bac77c4"
+checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e"
 dependencies = [
  "const-random-macro",
  "proc-macro-hack",
@@ -496,21 +546,21 @@ dependencies = [
 
 [[package]]
 name = "const-random-macro"
-version = "0.1.13"
+version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "615f6e27d000a2bffbc7f2f6a8669179378fa27ee4d0a509e985dfc0a7defb40"
+checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb"
 dependencies = [
  "getrandom",
- "lazy_static",
+ "once_cell",
  "proc-macro-hack",
  "tiny-keccak",
 ]
 
 [[package]]
 name = "constant_time_eq"
-version = "0.1.5"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
+checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279"
 
 [[package]]
 name = "core-foundation-sys"
@@ -576,9 +626,9 @@ dependencies = [
 
 [[package]]
 name = "cxx"
-version = "1.0.78"
+version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19f39818dcfc97d45b03953c1292efc4e80954e1583c4aa770bac1383e2310a4"
+checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579"
 dependencies = [
  "cc",
  "cxxbridge-flags",
@@ -588,9 +638,9 @@ dependencies = [
 
 [[package]]
 name = "cxx-build"
-version = "1.0.78"
+version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e580d70777c116df50c390d1211993f62d40302881e54d4b79727acb83d0199"
+checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70"
 dependencies = [
  "cc",
  "codespan-reporting",
@@ -603,15 +653,15 @@ dependencies = [
 
 [[package]]
 name = "cxxbridge-flags"
-version = "1.0.78"
+version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56a46460b88d1cec95112c8c363f0e2c39afdb237f60583b0b36343bf627ea9c"
+checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c"
 
 [[package]]
 name = "cxxbridge-macro"
-version = "1.0.78"
+version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "747b608fecf06b0d72d440f27acc99288207324b793be2c17991839f3d4995ea"
+checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -634,8 +684,7 @@ dependencies = [
 [[package]]
 name = "datafusion"
 version = "16.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f166d67281ee90d0b35d93a9d13b3b32c26f35348da9e5101a12e9de6093bd06"
+source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "ahash",
  "apache-avro",
@@ -655,7 +704,7 @@ dependencies = [
  "flate2",
  "futures",
  "glob",
- "hashbrown 0.13.1",
+ "hashbrown 0.13.2",
  "indexmap",
  "itertools",
  "lazy_static",
@@ -677,15 +726,14 @@ dependencies = [
  "tokio-stream",
  "tokio-util",
  "url",
- "uuid 1.2.1",
+ "uuid 1.2.2",
  "xz2",
 ]
 
 [[package]]
 name = "datafusion-common"
 version = "16.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cb2144c73ca974b00bf735a6e8692efe22c65731097bc49018a0edfbd1d0120"
+source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "apache-avro",
  "arrow",
@@ -700,8 +748,7 @@ dependencies = [
 [[package]]
 name = "datafusion-expr"
 version = "16.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6b988765372fdee77d805dda00cb7ffb28dfda831cc0b79aff9e09527b70402"
+source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "ahash",
  "arrow",
@@ -713,8 +760,7 @@ dependencies = [
 [[package]]
 name = "datafusion-optimizer"
 version = "16.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50b44aa5128e3e6f76d7ae8efc3a595133abd74aa60c3f50f39f93ea00ef302c"
+source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "arrow",
  "async-trait",
@@ -722,7 +768,7 @@ dependencies = [
  "datafusion-common",
  "datafusion-expr",
  "datafusion-physical-expr",
- "hashbrown 0.13.1",
+ "hashbrown 0.13.2",
  "log",
  "regex-syntax",
 ]
@@ -730,8 +776,7 @@ dependencies = [
 [[package]]
 name = "datafusion-physical-expr"
 version = "16.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fd7e12ce4d61eb698a83fde3bf14417fe67a23d7131b1f00c28dc70183aeed7"
+source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "ahash",
  "arrow",
@@ -744,7 +789,7 @@ dependencies = [
  "datafusion-expr",
  "datafusion-row",
  "half",
- "hashbrown 0.13.1",
+ "hashbrown 0.13.2",
  "indexmap",
  "itertools",
  "lazy_static",
@@ -755,7 +800,7 @@ dependencies = [
  "regex",
  "sha2",
  "unicode-segmentation",
- "uuid 1.2.1",
+ "uuid 1.2.2",
 ]
 
 [[package]]
@@ -767,6 +812,7 @@ dependencies = [
  "datafusion-common",
  "datafusion-expr",
  "datafusion-optimizer",
+ "datafusion-substrait",
  "futures",
  "mimalloc",
  "object_store",
@@ -781,8 +827,7 @@ dependencies = [
 [[package]]
 name = "datafusion-row"
 version = "16.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e34f6c09f1458190bb90305d70c2075bf2dd4cf8c51a65d5635e5217a3bb8bff"
+source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -793,8 +838,7 @@ dependencies = [
 [[package]]
 name = "datafusion-sql"
 version = "16.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b42f29fd2c98e0e0030db4638f971c91145ef5e67ab139f0426b2891e14b9bf5"
+source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "arrow-schema",
  "datafusion-common",
@@ -803,11 +847,25 @@ dependencies = [
  "sqlparser",
 ]
 
+[[package]]
+name = "datafusion-substrait"
+version = "16.0.0"
+source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
+dependencies = [
+ "async-recursion",
+ "datafusion",
+ "prost 0.11.6",
+ "prost-build 0.9.0",
+ "prost-types 0.11.6",
+ "substrait",
+ "tokio",
+]
+
 [[package]]
 name = "digest"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c"
+checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
 dependencies = [
  "block-buffer",
  "crypto-common",
@@ -820,6 +878,12 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
 
+[[package]]
+name = "dyn-clone"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60"
+
 [[package]]
 name = "either"
 version = "1.8.0"
@@ -844,6 +908,12 @@ dependencies = [
  "instant",
 ]
 
+[[package]]
+name = "fixedbitset"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
+
 [[package]]
 name = "flatbuffers"
 version = "22.9.29"
@@ -856,9 +926,9 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.0.24"
+version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
+checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
@@ -980,9 +1050,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.7"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
+checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
 dependencies = [
  "cfg-if",
  "libc",
@@ -991,15 +1061,15 @@ dependencies = [
 
 [[package]]
 name = "glob"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
 [[package]]
 name = "h2"
-version = "0.3.14"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be"
+checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4"
 dependencies = [
  "bytes",
  "fnv",
@@ -1016,9 +1086,9 @@ dependencies = [
 
 [[package]]
 name = "half"
-version = "2.1.0"
+version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad6a9459c9c30b177b925162351f97e7d967c7ea8bab3b8352805327daf45554"
+checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0"
 dependencies = [
  "crunchy",
  "num-traits",
@@ -1032,13 +1102,22 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
 [[package]]
 name = "hashbrown"
-version = "0.13.1"
+version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038"
+checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
 dependencies = [
  "ahash",
 ]
 
+[[package]]
+name = "heck"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "heck"
 version = "0.4.0"
@@ -1047,13 +1126,22 @@ checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.19"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
 dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "home"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "747309b4b440c06d57b0b25f2aee03ee9b5e5397d288c60e21fc709bb98a7408"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "http"
 version = "0.2.8"
@@ -1062,7 +1150,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
 dependencies = [
  "bytes",
  "fnv",
- "itoa 1.0.4",
+ "itoa 1.0.5",
 ]
 
 [[package]]
@@ -1090,9 +1178,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
 
 [[package]]
 name = "hyper"
-version = "0.14.20"
+version = "0.14.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac"
+checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -1103,7 +1191,7 @@ dependencies = [
  "http-body",
  "httparse",
  "httpdate",
- "itoa 1.0.4",
+ "itoa 1.0.5",
  "pin-project-lite",
  "socket2",
  "tokio",
@@ -1114,9 +1202,9 @@ dependencies = [
 
 [[package]]
 name = "hyper-rustls"
-version = "0.23.0"
+version = "0.23.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac"
+checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"
 dependencies = [
  "http",
  "hyper",
@@ -1127,9 +1215,9 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.51"
+version = "0.1.53"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed"
+checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
@@ -1141,9 +1229,9 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone-haiku"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fde6edd6cef363e9359ed3c98ba64590ba9eecba2293eb5a723ab32aee8926aa"
+checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
 dependencies = [
  "cxx",
  "cxx-build",
@@ -1171,9 +1259,9 @@ dependencies = [
 
 [[package]]
 name = "indoc"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adab1eaa3408fb7f0c777a73e7465fd5656136fc93b670eb6df3c88c2c1344e3"
+checksum = "da2d6f23ffea9d7e76c53eee25dfb67bcd8fde7f1198b0855350698c9f07c780"
 
 [[package]]
 name = "instant"
@@ -1192,9 +1280,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
 [[package]]
 name = "ipnet"
-version = "2.5.0"
+version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
+checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"
 
 [[package]]
 name = "itertools"
@@ -1213,9 +1301,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
 
 [[package]]
 name = "itoa"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
+checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
 
 [[package]]
 name = "jobserver"
@@ -1307,9 +1395,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.135"
+version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
+checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
 
 [[package]]
 name = "libflate"
@@ -1333,9 +1421,9 @@ dependencies = [
 
 [[package]]
 name = "libm"
-version = "0.2.5"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "292a948cd991e376cf75541fe5b97a1081d713c618b4f1b9500f8844e49eb565"
+checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
 
 [[package]]
 name = "libmimalloc-sys"
@@ -1349,9 +1437,9 @@ dependencies = [
 
 [[package]]
 name = "link-cplusplus"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369"
+checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
 dependencies = [
  "cc",
 ]
@@ -1447,44 +1535,30 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.5.4"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
+checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
 dependencies = [
  "adler",
 ]
 
 [[package]]
 name = "mio"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf"
+checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
 dependencies = [
  "libc",
  "log",
  "wasi",
- "windows-sys 0.36.1",
+ "windows-sys",
 ]
 
 [[package]]
-name = "multiversion"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "025c962a3dd3cc5e0e520aa9c612201d127dcdf28616974961a649dca64f5373"
-dependencies = [
- "multiversion-macros",
-]
-
-[[package]]
-name = "multiversion-macros"
-version = "0.6.1"
+name = "multimap"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8a3e2bde382ebf960c1f3e79689fa5941625fe9bf694a1cb64af3e85faff3af"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
+checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 
 [[package]]
 name = "num"
@@ -1565,9 +1639,9 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.13.1"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
 dependencies = [
  "hermit-abi",
  "libc",
@@ -1603,9 +1677,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.15.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
+checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
 
 [[package]]
 name = "ordered-float"
@@ -1628,22 +1702,22 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.3"
+version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
+checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf"
 dependencies = [
  "cfg-if",
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-sys 0.36.1",
+ "windows-sys",
 ]
 
 [[package]]
 name = "parquet"
-version = "29.0.0"
+version = "31.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d906343fd18ace6b998d5074697743e8e9358efa8c3c796a1381b98cba813338"
+checksum = "6b4ee1ffc0778395c9783a5c74f2cad2fb1a128ade95a965212d31b7b13e3d45"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -1653,13 +1727,13 @@ dependencies = [
  "arrow-ipc",
  "arrow-schema",
  "arrow-select",
- "base64 0.13.0",
+ "base64 0.21.0",
  "brotli",
  "bytes",
  "chrono",
  "flate2",
  "futures",
- "hashbrown 0.13.1",
+ "hashbrown 0.13.2",
  "lz4",
  "num",
  "num-bigint",
@@ -1674,9 +1748,9 @@ dependencies = [
 
 [[package]]
 name = "paste"
-version = "1.0.9"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1"
+checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
 
 [[package]]
 name = "percent-encoding"
@@ -1684,6 +1758,26 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
 
+[[package]]
+name = "pest"
+version = "2.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4257b4a04d91f7e9e6290be5d3da4804dd5784fafde3a497d73eb2b4a158c30a"
+dependencies = [
+ "thiserror",
+ "ucd-trie",
+]
+
+[[package]]
+name = "petgraph"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+]
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.9"
@@ -1704,25 +1798,131 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
 
 [[package]]
 name = "ppv-lite86"
-version = "0.2.16"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro-hack"
-version = "0.5.19"
+version = "0.5.20+deprecated"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
+checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.46"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b"
+checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2"
 dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prost"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
+dependencies = [
+ "bytes",
+ "prost-derive 0.9.0",
+]
+
+[[package]]
+name = "prost"
+version = "0.11.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698"
+dependencies = [
+ "bytes",
+ "prost-derive 0.11.6",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
+dependencies = [
+ "bytes",
+ "heck 0.3.3",
+ "itertools",
+ "lazy_static",
+ "log",
+ "multimap",
+ "petgraph",
+ "prost 0.9.0",
+ "prost-types 0.9.0",
+ "regex",
+ "tempfile",
+ "which",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.11.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3f8ad728fb08fe212df3c05169e940fbb6d9d16a877ddde14644a983ba2012e"
+dependencies = [
+ "bytes",
+ "heck 0.4.0",
+ "itertools",
+ "lazy_static",
+ "log",
+ "multimap",
+ "petgraph",
+ "prost 0.11.6",
+ "prost-types 0.11.6",
+ "regex",
+ "tempfile",
+ "which",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.11.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8bda8c0881ea9f722eb9629376db3d0b903b462477c1aafcb0566610ac28ac5d"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
+dependencies = [
+ "bytes",
+ "prost 0.9.0",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.11.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788"
+dependencies = [
+ "bytes",
+ "prost 0.11.6",
+]
+
 [[package]]
 name = "pyo3"
 version = "0.17.3"
@@ -1801,9 +2001,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.21"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
 dependencies = [
  "proc-macro2",
 ]
@@ -1849,9 +2049,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.7.0"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
+checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1870,6 +2070,15 @@ version = "0.6.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
 
+[[package]]
+name = "regress"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a92ff21fe8026ce3f2627faaf43606f0b67b014dbc9ccf027181a804f75d92e"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "remove_dir_all"
 version = "0.5.3"
@@ -1881,11 +2090,11 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.12"
+version = "0.11.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc"
+checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c"
 dependencies = [
- "base64 0.13.0",
+ "base64 0.13.1",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -1940,11 +2149,24 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
 
+[[package]]
+name = "rustfmt-wrapper"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed729e3bee08ec2befd593c27e90ca9fdd25efdc83c94c3b82eaef16e4f7406e"
+dependencies = [
+ "serde",
+ "tempfile",
+ "thiserror",
+ "toml",
+ "toolchain_find",
+]
+
 [[package]]
 name = "rustls"
-version = "0.20.6"
+version = "0.20.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033"
+checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f"
 dependencies = [
  "log",
  "ring",
@@ -1954,24 +2176,24 @@ dependencies = [
 
 [[package]]
 name = "rustls-pemfile"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55"
+checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b"
 dependencies = [
- "base64 0.13.0",
+ "base64 0.21.0",
 ]
 
 [[package]]
 name = "rustversion"
-version = "1.0.9"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
+checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70"
 
 [[package]]
 name = "ryu"
-version = "1.0.11"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
 
 [[package]]
 name = "same-file"
@@ -1982,6 +2204,30 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "schemars"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a5fb6c61f29e723026dc8e923d94c694313212abbecbbe5f55a7748eec5b307"
+dependencies = [
+ "dyn-clone",
+ "schemars_derive",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "schemars_derive"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f188d036977451159430f3b8dc82ec76364a42b7e289c2b18a9a18f4470058e9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "serde_derive_internals",
+ "syn",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -1990,9 +2236,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "scratch"
-version = "1.0.2"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898"
+checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2"
 
 [[package]]
 name = "sct"
@@ -2004,26 +2250,55 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "semver"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6"
+dependencies = [
+ "semver-parser",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7"
+dependencies = [
+ "pest",
+]
+
 [[package]]
 name = "seq-macro"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0772c5c30e1a0d91f6834f8e545c69281c099dfa9a3ac58d96a9fd629c8d4898"
+checksum = "1685deded9b272198423bdbdb907d8519def2f26cf3699040e54e8c4fbd5c5ce"
 
 [[package]]
 name = "serde"
-version = "1.0.145"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b"
+checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.145"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c"
+checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_derive_internals"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2032,15 +2307,26 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.86"
+version = "1.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074"
+checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
 dependencies = [
- "itoa 1.0.4",
+ "itoa 1.0.5",
  "ryu",
  "serde",
 ]
 
+[[package]]
+name = "serde_tokenstream"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "274f512d6748a01e67cbcde5b4307ab2c9d52a98a2b870a980ef0793a351deff"
+dependencies = [
+ "proc-macro2",
+ "serde",
+ "syn",
+]
+
 [[package]]
 name = "serde_urlencoded"
 version = "0.7.1"
@@ -2048,11 +2334,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
 dependencies = [
  "form_urlencoded",
- "itoa 1.0.4",
+ "itoa 1.0.5",
  "ryu",
  "serde",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.9.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92b5b431e8907b50339b51223b97d102db8d987ced36f6e4d03621db9316c834"
+dependencies = [
+ "indexmap",
+ "itoa 1.0.5",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "sha2"
 version = "0.10.6"
@@ -2081,9 +2380,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
 [[package]]
 name = "snafu"
-version = "0.7.2"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd726aec4ebad65756394ff89a9b9598793d4e30121cd71690244c1e497b3aee"
+checksum = "cb0656e7e3ffb70f6c39b3c2a86332bb74aa3c679da781642590f3c1118c5045"
 dependencies = [
  "doc-comment",
  "snafu-derive",
@@ -2091,11 +2390,11 @@ dependencies = [
 
 [[package]]
 name = "snafu-derive"
-version = "0.7.2"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712529e9b0b014eabaa345b38e06032767e3dc393e8b017e853b1d7247094e74"
+checksum = "475b3bbe5245c26f2d8a6f62d67c1f30eb9fffeccee721c45d162c3ebbdf81b2"
 dependencies = [
- "heck",
+ "heck 0.4.0",
  "proc-macro2",
  "quote",
  "syn",
@@ -2103,9 +2402,9 @@ dependencies = [
 
 [[package]]
 name = "snap"
-version = "1.0.5"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451"
+checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831"
 
 [[package]]
 name = "socket2"
@@ -2162,13 +2461,31 @@ version = "0.24.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
 dependencies = [
- "heck",
+ "heck 0.4.0",
  "proc-macro2",
  "quote",
  "rustversion",
  "syn",
 ]
 
+[[package]]
+name = "substrait"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2feb96a6a106e21161551af32dc4e0fdab3aceb926b940d7e92a086b640fc7c"
+dependencies = [
+ "heck 0.4.0",
+ "prost 0.11.6",
+ "prost-build 0.11.6",
+ "prost-types 0.11.6",
+ "schemars",
+ "serde",
+ "serde_json",
+ "serde_yaml",
+ "typify",
+ "walkdir",
+]
+
 [[package]]
 name = "subtle"
 version = "2.4.1"
@@ -2177,9 +2494,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.102"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1"
+checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2188,9 +2505,9 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.4"
+version = "0.12.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1"
+checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d"
 
 [[package]]
 name = "tempfile"
@@ -2208,27 +2525,27 @@ dependencies = [
 
 [[package]]
 name = "termcolor"
-version = "1.1.3"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
 dependencies = [
  "winapi-util",
 ]
 
 [[package]]
 name = "thiserror"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
+checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
+checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2272,9 +2589,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.24.1"
+version = "1.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae"
+checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb"
 dependencies = [
  "autocfg",
  "bytes",
@@ -2286,14 +2603,14 @@ dependencies = [
  "pin-project-lite",
  "socket2",
  "tokio-macros",
- "windows-sys 0.42.0",
+ "windows-sys",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "1.8.0"
+version = "1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
+checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2336,6 +2653,28 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "toml"
+version = "0.5.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toolchain_find"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e85654a10e7a07a47c6f19d93818f3f343e22927f2fa280c84f7c8042743413"
+dependencies = [
+ "home",
+ "lazy_static",
+ "regex",
+ "semver",
+ "walkdir",
+]
+
 [[package]]
 name = "tower-service"
 version = "0.3.2"
@@ -2376,9 +2715,9 @@ dependencies = [
 
 [[package]]
 name = "try-lock"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
+checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
 
 [[package]]
 name = "twox-hash"
@@ -2403,21 +2742,72 @@ dependencies = [
 
 [[package]]
 name = "typenum"
-version = "1.15.0"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
+
+[[package]]
+name = "typify"
+version = "0.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+checksum = "2e8486352f3c946e69f983558cfc09b295250b01e01b381ec67a05a812d01d63"
+dependencies = [
+ "typify-impl",
+ "typify-macro",
+]
+
+[[package]]
+name = "typify-impl"
+version = "0.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7624d0b911df6e2bbf34a236f76281f93b294cdde1d4df1dbdb748e5a7fefa5"
+dependencies = [
+ "heck 0.4.0",
+ "log",
+ "proc-macro2",
+ "quote",
+ "regress",
+ "rustfmt-wrapper",
+ "schemars",
+ "serde_json",
+ "syn",
+ "thiserror",
+ "unicode-ident",
+]
+
+[[package]]
+name = "typify-macro"
+version = "0.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c42802aa033cee7650a4e1509ba7d5848a56f84be7c4b31e4385ee12445e942"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "schemars",
+ "serde",
+ "serde_json",
+ "serde_tokenstream",
+ "syn",
+ "typify-impl",
+]
+
+[[package]]
+name = "ucd-trie"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81"
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.8"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
+checksum = "0046be40136ef78dc325e0edefccf84ccddacd0afcc1ca54103fa3c61bbdab1d"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.5"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
+checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
 
 [[package]]
 name = "unicode-normalization"
@@ -2442,9 +2832,15 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
 
 [[package]]
 name = "unindent"
-version = "0.1.10"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c"
+
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58ee9362deb4a96cef4d437d1ad49cffc9b9e92d202b6995674e928ce684f112"
+checksum = "bc7ed8ba44ca06be78ea1ad2c3682a43349126c8818054231ee6f4748012aed2"
 
 [[package]]
 name = "untrusted"
@@ -2474,9 +2870,9 @@ dependencies = [
 
 [[package]]
 name = "uuid"
-version = "1.2.1"
+version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "feb41e78f93363bb2df8b0e86a2ca30eed7806ea16ea0c790d757cf93f79be83"
+checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c"
 dependencies = [
  "getrandom",
  "serde",
@@ -2603,13 +2999,24 @@ dependencies = [
 
 [[package]]
 name = "webpki-roots"
-version = "0.22.5"
+version = "0.22.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be"
+checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87"
 dependencies = [
  "webpki",
 ]
 
+[[package]]
+name = "which"
+version = "4.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b"
+dependencies = [
+ "either",
+ "libc",
+ "once_cell",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -2641,19 +3048,6 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
-[[package]]
-name = "windows-sys"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
-dependencies = [
- "windows_aarch64_msvc 0.36.1",
- "windows_i686_gnu 0.36.1",
- "windows_i686_msvc 0.36.1",
- "windows_x86_64_gnu 0.36.1",
- "windows_x86_64_msvc 0.36.1",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.42.0"
@@ -2661,85 +3055,55 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
 dependencies = [
  "windows_aarch64_gnullvm",
- "windows_aarch64_msvc 0.42.0",
- "windows_i686_gnu 0.42.0",
- "windows_i686_msvc 0.42.0",
- "windows_x86_64_gnu 0.42.0",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
  "windows_x86_64_gnullvm",
- "windows_x86_64_msvc 0.42.0",
+ "windows_x86_64_msvc",
 ]
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.42.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.42.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.42.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.42.0"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"
+checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.42.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"
+checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.42.0"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"
+checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.42.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
+checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
 [[package]]
 name = "winreg"
@@ -2782,9 +3146,9 @@ dependencies = [
 
 [[package]]
 name = "zstd"
-version = "0.12.0+zstd.1.5.2"
+version = "0.12.2+zstd.1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8148aa921e9d53217ab9322f8553bd130f7ae33489db68b381d76137d2e6374"
+checksum = "e9262a83dc741c0b0ffec209881b45dbc232c21b02a2b9cb1adb93266e41303d"
 dependencies = [
  "zstd-safe",
 ]
@@ -2801,10 +3165,11 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.1+zstd.1.5.2"
+version = "2.0.5+zstd.1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
+checksum = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596"
 dependencies = [
  "cc",
  "libc",
+ "pkg-config",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 956938487..078fcaa2f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -34,10 +34,11 @@ default = ["mimalloc"]
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 rand = "0.8"
 pyo3 = { version = "~0.17.3", features = ["extension-module", "abi3", "abi3-py37"] }
-datafusion = { version = "16.0.0", features = ["pyarrow", "avro"] }
-datafusion-expr = { version = "16.0.0" }
-datafusion-optimizer = { version = "16.0.0" }
-datafusion-common = { version = "16.0.0", features = ["pyarrow"] }
+datafusion = { git = "https://github.com/apache/arrow-datafusion", branch = "master", features = ["pyarrow", "avro"] }
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", branch = "master" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", branch = "master" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion", branch = "master", features = ["pyarrow"] }
+datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", branch = "master" }
 uuid = { version = "0.8", features = ["v4"] }
 mimalloc = { version = "*", optional = true, default-features = false }
 async-trait = "0.1"
@@ -55,4 +56,4 @@ name = "datafusion._internal"
 
 [profile.release]
 lto = true
-codegen-units = 1
\ No newline at end of file
+codegen-units = 1
diff --git a/datafusion/substrait.py b/datafusion/substrait.py
new file mode 100644
index 000000000..eff809a0c
--- /dev/null
+++ b/datafusion/substrait.py
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+from ._internal import substrait
+
+
+def __getattr__(name):
+    return getattr(substrait, name)
diff --git a/datafusion/tests/test_substrait.py b/datafusion/tests/test_substrait.py
new file mode 100644
index 000000000..1e1313f99
--- /dev/null
+++ b/datafusion/tests/test_substrait.py
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import pyarrow as pa
+import pyarrow.dataset as ds
+
+from datafusion import column, literal, SessionContext
+import pytest
+
+
+@pytest.fixture
+def config():
+    return Config()
+
+def test_create_context_no_args():
+    SessionContext()
+
+
+def test_register_table(ctx, database):
+    default = ctx.catalog()
+    public = default.database("public")
+    assert public.names() == {"csv", "csv1", "csv2"}
+    table = public.table("csv")
+
+    ctx.register_table("csv3", table)
+    assert public.names() == {"csv", "csv1", "csv2", "csv3"}
+
+
+def test_register_dataset(ctx):
+    # create a RecordBatch and register it as a pyarrow.dataset.Dataset
+    batch = pa.RecordBatch.from_arrays(
+        [pa.array([1, 2, 3]), pa.array([4, 5, 6])],
+        names=["a", "b"],
+    )
+    dataset = ds.dataset([batch])
+    ctx.register_dataset("t", dataset)
+
+    assert ctx.tables() == {"t"}
+
+    result = ctx.sql("SELECT a+b, a-b FROM t").collect()
+
+    assert result[0].column(0) == pa.array([5, 7, 9])
+    assert result[0].column(1) == pa.array([-3, -3, -3])
+
diff --git a/src/context.rs b/src/context.rs
index 2ffa3da3c..71f99f55a 100644
--- a/src/context.rs
+++ b/src/context.rs
@@ -48,8 +48,9 @@ use datafusion_common::ScalarValue;
 /// It has a powerful optimizer, a physical planner for local execution, and a
 /// multi-threaded execution engine to perform the execution.
 #[pyclass(name = "SessionContext", module = "datafusion", subclass, unsendable)]
+#[derive(Clone)]
 pub(crate) struct PySessionContext {
-    ctx: SessionContext,
+    pub(crate) ctx: SessionContext,
 }
 
 #[pymethods]
@@ -481,3 +482,15 @@ fn convert_table_partition_cols(
         })
         .collect::<Result<Vec<_>, _>>()
 }
+
+impl From<PySessionContext> for SessionContext {
+    fn from(ctx: PySessionContext) -> SessionContext {
+        ctx.ctx
+    }
+}
+
+impl From<SessionContext> for PySessionContext {
+    fn from(ctx: SessionContext) -> PySessionContext {
+        PySessionContext { ctx }
+    }
+}
diff --git a/src/errors.rs b/src/errors.rs
index 29d3e8f23..69a476d89 100644
--- a/src/errors.rs
+++ b/src/errors.rs
@@ -17,6 +17,7 @@
 
 use core::fmt;
 use std::error::Error;
+use std::fmt::Debug;
 
 use datafusion::arrow::error::ArrowError;
 use datafusion::error::DataFusionError as InnerDataFusionError;
@@ -69,3 +70,15 @@ impl From<DataFusionError> for PyErr {
 }
 
 impl Error for DataFusionError {}
+
+pub fn py_type_err(e: impl Debug) -> PyErr {
+    PyErr::new::<pyo3::exceptions::PyTypeError, _>(format!("{:?}", e))
+}
+
+pub fn py_runtime_err(e: impl Debug) -> PyErr {
+    PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(format!("{:?}", e))
+}
+
+pub fn py_datafusion_err(e: impl Debug) -> PyErr {
+    PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(format!("{:?}", e))
+}
diff --git a/src/lib.rs b/src/lib.rs
index 28544f5d3..21b47f449 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -34,8 +34,10 @@ pub mod errors;
 mod expression;
 #[allow(clippy::borrow_deref_ref)]
 mod functions;
+pub mod logical;
 mod pyarrow_filter_expression;
 pub mod store;
+pub mod substrait;
 #[allow(clippy::borrow_deref_ref)]
 mod udaf;
 #[allow(clippy::borrow_deref_ref)]
@@ -62,6 +64,7 @@ fn _internal(py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<udf::PyScalarUDF>()?;
     m.add_class::<udaf::PyAggregateUDF>()?;
     m.add_class::<config::PyConfig>()?;
+    m.add_class::<logical::PyLogicalPlan>()?;
 
     // Register the functions as a submodule
     let funcs = PyModule::new(py, "functions")?;
@@ -72,5 +75,10 @@ fn _internal(py: Python, m: &PyModule) -> PyResult<()> {
     store::init_module(store)?;
     m.add_submodule(store)?;
 
+    // Register substrait as a submodule
+    let substrait = PyModule::new(py, "substrait")?;
+    substrait::init_module(substrait)?;
+    m.add_submodule(substrait)?;
+
     Ok(())
 }
diff --git a/src/logical.rs b/src/logical.rs
new file mode 100644
index 000000000..8c3acf717
--- /dev/null
+++ b/src/logical.rs
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use datafusion_expr::LogicalPlan;
+use pyo3::prelude::*;
+
+#[pyclass(name = "LogicalPlan", module = "substrait", subclass)]
+#[derive(Debug, Clone)]
+pub struct PyLogicalPlan {
+    pub(crate) plan: Arc<LogicalPlan>,
+}
+
+impl PyLogicalPlan {
+    /// creates a new PyLogicalPlan
+    pub fn new(plan: LogicalPlan) -> Self {
+        Self {
+            plan: Arc::new(plan),
+        }
+    }
+}
+
+impl From<PyLogicalPlan> for LogicalPlan {
+    fn from(logical_plan: PyLogicalPlan) -> LogicalPlan {
+        logical_plan.plan.as_ref().clone()
+    }
+}
+
+impl From<LogicalPlan> for PyLogicalPlan {
+    fn from(logical_plan: LogicalPlan) -> PyLogicalPlan {
+        PyLogicalPlan {
+            plan: Arc::new(logical_plan),
+        }
+    }
+}
diff --git a/src/substrait.rs b/src/substrait.rs
new file mode 100644
index 000000000..6a3d38018
--- /dev/null
+++ b/src/substrait.rs
@@ -0,0 +1,155 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use pyo3::prelude::*;
+
+use crate::context::PySessionContext;
+use crate::errors::py_datafusion_err;
+use crate::{logical::PyLogicalPlan, utils::wait_for_future};
+
+use crate::errors::DataFusionError;
+use datafusion_substrait::{consumer, producer, serializer, substrait::proto::Plan};
+
+#[pyclass(name = "plan", module = "datafusion.substrait", subclass, unsendable)]
+#[derive(Debug, Clone)]
+pub(crate) struct PyPlan {
+    pub(crate) plan: Plan,
+}
+
+impl From<PyPlan> for Plan {
+    fn from(plan: PyPlan) -> Plan {
+        plan.plan
+    }
+}
+
+impl From<Plan> for PyPlan {
+    fn from(plan: Plan) -> PyPlan {
+        PyPlan { plan }
+    }
+}
+
+/// A PySubstraitSerializer is a representation of a Serializer that is capable of both serializing
+/// a `LogicalPlan` instance to Substrait Protobuf bytes and also deserialize Substrait Protobuf bytes
+/// to a valid `LogicalPlan` instance.
+#[pyclass(
+    name = "serde",
+    module = "datafusion.substrait",
+    subclass,
+    unsendable
+)]
+#[derive(Debug, Clone)]
+pub(crate) struct PySubstraitSerializer;
+
+#[pymethods]
+impl PySubstraitSerializer {
+    #[staticmethod]
+    pub fn serialize(
+        sql: &str,
+        ctx: PySessionContext,
+        path: &str,
+        py: Python,
+    ) -> PyResult<()> {
+        wait_for_future(py, serializer::serialize(sql, &ctx.ctx, path))
+            .map_err(DataFusionError::from)?;
+        Ok(())
+    }
+
+    #[staticmethod]
+    pub fn serialize_to_plan(
+        sql: &str,
+        ctx: PySessionContext,
+        py: Python,
+    ) -> PyResult<PyPlan> {
+        match PySubstraitSerializer::serialize_bytes(sql, ctx, py) {
+            Ok(proto_bytes) => PySubstraitSerializer::deserialize_bytes(proto_bytes, py),
+            Err(e) => Err(py_datafusion_err(e)),
+        }
+    }
+
+    #[staticmethod]
+    pub fn serialize_bytes(
+        sql: &str,
+        ctx: PySessionContext,
+        py: Python,
+    ) -> PyResult<Vec<u8>> {
+        let proto_bytes: Vec<u8> = wait_for_future(py, serializer::serialize_bytes(sql, &ctx.ctx))
+            .map_err(DataFusionError::from)?;
+        Ok(proto_bytes)
+    }
+
+    #[staticmethod]
+    pub fn deserialize(path: &str, py: Python) -> PyResult<PyPlan> {
+        let plan =
+            wait_for_future(py, serializer::deserialize(path)).map_err(DataFusionError::from)?;
+        Ok(PyPlan { plan: *plan })
+    }
+
+    #[staticmethod]
+    pub fn deserialize_bytes(proto_bytes: Vec<u8>, py: Python) -> PyResult<PyPlan> {
+        let plan = wait_for_future(py, serializer::deserialize_bytes(proto_bytes))
+            .map_err(DataFusionError::from)?;
+        Ok(PyPlan { plan: *plan })
+    }
+}
+
+#[pyclass(name = "producer", module = "datafusion.substrait", subclass, unsendable)]
+#[derive(Debug, Clone)]
+pub(crate) struct PySubstraitProducer;
+
+#[pymethods]
+impl PySubstraitProducer {
+    /// Convert DataFusion LogicalPlan to Substrait Plan
+    #[staticmethod]
+    pub fn to_substrait_plan(plan: PyLogicalPlan) -> PyResult<PyPlan> {
+        match producer::to_substrait_plan(&plan.plan) {
+            Ok(plan) => Ok(PyPlan { plan: *plan }),
+            Err(e) => Err(py_datafusion_err(e)),
+        }
+    }
+}
+
+#[pyclass(
+    name = "consumer",
+    module = "datafusion.substrait",
+    subclass,
+    unsendable
+)]
+#[derive(Debug, Clone)]
+pub(crate) struct PySubstraitConsumer;
+
+#[pymethods]
+impl PySubstraitConsumer {
+    /// Convert Substrait Plan to DataFusion DataFrame
+    #[staticmethod]
+    pub fn from_substrait_plan(
+        ctx: &mut PySessionContext,
+        plan: PyPlan,
+        py: Python,
+    ) -> PyResult<PyLogicalPlan> {
+        let result = consumer::from_substrait_plan(&mut ctx.ctx, &plan.plan);
+        let logical_plan = wait_for_future(py, result).map_err(DataFusionError::from)?;
+        Ok(PyLogicalPlan::new(logical_plan))
+    }
+}
+
+
+pub(crate) fn init_module(m: &PyModule) -> PyResult<()> {
+    m.add_class::<PySubstraitConsumer>()?;
+    m.add_class::<PySubstraitProducer>()?;
+    m.add_class::<PySubstraitSerializer>()?;
+    Ok(())
+}

From 5e398a5388d0f83abbe6e51a65575d7806574c53 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 15:57:22 -0500
Subject: [PATCH 02/20] documention for the substrait module including examples

---
 README.md | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/README.md b/README.md
index 17e872af1..ab89ff6dd 100644
--- a/README.md
+++ b/README.md
@@ -85,6 +85,43 @@ This produces the following chart:
 
 ![Chart](examples/chart.png)
 
+## Substrait Support
+
+`arrow-datafusion-python` has bindings which allow for serializing a SQL query to substrait protobuf format and deserializing substrait protobuf bytes to a DataFusion `LogicalPlan`, `PyLogicalPlan` in a Python context, which can then be executed.
+
+### Example of Serializing/Deserializing Substrait Plans
+
+```python
+from datafusion import SessionContext
+from datafusion import substrait as ss
+
+# Create a DataFusion context
+ctx = SessionContext()
+
+# Register table with context
+ctx.register_parquet('aggregate_test_data', './testing/data/csv/aggregate_test_100.csv')
+
+substrait_plan = ss.substrait.serde.serialize_to_plan("SELECT * FROM aggregate_test_data", ctx)
+# type(substrait_plan) -> <class 'datafusion.substrait.plan'>
+
+# Alternative serialization approaches
+# type(substrait_bytes) -> <class 'list'>, at this point the bytes can be distributed to file, network, etc safely
+# where they could subsequently be deserialized on the receiving end.
+substrait_bytes = ss.substrait.serde.serialize_bytes("SELECT * FROM aggregate_test_data", ctx)
+
+# Imagine here bytes would be read from network, file, etc ... for example brevity this is omitted and variable is simply reused
+# type(substrait_plan) -> <class 'datafusion.substrait.plan'>
+substrait_plan = ss.substrait.serde.deserialize_bytes(substrait_bytes)
+
+# type(df_logical_plan) -> <class 'substrait.LogicalPlan'>
+df_logical_plan = ss.substrait.consumer.from_substrait_plan(ctx, substrait_plan)
+
+# Back to Substrait Plan just for demonstration purposes
+# type(substrait_plan) -> <class 'datafusion.substrait.plan'>
+substrait_plan = ss.substrait.producer.to_substrait_plan(df_logical_plan)
+
+```
+
 ## How to install (from pip)
 
 ### Pip

From 679a539c9dbd731bcae48704ff0d50a32d6f3f77 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 16:34:48 -0500
Subject: [PATCH 03/20] add pytest for substrait

---
 datafusion/tests/test_substrait.py | 38 ++++++++++--------------------
 1 file changed, 12 insertions(+), 26 deletions(-)

diff --git a/datafusion/tests/test_substrait.py b/datafusion/tests/test_substrait.py
index 1e1313f99..7f31c5a99 100644
--- a/datafusion/tests/test_substrait.py
+++ b/datafusion/tests/test_substrait.py
@@ -15,46 +15,32 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import os
-
 import pyarrow as pa
 import pyarrow.dataset as ds
 
 from datafusion import column, literal, SessionContext
+from datafusion import substrait as ss
 import pytest
 
 
 @pytest.fixture
-def config():
-    return Config()
-
-def test_create_context_no_args():
-    SessionContext()
-
-
-def test_register_table(ctx, database):
-    default = ctx.catalog()
-    public = default.database("public")
-    assert public.names() == {"csv", "csv1", "csv2"}
-    table = public.table("csv")
+def ctx():
+    return SessionContext()
 
-    ctx.register_table("csv3", table)
-    assert public.names() == {"csv", "csv1", "csv2", "csv3"}
 
-
-def test_register_dataset(ctx):
-    # create a RecordBatch and register it as a pyarrow.dataset.Dataset
+def test_substrait_serialization(ctx):
     batch = pa.RecordBatch.from_arrays(
         [pa.array([1, 2, 3]), pa.array([4, 5, 6])],
         names=["a", "b"],
     )
-    dataset = ds.dataset([batch])
-    ctx.register_dataset("t", dataset)
-
-    assert ctx.tables() == {"t"}
 
-    result = ctx.sql("SELECT a+b, a-b FROM t").collect()
+    ctx.register_record_batches("t", [[batch]])
 
-    assert result[0].column(0) == pa.array([5, 7, 9])
-    assert result[0].column(1) == pa.array([-3, -3, -3])
+    assert ctx.tables() == {"t"}
 
+    # For now just make sure the method calls blow up
+    substrait_plan = ss.substrait.serde.serialize_to_plan("SELECT * FROM t", ctx)
+    substrait_bytes = ss.substrait.serde.serialize_bytes("SELECT * FROM t", ctx)
+    substrait_plan = ss.substrait.serde.deserialize_bytes(substrait_bytes)
+    df_logical_plan = ss.substrait.consumer.from_substrait_plan(ctx, substrait_plan)
+    substrait_plan = ss.substrait.producer.to_substrait_plan(df_logical_plan)

From eb3f82b154dcdb2e4faf67f412fe8471d301dcef Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 16:38:02 -0500
Subject: [PATCH 04/20] cargo fmt

---
 src/substrait.rs | 34 ++++++++++------------------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/src/substrait.rs b/src/substrait.rs
index 6a3d38018..0c42e2104 100644
--- a/src/substrait.rs
+++ b/src/substrait.rs
@@ -45,35 +45,21 @@ impl From<Plan> for PyPlan {
 /// A PySubstraitSerializer is a representation of a Serializer that is capable of both serializing
 /// a `LogicalPlan` instance to Substrait Protobuf bytes and also deserialize Substrait Protobuf bytes
 /// to a valid `LogicalPlan` instance.
-#[pyclass(
-    name = "serde",
-    module = "datafusion.substrait",
-    subclass,
-    unsendable
-)]
+#[pyclass(name = "serde", module = "datafusion.substrait", subclass, unsendable)]
 #[derive(Debug, Clone)]
 pub(crate) struct PySubstraitSerializer;
 
 #[pymethods]
 impl PySubstraitSerializer {
     #[staticmethod]
-    pub fn serialize(
-        sql: &str,
-        ctx: PySessionContext,
-        path: &str,
-        py: Python,
-    ) -> PyResult<()> {
+    pub fn serialize(sql: &str, ctx: PySessionContext, path: &str, py: Python) -> PyResult<()> {
         wait_for_future(py, serializer::serialize(sql, &ctx.ctx, path))
             .map_err(DataFusionError::from)?;
         Ok(())
     }
 
     #[staticmethod]
-    pub fn serialize_to_plan(
-        sql: &str,
-        ctx: PySessionContext,
-        py: Python,
-    ) -> PyResult<PyPlan> {
+    pub fn serialize_to_plan(sql: &str, ctx: PySessionContext, py: Python) -> PyResult<PyPlan> {
         match PySubstraitSerializer::serialize_bytes(sql, ctx, py) {
             Ok(proto_bytes) => PySubstraitSerializer::deserialize_bytes(proto_bytes, py),
             Err(e) => Err(py_datafusion_err(e)),
@@ -81,11 +67,7 @@ impl PySubstraitSerializer {
     }
 
     #[staticmethod]
-    pub fn serialize_bytes(
-        sql: &str,
-        ctx: PySessionContext,
-        py: Python,
-    ) -> PyResult<Vec<u8>> {
+    pub fn serialize_bytes(sql: &str, ctx: PySessionContext, py: Python) -> PyResult<Vec<u8>> {
         let proto_bytes: Vec<u8> = wait_for_future(py, serializer::serialize_bytes(sql, &ctx.ctx))
             .map_err(DataFusionError::from)?;
         Ok(proto_bytes)
@@ -106,7 +88,12 @@ impl PySubstraitSerializer {
     }
 }
 
-#[pyclass(name = "producer", module = "datafusion.substrait", subclass, unsendable)]
+#[pyclass(
+    name = "producer",
+    module = "datafusion.substrait",
+    subclass,
+    unsendable
+)]
 #[derive(Debug, Clone)]
 pub(crate) struct PySubstraitProducer;
 
@@ -146,7 +133,6 @@ impl PySubstraitConsumer {
     }
 }
 
-
 pub(crate) fn init_module(m: &PyModule) -> PyResult<()> {
     m.add_class::<PySubstraitConsumer>()?;
     m.add_class::<PySubstraitProducer>()?;

From a8c6b45a2c3dfcf8db07d3379abad5fb96f226db Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 17:02:21 -0500
Subject: [PATCH 05/20] Add action for setting up build environment and also
 install protobuf compiler

---
 .github/actions/setup-builder/action.yaml | 39 +++++++++++++++++++++++
 .github/workflows/test.yaml               |  9 +++---
 2 files changed, 43 insertions(+), 5 deletions(-)
 create mode 100644 .github/actions/setup-builder/action.yaml

diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml
new file mode 100644
index 000000000..13a3008b7
--- /dev/null
+++ b/.github/actions/setup-builder/action.yaml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder
+description: 'Prepare Rust Build Environment'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. stable)'
+    required: true
+    default: 'stable'
+runs:
+  using: "composite"
+  steps:
+    - name: Install Build Dependencies
+      shell: bash
+      run: |
+        apt-get update
+        apt-get install -y protobuf-compiler
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        echo "Installing ${{ inputs.rust-version }}"
+        rustup toolchain install ${{ inputs.rust-version }}
+        rustup default ${{ inputs.rust-version }}
+        rustup component add rustfmt
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 327b1cf45..dffaf7b99 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -46,12 +46,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Rust Toolchain
-        uses: actions-rs/toolchain@v1
-        id: rust-toolchain
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
         with:
-          toolchain: ${{ matrix.toolchain }}
-          override: true
+          rust-version: ${{ matrix.toolchain }}
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -103,5 +101,6 @@ jobs:
         run: |
           git submodule update --init
           source venv/bin/activate
+          export PATH=$PATH:$HOME/d/protoc/bin
           maturin develop --locked
           RUST_BACKTRACE=1 pytest -v .

From fca3cc688df075a78908562e61ad34b9d6a24ebc Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 17:11:55 -0500
Subject: [PATCH 06/20] pin to a specific revision instead of master

---
 .github/workflows/test.yaml | 19 ++++++++++++++++---
 Cargo.toml                  | 10 +++++-----
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index dffaf7b99..f648041cd 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -46,10 +46,23 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
+      - name: Setup Rust Toolchain
+        uses: actions-rs/toolchain@v1
+        id: rust-toolchain
         with:
-          rust-version: ${{ matrix.toolchain }}
+          toolchain: ${{ matrix.toolchain }}
+          override: true
+
+      - name: Install protobuf compiler
+        shell: bash
+        run: |
+          mkdir -p $HOME/d/protoc
+          cd $HOME/d/protoc
+          export PROTO_ZIP="protoc-21.4-linux-x86_64.zip"
+          curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+          unzip $PROTO_ZIP
+          export PATH=$PATH:$HOME/d/protoc/bin
+          protoc --version
 
       - name: Setup Python
         uses: actions/setup-python@v4
diff --git a/Cargo.toml b/Cargo.toml
index c11eda0e8..f20a6aa06 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -34,11 +34,11 @@ default = ["mimalloc"]
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 rand = "0.8"
 pyo3 = { version = "~0.17.3", features = ["extension-module", "abi3", "abi3-py37"] }
-datafusion = { git = "https://github.com/apache/arrow-datafusion", branch = "master", features = ["pyarrow", "avro"] }
-datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", branch = "master" }
-datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", branch = "master" }
-datafusion-common = { git = "https://github.com/apache/arrow-datafusion", branch = "master", features = ["pyarrow"] }
-datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", branch = "master" }
+datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "e6a4d4da853d02b346e1b5884c2f0226b28bec66", features = ["pyarrow", "avro"] }
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "e6a4d4da853d02b346e1b5884c2f0226b28bec66" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "e6a4d4da853d02b346e1b5884c2f0226b28bec66" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "e6a4d4da853d02b346e1b5884c2f0226b28bec66", features = ["pyarrow"] }
+datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", rev = "e6a4d4da853d02b346e1b5884c2f0226b28bec66" }
 uuid = { version = "0.8", features = ["v4"] }
 mimalloc = { version = "*", optional = true, default-features = false }
 async-trait = "0.1"

From 8f47853ab6183dee972c8c93ad17e2def108a33a Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 17:14:24 -0500
Subject: [PATCH 07/20] pin to specific version, wrong version last time

---
 Cargo.toml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index f20a6aa06..0ab8e98db 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -34,11 +34,11 @@ default = ["mimalloc"]
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 rand = "0.8"
 pyo3 = { version = "~0.17.3", features = ["extension-module", "abi3", "abi3-py37"] }
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "e6a4d4da853d02b346e1b5884c2f0226b28bec66", features = ["pyarrow", "avro"] }
-datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "e6a4d4da853d02b346e1b5884c2f0226b28bec66" }
-datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "e6a4d4da853d02b346e1b5884c2f0226b28bec66" }
-datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "e6a4d4da853d02b346e1b5884c2f0226b28bec66", features = ["pyarrow"] }
-datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", rev = "e6a4d4da853d02b346e1b5884c2f0226b28bec66" }
+datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "6dce728a3c7130ca3590a16f413c7c6ccb7209b7", features = ["pyarrow", "avro"] }
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "6dce728a3c7130ca3590a16f413c7c6ccb7209b7" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "6dce728a3c7130ca3590a16f413c7c6ccb7209b7" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "6dce728a3c7130ca3590a16f413c7c6ccb7209b7", features = ["pyarrow"] }
+datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", rev = "6dce728a3c7130ca3590a16f413c7c6ccb7209b7" }
 uuid = { version = "0.8", features = ["v4"] }
 mimalloc = { version = "*", optional = true, default-features = false }
 async-trait = "0.1"

From c91c5fe2c7e8ea6f79e439189a74ac63a1966af0 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 17:21:55 -0500
Subject: [PATCH 08/20] Remove --locked since building from github revisions

---
 .github/workflows/build.yml | 4 ++--
 .github/workflows/test.yaml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 868fd8953..2398e6b9e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -73,7 +73,7 @@ jobs:
           path: .
 
       - name: Build Python package
-        run: maturin build --release --strip --locked
+        run: maturin build --release --strip
 
       - name: List Windows wheels
         if: matrix.os == 'windows-latest'
@@ -108,7 +108,7 @@ jobs:
           docker run --rm -v $(pwd):/io \
             --workdir /io \
             ghcr.io/pyo3/maturin:v0.14.2 \
-            build --release --manylinux 2014 --locked
+            build --release --manylinux 2014
       - name: Archive wheels
         uses: actions/upload-artifact@v3
         with:
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index f648041cd..23cb02b31 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -115,5 +115,5 @@ jobs:
           git submodule update --init
           source venv/bin/activate
           export PATH=$PATH:$HOME/d/protoc/bin
-          maturin develop --locked
+          maturin develop
           RUST_BACKTRACE=1 pytest -v .

From 72f81034af8488e8195050298fc2a604711b95c4 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 17:40:41 -0500
Subject: [PATCH 09/20] add lock file

---
 Cargo.lock | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7612037f8..918631ab9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -678,7 +678,7 @@ dependencies = [
 [[package]]
 name = "datafusion"
 version = "16.0.0"
-source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
+source = "git+https://github.com/apache/arrow-datafusion?rev=6dce728a3c7130ca3590a16f413c7c6ccb7209b7#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "ahash",
  "apache-avro",
@@ -727,7 +727,7 @@ dependencies = [
 [[package]]
 name = "datafusion-common"
 version = "16.0.0"
-source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
+source = "git+https://github.com/apache/arrow-datafusion?rev=6dce728a3c7130ca3590a16f413c7c6ccb7209b7#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "apache-avro",
  "arrow",
@@ -742,7 +742,7 @@ dependencies = [
 [[package]]
 name = "datafusion-expr"
 version = "16.0.0"
-source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
+source = "git+https://github.com/apache/arrow-datafusion?rev=6dce728a3c7130ca3590a16f413c7c6ccb7209b7#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "ahash",
  "arrow",
@@ -754,7 +754,7 @@ dependencies = [
 [[package]]
 name = "datafusion-optimizer"
 version = "16.0.0"
-source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
+source = "git+https://github.com/apache/arrow-datafusion?rev=6dce728a3c7130ca3590a16f413c7c6ccb7209b7#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "arrow",
  "async-trait",
@@ -770,7 +770,7 @@ dependencies = [
 [[package]]
 name = "datafusion-physical-expr"
 version = "16.0.0"
-source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
+source = "git+https://github.com/apache/arrow-datafusion?rev=6dce728a3c7130ca3590a16f413c7c6ccb7209b7#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "ahash",
  "arrow",
@@ -821,7 +821,7 @@ dependencies = [
 [[package]]
 name = "datafusion-row"
 version = "16.0.0"
-source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
+source = "git+https://github.com/apache/arrow-datafusion?rev=6dce728a3c7130ca3590a16f413c7c6ccb7209b7#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -832,7 +832,7 @@ dependencies = [
 [[package]]
 name = "datafusion-sql"
 version = "16.0.0"
-source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
+source = "git+https://github.com/apache/arrow-datafusion?rev=6dce728a3c7130ca3590a16f413c7c6ccb7209b7#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "arrow-schema",
  "datafusion-common",
@@ -844,7 +844,7 @@ dependencies = [
 [[package]]
 name = "datafusion-substrait"
 version = "16.0.0"
-source = "git+https://github.com/apache/arrow-datafusion?branch=master#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
+source = "git+https://github.com/apache/arrow-datafusion?rev=6dce728a3c7130ca3590a16f413c7c6ccb7209b7#6dce728a3c7130ca3590a16f413c7c6ccb7209b7"
 dependencies = [
  "async-recursion",
  "datafusion",

From 19614151047d2e83fd0288f64fffc8e065c072a7 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 17:54:55 -0500
Subject: [PATCH 10/20] adjust path to include protoc

---
 .github/workflows/build.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2398e6b9e..cb4b76538 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -105,6 +105,7 @@ jobs:
       - name: Build wheels
         run: |
           export RUSTFLAGS='-C target-cpu=skylake'
+          export PATH=$PATH:$HOME/d/protoc/bin
           docker run --rm -v $(pwd):/io \
             --workdir /io \
             ghcr.io/pyo3/maturin:v0.14.2 \
@@ -130,10 +131,11 @@ jobs:
       - run: cat LICENSE.txt
       - name: Build sdist
         run: |
+          export PATH=$PATH:$HOME/d/protoc/bin
           docker run --rm -v $(pwd):/io \
             --workdir /io \
             ghcr.io/pyo3/maturin:v0.14.2 \
-            build --release --sdist --locked
+            build --release --sdist
           find target/wheels/
       - name: Archive wheels
         uses: actions/upload-artifact@v2

From 94b00de422211adeb3d30c225dcbce5e776a5df4 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 18:08:43 -0500
Subject: [PATCH 11/20] set PROTOC evn variable

---
 .github/workflows/build.yml | 3 ++-
 .github/workflows/test.yaml | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index cb4b76538..cf6f61d01 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -132,8 +132,9 @@ jobs:
       - name: Build sdist
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
+          export PROTOC=$HOME/d/protoc/bin
           docker run --rm -v $(pwd):/io \
-            --workdir /io \
+            -e PROTOC='$(HOME)/d/protoc/bin' --workdir /io \
             ghcr.io/pyo3/maturin:v0.14.2 \
             build --release --sdist
           find target/wheels/
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 23cb02b31..4f42db557 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -62,6 +62,7 @@ jobs:
           curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
           unzip $PROTO_ZIP
           export PATH=$PATH:$HOME/d/protoc/bin
+          export PROTOC=$HOME/d/protoc/bin
           protoc --version
 
       - name: Setup Python
@@ -115,5 +116,6 @@ jobs:
           git submodule update --init
           source venv/bin/activate
           export PATH=$PATH:$HOME/d/protoc/bin
+          export PROTOC=$HOME/d/protoc/bin
           maturin develop
           RUST_BACKTRACE=1 pytest -v .

From 502de987863467219e6488f1339d46fb8654c9d1 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 19:54:12 -0500
Subject: [PATCH 12/20] change permissions of directory containing protobuf
 compiler since permissions error was encountered

---
 .github/workflows/test.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 4f42db557..7dd2fd8dd 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -63,6 +63,7 @@ jobs:
           unzip $PROTO_ZIP
           export PATH=$PATH:$HOME/d/protoc/bin
           export PROTOC=$HOME/d/protoc/bin
+          sudo chown -R $(whoami) $HOME/d/protoc
           protoc --version
 
       - name: Setup Python

From 29c612eebda989f5eb9e607f7d48ef65e8a4479e Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 20:02:17 -0500
Subject: [PATCH 13/20] Try permissions again

---
 .github/workflows/test.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 7dd2fd8dd..a94aec815 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -118,5 +118,7 @@ jobs:
           source venv/bin/activate
           export PATH=$PATH:$HOME/d/protoc/bin
           export PROTOC=$HOME/d/protoc/bin
-          maturin develop
+          sudo chown -R $(whoami) $HOME/d/protoc
+          sudo cargo clean
+          sudo maturin develop
           RUST_BACKTRACE=1 pytest -v .

From 10ea03a7a5d1602c00d908217f68410ea6900f6c Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 20:05:22 -0500
Subject: [PATCH 14/20] Try permissions again

---
 .github/workflows/test.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index a94aec815..ef7ba262e 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -119,6 +119,6 @@ jobs:
           export PATH=$PATH:$HOME/d/protoc/bin
           export PROTOC=$HOME/d/protoc/bin
           sudo chown -R $(whoami) $HOME/d/protoc
-          sudo cargo clean
-          sudo maturin develop
+          cargo clean
+          maturin develop
           RUST_BACKTRACE=1 pytest -v .

From ed3461186322fdf9cf4f36abadd22baf950c693e Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 20:27:02 -0500
Subject: [PATCH 15/20] Try permissions again

---
 .github/workflows/test.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index ef7ba262e..5198ec06e 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -119,6 +119,8 @@ jobs:
           export PATH=$PATH:$HOME/d/protoc/bin
           export PROTOC=$HOME/d/protoc/bin
           sudo chown -R $(whoami) $HOME/d/protoc
+          ls -l $HOME/d/protoc/
+          ls -l $HOME/d/protoc/bin
           cargo clean
           maturin develop
           RUST_BACKTRACE=1 pytest -v .

From 64517a8edc45a17e0649238dabdf4c0160864541 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 20:40:57 -0500
Subject: [PATCH 16/20] Try permissions again

---
 .github/workflows/test.yaml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 5198ec06e..dc44d6e2c 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -112,15 +112,24 @@ jobs:
           flake8 --exclude venv --ignore=E501,W503
           black --line-length 79 --diff --check .
 
+      -name: Build wheels
+       uses: PyO3/maturin-action@v1
+       with:
+        rust-toolchain: stable
+        target: x86_64
+        manylinux: auto
+        args: --release --out dist
+
       - name: Run tests
         run: |
           git submodule update --init
-          source venv/bin/activate
           export PATH=$PATH:$HOME/d/protoc/bin
           export PROTOC=$HOME/d/protoc/bin
           sudo chown -R $(whoami) $HOME/d/protoc
           ls -l $HOME/d/protoc/
           ls -l $HOME/d/protoc/bin
+          pip install datafusion-python --no-index --find-links dist --force-reinstall
+          pip install pytest
           cargo clean
           maturin develop
           RUST_BACKTRACE=1 pytest -v .

From 8fdcfeb9fc8037f79ee9ee87daf64a99de2d0c21 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 20:57:02 -0500
Subject: [PATCH 17/20] Try permissions again

---
 .github/workflows/build.yml | 41 +++++++++++++++++++++++--------------
 1 file changed, 26 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index cf6f61d01..60f1d9ef0 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -103,13 +103,19 @@ jobs:
           path: .
       - run: cat LICENSE.txt
       - name: Build wheels
-        run: |
-          export RUSTFLAGS='-C target-cpu=skylake'
-          export PATH=$PATH:$HOME/d/protoc/bin
-          docker run --rm -v $(pwd):/io \
-            --workdir /io \
-            ghcr.io/pyo3/maturin:v0.14.2 \
-            build --release --manylinux 2014
+        uses: PyO3/maturin-action@v1
+        with:
+          rust-toolchain: nightly
+          target: ${{ matrix.target }}
+          manylinux: auto
+          args: --release --manylinux 2014
+        # run: |
+        #   export RUSTFLAGS='-C target-cpu=skylake'
+        #   export PATH=$PATH:$HOME/d/protoc/bin
+        #   docker run --rm -v $(pwd):/io \
+        #     --workdir /io \
+        #     ghcr.io/pyo3/maturin:v0.14.2 \
+        #     build --release --manylinux 2014
       - name: Archive wheels
         uses: actions/upload-artifact@v3
         with:
@@ -130,14 +136,19 @@ jobs:
           path: .
       - run: cat LICENSE.txt
       - name: Build sdist
-        run: |
-          export PATH=$PATH:$HOME/d/protoc/bin
-          export PROTOC=$HOME/d/protoc/bin
-          docker run --rm -v $(pwd):/io \
-            -e PROTOC='$(HOME)/d/protoc/bin' --workdir /io \
-            ghcr.io/pyo3/maturin:v0.14.2 \
-            build --release --sdist
-          find target/wheels/
+        uses: PyO3/maturin-action@v1
+        with:
+          rust-toolchain: stable
+          manylinux: auto
+          args: --release --sdist --out dist
+        # run: |
+        #   export PATH=$PATH:$HOME/d/protoc/bin
+        #   export PROTOC=$HOME/d/protoc/bin
+        #   docker run --rm -v $(pwd):/io \
+        #     -e PROTOC='$(HOME)/d/protoc/bin' --workdir /io \
+        #     ghcr.io/pyo3/maturin:v0.14.2 \
+        #     build --release --sdist
+        #   find target/wheels/
       - name: Archive wheels
         uses: actions/upload-artifact@v2
         with:

From 63ea60a3ebb4305a73c3c5eedb1888f05fa0bd9a Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 21:00:47 -0500
Subject: [PATCH 18/20] Alter permissions

---
 .github/workflows/build.yml | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 60f1d9ef0..0c56e4a95 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -60,11 +60,6 @@ jobs:
         with:
           toolchain: stable
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install maturin==0.14.2
-
       - run: rm LICENSE.txt
       - name: Download LICENSE.txt
         uses: actions/download-artifact@v3
@@ -106,16 +101,9 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           rust-toolchain: nightly
-          target: ${{ matrix.target }}
+          target: x86_64
           manylinux: auto
           args: --release --manylinux 2014
-        # run: |
-        #   export RUSTFLAGS='-C target-cpu=skylake'
-        #   export PATH=$PATH:$HOME/d/protoc/bin
-        #   docker run --rm -v $(pwd):/io \
-        #     --workdir /io \
-        #     ghcr.io/pyo3/maturin:v0.14.2 \
-        #     build --release --manylinux 2014
       - name: Archive wheels
         uses: actions/upload-artifact@v3
         with:
@@ -141,14 +129,6 @@ jobs:
           rust-toolchain: stable
           manylinux: auto
           args: --release --sdist --out dist
-        # run: |
-        #   export PATH=$PATH:$HOME/d/protoc/bin
-        #   export PROTOC=$HOME/d/protoc/bin
-        #   docker run --rm -v $(pwd):/io \
-        #     -e PROTOC='$(HOME)/d/protoc/bin' --workdir /io \
-        #     ghcr.io/pyo3/maturin:v0.14.2 \
-        #     build --release --sdist
-        #   find target/wheels/
       - name: Archive wheels
         uses: actions/upload-artifact@v2
         with:

From 1080251b6eadc877b87146a0f97f706c22a4932b Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 21:14:27 -0500
Subject: [PATCH 19/20] Alter permissions

---
 .github/workflows/test.yaml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index dc44d6e2c..e6735e332 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -112,13 +112,13 @@ jobs:
           flake8 --exclude venv --ignore=E501,W503
           black --line-length 79 --diff --check .
 
-      -name: Build wheels
-       uses: PyO3/maturin-action@v1
-       with:
-        rust-toolchain: stable
-        target: x86_64
-        manylinux: auto
-        args: --release --out dist
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          rust-toolchain: stable
+          target: x86_64
+          manylinux: auto
+          args: --release --out dist
 
       - name: Run tests
         run: |

From b99df2c1941df5f78d5d9c5e3be2b970e9c60a57 Mon Sep 17 00:00:00 2001
From: Jeremy Dyer <jdye64@gmail.com>
Date: Thu, 19 Jan 2023 21:21:58 -0500
Subject: [PATCH 20/20] Alter permissions

---
 .github/workflows/test.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index e6735e332..ffa983257 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -115,9 +115,7 @@ jobs:
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
-          rust-toolchain: stable
-          target: x86_64
-          manylinux: auto
+          command: build
           args: --release --out dist
 
       - name: Run tests