general-kroll-4-life · Mar 21, 2022 · Mar 14, 2022 · Mar 21, 2022
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -36,6 +36,11 @@ jobs:
       uses: egor-tensin/setup-mingw@v2
       id: gccsetup
 
+    - name: Choco install postgres
+      uses: crazy-max/ghaction-chocolatey@v1
+      with:
+        args: "install --force postgresql13 openssl"
+
     - name: Get rid of disruptive line endings before checkout
       run: |
         git config --global core.autocrlf false
@@ -117,8 +122,16 @@ jobs:
         '-Dartifact=org.mock-server:mockserver-netty:5.12.0:jar:shaded' `
         '-DoutputDirectory=test/downloads'
 
+    - name: Create certificates for robot tests
+      run: |
+        openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
+        openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
+        openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 
+
     - name: Run robot tests
       if: success()
+      env:
+        PSQL_EXE: C:\Program Files\PostgreSQL\13\bin\psql
       run: |
         cmd /c "robot test\functional"
 
@@ -153,6 +166,11 @@ jobs:
       with:
         platform: x64
         cygwin: 0
+
+    - name: Install psql
+      run: |
+          sudo apt-get update
+          sudo apt-get install --yes --no-install-recommends postgresql-client
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
@@ -221,6 +239,12 @@ jobs:
         -Dartifact=org.mock-server:mockserver-netty:5.12.0:jar:shaded \
         -DoutputDirectory=test/downloads
 
+    - name: Create certificates for robot tests
+      run: |
+        openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
+        openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
+        openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 
+
     - name: Run robot tests
       if: success()
       run: |
@@ -322,6 +346,12 @@ jobs:
         -Dartifact=org.mock-server:mockserver-netty:5.12.0:jar:shaded \
         -DoutputDirectory=test/downloads
 
+    - name: Create certificates for robot tests
+      run: |
+        openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
+        openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
+        openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 
+
     - name: Run robot tests
       if: success()
       run: |

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -115,6 +115,18 @@
         "json"
       ],
     },
+    {
+      "name": "DEBUG run server;",
+      "type": "go",
+      "request": "launch",
+      "mode": "debug",
+      "program": "${workspaceFolder}/stackql",
+      "args": [
+        "srv",
+        "--auth={ \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/keys/sa-key.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/keys/okta-token.txt\", \"type\": \"api_key\" } }",
+        "{ \"url\": \"file://${workspaceFolder}/examples/registry\", \"localDocRoot\": \"${workspaceFolder}/examples/registry\", \"verifyConfig\": {\"nopVerify\": true } }"
+      ],
+    },
     {
       "name": "DEBUG delete from google.storage.buckets ... ;",
       "type": "go",

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -69,6 +69,33 @@ add_custom_command(TARGET ${TARGET}
   COMMENT "set up dummy regsitry for server simulations"
 )
 
+add_custom_command(TARGET ${TARGET}
+  PRE_BUILD
+  DEPENDS ${SRCS}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND
+    openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
+  COMMENT "set up server TLS credentials for robot tests"
+)
+
+add_custom_command(TARGET ${TARGET}
+  PRE_BUILD
+  DEPENDS ${SRCS}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND
+    openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
+  COMMENT "set up client TLS credentials for robot tests"
+)
+
+add_custom_command(TARGET ${TARGET}
+  PRE_BUILD
+  DEPENDS ${SRCS}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND
+    openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 
+  COMMENT "set up rubbish TLS credentials for robot tests"
+)
+
 add_custom_command(TARGET ${TARGET}
   PRE_BUILD
   DEPENDS ${SRCS}

diff --git a/README.md b/README.md
@@ -38,24 +38,35 @@ Please see [registry_contribution.md](/docs/registry_contribution.md).
 
 ## Providers
 
+Please see [the stackql-provider-registry repository](https://github.com/stackql/stackql-provider-registry)
+
+Providers include:
+
 - Google.
 - Okta.
+- ...
 
 ---
 
 ## Build
 
-With cmake:
+Presuming you have all of [the system requirements](#system-requirements-for-local-devlopment-build-and-test), then build/test with cmake:
 
 ```bash
 cd build
 cmake ..
 cmake --build .
 ```
 
-
 Executable `build/stackql` will be created.
 
+### System requirements for local development, build and test
+
+- cmake>=3.22.3
+- golang>=1.16
+- openssl>=1.1.1
+- python>=3.5
+
 
 ## Run
 
@@ -84,6 +95,10 @@ More examples in [docs/examples.md](/docs/examples.md).
 
 [docs/integration_testing.md](/docs/integration_testing.md).
 
+## Server mode
+
+Please see [the server mode section of the developer docs](/docs/developer_guide.md#server-mode).
+
 ## Acknowledgements
 
 Forks of the following support our work:
@@ -94,6 +109,8 @@ Forks of the following support our work:
 
 We gratefully acknowledge these pieces of work.
 
-## License
+## Licensing
+
+Please see the [stackql LICENSE](/LICENSE).
 
-See [/LICENSE](/LICENSE)
+Licenses for third party software we are using are included in the [/licenses](/licenses) directory.
diff --git a/docs/developer_guide.md b/docs/developer_guide.md
@@ -21,6 +21,16 @@ If you want further auth types or discover bugs, please raise an issue.
 
 Examples are present [here](/docs/examples.md).
 
+## Server mode
+
+**Note that this feature is in alpha**.  We will update timelines for General Availability after a period of analysis and testing.  At the time of writing, server mode is most useful for R&D purposes: 
+- experimentation.
+- tooling / system integrations and design thereof.
+- development of `stackql` itself.
+- development of use cases for the product.
+
+The `stackql` server leverages the `postgres` wire protocol and can be used with the `psql` client, including mTLS auth / encryption in transit.  Please see [the relevant examples](/docs/examples.md#running-in-server-mode) for further details.
+
 ## Concurrency considerations
 
 In server mode, a thread pool issues one thread to handle each connection.
@@ -205,4 +215,12 @@ is the guts of query processing.
 
 ```
 time ./stackql exec --cpuprofile=./select-disks-improved-05.profile --auth='{ "google": { "credentialsfilepath": "'${HOME}'/stackql/stackql-devel/keys/sa-key.json" }, "okta": { "credentialsfilepath": "'${HOME}'/stackql/stackql-devel/keys/okta-token.txt", "type": "api_key" } } ' "select name from google.compute.disks where project = 'lab-kr-network-01' and zone = 'australia-southeast1-a';"
-```
+```
+
+## Postgres Server Implementation
+
+1. Heavy duty option as per cockroachdb:
+    - https://github.com/cockroachdb/cockroach/tree/e6a0d23d516203bf5e8d1c8b3c3c26ddfaddc388/pkg/sql/pgwire
+2. Light touch option can be based upon:
+    - https://github.com/jeroenrinzema/psql-wire
+
diff --git a/docs/examples.md b/docs/examples.md
@@ -10,6 +10,10 @@
 
 If using `service account` auth against the `google` provider, then no ancillary information is required.  If howevere, you are using another key type / provider, then more runtime information is required, eg:
 
+## Running stackql
+
+Simplest example is using the interactive shell.
+
 Google:
 
 ```sh
@@ -23,6 +27,41 @@ export AUTH_STR='{ "google": { "credentialsfilepath": "'${HOME}'/stackql/stackql
 
 ```
 
+## Running in server mode
+
+**Note that this feature is in alpha**, as discussed in [the developer guide](/docs/developer_guide.md#server-mode).
+
+
+To run a `stackql` server over the `postgres` wire protocol (without client authentication), from the `build` directory.
+
+```bash
+./stackql --auth="${AUTH_STR}" --registry="${REG_STR}" srv
+```
+
+And then, using the `psql` client:
+
+```bash
+psql -d "host=127.0.0.1 port=5466 user=silly dbname=silly"
+```
+
+To run using mTLS auth, first prepare collateral as per [the mTLS setup README](/test/server/mtls/README.md).  Important to define the env var `CLIENT_CERT` in the shell session you will use to run the server.
+
+Then:
+
+```bash
+STACKQL_SRV_TLS_CFG='{ "keyFilePath": "../test/server/mtls/credentials/pg_server_key.pem", "certFilePath": "../test/server/mtls/credentials/pg_server_cert.pem", "clientCAs": [ "'${CLIENT_CERT}'" ] }'
+
+./stackql --auth="${AUTH_STR}" --registry="${REG_STR}"  --pg.tls="${STACKQL_SRV_TLS_CFG}" 
+```
+
+And then, using the `psql` client (from same directory; `build`):
+
+```bash
+psql -d "host=127.0.0.1 port=5466 user=silly dbname=silly sslmode=verify-full sslcert=../test/server/mtls/credentials/pg_client_cert.pem sslkey=../test/server/mtls/credentials/pg_client_key.pem sslrootcert=../test/server/mtls/credentials/pg_server_cert.pem"
+```
+
+## Queries
+
 ### SELECT
 
 ```

diff --git a/go.mod b/go.mod
@@ -3,28 +3,59 @@ module github.com/stackql/stackql
 go 1.16
 
 require (
+	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect
+	github.com/axiomhq/hyperloglog v0.0.0-20220105174342-98591331716a // indirect
+	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
+	github.com/cenk/backoff v2.2.1+incompatible // indirect
+	github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
+	github.com/cockroachdb/circuitbreaker v2.2.1+incompatible // indirect
+	github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 // indirect
+	github.com/cockroachdb/cockroach v20.1.17+incompatible
+	github.com/cockroachdb/cockroach-go v2.0.1+incompatible // indirect
+	github.com/cockroachdb/errors v1.9.0 // indirect
+	github.com/cockroachdb/ttycolor v0.0.0-20210902133924-c7d7dcdde4e8 // indirect
+	github.com/edsrzf/mmap-go v1.1.0 // indirect
+	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
 	github.com/fatih/color v1.13.0
 	github.com/getkin/kin-openapi v0.88.0
+	github.com/getsentry/raven-go v0.2.0 // indirect
+	github.com/google/flatbuffers v2.0.6+incompatible // indirect
 	github.com/google/go-jsonnet v0.17.0
-	github.com/magiconair/properties v1.8.5
+	github.com/jackc/pgtype v1.10.0
+	github.com/jackc/pgx v3.6.2+incompatible // indirect
+	github.com/jaegertracing/jaeger v1.32.0 // indirect
+	github.com/jeroenrinzema/psql-wire v0.0.1-stackqlalpha3
+	github.com/knz/strtime v0.0.0-20200924090105-187c67f2bf5e // indirect
+	github.com/lib/pq v1.10.4
+	github.com/lightstep/lightstep-tracer-go v0.25.0 // indirect
+	github.com/magiconair/properties v1.8.6
+	github.com/marusama/semaphore v0.0.0-20190110074507-6952cef993b2 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/montanaflynn/stats v0.6.6 // indirect
 	github.com/olekukonko/tablewriter v0.0.0-20180130162743-b8a9be070da4
+	github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5 // indirect
+	github.com/petermattis/goid v0.0.0-20220302125637-5f11c28912df // indirect
 	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/cobra v1.3.0
+	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.10.1
-	github.com/stackql/go-openapistackql v0.0.5-alpha05
-	github.com/stackql/go-sqlite3 v0.0.1-stackqlalpha
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
+	github.com/stackql/go-openapistackql v0.0.5-rc02
+	github.com/stackql/go-sqlite3 v0.0.1-stackqlrc01
+	go.uber.org/zap v1.21.0
+	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
-	gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485
+	gonum.org/v1/gonum v0.9.3
 	gopkg.in/yaml.v2 v2.4.0
+	gotest.tools v2.2.0+incompatible
 	readline v0.0.0-00010101000000-000000000000
-	vitess.io/vitess v0.0.10-alpha5
+	vitess.io/vitess v0.0.10-rc01
 )
 
-replace readline => github.com/stackql/readline v0.0.0-20210418072316-6e4ad520d2b4
+replace readline => github.com/stackql/readline v0.0.1-rc01
 
-replace github.com/fatih/color => github.com/stackql/color v1.10.1-0.20210418074258-4aa529ee76ed
+replace github.com/fatih/color => github.com/stackql/color v0.0.1-rc01
 
-replace vitess.io/vitess => github.com/stackql/vitess v0.0.10-alpha5
+replace vitess.io/vitess => github.com/stackql/vitess v0.0.10-rc01
+
+replace github.com/jeroenrinzema/psql-wire => github.com/stackql/psql-wire v0.0.1-stackqlrc01