general-kroll-4-life · Mar 8, 2022 · Feb 8, 2022
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -36,9 +36,22 @@ jobs:
       uses: egor-tensin/setup-mingw@v2
       id: gccsetup
 
+    - name: Get rid of disruptive line endings before checkout
+      run: |
+        git config --global core.autocrlf false
+
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
 
+    - name: Install Python dependencies
+      run: |
+        pip3 install PyYaml
+        pip3 install robotframework
+
+    - name: Generate rewritten registry for simulations
+      run: |
+        python3 test\python\registry-rewrite.py
+
     - name: Get dependencies
       run: |
         go env -w GOPRIVATE="github.com/stackql/*"
@@ -97,6 +110,17 @@ jobs:
     - name: Test
       if: success()
       run: go test -timeout 1200s --tags "json1" -v ./...
+
+    - name: Mock Server Download
+      run: |
+        mvn org.apache.maven.plugins:maven-dependency-plugin:3.0.2:copy `
+        '-Dartifact=org.mock-server:mockserver-netty:5.12.0:jar:shaded' `
+        '-DoutputDirectory=test/downloads'
+
+    - name: Run robot tests
+      if: success()
+      run: |
+        cmd /c "robot test\functional"
 
     - name: Prepare Test DB
       if: success()
@@ -132,6 +156,15 @@ jobs:
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
+
+    - name: Install Python dependencies
+      run: |
+        pip3 install PyYaml
+        pip3 install robotframework
+
+    - name: Generate rewritten registry for simulations
+      run: |
+        python3 test/python/registry-rewrite.py
 
     - name: Get dependencies
       run: |
@@ -180,6 +213,18 @@ jobs:
     - name: Test
       if: success()
       run: go test -timeout 1200s --tags "json1" -v ./...
+
+    - name: Mock Server Download
+      run: |
+        mvn \
+        org.apache.maven.plugins:maven-dependency-plugin:3.0.2:copy \
+        -Dartifact=org.mock-server:mockserver-netty:5.12.0:jar:shaded \
+        -DoutputDirectory=test/downloads
+
+    - name: Run robot tests
+      if: success()
+      run: |
+        robot test/functional
 
     - name: Prepare Test DB
       if: success()
@@ -211,6 +256,15 @@ jobs:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
 
+    - name: Install Python dependencies
+      run: |
+        pip3 install PyYaml
+        pip3 install robotframework
+
+    - name: Generate rewritten registry for simulations
+      run: |
+        python3 test/python/registry-rewrite.py
+
     - name: Get dependencies
       run: |
         git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/
@@ -260,6 +314,18 @@ jobs:
     - name: Test
       if: success()
       run: go test -timeout 1200s --tags "json1" -v ./...
+
+    - name: Mock Server Download
+      run: |
+        mvn \
+        org.apache.maven.plugins:maven-dependency-plugin:3.0.2:copy \
+        -Dartifact=org.mock-server:mockserver-netty:5.12.0:jar:shaded \
+        -DoutputDirectory=test/downloads
+
+    - name: Run robot tests
+      if: success()
+      run: |
+        robot test/functional
 
     - name: Prepare Test DB
       if: success()

diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml
@@ -10,6 +10,12 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
+
+
+      - name: Get rid of dissruptive line endings before checkout
+        run: |
+          git config --global core.autocrlf false
+
       -
         name: Checkout
         uses: actions/checkout@v2

diff --git a/.vscode/launch.json b/.vscode/launch.json
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -33,10 +33,42 @@ set(INT_TEST_DB_DEST_FILE "test/db/tmp/python-tests-tmp-db.sqlite")
 
 enable_testing()
 
+find_program(Python_EXECUTABLE python3)
+find_program(Java_EXECUTABLE java)
+find_program(Robot_EXECUTABLE robot)
+find_program(Maven_EXECUTABLE mvn)
+
 set(TEST_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/test/python/main.py)
+set(REG_RW_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/test/python/registry-rewrite.py)
+set(ROBOT_TEST_DIR_PATH ${CMAKE_CURRENT_SOURCE_DIR}/test/functional)
 
 add_custom_target(${TARGET} ALL)
 
+
+if(NOT Maven_EXECUTABLE)
+    message(FATAL_ERROR "Cannot find the mvn executable!")
+endif()
+
+if(NOT Python_EXECUTABLE)
+    message(FATAL_ERROR "Cannot find the python3 executable!")
+endif()
+
+add_custom_command(TARGET ${TARGET}
+  PRE_BUILD
+  DEPENDS ${SRCS}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND mvn org.apache.maven.plugins:maven-dependency-plugin:3.2.0:copy -Dartifact=org.mock-server:mockserver-netty:5.12.0:jar:shaded -DoutputDirectory=${CMAKE_CURRENT_SOURCE_DIR}/test/downloads -DdestFileName=mockserver-netty.jar -DoverWrite=true
+  COMMENT "maven copy mock server artifact"
+)
+
+add_custom_command(TARGET ${TARGET}
+  PRE_BUILD
+  DEPENDS ${SRCS}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND ${Python_EXECUTABLE} ${REG_RW_SCRIPT}
+  COMMENT "set up dummy regsitry for server simulations"
+)
+
 add_custom_command(TARGET ${TARGET}
   PRE_BUILD
   DEPENDS ${SRCS}
@@ -125,8 +157,6 @@ add_custom_command(TARGET ${TARGET}
   COMMENT "Building Go executable for ${OS_STR_LINUX}"
 )
 
-find_program(Python_EXECUTABLE python3)
-
 
 if(DEFINED ENV{STACKQL_TEST_LOG_LEVEL})
   set(STACKQL_TEST_LOG_LEVEL $ENV{STACKQL_TEST_LOG_LEVEL})
@@ -141,13 +171,20 @@ else()
   message(STATUS "LIVE_INTEGRATION_TESTS cmake variable is empty, only 'always' tests will be run")
 endif()
 
-if(NOT Python_EXECUTABLE)
-    message(FATAL_ERROR "Cannot find the python3 executable!")
-endif()
 add_test(NAME int-tests
     COMMAND ${Python_EXECUTABLE} ${TEST_SCRIPT} --additionalintegrationtestdir=${LIVE_INTEGRATION_TESTS} --loglevel=${STACKQL_TEST_LOG_LEVEL}
 )
 
+if(NOT Java_EXECUTABLE)
+    message(FATAL_ERROR "Cannot find the java executable!")
+endif()
+if(NOT Robot_EXECUTABLE)
+    message(FATAL_ERROR "Cannot find the robot executable!")
+endif()
+add_test(NAME robot-tests
+    COMMAND ${Robot_EXECUTABLE} ${ROBOT_TEST_DIR_PATH}
+)
+
 add_custom_command(TARGET ${TARGET}
   POST_BUILD
   COMMAND ctest --verbose)

diff --git a/docs/developer_guide.md b/docs/developer_guide.md
@@ -70,6 +70,8 @@ Building locally or in cloud will automatically:
 2. Build the executable.
 3. Run integration tests.
 
+[This article](https://medium.com/cbi-engineering/mocking-techniques-for-go-805c10f1676b) gives a nice overview of mocking in golang.
+
 ### gotest
 
 Test coverage is sparse.  Regressions are mitigated by `gotest` integration testing in the [driver](/internal/stackql/driver/driver_integration_test.go) and [stackql](/stackql/main_integration_test.go) packages.  Some testing functionality is supported through convenience functionality inside the [test](/internal/test) packages.
@@ -133,7 +135,7 @@ Then, on OSX > 10, you will need to whitelist the executable for execution even
 
 Then, run test commands, such as:
 ```
-~/Downloads/stackql --keyfilepath=$HOME/stackql/stackql-devel/keys/sa-key.json exec "select group_concat(substr(name, 0, 5)) || ' lalala' as cc from google.compute.disks where project = 'lab-kr-network-01' and zone = 'australia-southeast1-b';" -o text
+~/Downloads/stackql --credentialsfilepath=$HOME/stackql/stackql-devel/keys/sa-key.json exec "select group_concat(substr(name, 0, 5)) || ' lalala' as cc from google.compute.disks where project = 'lab-kr-network-01' and zone = 'australia-southeast1-b';" -o text
 ```
 
 ## Notes on vitess
@@ -180,5 +182,5 @@ is the guts of query processing.
 
 
 ```
-time ./stackql exec --cpuprofile=./select-disks-improved-05.profile --auth='{ "google": { "keyfilepath": "'${HOME}'/stackql/stackql-devel/keys/sa-key.json" }, "okta": { "keyfilepath": "'${HOME}'/stackql/stackql-devel/keys/okta-token.txt", "keyfiletype": "api_key" } } ' "select name from google.compute.disks where project = 'lab-kr-network-01' and zone = 'australia-southeast1-a';"
+time ./stackql exec --cpuprofile=./select-disks-improved-05.profile --auth='{ "google": { "credentialsfilepath": "'${HOME}'/stackql/stackql-devel/keys/sa-key.json" }, "okta": { "credentialsfilepath": "'${HOME}'/stackql/stackql-devel/keys/okta-token.txt", "type": "api_key" } } ' "select name from google.compute.disks where project = 'lab-kr-network-01' and zone = 'australia-southeast1-a';"
 ```
diff --git a/docs/examples.md b/docs/examples.md
@@ -4,20 +4,19 @@
 ## Assumptions
 
 - `stackql` is in your `${PATH}`.
-- Authentication particulars are supplied as a json string in the arg `--auth`.  Per provider, you supply a key/val pair.  The val iteslf is a json string, optionally specifying `keyfiletype` (defaulted to `serviceaccount`, which represents a google service account key). The val minimally contains either:
-    - An appropriate key file at the file location `{ "keyfilepath": "/PATH/TO/KEY/FILE" }`.  For example, with the google provider, one might use a service account json key.
-    - An appropriate key plaintext stored in an (exported) environment variable.  Eg: `{ "keyenvvar": "OKTA_SECRET_KEY" }`.  For example, with the google provider, one might use a service account json key.
+- Authentication particulars are supplied as a json string in the arg `--auth`.  Per provider, you supply a key/val pair.  The val iteslf is a json string, optionally specifying `type` (defaulted to `service_account`, which represents a google service account key). The val minimally contains either:
+    - An appropriate key file at the file location `{ "credentialsfilepath": "/PATH/TO/KEY/FILE" }`.  For example, with the google provider, one might use a service account json key.
+    - An appropriate key plaintext stored in an (exported) environment variable.  Eg: `{ "credentialsenvvar": "OKTA_SECRET_KEY" }`.  For example, with the google provider, one might use a service account json key.
 
 If using `service account` auth against the `google` provider, then no ancillary information is required.  If howevere, you are using another key type / provider, then more runtime information is required, eg:
 
 Google:
 
 ```sh
 
-export 
-OKTA_SECRET_KEY="$(cat ${HOME}/stackql/stackql-devel/keys/okta-token.txt)"
+export OKTA_SECRET_KEY="$(cat ${HOME}/stackql/stackql-devel/keys/okta-token.txt)"
 
-AUTH_STR='{ "google": { "keyfilepath": "'${HOME}'/stackql/stackql-devel/keys/sa-key.json" }, "okta": { "keyenvvar": "OKTA_SECRET_KEY", "keyfiletype": "api_key" } }'
+export AUTH_STR='{ "google": { "credentialsfilepath": "'${HOME}'/stackql/stackql-devel/keys/sa-key.json", "type": "service_account" }, "okta": { "credentialsenvvar": "OKTA_SECRET_KEY", "type": "api_key" } }'
 
 ./stackql shell --auth="${AUTH_STR}"
 
@@ -45,8 +44,8 @@ stackql \
 ### SHOW SERVICES
 
 ```
-stackql --providerroot=../test/.stackql \
-  --configfile=../test/.iqlrc exec \
+stackql --approot=../test/.stackql \
+  --configfile=../test/.stackqlrc exec \
   "SHOW SERVICES from google ;" ; echo
 
 ```
@@ -63,4 +62,10 @@ insert into google.compute.disks(project, zone, data__name) SELECT 'lab-kr-netwo
 
 ```
 insert into okta.application.apps(subdomain, data__name, data__label, data__signOnMode, data__settings) SELECT 'dev-79923018-admin', 'template_basic_auth', 'some other4 new app', 'BASIC_AUTH', '{ "app": { "authURL": "https://example.com/auth.html", "url": "https://example.com/bookmark.html" } }';
+```
+
+### aliased table select
+
+```
+select * from okta.application.apps;
 ```
diff --git a/examples/README.md b/examples/README.md
@@ -0,0 +1,23 @@
+
+
+## BYO Registry
+
+
+```bash
+
+PROVIDER_REGISTRY_ROOT_DIR="$(pwd)/../examples/registry"
+
+REG_STR='{ "url": "file://'${PROVIDER_REGISTRY_ROOT_DIR}'", "localDocRoot": "'${PROVIDER_REGISTRY_ROOT_DIR}'",  "useEmbedded": false, "verifyConfig": {"nopVerify": true } }'
+
+
+## All auth required ahead of time at this stage, even for no auth providers
+AUTH_STR='{ "publicapis": { "type": "null_auth" }  }'
+
+./stackql --auth="${AUTH_STR}" --registry="${REG_STR}" exec "select API from publicapis.api.apis where API like 'Dog%' limit 10;"
+
+./stackql --auth="${AUTH_STR}" --registry="${REG_STR}" exec "select API from publicapis.api.random where title =  'Dog';"
+
+## The single, anonmyous column returned from selecing an array of strings is a core issue to fix separate to this
+./stackql --auth="${AUTH_STR}" --registry="${REG_STR}" exec "select * from publicapis.api.categories limit 5;"
+
+```
diff --git a/examples/empty-registry/.gitignore b/examples/empty-registry/.gitignore
@@ -0,0 +1,3 @@
+*
+!.gitignore
+!src
diff --git a/examples/empty-registry/src/.gitignore b/examples/empty-registry/src/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/examples/registry/src/publicapis/v1/provider.yaml b/examples/registry/src/publicapis/v1/provider.yaml
@@ -0,0 +1,16 @@
+id: publicapis
+name: publicapis
+version: v1
+description: Catalogue of public http APIs.
+title: Public APIs
+providerServices:
+  api:
+    description: Public API
+      Catalogue of known public APIs.
+    id: api:v1
+    name: api
+    preferred: true
+    service:
+      $ref: publicapis/v1/services/api-v1.yaml
+    title: API catalogue
+    version: v1