diff --git a/.github/goreleaser/.goreleaser-for-darwin.yaml b/.github/goreleaser/.goreleaser-for-darwin.yaml index dc4d9698..b5b3da29 100644 --- a/.github/goreleaser/.goreleaser-for-darwin.yaml +++ b/.github/goreleaser/.goreleaser-for-darwin.yaml @@ -17,7 +17,7 @@ builds: - amd64 - arm64 env: - - CGO_ENABLED=1 + - CGO_ENABLED=0 flags: - -trimpath ldflags: >- diff --git a/.github/goreleaser/.goreleaser-for-linux.yaml b/.github/goreleaser/.goreleaser-for-linux.yaml index df6d76e1..fcc7385b 100644 --- a/.github/goreleaser/.goreleaser-for-linux.yaml +++ b/.github/goreleaser/.goreleaser-for-linux.yaml @@ -17,13 +17,7 @@ builds: - amd64 - arm64 env: - - CGO_ENABLED=1 - - >- - {{- if eq .Arch "amd64"}}CC=x86_64-linux-gnu-gcc{{- end }} - {{- if eq .Arch "arm64"}}CC=aarch64-linux-gnu-gcc{{- end }} - - >- - {{- if eq .Arch "amd64"}}CXX=x86_64-linux-gnu-g++{{- end }} - {{- if eq .Arch "arm64"}}CXX=aarch64-linux-gnu-gcc{{- end }} + - CGO_ENABLED=0 flags: - -trimpath ldflags: >- diff --git a/.github/goreleaser/.goreleaser-for-windows.yaml b/.github/goreleaser/.goreleaser-for-windows.yaml index a17db0b1..5662f800 100644 --- a/.github/goreleaser/.goreleaser-for-windows.yaml +++ b/.github/goreleaser/.goreleaser-for-windows.yaml @@ -16,9 +16,7 @@ builds: goarch: - amd64 env: - - CGO_ENABLED=1 - - CC=x86_64-w64-mingw32-gcc - - CXX=x86_64-w64-mingw32-g++ + - CGO_ENABLED=0 flags: - -trimpath ldflags: >- diff --git a/.github/workflows/build-expt.yaml b/.github/workflows/build-expt.yaml index fa8a3221..199f2114 100644 --- a/.github/workflows/build-expt.yaml +++ b/.github/workflows/build-expt.yaml @@ -20,8 +20,6 @@ jobs: uses: actions/setup-go@v4 with: go-version: ^1.19 - - name: Install cross-compiler for linux/arm64 - run: sudo apt-get install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - name: Run Goreleaser uses: goreleaser/goreleaser-action@v5 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cdfcdf04..f4fd9472 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,7 @@ on: - '.github/workflows/mcp-registry-publish.yml' - '.github/workflows/contributors.yml' env: - GOTESTCMD: "go test -timeout 1200s --tags \"sqlite_stackql\" -v ./..." + GOTESTCMD: "go test -timeout 1200s -v ./..." TESTSCRIPT: "test/deprecated/python/main.py" GOPRIVATE: github.com/stackql/* GH_ACCESS_TOKEN: ${{ secrets.ACTIONS_PRIVATE_PACKAGE_SECRET }} @@ -146,35 +146,6 @@ jobs: run: | dir "C:\Users\${{ env.username }}\AppData\Local\Temp\chocolatey\" - - name: Set up mingw - # This action installs mingw via Chocolatey's public feed, which occasionally returns - # transient errors (e.g. 406 Not Acceptable) unrelated to this repo. It is a composite - # action, so it can't be wrapped by generic JS-action retry actions - fall back to a - # native retry step below instead. - id: gccsetup - continue-on-error: true - uses: egor-tensin/setup-mingw@v2.2.0 - timeout-minutes: ${{ vars.DEFAULT_STEP_TIMEOUT_MIN == '' && 20 || vars.DEFAULT_STEP_TIMEOUT_MIN }} - with: - version: '8.1.0' - - - name: Set up mingw (retry 1) - if: steps.gccsetup.outcome == 'failure' - id: gccsetup_retry1 - continue-on-error: true - uses: egor-tensin/setup-mingw@v2.2.0 - timeout-minutes: ${{ vars.DEFAULT_STEP_TIMEOUT_MIN == '' && 20 || vars.DEFAULT_STEP_TIMEOUT_MIN }} - with: - version: '8.1.0' - - - name: Set up mingw (retry 2) - if: steps.gccsetup.outcome == 'failure' && steps.gccsetup_retry1.outcome == 'failure' - id: gccsetup_retry2 - uses: egor-tensin/setup-mingw@v2.2.0 - timeout-minutes: ${{ vars.DEFAULT_STEP_TIMEOUT_MIN == '' && 20 || vars.DEFAULT_STEP_TIMEOUT_MIN }} - with: - version: '8.1.0' - - name: Git Ref Parse id: git_ref_parse run: | @@ -212,7 +183,7 @@ jobs: git --no-pager config --list go get -v -t -d ./... env: - CGO_ENABLED: 1 + CGO_ENABLED: 0 GHACCESSTOKEN: ${{env.GH_ACCESS_TOKEN}} GOPRIVATE: ${{env.GOPRIVATE}} @@ -222,32 +193,12 @@ jobs: BUILDBRANCH: ${{github.ref}} BUILDPLATFORM: ${{runner.os}} BUILDPATCHVERSION: ${{github.run_number}} - CGO_ENABLED: '1' + CGO_ENABLED: '0' GH_ACCESS_TOKEN: ${{env.GH_ACCESS_TOKEN}} GOPRIVATE: ${{env.GOPRIVATE}} GOOS: windows GOARCH: amd64 - GOMINGWID: 'x86_64-w64-mingw32' run: | - # Explicitly prioritize the correct MinGW in the PATH - $mingwPath = "C:\mingw64" - if (-not (Test-Path -Path $mingwPath)) { - # Fallback to the chocolatey path, which we know exists - $mingwPath = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64" - } - - # Use Resolve-Path to get the guaranteed absolute path - # Now that we've ensured $mingwPath isn't empty, Resolve-Path will succeed. - $mingwPath = Resolve-Path $mingwPath - $env:Path = "$mingwPath\bin;" + $env:Path - - # Set CC and CXX with the fully resolved absolute path - $env:CC = "$mingwPath\bin\x86_64-w64-mingw32-gcc" - $env:CXX = "$mingwPath\bin\x86_64-w64-mingw32-g++" - echo "Current PATH:" - echo $env:PATH - - git config --global url.https://$env:GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ $Version = convertfrom-stringdata (get-content ./cicd/version.txt -raw) $BuildMajorVersion = $Version.'MajorVersion' @@ -262,14 +213,6 @@ jobs: echo "BUILDMINORVERSION=$env:BUILDMINORVERSION" >> $GITHUB_ENV echo "BUILDPATCHVERSION=$env:BUILDPATCHVERSION" >> $GITHUB_ENV - # This creates a PowerShell function to intercept the 'go build' command - # and inject the static linker flags. - # The function calls the actual go.exe with the added arguments. - $originalGoPath = (Get-Command go.exe).Path - function go { - & $originalGoPath @args -ldflags "-linkmode external -extldflags '-static -s -w'" - } - python cicd/python/build.py --verbose --build python cicd/python/build.py --verbose --build-mcp-client @@ -369,13 +312,30 @@ jobs: name: stackql_windows_amd64 path: build/stackql.exe + # Consumed by winarmsmoke, so that the MCP robot scenarios run there too. + - name: Upload MCP client artifact + uses: actions/upload-artifact@v7.0.1 + if: success() + with: + name: stackql_mcp_client_windows_amd64 + path: build/stackql_mcp_client.exe + winarmsmoke: name: Windows arm64 Smoke (x64 emulation) needs: winbuild runs-on: windows-11-arm - timeout-minutes: ${{ vars.DEFAULT_SHORT_JOB_TIMEOUT_MIN == '' && 15 || vars.DEFAULT_SHORT_JOB_TIMEOUT_MIN }} + timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }} steps: + - name: Get rid of disruptive line endings before checkout + run: | + git config --global core.autocrlf false + + # The checkout supplies the robot suite and must precede the artifact + # downloads, because it cleans the workspace. + - name: Check out code into the Go module directory + uses: actions/checkout@v7.0.1 + # We deliberately do NOT build here: there is no native windows/arm64 build # (goreleaser only targets windows/amd64). This job downloads the amd64 .exe # produced by winbuild and runs it on a native arm64 host, validating that it @@ -387,6 +347,12 @@ jobs: name: stackql_windows_amd64 path: build + - name: Download Windows amd64 MCP client artifact + uses: actions/download-artifact@v8.0.1 + with: + name: stackql_mcp_client_windows_amd64 + path: build + - name: Confirm host is arm64 (binary under test is x64) shell: pwsh run: | @@ -406,7 +372,7 @@ jobs: if ($LASTEXITCODE -ne 0) { throw "stackql --version failed (exit $LASTEXITCODE)" } # 'exec' runs a query in batch mode (no TTY needed) and exercises the same - # cgo/sqlite engine the interactive shell uses, so it fully covers the + # embedded sqlite engine the interactive shell uses, so it fully covers the # "x64 binary runs under emulation" claim. The interactive shell is not # smoke-tested here: it's a REPL that requires a TTY, so piping a query to # it yields EOF + "goodbye" without ever executing the query. @@ -420,6 +386,63 @@ jobs: Write-Host "Windows arm64 (x64 emulation) smoke PASSED" + # The robot harness below mirrors winbuild, run against the emulated binary. + # Python is x64 (so itself emulated) because the pinned test dependencies + # psycopg2-binary, psycopg-binary and cryptography publish no win_arm64 wheels. + - name: Setup Python + uses: actions/setup-python@v7.0.0 + with: + cache: pip + python-version: '3.12' + architecture: x64 + + - name: Choco install openssl + uses: crazy-max/ghaction-chocolatey@v4.1.0 + timeout-minutes: ${{ vars.DEFAULT_LONG_STEP_TIMEOUT_MIN == '' && 40 || vars.DEFAULT_LONG_STEP_TIMEOUT_MIN }} + with: + args: "install --force openssl --version 3.1.1" + + - name: Choco install packages + uses: crazy-max/ghaction-chocolatey@v4.1.0 + timeout-minutes: ${{ vars.DEFAULT_LONG_STEP_TIMEOUT_MIN == '' && 40 || vars.DEFAULT_LONG_STEP_TIMEOUT_MIN }} + with: + args: "install --force postgresql13 sqlite" + + - name: Install Python dependencies + run: | + pip3 install -r cicd/requirements.txt + + - name: Generate rewritten registry for simulations + run: | + python3 test\python\stackql_test_tooling\registry_rewrite.py --srcdir "$pwd\test\registry\src" --destdir "$pwd\test\registry-mocked\src" + + # winbuild leaves the MCP client under both names; the robot tooling + # addresses it without the extension. + - name: Mirror winbuild binary layout + shell: pwsh + run: | + Copy-Item -Path ".\build\stackql_mcp_client.exe" -Destination ".\build\stackql_mcp_client" -Force + ls ".\build" + + - name: Create certificates for robot tests + run: | + openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365 + openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365 + openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 + + - name: Run robot mocked functional tests + env: + PSQL_EXE: C:\Program Files\PostgreSQL\13\bin\psql + SQLITE_EXE: C:\ProgramData\chocolatey\lib\SQLite\tools\sqlite3.exe + PYTHONPATH: '${{ env.PYTHONPATH }};${{ github.workspace }}\test\python' + run: | + python cicd/python/build.py --robot-test + + - name: Output from mocked functional tests + if: always() + run: | + cat ./test/robot/reports/output.xml || echo "no functional test output present" + linuxbuild: name: Linux Build runs-on: ubuntu-latest @@ -464,13 +487,6 @@ jobs: echo "SOURCE_TAG=${GITHUB_REF#refs/tags/}" } >> "${GITHUB_STATE}" - - name: Set up GCC - uses: egor-tensin/setup-gcc@v1 - id: gccsetup - with: - platform: x64 - cygwin: 0 - - name: Install psql run: | sudo apt-get update @@ -499,8 +515,7 @@ jobs: BUILDBRANCH: ${{github.ref}} BUILDPLATFORM: ${{runner.os}} BUILDPATCHVERSION: ${{github.run_number}} - CGO_ENABLED: 1 - CGO_LDFLAGS: '-static' + CGO_ENABLED: 0 run: | source cicd/version.txt export BUILDMAJORVERSION="$MajorVersion" @@ -536,7 +551,7 @@ jobs: BUILDBRANCH: ${{github.ref}} BUILDPLATFORM: ${{runner.os}} BUILDPATCHVERSION: ${{github.run_number}} - CGO_ENABLED: 1 + CGO_ENABLED: 0 run: | python cicd/python/build.py --verbose --build-mcp-client @@ -551,7 +566,7 @@ jobs: baselineRefFile="cicd/ref/bench/baseline-go-bench.log" thresholdRefFile="cicd/ref/bench/max-threshold-go-bench.log" go test -run='^$' -bench . -benchtime=100x -count=6 \ - --tags "sqlite_stackql" --ldflags "-X stackql/internal/stackql/planbuilder.PlanCacheEnabled=false" \ + --ldflags "-X stackql/internal/stackql/planbuilder.PlanCacheEnabled=false" \ ./... | tee ${outFile} # shellcheck disable=SC2181 if [ "$?" = "0" ]; then @@ -736,8 +751,7 @@ jobs: BUILDBRANCH: ${{github.ref}} BUILDPLATFORM: ${{runner.os}} BUILDPATCHVERSION: ${{github.run_number}} - CGO_ENABLED: 1 - CGO_LDFLAGS: '-static' + CGO_ENABLED: 0 run: | source cicd/version.txt export BUILDMAJORVERSION="$MajorVersion" @@ -934,8 +948,7 @@ jobs: BUILDBRANCH: ${{github.ref}} BUILDPLATFORM: ${{runner.os}} BUILDPATCHVERSION: ${{github.run_number}} - CGO_ENABLED: 1 - CGO_LDFLAGS: '-static' + CGO_ENABLED: 0 run: | if [ "${{ matrix.registry }}" = "test/registry" ]; then python3 test/python/stackql_test_tooling/tcp_lb.py --generate-hosts-entries | sudo tee -a /etc/hosts @@ -1088,10 +1101,7 @@ jobs: - name: Install psql run: | sudo apt-get update - sudo apt-get install --yes build-essential sudo apt-get install --yes --no-install-recommends postgresql-client - gccVersion="$(gcc --version 2>&1 | head -n 1)" - echo "gcc version: ${gccVersion}" - name: Install Python dependencies run: | @@ -1116,8 +1126,7 @@ jobs: BUILDBRANCH: ${{github.ref}} BUILDPLATFORM: ${{runner.os}} BUILDPATCHVERSION: ${{github.run_number}} - CGO_ENABLED: 1 - CGO_LDFLAGS: '-static' + CGO_ENABLED: 0 run: | source cicd/version.txt export BUILDMAJORVERSION="$MajorVersion" @@ -1153,7 +1162,7 @@ jobs: BUILDBRANCH: ${{github.ref}} BUILDPLATFORM: ${{runner.os}} BUILDPATCHVERSION: ${{github.run_number}} - CGO_ENABLED: 1 + CGO_ENABLED: 0 run: | python cicd/python/build.py --verbose --build-mcp-client @@ -1168,7 +1177,7 @@ jobs: baselineRefFile="cicd/ref/bench/baseline-go-bench.log" thresholdRefFile="cicd/ref/bench/max-threshold-go-bench.log" go test -run='^$' -bench . -benchtime=100x -count=6 \ - --tags "sqlite_stackql" --ldflags "-X stackql/internal/stackql/planbuilder.PlanCacheEnabled=false" \ + --ldflags "-X stackql/internal/stackql/planbuilder.PlanCacheEnabled=false" \ ./... | tee ${outFile} # shellcheck disable=SC2181 if [ "$?" = "0" ]; then @@ -1539,7 +1548,7 @@ jobs: git config --global "url.https://${GH_ACCESS_TOKEN}@github.com/.insteadOf" https://github.com/ go get -v -t -d ./... env: - CGO_ENABLED: 1 + CGO_ENABLED: 0 GH_ACCESS_TOKEN: ${{env.GH_ACCESS_TOKEN}} GOPRIVATE: ${{env.GOPRIVATE}} @@ -1549,7 +1558,7 @@ jobs: BUILDBRANCH: ${{github.ref}} BUILDPLATFORM: ${{runner.os}} BUILDPATCHVERSION: ${{github.run_number}} - CGO_ENABLED: 1 + CGO_ENABLED: 0 run: | source cicd/version.txt export BUILDMAJORVERSION="$MajorVersion" @@ -1672,12 +1681,32 @@ jobs: cache: true id: go + # Mirrors macosbuild, except that the libpq location is resolved rather + # than hardcoded: homebrew's prefix differs on Apple Silicon. + - name: Install Various dependencies + run: | + brew install libpq@16 + + # Use overwrite to force the links for version 16 + brew link --force --overwrite libpq@16 + + # Update PATH so the CI finds the brew version of psql + echo "$(brew --prefix libpq@16)/bin" >> "$GITHUB_PATH" + + pip3 install -r cicd/requirements.txt + + "$(brew --prefix libpq@16)/bin/psql" --version + + - name: Generate rewritten registry for simulations + run: | + python3 test/python/stackql_test_tooling/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src" + - name: Get dependencies run: | git config --global "url.https://${GH_ACCESS_TOKEN}@github.com/.insteadOf" https://github.com/ go get -v -t -d ./... env: - CGO_ENABLED: 1 + CGO_ENABLED: 0 GH_ACCESS_TOKEN: ${{env.GH_ACCESS_TOKEN}} GOPRIVATE: ${{env.GOPRIVATE}} @@ -1687,7 +1716,7 @@ jobs: BUILDBRANCH: ${{github.ref}} BUILDPLATFORM: "darwin_arm64" BUILDPATCHVERSION: ${{github.run_number}} - CGO_ENABLED: 1 + CGO_ENABLED: 0 run: | source cicd/version.txt export BUILDMAJORVERSION="$MajorVersion" @@ -1721,6 +1750,38 @@ jobs: python cicd/python/build.py --verbose --build python cicd/python/build.py --verbose --build-mcp-client + - name: Create certificates for robot tests + run: | + openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365 + openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365 + openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 + + - name: Run robot mocked functional tests + env: + PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python' + if: success() + run: | + python cicd/python/build.py --robot-test + + - name: Output from mocked functional tests + if: always() + run: | + cat ./test/robot/reports/output.xml + + - name: Upload Robot Reports Artifact + uses: actions/upload-artifact@v7.0.1 + if: always() && github.repository == 'stackql/stackql-devel' + with: + name: stackql_darwin_arm64_robot_reports + path: test/robot/reports + + - name: Upload Robot Tmp Artifact + uses: actions/upload-artifact@v7.0.1 + if: always() && github.repository == 'stackql/stackql-devel' + with: + name: stackql_darwin_arm64_robot_tmp + path: test/robot/functional/tmp + - name: Upload Artifact uses: actions/upload-artifact@v7.0.1 if: success() @@ -1914,7 +1975,7 @@ jobs: dockertest: env: IS_DOCKER: 'true' - name: Docker Test + name: ${{ matrix.job_name }} needs: - dockerbuild runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} @@ -1922,12 +1983,39 @@ jobs: strategy: fail-fast: false matrix: - platform: - - linux/amd64 - - linux/arm64 - db_backend: - - sqlite - - postgres_tcp + include: + - job_name: Docker Test (linux/amd64, sqlite) + platform: linux/amd64 + db_backend: sqlite + run_integration: true + - job_name: Docker Test (linux/amd64, postgres_tcp, shard 1 of 4) + platform: linux/amd64 + db_backend: postgres_tcp + robot_shard_index: 1 + robot_shard_count: 4 + run_integration: false + - job_name: Docker Test (linux/amd64, postgres_tcp, shard 2 of 4) + platform: linux/amd64 + db_backend: postgres_tcp + robot_shard_index: 2 + robot_shard_count: 4 + run_integration: false + - job_name: Docker Test (linux/amd64, postgres_tcp, shard 3 of 4) + platform: linux/amd64 + db_backend: postgres_tcp + robot_shard_index: 3 + robot_shard_count: 4 + run_integration: false + - job_name: Docker Test (linux/amd64, postgres_tcp, shard 4 of 4) + platform: linux/amd64 + db_backend: postgres_tcp + robot_shard_index: 4 + robot_shard_count: 4 + run_integration: false + - job_name: Docker Test (linux/arm64, sqlite) + platform: linux/arm64 + db_backend: sqlite + run_integration: false steps: - name: Prepare run: | @@ -2110,7 +2198,7 @@ jobs: PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python' IS_SKIP_MCP_TEST: 'true' if: success() && env.CI_IS_EXPRESS != 'true' && matrix.platform == 'linux/amd64' && env.BUILD_IMAGE_REQUIRED == 'true' && matrix.db_backend == 'postgres_tcp' - timeout-minutes: ${{ vars.DEFAULT_LONG_STEP_TIMEOUT_MIN == '' && 45 || vars.DEFAULT_LONG_STEP_TIMEOUT_MIN }} + timeout-minutes: ${{ vars.DEFAULT_STEP_TIMEOUT_MIN == '' && 20 || vars.DEFAULT_STEP_TIMEOUT_MIN }} run: | sudo rm -rf test/tmp || true mkdir -p test/tmp @@ -2126,15 +2214,30 @@ jobs: echo "## Stray docker containers after clean up commands ##" docker container ls --format "table {{.ID}}\t{{.Names}}\t{{.Ports}}" -a echo "## End ##" - python cicd/python/build.py --robot-test --config='{ "variables": { "EXECUTION_PLATFORM": "docker", "SHOULD_RUN_DOCKER_EXTERNAL_TESTS": true, "SQL_BACKEND": "postgres_tcp" } }' + python cicd/python/build.py \ + --robot-test \ + --robot-shard-index "${{ matrix.robot_shard_index }}" \ + --robot-shard-count "${{ matrix.robot_shard_count }}" \ + --config='{ "variables": { "EXECUTION_PLATFORM": "docker", "SHOULD_RUN_DOCKER_EXTERNAL_TESTS": true, "SQL_BACKEND": "postgres_tcp" } }' - name: Output from mocked functional tests if: always() && env.CI_IS_EXPRESS != 'true' && matrix.platform == 'linux/amd64' && env.BUILD_IMAGE_REQUIRED == 'true' run: | cat ./test/robot/reports/output.xml + # Consumed by dockertestverify, which asserts that the shards of each + # (platform, db backend) group together ran every functional test. + # Unsharded legs upload as shard 1 of 1. + - name: Upload robot output for shard verification + if: always() && env.CI_IS_EXPRESS != 'true' && matrix.platform == 'linux/amd64' && env.BUILD_IMAGE_REQUIRED == 'true' + uses: actions/upload-artifact@v7.0.1 + with: + name: robot-output-${{ env.PLATFORM_PAIR }}-${{ matrix.db_backend }}-shard-${{ matrix.robot_shard_index || 1 }}-of-${{ matrix.robot_shard_count || 1 }} + path: test/robot/reports/output.xml + if-no-files-found: error + - name: Run robot integration tests - if: env.AZURE_CLIENT_SECRET != '' && startsWith(env.STATE_SOURCE_TAG, 'build-release') && matrix.platform == 'linux/amd64' && env.BUILD_IMAGE_REQUIRED == 'true' + if: env.AZURE_CLIENT_SECRET != '' && startsWith(env.STATE_SOURCE_TAG, 'build-release') && matrix.run_integration && env.BUILD_IMAGE_REQUIRED == 'true' env: AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} @@ -2149,11 +2252,73 @@ jobs: echo "## End ##" python cicd/python/build.py --robot-test-integration --config='{ "variables": { "EXECUTION_PLATFORM": "docker" } }' + ## Sharding splits one robot run across several dockertest legs, so no single + ## leg can attest that the whole suite ran. This job runs once all legs have + ## concluded and fails unless, for each (platform, db backend) group, the + ## shards together account for every functional test exactly once: both the + ## total count and each individual test are checked against the test source, + ## parsed independently of the shard modifier. Anything that prevents the + ## check (absent shard, absent output, unlisted group) is itself a failure. + dockertestverify: + name: Docker Test Shard Verification + needs: + - dockertest + runs-on: ubuntu-latest + timeout-minutes: ${{ vars.DEFAULT_SHORT_JOB_TIMEOUT_MIN == '' && 15 || vars.DEFAULT_SHORT_JOB_TIMEOUT_MIN }} + steps: + + - name: Check out code into the Go module directory + uses: actions/checkout@v7.0.1 + + # Must mirror the conditions under which dockertest runs the robot + # functional tests (BUILD_IMAGE_REQUIRED and CI_IS_EXPRESS there). If the + # two drift, the safe outcome is the one that happens: this job expects + # reports that were never produced, and fails. + - name: Robot run env sanitize + run: | + ROBOT_RUN_REQUIRED="true" + if [ "$( grep '^build-elide.*' <<< '${{ github.ref_name }}' )" != "" ] || [ "${{ env.IS_FORK_PR }}" = "true" ] || [ "${{ env.CI_IS_EXPRESS }}" = "true" ]; then + ROBOT_RUN_REQUIRED="false" + fi + { + echo "ROBOT_RUN_REQUIRED=${ROBOT_RUN_REQUIRED}"; + } | tee -a "${GITHUB_ENV}" + + - name: Setup Python + uses: actions/setup-python@v7.0.0 + with: + cache: pip + python-version: '3.12' + + - name: Install Python dependencies + run: | + pip3 install -r cicd/requirements.txt + + - name: Download robot outputs + if: env.ROBOT_RUN_REQUIRED == 'true' + uses: actions/download-artifact@v8.0.1 + with: + path: ${{ runner.temp }}/robot-outputs + pattern: robot-output-* + + # The expected groups are listed explicitly: a group that uploads reports + # without being listed here fails verification, so a new robot-running + # dockertest leg cannot go unverified. + - name: Verify every functional test ran in each group + if: env.ROBOT_RUN_REQUIRED == 'true' + run: | + python test/python/stackql_test_tooling/robot_shard.py \ + --source test/robot/functional \ + --reports-dir "${{ runner.temp }}/robot-outputs" \ + --expected-group linux-amd64-sqlite \ + --expected-group linux-amd64-postgres_tcp + dockermerge: runs-on: ubuntu-latest needs: - dockerbuild - dockertest + - dockertestverify steps: - name: Check out code into the Go module directory @@ -2285,4 +2450,3 @@ jobs: if: env.PUSH_IMAGE_REQUIRED == 'true' run: | docker buildx imagetools inspect ${{ env.STACKQL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} - diff --git a/.github/workflows/mcp-packaging.yml b/.github/workflows/mcp-packaging.yml index d42b2fb2..ad738b1f 100644 --- a/.github/workflows/mcp-packaging.yml +++ b/.github/workflows/mcp-packaging.yml @@ -5,7 +5,7 @@ # release event fires before they exist. Flow: # push tag -> create release -> attach signed assets -> dispatch. # The npm / PyPI wrappers need interactive (2FA) credentials and are -# published from a local clone - see step 7 of docs/stackql release process.md. +# published from a local clone - see step 7 of docs/stackql_release_process.md. # The three embedded SDK wrappers (cargo, go, dotnet) are validated here on # every packaging PR (against the latest release, smoke tests fed the freshly # built bundle via STACKQL_MCP_BUNDLE) and, on dispatch, validated with real diff --git a/.github/workflows/mcp-registry-publish.yml b/.github/workflows/mcp-registry-publish.yml index 3a88fa8e..a635233b 100644 --- a/.github/workflows/mcp-registry-publish.yml +++ b/.github/workflows/mcp-registry-publish.yml @@ -4,7 +4,7 @@ # registry at the exact version, so the .mcpb release assets, the OCI image # (mcp-packaging), and the npm + PyPI wrappers (manual, 2FA) must ALL be live # before this runs. Dispatch it after steps 7a/7b of -# docs/stackql release process.md. +# docs/stackql_release_process.md. # # Auth is GitHub Actions OIDC: the registry grants io.github./* to # OIDC tokens from this repo's workflows, so no secret is required. diff --git a/.github/workflows/openai-plugin.yml b/.github/workflows/openai-plugin.yml index 43b7b221..8711e47e 100644 --- a/.github/workflows/openai-plugin.yml +++ b/.github/workflows/openai-plugin.yml @@ -37,24 +37,10 @@ jobs: - uses: actions/setup-go@v6 with: go-version-file: go.mod - - name: Configure Windows C toolchain - if: runner.os == 'Windows' - shell: pwsh - run: | - $mingwPath = 'C:\mingw64' - if (-not (Test-Path $mingwPath)) { - $mingwPath = 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64' - } - if (-not (Test-Path $mingwPath)) { - throw 'MinGW toolchain not found' - } - "$mingwPath\bin" | Out-File $env:GITHUB_PATH -Append - "CC=$mingwPath\bin\x86_64-w64-mingw32-gcc" | Out-File $env:GITHUB_ENV -Append - "CXX=$mingwPath\bin\x86_64-w64-mingw32-g++" | Out-File $env:GITHUB_ENV -Append - name: Build StackQL env: - CGO_ENABLED: '1' - run: go build --tags sqlite_stackql -o ${{ matrix.binary }} ./stackql + CGO_ENABLED: '0' + run: go build -o ${{ matrix.binary }} ./stackql - name: Validate package run: ${{ matrix.python }} packaging/openai-plugin/scripts/validate.py - name: Test stdio connector diff --git a/.vscode/launch.json b/.vscode/launch.json index 815a876e..e918f076 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -245,15 +245,6 @@ "{}" ] }, - { - "type": "pickString", - "id": "buildFlags", - "description": "Extra Build Flags for debug; cannot place elsewhere", - "default": "-tags=sqlite_stackql", - "options": [ - "-tags=sqlite_stackql" - ] - }, { "type": "pickString", "id": "appRoot", @@ -460,7 +451,6 @@ "request": "launch", "mode": "debug", "program": "${workspaceFolder}/stackql", - "buildFlags": "${input:buildFlags}", "args": [ "exec", "--registry=${input:registryString}", diff --git a/.vscode/settings.json b/.vscode/settings.json index fa0f5914..27cb218c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,10 +9,6 @@ ], "editor.insertSpaces": true, "cmake.configureOnOpen": false, - "go.buildFlags": [ - "-tags=sqlite_stackql" - ], - "go.testTags": "sqlite_stackql", "todohighlight.keywordsPattern": "TODO:|FIXME:|BLOCK|END_BLOCK", //highlight `TODO:`,`FIXME:` or commented content between parentheses "todohighlight.exclude": [ "**/*.json" diff --git a/AGENTS.md b/AGENTS.md index bdf98d2f..ce79a50f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ Authoritative references: [developer guide](/docs/developer_guide.md), [test sum Common commands (run from repo root): - Build: `python cicd/python/build.py --build` (output at `./build/stackql`). -- Unit tests: `python cicd/python/build.py --test` (CI uses `go test -timeout 1200s --tags "sqlite_stackql" -v ./...`). +- Unit tests: `python cicd/python/build.py --test` (CI uses `go test -timeout 1200s -v ./...`). - Robot tests: `python cicd/python/build.py --robot-test` (requires the binary from the build step). - Lint: `golangci-lint run` (CI pins the version in [`.github/workflows/lint.yml`](/.github/workflows/lint.yml); config in [`.golangci.yml`](/.golangci.yml)). @@ -58,6 +58,30 @@ StackQL supports flexible configuration management, including authentication sec --- +## SQLite backend + +- The embedded backend is `modernc.org/sqlite` (pure Go, no cgo), pinned >= v1.57.0 + (caller-constructed `Driver` function registration). It is consumed via + [`any-sdk`](https://github.com/stackql/any-sdk) `public/sqlengine`, which registers + the driver under the name `stackql-sqlite`. +- DSN construction happens only via the central `BuildDSN` in any-sdk `public/sqlengine` + (legacy mattn-style params such as `_busy_timeout=5000` are translated to modernc + `_pragma=busy_timeout(5000)` syntax there); driver errors are inspected only via the + error predicates in that package (`isBusy`, `isConstraintViolation`); the custom + extension functions (`split_part`, `regexp_like`, `regexp_substr`, `regexp_replace`, + `json_equal`, `aws_policy_equal`) live only in any-sdk `public/sqlfuncs` and are + registered on the constructed driver, never globally. +- Never reintroduce `mattn/go-sqlite3`, cgo, the `sqlite_stackql` build tag, or + package-global function registration. All builds and release artifacts are + `CGO_ENABLED=0`. +- Bumps of `modernc.org/sqlite` merge only after the full robot suite passes + (transpile regenerations occasionally regress). +- Rollback anchor: the tag `v0.11.707-final-cgo` marks the last cgo commit on `main` + (the `stackql-go-sqlite3` fork plus the `sqlite_stackql` build tag). To roll back + the pure Go migration, branch from that tag. + +--- + ## StackQL Resource Key Encoding Quirk ### Hierarchical Keys and URL Encoding diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2cf0027e..558a5759 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,10 @@ Before beginning implementation work, please raise an issue and receive a consen Please raise pull requests (PRs) from a fork of this repository, using the provided PR template. The richer the detail / evidence supplied, the easier the review process. A usefully named feature branch for the PR source is appreciated. +## Dependency notes + +- Bumps of `modernc.org/sqlite` (the pure Go embedded SQLite backend, consumed via `any-sdk`) merge only after the full robot test suite passes; its transpile regenerations occasionally regress. + Finally, in respect of all your efforts; we thank you and ask that you please give us a star while you are at it. Many Thanks, diff --git a/Dockerfile b/Dockerfile index d279b1d2..2d3cf778 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,8 +46,8 @@ ENV SRC_DIR=/work/stackql/src ENV BUILD_DIR=/work/stackql/build RUN cd ${SRC_DIR} \ - && go test --tags "sqlite_stackql" ./... \ - && go build -ldflags "-X github.com/stackql/stackql/internal/stackql/cmd.BuildMajorVersion=$BUILDMAJORVERSION \ + && CGO_ENABLED=0 go test ./... \ + && CGO_ENABLED=0 go build -ldflags "-X github.com/stackql/stackql/internal/stackql/cmd.BuildMajorVersion=$BUILDMAJORVERSION \ -X github.com/stackql/stackql/internal/stackql/cmd.BuildMinorVersion=$BUILDMINORVERSION \ -X github.com/stackql/stackql/internal/stackql/cmd.BuildPatchVersion=$BUILDPATCHVERSION \ -X github.com/stackql/stackql/internal/stackql/cmd.BuildCommitSHA=$BUILDCOMMITSHA \ @@ -55,11 +55,10 @@ RUN cd ${SRC_DIR} \ -X \"github.com/stackql/stackql/internal/stackql/cmd.BuildDate=$BUILDDATE\" \ -X \"stackql/internal/stackql/planbuilder.PlanCacheEnabled=$PLANCACHEENABLED\" \ -X github.com/stackql/stackql/internal/stackql/cmd.BuildPlatform=$BUILDPLATFORM" \ - --tags "sqlite_stackql" \ -o ${BUILD_DIR}/stackql ./stackql RUN cd ${SRC_DIR} \ - && go build \ + && CGO_ENABLED=0 go build \ -o ${BUILD_DIR}/stackql_mcp_client ./mcp_client/cmd # bookworm, not bullseye: bullseye LTS ended 2026-08-31 and its security suite diff --git a/cicd/python/build.py b/cicd/python/build.py index 0409cfe1..0643d74d 100644 --- a/cicd/python/build.py +++ b/cicd/python/build.py @@ -13,9 +13,9 @@ def build_stackql(verbose :bool) -> int: os.environ['BUILDMAJORVERSION'] = os.environ.get('BUILDMAJORVERSION', '1') os.environ['BUILDMINORVERSION'] = os.environ.get('BUILDMINORVERSION', '1') os.environ['BUILDPATCHVERSION'] = os.environ.get('BUILDPATCHVERSION', '1') - os.environ['CGO_ENABLED'] = os.environ.get('CGO_ENABLED', '1') + os.environ['CGO_ENABLED'] = os.environ.get('CGO_ENABLED', '0') return subprocess.call( - f'go build {"-x -v" if verbose else ""} --tags "sqlite_stackql" -ldflags "-X github.com/stackql/stackql/internal/stackql/cmd.BuildMajorVersion={os.environ.get("BUILDMAJORVERSION")} ' + f'go build {"-x -v" if verbose else ""} -ldflags "-X github.com/stackql/stackql/internal/stackql/cmd.BuildMajorVersion={os.environ.get("BUILDMAJORVERSION")} ' f'-X github.com/stackql/stackql/internal/stackql/cmd.BuildMinorVersion={os.environ.get("BUILDMINORVERSION")} ' f'-X github.com/stackql/stackql/internal/stackql/cmd.BuildPatchVersion={os.environ.get("BUILDPATCHVERSION")} ' f'-X github.com/stackql/stackql/internal/stackql/cmd.BuildCommitSHA={os.environ.get("BUILDCOMMITSHA", "")} ' @@ -31,7 +31,7 @@ def build_stackql_mcp_client(verbose :bool) -> int: os.environ['BUILDMAJORVERSION'] = os.environ.get('BUILDMAJORVERSION', '1') os.environ['BUILDMINORVERSION'] = os.environ.get('BUILDMINORVERSION', '1') os.environ['BUILDPATCHVERSION'] = os.environ.get('BUILDPATCHVERSION', '1') - os.environ['CGO_ENABLED'] = os.environ.get('CGO_ENABLED', '1') + os.environ['CGO_ENABLED'] = os.environ.get('CGO_ENABLED', '0') return subprocess.call( 'go build ' f'{"-x -v" if verbose else ""} ' @@ -42,7 +42,7 @@ def build_stackql_mcp_client(verbose :bool) -> int: def unit_test_stackql(verbose :bool) -> int: return subprocess.call( - f'go test -timeout 1200s {"-v" if verbose else ""} --tags "sqlite_stackql" ./...', + f'go test -timeout 1200s {"-v" if verbose else ""} ./...', shell=True ) @@ -54,9 +54,22 @@ def sanitise_val(val) -> str: def run_robot_mocked_functional_tests_stackql(*args, **kwargs) -> int: variables = ' '.join([f'--variable {key}:{sanitise_val(value)} ' for key, value in kwargs.get("variables", {}).items() ]) + shard_index = kwargs.get('shard_index') + shard_count = kwargs.get('shard_count') + if (shard_index is None) != (shard_count is None): + raise ValueError('robot shard index and count must be specified together') + shard_option = '' + if shard_index is not None: + if shard_count < 1 or shard_index < 1 or shard_index > shard_count: + raise ValueError('robot shard index must be between 1 and the shard count') + shard_option = ( + '--prerunmodifier ' + f'stackql_test_tooling.robot_shard.ShardByTest:{shard_index}:{shard_count} ' + ) return subprocess.call( 'robot ' - f'{variables} ' + f'{variables} ' + f'{shard_option}' '-d test/robot/reports ' 'test/robot/functional', shell=True @@ -109,6 +122,8 @@ def main(): parser.add_argument('--build-mcp-client', action='store_true') parser.add_argument('--test', action='store_true') parser.add_argument('--robot-test', action='store_true') + parser.add_argument('--robot-shard-index', type=int) + parser.add_argument('--robot-shard-count', type=int) parser.add_argument('--robot-test-integration', action='store_true') parser.add_argument('--robot-test-traffic-lights-integration', action='store_true') parser.add_argument('--robot-test-id-fed-traffic-lights-integration', action='store_true') @@ -129,7 +144,11 @@ def main(): if ret_code != 0: exit(ret_code) if args.robot_test: - ret_code = run_robot_mocked_functional_tests_stackql(**args.config) + ret_code = run_robot_mocked_functional_tests_stackql( + shard_index=args.robot_shard_index, + shard_count=args.robot_shard_count, + **args.config + ) if ret_code != 0: exit(ret_code) if args.robot_test_integration: diff --git a/docs/developer_guide.md b/docs/developer_guide.md index dbf05d4a..8235e21a 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -48,8 +48,7 @@ As of now, `stackql` handles `xml` SERDE through the core, and does not route th ## Building locally ```bash -env CGO_ENABLED=1 PLANCACHEENABLED=false go build \ - --tags "sqlite_stackql" \ +env CGO_ENABLED=0 PLANCACHEENABLED=false go build \ -ldflags "-X github.com/stackql/stackql/internal/stackql/cmd.BuildMajorVersion=${BUILDMAJORVERSION:-1} \ -X github.com/stackql/stackql/internal/stackql/cmd.BuildMinorVersion=${BUILDMINORVERSION:-1} \ -X github.com/stackql/stackql/internal/stackql/cmd.BuildPatchVersion=${BUILDPATCHVERSION:-1} \ @@ -71,7 +70,7 @@ We have upgraded to golang `>= 1.25.3` in order to take advantage of [time simul To run all unit tests: ```bash -go test -timeout 1200s --tags "sqlite_stackql" ./... +go test -timeout 1200s ./... ``` ### Robot tests @@ -186,7 +185,7 @@ Requirements are [detailed in the root README](/README.md#system-requirements). Local testing of the application: -1. Run `go test --tags "sqlite_stackql" ./...` tests. +1. Run `go test ./...` tests. 2. Build the executable [as per the root README](/README.md#build) 3. Perform registry rewrites as needed for mocking `python3 test/python/stackql_test_tooling/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"`. 3. Run robot tests: @@ -205,7 +204,7 @@ Test coverage is sparse. Regressions are mitigated by `go test` integration tes #### Point in time gotest coverage If not already done, then install 'cover' with `go get golang.org/x/tools/cmd/cover`. -Then: `go test --tags "sqlite_stackql" -cover ../...`. +Then: `go test -cover ../...`. ### Functional and Integration testing diff --git a/docs/diagrams/plantuml/hld-components.puml b/docs/diagrams/plantuml/hld-components.puml index 84965736..1b5f99cf 100644 --- a/docs/diagrams/plantuml/hld-components.puml +++ b/docs/diagrams/plantuml/hld-components.puml @@ -4,7 +4,7 @@ node "any-sdk" as AnySdk node "stackql-parser" as StackQLParser node "psql-wire" as PsqlWire node "stackql-provider-registry" as ProviderRegistry -node "stackql-go-sqlite3" as StackqlSqlite +node "modernc.org/sqlite" as StackqlSqlite node "postgres" as Postgres node "postgres clients\neg: libpq" as PostgresClient node "AI agents\neg: Claude Code" as AIAgent diff --git a/docs/licenses/README.md b/docs/licenses/README.md index 93c02a1e..74eb038b 100644 --- a/docs/licenses/README.md +++ b/docs/licenses/README.md @@ -8,7 +8,7 @@ Some deep links to original licenses are supplied for convenience: | Name | Source | Local copy if present | Original License | | ----- | ----- | ------ | ------ | -| go-sqlite3 | https://github.com/mattn/go-sqlite3 | [go-sqlite3_license](/docs/licenses/go-sqlite3_license) | https://github.com/mattn/go-sqlite3/blob/master/LICENSE | +| modernc.org/sqlite | https://gitlab.com/cznic/sqlite | [modernc-sqlite_license](/docs/licenses/modernc-sqlite_license) | https://gitlab.com/cznic/sqlite/-/blob/master/LICENSE | | psql-wire | https://github.com/jeroenrinzema/psql-wire | [psql-wire_license](/docs/licenses/psql-wire_license) | https://github.com/jeroenrinzema/psql-wire/blob/ccdd1b552fc805192177fb5263a67b735603406e/LICENSE | | readline | https://github.com/chzyer/readline | [readline_license](/docs/licenses/readline_license) | https://github.com/chzyer/readline/blob/master/LICENSE | | [vitess](https://vitess.io) | https://github.com/vitessio/vitess | [vitess_license](/docs/licenses/vitess_license) | https://github.com/vitessio/vitess/blob/main/LICENSE | diff --git a/docs/licenses/go-sqlite3_license b/docs/licenses/go-sqlite3_license deleted file mode 100644 index cc154027..00000000 --- a/docs/licenses/go-sqlite3_license +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Yasuhiro Matsumoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/docs/licenses/modernc-sqlite_license b/docs/licenses/modernc-sqlite_license new file mode 100644 index 00000000..867b0f37 --- /dev/null +++ b/docs/licenses/modernc-sqlite_license @@ -0,0 +1,26 @@ +Copyright (c) 2017 The Sqlite Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/sql_backends.md b/docs/sql_backends.md index 6e4b97a3..edab14e7 100644 --- a/docs/sql_backends.md +++ b/docs/sql_backends.md @@ -53,5 +53,5 @@ List of drivers: ### Data Source Name (DSN) strings -- [SQLite as per golang](https://github.com/mattn/go-sqlite3#dsn-examples). +- [SQLite as per golang (modernc.org/sqlite)](https://pkg.go.dev/modernc.org/sqlite#Driver.Open). - [Postgres URI](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING). diff --git a/docs/stackql release process.md b/docs/stackql_release_process.md similarity index 82% rename from docs/stackql release process.md rename to docs/stackql_release_process.md index 13b96e31..3ec3967e 100644 --- a/docs/stackql release process.md +++ b/docs/stackql_release_process.md @@ -1,5 +1,15 @@ ## StackQL release process +### Release notes: pure Go SQLite backend (first release after the modernc migration) + +Include the following in the notes of the first release cut after the modernc.org/sqlite migration: + +> The embedded SQLite backend is now [`modernc.org/sqlite`](https://gitlab.com/cznic/sqlite) (pure Go). All binaries are built with `CGO_ENABLED=0` and are statically linked; no C toolchain is involved anywhere in the build. +> The custom extension functions (`split_part`, `regexp_like`, `regexp_substr`, `regexp_replace`, `json_equal`, `aws_policy_equal`) are ported to pure Go in [`any-sdk`](https://github.com/stackql/any-sdk); documented behaviour divergences (RE2 regexp limits) are listed in any-sdk `public/sqlfuncs/DIVERGENCES.md`. +> Library embedders now inherit the `modernc.org/sqlite` dependency tree instead of `mattn/go-sqlite3`/cgo. +> External loadable SQLite extensions (`.load`) are no longer supported. +> Rollback anchor: the tag `v0.11.707-final-cgo` marks the last cgo commit on `main`. + 1. Download Artifacts from Latest Build Download the following artifacts from the latest build on the `main` branch including: diff --git a/go.mod b/go.mod index 998cb418..d7015d24 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.10.1 github.com/stackql-labs/omnisdk v0.1.2-beta02 - github.com/stackql/any-sdk v0.5.5-alpha01 + github.com/stackql/any-sdk v0.5.6-alpha02-pure-go-sqlite-migration github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.2-beta01 github.com/stackql/stackql-parser v0.0.16-alpha02 @@ -74,6 +74,7 @@ require ( github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.7.0 // indirect github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect @@ -109,10 +110,11 @@ require ( github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/mattn/go-sqlite3 v1.14.31 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oracle/oci-go-sdk/v65 v65.120.0 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect @@ -121,6 +123,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/segmentio/asm v1.1.3 // indirect @@ -158,8 +161,10 @@ require ( google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.75.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect + modernc.org/sqlite v1.59.0 // indirect ) replace github.com/chzyer/readline => github.com/stackql/readline v0.0.2-alpha05 - -replace github.com/mattn/go-sqlite3 => github.com/stackql/stackql-go-sqlite3 v1.0.5-stackql diff --git a/go.sum b/go.sum index 0dc5eb46..fb6566cd 100644 --- a/go.sum +++ b/go.sum @@ -109,6 +109,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= github.com/dvsekhvalnov/jose2go v1.7.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= @@ -166,6 +168,8 @@ github.com/google/go-jsonnet v0.17.0 h1:/9NIEfhK1NQRKl3sP2536b2+x5HnZMdql7x3yK/l github.com/google/go-jsonnet v0.17.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw= github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -173,6 +177,9 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -276,6 +283,8 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= @@ -288,6 +297,8 @@ github.com/modelcontextprotocol/go-sdk v1.7.0 h1:yqjY2dsbKAC0LSuWZVBMrHgiG8ukXv6 github.com/modelcontextprotocol/go-sdk v1.7.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/olekukonko/tablewriter v0.0.0-20180130162743-b8a9be070da4 h1:Mm4XQCBICntJzH8fKglsRuEiFUJYnTnM4BBFvpP5BWs= github.com/olekukonko/tablewriter v0.0.0-20180130162743-b8a9be070da4/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -310,6 +321,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -355,16 +368,14 @@ github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcG github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/stackql-labs/omnisdk v0.1.2-beta02 h1:4UpcSdZtsqyPFhv5ZZ9Mz1siUvAXXeGah0R64tczMww= github.com/stackql-labs/omnisdk v0.1.2-beta02/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= -github.com/stackql/any-sdk v0.5.5-alpha01 h1:Omj/EuF0hx8oai54aRZMrWkjsV11YK532lq4Spfvjiw= -github.com/stackql/any-sdk v0.5.5-alpha01/go.mod h1:DV6KrDVMIpbuWHm4YAdMtCsszbpdzLL0JUXpEX3wDMU= +github.com/stackql/any-sdk v0.5.6-alpha02-pure-go-sqlite-migration h1:UZBBzLF9lrZjbsGQoRPBZ4HhaIChini+sDX3VmDwvaQ= +github.com/stackql/any-sdk v0.5.6-alpha02-pure-go-sqlite-migration/go.mod h1:NjQV7GMMEk6aBwQUSn8PeIEKqM8CtRwlAsyZ5GaulTs= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g= github.com/stackql/psql-wire v0.1.2-beta01 h1:+0xeLLam3nbI1iv84pWJYGYw5VO5S/+eICRbdu6+dj8= github.com/stackql/psql-wire v0.1.2-beta01/go.mod h1:a44Wd8kDC3irFLpGutarKDBqhJ/aqXlj1aMzO5bVJYg= github.com/stackql/readline v0.0.2-alpha05 h1:ID4QzGdplFBsrSnTuz8pvKzWw96JbrJg8fsLry2UriU= github.com/stackql/readline v0.0.2-alpha05/go.mod h1:OFAYOdXk/X4+5GYiDXFfaGrk+bCN6Qv0SYY5HNzD2E0= -github.com/stackql/stackql-go-sqlite3 v1.0.5-stackql h1:cgy71EPxlbURHAjAn+gwNmDgjhWdm6V3zGY5QvB78eY= -github.com/stackql/stackql-go-sqlite3 v1.0.5-stackql/go.mod h1:HemqCrcMK2xyhMMMt6oZ7ERDtoSmyyDsw5LBBcTZ+Rk= github.com/stackql/stackql-parser v0.0.16-alpha02 h1:kU3THgY23jS39Uv8b0xnFp329gWWWsKbsyvbFWeGrL8= github.com/stackql/stackql-parser v0.0.16-alpha02/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ= github.com/stackql/stackql-provider-registry v0.0.2-alpha01 h1:fPz6MhfXaMC/VPNJnjUo0sZyjTYhlifzaUWZ9Yj8NVE= @@ -556,3 +567,31 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/packaging/mcpb/CLAUDE.md b/packaging/mcpb/CLAUDE.md index 85b0da6b..177e689e 100644 --- a/packaging/mcpb/CLAUDE.md +++ b/packaging/mcpb/CLAUDE.md @@ -37,7 +37,7 @@ The contract every wrapper follows: - Canonical launch argv `mcp --mcp.server.type=stdio --approot /.stackql --mcp.config {"server":{"mode":"","audit":{"disabled":true}}} [--auth=]`; SDKs default to `read_only`. - Every SDK ships a launcher `scripts/smoke-test.py --cmd` can drive with no code (`make -smoke`): cargo `cargo run --example launcher --`, go `go run ./cmd/stackql-mcp-launch`, dotnet `dotnet run --project samples/Launcher --`. Extra argv passes through to the server. -Ordering rules (one-way, see `docs/stackql release process.md` step 7): +Ordering rules (one-way, see `docs/stackql_release_process.md` step 7): 1. Bundles first: `.mcpb` + `.sha256` on the GitHub release (`mcp-packaging` dispatch, or `make publish`). 2. Only then render (`make manifests VERSION=`) - the renderer fetches the canonical `.sha256` from the release; never pin locally built bundle hashes. diff --git a/test/assets/expected/github/repos/select-github-repos-functions.txt b/test/assets/expected/github/repos/select-github-repos-functions.txt index 586400b7..7db447af 100644 --- a/test/assets/expected/github/repos/select-github-repos-functions.txt +++ b/test/assets/expected/github/repos/select-github-repos-functions.txt @@ -1,9 +1,9 @@ |---------------------------|----------------|---------------------|---------------|--------------------| | name | extracted_team | user_suffix | is_docusaurus | sqlite_lib_version | |---------------------------|----------------|---------------------|---------------|--------------------| -| docusaurus-plugin-hubspot | dummyorg | username = dummyorg | 1 | 3.45.1 | +| docusaurus-plugin-hubspot | dummyorg | username = dummyorg | 1 | 3.53.4 | |---------------------------|----------------|---------------------|---------------|--------------------| -| dummyapp.io | dummyorg | username = dummyorg | 0 | 3.45.1 | +| dummyapp.io | dummyorg | username = dummyorg | 0 | 3.53.4 | |---------------------------|----------------|---------------------|---------------|--------------------| -| gatsby-plugin-smartlook | dummyorg | username = dummyorg | 0 | 3.45.1 | +| gatsby-plugin-smartlook | dummyorg | username = dummyorg | 0 | 3.53.4 | |---------------------------|----------------|---------------------|---------------|--------------------| diff --git a/test/python/stackql_test_tooling/robot_shard.py b/test/python/stackql_test_tooling/robot_shard.py new file mode 100644 index 00000000..e731d6ad --- /dev/null +++ b/test/python/stackql_test_tooling/robot_shard.py @@ -0,0 +1,162 @@ +import argparse +import collections +import hashlib +import os +import re +import sys + +from robot.api import ExecutionResult, SuiteVisitor, TestSuiteBuilder + + +_SHARD_GROUP_TAG_PREFIX = 'shard-group:' + +# Report directories are named after the uploaded artifact, which encodes the +# (platform, db backend) group plus the shard coordinates of the run. +_REPORT_DIR_PATTERN = re.compile( + r'^robot-output-(?P.+)-shard-(?P\d+)-of-(?P\d+)$' +) + +_OUTPUT_FILE_NAME = 'output.xml' + +_NOT_RUN_STATUS = 'NOT RUN' + + +class ShardByTest(SuiteVisitor): + """Select one deterministic shard while keeping tagged tests together.""" + + def __init__(self, shard_index, shard_count): + self.shard_index = int(shard_index) + self.shard_count = int(shard_count) + if self.shard_count < 1: + raise ValueError('shard count must be positive') + if not 1 <= self.shard_index <= self.shard_count: + raise ValueError('shard index must be between 1 and the shard count') + + def start_suite(self, suite): + suite.tests = [ + test for test in suite.tests + if self._shard_for(self._shard_key(test)) == self.shard_index + ] + + def end_suite(self, suite): + suite.suites = [child for child in suite.suites if child.test_count > 0] + + def _shard_for(self, test_name): + digest = hashlib.sha256(test_name.encode('utf-8')).digest() + return int.from_bytes(digest[:8], 'big') % self.shard_count + 1 + + def _shard_key(self, test): + group_tags = [ + str(tag).lower() for tag in test.tags + if str(tag).lower().startswith(_SHARD_GROUP_TAG_PREFIX) + ] + if len(group_tags) > 1: + raise ValueError(f'test {test.full_name} has multiple shard group tags') + return group_tags[0] if group_tags else test.full_name + + +def _expected_tests(source): + """Every test in the source tree, parsed with no shard modifier applied.""" + suite = TestSuiteBuilder().build(source) + return collections.Counter(test.full_name for test in suite.all_tests) + + +def _collect_report_dirs(reports_dir): + """Map group name to {shard index: (shard count, report dir)}.""" + errors = [] + groups = collections.defaultdict(dict) + if not os.path.isdir(reports_dir): + return groups, [f'reports dir not found: {reports_dir}'] + for entry in sorted(os.listdir(reports_dir)): + matched = _REPORT_DIR_PATTERN.match(entry) + if matched is None: + errors.append(f'unrecognised entry in reports dir: {entry}') + continue + index = int(matched.group('index')) + if index in groups[matched.group('group')]: + errors.append(f'duplicate report for shard {index}: {entry}') + continue + groups[matched.group('group')][index] = ( + int(matched.group('count')), + os.path.join(reports_dir, entry), + ) + return groups, errors + + +def _verify_group(group, shards, expected): + """Assert that the shards of one group ran exactly the expected tests.""" + errors = [] + shard_counts = {count for count, _ in shards.values()} + if len(shard_counts) != 1: + return [f'{group}: inconsistent shard counts {sorted(shard_counts)}'] + shard_count = shard_counts.pop() + absent_shards = sorted(set(range(1, shard_count + 1)) - set(shards)) + surplus_shards = sorted(set(shards) - set(range(1, shard_count + 1))) + if absent_shards: + errors.append(f'{group}: no report for shards {absent_shards} of {shard_count}') + if surplus_shards: + errors.append(f'{group}: reports for out of range shards {surplus_shards} of {shard_count}') + observed = collections.Counter() + for index in sorted(shards): + output_path = os.path.join(shards[index][1], _OUTPUT_FILE_NAME) + if not os.path.isfile(output_path): + errors.append(f'{group}: shard {index} has no {_OUTPUT_FILE_NAME}') + continue + shard_tests = list(ExecutionResult(output_path).suite.all_tests) + print(f'{group}: shard {index} of {shard_count} reported {len(shard_tests)} tests') + for test in shard_tests: + observed[test.full_name] += 1 + if test.status == _NOT_RUN_STATUS: + errors.append(f'{group}: test not run: {test.full_name}') + for name in sorted((expected - observed).elements()): + errors.append(f'{group}: expected test absent from every shard: {name}') + for name in sorted((observed - expected).elements()): + errors.append(f'{group}: test reported more often than expected: {name}') + expected_total = sum(expected.values()) + observed_total = sum(observed.values()) + print(f'{group}: {observed_total} tests reported, {expected_total} expected') + if observed_total != expected_total: + errors.append( + f'{group}: total test count {observed_total} does not match expected {expected_total}' + ) + return errors + + +def verify_shards(source, reports_dir, expected_groups): + """Return the list of reasons the shard reports fail to cover the source. + + Each expected group must account for every test in the source exactly + once across its shards, by name and by total count. An empty list + means verified; anything that prevents verification is itself an error. + """ + expected = _expected_tests(source) + if not expected: + return [f'no tests found in {source}'] + groups, errors = _collect_report_dirs(reports_dir) + for group in sorted(set(expected_groups) - set(groups)): + errors.append(f'{group}: no reports found') + for group in sorted(set(groups) - set(expected_groups)): + errors.append(f'{group}: reports found for a group that is not expected') + for group in sorted(set(groups) & set(expected_groups)): + errors.extend(_verify_group(group, groups[group], expected)) + return errors + + +def main(): + parser = argparse.ArgumentParser( + description='Assert that sharded robot runs cover every test in the source.' + ) + parser.add_argument('--source', required=True) + parser.add_argument('--reports-dir', required=True) + parser.add_argument('--expected-group', action='append', required=True) + args = parser.parse_args() + errors = verify_shards(args.source, args.reports_dir, args.expected_group) + for error in errors: + print(f'SHARD VERIFICATION FAILURE: {error}', file=sys.stderr) + if errors: + sys.exit(1) + print('shard verification PASSED') + + +if __name__ == '__main__': + main() diff --git a/test/robot/functional/stackql.resource b/test/robot/functional/stackql.resource index 0ad34125..b52b94e7 100644 --- a/test/robot/functional/stackql.resource +++ b/test/robot/functional/stackql.resource @@ -187,12 +187,8 @@ Start Postgres External Source If Viable Stackql Per Test Teardown IF "${EXECUTION_PLATFORM}" == "docker" and "${SQL_BACKEND}" == "postgres_tcp" - ${res} = Run Process bash \-c docker kill $(docker ps \-\-filter name\=execrun \-q) - Log Container killed - # Should Be Equal As Integers ${res.rc} 0 - ${restwo} = Run Process bash \-c docker rm $(docker ps \-\-filter status\=exited \-q) - Log Container removed - # Should Be Equal As Integers ${restwo.rc} 0 + ${result} = Run Process docker compose \-p execrun down \-\-remove\-orphans + Log Execution containers cleaned up END Remove Preview Mock Environment @@ -204,4 +200,3 @@ Remove Preview Mock Environment ... GOOGLE_APPLICATION_CREDENTIALS Remove Environment Variable ${name} END - diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index eeb16e77..d1fec1cf 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -1207,6 +1207,32 @@ Split Part Negative Index Invocation Working ... ${outputStr} ... ${CURDIR}/tmp/Split-Part-Negative-Index-Invocation-Working.tmp +Sqlite Extension Functions Smoke Working + [Documentation] Exercises every custom sqlite extension function (split_part, + ... regexp_like, regexp_substr, regexp_replace, json_equal, aws_policy_equal) + ... in a single scalar select, guarding the pure Go (modernc.org/sqlite) port + ... of the former cgo extension functions. This is the per-platform check: + ... it runs wherever the functional suite runs, in place of workflow-level + ... smoke steps. + Pass Execution If "${SQL_BACKEND}" == "postgres_tcp" Skipping postgres backend test due to unsupported sqlite extension functions + ${outputStr} = Catenate SEPARATOR=\n + ... |----|----|-----|--------|----|-----| + ... |${SPACE}sp${SPACE}|${SPACE}rl${SPACE}|${SPACE}rs${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}rr${SPACE}${SPACE}${SPACE}|${SPACE}je${SPACE}|${SPACE}ape${SPACE}| + ... |----|----|-----|--------|----|-----| + ... |${SPACE}b${SPACE}${SPACE}|${SPACE}${SPACE}1${SPACE}|${SPACE}123${SPACE}|${SPACE}abcXXX${SPACE}|${SPACE}${SPACE}1${SPACE}|${SPACE}${SPACE}${SPACE}1${SPACE}| + ... |----|----|-----|--------|----|-----| + Should Horrid Query StackQL Inline Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... select split_part('a,b,c', ',', 2) as sp, regexp_like('abc123', '[0-9]+') as rl, regexp_substr('abc123def', '[0-9]+') as rs, regexp_replace('abc123', '[0-9]', 'X') as rr, json_equal('{"a":1}', '{ "a" : 1.0 }') as je, aws_policy_equal('{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}', '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["*"]}]}') as ape; + ... ${outputStr} + ... ${CURDIR}/tmp/Sqlite-Extension-Functions-Smoke-Working.tmp + Create Table Scenario Working ${inputStr} = Catenate ... create table phystab_one(t_id int, z text); @@ -2927,6 +2953,7 @@ Page Limited Select Github ... stdout=${CURDIR}/tmp/Page-Limited-Select-Github.tmp Basic Query mTLS Returns OK + [Tags] shard-group:pg-server-generation Should PG Client Inline Contain ... ${CURDIR} ... ${PSQL_EXE} @@ -2935,6 +2962,7 @@ Basic Query mTLS Returns OK ... ipCidrRange Basic Error Query mTLS Returns Error Message + [Tags] shard-group:pg-server-generation Should PG Client StdErr Inline Contain ... ${CURDIR} ... ${PSQL_EXE} diff --git a/test/robot/functional/stackql_sessions.robot b/test/robot/functional/stackql_sessions.robot index 39fcf552..102d0e7d 100644 --- a/test/robot/functional/stackql_sessions.robot +++ b/test/robot/functional/stackql_sessions.robot @@ -97,6 +97,7 @@ Shell Session Invalid Registry Config Exits Cleanly [Teardown] NONE PG Session GC Manual Behaviour Canonical + [Tags] shard-group:pg-server-generation Should PG Client Session Inline Equal Strict ... ${PSQL_MTLS_CONN_STR_UNIX} ... ${SHELL_COMMANDS_GC_SEQUENCE_CANONICAL}