Skip to content

Commit e24a55c

Browse files
shuyanglishuyangli
authored
Add config-in-database live-tests workflow and config editing e2e tests (#7231)
1 parent 1a4ebe6 commit e24a55c

28 files changed

Lines changed: 3352 additions & 1 deletion

‎.github/workflows/general.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1407,6 +1407,23 @@ jobs:
14071407
uses: ./.github/workflows/live-tests.yml
14081408
secrets: inherit
14091409

1410+
live-tests-config-in-database:
1411+
if: ${{ needs.detect-changes.outputs.live-tests == 'true' && (github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) }}
1412+
needs:
1413+
[
1414+
detect-changes,
1415+
build-gateway-container,
1416+
build-gateway-e2e-container,
1417+
build-fixtures-container,
1418+
build-provider-proxy-container,
1419+
build-live-tests-container,
1420+
]
1421+
permissions:
1422+
contents: read
1423+
actions: read
1424+
uses: ./.github/workflows/live-tests-config-in-database.yml
1425+
secrets: inherit
1426+
14101427
inference-cache-tests:
14111428
if: ${{ needs.detect-changes.outputs.code == 'true' && (github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) }}
14121429
needs:
@@ -1547,6 +1564,7 @@ jobs:
15471564
inference-cache-tests,
15481565
lint-rust,
15491566
live-tests,
1567+
live-tests-config-in-database,
15501568
k3d,
15511569
mock-optimization-tests,
15521570
mocked-batch-tests,

‎.github/workflows/live-tests-config-in-database.yml‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
name: Live Tests (Config in Database)
2+
3+
on:
4+
workflow_call:
5+
6+
env:
7+
CARGO_INCREMENTAL: 0
8+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
9+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
10+
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
11+
AWS_REGION: "us-east-1"
12+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
13+
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY_NEW }}
14+
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
15+
FIREWORKS_ACCOUNT_ID: ${{ secrets.FIREWORKS_ACCOUNT_ID }}
16+
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
17+
FORCE_COLOR: 1
18+
GCP_STORAGE_ACCESS_KEY_ID: ${{ secrets.GCP_STORAGE_ACCESS_KEY_ID }}
19+
GCP_STORAGE_SECRET_ACCESS_KEY: ${{ secrets.GCP_STORAGE_SECRET_ACCESS_KEY }}
20+
GCP_VERTEX_CREDENTIALS_PATH: ${{ github.workspace }}/gcp_jwt_key.json
21+
GOOGLE_AI_STUDIO_API_KEY: ${{ secrets.GOOGLE_AI_STUDIO_API_KEY }}
22+
GOOGLE_APPLICATION_CREDENTIALS: ${{ github.workspace }}/gcp_jwt_key.json
23+
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
24+
HYPERBOLIC_API_KEY: ${{secrets.HYPERBOLIC_API_KEY}}
25+
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
26+
MODAL_KEY: ${{ secrets.MODAL_KEY }}
27+
MODAL_SECRET: ${{ secrets.MODAL_SECRET }}
28+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
29+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
30+
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
31+
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
32+
SGLANG_API_KEY: ${{ secrets.SGLANG_API_KEY }}
33+
TGI_API_KEY: ${{ secrets.TGI_API_KEY }}
34+
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
35+
VLLM_API_BASE: ${{ secrets.VLLM_API_BASE }}
36+
VLLM_API_KEY: ${{ secrets.VLLM_API_KEY }}
37+
VLLM_MODEL_NAME: "microsoft/Phi-3.5-mini-instruct"
38+
VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
39+
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
40+
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: https://localhost:4316
41+
SQLX_OFFLINE: 1
42+
TENSORZERO_E2E_PROXY: http://localhost:3003
43+
TENSORZERO_COMMIT_TAG: sha-${{ github.sha }}
44+
TENSORZERO_GATEWAY_TAG: sha-${{ github.sha }}
45+
TENSORZERO_MOCK_PROVIDER_API_TAG: sha-${{ github.sha }}
46+
TENSORZERO_CI: 1
47+
CARGO_NEXTEST_FLAKY_TESTS: ${{ vars.CARGO_NEXTEST_FLAKY_TESTS }}
48+
# Force Postgres as the primary datastore — the config-in-database feature
49+
# only supports Postgres.
50+
TENSORZERO_INTERNAL_TEST_OBSERVABILITY_BACKEND: postgres
51+
# Two compose files are merged: the standard live setup plus the override
52+
# that swaps the gateway onto the config-in-database boot path.
53+
COMPOSE_FILES: "-f crates/tensorzero-core/tests/e2e/docker-compose.live.yml -f crates/tensorzero-core/tests/e2e/docker-compose.config-in-db.yml"
54+
55+
jobs:
56+
live-tests-config-in-database:
57+
if: github.repository == 'tensorzero/tensorzero'
58+
name: "live-tests-config-in-database"
59+
runs-on: ubuntu-latest
60+
permissions:
61+
contents: read
62+
actions: read
63+
timeout-minutes: 45
64+
65+
steps:
66+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
67+
68+
- name: Cleanup disk space
69+
run: bash ci/free-disk-space.sh
70+
71+
- name: Login to DockerHub
72+
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772
73+
with:
74+
username: tensorzero
75+
password: ${{ secrets.DOCKERHUB_LIMITED_TOKEN }}
76+
77+
- name: Pull container images
78+
run: |
79+
./ci/docker-pull-with-retry.sh \
80+
tensorzero/gateway-dev:sha-${{ github.sha }} \
81+
tensorzero/provider-proxy:sha-${{ github.sha }} \
82+
tensorzero/gateway-e2e:sha-${{ github.sha }} \
83+
tensorzero/live-tests:sha-${{ github.sha }} \
84+
tensorzero/fixtures:sha-${{ github.sha }}
85+
docker tag tensorzero/gateway-dev:sha-${{ github.sha }} tensorzero/gateway:sha-${{ github.sha }}
86+
docker tag tensorzero/fixtures:sha-${{ github.sha }} tensorzero/fixtures-postgres:sha-${{ github.sha }}
87+
88+
- name: Restore provider-proxy cache
89+
if: github.event_name != 'schedule'
90+
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684
91+
with:
92+
path: ./ci/provider-proxy-cache/
93+
key: provider-proxy-cache-${{ github.run_id }}
94+
restore-keys: provider-proxy-cache-
95+
96+
- name: Download provider-proxy cache
97+
if: github.event_name != 'schedule'
98+
run: |
99+
AWS_ACCESS_KEY_ID=$R2_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$R2_SECRET_ACCESS_KEY PROVIDER_PROXY_CACHE_BUCKET=provider-proxy-cache ./ci/download-provider-proxy-cache.sh
100+
101+
- name: Write GCP JWT key to file
102+
env:
103+
GCP_JWT_KEY: ${{ secrets.GCP_JWT_KEY }}
104+
run: echo "$GCP_JWT_KEY" > $GITHUB_WORKSPACE/gcp_jwt_key.json
105+
106+
- name: Pull images referenced by the compose files
107+
run: docker compose ${{ env.COMPOSE_FILES }} --profile provider-proxy pull --ignore-pull-failures
108+
109+
- name: Run live tests against config-in-database gateway
110+
run: |
111+
DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose ${{ env.COMPOSE_FILES }} --profile provider-proxy run --rm \
112+
-e TENSORZERO_CI=1 \
113+
-e TENSORZERO_CLICKHOUSE_BATCH_WRITES=false \
114+
-e TENSORZERO_INTERNAL_TEST_OBSERVABILITY_BACKEND=postgres \
115+
live-tests
116+
117+
- name: Run config editing e2e tests
118+
# The config-editing profile filters down to tests that mutate
119+
# gateway-wide config via `/internal/config_toml/apply`. They run
120+
# after the main live suite so they don't race with concurrent tests.
121+
run: |
122+
DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose ${{ env.COMPOSE_FILES }} --profile provider-proxy run --rm \
123+
-e TENSORZERO_CI=1 \
124+
-e TENSORZERO_INTERNAL_TEST_OBSERVABILITY_BACKEND=postgres \
125+
live-tests \
126+
sh -c "cargo-nextest nextest run --no-fail-fast --archive-file tests.tar.zst --workspace-remap /app/crates --config-file /app/crates/.config/nextest.toml --profile config-editing -j 1"
127+
128+
- name: Print gateway and test logs
129+
if: always()
130+
run: docker compose ${{ env.COMPOSE_FILES }} --profile provider-proxy logs -t
131+
132+
- name: Check e2e logs for impossible error messages
133+
run: |
134+
LOGS=$(docker compose ${{ env.COMPOSE_FILES }} --profile provider-proxy logs gateway)
135+
if [ -z "$LOGS" ]; then
136+
echo "ERROR: Gateway logs are empty"
137+
exit 1
138+
fi
139+
grep --invert-match -i "please file a bug report" <<< "$LOGS"
140+
141+
- name: Check gateway loaded config from database
142+
run: |
143+
LOGS=$(docker compose ${{ env.COMPOSE_FILES }} --profile provider-proxy logs gateway)
144+
if [ -z "$LOGS" ]; then
145+
echo "ERROR: Gateway logs are empty"
146+
exit 1
147+
fi
148+
# When ENABLE_CONFIG_IN_DATABASE is on and no --config-file is passed,
149+
# main.rs prints a "Configuration: database" banner line. Fail fast
150+
# if that path wasn't exercised (means the override wasn't applied).
151+
grep -q "Configuration: database" <<< "$LOGS" || {
152+
echo "ERROR: Gateway does not appear to have loaded its config from the database"
153+
exit 1
154+
}
155+
156+
- name: Upload provider-proxy cache
157+
if: github.event_name == 'merge_group' || github.event_name == 'schedule'
158+
run: |
159+
AWS_ACCESS_KEY_ID=$R2_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$R2_SECRET_ACCESS_KEY PROVIDER_PROXY_CACHE_BUCKET=provider-proxy-cache ./ci/upload-provider-proxy-cache.sh

‎ci/check-all-general-jobs-passed.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ ALLOWED_SKIP=(
4545
"Endpoint tests"
4646
"lint-rust"
4747
"live-tests"
48+
"live-tests-config-in-database"
4849
"minikube"
4950
"mock-optimization-tests"
5051
"mocked-batch-tests"

‎crates/.config/nextest.toml‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,28 @@ slow-timeout = { period = "20s", terminate-after = 5 }
3131
# - optimization mock tests run separately in mock-optimization-tests with the mock gateway config
3232
# - binding generation
3333
# - evaluation tests run separately with a database matrix in the evaluation-tests CI job
34+
# - config editing tests run separately in the live-tests-config-in-database CI job because
35+
# they mutate gateway-wide config state
3436
[profile.e2e]
3537
retries = { backoff = "exponential", count = 4, delay = "5s", jitter = true, max-delay = "60s" }
3638
slow-timeout = { period = "30s", terminate-after = 2 }
3739
default-filter = '''
38-
not (test(batch) | test(test_dummy_only) | test(clickhouse::) | (test(db::) & !test(postgres) & !test(valkey)) | test(export_bindings_) | test(export_schema_) | binary(optimization-mock) | package(tensorzero-derive) | package(evaluations))
40+
not (test(batch) | test(test_dummy_only) | test(clickhouse::) | (test(db::) & !test(postgres) & !test(valkey)) | test(export_bindings_) | test(export_schema_) | binary(optimization-mock) | package(tensorzero-derive) | package(evaluations) | test(config_editing::))
3941
'''
4042
junit.path = "junit.xml"
4143

44+
# Run only the config editing e2e tests. These mutate gateway-wide config state
45+
# via `/internal/config_toml/apply`, so they must not race with the main e2e
46+
# suite and run after it finishes in the live-tests-config-in-database workflow.
47+
[profile.config-editing]
48+
default-filter = 'binary(e2e) and test(config_editing::)'
49+
# These tests all mutate gateway-wide config state, so they must run one at a
50+
# time even within the profile.
51+
test-threads = 1
52+
retries = { backoff = "exponential", count = 2, delay = "5s", jitter = true, max-delay = "30s" }
53+
slow-timeout = { period = "30s", terminate-after = 2 }
54+
junit.path = "junit.xml"
55+
4256
[profile.live-batch]
4357
default-filter = 'test(batch) and not test(mock) and not test(postgres)'
4458

‎crates/tensorzero-core/fixtures/config/functions/basic_test/prompt/simple_system_template.minijinja‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
You are a helpful and friendly assistant named {{ assistant_name }}

‎crates/tensorzero-core/tests/e2e/config-in-db/cache-clickhouse.gateway.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../config/cache-clickhouse.gateway.toml

‎crates/tensorzero-core/tests/e2e/config-in-db/cache-valkey.gateway.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../config/cache-valkey.gateway.toml

‎crates/tensorzero-core/tests/e2e/config-in-db/postgres.gateway.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../config/postgres.gateway.toml

‎crates/tensorzero-core/tests/e2e/config-in-db/tensorzero.evaluations.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../config/tensorzero.evaluations.toml

‎crates/tensorzero-core/tests/e2e/config-in-db/tensorzero.functions.anthropic_beta.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../config/tensorzero.functions.anthropic_beta.toml

0 commit comments

Comments
 (0)