Skip to content

Make e2e tests deterministic for provider-proxy cache #7248

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
AntoineToussaint wants to merge 6 commits into from
Closed

Make e2e tests deterministic for provider-proxy cache #7248

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9a6feab
Make e2e tests deterministic for provider-proxy cache
AntoineToussaint Apr 8, 2026
02a5a18
Revert embeddings cache tests to use random inputs
AntoineToussaint Apr 8, 2026
98afdd0
Fix embeddings cache tests: use write_only instead of random inputs
AntoineToussaint Apr 9, 2026
39952f2
Fix image URL tests and bypass raw.githubusercontent.com
AntoineToussaint Apr 9, 2026
0fbe2ab
Fix embeddings cache test and image server port conflicts
AntoineToussaint Apr 9, 2026
9d94b58
Revert cache test changes to split into a separate PR
AntoineToussaint Apr 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Make e2e tests deterministic for provider-proxy cache 
Replace random values (UUIDs, random integers) in LLM provider request
bodies with fixed strings. This makes provider-proxy cache keys
deterministic across test runs, enabling read-only cache mode in CI.

Changes:
- providers/common.rs: Remove UUID from bad auth test prompt
- providers/anthropic.rs: Remove UUID from thinking test prompt
- aggregated_response/mod.rs: Remove UUID suffix from prompt
- otel.rs: Use fixed string for tag value
- raw_response/embeddings.rs: Use fixed string for embedding input
- test_client.py: Remove UUID from extra headers test prompt
- test_embeddings.py: Use fixed strings for cache test inputs

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
  • Loading branch information
@AntoineToussaint @claude
AntoineToussaint and claude committed Apr 8, 2026
commit 9a6feab46ea35ee2a39de7ec4ae18be5ab24c70b
3 changes: 1 addition & 2 deletions crates/tensorzero-core/tests/e2e/aggregated_response/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ use crate::common::get_gateway_endpoint;
#[tokio::test]
async fn test_aggregated_response_chat_streaming() {
let episode_id = Uuid::now_v7();
let random_suffix = Uuid::now_v7();

let payload = json!({
"function_name": "basic_test",
Expand All @@ -26,7 +25,7 @@ async fn test_aggregated_response_chat_streaming() {
"messages": [
{
"role": "user",
"content": format!("Say hello in one sentence. {random_suffix}")
"content": "Say hello in one sentence. deterministic-cache-test"
}
]
},
Expand Down
2 changes: 1 addition & 1 deletion crates/tensorzero-core/tests/e2e/otel.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -943,7 +943,7 @@ pub fn test_capture_model_error(mode: OtlpTracesFormat, config_mode: &str) {
tags: HashMap::from([
("first_tag".to_string(), "first_value".into()),
("second_tag".to_string(), "second_value".into()),
("user_id".to_string(), Uuid::now_v7().to_string()),
Copy link
Copy Markdown
Member

@Aaron1011 Aaron1011 Apr 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this should be needed, since the tags don't get included in model inference requests

("user_id".to_string(), "deterministic-cache-test".into()),
]),
..Default::default()
})
Expand Down
3 changes: 1 addition & 2 deletions crates/tensorzero-core/tests/e2e/providers/anthropic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,14 +202,13 @@ async fn test_thinking_rejected_128k() {
// does not cache. Therefore, this test will always hit live providers.
// Consider moving to periodic live tests if this causes merge queue instability.
// See: https://github.com/tensorzero/tensorzero/issues/5380
let random = Uuid::now_v7();
let payload = json!({
"model_name": "anthropic::claude-sonnet-4-5",
"input":{
"messages": [
{
"role": "user",
"content": format!("Output a haiku that ends in the word 'my_custom_stop': {random}"),
"content": "Output a haiku that ends in the word 'my_custom_stop': deterministic-cache-test",
}
]},
"params": {
Expand Down
2 changes: 1 addition & 1 deletion crates/tensorzero-core/tests/e2e/providers/common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2405,7 +2405,7 @@ pub async fn test_bad_auth_extra_headers_with_provider_and_stream(
"messages": [
{
"role": "user",
"content": format!("If you see this, something has gone wrong - the request should have failed: {}", Uuid::now_v7())
"content": "If you see this, something has gone wrong - the request should have failed: deterministic-cache-test"
}
]},
"stream": stream,
Expand Down
5 changes: 1 addition & 4 deletions crates/tensorzero-core/tests/e2e/raw_response/embeddings.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,7 @@ async fn test_embeddings_raw_response_batch() {

#[tokio::test]
async fn test_embeddings_raw_response_with_cache() {
let input_text = format!(
"This is a cache test for embeddings raw_response - {}",
rand::random::<u32>()
);
let input_text = "This is a cache test for embeddings raw_response - deterministic-cache-test";

// First request: populate cache with raw_response enabled
let payload = json!({
Expand Down
3 changes: 1 addition & 2 deletions crates/tensorzero-python/tests/test_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2413,13 +2413,12 @@ def test_prepare_inference_request(sync_client: TensorZeroGateway):

def test_extra_headers_raw(sync_client: TensorZeroGateway):
with pytest.raises(TensorZeroError) as exc_info:
id = uuid7()
sync_client.inference(
function_name="basic_test",
variant_name="openai",
input={
"system": {"assistant_name": "Alfred Pennyworth"},
"messages": [{"role": "user", "content": f"Write me a haiku {id}"}],
"messages": [{"role": "user", "content": "Write me a haiku about extra headers"}],
},
extra_headers=[
{
Expand Down
7 changes: 2 additions & 5 deletions crates/tensorzero-python/tests/test_embeddings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
"""

import asyncio
import random

import pytest

Expand Down Expand Up @@ -217,8 +216,7 @@ async def test_embeddings_consistency(async_openai_client):
@pytest.mark.asyncio
async def test_embeddings_cache_with_float_encoding(async_openai_client):
"""Test that caching works correctly with float encoding format"""
# Use a unique input to ensure we're not hitting existing cache
input_text = f"Cache test with float encoding - {random.randint(0, 1000000)}"
input_text = "Cache test with float encoding - deterministic"

# First request with float encoding and cache enabled
result1 = await async_openai_client.embeddings.create(
Expand Down Expand Up @@ -254,8 +252,7 @@ async def test_embeddings_cache_with_float_encoding(async_openai_client):
@pytest.mark.asyncio
async def test_embeddings_cache_with_base64_encoding(async_openai_client):
"""Test that caching works correctly with base64 encoding format"""
# Use a unique input to ensure we're not hitting existing cache
input_text = f"Cache test with base64 encoding - {random.randint(0, 1000000)}"
input_text = "Cache test with base64 encoding - deterministic"

# First request with base64 encoding and cache enabled
result1 = await async_openai_client.embeddings.create(
Expand Down
Loading