Skip to content

fix(ai): Gate response tool calls on the gen_ai inputs setting #7174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ericapisani wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix(ai): Gate response tool calls on the gen_ai inputs setting #7174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions sentry_sdk/integrations/anthropic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -608,14 +608,16 @@ def _set_output_data(
set_on_span(SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS, [finish_reason])

client = sentry_sdk.get_client()
record_inputs = False
record_outputs = False
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"]["outputs"]:
record_outputs = True
record_inputs = client.options["data_collection"]["gen_ai"]["inputs"]
record_outputs = client.options["data_collection"]["gen_ai"]["outputs"]
elif should_send_default_pii() and integration.include_prompts:
record_inputs = True
record_outputs = True

if record_outputs:
if record_inputs or record_outputs:
output_messages: "dict[str, list[Any]]" = {
"response": [],
"tool": [],
Expand All @@ -627,15 +629,15 @@ def _set_output_data(
elif output["type"] == "tool_use":
output_messages["tool"].append(output)

if len(output_messages["tool"]) > 0:
if record_inputs and len(output_messages["tool"]) > 0:
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
output_messages["tool"],
unpack=False,
)

if len(output_messages["response"]) > 0:
if record_outputs and len(output_messages["response"]) > 0:
set_data_normalized(
span, SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"]
)
Expand Down
2 changes: 1 addition & 1 deletion sentry_sdk/integrations/google_genai/streaming.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ def set_span_data_for_streaming_response(

if accumulated_response.get("tool_calls"):
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"]["outputs"]:
if client.options["data_collection"]["gen_ai"]["inputs"]:
set_on_span(
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
safe_serialize(accumulated_response["tool_calls"]),
Expand Down
2 changes: 1 addition & 1 deletion sentry_sdk/integrations/google_genai/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1034,7 +1034,7 @@ def set_span_data_for_response(
tool_calls = extract_tool_calls(response)
if tool_calls:
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"]["outputs"]:
if client.options["data_collection"]["gen_ai"]["inputs"]:
set_on_span(
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, safe_serialize(tool_calls)
)
Expand Down
9 changes: 6 additions & 3 deletions sentry_sdk/integrations/openai.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,10 @@ def _set_common_output_data(
}

if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"]["outputs"]:
record_inputs = client.options["data_collection"]["gen_ai"]["inputs"]
record_outputs = client.options["data_collection"]["gen_ai"]["outputs"]

if record_inputs or record_outputs:
for output in response.output:
if output.type == "function_call":
output_messages["tool"].append(output.dict())
Expand All @@ -726,15 +729,15 @@ def _set_common_output_data(
output_message.dict()
)

if len(output_messages["tool"]) > 0:
if record_inputs and len(output_messages["tool"]) > 0:
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
output_messages["tool"],
unpack=False,
)

if len(output_messages["response"]) > 0:
if record_outputs and len(output_messages["response"]) > 0:
set_data_normalized(
span, SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"]
)
Expand Down
16 changes: 11 additions & 5 deletions sentry_sdk/integrations/openai_agents/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,10 +209,16 @@ def _set_output_data(
span: "Union[sentry_sdk.tracing.Span, StreamedSpan]", result: "Any"
) -> None:
client = sentry_sdk.get_client()
record_inputs = False
record_outputs = False
if has_data_collection_enabled(client.options):
if not client.options["data_collection"]["gen_ai"]["outputs"]:
return
elif not should_send_default_pii():
record_inputs = client.options["data_collection"]["gen_ai"]["inputs"]
record_outputs = client.options["data_collection"]["gen_ai"]["outputs"]
elif should_send_default_pii():
record_inputs = True
record_outputs = True

if not record_inputs and not record_outputs:
return

output_messages: "dict[str, list[Any]]" = {
Expand All @@ -231,7 +237,7 @@ def _set_output_data(
# Unknown output message type, just return the json
output_messages["response"].append(output_message.dict())

if len(output_messages["tool"]) > 0:
if record_inputs and len(output_messages["tool"]) > 0:
if isinstance(span, StreamedSpan):
span.set_attribute(
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
Expand All @@ -243,7 +249,7 @@ def _set_output_data(
safe_serialize(output_messages["tool"]),
)

if len(output_messages["response"]) > 0:
if record_outputs and len(output_messages["response"]) > 0:
set_data_normalized(
span, SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"]
)
66 changes: 58 additions & 8 deletions tests/integrations/anthropic/test_anthropic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -639,41 +639,62 @@ async def test_nonstreaming_create_message_data_collection_async(
@pytest.mark.parametrize("span_streaming", [True, False])
@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
@pytest.mark.parametrize(
"data_collection,send_default_pii,include_prompts,outputs_collected",
"data_collection,send_default_pii,include_prompts,outputs_collected,tool_calls_collected",
[
pytest.param(
{"gen_ai": {"outputs": True}},
False,
False,
True,
True,
id="gen-ai-outputs-enabled-overrides-pii-and-include-prompts",
),
pytest.param(
{"gen_ai": {"outputs": False}},
True,
True,
False,
id="gen-ai-outputs-disabled-overrides-pii-and-include-prompts",
True,
id="gen-ai-outputs-disabled-still-collects-tool-calls-gated-on-inputs",
),
pytest.param(
{"gen_ai": {"inputs": False}},
True,
True,
True,
False,
id="gen-ai-inputs-disabled-drops-tool-calls-only",
),
pytest.param(
{"gen_ai": {"inputs": False, "outputs": False}},
True,
True,
False,
False,
id="gen-ai-inputs-and-outputs-disabled-overrides-pii-and-include-prompts",
),
pytest.param(
{"gen_ai": {}},
False,
False,
True,
id="gen-ai-outputs-omitted-defaults-to-enabled",
True,
id="gen-ai-inputs-and-outputs-omitted-defaults-to-enabled",
),
pytest.param(
None,
True,
True,
True,
True,
id="legacy-pii-and-include-prompts-enabled",
),
pytest.param(
None,
False,
True,
False,
False,
id="legacy-pii-disabled",
),
],
Expand All @@ -686,6 +707,7 @@ def test_nonstreaming_create_message_data_collection_outputs(
send_default_pii,
include_prompts,
outputs_collected,
tool_calls_collected,
stream_gen_ai_spans,
span_streaming,
):
Expand Down Expand Up @@ -743,12 +765,15 @@ def test_nonstreaming_create_message_data_collection_outputs(
span_data[SPANDATA.GEN_AI_RESPONSE_TEXT]
== DATA_COLLECTION_EXPECTED_RESPONSE_TEXT
)
else:
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data

if tool_calls_collected:
assert (
json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS])
== DATA_COLLECTION_EXPECTED_TOOL_CALLS
)
else:
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data


Expand All @@ -760,41 +785,62 @@ def test_nonstreaming_create_message_data_collection_outputs(
@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
@pytest.mark.asyncio
@pytest.mark.parametrize(
"data_collection,send_default_pii,include_prompts,outputs_collected",
"data_collection,send_default_pii,include_prompts,outputs_collected,tool_calls_collected",
[
pytest.param(
{"gen_ai": {"outputs": True}},
False,
False,
True,
True,
id="gen-ai-outputs-enabled-overrides-pii-and-include-prompts",
),
pytest.param(
{"gen_ai": {"outputs": False}},
True,
True,
False,
id="gen-ai-outputs-disabled-overrides-pii-and-include-prompts",
True,
id="gen-ai-outputs-disabled-still-collects-tool-calls-gated-on-inputs",
),
pytest.param(
{"gen_ai": {"inputs": False}},
True,
True,
True,
False,
id="gen-ai-inputs-disabled-drops-tool-calls-only",
),
pytest.param(
{"gen_ai": {"inputs": False, "outputs": False}},
True,
True,
False,
False,
id="gen-ai-inputs-and-outputs-disabled-overrides-pii-and-include-prompts",
),
pytest.param(
{"gen_ai": {}},
False,
False,
True,
id="gen-ai-outputs-omitted-defaults-to-enabled",
True,
id="gen-ai-inputs-and-outputs-omitted-defaults-to-enabled",
),
pytest.param(
None,
True,
True,
True,
True,
id="legacy-pii-and-include-prompts-enabled",
),
pytest.param(
None,
False,
True,
False,
False,
id="legacy-pii-disabled",
),
],
Expand All @@ -807,6 +853,7 @@ async def test_nonstreaming_create_message_data_collection_outputs_async(
send_default_pii,
include_prompts,
outputs_collected,
tool_calls_collected,
stream_gen_ai_spans,
span_streaming,
):
Expand Down Expand Up @@ -864,12 +911,15 @@ async def test_nonstreaming_create_message_data_collection_outputs_async(
span_data[SPANDATA.GEN_AI_RESPONSE_TEXT]
== DATA_COLLECTION_EXPECTED_RESPONSE_TEXT
)
else:
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data

if tool_calls_collected:
assert (
json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS])
== DATA_COLLECTION_EXPECTED_TOOL_CALLS
)
else:
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data


Expand Down
20 changes: 18 additions & 2 deletions tests/integrations/google_genai/test_google_genai.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3843,11 +3843,19 @@ def test_generate_content_data_collection(
{"gen_ai": {"inputs": True, "outputs": False}},
[
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
],
[],
id="gen-ai-inputs-enabled-outputs-disabled-tools-collected",
),
pytest.param(
{"gen_ai": {"inputs": False, "outputs": True}},
[],
[
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
],
id="gen-ai-inputs-enabled-outputs-disabled-available-tools-only",
id="gen-ai-inputs-disabled-outputs-enabled-tools-not-collected",
),
pytest.param(
{"gen_ai": {}},
Expand Down Expand Up @@ -4195,11 +4203,19 @@ def test_streaming_generate_content_data_collection(
{"gen_ai": {"inputs": True, "outputs": False}},
[
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
],
[],
id="gen-ai-inputs-enabled-outputs-disabled-tools-collected",
),
pytest.param(
{"gen_ai": {"inputs": False, "outputs": True}},
[],
[
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
],
id="gen-ai-inputs-enabled-outputs-disabled-available-tools-only",
id="gen-ai-inputs-disabled-outputs-enabled-tools-not-collected",
),
pytest.param(
{"gen_ai": {}},
Expand Down
Loading
Loading