Skip to content

fix(langgraph,huggingface_hub): Gate gen_ai.response.tool_calls on outputs, not inputs #7208

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ericapisani wants to merge 2 commits into from
+35 −44
Open

fix(langgraph,huggingface_hub): Gate gen_ai.response.tool_calls on outputs, not inputs #7208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7b086bb
fix(langgraph): Gate `gen_ai.response.tool_calls` on outputs, not inputs
ericapisani Aug 14, 2026
6a7ebed
fix(huggingface_hub): Gate `gen_ai.response.tool_calls` on outputs, n…
ericapisani Aug 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions sentry_sdk/integrations/huggingface_hub.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ def new_huggingface_task(*args: "Any", **kwargs: "Any") -> "Any":

if tool_calls is not None and len(tool_calls) > 0:
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"]["inputs"]:
if client.options["data_collection"]["gen_ai"]["outputs"]:
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
Expand Down Expand Up @@ -402,7 +402,7 @@ def new_iterator() -> "Iterable[ChatCompletionStreamOutput]":
if tool_calls is not None and len(tool_calls) > 0:
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"][
"inputs"
"outputs"
]:
set_data_normalized(
span,
Expand Down
2 changes: 0 additions & 2 deletions sentry_sdk/integrations/langgraph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -507,8 +507,6 @@ def _set_response_attributes(
else:
set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result)

# Tool calls are an input to the model, so they're gated on inputs
if _should_record_inputs(integration):
tool_calls = _extract_tool_calls(new_messages)
if tool_calls:
set_data_normalized(
Expand Down
14 changes: 8 additions & 6 deletions tests/integrations/huggingface_hub/test_huggingface_hub.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1921,20 +1921,22 @@ def test_text_generation_streaming_data_collection(
False,
[
SPANDATA.GEN_AI_REQUEST_MESSAGES,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
],
[],
[
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
],
id="gen-ai-inputs-enabled-outputs-disabled",
),
pytest.param(
{"gen_ai": {"inputs": False, "outputs": True}},
False,
False,
[],
[
SPANDATA.GEN_AI_REQUEST_MESSAGES,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
],
[
SPANDATA.GEN_AI_REQUEST_MESSAGES,
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
],
id="gen-ai-outputs-enabled-inputs-disabled",
Expand Down Expand Up @@ -2090,11 +2092,11 @@ def test_chat_completion_data_collection_tools(
False,
[
SPANDATA.GEN_AI_REQUEST_MESSAGES,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
],
[
SPANDATA.GEN_AI_RESPONSE_TEXT,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
],
id="gen-ai-inputs-enabled-outputs-disabled",
),
Expand All @@ -2104,10 +2106,10 @@ def test_chat_completion_data_collection_tools(
False,
[
SPANDATA.GEN_AI_RESPONSE_TEXT,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
],
[
SPANDATA.GEN_AI_REQUEST_MESSAGES,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
],
id="gen-ai-outputs-enabled-inputs-disabled",
Expand Down
59 changes: 25 additions & 34 deletions tests/integrations/langgraph/test_langgraph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import asyncio
import json
import sys
from unittest.mock import MagicMock, patch

Expand Down Expand Up @@ -347,8 +348,6 @@ def original_invoke(self, *args, **kwargs):
]

if isinstance(request_messages, str):
import json

request_messages = json.loads(request_messages)
assert len(request_messages) == 2
assert request_messages[0]["content"] == "Hello, can you help me?"
Expand All @@ -363,8 +362,6 @@ def original_invoke(self, *args, **kwargs):
]

if isinstance(tool_calls_data, str):
import json

tool_calls_data = json.loads(tool_calls_data)

assert len(tool_calls_data) == 1
Expand Down Expand Up @@ -411,8 +408,6 @@ def original_invoke(self, *args, **kwargs):
request_messages = invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]

if isinstance(request_messages, str):
import json

request_messages = json.loads(request_messages)
assert len(request_messages) == 1
assert request_messages[0]["content"] == "Of course! How can I assist you?"
Expand All @@ -424,8 +419,6 @@ def original_invoke(self, *args, **kwargs):
tool_calls_data = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]

if isinstance(tool_calls_data, str):
import json

tool_calls_data = json.loads(tool_calls_data)

assert len(tool_calls_data) == 1
Expand Down Expand Up @@ -534,8 +527,6 @@ async def run_test():
]

if isinstance(tool_calls_data, str):
import json

tool_calls_data = json.loads(tool_calls_data)

assert len(tool_calls_data) == 1
Expand Down Expand Up @@ -583,8 +574,6 @@ async def run_test():
tool_calls_data = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]

if isinstance(tool_calls_data, str):
import json

tool_calls_data = json.loads(tool_calls_data)

assert len(tool_calls_data) == 1
Expand Down Expand Up @@ -1917,7 +1906,6 @@ def original_invoke(self, *args, **kwargs):
assert response_text == "Final response"

assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in invoke_span["attributes"]
import json

tool_calls_data = invoke_span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]
else:
Expand All @@ -1941,7 +1929,6 @@ def original_invoke(self, *args, **kwargs):
assert response_text == "Final response"

assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in invoke_span["data"]
import json

tool_calls_data = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]

Expand Down Expand Up @@ -2012,8 +1999,6 @@ def __init__(self, content, message_type="human"):

# If messages were captured, verify role mapping
if SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"]:
import json

stored_messages = json.loads(
span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]
)
Expand All @@ -2037,8 +2022,6 @@ def __init__(self, content, message_type="human"):

# If messages were captured, verify role mapping
if SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"]:
import json

stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES])

# Find messages with specific content to verify role mapping
Expand All @@ -2065,7 +2048,6 @@ def __init__(self, content, message_type="human"):

def test_langgraph_message_truncation(sentry_init, capture_events):
"""Test that large messages are truncated properly in Langgraph integration."""
import json

sentry_init(
integrations=[LanggraphIntegration(include_prompts=True)],
Expand Down Expand Up @@ -2196,7 +2178,7 @@ def _invoke_span_data(items_or_events, span_streaming):
),
],
)
def test_pregel_invoke_gates_request_messages_and_tool_calls_on_inputs_setting(
def test_pregel_invoke_gates_request_messages_on_inputs_setting(
sentry_init,
capture_events,
capture_items,
Expand All @@ -2219,13 +2201,6 @@ def test_pregel_invoke_gates_request_messages_and_tool_calls_on_inputs_setting(

test_state = {"messages": [MockMessage("Hello, can you help me?", name="user")]}
pregel = MockPregelInstance("test_graph")
expected_tool_calls = [
{
"id": "call_test_123",
"type": "function",
"function": {"name": "search_tool", "arguments": '{"query": "help"}'},
}
]

def original_invoke(self, *args, **kwargs):
return {
Expand All @@ -2234,7 +2209,6 @@ def original_invoke(self, *args, **kwargs):
MockMessage(
content="I'll help you with that task!",
name="assistant",
tool_calls=expected_tool_calls,
)
]
}
Expand All @@ -2249,10 +2223,8 @@ def original_invoke(self, *args, **kwargs):

if expect_inputs:
assert SPANDATA.GEN_AI_REQUEST_MESSAGES in data
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in data
else:
assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in data
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data


@pytest.mark.parametrize("span_streaming", [True, False])
Expand Down Expand Up @@ -2297,7 +2269,7 @@ def original_invoke(self, *args, **kwargs):
),
],
)
def test_pregel_invoke_gates_response_text_on_outputs_setting(
def test_pregel_invoke_gates_response_text_and_tool_calls_on_outputs_setting(
sentry_init,
capture_events,
capture_items,
Expand All @@ -2321,11 +2293,24 @@ def test_pregel_invoke_gates_response_text_on_outputs_setting(
test_state = {"messages": [MockMessage("Hello, can you help me?", name="user")]}
pregel = MockPregelInstance("test_graph")
expected_assistant_response = "I'll help you with that task!"
expected_tool_calls = [
{
"id": "call_test_123",
"type": "function",
"function": {"name": "search_tool", "arguments": '{"query": "help"}'},
}
]

def original_invoke(self, *args, **kwargs):
return {
"messages": args[0].get("messages", [])
+ [MockMessage(content=expected_assistant_response, name="assistant")]
+ [
MockMessage(
content=expected_assistant_response,
name="assistant",
tool_calls=expected_tool_calls,
)
]
}

captured = capture_items("span") if span_streaming else capture_events()
Expand All @@ -2338,8 +2323,12 @@ def original_invoke(self, *args, **kwargs):

if expect_outputs:
assert data[SPANDATA.GEN_AI_RESPONSE_TEXT] == expected_assistant_response
assert (
json.loads(data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == expected_tool_calls
)
else:
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in data
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data


@pytest.mark.parametrize("span_streaming", [True, False])
Expand Down Expand Up @@ -2434,15 +2423,17 @@ async def run_test():

if expect_inputs:
assert SPANDATA.GEN_AI_REQUEST_MESSAGES in data
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in data
else:
assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in data
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data

if expect_outputs:
assert data[SPANDATA.GEN_AI_RESPONSE_TEXT] == expected_assistant_response
assert (
json.loads(data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == expected_tool_calls
)
else:
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in data
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data


@pytest.mark.parametrize("span_streaming", [True, False])
Expand Down
Loading