Skip to content

feat(huggingface_hub): Gate prompt/response collection on data_collection option #7172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ericapisani wants to merge 5 commits into
base: master
Choose a base branch
from
+749 −22
Open

feat(huggingface_hub): Gate prompt/response collection on data_collection option #7172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2ec0d48
feat(huggingface_hub): Gate prompt/response collection on data_collec…
ericapisani Aug 11, 2026
4962948
test(huggingface_hub): Inline data collection parametrize cases
ericapisani Aug 11, 2026
225aa79
test(huggingface_hub): Inline data collection test helpers
ericapisani Aug 11, 2026
85f993b
test(huggingface_hub): Group data collection test constants together
ericapisani Aug 11, 2026
3755845
test(huggingface_hub): Inline single-use expected value dicts
ericapisani Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 84 additions & 22 deletions sentry_sdk/integrations/huggingface_hub.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from sentry_sdk.utils import (
capture_internal_exceptions,
event_from_exception,
has_data_collection_enabled,
reraise,
)

Expand Down Expand Up @@ -74,7 +75,8 @@
def _wrap_huggingface_task(f: "Callable[..., Any]", op: str) -> "Callable[..., Any]":
@wraps(f)
def new_huggingface_task(*args: "Any", **kwargs: "Any") -> "Any":
integration = sentry_sdk.get_client().get_integration(HuggingfaceHubIntegration)
client = sentry_sdk.get_client()
integration = client.get_integration(HuggingfaceHubIntegration)
if integration is None:
return f(*args, **kwargs)

Expand All @@ -91,12 +93,12 @@
# invalid call, dont instrument, let it return error
return f(*args, **kwargs)

client = args[0]
model = client.model or kwargs.get("model") or ""
hf_client = args[0]
model = hf_client.model or kwargs.get("model") or ""
operation_name = op.split(".")[-1]

span: "Union[Span, StreamedSpan]"
if has_span_streaming_enabled(sentry_sdk.get_client().options):
if has_span_streaming_enabled(client.options):
span = sentry_sdk.traces.start_span(
name=f"{operation_name} {model}",
attributes={
Expand All @@ -117,14 +119,7 @@
if model:
_set_span_data_attribute(span, SPANDATA.GEN_AI_REQUEST_MODEL, model)

# Input attributes
if should_send_default_pii() and integration.include_prompts:
set_data_normalized(
span, SPANDATA.GEN_AI_REQUEST_MESSAGES, prompt, unpack=False
)

attribute_mapping = {
"tools": SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
"frequency_penalty": SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY,
"max_tokens": SPANDATA.GEN_AI_REQUEST_MAX_TOKENS,
"presence_penalty": SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY,
Expand All @@ -134,6 +129,24 @@
"stream": SPANDATA.GEN_AI_RESPONSE_STREAMING,
}

if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"]["inputs"]:
attribute_mapping["tools"] = SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS
else:
# Legacy behaviour where we unconditionally set this. Remove when data collection is fully rolled out
attribute_mapping["tools"] = SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS

# Input attributes
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"]["inputs"]:
set_data_normalized(
span, SPANDATA.GEN_AI_REQUEST_MESSAGES, prompt, unpack=False
)
elif should_send_default_pii() and integration.include_prompts:
set_data_normalized(
span, SPANDATA.GEN_AI_REQUEST_MESSAGES, prompt, unpack=False
)

for attribute, span_attribute in attribute_mapping.items():
value = kwargs.get(attribute, None)
if value is not None:
Expand Down Expand Up @@ -209,19 +222,35 @@
SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS,
finish_reason,
)

if should_send_default_pii() and integration.include_prompts:
if tool_calls is not None and len(tool_calls) > 0:
if tool_calls is not None and len(tool_calls) > 0:
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"]["inputs"]:
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,

Check warning on line 231 in sentry_sdk/integrations/huggingface_hub.py

View check run for this annotation

@sentry/warden / warden: find-bugs

Response tool calls gated on data_collection gen_ai.inputs instead of outputs 

Response tool calls are gated on `gen_ai.inputs` instead of `gen_ai.outputs`, causing model-generated tool call data to be collected even when a user explicitly disables `outputs`.
tool_calls,
unpack=False,
)
elif should_send_default_pii() and integration.include_prompts:
Comment thread
ericapisani marked this conversation as resolved.
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
tool_calls,
unpack=False,
)

if len(response_text_buffer) > 0:
text_response = "".join(response_text_buffer)
if text_response:
if len(response_text_buffer) > 0:
text_response = "".join(response_text_buffer)
if text_response:
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"]["outputs"]:
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TEXT,
text_response,
)
elif should_send_default_pii() and integration.include_prompts:
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TEXT,
Expand Down Expand Up @@ -284,7 +313,14 @@
finish_reason,
)

if should_send_default_pii() and integration.include_prompts:
should_set_response_text = False
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"]["outputs"]:
should_set_response_text = True
elif should_send_default_pii() and integration.include_prompts:
should_set_response_text = True

if should_set_response_text:
if len(response_text_buffer) > 0:
text_response = "".join(response_text_buffer)
if text_response:
Expand Down Expand Up @@ -363,18 +399,44 @@
finish_reason,
)

if should_send_default_pii() and integration.include_prompts:
if tool_calls is not None and len(tool_calls) > 0:
if tool_calls is not None and len(tool_calls) > 0:
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"][
"inputs"
]:
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
tool_calls,
unpack=False,
)
elif (
should_send_default_pii()
and integration.include_prompts
):
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
tool_calls,
unpack=False,
)

if len(response_text_buffer) > 0:
text_response = "".join(response_text_buffer)
if text_response:
if len(response_text_buffer) > 0:
Comment thread
sentry[bot] marked this conversation as resolved.
text_response = "".join(response_text_buffer)
if text_response:
if has_data_collection_enabled(client.options):
if client.options["data_collection"]["gen_ai"][
"outputs"
]:
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TEXT,
text_response,
)
elif (
should_send_default_pii()
and integration.include_prompts
):
set_data_normalized(
span,
SPANDATA.GEN_AI_RESPONSE_TEXT,
Expand Down
Loading
Loading