Repository navigation
Expand file tree
/
Copy pathtest_observability.py
More file actions
136 lines (99 loc) · 5.33 KB
/
Copy pathtest_observability.py
File metadata and controls
136 lines (99 loc) · 5.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
import time
import pytest
from buffdata.observability import ObservabilityRegistry, disabled_registry
# --- disabled by default / opt-out ---------------------------------------------------------
def test_disabled_registry_every_method_is_a_real_noop():
obs = ObservabilityRegistry(enabled=False)
assert obs.enabled is False
assert obs.registry is None
with obs.stage_span("validate"):
pass # must not raise
obs.record_stage_outcome("validate", accepted=10, rejected=2)
obs.record_llm_error("gemini")
obs.record_tokens("gemini", "gemini-3.7-flash", input_tokens=100, output_tokens=50)
assert obs.export_prometheus_text() == ""
def test_disabled_registry_helper_is_shared_and_disabled():
a = disabled_registry()
b = disabled_registry()
assert a is b
assert a.enabled is False
# --- enabled: Prometheus metrics ------------------------------------------------------------
def test_enabled_registry_reports_both_backends_available():
obs = ObservabilityRegistry(enabled=True)
# Both opentelemetry-sdk and prometheus_client are installed in this environment.
assert obs.prometheus_enabled is True
assert obs.otel_enabled is True
def test_stage_span_records_duration_histogram():
obs = ObservabilityRegistry(enabled=True)
with obs.stage_span("validate"):
time.sleep(0.01)
text = obs.export_prometheus_text()
assert 'buffdata_stage_duration_seconds_count{stage="validate"} 1.0' in text
assert "buffdata_stage_duration_seconds_sum" in text
def test_stage_span_records_even_when_body_raises():
obs = ObservabilityRegistry(enabled=True)
with pytest.raises(ValueError):
with obs.stage_span("dedup"):
raise ValueError("boom")
text = obs.export_prometheus_text()
assert 'buffdata_stage_duration_seconds_count{stage="dedup"} 1.0' in text
def test_record_stage_outcome_increments_correct_labels():
obs = ObservabilityRegistry(enabled=True)
obs.record_stage_outcome("validate", accepted=8, rejected=2)
obs.record_stage_outcome("validate", accepted=3, rejected=0)
text = obs.export_prometheus_text()
assert 'buffdata_records_total{outcome="accepted",stage="validate"} 11.0' in text
assert 'buffdata_records_total{outcome="rejected",stage="validate"} 2.0' in text
def test_record_stage_outcome_skips_zero_values_without_erroring():
obs = ObservabilityRegistry(enabled=True)
obs.record_stage_outcome("filter", accepted=0, rejected=0)
# No exception, and no bogus zero-valued series forced into existence for labels that
# never actually occurred.
text = obs.export_prometheus_text()
assert 'stage="filter"' not in text
def test_record_llm_error_increments_by_provider():
obs = ObservabilityRegistry(enabled=True)
obs.record_llm_error("gemini")
obs.record_llm_error("gemini")
obs.record_llm_error("anthropic")
text = obs.export_prometheus_text()
assert 'buffdata_llm_errors_total{provider="gemini"} 2.0' in text
assert 'buffdata_llm_errors_total{provider="anthropic"} 1.0' in text
def test_record_tokens_splits_by_direction():
obs = ObservabilityRegistry(enabled=True)
obs.record_tokens("gemini", "gemini-3.7-flash", input_tokens=100, output_tokens=40)
text = obs.export_prometheus_text()
assert 'buffdata_tokens_total{direction="input",model="gemini-3.7-flash",provider="gemini"} 100.0' in text
assert 'buffdata_tokens_total{direction="output",model="gemini-3.7-flash",provider="gemini"} 40.0' in text
def test_two_registries_do_not_collide():
# Each ObservabilityRegistry must own its own CollectorRegistry -- using Prometheus's
# shared global default registry would make a second pipeline run (or a second test)
# raise "Duplicated timeseries" on construction.
first = ObservabilityRegistry(enabled=True)
second = ObservabilityRegistry(enabled=True) # must not raise
first.record_llm_error("gemini")
second.record_llm_error("openai")
assert "gemini" in first.export_prometheus_text()
assert "gemini" not in second.export_prometheus_text()
assert "openai" in second.export_prometheus_text()
# --- enabled: OpenTelemetry spans ------------------------------------------------------------
def test_stage_span_creates_a_real_otel_span_when_a_provider_is_configured():
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
# OpenTelemetry's TracerProvider can only be set once per process (later calls are a
# documented no-op), so this test relies on being the only one in the suite that sets
# one -- get_tracer("buffdata") is called *after* this, and correctly resolves against
# whatever the current global provider is via OTel's proxy-tracer mechanism, which is
# exactly what makes this work regardless of when a real SDK provider gets configured.
provider = TracerProvider()
exporter = InMemorySpanExporter()
provider.add_span_processor(SimpleSpanProcessor(exporter))
trace.set_tracer_provider(provider)
obs = ObservabilityRegistry(enabled=True)
with obs.stage_span("classify"):
pass
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "buffdata.stage.classify"