Skip to content

Commit 797e82f

Browse files
ZylphrexZylphrex
authored
feat(profiling): Continuous profiling sample rate (#4002)
This introduces a new top level setting for the continuous profiling session sample rate. The sample rate is evaluated once at the beginning and is used to determine whether or not the profiler will be run for the remainder of the process.
1 parent bba389e commit 797e82f

3 files changed

Lines changed: 134 additions & 43 deletions

File tree

‎sentry_sdk/consts.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -528,6 +528,7 @@ def __init__(
528528
profiles_sample_rate=None, # type: Optional[float]
529529
profiles_sampler=None, # type: Optional[TracesSampler]
530530
profiler_mode=None, # type: Optional[ProfilerMode]
531+
profile_session_sample_rate=None, # type: Optional[float]
531532
auto_enabling_integrations=True, # type: bool
532533
disabled_integrations=None, # type: Optional[Sequence[sentry_sdk.integrations.Integration]]
533534
auto_session_tracking=True, # type: bool

‎sentry_sdk/profiler/continuous_profiler.py‎

Lines changed: 52 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import atexit
22
import os
3+
import random
34
import sys
45
import threading
56
import time
@@ -83,11 +84,15 @@ def setup_continuous_profiler(options, sdk_info, capture_func):
8384
else:
8485
default_profiler_mode = ThreadContinuousScheduler.mode
8586

86-
experiments = options.get("_experiments", {})
87+
if options.get("profiler_mode") is not None:
88+
profiler_mode = options["profiler_mode"]
89+
else:
90+
# TODO: deprecate this and just use the existing `profiler_mode`
91+
experiments = options.get("_experiments", {})
8792

88-
profiler_mode = (
89-
experiments.get("continuous_profiling_mode") or default_profiler_mode
90-
)
93+
profiler_mode = (
94+
experiments.get("continuous_profiling_mode") or default_profiler_mode
95+
)
9196

9297
frequency = DEFAULT_SAMPLING_FREQUENCY
9398

@@ -118,35 +123,26 @@ def try_autostart_continuous_profiler():
118123
if _scheduler is None:
119124
return
120125

121-
# Ensure that the scheduler only autostarts once per process.
122-
# This is necessary because many web servers use forks to spawn
123-
# additional processes. And the profiler is only spawned on the
124-
# master process, then it often only profiles the main process
125-
# and not the ones where the requests are being handled.
126-
#
127-
# Additionally, we only want this autostart behaviour once per
128-
# process. If the user explicitly calls `stop_profiler`, it should
129-
# be respected and not start the profiler again.
130-
if not _scheduler.should_autostart():
126+
if not _scheduler.is_auto_start_enabled():
131127
return
132128

133-
_scheduler.ensure_running()
129+
_scheduler.manual_start()
134130

135131

136132
def start_profiler():
137133
# type: () -> None
138134
if _scheduler is None:
139135
return
140136

141-
_scheduler.ensure_running()
137+
_scheduler.manual_start()
142138

143139

144140
def stop_profiler():
145141
# type: () -> None
146142
if _scheduler is None:
147143
return
148144

149-
_scheduler.teardown()
145+
_scheduler.manual_stop()
150146

151147

152148
def teardown_continuous_profiler():
@@ -164,6 +160,16 @@ def get_profiler_id():
164160
return _scheduler.profiler_id
165161

166162

163+
def determine_profile_session_sampling_decision(sample_rate):
164+
# type: (Union[float, None]) -> bool
165+
166+
# `None` is treated as `0.0`
167+
if not sample_rate:
168+
return False
169+
170+
return random.random() < float(sample_rate)
171+
172+
167173
class ContinuousScheduler:
168174
mode = "unknown" # type: ContinuousProfilerMode
169175

@@ -175,16 +181,43 @@ def __init__(self, frequency, options, sdk_info, capture_func):
175181
self.capture_func = capture_func
176182
self.sampler = self.make_sampler()
177183
self.buffer = None # type: Optional[ProfileBuffer]
184+
self.pid = None # type: Optional[int]
178185

179186
self.running = False
180187

181-
def should_autostart(self):
188+
profile_session_sample_rate = self.options.get("profile_session_sample_rate")
189+
self.sampled = determine_profile_session_sampling_decision(
190+
profile_session_sample_rate
191+
)
192+
193+
def is_auto_start_enabled(self):
182194
# type: () -> bool
195+
196+
# Ensure that the scheduler only autostarts once per process.
197+
# This is necessary because many web servers use forks to spawn
198+
# additional processes. And the profiler is only spawned on the
199+
# master process, then it often only profiles the main process
200+
# and not the ones where the requests are being handled.
201+
if self.pid == os.getpid():
202+
return False
203+
183204
experiments = self.options.get("_experiments")
184205
if not experiments:
185206
return False
207+
186208
return experiments.get("continuous_profiling_auto_start")
187209

210+
def manual_start(self):
211+
# type: () -> None
212+
if not self.sampled:
213+
return
214+
215+
self.ensure_running()
216+
217+
def manual_stop(self):
218+
# type: () -> None
219+
self.teardown()
220+
188221
def ensure_running(self):
189222
# type: () -> None
190223
raise NotImplementedError
@@ -277,15 +310,11 @@ def __init__(self, frequency, options, sdk_info, capture_func):
277310
super().__init__(frequency, options, sdk_info, capture_func)
278311

279312
self.thread = None # type: Optional[threading.Thread]
280-
self.pid = None # type: Optional[int]
281313
self.lock = threading.Lock()
282314

283-
def should_autostart(self):
284-
# type: () -> bool
285-
return super().should_autostart() and self.pid != os.getpid()
286-
287315
def ensure_running(self):
288316
# type: () -> None
317+
289318
pid = os.getpid()
290319

291320
# is running on the right process
@@ -356,13 +385,8 @@ def __init__(self, frequency, options, sdk_info, capture_func):
356385
super().__init__(frequency, options, sdk_info, capture_func)
357386

358387
self.thread = None # type: Optional[_ThreadPool]
359-
self.pid = None # type: Optional[int]
360388
self.lock = threading.Lock()
361389

362-
def should_autostart(self):
363-
# type: () -> bool
364-
return super().should_autostart() and self.pid != os.getpid()
365-
366390
def ensure_running(self):
367391
# type: () -> None
368392
pid = os.getpid()
@@ -393,7 +417,6 @@ def ensure_running(self):
393417
# longer allows us to spawn a thread and we have to bail.
394418
self.running = False
395419
self.thread = None
396-
return
397420

398421
def teardown(self):
399422
# type: () -> None

‎tests/profiler/test_continuous_profiler.py‎

Lines changed: 81 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,25 @@
2323
requires_gevent = pytest.mark.skipif(gevent is None, reason="gevent not enabled")
2424

2525

26-
def experimental_options(mode=None, auto_start=None):
27-
return {
28-
"_experiments": {
29-
"continuous_profiling_auto_start": auto_start,
30-
"continuous_profiling_mode": mode,
26+
def get_client_options(use_top_level_profiler_mode):
27+
def client_options(mode=None, auto_start=None, profile_session_sample_rate=1.0):
28+
if use_top_level_profiler_mode:
29+
return {
30+
"profiler_mode": mode,
31+
"profile_session_sample_rate": profile_session_sample_rate,
32+
"_experiments": {
33+
"continuous_profiling_auto_start": auto_start,
34+
},
35+
}
36+
return {
37+
"profile_session_sample_rate": profile_session_sample_rate,
38+
"_experiments": {
39+
"continuous_profiling_auto_start": auto_start,
40+
"continuous_profiling_mode": mode,
41+
},
3142
}
32-
}
43+
44+
return client_options
3345

3446

3547
mock_sdk_info = {
@@ -42,7 +54,10 @@ def experimental_options(mode=None, auto_start=None):
4254
@pytest.mark.parametrize("mode", [pytest.param("foo")])
4355
@pytest.mark.parametrize(
4456
"make_options",
45-
[pytest.param(experimental_options, id="experiment")],
57+
[
58+
pytest.param(get_client_options(True), id="non-experiment"),
59+
pytest.param(get_client_options(False), id="experiment"),
60+
],
4661
)
4762
def test_continuous_profiler_invalid_mode(mode, make_options, teardown_profiling):
4863
with pytest.raises(ValueError):
@@ -62,7 +77,10 @@ def test_continuous_profiler_invalid_mode(mode, make_options, teardown_profiling
6277
)
6378
@pytest.mark.parametrize(
6479
"make_options",
65-
[pytest.param(experimental_options, id="experiment")],
80+
[
81+
pytest.param(get_client_options(True), id="non-experiment"),
82+
pytest.param(get_client_options(False), id="experiment"),
83+
],
6684
)
6785
def test_continuous_profiler_valid_mode(mode, make_options, teardown_profiling):
6886
options = make_options(mode=mode)
@@ -82,7 +100,10 @@ def test_continuous_profiler_valid_mode(mode, make_options, teardown_profiling):
82100
)
83101
@pytest.mark.parametrize(
84102
"make_options",
85-
[pytest.param(experimental_options, id="experiment")],
103+
[
104+
pytest.param(get_client_options(True), id="non-experiment"),
105+
pytest.param(get_client_options(False), id="experiment"),
106+
],
86107
)
87108
def test_continuous_profiler_setup_twice(mode, make_options, teardown_profiling):
88109
options = make_options(mode=mode)
@@ -178,7 +199,10 @@ def assert_single_transaction_without_profile_chunks(envelopes):
178199
)
179200
@pytest.mark.parametrize(
180201
"make_options",
181-
[pytest.param(experimental_options, id="experiment")],
202+
[
203+
pytest.param(get_client_options(True), id="non-experiment"),
204+
pytest.param(get_client_options(False), id="experiment"),
205+
],
182206
)
183207
@mock.patch("sentry_sdk.profiler.continuous_profiler.PROFILE_BUFFER_SECONDS", 0.01)
184208
def test_continuous_profiler_auto_start_and_manual_stop(
@@ -191,7 +215,7 @@ def test_continuous_profiler_auto_start_and_manual_stop(
191215
options = make_options(mode=mode, auto_start=True)
192216
sentry_init(
193217
traces_sample_rate=1.0,
194-
_experiments=options.get("_experiments", {}),
218+
**options,
195219
)
196220

197221
envelopes = capture_envelopes()
@@ -235,10 +259,13 @@ def test_continuous_profiler_auto_start_and_manual_stop(
235259
)
236260
@pytest.mark.parametrize(
237261
"make_options",
238-
[pytest.param(experimental_options, id="experiment")],
262+
[
263+
pytest.param(get_client_options(True), id="non-experiment"),
264+
pytest.param(get_client_options(False), id="experiment"),
265+
],
239266
)
240267
@mock.patch("sentry_sdk.profiler.continuous_profiler.PROFILE_BUFFER_SECONDS", 0.01)
241-
def test_continuous_profiler_manual_start_and_stop(
268+
def test_continuous_profiler_manual_start_and_stop_sampled(
242269
sentry_init,
243270
capture_envelopes,
244271
mode,
@@ -248,7 +275,7 @@ def test_continuous_profiler_manual_start_and_stop(
248275
options = make_options(mode=mode)
249276
sentry_init(
250277
traces_sample_rate=1.0,
251-
_experiments=options.get("_experiments", {}),
278+
**options,
252279
)
253280

254281
envelopes = capture_envelopes()
@@ -275,3 +302,43 @@ def test_continuous_profiler_manual_start_and_stop(
275302
time.sleep(0.05)
276303

277304
assert_single_transaction_without_profile_chunks(envelopes)
305+
306+
307+
@pytest.mark.parametrize(
308+
"mode",
309+
[
310+
pytest.param("thread"),
311+
pytest.param("gevent", marks=requires_gevent),
312+
],
313+
)
314+
@pytest.mark.parametrize(
315+
"make_options",
316+
[
317+
pytest.param(get_client_options(True), id="non-experiment"),
318+
pytest.param(get_client_options(False), id="experiment"),
319+
],
320+
)
321+
def test_continuous_profiler_manual_start_and_stop_unsampled(
322+
sentry_init,
323+
capture_envelopes,
324+
mode,
325+
make_options,
326+
teardown_profiling,
327+
):
328+
options = make_options(mode=mode, profile_session_sample_rate=0.0)
329+
sentry_init(
330+
traces_sample_rate=1.0,
331+
**options,
332+
)
333+
334+
envelopes = capture_envelopes()
335+
336+
start_profiler()
337+
338+
with sentry_sdk.start_transaction(name="profiling"):
339+
with sentry_sdk.start_span(op="op"):
340+
time.sleep(0.05)
341+
342+
assert_single_transaction_without_profile_chunks(envelopes)
343+
344+
stop_profiler()

0 commit comments

Comments
 (0)