Skip to content

gh-146256: Add --jsonl collector to the profiling.sampling #146257

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
maurycy wants to merge 41 commits into
base: main
Choose a base branch
from
Open

gh-146256: Add --jsonl collector to the profiling.sampling #146257

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
2749a44
first stab
maurycy Mar 21, 2026
f13d34c
s/ndjson/jsonl/
maurycy Mar 21, 2026
c15d318
printing to stdout isn't a great idea
maurycy Mar 22, 2026
6a0ea81
Merge remote-tracking branch 'upstream/main' into tachyon-ndjson-kole…
maurycy Mar 30, 2026
cb27fc0
even a basic test
maurycy Mar 30, 2026
59cbb4a
separate func for end record
maurycy Mar 30, 2026
25c6922
proper name
maurycy Mar 30, 2026
67cd39a
test_jsonl_collector_with_location_info
maurycy Mar 31, 2026
7c85d47
test synthetic frames
maurycy Mar 31, 2026
3eddae8
too many new lines
maurycy Mar 31, 2026
f71252e
BUG? confusing... two ways to set skip_idle?
maurycy Mar 31, 2026
9836ffa
Merge branch 'main' into tachyon-ndjson-kolektor
maurycy Mar 31, 2026
c183109
ok, thx b4fac15613a16f9cd7b2ee32840523b399f4621f
maurycy Mar 31, 2026
f20eb52
check if it works fine with (file, loc, func, op)
maurycy Mar 31, 2026
546ce90
missing new line
maurycy Mar 31, 2026
350ad99
filter out sync coordinator
maurycy Mar 31, 2026
942d821
s/collapsed_out/jsonl_out/, less copying :D
maurycy Mar 31, 2026
bd9aefe
nicer reading
maurycy Mar 31, 2026
311a4e3
typo
maurycy Mar 31, 2026
749a868
too much copying, left-over
maurycy Mar 31, 2026
85ce978
just Counter
maurycy Mar 31, 2026
820d3b9
ruff
maurycy Mar 31, 2026
aad4b18
future-proof name
maurycy Mar 31, 2026
da3e754
future-proof iter for streaming
maurycy Mar 31, 2026
cb6ed34
truth to be told, this should be layer above
maurycy Mar 31, 2026
5a59e0b
helper
maurycy Mar 31, 2026
192e54b
reorder
maurycy Mar 31, 2026
3189a8f
eh, just copy from heatmap
maurycy Mar 31, 2026
935779f
smaller chunk; matter of taste
maurycy Mar 31, 2026
e3d8aff
test actual chunking
maurycy Mar 31, 2026
d37f07a
test edge cases
maurycy Mar 31, 2026
aaaa972
ruff
maurycy Mar 31, 2026
a9b6ccd
match pep8
maurycy Mar 31, 2026
4fb3ade
style
maurycy Mar 31, 2026
a0decb5
too defensive
maurycy Mar 31, 2026
5f1704b
too many style changes
maurycy Mar 31, 2026
f2a21fb
less style
maurycy Mar 31, 2026
15b07ba
ha! even less style...
maurycy Mar 31, 2026
148f4e2
news
maurycy Mar 31, 2026
69c5768
news: proper formatting
maurycy Mar 31, 2026
f0aa26c
claim credit!
maurycy Apr 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
s/ndjson/jsonl/
  • Loading branch information
@maurycy
maurycy committed Mar 21, 2026
commit f13d34c02b4b3a3e507e8863253c8f3c672484e9
4 changes: 2 additions & 2 deletions Lib/profiling/sampling/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from .stack_collector import CollapsedStackCollector
from .heatmap_collector import HeatmapCollector
from .gecko_collector import GeckoCollector
from .ndjson_collector import NdjsonCollector
from .jsonl_collector import JsonlCollector
from .string_table import StringTable

__all__ = (
Expand All @@ -18,6 +18,6 @@
"CollapsedStackCollector",
"HeatmapCollector",
"GeckoCollector",
"NdjsonCollector",
"JsonlCollector",
"StringTable",
)
6 changes: 3 additions & 3 deletions Lib/profiling/sampling/binary_reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from .gecko_collector import GeckoCollector
from .stack_collector import FlamegraphCollector, CollapsedStackCollector
from .ndjson_collector import NdjsonCollector
from .jsonl_collector import JsonlCollector
from .pstats_collector import PstatsCollector


Expand Down Expand Up @@ -118,8 +118,8 @@ def convert_binary_to_format(input_file, output_file, output_format,
collector = PstatsCollector(interval)
elif output_format == 'gecko':
collector = GeckoCollector(interval)
elif output_format == 'ndjson':
collector = NdjsonCollector(interval)
elif output_format == 'jsonl':
collector = JsonlCollector(interval)
else:
raise ValueError(f"Unknown output format: {output_format}")

Expand Down
16 changes: 8 additions & 8 deletions Lib/profiling/sampling/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from .stack_collector import CollapsedStackCollector, FlamegraphCollector
from .heatmap_collector import HeatmapCollector
from .gecko_collector import GeckoCollector
from .ndjson_collector import NdjsonCollector
from .jsonl_collector import JsonlCollector
from .binary_collector import BinaryCollector
from .binary_reader import BinaryReader
from .constants import (
Expand Down Expand Up @@ -88,7 +88,7 @@ class CustomFormatter(
"flamegraph": "html",
"gecko": "json",
"heatmap": "html",
"ndjson": "ndjson",
"jsonl": "jsonl",
"binary": "bin",
}

Expand All @@ -98,7 +98,7 @@ class CustomFormatter(
"flamegraph": FlamegraphCollector,
"gecko": GeckoCollector,
"heatmap": HeatmapCollector,
"ndjson": NdjsonCollector,
"jsonl": JsonlCollector,
"binary": BinaryCollector,
}

Expand Down Expand Up @@ -471,11 +471,11 @@ def _add_format_options(parser, include_compression=True, include_binary=True):
help="Generate interactive HTML heatmap visualization with line-level sample counts",
)
format_group.add_argument(
"--ndjson",
"--jsonl",
action="store_const",
const="ndjson",
const="jsonl",
dest="format",
help="Generate NDJSON snapshot output for external consumers",
help="Generate JSONL snapshot output for external consumers",
)
if include_binary:
format_group.add_argument(
Expand Down Expand Up @@ -560,7 +560,7 @@ def _create_collector(format_type, sample_interval_usec, skip_idle, opcodes=Fals

Args:
format_type: The output format ('pstats', 'collapsed', 'flamegraph',
'gecko', 'heatmap', 'ndjson', 'binary')
'gecko', 'heatmap', 'jsonl', 'binary')
sample_interval_usec: Sampling interval in microseconds
skip_idle: Whether to skip idle samples
opcodes: Whether to collect opcode information (only used by gecko format
Expand Down Expand Up @@ -589,7 +589,7 @@ def _create_collector(format_type, sample_interval_usec, skip_idle, opcodes=Fals
skip_idle = False
return collector_class(sample_interval_usec, skip_idle=skip_idle, opcodes=opcodes)

if format_type == "ndjson":
if format_type == "jsonl":
return collector_class(
sample_interval_usec, skip_idle=skip_idle, mode=mode
)
Expand Down
8 changes: 4 additions & 4 deletions Lib/profiling/sampling/ndjson_collector.py → Lib/profiling/sampling/jsonl_collector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""NDJSON collector."""
"""JSONL collector."""

import json
import uuid
Expand All @@ -25,8 +25,8 @@
}


class NdjsonCollector(StackTraceCollector):
"""Collector that exports finalized profiling data as NDJSON."""
class JsonlCollector(StackTraceCollector):
Copy link
Copy Markdown
Contributor Author

@maurycy maurycy Mar 31, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe the collectors should be separate from renderers?

"""Collector that exports finalized profiling data as JSONL."""

def __init__(self, sample_interval_usec, *, skip_idle=False, mode=None):
super().__init__(sample_interval_usec, skip_idle=skip_idle)
Expand Down Expand Up @@ -81,7 +81,7 @@ def export(self, filename):
},
)

print(f"NDJSON profile written to {filename}")
print(f"JSONL profile written to {filename}")

def _build_meta_record(self):
record = {
Expand Down
Loading