Skip to content

GH-146527: Add more data to GC statistics and add it to PyDebugOffsets #146532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pablogsal merged 8 commits into from
Mar 28, 2026
Merged

GH-146527: Add more data to GC statistics and add it to PyDebugOffsets #146532

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
37c3d44
Extend statics in the GC
sergey-miryanov Mar 18, 2026
2a04eb4
Add GC stats to _PyDebugOffsets
sergey-miryanov Mar 18, 2026
2b6cfaa
Fix object_visits calculation
sergey-miryanov Mar 18, 2026
2db2df6
Add heap_size and work_to_do
sergey-miryanov Mar 27, 2026
f86796d
Reduce size of stats buffers
sergey-miryanov Mar 27, 2026
5da8800
Revert OBJECT_STAT_INC(object_visits)
sergey-miryanov Mar 28, 2026
d09c49a
Revert duration because we need total duration in tests
sergey-miryanov Mar 28, 2026
0bac6ac
Fix gc_free_threading compilation
sergey-miryanov Mar 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Extend statics in the GC
  • Loading branch information
@sergey-miryanov
sergey-miryanov committed Mar 27, 2026
commit 37c3d44f5d7ab7ee42cccb980c5ff9b6c9ad2e08
29 changes: 17 additions & 12 deletions Include/internal/pycore_interp_structs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,31 +177,32 @@ struct gc_generation {
generations */
};

struct gc_collection_stats {
/* number of collected objects */
Py_ssize_t collected;
/* total number of uncollectable objects (put into gc.garbage) */
Py_ssize_t uncollectable;
// Total number of objects considered for collection and traversed:
Py_ssize_t candidates;
// Duration of the collection in seconds:
double duration;
};

/* Running stats per generation */
struct gc_generation_stats {
PyTime_t ts;
/* total number of collections */
Py_ssize_t collections;

/* total number of collected objects */
Py_ssize_t collected;
/* total number of uncollectable objects (put into gc.garbage) */
Py_ssize_t uncollectable;
// Total number of objects considered for collection and traversed:
Py_ssize_t candidates;
// Duration of the collection in seconds:

Py_ssize_t object_visits;
Py_ssize_t objects_transitively_reachable;
Py_ssize_t objects_not_transitively_reachable;

double duration;
};

struct gc_generation_stats_buffer {
struct gc_generation_stats items[11];
int8_t index;
};

enum _GCPhase {
GC_PHASE_MARK = 0,
GC_PHASE_COLLECT = 1
Expand All @@ -211,6 +212,10 @@ enum _GCPhase {
signature of gc.collect and change the size of PyStats.gc_stats */
#define NUM_GENERATIONS 3

struct gc_stats {
struct gc_generation_stats_buffer gen[NUM_GENERATIONS];
};

struct _gc_runtime_state {
/* Is automatic collection enabled? */
int enabled;
Expand All @@ -220,7 +225,7 @@ struct _gc_runtime_state {
struct gc_generation old[2];
/* a permanent generation which won't be collected */
struct gc_generation permanent_generation;
struct gc_generation_stats generation_stats[NUM_GENERATIONS];
struct gc_stats generation_stats;
/* true if we are currently running the collector */
int collecting;
// The frame that started the current collection. It might be NULL even when
Expand Down
4 changes: 3 additions & 1 deletion Modules/gcmodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,8 +347,10 @@ gc_get_stats_impl(PyObject *module)
/* To get consistent values despite allocations while constructing
the result list, we use a snapshot of the running stats. */
GCState *gcstate = get_gc_state();
struct gc_generation_stats_buffer *buffer;
for (i = 0; i < NUM_GENERATIONS; i++) {
stats[i] = gcstate->generation_stats[i];
buffer = &gcstate->generation_stats.gen[i];
stats[i] = buffer->items[buffer->index];
}

PyObject *result = PyList_New(0);
Expand Down
69 changes: 45 additions & 24 deletions Python/gc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1331,7 +1331,7 @@ static void
gc_collect_region(PyThreadState *tstate,
PyGC_Head *from,
PyGC_Head *to,
struct gc_collection_stats *stats);
struct gc_generation_stats *stats);

static inline Py_ssize_t
gc_list_set_space(PyGC_Head *list, int space)
Expand Down Expand Up @@ -1364,26 +1364,49 @@ gc_list_set_space(PyGC_Head *list, int space)
* scans objects at 1% of the heap size */
#define SCAN_RATE_DIVISOR 10

static struct gc_generation_stats *
gc_get_stats(GCState *gcstate, int gen)
{
struct gc_generation_stats_buffer *buffer = &gcstate->generation_stats.gen[gen];
buffer->index = (buffer->index + 1) % 11;
struct gc_generation_stats *stats = &buffer->items[buffer->index];
return stats;
}

static struct gc_generation_stats *
gc_get_prev_stats(GCState *gcstate, int gen)
{
struct gc_generation_stats_buffer *buffer = &gcstate->generation_stats.gen[gen];
struct gc_generation_stats *stats = &buffer->items[buffer->index];
return stats;
}

static void
add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats)
add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats)
{
gcstate->generation_stats[gen].duration += stats->duration;
gcstate->generation_stats[gen].collected += stats->collected;
gcstate->generation_stats[gen].uncollectable += stats->uncollectable;
gcstate->generation_stats[gen].candidates += stats->candidates;
gcstate->generation_stats[gen].collections += 1;
struct gc_generation_stats *prev_stats = gc_get_prev_stats(gcstate, gen);
struct gc_generation_stats *cur_stats = gc_get_stats(gcstate, gen);

cur_stats->ts = stats->ts;
cur_stats->collections = prev_stats->collections + 1;
cur_stats->object_visits = prev_stats->object_visits + stats->object_visits;
cur_stats->collected = prev_stats->collected + stats->collected;
cur_stats->objects_transitively_reachable = prev_stats->objects_transitively_reachable + stats->objects_transitively_reachable;
cur_stats->objects_not_transitively_reachable = prev_stats->objects_not_transitively_reachable + stats->objects_not_transitively_reachable;
cur_stats->uncollectable = prev_stats->uncollectable + stats->uncollectable;
cur_stats->candidates = prev_stats->candidates + stats->candidates;
cur_stats->duration = prev_stats->duration + stats->duration;
}

static void
gc_collect_young(PyThreadState *tstate,
struct gc_collection_stats *stats)
struct gc_generation_stats *stats)
{
GCState *gcstate = &tstate->interp->gc;
validate_spaces(gcstate);
PyGC_Head *young = &gcstate->young.head;
PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head;
untrack_tuples(young);
GC_STAT_ADD(0, collections, 1);

PyGC_Head survivors;
gc_list_init(&survivors);
Expand Down Expand Up @@ -1654,9 +1677,8 @@ assess_work_to_do(GCState *gcstate)
}

static void
gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
gc_collect_increment(PyThreadState *tstate, struct gc_generation_stats *stats)
{
GC_STAT_ADD(1, collections, 1);
GCState *gcstate = &tstate->interp->gc;
gcstate->work_to_do += assess_work_to_do(gcstate);
if (gcstate->work_to_do < 0) {
Expand All @@ -1665,9 +1687,9 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
untrack_tuples(&gcstate->young.head);
if (gcstate->phase == GC_PHASE_MARK) {
Py_ssize_t objects_marked = mark_at_start(tstate);
GC_STAT_ADD(1, objects_transitively_reachable, objects_marked);
gcstate->work_to_do -= objects_marked;
stats->objects_transitively_reachable += objects_marked;
stats->candidates += objects_marked;
gcstate->work_to_do -= objects_marked;
validate_spaces(gcstate);
return;
}
Expand All @@ -1680,7 +1702,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
scale_factor = 2;
}
intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false);
GC_STAT_ADD(1, objects_transitively_reachable, objects_marked);
stats->objects_transitively_reachable += objects_marked;
gcstate->work_to_do -= objects_marked;
gc_list_set_space(&gcstate->young.head, gcstate->visited_space);
gc_list_merge(&gcstate->young.head, &increment);
Expand All @@ -1697,7 +1719,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
gc_set_old_space(gc, gcstate->visited_space);
increment_size += expand_region_transitively_reachable(&increment, gc, gcstate);
}
GC_STAT_ADD(1, objects_not_transitively_reachable, increment_size);
stats->objects_not_transitively_reachable += increment_size;
validate_list(&increment, collecting_clear_unreachable_clear);
gc_list_validate_space(&increment, gcstate->visited_space);
PyGC_Head survivors;
Expand All @@ -1715,9 +1737,8 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)

static void
gc_collect_full(PyThreadState *tstate,
struct gc_collection_stats *stats)
struct gc_generation_stats *stats)
{
GC_STAT_ADD(2, collections, 1);
GCState *gcstate = &tstate->interp->gc;
validate_spaces(gcstate);
PyGC_Head *young = &gcstate->young.head;
Expand Down Expand Up @@ -1749,7 +1770,7 @@ static void
gc_collect_region(PyThreadState *tstate,
PyGC_Head *from,
PyGC_Head *to,
struct gc_collection_stats *stats)
struct gc_generation_stats *stats)
{
PyGC_Head unreachable; /* non-problematic unreachable trash */
PyGC_Head finalizers; /* objects with, & reachable from, __del__ */
Expand Down Expand Up @@ -1842,7 +1863,7 @@ gc_collect_region(PyThreadState *tstate,
*/
static void
do_gc_callback(GCState *gcstate, const char *phase,
int generation, struct gc_collection_stats *stats)
int generation, struct gc_generation_stats *stats)
{
assert(!PyErr_Occurred());

Expand Down Expand Up @@ -1890,7 +1911,7 @@ do_gc_callback(GCState *gcstate, const char *phase,

static void
invoke_gc_callback(GCState *gcstate, const char *phase,
int generation, struct gc_collection_stats *stats)
int generation, struct gc_generation_stats *stats)
{
if (gcstate->callbacks == NULL) {
return;
Expand Down Expand Up @@ -2082,7 +2103,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason)
}
gcstate->frame = tstate->current_frame;

struct gc_collection_stats stats = { 0 };
struct gc_generation_stats stats = { 0 };
if (reason != _Py_GC_REASON_SHUTDOWN) {
invoke_gc_callback(gcstate, "start", generation, &stats);
}
Expand All @@ -2093,8 +2114,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason)
if (PyDTrace_GC_START_ENABLED()) {
PyDTrace_GC_START(generation);
}
PyTime_t start, stop;
(void)PyTime_PerfCounterRaw(&start);
(void)PyTime_PerfCounterRaw(&stats.ts);
PyObject *exc = _PyErr_GetRaisedException(tstate);
switch(generation) {
case 0:
Expand All @@ -2109,8 +2129,9 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason)
default:
Py_UNREACHABLE();
}
PyTime_t stop;
(void)PyTime_PerfCounterRaw(&stop);
stats.duration = PyTime_AsSecondsDouble(stop - start);
stats.duration = PyTime_AsSecondsDouble(stop - stats.ts);
add_stats(gcstate, generation, &stats);
if (PyDTrace_GC_DONE_ENABLED()) {
PyDTrace_GC_DONE(stats.uncollectable + stats.collected);
Expand Down