Skip to content

Commit 793dd86

Browse files
encukouericsnowcurrently
authored and
mrahtz
committed
pythongh-113993: Allow interned strings to be mortal, and fix related issues (pythonGH-120520)
 * Add an InternalDocs file describing how interning should work and how to use it. * Add internal functions to *explicitly* request what kind of interning is done: - `_PyUnicode_InternMortal` - `_PyUnicode_InternImmortal` - `_PyUnicode_InternStatic` * Switch uses of `PyUnicode_InternInPlace` to those. * Disallow using `_Py_SetImmortal` on strings directly. You should use `_PyUnicode_InternImmortal` instead: - Strings should be interned before immortalization, otherwise you're possibly interning a immortalizing copy. - `_Py_SetImmortal` doesn't handle the `SSTATE_INTERNED_MORTAL` to `SSTATE_INTERNED_IMMORTAL` update, and those flags can't be changed in backports, as they are now part of public API and version-specific ABI. * Add private `_only_immortal` argument for `sys.getunicodeinternedsize`, used in refleak test machinery. * Make sure the statically allocated string singletons are unique. This means these sets are now disjoint: - `_Py_ID` - `_Py_STR` (including the empty string) - one-character latin-1 singletons Now, when you intern a singleton, that exact singleton will be interned. * Add a `_Py_LATIN1_CHR` macro, use it instead of `_Py_ID`/`_Py_STR` for one-character latin-1 singletons everywhere (including Clinic). * Intern `_Py_STR` singletons at startup. * For free-threaded builds, intern `_Py_LATIN1_CHR` singletons at startup. * Beef up the tests. Cover internal details (marked with `@cpython_only`). * Add lots of assertions Co-Authored-By: Eric Snow <ericsnowcurrently@gmail.com>
1 parent 97b4b08 commit 793dd86

42 files changed

Lines changed: 2460 additions & 1136 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Include/internal/pycore_global_objects_fini_generated.h‎

Lines changed: 1 addition & 19 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_global_strings.h‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -37,22 +37,17 @@ struct _Py_global_strings {
3737
STRUCT_FOR_STR(anon_setcomp, "<setcomp>")
3838
STRUCT_FOR_STR(anon_string, "<string>")
3939
STRUCT_FOR_STR(anon_unknown, "<unknown>")
40-
STRUCT_FOR_STR(close_br, "}")
4140
STRUCT_FOR_STR(dbl_close_br, "}}")
4241
STRUCT_FOR_STR(dbl_open_br, "{{")
4342
STRUCT_FOR_STR(dbl_percent, "%%")
4443
STRUCT_FOR_STR(defaults, ".defaults")
45-
STRUCT_FOR_STR(dot, ".")
4644
STRUCT_FOR_STR(dot_locals, ".<locals>")
4745
STRUCT_FOR_STR(empty, "")
4846
STRUCT_FOR_STR(format, ".format")
4947
STRUCT_FOR_STR(generic_base, ".generic_base")
5048
STRUCT_FOR_STR(json_decoder, "json.decoder")
5149
STRUCT_FOR_STR(kwdefaults, ".kwdefaults")
5250
STRUCT_FOR_STR(list_err, "list index out of range")
53-
STRUCT_FOR_STR(newline, "\n")
54-
STRUCT_FOR_STR(open_br, "{")
55-
STRUCT_FOR_STR(percent, "%")
5651
STRUCT_FOR_STR(type_params, ".type_params")
5752
STRUCT_FOR_STR(utf_8, "utf-8")
5853
} literals;
@@ -67,7 +62,6 @@ struct _Py_global_strings {
6762
STRUCT_FOR_ID(TextIOWrapper)
6863
STRUCT_FOR_ID(True)
6964
STRUCT_FOR_ID(WarningMessage)
70-
STRUCT_FOR_ID(_)
7165
STRUCT_FOR_ID(_WindowsConsoleIO)
7266
STRUCT_FOR_ID(__IOBase_closed)
7367
STRUCT_FOR_ID(__abc_tpflags__)
@@ -261,6 +255,7 @@ struct _Py_global_strings {
261255
STRUCT_FOR_ID(_lock_unlock_module)
262256
STRUCT_FOR_ID(_loop)
263257
STRUCT_FOR_ID(_needs_com_addref_)
258+
STRUCT_FOR_ID(_only_immortal)
264259
STRUCT_FOR_ID(_pack_)
265260
STRUCT_FOR_ID(_restype_)
266261
STRUCT_FOR_ID(_showwarnmsg)
@@ -273,7 +268,6 @@ struct _Py_global_strings {
273268
STRUCT_FOR_ID(_uninitialized_submodules)
274269
STRUCT_FOR_ID(_warn_unawaited_coroutine)
275270
STRUCT_FOR_ID(_xoptions)
276-
STRUCT_FOR_ID(a)
277271
STRUCT_FOR_ID(abs_tol)
278272
STRUCT_FOR_ID(access)
279273
STRUCT_FOR_ID(aclose)
@@ -297,7 +291,6 @@ struct _Py_global_strings {
297291
STRUCT_FOR_ID(attribute)
298292
STRUCT_FOR_ID(authorizer_callback)
299293
STRUCT_FOR_ID(autocommit)
300-
STRUCT_FOR_ID(b)
301294
STRUCT_FOR_ID(backtick)
302295
STRUCT_FOR_ID(base)
303296
STRUCT_FOR_ID(before)
@@ -315,7 +308,6 @@ struct _Py_global_strings {
315308
STRUCT_FOR_ID(byteorder)
316309
STRUCT_FOR_ID(bytes)
317310
STRUCT_FOR_ID(bytes_per_sep)
318-
STRUCT_FOR_ID(c)
319311
STRUCT_FOR_ID(c_call)
320312
STRUCT_FOR_ID(c_exception)
321313
STRUCT_FOR_ID(c_return)
@@ -371,7 +363,6 @@ struct _Py_global_strings {
371363
STRUCT_FOR_ID(count)
372364
STRUCT_FOR_ID(covariant)
373365
STRUCT_FOR_ID(cwd)
374-
STRUCT_FOR_ID(d)
375366
STRUCT_FOR_ID(data)
376367
STRUCT_FOR_ID(database)
377368
STRUCT_FOR_ID(day)
@@ -400,7 +391,6 @@ struct _Py_global_strings {
400391
STRUCT_FOR_ID(dont_inherit)
401392
STRUCT_FOR_ID(dst)
402393
STRUCT_FOR_ID(dst_dir_fd)
403-
STRUCT_FOR_ID(e)
404394
STRUCT_FOR_ID(eager_start)
405395
STRUCT_FOR_ID(effective_ids)
406396
STRUCT_FOR_ID(element_factory)
@@ -424,7 +414,6 @@ struct _Py_global_strings {
424414
STRUCT_FOR_ID(exp)
425415
STRUCT_FOR_ID(extend)
426416
STRUCT_FOR_ID(extra_tokens)
427-
STRUCT_FOR_ID(f)
428417
STRUCT_FOR_ID(facility)
429418
STRUCT_FOR_ID(factory)
430419
STRUCT_FOR_ID(false)
@@ -457,7 +446,6 @@ struct _Py_global_strings {
457446
STRUCT_FOR_ID(fset)
458447
STRUCT_FOR_ID(func)
459448
STRUCT_FOR_ID(future)
460-
STRUCT_FOR_ID(g)
461449
STRUCT_FOR_ID(generation)
462450
STRUCT_FOR_ID(genexpr)
463451
STRUCT_FOR_ID(get)
@@ -471,7 +459,6 @@ struct _Py_global_strings {
471459
STRUCT_FOR_ID(globals)
472460
STRUCT_FOR_ID(groupindex)
473461
STRUCT_FOR_ID(groups)
474-
STRUCT_FOR_ID(h)
475462
STRUCT_FOR_ID(handle)
476463
STRUCT_FOR_ID(handle_seq)
477464
STRUCT_FOR_ID(has_location)
@@ -582,7 +569,6 @@ struct _Py_global_strings {
582569
STRUCT_FOR_ID(msg)
583570
STRUCT_FOR_ID(mutex)
584571
STRUCT_FOR_ID(mycmp)
585-
STRUCT_FOR_ID(n)
586572
STRUCT_FOR_ID(n_arg)
587573
STRUCT_FOR_ID(n_fields)
588574
STRUCT_FOR_ID(n_sequence_fields)
@@ -628,7 +614,6 @@ struct _Py_global_strings {
628614
STRUCT_FOR_ID(outgoing)
629615
STRUCT_FOR_ID(overlapped)
630616
STRUCT_FOR_ID(owner)
631-
STRUCT_FOR_ID(p)
632617
STRUCT_FOR_ID(pages)
633618
STRUCT_FOR_ID(parent)
634619
STRUCT_FOR_ID(password)
@@ -656,7 +641,6 @@ struct _Py_global_strings {
656641
STRUCT_FOR_ID(ps2)
657642
STRUCT_FOR_ID(query)
658643
STRUCT_FOR_ID(quotetabs)
659-
STRUCT_FOR_ID(r)
660644
STRUCT_FOR_ID(raw)
661645
STRUCT_FOR_ID(read)
662646
STRUCT_FOR_ID(read1)
@@ -680,7 +664,6 @@ struct _Py_global_strings {
680664
STRUCT_FOR_ID(return)
681665
STRUCT_FOR_ID(reverse)
682666
STRUCT_FOR_ID(reversed)
683-
STRUCT_FOR_ID(s)
684667
STRUCT_FOR_ID(salt)
685668
STRUCT_FOR_ID(sched_priority)
686669
STRUCT_FOR_ID(scheduler)
@@ -787,7 +770,6 @@ struct _Py_global_strings {
787770
STRUCT_FOR_ID(writable)
788771
STRUCT_FOR_ID(write)
789772
STRUCT_FOR_ID(write_through)
790-
STRUCT_FOR_ID(x)
791773
STRUCT_FOR_ID(year)
792774
STRUCT_FOR_ID(zdict)
793775
} identifiers;
@@ -810,6 +792,10 @@ struct _Py_global_strings {
810792
(_Py_SINGLETON(strings.identifiers._py_ ## NAME._ascii.ob_base))
811793
#define _Py_STR(NAME) \
812794
(_Py_SINGLETON(strings.literals._py_ ## NAME._ascii.ob_base))
795+
#define _Py_LATIN1_CHR(CH) \
796+
((CH) < 128 \
797+
? (PyObject*)&_Py_SINGLETON(strings).ascii[(CH)] \
798+
: (PyObject*)&_Py_SINGLETON(strings).latin1[(CH) - 128])
813799

814800
/* _Py_DECLARE_STR() should precede all uses of _Py_STR() in a function.
815801

0 commit comments

Comments
 (0)