Skip to content

Commit 9fd6334

Browse files
encukouericsnowcurrently
encukou
and
ericsnowcurrently
committed
[3.12] pythongh-113993: Allow interned strings to be mortal, and fix related issues (pythonGH-120520)
* Add an InternalDocs file describing how interning should work and how to use it. * Add internal functions to *explicitly* request what kind of interning is done: - `_PyUnicode_InternMortal` - `_PyUnicode_InternImmortal` - `_PyUnicode_InternStatic` * Switch uses of `PyUnicode_InternInPlace` to those. * Disallow using `_Py_SetImmortal` on strings directly. You should use `_PyUnicode_InternImmortal` instead: - Strings should be interned before immortalization, otherwise you're possibly interning a immortalizing copy. - `_Py_SetImmortal` doesn't handle the `SSTATE_INTERNED_MORTAL` to `SSTATE_INTERNED_IMMORTAL` update, and those flags can't be changed in backports, as they are now part of public API and version-specific ABI. * Add private `_only_immortal` argument for `sys.getunicodeinternedsize`, used in refleak test machinery. * Make sure the statically allocated string singletons are unique. This means these sets are now disjoint: - `_Py_ID` - `_Py_STR` (including the empty string) - one-character latin-1 singletons Now, when you intern a singleton, that exact singleton will be interned. * Add a `_Py_LATIN1_CHR` macro, use it instead of `_Py_ID`/`_Py_STR` for one-character latin-1 singletons everywhere (including Clinic). * Intern `_Py_STR` singletons at startup. * Beef up the tests. Cover internal details (marked with `@cpython_only`). * Add lots of assertions Co-authored-by: Eric Snow <ericsnowcurrently@gmail.com>
1 parent 41090b7 commit 9fd6334

41 files changed

Lines changed: 2298 additions & 1033 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Include/internal/pycore_global_objects_fini_generated.h‎

Lines changed: 1 addition & 16 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_global_strings.h‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,21 +36,16 @@ struct _Py_global_strings {
3636
STRUCT_FOR_STR(anon_setcomp, "<setcomp>")
3737
STRUCT_FOR_STR(anon_string, "<string>")
3838
STRUCT_FOR_STR(anon_unknown, "<unknown>")
39-
STRUCT_FOR_STR(close_br, "}")
4039
STRUCT_FOR_STR(dbl_close_br, "}}")
4140
STRUCT_FOR_STR(dbl_open_br, "{{")
4241
STRUCT_FOR_STR(dbl_percent, "%%")
4342
STRUCT_FOR_STR(defaults, ".defaults")
44-
STRUCT_FOR_STR(dot, ".")
4543
STRUCT_FOR_STR(dot_locals, ".<locals>")
4644
STRUCT_FOR_STR(empty, "")
4745
STRUCT_FOR_STR(generic_base, ".generic_base")
4846
STRUCT_FOR_STR(json_decoder, "json.decoder")
4947
STRUCT_FOR_STR(kwdefaults, ".kwdefaults")
5048
STRUCT_FOR_STR(list_err, "list index out of range")
51-
STRUCT_FOR_STR(newline, "\n")
52-
STRUCT_FOR_STR(open_br, "{")
53-
STRUCT_FOR_STR(percent, "%")
5449
STRUCT_FOR_STR(shim_name, "<shim>")
5550
STRUCT_FOR_STR(type_params, ".type_params")
5651
STRUCT_FOR_STR(utf_8, "utf-8")
@@ -66,7 +61,6 @@ struct _Py_global_strings {
6661
STRUCT_FOR_ID(TextIOWrapper)
6762
STRUCT_FOR_ID(True)
6863
STRUCT_FOR_ID(WarningMessage)
69-
STRUCT_FOR_ID(_)
7064
STRUCT_FOR_ID(_WindowsConsoleIO)
7165
STRUCT_FOR_ID(__IOBase_closed)
7266
STRUCT_FOR_ID(__abc_tpflags__)
@@ -255,6 +249,7 @@ struct _Py_global_strings {
255249
STRUCT_FOR_ID(_lock_unlock_module)
256250
STRUCT_FOR_ID(_loop)
257251
STRUCT_FOR_ID(_needs_com_addref_)
252+
STRUCT_FOR_ID(_only_immortal)
258253
STRUCT_FOR_ID(_pack_)
259254
STRUCT_FOR_ID(_restype_)
260255
STRUCT_FOR_ID(_showwarnmsg)
@@ -266,7 +261,6 @@ struct _Py_global_strings {
266261
STRUCT_FOR_ID(_uninitialized_submodules)
267262
STRUCT_FOR_ID(_warn_unawaited_coroutine)
268263
STRUCT_FOR_ID(_xoptions)
269-
STRUCT_FOR_ID(a)
270264
STRUCT_FOR_ID(abs_tol)
271265
STRUCT_FOR_ID(access)
272266
STRUCT_FOR_ID(add)
@@ -286,7 +280,6 @@ struct _Py_global_strings {
286280
STRUCT_FOR_ID(attribute)
287281
STRUCT_FOR_ID(authorizer_callback)
288282
STRUCT_FOR_ID(autocommit)
289-
STRUCT_FOR_ID(b)
290283
STRUCT_FOR_ID(backtick)
291284
STRUCT_FOR_ID(base)
292285
STRUCT_FOR_ID(before)
@@ -304,7 +297,6 @@ struct _Py_global_strings {
304297
STRUCT_FOR_ID(byteorder)
305298
STRUCT_FOR_ID(bytes)
306299
STRUCT_FOR_ID(bytes_per_sep)
307-
STRUCT_FOR_ID(c)
308300
STRUCT_FOR_ID(c_call)
309301
STRUCT_FOR_ID(c_exception)
310302
STRUCT_FOR_ID(c_return)
@@ -357,7 +349,6 @@ struct _Py_global_strings {
357349
STRUCT_FOR_ID(count)
358350
STRUCT_FOR_ID(covariant)
359351
STRUCT_FOR_ID(cwd)
360-
STRUCT_FOR_ID(d)
361352
STRUCT_FOR_ID(data)
362353
STRUCT_FOR_ID(database)
363354
STRUCT_FOR_ID(decode)
@@ -385,7 +376,6 @@ struct _Py_global_strings {
385376
STRUCT_FOR_ID(dst)
386377
STRUCT_FOR_ID(dst_dir_fd)
387378
STRUCT_FOR_ID(duration)
388-
STRUCT_FOR_ID(e)
389379
STRUCT_FOR_ID(eager_start)
390380
STRUCT_FOR_ID(effective_ids)
391381
STRUCT_FOR_ID(element_factory)
@@ -546,7 +536,6 @@ struct _Py_global_strings {
546536
STRUCT_FOR_ID(mro)
547537
STRUCT_FOR_ID(msg)
548538
STRUCT_FOR_ID(mycmp)
549-
STRUCT_FOR_ID(n)
550539
STRUCT_FOR_ID(n_arg)
551540
STRUCT_FOR_ID(n_fields)
552541
STRUCT_FOR_ID(n_sequence_fields)
@@ -591,7 +580,6 @@ struct _Py_global_strings {
591580
STRUCT_FOR_ID(outgoing)
592581
STRUCT_FOR_ID(overlapped)
593582
STRUCT_FOR_ID(owner)
594-
STRUCT_FOR_ID(p)
595583
STRUCT_FOR_ID(pages)
596584
STRUCT_FOR_ID(parent)
597585
STRUCT_FOR_ID(password)
@@ -619,7 +607,6 @@ struct _Py_global_strings {
619607
STRUCT_FOR_ID(ps2)
620608
STRUCT_FOR_ID(query)
621609
STRUCT_FOR_ID(quotetabs)
622-
STRUCT_FOR_ID(r)
623610
STRUCT_FOR_ID(raw)
624611
STRUCT_FOR_ID(read)
625612
STRUCT_FOR_ID(read1)
@@ -643,7 +630,6 @@ struct _Py_global_strings {
643630
STRUCT_FOR_ID(return)
644631
STRUCT_FOR_ID(reverse)
645632
STRUCT_FOR_ID(reversed)
646-
STRUCT_FOR_ID(s)
647633
STRUCT_FOR_ID(salt)
648634
STRUCT_FOR_ID(sched_priority)
649635
STRUCT_FOR_ID(scheduler)
@@ -746,7 +732,6 @@ struct _Py_global_strings {
746732
STRUCT_FOR_ID(writable)
747733
STRUCT_FOR_ID(write)
748734
STRUCT_FOR_ID(write_through)
749-
STRUCT_FOR_ID(x)
750735
STRUCT_FOR_ID(year)
751736
STRUCT_FOR_ID(zdict)
752737
} identifiers;
@@ -769,6 +754,10 @@ struct _Py_global_strings {
769754
(_Py_SINGLETON(strings.identifiers._py_ ## NAME._ascii.ob_base))
770755
#define _Py_STR(NAME) \
771756
(_Py_SINGLETON(strings.literals._py_ ## NAME._ascii.ob_base))
757+
#define _Py_LATIN1_CHR(CH) \
758+
((CH) < 128 \
759+
? (PyObject*)&_Py_SINGLETON(strings).ascii[(CH)] \
760+
: (PyObject*)&_Py_SINGLETON(strings).latin1[(CH) - 128])
772761

773762
/* _Py_DECLARE_STR() should precede all uses of _Py_STR() in a function.
774763

‎Include/internal/pycore_runtime_init_generated.h‎

Lines changed: 1 addition & 16 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)