Skip to content

perf: move EG() and CG() in ZTS builds into __thread storage #23227

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
henderkes wants to merge 10 commits into
base: master
Choose a base branch
from
Open

perf: move EG() and CG() in ZTS builds into __thread storage #23227

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
b2dc217
perf: embed executor and compiler globals in the __thread TLS cache s…
henderkes Jul 14, 2026
2a7884d
fix typo
henderkes Jul 23, 2026
2e248cb
leave the option of initial-exec tls model even for apache2handler if…
henderkes Aug 11, 2026
4aef0ba
remove ts_allocate_fast_id_at again (unused, now that the four global…
henderkes Aug 11, 2026
4633dc0
guard ts_apply_for_id and ts_free_id for own threads globals too
henderkes Aug 12, 2026
d67443a
reword comment
henderkes Aug 12, 2026
c509919
simplify linux aarch64 jit generator
henderkes Aug 12, 2026
bd1730d
stupid typo
henderkes Aug 12, 2026
fdc77a4
clarification
henderkes Aug 12, 2026
06f5bba
abort, rather than assert
henderkes Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 24 additions & 39 deletions TSRM/TSRM.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ static int resource_types_table_size;
/* Reserved space for fast globals access */
static size_t tsrm_reserved_pos = 0;
static size_t tsrm_reserved_size = 0;
static size_t tsrm_reserved_front = 0;

static MUTEX_T tsmm_mutex; /* thread-safe memory manager mutex */
static MUTEX_T tsrm_env_mutex; /* tsrm environ mutex */
Expand Down Expand Up @@ -157,7 +156,6 @@ TSRM_API bool tsrm_startup(int expected_threads, int expected_resources, int deb

tsrm_reserved_pos = 0;
tsrm_reserved_size = 0;
tsrm_reserved_front = 0;

tsrm_env_mutex = tsrm_mutex_alloc();

Expand Down Expand Up @@ -216,7 +214,7 @@ TSRM_API void tsrm_shutdown(void)
} else {
free(p->storage);
}
free((char *) p - tsrm_reserved_front);
free(p);
p = next_p;
}
}
Expand All @@ -243,7 +241,6 @@ TSRM_API void tsrm_shutdown(void)

tsrm_reserved_pos = 0;
tsrm_reserved_size = 0;
tsrm_reserved_front = 0;
}/*}}}*/

/* {{{ */
Expand Down Expand Up @@ -335,20 +332,13 @@ TSRM_API void tsrm_reserve(size_t size)
}/*}}}*/


/* Carve a fixed-offset front region out of the reserved space. It is placed
* before the TLS entry, so the hot globals get compile-time-constant negative
* offsets from the cache pointer. */
TSRM_API void tsrm_reserve_fast_front(size_t size)
{
tsrm_reserved_front = TSRM_ALIGNED_SIZE(size);
tsrm_reserved_size -= tsrm_reserved_front;
}


/* allocates a new fast thread-safe-resource id */
TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor)
{/*{{{*/
TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtaining a new fast resource id, %d bytes", size));

tsrm_mutex_lock(tsmm_mutex);

size = TSRM_ALIGNED_SIZE(size);
if (tsrm_reserved_size - tsrm_reserved_pos < size) {
TSRM_ERROR((TSRM_ERROR_LEVEL_ERROR, "Unable to allocate space for fast resource"));
Expand All @@ -357,27 +347,13 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz
tsrm_mutex_unlock(tsmm_mutex);
return 0;
}
ptrdiff_t fixed_offset = TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_pos;
*offset = TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_pos;
tsrm_reserved_pos += size;
tsrm_mutex_unlock(tsmm_mutex);

return ts_allocate_fast_id_at(rsrc_id, offset, fixed_offset, size, ctor, dtor);
}/*}}}*/


TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, ptrdiff_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor)
{
TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtaining a new fast resource id, %d bytes", size));

tsrm_mutex_lock(tsmm_mutex);

/* obtain a resource id */
*rsrc_id = TSRM_SHUFFLE_RSRC_ID(id_count++);
TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtained resource id %d", *rsrc_id));

size = TSRM_ALIGNED_SIZE(size);
*offset = (size_t) fixed_offset;

/* store the new resource type in the resource sizes table */
if (resource_types_table_size < id_count) {
tsrm_resource_type *_tmp;
Expand All @@ -403,7 +379,7 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset,

TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Successfully allocated new resource id %d", *rsrc_id));
return *rsrc_id;
}
}/*}}}*/

/* allocates a resource id whose per-thread storage is a native __thread block */
TSRM_API ts_rsrc_id ts_allocate_tls_id(ts_rsrc_id *rsrc_id, void *(*tls_addr)(void), size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor)
Expand Down Expand Up @@ -450,10 +426,8 @@ static void set_thread_local_storage_resource_to(tsrm_tls_entry *thread_resource
static void allocate_new_resource(tsrm_tls_entry **thread_resources_ptr, THREAD_T thread_id)
{/*{{{*/
TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Creating data structures for thread %x", thread_id));
/* The entry follows the fixed-offset front region.
* hot globals live at negative offsets from the TLS cache pointer. */
char *block = (char *) malloc(tsrm_reserved_front + TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_size);
(*thread_resources_ptr) = (tsrm_tls_entry *) (block + tsrm_reserved_front);
/* Fast resources live in the reserved space right behind the entry. */
(*thread_resources_ptr) = (tsrm_tls_entry *) malloc(TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_size);
(*thread_resources_ptr)->storage = NULL;
if (id_count > 0) {
(*thread_resources_ptr)->storage = (void **) malloc(sizeof(void *)*id_count);
Expand Down Expand Up @@ -566,7 +540,7 @@ TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id)
* thread's blocks were never constructed, so keep tls dtors from running. */
thread_resources->thread_id = 0;
ts_free_resources(thread_resources);
free((char *) thread_resources - tsrm_reserved_front);
free(thread_resources);
/* Allocate a new resource at the same point in the linked list, and relink the next pointer */
allocate_new_resource(last_thread_resources, thread_id);
thread_resources = *last_thread_resources;
Expand Down Expand Up @@ -608,7 +582,7 @@ void ts_free_thread(void)
tsrm_tls_table[hash_value] = thread_resources->next;
}
tsrm_tls_set(0);
free((char *) thread_resources - tsrm_reserved_front);
free(thread_resources);
break;
}
if (thread_resources->next) {
Expand All @@ -623,6 +597,7 @@ void ts_free_thread(void)
void ts_free_id(ts_rsrc_id id)
{/*{{{*/
int rsrc_id = TSRM_UNSHUFFLE_RSRC_ID(id);
THREAD_T this_thread = tsrm_thread_id();

tsrm_mutex_lock(tsmm_mutex);

Expand All @@ -634,7 +609,9 @@ void ts_free_id(ts_rsrc_id id)

while (p) {
if (p->count > rsrc_id && p->storage[rsrc_id]) {
if (resource_types_table) {
/* A __thread block of a foreign thread is inaccessible. */
if (resource_types_table
&& (!resource_types_table[rsrc_id].tls_addr || p->thread_id == this_thread)) {
if (resource_types_table[rsrc_id].dtor) {
resource_types_table[rsrc_id].dtor(p->storage[rsrc_id]);
}
Expand All @@ -658,15 +635,20 @@ void ts_free_id(ts_rsrc_id id)
TSRM_API void ts_apply_for_id(ts_rsrc_id id, void (*cb)(void *))
{
int rsrc_id = TSRM_UNSHUFFLE_RSRC_ID(id);
THREAD_T this_thread = tsrm_thread_id();

tsrm_mutex_lock(tsmm_mutex);

if (tsrm_tls_table && resource_types_table) {
bool tls_backed = resource_types_table[rsrc_id].tls_addr != NULL;

for (int i = 0; i < tsrm_tls_table_size; i++) {
tsrm_tls_entry *p = tsrm_tls_table[i];

while (p) {
if (p->count > rsrc_id && p->storage[rsrc_id]) {
/* A __thread block of a foreign thread is inaccessible. */
if (p->count > rsrc_id && p->storage[rsrc_id]
&& (!tls_backed || p->thread_id == this_thread)) {
cb(p->storage[rsrc_id]);
}
p = p->next;
Expand Down Expand Up @@ -852,7 +834,10 @@ TSRM_API void *tsrm_get_ls_cache(void)
/* Returns offset of tsrm_ls_cache slot from Thread Control Block address */
TSRM_API size_t tsrm_get_ls_cache_tcb_offset(void)
{/*{{{*/
#if defined(__APPLE__) && defined(__x86_64__)
#if defined(TSRM_TLS_MODEL_GLOBAL_DYNAMIC)
/* No constant TCB offset under global-dynamic, can't use fast path */
return 0;
#elif defined(__APPLE__) && defined(__x86_64__)
// TODO: Implement support for fast JIT ZTS code ???
return 0;
#elif defined(__x86_64__) && defined(__GNUC__) && !defined(__FreeBSD__) && \
Expand Down
38 changes: 24 additions & 14 deletions TSRM/TSRM.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,11 +95,6 @@ TSRM_API ts_rsrc_id ts_allocate_id(ts_rsrc_id *rsrc_id, size_t size, ts_allocate
TSRM_API void tsrm_reserve(size_t size);
TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor);

/* Fast resources at caller-chosen, compile-time-constant offsets. The fixed
* front region must be reserved after tsrm_reserve() and before any fast id. */
TSRM_API void tsrm_reserve_fast_front(size_t size);
TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, ptrdiff_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor);

/* Resource whose per-thread storage is a native __thread block.
* Must be called at startup before any other thread exists. */
TSRM_API ts_rsrc_id ts_allocate_tls_id(ts_rsrc_id *rsrc_id, void *(*tls_addr)(void), size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor);
Expand Down Expand Up @@ -167,8 +162,13 @@ TSRM_API bool tsrm_is_managed_thread(void);
# define TSRM_TLS_MODEL_ATTR
# define TSRM_TLS_MODEL_DEFAULT
#elif defined(__PIC__) && !defined(__PIE__)
# define TSRM_TLS_MODEL_ATTR __attribute__((tls_model("initial-exec")))
# define TSRM_TLS_MODEL_INITIAL_EXEC
# if defined(TSRM_TLS_MODEL_USE_GLOBAL_DYNAMIC)
# define TSRM_TLS_MODEL_ATTR __attribute__((tls_model("global-dynamic")))
# define TSRM_TLS_MODEL_GLOBAL_DYNAMIC
# else
# define TSRM_TLS_MODEL_ATTR __attribute__((tls_model("initial-exec")))
# define TSRM_TLS_MODEL_INITIAL_EXEC
# endif
#else
# define TSRM_TLS_MODEL_ATTR __attribute__((tls_model("local-exec")))
# define TSRM_TLS_MODEL_LOCAL_EXEC
Expand All @@ -186,17 +186,27 @@ TSRM_API bool tsrm_is_managed_thread(void);
#define TSRMG_BULK_STATIC(id, type) ((type) (*((void ***) TSRMLS_CACHE))[TSRM_UNSHUFFLE_RSRC_ID(id)])
#define TSRMG_FAST_STATIC(offset, type, element) (TSRMG_FAST_BULK_STATIC(offset, type)->element)
#define TSRMG_FAST_BULK_STATIC(offset, type) ((type) (((char*) TSRMLS_CACHE)+(offset)))
struct _zend_tsrm_ls_cache;
#if defined(ZEND_WIN32) && !defined(LIBZEND_EXPORTS)
/* Windows can't dllexport __declspec(thread) symbols, so outside Zend each module
* keeps a per-module `void *` pointer and reaches EG/CG via the resource-id indirection. */
# define ZEND_TSRMLS_CACHE_T void *
# define TSRMLS_MAIN_CACHE_DEFINE() TSRM_TLS void *_tsrm_ls_cache TSRM_TLS_MODEL_ATTR = NULL;
# define TSRMLS_CACHE_DEFINE() TSRM_TLS void *_tsrm_ls_cache = NULL;
#else
# define ZEND_TSRMLS_CACHE_T struct _zend_tsrm_ls_cache
# define TSRMLS_MAIN_CACHE_DEFINE()
# define TSRMLS_CACHE_DEFINE()
#endif
#ifdef __cplusplus
#define TSRMLS_MAIN_CACHE_EXTERN() extern "C" { extern TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR; }
#define TSRMLS_CACHE_EXTERN() extern "C" { extern TSRM_TLS void *TSRMLS_CACHE; }
#define TSRMLS_MAIN_CACHE_EXTERN() extern "C" { extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache TSRM_TLS_MODEL_ATTR; }
#define TSRMLS_CACHE_EXTERN() extern "C" { extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache; }
#else
#define TSRMLS_MAIN_CACHE_EXTERN() extern TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR;
#define TSRMLS_CACHE_EXTERN() extern TSRM_TLS void *TSRMLS_CACHE;
#define TSRMLS_MAIN_CACHE_EXTERN() extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache TSRM_TLS_MODEL_ATTR;
#define TSRMLS_CACHE_EXTERN() extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache;
#endif
#define TSRMLS_MAIN_CACHE_DEFINE() TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR = NULL;
#define TSRMLS_CACHE_DEFINE() TSRM_TLS void *TSRMLS_CACHE = NULL;
#define TSRMLS_CACHE_UPDATE() TSRMLS_CACHE = tsrm_get_ls_cache()
#define TSRMLS_CACHE _tsrm_ls_cache
#define TSRMLS_CACHE (*(void **) &_tsrm_ls_cache)

#ifdef __cplusplus
}
Expand Down
6 changes: 6 additions & 0 deletions UPGRADING.INTERNALS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,8 @@ PHP 8.6 INTERNALS UPGRADE NOTES
. Added zend_string_ends_with() and related variants.
. Added trait support for internal classes.
. Added do_php_cli().
. The ZTS symbols compiler_globals_offset and executor_globals_offset have
been removed: CG()/EG() now reach the globals through a __thread struct.

========================
2. Build system changes
Expand Down Expand Up @@ -249,6 +251,10 @@ PHP 8.6 INTERNALS UPGRADE NOTES
HAVE_STRUCT_STAT_ST_BLOCKS).
. Added a new configure option --disable-apache2-conf to prevent apxs from
editing httpd.conf during installation.
. Added --with-tsrm-tls-model (initial-exec, global-dynamic, auto) to pick the
TLS model for the ZTS globals cache; auto uses global-dynamic for dlopen()ed
SAPIs. Forcing initial-exec there needs a larger static TLS surplus: set
GLIBC_TUNABLES=glibc.rtld.optional_static_tls=8192 for the host process.

- Windows build system changes:
. Function SETUP_OPENSSL() doesn't accept 6th argument anymore and doesn't
Expand Down
41 changes: 33 additions & 8 deletions Zend/zend.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,17 +52,43 @@ static bool startup_done = false;
#ifdef ZTS
ZEND_API int compiler_globals_id;
ZEND_API int executor_globals_id;
ZEND_API size_t compiler_globals_offset;
ZEND_API size_t executor_globals_offset;
ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_tsrm_ls_cache _tsrm_ls_cache = {0};
#if defined(_WIN64) && defined(ZEND_TLS_DIRECT)
/* Holds &_tsrm_ls_cache in a TEB TLS slot (filled by DllMain) so EG()/CG() reach it
* with a single gs:[] load rather than the 3-load __declspec(thread) lookup. */
ZEND_TLS_API unsigned long zend_win_tsrm_cache_slot = 0;
ZEND_API void zend_win_tsrm_cache_init(bool alloc)
{
if (alloc) {
zend_win_tsrm_cache_slot = TlsAlloc();
if (zend_win_tsrm_cache_slot == TLS_OUT_OF_INDEXES) {
fprintf(stderr, "PHP Startup: TlsAlloc() failed, no TLS slot available "
"for the ZTS globals cache\n");
abort();
}
if (zend_win_tsrm_cache_slot >= TLS_MINIMUM_AVAILABLE) {
/* Beyond the first TLS_MINIMUM_AVAILABLE slots the loader uses
* TEB->TlsExpansionSlots, which the inline gs:[] read in
* ZEND_TSRM_CACHE_PTR cannot reach. */
fprintf(stderr, "PHP Startup: TlsAlloc() returned slot %lu, but only the "
"first %d direct TEB slots are usable for the ZTS globals cache\n",
zend_win_tsrm_cache_slot, TLS_MINIMUM_AVAILABLE);
abort();
}
}
TlsSetValue(zend_win_tsrm_cache_slot, &_tsrm_ls_cache);
}
#endif
/* ts_allocate_tls_id takes a callback so each thread resolves its own block.
* A plain &language_scanner_globals would capture only the registering thread's address. */
* A plain &..._tls would capture only the registering thread's address. */
static void *executor_globals_tls_addr(void) { return &_tsrm_ls_cache.eg; }
static void *compiler_globals_tls_addr(void) { return &_tsrm_ls_cache.cg; }
static void *language_scanner_globals_tls_addr(void) { return &language_scanner_globals; }
static HashTable *global_function_table = NULL;
static HashTable *global_class_table = NULL;
static HashTable *global_constants_table = NULL;
static HashTable *global_auto_globals_table = NULL;
static HashTable *global_persistent_list = NULL;
TSRMLS_MAIN_CACHE_DEFINE()
# define GLOBAL_FUNCTION_TABLE global_function_table
# define GLOBAL_CLASS_TABLE global_class_table
# define GLOBAL_CONSTANTS_TABLE global_constants_table
Expand Down Expand Up @@ -804,6 +830,7 @@ static void compiler_globals_dtor(zend_compiler_globals *compiler_globals) /* {{

static void executor_globals_ctor(zend_executor_globals *executor_globals) /* {{{ */
{
_tsrm_ls_cache.self = &_tsrm_ls_cache;
zend_startup_constants();
zend_copy_constants(executor_globals->zend_constants, GLOBAL_CONSTANTS_TABLE);
zend_init_rsrc_plist();
Expand Down Expand Up @@ -1022,11 +1049,9 @@ void zend_startup(zend_utility_functions *utility_functions) /* {{{ */
zend_init_rsrc_list_dtors();

#ifdef ZTS
ts_allocate_fast_id_at(&compiler_globals_id, &compiler_globals_offset, ZEND_CG_OFFSET, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor);
ts_allocate_fast_id_at(&executor_globals_id, &executor_globals_offset, ZEND_EG_OFFSET, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor);
ts_allocate_tls_id(&compiler_globals_id, compiler_globals_tls_addr, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor);
ts_allocate_tls_id(&executor_globals_id, executor_globals_tls_addr, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor);
ts_allocate_tls_id(&language_scanner_globals_id, language_scanner_globals_tls_addr, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL);
ZEND_ASSERT(compiler_globals_offset == ZEND_CG_OFFSET);
ZEND_ASSERT(executor_globals_offset == ZEND_EG_OFFSET);
ts_allocate_fast_id(&ini_scanner_globals_id, &ini_scanner_globals_offset, sizeof(zend_ini_scanner_globals), (ts_allocate_ctor) ini_scanner_globals_ctor, NULL);
compiler_globals = ts_resource(compiler_globals_id);
executor_globals = ts_resource(executor_globals_id);
Expand Down
11 changes: 6 additions & 5 deletions Zend/zend_globals.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,6 @@
BEGIN_EXTERN_C()
ZEND_API extern int compiler_globals_id;
ZEND_API extern int executor_globals_id;
ZEND_API extern size_t compiler_globals_offset;
ZEND_API extern size_t executor_globals_offset;
END_EXTERN_C()

#endif
Expand Down Expand Up @@ -332,9 +330,12 @@ struct _zend_executor_globals {
};

#ifdef ZTS
/* Compile-time offsets of the hot globals, in a reserved region just before *_tsrm_ls_cache. */
# define ZEND_CG_OFFSET (-(ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)))
# define ZEND_EG_OFFSET (ZEND_CG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)))
struct _zend_tsrm_ls_cache {
void *cache;
void *self;
zend_executor_globals eg;
zend_compiler_globals cg;
};
#endif

#define EG_FLAGS_INITIAL (0)
Expand Down
Loading
Loading