diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c index a5032e456aae..9a2811e2c42c 100644 --- a/TSRM/TSRM.c +++ b/TSRM/TSRM.c @@ -59,7 +59,6 @@ static int resource_types_table_size; /* Reserved space for fast globals access */ static size_t tsrm_reserved_pos = 0; static size_t tsrm_reserved_size = 0; -static size_t tsrm_reserved_front = 0; static MUTEX_T tsmm_mutex; /* thread-safe memory manager mutex */ static MUTEX_T tsrm_env_mutex; /* tsrm environ mutex */ @@ -157,7 +156,6 @@ TSRM_API bool tsrm_startup(int expected_threads, int expected_resources, int deb tsrm_reserved_pos = 0; tsrm_reserved_size = 0; - tsrm_reserved_front = 0; tsrm_env_mutex = tsrm_mutex_alloc(); @@ -216,7 +214,7 @@ TSRM_API void tsrm_shutdown(void) } else { free(p->storage); } - free((char *) p - tsrm_reserved_front); + free(p); p = next_p; } } @@ -243,7 +241,6 @@ TSRM_API void tsrm_shutdown(void) tsrm_reserved_pos = 0; tsrm_reserved_size = 0; - tsrm_reserved_front = 0; }/*}}}*/ /* {{{ */ @@ -335,20 +332,13 @@ TSRM_API void tsrm_reserve(size_t size) }/*}}}*/ -/* Carve a fixed-offset front region out of the reserved space. It is placed - * before the TLS entry, so the hot globals get compile-time-constant negative - * offsets from the cache pointer. */ -TSRM_API void tsrm_reserve_fast_front(size_t size) -{ - tsrm_reserved_front = TSRM_ALIGNED_SIZE(size); - tsrm_reserved_size -= tsrm_reserved_front; -} - - /* allocates a new fast thread-safe-resource id */ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor) {/*{{{*/ + TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtaining a new fast resource id, %d bytes", size)); + tsrm_mutex_lock(tsmm_mutex); + size = TSRM_ALIGNED_SIZE(size); if (tsrm_reserved_size - tsrm_reserved_pos < size) { TSRM_ERROR((TSRM_ERROR_LEVEL_ERROR, "Unable to allocate space for fast resource")); @@ -357,27 +347,13 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz tsrm_mutex_unlock(tsmm_mutex); return 0; } - ptrdiff_t fixed_offset = TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_pos; + *offset = TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_pos; tsrm_reserved_pos += size; - tsrm_mutex_unlock(tsmm_mutex); - - return ts_allocate_fast_id_at(rsrc_id, offset, fixed_offset, size, ctor, dtor); -}/*}}}*/ - - -TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, ptrdiff_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor) -{ - TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtaining a new fast resource id, %d bytes", size)); - - tsrm_mutex_lock(tsmm_mutex); /* obtain a resource id */ *rsrc_id = TSRM_SHUFFLE_RSRC_ID(id_count++); TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtained resource id %d", *rsrc_id)); - size = TSRM_ALIGNED_SIZE(size); - *offset = (size_t) fixed_offset; - /* store the new resource type in the resource sizes table */ if (resource_types_table_size < id_count) { tsrm_resource_type *_tmp; @@ -403,7 +379,7 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Successfully allocated new resource id %d", *rsrc_id)); return *rsrc_id; -} +}/*}}}*/ /* allocates a resource id whose per-thread storage is a native __thread block */ TSRM_API ts_rsrc_id ts_allocate_tls_id(ts_rsrc_id *rsrc_id, void *(*tls_addr)(void), size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor) @@ -450,10 +426,8 @@ static void set_thread_local_storage_resource_to(tsrm_tls_entry *thread_resource static void allocate_new_resource(tsrm_tls_entry **thread_resources_ptr, THREAD_T thread_id) {/*{{{*/ TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Creating data structures for thread %x", thread_id)); - /* The entry follows the fixed-offset front region. - * hot globals live at negative offsets from the TLS cache pointer. */ - char *block = (char *) malloc(tsrm_reserved_front + TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_size); - (*thread_resources_ptr) = (tsrm_tls_entry *) (block + tsrm_reserved_front); + /* Fast resources live in the reserved space right behind the entry. */ + (*thread_resources_ptr) = (tsrm_tls_entry *) malloc(TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_size); (*thread_resources_ptr)->storage = NULL; if (id_count > 0) { (*thread_resources_ptr)->storage = (void **) malloc(sizeof(void *)*id_count); @@ -566,7 +540,7 @@ TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id) * thread's blocks were never constructed, so keep tls dtors from running. */ thread_resources->thread_id = 0; ts_free_resources(thread_resources); - free((char *) thread_resources - tsrm_reserved_front); + free(thread_resources); /* Allocate a new resource at the same point in the linked list, and relink the next pointer */ allocate_new_resource(last_thread_resources, thread_id); thread_resources = *last_thread_resources; @@ -608,7 +582,7 @@ void ts_free_thread(void) tsrm_tls_table[hash_value] = thread_resources->next; } tsrm_tls_set(0); - free((char *) thread_resources - tsrm_reserved_front); + free(thread_resources); break; } if (thread_resources->next) { @@ -623,6 +597,7 @@ void ts_free_thread(void) void ts_free_id(ts_rsrc_id id) {/*{{{*/ int rsrc_id = TSRM_UNSHUFFLE_RSRC_ID(id); + THREAD_T this_thread = tsrm_thread_id(); tsrm_mutex_lock(tsmm_mutex); @@ -634,7 +609,9 @@ void ts_free_id(ts_rsrc_id id) while (p) { if (p->count > rsrc_id && p->storage[rsrc_id]) { - if (resource_types_table) { + /* A __thread block of a foreign thread is inaccessible. */ + if (resource_types_table + && (!resource_types_table[rsrc_id].tls_addr || p->thread_id == this_thread)) { if (resource_types_table[rsrc_id].dtor) { resource_types_table[rsrc_id].dtor(p->storage[rsrc_id]); } @@ -658,15 +635,20 @@ void ts_free_id(ts_rsrc_id id) TSRM_API void ts_apply_for_id(ts_rsrc_id id, void (*cb)(void *)) { int rsrc_id = TSRM_UNSHUFFLE_RSRC_ID(id); + THREAD_T this_thread = tsrm_thread_id(); tsrm_mutex_lock(tsmm_mutex); if (tsrm_tls_table && resource_types_table) { + bool tls_backed = resource_types_table[rsrc_id].tls_addr != NULL; + for (int i = 0; i < tsrm_tls_table_size; i++) { tsrm_tls_entry *p = tsrm_tls_table[i]; while (p) { - if (p->count > rsrc_id && p->storage[rsrc_id]) { + /* A __thread block of a foreign thread is inaccessible. */ + if (p->count > rsrc_id && p->storage[rsrc_id] + && (!tls_backed || p->thread_id == this_thread)) { cb(p->storage[rsrc_id]); } p = p->next; @@ -852,7 +834,10 @@ TSRM_API void *tsrm_get_ls_cache(void) /* Returns offset of tsrm_ls_cache slot from Thread Control Block address */ TSRM_API size_t tsrm_get_ls_cache_tcb_offset(void) {/*{{{*/ -#if defined(__APPLE__) && defined(__x86_64__) +#if defined(TSRM_TLS_MODEL_GLOBAL_DYNAMIC) + /* No constant TCB offset under global-dynamic, can't use fast path */ + return 0; +#elif defined(__APPLE__) && defined(__x86_64__) // TODO: Implement support for fast JIT ZTS code ??? return 0; #elif defined(__x86_64__) && defined(__GNUC__) && !defined(__FreeBSD__) && \ diff --git a/TSRM/TSRM.h b/TSRM/TSRM.h index 6a72dd3c02f3..dae2df38e86b 100644 --- a/TSRM/TSRM.h +++ b/TSRM/TSRM.h @@ -95,11 +95,6 @@ TSRM_API ts_rsrc_id ts_allocate_id(ts_rsrc_id *rsrc_id, size_t size, ts_allocate TSRM_API void tsrm_reserve(size_t size); TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor); -/* Fast resources at caller-chosen, compile-time-constant offsets. The fixed - * front region must be reserved after tsrm_reserve() and before any fast id. */ -TSRM_API void tsrm_reserve_fast_front(size_t size); -TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, ptrdiff_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor); - /* Resource whose per-thread storage is a native __thread block. * Must be called at startup before any other thread exists. */ TSRM_API ts_rsrc_id ts_allocate_tls_id(ts_rsrc_id *rsrc_id, void *(*tls_addr)(void), size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor); @@ -167,8 +162,13 @@ TSRM_API bool tsrm_is_managed_thread(void); # define TSRM_TLS_MODEL_ATTR # define TSRM_TLS_MODEL_DEFAULT #elif defined(__PIC__) && !defined(__PIE__) -# define TSRM_TLS_MODEL_ATTR __attribute__((tls_model("initial-exec"))) -# define TSRM_TLS_MODEL_INITIAL_EXEC +# if defined(TSRM_TLS_MODEL_USE_GLOBAL_DYNAMIC) +# define TSRM_TLS_MODEL_ATTR __attribute__((tls_model("global-dynamic"))) +# define TSRM_TLS_MODEL_GLOBAL_DYNAMIC +# else +# define TSRM_TLS_MODEL_ATTR __attribute__((tls_model("initial-exec"))) +# define TSRM_TLS_MODEL_INITIAL_EXEC +# endif #else # define TSRM_TLS_MODEL_ATTR __attribute__((tls_model("local-exec"))) # define TSRM_TLS_MODEL_LOCAL_EXEC @@ -186,17 +186,27 @@ TSRM_API bool tsrm_is_managed_thread(void); #define TSRMG_BULK_STATIC(id, type) ((type) (*((void ***) TSRMLS_CACHE))[TSRM_UNSHUFFLE_RSRC_ID(id)]) #define TSRMG_FAST_STATIC(offset, type, element) (TSRMG_FAST_BULK_STATIC(offset, type)->element) #define TSRMG_FAST_BULK_STATIC(offset, type) ((type) (((char*) TSRMLS_CACHE)+(offset))) +struct _zend_tsrm_ls_cache; +#if defined(ZEND_WIN32) && !defined(LIBZEND_EXPORTS) +/* Windows can't dllexport __declspec(thread) symbols, so outside Zend each module + * keeps a per-module `void *` pointer and reaches EG/CG via the resource-id indirection. */ +# define ZEND_TSRMLS_CACHE_T void * +# define TSRMLS_MAIN_CACHE_DEFINE() TSRM_TLS void *_tsrm_ls_cache TSRM_TLS_MODEL_ATTR = NULL; +# define TSRMLS_CACHE_DEFINE() TSRM_TLS void *_tsrm_ls_cache = NULL; +#else +# define ZEND_TSRMLS_CACHE_T struct _zend_tsrm_ls_cache +# define TSRMLS_MAIN_CACHE_DEFINE() +# define TSRMLS_CACHE_DEFINE() +#endif #ifdef __cplusplus -#define TSRMLS_MAIN_CACHE_EXTERN() extern "C" { extern TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR; } -#define TSRMLS_CACHE_EXTERN() extern "C" { extern TSRM_TLS void *TSRMLS_CACHE; } +#define TSRMLS_MAIN_CACHE_EXTERN() extern "C" { extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache TSRM_TLS_MODEL_ATTR; } +#define TSRMLS_CACHE_EXTERN() extern "C" { extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache; } #else -#define TSRMLS_MAIN_CACHE_EXTERN() extern TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR; -#define TSRMLS_CACHE_EXTERN() extern TSRM_TLS void *TSRMLS_CACHE; +#define TSRMLS_MAIN_CACHE_EXTERN() extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache TSRM_TLS_MODEL_ATTR; +#define TSRMLS_CACHE_EXTERN() extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache; #endif -#define TSRMLS_MAIN_CACHE_DEFINE() TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR = NULL; -#define TSRMLS_CACHE_DEFINE() TSRM_TLS void *TSRMLS_CACHE = NULL; #define TSRMLS_CACHE_UPDATE() TSRMLS_CACHE = tsrm_get_ls_cache() -#define TSRMLS_CACHE _tsrm_ls_cache +#define TSRMLS_CACHE (*(void **) &_tsrm_ls_cache) #ifdef __cplusplus } diff --git a/UPGRADING.INTERNALS b/UPGRADING.INTERNALS index 4be8421dd7b1..de7231f34af1 100644 --- a/UPGRADING.INTERNALS +++ b/UPGRADING.INTERNALS @@ -219,6 +219,8 @@ PHP 8.6 INTERNALS UPGRADE NOTES . Added zend_string_ends_with() and related variants. . Added trait support for internal classes. . Added do_php_cli(). + . The ZTS symbols compiler_globals_offset and executor_globals_offset have + been removed: CG()/EG() now reach the globals through a __thread struct. ======================== 2. Build system changes @@ -249,6 +251,10 @@ PHP 8.6 INTERNALS UPGRADE NOTES HAVE_STRUCT_STAT_ST_BLOCKS). . Added a new configure option --disable-apache2-conf to prevent apxs from editing httpd.conf during installation. + . Added --with-tsrm-tls-model (initial-exec, global-dynamic, auto) to pick the + TLS model for the ZTS globals cache; auto uses global-dynamic for dlopen()ed + SAPIs. Forcing initial-exec there needs a larger static TLS surplus: set + GLIBC_TUNABLES=glibc.rtld.optional_static_tls=8192 for the host process. - Windows build system changes: . Function SETUP_OPENSSL() doesn't accept 6th argument anymore and doesn't diff --git a/Zend/zend.c b/Zend/zend.c index 8643c248e6be..d98e7f24609a 100644 --- a/Zend/zend.c +++ b/Zend/zend.c @@ -52,17 +52,43 @@ static bool startup_done = false; #ifdef ZTS ZEND_API int compiler_globals_id; ZEND_API int executor_globals_id; -ZEND_API size_t compiler_globals_offset; -ZEND_API size_t executor_globals_offset; +ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_tsrm_ls_cache _tsrm_ls_cache = {0}; +#if defined(_WIN64) && defined(ZEND_TLS_DIRECT) +/* Holds &_tsrm_ls_cache in a TEB TLS slot (filled by DllMain) so EG()/CG() reach it + * with a single gs:[] load rather than the 3-load __declspec(thread) lookup. */ +ZEND_TLS_API unsigned long zend_win_tsrm_cache_slot = 0; +ZEND_API void zend_win_tsrm_cache_init(bool alloc) +{ + if (alloc) { + zend_win_tsrm_cache_slot = TlsAlloc(); + if (zend_win_tsrm_cache_slot == TLS_OUT_OF_INDEXES) { + fprintf(stderr, "PHP Startup: TlsAlloc() failed, no TLS slot available " + "for the ZTS globals cache\n"); + abort(); + } + if (zend_win_tsrm_cache_slot >= TLS_MINIMUM_AVAILABLE) { + /* Beyond the first TLS_MINIMUM_AVAILABLE slots the loader uses + * TEB->TlsExpansionSlots, which the inline gs:[] read in + * ZEND_TSRM_CACHE_PTR cannot reach. */ + fprintf(stderr, "PHP Startup: TlsAlloc() returned slot %lu, but only the " + "first %d direct TEB slots are usable for the ZTS globals cache\n", + zend_win_tsrm_cache_slot, TLS_MINIMUM_AVAILABLE); + abort(); + } + } + TlsSetValue(zend_win_tsrm_cache_slot, &_tsrm_ls_cache); +} +#endif /* ts_allocate_tls_id takes a callback so each thread resolves its own block. - * A plain &language_scanner_globals would capture only the registering thread's address. */ + * A plain &..._tls would capture only the registering thread's address. */ +static void *executor_globals_tls_addr(void) { return &_tsrm_ls_cache.eg; } +static void *compiler_globals_tls_addr(void) { return &_tsrm_ls_cache.cg; } static void *language_scanner_globals_tls_addr(void) { return &language_scanner_globals; } static HashTable *global_function_table = NULL; static HashTable *global_class_table = NULL; static HashTable *global_constants_table = NULL; static HashTable *global_auto_globals_table = NULL; static HashTable *global_persistent_list = NULL; -TSRMLS_MAIN_CACHE_DEFINE() # define GLOBAL_FUNCTION_TABLE global_function_table # define GLOBAL_CLASS_TABLE global_class_table # define GLOBAL_CONSTANTS_TABLE global_constants_table @@ -804,6 +830,7 @@ static void compiler_globals_dtor(zend_compiler_globals *compiler_globals) /* {{ static void executor_globals_ctor(zend_executor_globals *executor_globals) /* {{{ */ { + _tsrm_ls_cache.self = &_tsrm_ls_cache; zend_startup_constants(); zend_copy_constants(executor_globals->zend_constants, GLOBAL_CONSTANTS_TABLE); zend_init_rsrc_plist(); @@ -1022,11 +1049,9 @@ void zend_startup(zend_utility_functions *utility_functions) /* {{{ */ zend_init_rsrc_list_dtors(); #ifdef ZTS - ts_allocate_fast_id_at(&compiler_globals_id, &compiler_globals_offset, ZEND_CG_OFFSET, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor); - ts_allocate_fast_id_at(&executor_globals_id, &executor_globals_offset, ZEND_EG_OFFSET, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor); + ts_allocate_tls_id(&compiler_globals_id, compiler_globals_tls_addr, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor); + ts_allocate_tls_id(&executor_globals_id, executor_globals_tls_addr, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor); ts_allocate_tls_id(&language_scanner_globals_id, language_scanner_globals_tls_addr, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL); - ZEND_ASSERT(compiler_globals_offset == ZEND_CG_OFFSET); - ZEND_ASSERT(executor_globals_offset == ZEND_EG_OFFSET); ts_allocate_fast_id(&ini_scanner_globals_id, &ini_scanner_globals_offset, sizeof(zend_ini_scanner_globals), (ts_allocate_ctor) ini_scanner_globals_ctor, NULL); compiler_globals = ts_resource(compiler_globals_id); executor_globals = ts_resource(executor_globals_id); diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h index 4d5e300e2859..7c5412bc4ad1 100644 --- a/Zend/zend_globals.h +++ b/Zend/zend_globals.h @@ -51,8 +51,6 @@ BEGIN_EXTERN_C() ZEND_API extern int compiler_globals_id; ZEND_API extern int executor_globals_id; -ZEND_API extern size_t compiler_globals_offset; -ZEND_API extern size_t executor_globals_offset; END_EXTERN_C() #endif @@ -332,9 +330,12 @@ struct _zend_executor_globals { }; #ifdef ZTS -/* Compile-time offsets of the hot globals, in a reserved region just before *_tsrm_ls_cache. */ -# define ZEND_CG_OFFSET (-(ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals))) -# define ZEND_EG_OFFSET (ZEND_CG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals))) +struct _zend_tsrm_ls_cache { + void *cache; + void *self; + zend_executor_globals eg; + zend_compiler_globals cg; +}; #endif #define EG_FLAGS_INITIAL (0) diff --git a/Zend/zend_globals_macros.h b/Zend/zend_globals_macros.h index adb3913ece71..2ddb8132c407 100644 --- a/Zend/zend_globals_macros.h +++ b/Zend/zend_globals_macros.h @@ -26,11 +26,39 @@ typedef struct _zend_executor_globals zend_executor_globals; typedef struct _zend_php_scanner_globals zend_php_scanner_globals; typedef struct _zend_ini_scanner_globals zend_ini_scanner_globals; +#ifdef ZEND_WIN32 +# define ZEND_TLS_API +# ifdef LIBZEND_EXPORTS +# define ZEND_TLS_DIRECT 1 +# endif +#else +# define ZEND_TLS_API ZEND_API +# define ZEND_TLS_DIRECT 1 +#endif + BEGIN_EXTERN_C() +#ifdef ZTS +typedef struct _zend_tsrm_ls_cache zend_tsrm_ls_cache; +# ifdef ZEND_TLS_DIRECT +extern ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_tsrm_ls_cache _tsrm_ls_cache; +/* See zend.c: zend_win_tsrm_cache_init */ +# if defined(_WIN64) +extern ZEND_TLS_API unsigned long zend_win_tsrm_cache_slot; +# define ZEND_TSRM_CACHE_PTR ((zend_tsrm_ls_cache*)__readgsqword(0x1480 + zend_win_tsrm_cache_slot * 8)) +# else +# define ZEND_TSRM_CACHE_PTR (&_tsrm_ls_cache) +# endif +# endif +#endif + /* Compiler */ #ifdef ZTS -# define CG(v) ZEND_TSRMG_FAST(ZEND_CG_OFFSET, zend_compiler_globals *, v) +# ifdef ZEND_TLS_DIRECT +# define CG(v) (ZEND_TSRM_CACHE_PTR->cg.v) +# else +# define CG(v) ZEND_TSRMG(compiler_globals_id, zend_compiler_globals *, v) +# endif #else # define CG(v) (compiler_globals.v) extern ZEND_API struct _zend_compiler_globals compiler_globals; @@ -40,7 +68,11 @@ ZEND_API int zendparse(void); /* Executor */ #ifdef ZTS -# define EG(v) ZEND_TSRMG_FAST(ZEND_EG_OFFSET, zend_executor_globals *, v) +# ifdef ZEND_TLS_DIRECT +# define EG(v) (ZEND_TSRM_CACHE_PTR->eg.v) +# else +# define EG(v) ZEND_TSRMG(executor_globals_id, zend_executor_globals *, v) +# endif #else # define EG(v) (executor_globals.v) extern ZEND_API zend_executor_globals executor_globals; diff --git a/configure.ac b/configure.ac index 3229e0dac040..ffcbac6fa2c2 100644 --- a/configure.ac +++ b/configure.ac @@ -284,6 +284,9 @@ SAPI_SHARED=libs/$SAPI_LIBNAME_SHARED SAPI_STATIC=libs/$SAPI_LIBNAME_STATIC SAPI_LIBTOOL=libphp.la +dnl Set by SAPIs whose host program dlopen()s libphp without linking against it. +php_tsrm_dlopened_sapi=no + PHP_CONFIGURE_PART([Configuring SAPI modules]) esyscmd(./build/config-stubs sapi) @@ -887,6 +890,38 @@ AS_VAR_IF([PHP_THREAD_SAFETY], [yes], [ AC_MSG_RESULT([yes]) ]) +PHP_ARG_WITH([tsrm-tls-model],, + [AS_HELP_STRING([[--with-tsrm-tls-model=MODEL]], + [TLS model for the ZTS globals cache: initial-exec, global-dynamic, or + auto for global-dynamic on dlopen()ed SAPIs and initial-exec elsewhere + [auto]])], + [auto], + [no]) + +AS_VAR_IF([PHP_THREAD_SAFETY], [yes], [ + AS_CASE([$PHP_TSRM_TLS_MODEL], + [auto], [ + AS_VAR_IF([php_tsrm_dlopened_sapi], [yes], + [php_tsrm_tls_model=global-dynamic], + [php_tsrm_tls_model=initial-exec]) + ], + [initial-exec|global-dynamic], [php_tsrm_tls_model=$PHP_TSRM_TLS_MODEL], + [AC_MSG_ERROR(m4_text_wrap([ + Unknown TLS model '$PHP_TSRM_TLS_MODEL' given to --with-tsrm-tls-model. + Valid values are: auto, initial-exec, global-dynamic. + ]))]) + + AC_MSG_CHECKING([which TLS model to use for the ZTS globals cache]) + AC_MSG_RESULT([$php_tsrm_tls_model]) + + dnl In php_config.h, not CFLAGS, so phpize builds use the same model. + AS_VAR_IF([php_tsrm_tls_model], [global-dynamic], [ + AC_DEFINE([TSRM_TLS_MODEL_USE_GLOBAL_DYNAMIC], [1], + [Define to 1 to use the global-dynamic TLS model for the ZTS globals + cache.]) + ]) +]) + PHP_ARG_ENABLE([rtld-now], [whether to dlopen extensions with RTLD_NOW instead of RTLD_LAZY], [AS_HELP_STRING([--enable-rtld-now], diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index fc4bb84f1e05..a7307c4f1653 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -5851,26 +5851,20 @@ static void ir_emit_tls(ir_ctx *ctx, ir_ref def, ir_insn *insn) ||#else || code = 0xd53bd040 | reg; // TODO: hard-coded: mrs reg, tpidr_el0 | .long code -||# ifdef __FreeBSD__ -|| if (insn->op3 == IR_NULL) { -| ldr Rx(reg), [Rx(reg), #insn->op2] -|| } else { -| ldr Rx(reg), [Rx(reg), #0] +||# ifdef __MUSL__ +||# define IR_DTV_OFFSET -8 +||# else +||# define IR_DTV_OFFSET 0 +||# endif +|| if (insn->op3 != IR_NULL) { +| ldr Rx(reg), [Rx(reg), #IR_DTV_OFFSET] | ldr Rx(reg), [Rx(reg), #insn->op2] | ldr Rx(reg), [Rx(reg), #insn->op3] -|| } -||# elif defined(__MUSL__) -|| if (insn->op3 == IR_NULL) { -| ldr Rx(reg), [Rx(reg), #insn->op2] -|| } else { -| ldr Rx(reg), [Rx(reg), #-8] +|| } else if (insn->op2 != 0) { +||//??? IR_ASSERT(insn->op2 <= LDR_STR_PIMM64); | ldr Rx(reg), [Rx(reg), #insn->op2] -| ldr Rx(reg), [Rx(reg), #insn->op3] || } -||# else -||//??? IR_ASSERT(insn->op2 <= LDR_STR_PIMM64); -| ldr Rx(reg), [Rx(reg), #insn->op2] -||# endif +||# undef IR_DTV_OFFSET ||#endif if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, IR_ADDR, def, reg); diff --git a/ext/opcache/jit/tls/zend_jit_tls_win.c b/ext/opcache/jit/tls/zend_jit_tls_win.c index 5646f3dcba0b..58fab12bf386 100644 --- a/ext/opcache/jit/tls/zend_jit_tls_win.c +++ b/ext/opcache/jit/tls/zend_jit_tls_win.c @@ -33,30 +33,21 @@ zend_result zend_jit_resolve_tsrm_ls_cache_offsets( size_t *module_index, size_t *module_offset ) { - /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ - /* Probably, it might be better solution */ + /* Offset of _tsrm_ls_cache within this module's TLS block. */ #ifdef _WIN64 void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index]; #else void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index]; #endif - void *val = _tsrm_ls_cache; - size_t offset = 0; size_t size = (char*)&_tls_end - (char*)&_tls_start; - - while (offset < size) { - if (*tls_mem == val) { - *module_index = _tls_index * sizeof(void*); - *module_offset = offset; - return SUCCESS; - } - tls_mem++; - offset += sizeof(void*); - } + size_t offset = (size_t)((char*)&_tsrm_ls_cache - (char*)tls_mem); if (offset >= size) { zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size"); + return FAILURE; } - return FAILURE; + *module_index = _tls_index * sizeof(void*); + *module_offset = offset; + return SUCCESS; } diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 2bbd7b0e3f4d..47f2e1879fcf 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -206,10 +206,10 @@ static size_t tsrm_tls_index = -1; static size_t tsrm_tls_offset = -1; # define EG_TLS_OFFSET(field) \ - (executor_globals_offset + offsetof(zend_executor_globals, field)) + (tsrm_ls_cache_tcb_offset + offsetof(zend_tsrm_ls_cache, eg) + offsetof(zend_executor_globals, field)) # define CG_TLS_OFFSET(field) \ - (compiler_globals_offset + offsetof(zend_compiler_globals, field)) + (tsrm_ls_cache_tcb_offset + offsetof(zend_tsrm_ls_cache, cg) + offsetof(zend_compiler_globals, field)) # define jit_EG(_field) \ ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field)) @@ -496,7 +496,7 @@ static const char* zend_reg_name(int8_t reg) #ifdef ZTS static void * ZEND_FASTCALL zend_jit_get_tsrm_ls_cache(void) { - return _tsrm_ls_cache; + return &_tsrm_ls_cache; } static ir_ref jit_TLS(zend_jit_ctx *jit) @@ -521,10 +521,10 @@ static ir_ref jit_TLS(zend_jit_ctx *jit) if (tsrm_ls_cache_tcb_offset == 0 && tsrm_tls_index == -1) { jit->tls = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_tsrm_ls_cache)); + } else if (tsrm_ls_cache_tcb_offset) { + jit->tls = ir_TLS(0, IR_NULL); } else { - jit->tls = ir_TLS( - tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index, - tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset); + jit->tls = ir_TLS(tsrm_tls_index, tsrm_tls_offset + offsetof(zend_tsrm_ls_cache, self)); } return jit->tls; diff --git a/main/main.c b/main/main.c index 0539220de362..b6fd413f25fa 100644 --- a/main/main.c +++ b/main/main.c @@ -2750,9 +2750,8 @@ PHPAPI zend_result php_lint_script(zend_file_handle *file) /* {{{ php_reserve_tsrm_memory */ PHPAPI void php_reserve_tsrm_memory(void) { + /* CG/EG live in native __thread storage and need no reserved TSRM space. */ tsrm_reserve( - TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)) + - TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)) + TSRM_ALIGNED_SIZE(sizeof(zend_ini_scanner_globals)) + TSRM_ALIGNED_SIZE(sizeof(virtual_cwd_globals)) + #ifdef ZEND_SIGNALS @@ -2774,10 +2773,6 @@ PHPAPI bool php_tsrm_startup_ex(int expected_threads) { bool ret = tsrm_startup(expected_threads, 1, 0, NULL); php_reserve_tsrm_memory(); - /* Must cover the total size of every ZEND_*_OFFSET global, or the furthest underflows the block. */ - tsrm_reserve_fast_front( - TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)) + - TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals))); (void)ts_resource(0); return ret; } diff --git a/sapi/apache2handler/config.m4 b/sapi/apache2handler/config.m4 index 3001a4d61d9a..c4498f1dae30 100644 --- a/sapi/apache2handler/config.m4 +++ b/sapi/apache2handler/config.m4 @@ -94,6 +94,12 @@ if test "$PHP_APXS2" != "no"; then LIBPHP_CFLAGS="-shared" PHP_SUBST([LIBPHP_CFLAGS]) + dnl httpd dlopen's libphp.so without linking against it, so we must + dnl use global-dynamic by default (_tsrm_ls_cache overflows the default + dnl static TLS surplus). Force --with-tsrm-tls-model=initil-exec to + dnl overwrite this behaviour, see UPGRADING.INTERNALS + php_tsrm_dlopened_sapi=yes + php_sapi_apache2handler_type=shared AS_CASE([$host_alias], [*aix*], [ diff --git a/win32/dllmain.c b/win32/dllmain.c index 168bef2baa22..bb9386b007ac 100644 --- a/win32/dllmain.c +++ b/win32/dllmain.c @@ -27,10 +27,22 @@ eq. initializing something before the DLL even is available to be called. */ +#if defined(_WIN64) && defined(ZTS) +ZEND_API void zend_win_tsrm_cache_init(bool alloc); +#endif + BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID dummy) { BOOL ret = TRUE; +#if defined(_WIN64) && defined(ZTS) + if (reason == DLL_PROCESS_ATTACH) { + zend_win_tsrm_cache_init(true); + } else if (reason == DLL_THREAD_ATTACH) { + zend_win_tsrm_cache_init(false); + } +#endif + #ifdef HAVE_LIBXML /* This imply that only LIBXML_STATIC_FOR_DLL is supported ATM. If that changes, this place will need some rework.