Skip to content

gh-119726: generate and patch AArch64 trampolines #123872

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brandtbucher merged 15 commits into from
Oct 2, 2024
+147 −59
Merged

gh-119726: generate and patch AArch64 trampolines #123872

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
b94d6ac
gh-119726: generate and patch AArch64 trampolines
diegorusso Sep 6, 2024
ef8c591
Fix Windows builds.
diegorusso Sep 10, 2024
ce3744f
fix pointer
diegorusso Sep 10, 2024
33f433b
Fix internal compiler error on Windows
diegorusso Sep 10, 2024
3693c00
Address Brandt's feedback.
diegorusso Sep 19, 2024
cc32850
📜🤖 Added by blurb_it.
blurb-it[bot] Sep 19, 2024
9c68b8c
Fix windows builds (again)
diegorusso Sep 19, 2024
473cb7d
Remove trampoline_state as global variable
diegorusso Sep 26, 2024
e252608
Fix linter error
diegorusso Sep 26, 2024
92c4151
Update news file
diegorusso Sep 27, 2024
645157c
Address Brandt's feedback
diegorusso Sep 30, 2024
345ba2e
Use TRAMPOLINE_SIZE instead of TRAMPOLINE_SIZE_AARCH64
diegorusso Oct 2, 2024
835ebf4
Change typehin for known_symbols data structure.
diegorusso Oct 2, 2024
6723f87
Fix issue in debug builds.
diegorusso Oct 2, 2024
27436f5
Remove unused file.
diegorusso Oct 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Address Brandt's feedback
  • Loading branch information
@diegorusso
diegorusso committed Oct 2, 2024
commit 645157cadc208cc853a7bc0e8c1dc9c884c9a1c7
36 changes: 15 additions & 21 deletions Python/jit.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,18 +116,18 @@ mark_executable(unsigned char *memory, size_t size)

#define SYMBOL_MASK_WORDS 4

typedef uint32_t SymbolMask[SYMBOL_MASK_WORDS];
typedef uint32_t symbol_mask[SYMBOL_MASK_WORDS];

typedef struct {
unsigned char *mem;
SymbolMask mask;
symbol_mask mask;
size_t size;
} TrampolineState;
} trampoline_state;

typedef struct {
TrampolineState trampolines;
trampoline_state trampolines;
uintptr_t instruction_starts[UOP_MAX_TRACE_LENGTH];
} CompileState;
} jit_state;

// Warning! AArch64 requires you to get your hands dirty. These are your gloves:

Expand Down Expand Up @@ -406,20 +406,16 @@ patch_x86_64_32rx(unsigned char *location, uint64_t value)
patch_32r(location, value);
}

void patch_aarch64_trampoline(unsigned char *location, int ordinal, CompileState *state);
void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state);

#include "jit_stencils.h"

#if defined(__aarch64__) || defined(_M_ARM64)
#define TRAMPOLINE_SIZE 16
#else
#define TRAMPOLINE_SIZE 0
#endif
#define TRAMPOLINE_SIZE_AARCH64 16

// Generate and patch AArch64 trampolines. The symbols to jump to are stored
// in the jit_stencils.h in the symbols_map.
void
patch_aarch64_trampoline(unsigned char *location, int ordinal, CompileState *state)
patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state)
{
// Masking is done modulo 32 as the mask is stored as an array of uint32_t
const uint32_t symbol_mask = 1 << (ordinal % 32);
Expand All @@ -433,8 +429,8 @@ patch_aarch64_trampoline(unsigned char *location, int ordinal, CompileState *sta
index += _Py_popcount32(state->trampolines.mask[i]);
}
Comment thread
diegorusso marked this conversation as resolved.

uint32_t *p = (uint32_t*)(state->trampolines.mem + index * TRAMPOLINE_SIZE);
assert((size_t)index * TRAMPOLINE_SIZE < state->trampolines.size);
uint32_t *p = (uint32_t*)(state->trampolines.mem + index * TRAMPOLINE_SIZE_AARCH64);
assert((size_t)(index + 1) * TRAMPOLINE_SIZE_AARCH64 < state->trampolines.size);

uint64_t value = (uintptr_t)symbols_map[ordinal];

Expand All @@ -453,7 +449,7 @@ patch_aarch64_trampoline(unsigned char *location, int ordinal, CompileState *sta
}

static void
combine_symbol_mask(const SymbolMask src, SymbolMask dest)
combine_symbol_mask(const symbol_mask src, symbol_mask dest)
{
// Calculate the union of the trampolines required by each StencilGroup
for (size_t i = 0; i < SYMBOL_MASK_WORDS; i++) {
Expand All @@ -469,7 +465,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz
// Loop once to find the total compiled size:
size_t code_size = 0;
size_t data_size = 0;
CompileState state = {};
jit_state state = {};
group = &trampoline;
code_size += group->code_size;
data_size += group->data_size;
Comment thread
diegorusso marked this conversation as resolved.
Expand All @@ -479,17 +475,15 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz
state.instruction_starts[i] = code_size;
code_size += group->code_size;
data_size += group->data_size;
combine_symbol_mask(group->trampoline_mask,
state.trampolines.mask);
combine_symbol_mask(group->trampoline_mask, state.trampolines.mask);
}
group = &stencil_groups[_FATAL_ERROR];
code_size += group->code_size;
data_size += group->data_size;
combine_symbol_mask(group->trampoline_mask,
state.trampolines.mask);
combine_symbol_mask(group->trampoline_mask, state.trampolines.mask);
// Calculate the size of the trampolines required by the whole trace
for (size_t i = 0; i < Py_ARRAY_LENGTH(state.trampolines.mask); i++) {
state.trampolines.size += _Py_popcount32(state.trampolines.mask[i]) * TRAMPOLINE_SIZE;
state.trampolines.size += _Py_popcount32(state.trampolines.mask[i]) * TRAMPOLINE_SIZE_AARCH64;
}
// Round up to the nearest page:
size_t page_size = get_page_size();
Expand Down
6 changes: 3 additions & 3 deletions Tools/jit/_writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ def _dump_footer(
yield "typedef struct {"
yield " void (*emit)("
yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor,"
yield " const _PyUOpInstruction *instruction, CompileState *state);"
yield " const _PyUOpInstruction *instruction, jit_state *state);"
yield " size_t code_size;"
yield " size_t data_size;"
yield " SymbolMask trampoline_mask;"
yield " symbol_mask trampoline_mask;"
yield "} StencilGroup;"
yield ""
yield f"static const StencilGroup trampoline = {groups['trampoline'].as_c('trampoline')};"
Expand All @@ -41,7 +41,7 @@ def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator
yield "void"
yield f"emit_{opname}("
yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor,"
yield " const _PyUOpInstruction *instruction, CompileState *state)"
yield " const _PyUOpInstruction *instruction, jit_state *state)"
yield "{"
for part, stencil in [("code", group.code), ("data", group.data)]:
for line in stencil.disassembly:
Expand Down