Skip to content

GH-99554: marshal bytecode more efficiently #99555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
brandtbucher wants to merge 9 commits into from
Closed

GH-99554: marshal bytecode more efficiently #99555

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
4034318
Compress marshalled bytecode
brandtbucher Nov 15, 2022
8b6b393
blurb add
brandtbucher Nov 16, 2022
5f81b82
Add opcode_for_build
brandtbucher Nov 16, 2022
e34c460
Catch up with main
brandtbucher Dec 15, 2022
23ba802
fixup
brandtbucher Dec 15, 2022
6ed83af
Catch up with main
brandtbucher Dec 17, 2022
5534112
Address review comments
brandtbucher Dec 17, 2022
ead8f26
Catch up with main
brandtbucher Dec 17, 2022
bcd7980
Terminate the compressed bytecode with two zeros
brandtbucher Dec 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Address review comments
  • Loading branch information
@brandtbucher
brandtbucher committed Dec 17, 2022
commit 5534112bb0d20ef4c1c6a773efef94245350cd74
7 changes: 5 additions & 2 deletions Python/marshal.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -972,10 +972,13 @@ r_bytecode(RFILE *p)
assert(0x00 <= opcode && opcode < 0x100);
assert(0x00 <= oparg && oparg < 0x100);
buffer[i].opcode = opcode;
buffer[i++].oparg = oparg;
buffer[i].oparg = oparg;
i++;
for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) {
assert(i < size);
buffer[i].opcode = CACHE;
buffer[i++].oparg = 0;
buffer[i].oparg = 0;
i++;
}
}
if (i != size) {
Expand Down
4 changes: 3 additions & 1 deletion Tools/build/deepfreeze.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
from typing import Dict, FrozenSet, TextIO, Tuple

import umarshal
import opcode_for_build as opcode
import opcode_for_build
from generate_global_objects import get_identifiers_and_strings

opcode = opcode_for_build.import_opcode()

verbose = False
identifiers, strings = get_identifiers_and_strings()

Expand Down
24 changes: 17 additions & 7 deletions Tools/build/opcode_for_build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,26 @@
Parts of our build process (looking at you, deepfreeze) need the opcode module
for the Python *being built*, not the Python *doing the building*.

This basically just loads ../../Lib/opcode.py and re-exports everything:
This basically just loads ../../Lib/opcode.py:

>>> import opcode_for_build as opcode
>>> import opcode_for_build
>>> opcode = opcode_for_build.import_opcode()
"""

import os
import types

_opcode_path = os.path.join(
os.path.dirname(__file__), os.pardir, os.pardir, "Lib", "opcode.py"
_OPCODE_PATH = os.path.realpath(
os.path.join(
os.path.dirname(__file__), os.pardir, os.pardir, "Lib", "opcode.py"
)
)
with open(_opcode_path, encoding="utf-8") as _opcode_file:
# Don't try this at home, kids:
exec(_opcode_file.read())

def import_opcode() -> types.ModuleType:
"""Import the current version of the opcode module (from Lib)."""
opcode_module = types.ModuleType("opcode")
opcode_module.__file__ = os.path.realpath(_OPCODE_PATH)
with open(_OPCODE_PATH, encoding="utf-8") as opcode_file:
# Don't try this at home, kids:
exec(opcode_file.read(), opcode_module.__dict__)
return opcode_module
6 changes: 4 additions & 2 deletions Tools/build/umarshal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
# Implementat marshal.loads() in pure Python

import ast
import opcode_for_build as opcode
import opcode_for_build

from typing import Any, Tuple

opcode = opcode_for_build.import_opcode()


class Type:
# Adapted from marshal.c
Expand Down Expand Up @@ -186,7 +188,7 @@ def r_bytecode(self) -> bytes:
bytecode = bytearray()
while len(bytecode) < nbytes:
opcode_byte = self.r_byte()
if opcode.HAVE_ARGUMENT <= opcode_byte:
if opcode_byte >= opcode.HAVE_ARGUMENT:
oparg_byte = self.r_byte()
else:
oparg_byte = 0
Expand Down