Skip to content

Commit 9b3e15f

Browse files
brettcannonbrettcannon
committed
Importlib was not matching import's handling of .pyc files where it had less
then 8 bytes total in the file. Fixes issues 7361 & 7875.
1 parent 055470a commit 9b3e15f

2 files changed

Lines changed: 109 additions & 29 deletions

File tree

‎Lib/importlib/_bootstrap.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -396,19 +396,24 @@ def get_code(self, fullname):
396396
bytecode_path = self.bytecode_path(fullname)
397397
if bytecode_path:
398398
data = self.get_data(bytecode_path)
399-
magic = data[:4]
400-
pyc_timestamp = marshal._r_long(data[4:8])
401-
bytecode = data[8:]
402399
try:
400+
magic = data[:4]
401+
if len(magic) < 4:
402+
raise ImportError("bad magic number in {}".format(fullname))
403+
raw_timestamp = data[4:8]
404+
if len(raw_timestamp) < 4:
405+
raise EOFError("bad timestamp in {}".format(fullname))
406+
pyc_timestamp = marshal._r_long(raw_timestamp)
407+
bytecode = data[8:]
403408
# Verify that the magic number is valid.
404409
if imp.get_magic() != magic:
405-
raise ImportError("bad magic number")
410+
raise ImportError("bad magic number in {}".format(fullname))
406411
# Verify that the bytecode is not stale (only matters when
407412
# there is source to fall back on.
408413
if source_timestamp:
409414
if pyc_timestamp < source_timestamp:
410415
raise ImportError("bytecode is stale")
411-
except ImportError:
416+
except (ImportError, EOFError):
412417
# If source is available give it a shot.
413418
if source_timestamp is not None:
414419
pass

‎Lib/importlib/test/source/test_file_loader.py‎

Lines changed: 99 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -113,42 +113,116 @@ def test_bad_syntax(self):
113113

114114
class BadBytecodeTest(unittest.TestCase):
115115

116-
"""But there are several things about the bytecode which might lead to the
117-
source being preferred. If the magic number differs from what the
118-
interpreter uses, then the source is used with the bytecode regenerated.
119-
If the timestamp is older than the modification time for the source then
120-
the bytecode is not used [bad timestamp].
121-
122-
But if the marshal data is bad, even if the magic number and timestamp
123-
work, a ValueError is raised and the source is not used [bad marshal].
124-
125-
The case of not being able to write out the bytecode must also be handled
126-
as it's possible it was made read-only. In that instance the attempt to
127-
write the bytecode should fail silently [bytecode read-only].
128-
129-
"""
130-
131116
def import_(self, file, module_name):
132117
loader = _bootstrap._PyPycFileLoader(module_name, file, False)
133118
module = loader.load_module(module_name)
134119
self.assertTrue(module_name in sys.modules)
135120

136-
# [bad magic]
121+
def manipulate_bytecode(self, name, mapping, manipulator, *,
122+
del_source=False):
123+
"""Manipulate the bytecode of a module by passing it into a callable
124+
that returns what to use as the new bytecode."""
125+
try:
126+
del sys.modules['_temp']
127+
except KeyError:
128+
pass
129+
py_compile.compile(mapping[name])
130+
bytecode_path = source_util.bytecode_path(mapping[name])
131+
with open(bytecode_path, 'rb') as file:
132+
bc = file.read()
133+
new_bc = manipulator(bc)
134+
with open(bytecode_path, 'wb') as file:
135+
if new_bc:
136+
file.write(new_bc)
137+
if del_source:
138+
os.unlink(mapping[name])
139+
return bytecode_path
140+
141+
@source_util.writes_bytecode_files
142+
def test_empty_file(self):
143+
# When a .pyc is empty, regenerate it if possible, else raise
144+
# ImportError.
145+
with source_util.create_modules('_temp') as mapping:
146+
bc_path = self.manipulate_bytecode('_temp', mapping,
147+
lambda bc: None)
148+
self.import_(mapping['_temp'], '_temp')
149+
with open(bc_path, 'rb') as file:
150+
self.assertGreater(len(file.read()), 8)
151+
self.manipulate_bytecode('_temp', mapping, lambda bc: None,
152+
del_source=True)
153+
with self.assertRaises(ImportError):
154+
self.import_(mapping['_temp'], '_temp')
155+
156+
@source_util.writes_bytecode_files
157+
def test_partial_magic(self):
158+
# When their are less than 4 bytes to a .pyc, regenerate it if
159+
# possible, else raise ImportError.
160+
with source_util.create_modules('_temp') as mapping:
161+
bc_path = self.manipulate_bytecode('_temp', mapping,
162+
lambda bc: bc[:3])
163+
self.import_(mapping['_temp'], '_temp')
164+
with open(bc_path, 'rb') as file:
165+
self.assertGreater(len(file.read()), 8)
166+
self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:3],
167+
del_source=True)
168+
with self.assertRaises(ImportError):
169+
self.import_(mapping['_temp'], '_temp')
170+
171+
@source_util.writes_bytecode_files
172+
def test_magic_only(self):
173+
# When there is only the magic number, regenerate the .pyc if possible,
174+
# else raise EOFError.
175+
with source_util.create_modules('_temp') as mapping:
176+
bc_path = self.manipulate_bytecode('_temp', mapping,
177+
lambda bc: bc[:4])
178+
self.import_(mapping['_temp'], '_temp')
179+
with open(bc_path, 'rb') as file:
180+
self.assertGreater(len(file.read()), 8)
181+
self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:4],
182+
del_source=True)
183+
with self.assertRaises(EOFError):
184+
self.import_(mapping['_temp'], '_temp')
185+
186+
@source_util.writes_bytecode_files
187+
def test_partial_timestamp(self):
188+
# When the timestamp is partial, regenerate the .pyc, else
189+
# raise EOFError.
190+
with source_util.create_modules('_temp') as mapping:
191+
bc_path = self.manipulate_bytecode('_temp', mapping,
192+
lambda bc: bc[:7])
193+
self.import_(mapping['_temp'], '_temp')
194+
with open(bc_path, 'rb') as file:
195+
self.assertGreater(len(file.read()), 8)
196+
self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:7],
197+
del_source=True)
198+
with self.assertRaises(EOFError):
199+
self.import_(mapping['_temp'], '_temp')
200+
201+
@source_util.writes_bytecode_files
202+
def test_no_marshal(self):
203+
# When there is only the magic number and timestamp, raise EOFError.
204+
with source_util.create_modules('_temp') as mapping:
205+
bc_path = self.manipulate_bytecode('_temp', mapping,
206+
lambda bc: bc[:8])
207+
with self.assertRaises(EOFError):
208+
self.import_(mapping['_temp'], '_temp')
209+
137210
@source_util.writes_bytecode_files
138211
def test_bad_magic(self):
212+
# When the magic number is different, the bytecode should be
213+
# regenerated.
139214
with source_util.create_modules('_temp') as mapping:
140-
py_compile.compile(mapping['_temp'])
141-
bytecode_path = source_util.bytecode_path(mapping['_temp'])
142-
with open(bytecode_path, 'r+b') as bytecode_file:
143-
bytecode_file.seek(0)
144-
bytecode_file.write(b'\x00\x00\x00\x00')
215+
bc_path = self.manipulate_bytecode('_temp', mapping,
216+
lambda bc: b'\x00\x00\x00\x00' + bc[4:])
145217
self.import_(mapping['_temp'], '_temp')
146-
with open(bytecode_path, 'rb') as bytecode_file:
218+
with open(bc_path, 'rb') as bytecode_file:
147219
self.assertEqual(bytecode_file.read(4), imp.get_magic())
148220

149221
# [bad timestamp]
150222
@source_util.writes_bytecode_files
151223
def test_bad_bytecode(self):
224+
# When the timestamp is older than the source, bytecode should be
225+
# regenerated.
152226
zeros = b'\x00\x00\x00\x00'
153227
with source_util.create_modules('_temp') as mapping:
154228
py_compile.compile(mapping['_temp'])
@@ -166,6 +240,7 @@ def test_bad_bytecode(self):
166240
# [bad marshal]
167241
@source_util.writes_bytecode_files
168242
def test_bad_marshal(self):
243+
# Bad marshal data should raise a ValueError.
169244
with source_util.create_modules('_temp') as mapping:
170245
bytecode_path = source_util.bytecode_path(mapping['_temp'])
171246
source_mtime = os.path.getmtime(mapping['_temp'])
@@ -181,6 +256,7 @@ def test_bad_marshal(self):
181256
# [bytecode read-only]
182257
@source_util.writes_bytecode_files
183258
def test_read_only_bytecode(self):
259+
# When bytecode is read-only but should be rewritten, fail silently.
184260
with source_util.create_modules('_temp') as mapping:
185261
# Create bytecode that will need to be re-created.
186262
py_compile.compile(mapping['_temp'])
@@ -201,8 +277,7 @@ def test_read_only_bytecode(self):
201277

202278
def test_main():
203279
from test.support import run_unittest
204-
run_unittest(SimpleTest, DontWriteBytecodeTest, BadDataTest,
205-
SourceBytecodeInteraction, BadBytecodeTest)
280+
run_unittest(SimpleTest, BadBytecodeTest)
206281

207282

208283
if __name__ == '__main__':

0 commit comments

Comments
 (0)