Skip to content

Commit f918089

Browse files
msimacekmsimacek
committed
Fix overrides mechanism
1 parent c3b2b26 commit f918089

1 file changed

Lines changed: 83 additions & 31 deletions

File tree

‎mx.graalpython/mx_graalpython_import.py‎

Lines changed: 83 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import abc
12
import os
23
import shutil
34
import sys
@@ -9,54 +10,99 @@
910
SUITE = mx.suite('graalpython')
1011

1112

12-
class CopyFrom:
13-
def __init__(self, source_path, copy_all=False):
13+
class AbstractRule(abc.ABC):
14+
def __init__(self, source_path):
1415
self.source_path = source_path
15-
self.copy_all = copy_all
1616

17-
def copy(self, basedir, graalpy_path, overrides):
18-
src = os.path.join(basedir, self.source_path)
17+
@abc.abstractmethod
18+
def copy(self, source_dir, graalpy_dir, graalpy_path, overrides):
19+
pass
20+
21+
22+
def match_overrides(graalpy_path, overrides):
23+
graalpy_prefix = f"{graalpy_path}/"
24+
matches = [f for f in overrides if f == graalpy_path or f.startswith(graalpy_prefix)]
25+
for match in matches:
26+
overrides.remove(match)
27+
return matches
28+
29+
30+
class Ignore(AbstractRule):
31+
"""
32+
A rule that ignores a file a directory.
33+
"""
34+
35+
def __init__(self):
36+
super().__init__(None)
37+
38+
def copy(self, source_dir, graalpy_dir, graalpy_path, overrides):
39+
match_overrides(graalpy_path, overrides)
40+
41+
42+
class CopyFrom(AbstractRule):
43+
"""
44+
A rule that copies a file or a whole directory recursively.
45+
"""
46+
47+
def copy(self, source_dir, graalpy_dir, graalpy_path, overrides):
48+
match_overrides(graalpy_path, overrides)
49+
src = os.path.join(source_dir, self.source_path)
50+
dst = os.path.join(graalpy_dir, graalpy_path)
1951
if os.path.isfile(src):
20-
os.makedirs(os.path.dirname(graalpy_path), exist_ok=True)
21-
shutil.copy(src, graalpy_path)
52+
os.makedirs(os.path.dirname(dst), exist_ok=True)
53+
shutil.copy(src, dst)
2254
elif os.path.isdir(src):
23-
if self.copy_all:
24-
shutil.copytree(src, graalpy_path, dirs_exist_ok=True)
25-
else:
26-
graalpy_prefix = f"{graalpy_path}/"
27-
matches = [f for f in overrides if f == graalpy_path or f.startswith(graalpy_prefix)]
28-
for f in matches:
29-
os.makedirs(os.path.dirname(f), exist_ok=True)
30-
src = os.path.join(basedir, self.source_path)
31-
if f.startswith(graalpy_prefix):
32-
src = os.path.join(src, f.removeprefix(graalpy_prefix))
33-
shutil.copy(src, f)
55+
shutil.copytree(src, dst, dirs_exist_ok=True)
3456
else:
3557
sys.exit(f"Source path {src} not found")
3658

3759

38-
class Ignore:
39-
def copy(self, basedir, graalpy_path, overrides):
40-
pass
60+
class CopyFromWithOverrides(AbstractRule):
61+
"""
62+
A rule that copies a directory using a list of files specified in the license header overrides list.
63+
"""
64+
65+
def copy(self, source_dir, graalpy_dir, graalpy_path, overrides):
66+
src = os.path.join(source_dir, self.source_path)
67+
if os.path.isdir(src):
68+
graalpy_prefix = f"{graalpy_path}/"
69+
matches = match_overrides(graalpy_path, overrides)
70+
for f in matches:
71+
dst = os.path.join(graalpy_dir, f)
72+
os.makedirs(os.path.dirname(dst), exist_ok=True)
73+
src = os.path.join(source_dir, self.source_path)
74+
if f.startswith(graalpy_prefix):
75+
src = os.path.join(src, f.removeprefix(graalpy_prefix))
76+
if not os.path.exists(src):
77+
sys.exit(f"File {f} from license overrides not found in {self.source_path}")
78+
shutil.copy(src, dst)
79+
elif os.path.exists(src):
80+
sys.exit(f"{type(self)} rule should only be used with directories")
81+
else:
82+
sys.exit(f"Source path {src} not found")
4183

4284

4385
#############################
4486
# Source file mapping rules #
4587
#############################
4688
CPYTHON_SOURCES_MAPPING = {
4789
# Standard library
48-
"graalpython/lib-python/3": CopyFrom("Lib", copy_all=True),
90+
"graalpython/lib-python/3": CopyFrom("Lib"),
4991

5092
# C API
51-
"graalpython/com.oracle.graal.python.cext/include": CopyFrom("Include"),
52-
"graalpython/com.oracle.graal.python.cext/expat": CopyFrom("Modules/expat"),
93+
"graalpython/com.oracle.graal.python.cext/include": CopyFromWithOverrides("Include"),
94+
"graalpython/com.oracle.graal.python.cext/expat": CopyFromWithOverrides("Modules/expat"),
5395
"graalpython/com.oracle.graal.python.cext/modules/_cpython_sre.c": CopyFrom("Modules/_sre.c"),
5496
"graalpython/com.oracle.graal.python.cext/modules/_cpython_unicodedata.c": CopyFrom("Modules/unicodedata.c"),
5597
"graalpython/com.oracle.graal.python.cext/modules/_bz2.c": CopyFrom("Modules/_bz2module.c"),
5698
"graalpython/com.oracle.graal.python.cext/modules/_mmap.c": CopyFrom("Modules/mmapmodule.c"),
5799
"graalpython/com.oracle.graal.python.cext/modules/_cpython_struct.c": CopyFrom("Modules/_struct.c"),
58100
"graalpython/com.oracle.graal.python.cext/modules/_testcapi.c": CopyFrom("Modules/_testcapimodule.c"),
59-
"graalpython/com.oracle.graal.python.cext/modules": CopyFrom("Modules"),
101+
"graalpython/com.oracle.graal.python.cext/modules/_ctypes_test.h": CopyFrom("Modules/_ctypes/_ctypes_test.h"),
102+
"graalpython/com.oracle.graal.python.cext/modules/_ctypes_test.c": CopyFrom("Modules/_ctypes/_ctypes_test.c"),
103+
"graalpython/com.oracle.graal.python.cext/modules/clinic/memoryobject.c.h": CopyFrom(
104+
"Objects/clinic/memoryobject.c.h"),
105+
"graalpython/com.oracle.graal.python.cext/modules": CopyFromWithOverrides("Modules"),
60106
"graalpython/com.oracle.graal.python.cext/src/getbuildinfo.c": CopyFrom("Modules/getbuildinfo.c"),
61107
"graalpython/com.oracle.graal.python.cext/src/capsule.c": CopyFrom("Objects/capsule.c"),
62108
"graalpython/com.oracle.graal.python.cext/src/complexobject.c": CopyFrom("Objects/complexobject.c"),
@@ -75,8 +121,7 @@ def copy(self, basedir, graalpy_path, overrides):
75121
"graalpython/com.oracle.graal.python.cext/posix/fork_exec.c": Ignore(),
76122

77123
# PEG Parser
78-
"graalpython/com.oracle.graal.python.pegparser.generator/pegen": CopyFrom("Tools/peg_generator/pegen",
79-
copy_all=True),
124+
"graalpython/com.oracle.graal.python.pegparser.generator/pegen": CopyFrom("Tools/peg_generator/pegen"),
80125
"graalpython/com.oracle.graal.python.pegparser.generator/asdl/asdl.py": CopyFrom("Parser/asdl.py"),
81126
"graalpython/com.oracle.graal.python.pegparser.generator/input_files/python.gram": CopyFrom("Grammar/python.gram"),
82127
"graalpython/com.oracle.graal.python.pegparser.generator/input_files/Tokens": CopyFrom("Grammar/Tokens"),
@@ -92,7 +137,11 @@ def copy(self, basedir, graalpy_path, overrides):
92137
}
93138

94139
PYPY_SOURCES_MAPPING = {
95-
"graalpython/lib-python/3": CopyFrom("lib_pypy"),
140+
"graalpython/lib-python/3/_md5.py": CopyFrom("lib_pypy/_md5.py"),
141+
"graalpython/lib-python/3/_sha1.py": CopyFrom("lib_pypy/_sha1.py"),
142+
"graalpython/lib-python/3/_sha256.py": CopyFrom("lib_pypy/_sha256.py"),
143+
"graalpython/lib-python/3/_sha512.py": CopyFrom("lib_pypy/_sha512.py"),
144+
"graalpython/com.oracle.graal.python.benchmarks": Ignore(),
96145
}
97146

98147

@@ -159,8 +208,8 @@ def import_python_sources(args):
159208
"""
160209

161210
with open(os.path.join(os.path.dirname(__file__), "copyrights", "overrides")) as f:
162-
entries = [line.split(',') for line in f if ',' in line]
163-
cpython_files = [file for file, license in entries if license == "python.copyright"]
211+
entries = [line.strip().split(',') for line in f if ',' in line]
212+
cpython_files = [file for file, license in entries if license == "python.copyright" and not file.endswith('.java')]
164213
pypy_files = [file for file, license in entries if license == "pypy.copyright"]
165214

166215
import_dir = os.path.abspath(os.path.join(SUITE.dir, 'python-import'))
@@ -169,11 +218,14 @@ def import_python_sources(args):
169218
SUITE.vc.git_command(SUITE.dir, ['fetch', 'origin', 'python-import:python-import'])
170219
# Checkout the python-import branch into a subdirectory
171220
SUITE.vc.git_command(SUITE.dir, ['clone', '-b', 'python-import', '.', 'python-import'])
221+
for file in os.listdir(import_dir):
222+
if not file.startswith('.'):
223+
shutil.rmtree(os.path.join(import_dir, file), ignore_errors=True)
172224

173225
def copy_inlined_files(mapping, source_directory, overrides):
174226
overrides = list(overrides)
175227
for path, rule in mapping.items():
176-
rule.copy(source_directory, os.path.join(import_dir, path), overrides)
228+
rule.copy(source_directory, import_dir, path, overrides)
177229
if overrides:
178230
lines = '\n'.join(overrides)
179231
sys.exit(f"ERROR: The following files were not matched by any rule:\n{lines}")

0 commit comments

Comments
 (0)