Skip to content

Commit 0cc8f63

Browse files
youknowoneyouknowone
authored
[update_lib] deps grouping (RustPython#6854)
1 parent 185f360 commit 0cc8f63

File tree

3 files changed

+394
-30
lines changed

3 files changed

+394
-30
lines changed

‎scripts/update_lib/deps.py‎

Lines changed: 264 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,67 @@ def clear_import_graph_caches() -> None:
6565
},
6666
# Rust-implemented modules (no lib file, only test)
6767
"int": {
68-
"lib": [], # No Python lib (Rust implementation)
68+
"lib": [],
6969
"hard_deps": ["_pylong.py"],
70+
"test": [
71+
"test/test_int.py",
72+
"test/test_long.py",
73+
],
74+
},
75+
"exception": {
76+
"lib": [],
77+
"test": [
78+
"test/test_exceptions.py",
79+
"test/test_baseexception.py",
80+
"test/test_except_star.py",
81+
"test/test_exception_group.py",
82+
"test/test_exception_hierarchy.py",
83+
"test/test_exception_variations.py",
84+
],
85+
},
86+
"dict": {
87+
"lib": [],
88+
"test": [
89+
"test/test_dict.py",
90+
"test/test_dictcomps.py",
91+
"test/test_dictviews.py",
92+
"test/test_userdict.py",
93+
],
94+
},
95+
"list": {
96+
"lib": [],
97+
"test": [
98+
"test/test_list.py",
99+
"test/test_listcomps.py",
100+
"test/test_userlist.py",
101+
],
102+
},
103+
"codecs": {
104+
"test": [
105+
"test/test_codecs.py",
106+
"test/test_codeccallbacks.py",
107+
"test/test_codecencodings_cn.py",
108+
"test/test_codecencodings_hk.py",
109+
"test/test_codecencodings_iso2022.py",
110+
"test/test_codecencodings_jp.py",
111+
"test/test_codecencodings_kr.py",
112+
"test/test_codecencodings_tw.py",
113+
"test/test_codecmaps_cn.py",
114+
"test/test_codecmaps_hk.py",
115+
"test/test_codecmaps_jp.py",
116+
"test/test_codecmaps_kr.py",
117+
"test/test_codecmaps_tw.py",
118+
"test/test_charmapcodec.py",
119+
"test/test_multibytecodec.py",
120+
],
70121
},
71122
# Non-pattern hard_deps (can't be auto-detected)
72123
"ast": {
73124
"hard_deps": ["_ast_unparse.py"],
125+
"test": [
126+
"test/test_ast.py",
127+
"test/test_unparse.py",
128+
],
74129
},
75130
# Data directories
76131
"pydoc": {
@@ -83,6 +138,206 @@ def clear_import_graph_caches() -> None:
83138
"support": {
84139
"lib": ["test/support"],
85140
"data": ["test/wheeldata"],
141+
"test": [
142+
"test/test_support.py",
143+
"test/test_script_helper.py",
144+
],
145+
},
146+
# test_htmlparser tests html.parser
147+
"html": {
148+
"test": ["test/test_html.py", "test/test_htmlparser.py"],
149+
},
150+
"xml": {
151+
"test": [
152+
"test/test_xml_etree.py",
153+
"test/test_xml_etree_c.py",
154+
"test/test_minidom.py",
155+
"test/test_pulldom.py",
156+
"test/test_pyexpat.py",
157+
"test/test_sax.py",
158+
],
159+
},
160+
"multiprocessing": {
161+
"test": [
162+
"test/test_multiprocessing_fork",
163+
"test/test_multiprocessing_forkserver",
164+
"test/test_multiprocessing_spawn",
165+
],
166+
},
167+
"urllib": {
168+
"test": [
169+
"test/test_urllib.py",
170+
"test/test_urllib2.py",
171+
"test/test_urllib2_localnet.py",
172+
"test/test_urllib2net.py",
173+
"test/test_urllibnet.py",
174+
"test/test_urlparse.py",
175+
"test/test_urllib_response.py",
176+
"test/test_robotparser.py",
177+
],
178+
},
179+
"collections": {
180+
"test": [
181+
"test/test_collections.py",
182+
"test/test_deque.py",
183+
"test/test_defaultdict.py",
184+
"test/test_ordered_dict.py",
185+
],
186+
},
187+
"http": {
188+
"test": [
189+
"test/test_httplib.py",
190+
"test/test_http_cookiejar.py",
191+
"test/test_http_cookies.py",
192+
"test/test_httpservers.py",
193+
],
194+
},
195+
"unicode": {
196+
"lib": [],
197+
"test": [
198+
"test/test_unicode_file.py",
199+
"test/test_unicode_file_functions.py",
200+
"test/test_unicode_identifiers.py",
201+
"test/test_unicodedata.py",
202+
],
203+
},
204+
"typing": {
205+
"test": [
206+
"test/test_typing.py",
207+
"test/test_type_aliases.py",
208+
"test/test_type_annotations.py",
209+
"test/test_type_params.py",
210+
"test/test_genericalias.py",
211+
],
212+
},
213+
"unpack": {
214+
"lib": [],
215+
"test": [
216+
"test/test_unpack.py",
217+
"test/test_unpack_ex.py",
218+
],
219+
},
220+
"zipimport": {
221+
"test": [
222+
"test/test_zipimport.py",
223+
"test/test_zipimport_support.py",
224+
],
225+
},
226+
"time": {
227+
"lib": [],
228+
"test": [
229+
"test/test_time.py",
230+
"test/test_strftime.py",
231+
],
232+
},
233+
"sys": {
234+
"lib": [],
235+
"test": [
236+
"test/test_sys.py",
237+
"test/test_syslog.py",
238+
"test/test_sys_setprofile.py",
239+
"test/test_sys_settrace.py",
240+
],
241+
},
242+
"str": {
243+
"lib": [],
244+
"test": [
245+
"test/test_str.py",
246+
"test/test_fstring.py",
247+
"test/test_string_literals.py",
248+
],
249+
},
250+
"thread": {
251+
"lib": [],
252+
"test": [
253+
"test/test_thread.py",
254+
"test/test_thread_local_bytecode.py",
255+
"test/test_threadsignals.py",
256+
],
257+
},
258+
"threading": {
259+
"test": [
260+
"test/test_threading.py",
261+
"test/test_threadedtempfile.py",
262+
"test/test_threading_local.py",
263+
],
264+
},
265+
"class": {
266+
"lib": [],
267+
"test": [
268+
"test/test_class.py",
269+
"test/test_genericclass.py",
270+
"test/test_subclassinit.py",
271+
],
272+
},
273+
"generator": {
274+
"lib": [],
275+
"test": [
276+
"test/test_generators.py",
277+
"test/test_genexps.py",
278+
"test/test_generator_stop.py",
279+
"test/test_yield_from.py",
280+
],
281+
},
282+
"descr": {
283+
"lib": [],
284+
"test": [
285+
"test/test_descr.py",
286+
"test/test_descrtut.py",
287+
],
288+
},
289+
"contextlib": {
290+
"test": [
291+
"test/test_contextlib.py",
292+
"test/test_contextlib_async.py",
293+
],
294+
},
295+
"io": {
296+
"test": [
297+
"test/test_io.py",
298+
"test/test_bufio.py",
299+
"test/test_fileio.py",
300+
"test/test_memoryio.py",
301+
],
302+
},
303+
"dbm": {
304+
"test": [
305+
"test/test_dbm.py",
306+
"test/test_dbm_gnu.py",
307+
"test/test_dbm_ndbm.py",
308+
],
309+
},
310+
"datetime": {
311+
"test": [
312+
"test/test_datetime.py",
313+
"test/test_strptime.py",
314+
],
315+
},
316+
"file": {
317+
"lib": [],
318+
"test": [
319+
"test/test_file.py",
320+
"test/test_largefile.py",
321+
],
322+
},
323+
"fcntl": {
324+
"lib": [],
325+
"test": [
326+
"test/test_fcntl.py",
327+
"test/test_ioctl.py",
328+
],
329+
},
330+
"xmlrpc": {
331+
"test": [
332+
"test/test_xmlrpc.py",
333+
"test/test_docxmlrpc.py",
334+
],
335+
},
336+
"ctypes": {
337+
"test": [
338+
"test/test_ctypes",
339+
"test/test_stable_abi_ctypes.py",
340+
],
86341
},
87342
}
88343

@@ -468,10 +723,13 @@ def is_up_to_date(
468723

469724
lib_paths = get_lib_paths(name, cpython_prefix)
470725

726+
found_any = False
471727
for cpython_path in lib_paths:
472728
if not cpython_path.exists():
473729
continue
474730

731+
found_any = True
732+
475733
# Convert cpython path to local path
476734
# cpython/Lib/foo.py -> Lib/foo.py
477735
rel_path = cpython_path.relative_to(cpython_prefix)
@@ -489,7 +747,11 @@ def is_up_to_date(
489747
if not _dircmp_is_same(dcmp):
490748
return False
491749

492-
return True
750+
if not found_any:
751+
dep_info = DEPENDENCIES.get(name, {})
752+
if dep_info.get("lib") == []:
753+
return True
754+
return found_any
493755

494756

495757
def get_test_dependencies(

‎scripts/update_lib/show_deps.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ def format_deps_tree(
8686
List of formatted lines
8787
"""
8888
from update_lib.deps import (
89+
get_lib_paths,
8990
get_rust_deps,
9091
get_soft_deps,
9192
is_up_to_date,
@@ -111,6 +112,11 @@ def format_deps_tree(
111112
dup_deps = []
112113

113114
for dep in soft_deps:
115+
# Skip if library doesn't exist in cpython
116+
lib_paths = get_lib_paths(dep, cpython_prefix)
117+
if not any(p.exists() for p in lib_paths):
118+
continue
119+
114120
up_to_date = is_up_to_date(dep, cpython_prefix, lib_prefix)
115121
if up_to_date:
116122
# Up-to-date modules collected compactly, no dup tracking needed

0 commit comments

Comments
 (0)