Skip to content

Commit b9d4ea3

Browse files
Andrej730Andrej730
committed
Script for packing pyodide wheel
1 parent c8f46cf commit b9d4ea3

2 files changed

Lines changed: 271 additions & 1 deletion

File tree

‎pyodide/pack_wheel.py‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
#
2+
# /// script
3+
# # Latest Pyodide build env versions are listed here:
4+
# # https://pyodide.github.io/pyodide/api/pyodide-cross-build-environments.json
5+
# # https://github.com/pyodide/pyodide-build/blob/main/pyodide_build/xbuildenv_releases.py
6+
# requires-python = "==3.13.2"
7+
# dependencies = [
8+
# "requests",
9+
# "setuptools",
10+
# ]
11+
# ///
12+
"""
13+
Pack an IfcOpenShell WASM wheel using Pyodide build system.
14+
15+
Usage:
16+
uv run make_wheel.py # Show this help
17+
uv run make_wheel.py --build # Build wheel
18+
uv run make_wheel.py --clean # Clean build artifacts and exit
19+
"""
20+
21+
import argparse
22+
import os
23+
import re
24+
import shutil
25+
import subprocess
26+
import time
27+
import zipfile
28+
from pathlib import Path
29+
from urllib.parse import quote
30+
31+
import requests
32+
33+
# Get repo root (parent of this script's parent directory)
34+
REPO_ROOT = Path(__file__).parent.parent
35+
PYODIDE_DIR = REPO_ROOT / "pyodide"
36+
BUILD_DIR = PYODIDE_DIR / "build"
37+
38+
# Hardcoded path (Windows packing workaround with --dev flag)
39+
PYODIDE_BUILD = Path(r"L:\Projects\Github\pyodide-build")
40+
41+
# Wheel platform tag (from PYODIDE_EMSCRIPTEN_VERSION in pyodide-build/Makefile.envs)
42+
WHEEL_PLATFORM_TAG = "emscripten_4_0_9_wasm32"
43+
44+
# Location where ifcopenshell will be extracted
45+
IFCOPENSHELL_DIR = PYODIDE_DIR / "ifcopenshell"
46+
47+
48+
class WheelBuilder:
49+
@staticmethod
50+
def extract_ifcopenshell_from_git(dst: Path) -> None:
51+
"""Extract ifcopenshell directory from git repo into destination."""
52+
Tools.rmrf(dst)
53+
54+
print(f"Extracting ifcopenshell from git to {dst}...")
55+
# Use git ls-files piped to git checkout-index to avoid copying
56+
# untracked or ignored files from the actual repo.
57+
ls_proc = subprocess.Popen(
58+
["git", "ls-files", "-z", "src/ifcopenshell-python/ifcopenshell"],
59+
cwd=REPO_ROOT,
60+
stdout=subprocess.PIPE,
61+
stderr=subprocess.PIPE,
62+
)
63+
checkout_proc = subprocess.Popen(
64+
["git", "checkout-index", "-z", "--prefix", "pyodide/", "--stdin"],
65+
cwd=REPO_ROOT,
66+
stdin=ls_proc.stdout,
67+
stdout=subprocess.PIPE,
68+
stderr=subprocess.PIPE,
69+
)
70+
assert ls_proc.stdout is not None
71+
ls_proc.stdout.close()
72+
checkout_proc.communicate()
73+
74+
if checkout_proc.returncode != 0:
75+
assert checkout_proc.stderr is not None
76+
raise RuntimeError(f"Failed to extract: {checkout_proc.stderr.decode()}")
77+
78+
# Move src/ifcopenshell-python/ifcopenshell to ifcopenshell.
79+
temp_src = PYODIDE_DIR / "src" / "ifcopenshell-python" / "ifcopenshell"
80+
shutil.move(temp_src, dst)
81+
82+
# Clean up temporary src directory.
83+
Tools.rmrf(PYODIDE_DIR / "src")
84+
85+
print("✓ Extracted ifcopenshell from git")
86+
87+
@staticmethod
88+
def get_wheel_url(makefile_path: Path) -> str:
89+
"""Get S3 wheel URL based on BINARY_VERSION and BUILD_COMMIT from Makefile."""
90+
91+
def parse_makefile_vars() -> dict[str, str]:
92+
content = makefile_path.read_text()
93+
vars: dict[str, str] = {}
94+
for match in re.finditer(r"^(BINARY_VERSION|BUILD_COMMIT):=(.+)$", content, re.MULTILINE):
95+
vars[match.group(1)] = match.group(2).strip()
96+
return vars
97+
98+
vars: dict[str, str] = parse_makefile_vars()
99+
binary_version = vars["BINARY_VERSION"]
100+
build_commit = vars["BUILD_COMMIT"]
101+
filename = f"ifcopenshell-{binary_version}+{build_commit}-cp313-cp313-pyodide_2025_0_wasm32.whl"
102+
encoded_filename = quote(filename, safe="")
103+
return f"https://s3.amazonaws.com/ifcopenshell-builds/{encoded_filename}"
104+
105+
@staticmethod
106+
def download_and_extract_so(url: str, build_dir: Path) -> tuple[Path, Path]:
107+
"""Download wheel from URL and extract .so and .py files."""
108+
py_wrapper_filename = "ifcopenshell_wrapper.py"
109+
build_dir.mkdir(parents=True, exist_ok=True)
110+
111+
wheel_path = build_dir / url.rsplit("/", 1)[-1]
112+
113+
if wheel_path.exists():
114+
print(f"Using cached wheel: {wheel_path}")
115+
else:
116+
print(f"Downloading {url}...")
117+
response = requests.get(url)
118+
response.raise_for_status()
119+
wheel_path.write_bytes(response.content)
120+
121+
print("Extracting _ifcopenshell_wrapper files...")
122+
with zipfile.ZipFile(wheel_path) as zf:
123+
so_files = [f for f in zf.namelist() if f.endswith(".so")]
124+
py_files = [f for f in zf.namelist() if f.endswith(py_wrapper_filename)]
125+
126+
assert so_files, "No .so file found in wheel"
127+
assert py_files, f"No {py_wrapper_filename} file found in wheel"
128+
129+
so_file = so_files[0]
130+
so_dst = build_dir / Path(so_file).name
131+
so_dst.write_bytes(zf.read(so_file))
132+
133+
py_file = py_files[0]
134+
py_dst = build_dir / Path(py_file).name
135+
py_dst.write_bytes(zf.read(py_file))
136+
137+
return so_dst, py_dst
138+
139+
140+
class Tools:
141+
@staticmethod
142+
def run(
143+
cmd: list[str],
144+
cwd: Path | None = None,
145+
) -> None:
146+
print(f"$ {' '.join(cmd)}")
147+
subprocess.check_call(cmd, cwd=cwd)
148+
149+
@staticmethod
150+
def create_symlink(dst: Path, src: Path) -> None:
151+
Tools.rmrf(dst)
152+
dst.symlink_to(src)
153+
154+
@staticmethod
155+
def rmrf(path: Path) -> None:
156+
if path.exists() or path.is_symlink():
157+
if path.is_dir() and not path.is_symlink():
158+
shutil.rmtree(path)
159+
else:
160+
path.unlink()
161+
162+
163+
def clean() -> None:
164+
"""Remove build artifacts."""
165+
paths_to_remove = (
166+
BUILD_DIR,
167+
PYODIDE_DIR / ".pyodide_build",
168+
PYODIDE_DIR / "dist",
169+
PYODIDE_DIR / "ifcopenshell.egg-info",
170+
PYODIDE_DIR / "src",
171+
IFCOPENSHELL_DIR,
172+
)
173+
for path in paths_to_remove:
174+
if path.exists() or path.is_symlink():
175+
print(f"Removing {path}...")
176+
Tools.rmrf(path)
177+
print("✓ Clean complete")
178+
179+
180+
def main() -> None:
181+
parser = argparse.ArgumentParser(description=__doc__, add_help=False)
182+
parser.add_argument("--build", action="store_true", help="Build the wheel")
183+
parser.add_argument("--clean", action="store_true", help="Clean build folder")
184+
parser.add_argument(
185+
"--dev",
186+
action="store_true",
187+
help="Use editable pyodide-build from hardcoded path (Windows packing workaround)",
188+
)
189+
args = parser.parse_args()
190+
191+
if not args.build and not args.clean:
192+
print(__doc__)
193+
return
194+
195+
if args.clean:
196+
clean()
197+
return
198+
199+
start_time = time.time()
200+
201+
WheelBuilder.extract_ifcopenshell_from_git(IFCOPENSHELL_DIR)
202+
203+
print("Downloading and extracting _ifcopenshell_wrapper files...")
204+
makefile = REPO_ROOT / "src" / "ifcopenshell-python" / "Makefile"
205+
wheel_url = WheelBuilder.get_wheel_url(makefile)
206+
so_file, py_file = WheelBuilder.download_and_extract_so(wheel_url, BUILD_DIR)
207+
208+
Tools.create_symlink(IFCOPENSHELL_DIR / Path(so_file).name, so_file)
209+
Tools.create_symlink(IFCOPENSHELL_DIR / Path(py_file).name, py_file)
210+
211+
print("Installing pyodide-build...")
212+
if args.dev:
213+
Tools.run(["uv", "pip", "install", "-e", str(PYODIDE_BUILD)])
214+
else:
215+
Tools.run(["uv", "pip", "install", "pyodide-build"])
216+
217+
print("Building with pyodide...")
218+
# Use --no-isolation due to pyodide-build Windows support issues:
219+
# symlink_unisolated_packages fails with missing `_sysconfigdata_$(CPYTHON_ABI_FLAGS)_emscripten_wasm32-emscripten.py`.
220+
# Hardcode platform name since pyodide doesn't yet support overriding wheel tags on Windows.
221+
#
222+
# Use `LEGACY_PLATFORM` since pyodide 0.34.1 introduced new tag for wheels `pyemscripten`,
223+
# which doesn't work with pyodide itself yet - https://github.com/pyodide/pyodide/issues/6177.
224+
os.environ["USE_LEGACY_PLATFORM"] = "1"
225+
Tools.run(["pyodide", "build", f"-C--build-option=--plat-name={WHEEL_PLATFORM_TAG}"])
226+
227+
elapsed = time.time() - start_time
228+
print(f"\n✓ Done! ({elapsed:.1f}s)")
229+
230+
231+
if __name__ == "__main__":
232+
main()

‎pyodide/setup.py‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,16 @@
22
# because `tool.setuptools.ext-modules` is still experimental in pyproject.toml
33
# and we need it to get the wheel suffix right.
44
import os
5+
import sys
56
from pathlib import Path
67

78
import tomllib
89
from setuptools import Extension, find_packages, setup
10+
from setuptools.command.build_ext import build_ext
911

10-
REPO_FOLDER = Path(__file__).parent
12+
# Detect repo folder: if setup.py is in pyodide folder, go to parent
13+
SETUP_DIR = Path(__file__).parent
14+
REPO_FOLDER = SETUP_DIR.parent if SETUP_DIR.name == "pyodide" else SETUP_DIR
1115

1216

1317
def get_version() -> str:
@@ -25,6 +29,39 @@ def get_dependencies() -> list[str]:
2529
return dependencies
2630

2731

32+
class UnixBuildExt(build_ext):
33+
"""Customize ``build_ext`` to support packing on Windows."""
34+
35+
def finalize_options(self):
36+
from distutils import sysconfig
37+
38+
super().finalize_options()
39+
if sys.platform == "win32":
40+
self.compiler = "unix"
41+
42+
# Configure sysconfig for Windows builds
43+
# CCSHARED is the only variable that's not customizable with env vars.
44+
# Basically avoiding this:
45+
# File ".venv\Lib\site-packages\setuptools\_distutils\sysconfig.py", line 366, in customize_compiler
46+
# compiler_so=cc_cmd + ' ' + ccshared,
47+
# ~~~~~~~~~~~~~^~~~~~~~~~
48+
# TypeError: can only concatenate str (not "NoneType") to str
49+
sysconfig.get_config_vars() # Initialize config cache
50+
if sysconfig._config_vars.get("CCSHARED") is None:
51+
sysconfig._config_vars["CCSHARED"] = "-fPIC"
52+
# Override compiler type before it's instantiated
53+
54+
# Set Emscripten compiler environment variables
55+
os.environ["CC"] = "emcc"
56+
os.environ["CXX"] = "em++"
57+
os.environ["CFLAGS"] = ""
58+
os.environ["CXXFLAGS"] = ""
59+
os.environ["LDSHARED"] = "emcc -shared"
60+
os.environ["AR"] = "emar"
61+
os.environ["ARFLAGS"] = "rcs"
62+
os.environ["SETUPTOOLS_EXT_SUFFIX"] = ".cpython-313-wasm32-emscripten.so"
63+
64+
2865
setup(
2966
name="ifcopenshell",
3067
version=get_version(),
@@ -44,4 +81,5 @@ def get_dependencies() -> list[str]:
4481
},
4582
# Has to provide extension to get the correct wheel suffix.
4683
ext_modules=[Extension("ifcopenshell._ifcopenshell_wrapper", sources=[])],
84+
cmdclass={"build_ext": UnixBuildExt},
4785
)

0 commit comments

Comments
 (0)