From 78b8ead87ffdc6bf23e21e17eb06d990cd55fa18 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 4 May 2026 15:05:43 +1000 Subject: [PATCH] py/makeqstrdefs.py: Gzip the qstr.i.last generated output. Generated `qstr.i.last` files can be very, very large due to inclusion of large HAL headers. The data is very repetitive text so compressing it can significantly reduce the size of the file, and hence reduce the size of the build directory and decrease wear of the filesystem. With a clean build from scratch, using gzip level 1 changes the build output by the following amounts: - stm32 PYBV10: 130M -> 48M (`qstr.i.last` reduced by 82M) - rp2 RPI_PICO: 85M -> 47M (`qstr.i.last` reduced by 38M) - esp32 ESP32_GENERIC: 304M -> 244M (`qstr.i.last` reduced by 60M) This can amount to significant savings when building all boards for a port, eg saving up to 6GB building all stm32 boards. It saves less on a rebuild because `qstr.i.last` only contains preprocessor output from files that changed since the previous build. Built time is increased slightly due to the compression: about 0.5-1 second slower (out of a total 30s build time, which includes 4s generating `qstr.i.last`) for stm32 PYBV10, which is around 3% slower. Using `compresslevel=9` saves about 2M more but almost doubles the time taken to generate `qstr.i.last` (going from 4s up to 8s or more). So using `compresslevel=1` is the best trade off here. Signed-off-by: Damien George --- ports/stm32/boards/plli2svalues.py | 3 ++- ports/stm32/boards/pllvalues.py | 3 ++- py/makeqstrdefs.py | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ports/stm32/boards/plli2svalues.py b/ports/stm32/boards/plli2svalues.py index f872c60ebb5c5..8fdf6c0d7a187 100644 --- a/ports/stm32/boards/plli2svalues.py +++ b/ports/stm32/boards/plli2svalues.py @@ -4,6 +4,7 @@ Those processors are listed below in the mcu_support_plli2s[] list. """ +import gzip import re from collections import namedtuple @@ -129,7 +130,7 @@ def generate_c_table(plli2s_table, hse, pllm): def search_header(filename, re_define, lookup): regex_define = re.compile(re_define) val = None - with open(filename) as f: + with gzip.open(filename, "rt") as f: for line in f: line = line.strip() m = regex_define.match(line) diff --git a/ports/stm32/boards/pllvalues.py b/ports/stm32/boards/pllvalues.py index 5558d04c5b9e1..2e607861e24b5 100644 --- a/ports/stm32/boards/pllvalues.py +++ b/ports/stm32/boards/pllvalues.py @@ -4,6 +4,7 @@ for the machine.freq() function. """ +import gzip import re @@ -231,7 +232,7 @@ def search_header_for_hsx_values(filename): regex_def = re.compile( r"static.* +(micropy_hw_hs[ei]_value) = +([0-9 +-/\*()]+);", ) - with open(filename) as f: + with gzip.open(filename, "rt") as f: for line in f: line = line.strip() m = regex_def.match(line) diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py index dd514c7033dd7..1f7bbc67a4689 100644 --- a/py/makeqstrdefs.py +++ b/py/makeqstrdefs.py @@ -5,7 +5,7 @@ This script works with Python 3.3+. """ -import io +import gzip import os import re import subprocess @@ -71,7 +71,7 @@ def run(files): except NotImplementedError: cpus = 1 p = multiprocessing.dummy.Pool(cpus) - with open(args.output[0], "wb") as out_file: + with gzip.open(args.output[0], "wb", compresslevel=1) as out_file: for flags, sources in ( (args.cflags, csources), (args.cxxflags, cxxsources), @@ -230,7 +230,7 @@ class Args: pass if args.command == "split": - with io.open(args.input_filename, encoding="utf-8") as infile: + with gzip.open(args.input_filename, "rt", encoding="utf-8") as infile: process_file(infile) if args.command == "cat":