py/persistentcode: Introduce .mpy sub-version.

jimmo · dpgeorge · commit d94141e1473a · 2022-09-19T23:19:55.000+10:00
The intent is to allow us to make breaking changes to the native ABI (e.g.
changes to dynruntime.h) without needing the bytecode version to increment.

With this commit the two bits previously used for the feature flags (but
now unused as of .mpy version 6) encode a sub-version.  A bytecode-only
.mpy file can be loaded as long as MPY_VERSION matches, but a native .mpy
(i.e. one with an arch set) must also match MPY_SUB_VERSION.  This allows 3
additional updates to the native ABI per bytecode revision.

The sub-version is set to 1 because the previous commits that changed the
layout of mp_obj_type_t have changed the native ABI.

Signed-off-by: Jim Mussared &lt;jim.mussared@gmail.com&gt;
Signed-off-by: Damien George &lt;damien@micropython.org&gt;
diff --git a/docs/reference/mpyfiles.rst b/docs/reference/mpyfiles.rst
@@ -27,6 +27,11 @@ Compatibility is based on the following:
 * Version of the .mpy file: the version of the file must match the version
   supported by the system loading it.
 
+* Sub-version of the .mpy file: if the .mpy file contains native machine code
+  then the sub-version of the file must match the version support by the
+  system loading it.  Otherwise, if there is no native machine code in the .mpy
+  file, then the sub-version is ignored when loading.
+
 * Small integer bits: the .mpy file will require a minimum number of bits in
   a small integer and the system loading it must support at least this many
   bits.
@@ -55,6 +60,7 @@ If importing an .mpy file fails then try the following:
         'armv6', 'armv6m', 'armv7m', 'armv7em', 'armv7emsp', 'armv7emdp',
         'xtensa', 'xtensawin'][sys_mpy >> 10]
     print('mpy version:', sys_mpy & 0xff)
+    print('mpy sub-version:', sys_mpy >> 8 & 3)
     print('mpy flags:', end='')
     if arch:
         print(' -march=' + arch, end='')
diff --git a/mpy-cross/main.c b/mpy-cross/main.c
@@ -228,7 +228,7 @@ MP_NOINLINE int main_(int argc, char **argv) {
                 a += 1;
             } else if (strcmp(argv[a], "--version") == 0) {
                 printf("MicroPython " MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE
-                    "; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "\n");
+                    "; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "." MP_STRINGIFY(MPY_SUB_VERSION) "\n");
                 return 0;
             } else if (strcmp(argv[a], "-v") == 0) {
                 mp_verbose_flag++;
diff --git a/py/persistentcode.c b/py/persistentcode.c
@@ -393,14 +393,14 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
 mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *context) {
     byte header[4];
     read_bytes(reader, header, sizeof(header));
+    byte arch = MPY_FEATURE_DECODE_ARCH(header[2]);
     if (header[0] != 'M'
         || header[1] != MPY_VERSION
-        || MPY_FEATURE_DECODE_FLAGS(header[2]) != MPY_FEATURE_FLAGS
+        || (arch != MP_NATIVE_ARCH_NONE && MPY_FEATURE_DECODE_SUB_VERSION(header[2]) != MPY_SUB_VERSION)
         || header[3] > MP_SMALL_INT_BITS) {
         mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file"));
     }
     if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) {
-        byte arch = MPY_FEATURE_DECODE_ARCH(header[2]);
         if (!MPY_FEATURE_ARCH_TEST(arch)) {
             if (MPY_FEATURE_ARCH_TEST(MP_NATIVE_ARCH_NONE)) {
                 // On supported ports this can be resolved by enabling feature, eg
@@ -596,7 +596,7 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) {
     byte header[4] = {
         'M',
         MPY_VERSION,
-        MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS_DYNAMIC),
+        MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION),
         #if MICROPY_DYNAMIC_COMPILER
         mp_dynamic_compiler.small_int_bits,
         #else
diff --git a/py/persistentcode.h b/py/persistentcode.h
@@ -30,24 +30,23 @@
 #include "py/reader.h"
 #include "py/emitglue.h"
 
-// The current version of .mpy files
+// The current version of .mpy files. A bytecode-only .mpy file can be loaded
+// as long as MPY_VERSION matches, but a native .mpy (i.e. one with an arch
+// set) must also match MPY_SUB_VERSION. This allows 3 additional updates to
+// the native ABI per bytecode revision.
 #define MPY_VERSION 6
+#define MPY_SUB_VERSION 1
 
-// Macros to encode/decode flags to/from the feature byte
-#define MPY_FEATURE_ENCODE_FLAGS(flags) (flags)
-#define MPY_FEATURE_DECODE_FLAGS(feat) ((feat) & 3)
+// Macros to encode/decode sub-version to/from the feature byte. This replaces
+// the bits previously used to encode the flags (map caching and unicode)
+// which are no longer used starting at .mpy version 6.
+#define MPY_FEATURE_ENCODE_SUB_VERSION(version) (version)
+#define MPY_FEATURE_DECODE_SUB_VERSION(feat) ((feat) & 3)
 
 // Macros to encode/decode native architecture to/from the feature byte
 #define MPY_FEATURE_ENCODE_ARCH(arch) ((arch) << 2)
 #define MPY_FEATURE_DECODE_ARCH(feat) ((feat) >> 2)
 
-// The feature flag bits encode the compile-time config options that affect
-// the generate bytecode. Note: no longer used.
-// (formerly MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE and MICROPY_PY_BUILTINS_STR_UNICODE).
-#define MPY_FEATURE_FLAGS (0)
-// This is a version of the flags that can be configured at runtime.
-#define MPY_FEATURE_FLAGS_DYNAMIC (0)
-
 // Define the host architecture
 #if MICROPY_EMIT_X86
     #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_X86)
@@ -82,7 +81,7 @@
 
 // 16-bit little-endian integer with the second and third bytes of supported .mpy files
 #define MPY_FILE_HEADER_INT (MPY_VERSION \
-    | (MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8)
+    | (MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8)
 
 enum {
     MP_NATIVE_ARCH_NONE = 0,
diff --git a/tests/micropython/import_mpy_native.py b/tests/micropython/import_mpy_native.py
@@ -11,7 +11,8 @@
     raise SystemExit
 
 mpy_arch = usys.implementation._mpy >> 8
-if mpy_arch == 0:
+if mpy_arch >> 2 == 0:
+    # This system does not support .mpy files containing native code
     print("SKIP")
     raise SystemExit
 
@@ -54,8 +55,8 @@ def open(self, path, mode):
 valid_header = bytes([77, 6, mpy_arch, 31])
 # fmt: off
 user_files = {
-    # bad architecture
-    '/mod0.mpy': b'M\x06\xfc\x1f',
+    # bad architecture (mpy_arch needed for sub-version)
+    '/mod0.mpy': bytes([77, 6, 0xfc | mpy_arch, 31]),
 
     # test loading of viper and asm
     '/mod1.mpy': valid_header + (
diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py
@@ -46,24 +46,26 @@ def open(self, path, mode):
 
 
 # Pre-compiled examples/natmod/features0 example for various architectures, keyed
-# by the required value of sys.implementation._mpy.
+# by the required value of sys.implementation._mpy (without sub-version).
 features0_file_contents = {
     # -march=x64
-    0x806: b'M\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff',
+    0x806: b'M\x06\x09\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff',
     # -march=armv6m
-    0x1006: b"M\x06\x10\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff",
+    0x1006: b"M\x06\x11\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff",
 }
 
 # Populate armv7m-derived archs based on armv6m.
 for arch in (0x1406, 0x1806, 0x1C06, 0x2006):
     features0_file_contents[arch] = features0_file_contents[0x1006]
 
-if sys.implementation._mpy not in features0_file_contents:
+# Check that a .mpy exists for the target (ignore sub-version in lookup).
+sys_implementation_mpy = sys.implementation._mpy & ~(3 << 8)
+if sys_implementation_mpy not in features0_file_contents:
     print("SKIP")
     raise SystemExit
 
 # These are the test .mpy files.
-user_files = {"/features0.mpy": features0_file_contents[sys.implementation._mpy]}
+user_files = {"/features0.mpy": features0_file_contents[sys_implementation_mpy]}
 
 # Create and mount a user filesystem.
 uos.mount(UserFS(user_files), "/userfs")
diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py
@@ -88,6 +88,7 @@ def __str__(self):
 
 class Config:
     MPY_VERSION = 6
+    MPY_SUB_VERSION = 1
     MICROPY_LONGINT_IMPL_NONE = 0
     MICROPY_LONGINT_IMPL_LONGLONG = 1
     MICROPY_LONGINT_IMPL_MPZ = 2
@@ -1335,6 +1336,9 @@ def read_mpy(filename):
         feature_byte = header[2]
         mpy_native_arch = feature_byte >> 2
         if mpy_native_arch != MP_NATIVE_ARCH_NONE:
+            mpy_sub_version = feature_byte & 3
+            if mpy_sub_version != config.MPY_SUB_VERSION:
+                raise MPYReadError(filename, "incompatible .mpy sub-version")
             if config.native_arch == MP_NATIVE_ARCH_NONE:
                 config.native_arch = mpy_native_arch
             elif config.native_arch != mpy_native_arch:
@@ -1658,7 +1662,9 @@ def merge_mpy(compiled_modules, output_file):
     else:
         main_cm_idx = None
         for idx, cm in enumerate(compiled_modules):
-            if cm.header[2]:
+            feature_byte = cm.header[2]
+            mpy_native_arch = feature_byte >> 2
+            if mpy_native_arch:
                 # Must use qstr_table and obj_table from this raw_code
                 if main_cm_idx is not None:
                     raise Exception("can't merge files when more than one contains native code")
@@ -1670,7 +1676,7 @@ def merge_mpy(compiled_modules, output_file):
         header = bytearray(4)
         header[0] = ord("M")
         header[1] = config.MPY_VERSION
-        header[2] = config.native_arch << 2
+        header[2] = config.native_arch << 2 | config.MPY_SUB_VERSION
         header[3] = config.mp_small_int_bits
         merged_mpy.extend(header)
 
diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py
@@ -36,6 +36,7 @@
 
 # MicroPython constants
 MPY_VERSION = 6
+MPY_SUB_VERSION = 1
 MP_CODE_BYTECODE = 2
 MP_CODE_NATIVE_VIPER = 4
 MP_NATIVE_ARCH_X86 = 1
@@ -917,7 +918,11 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs):
     out.open(fmpy)
 
     # MPY: header
-    out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS]))
+    out.write_bytes(
+        bytearray(
+            [ord("M"), MPY_VERSION, env.arch.mpy_feature | MPY_SUB_VERSION, MP_SMALL_INT_BITS]
+        )
+    )
 
     # MPY: n_qstr
     out.write_uint(1 + len(native_qstr_vals))
