Compact debug info. Work in progress.

python · markshannon · Apr 21, 2022 · Apr 8, 2022 · Apr 14, 2022 · Apr 14, 2022
commit 78cd7a3160214134e72e7aa28d20166409e41de9
diff --git a/Include/cpython/code.h b/Include/cpython/code.h
@@ -95,6 +95,7 @@ typedef uint16_t _Py_CODEUNIT;
     PyObject *co_columntable;     /* bytes object that holds start/end column  \
                                      offset each instruction */                \
                                                                                \
+    PyObject *co_locationtable;   /* bytes object that holds location info */ \
     PyObject *co_weakreflist;     /* to support weakrefs to code objects */    \
     /* Scratch space for extra data relating to the code object.               \
        Type is a void* to keep the format private in codeobject.c to force     \
@@ -153,13 +154,13 @@ PyAPI_FUNC(PyCodeObject *) PyCode_New(
         int, int, int, int, int, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, int, PyObject *,
-        PyObject *, PyObject *, PyObject *);
+        PyObject *, PyObject *, PyObject *, PyObject *);
 
 PyAPI_FUNC(PyCodeObject *) PyCode_NewWithPosOnlyArgs(
         int, int, int, int, int, int, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, int, PyObject *,
-        PyObject *, PyObject *, PyObject *);
+        PyObject *, PyObject *, PyObject *, PyObject *);
         /* same as struct above */
 
 /* Creates a new empty code object with the specified source location. */
@@ -176,8 +177,8 @@ PyAPI_FUNC(int) PyCode_Addr2Location(PyCodeObject *, int, int *, int *, int *, i
 /* for internal use only */
 struct _opaque {
     int computed_line;
-    const char *lo_next;
-    const char *limit;
+    const uint8_t *lo_next;
+    const uint8_t *limit;
 };
 
 typedef struct _line_offsets {
@@ -210,6 +211,19 @@ PyAPI_FUNC(int) _PyCode_GetExtra(PyObject *code, Py_ssize_t index,
 PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index,
                                  void *extra);
 
+
+typedef enum _PyCodeLocationInfoKind {
+    PY_CODE_LOCATION_INFO_SHORT0 = 0,
+    PY_CODE_LOCATION_INFO_SHORT1 = 1,
+    PY_CODE_LOCATION_INFO_SHORT2 = 2,
+    PY_CODE_LOCATION_INFO_SHORT3 = 3,
+    PY_CODE_LOCATION_INFO_SHORT4 = 4,
+    PY_CODE_LOCATION_INFO_SHORT5 = 5,
+
+    PYCODE_LOCATION_INFO_TWO_LINES = 14,
+    PYCODE_LOCATION_INFO_NONE = 15
+} _PyCodeLocationInfoKind;
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
@@ -178,6 +178,7 @@ struct _PyCodeConstructor {
     PyObject *linetable;
     PyObject *endlinetable;
     PyObject *columntable;
+    PyObject *locationtable;
 
     /* used by the code */
     PyObject *consts;
@@ -221,23 +222,12 @@ extern PyObject* _PyCode_GetCellvars(PyCodeObject *);
 extern PyObject* _PyCode_GetFreevars(PyCodeObject *);
 extern PyObject* _PyCode_GetCode(PyCodeObject *);
 
-/* Return the ending source code line number from a bytecode index. */
-extern int _PyCode_Addr2EndLine(PyCodeObject *, int);
-
-/* Return the ending source code line number from a bytecode index. */
-extern int _PyCode_Addr2EndLine(PyCodeObject *, int);
-/* Return the starting source code column offset from a bytecode index. */
-extern int _PyCode_Addr2Offset(PyCodeObject *, int);
-/* Return the ending source code column offset from a bytecode index. */
-extern int _PyCode_Addr2EndOffset(PyCodeObject *, int);
-
 /** API for initializing the line number tables. */
 extern int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds);
-extern int _PyCode_InitEndAddressRange(PyCodeObject* co, PyCodeAddressRange* bounds);
 
-/** Out of process API for initializing the line number table. */
-extern void _PyLineTable_InitAddressRange(
-    const char *linetable,
+/** Out of process API for initializing the location table. */
+extern void _PyLocationTable_InitAddressRange(
+    const char *locationtable,
     Py_ssize_t length,
     int firstlineno,
     PyCodeAddressRange *range);

@@ -400,6 +400,7 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH)
 #     Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH,
 #                         add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual)
+#     Python 3.11a7 3494 (New location info table)
 
 #     Python 3.12 will start with magic number 3500
 

diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py
@@ -234,6 +234,7 @@ def func(): pass
                         co.co_endlinetable,
                         co.co_columntable,
                         co.co_exceptiontable,
+                        co.co_locationtable,
                         co.co_freevars,
                         co.co_cellvars)
 
@@ -315,6 +316,7 @@ def func():
                          co.co_endlinetable,
                          co.co_columntable,
                          co.co_exceptiontable,
+                         co.co_locationtable,
                          co.co_freevars,
                          co.co_cellvars,
                          )
@@ -333,10 +335,10 @@ def func(arg):
         newcode = code.replace(co_name="func")  # Should not raise SystemError
         self.assertEqual(code, newcode)
 
-    def test_empty_linetable(self):
+    def test_empty_locationtable(self):
         def func():
             pass
-        new_code = code = func.__code__.replace(co_linetable=b'')
+        new_code = code = func.__code__.replace(co_locationtable=b'')
         self.assertEqual(list(new_code.co_lines()), [])
 
     @requires_debug_ranges()
@@ -383,9 +385,9 @@ def test_co_positions_artificial_instructions(self):
                 ("LOAD_CONST", None), # artificial 'None'
                 ("STORE_NAME", "e"),  # XX: we know the location for this
                 ("DELETE_NAME", "e"),
-                ("RERAISE", 1),
-                ("COPY", 3),
-                ("POP_EXCEPT", None),
+                ('RERAISE', 1),
+                ('COPY', 3),
+                ('POP_EXCEPT', None),
                 ("RERAISE", 1)
             ]
         )
@@ -416,40 +418,15 @@ def f():
     # co_positions behavior when info is missing.
 
     @requires_debug_ranges()
-    def test_co_positions_empty_linetable(self):
+    def test_co_positions_empty_locationtable(self):
         def func():
             x = 1
-        new_code = func.__code__.replace(co_linetable=b'')
+        new_code = func.__code__.replace(co_locationtable=b'')
         positions = new_code.co_positions()
-        next(positions) # Skip RESUME at start
         for line, end_line, column, end_column in positions:
             self.assertIsNone(line)
             self.assertEqual(end_line, new_code.co_firstlineno + 1)
 
-    @requires_debug_ranges()
-    def test_co_positions_empty_endlinetable(self):
-        def func():
-            x = 1
-        new_code = func.__code__.replace(co_endlinetable=b'')
-        positions = new_code.co_positions()
-        next(positions) # Skip RESUME at start
-        for line, end_line, column, end_column in positions:
-            self.assertEqual(line, new_code.co_firstlineno + 1)
-            self.assertIsNone(end_line)
-
-    @requires_debug_ranges()
-    def test_co_positions_empty_columntable(self):
-        def func():
-            x = 1
-        new_code = func.__code__.replace(co_columntable=b'')
-        positions = new_code.co_positions()
-        next(positions) # Skip RESUME at start
-        for line, end_line, column, end_column in positions:
-            self.assertEqual(line, new_code.co_firstlineno + 1)
-            self.assertEqual(end_line, new_code.co_firstlineno + 1)
-            self.assertIsNone(column)
-            self.assertIsNone(end_column)
-
 
 def isinterned(s):
     return s is sys.intern(('_' + s + '_')[1:-1])
@@ -527,6 +504,122 @@ def callback(code):
         self.assertFalse(bool(coderef()))
         self.assertTrue(self.called)
 
+# Python implementation of location table parsing algorithm
+def read(it):
+    return next(it)
+
+def read_varint(it):
+    b = read(it)
+    val = b & 63;
+    shift = 0;
+    while b & 64:
+        b = read(it)
+        shift += 6
+        val |= (b&63) << shift
+    return val
+
+def read_signed_varint(it):
+    uval = read_varint(it)
+    if uval & 1:
+        return -(uval >> 1)
+    else:
+        return uval >> 1
+
+def parse_location_table(code):
+    line = code.co_firstlineno
+    it = iter(code.co_locationtable)
+    while True:
+        try:
+            first_byte = read(it)
+        except StopIteration:
+            return
+        code = (first_byte >> 3) & 15
+        length = (first_byte & 7) + 1
+        if code == 15:
+            yield (code, length, None, None, None, None)
+        elif code == 14:
+            line_delta = read_signed_varint(it)
+            line += line_delta
+            end_line = line + read_varint(it)
+            col = read_varint(it)
+            if col == 0:
+                col = None
+            else:
+                col -= 1
+            end_col = read_varint(it)
+            if end_col == 0:
+                end_col = None
+            else:
+                end_col -= 1
+            yield (code, length, line, end_line, col, end_col)
+        elif code == 13: # No column
+            line_delta = read_signed_varint(it)
+            line += line_delta
+            yield (code, length, line, line, None, None)
+        elif code in (10, 11, 12): # new line
+            line_delta = code - 10
+            line += line_delta
+            column = read(it)
+            end_column = read(it)
+            yield (code, length, line, line, column, end_column)
+        else:
+            assert (0 <= code < 10)
+            second_byte = read(it)
+            column = code << 3 | (second_byte >> 4)
+            yield (code, length, line, line, column, column + (second_byte & 15))
+
+def positions_from_location_table(code):
+    for _, length, line, end_line, col, end_col in parse_location_table(code):
+        for _ in range(length):
+            yield (line, end_line, col, end_col)
+
+def misshappen():
+    """
+
+
+
+
+
+    """
+    x = (
+
+
+        4
+
+        +
+
+        y
+
+    )
+    y = (
+        a
+        +
+            b
+                +
+
+                d
+        )
+    return q if (
+
+        x
+
+        ) else p
+
+
+class CodeLocationTest(unittest.TestCase):
+
+    def check_positions(self, func):
+        pos1 = list(func.__code__.co_positions())
+        pos2 = list(positions_from_location_table(func.__code__))
+        for l1, l2 in zip(pos1, pos2):
+            self.assertEqual(l1, l2)
+        self.assertEqual(len(pos1), len(pos2))
+
+
+    def test_positions(self):
+        self.check_positions(parse_location_table)
+        self.check_positions(misshappen)
+
 
 if check_impl_detail(cpython=True) and ctypes is not None:
     py = ctypes.pythonapi

diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
@@ -158,7 +158,9 @@ def test_leading_newlines(self):
         s256 = "".join(["\n"] * 256 + ["spam"])
         co = compile(s256, 'fn', 'exec')
         self.assertEqual(co.co_firstlineno, 1)
-        self.assertEqual(list(co.co_lines()), [(0, 2, None), (2, 10, 257)])
+        lines = list(co.co_lines())
+        self.assertEqual(lines[0][2], None)
+        self.assertEqual(lines[1][2], 257)
 
     def test_literals_with_leading_zeroes(self):
         for arg in ["077787", "0xj", "0x.", "0e",  "090000000000000",
@@ -892,12 +894,19 @@ def no_code2():
             with self.subTest(func=func):
                 code = func.__code__
                 lines = list(code.co_lines())
-                self.assertEqual(len(lines), 1)
                 start, end, line = lines[0]
                 self.assertEqual(start, 0)
-                self.assertEqual(end, len(code.co_code))
                 self.assertEqual(line, code.co_firstlineno)
 
+    def get_code_lines(self, code):
+        last_line = -2
+        res = []
+        for _, _, line in code.co_lines():
+            if line is not None and line != last_line:
+                res.append(line - code.co_firstlineno)
+                last_line = line
+        return res
+
     def test_lineno_attribute(self):
         def load_attr():
             return (
@@ -939,9 +948,7 @@ def aug_store_attr():
 
         for func, lines in zip(funcs, func_lines, strict=True):
             with self.subTest(func=func):
-                code_lines = [ line-func.__code__.co_firstlineno
-                              for (_, _, line) in func.__code__.co_lines()
-                              if line is not None ]
+                code_lines = self.get_code_lines(func.__code__)
                 self.assertEqual(lines, code_lines)
 
     def test_line_number_genexp(self):
@@ -952,11 +959,10 @@ def return_genexp():
                     x
                     in
                     y)
-        genexp_lines = [1, 3, 1]
+        genexp_lines = [0, 2, 0]
 
         genexp_code = return_genexp.__code__.co_consts[1]
-        code_lines = [ None if line is None else line-return_genexp.__code__.co_firstlineno
-                      for (_, _, line) in genexp_code.co_lines() ]
+        code_lines = self.get_code_lines(genexp_code)
         self.assertEqual(genexp_lines, code_lines)
 
     def test_line_number_implicit_return_after_async_for(self):
@@ -966,8 +972,7 @@ async def test(aseq):
                 body
 
         expected_lines = [0, 1, 2, 1]
-        code_lines = [ None if line is None else line-test.__code__.co_firstlineno
-                      for (_, _, line) in test.__code__.co_lines() ]
+        code_lines = self.get_code_lines(test.__code__)
         self.assertEqual(expected_lines, code_lines)
 
     def test_big_dict_literal(self):
@@ -1112,14 +1117,14 @@ def test_multiline_expression(self):
             line=1, end_line=3, column=0, end_column=1)
 
     def test_very_long_line_end_offset(self):
-        # Make sure we get None for when the column offset is too large to
-        # store in a byte.
+        # Make sure we get the correct column offset for offsets
+        # too large to store in a byte.
         long_string = "a" * 1000
         snippet = f"g('{long_string}')"
 
         compiled_code, _ = self.check_positions_against_ast(snippet)
         self.assertOpcodeSourcePositionIs(compiled_code, 'CALL',
-            line=1, end_line=1, column=None, end_column=None)
+            line=1, end_line=1, column=0, end_column=1005)
 
     def test_complex_single_line_expression(self):
         snippet = "a - b @ (c * x['key'] + 23)"