Skip to content

Commit 85858e7

Browse files
jimmodpgeorge
jimmo
authored and
dpgeorge
committed
py/objexcept: Allow compression of exception message text.
The decompression of error-strings is only done if the string is accessed via printing or via er.args. Tests are added for this feature to ensure the decompression works.
1 parent 92c83bd commit 85858e7

12 files changed

+198
-30
lines changed

‎py/compile.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ STATIC void compile_error_set_line(compiler_t *comp, mp_parse_node_t pn) {
197197
}
198198
}
199199

200-
STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const char *msg) {
200+
STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, mp_rom_error_text_t msg) {
201201
// only register the error if there has been no other error
202202
if (comp->compile_error == MP_OBJ_NULL) {
203203
comp->compile_error = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg);

‎py/emitinlinethumb.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ struct _emit_inline_asm_t {
5959
qstr *label_lookup;
6060
};
6161

62-
STATIC void emit_inline_thumb_error_msg(emit_inline_asm_t *emit, const char *msg) {
62+
STATIC void emit_inline_thumb_error_msg(emit_inline_asm_t *emit, mp_rom_error_text_t msg) {
6363
*emit->error_slot = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg);
6464
}
6565

‎py/nativeglue.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ typedef struct _mp_fun_table_t {
145145
#if defined(__GNUC__)
146146
NORETURN // Only certain compilers support no-return attributes in function pointer declarations
147147
#endif
148-
void (*raise_msg)(const mp_obj_type_t *exc_type, const char *msg);
148+
void (*raise_msg)(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg);
149149
const mp_obj_type_t *(*obj_get_type)(mp_const_obj_t o_in);
150150
mp_obj_t (*obj_new_str)(const char *data, size_t len);
151151
mp_obj_t (*obj_new_bytes)(const byte *data, size_t len);

‎py/obj.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -731,10 +731,10 @@ mp_obj_t mp_obj_new_complex(mp_float_t real, mp_float_t imag);
731731
mp_obj_t mp_obj_new_exception(const mp_obj_type_t *exc_type);
732732
mp_obj_t mp_obj_new_exception_arg1(const mp_obj_type_t *exc_type, mp_obj_t arg);
733733
mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, const mp_obj_t *args);
734-
mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg);
735-
mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!)
734+
mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg);
735+
mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!)
736736
#ifdef va_start
737-
mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const char *fmt, va_list arg); // same fmt restrictions as above
737+
mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list arg); // same fmt restrictions as above
738738
#endif
739739
mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table);
740740
mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const void *fun_data, const mp_uint_t *const_table);

‎py/objexcept.c‎

Lines changed: 81 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,15 @@
3838
#include "py/gc.h"
3939
#include "py/mperrno.h"
4040

41+
// Extract the MP_MAX_UNCOMPRESSED_TEXT_LEN macro from "genhdr/compressed.data.h"
42+
#if MICROPY_ROM_TEXT_COMPRESSION
43+
#define MP_MATCH_COMPRESSED(...) // Ignore
44+
#define MP_COMPRESSED_DATA(...) // Ignore
45+
#include "genhdr/compressed.data.h"
46+
#undef MP_MATCH_COMPRESSED
47+
#undef MP_COMPRESSED_DATA
48+
#endif
49+
4150
// Number of items per traceback entry (file, line, block)
4251
#define TRACEBACK_ENTRY_LEN (3)
4352

@@ -57,6 +66,7 @@
5766
#define EMG_BUF_TUPLE_OFFSET (EMG_BUF_TRACEBACK_OFFSET + EMG_BUF_TRACEBACK_SIZE)
5867
#define EMG_BUF_TUPLE_SIZE(n_args) (sizeof(mp_obj_tuple_t) + n_args * sizeof(mp_obj_t))
5968
#define EMG_BUF_STR_OFFSET (EMG_BUF_TUPLE_OFFSET + EMG_BUF_TUPLE_SIZE(1))
69+
#define EMG_BUF_STR_BUF_OFFSET (EMG_BUF_STR_OFFSET + sizeof(mp_obj_str_t))
6070

6171
#if MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE > 0
6272
#define mp_emergency_exception_buf_size MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE
@@ -100,6 +110,40 @@ mp_obj_t mp_alloc_emergency_exception_buf(mp_obj_t size_in) {
100110
#endif
101111
#endif // MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF
102112

113+
STATIC void decompress_error_text_maybe(mp_obj_exception_t *o) {
114+
#if MICROPY_ROM_TEXT_COMPRESSION
115+
if (o->args->len == 1 && mp_obj_is_type(o->args->items[0], &mp_type_str)) {
116+
mp_obj_str_t *o_str = MP_OBJ_TO_PTR(o->args->items[0]);
117+
if (MP_IS_COMPRESSED_ROM_STRING(o_str->data)) {
118+
byte *buf = m_new_maybe(byte, MP_MAX_UNCOMPRESSED_TEXT_LEN + 1);
119+
if (!buf) {
120+
#if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF
121+
// Try and use the emergency exception buf if enough space is available.
122+
buf = (byte *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + EMG_BUF_STR_BUF_OFFSET);
123+
size_t avail = (uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + mp_emergency_exception_buf_size - buf;
124+
if (avail < MP_MAX_UNCOMPRESSED_TEXT_LEN + 1) {
125+
// No way to decompress, fallback to no message text.
126+
o->args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
127+
return;
128+
}
129+
#else
130+
o->args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
131+
return;
132+
#endif
133+
}
134+
mp_decompress_rom_string(buf, (mp_rom_error_text_t)o_str->data);
135+
o_str->data = buf;
136+
o_str->len = strlen((const char *)buf);
137+
o_str->hash = 0;
138+
}
139+
// Lazily compute the string hash.
140+
if (o_str->hash == 0) {
141+
o_str->hash = qstr_compute_hash(o_str->data, o_str->len);
142+
}
143+
}
144+
#endif
145+
}
146+
103147
void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) {
104148
mp_obj_exception_t *o = MP_OBJ_TO_PTR(o_in);
105149
mp_print_kind_t k = kind & ~PRINT_EXC_SUBCLASS;
@@ -112,6 +156,8 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin
112156
mp_print_str(print, ": ");
113157
}
114158

159+
decompress_error_text_maybe(o);
160+
115161
if (k == PRINT_STR || k == PRINT_EXC) {
116162
if (o->args == NULL || o->args->len == 0) {
117163
mp_print_str(print, "");
@@ -131,6 +177,7 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin
131177
return;
132178
}
133179
}
180+
134181
mp_obj_tuple_print(print, MP_OBJ_FROM_PTR(o->args), kind);
135182
}
136183

@@ -189,6 +236,7 @@ mp_obj_t mp_obj_exception_get_value(mp_obj_t self_in) {
189236
if (self->args->len == 0) {
190237
return mp_const_none;
191238
} else {
239+
decompress_error_text_maybe(self);
192240
return self->args->items[0];
193241
}
194242
}
@@ -210,6 +258,7 @@ void mp_obj_exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
210258
return;
211259
}
212260
if (attr == MP_QSTR_args) {
261+
decompress_error_text_maybe(self);
213262
dest[0] = MP_OBJ_FROM_PTR(self->args);
214263
} else if (self->base.type == &mp_type_StopIteration && attr == MP_QSTR_value) {
215264
dest[0] = mp_obj_exception_get_value(self_in);
@@ -323,7 +372,7 @@ mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args,
323372
return mp_obj_exception_make_new(exc_type, n_args, 0, args);
324373
}
325374

326-
mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg) {
375+
mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg) {
327376
// Check that the given type is an exception type
328377
assert(exc_type->make_new == mp_obj_exception_make_new);
329378

@@ -348,9 +397,13 @@ mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg
348397

349398
// Create the string object and call mp_obj_exception_make_new to create the exception
350399
o_str->base.type = &mp_type_str;
351-
o_str->len = strlen(msg);
400+
o_str->len = strlen((const char *)msg);
352401
o_str->data = (const byte *)msg;
402+
#if MICROPY_ROM_TEXT_COMPRESSION
403+
o_str->hash = 0; // will be computed only if string object is accessed
404+
#else
353405
o_str->hash = qstr_compute_hash(o_str->data, o_str->len);
406+
#endif
354407
mp_obj_t arg = MP_OBJ_FROM_PTR(o_str);
355408
return mp_obj_exception_make_new(exc_type, 1, 0, &arg);
356409
}
@@ -388,23 +441,23 @@ STATIC void exc_add_strn(void *data, const char *str, size_t len) {
388441
pr->len += len;
389442
}
390443

391-
mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...) {
444+
mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...) {
392445
va_list args;
393446
va_start(args, fmt);
394447
mp_obj_t exc = mp_obj_new_exception_msg_vlist(exc_type, fmt, args);
395448
va_end(args);
396449
return exc;
397450
}
398451

399-
mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const char *fmt, va_list args) {
452+
mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list args) {
400453
assert(fmt != NULL);
401454

402455
// Check that the given type is an exception type
403456
assert(exc_type->make_new == mp_obj_exception_make_new);
404457

405458
// Try to allocate memory for the message
406459
mp_obj_str_t *o_str = m_new_obj_maybe(mp_obj_str_t);
407-
size_t o_str_alloc = strlen(fmt) + 1;
460+
size_t o_str_alloc = strlen((const char *)fmt) + 1;
408461
byte *o_str_buf = m_new_maybe(byte, o_str_alloc);
409462

410463
bool used_emg_buf = false;
@@ -415,37 +468,51 @@ mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const cha
415468
if ((o_str == NULL || o_str_buf == NULL)
416469
&& mp_emergency_exception_buf_size >= EMG_BUF_STR_OFFSET + sizeof(mp_obj_str_t) + 16) {
417470
used_emg_buf = true;
418-
o_str = (mp_obj_str_t *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf)
419-
+ EMG_BUF_STR_OFFSET);
420-
o_str_buf = (byte *)&o_str[1];
421-
o_str_alloc = (uint8_t *)MP_STATE_VM(mp_emergency_exception_buf)
422-
+ mp_emergency_exception_buf_size - o_str_buf;
471+
o_str = (mp_obj_str_t *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + EMG_BUF_STR_OFFSET);
472+
o_str_buf = (byte *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + EMG_BUF_STR_BUF_OFFSET);
473+
o_str_alloc = (uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + mp_emergency_exception_buf_size - o_str_buf;
423474
}
424475
#endif
425476

426477
if (o_str == NULL) {
427-
// No memory for the string object so create the exception with no args
478+
// No memory for the string object so create the exception with no args.
479+
// The exception will only have a type and no message (compression is irrelevant).
428480
return mp_obj_exception_make_new(exc_type, 0, 0, NULL);
429481
}
430482

431483
if (o_str_buf == NULL) {
432484
// No memory for the string buffer: assume that the fmt string is in ROM
433-
// and use that data as the data of the string
485+
// and use that data as the data of the string.
486+
// The string will point directly to the compressed data -- will need to be decompressed
487+
// prior to display (this case is identical to mp_obj_new_exception_msg above).
434488
o_str->len = o_str_alloc - 1; // will be equal to strlen(fmt)
435489
o_str->data = (const byte *)fmt;
436490
} else {
437-
// We have some memory to format the string
491+
// We have some memory to format the string.
492+
// TODO: Optimise this to format-while-decompressing (and not require the temp stack space).
438493
struct _exc_printer_t exc_pr = {!used_emg_buf, o_str_alloc, 0, o_str_buf};
439494
mp_print_t print = {&exc_pr, exc_add_strn};
440-
mp_vprintf(&print, fmt, args);
495+
const char *fmt2 = (const char *)fmt;
496+
#if MICROPY_ROM_TEXT_COMPRESSION
497+
byte decompressed[MP_MAX_UNCOMPRESSED_TEXT_LEN];
498+
if (MP_IS_COMPRESSED_ROM_STRING(fmt)) {
499+
mp_decompress_rom_string(decompressed, fmt);
500+
fmt2 = (const char *)decompressed;
501+
}
502+
#endif
503+
mp_vprintf(&print, fmt2, args);
441504
exc_pr.buf[exc_pr.len] = '\0';
442505
o_str->len = exc_pr.len;
443506
o_str->data = exc_pr.buf;
444507
}
445508

446509
// Create the string object and call mp_obj_exception_make_new to create the exception
447510
o_str->base.type = &mp_type_str;
511+
#if MICROPY_ROM_TEXT_COMPRESSION
512+
o_str->hash = 0; // will be computed only if string object is accessed
513+
#else
448514
o_str->hash = qstr_compute_hash(o_str->data, o_str->len);
515+
#endif
449516
mp_obj_t arg = MP_OBJ_FROM_PTR(o_str);
450517
return mp_obj_exception_make_new(exc_type, 1, 0, &arg);
451518
}

‎py/runtime.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1500,35 +1500,35 @@ NORETURN void m_malloc_fail(size_t num_bytes) {
15001500
"memory allocation failed, allocating %u bytes", (uint)num_bytes);
15011501
}
15021502

1503-
NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg) {
1503+
NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg) {
15041504
if (msg == NULL) {
15051505
nlr_raise(mp_obj_new_exception(exc_type));
15061506
} else {
15071507
nlr_raise(mp_obj_new_exception_msg(exc_type, msg));
15081508
}
15091509
}
15101510

1511-
NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...) {
1511+
NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...) {
15121512
va_list args;
15131513
va_start(args, fmt);
15141514
mp_obj_t exc = mp_obj_new_exception_msg_vlist(exc_type, fmt, args);
15151515
va_end(args);
15161516
nlr_raise(exc);
15171517
}
15181518

1519-
NORETURN void mp_raise_ValueError(const char *msg) {
1519+
NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg) {
15201520
mp_raise_msg(&mp_type_ValueError, msg);
15211521
}
15221522

1523-
NORETURN void mp_raise_TypeError(const char *msg) {
1523+
NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg) {
15241524
mp_raise_msg(&mp_type_TypeError, msg);
15251525
}
15261526

15271527
NORETURN void mp_raise_OSError(int errno_) {
15281528
nlr_raise(mp_obj_new_exception_arg1(&mp_type_OSError, MP_OBJ_NEW_SMALL_INT(errno_)));
15291529
}
15301530

1531-
NORETURN void mp_raise_NotImplementedError(const char *msg) {
1531+
NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg) {
15321532
mp_raise_msg(&mp_type_NotImplementedError, msg);
15331533
}
15341534

‎py/runtime.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -162,11 +162,11 @@ mp_obj_t mp_import_from(mp_obj_t module, qstr name);
162162
void mp_import_all(mp_obj_t module);
163163

164164
#define mp_raise_type(exc_type) mp_raise_msg(exc_type, NULL)
165-
NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg);
166-
NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...);
167-
NORETURN void mp_raise_ValueError(const char *msg);
168-
NORETURN void mp_raise_TypeError(const char *msg);
169-
NORETURN void mp_raise_NotImplementedError(const char *msg);
165+
NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg);
166+
NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...);
167+
NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg);
168+
NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg);
169+
NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg);
170170
NORETURN void mp_raise_OSError(int errno_);
171171
NORETURN void mp_raise_recursion_depth(void);
172172

‎tests/micropython/heapalloc_exc_compressed.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import micropython
2+
3+
# Tests both code paths for built-in exception raising.
4+
# mp_obj_new_exception_msg_varg (exception requires decompression at raise-time to format)
5+
# mp_obj_new_exception_msg (decompression can be deferred)
6+
7+
# NameError uses mp_obj_new_exception_msg_varg for NameError("name '%q' isn't defined")
8+
# set.pop uses mp_obj_new_exception_msg for KeyError("pop from an empty set")
9+
10+
# Tests that deferred decompression works both via print(e) and accessing the message directly via e.args.
11+
12+
a = set()
13+
14+
# First test the regular case (can use heap for allocating the decompression buffer).
15+
try:
16+
name()
17+
except NameError as e:
18+
print(type(e).__name__, e)
19+
20+
try:
21+
a.pop()
22+
except KeyError as e:
23+
print(type(e).__name__, e)
24+
25+
try:
26+
name()
27+
except NameError as e:
28+
print(e.args[0])
29+
30+
try:
31+
a.pop()
32+
except KeyError as e:
33+
print(e.args[0])
34+
35+
# Then test that it still works when the heap is locked (i.e. in ISR context).
36+
micropython.heap_lock()
37+
38+
try:
39+
name()
40+
except NameError as e:
41+
print(type(e).__name__)
42+
43+
try:
44+
a.pop()
45+
except KeyError as e:
46+
print(type(e).__name__)
47+
48+
micropython.heap_unlock()

‎tests/micropython/heapalloc_exc_compressed.py.exp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
NameError name 'name' isn't defined
2+
KeyError pop from an empty set
3+
name 'name' isn't defined
4+
pop from an empty set
5+
NameError
6+
KeyError

0 commit comments

Comments
 (0)