py/objexcept: Allow compression of exception message text.

The decompression of error-strings is only done if the string is accessed
via printing or via er.args.  Tests are added for this feature to ensure
the decompression works.

py/compile.c

@@ -197,7 +197,7 @@ STATIC void compile_error_set_line(compiler_t *comp, mp_parse_node_t pn) {
     }
 }
 
-STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const char *msg) {
+STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, mp_rom_error_text_t msg) {
     // only register the error if there has been no other error
     if (comp->compile_error == MP_OBJ_NULL) {
         comp->compile_error = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg);

py/emitinlinethumb.c

@@ -59,7 +59,7 @@ struct _emit_inline_asm_t {
     qstr *label_lookup;
 };
 
-STATIC void emit_inline_thumb_error_msg(emit_inline_asm_t *emit, const char *msg) {
+STATIC void emit_inline_thumb_error_msg(emit_inline_asm_t *emit, mp_rom_error_text_t msg) {
     *emit->error_slot = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg);
 }
 

py/nativeglue.h

@@ -145,7 +145,7 @@ typedef struct _mp_fun_table_t {
     #if defined(__GNUC__)
     NORETURN // Only certain compilers support no-return attributes in function pointer declarations
     #endif
-    void (*raise_msg)(const mp_obj_type_t *exc_type, const char *msg);
+    void (*raise_msg)(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg);
     const mp_obj_type_t *(*obj_get_type)(mp_const_obj_t o_in);
     mp_obj_t (*obj_new_str)(const char *data, size_t len);
     mp_obj_t (*obj_new_bytes)(const byte *data, size_t len);

py/obj.h

@@ -731,10 +731,10 @@ mp_obj_t mp_obj_new_complex(mp_float_t real, mp_float_t imag);
 mp_obj_t mp_obj_new_exception(const mp_obj_type_t *exc_type);
 mp_obj_t mp_obj_new_exception_arg1(const mp_obj_type_t *exc_type, mp_obj_t arg);
 mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, const mp_obj_t *args);
-mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg);
+mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg);
-mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!)
+mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!)
 #ifdef va_start
-mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const char *fmt, va_list arg); // same fmt restrictions as above
+mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list arg); // same fmt restrictions as above
 #endif
 mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table);
 mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const void *fun_data, const mp_uint_t *const_table);

py/objexcept.c

@@ -38,6 +38,15 @@
 #include "py/gc.h"
 #include "py/mperrno.h"
 
+// Extract the MP_MAX_UNCOMPRESSED_TEXT_LEN macro from "genhdr/compressed.data.h"
+#if MICROPY_ROM_TEXT_COMPRESSION
+#define MP_MATCH_COMPRESSED(...) // Ignore
+#define MP_COMPRESSED_DATA(...) // Ignore
+#include "genhdr/compressed.data.h"
+#undef MP_MATCH_COMPRESSED
+#undef MP_COMPRESSED_DATA
+#endif
+
 // Number of items per traceback entry (file, line, block)
 #define TRACEBACK_ENTRY_LEN (3)
 
@@ -57,6 +66,7 @@
 #define EMG_BUF_TUPLE_OFFSET        (EMG_BUF_TRACEBACK_OFFSET + EMG_BUF_TRACEBACK_SIZE)
 #define EMG_BUF_TUPLE_SIZE(n_args)  (sizeof(mp_obj_tuple_t) + n_args * sizeof(mp_obj_t))
 #define EMG_BUF_STR_OFFSET          (EMG_BUF_TUPLE_OFFSET + EMG_BUF_TUPLE_SIZE(1))
+#define EMG_BUF_STR_BUF_OFFSET      (EMG_BUF_STR_OFFSET + sizeof(mp_obj_str_t))
 
 #if MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE > 0
 #define mp_emergency_exception_buf_size MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE
@@ -100,6 +110,40 @@ mp_obj_t mp_alloc_emergency_exception_buf(mp_obj_t size_in) {
 #endif
 #endif  // MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF
 
+STATIC void decompress_error_text_maybe(mp_obj_exception_t *o) {
+    #if MICROPY_ROM_TEXT_COMPRESSION
+    if (o->args->len == 1 && mp_obj_is_type(o->args->items[0], &mp_type_str)) {
+        mp_obj_str_t *o_str = MP_OBJ_TO_PTR(o->args->items[0]);
+        if (MP_IS_COMPRESSED_ROM_STRING(o_str->data)) {
+            byte *buf = m_new_maybe(byte, MP_MAX_UNCOMPRESSED_TEXT_LEN + 1);
+            if (!buf) {
+                #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF
+                // Try and use the emergency exception buf if enough space is available.
+                buf = (byte *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + EMG_BUF_STR_BUF_OFFSET);
+                size_t avail = (uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + mp_emergency_exception_buf_size - buf;
+                if (avail < MP_MAX_UNCOMPRESSED_TEXT_LEN + 1) {
+                    // No way to decompress, fallback to no message text.
+                    o->args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
+                    return;
+                }
+                #else
+                o->args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
+                return;
+                #endif
+            }
+            mp_decompress_rom_string(buf, (mp_rom_error_text_t)o_str->data);
+            o_str->data = buf;
+            o_str->len = strlen((const char *)buf);
+            o_str->hash = 0;
+        }
+        // Lazily compute the string hash.
+        if (o_str->hash == 0) {
+            o_str->hash = qstr_compute_hash(o_str->data, o_str->len);
+        }
+    }
+    #endif
+}
+
 void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) {
     mp_obj_exception_t *o = MP_OBJ_TO_PTR(o_in);
     mp_print_kind_t k = kind & ~PRINT_EXC_SUBCLASS;
@@ -112,6 +156,8 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin
         mp_print_str(print, ": ");
     }
 
+    decompress_error_text_maybe(o);
+
     if (k == PRINT_STR || k == PRINT_EXC) {
         if (o->args == NULL || o->args->len == 0) {
             mp_print_str(print, "");
@@ -131,6 +177,7 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin
             return;
         }
     }
+
     mp_obj_tuple_print(print, MP_OBJ_FROM_PTR(o->args), kind);
 }
 
@@ -189,6 +236,7 @@ mp_obj_t mp_obj_exception_get_value(mp_obj_t self_in) {
     if (self->args->len == 0) {
         return mp_const_none;
     } else {
+        decompress_error_text_maybe(self);
         return self->args->items[0];
     }
 }
@@ -210,6 +258,7 @@ void mp_obj_exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
         return;
     }
     if (attr == MP_QSTR_args) {
+        decompress_error_text_maybe(self);
         dest[0] = MP_OBJ_FROM_PTR(self->args);
     } else if (self->base.type == &mp_type_StopIteration && attr == MP_QSTR_value) {
         dest[0] = mp_obj_exception_get_value(self_in);
@@ -323,7 +372,7 @@ mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args,
     return mp_obj_exception_make_new(exc_type, n_args, 0, args);
 }
 
-mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg) {
+mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg) {
     // Check that the given type is an exception type
     assert(exc_type->make_new == mp_obj_exception_make_new);
 
@@ -348,9 +397,13 @@ mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg
 
     // Create the string object and call mp_obj_exception_make_new to create the exception
     o_str->base.type = &mp_type_str;
-    o_str->len = strlen(msg);
+    o_str->len = strlen((const char *)msg);
     o_str->data = (const byte *)msg;
+    #if MICROPY_ROM_TEXT_COMPRESSION
+    o_str->hash = 0; // will be computed only if string object is accessed
+    #else
     o_str->hash = qstr_compute_hash(o_str->data, o_str->len);
+    #endif
     mp_obj_t arg = MP_OBJ_FROM_PTR(o_str);
     return mp_obj_exception_make_new(exc_type, 1, 0, &arg);
 }
@@ -388,7 +441,7 @@ STATIC void exc_add_strn(void *data, const char *str, size_t len) {
     pr->len += len;
 }
 
-mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...) {
+mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...) {
     va_list args;
     va_start(args, fmt);
     mp_obj_t exc = mp_obj_new_exception_msg_vlist(exc_type, fmt, args);
@@ -396,7 +449,7 @@ mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char
     return exc;
 }
 
-mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const char *fmt, va_list args) {
+mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list args) {
     assert(fmt != NULL);
 
     // Check that the given type is an exception type
@@ -404,7 +457,7 @@ mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const cha
 
     // Try to allocate memory for the message
     mp_obj_str_t *o_str = m_new_obj_maybe(mp_obj_str_t);
-    size_t o_str_alloc = strlen(fmt) + 1;
+    size_t o_str_alloc = strlen((const char *)fmt) + 1;
     byte *o_str_buf = m_new_maybe(byte, o_str_alloc);
 
     bool used_emg_buf = false;
@@ -415,29 +468,39 @@ mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const cha
     if ((o_str == NULL || o_str_buf == NULL)
         && mp_emergency_exception_buf_size >= EMG_BUF_STR_OFFSET + sizeof(mp_obj_str_t) + 16) {
         used_emg_buf = true;
-        o_str = (mp_obj_str_t *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf)
+        o_str = (mp_obj_str_t *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + EMG_BUF_STR_OFFSET);
-            + EMG_BUF_STR_OFFSET);
+        o_str_buf = (byte *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + EMG_BUF_STR_BUF_OFFSET);
-        o_str_buf = (byte *)&o_str[1];
+        o_str_alloc = (uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + mp_emergency_exception_buf_size - o_str_buf;
-        o_str_alloc = (uint8_t *)MP_STATE_VM(mp_emergency_exception_buf)
-            + mp_emergency_exception_buf_size - o_str_buf;
     }
     #endif
 
     if (o_str == NULL) {
-        // No memory for the string object so create the exception with no args
+        // No memory for the string object so create the exception with no args.
+        // The exception will only have a type and no message (compression is irrelevant).
         return mp_obj_exception_make_new(exc_type, 0, 0, NULL);
     }
 
     if (o_str_buf == NULL) {
         // No memory for the string buffer: assume that the fmt string is in ROM
-        // and use that data as the data of the string
+        // and use that data as the data of the string.
+        // The string will point directly to the compressed data -- will need to be decompressed
+        // prior to display (this case is identical to mp_obj_new_exception_msg above).
         o_str->len = o_str_alloc - 1; // will be equal to strlen(fmt)
         o_str->data = (const byte *)fmt;
     } else {
-        // We have some memory to format the string
+        // We have some memory to format the string.
+        // TODO: Optimise this to format-while-decompressing (and not require the temp stack space).
         struct _exc_printer_t exc_pr = {!used_emg_buf, o_str_alloc, 0, o_str_buf};
         mp_print_t print = {&exc_pr, exc_add_strn};
-        mp_vprintf(&print, fmt, args);
+        const char *fmt2 = (const char *)fmt;
+        #if MICROPY_ROM_TEXT_COMPRESSION
+        byte decompressed[MP_MAX_UNCOMPRESSED_TEXT_LEN];
+        if (MP_IS_COMPRESSED_ROM_STRING(fmt)) {
+            mp_decompress_rom_string(decompressed, fmt);
+            fmt2 = (const char *)decompressed;
+        }
+        #endif
+        mp_vprintf(&print, fmt2, args);
         exc_pr.buf[exc_pr.len] = '\0';
         o_str->len = exc_pr.len;
         o_str->data = exc_pr.buf;
@@ -445,7 +508,11 @@ mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const cha
 
     // Create the string object and call mp_obj_exception_make_new to create the exception
     o_str->base.type = &mp_type_str;
+    #if MICROPY_ROM_TEXT_COMPRESSION
+    o_str->hash = 0; // will be computed only if string object is accessed
+    #else
     o_str->hash = qstr_compute_hash(o_str->data, o_str->len);
+    #endif
     mp_obj_t arg = MP_OBJ_FROM_PTR(o_str);
     return mp_obj_exception_make_new(exc_type, 1, 0, &arg);
 }

py/runtime.c

@@ -1500,7 +1500,7 @@ NORETURN void m_malloc_fail(size_t num_bytes) {
         "memory allocation failed, allocating %u bytes", (uint)num_bytes);
 }
 
-NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg) {
+NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg) {
     if (msg == NULL) {
         nlr_raise(mp_obj_new_exception(exc_type));
     } else {
@@ -1508,7 +1508,7 @@ NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg) {
     }
 }
 
-NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...) {
+NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...) {
     va_list args;
     va_start(args, fmt);
     mp_obj_t exc = mp_obj_new_exception_msg_vlist(exc_type, fmt, args);
@@ -1516,11 +1516,11 @@ NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt,
     nlr_raise(exc);
 }
 
-NORETURN void mp_raise_ValueError(const char *msg) {
+NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg) {
     mp_raise_msg(&mp_type_ValueError, msg);
 }
 
-NORETURN void mp_raise_TypeError(const char *msg) {
+NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg) {
     mp_raise_msg(&mp_type_TypeError, msg);
 }
 
@@ -1528,7 +1528,7 @@ NORETURN void mp_raise_OSError(int errno_) {
     nlr_raise(mp_obj_new_exception_arg1(&mp_type_OSError, MP_OBJ_NEW_SMALL_INT(errno_)));
 }
 
-NORETURN void mp_raise_NotImplementedError(const char *msg) {
+NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg) {
     mp_raise_msg(&mp_type_NotImplementedError, msg);
 }
 

py/runtime.h

@@ -162,11 +162,11 @@ mp_obj_t mp_import_from(mp_obj_t module, qstr name);
 void mp_import_all(mp_obj_t module);
 
 #define mp_raise_type(exc_type) mp_raise_msg(exc_type, NULL)
-NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg);
+NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg);
-NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...);
+NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...);
-NORETURN void mp_raise_ValueError(const char *msg);
+NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg);
-NORETURN void mp_raise_TypeError(const char *msg);
+NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg);
-NORETURN void mp_raise_NotImplementedError(const char *msg);
+NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg);
 NORETURN void mp_raise_OSError(int errno_);
 NORETURN void mp_raise_recursion_depth(void);
 

tests/micropython/heapalloc_exc_compressed.py

@@ -0,0 +1,48 @@
+import micropython
+
+# Tests both code paths for built-in exception raising.
+# mp_obj_new_exception_msg_varg (exception requires decompression at raise-time to format)
+# mp_obj_new_exception_msg (decompression can be deferred)
+
+# NameError uses mp_obj_new_exception_msg_varg for NameError("name '%q' isn't defined")
+# set.pop uses mp_obj_new_exception_msg for KeyError("pop from an empty set")
+
+# Tests that deferred decompression works both via print(e) and accessing the message directly via e.args.
+
+a = set()
+
+# First test the regular case (can use heap for allocating the decompression buffer).
+try:
+    name()
+except NameError as e:
+    print(type(e).__name__, e)
+
+try:
+    a.pop()
+except KeyError as e:
+    print(type(e).__name__, e)
+
+try:
+    name()
+except NameError as e:
+    print(e.args[0])
+
+try:
+    a.pop()
+except KeyError as e:
+    print(e.args[0])
+
+# Then test that it still works when the heap is locked (i.e. in ISR context).
+micropython.heap_lock()
+
+try:
+    name()
+except NameError as e:
+    print(type(e).__name__)
+
+try:
+    a.pop()
+except KeyError as e:
+    print(type(e).__name__)
+
+micropython.heap_unlock()

tests/micropython/heapalloc_exc_compressed.py.exp

@@ -0,0 +1,6 @@
+NameError name 'name' isn't defined
+KeyError pop from an empty set
+name 'name' isn't defined
+pop from an empty set
+NameError
+KeyError

tests/micropython/heapalloc_exc_compressed_emg_exc.py

@@ -0,0 +1,41 @@
+import micropython
+
+# Does the full test from heapalloc_exc_compressed.py but while the heap is
+# locked (this can only work when the emergency exception buf is enabled).
+
+# Some ports need to allocate heap for the emgergency exception buffer.
+try:
+    micropython.alloc_emergency_exception_buf(256)
+except AttributeError:
+    pass
+
+a = set()
+
+
+def test():
+    micropython.heap_lock()
+
+    try:
+        name()
+    except NameError as e:
+        print(type(e).__name__, e)
+
+    try:
+        a.pop()
+    except KeyError as e:
+        print(type(e).__name__, e)
+
+    try:
+        name()
+    except NameError as e:
+        print(e.args[0])
+
+    try:
+        a.pop()
+    except KeyError as e:
+        print(e.args[0])
+
+    micropython.heap_unlock()
+
+
+test()

tests/micropython/heapalloc_exc_compressed_emg_exc.py.exp

@@ -0,0 +1,4 @@
+NameError name 'name' isn't defined
+KeyError pop from an empty set
+name 'name' isn't defined
+pop from an empty set

tools/tinytest-codegen.py

@@ -89,6 +89,8 @@ exclude_tests = (
     # different filename in output
     "micropython/emg_exc.py",
     "micropython/heapalloc_traceback.py",
+    # don't have emergency exception buffer
+    "micropython/heapalloc_exc_compressed_emg_exc.py",
     # pattern matching in .exp
     "micropython/meminfo.py",
     # needs sys stdfiles