py: Fix adding of traceback so that it appends to existing info.

This makes exception traceback info self contained (ie doesn't rely on
list object, which was a bit of a hack), reduces code size, and reduces
RAM footprint of exception by eliminating the list object.

Addresses part of issue #1126.

py/objexcept.c

@@ -38,7 +38,7 @@
 #include "py/gc.h"
 
 // Instance of MemoryError exception - needed by mp_malloc_fail
-const mp_obj_exception_t mp_const_MemoryError_obj = {{&mp_type_MemoryError}, MP_OBJ_NULL, mp_const_empty_tuple};
+const mp_obj_exception_t mp_const_MemoryError_obj = {{&mp_type_MemoryError}, 0, 0, MP_OBJ_NULL, mp_const_empty_tuple};
 
 // Optionally allocated buffer for storing the first argument of an exception
 // allocated when the heap is locked.
@@ -88,7 +88,7 @@ mp_obj_t mp_alloc_emergency_exception_buf(mp_obj_t size_in) {
 // Instance of GeneratorExit exception - needed by generator.close()
 // This would belong to objgenerator.c, but to keep mp_obj_exception_t
 // definition module-private so far, have it here.
-const mp_obj_exception_t mp_const_GeneratorExit_obj = {{&mp_type_GeneratorExit}, MP_OBJ_NULL, mp_const_empty_tuple};
+const mp_obj_exception_t mp_const_GeneratorExit_obj = {{&mp_type_GeneratorExit}, 0, 0, MP_OBJ_NULL, mp_const_empty_tuple};
 
 STATIC void mp_obj_exception_print(void (*print)(void *env, const char *fmt, ...), void *env, mp_obj_t o_in, mp_print_kind_t kind) {
     mp_obj_exception_t *o = o_in;
@@ -126,7 +126,7 @@ mp_obj_t mp_obj_exception_make_new(mp_obj_t type_in, mp_uint_t n_args, mp_uint_t
         o->args = mp_obj_new_tuple(n_args, args);
     }
     o->base.type = type_in;
-    o->traceback = MP_OBJ_NULL;
+    o->traceback_data = NULL;
     return o;
 }
 
@@ -298,7 +298,7 @@ mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char
         // Unfortunately, we won't be able to format the string...
         o = &MP_STATE_VM(mp_emergency_exception_obj);
         o->base.type = exc_type;
-        o->traceback = MP_OBJ_NULL;
+        o->traceback_data = NULL;
         o->args = mp_const_empty_tuple;
 
 #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF
@@ -332,21 +332,17 @@ mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char
             offset += sizeof(void *) - 1;
             offset &= ~(sizeof(void *) - 1);
 
-            if ((mp_emergency_exception_buf_size - offset) > (sizeof(mp_obj_list_t) + sizeof(mp_obj_t) * 3)) {
+            if ((mp_emergency_exception_buf_size - offset) > (sizeof(mp_uint_t) * 3)) {
                 // We have room to store some traceback.
-                mp_obj_list_t *list = (mp_obj_list_t *)((byte *)MP_STATE_VM(mp_emergency_exception_buf) + offset);
+                o->traceback_data = (mp_uint_t*)((byte *)MP_STATE_VM(mp_emergency_exception_buf) + offset);
-                list->base.type = &mp_type_list;
+                o->traceback_alloc = (MP_STATE_VM(mp_emergency_exception_buf) + mp_emergency_exception_buf_size - (byte *)o->traceback_data) / sizeof(o->traceback_data[0]);
-                list->items = (mp_obj_t)&list[1];
+                o->traceback_len = 0;
-                list->alloc = (MP_STATE_VM(mp_emergency_exception_buf) + mp_emergency_exception_buf_size - (byte *)list->items) / sizeof(list->items[0]);
-                list->len = 0;
-
-                o->traceback = list;
             }
         }
 #endif // MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF
     } else {
         o->base.type = exc_type;
-        o->traceback = MP_OBJ_NULL;
+        o->traceback_data = NULL;
         o->args = mp_obj_new_tuple(1, NULL);
 
         if (fmt == NULL) {
@@ -416,50 +412,47 @@ void mp_obj_exception_clear_traceback(mp_obj_t self_in) {
     GET_NATIVE_EXCEPTION(self, self_in);
     // just set the traceback to the null object
     // we don't want to call any memory management functions here
-    self->traceback = MP_OBJ_NULL;
+    self->traceback_data = NULL;
 }
 
 void mp_obj_exception_add_traceback(mp_obj_t self_in, qstr file, mp_uint_t line, qstr block) {
     GET_NATIVE_EXCEPTION(self, self_in);
 
-    #if MICROPY_ENABLE_GC
+    // append this traceback info to traceback data
-    if (gc_is_locked()) {
+    // if memory allocation fails (eg because gc is locked), just return
-        if (self->traceback == MP_OBJ_NULL) {
-            // We can't allocate any memory, and no memory has been
-            // pre-allocated, so there is nothing else we can do.
-            return;
-        }
-        mp_obj_list_t *list = self->traceback;
-        if (list->alloc <= (list->len + 3)) {
-            // There is some preallocated memory, but not enough to store an
-            // entire record.
-            return;
-        }
-    }
-    #endif
 
-    // for traceback, we are just using the list object for convenience, it's not really a list of Python objects
+    if (self->traceback_data == NULL) {
-    if (self->traceback == MP_OBJ_NULL) {
+        self->traceback_data = m_new_maybe(mp_uint_t, 3);
-        self->traceback = mp_obj_new_list_maybe(3);
+        if (self->traceback_data == NULL) {
-        if (self->traceback == MP_OBJ_NULL) {
             return;
         }
+        self->traceback_alloc = 3;
+        self->traceback_len = 0;
+    } else if (self->traceback_len + 3 > self->traceback_alloc) {
+        // be conservative with growing traceback data
+        mp_uint_t *tb_data = m_renew_maybe(mp_uint_t, self->traceback_data, self->traceback_alloc, self->traceback_alloc + 3);
+        if (tb_data == NULL) {
+            return;
+        }
+        self->traceback_data = tb_data;
+        self->traceback_alloc += 3;
     }
-    mp_obj_list_t *list = self->traceback;
+
-    list->items[0] = (mp_obj_t)(mp_uint_t)file;
+    mp_uint_t *tb_data = &self->traceback_data[self->traceback_len];
-    list->items[1] = (mp_obj_t)(mp_uint_t)line;
+    self->traceback_len += 3;
-    list->items[2] = (mp_obj_t)(mp_uint_t)block;
+    tb_data[0] = (mp_uint_t)file;
+    tb_data[1] = (mp_uint_t)line;
+    tb_data[2] = (mp_uint_t)block;
 }
 
 void mp_obj_exception_get_traceback(mp_obj_t self_in, mp_uint_t *n, mp_uint_t **values) {
     GET_NATIVE_EXCEPTION(self, self_in);
 
-    if (self->traceback == MP_OBJ_NULL) {
+    if (self->traceback_data == NULL) {
         *n = 0;
         *values = NULL;
     } else {
-        mp_uint_t n2;
+        *n = self->traceback_len;
-        mp_obj_list_get(self->traceback, &n2, (mp_obj_t**)values);
+        *values = self->traceback_data;
-        *n = n2;
     }
 }

py/objexcept.h

@@ -31,7 +31,9 @@
 
 typedef struct _mp_obj_exception_t {
     mp_obj_base_t base;
-    mp_obj_t traceback; // a list object, holding (file,line,block) as numbers (not Python objects); a hack for now
+    mp_uint_t traceback_alloc : (BITS_PER_WORD / 2);
+    mp_uint_t traceback_len : (BITS_PER_WORD / 2);
+    mp_uint_t *traceback_data;
     mp_obj_tuple_t *args;
 } mp_obj_exception_t;
 

py/objlist.c

@@ -477,25 +477,6 @@ mp_obj_t mp_obj_new_list(mp_uint_t n, mp_obj_t *items) {
     return o;
 }
 
-// Special method for usage with exceptions
-// Doesn't initialize items, assumes they will be initialized by client.
-mp_obj_t mp_obj_new_list_maybe(mp_uint_t n) {
-    mp_obj_list_t *o = m_new_obj_maybe(mp_obj_list_t);
-    if (!o) {
-        return o;
-    }
-    o->items = m_new_maybe(mp_obj_t, n);
-    if (!o->items) {
-        m_del_obj(mp_obj_list_t, o);
-        return MP_OBJ_NULL;
-    }
-
-    o->base.type = &mp_type_list;
-    o->len = o->alloc = n;
-
-    return o;
-}
-
 void mp_obj_list_get(mp_obj_t self_in, mp_uint_t *len, mp_obj_t **items) {
     mp_obj_list_t *self = self_in;
     *len = self->len;

py/objlist.h

@@ -35,6 +35,4 @@ typedef struct _mp_obj_list_t {
     mp_obj_t *items;
 } mp_obj_list_t;
 
-mp_obj_t mp_obj_new_list_maybe(mp_uint_t n);
-
 #endif // __MICROPY_INCLUDED_PY_OBJLIST_H__

tests/misc/print_exception.py

@@ -6,19 +6,35 @@ else:
     import traceback
     print_exception = lambda e, f: traceback.print_exception(None, e, None, file=f)
 
-try:
+def print_exc(e):
-    XXX
-except Exception as e:
-    print('caught')
     buf = io.StringIO()
     print_exception(e, buf)
     s = buf.getvalue()
     for l in s.split("\n"):
         # uPy on pyboard prints <stdin> as file, so remove filename.
         if l.startswith("  File "):
-            print(l[:8], l[-23:])
+            l = l.split('"')
+            print(l[0], l[2])
         # uPy and CPy tracebacks differ in that CPy prints a source line for
         # each traceback entry. In this case, we know that offending line
         # has 4-space indent, so filter it out.
         elif not l.startswith("    "):
             print(l)
+
+# basic exception message
+try:
+    XXX
+except Exception as e:
+    print('caught')
+    print_exc(e)
+
+# exception message with more than 1 source-code line
+def f():
+    g()
+def g():
+    YYY
+try:
+    f()
+except Exception as e:
+    print('caught')
+    print_exc(e)