py/repl: Generalise REPL autocomplete to use qstr probing.

This patch changes the way REPL autocomplete finds matches.  It now probes
the target object for all qstrs via mp_load_method_maybe to look for a
match with the given input string.  Similar to how the builtin dir()
function works, this new algorithm now find all methods and instances of
user-defined classes including attributes of their parent classes.  This
helps a lot at the REPL prompt for user-discovery and to autocomplete names
even for classes that are derived.

The downside is that this new algorithm is slower than the previous one,
and in particular will be slower the more qstrs there are in the system.
But because REPL autocomplete is primarily used in an interactive way it is
not that important to make it fast, as long as it is "fast enough" compared
to human reaction.

On a slow microcontroller (CPU running at 16MHz) the autocomplete time for
a list of 35 names in the outer namespace (pressing tab at a bare prompt)
takes about 160ms with this algorithm, compared to about 40ms for the
previous implementation (this time includes the actual printing of the
names as well).  This time of 160ms is very reasonable especially given the
new functionality of listing all the names.

This patch also decreases code size by:

   bare-arm:    +0
minimal x86:  -128
   unix x64:  -128
unix nanbox:  -224
      stm32:   -88
     cc3200:   -80
    esp8266:   -92
      esp32:   -84

py/repl.c

@@ -27,6 +27,7 @@
 #include <string.h>
 #include "py/obj.h"
 #include "py/runtime.h"
+#include "py/builtin.h"
 #include "py/repl.h"
 
 #if MICROPY_HELPER_REPL
@@ -136,8 +137,11 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print
         }
     }
 
-    // begin search in locals dict
+    size_t nqstr = QSTR_TOTAL();
-    mp_obj_dict_t *dict = mp_locals_get();
+
+    // begin search in outer global dict which is accessed from __main__
+    mp_obj_t obj = MP_OBJ_FROM_PTR(&mp_module___main__);
+    mp_obj_t dest[2];
 
     for (;;) {
         // get next word in string to complete
@@ -148,43 +152,20 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print
         size_t s_len = str - s_start;
 
         if (str < top) {
-            // a complete word, lookup in current dict
+            // a complete word, lookup in current object
-
+            qstr q = qstr_find_strn(s_start, s_len);
-            mp_obj_t obj = MP_OBJ_NULL;
+            if (q == MP_QSTR_NULL) {
-            for (size_t i = 0; i < dict->map.alloc; i++) {
+                // lookup will fail
-                if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) {
+                return 0;
-                    size_t d_len;
-                    const char *d_str = mp_obj_str_get_data(dict->map.table[i].key, &d_len);
-                    if (s_len == d_len && strncmp(s_start, d_str, d_len) == 0) {
-                        obj = dict->map.table[i].value;
-                        break;
-                    }
-                }
             }
+            mp_load_method_maybe(obj, q, dest);
+            obj = dest[0]; // attribute, method, or MP_OBJ_NULL if nothing found
 
             if (obj == MP_OBJ_NULL) {
                 // lookup failed
                 return 0;
             }
 
-            // found an object of this name; try to get its dict
-            if (MP_OBJ_IS_TYPE(obj, &mp_type_module)) {
-                dict = mp_obj_module_get_globals(obj);
-            } else {
-                mp_obj_type_t *type;
-                if (MP_OBJ_IS_TYPE(obj, &mp_type_type)) {
-                    type = MP_OBJ_TO_PTR(obj);
-                } else {
-                    type = mp_obj_get_type(obj);
-                }
-                if (type->locals_dict != NULL && type->locals_dict->base.type == &mp_type_dict) {
-                    dict = type->locals_dict;
-                } else {
-                    // obj has no dict
-                    return 0;
-                }
-            }
-
             // skip '.' to move to next word
             ++str;
 
@@ -192,14 +173,15 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print
             // end of string, do completion on this partial name
 
             // look for matches
-            int n_found = 0;
             const char *match_str = NULL;
             size_t match_len = 0;
-            for (size_t i = 0; i < dict->map.alloc; i++) {
+            qstr q_first = 0, q_last;
-                if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) {
+            for (qstr q = 1; q < nqstr; ++q) {
                 size_t d_len;
-                    const char *d_str = mp_obj_str_get_data(dict->map.table[i].key, &d_len);
+                const char *d_str = (const char*)qstr_data(q, &d_len);
                 if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) {
+                    mp_load_method_maybe(obj, q, dest);
+                    if (dest[0] != MP_OBJ_NULL) {
                         if (match_str == NULL) {
                             match_str = d_str;
                             match_len = d_len;
@@ -213,13 +195,16 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print
                                 }
                             }
                         }
-                        ++n_found;
+                        if (q_first == 0) {
+                            q_first = q;
+                        }
+                        q_last = q;
                     }
                 }
             }
 
             // nothing found
-            if (n_found == 0) {
+            if (q_first == 0) {
                 // If there're no better alternatives, and if it's first word
                 // in the line, try to complete "import".
                 if (s_start == org_str) {
@@ -234,7 +219,7 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print
             }
 
             // 1 match found, or multiple matches with a common prefix
-            if (n_found == 1 || match_len > s_len) {
+            if (q_first == q_last || match_len > s_len) {
                 *compl_str = match_str + s_len;
                 return match_len - s_len;
             }
@@ -245,11 +230,12 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print
             #define MAX_LINE_LEN  (4 * WORD_SLOT_LEN)
 
             int line_len = MAX_LINE_LEN; // force a newline for first word
-            for (size_t i = 0; i < dict->map.alloc; i++) {
+            for (qstr q = q_first; q <= q_last; ++q) {
-                if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) {
                 size_t d_len;
-                    const char *d_str = mp_obj_str_get_data(dict->map.table[i].key, &d_len);
+                const char *d_str = (const char*)qstr_data(q, &d_len);
                 if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) {
+                    mp_load_method_maybe(obj, q, dest);
+                    if (dest[0] != MP_OBJ_NULL) {
                         int gap = (line_len + WORD_SLOT_LEN - 1) / WORD_SLOT_LEN * WORD_SLOT_LEN - line_len;
                         if (gap < 2) {
                             gap += WORD_SLOT_LEN;

tests/cmdline/repl_autocomplete.py

@@ -6,5 +6,4 @@ x = '123'
 1, x.isdi	()
 i = str
 i.lowe	('ABC')
-j = None
+None.	
-j.	

tests/cmdline/repl_autocomplete.py.exp

@@ -10,6 +10,5 @@ Use \.\+
 >>> i = str
 >>> i.lower('ABC')
 'abc'
->>> j = None
+>>> None.
->>> j.
 >>> 

tests/unix/extra_coverage.py.exp

@@ -24,11 +24,11 @@ RuntimeError:
 # repl
 ame__
 
-__name__        path            argv            version
+__class__       __name__        argv            byteorder
-version_info    implementation  platform        byteorder
+exc_info        exit            getsizeof       implementation
-maxsize         exit            stdin           stdout
+maxsize         modules         path            platform
-stderr          modules         exc_info        getsizeof
+print_exception                 stderr          stdin
-print_exception
+stdout          version         version_info
 ementation
 # attrtuple
 (start=1, stop=2, step=3)