Merge 2919d64ff1 into bd21820b4c

Compare the full absolute path instead of relying on the path form
passed by the user.

For instance, this will make

python3 run-tests.py -d basics
python3 run-tests.py -d ./basics
python3 run-tests.py -d ../tests/basics
python3 run-tests.py -d /full/path/to/basics

all behave the same by correctly treating the bytes_compare3 and
builtin_help tests as special, whereas previously only the first
invocation would do that and hence result in these tests to fail
when called with a different path form.

Signed-off-by: stijn <stijn@ignitron.net>

ports/samd/mcu/samd21/mpconfigmcu.h

@@ -14,20 +14,33 @@
 #define MICROPY_EMIT_THUMB_ARMV7M       (0)
 #define MICROPY_MODULE_BUILTIN_INIT     (1)
 
-#define MICROPY_FLOAT_IMPL              (MICROPY_FLOAT_IMPL_FLOAT)
+// Selected extensions beyond the basic features set.
-
+#define MICROPY_ENABLE_FINALISER        (1)
-#ifndef MICROPY_PY_MATH
+#define MICROPY_STACK_CHECK             (1)
-#define MICROPY_PY_MATH                 (1)
+#define MICROPY_KBD_EXCEPTION           (1)
-#define MP_NEED_LOG2                    (1)
+#define MICROPY_HELPER_REPL             (1)
-#endif
+#define MICROPY_REPL_AUTO_INDENT        (1)
-
+#define MICROPY_ENABLE_SOURCE_LINE      (1)
-#ifndef MICROPY_PY_BUILTINS_COMPLEX
+#define MICROPY_STREAMS_NON_BLOCK       (1)
-#define MICROPY_PY_BUILTINS_COMPLEX     (0)
+#define MICROPY_PY_BUILTINS_HELP        (1)
-#endif
+#define MICROPY_PY_BUILTINS_HELP_MODULES (1)
-
+#define MICROPY_ENABLE_SCHEDULER        (1)
-#ifndef MICROPY_PY_CMATH
+#define MICROPY_PY_BUILTINS_BYTES_HEX   (1)
-#define MICROPY_PY_CMATH                (0)
+#define MICROPY_PY_BUILTINS_MEMORYVIEW  (1)
-#endif
+#define MICROPY_PY_BUILTINS_INPUT       (1)
+#define MICROPY_PY_MICROPYTHON_MEM_INFO (1)
+#define MICROPY_PY_ARRAY_SLICE_ASSIGN   (1)
+#define MICROPY_PY_SYS_STDFILES         (1)
+#define MICROPY_PY_SYS_MAXSIZE          (1)
+#define MICROPY_PY_IO_IOBASE            (1)
+#define MICROPY_PY_OS                   (1)
+#define MICROPY_PY_JSON                 (1)
+#define MICROPY_PY_RE                   (1)
+#define MICROPY_PY_BINASCII             (1)
+#define MICROPY_PY_UCTYPES              (1)
+#define MICROPY_PY_HEAPQ                (1)
+#define MICROPY_PY_RANDOM               (1)
+#define MICROPY_PY_PLATFORM             (1)
 
 #define MICROPY_PY_RANDOM_SEED_INIT_FUNC (trng_random_u32(300))
 unsigned long trng_random_u32(int delay);

ports/samd/mcu/samd51/mpconfigmcu.h

@@ -10,21 +10,6 @@
 // Python internal features
 #define MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF  (1)
 
-#define MICROPY_FLOAT_IMPL              (MICROPY_FLOAT_IMPL_FLOAT)
-
-#ifndef MICROPY_PY_BUILTINS_COMPLEX
-#define MICROPY_PY_BUILTINS_COMPLEX     (0)
-#endif
-
-#ifndef MICROPY_PY_MATH
-#define MICROPY_PY_MATH                 (1)
-#define MP_NEED_LOG2                    (1)
-#endif
-
-#ifndef MICROPY_PY_CMATH
-#define MICROPY_PY_CMATH                (0)
-#endif
-
 #define MICROPY_PY_OS_SYNC              (1)
 #define MICROPY_PY_OS_URANDOM           (1)
 #define MICROPY_PY_ONEWIRE              (1)

ports/samd/mpconfigport.h

@@ -40,25 +40,18 @@
 // MicroPython emitters
 #define MICROPY_PERSISTENT_CODE_LOAD        (1)
 
-// Compiler configuration
-#define MICROPY_COMP_CONST                  (1)
-
 // Python internal features
 #define MICROPY_ENABLE_GC                   (1)
-#define MICROPY_ENABLE_FINALISER            (1)
-#define MICROPY_KBD_EXCEPTION               (1)
-#define MICROPY_HELPER_REPL                 (1)
-#define MICROPY_REPL_AUTO_INDENT            (1)
 #define MICROPY_LONGINT_IMPL                (MICROPY_LONGINT_IMPL_MPZ)
-#define MICROPY_ENABLE_SOURCE_LINE          (1)
+#define MICROPY_FLOAT_IMPL                  (MICROPY_FLOAT_IMPL_FLOAT)
-#define MICROPY_STREAMS_NON_BLOCK           (1)
+#ifndef MICROPY_PY_BUILTINS_COMPLEX
+#define MICROPY_PY_BUILTINS_COMPLEX         (0)
+#endif
 #define MICROPY_ERROR_REPORTING             (MICROPY_ERROR_REPORTING_TERSE)
-#define MICROPY_PY_BUILTINS_HELP            (1)
 #define MICROPY_PY_BUILTINS_HELP_TEXT       samd_help_text
-#define MICROPY_PY_BUILTINS_HELP_MODULES    (1)
 #define MICROPY_USE_INTERNAL_ERRNO          (1)
-#define MICROPY_ENABLE_SCHEDULER            (1)
 #define MICROPY_SCHEDULER_STATIC_NODES      (1)
+
 #define MICROPY_HW_ENABLE_USBDEV            (1)
 #define MICROPY_HW_USB_CDC_1200BPS_TOUCH    (1)
 
@@ -71,22 +64,9 @@
 #ifndef MICROPY_HW_USB_DESC_STR_MAX
 #define MICROPY_HW_USB_DESC_STR_MAX (32)
 #endif
-
 #endif
 
-// Control over Python builtins
-#define MICROPY_PY_BUILTINS_BYTES_HEX       (1)
-#define MICROPY_PY_BUILTINS_MEMORYVIEW      (1)
-#define MICROPY_PY_BUILTINS_INPUT           (1)
-#define MICROPY_PY_MICROPYTHON_MEM_INFO     (1)
-#define MICROPY_PY_ARRAY_SLICE_ASSIGN       (1)
-#define MICROPY_PY_SYS                      (1)
 #define MICROPY_PY_SYS_PLATFORM             "samd"
-#define MICROPY_PY_SYS_EXIT                 (1)
-#define MICROPY_PY_SYS_STDFILES             (1)
-#define MICROPY_PY_SYS_MAXSIZE              (1)
-#define MICROPY_PY_IO                       (1)
-#define MICROPY_PY_IO_IOBASE                (1)
 
 // Extended modules
 #define MICROPY_PY_TIME_GMTIME_LOCALTIME_MKTIME (1)
@@ -97,16 +77,9 @@
 #define MICROPY_PY_MACHINE_BARE_METAL_FUNCS (1)
 #define MICROPY_PY_MACHINE_BOOTLOADER       (1)
 #define MICROPY_PY_MACHINE_DISABLE_IRQ_ENABLE_IRQ (1)
-#define MICROPY_PY_OS                       (1)
 #define MICROPY_PY_OS_INCLUDEFILE           "ports/samd/modos.c"
 #define MICROPY_READER_VFS                  (1)
 #define MICROPY_VFS                         (1)
-#define MICROPY_PY_JSON                     (1)
-#define MICROPY_PY_RE                       (1)
-#define MICROPY_PY_BINASCII                 (1)
-#define MICROPY_PY_UCTYPES                  (1)
-#define MICROPY_PY_HEAPQ                    (1)
-#define MICROPY_PY_RANDOM                   (1)
 #ifndef MICROPY_PY_MACHINE_ADC
 #define MICROPY_PY_MACHINE_ADC              (1)
 #endif
@@ -148,7 +121,6 @@
 #define MICROPY_PY_MACHINE_WDT              (1)
 #define MICROPY_PY_MACHINE_WDT_INCLUDEFILE  "ports/samd/machine_wdt.c"
 #define MICROPY_PY_MACHINE_WDT_TIMEOUT_MS   (1)
-#define MICROPY_PY_PLATFORM                 (1)
 #define MICROPY_PLATFORM_VERSION            "ASF4"
 
 #define MP_STATE_PORT MP_STATE_VM
@@ -188,5 +160,8 @@ typedef int mp_int_t; // must be pointer size
 typedef unsigned mp_uint_t; // must be pointer size
 typedef long mp_off_t;
 
+// Need an implementation of the log2 function which is not a macro.
+#define MP_NEED_LOG2 (1)
+
 // Need to provide a declaration/definition of alloca()
 #include <alloca.h>

tests/run-tests.py

@@ -176,20 +176,25 @@ def run_script_on_remote_target(pyb, args, test_file, is_special):
     return had_crash, output_mupy
 
 
-def run_micropython(pyb, args, test_file, is_special=False):
+special_tests = [
-    special_tests = (
+    base_path(file)
+    for file in (
         "micropython/meminfo.py",
         "basics/bytes_compare3.py",
         "basics/builtin_help.py",
         "thread/thread_exc2.py",
         "esp32/partition_ota.py",
     )
+]
+
+
+def run_micropython(pyb, args, test_file, test_file_abspath, is_special=False):
     had_crash = False
     if pyb is None:
         # run on PC
         if (
-            test_file.startswith(("cmdline/", base_path("feature_check/")))
+            test_file_abspath.startswith((base_path("cmdline/"), base_path("feature_check/")))
-            or test_file in special_tests
+            or test_file_abspath in special_tests
         ):
             # special handling for tests of the unix cmdline program
             is_special = True
@@ -205,7 +210,7 @@ def run_micropython(pyb, args, test_file, is_special=False):
 
             # run the test, possibly with redirected input
             try:
-                if "repl_" in test_file:
+                if os.path.basename(test_file).startswith("repl_"):
                     # Need to use a PTY to test command line editing
                     try:
                         import pty
@@ -283,7 +288,7 @@ def run_micropython(pyb, args, test_file, is_special=False):
                 mpy_modname = os.path.splitext(os.path.basename(mpy_filename))[0]
                 cmdlist.extend(["-m", mpy_modname])
             else:
-                cmdlist.append(os.path.abspath(test_file))
+                cmdlist.append(test_file_abspath)
 
             # run the actual test
             try:
@@ -316,7 +321,7 @@ def run_micropython(pyb, args, test_file, is_special=False):
     if is_special and not had_crash and b"\nSKIP\n" in output_mupy:
         return b"SKIP\n"
 
-    if is_special or test_file in special_tests:
+    if is_special or test_file_abspath in special_tests:
         # convert parts of the output that are not stable across runs
         with open(test_file + ".exp", "rb") as f:
             lines_exp = []
@@ -360,11 +365,12 @@ def run_micropython(pyb, args, test_file, is_special=False):
     return output_mupy
 
 
-def run_feature_check(pyb, args, base_path, test_file):
+def run_feature_check(pyb, args, test_file):
     if pyb is not None and test_file.startswith("repl_"):
         # REPL feature tests will not run via pyboard because they require prompt interactivity
         return b""
-    return run_micropython(pyb, args, base_path("feature_check", test_file), is_special=True)
+    test_file_path = base_path("feature_check", test_file)
+    return run_micropython(pyb, args, test_file_path, test_file_path, is_special=True)
 
 
 class ThreadSafeCounter:
@@ -419,57 +425,57 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         # run-tests.py script itself so use base_path.
 
         # Check if micropython.native is supported, and skip such tests if it's not
-        output = run_feature_check(pyb, args, base_path, "native_check.py")
+        output = run_feature_check(pyb, args, "native_check.py")
         if output != b"native\n":
             skip_native = True
 
         # Check if arbitrary-precision integers are supported, and skip such tests if it's not
-        output = run_feature_check(pyb, args, base_path, "int_big.py")
+        output = run_feature_check(pyb, args, "int_big.py")
         if output != b"1000000000000000000000000000000000000000000000\n":
             skip_int_big = True
 
         # Check if bytearray is supported, and skip such tests if it's not
-        output = run_feature_check(pyb, args, base_path, "bytearray.py")
+        output = run_feature_check(pyb, args, "bytearray.py")
         if output != b"bytearray\n":
             skip_bytearray = True
 
         # Check if set type (and set literals) is supported, and skip such tests if it's not
-        output = run_feature_check(pyb, args, base_path, "set_check.py")
+        output = run_feature_check(pyb, args, "set_check.py")
         if output != b"{1}\n":
             skip_set_type = True
 
         # Check if slice is supported, and skip such tests if it's not
-        output = run_feature_check(pyb, args, base_path, "slice.py")
+        output = run_feature_check(pyb, args, "slice.py")
         if output != b"slice\n":
             skip_slice = True
 
         # Check if async/await keywords are supported, and skip such tests if it's not
-        output = run_feature_check(pyb, args, base_path, "async_check.py")
+        output = run_feature_check(pyb, args, "async_check.py")
         if output != b"async\n":
             skip_async = True
 
         # Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not
-        output = run_feature_check(pyb, args, base_path, "const.py")
+        output = run_feature_check(pyb, args, "const.py")
         if output != b"1\n":
             skip_const = True
 
         # Check if __rOP__ special methods are supported, and skip such tests if it's not
-        output = run_feature_check(pyb, args, base_path, "reverse_ops.py")
+        output = run_feature_check(pyb, args, "reverse_ops.py")
         if output == b"TypeError\n":
             skip_revops = True
 
         # Check if io module exists, and skip such tests if it doesn't
-        output = run_feature_check(pyb, args, base_path, "io_module.py")
+        output = run_feature_check(pyb, args, "io_module.py")
         if output != b"io\n":
             skip_io_module = True
 
         # Check if fstring feature is enabled, and skip such tests if it doesn't
-        output = run_feature_check(pyb, args, base_path, "fstring.py")
+        output = run_feature_check(pyb, args, "fstring.py")
         if output != b"a=1\n":
             skip_fstring = True
 
         # Check if @micropython.asm_thumb supports Thumb2 instructions, and skip such tests if it doesn't
-        output = run_feature_check(pyb, args, base_path, "inlineasm_thumb2.py")
+        output = run_feature_check(pyb, args, "inlineasm_thumb2.py")
         if output != b"thumb2\n":
             skip_tests.add("inlineasm/asmbcc.py")
             skip_tests.add("inlineasm/asmbitops.py")
@@ -484,23 +490,23 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
             skip_tests.add("inlineasm/asmspecialregs.py")
 
         # Check if emacs repl is supported, and skip such tests if it's not
-        t = run_feature_check(pyb, args, base_path, "repl_emacs_check.py")
+        t = run_feature_check(pyb, args, "repl_emacs_check.py")
         if "True" not in str(t, "ascii"):
             skip_tests.add("cmdline/repl_emacs_keys.py")
 
         # Check if words movement in repl is supported, and skip such tests if it's not
-        t = run_feature_check(pyb, args, base_path, "repl_words_move_check.py")
+        t = run_feature_check(pyb, args, "repl_words_move_check.py")
         if "True" not in str(t, "ascii"):
             skip_tests.add("cmdline/repl_words_move.py")
 
-        upy_byteorder = run_feature_check(pyb, args, base_path, "byteorder.py")
+        upy_byteorder = run_feature_check(pyb, args, "byteorder.py")
-        upy_float_precision = run_feature_check(pyb, args, base_path, "float.py")
+        upy_float_precision = run_feature_check(pyb, args, "float.py")
         try:
             upy_float_precision = int(upy_float_precision)
         except ValueError:
             upy_float_precision = 0
-        has_complex = run_feature_check(pyb, args, base_path, "complex.py") == b"complex\n"
+        has_complex = run_feature_check(pyb, args, "complex.py") == b"complex\n"
-        has_coverage = run_feature_check(pyb, args, base_path, "coverage.py") == b"coverage\n"
+        has_coverage = run_feature_check(pyb, args, "coverage.py") == b"coverage\n"
         cpy_byteorder = subprocess.check_output(
             CPYTHON3_CMD + [base_path("feature_check/byteorder.py")]
         )
@@ -673,6 +679,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
 
     def run_one_test(test_file):
         test_file = test_file.replace("\\", "/")
+        test_file_abspath = os.path.abspath(test_file).replace("\\", "/")
 
         if args.filters:
             # Default verdict is the opposit of the first action
@@ -733,7 +740,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
             # run CPython to work out expected output
             try:
                 output_expected = subprocess.check_output(
-                    CPYTHON3_CMD + [os.path.abspath(test_file)],
+                    CPYTHON3_CMD + [test_file_abspath],
                     cwd=os.path.dirname(test_file),
                     stderr=subprocess.STDOUT,
                 )
@@ -750,7 +757,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
             return
 
         # run MicroPython
-        output_mupy = run_micropython(pyb, args, test_file)
+        output_mupy = run_micropython(pyb, args, test_file, test_file_abspath)
 
         if output_mupy == b"SKIP\n":
             print("skip ", test_file)

tools/mpremote/mpremote/transport_serial.py

@@ -35,7 +35,7 @@
 # Once the API is stabilised, the idea is that mpremote can be used both
 # as a command line tool and a library for interacting with devices.
 
-import ast, io, errno, os, re, struct, sys, time
+import ast, io, errno, os, re, struct, sys, time, contextlib, urllib
 from collections import namedtuple
 from errno import EPERM
 from .console import VT_ENABLED
@@ -58,6 +58,24 @@ def reraise_filesystem_error(e, info):
     raise
 
 
+@contextlib.contextmanager
+def open_file(src : str):
+    """Open a file. Can be either on local disk or a HTTP(S) URL"""
+    is_http = src.startswith('https://') or src.startswith('http://')
+    if is_http:
+        conn = urllib.request.urlopen(src, timeout=60.0)
+        size = int(conn.headers['content-length'])
+        file_like = conn
+    else:
+        size = os.path.getsize(src)
+        file_like = open(src, "rb")
+
+    # attach a size() method, so consumers can know the file size
+    file_like.size = lambda : size
+    yield file_like
+    file_like.close()
+
+
 class SerialTransport(Transport):
     def __init__(self, device, baudrate=115200, wait=0, exclusive=True):
         self.in_raw_repl = False
@@ -405,11 +423,12 @@ class SerialTransport(Transport):
         self.exec("f.close()")
 
     def fs_put(self, src, dest, chunk_size=256, progress_callback=None):
-        if progress_callback:
+
-            src_size = os.path.getsize(src)
-            written = 0
         self.exec("f=open('%s','wb')\nw=f.write" % dest)
-        with open(src, "rb") as f:
+        with open_file(src) as f:
+            if progress_callback:
+                src_size = f.size()
+                written = 0
             while True:
                 data = f.read(chunk_size)
                 if not data: