Integrate new lexer stream with stm framework.

py/lexer.c

@@ -14,7 +14,7 @@ struct _py_lexer_t {
     const char *name;           // name of source
     void *stream_data;          // data for stream
     py_lexer_stream_next_char_t stream_next_char;   // stream callback to get next char
-    py_lexer_stream_free_t stream_free;             // stream callback to free
+    py_lexer_stream_close_t stream_close;           // stream callback to free
 
     unichar chr0, chr1, chr2;   // current cached characters from source
 
@@ -589,13 +589,13 @@ static void py_lexer_next_token_into(py_lexer_t *lex, py_token_t *tok, bool firs
     }
 }
 
-py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_free_t stream_free) {
+py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_close_t stream_close) {
     py_lexer_t *lex = m_new(py_lexer_t, 1);
 
     lex->name = src_name; // TODO do we need to strdup this?
     lex->stream_data = stream_data;
     lex->stream_next_char = stream_next_char;
-    lex->stream_free = stream_free;
+    lex->stream_close = stream_close;
     lex->line = 1;
     lex->column = 1;
     lex->emit_dent = 0;
@@ -632,8 +632,8 @@ py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_strea
 
 void py_lexer_free(py_lexer_t *lex) {
     if (lex) {
-        if (lex->stream_free) {
+        if (lex->stream_close) {
-            lex->stream_free(lex->stream_data);
+            lex->stream_close(lex->stream_data);
         }
         m_free(lex);
     }

py/lexer.h

@@ -122,7 +122,7 @@ typedef struct _py_token_t {
 // it can be called again after returning PY_LEXER_CHAR_EOF, and in that case must return PY_LEXER_CHAR_EOF
 #define PY_LEXER_CHAR_EOF (-1)
 typedef unichar (*py_lexer_stream_next_char_t)(void*);
-typedef void (*py_lexer_stream_free_t)(void*);
+typedef void (*py_lexer_stream_close_t)(void*);
 
 typedef struct _py_lexer_t py_lexer_t;
 
@@ -130,7 +130,7 @@ void py_token_show(const py_token_t *tok);
 void py_token_show_error_prefix(const py_token_t *tok);
 bool py_token_show_error(const py_token_t *tok, const char *msg);
 
-py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_free_t stream_free);
+py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_close_t stream_close);
 void py_lexer_free(py_lexer_t *lex);
 void py_lexer_to_next(py_lexer_t *lex);
 const py_token_t *py_lexer_cur(const py_lexer_t *lex);

stm/Makefile

@@ -24,6 +24,7 @@ SRC_C = \
 	systick.c  \
 	stm32fxxx_it.c \
 	usb.c \
+	lexerstm.c \
 #	sd.c \
 
 SRC_S = \
@@ -33,6 +34,7 @@ PY_O = \
 	nlrthumb.o \
 	malloc.o \
 	qstr.o \
+	vstr.o \
 	misc.o \
 	lexer.o \
 	parse.o \

stm/lexerstm.c

@@ -0,0 +1,60 @@
+#include <stdint.h>
+#include <stdio.h>
+
+#include "ff.h"
+
+#include "misc.h"
+#include "lexer.h"
+#include "lexerstm.h"
+
+unichar str_buf_next_char(py_lexer_str_buf_t *sb) {
+    if (sb->src_cur < sb->src_end) {
+        return *sb->src_cur++;
+    } else {
+        return PY_LEXER_CHAR_EOF;
+    }
+}
+
+void str_buf_free(py_lexer_str_buf_t *sb) {
+    if (sb->free) {
+        m_free((char*)sb->src_beg);
+    }
+}
+
+py_lexer_t *py_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, py_lexer_str_buf_t *sb) {
+    sb->free = free_str;
+    sb->src_beg = str;
+    sb->src_cur = str;
+    sb->src_end = str + len;
+    return py_lexer_new(src_name, sb, (py_lexer_stream_next_char_t)str_buf_next_char, (py_lexer_stream_close_t)str_buf_free);
+}
+
+unichar file_buf_next_char(py_lexer_file_buf_t *fb) {
+    if (fb->pos >= fb->len) {
+        if (fb->len < sizeof(fb->buf)) {
+            return PY_LEXER_CHAR_EOF;
+        } else {
+            UINT n;
+            f_read(&fb->fp, fb->buf, sizeof(fb->buf), &n);
+            fb->len = n;
+            fb->pos = 0;
+        }
+    }
+    return fb->buf[fb->pos++];
+}
+
+void file_buf_close(py_lexer_file_buf_t *fb) {
+    f_close(&fb->fp);
+}
+
+py_lexer_t *py_lexer_new_from_file(const char *filename, py_lexer_file_buf_t *fb) {
+    FRESULT res = f_open(&fb->fp, filename, FA_READ);
+    if (res != FR_OK) {
+        return NULL;
+    }
+    UINT n;
+    f_read(&fb->fp, fb->buf, sizeof(fb->buf), &n);
+    fb->len = n;
+    fb->pos = 0;
+    return py_lexer_new(filename, fb, (py_lexer_stream_next_char_t)file_buf_next_char, (py_lexer_stream_close_t)file_buf_close);
+}

stm/lexerstm.h

@@ -0,0 +1,16 @@
+typedef struct _py_lexer_str_buf_t {
+    bool free;                  // free src_beg when done
+    const char *src_beg;        // beginning of source
+    const char *src_cur;        // current location in source
+    const char *src_end;        // end (exclusive) of source
+} py_lexer_str_buf_t;
+
+typedef struct _py_lexer_file_buf_t {
+    FIL fp;
+    char buf[20];
+    uint16_t len;
+    uint16_t pos;
+} py_lexer_file_buf_t;
+
+py_lexer_t *py_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, py_lexer_str_buf_t *sb);
+py_lexer_t *py_lexer_new_from_file(const char *filename, py_lexer_file_buf_t *fb);

stm/main.c

@@ -11,6 +11,7 @@
 #include "storage.h"
 #include "mma.h"
 #include "usb.h"
+#include "ff.h"
 
 static void impl02_c_version() {
     int x = 0;
@@ -63,8 +64,10 @@ void __fatal_error(const char *msg) {
     }
 }
 
+#include "nlr.h"
 #include "misc.h"
 #include "lexer.h"
+#include "lexerstm.h"
 #include "mpyconfig.h"
 #include "parse.h"
 #include "compile.h"
@@ -88,10 +91,8 @@ py_obj_t pyb_sw() {
     }
 }
 
-#include "ff.h"
 FATFS fatfs0;
 
-#include "nlr.h"
 
 /*
 void g(uint i) {
@@ -293,7 +294,7 @@ int main() {
     //sys_tick_delay_ms(1000);
 
     // Python!
-    if (0) {
+    if (1) {
         //const char *pysrc = "def f():\n  x=x+1\nprint(42)\n";
         const char *pysrc =
             // impl01.py
@@ -323,7 +324,6 @@ int main() {
             "        x = x + 1\n"
             "f()\n";
             */
-            /*
             "print('in python!')\n"
             "x = 0\n"
             "while x < 4:\n"
@@ -331,11 +331,10 @@ int main() {
             "    pyb_delay(201)\n"
             "    pyb_led(False)\n"
             "    pyb_delay(201)\n"
-            "    x = x + 1\n"
+            "    x += 1\n"
             "print('press me!')\n"
             "while True:\n"
             "    pyb_led(pyb_sw())\n";
-            */
             /*
             // impl16.py
             "@micropython.asm_thumb\n"
@@ -372,6 +371,7 @@ int main() {
             "except:\n"
             "    print(x)\n";
             */
+            /*
             // impl19.py
             "# for loop\n"
             "def f():\n"
@@ -380,29 +380,27 @@ int main() {
             "            for z in range(400):\n"
             "                pass\n"
             "f()\n";
+            */
 
-        py_lexer_t *lex = py_lexer_from_str_len("<>", pysrc, strlen(pysrc), false);
+        py_lexer_str_buf_t py_lexer_str_buf;
+        py_lexer_t *lex = py_lexer_new_from_str_len("<stdin>", pysrc, strlen(pysrc), false, &py_lexer_str_buf);
 
-        if (0) {
+        // nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler
-            while (!py_lexer_is_kind(lex, PY_TOKEN_END)) {
+        printf("lex; al=%u\n", m_get_total_bytes_allocated());
-                py_token_show(py_lexer_cur(lex));
+        sys_tick_delay_ms(1000);
-                py_lexer_to_next(lex);
+        py_parse_node_t pn = py_parse(lex, PY_PARSE_FILE_INPUT);
-                sys_tick_delay_ms(1000);
+        py_lexer_free(lex);
-            }
+        if (pn != PY_PARSE_NODE_NULL) {
-        } else {
-            // nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler
-            printf("lex; al=%u\n", m_get_total_bytes_allocated());
-            sys_tick_delay_ms(1000);
-            py_parse_node_t pn = py_parse(lex, 0);
-            //printf("----------------\n");
             printf("pars;al=%u\n", m_get_total_bytes_allocated());
             sys_tick_delay_ms(1000);
             //parse_node_show(pn, 0);
-            py_compile(pn, false);
+            bool comp_ok = py_compile(pn, false);
             printf("comp;al=%u\n", m_get_total_bytes_allocated());
             sys_tick_delay_ms(1000);
 
-            if (1) {
+            if (!comp_ok) {
+                printf("compile error\n");
+            } else {
                 // execute it!
 
                 // add some functions to the python namespace

stm/std.h

@@ -8,6 +8,7 @@ void *calloc(size_t sz, size_t n);
 void *realloc(void *ptr, size_t n);
 
 void *memcpy(void *dest, const void *src, size_t n);
+void *memmove(void *dest, const void *src, size_t n);
 void *memset(void *s, int c, size_t n);
 
 int strlen(const char *str);

stm/string0.c

@@ -11,6 +11,21 @@ void *memcpy(void *dest, const void *src, size_t n) {
     return dest;
 }
 
+void *memmove(void *dest, const void *src, size_t n) {
+    if (src < dest && dest < src + n) {
+        // need to copy backwards
+        uint8_t *d = dest + n - 1;
+        const uint8_t *s = src + n - 1;
+        for (; n > 0; n--) {
+            *d-- = *s--;
+        }
+        return dest;
+    } else {
+        // can use normal memcpy
+        return memcpy(dest, src, n);
+    }
+}
+
 void *memset(void *s, int c, size_t n) {
     uint8_t *s2 = s;
     for (; n > 0; n--) {

unix/main.c

@@ -68,8 +68,11 @@ void do_repl() {
                 line = line3;
             }
         }
+
         py_lexer_t *lex = py_lexer_new_from_str_len("<stdin>", line, strlen(line), false);
         py_parse_node_t pn = py_parse(lex, PY_PARSE_SINGLE_INPUT);
+        py_lexer_free(lex);
+
         if (pn != PY_PARSE_NODE_NULL) {
             //py_parse_node_show(pn, 0);
             bool comp_ok = py_compile(pn, true);
@@ -111,6 +114,8 @@ void do_file(const char *file) {
         // compile
 
         py_parse_node_t pn = py_parse(lex, PY_PARSE_FILE_INPUT);
+        py_lexer_free(lex);
+
         if (pn != PY_PARSE_NODE_NULL) {
             //printf("----------------\n");
             //parse_node_show(pn, 0);
@@ -118,8 +123,6 @@ void do_file(const char *file) {
             bool comp_ok = py_compile(pn, false);
             //printf("----------------\n");
 
-            py_lexer_free(lex);
-
 #if MICROPY_EMIT_CPYTHON
             if (!comp_ok) {
                 printf("compile error\n");

unix/mpyconfig.h

@@ -1,7 +1,7 @@
 // options to control how Micro Python is built
 
 #define MICROPY_ENABLE_FLOAT        (1)
-#define MICROPY_EMIT_CPYTHON        (1)
+#define MICROPY_EMIT_CPYTHON        (0)
 #define MICROPY_EMIT_X64            (1)
 #define MICROPY_EMIT_THUMB          (0)
 #define MICROPY_EMIT_INLINE_THUMB   (0)