From c23a2eabf5e22ca7d3a07dbbe6dae62025558e5b Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 20 Jan 2016 23:51:33 +0000 Subject: [PATCH] update parser and compiler to align with latest master --- py/compile2.c | 158 +++++++++++++++++++++++++++++--------------------- py/emit.h | 2 - py/parse2.c | 115 +++++++++++++++++++++++------------- py/parse2.h | 12 ++-- 4 files changed, 171 insertions(+), 116 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index 188d0f3f5e..cb034c6f36 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -1,9 +1,9 @@ /* - * This file is part of the Micro Python project, http://micropython.org/ + * This file is part of the MicroPython project, http://micropython.org/ * * The MIT License (MIT) * - * Copyright (c) 2013-2015 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,9 +33,9 @@ #include "py/scope.h" #include "py/emit.h" #include "py/compile.h" -#include "py/smallint.h" #include "py/runtime.h" -#include "py/builtin.h" + +#if MICROPY_ENABLE_COMPILER // TODO need to mangle __attr names @@ -80,7 +80,7 @@ typedef struct _compiler_t { // try to keep compiler clean from nlr mp_obj_t compile_error; // set to an exception object if there's an error - mp_uint_t compile_error_line; // set to best guess of line of error + size_t compile_error_line; // set to best guess of line of error uint next_label; @@ -94,7 +94,7 @@ typedef struct _compiler_t { mp_uint_t *co_data; - mp_uint_t num_scopes; + size_t num_scopes; scope_t **scopes; scope_t *scope_cur; @@ -112,7 +112,7 @@ typedef struct _compiler_t { STATIC void compile_error_set_line(compiler_t *comp, const byte *p) { // if the line of the error is unknown then try to update it from the parse data if (comp->compile_error_line == 0 && p != NULL && pt_is_any_rule(p)) { - mp_uint_t rule_id, src_line; + size_t rule_id, src_line; const byte *ptop; pt_rule_extract(p, &rule_id, &src_line, &ptop); comp->compile_error_line = src_line; @@ -147,7 +147,7 @@ STATIC void compile_decrease_except_level(compiler_t *comp) { comp->cur_except_level -= 1; } -STATIC void scope_new_and_link(compiler_t *comp, mp_uint_t scope_idx, scope_kind_t kind, const byte *p, uint emit_options) { +STATIC void scope_new_and_link(compiler_t *comp, size_t scope_idx, scope_kind_t kind, const byte *p, uint emit_options) { scope_t *scope = scope_new(kind, p, comp->source_file, emit_options); scope->parent = comp->scope_cur; comp->scopes[scope_idx] = scope; @@ -284,14 +284,12 @@ STATIC const byte *c_if_cond(compiler_t *comp, const byte *p, bool jump_if, int if (jump_if == false) { EMIT_ARG(jump, label); } - } else if (pt_is_rule(pt_rule_first(p), PN_testlist_comp)) { + } else { + assert(pt_is_rule(pt_rule_first(p), PN_testlist_comp)); // non-empty tuple, acts as true for the condition if (jump_if == true) { EMIT_ARG(jump, label); } - } else { - // parenthesis around 1 item, is just that item - c_if_cond(comp, pt_rule_first(p), jump_if, label); } return pt_next(p); } @@ -416,7 +414,6 @@ STATIC void c_assign_tuple(compiler_t *comp, const byte *p_head, const byte *p_t // assigns top of stack to pn STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind) { - tail_recursion: assert(!pt_is_null(p)); if (pt_is_any_id(p)) { qstr arg; @@ -459,16 +456,13 @@ STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind) if (pt_is_null_with_top(p0, ptop)) { // empty tuple goto cannot_assign; - } else if (pt_is_rule(p0, PN_testlist_comp)) { + } else { + assert(pt_is_rule(p0, PN_testlist_comp)); if (assign_kind != ASSIGN_STORE) { goto bad_aug; } p = p0; goto testlist_comp; - } else { - // parenthesis around 1 item, is just that item - p = p0; - goto tail_recursion; } break; } @@ -670,6 +664,13 @@ STATIC void compile_funcdef_lambdef_param(compiler_t *comp, const byte *p) { } STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, const byte *p, pn_kind_t pn_list_kind) { + // When we call compile_funcdef_lambdef_param below it can compile an arbitrary + // expression for default arguments, which may contain a lambda. The lambda will + // call here in a nested way, so we must save and restore the relevant state. + bool orig_have_star = comp->have_star; + uint16_t orig_num_dict_params = comp->num_dict_params; + uint16_t orig_num_default_params = comp->num_default_params; + // compile default parameters comp->have_star = false; comp->num_dict_params = 0; @@ -689,6 +690,11 @@ STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, const byte // make the function close_over_variables_etc(comp, scope, comp->num_default_params, comp->num_dict_params); + + // restore state + comp->have_star = orig_have_star; + comp->num_dict_params = orig_num_dict_params; + comp->num_default_params = orig_num_default_params; } // leaves function object on stack @@ -897,8 +903,11 @@ STATIC void c_del_stmt(compiler_t *comp, const byte *p) { goto cannot_delete; } } else if (pt_is_rule(p, PN_atom_paren)) { - p = pt_rule_first(p); - if (pt_is_rule(p, PN_testlist_comp)) { + if (pt_is_rule_empty(p)) { + goto cannot_delete; + } else { + p = pt_rule_first(p); + assert(pt_is_rule(p, PN_testlist_comp)); // TODO perhaps factorise testlist_comp code with other uses of PN_testlist_comp // or, simplify the logic here my making the parser simplify everything to a list const byte *p0 = pt_rule_first(p); @@ -923,12 +932,9 @@ STATIC void c_del_stmt(compiler_t *comp, const byte *p) { // sequence with 2 items c_del_stmt(comp, p1); } - } else { - // tuple with 1 element - c_del_stmt(comp, p); } } else { - // TODO is there anything else to implement? + // some arbitrary statment that we can't delete (eg del 1) goto cannot_delete; } @@ -1062,7 +1068,7 @@ STATIC void do_import_name(compiler_t *comp, const byte *p, qstr *q_base) { } qstr qst; p2 = pt_extract_id(p2, &qst); - mp_uint_t str_src_len; + size_t str_src_len; const byte *str_src = qstr_data(qst, &str_src_len); memcpy(str_dest, str_src, str_src_len); str_dest += str_src_len; @@ -1495,6 +1501,19 @@ STATIC void compile_for_stmt(compiler_t *comp, const byte *p, const byte *ptop) } } } + // arguments must be able to be compiled as standard expressions + if (pt_is_any_rule(p_start)) { + int k = pt_rule_extract_rule_id(p_start); + if (k == PN_arglist_star || k == PN_arglist_dbl_star || k == PN_argument) { + goto optimise_fail; + } + } + if (pt_is_any_rule(p_end)) { + int k = pt_rule_extract_rule_id(p_end); + if (k == PN_arglist_star || k == PN_arglist_dbl_star || k == PN_argument) { + goto optimise_fail; + } + } // can optimise const byte *p_body = p_it_top; const byte *p_else = pt_next(p_body); @@ -2221,7 +2240,8 @@ STATIC void compile_atom_paren(compiler_t *comp, const byte *p, const byte *ptop if (pt_is_null_with_top(p, ptop)) { // an empty tuple c_tuple(comp, NULL, NULL, NULL); - } else if (pt_is_rule(p, PN_testlist_comp)) { + } else { + assert(pt_is_rule(p, PN_testlist_comp)); p = pt_rule_first(p); const byte *p1 = pt_next(p); if (pt_is_rule(p1, PN_testlist_comp_3b) || pt_is_rule(p1, PN_testlist_comp_3c)) { @@ -2234,9 +2254,6 @@ STATIC void compile_atom_paren(compiler_t *comp, const byte *p, const byte *ptop // tuple with 2 items c_tuple(comp, NULL, p, ptop); } - } else { - // parenthesis around a single item, is just that item - compile_node(comp, p); } } @@ -2499,7 +2516,7 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) { EMIT_ARG(load_const_obj, mp_const_none); } else { qstr qst = p[1] | (p[2] << 8); - mp_uint_t len; + size_t len; const byte *data = qstr_data(qst, &len); EMIT_ARG(load_const_obj, mp_obj_new_bytes(data, len)); } @@ -2510,13 +2527,13 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) { compile_load_id(comp, qst); return p; } else if (*p == MP_PT_CONST_OBJECT) { - mp_uint_t idx; + size_t idx; p = pt_extract_const_obj(p, &idx); EMIT_ARG(load_const_obj, (mp_obj_t)comp->co_data[idx]); return p; } else { assert(*p >= MP_PT_RULE_BASE); - mp_uint_t rule_id, src_line; + size_t rule_id, src_line; const byte *ptop; p = pt_rule_extract(p, &rule_id, &src_line, &ptop); EMIT_ARG(set_source_line, src_line); @@ -2542,7 +2559,12 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) { STATIC void compile_scope_func_lambda_param(compiler_t *comp, const byte *p, pn_kind_t pn_name, pn_kind_t pn_star, pn_kind_t pn_dbl_star) { (void)pn_dbl_star; - // TODO verify that *k and **k are last etc + // check that **kw is last + if ((comp->scope_cur->scope_flags & MP_SCOPE_FLAG_VARKEYWORDS) != 0) { + compile_syntax_error(comp, p, "invalid syntax"); + return; + } + qstr param_name = MP_QSTR_NULL; uint param_flag = ID_FLAG_IS_PARAM; if (pt_is_any_id(p)) { @@ -2565,6 +2587,11 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, const byte *p, pn_ comp->scope_cur->num_pos_args += 1; } } else if (pt_is_rule(p, pn_star)) { + if (comp->have_star) { + // more than one star + compile_syntax_error(comp, p, "invalid syntax"); + return; + } comp->have_star = true; param_flag = ID_FLAG_IS_PARAM | ID_FLAG_IS_STAR_PARAM; if (pt_is_rule_empty(p)) { @@ -2655,9 +2682,16 @@ STATIC void compile_scope_func_annotations(compiler_t *comp, const byte *p) { } #endif // MICROPY_EMIT_NATIVE -STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const byte *p_inner_expr, int l_top, int for_depth) { +STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_comp_for, const byte *p_comp_for_top, const byte *p_inner_expr, int for_depth) { + uint l_top = comp_next_label(comp); + uint l_end = comp_next_label(comp); + EMIT_ARG(label_assign, l_top); + EMIT_ARG(for_iter, l_end); + c_assign(comp, p_comp_for, ASSIGN_STORE); + const byte *p_iter = pt_next(pt_next(p_comp_for)); + tail_recursion: - if (p_iter == NULL) { + if (p_iter == p_comp_for_top) { // no more nested if/for; compile inner expression compile_node(comp, p_inner_expr); if (comp->scope_cur->kind == SCOPE_LIST_COMP) { @@ -2674,12 +2708,8 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const } } else if (pt_is_rule(p_iter, PN_comp_if)) { // if condition - const byte *ptop; - const byte *p0 = pt_rule_extract_top(p_iter, &ptop); + const byte *p0 = pt_rule_extract_top(p_iter, &p_comp_for_top); p_iter = c_if_cond(comp, p0, false, l_top); - if (p_iter == ptop) { - p_iter = NULL; - } goto tail_recursion; } else { assert(pt_is_rule(p_iter, PN_comp_for)); // should be @@ -2687,18 +2717,14 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const const byte *ptop; const byte *p0 = pt_rule_extract_top(p_iter, &ptop); p0 = pt_next(p0); // skip scope index - const byte *p2 = compile_node(comp, pt_next(p0)); - uint l_end2 = comp_next_label(comp); - uint l_top2 = comp_next_label(comp); + compile_node(comp, pt_next(p0)); EMIT(get_iter); - EMIT_ARG(label_assign, l_top2); - EMIT_ARG(for_iter, l_end2); - c_assign(comp, p0, ASSIGN_STORE); - compile_scope_comp_iter(comp, p2 == ptop ? NULL : p2, p_inner_expr, l_top2, for_depth + 1); - EMIT_ARG(jump, l_top2); - EMIT_ARG(label_assign, l_end2); - EMIT(for_iter_end); + compile_scope_comp_iter(comp, p0, ptop, p_inner_expr, for_depth + 1); } + + EMIT_ARG(jump, l_top); + EMIT_ARG(label_assign, l_end); + EMIT(for_iter_end); } #if 0 @@ -2865,20 +2891,8 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { #endif } - uint l_end = comp_next_label(comp); - uint l_top = comp_next_label(comp); compile_load_id(comp, qstr_arg); - EMIT_ARG(label_assign, l_top); - EMIT_ARG(for_iter, l_end); - c_assign(comp, p_comp_for, ASSIGN_STORE); - const byte *p_comp_for_p2 = pt_next(pt_next(p_comp_for)); - if (p_comp_for_p2 == p_comp_for_top) { - p_comp_for_p2 = NULL; - } - compile_scope_comp_iter(comp, p_comp_for_p2, p, l_top, 0); - EMIT_ARG(jump, l_top); - EMIT_ARG(label_assign, l_end); - EMIT(for_iter_end); + compile_scope_comp_iter(comp, p_comp_for, p_comp_for_top, p, 0); if (scope->kind == SCOPE_GEN_EXPR) { EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); @@ -3063,7 +3077,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind } if (comp->pass > MP_PASS_SCOPE) { - EMIT_INLINE_ASM(end_pass); + EMIT_INLINE_ASM_ARG(end_pass, 0); } if (comp->compile_error != MP_OBJ_NULL) { @@ -3154,7 +3168,10 @@ STATIC void scope_compute_things(scope_t *scope) { } } -mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) { +#if !MICROPY_PERSISTENT_CODE_SAVE +STATIC +#endif +mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) { // put compiler state on the stack, it's relatively small compiler_t comp_state = {0}; compiler_t *comp = &comp_state; @@ -3351,7 +3368,14 @@ mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt if (comp->compile_error != MP_OBJ_NULL) { nlr_raise(comp->compile_error); } else { - // return function that executes the outer module - return mp_make_function_from_raw_code(outer_raw_code, MP_OBJ_NULL, MP_OBJ_NULL); + return outer_raw_code; } } + +mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) { + mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, emit_opt, is_repl); + // return function that executes the outer module + return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL); +} + +#endif // MICROPY_ENABLE_COMPILER diff --git a/py/emit.h b/py/emit.h index d65864b503..652e6118fb 100644 --- a/py/emit.h +++ b/py/emit.h @@ -269,8 +269,6 @@ typedef struct _emit_inline_asm_t emit_inline_asm_t; typedef struct _emit_inline_asm_method_table_t { void (*start_pass)(emit_inline_asm_t *emit, pass_kind_t pass, scope_t *scope, mp_obj_t *error_slot); void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig); - mp_uint_t (*count_params)(emit_inline_asm_t *emit, mp_uint_t n_params, mp_parse_node_t *pn_params); - void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig); mp_uint_t (*count_params)(emit_inline_asm_t *emit, const byte *p, const byte *ptop); bool (*label)(emit_inline_asm_t *emit, mp_uint_t label_num, qstr label_id); void (*align)(emit_inline_asm_t *emit, mp_uint_t align); diff --git a/py/parse2.c b/py/parse2.c index ce649270c0..06b518784f 100644 --- a/py/parse2.c +++ b/py/parse2.c @@ -1,9 +1,9 @@ /* - * This file is part of the Micro Python project, http://micropython.org/ + * This file is part of the MicroPython project, http://micropython.org/ * * The MIT License (MIT) * - * Copyright (c) 2013-2015 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -104,16 +104,16 @@ STATIC const rule_t *rules[] = { }; typedef struct _rule_stack_t { - mp_uint_t src_line : BITS_PER_WORD - 8; // maximum bits storing source line number - mp_uint_t rule_id : 8; // this must be large enough to fit largest rule number - mp_uint_t arg_i; // this dictates the maximum nodes in a "list" of things + size_t src_line : 8 * sizeof(size_t) - 8; // maximum bits storing source line number + size_t rule_id : 8; // this must be large enough to fit largest rule number + size_t arg_i; // this dictates the maximum nodes in a "list" of things size_t pt_off; } rule_stack_t; typedef struct _mp_parse_chunk_t { - mp_uint_t alloc; + size_t alloc; union { - mp_uint_t used; + size_t used; struct _mp_parse_chunk_t *next; } union_; byte data[]; @@ -128,14 +128,14 @@ typedef enum { typedef struct _parser_t { parse_error_t parse_error; - mp_uint_t rule_stack_alloc; - mp_uint_t rule_stack_top; + size_t rule_stack_alloc; + size_t rule_stack_top; rule_stack_t *rule_stack; mp_uint_t cur_scope_id; - mp_uint_t co_alloc; - mp_uint_t co_used; + size_t co_alloc; + size_t co_used; mp_uint_t *co_data; mp_lexer_t *lexer; @@ -147,7 +147,7 @@ typedef struct _parser_t { #endif } parser_t; -STATIC void push_rule(parser_t *parser, mp_uint_t src_line, const rule_t *rule, mp_uint_t arg_i, size_t pt_off) { +STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t arg_i, size_t pt_off) { if (parser->parse_error) { return; } @@ -167,14 +167,14 @@ STATIC void push_rule(parser_t *parser, mp_uint_t src_line, const rule_t *rule, rs->pt_off = pt_off; } -STATIC void push_rule_from_arg(parser_t *parser, mp_uint_t arg) { +STATIC void push_rule_from_arg(parser_t *parser, size_t arg) { assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE); - mp_uint_t rule_id = arg & RULE_ARG_ARG_MASK; + size_t rule_id = arg & RULE_ARG_ARG_MASK; assert(rule_id < RULE_maximum_number_of); push_rule(parser, parser->lexer->tok_line, rules[rule_id], 0, 0); } -STATIC void pop_rule(parser_t *parser, const rule_t **rule, mp_uint_t *arg_i, mp_uint_t *src_line, size_t *pt_off) { +STATIC void pop_rule(parser_t *parser, const rule_t **rule, size_t *arg_i, size_t *src_line, size_t *pt_off) { assert(!parser->parse_error); parser->rule_stack_top -= 1; *rule = rules[parser->rule_stack[parser->rule_stack_top].rule_id]; @@ -205,7 +205,7 @@ STATIC void pt_raw_truncate_at(pt_t *pt, size_t pt_off) { pt->vv.len = pt_off; } -STATIC int vuint_nbytes(mp_uint_t val) { +STATIC int vuint_nbytes(size_t val) { int n = 0; do { n += 1; @@ -214,7 +214,7 @@ STATIC int vuint_nbytes(mp_uint_t val) { return n; } -STATIC void vuint_store(byte *p, int nbytes, mp_uint_t val) { +STATIC void vuint_store(byte *p, int nbytes, size_t val) { p += nbytes; *--p = val & 0x7f; for (--nbytes; nbytes > 0; --nbytes) { @@ -223,9 +223,9 @@ STATIC void vuint_store(byte *p, int nbytes, mp_uint_t val) { } } -STATIC mp_uint_t vuint_load(const byte **p_in) { +STATIC size_t vuint_load(const byte **p_in) { const byte *p = *p_in; - mp_uint_t val = 0; + size_t val = 0; do { val = (val << 7) + (*p & 0x7f); } while ((*p++ & 0x80) != 0); @@ -269,8 +269,28 @@ STATIC byte *pt_advance(const byte *p, bool full_rule) { return (byte*)p; } +bool mp_parse_node_get_int_maybe(const byte *p, mp_obj_t *o) { + if (pt_is_small_int(p)) { + *o = MP_OBJ_NEW_SMALL_INT(pt_small_int_value(p)); + return true; + #if 0 // TODO + } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, RULE_const_object)) { + mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; + #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D + // nodes are 32-bit pointers, but need to extract 64-bit object + *o = (uint64_t)pns->nodes[0] | ((uint64_t)pns->nodes[1] << 32); + #else + *o = (mp_obj_t)pns->nodes[0]; + #endif + return MP_OBJ_IS_INT(*o); + #endif + } else { + return false; + } +} + // TODO this could perhaps allow *p to be null and in that case return null? -const byte *mp_parse_node_extract_list(const byte **p, mp_uint_t pn_kind) { +const byte *mp_parse_node_extract_list(const byte **p, size_t pn_kind) { if (pt_is_null(*p)) { *p += 1; return *p; @@ -295,7 +315,7 @@ const byte *pt_extract_id(const byte *p, qstr *qst) { } */ -const byte *pt_extract_const_obj(const byte *p, mp_uint_t *idx) { +const byte *pt_extract_const_obj(const byte *p, size_t *idx) { assert(*p == MP_PT_CONST_OBJECT); p += 1; *idx = vuint_load(&p); @@ -646,7 +666,7 @@ STATIC bool fold_constants(parser_t *parser, pt_t *pt, size_t pt_off, const rule } mp_obj_t dest[2]; mp_load_method_maybe(elem->value, q_attr, dest); - if (!(MP_OBJ_IS_SMALL_INT(dest[0]) && dest[1] == NULL)) { + if (!(MP_OBJ_IS_SMALL_INT(dest[0]) && dest[1] == MP_OBJ_NULL)) { return false; } arg0 = MP_OBJ_SMALL_INT_VALUE(dest[0]); @@ -665,9 +685,24 @@ STATIC bool fold_constants(parser_t *parser, pt_t *pt, size_t pt_off, const rule } #endif -STATIC void pt_ins_rule(parser_t *parser, pt_t *pt, size_t pt_off, mp_uint_t src_line, const rule_t *rule, mp_uint_t num_args) { +STATIC void pt_ins_rule(parser_t *parser, pt_t *pt, size_t pt_off, size_t src_line, const rule_t *rule, size_t num_args) { (void)num_args; + // optimise away parenthesis around an expression if possible + if (rule->rule_id == RULE_atom_paren) { + // there should be just 1 arg for this rule + const byte *p = (byte*)pt->vv.buf + pt_off; + if (pt_is_null(p)) { + // need to keep parenthesis for () + } else if (pt_is_rule(p, RULE_testlist_comp)) { + // need to keep parenthesis for (a, b, ...) + } else { + // parenthesis around a single expression, so it's just the expression + //printf("opt!\n"); + return; + } + } + #if MICROPY_COMP_CONST_FOLDING if (fold_constants(parser, pt, pt_off, rule)) { // we folded this rule so return straight away @@ -826,7 +861,7 @@ folding_fail:; // insert small int node for scope index if (extra_node != 0) { dest[1 + nb1 + nb2] = MP_PT_SMALL_INT; - mp_uint_t val = ++parser->cur_scope_id; + size_t val = ++parser->cur_scope_id; for (size_t i = 0; i < BYTES_PER_WORD; ++i) { dest[1 + nb1 + nb2 + 1 + i] = val; val >>= 8; @@ -848,7 +883,7 @@ STATIC void make_node_const_object(parser_t *parser, pt_t *pt, mp_obj_t obj) { parser->co_data[parser->co_used++] = (mp_uint_t)obj; } -STATIC void make_node_string_bytes(parser_t *parser, pt_t *pt, mp_token_kind_t tok, const char *str, mp_uint_t len) { +STATIC void make_node_string_bytes(parser_t *parser, pt_t *pt, mp_token_kind_t tok, const char *str, size_t len) { mp_obj_t o; if (tok == MP_TOKEN_STRING) { o = mp_obj_new_str(str, len, false); @@ -926,16 +961,16 @@ const byte *pt_rule_extract_top(const byte *p, const byte **ptop) { assert(*p >= MP_PT_RULE_BASE); p++; vuint_load(&p); - mp_uint_t nbytes = vuint_load(&p); + size_t nbytes = vuint_load(&p); *ptop = p + nbytes; return p; } -const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, mp_uint_t *src_line, const byte **ptop) { +const byte *pt_rule_extract(const byte *p, size_t *rule_id, size_t *src_line, const byte **ptop) { assert(*p >= MP_PT_RULE_BASE); *rule_id = *p++ - MP_PT_RULE_BASE; *src_line = vuint_load(&p); - mp_uint_t nbytes = vuint_load(&p); + size_t nbytes = vuint_load(&p); *ptop = p + nbytes; return p; } @@ -977,7 +1012,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } // work out the top-level rule to use, and push it on the stack - mp_uint_t top_level_rule; + size_t top_level_rule; switch (input_kind) { case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break; case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break; @@ -987,9 +1022,9 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // parse! - mp_uint_t n, i; // state for the current rule + size_t n, i; // state for the current rule size_t pt_off = 0; // state for the current rule - mp_uint_t rule_src_line; // source line for the first token matched by the current rule + size_t rule_src_line; // source line for the first token matched by the current rule bool backtrack = false; const rule_t *rule = NULL; pt_t *pt = pt_new(); @@ -1109,7 +1144,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { i = 0; bool emit_rule = false; /* - for (mp_uint_t x = 0; x < n; ++x) { + for (size_t x = 0; x < n; ++x) { if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK; if (tok_kind >= MP_TOKEN_NAME) { @@ -1125,7 +1160,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } */ - for (mp_uint_t x = 0; x < n; ++x) { + for (size_t x = 0; x < n; ++x) { if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK; if (tok_kind >= MP_TOKEN_NAME) { @@ -1164,8 +1199,6 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // if a rule has the RULE_ACT_ALLOW_IDENT bit set then this // rule should not be emitted if it has only 1 argument - // NOTE: can't set this flag for atom_paren because we need it - // to distinguish, for example, [a,b] from [(a,b)] if (rule->act & RULE_ACT_ALLOW_IDENT) { emit_rule = false; } @@ -1179,10 +1212,10 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } // count number of non-null nodes - mp_uint_t num_not_null = 0; - mp_uint_t num_trail_null = 0; + size_t num_not_null = 0; + size_t num_trail_null = 0; { const byte *p = (byte*)pt->vv.buf + pt_off; - for (mp_uint_t x = 0; x < i; ++x) { + for (size_t x = 0; x < i; ++x) { if (*p != MP_PT_NULL) { num_not_null += 1; num_trail_null = 0; @@ -1199,7 +1232,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } else { // single result, leave it on stack const byte *p = (byte*)pt->vv.buf + pt_off; - for (mp_uint_t x = 0; x < i; ++x) { + for (size_t x = 0; x < i; ++x) { if (*p == MP_PT_NULL) { p = pt_del_byte(pt, p); } else { @@ -1249,7 +1282,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } else { for (;;) { - mp_uint_t arg = rule->arg[i & 1 & n]; + size_t arg = rule->arg[i & 1 & n]; switch (arg & RULE_ARG_KIND_MASK) { case RULE_ARG_TOK: if (lex->tok_kind == (arg & RULE_ARG_ARG_MASK)) { @@ -1312,7 +1345,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { pt_show(pt); { - mp_uint_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes; + size_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes; qstr_pool_info(&n_pool, &n_qstr, &n_str_data_bytes, &n_total_bytes); printf("qstr pool: n_pool=" UINT_FMT ", n_qstr=" UINT_FMT ", n_str_data_bytes=" UINT_FMT ", n_total_bytes=" UINT_FMT "\n", diff --git a/py/parse2.h b/py/parse2.h index 5090cc00c7..f50af0df45 100644 --- a/py/parse2.h +++ b/py/parse2.h @@ -1,9 +1,9 @@ /* - * This file is part of the Micro Python project, http://micropython.org/ + * This file is part of the MicroPython project, http://micropython.org/ * * The MIT License (MIT) * - * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,11 +26,10 @@ #ifndef __MICROPY_INCLUDED_PY_PARSE_H__ #define __MICROPY_INCLUDED_PY_PARSE_H__ +#include #include -#include "py/mpconfig.h" -#include "py/misc.h" -#include "py/qstr.h" +#include "py/obj.h" struct _mp_lexer_t; @@ -108,9 +107,10 @@ mp_int_t pt_small_int_value(const byte *p); const byte *pt_get_small_int(const byte *p, mp_int_t *val); const byte *pt_rule_first(const byte *p); const byte *pt_rule_extract_top(const byte *p, const byte **ptop); -const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, mp_uint_t *src_line, const byte **ptop); +const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, size_t *src_line, const byte **ptop); bool pt_is_rule_empty(const byte *p); +bool mp_parse_node_get_int_maybe(const byte *p, mp_obj_t *o); const byte *mp_parse_node_extract_list(const byte **p, mp_uint_t pn_kind); typedef enum {