mirrors
/
micropython
mirror of https://github.com/micropython/micropython.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update parser and compiler to align with latest master

This commit is contained in:
Damien George 2016-01-20 23:51:33 +00:00
parent 6cdf8167f7
commit c23a2eabf5
4 changed files with 171 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
py/compile2.c
View File

@ -1,9 +1,9 @@
/*
* This file is part of the Micro Python project, http://micropython.org/
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2013-2015 Damien P. George
* Copyright (c) 2013-2016 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -33,9 +33,9 @@
#include "py/scope.h"
#include "py/emit.h"
#include "py/compile.h"
#include "py/smallint.h"
#include "py/runtime.h"
#include "py/builtin.h"
#if MICROPY_ENABLE_COMPILER
// TODO need to mangle __attr names
@ -80,7 +80,7 @@ typedef struct _compiler_t {
// try to keep compiler clean from nlr
mp_obj_t compile_error; // set to an exception object if there's an error
mp_uint_t compile_error_line; // set to best guess of line of error
size_t compile_error_line; // set to best guess of line of error
uint next_label;
@ -94,7 +94,7 @@ typedef struct _compiler_t {
mp_uint_t *co_data;
mp_uint_t num_scopes;
size_t num_scopes;
scope_t **scopes;
scope_t *scope_cur;
@ -112,7 +112,7 @@ typedef struct _compiler_t {
STATIC void compile_error_set_line(compiler_t *comp, const byte *p) {
// if the line of the error is unknown then try to update it from the parse data
if (comp->compile_error_line == 0 && p != NULL && pt_is_any_rule(p)) {
mp_uint_t rule_id, src_line;
size_t rule_id, src_line;
const byte *ptop;
pt_rule_extract(p, &rule_id, &src_line, &ptop);
comp->compile_error_line = src_line;
@ -147,7 +147,7 @@ STATIC void compile_decrease_except_level(compiler_t *comp) {
comp->cur_except_level -= 1;
}
STATIC void scope_new_and_link(compiler_t *comp, mp_uint_t scope_idx, scope_kind_t kind, const byte *p, uint emit_options) {
STATIC void scope_new_and_link(compiler_t *comp, size_t scope_idx, scope_kind_t kind, const byte *p, uint emit_options) {
scope_t *scope = scope_new(kind, p, comp->source_file, emit_options);
scope->parent = comp->scope_cur;
comp->scopes[scope_idx] = scope;
@ -284,14 +284,12 @@ STATIC const byte *c_if_cond(compiler_t *comp, const byte *p, bool jump_if, int
if (jump_if == false) {
EMIT_ARG(jump, label);
}
} else if (pt_is_rule(pt_rule_first(p), PN_testlist_comp)) {
} else {
assert(pt_is_rule(pt_rule_first(p), PN_testlist_comp));
// non-empty tuple, acts as true for the condition
if (jump_if == true) {
EMIT_ARG(jump, label);
}
} else {
// parenthesis around 1 item, is just that item
c_if_cond(comp, pt_rule_first(p), jump_if, label);
}
return pt_next(p);
}
@ -416,7 +414,6 @@ STATIC void c_assign_tuple(compiler_t *comp, const byte *p_head, const byte *p_t
// assigns top of stack to pn
STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind) {
tail_recursion:
assert(!pt_is_null(p));
if (pt_is_any_id(p)) {
qstr arg;
@ -459,16 +456,13 @@ STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind)
if (pt_is_null_with_top(p0, ptop)) {
// empty tuple
goto cannot_assign;
} else if (pt_is_rule(p0, PN_testlist_comp)) {
} else {
assert(pt_is_rule(p0, PN_testlist_comp));
if (assign_kind != ASSIGN_STORE) {
goto bad_aug;
}
p = p0;
goto testlist_comp;
} else {
// parenthesis around 1 item, is just that item
p = p0;
goto tail_recursion;
}
break;
}
@ -670,6 +664,13 @@ STATIC void compile_funcdef_lambdef_param(compiler_t *comp, const byte *p) {
}
STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, const byte *p, pn_kind_t pn_list_kind) {
// When we call compile_funcdef_lambdef_param below it can compile an arbitrary
// expression for default arguments, which may contain a lambda. The lambda will
// call here in a nested way, so we must save and restore the relevant state.
bool orig_have_star = comp->have_star;
uint16_t orig_num_dict_params = comp->num_dict_params;
uint16_t orig_num_default_params = comp->num_default_params;
// compile default parameters
comp->have_star = false;
comp->num_dict_params = 0;
@ -689,6 +690,11 @@ STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, const byte
// make the function
close_over_variables_etc(comp, scope, comp->num_default_params, comp->num_dict_params);
// restore state
comp->have_star = orig_have_star;
comp->num_dict_params = orig_num_dict_params;
comp->num_default_params = orig_num_default_params;
}
// leaves function object on stack
@ -897,8 +903,11 @@ STATIC void c_del_stmt(compiler_t *comp, const byte *p) {
goto cannot_delete;
}
} else if (pt_is_rule(p, PN_atom_paren)) {
p = pt_rule_first(p);
if (pt_is_rule(p, PN_testlist_comp)) {
if (pt_is_rule_empty(p)) {
goto cannot_delete;
} else {
p = pt_rule_first(p);
assert(pt_is_rule(p, PN_testlist_comp));
// TODO perhaps factorise testlist_comp code with other uses of PN_testlist_comp
// or, simplify the logic here my making the parser simplify everything to a list
const byte *p0 = pt_rule_first(p);
@ -923,12 +932,9 @@ STATIC void c_del_stmt(compiler_t *comp, const byte *p) {
// sequence with 2 items
c_del_stmt(comp, p1);
}
} else {
// tuple with 1 element
c_del_stmt(comp, p);
}
} else {
// TODO is there anything else to implement?
// some arbitrary statment that we can't delete (eg del 1)
goto cannot_delete;
}
@ -1062,7 +1068,7 @@ STATIC void do_import_name(compiler_t *comp, const byte *p, qstr *q_base) {
}
qstr qst;
p2 = pt_extract_id(p2, &qst);
mp_uint_t str_src_len;
size_t str_src_len;
const byte *str_src = qstr_data(qst, &str_src_len);
memcpy(str_dest, str_src, str_src_len);
str_dest += str_src_len;
@ -1495,6 +1501,19 @@ STATIC void compile_for_stmt(compiler_t *comp, const byte *p, const byte *ptop)
}
}
}
// arguments must be able to be compiled as standard expressions
if (pt_is_any_rule(p_start)) {
int k = pt_rule_extract_rule_id(p_start);
if (k == PN_arglist_star || k == PN_arglist_dbl_star || k == PN_argument) {
goto optimise_fail;
}
}
if (pt_is_any_rule(p_end)) {
int k = pt_rule_extract_rule_id(p_end);
if (k == PN_arglist_star || k == PN_arglist_dbl_star || k == PN_argument) {
goto optimise_fail;
}
}
// can optimise
const byte *p_body = p_it_top;
const byte *p_else = pt_next(p_body);
@ -2221,7 +2240,8 @@ STATIC void compile_atom_paren(compiler_t *comp, const byte *p, const byte *ptop
if (pt_is_null_with_top(p, ptop)) {
// an empty tuple
c_tuple(comp, NULL, NULL, NULL);
} else if (pt_is_rule(p, PN_testlist_comp)) {
} else {
assert(pt_is_rule(p, PN_testlist_comp));
p = pt_rule_first(p);
const byte *p1 = pt_next(p);
if (pt_is_rule(p1, PN_testlist_comp_3b) || pt_is_rule(p1, PN_testlist_comp_3c)) {
@ -2234,9 +2254,6 @@ STATIC void compile_atom_paren(compiler_t *comp, const byte *p, const byte *ptop
// tuple with 2 items
c_tuple(comp, NULL, p, ptop);
}
} else {
// parenthesis around a single item, is just that item
compile_node(comp, p);
}
}
@ -2499,7 +2516,7 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) {
EMIT_ARG(load_const_obj, mp_const_none);
} else {
qstr qst = p[1] | (p[2] << 8);
mp_uint_t len;
size_t len;
const byte *data = qstr_data(qst, &len);
EMIT_ARG(load_const_obj, mp_obj_new_bytes(data, len));
}
@ -2510,13 +2527,13 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) {
compile_load_id(comp, qst);
return p;
} else if (*p == MP_PT_CONST_OBJECT) {
mp_uint_t idx;
size_t idx;
p = pt_extract_const_obj(p, &idx);
EMIT_ARG(load_const_obj, (mp_obj_t)comp->co_data[idx]);
return p;
} else {
assert(*p >= MP_PT_RULE_BASE);
mp_uint_t rule_id, src_line;
size_t rule_id, src_line;
const byte *ptop;
p = pt_rule_extract(p, &rule_id, &src_line, &ptop);
EMIT_ARG(set_source_line, src_line);
@ -2542,7 +2559,12 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) {
STATIC void compile_scope_func_lambda_param(compiler_t *comp, const byte *p, pn_kind_t pn_name, pn_kind_t pn_star, pn_kind_t pn_dbl_star) {
(void)pn_dbl_star;
// TODO verify that *k and **k are last etc
// check that **kw is last
if ((comp->scope_cur->scope_flags & MP_SCOPE_FLAG_VARKEYWORDS) != 0) {
compile_syntax_error(comp, p, "invalid syntax");
return;
}
qstr param_name = MP_QSTR_NULL;
uint param_flag = ID_FLAG_IS_PARAM;
if (pt_is_any_id(p)) {
@ -2565,6 +2587,11 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, const byte *p, pn_
comp->scope_cur->num_pos_args += 1;
}
} else if (pt_is_rule(p, pn_star)) {
if (comp->have_star) {
// more than one star
compile_syntax_error(comp, p, "invalid syntax");
return;
}
comp->have_star = true;
param_flag = ID_FLAG_IS_PARAM | ID_FLAG_IS_STAR_PARAM;
if (pt_is_rule_empty(p)) {
@ -2655,9 +2682,16 @@ STATIC void compile_scope_func_annotations(compiler_t *comp, const byte *p) {
}
#endif // MICROPY_EMIT_NATIVE
STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const byte *p_inner_expr, int l_top, int for_depth) {
STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_comp_for, const byte *p_comp_for_top, const byte *p_inner_expr, int for_depth) {
uint l_top = comp_next_label(comp);
uint l_end = comp_next_label(comp);
EMIT_ARG(label_assign, l_top);
EMIT_ARG(for_iter, l_end);
c_assign(comp, p_comp_for, ASSIGN_STORE);
const byte *p_iter = pt_next(pt_next(p_comp_for));
tail_recursion:
if (p_iter == NULL) {
if (p_iter == p_comp_for_top) {
// no more nested if/for; compile inner expression
compile_node(comp, p_inner_expr);
if (comp->scope_cur->kind == SCOPE_LIST_COMP) {
@ -2674,12 +2708,8 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const
}
} else if (pt_is_rule(p_iter, PN_comp_if)) {
// if condition
const byte *ptop;
const byte *p0 = pt_rule_extract_top(p_iter, &ptop);
const byte *p0 = pt_rule_extract_top(p_iter, &p_comp_for_top);
p_iter = c_if_cond(comp, p0, false, l_top);
if (p_iter == ptop) {
p_iter = NULL;
}
goto tail_recursion;
} else {
assert(pt_is_rule(p_iter, PN_comp_for)); // should be
@ -2687,18 +2717,14 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const
const byte *ptop;
const byte *p0 = pt_rule_extract_top(p_iter, &ptop);
p0 = pt_next(p0); // skip scope index
const byte *p2 = compile_node(comp, pt_next(p0));
uint l_end2 = comp_next_label(comp);
uint l_top2 = comp_next_label(comp);
compile_node(comp, pt_next(p0));
EMIT(get_iter);
EMIT_ARG(label_assign, l_top2);
EMIT_ARG(for_iter, l_end2);
c_assign(comp, p0, ASSIGN_STORE);
compile_scope_comp_iter(comp, p2 == ptop ? NULL : p2, p_inner_expr, l_top2, for_depth + 1);
EMIT_ARG(jump, l_top2);
EMIT_ARG(label_assign, l_end2);
EMIT(for_iter_end);
compile_scope_comp_iter(comp, p0, ptop, p_inner_expr, for_depth + 1);
}
EMIT_ARG(jump, l_top);
EMIT_ARG(label_assign, l_end);
EMIT(for_iter_end);
}
#if 0
@ -2865,20 +2891,8 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) {
#endif
}
uint l_end = comp_next_label(comp);
uint l_top = comp_next_label(comp);
compile_load_id(comp, qstr_arg);
EMIT_ARG(label_assign, l_top);
EMIT_ARG(for_iter, l_end);
c_assign(comp, p_comp_for, ASSIGN_STORE);
const byte *p_comp_for_p2 = pt_next(pt_next(p_comp_for));
if (p_comp_for_p2 == p_comp_for_top) {
p_comp_for_p2 = NULL;
}
compile_scope_comp_iter(comp, p_comp_for_p2, p, l_top, 0);
EMIT_ARG(jump, l_top);
EMIT_ARG(label_assign, l_end);
EMIT(for_iter_end);
compile_scope_comp_iter(comp, p_comp_for, p_comp_for_top, p, 0);
if (scope->kind == SCOPE_GEN_EXPR) {
EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
@ -3063,7 +3077,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind
}
if (comp->pass > MP_PASS_SCOPE) {
EMIT_INLINE_ASM(end_pass);
EMIT_INLINE_ASM_ARG(end_pass, 0);
}
if (comp->compile_error != MP_OBJ_NULL) {
@ -3154,7 +3168,10 @@ STATIC void scope_compute_things(scope_t *scope) {
}
}
mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) {
#if !MICROPY_PERSISTENT_CODE_SAVE
STATIC
#endif
mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) {
// put compiler state on the stack, it's relatively small
compiler_t comp_state = {0};
compiler_t *comp = &comp_state;
@ -3351,7 +3368,14 @@ mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt
if (comp->compile_error != MP_OBJ_NULL) {
nlr_raise(comp->compile_error);
} else {
// return function that executes the outer module
return mp_make_function_from_raw_code(outer_raw_code, MP_OBJ_NULL, MP_OBJ_NULL);
return outer_raw_code;
}
}
mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) {
mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, emit_opt, is_repl);
// return function that executes the outer module
return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL);
}
#endif // MICROPY_ENABLE_COMPILER

2
py/emit.h
View File

@ -269,8 +269,6 @@ typedef struct _emit_inline_asm_t emit_inline_asm_t;
typedef struct _emit_inline_asm_method_table_t {
void (*start_pass)(emit_inline_asm_t *emit, pass_kind_t pass, scope_t *scope, mp_obj_t *error_slot);
void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig);
mp_uint_t (*count_params)(emit_inline_asm_t *emit, mp_uint_t n_params, mp_parse_node_t *pn_params);
void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig);
mp_uint_t (*count_params)(emit_inline_asm_t *emit, const byte *p, const byte *ptop);
bool (*label)(emit_inline_asm_t *emit, mp_uint_t label_num, qstr label_id);
void (*align)(emit_inline_asm_t *emit, mp_uint_t align);

115
py/parse2.c
View File

@ -1,9 +1,9 @@
/*
* This file is part of the Micro Python project, http://micropython.org/
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2013-2015 Damien P. George
* Copyright (c) 2013-2016 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -104,16 +104,16 @@ STATIC const rule_t *rules[] = {
};
typedef struct _rule_stack_t {
mp_uint_t src_line : BITS_PER_WORD - 8; // maximum bits storing source line number
mp_uint_t rule_id : 8; // this must be large enough to fit largest rule number
mp_uint_t arg_i; // this dictates the maximum nodes in a "list" of things
size_t src_line : 8 * sizeof(size_t) - 8; // maximum bits storing source line number
size_t rule_id : 8; // this must be large enough to fit largest rule number
size_t arg_i; // this dictates the maximum nodes in a "list" of things
size_t pt_off;
} rule_stack_t;
typedef struct _mp_parse_chunk_t {
mp_uint_t alloc;
size_t alloc;
union {
mp_uint_t used;
size_t used;
struct _mp_parse_chunk_t *next;
} union_;
byte data[];
@ -128,14 +128,14 @@ typedef enum {
typedef struct _parser_t {
parse_error_t parse_error;
mp_uint_t rule_stack_alloc;
mp_uint_t rule_stack_top;
size_t rule_stack_alloc;
size_t rule_stack_top;
rule_stack_t *rule_stack;
mp_uint_t cur_scope_id;
mp_uint_t co_alloc;
mp_uint_t co_used;
size_t co_alloc;
size_t co_used;
mp_uint_t *co_data;
mp_lexer_t *lexer;
@ -147,7 +147,7 @@ typedef struct _parser_t {
#endif
} parser_t;
STATIC void push_rule(parser_t *parser, mp_uint_t src_line, const rule_t *rule, mp_uint_t arg_i, size_t pt_off) {
STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t arg_i, size_t pt_off) {
if (parser->parse_error) {
return;
}
@ -167,14 +167,14 @@ STATIC void push_rule(parser_t *parser, mp_uint_t src_line, const rule_t *rule,
rs->pt_off = pt_off;
}
STATIC void push_rule_from_arg(parser_t *parser, mp_uint_t arg) {
STATIC void push_rule_from_arg(parser_t *parser, size_t arg) {
assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE);
mp_uint_t rule_id = arg & RULE_ARG_ARG_MASK;
size_t rule_id = arg & RULE_ARG_ARG_MASK;
assert(rule_id < RULE_maximum_number_of);
push_rule(parser, parser->lexer->tok_line, rules[rule_id], 0, 0);
}
STATIC void pop_rule(parser_t *parser, const rule_t **rule, mp_uint_t *arg_i, mp_uint_t *src_line, size_t *pt_off) {
STATIC void pop_rule(parser_t *parser, const rule_t **rule, size_t *arg_i, size_t *src_line, size_t *pt_off) {
assert(!parser->parse_error);
parser->rule_stack_top -= 1;
*rule = rules[parser->rule_stack[parser->rule_stack_top].rule_id];
@ -205,7 +205,7 @@ STATIC void pt_raw_truncate_at(pt_t *pt, size_t pt_off) {
pt->vv.len = pt_off;
}
STATIC int vuint_nbytes(mp_uint_t val) {
STATIC int vuint_nbytes(size_t val) {
int n = 0;
do {
n += 1;
@ -214,7 +214,7 @@ STATIC int vuint_nbytes(mp_uint_t val) {
return n;
}
STATIC void vuint_store(byte *p, int nbytes, mp_uint_t val) {
STATIC void vuint_store(byte *p, int nbytes, size_t val) {
p += nbytes;
*--p = val & 0x7f;
for (--nbytes; nbytes > 0; --nbytes) {
@ -223,9 +223,9 @@ STATIC void vuint_store(byte *p, int nbytes, mp_uint_t val) {
}
}
STATIC mp_uint_t vuint_load(const byte **p_in) {
STATIC size_t vuint_load(const byte **p_in) {
const byte *p = *p_in;
mp_uint_t val = 0;
size_t val = 0;
do {
val = (val << 7) + (*p & 0x7f);
} while ((*p++ & 0x80) != 0);
@ -269,8 +269,28 @@ STATIC byte *pt_advance(const byte *p, bool full_rule) {
return (byte*)p;
}
bool mp_parse_node_get_int_maybe(const byte *p, mp_obj_t *o) {
if (pt_is_small_int(p)) {
*o = MP_OBJ_NEW_SMALL_INT(pt_small_int_value(p));
return true;
#if 0 // TODO
} else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, RULE_const_object)) {
mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D
// nodes are 32-bit pointers, but need to extract 64-bit object
*o = (uint64_t)pns->nodes[0] | ((uint64_t)pns->nodes[1] << 32);
#else
*o = (mp_obj_t)pns->nodes[0];
#endif
return MP_OBJ_IS_INT(*o);
#endif
} else {
return false;
}
}
// TODO this could perhaps allow *p to be null and in that case return null?
const byte *mp_parse_node_extract_list(const byte **p, mp_uint_t pn_kind) {
const byte *mp_parse_node_extract_list(const byte **p, size_t pn_kind) {
if (pt_is_null(*p)) {
*p += 1;
return *p;
@ -295,7 +315,7 @@ const byte *pt_extract_id(const byte *p, qstr *qst) {
}
*/
const byte *pt_extract_const_obj(const byte *p, mp_uint_t *idx) {
const byte *pt_extract_const_obj(const byte *p, size_t *idx) {
assert(*p == MP_PT_CONST_OBJECT);
p += 1;
*idx = vuint_load(&p);
@ -646,7 +666,7 @@ STATIC bool fold_constants(parser_t *parser, pt_t *pt, size_t pt_off, const rule
}
mp_obj_t dest[2];
mp_load_method_maybe(elem->value, q_attr, dest);
if (!(MP_OBJ_IS_SMALL_INT(dest[0]) && dest[1] == NULL)) {
if (!(MP_OBJ_IS_SMALL_INT(dest[0]) && dest[1] == MP_OBJ_NULL)) {
return false;
}
arg0 = MP_OBJ_SMALL_INT_VALUE(dest[0]);
@ -665,9 +685,24 @@ STATIC bool fold_constants(parser_t *parser, pt_t *pt, size_t pt_off, const rule
}
#endif
STATIC void pt_ins_rule(parser_t *parser, pt_t *pt, size_t pt_off, mp_uint_t src_line, const rule_t *rule, mp_uint_t num_args) {
STATIC void pt_ins_rule(parser_t *parser, pt_t *pt, size_t pt_off, size_t src_line, const rule_t *rule, size_t num_args) {
(void)num_args;
// optimise away parenthesis around an expression if possible
if (rule->rule_id == RULE_atom_paren) {
// there should be just 1 arg for this rule
const byte *p = (byte*)pt->vv.buf + pt_off;
if (pt_is_null(p)) {
// need to keep parenthesis for ()
} else if (pt_is_rule(p, RULE_testlist_comp)) {
// need to keep parenthesis for (a, b, ...)
} else {
// parenthesis around a single expression, so it's just the expression
//printf("opt!\n");
return;
}
}
#if MICROPY_COMP_CONST_FOLDING
if (fold_constants(parser, pt, pt_off, rule)) {
// we folded this rule so return straight away
@ -826,7 +861,7 @@ folding_fail:;
// insert small int node for scope index
if (extra_node != 0) {
dest[1 + nb1 + nb2] = MP_PT_SMALL_INT;
mp_uint_t val = ++parser->cur_scope_id;
size_t val = ++parser->cur_scope_id;
for (size_t i = 0; i < BYTES_PER_WORD; ++i) {
dest[1 + nb1 + nb2 + 1 + i] = val;
val >>= 8;
@ -848,7 +883,7 @@ STATIC void make_node_const_object(parser_t *parser, pt_t *pt, mp_obj_t obj) {
parser->co_data[parser->co_used++] = (mp_uint_t)obj;
}
STATIC void make_node_string_bytes(parser_t *parser, pt_t *pt, mp_token_kind_t tok, const char *str, mp_uint_t len) {
STATIC void make_node_string_bytes(parser_t *parser, pt_t *pt, mp_token_kind_t tok, const char *str, size_t len) {
mp_obj_t o;
if (tok == MP_TOKEN_STRING) {
o = mp_obj_new_str(str, len, false);
@ -926,16 +961,16 @@ const byte *pt_rule_extract_top(const byte *p, const byte **ptop) {
assert(*p >= MP_PT_RULE_BASE);
p++;
vuint_load(&p);
mp_uint_t nbytes = vuint_load(&p);
size_t nbytes = vuint_load(&p);
*ptop = p + nbytes;
return p;
}
const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, mp_uint_t *src_line, const byte **ptop) {
const byte *pt_rule_extract(const byte *p, size_t *rule_id, size_t *src_line, const byte **ptop) {
assert(*p >= MP_PT_RULE_BASE);
*rule_id = *p++ - MP_PT_RULE_BASE;
*src_line = vuint_load(&p);
mp_uint_t nbytes = vuint_load(&p);
size_t nbytes = vuint_load(&p);
*ptop = p + nbytes;
return p;
}
@ -977,7 +1012,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
}
// work out the top-level rule to use, and push it on the stack
mp_uint_t top_level_rule;
size_t top_level_rule;
switch (input_kind) {
case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break;
case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
@ -987,9 +1022,9 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// parse!
mp_uint_t n, i; // state for the current rule
size_t n, i; // state for the current rule
size_t pt_off = 0; // state for the current rule
mp_uint_t rule_src_line; // source line for the first token matched by the current rule
size_t rule_src_line; // source line for the first token matched by the current rule
bool backtrack = false;
const rule_t *rule = NULL;
pt_t *pt = pt_new();
@ -1109,7 +1144,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
i = 0;
bool emit_rule = false;
/*
for (mp_uint_t x = 0; x < n; ++x) {
for (size_t x = 0; x < n; ++x) {
if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK;
if (tok_kind >= MP_TOKEN_NAME) {
@ -1125,7 +1160,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
}
}
*/
for (mp_uint_t x = 0; x < n; ++x) {
for (size_t x = 0; x < n; ++x) {
if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK;
if (tok_kind >= MP_TOKEN_NAME) {
@ -1164,8 +1199,6 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// if a rule has the RULE_ACT_ALLOW_IDENT bit set then this
// rule should not be emitted if it has only 1 argument
// NOTE: can't set this flag for atom_paren because we need it
// to distinguish, for example, [a,b] from [(a,b)]
if (rule->act & RULE_ACT_ALLOW_IDENT) {
emit_rule = false;
}
@ -1179,10 +1212,10 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
}
// count number of non-null nodes
mp_uint_t num_not_null = 0;
mp_uint_t num_trail_null = 0;
size_t num_not_null = 0;
size_t num_trail_null = 0;
{ const byte *p = (byte*)pt->vv.buf + pt_off;
for (mp_uint_t x = 0; x < i; ++x) {
for (size_t x = 0; x < i; ++x) {
if (*p != MP_PT_NULL) {
num_not_null += 1;
num_trail_null = 0;
@ -1199,7 +1232,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
} else {
// single result, leave it on stack
const byte *p = (byte*)pt->vv.buf + pt_off;
for (mp_uint_t x = 0; x < i; ++x) {
for (size_t x = 0; x < i; ++x) {
if (*p == MP_PT_NULL) {
p = pt_del_byte(pt, p);
} else {
@ -1249,7 +1282,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
}
} else {
for (;;) {
mp_uint_t arg = rule->arg[i & 1 & n];
size_t arg = rule->arg[i & 1 & n];
switch (arg & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK:
if (lex->tok_kind == (arg & RULE_ARG_ARG_MASK)) {
@ -1312,7 +1345,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
pt_show(pt);
{
mp_uint_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes;
size_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes;
qstr_pool_info(&n_pool, &n_qstr, &n_str_data_bytes, &n_total_bytes);
printf("qstr pool: n_pool=" UINT_FMT ", n_qstr=" UINT_FMT ", n_str_data_bytes="
UINT_FMT ", n_total_bytes=" UINT_FMT "\n",

12
py/parse2.h
View File

@ -1,9 +1,9 @@
/*
* This file is part of the Micro Python project, http://micropython.org/
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2013, 2014 Damien P. George
* Copyright (c) 2013-2016 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -26,11 +26,10 @@
#ifndef __MICROPY_INCLUDED_PY_PARSE_H__
#define __MICROPY_INCLUDED_PY_PARSE_H__
#include <stddef.h>
#include <stdint.h>
#include "py/mpconfig.h"
#include "py/misc.h"
#include "py/qstr.h"
#include "py/obj.h"
struct _mp_lexer_t;
@ -108,9 +107,10 @@ mp_int_t pt_small_int_value(const byte *p);
const byte *pt_get_small_int(const byte *p, mp_int_t *val);
const byte *pt_rule_first(const byte *p);
const byte *pt_rule_extract_top(const byte *p, const byte **ptop);
const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, mp_uint_t *src_line, const byte **ptop);
const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, size_t *src_line, const byte **ptop);
bool pt_is_rule_empty(const byte *p);
bool mp_parse_node_get_int_maybe(const byte *p, mp_obj_t *o);
const byte *mp_parse_node_extract_list(const byte **p, mp_uint_t pn_kind);
typedef enum {