2014-05-03 23:27:38 +01:00
|
|
|
/*
|
|
|
|
* This file is part of the Micro Python project, http://micropython.org/
|
|
|
|
*
|
|
|
|
* The MIT License (MIT)
|
|
|
|
*
|
|
|
|
* Copyright (c) 2013, 2014 Damien P. George
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
|
|
* in the Software without restriction, including without limitation the rights
|
|
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
|
|
* furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice shall be included in
|
|
|
|
* all copies or substantial portions of the Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
|
|
* THE SOFTWARE.
|
|
|
|
*/
|
|
|
|
|
2014-03-16 07:14:26 +00:00
|
|
|
#include <stdbool.h>
|
2013-10-04 19:53:11 +01:00
|
|
|
#include <stdint.h>
|
|
|
|
#include <stdio.h>
|
|
|
|
#include <assert.h>
|
|
|
|
|
|
|
|
#include "misc.h"
|
2013-12-21 18:17:45 +00:00
|
|
|
#include "mpconfig.h"
|
2014-01-21 21:40:13 +00:00
|
|
|
#include "qstr.h"
|
2013-10-04 19:53:11 +01:00
|
|
|
#include "lexer.h"
|
2014-03-01 19:50:50 +00:00
|
|
|
#include "parsenumbase.h"
|
2013-10-04 19:53:11 +01:00
|
|
|
#include "parse.h"
|
|
|
|
|
|
|
|
#define RULE_ACT_KIND_MASK (0xf0)
|
|
|
|
#define RULE_ACT_ARG_MASK (0x0f)
|
|
|
|
#define RULE_ACT_OR (0x10)
|
|
|
|
#define RULE_ACT_AND (0x20)
|
|
|
|
#define RULE_ACT_LIST (0x30)
|
|
|
|
|
|
|
|
#define RULE_ARG_BLANK (0x0000)
|
|
|
|
#define RULE_ARG_KIND_MASK (0xf000)
|
|
|
|
#define RULE_ARG_ARG_MASK (0x0fff)
|
|
|
|
#define RULE_ARG_TOK (0x1000)
|
|
|
|
#define RULE_ARG_RULE (0x2000)
|
|
|
|
#define RULE_ARG_OPT_TOK (0x3000)
|
|
|
|
#define RULE_ARG_OPT_RULE (0x4000)
|
|
|
|
|
2014-01-25 13:51:19 +00:00
|
|
|
#define ADD_BLANK_NODE(rule_id) ((rule_id) == RULE_funcdef || (rule_id) == RULE_classdef || (rule_id) == RULE_comp_for || (rule_id) == RULE_lambdef || (rule_id) == RULE_lambdef_nocond)
|
|
|
|
|
2013-10-04 19:53:11 +01:00
|
|
|
// (un)comment to use rule names; for debugging
|
|
|
|
//#define USE_RULE_NAME (1)
|
|
|
|
|
|
|
|
typedef struct _rule_t {
|
|
|
|
byte rule_id;
|
|
|
|
byte act;
|
|
|
|
#ifdef USE_RULE_NAME
|
|
|
|
const char *rule_name;
|
|
|
|
#endif
|
|
|
|
uint16_t arg[];
|
|
|
|
} rule_t;
|
|
|
|
|
|
|
|
enum {
|
|
|
|
RULE_none = 0,
|
2014-01-23 00:00:53 +00:00
|
|
|
#define DEF_RULE(rule, comp, kind, ...) RULE_##rule,
|
2013-10-04 19:53:11 +01:00
|
|
|
#include "grammar.h"
|
|
|
|
#undef DEF_RULE
|
|
|
|
RULE_maximum_number_of,
|
|
|
|
};
|
|
|
|
|
|
|
|
#define or(n) (RULE_ACT_OR | n)
|
|
|
|
#define and(n) (RULE_ACT_AND | n)
|
|
|
|
#define one_or_more (RULE_ACT_LIST | 2)
|
|
|
|
#define list (RULE_ACT_LIST | 1)
|
|
|
|
#define list_with_end (RULE_ACT_LIST | 3)
|
2013-12-21 18:17:45 +00:00
|
|
|
#define tok(t) (RULE_ARG_TOK | MP_TOKEN_##t)
|
2013-10-04 19:53:11 +01:00
|
|
|
#define rule(r) (RULE_ARG_RULE | RULE_##r)
|
2013-12-21 18:17:45 +00:00
|
|
|
#define opt_tok(t) (RULE_ARG_OPT_TOK | MP_TOKEN_##t)
|
2013-10-04 19:53:11 +01:00
|
|
|
#define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r)
|
|
|
|
#ifdef USE_RULE_NAME
|
2014-01-23 00:00:53 +00:00
|
|
|
#define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } };
|
2013-10-04 19:53:11 +01:00
|
|
|
#else
|
2014-01-23 00:00:53 +00:00
|
|
|
#define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } };
|
2013-10-04 19:53:11 +01:00
|
|
|
#endif
|
|
|
|
#include "grammar.h"
|
|
|
|
#undef or
|
|
|
|
#undef and
|
|
|
|
#undef list
|
|
|
|
#undef list_with_end
|
|
|
|
#undef tok
|
|
|
|
#undef rule
|
|
|
|
#undef opt_tok
|
|
|
|
#undef opt_rule
|
|
|
|
#undef one_or_more
|
|
|
|
#undef DEF_RULE
|
|
|
|
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC const rule_t *rules[] = {
|
2013-10-04 19:53:11 +01:00
|
|
|
NULL,
|
2014-01-23 00:00:53 +00:00
|
|
|
#define DEF_RULE(rule, comp, kind, ...) &rule_##rule,
|
2013-10-04 19:53:11 +01:00
|
|
|
#include "grammar.h"
|
|
|
|
#undef DEF_RULE
|
|
|
|
};
|
|
|
|
|
|
|
|
typedef struct _rule_stack_t {
|
2014-01-18 23:24:36 +00:00
|
|
|
unsigned int src_line : 24;
|
|
|
|
unsigned int rule_id : 8;
|
2013-10-04 19:53:11 +01:00
|
|
|
int32_t arg_i; // what should be the size and signedness?
|
|
|
|
} rule_stack_t;
|
|
|
|
|
|
|
|
typedef struct _parser_t {
|
2014-04-10 15:27:31 +01:00
|
|
|
bool had_memory_error;
|
|
|
|
|
2013-10-04 19:53:11 +01:00
|
|
|
uint rule_stack_alloc;
|
|
|
|
uint rule_stack_top;
|
|
|
|
rule_stack_t *rule_stack;
|
|
|
|
|
2014-01-12 13:55:24 +00:00
|
|
|
uint result_stack_alloc;
|
2013-10-04 19:53:11 +01:00
|
|
|
uint result_stack_top;
|
2013-12-21 18:17:45 +00:00
|
|
|
mp_parse_node_t *result_stack;
|
2014-01-18 23:24:36 +00:00
|
|
|
|
|
|
|
mp_lexer_t *lexer;
|
2013-10-04 19:53:11 +01:00
|
|
|
} parser_t;
|
|
|
|
|
2014-04-10 15:27:31 +01:00
|
|
|
STATIC inline void memory_error(parser_t *parser) {
|
|
|
|
parser->had_memory_error = true;
|
|
|
|
}
|
|
|
|
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC void push_rule(parser_t *parser, int src_line, const rule_t *rule, int arg_i) {
|
2014-04-10 15:27:31 +01:00
|
|
|
if (parser->had_memory_error) {
|
|
|
|
return;
|
|
|
|
}
|
2013-10-04 19:53:11 +01:00
|
|
|
if (parser->rule_stack_top >= parser->rule_stack_alloc) {
|
py: Turn down amount of RAM parser and compiler use.
There are 2 locations in parser, and 1 in compiler, where memory
allocation is not precise. In the parser it's the rule stack and result
stack, in the compiler it's the array for the identifiers in the current
scope. All other mallocs are exact (ie they don't allocate more than is
needed).
This patch adds tuning options (MP_ALLOC_*) to mpconfig.h for these 3
inexact allocations.
The inexact allocations in the parser should actually be close to
logarithmic: you need an exponentially larger script (absent pathological
cases) to use up more room on the rule and result stacks. As such, the
default allocation policy for these is now to start with a modest sized
stack, but grow only in small increments.
For the identifier arrays in the compiler, these now start out quite
small (4 entries, since most functions don't have that many ids), and
grow incrementally by 6 (since if you have more ids than 4, you probably
have quite a few more, but it wouldn't be exponentially more).
Partially addresses issue #560.
2014-05-05 13:19:03 +01:00
|
|
|
rule_stack_t *rs = m_renew_maybe(rule_stack_t, parser->rule_stack, parser->rule_stack_alloc, parser->rule_stack_alloc + MP_ALLOC_PARSE_RULE_INC);
|
2014-04-10 15:27:31 +01:00
|
|
|
if (rs == NULL) {
|
|
|
|
memory_error(parser);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
parser->rule_stack = rs;
|
py: Turn down amount of RAM parser and compiler use.
There are 2 locations in parser, and 1 in compiler, where memory
allocation is not precise. In the parser it's the rule stack and result
stack, in the compiler it's the array for the identifiers in the current
scope. All other mallocs are exact (ie they don't allocate more than is
needed).
This patch adds tuning options (MP_ALLOC_*) to mpconfig.h for these 3
inexact allocations.
The inexact allocations in the parser should actually be close to
logarithmic: you need an exponentially larger script (absent pathological
cases) to use up more room on the rule and result stacks. As such, the
default allocation policy for these is now to start with a modest sized
stack, but grow only in small increments.
For the identifier arrays in the compiler, these now start out quite
small (4 entries, since most functions don't have that many ids), and
grow incrementally by 6 (since if you have more ids than 4, you probably
have quite a few more, but it wouldn't be exponentially more).
Partially addresses issue #560.
2014-05-05 13:19:03 +01:00
|
|
|
parser->rule_stack_alloc += MP_ALLOC_PARSE_RULE_INC;
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
2014-01-18 23:24:36 +00:00
|
|
|
rule_stack_t *rs = &parser->rule_stack[parser->rule_stack_top++];
|
|
|
|
rs->src_line = src_line;
|
|
|
|
rs->rule_id = rule->rule_id;
|
|
|
|
rs->arg_i = arg_i;
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC void push_rule_from_arg(parser_t *parser, uint arg) {
|
2013-10-04 19:53:11 +01:00
|
|
|
assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE);
|
|
|
|
uint rule_id = arg & RULE_ARG_ARG_MASK;
|
|
|
|
assert(rule_id < RULE_maximum_number_of);
|
2014-01-18 23:24:36 +00:00
|
|
|
push_rule(parser, mp_lexer_cur(parser->lexer)->src_line, rules[rule_id], 0);
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC void pop_rule(parser_t *parser, const rule_t **rule, uint *arg_i, uint *src_line) {
|
2014-04-10 15:27:31 +01:00
|
|
|
assert(!parser->had_memory_error);
|
2013-10-04 19:53:11 +01:00
|
|
|
parser->rule_stack_top -= 1;
|
|
|
|
*rule = rules[parser->rule_stack[parser->rule_stack_top].rule_id];
|
|
|
|
*arg_i = parser->rule_stack[parser->rule_stack_top].arg_i;
|
2014-01-18 23:24:36 +00:00
|
|
|
*src_line = parser->rule_stack[parser->rule_stack_top].src_line;
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
|
2013-12-21 18:17:45 +00:00
|
|
|
mp_parse_node_t mp_parse_node_new_leaf(machine_int_t kind, machine_int_t arg) {
|
2014-02-22 14:39:45 +00:00
|
|
|
if (kind == MP_PARSE_NODE_SMALL_INT) {
|
|
|
|
return (mp_parse_node_t)(kind | (arg << 1));
|
|
|
|
}
|
|
|
|
return (mp_parse_node_t)(kind | (arg << 5));
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
|
2014-01-25 13:51:19 +00:00
|
|
|
uint mp_parse_node_free(mp_parse_node_t pn) {
|
|
|
|
uint cnt = 0;
|
|
|
|
if (MP_PARSE_NODE_IS_STRUCT(pn)) {
|
|
|
|
mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn;
|
|
|
|
uint n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns);
|
|
|
|
uint rule_id = MP_PARSE_NODE_STRUCT_KIND(pns);
|
|
|
|
bool adjust = ADD_BLANK_NODE(rule_id);
|
|
|
|
if (adjust) {
|
|
|
|
n--;
|
|
|
|
}
|
|
|
|
for (uint i = 0; i < n; i++) {
|
|
|
|
cnt += mp_parse_node_free(pns->nodes[i]);
|
|
|
|
}
|
|
|
|
if (adjust) {
|
|
|
|
n++;
|
2014-01-24 22:56:19 +00:00
|
|
|
}
|
2014-01-25 13:51:19 +00:00
|
|
|
m_del_var(mp_parse_node_struct_t, mp_parse_node_t, n, pns);
|
2014-01-24 22:56:19 +00:00
|
|
|
cnt++;
|
|
|
|
}
|
|
|
|
return cnt;
|
|
|
|
}
|
|
|
|
|
2014-01-19 11:48:48 +00:00
|
|
|
#if MICROPY_DEBUG_PRINTERS
|
|
|
|
void mp_parse_node_print(mp_parse_node_t pn, int indent) {
|
2014-01-18 23:24:36 +00:00
|
|
|
if (MP_PARSE_NODE_IS_STRUCT(pn)) {
|
|
|
|
printf("[% 4d] ", (int)((mp_parse_node_struct_t*)pn)->source_line);
|
|
|
|
} else {
|
|
|
|
printf(" ");
|
|
|
|
}
|
2013-10-04 19:53:11 +01:00
|
|
|
for (int i = 0; i < indent; i++) {
|
|
|
|
printf(" ");
|
|
|
|
}
|
2013-12-21 18:17:45 +00:00
|
|
|
if (MP_PARSE_NODE_IS_NULL(pn)) {
|
2013-10-04 19:53:11 +01:00
|
|
|
printf("NULL\n");
|
2014-02-22 14:39:45 +00:00
|
|
|
} else if (MP_PARSE_NODE_IS_SMALL_INT(pn)) {
|
|
|
|
machine_int_t arg = MP_PARSE_NODE_LEAF_SMALL_INT(pn);
|
|
|
|
printf("int(" INT_FMT ")\n", arg);
|
2013-12-21 18:17:45 +00:00
|
|
|
} else if (MP_PARSE_NODE_IS_LEAF(pn)) {
|
2014-02-22 14:39:45 +00:00
|
|
|
machine_uint_t arg = MP_PARSE_NODE_LEAF_ARG(pn);
|
2013-12-21 18:17:45 +00:00
|
|
|
switch (MP_PARSE_NODE_LEAF_KIND(pn)) {
|
|
|
|
case MP_PARSE_NODE_ID: printf("id(%s)\n", qstr_str(arg)); break;
|
|
|
|
case MP_PARSE_NODE_INTEGER: printf("int(%s)\n", qstr_str(arg)); break;
|
|
|
|
case MP_PARSE_NODE_DECIMAL: printf("dec(%s)\n", qstr_str(arg)); break;
|
|
|
|
case MP_PARSE_NODE_STRING: printf("str(%s)\n", qstr_str(arg)); break;
|
|
|
|
case MP_PARSE_NODE_BYTES: printf("bytes(%s)\n", qstr_str(arg)); break;
|
2014-01-29 18:58:52 +00:00
|
|
|
case MP_PARSE_NODE_TOKEN: printf("tok(" INT_FMT ")\n", arg); break;
|
2013-10-04 19:53:11 +01:00
|
|
|
default: assert(0);
|
|
|
|
}
|
|
|
|
} else {
|
2014-01-25 13:51:19 +00:00
|
|
|
mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
|
|
|
|
uint n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns);
|
2013-10-04 19:53:11 +01:00
|
|
|
#ifdef USE_RULE_NAME
|
2014-01-25 13:51:19 +00:00
|
|
|
printf("%s(%d) (n=%d)\n", rules[MP_PARSE_NODE_STRUCT_KIND(pns)]->rule_name, MP_PARSE_NODE_STRUCT_KIND(pns), n);
|
2013-10-04 19:53:11 +01:00
|
|
|
#else
|
2014-01-25 13:51:19 +00:00
|
|
|
printf("rule(%u) (n=%d)\n", (uint)MP_PARSE_NODE_STRUCT_KIND(pns), n);
|
2013-10-04 19:53:11 +01:00
|
|
|
#endif
|
2014-01-25 13:51:19 +00:00
|
|
|
for (uint i = 0; i < n; i++) {
|
|
|
|
mp_parse_node_print(pns->nodes[i], indent + 2);
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2014-01-19 11:48:48 +00:00
|
|
|
#endif // MICROPY_DEBUG_PRINTERS
|
2013-10-04 19:53:11 +01:00
|
|
|
|
|
|
|
/*
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC void result_stack_show(parser_t *parser) {
|
2013-10-04 19:53:11 +01:00
|
|
|
printf("result stack, most recent first\n");
|
|
|
|
for (int i = parser->result_stack_top - 1; i >= 0; i--) {
|
2014-01-19 11:48:48 +00:00
|
|
|
mp_parse_node_print(parser->result_stack[i], 0);
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
*/
|
|
|
|
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC mp_parse_node_t pop_result(parser_t *parser) {
|
2014-04-10 15:27:31 +01:00
|
|
|
if (parser->had_memory_error) {
|
|
|
|
return MP_PARSE_NODE_NULL;
|
|
|
|
}
|
2013-10-04 19:53:11 +01:00
|
|
|
assert(parser->result_stack_top > 0);
|
|
|
|
return parser->result_stack[--parser->result_stack_top];
|
|
|
|
}
|
|
|
|
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC mp_parse_node_t peek_result(parser_t *parser, int pos) {
|
2014-04-10 15:27:31 +01:00
|
|
|
if (parser->had_memory_error) {
|
|
|
|
return MP_PARSE_NODE_NULL;
|
|
|
|
}
|
2013-10-04 19:53:11 +01:00
|
|
|
assert(parser->result_stack_top > pos);
|
|
|
|
return parser->result_stack[parser->result_stack_top - 1 - pos];
|
|
|
|
}
|
|
|
|
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC void push_result_node(parser_t *parser, mp_parse_node_t pn) {
|
2014-04-10 15:27:31 +01:00
|
|
|
if (parser->had_memory_error) {
|
|
|
|
return;
|
|
|
|
}
|
2014-01-12 13:55:24 +00:00
|
|
|
if (parser->result_stack_top >= parser->result_stack_alloc) {
|
py: Turn down amount of RAM parser and compiler use.
There are 2 locations in parser, and 1 in compiler, where memory
allocation is not precise. In the parser it's the rule stack and result
stack, in the compiler it's the array for the identifiers in the current
scope. All other mallocs are exact (ie they don't allocate more than is
needed).
This patch adds tuning options (MP_ALLOC_*) to mpconfig.h for these 3
inexact allocations.
The inexact allocations in the parser should actually be close to
logarithmic: you need an exponentially larger script (absent pathological
cases) to use up more room on the rule and result stacks. As such, the
default allocation policy for these is now to start with a modest sized
stack, but grow only in small increments.
For the identifier arrays in the compiler, these now start out quite
small (4 entries, since most functions don't have that many ids), and
grow incrementally by 6 (since if you have more ids than 4, you probably
have quite a few more, but it wouldn't be exponentially more).
Partially addresses issue #560.
2014-05-05 13:19:03 +01:00
|
|
|
mp_parse_node_t *pn = m_renew_maybe(mp_parse_node_t, parser->result_stack, parser->result_stack_alloc, parser->result_stack_alloc + MP_ALLOC_PARSE_RESULT_INC);
|
2014-04-10 15:27:31 +01:00
|
|
|
if (pn == NULL) {
|
|
|
|
memory_error(parser);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
parser->result_stack = pn;
|
py: Turn down amount of RAM parser and compiler use.
There are 2 locations in parser, and 1 in compiler, where memory
allocation is not precise. In the parser it's the rule stack and result
stack, in the compiler it's the array for the identifiers in the current
scope. All other mallocs are exact (ie they don't allocate more than is
needed).
This patch adds tuning options (MP_ALLOC_*) to mpconfig.h for these 3
inexact allocations.
The inexact allocations in the parser should actually be close to
logarithmic: you need an exponentially larger script (absent pathological
cases) to use up more room on the rule and result stacks. As such, the
default allocation policy for these is now to start with a modest sized
stack, but grow only in small increments.
For the identifier arrays in the compiler, these now start out quite
small (4 entries, since most functions don't have that many ids), and
grow incrementally by 6 (since if you have more ids than 4, you probably
have quite a few more, but it wouldn't be exponentially more).
Partially addresses issue #560.
2014-05-05 13:19:03 +01:00
|
|
|
parser->result_stack_alloc += MP_ALLOC_PARSE_RESULT_INC;
|
2014-01-12 13:55:24 +00:00
|
|
|
}
|
2013-10-04 19:53:11 +01:00
|
|
|
parser->result_stack[parser->result_stack_top++] = pn;
|
|
|
|
}
|
|
|
|
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC void push_result_token(parser_t *parser, const mp_lexer_t *lex) {
|
2013-12-21 18:17:45 +00:00
|
|
|
const mp_token_t *tok = mp_lexer_cur(lex);
|
|
|
|
mp_parse_node_t pn;
|
|
|
|
if (tok->kind == MP_TOKEN_NAME) {
|
2014-01-21 21:40:13 +00:00
|
|
|
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, qstr_from_strn(tok->str, tok->len));
|
2013-12-21 18:17:45 +00:00
|
|
|
} else if (tok->kind == MP_TOKEN_NUMBER) {
|
2013-10-04 19:53:11 +01:00
|
|
|
bool dec = false;
|
|
|
|
bool small_int = true;
|
2014-01-11 00:33:29 +00:00
|
|
|
machine_int_t int_val = 0;
|
2013-10-04 19:53:11 +01:00
|
|
|
int len = tok->len;
|
|
|
|
const char *str = tok->str;
|
2014-03-01 19:50:50 +00:00
|
|
|
int base = 0;
|
|
|
|
int i = mp_parse_num_base(str, len, &base);
|
2014-01-11 00:33:29 +00:00
|
|
|
bool overflow = false;
|
2013-10-04 19:53:11 +01:00
|
|
|
for (; i < len; i++) {
|
2014-01-11 00:33:29 +00:00
|
|
|
machine_int_t old_val = int_val;
|
2013-12-30 18:23:50 +00:00
|
|
|
if (unichar_isdigit(str[i]) && str[i] - '0' < base) {
|
2013-10-04 19:53:11 +01:00
|
|
|
int_val = base * int_val + str[i] - '0';
|
|
|
|
} else if (base == 16 && 'a' <= str[i] && str[i] <= 'f') {
|
|
|
|
int_val = base * int_val + str[i] - 'a' + 10;
|
2013-12-29 13:03:49 +00:00
|
|
|
} else if (base == 16 && 'A' <= str[i] && str[i] <= 'F') {
|
2013-10-04 19:53:11 +01:00
|
|
|
int_val = base * int_val + str[i] - 'A' + 10;
|
2013-11-02 19:47:57 +00:00
|
|
|
} else if (str[i] == '.' || str[i] == 'e' || str[i] == 'E' || str[i] == 'j' || str[i] == 'J') {
|
2013-10-04 19:53:11 +01:00
|
|
|
dec = true;
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
small_int = false;
|
|
|
|
break;
|
|
|
|
}
|
2014-01-11 00:33:29 +00:00
|
|
|
if (int_val < old_val) {
|
|
|
|
// If new value became less than previous, it's overflow
|
|
|
|
overflow = true;
|
|
|
|
} else if ((old_val ^ int_val) & WORD_MSBIT_HIGH) {
|
|
|
|
// If signed number changed sign - it's overflow
|
|
|
|
overflow = true;
|
|
|
|
}
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
if (dec) {
|
2014-01-21 21:40:13 +00:00
|
|
|
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_DECIMAL, qstr_from_strn(str, len));
|
2014-02-21 00:04:32 +00:00
|
|
|
} else if (small_int && !overflow && MP_PARSE_FITS_SMALL_INT(int_val)) {
|
2013-12-21 18:17:45 +00:00
|
|
|
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_SMALL_INT, int_val);
|
2013-10-04 19:53:11 +01:00
|
|
|
} else {
|
2014-01-21 21:40:13 +00:00
|
|
|
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_INTEGER, qstr_from_strn(str, len));
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
2013-12-21 18:17:45 +00:00
|
|
|
} else if (tok->kind == MP_TOKEN_STRING) {
|
2014-01-21 21:40:13 +00:00
|
|
|
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_STRING, qstr_from_strn(tok->str, tok->len));
|
2013-12-21 18:17:45 +00:00
|
|
|
} else if (tok->kind == MP_TOKEN_BYTES) {
|
2014-01-21 21:40:13 +00:00
|
|
|
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_BYTES, qstr_from_strn(tok->str, tok->len));
|
2013-10-04 19:53:11 +01:00
|
|
|
} else {
|
2013-12-21 18:17:45 +00:00
|
|
|
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_TOKEN, tok->kind);
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
push_result_node(parser, pn);
|
|
|
|
}
|
|
|
|
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC void push_result_rule(parser_t *parser, int src_line, const rule_t *rule, int num_args) {
|
2014-04-10 15:27:31 +01:00
|
|
|
mp_parse_node_struct_t *pn = m_new_obj_var_maybe(mp_parse_node_struct_t, mp_parse_node_t, num_args);
|
|
|
|
if (pn == NULL) {
|
|
|
|
memory_error(parser);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
pn->source_line = src_line;
|
|
|
|
pn->kind_num_nodes = (rule->rule_id & 0xff) | (num_args << 8);
|
2013-10-04 19:53:11 +01:00
|
|
|
for (int i = num_args; i > 0; i--) {
|
|
|
|
pn->nodes[i - 1] = pop_result(parser);
|
|
|
|
}
|
2013-12-21 18:17:45 +00:00
|
|
|
push_result_node(parser, (mp_parse_node_t)pn);
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
|
2014-02-15 16:10:44 +00:00
|
|
|
mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_parse_error_kind_t *parse_error_kind_out) {
|
2014-01-12 13:55:24 +00:00
|
|
|
|
2014-05-10 17:36:41 +01:00
|
|
|
// initialise parser and allocate memory for its stacks
|
2014-01-12 13:55:24 +00:00
|
|
|
|
2014-05-10 17:36:41 +01:00
|
|
|
parser_t parser;
|
2014-01-12 13:55:24 +00:00
|
|
|
|
2014-05-10 17:36:41 +01:00
|
|
|
parser.had_memory_error = false;
|
2014-04-10 15:27:31 +01:00
|
|
|
|
2014-05-10 17:36:41 +01:00
|
|
|
parser.rule_stack_alloc = MP_ALLOC_PARSE_RULE_INIT;
|
|
|
|
parser.rule_stack_top = 0;
|
|
|
|
parser.rule_stack = m_new_maybe(rule_stack_t, parser.rule_stack_alloc);
|
2013-10-04 19:53:11 +01:00
|
|
|
|
2014-05-10 17:36:41 +01:00
|
|
|
parser.result_stack_alloc = MP_ALLOC_PARSE_RESULT_INIT;
|
|
|
|
parser.result_stack_top = 0;
|
|
|
|
parser.result_stack = m_new_maybe(mp_parse_node_t, parser.result_stack_alloc);
|
2013-10-04 19:53:11 +01:00
|
|
|
|
2014-05-10 17:36:41 +01:00
|
|
|
parser.lexer = lex;
|
|
|
|
|
|
|
|
// check if we could allocate the stacks
|
|
|
|
if (parser.rule_stack == NULL || parser.result_stack == NULL) {
|
|
|
|
goto memory_error;
|
|
|
|
}
|
2014-01-18 23:24:36 +00:00
|
|
|
|
2014-01-12 13:55:24 +00:00
|
|
|
// work out the top-level rule to use, and push it on the stack
|
2013-10-18 19:58:12 +01:00
|
|
|
int top_level_rule;
|
|
|
|
switch (input_kind) {
|
2013-12-21 18:17:45 +00:00
|
|
|
case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break;
|
2014-01-15 22:14:03 +00:00
|
|
|
case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
|
2013-10-18 19:58:12 +01:00
|
|
|
default: top_level_rule = RULE_file_input;
|
|
|
|
}
|
2014-05-10 17:36:41 +01:00
|
|
|
push_rule(&parser, mp_lexer_cur(lex)->src_line, rules[top_level_rule], 0);
|
2013-10-04 19:53:11 +01:00
|
|
|
|
2014-01-12 13:55:24 +00:00
|
|
|
// parse!
|
|
|
|
|
2014-01-18 23:24:36 +00:00
|
|
|
uint n, i; // state for the current rule
|
|
|
|
uint rule_src_line; // source line for the first token matched by the current rule
|
2013-10-04 19:53:11 +01:00
|
|
|
bool backtrack = false;
|
2014-01-18 23:24:36 +00:00
|
|
|
const rule_t *rule = NULL;
|
2013-12-21 18:17:45 +00:00
|
|
|
mp_token_kind_t tok_kind;
|
2013-10-04 19:53:11 +01:00
|
|
|
bool emit_rule;
|
|
|
|
bool had_trailing_sep;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
next_rule:
|
2014-05-10 17:36:41 +01:00
|
|
|
if (parser.rule_stack_top == 0 || parser.had_memory_error) {
|
2013-10-04 19:53:11 +01:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2014-05-10 17:36:41 +01:00
|
|
|
pop_rule(&parser, &rule, &i, &rule_src_line);
|
2013-10-04 19:53:11 +01:00
|
|
|
n = rule->act & RULE_ACT_ARG_MASK;
|
|
|
|
|
|
|
|
/*
|
|
|
|
// debugging
|
2014-05-10 17:36:41 +01:00
|
|
|
printf("depth=%d ", parser.rule_stack_top);
|
|
|
|
for (int j = 0; j < parser.rule_stack_top; ++j) {
|
2013-10-04 19:53:11 +01:00
|
|
|
printf(" ");
|
|
|
|
}
|
|
|
|
printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack);
|
|
|
|
*/
|
|
|
|
|
|
|
|
switch (rule->act & RULE_ACT_KIND_MASK) {
|
|
|
|
case RULE_ACT_OR:
|
|
|
|
if (i > 0 && !backtrack) {
|
|
|
|
goto next_rule;
|
|
|
|
} else {
|
|
|
|
backtrack = false;
|
|
|
|
}
|
|
|
|
for (; i < n - 1; ++i) {
|
|
|
|
switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
|
|
|
|
case RULE_ARG_TOK:
|
2013-12-21 18:17:45 +00:00
|
|
|
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_token(&parser, lex);
|
2013-12-21 18:17:45 +00:00
|
|
|
mp_lexer_to_next(lex);
|
2013-10-04 19:53:11 +01:00
|
|
|
goto next_rule;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case RULE_ARG_RULE:
|
2014-05-10 17:36:41 +01:00
|
|
|
push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule
|
|
|
|
push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule
|
2013-10-04 19:53:11 +01:00
|
|
|
goto next_rule;
|
|
|
|
default:
|
|
|
|
assert(0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
|
2013-12-21 18:17:45 +00:00
|
|
|
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_token(&parser, lex);
|
2013-12-21 18:17:45 +00:00
|
|
|
mp_lexer_to_next(lex);
|
2013-10-04 19:53:11 +01:00
|
|
|
} else {
|
|
|
|
backtrack = true;
|
|
|
|
goto next_rule;
|
|
|
|
}
|
|
|
|
} else {
|
2014-05-10 17:36:41 +01:00
|
|
|
push_rule_from_arg(&parser, rule->arg[i]);
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case RULE_ACT_AND:
|
|
|
|
|
|
|
|
// failed, backtrack if we can, else syntax error
|
|
|
|
if (backtrack) {
|
|
|
|
assert(i > 0);
|
|
|
|
if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
|
|
|
|
// an optional rule that failed, so continue with next arg
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_node(&parser, MP_PARSE_NODE_NULL);
|
2013-10-04 19:53:11 +01:00
|
|
|
backtrack = false;
|
|
|
|
} else {
|
|
|
|
// a mandatory rule that failed, so propagate backtrack
|
|
|
|
if (i > 1) {
|
|
|
|
// already eaten tokens so can't backtrack
|
|
|
|
goto syntax_error;
|
|
|
|
} else {
|
|
|
|
goto next_rule;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// progress through the rule
|
|
|
|
for (; i < n; ++i) {
|
|
|
|
switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
|
|
|
|
case RULE_ARG_TOK:
|
|
|
|
// need to match a token
|
|
|
|
tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK;
|
2013-12-21 18:17:45 +00:00
|
|
|
if (mp_lexer_is_kind(lex, tok_kind)) {
|
2013-10-04 19:53:11 +01:00
|
|
|
// matched token
|
2013-12-21 18:17:45 +00:00
|
|
|
if (tok_kind == MP_TOKEN_NAME) {
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_token(&parser, lex);
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
2013-12-21 18:17:45 +00:00
|
|
|
mp_lexer_to_next(lex);
|
2013-10-04 19:53:11 +01:00
|
|
|
} else {
|
|
|
|
// failed to match token
|
|
|
|
if (i > 0) {
|
|
|
|
// already eaten tokens so can't backtrack
|
|
|
|
goto syntax_error;
|
|
|
|
} else {
|
|
|
|
// this rule failed, so backtrack
|
|
|
|
backtrack = true;
|
|
|
|
goto next_rule;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case RULE_ARG_RULE:
|
|
|
|
case RULE_ARG_OPT_RULE:
|
2014-05-10 17:36:41 +01:00
|
|
|
push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule
|
|
|
|
push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule
|
2013-10-04 19:53:11 +01:00
|
|
|
goto next_rule;
|
|
|
|
default:
|
|
|
|
assert(0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
assert(i == n);
|
|
|
|
|
|
|
|
// matched the rule, so now build the corresponding parse_node
|
|
|
|
|
|
|
|
// count number of arguments for the parse_node
|
|
|
|
i = 0;
|
|
|
|
emit_rule = false;
|
|
|
|
for (int x = 0; x < n; ++x) {
|
|
|
|
if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
|
|
|
|
tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK;
|
2013-12-21 18:17:45 +00:00
|
|
|
if (tok_kind >= MP_TOKEN_NAME) {
|
2013-10-04 19:53:11 +01:00
|
|
|
emit_rule = true;
|
|
|
|
}
|
2013-12-21 18:17:45 +00:00
|
|
|
if (tok_kind == MP_TOKEN_NAME) {
|
2013-10-04 19:53:11 +01:00
|
|
|
// only tokens which were names are pushed to stack
|
|
|
|
i += 1;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// rules are always pushed
|
|
|
|
i += 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-06 21:44:11 +01:00
|
|
|
#if 0 && !MICROPY_ENABLE_DOC_STRING
|
|
|
|
// this code discards lonely statement, such as doc strings
|
|
|
|
// problem is that doc strings have already been interned, so this doesn't really help reduce RAM usage
|
2014-05-10 17:36:41 +01:00
|
|
|
if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) {
|
|
|
|
mp_parse_node_t p = peek_result(&parser, 1);
|
2014-05-06 21:44:11 +01:00
|
|
|
if (MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) {
|
|
|
|
pop_result(parser);
|
|
|
|
pop_result(parser);
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0);
|
2014-05-06 21:44:11 +01:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2013-10-04 19:53:11 +01:00
|
|
|
// always emit these rules, even if they have only 1 argument
|
|
|
|
if (rule->rule_id == RULE_expr_stmt || rule->rule_id == RULE_yield_stmt) {
|
|
|
|
emit_rule = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
// never emit these rules if they have only 1 argument
|
|
|
|
// NOTE: can't put atom_paren here because we need it to distinguisg, for example, [a,b] from [(a,b)]
|
2013-10-06 00:28:28 +01:00
|
|
|
// TODO possibly put varargslist_name, varargslist_equal here as well
|
|
|
|
if (rule->rule_id == RULE_else_stmt || rule->rule_id == RULE_testlist_comp_3b || rule->rule_id == RULE_import_as_names_paren || rule->rule_id == RULE_typedargslist_name || rule->rule_id == RULE_typedargslist_colon || rule->rule_id == RULE_typedargslist_equal || rule->rule_id == RULE_dictorsetmaker_colon || rule->rule_id == RULE_classdef_2 || rule->rule_id == RULE_with_item_as || rule->rule_id == RULE_assert_stmt_extra || rule->rule_id == RULE_as_name || rule->rule_id == RULE_raise_stmt_from || rule->rule_id == RULE_vfpdef) {
|
2013-10-04 19:53:11 +01:00
|
|
|
emit_rule = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// always emit these rules, and add an extra blank node at the end (to be used by the compiler to store data)
|
2014-01-25 13:51:19 +00:00
|
|
|
if (ADD_BLANK_NODE(rule->rule_id)) {
|
2013-10-04 19:53:11 +01:00
|
|
|
emit_rule = true;
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_node(&parser, MP_PARSE_NODE_NULL);
|
2013-10-04 19:53:11 +01:00
|
|
|
i += 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
int num_not_nil = 0;
|
|
|
|
for (int x = 0; x < i; ++x) {
|
2014-05-10 17:36:41 +01:00
|
|
|
if (peek_result(&parser, x) != MP_PARSE_NODE_NULL) {
|
2013-10-04 19:53:11 +01:00
|
|
|
num_not_nil += 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
//printf("done and %s n=%d i=%d notnil=%d\n", rule->rule_name, n, i, num_not_nil);
|
|
|
|
if (emit_rule) {
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_rule(&parser, rule_src_line, rule, i);
|
2013-10-04 19:53:11 +01:00
|
|
|
} else if (num_not_nil == 0) {
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_rule(&parser, rule_src_line, rule, i); // needed for, eg, atom_paren, testlist_comp_3b
|
2013-10-04 19:53:11 +01:00
|
|
|
//result_stack_show(parser);
|
|
|
|
//assert(0);
|
|
|
|
} else if (num_not_nil == 1) {
|
|
|
|
// single result, leave it on stack
|
2013-12-21 18:17:45 +00:00
|
|
|
mp_parse_node_t pn = MP_PARSE_NODE_NULL;
|
2013-10-04 19:53:11 +01:00
|
|
|
for (int x = 0; x < i; ++x) {
|
2014-05-10 17:36:41 +01:00
|
|
|
mp_parse_node_t pn2 = pop_result(&parser);
|
2013-12-21 18:17:45 +00:00
|
|
|
if (pn2 != MP_PARSE_NODE_NULL) {
|
2013-10-04 19:53:11 +01:00
|
|
|
pn = pn2;
|
|
|
|
}
|
|
|
|
}
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_node(&parser, pn);
|
2013-10-04 19:53:11 +01:00
|
|
|
} else {
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_rule(&parser, rule_src_line, rule, i);
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case RULE_ACT_LIST:
|
|
|
|
// n=2 is: item item*
|
|
|
|
// n=1 is: item (sep item)*
|
|
|
|
// n=3 is: item (sep item)* [sep]
|
|
|
|
if (backtrack) {
|
|
|
|
list_backtrack:
|
|
|
|
had_trailing_sep = false;
|
|
|
|
if (n == 2) {
|
|
|
|
if (i == 1) {
|
|
|
|
// fail on item, first time round; propagate backtrack
|
|
|
|
goto next_rule;
|
|
|
|
} else {
|
|
|
|
// fail on item, in later rounds; finish with this rule
|
|
|
|
backtrack = false;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (i == 1) {
|
|
|
|
// fail on item, first time round; propagate backtrack
|
|
|
|
goto next_rule;
|
|
|
|
} else if ((i & 1) == 1) {
|
|
|
|
// fail on item, in later rounds; have eaten tokens so can't backtrack
|
|
|
|
if (n == 3) {
|
|
|
|
// list allows trailing separator; finish parsing list
|
|
|
|
had_trailing_sep = true;
|
|
|
|
backtrack = false;
|
|
|
|
} else {
|
|
|
|
// list doesn't allowing trailing separator; fail
|
|
|
|
goto syntax_error;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// fail on separator; finish parsing list
|
|
|
|
backtrack = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
for (;;) {
|
|
|
|
uint arg = rule->arg[i & 1 & n];
|
|
|
|
switch (arg & RULE_ARG_KIND_MASK) {
|
|
|
|
case RULE_ARG_TOK:
|
2013-12-21 18:17:45 +00:00
|
|
|
if (mp_lexer_is_kind(lex, arg & RULE_ARG_ARG_MASK)) {
|
2013-10-04 19:53:11 +01:00
|
|
|
if (i & 1 & n) {
|
|
|
|
// separators which are tokens are not pushed to result stack
|
|
|
|
} else {
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_token(&parser, lex);
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
2013-12-21 18:17:45 +00:00
|
|
|
mp_lexer_to_next(lex);
|
2013-10-04 19:53:11 +01:00
|
|
|
// got element of list, so continue parsing list
|
|
|
|
i += 1;
|
|
|
|
} else {
|
|
|
|
// couldn't get element of list
|
|
|
|
i += 1;
|
|
|
|
backtrack = true;
|
|
|
|
goto list_backtrack;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case RULE_ARG_RULE:
|
2014-05-10 17:36:41 +01:00
|
|
|
push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule
|
|
|
|
push_rule_from_arg(&parser, arg); // push child of list-rule
|
2013-10-04 19:53:11 +01:00
|
|
|
goto next_rule;
|
|
|
|
default:
|
|
|
|
assert(0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
assert(i >= 1);
|
|
|
|
|
|
|
|
// compute number of elements in list, result in i
|
|
|
|
i -= 1;
|
|
|
|
if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
|
|
|
|
// don't count separators when they are tokens
|
|
|
|
i = (i + 1) / 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i == 1) {
|
|
|
|
// list matched single item
|
|
|
|
if (had_trailing_sep) {
|
|
|
|
// if there was a trailing separator, make a list of a single item
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_rule(&parser, rule_src_line, rule, i);
|
2013-10-04 19:53:11 +01:00
|
|
|
} else {
|
|
|
|
// just leave single item on stack (ie don't wrap in a list)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
//printf("done list %s %d %d\n", rule->rule_name, n, i);
|
2014-05-10 17:36:41 +01:00
|
|
|
push_result_rule(&parser, rule_src_line, rule, i);
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
|
|
|
assert(0);
|
|
|
|
}
|
|
|
|
}
|
2013-10-09 15:09:52 +01:00
|
|
|
|
2014-04-10 15:27:31 +01:00
|
|
|
mp_parse_node_t result;
|
|
|
|
|
|
|
|
// check if we had a memory error
|
2014-05-10 17:36:41 +01:00
|
|
|
if (parser.had_memory_error) {
|
|
|
|
memory_error:
|
2014-04-10 15:27:31 +01:00
|
|
|
*parse_error_kind_out = MP_PARSE_ERROR_MEMORY;
|
|
|
|
result = MP_PARSE_NODE_NULL;
|
|
|
|
goto finished;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2013-10-09 15:09:52 +01:00
|
|
|
// check we are at the end of the token stream
|
2013-12-21 18:17:45 +00:00
|
|
|
if (!mp_lexer_is_kind(lex, MP_TOKEN_END)) {
|
2013-10-09 15:09:52 +01:00
|
|
|
goto syntax_error;
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|
2013-10-09 15:09:52 +01:00
|
|
|
|
2013-10-04 19:53:11 +01:00
|
|
|
//printf("--------------\n");
|
|
|
|
//result_stack_show(parser);
|
2014-05-10 17:36:41 +01:00
|
|
|
//printf("rule stack alloc: %d\n", parser.rule_stack_alloc);
|
|
|
|
//printf("result stack alloc: %d\n", parser.result_stack_alloc);
|
2013-10-04 19:53:11 +01:00
|
|
|
//printf("number of parse nodes allocated: %d\n", num_parse_nodes_allocated);
|
2014-01-12 13:55:24 +00:00
|
|
|
|
|
|
|
// get the root parse node that we created
|
2014-05-10 17:36:41 +01:00
|
|
|
assert(parser.result_stack_top == 1);
|
|
|
|
result = parser.result_stack[0];
|
2014-01-12 13:55:24 +00:00
|
|
|
|
|
|
|
finished:
|
|
|
|
// free the memory that we don't need anymore
|
2014-05-10 17:36:41 +01:00
|
|
|
m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc);
|
|
|
|
m_del(mp_parse_node_t, parser.result_stack, parser.result_stack_alloc);
|
2014-01-12 13:55:24 +00:00
|
|
|
|
|
|
|
// return the result
|
|
|
|
return result;
|
2013-10-04 19:53:11 +01:00
|
|
|
|
|
|
|
syntax_error:
|
2013-12-21 18:17:45 +00:00
|
|
|
if (mp_lexer_is_kind(lex, MP_TOKEN_INDENT)) {
|
2014-02-15 16:10:44 +00:00
|
|
|
*parse_error_kind_out = MP_PARSE_ERROR_UNEXPECTED_INDENT;
|
2013-12-21 18:17:45 +00:00
|
|
|
} else if (mp_lexer_is_kind(lex, MP_TOKEN_DEDENT_MISMATCH)) {
|
2014-02-15 16:10:44 +00:00
|
|
|
*parse_error_kind_out = MP_PARSE_ERROR_UNMATCHED_UNINDENT;
|
2013-10-09 15:09:52 +01:00
|
|
|
} else {
|
2014-02-15 16:10:44 +00:00
|
|
|
*parse_error_kind_out = MP_PARSE_ERROR_INVALID_SYNTAX;
|
2013-10-04 19:53:11 +01:00
|
|
|
#ifdef USE_RULE_NAME
|
2014-01-15 21:23:31 +00:00
|
|
|
// debugging: print the rule name that failed and the token
|
2014-02-15 16:10:44 +00:00
|
|
|
printf("rule: %s\n", rule->rule_name);
|
|
|
|
#if MICROPY_DEBUG_PRINTERS
|
2013-12-21 18:17:45 +00:00
|
|
|
mp_token_show(mp_lexer_cur(lex));
|
2014-02-15 16:10:44 +00:00
|
|
|
#endif
|
2014-01-15 21:23:31 +00:00
|
|
|
#endif
|
2013-10-09 15:09:52 +01:00
|
|
|
}
|
2014-01-12 13:55:24 +00:00
|
|
|
result = MP_PARSE_NODE_NULL;
|
|
|
|
goto finished;
|
2013-10-04 19:53:11 +01:00
|
|
|
}
|