1834 lines
65 KiB
Python
Executable File
1834 lines
65 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# This file is part of the MicroPython project, http://micropython.org/
|
|
#
|
|
# The MIT License (MIT)
|
|
#
|
|
# Copyright (c) 2016-2019 Damien P. George
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in
|
|
# all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
# THE SOFTWARE.
|
|
|
|
# Python 2/3 compatibility code
|
|
from __future__ import print_function
|
|
import platform
|
|
|
|
if platform.python_version_tuple()[0] == "2":
|
|
from binascii import hexlify as hexlify_py2
|
|
|
|
str_cons = lambda val, enc=None: str(val)
|
|
bytes_cons = lambda val, enc=None: bytearray(val)
|
|
is_str_type = lambda o: isinstance(o, str)
|
|
is_bytes_type = lambda o: type(o) is bytearray
|
|
is_int_type = lambda o: isinstance(o, int) or isinstance(o, long) # noqa: F821
|
|
|
|
def hexlify_to_str(b):
|
|
x = hexlify_py2(b)
|
|
return ":".join(x[i : i + 2] for i in range(0, len(x), 2))
|
|
|
|
else:
|
|
from binascii import hexlify
|
|
|
|
str_cons = str
|
|
bytes_cons = bytes
|
|
is_str_type = lambda o: isinstance(o, str)
|
|
is_bytes_type = lambda o: isinstance(o, bytes)
|
|
is_int_type = lambda o: isinstance(o, int)
|
|
|
|
def hexlify_to_str(b):
|
|
return str(hexlify(b, ":"), "ascii")
|
|
|
|
|
|
# end compatibility code
|
|
|
|
import sys
|
|
import struct
|
|
|
|
sys.path.append(sys.path[0] + "/../py")
|
|
import makeqstrdata as qstrutil
|
|
|
|
# Threshold of str length below which it will be turned into a qstr when freezing.
|
|
# This helps to reduce frozen code size because qstrs are more efficient to encode
|
|
# as objects than full mp_obj_str_t instances.
|
|
PERSISTENT_STR_INTERN_THRESHOLD = 25
|
|
|
|
|
|
class MPYReadError(Exception):
|
|
def __init__(self, filename, msg):
|
|
self.filename = filename
|
|
self.msg = msg
|
|
|
|
def __str__(self):
|
|
return "%s: %s" % (self.filename, self.msg)
|
|
|
|
|
|
class FreezeError(Exception):
|
|
def __init__(self, rawcode, msg):
|
|
self.rawcode = rawcode
|
|
self.msg = msg
|
|
|
|
def __str__(self):
|
|
return "error while freezing %s: %s" % (self.rawcode.source_file, self.msg)
|
|
|
|
|
|
class Config:
|
|
MPY_VERSION = 6
|
|
MPY_SUB_VERSION = 2
|
|
MICROPY_LONGINT_IMPL_NONE = 0
|
|
MICROPY_LONGINT_IMPL_LONGLONG = 1
|
|
MICROPY_LONGINT_IMPL_MPZ = 2
|
|
|
|
|
|
config = Config()
|
|
|
|
|
|
MP_CODE_BYTECODE = 2
|
|
MP_CODE_NATIVE_PY = 3
|
|
MP_CODE_NATIVE_VIPER = 4
|
|
MP_CODE_NATIVE_ASM = 5
|
|
|
|
MP_NATIVE_ARCH_NONE = 0
|
|
MP_NATIVE_ARCH_X86 = 1
|
|
MP_NATIVE_ARCH_X64 = 2
|
|
MP_NATIVE_ARCH_ARMV6 = 3
|
|
MP_NATIVE_ARCH_ARMV6M = 4
|
|
MP_NATIVE_ARCH_ARMV7M = 5
|
|
MP_NATIVE_ARCH_ARMV7EM = 6
|
|
MP_NATIVE_ARCH_ARMV7EMSP = 7
|
|
MP_NATIVE_ARCH_ARMV7EMDP = 8
|
|
MP_NATIVE_ARCH_XTENSA = 9
|
|
MP_NATIVE_ARCH_XTENSAWIN = 10
|
|
|
|
MP_PERSISTENT_OBJ_FUN_TABLE = 0
|
|
MP_PERSISTENT_OBJ_NONE = 1
|
|
MP_PERSISTENT_OBJ_FALSE = 2
|
|
MP_PERSISTENT_OBJ_TRUE = 3
|
|
MP_PERSISTENT_OBJ_ELLIPSIS = 4
|
|
MP_PERSISTENT_OBJ_STR = 5
|
|
MP_PERSISTENT_OBJ_BYTES = 6
|
|
MP_PERSISTENT_OBJ_INT = 7
|
|
MP_PERSISTENT_OBJ_FLOAT = 8
|
|
MP_PERSISTENT_OBJ_COMPLEX = 9
|
|
MP_PERSISTENT_OBJ_TUPLE = 10
|
|
|
|
MP_SCOPE_FLAG_VIPERRELOC = 0x10
|
|
MP_SCOPE_FLAG_VIPERRODATA = 0x20
|
|
MP_SCOPE_FLAG_VIPERBSS = 0x40
|
|
|
|
MP_BC_MASK_EXTRA_BYTE = 0x9E
|
|
|
|
MP_BC_FORMAT_BYTE = 0
|
|
MP_BC_FORMAT_QSTR = 1
|
|
MP_BC_FORMAT_VAR_UINT = 2
|
|
MP_BC_FORMAT_OFFSET = 3
|
|
|
|
mp_unary_op_method_name = (
|
|
"__pos__",
|
|
"__neg__",
|
|
"__invert__",
|
|
"<not>",
|
|
)
|
|
|
|
mp_binary_op_method_name = (
|
|
"__lt__",
|
|
"__gt__",
|
|
"__eq__",
|
|
"__le__",
|
|
"__ge__",
|
|
"__ne__",
|
|
"<in>",
|
|
"<is>",
|
|
"<exception match>",
|
|
"__ior__",
|
|
"__ixor__",
|
|
"__iand__",
|
|
"__ilshift__",
|
|
"__irshift__",
|
|
"__iadd__",
|
|
"__isub__",
|
|
"__imul__",
|
|
"__imatmul__",
|
|
"__ifloordiv__",
|
|
"__itruediv__",
|
|
"__imod__",
|
|
"__ipow__",
|
|
"__or__",
|
|
"__xor__",
|
|
"__and__",
|
|
"__lshift__",
|
|
"__rshift__",
|
|
"__add__",
|
|
"__sub__",
|
|
"__mul__",
|
|
"__matmul__",
|
|
"__floordiv__",
|
|
"__truediv__",
|
|
"__mod__",
|
|
"__pow__",
|
|
)
|
|
|
|
|
|
class Opcode:
|
|
# fmt: off
|
|
# Load, Store, Delete, Import, Make, Build, Unpack, Call, Jump, Exception, For, sTack, Return, Yield, Op
|
|
MP_BC_BASE_RESERVED = (0x00) # ----------------
|
|
MP_BC_BASE_QSTR_O = (0x10) # LLLLLLSSSDDII---
|
|
MP_BC_BASE_VINT_E = (0x20) # MMLLLLSSDDBBBBBB
|
|
MP_BC_BASE_VINT_O = (0x30) # UUMMCCCC--------
|
|
MP_BC_BASE_JUMP_E = (0x40) # J-JJJJJEEEEF----
|
|
MP_BC_BASE_BYTE_O = (0x50) # LLLLSSDTTTTTEEFF
|
|
MP_BC_BASE_BYTE_E = (0x60) # --BREEEYYI------
|
|
MP_BC_LOAD_CONST_SMALL_INT_MULTI = (0x70) # LLLLLLLLLLLLLLLL
|
|
# = (0x80) # LLLLLLLLLLLLLLLL
|
|
# = (0x90) # LLLLLLLLLLLLLLLL
|
|
# = (0xa0) # LLLLLLLLLLLLLLLL
|
|
MP_BC_LOAD_FAST_MULTI = (0xb0) # LLLLLLLLLLLLLLLL
|
|
MP_BC_STORE_FAST_MULTI = (0xc0) # SSSSSSSSSSSSSSSS
|
|
MP_BC_UNARY_OP_MULTI = (0xd0) # OOOOOOO
|
|
MP_BC_BINARY_OP_MULTI = (0xd7) # OOOOOOOOO
|
|
# = (0xe0) # OOOOOOOOOOOOOOOO
|
|
# = (0xf0) # OOOOOOOOOO------
|
|
|
|
MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM = 64
|
|
MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS = 16
|
|
MP_BC_LOAD_FAST_MULTI_NUM = 16
|
|
MP_BC_STORE_FAST_MULTI_NUM = 16
|
|
MP_BC_UNARY_OP_MULTI_NUM = 4 # MP_UNARY_OP_NUM_BYTECODE
|
|
MP_BC_BINARY_OP_MULTI_NUM = 35 # MP_BINARY_OP_NUM_BYTECODE
|
|
|
|
MP_BC_LOAD_CONST_FALSE = (MP_BC_BASE_BYTE_O + 0x00)
|
|
MP_BC_LOAD_CONST_NONE = (MP_BC_BASE_BYTE_O + 0x01)
|
|
MP_BC_LOAD_CONST_TRUE = (MP_BC_BASE_BYTE_O + 0x02)
|
|
MP_BC_LOAD_CONST_SMALL_INT = (MP_BC_BASE_VINT_E + 0x02) # signed var-int
|
|
MP_BC_LOAD_CONST_STRING = (MP_BC_BASE_QSTR_O + 0x00) # qstr
|
|
MP_BC_LOAD_CONST_OBJ = (MP_BC_BASE_VINT_E + 0x03) # ptr
|
|
MP_BC_LOAD_NULL = (MP_BC_BASE_BYTE_O + 0x03)
|
|
|
|
MP_BC_LOAD_FAST_N = (MP_BC_BASE_VINT_E + 0x04) # uint
|
|
MP_BC_LOAD_DEREF = (MP_BC_BASE_VINT_E + 0x05) # uint
|
|
MP_BC_LOAD_NAME = (MP_BC_BASE_QSTR_O + 0x01) # qstr
|
|
MP_BC_LOAD_GLOBAL = (MP_BC_BASE_QSTR_O + 0x02) # qstr
|
|
MP_BC_LOAD_ATTR = (MP_BC_BASE_QSTR_O + 0x03) # qstr
|
|
MP_BC_LOAD_METHOD = (MP_BC_BASE_QSTR_O + 0x04) # qstr
|
|
MP_BC_LOAD_SUPER_METHOD = (MP_BC_BASE_QSTR_O + 0x05) # qstr
|
|
MP_BC_LOAD_BUILD_CLASS = (MP_BC_BASE_BYTE_O + 0x04)
|
|
MP_BC_LOAD_SUBSCR = (MP_BC_BASE_BYTE_O + 0x05)
|
|
|
|
MP_BC_STORE_FAST_N = (MP_BC_BASE_VINT_E + 0x06) # uint
|
|
MP_BC_STORE_DEREF = (MP_BC_BASE_VINT_E + 0x07) # uint
|
|
MP_BC_STORE_NAME = (MP_BC_BASE_QSTR_O + 0x06) # qstr
|
|
MP_BC_STORE_GLOBAL = (MP_BC_BASE_QSTR_O + 0x07) # qstr
|
|
MP_BC_STORE_ATTR = (MP_BC_BASE_QSTR_O + 0x08) # qstr
|
|
MP_BC_STORE_SUBSCR = (MP_BC_BASE_BYTE_O + 0x06)
|
|
|
|
MP_BC_DELETE_FAST = (MP_BC_BASE_VINT_E + 0x08) # uint
|
|
MP_BC_DELETE_DEREF = (MP_BC_BASE_VINT_E + 0x09) # uint
|
|
MP_BC_DELETE_NAME = (MP_BC_BASE_QSTR_O + 0x09) # qstr
|
|
MP_BC_DELETE_GLOBAL = (MP_BC_BASE_QSTR_O + 0x0a) # qstr
|
|
|
|
MP_BC_DUP_TOP = (MP_BC_BASE_BYTE_O + 0x07)
|
|
MP_BC_DUP_TOP_TWO = (MP_BC_BASE_BYTE_O + 0x08)
|
|
MP_BC_POP_TOP = (MP_BC_BASE_BYTE_O + 0x09)
|
|
MP_BC_ROT_TWO = (MP_BC_BASE_BYTE_O + 0x0a)
|
|
MP_BC_ROT_THREE = (MP_BC_BASE_BYTE_O + 0x0b)
|
|
|
|
MP_BC_UNWIND_JUMP = (MP_BC_BASE_JUMP_E + 0x00) # signed relative bytecode offset; then a byte
|
|
MP_BC_JUMP = (MP_BC_BASE_JUMP_E + 0x02) # signed relative bytecode offset
|
|
MP_BC_POP_JUMP_IF_TRUE = (MP_BC_BASE_JUMP_E + 0x03) # signed relative bytecode offset
|
|
MP_BC_POP_JUMP_IF_FALSE = (MP_BC_BASE_JUMP_E + 0x04) # signed relative bytecode offset
|
|
MP_BC_JUMP_IF_TRUE_OR_POP = (MP_BC_BASE_JUMP_E + 0x05) # unsigned relative bytecode offset
|
|
MP_BC_JUMP_IF_FALSE_OR_POP = (MP_BC_BASE_JUMP_E + 0x06) # unsigned relative bytecode offset
|
|
MP_BC_SETUP_WITH = (MP_BC_BASE_JUMP_E + 0x07) # unsigned relative bytecode offset
|
|
MP_BC_SETUP_EXCEPT = (MP_BC_BASE_JUMP_E + 0x08) # unsigned relative bytecode offset
|
|
MP_BC_SETUP_FINALLY = (MP_BC_BASE_JUMP_E + 0x09) # unsigned relative bytecode offset
|
|
MP_BC_POP_EXCEPT_JUMP = (MP_BC_BASE_JUMP_E + 0x0a) # unsigned relative bytecode offset
|
|
MP_BC_FOR_ITER = (MP_BC_BASE_JUMP_E + 0x0b) # unsigned relative bytecode offset
|
|
MP_BC_WITH_CLEANUP = (MP_BC_BASE_BYTE_O + 0x0c)
|
|
MP_BC_END_FINALLY = (MP_BC_BASE_BYTE_O + 0x0d)
|
|
MP_BC_GET_ITER = (MP_BC_BASE_BYTE_O + 0x0e)
|
|
MP_BC_GET_ITER_STACK = (MP_BC_BASE_BYTE_O + 0x0f)
|
|
|
|
MP_BC_BUILD_TUPLE = (MP_BC_BASE_VINT_E + 0x0a) # uint
|
|
MP_BC_BUILD_LIST = (MP_BC_BASE_VINT_E + 0x0b) # uint
|
|
MP_BC_BUILD_MAP = (MP_BC_BASE_VINT_E + 0x0c) # uint
|
|
MP_BC_STORE_MAP = (MP_BC_BASE_BYTE_E + 0x02)
|
|
MP_BC_BUILD_SET = (MP_BC_BASE_VINT_E + 0x0d) # uint
|
|
MP_BC_BUILD_SLICE = (MP_BC_BASE_VINT_E + 0x0e) # uint
|
|
MP_BC_STORE_COMP = (MP_BC_BASE_VINT_E + 0x0f) # uint
|
|
MP_BC_UNPACK_SEQUENCE = (MP_BC_BASE_VINT_O + 0x00) # uint
|
|
MP_BC_UNPACK_EX = (MP_BC_BASE_VINT_O + 0x01) # uint
|
|
|
|
MP_BC_RETURN_VALUE = (MP_BC_BASE_BYTE_E + 0x03)
|
|
MP_BC_RAISE_LAST = (MP_BC_BASE_BYTE_E + 0x04)
|
|
MP_BC_RAISE_OBJ = (MP_BC_BASE_BYTE_E + 0x05)
|
|
MP_BC_RAISE_FROM = (MP_BC_BASE_BYTE_E + 0x06)
|
|
MP_BC_YIELD_VALUE = (MP_BC_BASE_BYTE_E + 0x07)
|
|
MP_BC_YIELD_FROM = (MP_BC_BASE_BYTE_E + 0x08)
|
|
|
|
MP_BC_MAKE_FUNCTION = (MP_BC_BASE_VINT_O + 0x02) # uint
|
|
MP_BC_MAKE_FUNCTION_DEFARGS = (MP_BC_BASE_VINT_O + 0x03) # uint
|
|
MP_BC_MAKE_CLOSURE = (MP_BC_BASE_VINT_E + 0x00) # uint; extra byte
|
|
MP_BC_MAKE_CLOSURE_DEFARGS = (MP_BC_BASE_VINT_E + 0x01) # uint; extra byte
|
|
MP_BC_CALL_FUNCTION = (MP_BC_BASE_VINT_O + 0x04) # uint
|
|
MP_BC_CALL_FUNCTION_VAR_KW = (MP_BC_BASE_VINT_O + 0x05) # uint
|
|
MP_BC_CALL_METHOD = (MP_BC_BASE_VINT_O + 0x06) # uint
|
|
MP_BC_CALL_METHOD_VAR_KW = (MP_BC_BASE_VINT_O + 0x07) # uint
|
|
|
|
MP_BC_IMPORT_NAME = (MP_BC_BASE_QSTR_O + 0x0b) # qstr
|
|
MP_BC_IMPORT_FROM = (MP_BC_BASE_QSTR_O + 0x0c) # qstr
|
|
MP_BC_IMPORT_STAR = (MP_BC_BASE_BYTE_E + 0x09)
|
|
# fmt: on
|
|
|
|
# Create sets of related opcodes.
|
|
ALL_OFFSET_SIGNED = (
|
|
MP_BC_UNWIND_JUMP,
|
|
MP_BC_JUMP,
|
|
MP_BC_POP_JUMP_IF_TRUE,
|
|
MP_BC_POP_JUMP_IF_FALSE,
|
|
)
|
|
|
|
# Create a dict mapping opcode value to opcode name.
|
|
mapping = ["unknown" for _ in range(256)]
|
|
for op_name in list(locals()):
|
|
if op_name.startswith("MP_BC_"):
|
|
mapping[locals()[op_name]] = op_name[len("MP_BC_") :]
|
|
for i in range(MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM):
|
|
name = "LOAD_CONST_SMALL_INT %d" % (i - MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS)
|
|
mapping[MP_BC_LOAD_CONST_SMALL_INT_MULTI + i] = name
|
|
for i in range(MP_BC_LOAD_FAST_MULTI_NUM):
|
|
mapping[MP_BC_LOAD_FAST_MULTI + i] = "LOAD_FAST %d" % i
|
|
for i in range(MP_BC_STORE_FAST_MULTI_NUM):
|
|
mapping[MP_BC_STORE_FAST_MULTI + i] = "STORE_FAST %d" % i
|
|
for i in range(MP_BC_UNARY_OP_MULTI_NUM):
|
|
mapping[MP_BC_UNARY_OP_MULTI + i] = "UNARY_OP %d %s" % (i, mp_unary_op_method_name[i])
|
|
for i in range(MP_BC_BINARY_OP_MULTI_NUM):
|
|
mapping[MP_BC_BINARY_OP_MULTI + i] = "BINARY_OP %d %s" % (i, mp_binary_op_method_name[i])
|
|
|
|
def __init__(self, offset, fmt, opcode_byte, arg, extra_arg):
|
|
self.offset = offset
|
|
self.fmt = fmt
|
|
self.opcode_byte = opcode_byte
|
|
self.arg = arg
|
|
self.extra_arg = extra_arg
|
|
|
|
|
|
# This definition of a small int covers all possible targets, in the sense that every
|
|
# target can encode as a small int, an integer that passes this test. The minimum is set
|
|
# by MICROPY_OBJ_REPR_B on a 16-bit machine, where there are 14 bits for the small int.
|
|
def mp_small_int_fits(i):
|
|
return -0x2000 <= i <= 0x1FFF
|
|
|
|
|
|
def mp_encode_uint(val, signed=False):
|
|
encoded = bytearray([val & 0x7F])
|
|
val >>= 7
|
|
while val != 0 and val != -1:
|
|
encoded.insert(0, 0x80 | (val & 0x7F))
|
|
val >>= 7
|
|
if signed:
|
|
if val == -1 and encoded[0] & 0x40 == 0:
|
|
encoded.insert(0, 0xFF)
|
|
elif val == 0 and encoded[0] & 0x40 != 0:
|
|
encoded.insert(0, 0x80)
|
|
return encoded
|
|
|
|
|
|
def mp_opcode_decode(bytecode, ip):
|
|
opcode = bytecode[ip]
|
|
ip_start = ip
|
|
f = (0x000003A4 >> (2 * ((opcode) >> 4))) & 3
|
|
ip += 1
|
|
arg = None
|
|
extra_arg = None
|
|
if f in (MP_BC_FORMAT_QSTR, MP_BC_FORMAT_VAR_UINT):
|
|
arg = bytecode[ip] & 0x7F
|
|
if opcode == Opcode.MP_BC_LOAD_CONST_SMALL_INT and arg & 0x40 != 0:
|
|
arg |= -1 << 7
|
|
while bytecode[ip] & 0x80 != 0:
|
|
ip += 1
|
|
arg = arg << 7 | bytecode[ip] & 0x7F
|
|
ip += 1
|
|
elif f == MP_BC_FORMAT_OFFSET:
|
|
if bytecode[ip] & 0x80 == 0:
|
|
arg = bytecode[ip]
|
|
ip += 1
|
|
if opcode in Opcode.ALL_OFFSET_SIGNED:
|
|
arg -= 0x40
|
|
else:
|
|
arg = bytecode[ip] & 0x7F | bytecode[ip + 1] << 7
|
|
ip += 2
|
|
if opcode in Opcode.ALL_OFFSET_SIGNED:
|
|
arg -= 0x4000
|
|
if opcode & MP_BC_MASK_EXTRA_BYTE == 0:
|
|
extra_arg = bytecode[ip]
|
|
ip += 1
|
|
return f, ip - ip_start, arg, extra_arg
|
|
|
|
|
|
def mp_opcode_encode(opcode):
|
|
overflow = False
|
|
encoded = bytearray([opcode.opcode_byte])
|
|
if opcode.fmt in (MP_BC_FORMAT_QSTR, MP_BC_FORMAT_VAR_UINT):
|
|
signed = opcode.opcode_byte == Opcode.MP_BC_LOAD_CONST_SMALL_INT
|
|
encoded.extend(mp_encode_uint(opcode.arg, signed))
|
|
elif opcode.fmt == MP_BC_FORMAT_OFFSET:
|
|
is_signed = opcode.opcode_byte in Opcode.ALL_OFFSET_SIGNED
|
|
|
|
# The -2 accounts for this jump opcode taking 2 bytes (at least).
|
|
bytecode_offset = opcode.target.offset - opcode.offset - 2
|
|
|
|
# Check if the bytecode_offset is small enough to use a 1-byte encoding.
|
|
if (is_signed and -64 <= bytecode_offset <= 63) or (
|
|
not is_signed and bytecode_offset <= 127
|
|
):
|
|
# Use a 1-byte jump offset.
|
|
if is_signed:
|
|
bytecode_offset += 0x40
|
|
overflow = not (0 <= bytecode_offset <= 0x7F)
|
|
encoded.append(bytecode_offset & 0x7F)
|
|
else:
|
|
bytecode_offset -= 1
|
|
if is_signed:
|
|
bytecode_offset += 0x4000
|
|
overflow = not (0 <= bytecode_offset <= 0x7FFF)
|
|
encoded.append(0x80 | (bytecode_offset & 0x7F))
|
|
encoded.append((bytecode_offset >> 7) & 0xFF)
|
|
if opcode.extra_arg is not None:
|
|
encoded.append(opcode.extra_arg)
|
|
return overflow, encoded
|
|
|
|
|
|
def read_prelude_sig(read_byte):
|
|
z = read_byte()
|
|
# xSSSSEAA
|
|
S = (z >> 3) & 0xF
|
|
E = (z >> 2) & 0x1
|
|
F = 0
|
|
A = z & 0x3
|
|
K = 0
|
|
D = 0
|
|
n = 0
|
|
while z & 0x80:
|
|
z = read_byte()
|
|
# xFSSKAED
|
|
S |= (z & 0x30) << (2 * n)
|
|
E |= (z & 0x02) << n
|
|
F |= ((z & 0x40) >> 6) << n
|
|
A |= (z & 0x4) << n
|
|
K |= ((z & 0x08) >> 3) << n
|
|
D |= (z & 0x1) << n
|
|
n += 1
|
|
S += 1
|
|
return S, E, F, A, K, D
|
|
|
|
|
|
def read_prelude_size(read_byte):
|
|
I = 0
|
|
C = 0
|
|
n = 0
|
|
while True:
|
|
z = read_byte()
|
|
# xIIIIIIC
|
|
I |= ((z & 0x7E) >> 1) << (6 * n)
|
|
C |= (z & 1) << n
|
|
if not (z & 0x80):
|
|
break
|
|
n += 1
|
|
return I, C
|
|
|
|
|
|
# See py/bc.h:MP_BC_PRELUDE_SIZE_ENCODE macro.
|
|
def encode_prelude_size(I, C):
|
|
# Encode bit-wise as: xIIIIIIC
|
|
encoded = bytearray()
|
|
while True:
|
|
z = (I & 0x3F) << 1 | (C & 1)
|
|
C >>= 1
|
|
I >>= 6
|
|
if C | I:
|
|
z |= 0x80
|
|
encoded.append(z)
|
|
if not C | I:
|
|
return encoded
|
|
|
|
|
|
def extract_prelude(bytecode, ip):
|
|
def local_read_byte():
|
|
b = bytecode[ip_ref[0]]
|
|
ip_ref[0] += 1
|
|
return b
|
|
|
|
ip_ref = [ip] # to close over ip in Python 2 and 3
|
|
|
|
# Read prelude signature.
|
|
(
|
|
n_state,
|
|
n_exc_stack,
|
|
scope_flags,
|
|
n_pos_args,
|
|
n_kwonly_args,
|
|
n_def_pos_args,
|
|
) = read_prelude_sig(local_read_byte)
|
|
|
|
offset_prelude_size = ip_ref[0]
|
|
|
|
# Read prelude size.
|
|
n_info, n_cell = read_prelude_size(local_read_byte)
|
|
|
|
offset_source_info = ip_ref[0]
|
|
|
|
# Extract simple_name and argument qstrs (var uints).
|
|
args = []
|
|
for arg_num in range(1 + n_pos_args + n_kwonly_args):
|
|
value = 0
|
|
while True:
|
|
b = local_read_byte()
|
|
value = (value << 7) | (b & 0x7F)
|
|
if b & 0x80 == 0:
|
|
break
|
|
args.append(value)
|
|
|
|
offset_line_info = ip_ref[0]
|
|
offset_closure_info = offset_source_info + n_info
|
|
offset_opcodes = offset_source_info + n_info + n_cell
|
|
|
|
return (
|
|
offset_prelude_size,
|
|
offset_source_info,
|
|
offset_line_info,
|
|
offset_closure_info,
|
|
offset_opcodes,
|
|
(n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args),
|
|
(n_info, n_cell),
|
|
args,
|
|
)
|
|
|
|
|
|
class QStrType:
|
|
def __init__(self, str):
|
|
self.str = str
|
|
self.qstr_esc = qstrutil.qstr_escape(self.str)
|
|
self.qstr_id = "MP_QSTR_" + self.qstr_esc
|
|
|
|
|
|
class GlobalQStrList:
|
|
def __init__(self):
|
|
# Initialise global list of qstrs with static qstrs
|
|
self.qstrs = [None] # MP_QSTRnull should never be referenced
|
|
for n in qstrutil.static_qstr_list:
|
|
self.qstrs.append(QStrType(n))
|
|
|
|
def add(self, s):
|
|
q = QStrType(s)
|
|
self.qstrs.append(q)
|
|
return q
|
|
|
|
def get_by_index(self, i):
|
|
return self.qstrs[i]
|
|
|
|
def find_by_str(self, s):
|
|
for q in self.qstrs:
|
|
if q is not None and q.str == s:
|
|
return q
|
|
return None
|
|
|
|
|
|
class MPFunTable:
|
|
def __repr__(self):
|
|
return "mp_fun_table"
|
|
|
|
|
|
class CompiledModule:
|
|
def __init__(
|
|
self,
|
|
mpy_source_file,
|
|
mpy_segments,
|
|
header,
|
|
qstr_table,
|
|
obj_table,
|
|
raw_code,
|
|
qstr_table_file_offset,
|
|
obj_table_file_offset,
|
|
raw_code_file_offset,
|
|
escaped_name,
|
|
):
|
|
self.mpy_source_file = mpy_source_file
|
|
self.mpy_segments = mpy_segments
|
|
self.source_file = qstr_table[0]
|
|
self.header = header
|
|
self.qstr_table = qstr_table
|
|
self.obj_table = obj_table
|
|
self.raw_code = raw_code
|
|
self.qstr_table_file_offset = qstr_table_file_offset
|
|
self.obj_table_file_offset = obj_table_file_offset
|
|
self.raw_code_file_offset = raw_code_file_offset
|
|
self.escaped_name = escaped_name
|
|
|
|
def hexdump(self):
|
|
with open(self.mpy_source_file, "rb") as f:
|
|
WIDTH = 16
|
|
COL_OFF = "\033[0m"
|
|
COL_TABLE = (
|
|
("", ""), # META
|
|
("\033[0;31m", "\033[0;91m"), # QSTR
|
|
("\033[0;32m", "\033[0;92m"), # OBJ
|
|
("\033[0;34m", "\033[0;94m"), # CODE
|
|
)
|
|
cur_col = ""
|
|
cur_col_index = 0
|
|
offset = 0
|
|
segment_index = 0
|
|
while True:
|
|
data = bytes_cons(f.read(WIDTH))
|
|
if not data:
|
|
break
|
|
|
|
# Print out the hex dump of this line of data.
|
|
line_hex = cur_col
|
|
line_chr = cur_col
|
|
line_comment = ""
|
|
for i in range(len(data)):
|
|
# Determine the colour of the data, if any, and the line comment.
|
|
while segment_index < len(self.mpy_segments):
|
|
if offset + i == self.mpy_segments[segment_index].start:
|
|
cur_col = COL_TABLE[self.mpy_segments[segment_index].kind][
|
|
cur_col_index
|
|
]
|
|
cur_col_index = 1 - cur_col_index
|
|
line_hex += cur_col
|
|
line_chr += cur_col
|
|
line_comment += " %s%s%s" % (
|
|
cur_col,
|
|
self.mpy_segments[segment_index].name,
|
|
COL_OFF,
|
|
)
|
|
if offset + i == self.mpy_segments[segment_index].end:
|
|
cur_col = ""
|
|
line_hex += COL_OFF
|
|
line_chr += COL_OFF
|
|
segment_index += 1
|
|
else:
|
|
break
|
|
|
|
# Add to the hex part of the line.
|
|
if i % 2 == 0:
|
|
line_hex += " "
|
|
line_hex += "%02x" % data[i]
|
|
|
|
# Add to the characters part of the line.
|
|
if 0x20 <= data[i] <= 0x7E:
|
|
line_chr += "%s" % chr(data[i])
|
|
else:
|
|
line_chr += "."
|
|
|
|
# Print out this line.
|
|
if cur_col:
|
|
line_hex += COL_OFF
|
|
line_chr += COL_OFF
|
|
pad = " " * ((WIDTH - len(data)) * 5 // 2)
|
|
print("%08x:%s%s %s %s" % (offset, line_hex, pad, line_chr, line_comment))
|
|
offset += WIDTH
|
|
|
|
def disassemble(self):
|
|
print("mpy_source_file:", self.mpy_source_file)
|
|
print("source_file:", self.source_file.str)
|
|
print("header:", hexlify_to_str(self.header))
|
|
print("qstr_table[%u]:" % len(self.qstr_table))
|
|
for q in self.qstr_table:
|
|
print(" %s" % q.str)
|
|
print("obj_table:", self.obj_table)
|
|
self.raw_code.disassemble()
|
|
|
|
def freeze(self, compiled_module_index):
|
|
print()
|
|
print("/" * 80)
|
|
print("// frozen module %s" % self.escaped_name)
|
|
print("// - original source file: %s" % self.mpy_source_file)
|
|
print("// - frozen file name: %s" % self.source_file.str)
|
|
print("// - .mpy header: %s" % ":".join("%02x" % b for b in self.header))
|
|
print()
|
|
|
|
self.raw_code.freeze()
|
|
print()
|
|
|
|
self.freeze_constants()
|
|
|
|
print()
|
|
print("static const mp_frozen_module_t frozen_module_%s = {" % self.escaped_name)
|
|
print(" .constants = {")
|
|
if len(self.qstr_table):
|
|
print(
|
|
" .qstr_table = (qstr_short_t *)&const_qstr_table_data_%s,"
|
|
% self.escaped_name
|
|
)
|
|
else:
|
|
print(" .qstr_table = NULL,")
|
|
if len(self.obj_table):
|
|
print(" .obj_table = (mp_obj_t *)&const_obj_table_data_%s," % self.escaped_name)
|
|
else:
|
|
print(" .obj_table = NULL,")
|
|
print(" },")
|
|
print(" .rc = &raw_code_%s," % self.raw_code.escaped_name)
|
|
print("};")
|
|
|
|
def freeze_constant_obj(self, obj_name, obj):
|
|
global const_str_content, const_int_content, const_obj_content
|
|
|
|
if isinstance(obj, MPFunTable):
|
|
return "&mp_fun_table"
|
|
elif obj is None:
|
|
return "MP_ROM_NONE"
|
|
elif obj is False:
|
|
return "MP_ROM_FALSE"
|
|
elif obj is True:
|
|
return "MP_ROM_TRUE"
|
|
elif obj is Ellipsis:
|
|
return "MP_ROM_PTR(&mp_const_ellipsis_obj)"
|
|
elif is_str_type(obj) or is_bytes_type(obj):
|
|
if len(obj) == 0:
|
|
if is_str_type(obj):
|
|
return "MP_ROM_QSTR(MP_QSTR_)"
|
|
else:
|
|
return "MP_ROM_PTR(&mp_const_empty_bytes_obj)"
|
|
if is_str_type(obj):
|
|
q = global_qstrs.find_by_str(obj)
|
|
if q:
|
|
return "MP_ROM_QSTR(%s)" % q.qstr_id
|
|
obj = bytes_cons(obj, "utf8")
|
|
obj_type = "mp_type_str"
|
|
else:
|
|
obj_type = "mp_type_bytes"
|
|
print(
|
|
'static const mp_obj_str_t %s = {{&%s}, %u, %u, (const byte*)"%s"};'
|
|
% (
|
|
obj_name,
|
|
obj_type,
|
|
qstrutil.compute_hash(obj, config.MICROPY_QSTR_BYTES_IN_HASH),
|
|
len(obj),
|
|
"".join(("\\x%02x" % b) for b in obj),
|
|
)
|
|
)
|
|
const_str_content += len(obj)
|
|
const_obj_content += 4 * 4
|
|
return "MP_ROM_PTR(&%s)" % obj_name
|
|
elif is_int_type(obj):
|
|
if mp_small_int_fits(obj):
|
|
# Encode directly as a small integer object.
|
|
return "MP_ROM_INT(%d)" % obj
|
|
elif config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_NONE:
|
|
raise FreezeError(self, "target does not support long int")
|
|
elif config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_LONGLONG:
|
|
# TODO
|
|
raise FreezeError(self, "freezing int to long-long is not implemented")
|
|
elif config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_MPZ:
|
|
neg = 0
|
|
if obj < 0:
|
|
obj = -obj
|
|
neg = 1
|
|
bits_per_dig = config.MPZ_DIG_SIZE
|
|
digs = []
|
|
z = obj
|
|
while z:
|
|
digs.append(z & ((1 << bits_per_dig) - 1))
|
|
z >>= bits_per_dig
|
|
ndigs = len(digs)
|
|
digs = ",".join(("%#x" % d) for d in digs)
|
|
print(
|
|
"static const mp_obj_int_t %s = {{&mp_type_int}, "
|
|
"{.neg=%u, .fixed_dig=1, .alloc=%u, .len=%u, .dig=(uint%u_t*)(const uint%u_t[]){%s}}};"
|
|
% (obj_name, neg, ndigs, ndigs, bits_per_dig, bits_per_dig, digs)
|
|
)
|
|
const_int_content += (digs.count(",") + 1) * bits_per_dig // 8
|
|
const_obj_content += 4 * 4
|
|
return "MP_ROM_PTR(&%s)" % obj_name
|
|
elif isinstance(obj, float):
|
|
macro_name = "%s_macro" % obj_name
|
|
print(
|
|
"#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B"
|
|
)
|
|
print(
|
|
"static const mp_obj_float_t %s = {{&mp_type_float}, (mp_float_t)%.16g};"
|
|
% (obj_name, obj)
|
|
)
|
|
print("#define %s MP_ROM_PTR(&%s)" % (macro_name, obj_name))
|
|
print("#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C")
|
|
n = struct.unpack("<I", struct.pack("<f", obj))[0]
|
|
n = ((n & ~0x3) | 2) + 0x80800000
|
|
print("#define %s ((mp_rom_obj_t)(0x%08x))" % (macro_name, n))
|
|
print("#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D")
|
|
n = struct.unpack("<Q", struct.pack("<d", obj))[0]
|
|
n += 0x8004000000000000
|
|
print("#define %s ((mp_rom_obj_t)(0x%016x))" % (macro_name, n))
|
|
print("#endif")
|
|
const_obj_content += 3 * 4
|
|
return macro_name
|
|
elif isinstance(obj, complex):
|
|
print(
|
|
"static const mp_obj_complex_t %s = {{&mp_type_complex}, (mp_float_t)%.16g, (mp_float_t)%.16g};"
|
|
% (obj_name, obj.real, obj.imag)
|
|
)
|
|
return "MP_ROM_PTR(&%s)" % obj_name
|
|
elif type(obj) is tuple:
|
|
if len(obj) == 0:
|
|
return "MP_ROM_PTR(&mp_const_empty_tuple_obj)"
|
|
else:
|
|
obj_refs = []
|
|
for i, sub_obj in enumerate(obj):
|
|
sub_obj_name = "%s_%u" % (obj_name, i)
|
|
obj_refs.append(self.freeze_constant_obj(sub_obj_name, sub_obj))
|
|
print(
|
|
"static const mp_rom_obj_tuple_t %s = {{&mp_type_tuple}, %d, {"
|
|
% (obj_name, len(obj))
|
|
)
|
|
for ref in obj_refs:
|
|
print(" %s," % ref)
|
|
print("}};")
|
|
return "MP_ROM_PTR(&%s)" % obj_name
|
|
else:
|
|
raise FreezeError(self, "freezing of object %r is not implemented" % (obj,))
|
|
|
|
def freeze_constants(self):
|
|
if len(self.qstr_table):
|
|
print(
|
|
"static const qstr_short_t const_qstr_table_data_%s[%u] = {"
|
|
% (self.escaped_name, len(self.qstr_table))
|
|
)
|
|
for q in self.qstr_table:
|
|
print(" %s," % q.qstr_id)
|
|
print("};")
|
|
|
|
if not len(self.obj_table):
|
|
return
|
|
|
|
# generate constant objects
|
|
print()
|
|
print("// constants")
|
|
obj_refs = []
|
|
for i, obj in enumerate(self.obj_table):
|
|
obj_name = "const_obj_%s_%u" % (self.escaped_name, i)
|
|
obj_refs.append(self.freeze_constant_obj(obj_name, obj))
|
|
|
|
# generate constant table
|
|
print()
|
|
print("// constant table")
|
|
print(
|
|
"static const mp_rom_obj_t const_obj_table_data_%s[%u] = {"
|
|
% (self.escaped_name, len(self.obj_table))
|
|
)
|
|
for ref in obj_refs:
|
|
print(" %s," % ref)
|
|
print("};")
|
|
|
|
global const_table_ptr_content
|
|
const_table_ptr_content += len(self.obj_table)
|
|
|
|
|
|
class RawCode(object):
|
|
# a set of all escaped names, to make sure they are unique
|
|
escaped_names = set()
|
|
|
|
# convert code kind number to string
|
|
code_kind_str = {
|
|
MP_CODE_BYTECODE: "MP_CODE_BYTECODE",
|
|
MP_CODE_NATIVE_PY: "MP_CODE_NATIVE_PY",
|
|
MP_CODE_NATIVE_VIPER: "MP_CODE_NATIVE_VIPER",
|
|
MP_CODE_NATIVE_ASM: "MP_CODE_NATIVE_ASM",
|
|
}
|
|
|
|
def __init__(self, parent_name, qstr_table, fun_data, prelude_offset, code_kind):
|
|
self.qstr_table = qstr_table
|
|
self.fun_data = fun_data
|
|
self.prelude_offset = prelude_offset
|
|
self.code_kind = code_kind
|
|
|
|
if code_kind in (MP_CODE_BYTECODE, MP_CODE_NATIVE_PY):
|
|
(
|
|
self.offset_prelude_size,
|
|
self.offset_source_info,
|
|
self.offset_line_info,
|
|
self.offset_closure_info,
|
|
self.offset_opcodes,
|
|
self.prelude_signature,
|
|
self.prelude_size,
|
|
self.names,
|
|
) = extract_prelude(self.fun_data, prelude_offset)
|
|
self.scope_flags = self.prelude_signature[2]
|
|
self.n_pos_args = self.prelude_signature[3]
|
|
self.simple_name = self.qstr_table[self.names[0]]
|
|
else:
|
|
self.simple_name = self.qstr_table[0]
|
|
|
|
escaped_name = parent_name + "_" + self.simple_name.qstr_esc
|
|
|
|
# make sure the escaped name is unique
|
|
i = 2
|
|
unique_escaped_name = escaped_name
|
|
while unique_escaped_name in self.escaped_names:
|
|
unique_escaped_name = escaped_name + str(i)
|
|
i += 1
|
|
self.escaped_names.add(unique_escaped_name)
|
|
self.escaped_name = unique_escaped_name
|
|
|
|
def disassemble_children(self):
|
|
print(" children:", [rc.simple_name.str for rc in self.children])
|
|
for rc in self.children:
|
|
rc.disassemble()
|
|
|
|
def freeze_children(self, prelude_ptr=None):
|
|
# Freeze children and generate table of children.
|
|
if len(self.children):
|
|
for rc in self.children:
|
|
print("// child of %s" % self.escaped_name)
|
|
rc.freeze()
|
|
print()
|
|
print("static const mp_raw_code_t *const children_%s[] = {" % self.escaped_name)
|
|
for rc in self.children:
|
|
print(" &raw_code_%s," % rc.escaped_name)
|
|
if prelude_ptr:
|
|
print(" (void *)%s," % prelude_ptr)
|
|
print("};")
|
|
print()
|
|
|
|
def freeze_raw_code(self, prelude_ptr=None, type_sig=0):
|
|
# Generate mp_raw_code_t.
|
|
print("static const mp_raw_code_t raw_code_%s = {" % self.escaped_name)
|
|
print(" .kind = %s," % RawCode.code_kind_str[self.code_kind])
|
|
print(" .scope_flags = 0x%02x," % self.scope_flags)
|
|
print(" .n_pos_args = %u," % self.n_pos_args)
|
|
print(" .fun_data = fun_data_%s," % self.escaped_name)
|
|
print(" #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS")
|
|
print(" .fun_data_len = %u," % len(self.fun_data))
|
|
print(" #endif")
|
|
if len(self.children):
|
|
print(" .children = (void *)&children_%s," % self.escaped_name)
|
|
elif prelude_ptr:
|
|
print(" .children = (void *)%s," % prelude_ptr)
|
|
else:
|
|
print(" .children = NULL,")
|
|
print(" #if MICROPY_PERSISTENT_CODE_SAVE")
|
|
print(" .n_children = %u," % len(self.children))
|
|
if self.code_kind == MP_CODE_BYTECODE:
|
|
print(" #if MICROPY_PY_SYS_SETTRACE")
|
|
print(" .prelude = {")
|
|
print(" .n_state = %u," % self.prelude_signature[0])
|
|
print(" .n_exc_stack = %u," % self.prelude_signature[1])
|
|
print(" .scope_flags = %u," % self.prelude_signature[2])
|
|
print(" .n_pos_args = %u," % self.prelude_signature[3])
|
|
print(" .n_kwonly_args = %u," % self.prelude_signature[4])
|
|
print(" .n_def_pos_args = %u," % self.prelude_signature[5])
|
|
print(" .qstr_block_name_idx = %u," % self.names[0])
|
|
print(
|
|
" .line_info = fun_data_%s + %u,"
|
|
% (self.escaped_name, self.offset_line_info)
|
|
)
|
|
print(
|
|
" .line_info_top = fun_data_%s + %u,"
|
|
% (self.escaped_name, self.offset_closure_info)
|
|
)
|
|
print(
|
|
" .opcodes = fun_data_%s + %u," % (self.escaped_name, self.offset_opcodes)
|
|
)
|
|
print(" },")
|
|
print(" .line_of_definition = %u," % 0) # TODO
|
|
print(" #endif")
|
|
print(" #if MICROPY_EMIT_MACHINE_CODE")
|
|
print(" .prelude_offset = %u," % self.prelude_offset)
|
|
print(" #endif")
|
|
print(" #endif")
|
|
print(" #if MICROPY_EMIT_MACHINE_CODE")
|
|
print(" .type_sig = %u," % type_sig)
|
|
print(" #endif")
|
|
print("};")
|
|
|
|
global raw_code_count, raw_code_content
|
|
raw_code_count += 1
|
|
raw_code_content += 4 * 4
|
|
|
|
|
|
class RawCodeBytecode(RawCode):
|
|
def __init__(self, parent_name, qstr_table, obj_table, fun_data):
|
|
self.obj_table = obj_table
|
|
super(RawCodeBytecode, self).__init__(
|
|
parent_name, qstr_table, fun_data, 0, MP_CODE_BYTECODE
|
|
)
|
|
|
|
def disassemble(self):
|
|
bc = self.fun_data
|
|
print("simple_name:", self.simple_name.str)
|
|
print(" raw bytecode:", len(bc), hexlify_to_str(bc))
|
|
print(" prelude:", self.prelude_signature)
|
|
print(" args:", [self.qstr_table[i].str for i in self.names[1:]])
|
|
print(" line info:", hexlify_to_str(bc[self.offset_line_info : self.offset_opcodes]))
|
|
ip = self.offset_opcodes
|
|
while ip < len(bc):
|
|
fmt, sz, arg, _ = mp_opcode_decode(bc, ip)
|
|
if bc[ip] == Opcode.MP_BC_LOAD_CONST_OBJ:
|
|
arg = repr(self.obj_table[arg])
|
|
if fmt == MP_BC_FORMAT_QSTR:
|
|
arg = self.qstr_table[arg].str
|
|
elif fmt in (MP_BC_FORMAT_VAR_UINT, MP_BC_FORMAT_OFFSET):
|
|
pass
|
|
else:
|
|
arg = ""
|
|
print(
|
|
" %-11s %s %s" % (hexlify_to_str(bc[ip : ip + sz]), Opcode.mapping[bc[ip]], arg)
|
|
)
|
|
ip += sz
|
|
self.disassemble_children()
|
|
|
|
def freeze(self):
|
|
# generate bytecode data
|
|
bc = self.fun_data
|
|
print(
|
|
"// frozen bytecode for file %s, scope %s"
|
|
% (self.qstr_table[0].str, self.escaped_name)
|
|
)
|
|
print("static const byte fun_data_%s[%u] = {" % (self.escaped_name, len(bc)))
|
|
|
|
print(" ", end="")
|
|
for b in bc[: self.offset_source_info]:
|
|
print("0x%02x," % b, end="")
|
|
print(" // prelude")
|
|
|
|
print(" ", end="")
|
|
for b in bc[self.offset_source_info : self.offset_line_info]:
|
|
print("0x%02x," % b, end="")
|
|
print(" // names: %s" % ", ".join(self.qstr_table[i].str for i in self.names))
|
|
|
|
print(" ", end="")
|
|
for b in bc[self.offset_line_info : self.offset_opcodes]:
|
|
print("0x%02x," % b, end="")
|
|
print(" // code info")
|
|
|
|
ip = self.offset_opcodes
|
|
while ip < len(bc):
|
|
fmt, sz, arg, _ = mp_opcode_decode(bc, ip)
|
|
opcode_name = Opcode.mapping[bc[ip]]
|
|
if fmt == MP_BC_FORMAT_QSTR:
|
|
opcode_name += " " + repr(self.qstr_table[arg].str)
|
|
elif fmt in (MP_BC_FORMAT_VAR_UINT, MP_BC_FORMAT_OFFSET):
|
|
opcode_name += " %u" % arg
|
|
print(
|
|
" %s, // %s" % (",".join("0x%02x" % b for b in bc[ip : ip + sz]), opcode_name)
|
|
)
|
|
ip += sz
|
|
|
|
print("};")
|
|
|
|
self.freeze_children()
|
|
self.freeze_raw_code()
|
|
|
|
global bc_content
|
|
bc_content += len(bc)
|
|
|
|
|
|
class RawCodeNative(RawCode):
|
|
def __init__(
|
|
self,
|
|
parent_name,
|
|
qstr_table,
|
|
kind,
|
|
fun_data,
|
|
prelude_offset,
|
|
scope_flags,
|
|
n_pos_args,
|
|
type_sig,
|
|
):
|
|
super(RawCodeNative, self).__init__(
|
|
parent_name, qstr_table, fun_data, prelude_offset, kind
|
|
)
|
|
|
|
if kind in (MP_CODE_NATIVE_VIPER, MP_CODE_NATIVE_ASM):
|
|
self.scope_flags = scope_flags
|
|
self.n_pos_args = n_pos_args
|
|
|
|
self.type_sig = type_sig
|
|
if config.native_arch in (
|
|
MP_NATIVE_ARCH_X86,
|
|
MP_NATIVE_ARCH_X64,
|
|
MP_NATIVE_ARCH_XTENSA,
|
|
MP_NATIVE_ARCH_XTENSAWIN,
|
|
):
|
|
self.fun_data_attributes = '__attribute__((section(".text,\\"ax\\",@progbits # ")))'
|
|
else:
|
|
self.fun_data_attributes = '__attribute__((section(".text,\\"ax\\",%progbits @ ")))'
|
|
|
|
# Allow single-byte alignment by default for x86/x64.
|
|
# ARM needs word alignment, ARM Thumb needs halfword, due to instruction size.
|
|
# Xtensa needs word alignment due to the 32-bit constant table embedded in the code.
|
|
if config.native_arch in (
|
|
MP_NATIVE_ARCH_ARMV6,
|
|
MP_NATIVE_ARCH_XTENSA,
|
|
MP_NATIVE_ARCH_XTENSAWIN,
|
|
):
|
|
# ARMV6 or Xtensa -- four byte align.
|
|
self.fun_data_attributes += " __attribute__ ((aligned (4)))"
|
|
elif MP_NATIVE_ARCH_ARMV6M <= config.native_arch <= MP_NATIVE_ARCH_ARMV7EMDP:
|
|
# ARMVxxM -- two byte align.
|
|
self.fun_data_attributes += " __attribute__ ((aligned (2)))"
|
|
|
|
def disassemble(self):
|
|
fun_data = self.fun_data
|
|
print("simple_name:", self.simple_name.str)
|
|
print(
|
|
" raw data:",
|
|
len(fun_data),
|
|
hexlify_to_str(fun_data[:32]),
|
|
"..." if len(fun_data) > 32 else "",
|
|
)
|
|
if self.code_kind != MP_CODE_NATIVE_PY:
|
|
return
|
|
print(" prelude:", self.prelude_signature)
|
|
print(" args:", [self.qstr_table[i].str for i in self.names[1:]])
|
|
print(" line info:", fun_data[self.offset_line_info : self.offset_opcodes])
|
|
ip = 0
|
|
while ip < self.prelude_offset:
|
|
sz = 16
|
|
print(" ", hexlify_to_str(fun_data[ip : min(ip + sz, self.prelude_offset)]))
|
|
ip += sz
|
|
self.disassemble_children()
|
|
|
|
def freeze(self):
|
|
if self.scope_flags & ~0x0F:
|
|
raise FreezeError("unable to freeze code with relocations")
|
|
|
|
# generate native code data
|
|
print()
|
|
print(
|
|
"// frozen native code for file %s, scope %s"
|
|
% (self.qstr_table[0].str, self.escaped_name)
|
|
)
|
|
print(
|
|
"static const byte fun_data_%s[%u] %s = {"
|
|
% (self.escaped_name, len(self.fun_data), self.fun_data_attributes)
|
|
)
|
|
|
|
i_top = len(self.fun_data)
|
|
i = 0
|
|
while i < i_top:
|
|
# copy machine code (max 16 bytes)
|
|
i16 = min(i + 16, i_top)
|
|
print(" ", end="")
|
|
for ii in range(i, i16):
|
|
print(" 0x%02x," % self.fun_data[ii], end="")
|
|
print()
|
|
i = i16
|
|
|
|
print("};")
|
|
|
|
prelude_ptr = None
|
|
if self.code_kind == MP_CODE_NATIVE_PY:
|
|
prelude_ptr = "fun_data_%s_prelude_macro" % self.escaped_name
|
|
print("#if MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE")
|
|
n = len(self.fun_data) - self.prelude_offset
|
|
print("static const byte fun_data_%s_prelude[%u] = {" % (self.escaped_name, n), end="")
|
|
for i in range(n):
|
|
print(" 0x%02x," % self.fun_data[self.prelude_offset + i], end="")
|
|
print("};")
|
|
print("#define %s &fun_data_%s_prelude[0]" % (prelude_ptr, self.escaped_name))
|
|
print("#else")
|
|
print(
|
|
"#define %s &fun_data_%s[%u]"
|
|
% (prelude_ptr, self.escaped_name, self.prelude_offset)
|
|
)
|
|
print("#endif")
|
|
|
|
self.freeze_children(prelude_ptr)
|
|
self.freeze_raw_code(prelude_ptr, self.type_sig)
|
|
|
|
|
|
class MPYSegment:
|
|
META = 0
|
|
QSTR = 1
|
|
OBJ = 2
|
|
CODE = 3
|
|
|
|
def __init__(self, kind, name, start, end):
|
|
self.kind = kind
|
|
self.name = name
|
|
self.start = start
|
|
self.end = end
|
|
|
|
|
|
class MPYReader:
|
|
def __init__(self, filename, fileobj):
|
|
self.filename = filename
|
|
self.fileobj = fileobj
|
|
|
|
def tell(self):
|
|
return self.fileobj.tell()
|
|
|
|
def read_byte(self):
|
|
return bytes_cons(self.fileobj.read(1))[0]
|
|
|
|
def read_bytes(self, n):
|
|
return bytes_cons(self.fileobj.read(n))
|
|
|
|
def read_uint(self):
|
|
i = 0
|
|
while True:
|
|
b = self.read_byte()
|
|
i = (i << 7) | (b & 0x7F)
|
|
if b & 0x80 == 0:
|
|
break
|
|
return i
|
|
|
|
|
|
def read_qstr(reader, segments):
|
|
start_pos = reader.tell()
|
|
ln = reader.read_uint()
|
|
if ln & 1:
|
|
# static qstr
|
|
q = global_qstrs.get_by_index(ln >> 1)
|
|
segments.append(MPYSegment(MPYSegment.META, q.str, start_pos, start_pos))
|
|
return q
|
|
ln >>= 1
|
|
start_pos = reader.tell()
|
|
data = str_cons(reader.read_bytes(ln), "utf8")
|
|
reader.read_byte() # read and discard null terminator
|
|
segments.append(MPYSegment(MPYSegment.QSTR, data, start_pos, reader.tell()))
|
|
return global_qstrs.add(data)
|
|
|
|
|
|
def read_obj(reader, segments):
|
|
obj_type = reader.read_byte()
|
|
if obj_type == MP_PERSISTENT_OBJ_FUN_TABLE:
|
|
return MPFunTable()
|
|
elif obj_type == MP_PERSISTENT_OBJ_NONE:
|
|
return None
|
|
elif obj_type == MP_PERSISTENT_OBJ_FALSE:
|
|
return False
|
|
elif obj_type == MP_PERSISTENT_OBJ_TRUE:
|
|
return True
|
|
elif obj_type == MP_PERSISTENT_OBJ_ELLIPSIS:
|
|
return Ellipsis
|
|
elif obj_type == MP_PERSISTENT_OBJ_TUPLE:
|
|
ln = reader.read_uint()
|
|
return tuple(read_obj(reader, segments) for _ in range(ln))
|
|
else:
|
|
ln = reader.read_uint()
|
|
start_pos = reader.tell()
|
|
buf = reader.read_bytes(ln)
|
|
if obj_type in (MP_PERSISTENT_OBJ_STR, MP_PERSISTENT_OBJ_BYTES):
|
|
reader.read_byte() # read and discard null terminator
|
|
if obj_type == MP_PERSISTENT_OBJ_STR:
|
|
obj = str_cons(buf, "utf8")
|
|
if len(obj) < PERSISTENT_STR_INTERN_THRESHOLD:
|
|
if not global_qstrs.find_by_str(obj):
|
|
global_qstrs.add(obj)
|
|
elif obj_type == MP_PERSISTENT_OBJ_BYTES:
|
|
obj = buf
|
|
elif obj_type == MP_PERSISTENT_OBJ_INT:
|
|
obj = int(str_cons(buf, "ascii"), 10)
|
|
elif obj_type == MP_PERSISTENT_OBJ_FLOAT:
|
|
obj = float(str_cons(buf, "ascii"))
|
|
elif obj_type == MP_PERSISTENT_OBJ_COMPLEX:
|
|
obj = complex(str_cons(buf, "ascii"))
|
|
else:
|
|
raise MPYReadError(reader.filename, "corrupt .mpy file")
|
|
segments.append(MPYSegment(MPYSegment.OBJ, obj, start_pos, reader.tell()))
|
|
return obj
|
|
|
|
|
|
def read_raw_code(reader, parent_name, qstr_table, obj_table, segments):
|
|
# Read raw code header.
|
|
kind_len = reader.read_uint()
|
|
kind = (kind_len & 3) + MP_CODE_BYTECODE
|
|
has_children = (kind_len >> 2) & 1
|
|
fun_data_len = kind_len >> 3
|
|
|
|
# Read the body of the raw code.
|
|
file_offset = reader.tell()
|
|
fun_data = reader.read_bytes(fun_data_len)
|
|
segments_len = len(segments)
|
|
|
|
if kind == MP_CODE_BYTECODE:
|
|
# Create bytecode raw code.
|
|
rc = RawCodeBytecode(parent_name, qstr_table, obj_table, fun_data)
|
|
else:
|
|
# Create native raw code.
|
|
native_scope_flags = 0
|
|
native_n_pos_args = 0
|
|
native_type_sig = 0
|
|
if kind == MP_CODE_NATIVE_PY:
|
|
prelude_offset = reader.read_uint()
|
|
else:
|
|
prelude_offset = 0
|
|
native_scope_flags = reader.read_uint()
|
|
if kind == MP_CODE_NATIVE_VIPER:
|
|
# Read any additional sections for native viper.
|
|
if native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA:
|
|
rodata_size = reader.read_uint()
|
|
if native_scope_flags & MP_SCOPE_FLAG_VIPERBSS:
|
|
reader.read_uint() # bss_size
|
|
if native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA:
|
|
reader.read_bytes(rodata_size)
|
|
if native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC:
|
|
while True:
|
|
op = reader.read_byte()
|
|
if op == 0xFF:
|
|
break
|
|
if op & 1:
|
|
reader.read_uint() # addr
|
|
op >>= 1
|
|
if op <= 5 and op & 1:
|
|
reader.read_uint() # n
|
|
else:
|
|
assert kind == MP_CODE_NATIVE_ASM
|
|
native_n_pos_args = reader.read_uint()
|
|
native_type_sig = reader.read_uint()
|
|
|
|
rc = RawCodeNative(
|
|
parent_name,
|
|
qstr_table,
|
|
kind,
|
|
fun_data,
|
|
prelude_offset,
|
|
native_scope_flags,
|
|
native_n_pos_args,
|
|
native_type_sig,
|
|
)
|
|
|
|
# Add a segment for the raw code data.
|
|
segments.insert(
|
|
segments_len,
|
|
MPYSegment(MPYSegment.CODE, rc.simple_name.str, file_offset, file_offset + fun_data_len),
|
|
)
|
|
|
|
# Read children, if there are any.
|
|
rc.children = []
|
|
if has_children:
|
|
# Make a pretty parent name (otherwise all identifiers will include _lt_module_gt_).
|
|
if not rc.escaped_name.endswith("_lt_module_gt_"):
|
|
parent_name = rc.escaped_name
|
|
|
|
# Read all the child raw codes.
|
|
n_children = reader.read_uint()
|
|
for _ in range(n_children):
|
|
rc.children.append(read_raw_code(reader, parent_name, qstr_table, obj_table, segments))
|
|
|
|
return rc
|
|
|
|
|
|
def read_mpy(filename):
|
|
with open(filename, "rb") as fileobj:
|
|
reader = MPYReader(filename, fileobj)
|
|
segments = []
|
|
|
|
# Read and verify the header.
|
|
header = reader.read_bytes(4)
|
|
if header[0] != ord("M"):
|
|
raise MPYReadError(filename, "not a valid .mpy file")
|
|
if header[1] != config.MPY_VERSION:
|
|
raise MPYReadError(filename, "incompatible .mpy version")
|
|
feature_byte = header[2]
|
|
mpy_native_arch = feature_byte >> 2
|
|
if mpy_native_arch != MP_NATIVE_ARCH_NONE:
|
|
mpy_sub_version = feature_byte & 3
|
|
if mpy_sub_version != config.MPY_SUB_VERSION:
|
|
raise MPYReadError(filename, "incompatible .mpy sub-version")
|
|
if config.native_arch == MP_NATIVE_ARCH_NONE:
|
|
config.native_arch = mpy_native_arch
|
|
elif config.native_arch != mpy_native_arch:
|
|
raise MPYReadError(filename, "native architecture mismatch")
|
|
config.mp_small_int_bits = header[3]
|
|
|
|
# Read number of qstrs, and number of objects.
|
|
n_qstr = reader.read_uint()
|
|
n_obj = reader.read_uint()
|
|
|
|
# Read qstrs and construct qstr table.
|
|
qstr_table_file_offset = reader.tell()
|
|
qstr_table = []
|
|
for i in range(n_qstr):
|
|
qstr_table.append(read_qstr(reader, segments))
|
|
|
|
# Read objects and construct object table.
|
|
obj_table_file_offset = reader.tell()
|
|
obj_table = []
|
|
for i in range(n_obj):
|
|
obj_table.append(read_obj(reader, segments))
|
|
|
|
# Compute the compiled-module escaped name.
|
|
cm_escaped_name = qstr_table[0].str.replace("/", "_")[:-3]
|
|
|
|
# Read the outer raw code, which will in turn read all its children.
|
|
raw_code_file_offset = reader.tell()
|
|
raw_code = read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments)
|
|
|
|
# Create the outer-level compiled module representing the whole .mpy file.
|
|
return CompiledModule(
|
|
filename,
|
|
segments,
|
|
header,
|
|
qstr_table,
|
|
obj_table,
|
|
raw_code,
|
|
qstr_table_file_offset,
|
|
obj_table_file_offset,
|
|
raw_code_file_offset,
|
|
cm_escaped_name,
|
|
)
|
|
|
|
|
|
def hexdump_mpy(compiled_modules):
|
|
for cm in compiled_modules:
|
|
cm.hexdump()
|
|
|
|
|
|
def disassemble_mpy(compiled_modules):
|
|
for cm in compiled_modules:
|
|
cm.disassemble()
|
|
|
|
|
|
def freeze_mpy(firmware_qstr_idents, compiled_modules):
|
|
# add to qstrs
|
|
new = {}
|
|
for q in global_qstrs.qstrs:
|
|
# don't add duplicates that are already in the firmware
|
|
if q is None or q.qstr_esc in firmware_qstr_idents or q.qstr_esc in new:
|
|
continue
|
|
new[q.qstr_esc] = (len(new), q.qstr_esc, q.str, bytes_cons(q.str, "utf8"))
|
|
# Sort by string value (because this is a sorted pool).
|
|
new = sorted(new.values(), key=lambda x: x[2])
|
|
|
|
print('#include "py/mpconfig.h"')
|
|
print('#include "py/objint.h"')
|
|
print('#include "py/objstr.h"')
|
|
print('#include "py/emitglue.h"')
|
|
print('#include "py/nativeglue.h"')
|
|
print()
|
|
|
|
print("#if MICROPY_LONGINT_IMPL != %u" % config.MICROPY_LONGINT_IMPL)
|
|
print('#error "incompatible MICROPY_LONGINT_IMPL"')
|
|
print("#endif")
|
|
print()
|
|
|
|
if config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_MPZ:
|
|
print("#if MPZ_DIG_SIZE != %u" % config.MPZ_DIG_SIZE)
|
|
print('#error "incompatible MPZ_DIG_SIZE"')
|
|
print("#endif")
|
|
print()
|
|
|
|
print("#if MICROPY_PY_BUILTINS_FLOAT")
|
|
print("typedef struct _mp_obj_float_t {")
|
|
print(" mp_obj_base_t base;")
|
|
print(" mp_float_t value;")
|
|
print("} mp_obj_float_t;")
|
|
print("#endif")
|
|
print()
|
|
|
|
print("#if MICROPY_PY_BUILTINS_COMPLEX")
|
|
print("typedef struct _mp_obj_complex_t {")
|
|
print(" mp_obj_base_t base;")
|
|
print(" mp_float_t real;")
|
|
print(" mp_float_t imag;")
|
|
print("} mp_obj_complex_t;")
|
|
print("#endif")
|
|
print()
|
|
|
|
if len(new) > 0:
|
|
print("enum {")
|
|
for i in range(len(new)):
|
|
if i == 0:
|
|
print(" MP_QSTR_%s = MP_QSTRnumber_of," % new[i][1])
|
|
else:
|
|
print(" MP_QSTR_%s," % new[i][1])
|
|
print("};")
|
|
|
|
# As in qstr.c, set so that the first dynamically allocated pool is twice this size; must be <= the len
|
|
qstr_pool_alloc = min(len(new), 10)
|
|
|
|
global \
|
|
bc_content, \
|
|
const_str_content, \
|
|
const_int_content, \
|
|
const_obj_content, \
|
|
const_table_qstr_content, \
|
|
const_table_ptr_content, \
|
|
raw_code_count, \
|
|
raw_code_content
|
|
qstr_content = 0
|
|
bc_content = 0
|
|
const_str_content = 0
|
|
const_int_content = 0
|
|
const_obj_content = 0
|
|
const_table_qstr_content = 0
|
|
const_table_ptr_content = 0
|
|
raw_code_count = 0
|
|
raw_code_content = 0
|
|
|
|
print()
|
|
print("const qstr_hash_t mp_qstr_frozen_const_hashes[] = {")
|
|
qstr_size = {"metadata": 0, "data": 0}
|
|
for _, _, _, qbytes in new:
|
|
qhash = qstrutil.compute_hash(qbytes, config.MICROPY_QSTR_BYTES_IN_HASH)
|
|
print(" %d," % qhash)
|
|
print("};")
|
|
print()
|
|
print("const qstr_len_t mp_qstr_frozen_const_lengths[] = {")
|
|
for _, _, _, qbytes in new:
|
|
print(" %d," % len(qbytes))
|
|
qstr_size["metadata"] += (
|
|
config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH
|
|
)
|
|
qstr_size["data"] += len(qbytes)
|
|
print("};")
|
|
print()
|
|
print("extern const qstr_pool_t mp_qstr_const_pool;")
|
|
print("const qstr_pool_t mp_qstr_frozen_const_pool = {")
|
|
print(" &mp_qstr_const_pool, // previous pool")
|
|
print(" MP_QSTRnumber_of, // previous pool size")
|
|
print(" true, // is_sorted")
|
|
print(" %u, // allocated entries" % qstr_pool_alloc)
|
|
print(" %u, // used entries" % len(new))
|
|
print(" (qstr_hash_t *)mp_qstr_frozen_const_hashes,")
|
|
print(" (qstr_len_t *)mp_qstr_frozen_const_lengths,")
|
|
print(" {")
|
|
for _, _, qstr, qbytes in new:
|
|
print(' "%s",' % qstrutil.escape_bytes(qstr, qbytes))
|
|
qstr_content += (
|
|
config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH + len(qbytes) + 1
|
|
)
|
|
print(" },")
|
|
print("};")
|
|
|
|
# Freeze all modules.
|
|
for idx, cm in enumerate(compiled_modules):
|
|
cm.freeze(idx)
|
|
|
|
# Print separator, separating individual modules from global data structures.
|
|
print()
|
|
print("/" * 80)
|
|
print("// collection of all frozen modules")
|
|
|
|
# Define the string of frozen module names.
|
|
print()
|
|
print("const char mp_frozen_names[] = {")
|
|
print(" #ifdef MP_FROZEN_STR_NAMES")
|
|
# makemanifest.py might also include some frozen string content.
|
|
print(" MP_FROZEN_STR_NAMES")
|
|
print(" #endif")
|
|
mp_frozen_mpy_names_content = 1
|
|
for cm in compiled_modules:
|
|
module_name = cm.source_file.str
|
|
print(' "%s\\0"' % module_name)
|
|
mp_frozen_mpy_names_content += len(cm.source_file.str) + 1
|
|
print(' "\\0"')
|
|
print("};")
|
|
|
|
# Define the array of pointers to frozen module content.
|
|
print()
|
|
print("const mp_frozen_module_t *const mp_frozen_mpy_content[] = {")
|
|
for cm in compiled_modules:
|
|
print(" &frozen_module_%s," % cm.escaped_name)
|
|
print("};")
|
|
mp_frozen_mpy_content_size = len(compiled_modules * 4)
|
|
|
|
# If a port defines MICROPY_FROZEN_LIST_ITEM then list all modules wrapped in that macro.
|
|
print()
|
|
print("#ifdef MICROPY_FROZEN_LIST_ITEM")
|
|
for cm in compiled_modules:
|
|
module_name = cm.source_file.str
|
|
if module_name.endswith("/__init__.py"):
|
|
short_name = module_name[: -len("/__init__.py")]
|
|
else:
|
|
short_name = module_name[: -len(".py")]
|
|
print('MICROPY_FROZEN_LIST_ITEM("%s", "%s")' % (short_name, module_name))
|
|
print("#endif")
|
|
|
|
print()
|
|
print("/*")
|
|
print("byte sizes:")
|
|
print("qstr content: %d unique, %d bytes" % (len(new), qstr_content))
|
|
print("bc content: %d" % bc_content)
|
|
print("const str content: %d" % const_str_content)
|
|
print("const int content: %d" % const_int_content)
|
|
print("const obj content: %d" % const_obj_content)
|
|
print(
|
|
"const table qstr content: %d entries, %d bytes"
|
|
% (const_table_qstr_content, const_table_qstr_content * 4)
|
|
)
|
|
print(
|
|
"const table ptr content: %d entries, %d bytes"
|
|
% (const_table_ptr_content, const_table_ptr_content * 4)
|
|
)
|
|
print("raw code content: %d * 4 = %d" % (raw_code_count, raw_code_content))
|
|
print("mp_frozen_mpy_names_content: %d" % mp_frozen_mpy_names_content)
|
|
print("mp_frozen_mpy_content_size: %d" % mp_frozen_mpy_content_size)
|
|
print(
|
|
"total: %d"
|
|
% (
|
|
qstr_content
|
|
+ bc_content
|
|
+ const_str_content
|
|
+ const_int_content
|
|
+ const_obj_content
|
|
+ const_table_qstr_content * 4
|
|
+ const_table_ptr_content * 4
|
|
+ raw_code_content
|
|
+ mp_frozen_mpy_names_content
|
|
+ mp_frozen_mpy_content_size
|
|
)
|
|
)
|
|
print("*/")
|
|
|
|
|
|
def adjust_bytecode_qstr_obj_indices(bytecode_in, qstr_table_base, obj_table_base):
|
|
# Expand bytcode to a list of opcodes.
|
|
opcodes = []
|
|
labels = {}
|
|
ip = 0
|
|
while ip < len(bytecode_in):
|
|
fmt, sz, arg, extra_arg = mp_opcode_decode(bytecode_in, ip)
|
|
opcode = Opcode(ip, fmt, bytecode_in[ip], arg, extra_arg)
|
|
labels[ip] = opcode
|
|
opcodes.append(opcode)
|
|
ip += sz
|
|
if fmt == MP_BC_FORMAT_OFFSET:
|
|
opcode.arg += ip
|
|
|
|
# Link jump opcodes to their destination.
|
|
for opcode in opcodes:
|
|
if opcode.fmt == MP_BC_FORMAT_OFFSET:
|
|
opcode.target = labels[opcode.arg]
|
|
|
|
# Adjust bytcode as required.
|
|
for opcode in opcodes:
|
|
if opcode.fmt == MP_BC_FORMAT_QSTR:
|
|
opcode.arg += qstr_table_base
|
|
elif opcode.opcode_byte == Opcode.MP_BC_LOAD_CONST_OBJ:
|
|
opcode.arg += obj_table_base
|
|
|
|
# Write out new bytecode.
|
|
offset_changed = True
|
|
while offset_changed:
|
|
offset_changed = False
|
|
overflow = False
|
|
bytecode_out = b""
|
|
for opcode in opcodes:
|
|
ip = len(bytecode_out)
|
|
if opcode.offset != ip:
|
|
offset_changed = True
|
|
opcode.offset = ip
|
|
opcode_overflow, encoded_opcode = mp_opcode_encode(opcode)
|
|
if opcode_overflow:
|
|
overflow = True
|
|
bytecode_out += encoded_opcode
|
|
|
|
if overflow:
|
|
raise Exception("bytecode overflow")
|
|
|
|
return bytecode_out
|
|
|
|
|
|
def rewrite_raw_code(rc, qstr_table_base, obj_table_base):
|
|
if rc.code_kind != MP_CODE_BYTECODE:
|
|
raise Exception("can only rewrite bytecode")
|
|
|
|
source_info = bytearray()
|
|
for arg in rc.names:
|
|
source_info.extend(mp_encode_uint(qstr_table_base + arg))
|
|
|
|
closure_info = rc.fun_data[rc.offset_closure_info : rc.offset_opcodes]
|
|
|
|
bytecode_in = memoryview(rc.fun_data)[rc.offset_opcodes :]
|
|
bytecode_out = adjust_bytecode_qstr_obj_indices(bytecode_in, qstr_table_base, obj_table_base)
|
|
|
|
prelude_signature = rc.fun_data[: rc.offset_prelude_size]
|
|
prelude_size = encode_prelude_size(len(source_info), len(closure_info))
|
|
|
|
fun_data = prelude_signature + prelude_size + source_info + closure_info + bytecode_out
|
|
|
|
output = mp_encode_uint(len(fun_data) << 3 | bool(len(rc.children)) << 2)
|
|
output += fun_data
|
|
|
|
if rc.children:
|
|
output += mp_encode_uint(len(rc.children))
|
|
for child in rc.children:
|
|
output += rewrite_raw_code(child, qstr_table_base, obj_table_base)
|
|
|
|
return output
|
|
|
|
|
|
def merge_mpy(compiled_modules, output_file):
|
|
merged_mpy = bytearray()
|
|
|
|
if len(compiled_modules) == 1:
|
|
with open(compiled_modules[0].mpy_source_file, "rb") as f:
|
|
merged_mpy.extend(f.read())
|
|
else:
|
|
main_cm_idx = None
|
|
for idx, cm in enumerate(compiled_modules):
|
|
feature_byte = cm.header[2]
|
|
mpy_native_arch = feature_byte >> 2
|
|
if mpy_native_arch:
|
|
# Must use qstr_table and obj_table from this raw_code
|
|
if main_cm_idx is not None:
|
|
raise Exception("can't merge files when more than one contains native code")
|
|
main_cm_idx = idx
|
|
if main_cm_idx is not None:
|
|
# Shift main_cm to front of list.
|
|
compiled_modules.insert(0, compiled_modules.pop(main_cm_idx))
|
|
|
|
header = bytearray(4)
|
|
header[0] = ord("M")
|
|
header[1] = config.MPY_VERSION
|
|
header[2] = config.native_arch << 2 | config.MPY_SUB_VERSION if config.native_arch else 0
|
|
header[3] = config.mp_small_int_bits
|
|
merged_mpy.extend(header)
|
|
|
|
n_qstr = 0
|
|
n_obj = 0
|
|
for cm in compiled_modules:
|
|
n_qstr += len(cm.qstr_table)
|
|
n_obj += len(cm.obj_table)
|
|
merged_mpy.extend(mp_encode_uint(n_qstr))
|
|
merged_mpy.extend(mp_encode_uint(n_obj))
|
|
|
|
# Copy verbatim the qstr and object tables from all compiled modules.
|
|
def copy_section(file, offset, offset2):
|
|
with open(file, "rb") as f:
|
|
f.seek(offset)
|
|
merged_mpy.extend(f.read(offset2 - offset))
|
|
|
|
for cm in compiled_modules:
|
|
copy_section(cm.mpy_source_file, cm.qstr_table_file_offset, cm.obj_table_file_offset)
|
|
for cm in compiled_modules:
|
|
copy_section(cm.mpy_source_file, cm.obj_table_file_offset, cm.raw_code_file_offset)
|
|
|
|
bytecode = bytearray()
|
|
bytecode.append(0b00000000) # prelude signature
|
|
bytecode.append(0b00000010) # prelude size (n_info=1, n_cell=0)
|
|
bytecode.extend(b"\x00") # simple_name: qstr index 0 (will use source filename)
|
|
for idx in range(len(compiled_modules)):
|
|
bytecode.append(0x32) # MP_BC_MAKE_FUNCTION
|
|
bytecode.append(idx) # index raw code
|
|
bytecode.extend(b"\x34\x00\x59") # MP_BC_CALL_FUNCTION, 0 args, MP_BC_POP_TOP
|
|
bytecode.extend(b"\x51\x63") # MP_BC_LOAD_NONE, MP_BC_RETURN_VALUE
|
|
|
|
merged_mpy.extend(mp_encode_uint(len(bytecode) << 3 | 1 << 2)) # length, has_children
|
|
merged_mpy.extend(bytecode)
|
|
merged_mpy.extend(mp_encode_uint(len(compiled_modules))) # n_children
|
|
|
|
qstr_table_base = 0
|
|
obj_table_base = 0
|
|
for cm in compiled_modules:
|
|
if qstr_table_base == 0 and obj_table_base == 0:
|
|
with open(cm.mpy_source_file, "rb") as f:
|
|
f.seek(cm.raw_code_file_offset)
|
|
merged_mpy.extend(f.read())
|
|
else:
|
|
merged_mpy.extend(rewrite_raw_code(cm.raw_code, qstr_table_base, obj_table_base))
|
|
qstr_table_base += len(cm.qstr_table)
|
|
obj_table_base += len(cm.obj_table)
|
|
|
|
if output_file is None:
|
|
sys.stdout.buffer.write(merged_mpy)
|
|
else:
|
|
with open(output_file, "wb") as f:
|
|
f.write(merged_mpy)
|
|
|
|
|
|
def main():
|
|
global global_qstrs
|
|
|
|
import argparse
|
|
|
|
cmd_parser = argparse.ArgumentParser(description="A tool to work with MicroPython .mpy files.")
|
|
cmd_parser.add_argument(
|
|
"-x", "--hexdump", action="store_true", help="output an annotated hex dump of files"
|
|
)
|
|
cmd_parser.add_argument(
|
|
"-d", "--disassemble", action="store_true", help="output disassembled contents of files"
|
|
)
|
|
cmd_parser.add_argument("-f", "--freeze", action="store_true", help="freeze files")
|
|
cmd_parser.add_argument(
|
|
"--merge", action="store_true", help="merge multiple .mpy files into one"
|
|
)
|
|
cmd_parser.add_argument("-q", "--qstr-header", help="qstr header file to freeze against")
|
|
cmd_parser.add_argument(
|
|
"-mlongint-impl",
|
|
choices=["none", "longlong", "mpz"],
|
|
default="mpz",
|
|
help="long-int implementation used by target (default mpz)",
|
|
)
|
|
cmd_parser.add_argument(
|
|
"-mmpz-dig-size",
|
|
metavar="N",
|
|
type=int,
|
|
default=16,
|
|
help="mpz digit size used by target (default 16)",
|
|
)
|
|
cmd_parser.add_argument("-o", "--output", default=None, help="output file")
|
|
cmd_parser.add_argument("files", nargs="+", help="input .mpy files")
|
|
args = cmd_parser.parse_args()
|
|
|
|
# set config values relevant to target machine
|
|
config.MICROPY_LONGINT_IMPL = {
|
|
"none": config.MICROPY_LONGINT_IMPL_NONE,
|
|
"longlong": config.MICROPY_LONGINT_IMPL_LONGLONG,
|
|
"mpz": config.MICROPY_LONGINT_IMPL_MPZ,
|
|
}[args.mlongint_impl]
|
|
config.MPZ_DIG_SIZE = args.mmpz_dig_size
|
|
config.native_arch = MP_NATIVE_ARCH_NONE
|
|
|
|
# set config values for qstrs, and get the existing base set of qstrs
|
|
# already in the firmware
|
|
if args.qstr_header:
|
|
qcfgs, extra_qstrs = qstrutil.parse_input_headers([args.qstr_header])
|
|
firmware_qstr_idents = set(qstrutil.static_qstr_list_ident) | set(extra_qstrs.keys())
|
|
config.MICROPY_QSTR_BYTES_IN_LEN = int(qcfgs["BYTES_IN_LEN"])
|
|
config.MICROPY_QSTR_BYTES_IN_HASH = int(qcfgs["BYTES_IN_HASH"])
|
|
else:
|
|
config.MICROPY_QSTR_BYTES_IN_LEN = 1
|
|
config.MICROPY_QSTR_BYTES_IN_HASH = 1
|
|
firmware_qstr_idents = set(qstrutil.static_qstr_list)
|
|
|
|
# Create initial list of global qstrs.
|
|
global_qstrs = GlobalQStrList()
|
|
|
|
# Load all .mpy files.
|
|
try:
|
|
compiled_modules = [read_mpy(file) for file in args.files]
|
|
except MPYReadError as er:
|
|
print(er, file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
if args.hexdump:
|
|
hexdump_mpy(compiled_modules)
|
|
|
|
if args.disassemble:
|
|
if args.hexdump:
|
|
print()
|
|
disassemble_mpy(compiled_modules)
|
|
|
|
if args.freeze:
|
|
try:
|
|
freeze_mpy(firmware_qstr_idents, compiled_modules)
|
|
except FreezeError as er:
|
|
print(er, file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
if args.merge:
|
|
merge_mpy(compiled_modules, args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|