diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 3989f7aaaa..61e396e291 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -59,8 +59,7 @@ def do_work(infiles): for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]): qhash = compute_hash(qstr) qlen = len(qstr) - qchlen = len(qstr.decode("utf-8")) - print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qchlen & 0xff, (qchlen >> 8) & 0xff, qstr)) + print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qstr)) return True diff --git a/py/objstr.c b/py/objstr.c index db3490facb..3e275afb9a 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -52,9 +52,6 @@ const mp_obj_t mp_const_empty_bytes; // use this macro to extract the string data and length #define GET_STR_DATA_LEN(str_obj_in, str_data, str_len) const byte *str_data; uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_data = ((mp_obj_str_t*)str_obj_in)->data; } -// use this macro to extract the string data and both lengths -#define GET_STR_INFO(str_obj_in, str_data, str_len, str_charlen) const byte *str_data; uint str_len, str_charlen; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len); str_charlen = qstr_charlen(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_charlen = ((mp_obj_str_t*)str_obj_in)->charlen; str_data = ((mp_obj_str_t*)str_obj_in)->data; } - STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str); STATIC mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str); STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in); @@ -365,7 +362,7 @@ uncomparable: STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { mp_obj_type_t *type = mp_obj_get_type(self_in); - GET_STR_INFO(self_in, self_data, self_len, self_charlen); + GET_STR_DATA_LEN(self_in, self_data, self_len); if (value == MP_OBJ_SENTINEL) { // load #if MICROPY_PY_BUILTINS_SLICE @@ -378,7 +375,8 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { return mp_obj_new_str_of_type(type, self_data + slice.start, slice.stop - slice.start); } #endif - uint index_val = mp_get_index(type, self_charlen, index, false); + // TODO: Don't use mp_get_index() here + uint index_val = mp_get_index(type, unichar_charlen((const char *)self_data, self_len), index, false); if (type == &mp_type_bytes) { return MP_OBJ_NEW_SMALL_INT((mp_small_int_t)self_data[index_val]); } else { @@ -1734,7 +1732,7 @@ const mp_obj_type_t mp_type_bytes = { }; // the zero-length bytes -STATIC const mp_obj_str_t empty_bytes_obj = {{&mp_type_bytes}, 0, 0, 0, NULL}; +STATIC const mp_obj_str_t empty_bytes_obj = {{&mp_type_bytes}, 0, 0, NULL}; const mp_obj_t mp_const_empty_bytes = (mp_obj_t)&empty_bytes_obj; mp_obj_t mp_obj_str_builder_start(const mp_obj_type_t *type, uint len, byte **data) { @@ -1761,20 +1759,6 @@ mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uin o->base.type = type; o->len = len; if (data) { - if (MP_OBJ_IS_STR(o)) { - // Count non-continuation bytes so we know how long the string is in characters. - const byte *endptr, *top = data + len; - uint charlen = 0; - for (endptr = data; endptr < top; ++endptr) { - if (!UTF8_IS_CONT(*endptr)) { - ++charlen; - } - } - o->charlen = charlen; - } else { - // For byte strings, the 'character' length (really the "exposed length" or "Python length") equals the byte length. - o->charlen = len; - } o->hash = qstr_compute_hash(data, len); byte *p = m_new(byte, len + 1); o->data = p; @@ -1844,8 +1828,8 @@ uint mp_obj_str_get_hash(mp_obj_t self_in) { uint mp_obj_str_get_len(mp_obj_t self_in) { // TODO This has a double check for the type, one in obj.c and one here if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) { - GET_STR_INFO(self_in, self_data, self_len, self_charlen); (void)self_data; - return self_charlen; + GET_STR_DATA_LEN(self_in, self_data, self_len); + return unichar_charlen((const char *)self_data, self_len); } else { bad_implicit_conversion(self_in); } diff --git a/py/objstr.h b/py/objstr.h index 98b0a5a316..3af5141be4 100644 --- a/py/objstr.h +++ b/py/objstr.h @@ -30,13 +30,10 @@ typedef struct _mp_obj_str_t { machine_uint_t hash : 16; // len == number of bytes used in data, alloc = len + 1 because (at the moment) we also append a null byte machine_uint_t len : 16; - // charlen == number of characters in the string - charlen <= len - 1, and is the value returned by len() in Python - machine_uint_t charlen : 16; const void *data; //Character data is encoded UTF-8 and should not be blindly indexed. } mp_obj_str_t; -// This is valid ONLY for pure-ASCII strings! -#define MP_DEFINE_STR_OBJ(obj_name, str) mp_obj_str_t obj_name = {{&mp_type_str}, 0, sizeof(str) - 1, sizeof(str) - 1, (const byte*)str}; +#define MP_DEFINE_STR_OBJ(obj_name, str) mp_obj_str_t obj_name = {{&mp_type_str}, 0, sizeof(str) - 1, (const byte*)str}; mp_obj_t mp_obj_str_format(uint n_args, const mp_obj_t *args); mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uint len); diff --git a/py/qstr.c b/py/qstr.c index 85f1bb4238..22b44a710a 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -46,15 +46,13 @@ // For now we use very simple encoding, just to get the framework correct: // - hash is 2 bytes (see function below) // - length is 2 bytes -// - character length is 2 bytes // - data follows // - \0 terminated (for now, so they can be printed using printf) #define Q_GET_HASH(q) ((q)[0] | ((q)[1] << 8)) -#define Q_GET_ALLOC(q) (6 + Q_GET_LENGTH(q) + 1) +#define Q_GET_ALLOC(q) (4 + Q_GET_LENGTH(q) + 1) #define Q_GET_LENGTH(q) ((q)[2] | ((q)[3] << 8)) -#define Q_GET_CHARLEN(q) ((q)[4] | ((q)[5] << 8)) -#define Q_GET_DATA(q) ((q) + 6) +#define Q_GET_DATA(q) ((q) + 4) // this must match the equivalent function in makeqstrdata.py // Note that this hashes the UTF-8 encoded data bytes. @@ -158,21 +156,13 @@ qstr qstr_from_strn(const char *str, uint len) { qstr q = qstr_find_strn(str, len); if (q == 0) { machine_uint_t hash = qstr_compute_hash((const byte*)str, len); - byte *q_ptr = m_new(byte, 6 + len + 1); - uint charlen = 0; - for (const char *s = str; s < str + len; ++s) { - if (!UTF8_IS_CONT(*s)) { - ++charlen; - } - } + byte *q_ptr = m_new(byte, 4 + len + 1); q_ptr[0] = hash; q_ptr[1] = hash >> 8; q_ptr[2] = len; q_ptr[3] = len >> 8; - q_ptr[4] = charlen; - q_ptr[5] = charlen >> 8; - memcpy(q_ptr + 6, str, len); - q_ptr[6 + len] = '\0'; + memcpy(q_ptr + 4, str, len); + q_ptr[4 + len] = '\0'; q = qstr_add(q_ptr); } return q; @@ -180,7 +170,7 @@ qstr qstr_from_strn(const char *str, uint len) { byte *qstr_build_start(uint len, byte **q_ptr) { assert(len <= 65535); - *q_ptr = m_new(byte, 7 + len + 1); + *q_ptr = m_new(byte, 4 + len + 1); (*q_ptr)[2] = len; (*q_ptr)[3] = len >> 8; return Q_GET_DATA(*q_ptr); @@ -194,15 +184,7 @@ qstr qstr_build_end(byte *q_ptr) { machine_uint_t hash = qstr_compute_hash(str, len); q_ptr[0] = hash; q_ptr[1] = hash >> 8; - uint charlen = 0; - for (const byte *s = str; s < str + len; ++s) { - if (!UTF8_IS_CONT(*s)) { - ++charlen; - } - } - q_ptr[4] = charlen; - q_ptr[5] = charlen >> 8; - q_ptr[6 + len] = '\0'; + q_ptr[4 + len] = '\0'; q = qstr_add(q_ptr); } else { m_del(byte, q_ptr, Q_GET_ALLOC(q_ptr)); @@ -219,11 +201,6 @@ uint qstr_len(qstr q) { return Q_GET_LENGTH(qd); } -uint qstr_charlen(qstr q) { - const byte *qd = find_qstr(q); - return Q_GET_CHARLEN(qd); -} - // XXX to remove! const char *qstr_str(qstr q) { const byte *qd = find_qstr(q); diff --git a/py/qstr.h b/py/qstr.h index b624e6f30d..9803e672ca 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -59,7 +59,6 @@ qstr qstr_build_end(byte *q_ptr); machine_uint_t qstr_hash(qstr q); const char* qstr_str(qstr q); uint qstr_len(qstr q); -uint qstr_charlen(qstr q); const byte* qstr_data(qstr q, uint *len); void qstr_pool_info(uint *n_pool, uint *n_qstr, uint *n_str_data_bytes, uint *n_total_bytes);