diff --git a/py/misc.h b/py/misc.h index 3f62e3198f..97e9b30edf 100644 --- a/py/misc.h +++ b/py/misc.h @@ -100,7 +100,9 @@ bool unichar_isupper(unichar c); bool unichar_islower(unichar c); unichar unichar_tolower(unichar c); unichar unichar_toupper(unichar c); -#define unichar_charlen(s, bytelen) (bytelen) +uint unichar_charlen(const char *str, uint len); +#define UTF8_IS_NONASCII(ch) ((ch) & 0x80) +#define UTF8_IS_CONT(ch) (((ch) & 0xC0) == 0x80) /** variable string *********************************************/ diff --git a/py/unicode.c b/py/unicode.c index 88f835131d..0da247889e 100644 --- a/py/unicode.c +++ b/py/unicode.c @@ -65,14 +65,39 @@ STATIC const uint8_t attr[] = { AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0 }; -unichar utf8_get_char(const byte *s) { - return *s; +unichar utf8_get_char(const char *s) { + unichar ord = *s++; + if (!UTF8_IS_NONASCII(ord)) return ord; + ord &= 0x7F; + for (unichar mask = 0x40; ord & mask; mask >>= 1) { + ord &= ~mask; + } + while (UTF8_IS_CONT(*s)) { + ord = (ord << 6) | (*s++ & 0x3F); + } + return ord; } -const byte *utf8_next_char(const byte *s) { - return s + 1; +char *utf8_next_char(const char *s) { + ++s; + while (UTF8_IS_CONT(*s)) { + ++s; + } + return (char *)s; } +uint unichar_charlen(const char *str, uint len) +{ + uint charlen = 0; + for (const char *top = str + len; str < top; ++str) { + if (!UTF8_IS_CONT(*str)) { + ++charlen; + } + } + return charlen; +} + +// Be aware: These unichar_is* functions are actually ASCII-only! bool unichar_isspace(unichar c) { return c < 128 && (attr[c] & FL_SPACE) != 0; }