diff options
author | 2016-12-23 14:48:09 +0100 | |
---|---|---|
committer | 2016-12-23 14:48:09 +0100 | |
commit | d0b0de1aec45c0f9b66c931efb9902874cb93728 (patch) | |
tree | 98a11fe89a65cc62fa8114a0b6c5312c21810ede /src/lib/util/unicode.cpp | |
parent | c0561727b406ed189039d81fac7c69fef4c79913 (diff) | |
parent | 63b7b9d521712404dc35156ef8d7e606440b4156 (diff) |
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'src/lib/util/unicode.cpp')
-rw-r--r-- | src/lib/util/unicode.cpp | 224 |
1 files changed, 133 insertions, 91 deletions
diff --git a/src/lib/util/unicode.cpp b/src/lib/util/unicode.cpp index e2c3956cedb..a6c5515fbf6 100644 --- a/src/lib/util/unicode.cpp +++ b/src/lib/util/unicode.cpp @@ -11,96 +11,123 @@ #include "unicode.h" -/*------------------------------------------------- - uchar_isvalid - return true if a given - character is a legitimate unicode character --------------------------------------------------*/ +//------------------------------------------------- +// uchar_isvalid - return true if a given +// character is a legitimate unicode character +//------------------------------------------------- -int uchar_isvalid(unicode_char uchar) +bool uchar_isvalid(char32_t uchar) { return (uchar < 0x110000) && !((uchar >= 0xd800) && (uchar <= 0xdfff)); } -/*------------------------------------------------- - uchar_from_utf8 - convert a UTF-8 sequence - into a unicode character --------------------------------------------------*/ +//------------------------------------------------- +// uchar_is_printable - tests to see if a unicode +// char is printable +//------------------------------------------------- -int uchar_from_utf8(unicode_char *uchar, const char *utf8char, size_t count) +bool uchar_is_printable(char32_t uchar) { - unicode_char c, minchar; + return + !(0x0001f >= uchar) && // C0 control + !((0x0007f <= uchar) && (0x0009f >= uchar)) && // DEL and C1 control + !((0x0fdd0 <= uchar) && (0x0fddf >= uchar)) && // noncharacters + !(0x0fffe == (uchar & 0x0ffff)) && // byte-order detection noncharacter + !(0x0ffff == (uchar & 0x0ffff)); // the other noncharacter +} + + +//------------------------------------------------- +// uchar_is_digit - tests to see if a unicode +// char is a digit +//------------------------------------------------- + +bool uchar_is_digit(char32_t uchar) +{ + return uchar >= '0' && uchar <= '9'; +} + + +//------------------------------------------------- +// uchar_from_utf8 - convert a UTF-8 sequence +// into a unicode character +//----------------------------------------------- + +int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count) +{ + char32_t c, minchar; int auxlen, i; char auxchar; - /* validate parameters */ + // validate parameters if (utf8char == nullptr || count == 0) return 0; - /* start with the first byte */ + // start with the first byte c = (unsigned char) *utf8char; count--; utf8char++; - /* based on that, determine how many additional bytes we need */ + // based on that, determine how many additional bytes we need if (c < 0x80) { - /* unicode char 0x00000000 - 0x0000007F */ + // unicode char 0x00000000 - 0x0000007F c &= 0x7f; auxlen = 0; minchar = 0x00000000; } else if (c >= 0xc0 && c < 0xe0) { - /* unicode char 0x00000080 - 0x000007FF */ + // unicode char 0x00000080 - 0x000007FF c &= 0x1f; auxlen = 1; minchar = 0x00000080; } else if (c >= 0xe0 && c < 0xf0) { - /* unicode char 0x00000800 - 0x0000FFFF */ + // unicode char 0x00000800 - 0x0000FFFF c &= 0x0f; auxlen = 2; minchar = 0x00000800; } else if (c >= 0xf0 && c < 0xf8) { - /* unicode char 0x00010000 - 0x001FFFFF */ + // unicode char 0x00010000 - 0x001FFFFF c &= 0x07; auxlen = 3; minchar = 0x00010000; } else if (c >= 0xf8 && c < 0xfc) { - /* unicode char 0x00200000 - 0x03FFFFFF */ + // unicode char 0x00200000 - 0x03FFFFFF c &= 0x03; auxlen = 4; minchar = 0x00200000; } else if (c >= 0xfc && c < 0xfe) { - /* unicode char 0x04000000 - 0x7FFFFFFF */ + // unicode char 0x04000000 - 0x7FFFFFFF c &= 0x01; auxlen = 5; minchar = 0x04000000; } else { - /* invalid */ + // invalid return -1; } - /* exceeds the count? */ + // exceeds the count? if (auxlen > count) return -1; - /* we now know how long the char is, now compute it */ + // we now know how long the char is, now compute it for (i = 0; i < auxlen; i++) { auxchar = utf8char[i]; - /* all auxillary chars must be between 0x80-0xbf */ + // all auxillary chars must be between 0x80-0xbf if ((auxchar & 0xc0) != 0x80) return -1; @@ -108,7 +135,7 @@ int uchar_from_utf8(unicode_char *uchar, const char *utf8char, size_t count) c |= auxchar & 0x3f; } - /* make sure that this char is above the minimum */ + // make sure that this char is above the minimum if (c < minchar) return -1; @@ -117,32 +144,32 @@ int uchar_from_utf8(unicode_char *uchar, const char *utf8char, size_t count) } -/*------------------------------------------------- - uchar_from_utf16 - convert a UTF-16 sequence - into a unicode character --------------------------------------------------*/ +//------------------------------------------------- +// uchar_from_utf16 - convert a UTF-16 sequence +// into a unicode character +//------------------------------------------------- -int uchar_from_utf16(unicode_char *uchar, const utf16_char *utf16char, size_t count) +int uchar_from_utf16(char32_t *uchar, const char16_t *utf16char, size_t count) { int rc = -1; - /* validate parameters */ + // validate parameters if (utf16char == nullptr || count == 0) - return 0; - - /* handle the two-byte case */ + { + rc = 0; + } if (utf16char[0] >= 0xd800 && utf16char[0] <= 0xdbff) { + // handle the two-byte case if (count > 1 && utf16char[1] >= 0xdc00 && utf16char[1] <= 0xdfff) { *uchar = 0x10000 + ((utf16char[0] & 0x3ff) * 0x400) + (utf16char[1] & 0x3ff); rc = 2; } } - - /* handle the one-byte case */ else if (utf16char[0] < 0xdc00 || utf16char[0] > 0xdfff) { + // handle the one-byte case *uchar = utf16char[0]; rc = 1; } @@ -151,47 +178,47 @@ int uchar_from_utf16(unicode_char *uchar, const utf16_char *utf16char, size_t co } -/*------------------------------------------------- - uchar_from_utf16f - convert a UTF-16 sequence - into a unicode character from a flipped - byte order --------------------------------------------------*/ +//------------------------------------------------- +// uchar_from_utf16f - convert a UTF-16 sequence +// into a unicode character from a flipped +// byte order +//------------------------------------------------- -int uchar_from_utf16f(unicode_char *uchar, const utf16_char *utf16char, size_t count) +int uchar_from_utf16f(char32_t *uchar, const char16_t *utf16char, size_t count) { - utf16_char buf[2] = {0}; + char16_t buf[2] = {0}; if (count > 0) - buf[0] = FLIPENDIAN_INT16(utf16char[0]); + buf[0] = flipendian_int16(utf16char[0]); if (count > 1) - buf[1] = FLIPENDIAN_INT16(utf16char[1]); + buf[1] = flipendian_int16(utf16char[1]); return uchar_from_utf16(uchar, buf, count); } -/*------------------------------------------------- - utf8_from_uchar - convert a unicode character - into a UTF-8 sequence --------------------------------------------------*/ +//------------------------------------------------- +// utf8_from_uchar - convert a unicode character +// into a UTF-8 sequence +//------------------------------------------------- -int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar) +int utf8_from_uchar(char *utf8string, size_t count, char32_t uchar) { int rc = 0; - /* error on invalid characters */ + // error on invalid characters if (!uchar_isvalid(uchar)) return -1; - /* based on the value, output the appropriate number of bytes */ + // based on the value, output the appropriate number of bytes if (uchar < 0x80) { - /* unicode char 0x00000000 - 0x0000007F */ + // unicode char 0x00000000 - 0x0000007F if (count < 1) return -1; utf8string[rc++] = (char) uchar; } else if (uchar < 0x800) { - /* unicode char 0x00000080 - 0x000007FF */ + // unicode char 0x00000080 - 0x000007FF if (count < 2) return -1; utf8string[rc++] = ((char) (uchar >> 6)) | 0xC0; @@ -199,7 +226,7 @@ int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar) } else if (uchar < 0x10000) { - /* unicode char 0x00000800 - 0x0000FFFF */ + // unicode char 0x00000800 - 0x0000FFFF if (count < 3) return -1; utf8string[rc++] = ((char) (uchar >> 12)) | 0xE0; @@ -208,7 +235,7 @@ int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar) } else if (uchar < 0x00200000) { - /* unicode char 0x00010000 - 0x001FFFFF */ + // unicode char 0x00010000 - 0x001FFFFF if (count < 4) return -1; utf8string[rc++] = ((char) (uchar >> 18)) | 0xF0; @@ -218,7 +245,7 @@ int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar) } else if (uchar < 0x04000000) { - /* unicode char 0x00200000 - 0x03FFFFFF */ + // unicode char 0x00200000 - 0x03FFFFFF if (count < 5) return -1; utf8string[rc++] = ((char) (uchar >> 24)) | 0xF8; @@ -229,7 +256,7 @@ int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar) } else if (uchar < 0x80000000) { - /* unicode char 0x04000000 - 0x7FFFFFFF */ + // unicode char 0x04000000 - 0x7FFFFFFF if (count < 6) return -1; utf8string[rc++] = ((char) (uchar >> 30)) | 0xFC; @@ -246,67 +273,82 @@ int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar) } -/*------------------------------------------------- - utf16_from_uchar - convert a unicode character - into a UTF-16 sequence --------------------------------------------------*/ +//------------------------------------------------- +// utf8_from_uchar - convert a unicode character +// into a UTF-8 sequence +//------------------------------------------------- -int utf16_from_uchar(utf16_char *utf16string, size_t count, unicode_char uchar) +std::string utf8_from_uchar(char32_t uchar) +{ + char buffer[UTF8_CHAR_MAX]; + auto len = utf8_from_uchar(buffer, ARRAY_LENGTH(buffer), uchar); + return std::string(buffer, len); +} + + +//------------------------------------------------- +// utf16_from_uchar - convert a unicode character +// into a UTF-16 sequence +//------------------------------------------------- + +int utf16_from_uchar(char16_t *utf16string, size_t count, char32_t uchar) { int rc; - /* error on invalid characters */ + // error on invalid characters if (!uchar_isvalid(uchar)) return -1; - /* single word case */ if (uchar < 0x10000) { + // single word case if (count < 1) return -1; - utf16string[0] = (utf16_char) uchar; + utf16string[0] = (char16_t) uchar; rc = 1; } - - /* double word case */ else if (uchar < 0x100000) { + // double word case if (count < 2) return -1; + uchar -= 0x10000; utf16string[0] = ((uchar >> 10) & 0x03ff) | 0xd800; utf16string[1] = ((uchar >> 0) & 0x03ff) | 0xdc00; rc = 2; } else + { return -1; + } return rc; } -/*------------------------------------------------- - utf16_from_uchar - convert a unicode character - into a UTF-16 sequence with flipped endianness --------------------------------------------------*/ +//------------------------------------------------- +// utf16_from_uchar - convert a unicode character +// into a UTF-16 sequence with flipped endianness +//------------------------------------------------- -int utf16f_from_uchar(utf16_char *utf16string, size_t count, unicode_char uchar) +int utf16f_from_uchar(char16_t *utf16string, size_t count, char32_t uchar) { int rc; - utf16_char buf[2] = { 0, 0 }; + char16_t buf[2] = { 0, 0 }; rc = utf16_from_uchar(buf, count, uchar); if (rc >= 1) - utf16string[0] = FLIPENDIAN_INT16(buf[0]); + utf16string[0] = flipendian_int16(buf[0]); if (rc >= 2) - utf16string[1] = FLIPENDIAN_INT16(buf[1]); + utf16string[1] = flipendian_int16(buf[1]); return rc; } -/*------------------------------------------------- - utf8_previous_char - return a pointer to the - previous character in a string --------------------------------------------------*/ +//------------------------------------------------- +// utf8_previous_char - return a pointer to the +// previous character in a string +//------------------------------------------------- /** * @fn const char *utf8_previous_char(const char *utf8string) @@ -326,11 +368,11 @@ const char *utf8_previous_char(const char *utf8string) } -/*------------------------------------------------- - utf8_is_valid_string - return true if the - given string is a properly formed sequence of - UTF-8 characters --------------------------------------------------*/ +//------------------------------------------------- +// utf8_is_valid_string - return true if the +// given string is a properly formed sequence of +// UTF-8 characters +//------------------------------------------------- /** * @fn int utf8_is_valid_string(const char *utf8string) @@ -342,24 +384,24 @@ const char *utf8_previous_char(const char *utf8string) * @return An int. */ -int utf8_is_valid_string(const char *utf8string) +bool utf8_is_valid_string(const char *utf8string) { int remaining_length = strlen(utf8string); while (*utf8string != 0) { - unicode_char uchar = 0; + char32_t uchar = 0; int charlen; - /* extract the current character and verify it */ + // extract the current character and verify it charlen = uchar_from_utf8(&uchar, utf8string, remaining_length); if (charlen <= 0 || uchar == 0 || !uchar_isvalid(uchar)) - return FALSE; + return false; - /* advance */ + // advance utf8string += charlen; remaining_length -= charlen; } - return TRUE; + return true; } |