summaryrefslogtreecommitdiffstatshomepage
path: root/src/lib/util/unicode.cpp
diff options
context:
space:
mode:
author sparrowred <redsparrow@freenet.de>2016-12-23 14:48:09 +0100
committer sparrowred <redsparrow@freenet.de>2016-12-23 14:48:09 +0100
commitd0b0de1aec45c0f9b66c931efb9902874cb93728 (patch)
tree98a11fe89a65cc62fa8114a0b6c5312c21810ede /src/lib/util/unicode.cpp
parentc0561727b406ed189039d81fac7c69fef4c79913 (diff)
parent63b7b9d521712404dc35156ef8d7e606440b4156 (diff)
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'src/lib/util/unicode.cpp')
-rw-r--r--src/lib/util/unicode.cpp224
1 files changed, 133 insertions, 91 deletions
diff --git a/src/lib/util/unicode.cpp b/src/lib/util/unicode.cpp
index e2c3956cedb..a6c5515fbf6 100644
--- a/src/lib/util/unicode.cpp
+++ b/src/lib/util/unicode.cpp
@@ -11,96 +11,123 @@
#include "unicode.h"
-/*-------------------------------------------------
- uchar_isvalid - return true if a given
- character is a legitimate unicode character
--------------------------------------------------*/
+//-------------------------------------------------
+// uchar_isvalid - return true if a given
+// character is a legitimate unicode character
+//-------------------------------------------------
-int uchar_isvalid(unicode_char uchar)
+bool uchar_isvalid(char32_t uchar)
{
return (uchar < 0x110000) && !((uchar >= 0xd800) && (uchar <= 0xdfff));
}
-/*-------------------------------------------------
- uchar_from_utf8 - convert a UTF-8 sequence
- into a unicode character
--------------------------------------------------*/
+//-------------------------------------------------
+// uchar_is_printable - tests to see if a unicode
+// char is printable
+//-------------------------------------------------
-int uchar_from_utf8(unicode_char *uchar, const char *utf8char, size_t count)
+bool uchar_is_printable(char32_t uchar)
{
- unicode_char c, minchar;
+ return
+ !(0x0001f >= uchar) && // C0 control
+ !((0x0007f <= uchar) && (0x0009f >= uchar)) && // DEL and C1 control
+ !((0x0fdd0 <= uchar) && (0x0fddf >= uchar)) && // noncharacters
+ !(0x0fffe == (uchar & 0x0ffff)) && // byte-order detection noncharacter
+ !(0x0ffff == (uchar & 0x0ffff)); // the other noncharacter
+}
+
+
+//-------------------------------------------------
+// uchar_is_digit - tests to see if a unicode
+// char is a digit
+//-------------------------------------------------
+
+bool uchar_is_digit(char32_t uchar)
+{
+ return uchar >= '0' && uchar <= '9';
+}
+
+
+//-------------------------------------------------
+// uchar_from_utf8 - convert a UTF-8 sequence
+// into a unicode character
+//-----------------------------------------------
+
+int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count)
+{
+ char32_t c, minchar;
int auxlen, i;
char auxchar;
- /* validate parameters */
+ // validate parameters
if (utf8char == nullptr || count == 0)
return 0;
- /* start with the first byte */
+ // start with the first byte
c = (unsigned char) *utf8char;
count--;
utf8char++;
- /* based on that, determine how many additional bytes we need */
+ // based on that, determine how many additional bytes we need
if (c < 0x80)
{
- /* unicode char 0x00000000 - 0x0000007F */
+ // unicode char 0x00000000 - 0x0000007F
c &= 0x7f;
auxlen = 0;
minchar = 0x00000000;
}
else if (c >= 0xc0 && c < 0xe0)
{
- /* unicode char 0x00000080 - 0x000007FF */
+ // unicode char 0x00000080 - 0x000007FF
c &= 0x1f;
auxlen = 1;
minchar = 0x00000080;
}
else if (c >= 0xe0 && c < 0xf0)
{
- /* unicode char 0x00000800 - 0x0000FFFF */
+ // unicode char 0x00000800 - 0x0000FFFF
c &= 0x0f;
auxlen = 2;
minchar = 0x00000800;
}
else if (c >= 0xf0 && c < 0xf8)
{
- /* unicode char 0x00010000 - 0x001FFFFF */
+ // unicode char 0x00010000 - 0x001FFFFF
c &= 0x07;
auxlen = 3;
minchar = 0x00010000;
}
else if (c >= 0xf8 && c < 0xfc)
{
- /* unicode char 0x00200000 - 0x03FFFFFF */
+ // unicode char 0x00200000 - 0x03FFFFFF
c &= 0x03;
auxlen = 4;
minchar = 0x00200000;
}
else if (c >= 0xfc && c < 0xfe)
{
- /* unicode char 0x04000000 - 0x7FFFFFFF */
+ // unicode char 0x04000000 - 0x7FFFFFFF
c &= 0x01;
auxlen = 5;
minchar = 0x04000000;
}
else
{
- /* invalid */
+ // invalid
return -1;
}
- /* exceeds the count? */
+ // exceeds the count?
if (auxlen > count)
return -1;
- /* we now know how long the char is, now compute it */
+ // we now know how long the char is, now compute it
for (i = 0; i < auxlen; i++)
{
auxchar = utf8char[i];
- /* all auxillary chars must be between 0x80-0xbf */
+ // all auxillary chars must be between 0x80-0xbf
if ((auxchar & 0xc0) != 0x80)
return -1;
@@ -108,7 +135,7 @@ int uchar_from_utf8(unicode_char *uchar, const char *utf8char, size_t count)
c |= auxchar & 0x3f;
}
- /* make sure that this char is above the minimum */
+ // make sure that this char is above the minimum
if (c < minchar)
return -1;
@@ -117,32 +144,32 @@ int uchar_from_utf8(unicode_char *uchar, const char *utf8char, size_t count)
}
-/*-------------------------------------------------
- uchar_from_utf16 - convert a UTF-16 sequence
- into a unicode character
--------------------------------------------------*/
+//-------------------------------------------------
+// uchar_from_utf16 - convert a UTF-16 sequence
+// into a unicode character
+//-------------------------------------------------
-int uchar_from_utf16(unicode_char *uchar, const utf16_char *utf16char, size_t count)
+int uchar_from_utf16(char32_t *uchar, const char16_t *utf16char, size_t count)
{
int rc = -1;
- /* validate parameters */
+ // validate parameters
if (utf16char == nullptr || count == 0)
- return 0;
-
- /* handle the two-byte case */
+ {
+ rc = 0;
+ }
if (utf16char[0] >= 0xd800 && utf16char[0] <= 0xdbff)
{
+ // handle the two-byte case
if (count > 1 && utf16char[1] >= 0xdc00 && utf16char[1] <= 0xdfff)
{
*uchar = 0x10000 + ((utf16char[0] & 0x3ff) * 0x400) + (utf16char[1] & 0x3ff);
rc = 2;
}
}
-
- /* handle the one-byte case */
else if (utf16char[0] < 0xdc00 || utf16char[0] > 0xdfff)
{
+ // handle the one-byte case
*uchar = utf16char[0];
rc = 1;
}
@@ -151,47 +178,47 @@ int uchar_from_utf16(unicode_char *uchar, const utf16_char *utf16char, size_t co
}
-/*-------------------------------------------------
- uchar_from_utf16f - convert a UTF-16 sequence
- into a unicode character from a flipped
- byte order
--------------------------------------------------*/
+//-------------------------------------------------
+// uchar_from_utf16f - convert a UTF-16 sequence
+// into a unicode character from a flipped
+// byte order
+//-------------------------------------------------
-int uchar_from_utf16f(unicode_char *uchar, const utf16_char *utf16char, size_t count)
+int uchar_from_utf16f(char32_t *uchar, const char16_t *utf16char, size_t count)
{
- utf16_char buf[2] = {0};
+ char16_t buf[2] = {0};
if (count > 0)
- buf[0] = FLIPENDIAN_INT16(utf16char[0]);
+ buf[0] = flipendian_int16(utf16char[0]);
if (count > 1)
- buf[1] = FLIPENDIAN_INT16(utf16char[1]);
+ buf[1] = flipendian_int16(utf16char[1]);
return uchar_from_utf16(uchar, buf, count);
}
-/*-------------------------------------------------
- utf8_from_uchar - convert a unicode character
- into a UTF-8 sequence
--------------------------------------------------*/
+//-------------------------------------------------
+// utf8_from_uchar - convert a unicode character
+// into a UTF-8 sequence
+//-------------------------------------------------
-int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar)
+int utf8_from_uchar(char *utf8string, size_t count, char32_t uchar)
{
int rc = 0;
- /* error on invalid characters */
+ // error on invalid characters
if (!uchar_isvalid(uchar))
return -1;
- /* based on the value, output the appropriate number of bytes */
+ // based on the value, output the appropriate number of bytes
if (uchar < 0x80)
{
- /* unicode char 0x00000000 - 0x0000007F */
+ // unicode char 0x00000000 - 0x0000007F
if (count < 1)
return -1;
utf8string[rc++] = (char) uchar;
}
else if (uchar < 0x800)
{
- /* unicode char 0x00000080 - 0x000007FF */
+ // unicode char 0x00000080 - 0x000007FF
if (count < 2)
return -1;
utf8string[rc++] = ((char) (uchar >> 6)) | 0xC0;
@@ -199,7 +226,7 @@ int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar)
}
else if (uchar < 0x10000)
{
- /* unicode char 0x00000800 - 0x0000FFFF */
+ // unicode char 0x00000800 - 0x0000FFFF
if (count < 3)
return -1;
utf8string[rc++] = ((char) (uchar >> 12)) | 0xE0;
@@ -208,7 +235,7 @@ int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar)
}
else if (uchar < 0x00200000)
{
- /* unicode char 0x00010000 - 0x001FFFFF */
+ // unicode char 0x00010000 - 0x001FFFFF
if (count < 4)
return -1;
utf8string[rc++] = ((char) (uchar >> 18)) | 0xF0;
@@ -218,7 +245,7 @@ int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar)
}
else if (uchar < 0x04000000)
{
- /* unicode char 0x00200000 - 0x03FFFFFF */
+ // unicode char 0x00200000 - 0x03FFFFFF
if (count < 5)
return -1;
utf8string[rc++] = ((char) (uchar >> 24)) | 0xF8;
@@ -229,7 +256,7 @@ int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar)
}
else if (uchar < 0x80000000)
{
- /* unicode char 0x04000000 - 0x7FFFFFFF */
+ // unicode char 0x04000000 - 0x7FFFFFFF
if (count < 6)
return -1;
utf8string[rc++] = ((char) (uchar >> 30)) | 0xFC;
@@ -246,67 +273,82 @@ int utf8_from_uchar(char *utf8string, size_t count, unicode_char uchar)
}
-/*-------------------------------------------------
- utf16_from_uchar - convert a unicode character
- into a UTF-16 sequence
--------------------------------------------------*/
+//-------------------------------------------------
+// utf8_from_uchar - convert a unicode character
+// into a UTF-8 sequence
+//-------------------------------------------------
-int utf16_from_uchar(utf16_char *utf16string, size_t count, unicode_char uchar)
+std::string utf8_from_uchar(char32_t uchar)
+{
+ char buffer[UTF8_CHAR_MAX];
+ auto len = utf8_from_uchar(buffer, ARRAY_LENGTH(buffer), uchar);
+ return std::string(buffer, len);
+}
+
+
+//-------------------------------------------------
+// utf16_from_uchar - convert a unicode character
+// into a UTF-16 sequence
+//-------------------------------------------------
+
+int utf16_from_uchar(char16_t *utf16string, size_t count, char32_t uchar)
{
int rc;
- /* error on invalid characters */
+ // error on invalid characters
if (!uchar_isvalid(uchar))
return -1;
- /* single word case */
if (uchar < 0x10000)
{
+ // single word case
if (count < 1)
return -1;
- utf16string[0] = (utf16_char) uchar;
+ utf16string[0] = (char16_t) uchar;
rc = 1;
}
-
- /* double word case */
else if (uchar < 0x100000)
{
+ // double word case
if (count < 2)
return -1;
+ uchar -= 0x10000;
utf16string[0] = ((uchar >> 10) & 0x03ff) | 0xd800;
utf16string[1] = ((uchar >> 0) & 0x03ff) | 0xdc00;
rc = 2;
}
else
+ {
return -1;
+ }
return rc;
}
-/*-------------------------------------------------
- utf16_from_uchar - convert a unicode character
- into a UTF-16 sequence with flipped endianness
--------------------------------------------------*/
+//-------------------------------------------------
+// utf16_from_uchar - convert a unicode character
+// into a UTF-16 sequence with flipped endianness
+//-------------------------------------------------
-int utf16f_from_uchar(utf16_char *utf16string, size_t count, unicode_char uchar)
+int utf16f_from_uchar(char16_t *utf16string, size_t count, char32_t uchar)
{
int rc;
- utf16_char buf[2] = { 0, 0 };
+ char16_t buf[2] = { 0, 0 };
rc = utf16_from_uchar(buf, count, uchar);
if (rc >= 1)
- utf16string[0] = FLIPENDIAN_INT16(buf[0]);
+ utf16string[0] = flipendian_int16(buf[0]);
if (rc >= 2)
- utf16string[1] = FLIPENDIAN_INT16(buf[1]);
+ utf16string[1] = flipendian_int16(buf[1]);
return rc;
}
-/*-------------------------------------------------
- utf8_previous_char - return a pointer to the
- previous character in a string
--------------------------------------------------*/
+//-------------------------------------------------
+// utf8_previous_char - return a pointer to the
+// previous character in a string
+//-------------------------------------------------
/**
* @fn const char *utf8_previous_char(const char *utf8string)
@@ -326,11 +368,11 @@ const char *utf8_previous_char(const char *utf8string)
}
-/*-------------------------------------------------
- utf8_is_valid_string - return true if the
- given string is a properly formed sequence of
- UTF-8 characters
--------------------------------------------------*/
+//-------------------------------------------------
+// utf8_is_valid_string - return true if the
+// given string is a properly formed sequence of
+// UTF-8 characters
+//-------------------------------------------------
/**
* @fn int utf8_is_valid_string(const char *utf8string)
@@ -342,24 +384,24 @@ const char *utf8_previous_char(const char *utf8string)
* @return An int.
*/
-int utf8_is_valid_string(const char *utf8string)
+bool utf8_is_valid_string(const char *utf8string)
{
int remaining_length = strlen(utf8string);
while (*utf8string != 0)
{
- unicode_char uchar = 0;
+ char32_t uchar = 0;
int charlen;
- /* extract the current character and verify it */
+ // extract the current character and verify it
charlen = uchar_from_utf8(&uchar, utf8string, remaining_length);
if (charlen <= 0 || uchar == 0 || !uchar_isvalid(uchar))
- return FALSE;
+ return false;
- /* advance */
+ // advance
utf8string += charlen;
remaining_length -= charlen;
}
- return TRUE;
+ return true;
}