summaryrefslogtreecommitdiffstatshomepage
path: root/src/lib/util/unicode.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/util/unicode.cpp')
-rw-r--r--src/lib/util/unicode.cpp170
1 files changed, 101 insertions, 69 deletions
diff --git a/src/lib/util/unicode.cpp b/src/lib/util/unicode.cpp
index 5151a1e101d..8e6478beee3 100644
--- a/src/lib/util/unicode.cpp
+++ b/src/lib/util/unicode.cpp
@@ -21,6 +21,67 @@
#include <locale>
+namespace {
+
+//-------------------------------------------------
+// internal_normalize_unicode - uses utf8proc to
+// normalize unicode
+//-------------------------------------------------
+
+std::string internal_normalize_unicode(
+ char const *s,
+ size_t length,
+ unicode_normalization_form normalization_form,
+ bool fold_case,
+ bool null_terminated)
+{
+ // convert the normalization form
+ int options;
+ switch (normalization_form)
+ {
+ case unicode_normalization_form::C:
+ options = UTF8PROC_STABLE | UTF8PROC_COMPOSE;
+ break;
+ case unicode_normalization_form::D:
+ options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE;
+ break;
+ case unicode_normalization_form::KC:
+ options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT;
+ break;
+ case unicode_normalization_form::KD:
+ options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT;
+ break;
+ default:
+ throw false;
+ }
+
+ // perform case folding?
+ if (fold_case)
+ options |= UTF8PROC_CASEFOLD;
+
+ // use NUL terminator to determine length?
+ if (null_terminated)
+ options |= UTF8PROC_NULLTERM;
+
+ // invoke utf8proc
+ utf8proc_uint8_t *utf8proc_result(nullptr);
+ utf8proc_ssize_t const utf8proc_result_length(utf8proc_map(reinterpret_cast<utf8proc_uint8_t const *>(s), length, &utf8proc_result, utf8proc_option_t(options)));
+
+ // conver the result
+ std::string result;
+ if (utf8proc_result)
+ {
+ if (utf8proc_result_length > 0)
+ result.assign(reinterpret_cast<char const *>(utf8proc_result), utf8proc_result_length);
+ free(utf8proc_result);
+ }
+
+ return result;
+}
+
+} // anonymous namespace
+
+
//-------------------------------------------------
// uchar_isvalid - return true if a given
// character is a legitimate unicode character
@@ -66,56 +127,53 @@ bool uchar_is_digit(char32_t uchar)
int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count)
{
- char32_t c, minchar;
- int auxlen, i;
- char auxchar;
-
// validate parameters
- if (utf8char == nullptr || count == 0)
+ if (!utf8char || !count)
return 0;
// start with the first byte
- c = (unsigned char) *utf8char;
+ char32_t c = (unsigned char)*utf8char;
count--;
utf8char++;
// based on that, determine how many additional bytes we need
- if (c < 0x80)
+ char32_t minchar;
+ int auxlen;
+ if ((c & 0x80) == 0x00)
{
// unicode char 0x00000000 - 0x0000007F
- c &= 0x7f;
auxlen = 0;
minchar = 0x00000000;
}
- else if (c >= 0xc0 && c < 0xe0)
+ else if ((c & 0xe0) == 0xc0)
{
// unicode char 0x00000080 - 0x000007FF
c &= 0x1f;
auxlen = 1;
minchar = 0x00000080;
}
- else if (c >= 0xe0 && c < 0xf0)
+ else if ((c & 0xf0) == 0xe0)
{
// unicode char 0x00000800 - 0x0000FFFF
c &= 0x0f;
auxlen = 2;
minchar = 0x00000800;
}
- else if (c >= 0xf0 && c < 0xf8)
+ else if ((c & 0xf8) == 0xf0)
{
// unicode char 0x00010000 - 0x001FFFFF
c &= 0x07;
auxlen = 3;
minchar = 0x00010000;
}
- else if (c >= 0xf8 && c < 0xfc)
+ else if ((c & 0xfc) == 0xf8)
{
// unicode char 0x00200000 - 0x03FFFFFF
c &= 0x03;
auxlen = 4;
minchar = 0x00200000;
}
- else if (c >= 0xfc && c < 0xfe)
+ else if ((c & 0xfe) == 0xfc)
{
// unicode char 0x04000000 - 0x7FFFFFFF
c &= 0x01;
@@ -133,9 +191,9 @@ int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count)
return -1;
// we now know how long the char is, now compute it
- for (i = 0; i < auxlen; i++)
+ for (int i = 0; i < auxlen; i++)
{
- auxchar = utf8char[i];
+ char32_t const auxchar = (unsigned char)utf8char[i];
// all auxillary chars must be between 0x80-0xbf
if ((auxchar & 0xc0) != 0x80)
@@ -206,6 +264,28 @@ int uchar_from_utf16f(char32_t *uchar, const char16_t *utf16char, size_t count)
//-------------------------------------------------
+// ustr_from_utf8 - convert a UTF-8 sequence into
+// into a Unicode string
+//-------------------------------------------------
+
+std::u32string ustr_from_utf8(const std::string &utf8str)
+{
+ std::u32string result;
+ char const *utf8char(utf8str.c_str());
+ size_t remaining(utf8str.length());
+ while (remaining)
+ {
+ char32_t ch;
+ int const consumed(uchar_from_utf8(&ch, utf8char, remaining));
+ result.append(1, (consumed > 0) ? ch : char32_t(0x00fffdU));
+ utf8char += (consumed > 0) ? consumed : 1;
+ remaining -= (consumed > 0) ? consumed : 1;
+ }
+ return result;
+}
+
+
+//-------------------------------------------------
// utf8_from_uchar - convert a unicode character
// into a UTF-8 sequence
//-------------------------------------------------
@@ -388,61 +468,13 @@ std::string utf8_from_wstring(const std::wstring &string)
//-------------------------------------------------
-// internal_normalize_unicode - uses utf8proc to
-// normalize unicode
-//-------------------------------------------------
-
-static std::string internal_normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool null_terminated)
-{
- // convert the normalization form
- int options;
- switch (normalization_form)
- {
- case unicode_normalization_form::C:
- options = UTF8PROC_STABLE | UTF8PROC_COMPOSE;
- break;
- case unicode_normalization_form::D:
- options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE;
- break;
- case unicode_normalization_form::KC:
- options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT;
- break;
- case unicode_normalization_form::KD:
- options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT;
- break;
- default:
- throw false;
- }
-
- // was this null terminated?
- if (null_terminated)
- options |= UTF8PROC_NULLTERM;
-
- // invoke utf8proc
- utf8proc_uint8_t *utf8proc_result;
- utf8proc_ssize_t utf8proc_result_length = utf8proc_map((utf8proc_uint8_t *) s, length, &utf8proc_result, (utf8proc_option_t)options);
-
- // conver the result
- std::string result;
- if (utf8proc_result)
- {
- if (utf8proc_result_length > 0)
- result = std::string((const char *)utf8proc_result, utf8proc_result_length);
- free(utf8proc_result);
- }
-
- return result;
-}
-
-
-//-------------------------------------------------
// normalize_unicode - uses utf8proc to normalize
// unicode
//-------------------------------------------------
-std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form)
+std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form, bool fold_case)
{
- return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, false);
+ return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, fold_case, false);
}
@@ -451,9 +483,9 @@ std::string normalize_unicode(const std::string &s, unicode_normalization_form n
// unicode
//-------------------------------------------------
-std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form)
+std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form, bool fold_case)
{
- return internal_normalize_unicode(s, 0, normalization_form, true);
+ return internal_normalize_unicode(s, 0, normalization_form, fold_case, true);
}
@@ -462,9 +494,9 @@ std::string normalize_unicode(const char *s, unicode_normalization_form normaliz
// unicode
//-------------------------------------------------
-std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form)
+std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool fold_case)
{
- return internal_normalize_unicode(s, length, normalization_form, false);
+ return internal_normalize_unicode(s, length, normalization_form, fold_case, false);
}