diff options
author | 2019-03-25 23:13:40 +0100 | |
---|---|---|
committer | 2019-03-25 23:13:40 +0100 | |
commit | b380514764cf857469bae61c11143a19f79a74c5 (patch) | |
tree | 63c8012e262618f08a332da31dd714281aa2c5ed /src/lib/util/unicode.cpp | |
parent | c24473ddff715ecec2e258a6eb38960cf8c8e98e (diff) |
Revert "conflict resolution (nw)"
This reverts commit c24473ddff715ecec2e258a6eb38960cf8c8e98e, reversing
changes made to 009cba4fb8102102168ef32870892438327f3705.
Diffstat (limited to 'src/lib/util/unicode.cpp')
-rw-r--r-- | src/lib/util/unicode.cpp | 170 |
1 files changed, 69 insertions, 101 deletions
diff --git a/src/lib/util/unicode.cpp b/src/lib/util/unicode.cpp index 8e6478beee3..5151a1e101d 100644 --- a/src/lib/util/unicode.cpp +++ b/src/lib/util/unicode.cpp @@ -21,67 +21,6 @@ #include <locale> -namespace { - -//------------------------------------------------- -// internal_normalize_unicode - uses utf8proc to -// normalize unicode -//------------------------------------------------- - -std::string internal_normalize_unicode( - char const *s, - size_t length, - unicode_normalization_form normalization_form, - bool fold_case, - bool null_terminated) -{ - // convert the normalization form - int options; - switch (normalization_form) - { - case unicode_normalization_form::C: - options = UTF8PROC_STABLE | UTF8PROC_COMPOSE; - break; - case unicode_normalization_form::D: - options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE; - break; - case unicode_normalization_form::KC: - options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT; - break; - case unicode_normalization_form::KD: - options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT; - break; - default: - throw false; - } - - // perform case folding? - if (fold_case) - options |= UTF8PROC_CASEFOLD; - - // use NUL terminator to determine length? - if (null_terminated) - options |= UTF8PROC_NULLTERM; - - // invoke utf8proc - utf8proc_uint8_t *utf8proc_result(nullptr); - utf8proc_ssize_t const utf8proc_result_length(utf8proc_map(reinterpret_cast<utf8proc_uint8_t const *>(s), length, &utf8proc_result, utf8proc_option_t(options))); - - // conver the result - std::string result; - if (utf8proc_result) - { - if (utf8proc_result_length > 0) - result.assign(reinterpret_cast<char const *>(utf8proc_result), utf8proc_result_length); - free(utf8proc_result); - } - - return result; -} - -} // anonymous namespace - - //------------------------------------------------- // uchar_isvalid - return true if a given // character is a legitimate unicode character @@ -127,53 +66,56 @@ bool uchar_is_digit(char32_t uchar) int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count) { + char32_t c, minchar; + int auxlen, i; + char auxchar; + // validate parameters - if (!utf8char || !count) + if (utf8char == nullptr || count == 0) return 0; // start with the first byte - char32_t c = (unsigned char)*utf8char; + c = (unsigned char) *utf8char; count--; utf8char++; // based on that, determine how many additional bytes we need - char32_t minchar; - int auxlen; - if ((c & 0x80) == 0x00) + if (c < 0x80) { // unicode char 0x00000000 - 0x0000007F + c &= 0x7f; auxlen = 0; minchar = 0x00000000; } - else if ((c & 0xe0) == 0xc0) + else if (c >= 0xc0 && c < 0xe0) { // unicode char 0x00000080 - 0x000007FF c &= 0x1f; auxlen = 1; minchar = 0x00000080; } - else if ((c & 0xf0) == 0xe0) + else if (c >= 0xe0 && c < 0xf0) { // unicode char 0x00000800 - 0x0000FFFF c &= 0x0f; auxlen = 2; minchar = 0x00000800; } - else if ((c & 0xf8) == 0xf0) + else if (c >= 0xf0 && c < 0xf8) { // unicode char 0x00010000 - 0x001FFFFF c &= 0x07; auxlen = 3; minchar = 0x00010000; } - else if ((c & 0xfc) == 0xf8) + else if (c >= 0xf8 && c < 0xfc) { // unicode char 0x00200000 - 0x03FFFFFF c &= 0x03; auxlen = 4; minchar = 0x00200000; } - else if ((c & 0xfe) == 0xfc) + else if (c >= 0xfc && c < 0xfe) { // unicode char 0x04000000 - 0x7FFFFFFF c &= 0x01; @@ -191,9 +133,9 @@ int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count) return -1; // we now know how long the char is, now compute it - for (int i = 0; i < auxlen; i++) + for (i = 0; i < auxlen; i++) { - char32_t const auxchar = (unsigned char)utf8char[i]; + auxchar = utf8char[i]; // all auxillary chars must be between 0x80-0xbf if ((auxchar & 0xc0) != 0x80) @@ -264,28 +206,6 @@ int uchar_from_utf16f(char32_t *uchar, const char16_t *utf16char, size_t count) //------------------------------------------------- -// ustr_from_utf8 - convert a UTF-8 sequence into -// into a Unicode string -//------------------------------------------------- - -std::u32string ustr_from_utf8(const std::string &utf8str) -{ - std::u32string result; - char const *utf8char(utf8str.c_str()); - size_t remaining(utf8str.length()); - while (remaining) - { - char32_t ch; - int const consumed(uchar_from_utf8(&ch, utf8char, remaining)); - result.append(1, (consumed > 0) ? ch : char32_t(0x00fffdU)); - utf8char += (consumed > 0) ? consumed : 1; - remaining -= (consumed > 0) ? consumed : 1; - } - return result; -} - - -//------------------------------------------------- // utf8_from_uchar - convert a unicode character // into a UTF-8 sequence //------------------------------------------------- @@ -468,13 +388,61 @@ std::string utf8_from_wstring(const std::wstring &string) //------------------------------------------------- +// internal_normalize_unicode - uses utf8proc to +// normalize unicode +//------------------------------------------------- + +static std::string internal_normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool null_terminated) +{ + // convert the normalization form + int options; + switch (normalization_form) + { + case unicode_normalization_form::C: + options = UTF8PROC_STABLE | UTF8PROC_COMPOSE; + break; + case unicode_normalization_form::D: + options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE; + break; + case unicode_normalization_form::KC: + options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT; + break; + case unicode_normalization_form::KD: + options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT; + break; + default: + throw false; + } + + // was this null terminated? + if (null_terminated) + options |= UTF8PROC_NULLTERM; + + // invoke utf8proc + utf8proc_uint8_t *utf8proc_result; + utf8proc_ssize_t utf8proc_result_length = utf8proc_map((utf8proc_uint8_t *) s, length, &utf8proc_result, (utf8proc_option_t)options); + + // conver the result + std::string result; + if (utf8proc_result) + { + if (utf8proc_result_length > 0) + result = std::string((const char *)utf8proc_result, utf8proc_result_length); + free(utf8proc_result); + } + + return result; +} + + +//------------------------------------------------- // normalize_unicode - uses utf8proc to normalize // unicode //------------------------------------------------- -std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form, bool fold_case) +std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form) { - return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, fold_case, false); + return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, false); } @@ -483,9 +451,9 @@ std::string normalize_unicode(const std::string &s, unicode_normalization_form n // unicode //------------------------------------------------- -std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form, bool fold_case) +std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form) { - return internal_normalize_unicode(s, 0, normalization_form, fold_case, true); + return internal_normalize_unicode(s, 0, normalization_form, true); } @@ -494,9 +462,9 @@ std::string normalize_unicode(const char *s, unicode_normalization_form normaliz // unicode //------------------------------------------------- -std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool fold_case) +std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form) { - return internal_normalize_unicode(s, length, normalization_form, fold_case, false); + return internal_normalize_unicode(s, length, normalization_form, false); } |