From 97b67170277437131adf6ed4d60139c172529e4f Mon Sep 17 00:00:00 2001 From: Vas Crabb Date: Tue, 26 Mar 2019 11:13:37 +1100 Subject: (nw) Clean up the mess on master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This effectively reverts b380514764cf857469bae61c11143a19f79a74c5 and c24473ddff715ecec2e258a6eb38960cf8c8e98e, restoring the state at 598cd5227223c3b04ca31f0dbc1981256d9ea3ff. Before pushing, please check that what you're about to push is sane. Check your local commit log and ensure there isn't anything out-of-place before pushing to mainline. When things like this happen, it wastes everyone's time. I really don't need this in a week when real work™ is busting my balls and I'm behind where I want to be with preparing for MAME release. --- src/lib/util/unicode.cpp | 170 ++++++++++++++++++++++++++++------------------- 1 file changed, 101 insertions(+), 69 deletions(-) (limited to 'src/lib/util/unicode.cpp') diff --git a/src/lib/util/unicode.cpp b/src/lib/util/unicode.cpp index 5151a1e101d..8e6478beee3 100644 --- a/src/lib/util/unicode.cpp +++ b/src/lib/util/unicode.cpp @@ -21,6 +21,67 @@ #include +namespace { + +//------------------------------------------------- +// internal_normalize_unicode - uses utf8proc to +// normalize unicode +//------------------------------------------------- + +std::string internal_normalize_unicode( + char const *s, + size_t length, + unicode_normalization_form normalization_form, + bool fold_case, + bool null_terminated) +{ + // convert the normalization form + int options; + switch (normalization_form) + { + case unicode_normalization_form::C: + options = UTF8PROC_STABLE | UTF8PROC_COMPOSE; + break; + case unicode_normalization_form::D: + options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE; + break; + case unicode_normalization_form::KC: + options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT; + break; + case unicode_normalization_form::KD: + options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT; + break; + default: + throw false; + } + + // perform case folding? + if (fold_case) + options |= UTF8PROC_CASEFOLD; + + // use NUL terminator to determine length? + if (null_terminated) + options |= UTF8PROC_NULLTERM; + + // invoke utf8proc + utf8proc_uint8_t *utf8proc_result(nullptr); + utf8proc_ssize_t const utf8proc_result_length(utf8proc_map(reinterpret_cast(s), length, &utf8proc_result, utf8proc_option_t(options))); + + // conver the result + std::string result; + if (utf8proc_result) + { + if (utf8proc_result_length > 0) + result.assign(reinterpret_cast(utf8proc_result), utf8proc_result_length); + free(utf8proc_result); + } + + return result; +} + +} // anonymous namespace + + //------------------------------------------------- // uchar_isvalid - return true if a given // character is a legitimate unicode character @@ -66,56 +127,53 @@ bool uchar_is_digit(char32_t uchar) int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count) { - char32_t c, minchar; - int auxlen, i; - char auxchar; - // validate parameters - if (utf8char == nullptr || count == 0) + if (!utf8char || !count) return 0; // start with the first byte - c = (unsigned char) *utf8char; + char32_t c = (unsigned char)*utf8char; count--; utf8char++; // based on that, determine how many additional bytes we need - if (c < 0x80) + char32_t minchar; + int auxlen; + if ((c & 0x80) == 0x00) { // unicode char 0x00000000 - 0x0000007F - c &= 0x7f; auxlen = 0; minchar = 0x00000000; } - else if (c >= 0xc0 && c < 0xe0) + else if ((c & 0xe0) == 0xc0) { // unicode char 0x00000080 - 0x000007FF c &= 0x1f; auxlen = 1; minchar = 0x00000080; } - else if (c >= 0xe0 && c < 0xf0) + else if ((c & 0xf0) == 0xe0) { // unicode char 0x00000800 - 0x0000FFFF c &= 0x0f; auxlen = 2; minchar = 0x00000800; } - else if (c >= 0xf0 && c < 0xf8) + else if ((c & 0xf8) == 0xf0) { // unicode char 0x00010000 - 0x001FFFFF c &= 0x07; auxlen = 3; minchar = 0x00010000; } - else if (c >= 0xf8 && c < 0xfc) + else if ((c & 0xfc) == 0xf8) { // unicode char 0x00200000 - 0x03FFFFFF c &= 0x03; auxlen = 4; minchar = 0x00200000; } - else if (c >= 0xfc && c < 0xfe) + else if ((c & 0xfe) == 0xfc) { // unicode char 0x04000000 - 0x7FFFFFFF c &= 0x01; @@ -133,9 +191,9 @@ int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count) return -1; // we now know how long the char is, now compute it - for (i = 0; i < auxlen; i++) + for (int i = 0; i < auxlen; i++) { - auxchar = utf8char[i]; + char32_t const auxchar = (unsigned char)utf8char[i]; // all auxillary chars must be between 0x80-0xbf if ((auxchar & 0xc0) != 0x80) @@ -205,6 +263,28 @@ int uchar_from_utf16f(char32_t *uchar, const char16_t *utf16char, size_t count) } +//------------------------------------------------- +// ustr_from_utf8 - convert a UTF-8 sequence into +// into a Unicode string +//------------------------------------------------- + +std::u32string ustr_from_utf8(const std::string &utf8str) +{ + std::u32string result; + char const *utf8char(utf8str.c_str()); + size_t remaining(utf8str.length()); + while (remaining) + { + char32_t ch; + int const consumed(uchar_from_utf8(&ch, utf8char, remaining)); + result.append(1, (consumed > 0) ? ch : char32_t(0x00fffdU)); + utf8char += (consumed > 0) ? consumed : 1; + remaining -= (consumed > 0) ? consumed : 1; + } + return result; +} + + //------------------------------------------------- // utf8_from_uchar - convert a unicode character // into a UTF-8 sequence @@ -387,62 +467,14 @@ std::string utf8_from_wstring(const std::wstring &string) } -//------------------------------------------------- -// internal_normalize_unicode - uses utf8proc to -// normalize unicode -//------------------------------------------------- - -static std::string internal_normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool null_terminated) -{ - // convert the normalization form - int options; - switch (normalization_form) - { - case unicode_normalization_form::C: - options = UTF8PROC_STABLE | UTF8PROC_COMPOSE; - break; - case unicode_normalization_form::D: - options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE; - break; - case unicode_normalization_form::KC: - options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT; - break; - case unicode_normalization_form::KD: - options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT; - break; - default: - throw false; - } - - // was this null terminated? - if (null_terminated) - options |= UTF8PROC_NULLTERM; - - // invoke utf8proc - utf8proc_uint8_t *utf8proc_result; - utf8proc_ssize_t utf8proc_result_length = utf8proc_map((utf8proc_uint8_t *) s, length, &utf8proc_result, (utf8proc_option_t)options); - - // conver the result - std::string result; - if (utf8proc_result) - { - if (utf8proc_result_length > 0) - result = std::string((const char *)utf8proc_result, utf8proc_result_length); - free(utf8proc_result); - } - - return result; -} - - //------------------------------------------------- // normalize_unicode - uses utf8proc to normalize // unicode //------------------------------------------------- -std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form) +std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form, bool fold_case) { - return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, false); + return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, fold_case, false); } @@ -451,9 +483,9 @@ std::string normalize_unicode(const std::string &s, unicode_normalization_form n // unicode //------------------------------------------------- -std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form) +std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form, bool fold_case) { - return internal_normalize_unicode(s, 0, normalization_form, true); + return internal_normalize_unicode(s, 0, normalization_form, fold_case, true); } @@ -462,9 +494,9 @@ std::string normalize_unicode(const char *s, unicode_normalization_form normaliz // unicode //------------------------------------------------- -std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form) +std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool fold_case) { - return internal_normalize_unicode(s, length, normalization_form, false); + return internal_normalize_unicode(s, length, normalization_form, fold_case, false); } -- cgit v1.2.3-70-g09d2