diff options
author | andreasnaive <andreasnaive@gmail.com> | 2019-03-25 23:13:40 +0100 |
---|---|---|
committer | andreasnaive <andreasnaive@gmail.com> | 2019-03-25 23:13:40 +0100 |
commit | b380514764cf857469bae61c11143a19f79a74c5 (patch) | |
tree | 63c8012e262618f08a332da31dd714281aa2c5ed /src/lib/util | |
parent | c24473ddff715ecec2e258a6eb38960cf8c8e98e (diff) |
Revert "conflict resolution (nw)"
This reverts commit c24473ddff715ecec2e258a6eb38960cf8c8e98e, reversing
changes made to 009cba4fb8102102168ef32870892438327f3705.
Diffstat (limited to 'src/lib/util')
-rw-r--r-- | src/lib/util/corefile.cpp | 22 | ||||
-rw-r--r-- | src/lib/util/corefile.h | 21 | ||||
-rw-r--r-- | src/lib/util/corestr.cpp | 95 | ||||
-rw-r--r-- | src/lib/util/corestr.h | 7 | ||||
-rw-r--r-- | src/lib/util/unicode.cpp | 170 | ||||
-rw-r--r-- | src/lib/util/unicode.h | 17 |
6 files changed, 97 insertions, 235 deletions
diff --git a/src/lib/util/corefile.cpp b/src/lib/util/corefile.cpp index 78a484c5511..264d80f4113 100644 --- a/src/lib/util/corefile.cpp +++ b/src/lib/util/corefile.cpp @@ -23,10 +23,9 @@ #include <ctype.h> -namespace util { +namespace util { namespace { - /*************************************************************************** VALIDATION ***************************************************************************/ @@ -337,6 +336,23 @@ private: /*************************************************************************** + INLINE FUNCTIONS +***************************************************************************/ + +/*------------------------------------------------- + is_directory_separator - is a given character + a directory separator? The following logic + works for most platforms +-------------------------------------------------*/ + +inline int is_directory_separator(char c) +{ + return (c == '\\' || c == '/' || c == ':'); +} + + + +/*************************************************************************** core_text_file ***************************************************************************/ @@ -1259,7 +1275,7 @@ core_file::core_file() std::string core_filename_extract_base(const std::string &name, bool strip_extension) { // find the start of the basename - auto const start = std::find_if(name.rbegin(), name.rend(), &util::is_directory_separator); + auto const start = std::find_if(name.rbegin(), name.rend(), [](char c) { return util::is_directory_separator(c); }); // find the end of the basename auto const chop_position = strip_extension diff --git a/src/lib/util/corefile.h b/src/lib/util/corefile.h index d286cb866ac..2c4a5da8217 100644 --- a/src/lib/util/corefile.h +++ b/src/lib/util/corefile.h @@ -7,11 +7,11 @@ Core file I/O interface functions and definitions. ***************************************************************************/ -#ifndef MAME_LIB_UTIL_COREFILE_H -#define MAME_LIB_UTIL_COREFILE_H #pragma once +#ifndef MAME_LIB_UTIL_COREFILE_H +#define MAME_LIB_UTIL_COREFILE_H #include "corestr.h" #include "coretmpl.h" @@ -23,7 +23,6 @@ namespace util { - /*************************************************************************** ADDITIONAL OPEN FLAGS ***************************************************************************/ @@ -131,22 +130,6 @@ protected: core_file(); }; - -/*************************************************************************** - INLINE FUNCTIONS -***************************************************************************/ - -// is a given character a directory separator? - -constexpr bool is_directory_separator(char c) -{ -#if defined(WIN32) - return ('\\' == c) || ('/' == c) || (':' == c); -#else - return '/' == c; -#endif -} - } // namespace util diff --git a/src/lib/util/corestr.cpp b/src/lib/util/corestr.cpp index 5add3ed302f..a2fef33a87a 100644 --- a/src/lib/util/corestr.cpp +++ b/src/lib/util/corestr.cpp @@ -10,10 +10,6 @@ #include "corestr.h" #include "osdcore.h" - -#include <algorithm> -#include <memory> - #include <ctype.h> #include <stdlib.h> @@ -240,94 +236,3 @@ int strreplace(std::string &str, const std::string& search, const std::string& r } return matches; } - -namespace util { - -/** - * @fn double edit_distance(std::u32string const &lhs, std::u32string const &rhs) - * - * @brief Compares strings and returns prefix-weighted similarity score (smaller is more similar). - * - * @param lhs First input. - * @param rhs Second input. - * - * @return Similarity score ranging from 0.0 (totally dissimilar) to 1.0 (identical). - */ - -double edit_distance(std::u32string const &lhs, std::u32string const &rhs) -{ - // based on Jaro-Winkler distance - // TODO: this breaks if the lengths don't fit in a long int, but that's not a big limitation - constexpr long MAX_PREFIX(4); - constexpr double PREFIX_WEIGHT(0.1); - constexpr double PREFIX_THRESHOLD(0.7); - - std::u32string const &longer((lhs.length() >= rhs.length()) ? lhs : rhs); - std::u32string const &shorter((lhs.length() < rhs.length()) ? lhs : rhs); - - // find matches - long const range((std::max)(long(longer.length() / 2) - 1, 0L)); - std::unique_ptr<long []> match_idx(std::make_unique<long []>(shorter.length())); - std::unique_ptr<bool []> match_flg(std::make_unique<bool []>(longer.length())); - std::fill_n(match_idx.get(), shorter.length(), -1); - std::fill_n(match_flg.get(), longer.length(), false); - long match_cnt(0); - for (long i = 0; shorter.length() > i; ++i) - { - char32_t const ch(shorter[i]); - long const n((std::min)(i + range + 1L, long(longer.length()))); - for (long j = (std::max)(i - range, 0L); n > j; ++j) - { - if (!match_flg[j] && (ch == longer[j])) - { - match_idx[i] = j; - match_flg[j] = true; - ++match_cnt; - break; - } - } - } - - // early exit if strings are very dissimilar - if (!match_cnt) - return 1.0; - - // now find transpositions - std::unique_ptr<char32_t []> ms(std::make_unique<char32_t []>(2 * match_cnt)); - std::fill_n(ms.get(), 2 * match_cnt, char32_t(0)); - char32_t *const ms1(&ms[0]); - char32_t *const ms2(&ms[match_cnt]); - for (long i = 0, j = 0; shorter.length() > i; ++i) - { - if (0 <= match_idx[i]) - ms1[j++] = shorter[i]; - } - match_idx.reset(); - for (long i = 0, j = 0; longer.length() > i; ++i) - { - if (match_flg[i]) - ms2[j++] = longer[i]; - } - match_flg.reset(); - long halftrans_cnt(0); - for (long i = 0; match_cnt > i; ++i) - { - if (ms1[i] != ms2[i]) - ++halftrans_cnt; - } - ms.reset(); - - // simple prefix detection - long prefix_len(0); - for (long i = 0; ((std::min)(long(shorter.length()), MAX_PREFIX) > i) && (lhs[i] == rhs[i]); ++i) - ++prefix_len; - - // do the weighting - double const m(match_cnt); - double const t(double(halftrans_cnt) / 2); - double const jaro(((m / lhs.length()) + (m / rhs.length()) + ((m - t) / m)) / 3); - double const jaro_winkler((PREFIX_THRESHOLD > jaro) ? jaro : (jaro + (PREFIX_WEIGHT * prefix_len * (1.0 - jaro)))); - return 1.0 - jaro_winkler; -} - -} // namespace util diff --git a/src/lib/util/corestr.h b/src/lib/util/corestr.h index c0bf4ef16d5..d7734d50b29 100644 --- a/src/lib/util/corestr.h +++ b/src/lib/util/corestr.h @@ -64,11 +64,4 @@ std::string &strmakeupper(std::string& str); std::string &strmakelower(std::string& str); int strreplace(std::string &str, const std::string& search, const std::string& replace); -namespace util { - -// based on Jaro-Winkler distance - returns value from 0.0 (totally dissimilar) to 1.0 (identical) -double edit_distance(std::u32string const &lhs, std::u32string const &rhs); - -} // namespace util - #endif // MAME_UTIL_CORESTR_H diff --git a/src/lib/util/unicode.cpp b/src/lib/util/unicode.cpp index 8e6478beee3..5151a1e101d 100644 --- a/src/lib/util/unicode.cpp +++ b/src/lib/util/unicode.cpp @@ -21,67 +21,6 @@ #include <locale> -namespace { - -//------------------------------------------------- -// internal_normalize_unicode - uses utf8proc to -// normalize unicode -//------------------------------------------------- - -std::string internal_normalize_unicode( - char const *s, - size_t length, - unicode_normalization_form normalization_form, - bool fold_case, - bool null_terminated) -{ - // convert the normalization form - int options; - switch (normalization_form) - { - case unicode_normalization_form::C: - options = UTF8PROC_STABLE | UTF8PROC_COMPOSE; - break; - case unicode_normalization_form::D: - options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE; - break; - case unicode_normalization_form::KC: - options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT; - break; - case unicode_normalization_form::KD: - options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT; - break; - default: - throw false; - } - - // perform case folding? - if (fold_case) - options |= UTF8PROC_CASEFOLD; - - // use NUL terminator to determine length? - if (null_terminated) - options |= UTF8PROC_NULLTERM; - - // invoke utf8proc - utf8proc_uint8_t *utf8proc_result(nullptr); - utf8proc_ssize_t const utf8proc_result_length(utf8proc_map(reinterpret_cast<utf8proc_uint8_t const *>(s), length, &utf8proc_result, utf8proc_option_t(options))); - - // conver the result - std::string result; - if (utf8proc_result) - { - if (utf8proc_result_length > 0) - result.assign(reinterpret_cast<char const *>(utf8proc_result), utf8proc_result_length); - free(utf8proc_result); - } - - return result; -} - -} // anonymous namespace - - //------------------------------------------------- // uchar_isvalid - return true if a given // character is a legitimate unicode character @@ -127,53 +66,56 @@ bool uchar_is_digit(char32_t uchar) int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count) { + char32_t c, minchar; + int auxlen, i; + char auxchar; + // validate parameters - if (!utf8char || !count) + if (utf8char == nullptr || count == 0) return 0; // start with the first byte - char32_t c = (unsigned char)*utf8char; + c = (unsigned char) *utf8char; count--; utf8char++; // based on that, determine how many additional bytes we need - char32_t minchar; - int auxlen; - if ((c & 0x80) == 0x00) + if (c < 0x80) { // unicode char 0x00000000 - 0x0000007F + c &= 0x7f; auxlen = 0; minchar = 0x00000000; } - else if ((c & 0xe0) == 0xc0) + else if (c >= 0xc0 && c < 0xe0) { // unicode char 0x00000080 - 0x000007FF c &= 0x1f; auxlen = 1; minchar = 0x00000080; } - else if ((c & 0xf0) == 0xe0) + else if (c >= 0xe0 && c < 0xf0) { // unicode char 0x00000800 - 0x0000FFFF c &= 0x0f; auxlen = 2; minchar = 0x00000800; } - else if ((c & 0xf8) == 0xf0) + else if (c >= 0xf0 && c < 0xf8) { // unicode char 0x00010000 - 0x001FFFFF c &= 0x07; auxlen = 3; minchar = 0x00010000; } - else if ((c & 0xfc) == 0xf8) + else if (c >= 0xf8 && c < 0xfc) { // unicode char 0x00200000 - 0x03FFFFFF c &= 0x03; auxlen = 4; minchar = 0x00200000; } - else if ((c & 0xfe) == 0xfc) + else if (c >= 0xfc && c < 0xfe) { // unicode char 0x04000000 - 0x7FFFFFFF c &= 0x01; @@ -191,9 +133,9 @@ int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count) return -1; // we now know how long the char is, now compute it - for (int i = 0; i < auxlen; i++) + for (i = 0; i < auxlen; i++) { - char32_t const auxchar = (unsigned char)utf8char[i]; + auxchar = utf8char[i]; // all auxillary chars must be between 0x80-0xbf if ((auxchar & 0xc0) != 0x80) @@ -264,28 +206,6 @@ int uchar_from_utf16f(char32_t *uchar, const char16_t *utf16char, size_t count) //------------------------------------------------- -// ustr_from_utf8 - convert a UTF-8 sequence into -// into a Unicode string -//------------------------------------------------- - -std::u32string ustr_from_utf8(const std::string &utf8str) -{ - std::u32string result; - char const *utf8char(utf8str.c_str()); - size_t remaining(utf8str.length()); - while (remaining) - { - char32_t ch; - int const consumed(uchar_from_utf8(&ch, utf8char, remaining)); - result.append(1, (consumed > 0) ? ch : char32_t(0x00fffdU)); - utf8char += (consumed > 0) ? consumed : 1; - remaining -= (consumed > 0) ? consumed : 1; - } - return result; -} - - -//------------------------------------------------- // utf8_from_uchar - convert a unicode character // into a UTF-8 sequence //------------------------------------------------- @@ -468,13 +388,61 @@ std::string utf8_from_wstring(const std::wstring &string) //------------------------------------------------- +// internal_normalize_unicode - uses utf8proc to +// normalize unicode +//------------------------------------------------- + +static std::string internal_normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool null_terminated) +{ + // convert the normalization form + int options; + switch (normalization_form) + { + case unicode_normalization_form::C: + options = UTF8PROC_STABLE | UTF8PROC_COMPOSE; + break; + case unicode_normalization_form::D: + options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE; + break; + case unicode_normalization_form::KC: + options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT; + break; + case unicode_normalization_form::KD: + options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT; + break; + default: + throw false; + } + + // was this null terminated? + if (null_terminated) + options |= UTF8PROC_NULLTERM; + + // invoke utf8proc + utf8proc_uint8_t *utf8proc_result; + utf8proc_ssize_t utf8proc_result_length = utf8proc_map((utf8proc_uint8_t *) s, length, &utf8proc_result, (utf8proc_option_t)options); + + // conver the result + std::string result; + if (utf8proc_result) + { + if (utf8proc_result_length > 0) + result = std::string((const char *)utf8proc_result, utf8proc_result_length); + free(utf8proc_result); + } + + return result; +} + + +//------------------------------------------------- // normalize_unicode - uses utf8proc to normalize // unicode //------------------------------------------------- -std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form, bool fold_case) +std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form) { - return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, fold_case, false); + return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, false); } @@ -483,9 +451,9 @@ std::string normalize_unicode(const std::string &s, unicode_normalization_form n // unicode //------------------------------------------------- -std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form, bool fold_case) +std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form) { - return internal_normalize_unicode(s, 0, normalization_form, fold_case, true); + return internal_normalize_unicode(s, 0, normalization_form, true); } @@ -494,9 +462,9 @@ std::string normalize_unicode(const char *s, unicode_normalization_form normaliz // unicode //------------------------------------------------- -std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool fold_case) +std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form) { - return internal_normalize_unicode(s, length, normalization_form, fold_case, false); + return internal_normalize_unicode(s, length, normalization_form, false); } diff --git a/src/lib/util/unicode.h b/src/lib/util/unicode.h index 46e1aec3496..5e30a74916e 100644 --- a/src/lib/util/unicode.h +++ b/src/lib/util/unicode.h @@ -14,16 +14,14 @@ singular 32-bit Unicode chars. ***************************************************************************/ -#ifndef MAME_LIB_UTIL_UNICODE_H -#define MAME_LIB_UTIL_UNICODE_H #pragma once -#include "osdcore.h" - -#include <string> +#ifndef UNICODE_H +#define UNICODE_H #include <stdlib.h> +#include "osdcore.h" @@ -97,7 +95,6 @@ bool uchar_is_digit(char32_t uchar); int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count); int uchar_from_utf16(char32_t *uchar, const char16_t *utf16char, size_t count); int uchar_from_utf16f(char32_t *uchar, const char16_t *utf16char, size_t count); -std::u32string ustr_from_utf8(const std::string &utf8str); // converting 32-bit Unicode chars to strings int utf8_from_uchar(char *utf8string, size_t count, char32_t uchar); @@ -110,9 +107,9 @@ std::wstring wstring_from_utf8(const std::string &utf8string); std::string utf8_from_wstring(const std::wstring &string); // unicode normalization -std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form, bool fold_case = false); -std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form, bool fold_case = false); -std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool fold_case = false); +std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form); +std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form); +std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form); // upper and lower case char32_t uchar_toupper(char32_t ch); @@ -140,4 +137,4 @@ bool utf8_is_valid_string(const char *utf8string); #define utf16le_from_uchar utf16f_from_uchar #endif -#endif // MAME_LIB_UTIL_UNICODE_H +#endif /* UNICODE_H */ |