summaryrefslogtreecommitdiffstats
path: root/src/lib/util
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/util')
-rw-r--r--src/lib/util/corefile.cpp22
-rw-r--r--src/lib/util/corefile.h21
-rw-r--r--src/lib/util/corestr.cpp95
-rw-r--r--src/lib/util/corestr.h7
-rw-r--r--src/lib/util/unicode.cpp170
-rw-r--r--src/lib/util/unicode.h17
6 files changed, 97 insertions, 235 deletions
diff --git a/src/lib/util/corefile.cpp b/src/lib/util/corefile.cpp
index 78a484c5511..264d80f4113 100644
--- a/src/lib/util/corefile.cpp
+++ b/src/lib/util/corefile.cpp
@@ -23,10 +23,9 @@
#include <ctype.h>
-namespace util {
+namespace util {
namespace {
-
/***************************************************************************
VALIDATION
***************************************************************************/
@@ -337,6 +336,23 @@ private:
/***************************************************************************
+ INLINE FUNCTIONS
+***************************************************************************/
+
+/*-------------------------------------------------
+ is_directory_separator - is a given character
+ a directory separator? The following logic
+ works for most platforms
+-------------------------------------------------*/
+
+inline int is_directory_separator(char c)
+{
+ return (c == '\\' || c == '/' || c == ':');
+}
+
+
+
+/***************************************************************************
core_text_file
***************************************************************************/
@@ -1259,7 +1275,7 @@ core_file::core_file()
std::string core_filename_extract_base(const std::string &name, bool strip_extension)
{
// find the start of the basename
- auto const start = std::find_if(name.rbegin(), name.rend(), &util::is_directory_separator);
+ auto const start = std::find_if(name.rbegin(), name.rend(), [](char c) { return util::is_directory_separator(c); });
// find the end of the basename
auto const chop_position = strip_extension
diff --git a/src/lib/util/corefile.h b/src/lib/util/corefile.h
index d286cb866ac..2c4a5da8217 100644
--- a/src/lib/util/corefile.h
+++ b/src/lib/util/corefile.h
@@ -7,11 +7,11 @@
Core file I/O interface functions and definitions.
***************************************************************************/
-#ifndef MAME_LIB_UTIL_COREFILE_H
-#define MAME_LIB_UTIL_COREFILE_H
#pragma once
+#ifndef MAME_LIB_UTIL_COREFILE_H
+#define MAME_LIB_UTIL_COREFILE_H
#include "corestr.h"
#include "coretmpl.h"
@@ -23,7 +23,6 @@
namespace util {
-
/***************************************************************************
ADDITIONAL OPEN FLAGS
***************************************************************************/
@@ -131,22 +130,6 @@ protected:
core_file();
};
-
-/***************************************************************************
- INLINE FUNCTIONS
-***************************************************************************/
-
-// is a given character a directory separator?
-
-constexpr bool is_directory_separator(char c)
-{
-#if defined(WIN32)
- return ('\\' == c) || ('/' == c) || (':' == c);
-#else
- return '/' == c;
-#endif
-}
-
} // namespace util
diff --git a/src/lib/util/corestr.cpp b/src/lib/util/corestr.cpp
index 5add3ed302f..a2fef33a87a 100644
--- a/src/lib/util/corestr.cpp
+++ b/src/lib/util/corestr.cpp
@@ -10,10 +10,6 @@
#include "corestr.h"
#include "osdcore.h"
-
-#include <algorithm>
-#include <memory>
-
#include <ctype.h>
#include <stdlib.h>
@@ -240,94 +236,3 @@ int strreplace(std::string &str, const std::string& search, const std::string& r
}
return matches;
}
-
-namespace util {
-
-/**
- * @fn double edit_distance(std::u32string const &lhs, std::u32string const &rhs)
- *
- * @brief Compares strings and returns prefix-weighted similarity score (smaller is more similar).
- *
- * @param lhs First input.
- * @param rhs Second input.
- *
- * @return Similarity score ranging from 0.0 (totally dissimilar) to 1.0 (identical).
- */
-
-double edit_distance(std::u32string const &lhs, std::u32string const &rhs)
-{
- // based on Jaro-Winkler distance
- // TODO: this breaks if the lengths don't fit in a long int, but that's not a big limitation
- constexpr long MAX_PREFIX(4);
- constexpr double PREFIX_WEIGHT(0.1);
- constexpr double PREFIX_THRESHOLD(0.7);
-
- std::u32string const &longer((lhs.length() >= rhs.length()) ? lhs : rhs);
- std::u32string const &shorter((lhs.length() < rhs.length()) ? lhs : rhs);
-
- // find matches
- long const range((std::max)(long(longer.length() / 2) - 1, 0L));
- std::unique_ptr<long []> match_idx(std::make_unique<long []>(shorter.length()));
- std::unique_ptr<bool []> match_flg(std::make_unique<bool []>(longer.length()));
- std::fill_n(match_idx.get(), shorter.length(), -1);
- std::fill_n(match_flg.get(), longer.length(), false);
- long match_cnt(0);
- for (long i = 0; shorter.length() > i; ++i)
- {
- char32_t const ch(shorter[i]);
- long const n((std::min)(i + range + 1L, long(longer.length())));
- for (long j = (std::max)(i - range, 0L); n > j; ++j)
- {
- if (!match_flg[j] && (ch == longer[j]))
- {
- match_idx[i] = j;
- match_flg[j] = true;
- ++match_cnt;
- break;
- }
- }
- }
-
- // early exit if strings are very dissimilar
- if (!match_cnt)
- return 1.0;
-
- // now find transpositions
- std::unique_ptr<char32_t []> ms(std::make_unique<char32_t []>(2 * match_cnt));
- std::fill_n(ms.get(), 2 * match_cnt, char32_t(0));
- char32_t *const ms1(&ms[0]);
- char32_t *const ms2(&ms[match_cnt]);
- for (long i = 0, j = 0; shorter.length() > i; ++i)
- {
- if (0 <= match_idx[i])
- ms1[j++] = shorter[i];
- }
- match_idx.reset();
- for (long i = 0, j = 0; longer.length() > i; ++i)
- {
- if (match_flg[i])
- ms2[j++] = longer[i];
- }
- match_flg.reset();
- long halftrans_cnt(0);
- for (long i = 0; match_cnt > i; ++i)
- {
- if (ms1[i] != ms2[i])
- ++halftrans_cnt;
- }
- ms.reset();
-
- // simple prefix detection
- long prefix_len(0);
- for (long i = 0; ((std::min)(long(shorter.length()), MAX_PREFIX) > i) && (lhs[i] == rhs[i]); ++i)
- ++prefix_len;
-
- // do the weighting
- double const m(match_cnt);
- double const t(double(halftrans_cnt) / 2);
- double const jaro(((m / lhs.length()) + (m / rhs.length()) + ((m - t) / m)) / 3);
- double const jaro_winkler((PREFIX_THRESHOLD > jaro) ? jaro : (jaro + (PREFIX_WEIGHT * prefix_len * (1.0 - jaro))));
- return 1.0 - jaro_winkler;
-}
-
-} // namespace util
diff --git a/src/lib/util/corestr.h b/src/lib/util/corestr.h
index c0bf4ef16d5..d7734d50b29 100644
--- a/src/lib/util/corestr.h
+++ b/src/lib/util/corestr.h
@@ -64,11 +64,4 @@ std::string &strmakeupper(std::string& str);
std::string &strmakelower(std::string& str);
int strreplace(std::string &str, const std::string& search, const std::string& replace);
-namespace util {
-
-// based on Jaro-Winkler distance - returns value from 0.0 (totally dissimilar) to 1.0 (identical)
-double edit_distance(std::u32string const &lhs, std::u32string const &rhs);
-
-} // namespace util
-
#endif // MAME_UTIL_CORESTR_H
diff --git a/src/lib/util/unicode.cpp b/src/lib/util/unicode.cpp
index 8e6478beee3..5151a1e101d 100644
--- a/src/lib/util/unicode.cpp
+++ b/src/lib/util/unicode.cpp
@@ -21,67 +21,6 @@
#include <locale>
-namespace {
-
-//-------------------------------------------------
-// internal_normalize_unicode - uses utf8proc to
-// normalize unicode
-//-------------------------------------------------
-
-std::string internal_normalize_unicode(
- char const *s,
- size_t length,
- unicode_normalization_form normalization_form,
- bool fold_case,
- bool null_terminated)
-{
- // convert the normalization form
- int options;
- switch (normalization_form)
- {
- case unicode_normalization_form::C:
- options = UTF8PROC_STABLE | UTF8PROC_COMPOSE;
- break;
- case unicode_normalization_form::D:
- options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE;
- break;
- case unicode_normalization_form::KC:
- options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT;
- break;
- case unicode_normalization_form::KD:
- options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT;
- break;
- default:
- throw false;
- }
-
- // perform case folding?
- if (fold_case)
- options |= UTF8PROC_CASEFOLD;
-
- // use NUL terminator to determine length?
- if (null_terminated)
- options |= UTF8PROC_NULLTERM;
-
- // invoke utf8proc
- utf8proc_uint8_t *utf8proc_result(nullptr);
- utf8proc_ssize_t const utf8proc_result_length(utf8proc_map(reinterpret_cast<utf8proc_uint8_t const *>(s), length, &utf8proc_result, utf8proc_option_t(options)));
-
- // conver the result
- std::string result;
- if (utf8proc_result)
- {
- if (utf8proc_result_length > 0)
- result.assign(reinterpret_cast<char const *>(utf8proc_result), utf8proc_result_length);
- free(utf8proc_result);
- }
-
- return result;
-}
-
-} // anonymous namespace
-
-
//-------------------------------------------------
// uchar_isvalid - return true if a given
// character is a legitimate unicode character
@@ -127,53 +66,56 @@ bool uchar_is_digit(char32_t uchar)
int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count)
{
+ char32_t c, minchar;
+ int auxlen, i;
+ char auxchar;
+
// validate parameters
- if (!utf8char || !count)
+ if (utf8char == nullptr || count == 0)
return 0;
// start with the first byte
- char32_t c = (unsigned char)*utf8char;
+ c = (unsigned char) *utf8char;
count--;
utf8char++;
// based on that, determine how many additional bytes we need
- char32_t minchar;
- int auxlen;
- if ((c & 0x80) == 0x00)
+ if (c < 0x80)
{
// unicode char 0x00000000 - 0x0000007F
+ c &= 0x7f;
auxlen = 0;
minchar = 0x00000000;
}
- else if ((c & 0xe0) == 0xc0)
+ else if (c >= 0xc0 && c < 0xe0)
{
// unicode char 0x00000080 - 0x000007FF
c &= 0x1f;
auxlen = 1;
minchar = 0x00000080;
}
- else if ((c & 0xf0) == 0xe0)
+ else if (c >= 0xe0 && c < 0xf0)
{
// unicode char 0x00000800 - 0x0000FFFF
c &= 0x0f;
auxlen = 2;
minchar = 0x00000800;
}
- else if ((c & 0xf8) == 0xf0)
+ else if (c >= 0xf0 && c < 0xf8)
{
// unicode char 0x00010000 - 0x001FFFFF
c &= 0x07;
auxlen = 3;
minchar = 0x00010000;
}
- else if ((c & 0xfc) == 0xf8)
+ else if (c >= 0xf8 && c < 0xfc)
{
// unicode char 0x00200000 - 0x03FFFFFF
c &= 0x03;
auxlen = 4;
minchar = 0x00200000;
}
- else if ((c & 0xfe) == 0xfc)
+ else if (c >= 0xfc && c < 0xfe)
{
// unicode char 0x04000000 - 0x7FFFFFFF
c &= 0x01;
@@ -191,9 +133,9 @@ int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count)
return -1;
// we now know how long the char is, now compute it
- for (int i = 0; i < auxlen; i++)
+ for (i = 0; i < auxlen; i++)
{
- char32_t const auxchar = (unsigned char)utf8char[i];
+ auxchar = utf8char[i];
// all auxillary chars must be between 0x80-0xbf
if ((auxchar & 0xc0) != 0x80)
@@ -264,28 +206,6 @@ int uchar_from_utf16f(char32_t *uchar, const char16_t *utf16char, size_t count)
//-------------------------------------------------
-// ustr_from_utf8 - convert a UTF-8 sequence into
-// into a Unicode string
-//-------------------------------------------------
-
-std::u32string ustr_from_utf8(const std::string &utf8str)
-{
- std::u32string result;
- char const *utf8char(utf8str.c_str());
- size_t remaining(utf8str.length());
- while (remaining)
- {
- char32_t ch;
- int const consumed(uchar_from_utf8(&ch, utf8char, remaining));
- result.append(1, (consumed > 0) ? ch : char32_t(0x00fffdU));
- utf8char += (consumed > 0) ? consumed : 1;
- remaining -= (consumed > 0) ? consumed : 1;
- }
- return result;
-}
-
-
-//-------------------------------------------------
// utf8_from_uchar - convert a unicode character
// into a UTF-8 sequence
//-------------------------------------------------
@@ -468,13 +388,61 @@ std::string utf8_from_wstring(const std::wstring &string)
//-------------------------------------------------
+// internal_normalize_unicode - uses utf8proc to
+// normalize unicode
+//-------------------------------------------------
+
+static std::string internal_normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool null_terminated)
+{
+ // convert the normalization form
+ int options;
+ switch (normalization_form)
+ {
+ case unicode_normalization_form::C:
+ options = UTF8PROC_STABLE | UTF8PROC_COMPOSE;
+ break;
+ case unicode_normalization_form::D:
+ options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE;
+ break;
+ case unicode_normalization_form::KC:
+ options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT;
+ break;
+ case unicode_normalization_form::KD:
+ options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT;
+ break;
+ default:
+ throw false;
+ }
+
+ // was this null terminated?
+ if (null_terminated)
+ options |= UTF8PROC_NULLTERM;
+
+ // invoke utf8proc
+ utf8proc_uint8_t *utf8proc_result;
+ utf8proc_ssize_t utf8proc_result_length = utf8proc_map((utf8proc_uint8_t *) s, length, &utf8proc_result, (utf8proc_option_t)options);
+
+ // conver the result
+ std::string result;
+ if (utf8proc_result)
+ {
+ if (utf8proc_result_length > 0)
+ result = std::string((const char *)utf8proc_result, utf8proc_result_length);
+ free(utf8proc_result);
+ }
+
+ return result;
+}
+
+
+//-------------------------------------------------
// normalize_unicode - uses utf8proc to normalize
// unicode
//-------------------------------------------------
-std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form, bool fold_case)
+std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form)
{
- return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, fold_case, false);
+ return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, false);
}
@@ -483,9 +451,9 @@ std::string normalize_unicode(const std::string &s, unicode_normalization_form n
// unicode
//-------------------------------------------------
-std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form, bool fold_case)
+std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form)
{
- return internal_normalize_unicode(s, 0, normalization_form, fold_case, true);
+ return internal_normalize_unicode(s, 0, normalization_form, true);
}
@@ -494,9 +462,9 @@ std::string normalize_unicode(const char *s, unicode_normalization_form normaliz
// unicode
//-------------------------------------------------
-std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool fold_case)
+std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form)
{
- return internal_normalize_unicode(s, length, normalization_form, fold_case, false);
+ return internal_normalize_unicode(s, length, normalization_form, false);
}
diff --git a/src/lib/util/unicode.h b/src/lib/util/unicode.h
index 46e1aec3496..5e30a74916e 100644
--- a/src/lib/util/unicode.h
+++ b/src/lib/util/unicode.h
@@ -14,16 +14,14 @@
singular 32-bit Unicode chars.
***************************************************************************/
-#ifndef MAME_LIB_UTIL_UNICODE_H
-#define MAME_LIB_UTIL_UNICODE_H
#pragma once
-#include "osdcore.h"
-
-#include <string>
+#ifndef UNICODE_H
+#define UNICODE_H
#include <stdlib.h>
+#include "osdcore.h"
@@ -97,7 +95,6 @@ bool uchar_is_digit(char32_t uchar);
int uchar_from_utf8(char32_t *uchar, const char *utf8char, size_t count);
int uchar_from_utf16(char32_t *uchar, const char16_t *utf16char, size_t count);
int uchar_from_utf16f(char32_t *uchar, const char16_t *utf16char, size_t count);
-std::u32string ustr_from_utf8(const std::string &utf8str);
// converting 32-bit Unicode chars to strings
int utf8_from_uchar(char *utf8string, size_t count, char32_t uchar);
@@ -110,9 +107,9 @@ std::wstring wstring_from_utf8(const std::string &utf8string);
std::string utf8_from_wstring(const std::wstring &string);
// unicode normalization
-std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form, bool fold_case = false);
-std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form, bool fold_case = false);
-std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool fold_case = false);
+std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form);
+std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form);
+std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form);
// upper and lower case
char32_t uchar_toupper(char32_t ch);
@@ -140,4 +137,4 @@ bool utf8_is_valid_string(const char *utf8string);
#define utf16le_from_uchar utf16f_from_uchar
#endif
-#endif // MAME_LIB_UTIL_UNICODE_H
+#endif /* UNICODE_H */