diff options
Diffstat (limited to 'src/lib/util')
-rw-r--r-- | src/lib/util/unicode.cpp | 89 | ||||
-rw-r--r-- | src/lib/util/unicode.h | 8 |
2 files changed, 96 insertions, 1 deletions
diff --git a/src/lib/util/unicode.cpp b/src/lib/util/unicode.cpp index a6c5515fbf6..b3e92e4094e 100644 --- a/src/lib/util/unicode.cpp +++ b/src/lib/util/unicode.cpp @@ -2,7 +2,7 @@ // copyright-holders:Aaron Giles /********************************************************************* - unicode.c + unicode.cpp Unicode related functions @@ -10,6 +10,12 @@ #include "unicode.h" +#ifdef _WIN32 +#define UTF8PROC_DLLEXPORT +#endif + +#include "utf8proc/utf8proc.h" + //------------------------------------------------- // uchar_isvalid - return true if a given @@ -346,6 +352,87 @@ int utf16f_from_uchar(char16_t *utf16string, size_t count, char32_t uchar) //------------------------------------------------- +// internal_normalize_unicode - uses utf8proc to +// normalize unicode +//------------------------------------------------- + +static std::string internal_normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool null_terminated) +{ + // convert the normalization form + int options; + switch (normalization_form) + { + case unicode_normalization_form::C: + options = UTF8PROC_STABLE | UTF8PROC_COMPOSE; + break; + case unicode_normalization_form::D: + options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE; + break; + case unicode_normalization_form::KC: + options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT; + break; + case unicode_normalization_form::KD: + options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT; + break; + default: + throw false; + } + + // was this null terminated? + if (null_terminated) + options |= UTF8PROC_NULLTERM; + + // invoke utf8proc + utf8proc_uint8_t *utf8proc_result; + utf8proc_ssize_t utf8proc_result_length = utf8proc_map((utf8proc_uint8_t *) s, length, &utf8proc_result, (utf8proc_option_t)options); + + // conver the result + std::string result; + if (utf8proc_result) + { + if (utf8proc_result_length > 0) + result = std::string((const char *)utf8proc_result, utf8proc_result_length); + free(utf8proc_result); + } + + return result; +} + + +//------------------------------------------------- +// normalize_unicode - uses utf8proc to normalize +// unicode +//------------------------------------------------- + +std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form) +{ + return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, false); +} + + +//------------------------------------------------- +// normalize_unicode - uses utf8proc to normalize +// unicode +//------------------------------------------------- + +std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form) +{ + return internal_normalize_unicode(s, 0, normalization_form, true); +} + + +//------------------------------------------------- +// normalize_unicode - uses utf8proc to normalize +// unicode +//------------------------------------------------- + +std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form) +{ + return internal_normalize_unicode(s, length, normalization_form, false); +} + + +//------------------------------------------------- // utf8_previous_char - return a pointer to the // previous character in a string //------------------------------------------------- diff --git a/src/lib/util/unicode.h b/src/lib/util/unicode.h index 5cc6db383ee..ef5267794e6 100644 --- a/src/lib/util/unicode.h +++ b/src/lib/util/unicode.h @@ -72,6 +72,9 @@ #define UTF8_UP "\xe2\x86\x91" /* cursor up */ #define UTF8_DOWN "\xe2\x86\x93" /* cursor down */ +enum class unicode_normalization_form { C, D, KC, KD }; + + /*************************************************************************** FUNCTION PROTOTYPES @@ -97,6 +100,11 @@ std::string utf8_from_uchar(char32_t uchar); int utf16_from_uchar(char16_t *utf16string, size_t count, char32_t uchar); int utf16f_from_uchar(char16_t *utf16string, size_t count, char32_t uchar); +// unicode normalization +std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form); +std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form); +std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form); + // misc UTF-8 helpers const char *utf8_previous_char(const char *utf8string); bool utf8_is_valid_string(const char *utf8string); |