summaryrefslogtreecommitdiffstatshomepage
path: root/src/lib/util
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/util')
-rw-r--r--src/lib/util/unicode.cpp89
-rw-r--r--src/lib/util/unicode.h8
2 files changed, 96 insertions, 1 deletions
diff --git a/src/lib/util/unicode.cpp b/src/lib/util/unicode.cpp
index a6c5515fbf6..b3e92e4094e 100644
--- a/src/lib/util/unicode.cpp
+++ b/src/lib/util/unicode.cpp
@@ -2,7 +2,7 @@
// copyright-holders:Aaron Giles
/*********************************************************************
- unicode.c
+ unicode.cpp
Unicode related functions
@@ -10,6 +10,12 @@
#include "unicode.h"
+#ifdef _WIN32
+#define UTF8PROC_DLLEXPORT
+#endif
+
+#include "utf8proc/utf8proc.h"
+
//-------------------------------------------------
// uchar_isvalid - return true if a given
@@ -346,6 +352,87 @@ int utf16f_from_uchar(char16_t *utf16string, size_t count, char32_t uchar)
//-------------------------------------------------
+// internal_normalize_unicode - uses utf8proc to
+// normalize unicode
+//-------------------------------------------------
+
+static std::string internal_normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool null_terminated)
+{
+ // convert the normalization form
+ int options;
+ switch (normalization_form)
+ {
+ case unicode_normalization_form::C:
+ options = UTF8PROC_STABLE | UTF8PROC_COMPOSE;
+ break;
+ case unicode_normalization_form::D:
+ options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE;
+ break;
+ case unicode_normalization_form::KC:
+ options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT;
+ break;
+ case unicode_normalization_form::KD:
+ options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT;
+ break;
+ default:
+ throw false;
+ }
+
+ // was this null terminated?
+ if (null_terminated)
+ options |= UTF8PROC_NULLTERM;
+
+ // invoke utf8proc
+ utf8proc_uint8_t *utf8proc_result;
+ utf8proc_ssize_t utf8proc_result_length = utf8proc_map((utf8proc_uint8_t *) s, length, &utf8proc_result, (utf8proc_option_t)options);
+
+ // conver the result
+ std::string result;
+ if (utf8proc_result)
+ {
+ if (utf8proc_result_length > 0)
+ result = std::string((const char *)utf8proc_result, utf8proc_result_length);
+ free(utf8proc_result);
+ }
+
+ return result;
+}
+
+
+//-------------------------------------------------
+// normalize_unicode - uses utf8proc to normalize
+// unicode
+//-------------------------------------------------
+
+std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form)
+{
+ return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, false);
+}
+
+
+//-------------------------------------------------
+// normalize_unicode - uses utf8proc to normalize
+// unicode
+//-------------------------------------------------
+
+std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form)
+{
+ return internal_normalize_unicode(s, 0, normalization_form, true);
+}
+
+
+//-------------------------------------------------
+// normalize_unicode - uses utf8proc to normalize
+// unicode
+//-------------------------------------------------
+
+std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form)
+{
+ return internal_normalize_unicode(s, length, normalization_form, false);
+}
+
+
+//-------------------------------------------------
// utf8_previous_char - return a pointer to the
// previous character in a string
//-------------------------------------------------
diff --git a/src/lib/util/unicode.h b/src/lib/util/unicode.h
index 5cc6db383ee..ef5267794e6 100644
--- a/src/lib/util/unicode.h
+++ b/src/lib/util/unicode.h
@@ -72,6 +72,9 @@
#define UTF8_UP "\xe2\x86\x91" /* cursor up */
#define UTF8_DOWN "\xe2\x86\x93" /* cursor down */
+enum class unicode_normalization_form { C, D, KC, KD };
+
+
/***************************************************************************
FUNCTION PROTOTYPES
@@ -97,6 +100,11 @@ std::string utf8_from_uchar(char32_t uchar);
int utf16_from_uchar(char16_t *utf16string, size_t count, char32_t uchar);
int utf16f_from_uchar(char16_t *utf16string, size_t count, char32_t uchar);
+// unicode normalization
+std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form);
+std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form);
+std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form);
+
// misc UTF-8 helpers
const char *utf8_previous_char(const char *utf8string);
bool utf8_is_valid_string(const char *utf8string);