diff options
Diffstat (limited to '3rdparty/utf8proc/utf8proc.h')
-rw-r--r-- | 3rdparty/utf8proc/utf8proc.h | 108 |
1 files changed, 72 insertions, 36 deletions
diff --git a/3rdparty/utf8proc/utf8proc.h b/3rdparty/utf8proc/utf8proc.h index 371d39fd8b7..a39874a3896 100644 --- a/3rdparty/utf8proc/utf8proc.h +++ b/3rdparty/utf8proc/utf8proc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors. + * Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors. * Copyright (c) 2009 Public Software Group e. V., Berlin, Germany * * Permission is hereby granted, free of charge, to any person obtaining a @@ -27,8 +27,8 @@ * * utf8proc is a free/open-source (MIT/expat licensed) C library * providing Unicode normalization, case-folding, and other operations - * for strings in the UTF-8 encoding, supporting Unicode version - * 8.0.0. See the utf8proc home page (http://julialang.org/utf8proc/) + * for strings in the UTF-8 encoding, supporting up-to-date Unicode versions. + * See the utf8proc home page (http://julialang.org/utf8proc/) * for downloads and other information, or the source code on github * (https://github.com/JuliaLang/utf8proc). * @@ -71,13 +71,13 @@ /** The MAJOR version number (increased when backwards API compatibility is broken). */ #define UTF8PROC_VERSION_MAJOR 2 /** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */ -#define UTF8PROC_VERSION_MINOR 1 +#define UTF8PROC_VERSION_MINOR 9 /** The PATCH version (increased for fixes that do not change the API). */ #define UTF8PROC_VERSION_PATCH 0 /** @} */ #include <stdlib.h> -#include <sys/types.h> + #if defined(_MSC_VER) && _MSC_VER < 1800 // MSVC prior to 2013 lacked stdbool.h and inttypes.h typedef signed char utf8proc_int8_t; @@ -120,23 +120,19 @@ typedef bool utf8proc_bool; #endif #include <limits.h> -#ifdef _WIN32 -# ifdef UTF8PROC_EXPORTS -# ifndef UTF8PROC_DLLEXPORT -# define UTF8PROC_DLLEXPORT __declspec(dllexport) +#ifdef UTF8PROC_STATIC +# define UTF8PROC_DLLEXPORT +#else +# ifdef _WIN32 +# ifdef UTF8PROC_EXPORTS +# define UTF8PROC_DLLEXPORT __declspec(dllexport) +# else +# define UTF8PROC_DLLEXPORT __declspec(dllimport) # endif +# elif __GNUC__ >= 4 +# define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default"))) # else -# ifndef UTF8PROC_DLLEXPORT -# define UTF8PROC_DLLEXPORT __declspec(dllimport) -# endif -# endif -#elif __GNUC__ >= 4 -# ifndef UTF8PROC_DLLEXPORT -# define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default"))) -# endif -#else -# ifndef UTF8PROC_DLLEXPORT -# define UTF8PROC_DLLEXPORT +# define UTF8PROC_DLLEXPORT # endif #endif @@ -144,14 +140,6 @@ typedef bool utf8proc_bool; extern "C" { #endif -#ifndef SSIZE_MAX -#define SSIZE_MAX ((size_t)SIZE_MAX/2) -#endif - -#ifndef UINT16_MAX -# define UINT16_MAX 65535U -#endif - /** * Option flags used by several functions in the library. */ @@ -217,6 +205,10 @@ typedef enum { * @ref UTF8PROC_DECOMPOSE */ UTF8PROC_STRIPMARK = (1<<13), + /** + * Strip unassigned codepoints. + */ + UTF8PROC_STRIPNA = (1<<14), } utf8proc_option_t; /** @name Error codes @@ -281,7 +273,8 @@ typedef struct utf8proc_property_struct { * Boundclass. * @see utf8proc_boundclass_t. */ - unsigned boundclass:8; + unsigned boundclass:6; + unsigned indic_conjunct_break:2; } utf8proc_property_t; /** Unicode categories. */ @@ -382,12 +375,28 @@ typedef enum { UTF8PROC_BOUNDCLASS_SPACINGMARK = 12, /**< Spacingmark */ UTF8PROC_BOUNDCLASS_PREPEND = 13, /**< Prepend */ UTF8PROC_BOUNDCLASS_ZWJ = 14, /**< Zero Width Joiner */ + + /* the following are no longer used in Unicode 11, but we keep + the constants here for backward compatibility */ UTF8PROC_BOUNDCLASS_E_BASE = 15, /**< Emoji Base */ UTF8PROC_BOUNDCLASS_E_MODIFIER = 16, /**< Emoji Modifier */ UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ = 17, /**< Glue_After_ZWJ */ UTF8PROC_BOUNDCLASS_E_BASE_GAZ = 18, /**< E_BASE + GLUE_AFTER_ZJW */ + + /* the Extended_Pictographic property is used in the Unicode 11 + grapheme-boundary rules, so we store it in the boundclass field */ + UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC = 19, + UTF8PROC_BOUNDCLASS_E_ZWG = 20, /* UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC + ZWJ */ } utf8proc_boundclass_t; +/** Indic_Conjunct_Break property. (TR44) */ +typedef enum { + UTF8PROC_INDIC_CONJUNCT_BREAK_NONE = 0, + UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER = 1, + UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT = 2, + UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND = 3, +} utf8proc_indic_conjunct_break_t; + /** * Function pointer type passed to @ref utf8proc_map_custom and * @ref utf8proc_decompose_custom, which is used to specify a user-defined @@ -409,6 +418,11 @@ UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256]; UTF8PROC_DLLEXPORT const char *utf8proc_version(void); /** + * Returns the utf8proc supported Unicode version as a string MAJOR.MINOR.PATCH. + */ +UTF8PROC_DLLEXPORT const char *utf8proc_unicode_version(void); + +/** * Returns an informative error string for the given utf8proc error code * (e.g. the error codes returned by @ref utf8proc_map). */ @@ -473,10 +487,12 @@ UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int * - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster * - @ref UTF8PROC_LUMP - lump certain different codepoints together * - @ref UTF8PROC_STRIPMARK - remove all character marks + * - @ref UTF8PROC_STRIPNA - remove unassigned codepoints * @param last_boundclass * Pointer to an integer variable containing - * the previous codepoint's boundary class if the @ref UTF8PROC_CHARBOUND - * option is used. Otherwise, this parameter is ignored. + * the previous codepoint's (boundclass + indic_conjunct_break << 1) if the @ref UTF8PROC_CHARBOUND + * option is used. If the string is being processed in order, this can be initialized to 0 for + * the beginning of the string, and is thereafter updated automatically. Otherwise, this parameter is ignored. * * @return * In case of success, the number of codepoints written is returned; in case @@ -496,7 +512,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char( * string and orders the decomposed sequences correctly. * * If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing - * will be stopped, when a NULL byte is encounted, otherwise `strlen` + * will be stopped, when a NULL byte is encountered, otherwise `strlen` * bytes are processed. The result (in the form of 32-bit unicode * codepoints) is written into the buffer being pointed to by * `buffer` (which must contain at least `bufsize` entries). In case of @@ -584,6 +600,8 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, * Given a pair of consecutive codepoints, return whether a grapheme break is * permitted between them (as defined by the extended grapheme clusters in UAX#29). * + * @param codepoint1 The first codepoint. + * @param codepoint2 The second codepoint, occurring consecutively after `codepoint1`. * @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires * state to break graphemes. This state can be passed in as a pointer * in the `state` argument and should initially be set to 0. If the @@ -592,7 +610,8 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, * matching the rules in Unicode 8.0.0. * * @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must - * be called IN ORDER on ALL potential breaks in a string. + * be called IN ORDER on ALL potential breaks in a string. However, it + * is safe to reset the state to zero after a grapheme break. */ UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful( utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state); @@ -627,6 +646,18 @@ UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c); UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c); /** + * Given a codepoint `c`, return `1` if the codepoint corresponds to a lower-case character + * and `0` otherwise. + */ +UTF8PROC_DLLEXPORT int utf8proc_islower(utf8proc_int32_t c); + +/** + * Given a codepoint `c`, return `1` if the codepoint corresponds to an upper-case character + * and `0` otherwise. + */ +UTF8PROC_DLLEXPORT int utf8proc_isupper(utf8proc_int32_t c); + +/** * Given a codepoint, return a character width analogous to `wcwidth(codepoint)`, * except that a width of 0 is returned for non-printable codepoints * instead of -1 as in `wcwidth`. @@ -659,7 +690,7 @@ UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoi * contain NULL characters with the string if `str` contained NULL * characters). Other flags in the `options` field are passed to the * functions defined above, and regarded as described. See also - * @ref utfproc_map_custom to supply a custom codepoint transformation. + * @ref utf8proc_map_custom to supply a custom codepoint transformation. * * In case of success the length of the new string is returned, * otherwise a negative error code is returned. @@ -684,8 +715,8 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom( /** @name Unicode normalization * - * Returns a pointer to newly allocated memory of a NFD, NFC, NFKD or NFKC - * normalized version of the null-terminated string `str`. These + * Returns a pointer to newly allocated memory of a NFD, NFC, NFKD, NFKC or + * NFKC_Casefold normalized version of the null-terminated string `str`. These * are shortcuts to calling @ref utf8proc_map with @ref UTF8PROC_NULLTERM * combined with @ref UTF8PROC_STABLE and flags indicating the normalization. */ @@ -698,6 +729,11 @@ UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str); UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str); /** NFKC normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */ UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str); +/** + * NFKC_Casefold normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT + * and @ref UTF8PROC_CASEFOLD and @ref UTF8PROC_IGNORE). + **/ +UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC_Casefold(const utf8proc_uint8_t *str); /** @} */ #ifdef __cplusplus |