summaryrefslogtreecommitdiffstatshomepage
path: root/3rdparty/utf8proc/test
diff options
context:
space:
mode:
Diffstat (limited to '3rdparty/utf8proc/test')
-rw-r--r--3rdparty/utf8proc/test/case.c40
-rw-r--r--3rdparty/utf8proc/test/charwidth.c120
-rw-r--r--3rdparty/utf8proc/test/custom.c1
-rw-r--r--3rdparty/utf8proc/test/fuzz_main.c54
-rw-r--r--3rdparty/utf8proc/test/fuzzer.c84
-rw-r--r--3rdparty/utf8proc/test/graphemetest.c183
-rw-r--r--3rdparty/utf8proc/test/iscase.c62
-rw-r--r--3rdparty/utf8proc/test/iterate.c16
-rw-r--r--3rdparty/utf8proc/test/misc.c51
-rw-r--r--3rdparty/utf8proc/test/normtest.c11
-rwxr-xr-x3rdparty/utf8proc/test/ossfuzz.sh13
-rw-r--r--3rdparty/utf8proc/test/printproperty.c97
-rw-r--r--3rdparty/utf8proc/test/tests.c21
-rw-r--r--3rdparty/utf8proc/test/tests.h10
14 files changed, 577 insertions, 186 deletions
diff --git a/3rdparty/utf8proc/test/case.c b/3rdparty/utf8proc/test/case.c
index 39958e3e933..c1630f0d9d5 100644
--- a/3rdparty/utf8proc/test/case.c
+++ b/3rdparty/utf8proc/test/case.c
@@ -13,37 +13,63 @@ int main(int argc, char **argv)
for (c = 0; c <= 0x110000; ++c) {
utf8proc_int32_t l = utf8proc_tolower(c);
utf8proc_int32_t u = utf8proc_toupper(c);
+ utf8proc_int32_t t = utf8proc_totitle(c);
check(l == c || utf8proc_codepoint_valid(l), "invalid tolower");
check(u == c || utf8proc_codepoint_valid(u), "invalid toupper");
+ check(t == c || utf8proc_codepoint_valid(t), "invalid totitle");
+
+ if (utf8proc_codepoint_valid(c) && (l == u) != (l == t) &&
+ /* Unicode 11: Georgian Mkhedruli chars have uppercase but no titlecase. */
+ !(((c >= 0x10d0 && c <= 0x10fa) || c >= (0x10fd && c <= 0x10ff)) && l != u)) {
+ fprintf(stderr, "unexpected titlecase %x for lowercase %x / uppercase %x\n", t, l, c);
+ ++error;
+ }
+
+ if (sizeof(wint_t) > 2 || (c < (1<<16) && u < (1<<16) && l < (1<<16))) {
+ wint_t l0 = towlower((wint_t)c), u0 = towupper((wint_t)c);
- if (sizeof(wint_t) > 2 || c < (1<<16)) {
- wint_t l0 = towlower(c), u0 = towupper(c);
-
/* OS unicode tables may be out of date. But if they
do have a lower/uppercase mapping, hopefully it
is correct? */
- if (l0 != c && l0 != l) {
+ if (l0 != (wint_t)c && l0 != (wint_t)l) {
fprintf(stderr, "MISMATCH %x != towlower(%x) == %x\n",
l, c, l0);
++error;
}
- else if (l0 != l) { /* often true for out-of-date OS unicode */
+ else if (l0 != (wint_t)l) { /* often true for out-of-date OS unicode */
++better;
/* printf("%x != towlower(%x) == %x\n", l, c, l0); */
}
- if (u0 != c && u0 != u) {
+ if (u0 != (wint_t)c && u0 != (wint_t)u) {
fprintf(stderr, "MISMATCH %x != towupper(%x) == %x\n",
u, c, u0);
++error;
}
- else if (u0 != u) { /* often true for out-of-date OS unicode */
+ else if (u0 != (wint_t)u) { /* often true for out-of-date OS unicode */
++better;
/* printf("%x != towupper(%x) == %x\n", u, c, u0); */
}
}
}
check(!error, "utf8proc case conversion FAILED %d tests.", error);
+
+ /* issue #130 */
+ check(utf8proc_toupper(0x00df) == 0x1e9e &&
+ utf8proc_totitle(0x00df) == 0x1e9e &&
+ utf8proc_tolower(0x00df) == 0x00df &&
+ utf8proc_tolower(0x1e9e) == 0x00df &&
+ utf8proc_toupper(0x1e9e) == 0x1e9e,
+ "incorrect 0x00df/0x1e9e case conversions");
+ utf8proc_uint8_t str_00df[] = {0xc3, 0x9f, 0x00};
+ utf8proc_uint8_t str_1e9e[] = {0xe1, 0xba, 0x9e, 0x00};
+ utf8proc_uint8_t *s1 = utf8proc_NFKC_Casefold(str_00df);
+ utf8proc_uint8_t *s2 = utf8proc_NFKC_Casefold(str_1e9e);
+ check(!strcmp((char*)s1, "ss") &&
+ !strcmp((char*)s2, "ss"),
+ "incorrect 0x00df/0x1e9e casefold normalization");
+ free(s1);
+ free(s2);
printf("More up-to-date than OS unicode tables for %d tests.\n", better);
printf("utf8proc case conversion tests SUCCEEDED.\n");
return 0;
diff --git a/3rdparty/utf8proc/test/charwidth.c b/3rdparty/utf8proc/test/charwidth.c
index 330f18eebb4..c5cbbd7cdc1 100644
--- a/3rdparty/utf8proc/test/charwidth.c
+++ b/3rdparty/utf8proc/test/charwidth.c
@@ -2,70 +2,76 @@
#include <ctype.h>
#include <wchar.h>
+static int my_unassigned(int c) {
+ int cat = utf8proc_get_property(c)->category;
+ return (cat == UTF8PROC_CATEGORY_CN) || (cat == UTF8PROC_CATEGORY_CO);
+}
+
static int my_isprint(int c) {
- int cat = utf8proc_get_property(c)->category;
- return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) ||
- (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd);
+ int cat = utf8proc_get_property(c)->category;
+ return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) ||
+ (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd || c == 0x00ad) ||
+ (cat == UTF8PROC_CATEGORY_CN) || (cat == UTF8PROC_CATEGORY_CO);
}
int main(int argc, char **argv)
{
- int c, error = 0, updates = 0;
+ int c, error = 0, updates = 0;
+
+ (void) argc; /* unused */
+ (void) argv; /* unused */
- (void) argc; /* unused */
- (void) argv; /* unused */
+ /* some simple sanity tests of the character widths */
+ for (c = 0; c <= 0x110000; ++c) {
+ int cat = utf8proc_get_property(c)->category;
+ int w = utf8proc_charwidth(c);
+ if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) && w > 0) {
+ fprintf(stderr, "nonzero width %d for combining char %x\n", w, c);
+ error += 1;
+ }
+ if (w == 0 &&
+ ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) ||
+ (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) ||
+ (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) {
+ fprintf(stderr, "zero width for symbol-like char %x\n", c);
+ error += 1;
+ }
+ if (c <= 127 && ((!isprint(c) && w > 0) || (isprint(c) && wcwidth(c) != w))) {
+ fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n",
+ wcwidth(c), w,
+ isprint(c) ? "printable" : "non-printable", c);
+ error += 1;
+ }
+ if (!my_isprint(c) && w > 0) {
+ fprintf(stderr, "non-printing %x had width %d\n", c, w);
+ error += 1;
+ }
+ if (my_unassigned(c) && w != 1) {
+ fprintf(stderr, "unexpected width %d for unassigned char %x\n", w, c);
+ error += 1;
+ }
+ }
+ check(!error, "utf8proc_charwidth FAILED %d tests.", error);
- /* some simple sanity tests of the character widths */
- for (c = 0; c <= 0x110000; ++c) {
- int cat = utf8proc_get_property(c)->category;
- int w = utf8proc_charwidth(c);
- if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) &&
- w > 0) {
- fprintf(stderr, "nonzero width %d for combining char %x\n", w, c);
- error = 1;
- }
- if (w == 0 &&
- ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) ||
- (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) ||
- (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) {
- fprintf(stderr, "zero width for symbol-like char %x\n", c);
- error = 1;
- }
- if (c <= 127 && ((!isprint(c) && w > 0) ||
- (isprint(c) && wcwidth(c) != w))) {
- fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n",
- wcwidth(c), w,
- isprint(c) ? "printable" : "non-printable", c);
- error = 1;
- }
- if (!my_isprint(c) && w > 0) {
- fprintf(stderr, "non-printing %x had width %d\n", c, w);
- error = 1;
- }
- }
- check(!error, "utf8proc_charwidth FAILED tests.");
+ check(utf8proc_charwidth(0x00ad) == 1, "incorrect width for U+00AD (soft hyphen)");
+ check(utf8proc_charwidth(0xe000) == 1, "incorrect width for U+e000 (PUA)");
- /* print some other information by compariing with system wcwidth */
- printf("Mismatches with system wcwidth (not necessarily errors):\n");
- for (c = 0; c <= 0x110000; ++c) {
- int w = utf8proc_charwidth(c);
- int wc = wcwidth(c);
- if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue;
- /* lots of these errors for out-of-date system unicode tables */
- if (wc == -1 && my_isprint(c) && w > 0) {
- updates += 1;
-#if 0
- printf(" wcwidth(%x) = -1 for printable char\n", c);
-#endif
- }
- if (wc == -1 && !my_isprint(c) && w > 0)
- printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w);
- if (wc >= 0 && wc != w)
- printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w);
- }
- printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n",
- updates);
- printf("Character-width tests SUCCEEDED.\n");
+ /* print some other information by compariing with system wcwidth */
+ printf("Mismatches with system wcwidth (not necessarily errors):\n");
+ for (c = 0; c <= 0x110000; ++c) {
+ int w = utf8proc_charwidth(c);
+ int wc = wcwidth(c);
+ if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue;
+ /* lots of these errors for out-of-date system unicode tables */
+ if (wc == -1 && my_isprint(c) && !my_unassigned(c) && w > 0)
+ updates += 1;
+ if (wc == -1 && !my_isprint(c) && w > 0)
+ printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w);
+ if (wc >= 0 && wc != w)
+ printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w);
+ }
+ printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n", updates);
+ printf("Character-width tests SUCCEEDED.\n");
- return 0;
+ return 0;
}
diff --git a/3rdparty/utf8proc/test/custom.c b/3rdparty/utf8proc/test/custom.c
index f85b3cc2bc6..fe4239d91b4 100644
--- a/3rdparty/utf8proc/test/custom.c
+++ b/3rdparty/utf8proc/test/custom.c
@@ -23,5 +23,6 @@ int main(void)
check(strlen((char*) output) == 6, "incorrect output length");
check(!memcmp(correct, output, 7), "incorrect output data");
free(output);
+ printf("map_custom tests SUCCEEDED.\n");
return 0;
}
diff --git a/3rdparty/utf8proc/test/fuzz_main.c b/3rdparty/utf8proc/test/fuzz_main.c
new file mode 100644
index 00000000000..39bf14738c2
--- /dev/null
+++ b/3rdparty/utf8proc/test/fuzz_main.c
@@ -0,0 +1,54 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+/* Fuzz target entry point, works without libFuzzer */
+
+int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size);
+
+int main(int argc, char **argv)
+{
+ FILE *f;
+ char *buf = NULL;
+ long siz_buf;
+
+ if(argc < 2)
+ {
+ fprintf(stderr, "no input file\n");
+ goto err;
+ }
+
+ f = fopen(argv[1], "rb");
+ if(f == NULL)
+ {
+ fprintf(stderr, "error opening input file %s\n", argv[1]);
+ goto err;
+ }
+
+ fseek(f, 0, SEEK_END);
+
+ siz_buf = ftell(f);
+ rewind(f);
+
+ if(siz_buf < 1) goto err;
+
+ buf = (char*)malloc(siz_buf);
+ if(buf == NULL)
+ {
+ fprintf(stderr, "malloc() failed\n");
+ goto err;
+ }
+
+ if(fread(buf, siz_buf, 1, f) != 1)
+ {
+ fprintf(stderr, "fread() failed\n");
+ goto err;
+ }
+
+ (void)LLVMFuzzerTestOneInput((uint8_t*)buf, siz_buf);
+
+err:
+ free(buf);
+
+ return 0;
+}
diff --git a/3rdparty/utf8proc/test/fuzzer.c b/3rdparty/utf8proc/test/fuzzer.c
new file mode 100644
index 00000000000..b5349a322c9
--- /dev/null
+++ b/3rdparty/utf8proc/test/fuzzer.c
@@ -0,0 +1,84 @@
+#include <utf8proc.h>
+#include <string.h>
+
+int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
+{
+ if(size < 1) return 0;
+
+ /* Avoid timeout with long inputs */
+ if(size > (64 * 1024)) return 0;
+
+ if(data[size-1] != '\0') return 0;
+
+ const uint8_t* ptr = data;
+ utf8proc_int32_t c = 0, c_prev = 0, state = 0;
+ utf8proc_option_t options;
+ utf8proc_ssize_t ret, bytes = 0;
+ size_t len = strlen((const char*)data);
+
+ while(bytes != len)
+ {
+ ret = utf8proc_iterate(ptr, -1, &c);
+
+ if(ret < 0 || ret == 0) break;
+
+ bytes += ret;
+ ptr += ret;
+
+ utf8proc_tolower(c);
+ utf8proc_toupper(c);
+ utf8proc_totitle(c);
+ utf8proc_islower(c);
+ utf8proc_isupper(c);
+ utf8proc_charwidth(c);
+ utf8proc_category(c);
+ utf8proc_category_string(c);
+ utf8proc_codepoint_valid(c);
+
+ utf8proc_grapheme_break(c_prev, c);
+ utf8proc_grapheme_break_stateful(c_prev, c, &state);
+
+ c_prev = c;
+ }
+
+ utf8proc_int32_t *copy = size >= 4 ? NULL : malloc(size);
+
+ if(copy)
+ {
+ size /= 4;
+
+ options = UTF8PROC_STRIPCC | UTF8PROC_NLF2LS | UTF8PROC_NLF2PS;
+ memcpy(copy, data, size);
+ utf8proc_normalize_utf32(copy, size, options);
+
+ options = UTF8PROC_STRIPCC | UTF8PROC_NLF2LS;
+ memcpy(copy, data, size);
+ utf8proc_normalize_utf32(copy, size, options);
+
+ options = UTF8PROC_STRIPCC | UTF8PROC_NLF2PS;
+ memcpy(copy, data, size);
+ utf8proc_normalize_utf32(copy, size, options);
+
+ options = UTF8PROC_STRIPCC;
+ memcpy(copy, data, size);
+ utf8proc_normalize_utf32(copy, size, options);
+
+ options = UTF8PROC_LUMP;
+ memcpy(copy, data, size);
+ utf8proc_normalize_utf32(copy, size, options);
+
+ options = 0;
+ memcpy(copy, data, size);
+ utf8proc_normalize_utf32(copy, size, options);
+
+ free(copy);
+ }
+
+ free(utf8proc_NFD(data));
+ free(utf8proc_NFC(data));
+ free(utf8proc_NFKD(data));
+ free(utf8proc_NFKC(data));
+ free(utf8proc_NFKC_Casefold(data));
+
+ return 0;
+} \ No newline at end of file
diff --git a/3rdparty/utf8proc/test/graphemetest.c b/3rdparty/utf8proc/test/graphemetest.c
index eb3645b9a0c..6fbe5bdaa97 100644
--- a/3rdparty/utf8proc/test/graphemetest.c
+++ b/3rdparty/utf8proc/test/graphemetest.c
@@ -1,74 +1,135 @@
#include "tests.h"
+/* check one line in the format of GraphemeBreakTest.txt */
+void checkline(const char *_buf, bool verbose) {
+ size_t bi = 0, si = 0;
+ utf8proc_uint8_t src[1024]; /* more than long enough for all of our tests */
+ const unsigned char *buf = (const unsigned char *) _buf;
+
+ while (buf[bi]) {
+ bi = skipspaces(buf, bi);
+ if (buf[bi] == 0xc3 && buf[bi+1] == 0xb7) { /* U+00f7 = grapheme break */
+ src[si++] = '/';
+ bi += 2;
+ }
+ else if (buf[bi] == 0xc3 && buf[bi+1] == 0x97) { /* U+00d7 = no break */
+ bi += 2;
+ }
+ else if (buf[bi] == '#') { /* start of comments */
+ break;
+ }
+ else if (buf[bi] == '/') { /* for convenience, also accept / as grapheme break */
+ src[si++] = '/';
+ bi += 1;
+ }
+ else { /* hex-encoded codepoint */
+ size_t len = encode((unsigned char*) (src + si), buf + bi) - 1;
+ while (src[si]) ++si; /* advance to NUL termination */
+ bi += len;
+ }
+ }
+ if (si && src[si-1] == '/')
+ --si; /* no break after final grapheme */
+ src[si] = 0; /* NUL-terminate */
+
+ if (si) { /* test utf8proc_map */
+ utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */
+ size_t i = 0, j = 0;
+ utf8proc_ssize_t glen, k;
+ utf8proc_uint8_t *g; /* utf8proc_map grapheme results */
+ while (i < si) {
+ if (src[i] != '/')
+ utf8[j++] = src[i++];
+ else
+ i++;
+ }
+ glen = utf8proc_map(utf8, (utf8proc_ssize_t)j, &g, UTF8PROC_CHARBOUND);
+ if (glen == UTF8PROC_ERROR_INVALIDUTF8) {
+ /* the test file contains surrogate codepoints, which are only for UTF-16 */
+ printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno);
+ }
+ else {
+ check(glen >= 0, "utf8proc_map error = %s",
+ utf8proc_errmsg(glen));
+ for (k = 0; k <= glen; ++k)
+ if (g[k] == 0xff)
+ g[k] = '/'; /* easier-to-read output (/ is not in test strings) */
+ check(!strcmp((char*)g, (char*)src),
+ "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
+ }
+ free(g);
+ }
+
+ if (si) { /* test manual calls to utf8proc_grapheme_break_stateful */
+ utf8proc_int32_t state = 0, prev_codepoint = 0;
+ size_t i = 0;
+ utf8proc_bool expectbreak = false;
+ do {
+ utf8proc_int32_t codepoint;
+ i += (size_t)utf8proc_iterate(src + i, (utf8proc_ssize_t)(si - i), &codepoint);
+ check(codepoint >= 0, "invalid UTF-8 data");
+ if (codepoint == 0x002F)
+ expectbreak = true;
+ else {
+ if (prev_codepoint != 0) {
+ check(expectbreak == utf8proc_grapheme_break_stateful(prev_codepoint, codepoint, &state),
+ "grapheme mismatch: between 0x%04x and 0x%04x in \"%s\"", prev_codepoint, codepoint, (char*) src);
+ }
+ expectbreak = false;
+ prev_codepoint = codepoint;
+ }
+ } while (i < si);
+ }
+
+ if (verbose)
+ printf("passed grapheme test: \"%s\"\n", (char*) src);
+}
+
int main(int argc, char **argv)
{
- char *buf = NULL;
- size_t bufsize = 0;
+ unsigned char buf[8192];
FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
- utf8proc_uint8_t src[1024];
- int len;
-
+
check(f != NULL, "error opening GraphemeBreakTest.txt");
- while (getline(&buf, &bufsize, f) > 0) {
- size_t bi = 0, si = 0;
- lineno += 1;
-
- if (lineno % 100 == 0)
+ while (simple_getline(buf, f) > 0) {
+ if ((++lineno) % 100 == 0)
printf("checking line %zd...\n", lineno);
-
if (buf[0] == '#') continue;
-
- while (buf[bi]) {
- bi = skipspaces(buf, bi);
- if (buf[bi] == '/') { /* grapheme break */
- src[si++] = '/';
- bi++;
- }
- else if (buf[bi] == '+') { /* no break */
- bi++;
- }
- else if (buf[bi] == '#') { /* start of comments */
- break;
- }
- else { /* hex-encoded codepoint */
- len = encode((char*) (src + si), buf + bi) - 1;
- while (src[si]) ++si; /* advance to NUL termination */
- bi += len;
- }
- }
- if (si && src[si-1] == '/')
- --si; /* no break after final grapheme */
- src[si] = 0; /* NUL-terminate */
-
- if (si) {
- utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */
- size_t i = 0, j = 0;
- utf8proc_ssize_t glen;
- utf8proc_uint8_t *g; /* utf8proc_map grapheme results */
- while (i < si) {
- if (src[i] != '/')
- utf8[j++] = src[i++];
- else
- i++;
- }
- glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND);
- if (glen == UTF8PROC_ERROR_INVALIDUTF8) {
- /* the test file contains surrogate codepoints, which are only for UTF-16 */
- printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno);
- }
- else {
- check(glen >= 0, "utf8proc_map error = %s",
- utf8proc_errmsg(glen));
- for (i = 0; i <= glen; ++i)
- if (g[i] == 0xff)
- g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
- check(!strcmp((char*)g, (char*)src),
- "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
- }
- free(g);
- }
+ checkline((char *) buf, false);
}
fclose(f);
printf("Passed tests after %zd lines!\n", lineno);
+
+ printf("Performing regression tests...\n");
+
+ /* issue 144 */
+ {
+ utf8proc_uint8_t input[] = {0xef,0xbf,0xbf,0xef,0xbf,0xbe,0x00}; /* "\uffff\ufffe" */
+ utf8proc_uint8_t output[] = {0xff,0xef,0xbf,0xbf,0xff,0xef,0xbf,0xbe,0x00}; /* with 0xff grapheme markers */
+ utf8proc_ssize_t glen;
+ utf8proc_uint8_t *g;
+ glen = utf8proc_map(input, 6, &g, UTF8PROC_CHARBOUND);
+ check(!strcmp((char*)g, (char*)output), "mishandled u+ffff and u+fffe grapheme breaks");
+ check(glen != 6, "mishandled u+ffff and u+fffe grapheme breaks");
+ free(g);
+ };
+
+ /* https://github.com/JuliaLang/julia/issues/37680 */
+ checkline("/ 1f1f8 1f1ea / 1f1f8 1f1ea /", true); /* Two swedish flags after each other */
+ checkline("/ 1f926 1f3fc 200d 2642 fe0f /", true); /* facepalm + pale skin + zwj + male sign + FE0F */
+ checkline("/ 1f468 1f3fb 200d 1f91d 200d 1f468 1f3fd /", true); /* man face + pale skin + zwj + hand holding + zwj + man face + dark skin */
+
+ /* more GB9c tests */
+ checkline("/ 0915 0300 094d 0300 0924 / 0915 /", true);
+ checkline("/ 0915 0300 094d 0300 094d 0924 / 0915 /", true);
+ checkline("/ 0915 0300 0300 / 0924 / 0915 /", true);
+ checkline("/ 0915 0300 094d 0300 / 0078 /", true);
+ checkline("/ 0300 094d 0300 / 0924 / 0915 /", true);
+
+ check(utf8proc_grapheme_break(0x03b1, 0x03b2), "failed 03b1 / 03b2 test");
+ check(!utf8proc_grapheme_break(0x03b1, 0x0302), "failed 03b1 0302 test");
+
+ printf("Passed regression tests!\n");
+
return 0;
}
diff --git a/3rdparty/utf8proc/test/iscase.c b/3rdparty/utf8proc/test/iscase.c
new file mode 100644
index 00000000000..f3f8cf31b40
--- /dev/null
+++ b/3rdparty/utf8proc/test/iscase.c
@@ -0,0 +1,62 @@
+#include "tests.h"
+
+int read_range(FILE *f, utf8proc_int32_t *start, utf8proc_int32_t *end)
+{
+ unsigned char buf[8192];
+ size_t len = simple_getline(buf, f);
+ size_t pos = skipspaces(buf, 0);
+ unsigned char s[16];
+ if (pos == len || buf[pos] == '#') return 0;
+ pos += encode(s, buf + pos) - 1;
+ check(s[0], "invalid line %s in data", buf);
+ utf8proc_iterate((utf8proc_uint8_t*) s, -1, start);
+ if (buf[pos] == '.' && buf[pos+1] == '.') {
+ encode(s, buf + pos + 2);
+ check(s[0], "invalid line %s in data", buf);
+ utf8proc_iterate((utf8proc_uint8_t*) s, -1, end);
+ }
+ else
+ *end = *start;
+ return 1;
+}
+
+int test_iscase(const char *fname, int (*iscase)(utf8proc_int32_t),
+ utf8proc_int32_t (*thatcase)(utf8proc_int32_t))
+{
+ FILE *f = fopen(fname, "r");
+ int lines = 0, tests = 0, success = 1;
+ utf8proc_int32_t c = 0;
+
+ check(f != NULL, "error opening data file \"%s\"\n", fname);
+
+ while (success && !feof(f)) {
+ utf8proc_int32_t start, end;
+ if (read_range(f, &start, &end)) {
+ for (; c < start; ++c) {
+ check(!iscase(c), "failed !iscase(%04x) in %s\n", c, fname);
+ }
+ for (; c <= end; ++c) {
+ check(iscase(c), "failed iscase(%04x) in %s\n", c, fname);
+ check(thatcase(c) == c, "inconsistent thatcase(%04x) in %s\n", c, fname);
+ ++tests;
+ }
+ }
+ ++lines;
+ }
+ for (; c <= 0x110000; ++c) {
+ check(!iscase(c), "failed !iscase(%04x) in %s\n", c, fname);
+ }
+
+ printf("Checked %d characters from %d lines of %s\n", tests, lines, fname);
+ fclose(f);
+ return success;
+}
+
+int main(int argc, char **argv)
+{
+ check(argc == 3, "Expected Lowercase.txt and Uppercase.txt as arguments");
+ check(test_iscase(argv[1], utf8proc_islower, utf8proc_tolower), "Lowercase tests failed");
+ check(test_iscase(argv[2], utf8proc_isupper, utf8proc_toupper), "Uppercase tests failed");
+ printf("utf8proc iscase tests SUCCEEDED.\n");
+ return 0;
+}
diff --git a/3rdparty/utf8proc/test/iterate.c b/3rdparty/utf8proc/test/iterate.c
index c1674b79952..a923afdbf09 100644
--- a/3rdparty/utf8proc/test/iterate.c
+++ b/3rdparty/utf8proc/test/iterate.c
@@ -8,7 +8,7 @@ static int error;
#define CHECKVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,len,__LINE__)
#define CHECKINVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,UTF8PROC_ERROR_INVALIDUTF8,__LINE__)
-static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int line)
+static void testbytes(utf8proc_uint8_t *buf, utf8proc_ssize_t len, utf8proc_ssize_t retval, int line)
{
utf8proc_int32_t out[16];
utf8proc_ssize_t ret;
@@ -16,13 +16,13 @@ static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int
/* Make a copy to ensure that memory is left uninitialized after "len"
* bytes. This way, Valgrind can detect overreads.
*/
- unsigned char tmp[16];
- memcpy(tmp, buf, len);
+ utf8proc_uint8_t tmp[16];
+ memcpy(tmp, buf, (unsigned long int)len);
tests++;
if ((ret = utf8proc_iterate(tmp, len, out)) != retval) {
fprintf(stderr, "Failed (%d):", line);
- for (int i = 0; i < len ; i++) {
+ for (utf8proc_ssize_t i = 0; i < len ; i++) {
fprintf(stderr, " 0x%02x", tmp[i]);
}
fprintf(stderr, " -> %zd\n", ret);
@@ -32,8 +32,10 @@ static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int
int main(int argc, char **argv)
{
- uint32_t byt;
- unsigned char buf[16];
+ utf8proc_int32_t byt;
+ utf8proc_uint8_t buf[16];
+
+ (void) argc; (void) argv; /* unused */
tests = error = 0;
@@ -54,7 +56,7 @@ int main(int argc, char **argv)
CHECKVALID(3, 0xbe, 4);
CHECKVALID(3, 0xbf, 4);
}
-
+
// Continuation byte not after lead
for (byt = 0x80; byt < 0xc0; byt++) {
CHECKINVALID(0, byt, 1);
diff --git a/3rdparty/utf8proc/test/misc.c b/3rdparty/utf8proc/test/misc.c
new file mode 100644
index 00000000000..9156f95541d
--- /dev/null
+++ b/3rdparty/utf8proc/test/misc.c
@@ -0,0 +1,51 @@
+/* Miscellaneous tests, e.g. regression tests */
+
+#include "tests.h"
+
+static void issue128(void) /* #128 */
+{
+ utf8proc_uint8_t input[] = {0x72, 0xcc, 0x87, 0xcc, 0xa3, 0x00}; /* "r\u0307\u0323" */
+ utf8proc_uint8_t nfc[] = {0xe1, 0xb9, 0x9b, 0xcc, 0x87, 0x00}; /* "\u1E5B\u0307" */
+ utf8proc_uint8_t nfd[] = {0x72, 0xcc, 0xa3, 0xcc, 0x87, 0x00}; /* "r\u0323\u0307" */
+ utf8proc_uint8_t *nfc_out, *nfd_out;
+ nfc_out = utf8proc_NFC(input);
+ printf("NFC \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)nfc_out, (char*)nfc);
+ check(strlen((char*) nfc_out) == 5, "incorrect nfc length");
+ check(!memcmp(nfc, nfc_out, 6), "incorrect nfc data");
+ nfd_out = utf8proc_NFD(input);
+ printf("NFD \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)nfd_out, (char*)nfd);
+ check(strlen((char*) nfd_out) == 5, "incorrect nfd length");
+ check(!memcmp(nfd, nfd_out, 6), "incorrect nfd data");
+ free(nfd_out); free(nfc_out);
+}
+
+static void issue102(void) /* #128 */
+{
+ utf8proc_uint8_t input[] = {0x58, 0xe2, 0x81, 0xa5, 0x45, 0xcc, 0x80, 0xc2, 0xad, 0xe1, 0xb4, 0xac, 0x00}; /* "X\u2065E\u0300\u00ad\u1d2c" */
+ utf8proc_uint8_t stripna[] = {0x78, 0xc3, 0xa8, 0x61, 0x00}; /* "x\u00e8a" */
+ utf8proc_uint8_t correct[] = {0x78, 0xe2, 0x81, 0xa5, 0xc3, 0xa8, 0x61, 0x00}; /* "x\u2065\u00e8a" */
+ utf8proc_uint8_t *output;
+ utf8proc_map(input, 0, &output, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+ UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD | UTF8PROC_IGNORE | UTF8PROC_STRIPNA);
+ printf("NFKC_Casefold \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)output, (char*)stripna);
+ check(strlen((char*) output) == 4, "incorrect NFKC_Casefold+stripna length");
+ check(!memcmp(stripna, output, 5), "incorrect NFKC_Casefold+stripna data");
+ free(output);
+ output = utf8proc_NFKC_Casefold(input);
+ printf("NFKC_Casefold \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)output, (char*)correct);
+ check(strlen((char*) output) == 7, "incorrect NFKC_Casefold length");
+ check(!memcmp(correct, output, 8), "incorrect NFKC_Casefold data");
+ free(output);
+}
+
+int main(void)
+{
+ issue128();
+ issue102();
+#ifdef UNICODE_VERSION
+ printf("Unicode version: Makefile has %s, has API %s\n", UNICODE_VERSION, utf8proc_unicode_version());
+ check(!strcmp(UNICODE_VERSION, utf8proc_unicode_version()), "utf8proc_unicode_version mismatch");
+#endif
+ printf("Misc tests SUCCEEDED.\n");
+ return 0;
+}
diff --git a/3rdparty/utf8proc/test/normtest.c b/3rdparty/utf8proc/test/normtest.c
index 555c14c84bf..627ee79fd26 100644
--- a/3rdparty/utf8proc/test/normtest.c
+++ b/3rdparty/utf8proc/test/normtest.c
@@ -1,21 +1,20 @@
#include "tests.h"
#define CHECK_NORM(NRM, norm, src) { \
- char *src_norm = (char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \
- check(!strcmp(norm, src_norm), \
+ unsigned char *src_norm = (unsigned char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \
+ check(!strcmp((char *) norm, (char *) src_norm), \
"normalization failed for %s -> %s", src, norm); \
free(src_norm); \
}
int main(int argc, char **argv)
{
- char *buf = NULL;
- size_t bufsize = 0;
+ unsigned char buf[8192];
FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
- char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024];
+ unsigned char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024];
check(f != NULL, "error opening NormalizationTest.txt");
- while (getline(&buf, &bufsize, f) > 0) {
+ while (simple_getline(buf, f) > 0) {
size_t offset;
lineno += 1;
diff --git a/3rdparty/utf8proc/test/ossfuzz.sh b/3rdparty/utf8proc/test/ossfuzz.sh
new file mode 100755
index 00000000000..3e3b33b049a
--- /dev/null
+++ b/3rdparty/utf8proc/test/ossfuzz.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -eu
+# This script is meant to be run by
+# https://github.com/google/oss-fuzz/blob/master/projects/utf8proc/Dockerfile
+
+mkdir build
+cd build
+cmake .. -DUTF8PROC_ENABLE_TESTING=ON -DLIB_FUZZING_ENGINE="$LIB_FUZZING_ENGINE"
+make -j$(nproc)
+
+cp $SRC/utf8proc/build/fuzzer $OUT/utf8proc_fuzzer
+
+find $SRC/utf8proc/test -name "*.txt" | \
+ xargs zip $OUT/utf8proc_fuzzer_seed_corpus.zip
diff --git a/3rdparty/utf8proc/test/printproperty.c b/3rdparty/utf8proc/test/printproperty.c
index 2819aa1e881..89949355051 100644
--- a/3rdparty/utf8proc/test/printproperty.c
+++ b/3rdparty/utf8proc/test/printproperty.c
@@ -4,46 +4,61 @@
int main(int argc, char **argv)
{
- int i;
+ int i;
- for (i = 1; i < argc; ++i) {
- unsigned int c;
- if (!strcmp(argv[i], "-V")) {
- printf("utf8proc version %s\n", utf8proc_version());
- continue;
- }
- check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]);
- const utf8proc_property_t *p = utf8proc_get_property(c);
- printf("U+%s:\n"
- " category = %s\n"
- " combining_class = %d\n"
- " bidi_class = %d\n"
- " decomp_type = %d\n"
- " uppercase_mapping = %x\n"
- " lowercase_mapping = %x\n"
- " titlecase_mapping = %x\n"
- " comb_index = %d\n"
- " bidi_mirrored = %d\n"
- " comp_exclusion = %d\n"
- " ignorable = %d\n"
- " control_boundary = %d\n"
- " boundclass = %d\n"
- " charwidth = %d\n",
- argv[i],
- utf8proc_category_string(c),
- p->combining_class,
- p->bidi_class,
- p->decomp_type,
- utf8proc_toupper(c),
- utf8proc_tolower(c),
- utf8proc_totitle(c),
- p->comb_index,
- p->bidi_mirrored,
- p->comp_exclusion,
- p->ignorable,
- p->control_boundary,
- p->boundclass,
- utf8proc_charwidth(c));
- }
- return 0;
+ for (i = 1; i < argc; ++i) {
+ utf8proc_uint8_t cstr[16], *map;
+ utf8proc_uint32_t x;
+ utf8proc_int32_t c;
+ if (!strcmp(argv[i], "-V")) {
+ printf("utf8proc version %s\n", utf8proc_version());
+ continue;
+ }
+ check(sscanf(argv[i],"%x", &x) == 1, "invalid hex input %s", argv[i]);
+ c = (utf8proc_int32_t)x;
+ const utf8proc_property_t *p = utf8proc_get_property(c);
+
+ if (utf8proc_codepoint_valid(c))
+ cstr[utf8proc_encode_char(c, cstr)] = 0;
+ else
+ strcat((char*)cstr, "N/A");
+ utf8proc_map(cstr, 0, &map, UTF8PROC_NULLTERM | UTF8PROC_CASEFOLD);
+
+ printf("U+%s: %s\n"
+ " category = %s\n"
+ " combining_class = %d\n"
+ " bidi_class = %d\n"
+ " decomp_type = %d\n"
+ " uppercase_mapping = %04x (seqindex %04x)%s\n"
+ " lowercase_mapping = %04x (seqindex %04x)%s\n"
+ " titlecase_mapping = %04x (seqindex %04x)\n"
+ " casefold = %s\n"
+ " comb_index = %d\n"
+ " bidi_mirrored = %d\n"
+ " comp_exclusion = %d\n"
+ " ignorable = %d\n"
+ " control_boundary = %d\n"
+ " boundclass = %d\n"
+ " indic_conjunct_break = %d\n"
+ " charwidth = %d\n",
+ argv[i], (char*) cstr,
+ utf8proc_category_string(c),
+ p->combining_class,
+ p->bidi_class,
+ p->decomp_type,
+ utf8proc_toupper(c), p->uppercase_seqindex, utf8proc_isupper(c) ? " (isupper)" : "",
+ utf8proc_tolower(c), p->lowercase_seqindex, utf8proc_islower(c) ? " (islower)" : "",
+ utf8proc_totitle(c), p->titlecase_seqindex,
+ (char *) map,
+ p->comb_index,
+ p->bidi_mirrored,
+ p->comp_exclusion,
+ p->ignorable,
+ p->control_boundary,
+ p->boundclass,
+ p->indic_conjunct_break,
+ utf8proc_charwidth(c));
+ free(map);
+ }
+ return 0;
}
diff --git a/3rdparty/utf8proc/test/tests.c b/3rdparty/utf8proc/test/tests.c
index 0fb0da36305..68b21e7f02a 100644
--- a/3rdparty/utf8proc/test/tests.c
+++ b/3rdparty/utf8proc/test/tests.c
@@ -17,7 +17,7 @@ void check(int cond, const char *format, ...)
}
}
-size_t skipspaces(const char *buf, size_t i)
+size_t skipspaces(const unsigned char *buf, size_t i)
{
while (isspace(buf[i])) ++i;
return i;
@@ -27,9 +27,10 @@ size_t skipspaces(const char *buf, size_t i)
separated by whitespace, and terminated by any character not in
[0-9a-fA-F] or whitespace, then stores the corresponding utf8 string
in dest, returning the number of bytes read from buf */
-size_t encode(char *dest, const char *buf)
+size_t encode(unsigned char *dest, const unsigned char *buf)
{
- size_t i = 0, j, d = 0;
+ size_t i = 0, j;
+ utf8proc_ssize_t d = 0;
for (;;) {
int c;
i = skipspaces(buf, i);
@@ -39,8 +40,20 @@ size_t encode(char *dest, const char *buf)
dest[d] = 0; /* NUL-terminate destination string */
return i + 1;
}
- check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
+ check(sscanf((char *) (buf + i), "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
i = j; /* skip to char after hex input */
d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d));
}
}
+
+/* simplistic, portable replacement for getline, sufficient for our tests */
+size_t simple_getline(unsigned char buf[8192], FILE *f) {
+ size_t i = 0;
+ while (i < 8191) {
+ int c = getc(f);
+ if (c == EOF || c == '\n') break;
+ buf[i++] = (unsigned char) c;
+ }
+ buf[i] = 0;
+ return i;
+}
diff --git a/3rdparty/utf8proc/test/tests.h b/3rdparty/utf8proc/test/tests.h
index 1811a734a5f..a30510100ed 100644
--- a/3rdparty/utf8proc/test/tests.h
+++ b/3rdparty/utf8proc/test/tests.h
@@ -1,13 +1,16 @@
/* Common functions and includes for our test programs. */
/*
- * Set feature macro to enable getline() and wcwidth().
+ * Set feature macro to enable wcwidth().
*
* Please refer to section 2.2.1 of POSIX.1-2008:
* http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_02
*/
#define _XOPEN_SOURCE 700
+/* silence warnings about sscanf on Windows */
+#define _CRT_SECURE_NO_WARNINGS
+
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
@@ -19,5 +22,6 @@
extern size_t lineno;
void check(int cond, const char *format, ...);
-size_t skipspaces(const char *buf, size_t i);
-size_t encode(char *dest, const char *buf);
+size_t skipspaces(const unsigned char *buf, size_t i);
+size_t encode(unsigned char *dest, const unsigned char *buf);
+size_t simple_getline(unsigned char buf[8192], FILE *f);