diff options
Diffstat (limited to '3rdparty/utf8proc/test')
-rw-r--r-- | 3rdparty/utf8proc/test/case.c | 40 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/charwidth.c | 120 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/custom.c | 1 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/fuzz_main.c | 54 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/fuzzer.c | 84 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/graphemetest.c | 183 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/iscase.c | 62 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/iterate.c | 16 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/misc.c | 51 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/normtest.c | 11 | ||||
-rwxr-xr-x | 3rdparty/utf8proc/test/ossfuzz.sh | 13 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/printproperty.c | 97 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/tests.c | 21 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/tests.h | 10 |
14 files changed, 577 insertions, 186 deletions
diff --git a/3rdparty/utf8proc/test/case.c b/3rdparty/utf8proc/test/case.c index 39958e3e933..c1630f0d9d5 100644 --- a/3rdparty/utf8proc/test/case.c +++ b/3rdparty/utf8proc/test/case.c @@ -13,37 +13,63 @@ int main(int argc, char **argv) for (c = 0; c <= 0x110000; ++c) { utf8proc_int32_t l = utf8proc_tolower(c); utf8proc_int32_t u = utf8proc_toupper(c); + utf8proc_int32_t t = utf8proc_totitle(c); check(l == c || utf8proc_codepoint_valid(l), "invalid tolower"); check(u == c || utf8proc_codepoint_valid(u), "invalid toupper"); + check(t == c || utf8proc_codepoint_valid(t), "invalid totitle"); + + if (utf8proc_codepoint_valid(c) && (l == u) != (l == t) && + /* Unicode 11: Georgian Mkhedruli chars have uppercase but no titlecase. */ + !(((c >= 0x10d0 && c <= 0x10fa) || c >= (0x10fd && c <= 0x10ff)) && l != u)) { + fprintf(stderr, "unexpected titlecase %x for lowercase %x / uppercase %x\n", t, l, c); + ++error; + } + + if (sizeof(wint_t) > 2 || (c < (1<<16) && u < (1<<16) && l < (1<<16))) { + wint_t l0 = towlower((wint_t)c), u0 = towupper((wint_t)c); - if (sizeof(wint_t) > 2 || c < (1<<16)) { - wint_t l0 = towlower(c), u0 = towupper(c); - /* OS unicode tables may be out of date. But if they do have a lower/uppercase mapping, hopefully it is correct? */ - if (l0 != c && l0 != l) { + if (l0 != (wint_t)c && l0 != (wint_t)l) { fprintf(stderr, "MISMATCH %x != towlower(%x) == %x\n", l, c, l0); ++error; } - else if (l0 != l) { /* often true for out-of-date OS unicode */ + else if (l0 != (wint_t)l) { /* often true for out-of-date OS unicode */ ++better; /* printf("%x != towlower(%x) == %x\n", l, c, l0); */ } - if (u0 != c && u0 != u) { + if (u0 != (wint_t)c && u0 != (wint_t)u) { fprintf(stderr, "MISMATCH %x != towupper(%x) == %x\n", u, c, u0); ++error; } - else if (u0 != u) { /* often true for out-of-date OS unicode */ + else if (u0 != (wint_t)u) { /* often true for out-of-date OS unicode */ ++better; /* printf("%x != towupper(%x) == %x\n", u, c, u0); */ } } } check(!error, "utf8proc case conversion FAILED %d tests.", error); + + /* issue #130 */ + check(utf8proc_toupper(0x00df) == 0x1e9e && + utf8proc_totitle(0x00df) == 0x1e9e && + utf8proc_tolower(0x00df) == 0x00df && + utf8proc_tolower(0x1e9e) == 0x00df && + utf8proc_toupper(0x1e9e) == 0x1e9e, + "incorrect 0x00df/0x1e9e case conversions"); + utf8proc_uint8_t str_00df[] = {0xc3, 0x9f, 0x00}; + utf8proc_uint8_t str_1e9e[] = {0xe1, 0xba, 0x9e, 0x00}; + utf8proc_uint8_t *s1 = utf8proc_NFKC_Casefold(str_00df); + utf8proc_uint8_t *s2 = utf8proc_NFKC_Casefold(str_1e9e); + check(!strcmp((char*)s1, "ss") && + !strcmp((char*)s2, "ss"), + "incorrect 0x00df/0x1e9e casefold normalization"); + free(s1); + free(s2); printf("More up-to-date than OS unicode tables for %d tests.\n", better); printf("utf8proc case conversion tests SUCCEEDED.\n"); return 0; diff --git a/3rdparty/utf8proc/test/charwidth.c b/3rdparty/utf8proc/test/charwidth.c index 330f18eebb4..c5cbbd7cdc1 100644 --- a/3rdparty/utf8proc/test/charwidth.c +++ b/3rdparty/utf8proc/test/charwidth.c @@ -2,70 +2,76 @@ #include <ctype.h> #include <wchar.h> +static int my_unassigned(int c) { + int cat = utf8proc_get_property(c)->category; + return (cat == UTF8PROC_CATEGORY_CN) || (cat == UTF8PROC_CATEGORY_CO); +} + static int my_isprint(int c) { - int cat = utf8proc_get_property(c)->category; - return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) || - (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd); + int cat = utf8proc_get_property(c)->category; + return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) || + (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd || c == 0x00ad) || + (cat == UTF8PROC_CATEGORY_CN) || (cat == UTF8PROC_CATEGORY_CO); } int main(int argc, char **argv) { - int c, error = 0, updates = 0; + int c, error = 0, updates = 0; + + (void) argc; /* unused */ + (void) argv; /* unused */ - (void) argc; /* unused */ - (void) argv; /* unused */ + /* some simple sanity tests of the character widths */ + for (c = 0; c <= 0x110000; ++c) { + int cat = utf8proc_get_property(c)->category; + int w = utf8proc_charwidth(c); + if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) && w > 0) { + fprintf(stderr, "nonzero width %d for combining char %x\n", w, c); + error += 1; + } + if (w == 0 && + ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) || + (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) || + (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) { + fprintf(stderr, "zero width for symbol-like char %x\n", c); + error += 1; + } + if (c <= 127 && ((!isprint(c) && w > 0) || (isprint(c) && wcwidth(c) != w))) { + fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n", + wcwidth(c), w, + isprint(c) ? "printable" : "non-printable", c); + error += 1; + } + if (!my_isprint(c) && w > 0) { + fprintf(stderr, "non-printing %x had width %d\n", c, w); + error += 1; + } + if (my_unassigned(c) && w != 1) { + fprintf(stderr, "unexpected width %d for unassigned char %x\n", w, c); + error += 1; + } + } + check(!error, "utf8proc_charwidth FAILED %d tests.", error); - /* some simple sanity tests of the character widths */ - for (c = 0; c <= 0x110000; ++c) { - int cat = utf8proc_get_property(c)->category; - int w = utf8proc_charwidth(c); - if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) && - w > 0) { - fprintf(stderr, "nonzero width %d for combining char %x\n", w, c); - error = 1; - } - if (w == 0 && - ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) || - (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) || - (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) { - fprintf(stderr, "zero width for symbol-like char %x\n", c); - error = 1; - } - if (c <= 127 && ((!isprint(c) && w > 0) || - (isprint(c) && wcwidth(c) != w))) { - fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n", - wcwidth(c), w, - isprint(c) ? "printable" : "non-printable", c); - error = 1; - } - if (!my_isprint(c) && w > 0) { - fprintf(stderr, "non-printing %x had width %d\n", c, w); - error = 1; - } - } - check(!error, "utf8proc_charwidth FAILED tests."); + check(utf8proc_charwidth(0x00ad) == 1, "incorrect width for U+00AD (soft hyphen)"); + check(utf8proc_charwidth(0xe000) == 1, "incorrect width for U+e000 (PUA)"); - /* print some other information by compariing with system wcwidth */ - printf("Mismatches with system wcwidth (not necessarily errors):\n"); - for (c = 0; c <= 0x110000; ++c) { - int w = utf8proc_charwidth(c); - int wc = wcwidth(c); - if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue; - /* lots of these errors for out-of-date system unicode tables */ - if (wc == -1 && my_isprint(c) && w > 0) { - updates += 1; -#if 0 - printf(" wcwidth(%x) = -1 for printable char\n", c); -#endif - } - if (wc == -1 && !my_isprint(c) && w > 0) - printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w); - if (wc >= 0 && wc != w) - printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w); - } - printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n", - updates); - printf("Character-width tests SUCCEEDED.\n"); + /* print some other information by compariing with system wcwidth */ + printf("Mismatches with system wcwidth (not necessarily errors):\n"); + for (c = 0; c <= 0x110000; ++c) { + int w = utf8proc_charwidth(c); + int wc = wcwidth(c); + if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue; + /* lots of these errors for out-of-date system unicode tables */ + if (wc == -1 && my_isprint(c) && !my_unassigned(c) && w > 0) + updates += 1; + if (wc == -1 && !my_isprint(c) && w > 0) + printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w); + if (wc >= 0 && wc != w) + printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w); + } + printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n", updates); + printf("Character-width tests SUCCEEDED.\n"); - return 0; + return 0; } diff --git a/3rdparty/utf8proc/test/custom.c b/3rdparty/utf8proc/test/custom.c index f85b3cc2bc6..fe4239d91b4 100644 --- a/3rdparty/utf8proc/test/custom.c +++ b/3rdparty/utf8proc/test/custom.c @@ -23,5 +23,6 @@ int main(void) check(strlen((char*) output) == 6, "incorrect output length"); check(!memcmp(correct, output, 7), "incorrect output data"); free(output); + printf("map_custom tests SUCCEEDED.\n"); return 0; } diff --git a/3rdparty/utf8proc/test/fuzz_main.c b/3rdparty/utf8proc/test/fuzz_main.c new file mode 100644 index 00000000000..39bf14738c2 --- /dev/null +++ b/3rdparty/utf8proc/test/fuzz_main.c @@ -0,0 +1,54 @@ +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> + +/* Fuzz target entry point, works without libFuzzer */ + +int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size); + +int main(int argc, char **argv) +{ + FILE *f; + char *buf = NULL; + long siz_buf; + + if(argc < 2) + { + fprintf(stderr, "no input file\n"); + goto err; + } + + f = fopen(argv[1], "rb"); + if(f == NULL) + { + fprintf(stderr, "error opening input file %s\n", argv[1]); + goto err; + } + + fseek(f, 0, SEEK_END); + + siz_buf = ftell(f); + rewind(f); + + if(siz_buf < 1) goto err; + + buf = (char*)malloc(siz_buf); + if(buf == NULL) + { + fprintf(stderr, "malloc() failed\n"); + goto err; + } + + if(fread(buf, siz_buf, 1, f) != 1) + { + fprintf(stderr, "fread() failed\n"); + goto err; + } + + (void)LLVMFuzzerTestOneInput((uint8_t*)buf, siz_buf); + +err: + free(buf); + + return 0; +} diff --git a/3rdparty/utf8proc/test/fuzzer.c b/3rdparty/utf8proc/test/fuzzer.c new file mode 100644 index 00000000000..b5349a322c9 --- /dev/null +++ b/3rdparty/utf8proc/test/fuzzer.c @@ -0,0 +1,84 @@ +#include <utf8proc.h> +#include <string.h> + +int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) +{ + if(size < 1) return 0; + + /* Avoid timeout with long inputs */ + if(size > (64 * 1024)) return 0; + + if(data[size-1] != '\0') return 0; + + const uint8_t* ptr = data; + utf8proc_int32_t c = 0, c_prev = 0, state = 0; + utf8proc_option_t options; + utf8proc_ssize_t ret, bytes = 0; + size_t len = strlen((const char*)data); + + while(bytes != len) + { + ret = utf8proc_iterate(ptr, -1, &c); + + if(ret < 0 || ret == 0) break; + + bytes += ret; + ptr += ret; + + utf8proc_tolower(c); + utf8proc_toupper(c); + utf8proc_totitle(c); + utf8proc_islower(c); + utf8proc_isupper(c); + utf8proc_charwidth(c); + utf8proc_category(c); + utf8proc_category_string(c); + utf8proc_codepoint_valid(c); + + utf8proc_grapheme_break(c_prev, c); + utf8proc_grapheme_break_stateful(c_prev, c, &state); + + c_prev = c; + } + + utf8proc_int32_t *copy = size >= 4 ? NULL : malloc(size); + + if(copy) + { + size /= 4; + + options = UTF8PROC_STRIPCC | UTF8PROC_NLF2LS | UTF8PROC_NLF2PS; + memcpy(copy, data, size); + utf8proc_normalize_utf32(copy, size, options); + + options = UTF8PROC_STRIPCC | UTF8PROC_NLF2LS; + memcpy(copy, data, size); + utf8proc_normalize_utf32(copy, size, options); + + options = UTF8PROC_STRIPCC | UTF8PROC_NLF2PS; + memcpy(copy, data, size); + utf8proc_normalize_utf32(copy, size, options); + + options = UTF8PROC_STRIPCC; + memcpy(copy, data, size); + utf8proc_normalize_utf32(copy, size, options); + + options = UTF8PROC_LUMP; + memcpy(copy, data, size); + utf8proc_normalize_utf32(copy, size, options); + + options = 0; + memcpy(copy, data, size); + utf8proc_normalize_utf32(copy, size, options); + + free(copy); + } + + free(utf8proc_NFD(data)); + free(utf8proc_NFC(data)); + free(utf8proc_NFKD(data)); + free(utf8proc_NFKC(data)); + free(utf8proc_NFKC_Casefold(data)); + + return 0; +}
\ No newline at end of file diff --git a/3rdparty/utf8proc/test/graphemetest.c b/3rdparty/utf8proc/test/graphemetest.c index eb3645b9a0c..6fbe5bdaa97 100644 --- a/3rdparty/utf8proc/test/graphemetest.c +++ b/3rdparty/utf8proc/test/graphemetest.c @@ -1,74 +1,135 @@ #include "tests.h" +/* check one line in the format of GraphemeBreakTest.txt */ +void checkline(const char *_buf, bool verbose) { + size_t bi = 0, si = 0; + utf8proc_uint8_t src[1024]; /* more than long enough for all of our tests */ + const unsigned char *buf = (const unsigned char *) _buf; + + while (buf[bi]) { + bi = skipspaces(buf, bi); + if (buf[bi] == 0xc3 && buf[bi+1] == 0xb7) { /* U+00f7 = grapheme break */ + src[si++] = '/'; + bi += 2; + } + else if (buf[bi] == 0xc3 && buf[bi+1] == 0x97) { /* U+00d7 = no break */ + bi += 2; + } + else if (buf[bi] == '#') { /* start of comments */ + break; + } + else if (buf[bi] == '/') { /* for convenience, also accept / as grapheme break */ + src[si++] = '/'; + bi += 1; + } + else { /* hex-encoded codepoint */ + size_t len = encode((unsigned char*) (src + si), buf + bi) - 1; + while (src[si]) ++si; /* advance to NUL termination */ + bi += len; + } + } + if (si && src[si-1] == '/') + --si; /* no break after final grapheme */ + src[si] = 0; /* NUL-terminate */ + + if (si) { /* test utf8proc_map */ + utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */ + size_t i = 0, j = 0; + utf8proc_ssize_t glen, k; + utf8proc_uint8_t *g; /* utf8proc_map grapheme results */ + while (i < si) { + if (src[i] != '/') + utf8[j++] = src[i++]; + else + i++; + } + glen = utf8proc_map(utf8, (utf8proc_ssize_t)j, &g, UTF8PROC_CHARBOUND); + if (glen == UTF8PROC_ERROR_INVALIDUTF8) { + /* the test file contains surrogate codepoints, which are only for UTF-16 */ + printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno); + } + else { + check(glen >= 0, "utf8proc_map error = %s", + utf8proc_errmsg(glen)); + for (k = 0; k <= glen; ++k) + if (g[k] == 0xff) + g[k] = '/'; /* easier-to-read output (/ is not in test strings) */ + check(!strcmp((char*)g, (char*)src), + "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src); + } + free(g); + } + + if (si) { /* test manual calls to utf8proc_grapheme_break_stateful */ + utf8proc_int32_t state = 0, prev_codepoint = 0; + size_t i = 0; + utf8proc_bool expectbreak = false; + do { + utf8proc_int32_t codepoint; + i += (size_t)utf8proc_iterate(src + i, (utf8proc_ssize_t)(si - i), &codepoint); + check(codepoint >= 0, "invalid UTF-8 data"); + if (codepoint == 0x002F) + expectbreak = true; + else { + if (prev_codepoint != 0) { + check(expectbreak == utf8proc_grapheme_break_stateful(prev_codepoint, codepoint, &state), + "grapheme mismatch: between 0x%04x and 0x%04x in \"%s\"", prev_codepoint, codepoint, (char*) src); + } + expectbreak = false; + prev_codepoint = codepoint; + } + } while (i < si); + } + + if (verbose) + printf("passed grapheme test: \"%s\"\n", (char*) src); +} + int main(int argc, char **argv) { - char *buf = NULL; - size_t bufsize = 0; + unsigned char buf[8192]; FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL; - utf8proc_uint8_t src[1024]; - int len; - + check(f != NULL, "error opening GraphemeBreakTest.txt"); - while (getline(&buf, &bufsize, f) > 0) { - size_t bi = 0, si = 0; - lineno += 1; - - if (lineno % 100 == 0) + while (simple_getline(buf, f) > 0) { + if ((++lineno) % 100 == 0) printf("checking line %zd...\n", lineno); - if (buf[0] == '#') continue; - - while (buf[bi]) { - bi = skipspaces(buf, bi); - if (buf[bi] == '/') { /* grapheme break */ - src[si++] = '/'; - bi++; - } - else if (buf[bi] == '+') { /* no break */ - bi++; - } - else if (buf[bi] == '#') { /* start of comments */ - break; - } - else { /* hex-encoded codepoint */ - len = encode((char*) (src + si), buf + bi) - 1; - while (src[si]) ++si; /* advance to NUL termination */ - bi += len; - } - } - if (si && src[si-1] == '/') - --si; /* no break after final grapheme */ - src[si] = 0; /* NUL-terminate */ - - if (si) { - utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */ - size_t i = 0, j = 0; - utf8proc_ssize_t glen; - utf8proc_uint8_t *g; /* utf8proc_map grapheme results */ - while (i < si) { - if (src[i] != '/') - utf8[j++] = src[i++]; - else - i++; - } - glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND); - if (glen == UTF8PROC_ERROR_INVALIDUTF8) { - /* the test file contains surrogate codepoints, which are only for UTF-16 */ - printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno); - } - else { - check(glen >= 0, "utf8proc_map error = %s", - utf8proc_errmsg(glen)); - for (i = 0; i <= glen; ++i) - if (g[i] == 0xff) - g[i] = '/'; /* easier-to-read output (/ is not in test strings) */ - check(!strcmp((char*)g, (char*)src), - "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src); - } - free(g); - } + checkline((char *) buf, false); } fclose(f); printf("Passed tests after %zd lines!\n", lineno); + + printf("Performing regression tests...\n"); + + /* issue 144 */ + { + utf8proc_uint8_t input[] = {0xef,0xbf,0xbf,0xef,0xbf,0xbe,0x00}; /* "\uffff\ufffe" */ + utf8proc_uint8_t output[] = {0xff,0xef,0xbf,0xbf,0xff,0xef,0xbf,0xbe,0x00}; /* with 0xff grapheme markers */ + utf8proc_ssize_t glen; + utf8proc_uint8_t *g; + glen = utf8proc_map(input, 6, &g, UTF8PROC_CHARBOUND); + check(!strcmp((char*)g, (char*)output), "mishandled u+ffff and u+fffe grapheme breaks"); + check(glen != 6, "mishandled u+ffff and u+fffe grapheme breaks"); + free(g); + }; + + /* https://github.com/JuliaLang/julia/issues/37680 */ + checkline("/ 1f1f8 1f1ea / 1f1f8 1f1ea /", true); /* Two swedish flags after each other */ + checkline("/ 1f926 1f3fc 200d 2642 fe0f /", true); /* facepalm + pale skin + zwj + male sign + FE0F */ + checkline("/ 1f468 1f3fb 200d 1f91d 200d 1f468 1f3fd /", true); /* man face + pale skin + zwj + hand holding + zwj + man face + dark skin */ + + /* more GB9c tests */ + checkline("/ 0915 0300 094d 0300 0924 / 0915 /", true); + checkline("/ 0915 0300 094d 0300 094d 0924 / 0915 /", true); + checkline("/ 0915 0300 0300 / 0924 / 0915 /", true); + checkline("/ 0915 0300 094d 0300 / 0078 /", true); + checkline("/ 0300 094d 0300 / 0924 / 0915 /", true); + + check(utf8proc_grapheme_break(0x03b1, 0x03b2), "failed 03b1 / 03b2 test"); + check(!utf8proc_grapheme_break(0x03b1, 0x0302), "failed 03b1 0302 test"); + + printf("Passed regression tests!\n"); + return 0; } diff --git a/3rdparty/utf8proc/test/iscase.c b/3rdparty/utf8proc/test/iscase.c new file mode 100644 index 00000000000..f3f8cf31b40 --- /dev/null +++ b/3rdparty/utf8proc/test/iscase.c @@ -0,0 +1,62 @@ +#include "tests.h" + +int read_range(FILE *f, utf8proc_int32_t *start, utf8proc_int32_t *end) +{ + unsigned char buf[8192]; + size_t len = simple_getline(buf, f); + size_t pos = skipspaces(buf, 0); + unsigned char s[16]; + if (pos == len || buf[pos] == '#') return 0; + pos += encode(s, buf + pos) - 1; + check(s[0], "invalid line %s in data", buf); + utf8proc_iterate((utf8proc_uint8_t*) s, -1, start); + if (buf[pos] == '.' && buf[pos+1] == '.') { + encode(s, buf + pos + 2); + check(s[0], "invalid line %s in data", buf); + utf8proc_iterate((utf8proc_uint8_t*) s, -1, end); + } + else + *end = *start; + return 1; +} + +int test_iscase(const char *fname, int (*iscase)(utf8proc_int32_t), + utf8proc_int32_t (*thatcase)(utf8proc_int32_t)) +{ + FILE *f = fopen(fname, "r"); + int lines = 0, tests = 0, success = 1; + utf8proc_int32_t c = 0; + + check(f != NULL, "error opening data file \"%s\"\n", fname); + + while (success && !feof(f)) { + utf8proc_int32_t start, end; + if (read_range(f, &start, &end)) { + for (; c < start; ++c) { + check(!iscase(c), "failed !iscase(%04x) in %s\n", c, fname); + } + for (; c <= end; ++c) { + check(iscase(c), "failed iscase(%04x) in %s\n", c, fname); + check(thatcase(c) == c, "inconsistent thatcase(%04x) in %s\n", c, fname); + ++tests; + } + } + ++lines; + } + for (; c <= 0x110000; ++c) { + check(!iscase(c), "failed !iscase(%04x) in %s\n", c, fname); + } + + printf("Checked %d characters from %d lines of %s\n", tests, lines, fname); + fclose(f); + return success; +} + +int main(int argc, char **argv) +{ + check(argc == 3, "Expected Lowercase.txt and Uppercase.txt as arguments"); + check(test_iscase(argv[1], utf8proc_islower, utf8proc_tolower), "Lowercase tests failed"); + check(test_iscase(argv[2], utf8proc_isupper, utf8proc_toupper), "Uppercase tests failed"); + printf("utf8proc iscase tests SUCCEEDED.\n"); + return 0; +} diff --git a/3rdparty/utf8proc/test/iterate.c b/3rdparty/utf8proc/test/iterate.c index c1674b79952..a923afdbf09 100644 --- a/3rdparty/utf8proc/test/iterate.c +++ b/3rdparty/utf8proc/test/iterate.c @@ -8,7 +8,7 @@ static int error; #define CHECKVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,len,__LINE__) #define CHECKINVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,UTF8PROC_ERROR_INVALIDUTF8,__LINE__) -static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int line) +static void testbytes(utf8proc_uint8_t *buf, utf8proc_ssize_t len, utf8proc_ssize_t retval, int line) { utf8proc_int32_t out[16]; utf8proc_ssize_t ret; @@ -16,13 +16,13 @@ static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int /* Make a copy to ensure that memory is left uninitialized after "len" * bytes. This way, Valgrind can detect overreads. */ - unsigned char tmp[16]; - memcpy(tmp, buf, len); + utf8proc_uint8_t tmp[16]; + memcpy(tmp, buf, (unsigned long int)len); tests++; if ((ret = utf8proc_iterate(tmp, len, out)) != retval) { fprintf(stderr, "Failed (%d):", line); - for (int i = 0; i < len ; i++) { + for (utf8proc_ssize_t i = 0; i < len ; i++) { fprintf(stderr, " 0x%02x", tmp[i]); } fprintf(stderr, " -> %zd\n", ret); @@ -32,8 +32,10 @@ static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int int main(int argc, char **argv) { - uint32_t byt; - unsigned char buf[16]; + utf8proc_int32_t byt; + utf8proc_uint8_t buf[16]; + + (void) argc; (void) argv; /* unused */ tests = error = 0; @@ -54,7 +56,7 @@ int main(int argc, char **argv) CHECKVALID(3, 0xbe, 4); CHECKVALID(3, 0xbf, 4); } - + // Continuation byte not after lead for (byt = 0x80; byt < 0xc0; byt++) { CHECKINVALID(0, byt, 1); diff --git a/3rdparty/utf8proc/test/misc.c b/3rdparty/utf8proc/test/misc.c new file mode 100644 index 00000000000..9156f95541d --- /dev/null +++ b/3rdparty/utf8proc/test/misc.c @@ -0,0 +1,51 @@ +/* Miscellaneous tests, e.g. regression tests */ + +#include "tests.h" + +static void issue128(void) /* #128 */ +{ + utf8proc_uint8_t input[] = {0x72, 0xcc, 0x87, 0xcc, 0xa3, 0x00}; /* "r\u0307\u0323" */ + utf8proc_uint8_t nfc[] = {0xe1, 0xb9, 0x9b, 0xcc, 0x87, 0x00}; /* "\u1E5B\u0307" */ + utf8proc_uint8_t nfd[] = {0x72, 0xcc, 0xa3, 0xcc, 0x87, 0x00}; /* "r\u0323\u0307" */ + utf8proc_uint8_t *nfc_out, *nfd_out; + nfc_out = utf8proc_NFC(input); + printf("NFC \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)nfc_out, (char*)nfc); + check(strlen((char*) nfc_out) == 5, "incorrect nfc length"); + check(!memcmp(nfc, nfc_out, 6), "incorrect nfc data"); + nfd_out = utf8proc_NFD(input); + printf("NFD \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)nfd_out, (char*)nfd); + check(strlen((char*) nfd_out) == 5, "incorrect nfd length"); + check(!memcmp(nfd, nfd_out, 6), "incorrect nfd data"); + free(nfd_out); free(nfc_out); +} + +static void issue102(void) /* #128 */ +{ + utf8proc_uint8_t input[] = {0x58, 0xe2, 0x81, 0xa5, 0x45, 0xcc, 0x80, 0xc2, 0xad, 0xe1, 0xb4, 0xac, 0x00}; /* "X\u2065E\u0300\u00ad\u1d2c" */ + utf8proc_uint8_t stripna[] = {0x78, 0xc3, 0xa8, 0x61, 0x00}; /* "x\u00e8a" */ + utf8proc_uint8_t correct[] = {0x78, 0xe2, 0x81, 0xa5, 0xc3, 0xa8, 0x61, 0x00}; /* "x\u2065\u00e8a" */ + utf8proc_uint8_t *output; + utf8proc_map(input, 0, &output, UTF8PROC_NULLTERM | UTF8PROC_STABLE | + UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD | UTF8PROC_IGNORE | UTF8PROC_STRIPNA); + printf("NFKC_Casefold \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)output, (char*)stripna); + check(strlen((char*) output) == 4, "incorrect NFKC_Casefold+stripna length"); + check(!memcmp(stripna, output, 5), "incorrect NFKC_Casefold+stripna data"); + free(output); + output = utf8proc_NFKC_Casefold(input); + printf("NFKC_Casefold \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)output, (char*)correct); + check(strlen((char*) output) == 7, "incorrect NFKC_Casefold length"); + check(!memcmp(correct, output, 8), "incorrect NFKC_Casefold data"); + free(output); +} + +int main(void) +{ + issue128(); + issue102(); +#ifdef UNICODE_VERSION + printf("Unicode version: Makefile has %s, has API %s\n", UNICODE_VERSION, utf8proc_unicode_version()); + check(!strcmp(UNICODE_VERSION, utf8proc_unicode_version()), "utf8proc_unicode_version mismatch"); +#endif + printf("Misc tests SUCCEEDED.\n"); + return 0; +} diff --git a/3rdparty/utf8proc/test/normtest.c b/3rdparty/utf8proc/test/normtest.c index 555c14c84bf..627ee79fd26 100644 --- a/3rdparty/utf8proc/test/normtest.c +++ b/3rdparty/utf8proc/test/normtest.c @@ -1,21 +1,20 @@ #include "tests.h" #define CHECK_NORM(NRM, norm, src) { \ - char *src_norm = (char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \ - check(!strcmp(norm, src_norm), \ + unsigned char *src_norm = (unsigned char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \ + check(!strcmp((char *) norm, (char *) src_norm), \ "normalization failed for %s -> %s", src, norm); \ free(src_norm); \ } int main(int argc, char **argv) { - char *buf = NULL; - size_t bufsize = 0; + unsigned char buf[8192]; FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL; - char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024]; + unsigned char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024]; check(f != NULL, "error opening NormalizationTest.txt"); - while (getline(&buf, &bufsize, f) > 0) { + while (simple_getline(buf, f) > 0) { size_t offset; lineno += 1; diff --git a/3rdparty/utf8proc/test/ossfuzz.sh b/3rdparty/utf8proc/test/ossfuzz.sh new file mode 100755 index 00000000000..3e3b33b049a --- /dev/null +++ b/3rdparty/utf8proc/test/ossfuzz.sh @@ -0,0 +1,13 @@ +#!/bin/bash -eu +# This script is meant to be run by +# https://github.com/google/oss-fuzz/blob/master/projects/utf8proc/Dockerfile + +mkdir build +cd build +cmake .. -DUTF8PROC_ENABLE_TESTING=ON -DLIB_FUZZING_ENGINE="$LIB_FUZZING_ENGINE" +make -j$(nproc) + +cp $SRC/utf8proc/build/fuzzer $OUT/utf8proc_fuzzer + +find $SRC/utf8proc/test -name "*.txt" | \ + xargs zip $OUT/utf8proc_fuzzer_seed_corpus.zip diff --git a/3rdparty/utf8proc/test/printproperty.c b/3rdparty/utf8proc/test/printproperty.c index 2819aa1e881..89949355051 100644 --- a/3rdparty/utf8proc/test/printproperty.c +++ b/3rdparty/utf8proc/test/printproperty.c @@ -4,46 +4,61 @@ int main(int argc, char **argv) { - int i; + int i; - for (i = 1; i < argc; ++i) { - unsigned int c; - if (!strcmp(argv[i], "-V")) { - printf("utf8proc version %s\n", utf8proc_version()); - continue; - } - check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]); - const utf8proc_property_t *p = utf8proc_get_property(c); - printf("U+%s:\n" - " category = %s\n" - " combining_class = %d\n" - " bidi_class = %d\n" - " decomp_type = %d\n" - " uppercase_mapping = %x\n" - " lowercase_mapping = %x\n" - " titlecase_mapping = %x\n" - " comb_index = %d\n" - " bidi_mirrored = %d\n" - " comp_exclusion = %d\n" - " ignorable = %d\n" - " control_boundary = %d\n" - " boundclass = %d\n" - " charwidth = %d\n", - argv[i], - utf8proc_category_string(c), - p->combining_class, - p->bidi_class, - p->decomp_type, - utf8proc_toupper(c), - utf8proc_tolower(c), - utf8proc_totitle(c), - p->comb_index, - p->bidi_mirrored, - p->comp_exclusion, - p->ignorable, - p->control_boundary, - p->boundclass, - utf8proc_charwidth(c)); - } - return 0; + for (i = 1; i < argc; ++i) { + utf8proc_uint8_t cstr[16], *map; + utf8proc_uint32_t x; + utf8proc_int32_t c; + if (!strcmp(argv[i], "-V")) { + printf("utf8proc version %s\n", utf8proc_version()); + continue; + } + check(sscanf(argv[i],"%x", &x) == 1, "invalid hex input %s", argv[i]); + c = (utf8proc_int32_t)x; + const utf8proc_property_t *p = utf8proc_get_property(c); + + if (utf8proc_codepoint_valid(c)) + cstr[utf8proc_encode_char(c, cstr)] = 0; + else + strcat((char*)cstr, "N/A"); + utf8proc_map(cstr, 0, &map, UTF8PROC_NULLTERM | UTF8PROC_CASEFOLD); + + printf("U+%s: %s\n" + " category = %s\n" + " combining_class = %d\n" + " bidi_class = %d\n" + " decomp_type = %d\n" + " uppercase_mapping = %04x (seqindex %04x)%s\n" + " lowercase_mapping = %04x (seqindex %04x)%s\n" + " titlecase_mapping = %04x (seqindex %04x)\n" + " casefold = %s\n" + " comb_index = %d\n" + " bidi_mirrored = %d\n" + " comp_exclusion = %d\n" + " ignorable = %d\n" + " control_boundary = %d\n" + " boundclass = %d\n" + " indic_conjunct_break = %d\n" + " charwidth = %d\n", + argv[i], (char*) cstr, + utf8proc_category_string(c), + p->combining_class, + p->bidi_class, + p->decomp_type, + utf8proc_toupper(c), p->uppercase_seqindex, utf8proc_isupper(c) ? " (isupper)" : "", + utf8proc_tolower(c), p->lowercase_seqindex, utf8proc_islower(c) ? " (islower)" : "", + utf8proc_totitle(c), p->titlecase_seqindex, + (char *) map, + p->comb_index, + p->bidi_mirrored, + p->comp_exclusion, + p->ignorable, + p->control_boundary, + p->boundclass, + p->indic_conjunct_break, + utf8proc_charwidth(c)); + free(map); + } + return 0; } diff --git a/3rdparty/utf8proc/test/tests.c b/3rdparty/utf8proc/test/tests.c index 0fb0da36305..68b21e7f02a 100644 --- a/3rdparty/utf8proc/test/tests.c +++ b/3rdparty/utf8proc/test/tests.c @@ -17,7 +17,7 @@ void check(int cond, const char *format, ...) } } -size_t skipspaces(const char *buf, size_t i) +size_t skipspaces(const unsigned char *buf, size_t i) { while (isspace(buf[i])) ++i; return i; @@ -27,9 +27,10 @@ size_t skipspaces(const char *buf, size_t i) separated by whitespace, and terminated by any character not in [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string in dest, returning the number of bytes read from buf */ -size_t encode(char *dest, const char *buf) +size_t encode(unsigned char *dest, const unsigned char *buf) { - size_t i = 0, j, d = 0; + size_t i = 0, j; + utf8proc_ssize_t d = 0; for (;;) { int c; i = skipspaces(buf, i); @@ -39,8 +40,20 @@ size_t encode(char *dest, const char *buf) dest[d] = 0; /* NUL-terminate destination string */ return i + 1; } - check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i); + check(sscanf((char *) (buf + i), "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i); i = j; /* skip to char after hex input */ d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d)); } } + +/* simplistic, portable replacement for getline, sufficient for our tests */ +size_t simple_getline(unsigned char buf[8192], FILE *f) { + size_t i = 0; + while (i < 8191) { + int c = getc(f); + if (c == EOF || c == '\n') break; + buf[i++] = (unsigned char) c; + } + buf[i] = 0; + return i; +} diff --git a/3rdparty/utf8proc/test/tests.h b/3rdparty/utf8proc/test/tests.h index 1811a734a5f..a30510100ed 100644 --- a/3rdparty/utf8proc/test/tests.h +++ b/3rdparty/utf8proc/test/tests.h @@ -1,13 +1,16 @@ /* Common functions and includes for our test programs. */ /* - * Set feature macro to enable getline() and wcwidth(). + * Set feature macro to enable wcwidth(). * * Please refer to section 2.2.1 of POSIX.1-2008: * http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_02 */ #define _XOPEN_SOURCE 700 +/* silence warnings about sscanf on Windows */ +#define _CRT_SECURE_NO_WARNINGS + #include <stdio.h> #include <stdlib.h> #include <ctype.h> @@ -19,5 +22,6 @@ extern size_t lineno; void check(int cond, const char *format, ...); -size_t skipspaces(const char *buf, size_t i); -size_t encode(char *dest, const char *buf); +size_t skipspaces(const unsigned char *buf, size_t i); +size_t encode(unsigned char *dest, const unsigned char *buf); +size_t simple_getline(unsigned char buf[8192], FILE *f); |