diff options
author | 2021-10-09 12:16:17 +1100 | |
---|---|---|
committer | 2021-10-09 12:16:17 +1100 | |
commit | 38082ccbee749d650ccea886ae376a5d1dec337c (patch) | |
tree | 9ba9a900ba826bda58832834278025ced17f42f5 /3rdparty/utf8proc/test | |
parent | 34b3bf701098082feb9077db49987507962c1578 (diff) |
Overdue internal UI enhancements (#8674)
* frontend: Added support for message context to localisations.
* frontend: Added string_view versions of the message lookup functions.
* frontend: Added a few more folder options to the internal UI.
* emu/softlist.cpp: Use more appropriate containers.
* Switched to Python 3 by default - this will become a requirement.
* Updated msgfmt.py for message context support.
* frontend: Show all software item info in the internal UI.
* frontend: Search alternate titles in software selection menu.
* 3rdparty/utf8proc: Updated to v2.6.1 (has several fixes).
* frontend: Added software filters for common info fields.
* frontend: Allow UI manager to hold onto persistent session data.
* frontend: Cache software lists for eight machines.
* frontend: Added support for loading localised system names.
* frontend: Add UI for selecting localised system names.
Diffstat (limited to '3rdparty/utf8proc/test')
-rw-r--r-- | 3rdparty/utf8proc/test/case.c | 28 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/charwidth.c | 120 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/custom.c | 1 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/graphemetest.c | 175 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/iscase.c | 62 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/iterate.c | 4 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/misc.c | 51 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/normtest.c | 11 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/printproperty.c | 93 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/tests.c | 18 | ||||
-rw-r--r-- | 3rdparty/utf8proc/test/tests.h | 10 |
11 files changed, 400 insertions, 173 deletions
diff --git a/3rdparty/utf8proc/test/case.c b/3rdparty/utf8proc/test/case.c index 39958e3e933..4a98a6390b5 100644 --- a/3rdparty/utf8proc/test/case.c +++ b/3rdparty/utf8proc/test/case.c @@ -13,13 +13,22 @@ int main(int argc, char **argv) for (c = 0; c <= 0x110000; ++c) { utf8proc_int32_t l = utf8proc_tolower(c); utf8proc_int32_t u = utf8proc_toupper(c); + utf8proc_int32_t t = utf8proc_totitle(c); check(l == c || utf8proc_codepoint_valid(l), "invalid tolower"); check(u == c || utf8proc_codepoint_valid(u), "invalid toupper"); + check(t == c || utf8proc_codepoint_valid(t), "invalid totitle"); + + if (utf8proc_codepoint_valid(c) && (l == u) != (l == t) && + /* Unicode 11: Georgian Mkhedruli chars have uppercase but no titlecase. */ + !(((c >= 0x10d0 && c <= 0x10fa) || c >= (0x10fd && c <= 0x10ff)) && l != u)) { + fprintf(stderr, "unexpected titlecase %x for lowercase %x / uppercase %x\n", t, l, c); + ++error; + } if (sizeof(wint_t) > 2 || c < (1<<16)) { wint_t l0 = towlower(c), u0 = towupper(c); - + /* OS unicode tables may be out of date. But if they do have a lower/uppercase mapping, hopefully it is correct? */ @@ -44,6 +53,23 @@ int main(int argc, char **argv) } } check(!error, "utf8proc case conversion FAILED %d tests.", error); + + /* issue #130 */ + check(utf8proc_toupper(0x00df) == 0x1e9e && + utf8proc_totitle(0x00df) == 0x1e9e && + utf8proc_tolower(0x00df) == 0x00df && + utf8proc_tolower(0x1e9e) == 0x00df && + utf8proc_toupper(0x1e9e) == 0x1e9e, + "incorrect 0x00df/0x1e9e case conversions"); + utf8proc_uint8_t str_00df[] = {0xc3, 0x9f, 0x00}; + utf8proc_uint8_t str_1e9e[] = {0xe1, 0xba, 0x9e, 0x00}; + utf8proc_uint8_t *s1 = utf8proc_NFKC_Casefold(str_00df); + utf8proc_uint8_t *s2 = utf8proc_NFKC_Casefold(str_1e9e); + check(!strcmp((char*)s1, "ss") && + !strcmp((char*)s2, "ss"), + "incorrect 0x00df/0x1e9e casefold normalization"); + free(s1); + free(s2); printf("More up-to-date than OS unicode tables for %d tests.\n", better); printf("utf8proc case conversion tests SUCCEEDED.\n"); return 0; diff --git a/3rdparty/utf8proc/test/charwidth.c b/3rdparty/utf8proc/test/charwidth.c index 330f18eebb4..c5cbbd7cdc1 100644 --- a/3rdparty/utf8proc/test/charwidth.c +++ b/3rdparty/utf8proc/test/charwidth.c @@ -2,70 +2,76 @@ #include <ctype.h> #include <wchar.h> +static int my_unassigned(int c) { + int cat = utf8proc_get_property(c)->category; + return (cat == UTF8PROC_CATEGORY_CN) || (cat == UTF8PROC_CATEGORY_CO); +} + static int my_isprint(int c) { - int cat = utf8proc_get_property(c)->category; - return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) || - (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd); + int cat = utf8proc_get_property(c)->category; + return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) || + (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd || c == 0x00ad) || + (cat == UTF8PROC_CATEGORY_CN) || (cat == UTF8PROC_CATEGORY_CO); } int main(int argc, char **argv) { - int c, error = 0, updates = 0; + int c, error = 0, updates = 0; + + (void) argc; /* unused */ + (void) argv; /* unused */ - (void) argc; /* unused */ - (void) argv; /* unused */ + /* some simple sanity tests of the character widths */ + for (c = 0; c <= 0x110000; ++c) { + int cat = utf8proc_get_property(c)->category; + int w = utf8proc_charwidth(c); + if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) && w > 0) { + fprintf(stderr, "nonzero width %d for combining char %x\n", w, c); + error += 1; + } + if (w == 0 && + ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) || + (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) || + (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) { + fprintf(stderr, "zero width for symbol-like char %x\n", c); + error += 1; + } + if (c <= 127 && ((!isprint(c) && w > 0) || (isprint(c) && wcwidth(c) != w))) { + fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n", + wcwidth(c), w, + isprint(c) ? "printable" : "non-printable", c); + error += 1; + } + if (!my_isprint(c) && w > 0) { + fprintf(stderr, "non-printing %x had width %d\n", c, w); + error += 1; + } + if (my_unassigned(c) && w != 1) { + fprintf(stderr, "unexpected width %d for unassigned char %x\n", w, c); + error += 1; + } + } + check(!error, "utf8proc_charwidth FAILED %d tests.", error); - /* some simple sanity tests of the character widths */ - for (c = 0; c <= 0x110000; ++c) { - int cat = utf8proc_get_property(c)->category; - int w = utf8proc_charwidth(c); - if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) && - w > 0) { - fprintf(stderr, "nonzero width %d for combining char %x\n", w, c); - error = 1; - } - if (w == 0 && - ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) || - (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) || - (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) { - fprintf(stderr, "zero width for symbol-like char %x\n", c); - error = 1; - } - if (c <= 127 && ((!isprint(c) && w > 0) || - (isprint(c) && wcwidth(c) != w))) { - fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n", - wcwidth(c), w, - isprint(c) ? "printable" : "non-printable", c); - error = 1; - } - if (!my_isprint(c) && w > 0) { - fprintf(stderr, "non-printing %x had width %d\n", c, w); - error = 1; - } - } - check(!error, "utf8proc_charwidth FAILED tests."); + check(utf8proc_charwidth(0x00ad) == 1, "incorrect width for U+00AD (soft hyphen)"); + check(utf8proc_charwidth(0xe000) == 1, "incorrect width for U+e000 (PUA)"); - /* print some other information by compariing with system wcwidth */ - printf("Mismatches with system wcwidth (not necessarily errors):\n"); - for (c = 0; c <= 0x110000; ++c) { - int w = utf8proc_charwidth(c); - int wc = wcwidth(c); - if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue; - /* lots of these errors for out-of-date system unicode tables */ - if (wc == -1 && my_isprint(c) && w > 0) { - updates += 1; -#if 0 - printf(" wcwidth(%x) = -1 for printable char\n", c); -#endif - } - if (wc == -1 && !my_isprint(c) && w > 0) - printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w); - if (wc >= 0 && wc != w) - printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w); - } - printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n", - updates); - printf("Character-width tests SUCCEEDED.\n"); + /* print some other information by compariing with system wcwidth */ + printf("Mismatches with system wcwidth (not necessarily errors):\n"); + for (c = 0; c <= 0x110000; ++c) { + int w = utf8proc_charwidth(c); + int wc = wcwidth(c); + if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue; + /* lots of these errors for out-of-date system unicode tables */ + if (wc == -1 && my_isprint(c) && !my_unassigned(c) && w > 0) + updates += 1; + if (wc == -1 && !my_isprint(c) && w > 0) + printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w); + if (wc >= 0 && wc != w) + printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w); + } + printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n", updates); + printf("Character-width tests SUCCEEDED.\n"); - return 0; + return 0; } diff --git a/3rdparty/utf8proc/test/custom.c b/3rdparty/utf8proc/test/custom.c index f85b3cc2bc6..fe4239d91b4 100644 --- a/3rdparty/utf8proc/test/custom.c +++ b/3rdparty/utf8proc/test/custom.c @@ -23,5 +23,6 @@ int main(void) check(strlen((char*) output) == 6, "incorrect output length"); check(!memcmp(correct, output, 7), "incorrect output data"); free(output); + printf("map_custom tests SUCCEEDED.\n"); return 0; } diff --git a/3rdparty/utf8proc/test/graphemetest.c b/3rdparty/utf8proc/test/graphemetest.c index eb3645b9a0c..22880fc9f14 100644 --- a/3rdparty/utf8proc/test/graphemetest.c +++ b/3rdparty/utf8proc/test/graphemetest.c @@ -1,74 +1,127 @@ #include "tests.h" +/* check one line in the format of GraphemeBreakTest.txt */ +void checkline(const char *_buf, bool verbose) { + size_t bi = 0, si = 0; + utf8proc_uint8_t src[1024]; /* more than long enough for all of our tests */ + const unsigned char *buf = (const unsigned char *) _buf; + + while (buf[bi]) { + bi = skipspaces(buf, bi); + if (buf[bi] == 0xc3 && buf[bi+1] == 0xb7) { /* U+00f7 = grapheme break */ + src[si++] = '/'; + bi += 2; + } + else if (buf[bi] == 0xc3 && buf[bi+1] == 0x97) { /* U+00d7 = no break */ + bi += 2; + } + else if (buf[bi] == '#') { /* start of comments */ + break; + } + else if (buf[bi] == '/') { /* for convenience, also accept / as grapheme break */ + src[si++] = '/'; + bi += 1; + } + else { /* hex-encoded codepoint */ + size_t len = encode((unsigned char*) (src + si), buf + bi) - 1; + while (src[si]) ++si; /* advance to NUL termination */ + bi += len; + } + } + if (si && src[si-1] == '/') + --si; /* no break after final grapheme */ + src[si] = 0; /* NUL-terminate */ + + if (si) { /* test utf8proc_map */ + utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */ + size_t i = 0, j = 0; + utf8proc_ssize_t glen, k; + utf8proc_uint8_t *g; /* utf8proc_map grapheme results */ + while (i < si) { + if (src[i] != '/') + utf8[j++] = src[i++]; + else + i++; + } + glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND); + if (glen == UTF8PROC_ERROR_INVALIDUTF8) { + /* the test file contains surrogate codepoints, which are only for UTF-16 */ + printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno); + } + else { + check(glen >= 0, "utf8proc_map error = %s", + utf8proc_errmsg(glen)); + for (k = 0; k <= glen; ++k) + if (g[k] == 0xff) + g[k] = '/'; /* easier-to-read output (/ is not in test strings) */ + check(!strcmp((char*)g, (char*)src), + "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src); + } + free(g); + } + + if (si) { /* test manual calls to utf8proc_grapheme_break_stateful */ + utf8proc_int32_t state = 0, prev_codepoint = 0; + size_t i = 0; + utf8proc_bool expectbreak = false; + do { + utf8proc_int32_t codepoint; + i += utf8proc_iterate(src + i, si - i, &codepoint); + check(codepoint >= 0, "invalid UTF-8 data"); + if (codepoint == 0x002F) + expectbreak = true; + else { + if (prev_codepoint != 0) { + check(expectbreak == utf8proc_grapheme_break_stateful(prev_codepoint, codepoint, &state), + "grapheme mismatch: between 0x%04x and 0x%04x in \"%s\"", prev_codepoint, codepoint, (char*) src); + } + expectbreak = false; + prev_codepoint = codepoint; + } + } while (i < si); + } + + if (verbose) + printf("passed grapheme test: \"%s\"\n", (char*) src); +} + int main(int argc, char **argv) { - char *buf = NULL; - size_t bufsize = 0; + unsigned char buf[8192]; FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL; - utf8proc_uint8_t src[1024]; - int len; - + check(f != NULL, "error opening GraphemeBreakTest.txt"); - while (getline(&buf, &bufsize, f) > 0) { - size_t bi = 0, si = 0; - lineno += 1; - - if (lineno % 100 == 0) + while (simple_getline(buf, f) > 0) { + if ((++lineno) % 100 == 0) printf("checking line %zd...\n", lineno); - if (buf[0] == '#') continue; - - while (buf[bi]) { - bi = skipspaces(buf, bi); - if (buf[bi] == '/') { /* grapheme break */ - src[si++] = '/'; - bi++; - } - else if (buf[bi] == '+') { /* no break */ - bi++; - } - else if (buf[bi] == '#') { /* start of comments */ - break; - } - else { /* hex-encoded codepoint */ - len = encode((char*) (src + si), buf + bi) - 1; - while (src[si]) ++si; /* advance to NUL termination */ - bi += len; - } - } - if (si && src[si-1] == '/') - --si; /* no break after final grapheme */ - src[si] = 0; /* NUL-terminate */ - - if (si) { - utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */ - size_t i = 0, j = 0; - utf8proc_ssize_t glen; - utf8proc_uint8_t *g; /* utf8proc_map grapheme results */ - while (i < si) { - if (src[i] != '/') - utf8[j++] = src[i++]; - else - i++; - } - glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND); - if (glen == UTF8PROC_ERROR_INVALIDUTF8) { - /* the test file contains surrogate codepoints, which are only for UTF-16 */ - printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno); - } - else { - check(glen >= 0, "utf8proc_map error = %s", - utf8proc_errmsg(glen)); - for (i = 0; i <= glen; ++i) - if (g[i] == 0xff) - g[i] = '/'; /* easier-to-read output (/ is not in test strings) */ - check(!strcmp((char*)g, (char*)src), - "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src); - } - free(g); - } + checkline((char *) buf, false); } fclose(f); printf("Passed tests after %zd lines!\n", lineno); + + printf("Performing regression tests...\n"); + + /* issue 144 */ + { + utf8proc_uint8_t input[] = {0xef,0xbf,0xbf,0xef,0xbf,0xbe,0x00}; /* "\uffff\ufffe" */ + utf8proc_uint8_t output[] = {0xff,0xef,0xbf,0xbf,0xff,0xef,0xbf,0xbe,0x00}; /* with 0xff grapheme markers */ + utf8proc_ssize_t glen; + utf8proc_uint8_t *g; + glen = utf8proc_map(input, 6, &g, UTF8PROC_CHARBOUND); + check(!strcmp((char*)g, (char*)output), "mishandled u+ffff and u+fffe grapheme breaks"); + free(g); + }; + + /* https://github.com/JuliaLang/julia/issues/37680 */ + checkline("/ 1f1f8 1f1ea / 1f1f8 1f1ea /", true); /* Two swedish flags after each other */ + checkline("/ 1f926 1f3fc 200d 2642 fe0f /", true); /* facepalm + pale skin + zwj + male sign + FE0F */ + checkline("/ 1f468 1f3fb 200d 1f91d 200d 1f468 1f3fd /", true); /* man face + pale skin + zwj + hand holding + zwj + man face + dark skin */ + + check(utf8proc_grapheme_break(0x03b1, 0x03b2), "failed 03b1 / 03b2 test"); + check(!utf8proc_grapheme_break(0x03b1, 0x0302), "failed 03b1 0302 test"); + + printf("Passed regression tests!\n"); + return 0; } diff --git a/3rdparty/utf8proc/test/iscase.c b/3rdparty/utf8proc/test/iscase.c new file mode 100644 index 00000000000..f3f8cf31b40 --- /dev/null +++ b/3rdparty/utf8proc/test/iscase.c @@ -0,0 +1,62 @@ +#include "tests.h" + +int read_range(FILE *f, utf8proc_int32_t *start, utf8proc_int32_t *end) +{ + unsigned char buf[8192]; + size_t len = simple_getline(buf, f); + size_t pos = skipspaces(buf, 0); + unsigned char s[16]; + if (pos == len || buf[pos] == '#') return 0; + pos += encode(s, buf + pos) - 1; + check(s[0], "invalid line %s in data", buf); + utf8proc_iterate((utf8proc_uint8_t*) s, -1, start); + if (buf[pos] == '.' && buf[pos+1] == '.') { + encode(s, buf + pos + 2); + check(s[0], "invalid line %s in data", buf); + utf8proc_iterate((utf8proc_uint8_t*) s, -1, end); + } + else + *end = *start; + return 1; +} + +int test_iscase(const char *fname, int (*iscase)(utf8proc_int32_t), + utf8proc_int32_t (*thatcase)(utf8proc_int32_t)) +{ + FILE *f = fopen(fname, "r"); + int lines = 0, tests = 0, success = 1; + utf8proc_int32_t c = 0; + + check(f != NULL, "error opening data file \"%s\"\n", fname); + + while (success && !feof(f)) { + utf8proc_int32_t start, end; + if (read_range(f, &start, &end)) { + for (; c < start; ++c) { + check(!iscase(c), "failed !iscase(%04x) in %s\n", c, fname); + } + for (; c <= end; ++c) { + check(iscase(c), "failed iscase(%04x) in %s\n", c, fname); + check(thatcase(c) == c, "inconsistent thatcase(%04x) in %s\n", c, fname); + ++tests; + } + } + ++lines; + } + for (; c <= 0x110000; ++c) { + check(!iscase(c), "failed !iscase(%04x) in %s\n", c, fname); + } + + printf("Checked %d characters from %d lines of %s\n", tests, lines, fname); + fclose(f); + return success; +} + +int main(int argc, char **argv) +{ + check(argc == 3, "Expected Lowercase.txt and Uppercase.txt as arguments"); + check(test_iscase(argv[1], utf8proc_islower, utf8proc_tolower), "Lowercase tests failed"); + check(test_iscase(argv[2], utf8proc_isupper, utf8proc_toupper), "Uppercase tests failed"); + printf("utf8proc iscase tests SUCCEEDED.\n"); + return 0; +} diff --git a/3rdparty/utf8proc/test/iterate.c b/3rdparty/utf8proc/test/iterate.c index c1674b79952..b8e9feb01b5 100644 --- a/3rdparty/utf8proc/test/iterate.c +++ b/3rdparty/utf8proc/test/iterate.c @@ -35,6 +35,8 @@ int main(int argc, char **argv) uint32_t byt; unsigned char buf[16]; + (void) argc; (void) argv; /* unused */ + tests = error = 0; // Check valid sequences that were considered valid erroneously before @@ -54,7 +56,7 @@ int main(int argc, char **argv) CHECKVALID(3, 0xbe, 4); CHECKVALID(3, 0xbf, 4); } - + // Continuation byte not after lead for (byt = 0x80; byt < 0xc0; byt++) { CHECKINVALID(0, byt, 1); diff --git a/3rdparty/utf8proc/test/misc.c b/3rdparty/utf8proc/test/misc.c new file mode 100644 index 00000000000..9156f95541d --- /dev/null +++ b/3rdparty/utf8proc/test/misc.c @@ -0,0 +1,51 @@ +/* Miscellaneous tests, e.g. regression tests */ + +#include "tests.h" + +static void issue128(void) /* #128 */ +{ + utf8proc_uint8_t input[] = {0x72, 0xcc, 0x87, 0xcc, 0xa3, 0x00}; /* "r\u0307\u0323" */ + utf8proc_uint8_t nfc[] = {0xe1, 0xb9, 0x9b, 0xcc, 0x87, 0x00}; /* "\u1E5B\u0307" */ + utf8proc_uint8_t nfd[] = {0x72, 0xcc, 0xa3, 0xcc, 0x87, 0x00}; /* "r\u0323\u0307" */ + utf8proc_uint8_t *nfc_out, *nfd_out; + nfc_out = utf8proc_NFC(input); + printf("NFC \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)nfc_out, (char*)nfc); + check(strlen((char*) nfc_out) == 5, "incorrect nfc length"); + check(!memcmp(nfc, nfc_out, 6), "incorrect nfc data"); + nfd_out = utf8proc_NFD(input); + printf("NFD \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)nfd_out, (char*)nfd); + check(strlen((char*) nfd_out) == 5, "incorrect nfd length"); + check(!memcmp(nfd, nfd_out, 6), "incorrect nfd data"); + free(nfd_out); free(nfc_out); +} + +static void issue102(void) /* #128 */ +{ + utf8proc_uint8_t input[] = {0x58, 0xe2, 0x81, 0xa5, 0x45, 0xcc, 0x80, 0xc2, 0xad, 0xe1, 0xb4, 0xac, 0x00}; /* "X\u2065E\u0300\u00ad\u1d2c" */ + utf8proc_uint8_t stripna[] = {0x78, 0xc3, 0xa8, 0x61, 0x00}; /* "x\u00e8a" */ + utf8proc_uint8_t correct[] = {0x78, 0xe2, 0x81, 0xa5, 0xc3, 0xa8, 0x61, 0x00}; /* "x\u2065\u00e8a" */ + utf8proc_uint8_t *output; + utf8proc_map(input, 0, &output, UTF8PROC_NULLTERM | UTF8PROC_STABLE | + UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD | UTF8PROC_IGNORE | UTF8PROC_STRIPNA); + printf("NFKC_Casefold \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)output, (char*)stripna); + check(strlen((char*) output) == 4, "incorrect NFKC_Casefold+stripna length"); + check(!memcmp(stripna, output, 5), "incorrect NFKC_Casefold+stripna data"); + free(output); + output = utf8proc_NFKC_Casefold(input); + printf("NFKC_Casefold \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)output, (char*)correct); + check(strlen((char*) output) == 7, "incorrect NFKC_Casefold length"); + check(!memcmp(correct, output, 8), "incorrect NFKC_Casefold data"); + free(output); +} + +int main(void) +{ + issue128(); + issue102(); +#ifdef UNICODE_VERSION + printf("Unicode version: Makefile has %s, has API %s\n", UNICODE_VERSION, utf8proc_unicode_version()); + check(!strcmp(UNICODE_VERSION, utf8proc_unicode_version()), "utf8proc_unicode_version mismatch"); +#endif + printf("Misc tests SUCCEEDED.\n"); + return 0; +} diff --git a/3rdparty/utf8proc/test/normtest.c b/3rdparty/utf8proc/test/normtest.c index 555c14c84bf..627ee79fd26 100644 --- a/3rdparty/utf8proc/test/normtest.c +++ b/3rdparty/utf8proc/test/normtest.c @@ -1,21 +1,20 @@ #include "tests.h" #define CHECK_NORM(NRM, norm, src) { \ - char *src_norm = (char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \ - check(!strcmp(norm, src_norm), \ + unsigned char *src_norm = (unsigned char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \ + check(!strcmp((char *) norm, (char *) src_norm), \ "normalization failed for %s -> %s", src, norm); \ free(src_norm); \ } int main(int argc, char **argv) { - char *buf = NULL; - size_t bufsize = 0; + unsigned char buf[8192]; FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL; - char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024]; + unsigned char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024]; check(f != NULL, "error opening NormalizationTest.txt"); - while (getline(&buf, &bufsize, f) > 0) { + while (simple_getline(buf, f) > 0) { size_t offset; lineno += 1; diff --git a/3rdparty/utf8proc/test/printproperty.c b/3rdparty/utf8proc/test/printproperty.c index 2819aa1e881..709e6a72b15 100644 --- a/3rdparty/utf8proc/test/printproperty.c +++ b/3rdparty/utf8proc/test/printproperty.c @@ -4,46 +4,57 @@ int main(int argc, char **argv) { - int i; + int i; - for (i = 1; i < argc; ++i) { - unsigned int c; - if (!strcmp(argv[i], "-V")) { - printf("utf8proc version %s\n", utf8proc_version()); - continue; - } - check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]); - const utf8proc_property_t *p = utf8proc_get_property(c); - printf("U+%s:\n" - " category = %s\n" - " combining_class = %d\n" - " bidi_class = %d\n" - " decomp_type = %d\n" - " uppercase_mapping = %x\n" - " lowercase_mapping = %x\n" - " titlecase_mapping = %x\n" - " comb_index = %d\n" - " bidi_mirrored = %d\n" - " comp_exclusion = %d\n" - " ignorable = %d\n" - " control_boundary = %d\n" - " boundclass = %d\n" - " charwidth = %d\n", - argv[i], - utf8proc_category_string(c), - p->combining_class, - p->bidi_class, - p->decomp_type, - utf8proc_toupper(c), - utf8proc_tolower(c), - utf8proc_totitle(c), - p->comb_index, - p->bidi_mirrored, - p->comp_exclusion, - p->ignorable, - p->control_boundary, - p->boundclass, - utf8proc_charwidth(c)); - } - return 0; + for (i = 1; i < argc; ++i) { + utf8proc_uint8_t cstr[16], *map; + unsigned int c; + if (!strcmp(argv[i], "-V")) { + printf("utf8proc version %s\n", utf8proc_version()); + continue; + } + check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]); + const utf8proc_property_t *p = utf8proc_get_property(c); + + if (utf8proc_codepoint_valid(c)) + cstr[utf8proc_encode_char(c, cstr)] = 0; + else + strcat((char*)cstr, "N/A"); + utf8proc_map(cstr, 0, &map, UTF8PROC_NULLTERM | UTF8PROC_CASEFOLD); + + printf("U+%s: %s\n" + " category = %s\n" + " combining_class = %d\n" + " bidi_class = %d\n" + " decomp_type = %d\n" + " uppercase_mapping = %04x (seqindex %04x)%s\n" + " lowercase_mapping = %04x (seqindex %04x)%s\n" + " titlecase_mapping = %04x (seqindex %04x)\n" + " casefold = %s\n" + " comb_index = %d\n" + " bidi_mirrored = %d\n" + " comp_exclusion = %d\n" + " ignorable = %d\n" + " control_boundary = %d\n" + " boundclass = %d\n" + " charwidth = %d\n", + argv[i], (char*) cstr, + utf8proc_category_string(c), + p->combining_class, + p->bidi_class, + p->decomp_type, + utf8proc_toupper(c), p->uppercase_seqindex, utf8proc_isupper(c) ? " (isupper)" : "", + utf8proc_tolower(c), p->lowercase_seqindex, utf8proc_islower(c) ? " (islower)" : "", + utf8proc_totitle(c), p->titlecase_seqindex, + (char *) map, + p->comb_index, + p->bidi_mirrored, + p->comp_exclusion, + p->ignorable, + p->control_boundary, + p->boundclass, + utf8proc_charwidth(c)); + free(map); + } + return 0; } diff --git a/3rdparty/utf8proc/test/tests.c b/3rdparty/utf8proc/test/tests.c index 0fb0da36305..629edfff373 100644 --- a/3rdparty/utf8proc/test/tests.c +++ b/3rdparty/utf8proc/test/tests.c @@ -17,7 +17,7 @@ void check(int cond, const char *format, ...) } } -size_t skipspaces(const char *buf, size_t i) +size_t skipspaces(const unsigned char *buf, size_t i) { while (isspace(buf[i])) ++i; return i; @@ -27,7 +27,7 @@ size_t skipspaces(const char *buf, size_t i) separated by whitespace, and terminated by any character not in [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string in dest, returning the number of bytes read from buf */ -size_t encode(char *dest, const char *buf) +size_t encode(unsigned char *dest, const unsigned char *buf) { size_t i = 0, j, d = 0; for (;;) { @@ -39,8 +39,20 @@ size_t encode(char *dest, const char *buf) dest[d] = 0; /* NUL-terminate destination string */ return i + 1; } - check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i); + check(sscanf((char *) (buf + i), "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i); i = j; /* skip to char after hex input */ d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d)); } } + +/* simplistic, portable replacement for getline, sufficient for our tests */ +size_t simple_getline(unsigned char buf[8192], FILE *f) { + size_t i = 0; + while (i < 8191) { + int c = getc(f); + if (c == EOF || c == '\n') break; + buf[i++] = (unsigned char) c; + } + buf[i] = 0; + return i; +} diff --git a/3rdparty/utf8proc/test/tests.h b/3rdparty/utf8proc/test/tests.h index 1811a734a5f..a30510100ed 100644 --- a/3rdparty/utf8proc/test/tests.h +++ b/3rdparty/utf8proc/test/tests.h @@ -1,13 +1,16 @@ /* Common functions and includes for our test programs. */ /* - * Set feature macro to enable getline() and wcwidth(). + * Set feature macro to enable wcwidth(). * * Please refer to section 2.2.1 of POSIX.1-2008: * http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_02 */ #define _XOPEN_SOURCE 700 +/* silence warnings about sscanf on Windows */ +#define _CRT_SECURE_NO_WARNINGS + #include <stdio.h> #include <stdlib.h> #include <ctype.h> @@ -19,5 +22,6 @@ extern size_t lineno; void check(int cond, const char *format, ...); -size_t skipspaces(const char *buf, size_t i); -size_t encode(char *dest, const char *buf); +size_t skipspaces(const unsigned char *buf, size_t i); +size_t encode(unsigned char *dest, const unsigned char *buf); +size_t simple_getline(unsigned char buf[8192], FILE *f); |