diff options
author | 2021-10-09 12:16:17 +1100 | |
---|---|---|
committer | 2021-10-09 12:16:17 +1100 | |
commit | 38082ccbee749d650ccea886ae376a5d1dec337c (patch) | |
tree | 9ba9a900ba826bda58832834278025ced17f42f5 /3rdparty/utf8proc/test/graphemetest.c | |
parent | 34b3bf701098082feb9077db49987507962c1578 (diff) |
Overdue internal UI enhancements (#8674)
* frontend: Added support for message context to localisations.
* frontend: Added string_view versions of the message lookup functions.
* frontend: Added a few more folder options to the internal UI.
* emu/softlist.cpp: Use more appropriate containers.
* Switched to Python 3 by default - this will become a requirement.
* Updated msgfmt.py for message context support.
* frontend: Show all software item info in the internal UI.
* frontend: Search alternate titles in software selection menu.
* 3rdparty/utf8proc: Updated to v2.6.1 (has several fixes).
* frontend: Added software filters for common info fields.
* frontend: Allow UI manager to hold onto persistent session data.
* frontend: Cache software lists for eight machines.
* frontend: Added support for loading localised system names.
* frontend: Add UI for selecting localised system names.
Diffstat (limited to '3rdparty/utf8proc/test/graphemetest.c')
-rw-r--r-- | 3rdparty/utf8proc/test/graphemetest.c | 175 |
1 files changed, 114 insertions, 61 deletions
diff --git a/3rdparty/utf8proc/test/graphemetest.c b/3rdparty/utf8proc/test/graphemetest.c index eb3645b9a0c..22880fc9f14 100644 --- a/3rdparty/utf8proc/test/graphemetest.c +++ b/3rdparty/utf8proc/test/graphemetest.c @@ -1,74 +1,127 @@ #include "tests.h" +/* check one line in the format of GraphemeBreakTest.txt */ +void checkline(const char *_buf, bool verbose) { + size_t bi = 0, si = 0; + utf8proc_uint8_t src[1024]; /* more than long enough for all of our tests */ + const unsigned char *buf = (const unsigned char *) _buf; + + while (buf[bi]) { + bi = skipspaces(buf, bi); + if (buf[bi] == 0xc3 && buf[bi+1] == 0xb7) { /* U+00f7 = grapheme break */ + src[si++] = '/'; + bi += 2; + } + else if (buf[bi] == 0xc3 && buf[bi+1] == 0x97) { /* U+00d7 = no break */ + bi += 2; + } + else if (buf[bi] == '#') { /* start of comments */ + break; + } + else if (buf[bi] == '/') { /* for convenience, also accept / as grapheme break */ + src[si++] = '/'; + bi += 1; + } + else { /* hex-encoded codepoint */ + size_t len = encode((unsigned char*) (src + si), buf + bi) - 1; + while (src[si]) ++si; /* advance to NUL termination */ + bi += len; + } + } + if (si && src[si-1] == '/') + --si; /* no break after final grapheme */ + src[si] = 0; /* NUL-terminate */ + + if (si) { /* test utf8proc_map */ + utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */ + size_t i = 0, j = 0; + utf8proc_ssize_t glen, k; + utf8proc_uint8_t *g; /* utf8proc_map grapheme results */ + while (i < si) { + if (src[i] != '/') + utf8[j++] = src[i++]; + else + i++; + } + glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND); + if (glen == UTF8PROC_ERROR_INVALIDUTF8) { + /* the test file contains surrogate codepoints, which are only for UTF-16 */ + printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno); + } + else { + check(glen >= 0, "utf8proc_map error = %s", + utf8proc_errmsg(glen)); + for (k = 0; k <= glen; ++k) + if (g[k] == 0xff) + g[k] = '/'; /* easier-to-read output (/ is not in test strings) */ + check(!strcmp((char*)g, (char*)src), + "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src); + } + free(g); + } + + if (si) { /* test manual calls to utf8proc_grapheme_break_stateful */ + utf8proc_int32_t state = 0, prev_codepoint = 0; + size_t i = 0; + utf8proc_bool expectbreak = false; + do { + utf8proc_int32_t codepoint; + i += utf8proc_iterate(src + i, si - i, &codepoint); + check(codepoint >= 0, "invalid UTF-8 data"); + if (codepoint == 0x002F) + expectbreak = true; + else { + if (prev_codepoint != 0) { + check(expectbreak == utf8proc_grapheme_break_stateful(prev_codepoint, codepoint, &state), + "grapheme mismatch: between 0x%04x and 0x%04x in \"%s\"", prev_codepoint, codepoint, (char*) src); + } + expectbreak = false; + prev_codepoint = codepoint; + } + } while (i < si); + } + + if (verbose) + printf("passed grapheme test: \"%s\"\n", (char*) src); +} + int main(int argc, char **argv) { - char *buf = NULL; - size_t bufsize = 0; + unsigned char buf[8192]; FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL; - utf8proc_uint8_t src[1024]; - int len; - + check(f != NULL, "error opening GraphemeBreakTest.txt"); - while (getline(&buf, &bufsize, f) > 0) { - size_t bi = 0, si = 0; - lineno += 1; - - if (lineno % 100 == 0) + while (simple_getline(buf, f) > 0) { + if ((++lineno) % 100 == 0) printf("checking line %zd...\n", lineno); - if (buf[0] == '#') continue; - - while (buf[bi]) { - bi = skipspaces(buf, bi); - if (buf[bi] == '/') { /* grapheme break */ - src[si++] = '/'; - bi++; - } - else if (buf[bi] == '+') { /* no break */ - bi++; - } - else if (buf[bi] == '#') { /* start of comments */ - break; - } - else { /* hex-encoded codepoint */ - len = encode((char*) (src + si), buf + bi) - 1; - while (src[si]) ++si; /* advance to NUL termination */ - bi += len; - } - } - if (si && src[si-1] == '/') - --si; /* no break after final grapheme */ - src[si] = 0; /* NUL-terminate */ - - if (si) { - utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */ - size_t i = 0, j = 0; - utf8proc_ssize_t glen; - utf8proc_uint8_t *g; /* utf8proc_map grapheme results */ - while (i < si) { - if (src[i] != '/') - utf8[j++] = src[i++]; - else - i++; - } - glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND); - if (glen == UTF8PROC_ERROR_INVALIDUTF8) { - /* the test file contains surrogate codepoints, which are only for UTF-16 */ - printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno); - } - else { - check(glen >= 0, "utf8proc_map error = %s", - utf8proc_errmsg(glen)); - for (i = 0; i <= glen; ++i) - if (g[i] == 0xff) - g[i] = '/'; /* easier-to-read output (/ is not in test strings) */ - check(!strcmp((char*)g, (char*)src), - "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src); - } - free(g); - } + checkline((char *) buf, false); } fclose(f); printf("Passed tests after %zd lines!\n", lineno); + + printf("Performing regression tests...\n"); + + /* issue 144 */ + { + utf8proc_uint8_t input[] = {0xef,0xbf,0xbf,0xef,0xbf,0xbe,0x00}; /* "\uffff\ufffe" */ + utf8proc_uint8_t output[] = {0xff,0xef,0xbf,0xbf,0xff,0xef,0xbf,0xbe,0x00}; /* with 0xff grapheme markers */ + utf8proc_ssize_t glen; + utf8proc_uint8_t *g; + glen = utf8proc_map(input, 6, &g, UTF8PROC_CHARBOUND); + check(!strcmp((char*)g, (char*)output), "mishandled u+ffff and u+fffe grapheme breaks"); + free(g); + }; + + /* https://github.com/JuliaLang/julia/issues/37680 */ + checkline("/ 1f1f8 1f1ea / 1f1f8 1f1ea /", true); /* Two swedish flags after each other */ + checkline("/ 1f926 1f3fc 200d 2642 fe0f /", true); /* facepalm + pale skin + zwj + male sign + FE0F */ + checkline("/ 1f468 1f3fb 200d 1f91d 200d 1f468 1f3fd /", true); /* man face + pale skin + zwj + hand holding + zwj + man face + dark skin */ + + check(utf8proc_grapheme_break(0x03b1, 0x03b2), "failed 03b1 / 03b2 test"); + check(!utf8proc_grapheme_break(0x03b1, 0x0302), "failed 03b1 0302 test"); + + printf("Passed regression tests!\n"); + return 0; } |