summaryrefslogtreecommitdiffstatshomepage
path: root/3rdparty/utf8proc/test
diff options
context:
space:
mode:
Diffstat (limited to '3rdparty/utf8proc/test')
-rw-r--r--3rdparty/utf8proc/test/case.c28
-rw-r--r--3rdparty/utf8proc/test/charwidth.c120
-rw-r--r--3rdparty/utf8proc/test/custom.c1
-rw-r--r--3rdparty/utf8proc/test/graphemetest.c175
-rw-r--r--3rdparty/utf8proc/test/iscase.c62
-rw-r--r--3rdparty/utf8proc/test/iterate.c4
-rw-r--r--3rdparty/utf8proc/test/misc.c51
-rw-r--r--3rdparty/utf8proc/test/normtest.c11
-rw-r--r--3rdparty/utf8proc/test/printproperty.c93
-rw-r--r--3rdparty/utf8proc/test/tests.c18
-rw-r--r--3rdparty/utf8proc/test/tests.h10
11 files changed, 400 insertions, 173 deletions
diff --git a/3rdparty/utf8proc/test/case.c b/3rdparty/utf8proc/test/case.c
index 39958e3e933..4a98a6390b5 100644
--- a/3rdparty/utf8proc/test/case.c
+++ b/3rdparty/utf8proc/test/case.c
@@ -13,13 +13,22 @@ int main(int argc, char **argv)
for (c = 0; c <= 0x110000; ++c) {
utf8proc_int32_t l = utf8proc_tolower(c);
utf8proc_int32_t u = utf8proc_toupper(c);
+ utf8proc_int32_t t = utf8proc_totitle(c);
check(l == c || utf8proc_codepoint_valid(l), "invalid tolower");
check(u == c || utf8proc_codepoint_valid(u), "invalid toupper");
+ check(t == c || utf8proc_codepoint_valid(t), "invalid totitle");
+
+ if (utf8proc_codepoint_valid(c) && (l == u) != (l == t) &&
+ /* Unicode 11: Georgian Mkhedruli chars have uppercase but no titlecase. */
+ !(((c >= 0x10d0 && c <= 0x10fa) || c >= (0x10fd && c <= 0x10ff)) && l != u)) {
+ fprintf(stderr, "unexpected titlecase %x for lowercase %x / uppercase %x\n", t, l, c);
+ ++error;
+ }
if (sizeof(wint_t) > 2 || c < (1<<16)) {
wint_t l0 = towlower(c), u0 = towupper(c);
-
+
/* OS unicode tables may be out of date. But if they
do have a lower/uppercase mapping, hopefully it
is correct? */
@@ -44,6 +53,23 @@ int main(int argc, char **argv)
}
}
check(!error, "utf8proc case conversion FAILED %d tests.", error);
+
+ /* issue #130 */
+ check(utf8proc_toupper(0x00df) == 0x1e9e &&
+ utf8proc_totitle(0x00df) == 0x1e9e &&
+ utf8proc_tolower(0x00df) == 0x00df &&
+ utf8proc_tolower(0x1e9e) == 0x00df &&
+ utf8proc_toupper(0x1e9e) == 0x1e9e,
+ "incorrect 0x00df/0x1e9e case conversions");
+ utf8proc_uint8_t str_00df[] = {0xc3, 0x9f, 0x00};
+ utf8proc_uint8_t str_1e9e[] = {0xe1, 0xba, 0x9e, 0x00};
+ utf8proc_uint8_t *s1 = utf8proc_NFKC_Casefold(str_00df);
+ utf8proc_uint8_t *s2 = utf8proc_NFKC_Casefold(str_1e9e);
+ check(!strcmp((char*)s1, "ss") &&
+ !strcmp((char*)s2, "ss"),
+ "incorrect 0x00df/0x1e9e casefold normalization");
+ free(s1);
+ free(s2);
printf("More up-to-date than OS unicode tables for %d tests.\n", better);
printf("utf8proc case conversion tests SUCCEEDED.\n");
return 0;
diff --git a/3rdparty/utf8proc/test/charwidth.c b/3rdparty/utf8proc/test/charwidth.c
index 330f18eebb4..c5cbbd7cdc1 100644
--- a/3rdparty/utf8proc/test/charwidth.c
+++ b/3rdparty/utf8proc/test/charwidth.c
@@ -2,70 +2,76 @@
#include <ctype.h>
#include <wchar.h>
+static int my_unassigned(int c) {
+ int cat = utf8proc_get_property(c)->category;
+ return (cat == UTF8PROC_CATEGORY_CN) || (cat == UTF8PROC_CATEGORY_CO);
+}
+
static int my_isprint(int c) {
- int cat = utf8proc_get_property(c)->category;
- return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) ||
- (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd);
+ int cat = utf8proc_get_property(c)->category;
+ return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) ||
+ (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd || c == 0x00ad) ||
+ (cat == UTF8PROC_CATEGORY_CN) || (cat == UTF8PROC_CATEGORY_CO);
}
int main(int argc, char **argv)
{
- int c, error = 0, updates = 0;
+ int c, error = 0, updates = 0;
+
+ (void) argc; /* unused */
+ (void) argv; /* unused */
- (void) argc; /* unused */
- (void) argv; /* unused */
+ /* some simple sanity tests of the character widths */
+ for (c = 0; c <= 0x110000; ++c) {
+ int cat = utf8proc_get_property(c)->category;
+ int w = utf8proc_charwidth(c);
+ if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) && w > 0) {
+ fprintf(stderr, "nonzero width %d for combining char %x\n", w, c);
+ error += 1;
+ }
+ if (w == 0 &&
+ ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) ||
+ (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) ||
+ (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) {
+ fprintf(stderr, "zero width for symbol-like char %x\n", c);
+ error += 1;
+ }
+ if (c <= 127 && ((!isprint(c) && w > 0) || (isprint(c) && wcwidth(c) != w))) {
+ fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n",
+ wcwidth(c), w,
+ isprint(c) ? "printable" : "non-printable", c);
+ error += 1;
+ }
+ if (!my_isprint(c) && w > 0) {
+ fprintf(stderr, "non-printing %x had width %d\n", c, w);
+ error += 1;
+ }
+ if (my_unassigned(c) && w != 1) {
+ fprintf(stderr, "unexpected width %d for unassigned char %x\n", w, c);
+ error += 1;
+ }
+ }
+ check(!error, "utf8proc_charwidth FAILED %d tests.", error);
- /* some simple sanity tests of the character widths */
- for (c = 0; c <= 0x110000; ++c) {
- int cat = utf8proc_get_property(c)->category;
- int w = utf8proc_charwidth(c);
- if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) &&
- w > 0) {
- fprintf(stderr, "nonzero width %d for combining char %x\n", w, c);
- error = 1;
- }
- if (w == 0 &&
- ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) ||
- (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) ||
- (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) {
- fprintf(stderr, "zero width for symbol-like char %x\n", c);
- error = 1;
- }
- if (c <= 127 && ((!isprint(c) && w > 0) ||
- (isprint(c) && wcwidth(c) != w))) {
- fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n",
- wcwidth(c), w,
- isprint(c) ? "printable" : "non-printable", c);
- error = 1;
- }
- if (!my_isprint(c) && w > 0) {
- fprintf(stderr, "non-printing %x had width %d\n", c, w);
- error = 1;
- }
- }
- check(!error, "utf8proc_charwidth FAILED tests.");
+ check(utf8proc_charwidth(0x00ad) == 1, "incorrect width for U+00AD (soft hyphen)");
+ check(utf8proc_charwidth(0xe000) == 1, "incorrect width for U+e000 (PUA)");
- /* print some other information by compariing with system wcwidth */
- printf("Mismatches with system wcwidth (not necessarily errors):\n");
- for (c = 0; c <= 0x110000; ++c) {
- int w = utf8proc_charwidth(c);
- int wc = wcwidth(c);
- if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue;
- /* lots of these errors for out-of-date system unicode tables */
- if (wc == -1 && my_isprint(c) && w > 0) {
- updates += 1;
-#if 0
- printf(" wcwidth(%x) = -1 for printable char\n", c);
-#endif
- }
- if (wc == -1 && !my_isprint(c) && w > 0)
- printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w);
- if (wc >= 0 && wc != w)
- printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w);
- }
- printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n",
- updates);
- printf("Character-width tests SUCCEEDED.\n");
+ /* print some other information by compariing with system wcwidth */
+ printf("Mismatches with system wcwidth (not necessarily errors):\n");
+ for (c = 0; c <= 0x110000; ++c) {
+ int w = utf8proc_charwidth(c);
+ int wc = wcwidth(c);
+ if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue;
+ /* lots of these errors for out-of-date system unicode tables */
+ if (wc == -1 && my_isprint(c) && !my_unassigned(c) && w > 0)
+ updates += 1;
+ if (wc == -1 && !my_isprint(c) && w > 0)
+ printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w);
+ if (wc >= 0 && wc != w)
+ printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w);
+ }
+ printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n", updates);
+ printf("Character-width tests SUCCEEDED.\n");
- return 0;
+ return 0;
}
diff --git a/3rdparty/utf8proc/test/custom.c b/3rdparty/utf8proc/test/custom.c
index f85b3cc2bc6..fe4239d91b4 100644
--- a/3rdparty/utf8proc/test/custom.c
+++ b/3rdparty/utf8proc/test/custom.c
@@ -23,5 +23,6 @@ int main(void)
check(strlen((char*) output) == 6, "incorrect output length");
check(!memcmp(correct, output, 7), "incorrect output data");
free(output);
+ printf("map_custom tests SUCCEEDED.\n");
return 0;
}
diff --git a/3rdparty/utf8proc/test/graphemetest.c b/3rdparty/utf8proc/test/graphemetest.c
index eb3645b9a0c..22880fc9f14 100644
--- a/3rdparty/utf8proc/test/graphemetest.c
+++ b/3rdparty/utf8proc/test/graphemetest.c
@@ -1,74 +1,127 @@
#include "tests.h"
+/* check one line in the format of GraphemeBreakTest.txt */
+void checkline(const char *_buf, bool verbose) {
+ size_t bi = 0, si = 0;
+ utf8proc_uint8_t src[1024]; /* more than long enough for all of our tests */
+ const unsigned char *buf = (const unsigned char *) _buf;
+
+ while (buf[bi]) {
+ bi = skipspaces(buf, bi);
+ if (buf[bi] == 0xc3 && buf[bi+1] == 0xb7) { /* U+00f7 = grapheme break */
+ src[si++] = '/';
+ bi += 2;
+ }
+ else if (buf[bi] == 0xc3 && buf[bi+1] == 0x97) { /* U+00d7 = no break */
+ bi += 2;
+ }
+ else if (buf[bi] == '#') { /* start of comments */
+ break;
+ }
+ else if (buf[bi] == '/') { /* for convenience, also accept / as grapheme break */
+ src[si++] = '/';
+ bi += 1;
+ }
+ else { /* hex-encoded codepoint */
+ size_t len = encode((unsigned char*) (src + si), buf + bi) - 1;
+ while (src[si]) ++si; /* advance to NUL termination */
+ bi += len;
+ }
+ }
+ if (si && src[si-1] == '/')
+ --si; /* no break after final grapheme */
+ src[si] = 0; /* NUL-terminate */
+
+ if (si) { /* test utf8proc_map */
+ utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */
+ size_t i = 0, j = 0;
+ utf8proc_ssize_t glen, k;
+ utf8proc_uint8_t *g; /* utf8proc_map grapheme results */
+ while (i < si) {
+ if (src[i] != '/')
+ utf8[j++] = src[i++];
+ else
+ i++;
+ }
+ glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND);
+ if (glen == UTF8PROC_ERROR_INVALIDUTF8) {
+ /* the test file contains surrogate codepoints, which are only for UTF-16 */
+ printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno);
+ }
+ else {
+ check(glen >= 0, "utf8proc_map error = %s",
+ utf8proc_errmsg(glen));
+ for (k = 0; k <= glen; ++k)
+ if (g[k] == 0xff)
+ g[k] = '/'; /* easier-to-read output (/ is not in test strings) */
+ check(!strcmp((char*)g, (char*)src),
+ "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
+ }
+ free(g);
+ }
+
+ if (si) { /* test manual calls to utf8proc_grapheme_break_stateful */
+ utf8proc_int32_t state = 0, prev_codepoint = 0;
+ size_t i = 0;
+ utf8proc_bool expectbreak = false;
+ do {
+ utf8proc_int32_t codepoint;
+ i += utf8proc_iterate(src + i, si - i, &codepoint);
+ check(codepoint >= 0, "invalid UTF-8 data");
+ if (codepoint == 0x002F)
+ expectbreak = true;
+ else {
+ if (prev_codepoint != 0) {
+ check(expectbreak == utf8proc_grapheme_break_stateful(prev_codepoint, codepoint, &state),
+ "grapheme mismatch: between 0x%04x and 0x%04x in \"%s\"", prev_codepoint, codepoint, (char*) src);
+ }
+ expectbreak = false;
+ prev_codepoint = codepoint;
+ }
+ } while (i < si);
+ }
+
+ if (verbose)
+ printf("passed grapheme test: \"%s\"\n", (char*) src);
+}
+
int main(int argc, char **argv)
{
- char *buf = NULL;
- size_t bufsize = 0;
+ unsigned char buf[8192];
FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
- utf8proc_uint8_t src[1024];
- int len;
-
+
check(f != NULL, "error opening GraphemeBreakTest.txt");
- while (getline(&buf, &bufsize, f) > 0) {
- size_t bi = 0, si = 0;
- lineno += 1;
-
- if (lineno % 100 == 0)
+ while (simple_getline(buf, f) > 0) {
+ if ((++lineno) % 100 == 0)
printf("checking line %zd...\n", lineno);
-
if (buf[0] == '#') continue;
-
- while (buf[bi]) {
- bi = skipspaces(buf, bi);
- if (buf[bi] == '/') { /* grapheme break */
- src[si++] = '/';
- bi++;
- }
- else if (buf[bi] == '+') { /* no break */
- bi++;
- }
- else if (buf[bi] == '#') { /* start of comments */
- break;
- }
- else { /* hex-encoded codepoint */
- len = encode((char*) (src + si), buf + bi) - 1;
- while (src[si]) ++si; /* advance to NUL termination */
- bi += len;
- }
- }
- if (si && src[si-1] == '/')
- --si; /* no break after final grapheme */
- src[si] = 0; /* NUL-terminate */
-
- if (si) {
- utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */
- size_t i = 0, j = 0;
- utf8proc_ssize_t glen;
- utf8proc_uint8_t *g; /* utf8proc_map grapheme results */
- while (i < si) {
- if (src[i] != '/')
- utf8[j++] = src[i++];
- else
- i++;
- }
- glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND);
- if (glen == UTF8PROC_ERROR_INVALIDUTF8) {
- /* the test file contains surrogate codepoints, which are only for UTF-16 */
- printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno);
- }
- else {
- check(glen >= 0, "utf8proc_map error = %s",
- utf8proc_errmsg(glen));
- for (i = 0; i <= glen; ++i)
- if (g[i] == 0xff)
- g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
- check(!strcmp((char*)g, (char*)src),
- "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
- }
- free(g);
- }
+ checkline((char *) buf, false);
}
fclose(f);
printf("Passed tests after %zd lines!\n", lineno);
+
+ printf("Performing regression tests...\n");
+
+ /* issue 144 */
+ {
+ utf8proc_uint8_t input[] = {0xef,0xbf,0xbf,0xef,0xbf,0xbe,0x00}; /* "\uffff\ufffe" */
+ utf8proc_uint8_t output[] = {0xff,0xef,0xbf,0xbf,0xff,0xef,0xbf,0xbe,0x00}; /* with 0xff grapheme markers */
+ utf8proc_ssize_t glen;
+ utf8proc_uint8_t *g;
+ glen = utf8proc_map(input, 6, &g, UTF8PROC_CHARBOUND);
+ check(!strcmp((char*)g, (char*)output), "mishandled u+ffff and u+fffe grapheme breaks");
+ free(g);
+ };
+
+ /* https://github.com/JuliaLang/julia/issues/37680 */
+ checkline("/ 1f1f8 1f1ea / 1f1f8 1f1ea /", true); /* Two swedish flags after each other */
+ checkline("/ 1f926 1f3fc 200d 2642 fe0f /", true); /* facepalm + pale skin + zwj + male sign + FE0F */
+ checkline("/ 1f468 1f3fb 200d 1f91d 200d 1f468 1f3fd /", true); /* man face + pale skin + zwj + hand holding + zwj + man face + dark skin */
+
+ check(utf8proc_grapheme_break(0x03b1, 0x03b2), "failed 03b1 / 03b2 test");
+ check(!utf8proc_grapheme_break(0x03b1, 0x0302), "failed 03b1 0302 test");
+
+ printf("Passed regression tests!\n");
+
return 0;
}
diff --git a/3rdparty/utf8proc/test/iscase.c b/3rdparty/utf8proc/test/iscase.c
new file mode 100644
index 00000000000..f3f8cf31b40
--- /dev/null
+++ b/3rdparty/utf8proc/test/iscase.c
@@ -0,0 +1,62 @@
+#include "tests.h"
+
+int read_range(FILE *f, utf8proc_int32_t *start, utf8proc_int32_t *end)
+{
+ unsigned char buf[8192];
+ size_t len = simple_getline(buf, f);
+ size_t pos = skipspaces(buf, 0);
+ unsigned char s[16];
+ if (pos == len || buf[pos] == '#') return 0;
+ pos += encode(s, buf + pos) - 1;
+ check(s[0], "invalid line %s in data", buf);
+ utf8proc_iterate((utf8proc_uint8_t*) s, -1, start);
+ if (buf[pos] == '.' && buf[pos+1] == '.') {
+ encode(s, buf + pos + 2);
+ check(s[0], "invalid line %s in data", buf);
+ utf8proc_iterate((utf8proc_uint8_t*) s, -1, end);
+ }
+ else
+ *end = *start;
+ return 1;
+}
+
+int test_iscase(const char *fname, int (*iscase)(utf8proc_int32_t),
+ utf8proc_int32_t (*thatcase)(utf8proc_int32_t))
+{
+ FILE *f = fopen(fname, "r");
+ int lines = 0, tests = 0, success = 1;
+ utf8proc_int32_t c = 0;
+
+ check(f != NULL, "error opening data file \"%s\"\n", fname);
+
+ while (success && !feof(f)) {
+ utf8proc_int32_t start, end;
+ if (read_range(f, &start, &end)) {
+ for (; c < start; ++c) {
+ check(!iscase(c), "failed !iscase(%04x) in %s\n", c, fname);
+ }
+ for (; c <= end; ++c) {
+ check(iscase(c), "failed iscase(%04x) in %s\n", c, fname);
+ check(thatcase(c) == c, "inconsistent thatcase(%04x) in %s\n", c, fname);
+ ++tests;
+ }
+ }
+ ++lines;
+ }
+ for (; c <= 0x110000; ++c) {
+ check(!iscase(c), "failed !iscase(%04x) in %s\n", c, fname);
+ }
+
+ printf("Checked %d characters from %d lines of %s\n", tests, lines, fname);
+ fclose(f);
+ return success;
+}
+
+int main(int argc, char **argv)
+{
+ check(argc == 3, "Expected Lowercase.txt and Uppercase.txt as arguments");
+ check(test_iscase(argv[1], utf8proc_islower, utf8proc_tolower), "Lowercase tests failed");
+ check(test_iscase(argv[2], utf8proc_isupper, utf8proc_toupper), "Uppercase tests failed");
+ printf("utf8proc iscase tests SUCCEEDED.\n");
+ return 0;
+}
diff --git a/3rdparty/utf8proc/test/iterate.c b/3rdparty/utf8proc/test/iterate.c
index c1674b79952..b8e9feb01b5 100644
--- a/3rdparty/utf8proc/test/iterate.c
+++ b/3rdparty/utf8proc/test/iterate.c
@@ -35,6 +35,8 @@ int main(int argc, char **argv)
uint32_t byt;
unsigned char buf[16];
+ (void) argc; (void) argv; /* unused */
+
tests = error = 0;
// Check valid sequences that were considered valid erroneously before
@@ -54,7 +56,7 @@ int main(int argc, char **argv)
CHECKVALID(3, 0xbe, 4);
CHECKVALID(3, 0xbf, 4);
}
-
+
// Continuation byte not after lead
for (byt = 0x80; byt < 0xc0; byt++) {
CHECKINVALID(0, byt, 1);
diff --git a/3rdparty/utf8proc/test/misc.c b/3rdparty/utf8proc/test/misc.c
new file mode 100644
index 00000000000..9156f95541d
--- /dev/null
+++ b/3rdparty/utf8proc/test/misc.c
@@ -0,0 +1,51 @@
+/* Miscellaneous tests, e.g. regression tests */
+
+#include "tests.h"
+
+static void issue128(void) /* #128 */
+{
+ utf8proc_uint8_t input[] = {0x72, 0xcc, 0x87, 0xcc, 0xa3, 0x00}; /* "r\u0307\u0323" */
+ utf8proc_uint8_t nfc[] = {0xe1, 0xb9, 0x9b, 0xcc, 0x87, 0x00}; /* "\u1E5B\u0307" */
+ utf8proc_uint8_t nfd[] = {0x72, 0xcc, 0xa3, 0xcc, 0x87, 0x00}; /* "r\u0323\u0307" */
+ utf8proc_uint8_t *nfc_out, *nfd_out;
+ nfc_out = utf8proc_NFC(input);
+ printf("NFC \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)nfc_out, (char*)nfc);
+ check(strlen((char*) nfc_out) == 5, "incorrect nfc length");
+ check(!memcmp(nfc, nfc_out, 6), "incorrect nfc data");
+ nfd_out = utf8proc_NFD(input);
+ printf("NFD \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)nfd_out, (char*)nfd);
+ check(strlen((char*) nfd_out) == 5, "incorrect nfd length");
+ check(!memcmp(nfd, nfd_out, 6), "incorrect nfd data");
+ free(nfd_out); free(nfc_out);
+}
+
+static void issue102(void) /* #128 */
+{
+ utf8proc_uint8_t input[] = {0x58, 0xe2, 0x81, 0xa5, 0x45, 0xcc, 0x80, 0xc2, 0xad, 0xe1, 0xb4, 0xac, 0x00}; /* "X\u2065E\u0300\u00ad\u1d2c" */
+ utf8proc_uint8_t stripna[] = {0x78, 0xc3, 0xa8, 0x61, 0x00}; /* "x\u00e8a" */
+ utf8proc_uint8_t correct[] = {0x78, 0xe2, 0x81, 0xa5, 0xc3, 0xa8, 0x61, 0x00}; /* "x\u2065\u00e8a" */
+ utf8proc_uint8_t *output;
+ utf8proc_map(input, 0, &output, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+ UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD | UTF8PROC_IGNORE | UTF8PROC_STRIPNA);
+ printf("NFKC_Casefold \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)output, (char*)stripna);
+ check(strlen((char*) output) == 4, "incorrect NFKC_Casefold+stripna length");
+ check(!memcmp(stripna, output, 5), "incorrect NFKC_Casefold+stripna data");
+ free(output);
+ output = utf8proc_NFKC_Casefold(input);
+ printf("NFKC_Casefold \"%s\" -> \"%s\" vs. \"%s\"\n", (char*)input, (char*)output, (char*)correct);
+ check(strlen((char*) output) == 7, "incorrect NFKC_Casefold length");
+ check(!memcmp(correct, output, 8), "incorrect NFKC_Casefold data");
+ free(output);
+}
+
+int main(void)
+{
+ issue128();
+ issue102();
+#ifdef UNICODE_VERSION
+ printf("Unicode version: Makefile has %s, has API %s\n", UNICODE_VERSION, utf8proc_unicode_version());
+ check(!strcmp(UNICODE_VERSION, utf8proc_unicode_version()), "utf8proc_unicode_version mismatch");
+#endif
+ printf("Misc tests SUCCEEDED.\n");
+ return 0;
+}
diff --git a/3rdparty/utf8proc/test/normtest.c b/3rdparty/utf8proc/test/normtest.c
index 555c14c84bf..627ee79fd26 100644
--- a/3rdparty/utf8proc/test/normtest.c
+++ b/3rdparty/utf8proc/test/normtest.c
@@ -1,21 +1,20 @@
#include "tests.h"
#define CHECK_NORM(NRM, norm, src) { \
- char *src_norm = (char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \
- check(!strcmp(norm, src_norm), \
+ unsigned char *src_norm = (unsigned char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \
+ check(!strcmp((char *) norm, (char *) src_norm), \
"normalization failed for %s -> %s", src, norm); \
free(src_norm); \
}
int main(int argc, char **argv)
{
- char *buf = NULL;
- size_t bufsize = 0;
+ unsigned char buf[8192];
FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
- char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024];
+ unsigned char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024];
check(f != NULL, "error opening NormalizationTest.txt");
- while (getline(&buf, &bufsize, f) > 0) {
+ while (simple_getline(buf, f) > 0) {
size_t offset;
lineno += 1;
diff --git a/3rdparty/utf8proc/test/printproperty.c b/3rdparty/utf8proc/test/printproperty.c
index 2819aa1e881..709e6a72b15 100644
--- a/3rdparty/utf8proc/test/printproperty.c
+++ b/3rdparty/utf8proc/test/printproperty.c
@@ -4,46 +4,57 @@
int main(int argc, char **argv)
{
- int i;
+ int i;
- for (i = 1; i < argc; ++i) {
- unsigned int c;
- if (!strcmp(argv[i], "-V")) {
- printf("utf8proc version %s\n", utf8proc_version());
- continue;
- }
- check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]);
- const utf8proc_property_t *p = utf8proc_get_property(c);
- printf("U+%s:\n"
- " category = %s\n"
- " combining_class = %d\n"
- " bidi_class = %d\n"
- " decomp_type = %d\n"
- " uppercase_mapping = %x\n"
- " lowercase_mapping = %x\n"
- " titlecase_mapping = %x\n"
- " comb_index = %d\n"
- " bidi_mirrored = %d\n"
- " comp_exclusion = %d\n"
- " ignorable = %d\n"
- " control_boundary = %d\n"
- " boundclass = %d\n"
- " charwidth = %d\n",
- argv[i],
- utf8proc_category_string(c),
- p->combining_class,
- p->bidi_class,
- p->decomp_type,
- utf8proc_toupper(c),
- utf8proc_tolower(c),
- utf8proc_totitle(c),
- p->comb_index,
- p->bidi_mirrored,
- p->comp_exclusion,
- p->ignorable,
- p->control_boundary,
- p->boundclass,
- utf8proc_charwidth(c));
- }
- return 0;
+ for (i = 1; i < argc; ++i) {
+ utf8proc_uint8_t cstr[16], *map;
+ unsigned int c;
+ if (!strcmp(argv[i], "-V")) {
+ printf("utf8proc version %s\n", utf8proc_version());
+ continue;
+ }
+ check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]);
+ const utf8proc_property_t *p = utf8proc_get_property(c);
+
+ if (utf8proc_codepoint_valid(c))
+ cstr[utf8proc_encode_char(c, cstr)] = 0;
+ else
+ strcat((char*)cstr, "N/A");
+ utf8proc_map(cstr, 0, &map, UTF8PROC_NULLTERM | UTF8PROC_CASEFOLD);
+
+ printf("U+%s: %s\n"
+ " category = %s\n"
+ " combining_class = %d\n"
+ " bidi_class = %d\n"
+ " decomp_type = %d\n"
+ " uppercase_mapping = %04x (seqindex %04x)%s\n"
+ " lowercase_mapping = %04x (seqindex %04x)%s\n"
+ " titlecase_mapping = %04x (seqindex %04x)\n"
+ " casefold = %s\n"
+ " comb_index = %d\n"
+ " bidi_mirrored = %d\n"
+ " comp_exclusion = %d\n"
+ " ignorable = %d\n"
+ " control_boundary = %d\n"
+ " boundclass = %d\n"
+ " charwidth = %d\n",
+ argv[i], (char*) cstr,
+ utf8proc_category_string(c),
+ p->combining_class,
+ p->bidi_class,
+ p->decomp_type,
+ utf8proc_toupper(c), p->uppercase_seqindex, utf8proc_isupper(c) ? " (isupper)" : "",
+ utf8proc_tolower(c), p->lowercase_seqindex, utf8proc_islower(c) ? " (islower)" : "",
+ utf8proc_totitle(c), p->titlecase_seqindex,
+ (char *) map,
+ p->comb_index,
+ p->bidi_mirrored,
+ p->comp_exclusion,
+ p->ignorable,
+ p->control_boundary,
+ p->boundclass,
+ utf8proc_charwidth(c));
+ free(map);
+ }
+ return 0;
}
diff --git a/3rdparty/utf8proc/test/tests.c b/3rdparty/utf8proc/test/tests.c
index 0fb0da36305..629edfff373 100644
--- a/3rdparty/utf8proc/test/tests.c
+++ b/3rdparty/utf8proc/test/tests.c
@@ -17,7 +17,7 @@ void check(int cond, const char *format, ...)
}
}
-size_t skipspaces(const char *buf, size_t i)
+size_t skipspaces(const unsigned char *buf, size_t i)
{
while (isspace(buf[i])) ++i;
return i;
@@ -27,7 +27,7 @@ size_t skipspaces(const char *buf, size_t i)
separated by whitespace, and terminated by any character not in
[0-9a-fA-F] or whitespace, then stores the corresponding utf8 string
in dest, returning the number of bytes read from buf */
-size_t encode(char *dest, const char *buf)
+size_t encode(unsigned char *dest, const unsigned char *buf)
{
size_t i = 0, j, d = 0;
for (;;) {
@@ -39,8 +39,20 @@ size_t encode(char *dest, const char *buf)
dest[d] = 0; /* NUL-terminate destination string */
return i + 1;
}
- check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
+ check(sscanf((char *) (buf + i), "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
i = j; /* skip to char after hex input */
d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d));
}
}
+
+/* simplistic, portable replacement for getline, sufficient for our tests */
+size_t simple_getline(unsigned char buf[8192], FILE *f) {
+ size_t i = 0;
+ while (i < 8191) {
+ int c = getc(f);
+ if (c == EOF || c == '\n') break;
+ buf[i++] = (unsigned char) c;
+ }
+ buf[i] = 0;
+ return i;
+}
diff --git a/3rdparty/utf8proc/test/tests.h b/3rdparty/utf8proc/test/tests.h
index 1811a734a5f..a30510100ed 100644
--- a/3rdparty/utf8proc/test/tests.h
+++ b/3rdparty/utf8proc/test/tests.h
@@ -1,13 +1,16 @@
/* Common functions and includes for our test programs. */
/*
- * Set feature macro to enable getline() and wcwidth().
+ * Set feature macro to enable wcwidth().
*
* Please refer to section 2.2.1 of POSIX.1-2008:
* http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_02
*/
#define _XOPEN_SOURCE 700
+/* silence warnings about sscanf on Windows */
+#define _CRT_SECURE_NO_WARNINGS
+
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
@@ -19,5 +22,6 @@
extern size_t lineno;
void check(int cond, const char *format, ...);
-size_t skipspaces(const char *buf, size_t i);
-size_t encode(char *dest, const char *buf);
+size_t skipspaces(const unsigned char *buf, size_t i);
+size_t encode(unsigned char *dest, const unsigned char *buf);
+size_t simple_getline(unsigned char buf[8192], FILE *f);