diff options
Diffstat (limited to '3rdparty/lzma/CPP/Common/UTFConvert.cpp')
-rw-r--r-- | 3rdparty/lzma/CPP/Common/UTFConvert.cpp | 711 |
1 files changed, 643 insertions, 68 deletions
diff --git a/3rdparty/lzma/CPP/Common/UTFConvert.cpp b/3rdparty/lzma/CPP/Common/UTFConvert.cpp index b772164aab4..ac069dbadb9 100644 --- a/3rdparty/lzma/CPP/Common/UTFConvert.cpp +++ b/3rdparty/lzma/CPP/Common/UTFConvert.cpp @@ -2,11 +2,17 @@ #include "StdAfx.h" +// #include <stdio.h> + #include "MyTypes.h" #include "UTFConvert.h" -#ifdef _WIN32 -#define _WCHART_IS_16BIT 1 + +#ifndef _WCHART_IS_16BIT +#ifndef __APPLE__ + // we define it if the system supports files with non-utf8 symbols: + #define _UTF8_RAW_NON_UTF8_SUPPORTED +#endif #endif /* @@ -18,78 +24,332 @@ 1 : 0xC0 : 11 : 2 : 0xE0 : 16 : Basic Multilingual Plane 3 : 0xF0 : 21 : Unicode space - 3 : 0xF8 : 26 : - 5 : 0xFC : 31 : UCS-4 + 4 : 0xF8 : 26 : + 5 : 0xFC : 31 : UCS-4 : wcstombs() in ubuntu is limited to that value 6 : 0xFE : 36 : We can use it, if we want to encode any 32-bit value 7 : 0xFF : */ #define _UTF8_START(n) (0x100 - (1 << (7 - (n)))) -#define _UTF8_HEAD_PARSE2(n) if (c < _UTF8_START((n) + 1)) { numBytes = (n); c -= _UTF8_START(n); } +#define _UTF8_HEAD_PARSE2(n) \ + if (c < _UTF8_START((n) + 1)) \ + { numBytes = (n); val -= _UTF8_START(n); } + +#ifndef _WCHART_IS_16BIT + +/* + if (wchar_t is 32-bit), we can support large points in long UTF-8 sequence, + when we convert wchar_t strings to UTF-8: + (_UTF8_NUM_TAIL_BYTES_MAX == 3) : (21-bits points) - Unicode + (_UTF8_NUM_TAIL_BYTES_MAX == 5) : (31-bits points) - UCS-4 + (_UTF8_NUM_TAIL_BYTES_MAX == 6) : (36-bit hack) +*/ + +#define _UTF8_NUM_TAIL_BYTES_MAX 5 +#endif +/* #define _UTF8_HEAD_PARSE \ + UInt32 val = c; \ _UTF8_HEAD_PARSE2(1) \ else _UTF8_HEAD_PARSE2(2) \ else _UTF8_HEAD_PARSE2(3) \ else _UTF8_HEAD_PARSE2(4) \ else _UTF8_HEAD_PARSE2(5) \ + #if _UTF8_NUM_TAIL_BYTES_MAX >= 6 + else _UTF8_HEAD_PARSE2(6) + #endif +*/ + +#define _UTF8_HEAD_PARSE_MAX_3_BYTES \ + UInt32 val = c; \ + _UTF8_HEAD_PARSE2(1) \ + else _UTF8_HEAD_PARSE2(2) \ + else { numBytes = 3; val -= _UTF8_START(3); } + + +#define _UTF8_RANGE(n) (((UInt32)1) << ((n) * 5 + 6)) + + +#define START_POINT_FOR_SURROGATE 0x10000 + + +/* we use 128 bytes block in 16-bit BMP-PLANE to encode non-UTF-8 Escapes + Also we can use additional HIGH-PLANE (we use 21-bit points above 0x1f0000) + to simplify internal intermediate conversion in Linux: + RAW-UTF-8 <-> internal wchar_t utf-16 strings <-> RAW-UTF-UTF-8 +*/ + + +#if defined(_WCHART_IS_16BIT) + +#define UTF_ESCAPE_PLANE 0 + +#else + +/* +we can place 128 ESCAPE chars to + ef 80 - ee be 80 (3-bytes utf-8) : similar to WSL + ef ff - ee bf bf + +1f ef 80 - f7 be be 80 (4-bytes utf-8) : last 4-bytes utf-8 plane (out of Unicode) +1f ef ff - f7 be bf bf (4-bytes utf-8) : last 4-bytes utf-8 plane (out of Unicode) +*/ + +// #define UTF_ESCAPE_PLANE_HIGH (0x1f << 16) +// #define UTF_ESCAPE_PLANE UTF_ESCAPE_PLANE_HIGH +#define UTF_ESCAPE_PLANE 0 + +/* + if (UTF_FLAG__FROM_UTF8__USE_ESCAPE is set) + { + if (UTF_ESCAPE_PLANE is UTF_ESCAPE_PLANE_HIGH) + { + we can restore any 8-bit Escape from ESCAPE-PLANE-21 plane. + But ESCAPE-PLANE-21 point cannot be stored to utf-16 (7z archive) + So we still need a way to extract 8-bit Escapes and BMP-Escapes-8 + from same BMP-Escapes-16 stored in 7z. + And if we want to restore any 8-bit from 7z archive, + we still must use UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT for (utf-8 -> utf-16) + Also we need additional Conversions to tranform from utf-16 to utf-16-With-Escapes-21 + } + else (UTF_ESCAPE_PLANE == 0) + { + we must convert original 3-bytes utf-8 BMP-Escape point to sequence + of 3 BMP-Escape-16 points with UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + so we can extract original RAW-UTF-8 from UTFD-16 later. + } + } +*/ + +#endif + + + +#define UTF_ESCAPE_BASE 0xef00 + + +#ifdef UTF_ESCAPE_BASE +#define IS_ESCAPE_POINT(v, plane) (((v) & (UInt32)0xffffff80) == (plane) + UTF_ESCAPE_BASE + 0x80) +#endif + +#define IS_SURROGATE_POINT(v) (((v) & (UInt32)0xfffff800) == 0xd800) +#define IS_LOW_SURROGATE_POINT(v) (((v) & (UInt32)0xfffffC00) == 0xdc00) + + +#define _ERROR_UTF8_CHECK \ + { NonUtf = true; continue; } + +void CUtf8Check::Check_Buf(const char *src, size_t size) throw() +{ + Clear(); + // Byte maxByte = 0; + + for (;;) + { + if (size == 0) + break; + + const Byte c = (Byte)(*src++); + size--; + + if (c == 0) + { + ZeroChar = true; + continue; + } + + /* + if (c > maxByte) + maxByte = c; + */ + + if (c < 0x80) + continue; + + if (c < 0xc0 + 2)// it's limit for 0x140000 unicode codes : win32 compatibility + _ERROR_UTF8_CHECK + + unsigned numBytes; - // else _UTF8_HEAD_PARSE2(6) + UInt32 val = c; + _UTF8_HEAD_PARSE2(1) + else _UTF8_HEAD_PARSE2(2) + else _UTF8_HEAD_PARSE2(4) + else _UTF8_HEAD_PARSE2(5) + else + { + _ERROR_UTF8_CHECK + } + unsigned pos = 0; + do + { + if (pos == size) + break; + unsigned c2 = (Byte)src[pos]; + c2 -= 0x80; + if (c2 >= 0x40) + break; + val <<= 6; + val |= c2; + if (pos == 0) + if (val < (((unsigned)1 << 7) >> numBytes)) + break; + pos++; + } + while (--numBytes); + + if (numBytes != 0) + { + if (pos == size) + Truncated = true; + else + _ERROR_UTF8_CHECK + } + + #ifdef UTF_ESCAPE_BASE + if (IS_ESCAPE_POINT(val, 0)) + Escape = true; + #endif + + if (MaxHighPoint < val) + MaxHighPoint = val; + + if (IS_SURROGATE_POINT(val)) + SingleSurrogate = true; + + src += pos; + size -= pos; + } + + // MaxByte = maxByte; +} + +bool Check_UTF8_Buf(const char *src, size_t size, bool allowReduced) throw() +{ + CUtf8Check check; + check.Check_Buf(src, size); + return check.IsOK(allowReduced); +} + +/* +bool CheckUTF8_chars(const char *src, bool allowReduced) throw() +{ + CUtf8Check check; + check.CheckBuf(src, strlen(src)); + return check.IsOK(allowReduced); +} +*/ + +bool CheckUTF8_AString(const AString &s) throw() +{ + CUtf8Check check; + check.Check_AString(s); + return check.IsOK(); +} + + +/* bool CheckUTF8(const char *src, bool allowReduced) throw() { + // return Check_UTF8_Buf(src, strlen(src), allowReduced); + for (;;) { - Byte c = *src++; + const Byte c = (Byte)(*src++); if (c == 0) return true; if (c < 0x80) continue; - if (c < 0xC0) // (c < 0xC0 + 2) // if we support only optimal encoding chars + if (c < 0xC0 + 2 || c >= 0xf5) return false; unsigned numBytes; _UTF8_HEAD_PARSE else return false; - - UInt32 val = c; + unsigned pos = 0; + do { - Byte c2 = *src++; + Byte c2 = (Byte)(*src++); if (c2 < 0x80 || c2 >= 0xC0) return allowReduced && c2 == 0; val <<= 6; val |= (c2 - 0x80); + pos++; } while (--numBytes); - + + if (val < _UTF8_RANGE(pos - 1)) + return false; + if (val >= 0x110000) return false; } } +*/ + +// in case of UTF-8 error we have two ways: +// 21.01- : old : 0xfffd: REPLACEMENT CHARACTER : old version +// 21.02+ : new : 0xef00 + (c) : similar to WSL scheme for low symbols + +#define UTF_REPLACEMENT_CHAR 0xfffd + +#define UTF_ESCAPE(c) \ + ((flags & UTF_FLAG__FROM_UTF8__USE_ESCAPE) ? \ + UTF_ESCAPE_PLANE + UTF_ESCAPE_BASE + (c) : UTF_REPLACEMENT_CHAR) + +/* +#define _HARD_ERROR_UTF8 + { if (dest) dest[destPos] = (wchar_t)UTF_ESCAPE(c); \ + destPos++; ok = false; continue; } +*/ + +// we ignore utf errors, and don't change (ok) variable! + #define _ERROR_UTF8 \ - { if (dest) dest[destPos] = (wchar_t)0xFFFD; destPos++; ok = false; continue; } + { if (dest) dest[destPos] = (wchar_t)UTF_ESCAPE(c); \ + destPos++; continue; } -static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const char *srcLim) throw() +// we store UTF-16 in wchar_t strings. So we use surrogates for big unicode points: + +// for debug puposes only we can store UTF-32 in wchar_t: +// #define START_POINT_FOR_SURROGATE ((UInt32)0 - 1) + + +/* + WIN32 MultiByteToWideChar(CP_UTF8) emits 0xfffd point, if utf-8 error was found. + Ant it can emit single 0xfffd from 2 src bytes. + It doesn't emit single 0xfffd from 3-4 src bytes. + We can + 1) emit Escape point for each incorrect byte. So we can data recover later + 2) emit 0xfffd for each incorrect byte. + That scheme is similar to Escape scheme, but we emit 0xfffd + instead of each Escape point. + 3) emit single 0xfffd from 1-2 incorrect bytes, as WIN32 MultiByteToWideChar scheme +*/ + +static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const char *srcLim, unsigned flags) throw() { size_t destPos = 0; bool ok = true; for (;;) { - Byte c; if (src == srcLim) { *destLen = destPos; return ok; } - c = *src++; + + const Byte c = (Byte)(*src++); if (c < 0x80) { @@ -98,68 +358,127 @@ static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const destPos++; continue; } - if (c < 0xC0) + + if (c < 0xc0 + 2 + || c >= 0xf5) // it's limit for 0x140000 unicode codes : win32 compatibility + { _ERROR_UTF8 + } unsigned numBytes; - _UTF8_HEAD_PARSE - else - _ERROR_UTF8 - - UInt32 val = c; + _UTF8_HEAD_PARSE_MAX_3_BYTES + + unsigned pos = 0; do { - Byte c2; - if (src == srcLim) + if (src + pos == srcLim) break; - c2 = *src; - if (c2 < 0x80 || c2 >= 0xC0) + unsigned c2 = (Byte)src[pos]; + c2 -= 0x80; + if (c2 >= 0x40) break; - src++; val <<= 6; - val |= (c2 - 0x80); + val |= c2; + pos++; + if (pos == 1) + { + if (val < (((unsigned)1 << 7) >> numBytes)) + break; + if (numBytes == 2) + { + if (flags & UTF_FLAG__FROM_UTF8__SURROGATE_ERROR) + if ((val & (0xF800 >> 6)) == (0xd800 >> 6)) + break; + } + else if (numBytes == 3 && val >= (0x110000 >> 12)) + break; + } } while (--numBytes); if (numBytes != 0) + { + if ((flags & UTF_FLAG__FROM_UTF8__USE_ESCAPE) == 0) + { + // the following code to emit the 0xfffd chars as win32 Utf8 function. + // disable the folling line, if you need 0xfffd for each incorrect byte as in Escape mode + src += pos; + } + _ERROR_UTF8 + } + + /* + if (val < _UTF8_RANGE(pos - 1)) _ERROR_UTF8 + */ + + #ifdef UTF_ESCAPE_BASE + + if ((flags & UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT) + && IS_ESCAPE_POINT(val, 0)) + { + // We will emit 3 utf16-Escape-16-21 points from one Escape-16 point (3 bytes) + _ERROR_UTF8 + } + + #endif - if (val < 0x10000) + /* + We don't expect virtual Escape-21 points in UTF-8 stream. + And we don't check for Escape-21. + So utf8-Escape-21 will be converted to another 3 utf16-Escape-21 points. + Maybe we could convert virtual utf8-Escape-21 to one utf16-Escape-21 point in some cases? + */ + + if (val < START_POINT_FOR_SURROGATE) { + /* + if ((flags & UTF_FLAG__FROM_UTF8__SURROGATE_ERROR) + && IS_SURROGATE_POINT(val)) + { + // We will emit 3 utf16-Escape-16-21 points from one Surrogate-16 point (3 bytes) + _ERROR_UTF8 + } + */ if (dest) dest[destPos] = (wchar_t)val; destPos++; } else { - val -= 0x10000; - if (val >= 0x100000) + /* + if (val >= 0x110000) + { + // We will emit utf16-Escape-16-21 point from each source byte _ERROR_UTF8 + } + */ if (dest) { - dest[destPos + 0] = (wchar_t)(0xD800 + (val >> 10)); - dest[destPos + 1] = (wchar_t)(0xDC00 + (val & 0x3FF)); + dest[destPos + 0] = (wchar_t)(0xd800 - (0x10000 >> 10) + (val >> 10)); + dest[destPos + 1] = (wchar_t)(0xdc00 + (val & 0x3ff)); } destPos += 2; } + src += pos; } } -#define _UTF8_RANGE(n) (((UInt32)1) << ((n) * 5 + 6)) + #define _UTF8_HEAD(n, val) ((char)(_UTF8_START(n) + (val >> (6 * (n))))) #define _UTF8_CHAR(n, val) ((char)(0x80 + (((val) >> (6 * (n))) & 0x3F))) -static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim) +static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim, unsigned flags) { - size_t size = srcLim - src; + size_t size = (size_t)(srcLim - src); for (;;) { if (src == srcLim) return size; - UInt32 val = *src++; + UInt32 val = (UInt32)(*src++); if (val < 0x80) continue; @@ -170,15 +489,32 @@ static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim) continue; } - if (val >= 0xD800 && val < 0xDC00 && src != srcLim) + #ifdef UTF_ESCAPE_BASE + + #if UTF_ESCAPE_PLANE != 0 + if (flags & UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE) + if (IS_ESCAPE_POINT(val, UTF_ESCAPE_PLANE)) + continue; + #endif + + if (flags & UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE) + if (IS_ESCAPE_POINT(val, 0)) + continue; + + #endif + + if (IS_SURROGATE_POINT(val)) { - UInt32 c2 = *src; - if (c2 >= 0xDC00 && c2 < 0xE000) + // it's hack to UTF-8 encoding + + if (val < 0xdc00 && src != srcLim) { - src++; - size += 2; - continue; + const UInt32 c2 = (UInt32)*src; + if (c2 >= 0xdc00 && c2 < 0xe000) + src++; } + size += 2; + continue; } #ifdef _WCHART_IS_16BIT @@ -191,20 +527,26 @@ static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim) else if (val < _UTF8_RANGE(3)) size += 3; else if (val < _UTF8_RANGE(4)) size += 4; else if (val < _UTF8_RANGE(5)) size += 5; - else size += 6; + else + #if _UTF8_NUM_TAIL_BYTES_MAX >= 6 + size += 6; + #else + size += 3; + #endif #endif } } -static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim) + +static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim, unsigned flags) { for (;;) { if (src == srcLim) return dest; - UInt32 val = *src++; + UInt32 val = (UInt32)*src++; if (val < 0x80) { @@ -220,22 +562,57 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim continue; } - if (val >= 0xD800 && val < 0xDC00 && src != srcLim) - { - UInt32 c2 = *src; - if (c2 >= 0xDC00 && c2 < 0xE000) + #ifdef UTF_ESCAPE_BASE + + #if UTF_ESCAPE_PLANE != 0 + /* + if (wchar_t is 32-bit) + && (UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE is set) + && (point is virtual escape plane) + we extract 8-bit byte from virtual HIGH-ESCAPE PLANE. + */ + if (flags & UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE) + if (IS_ESCAPE_POINT(val, UTF_ESCAPE_PLANE)) { - src++; - val = (((val - 0xD800) << 10) | (c2 - 0xDC00)) + 0x10000; - dest[0] = _UTF8_HEAD(3, val); - dest[1] = _UTF8_CHAR(2, val); - dest[2] = _UTF8_CHAR(1, val); - dest[3] = _UTF8_CHAR(0, val); - dest += 4; + *dest++ = (char)(val); + continue; + } + #endif // UTF_ESCAPE_PLANE != 0 + + /* if (UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE is defined) + we extract 8-bit byte from BMP-ESCAPE PLANE. */ + + if (flags & UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE) + if (IS_ESCAPE_POINT(val, 0)) + { + *dest++ = (char)(val); continue; } - } + #endif // UTF_ESCAPE_BASE + + if (IS_SURROGATE_POINT(val)) + { + // it's hack to UTF-8 encoding + if (val < 0xdc00 && src != srcLim) + { + const UInt32 c2 = (UInt32)*src; + if (IS_LOW_SURROGATE_POINT(c2)) + { + src++; + val = (((val - 0xd800) << 10) | (c2 - 0xdc00)) + 0x10000; + dest[0] = _UTF8_HEAD(3, val); + dest[1] = _UTF8_CHAR(2, val); + dest[2] = _UTF8_CHAR(1, val); + dest[3] = _UTF8_CHAR(0, val); + dest += 4; + continue; + } + } + if (flags & UTF_FLAG__TO_UTF8__SURROGATE_ERROR) + val = UTF_REPLACEMENT_CHAR; // WIN32 function does it + } + #ifndef _WCHART_IS_16BIT if (val < _UTF8_RANGE(2)) #endif @@ -249,14 +626,25 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim #ifndef _WCHART_IS_16BIT - UInt32 b; + // we don't expect this case. so we can throw exception + // throw 20210407; + + char b; unsigned numBits; if (val < _UTF8_RANGE(3)) { numBits = 6 * 3; b = _UTF8_HEAD(3, val); } else if (val < _UTF8_RANGE(4)) { numBits = 6 * 4; b = _UTF8_HEAD(4, val); } else if (val < _UTF8_RANGE(5)) { numBits = 6 * 5; b = _UTF8_HEAD(5, val); } - else { numBits = 6 * 6; b = _UTF8_START(6); } - - *dest++ = (Byte)b; + #if _UTF8_NUM_TAIL_BYTES_MAX >= 6 + else { numBits = 6 * 6; b = (char)_UTF8_START(6); } + #else + else + { + val = UTF_REPLACEMENT_CHAR; + { numBits = 6 * 3; b = _UTF8_HEAD(3, val); } + } + #endif + + *dest++ = b; do { @@ -269,20 +657,207 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim } } -bool ConvertUTF8ToUnicode(const AString &src, UString &dest) +bool Convert_UTF8_Buf_To_Unicode(const char *src, size_t srcSize, UString &dest, unsigned flags) { dest.Empty(); size_t destLen = 0; - Utf8_To_Utf16(NULL, &destLen, src, src.Ptr(src.Len())); - bool res = Utf8_To_Utf16(dest.GetBuf((unsigned)destLen), &destLen, src, src.Ptr(src.Len())); + Utf8_To_Utf16(NULL, &destLen, src, src + srcSize, flags); + bool res = Utf8_To_Utf16(dest.GetBuf((unsigned)destLen), &destLen, src, src + srcSize, flags); dest.ReleaseBuf_SetEnd((unsigned)destLen); return res; } -void ConvertUnicodeToUTF8(const UString &src, AString &dest) +bool ConvertUTF8ToUnicode_Flags(const AString &src, UString &dest, unsigned flags) +{ + return Convert_UTF8_Buf_To_Unicode(src, src.Len(), dest, flags); +} + + +static +unsigned g_UTF8_To_Unicode_Flags = + UTF_FLAG__FROM_UTF8__USE_ESCAPE + #ifndef _WCHART_IS_16BIT + | UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + #ifdef _UTF8_RAW_NON_UTF8_SUPPORTED + | UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + #endif + #endif + ; + + +/* +bool ConvertUTF8ToUnicode_boolRes(const AString &src, UString &dest) +{ + return ConvertUTF8ToUnicode_Flags(src, dest, g_UTF8_To_Unicode_Flags); +} +*/ + +bool ConvertUTF8ToUnicode(const AString &src, UString &dest) { + return ConvertUTF8ToUnicode_Flags(src, dest, g_UTF8_To_Unicode_Flags); +} + +void Print_UString(const UString &a); + +void ConvertUnicodeToUTF8_Flags(const UString &src, AString &dest, unsigned flags) +{ + /* + if (src.Len()== 24) + throw "202104"; + */ dest.Empty(); - size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len())); - Utf16_To_Utf8(dest.GetBuf((unsigned)destLen), src, src.Ptr(src.Len())); + const size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len()), flags); + char *destStart = dest.GetBuf((unsigned)destLen); + const char *destEnd = Utf16_To_Utf8(destStart, src, src.Ptr(src.Len()), flags); dest.ReleaseBuf_SetEnd((unsigned)destLen); + // printf("\nlen = %d\n", src.Len()); + if (destLen != (size_t)(destEnd - destStart)) + { + /* + // dest.ReleaseBuf_SetEnd((unsigned)(destEnd - destStart)); + printf("\nlen = %d\n", (unsigned)destLen); + printf("\n(destEnd - destStart) = %d\n", (unsigned)(destEnd - destStart)); + printf("\n"); + // Print_UString(src); + printf("\n"); + // printf("\nlen = %d\n", destLen); + */ + throw 20210406; + } +} + + + +unsigned g_Unicode_To_UTF8_Flags = + // UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE + 0 + #ifndef _WIN32 + #ifdef _UTF8_RAW_NON_UTF8_SUPPORTED + | UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE + #else + | UTF_FLAG__TO_UTF8__SURROGATE_ERROR; + #endif + #endif + ; + +void ConvertUnicodeToUTF8(const UString &src, AString &dest) +{ + ConvertUnicodeToUTF8_Flags(src, dest, g_Unicode_To_UTF8_Flags); +} + +void Convert_Unicode_To_UTF8_Buf(const UString &src, CByteBuffer &dest) +{ + const unsigned flags = g_Unicode_To_UTF8_Flags; + dest.Free(); + const size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len()), flags); + dest.Alloc(destLen); + const char *destEnd = Utf16_To_Utf8((char *)(void *)(Byte *)dest, src, src.Ptr(src.Len()), flags); + if (destLen != (size_t)(destEnd - (char *)(void *)(Byte *)dest)) + throw 202104; } + +/* + +#ifndef _WIN32 +void Convert_UTF16_To_UTF32(const UString &src, UString &dest) +{ + dest.Empty(); + for (size_t i = 0; i < src.Len();) + { + wchar_t c = src[i++]; + if (c >= 0xd800 && c < 0xdc00 && i < src.Len()) + { + const wchar_t c2 = src[i]; + if (c2 >= 0xdc00 && c2 < 0x10000) + { + // printf("\nSurragate [%d]: %4x %4x -> ", i, (int)c, (int)c2); + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + // printf("%4x\n", (int)c); + i++; + } + } + dest += c; + } +} + +void Convert_UTF32_To_UTF16(const UString &src, UString &dest) +{ + dest.Empty(); + for (size_t i = 0; i < src.Len();) + { + wchar_t w = src[i++]; + if (w >= 0x10000 && w < 0x110000) + { + w -= 0x10000; + dest += (wchar_t)((unsigned)0xd800 + (((unsigned)w >> 10) & 0x3ff)); + w = 0xdc00 + (w & 0x3ff); + } + dest += w; + } +} + +bool UTF32_IsThere_BigPoint(const UString &src) +{ + for (size_t i = 0; i < src.Len();) + { + const UInt32 c = (UInt32)src[i++]; + if (c >= 0x110000) + return true; + } + return false; +} + +bool Unicode_IsThere_BmpEscape(const UString &src) +{ + for (size_t i = 0; i < src.Len();) + { + const UInt32 c = (UInt32)src[i++]; + if (IS_ESCAPE_POINT(c, 0)) + return true; + } + return false; +} + + +#endif + +bool Unicode_IsThere_Utf16SurrogateError(const UString &src) +{ + for (size_t i = 0; i < src.Len();) + { + const UInt32 val = (UInt32)src[i++]; + if (IS_SURROGATE_POINT(val)) + { + // it's hack to UTF-8 encoding + if (val >= 0xdc00 || i == src.Len()) + return true; + const UInt32 c2 = (UInt32)*src; + if (!IS_LOW_SURROGATE_POINT(c2)) + return true; + } + } + return false; +} +*/ + +#ifndef _WCHART_IS_16BIT + +void Convert_UnicodeEsc16_To_UnicodeEscHigh +#if UTF_ESCAPE_PLANE == 0 + (UString &) {} +#else + (UString &s) +{ + const unsigned len = s.Len(); + for (unsigned i = 0; i < len; i++) + { + wchar_t c = s[i]; + if (IS_ESCAPE_POINT(c, 0)) + { + c += UTF_ESCAPE_PLANE; + s.ReplaceOneCharAtPos(i, c); + } + } +} +#endif +#endif |