diff options
author | 2023-05-04 02:41:16 +1000 | |
---|---|---|
committer | 2023-05-04 02:41:16 +1000 | |
commit | a504bde3a7462b54fafd5cfc2f52e58d0f3218e1 (patch) | |
tree | 8b3108d572b1a0873a6cdbb4e8af1f17179545c3 /3rdparty/lzma/CPP/Common/StringConvert.cpp | |
parent | befb9bf4a8cfdb2b693a32e32535b5eac522c5d8 (diff) |
3rdparty/lzma: Updated to LZMA SDK version 22.01
Diffstat (limited to '3rdparty/lzma/CPP/Common/StringConvert.cpp')
-rw-r--r-- | 3rdparty/lzma/CPP/Common/StringConvert.cpp | 474 |
1 files changed, 459 insertions, 15 deletions
diff --git a/3rdparty/lzma/CPP/Common/StringConvert.cpp b/3rdparty/lzma/CPP/Common/StringConvert.cpp index 30d95aa7b56..c0bde0fa121 100644 --- a/3rdparty/lzma/CPP/Common/StringConvert.cpp +++ b/3rdparty/lzma/CPP/Common/StringConvert.cpp @@ -5,9 +5,18 @@ #include "StringConvert.h" #ifndef _WIN32 +// #include <stdio.h> #include <stdlib.h> #endif +#if !defined(_WIN32) || defined(ENV_HAVE_LOCALE) +#include "UTFConvert.h" +#endif + +#ifdef ENV_HAVE_LOCALE +#include <locale.h> +#endif + static const char k_DefultChar = '_'; #ifdef _WIN32 @@ -71,7 +80,7 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) d[i] = 0; dest.ReleaseBuf_SetLen(i); */ - unsigned len = MultiByteToWideChar(codePage, 0, src, src.Len(), NULL, 0); + unsigned len = (unsigned)MultiByteToWideChar(codePage, 0, src, (int)src.Len(), NULL, 0); if (len == 0) { if (GetLastError() != 0) @@ -79,7 +88,7 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) } else { - len = MultiByteToWideChar(codePage, 0, src, src.Len(), dest.GetBuf(len), len); + len = (unsigned)MultiByteToWideChar(codePage, 0, src, (int)src.Len(), dest.GetBuf(len), (int)len); if (len == 0) throw 282228; dest.ReleaseBuf_SetEnd(len); @@ -175,7 +184,7 @@ static void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT co } */ - unsigned len = WideCharToMultiByte(codePage, 0, src, src.Len(), NULL, 0, NULL, NULL); + unsigned len = (unsigned)WideCharToMultiByte(codePage, 0, src, (int)src.Len(), NULL, 0, NULL, NULL); if (len == 0) { if (GetLastError() != 0) @@ -186,8 +195,8 @@ static void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT co BOOL defUsed = FALSE; bool isUtf = (codePage == CP_UTF8 || codePage == CP_UTF7); // defaultChar = defaultChar; - len = WideCharToMultiByte(codePage, 0, src, src.Len(), - dest.GetBuf(len), len, + len = (unsigned)WideCharToMultiByte(codePage, 0, src, (int)src.Len(), + dest.GetBuf(len), (int)len, (isUtf ? NULL : &defaultChar), (isUtf ? NULL : &defUsed) ); @@ -213,23 +222,137 @@ AString SystemStringToOemString(const CSysString &src) #endif */ -#else +#else // _WIN32 + +// #include <stdio.h> +/* + if (wchar_t is 32-bit (#if WCHAR_MAX > 0xffff), + and utf-8 string contains big unicode character > 0xffff), + then we still use 16-bit surrogate pair in UString. + It simplifies another code where utf-16 encoding is used. + So we use surrogate-conversion code only in is file. +*/ + +/* + mbstowcs() returns error if there is error in utf-8 stream, + mbstowcs() returns error if there is single surrogates point (d800-dfff) in utf-8 stream +*/ + +/* +static void MultiByteToUnicodeString2_Native(UString &dest, const AString &src) +{ + dest.Empty(); + if (src.IsEmpty()) + return; + + const size_t limit = ((size_t)src.Len() + 1) * 2; + wchar_t *d = dest.GetBuf((unsigned)limit); + const size_t len = mbstowcs(d, src, limit); + if (len != (size_t)-1) + { + dest.ReleaseBuf_SetEnd((unsigned)len); + return; + } + dest.ReleaseBuf_SetEnd(0); +} +*/ + +bool g_ForceToUTF8 = true; // false; -void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT /* codePage */) +void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) { dest.Empty(); if (src.IsEmpty()) return; - size_t limit = ((size_t)src.Len() + 1) * 2; + if (codePage == CP_UTF8 || g_ForceToUTF8) + { + ConvertUTF8ToUnicode(src, dest); + return; + } + + const size_t limit = ((size_t)src.Len() + 1) * 2; wchar_t *d = dest.GetBuf((unsigned)limit); - size_t len = mbstowcs(d, src, limit); + const size_t len = mbstowcs(d, src, limit); if (len != (size_t)-1) { dest.ReleaseBuf_SetEnd((unsigned)len); + + #if WCHAR_MAX > 0xffff + d = dest.GetBuf(); + for (size_t i = 0;; i++) + { + // wchar_t c = dest[i]; + wchar_t c = d[i]; + if (c == 0) + break; + if (c >= 0x10000 && c < 0x110000) + { + /* + c -= 0x10000; + unsigned c0 = 0xd800 + ((c >> 10) & 0x3FF); + dest.ReplaceOneCharAtPos(i, c0); + i++; + c = 0xdc00 + (c & 0x3FF); + dest.Insert_wchar_t(i, c); + */ + UString temp = d + i; + + for (size_t t = 0;; t++) + { + wchar_t w = temp[t]; + if (w == 0) + break; + if (i == limit) + break; // unexpected error + if (w >= 0x10000 && w < 0x110000) + { + if (i + 1 == limit) + break; // unexpected error + w -= 0x10000; + d[i++] = (unsigned)0xd800 + (((unsigned)w >> 10) & 0x3FF); + w = 0xdc00 + (w & 0x3FF); + } + d[i++] = w; + } + dest.ReleaseBuf_SetEnd((unsigned)i); + } + } + + #endif + + /* + printf("\nMultiByteToUnicodeString2 (%d) %s\n", (int)src.Len(), src.Ptr()); + printf("char: "); + for (unsigned i = 0; i < src.Len(); i++) + printf (" %02x", (int)(Byte)src[i]); + printf("\n"); + printf("\n-> (%d) %ls\n", (int)dest.Len(), dest.Ptr()); + printf("wchar_t: "); + for (unsigned i = 0; i < dest.Len(); i++) + { + printf (" %02x", (int)dest[i]); + } + printf("\n"); + */ + return; } + + /* if there is mbstowcs() error, we have two ways: + + 1) change 0x80+ characters to some character: '_' + in that case we lose data, but we have correct UString() + and that scheme can show errors to user in early stages, + when file converted back to mbs() cannot be found + + 2) transfer bad characters in some UTF-16 range. + it can be non-original Unicode character. + but later we still can restore original character. + */ + + // printf("\nmbstowcs ERROR !!!!!! s=%s\n", src.Ptr()); { unsigned i; const char *s = (const char *)src; @@ -238,6 +361,8 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT /* codePa Byte c = (Byte)s[i]; if (c == 0) break; + // we can use ascii compatibilty character '_' + // if (c > 0x7F) c = '_'; // we replace "bad: character d[i++] = (wchar_t)c; } d[i] = 0; @@ -245,43 +370,131 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT /* codePa } } -static void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT /* codePage */, char defaultChar, bool &defaultCharWasUsed) +static void UnicodeStringToMultiByte2_Native(AString &dest, const UString &src) { dest.Empty(); - defaultCharWasUsed = false; if (src.IsEmpty()) return; - size_t limit = ((size_t)src.Len() + 1) * 6; + const size_t limit = ((size_t)src.Len() + 1) * 6; char *d = dest.GetBuf((unsigned)limit); - size_t len = wcstombs(d, src, limit); + + const size_t len = wcstombs(d, src, limit); + if (len != (size_t)-1) { dest.ReleaseBuf_SetEnd((unsigned)len); return; } + dest.ReleaseBuf_SetEnd(0); +} + + +static void UnicodeStringToMultiByte2(AString &dest, const UString &src2, UINT codePage, char defaultChar, bool &defaultCharWasUsed) +{ + // if (codePage == 1234567) // for debug purposes + if (codePage == CP_UTF8 || g_ForceToUTF8) + { + defaultCharWasUsed = false; + ConvertUnicodeToUTF8(src2, dest); + return; + } + + UString src = src2; + #if WCHAR_MAX > 0xffff + { + src.Empty(); + for (unsigned i = 0; i < src2.Len();) + { + wchar_t c = src2[i]; + if (c >= 0xd800 && c < 0xdc00 && i + 1 != src2.Len()) + { + const wchar_t c2 = src2[i + 1]; + if (c2 >= 0xdc00 && c2 < 0x10000) + { + // printf("\nSurragate [%d]: %4x %4x -> ", i, (int)c, (int)c2); + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + // printf("%4x\n", (int)c); + i++; + } + } + src += c; + i++; + } + } + #endif + + dest.Empty(); + defaultCharWasUsed = false; + if (src.IsEmpty()) + return; + + const size_t len = wcstombs(NULL, src, 0); + + if (len != (size_t)-1) + { + const unsigned limit = ((unsigned)len); + if (limit == len) + { + char *d = dest.GetBuf(limit); + + /* + { + printf("\nwcstombs; len = %d %ls \n", (int)src.Len(), src.Ptr()); + for (unsigned i = 0; i < src.Len(); i++) + printf (" %02x", (int)src[i]); + printf("\n"); + printf("\ndest Limit = %d \n", limit); + } + */ + + const size_t len2 = wcstombs(d, src, len + 1); + + if (len2 != (size_t)-1 && len2 <= limit) + { + /* + printf("\nOK : destLen = %d : %s\n", (int)len, dest.Ptr()); + for (unsigned i = 0; i < len2; i++) + printf(" %02x", (int)(Byte)dest[i]); + printf("\n"); + */ + dest.ReleaseBuf_SetEnd((unsigned)len2); + return; + } + } + } { const wchar_t *s = (const wchar_t *)src; + char *d = dest.GetBuf(src.Len()); + unsigned i; for (i = 0;;) { wchar_t c = s[i]; if (c == 0) break; - if (c >= 0x100) + if (c >= + 0x100 + // 0x80 + ) { c = defaultChar; defaultCharWasUsed = true; } + d[i++] = (char)c; } d[i] = 0; dest.ReleaseBuf_SetLen(i); + /* + printf("\nUnicodeStringToMultiByte2; len = %d \n", (int)src.Len()); + printf("ERROR: %s\n", dest.Ptr()); + */ } } -#endif +#endif // _WIN32 UString MultiByteToUnicodeString(const AString &src, UINT codePage) @@ -291,6 +504,12 @@ UString MultiByteToUnicodeString(const AString &src, UINT codePage) return dest; } +UString MultiByteToUnicodeString(const char *src, UINT codePage) +{ + return MultiByteToUnicodeString(AString(src), codePage); +} + + void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT codePage) { bool defaultCharWasUsed; @@ -311,3 +530,228 @@ AString UnicodeStringToMultiByte(const UString &src, UINT codePage) UnicodeStringToMultiByte2(dest, src, codePage, k_DefultChar, defaultCharWasUsed); return dest; } + + + + + +#ifdef _WIN32 +#define U_to_A(a, b, c) UnicodeStringToMultiByte2 +// #define A_to_U(a, b, c) MultiByteToUnicodeString2 +#else +// void MultiByteToUnicodeString2_Native(UString &dest, const AString &src); +#define U_to_A(a, b, c) UnicodeStringToMultiByte2_Native(a, b) +// #define A_to_U(a, b, c) MultiByteToUnicodeString2_Native(a, b) +#endif + +#if !defined(_WIN32) || defined(ENV_HAVE_LOCALE) + +bool IsNativeUTF8() +{ + UString u; + AString a, a2; + // for (unsigned c = 0x80; c < (UInt32)0x10000; c += (c >> 9) + 1) + for (unsigned c = 0x80; c < (UInt32)0xD000; c += (c >> 2) + 1) + { + u.Empty(); + u += (wchar_t)c; + /* + if (Unicode_Is_There_Utf16SurrogateError(u)) + continue; + #ifndef _WIN32 + if (Unicode_Is_There_BmpEscape(u)) + continue; + #endif + */ + ConvertUnicodeToUTF8(u, a); + U_to_A(a2, u, CP_OEMCP); + if (a != a2) + return false; + } + return true; +} + +#endif + + +#ifdef ENV_HAVE_LOCALE + +const char *GetLocale(void) +{ + #ifdef ENV_HAVE_LOCALE + // printf("\n\nsetlocale(LC_CTYPE, NULL) : return : "); + const char *s = setlocale(LC_CTYPE, NULL); + if (!s) + { + // printf("[NULL]\n"); + s = "C"; + } + else + { + // ubuntu returns "C" after program start + // printf("\"%s\"\n", s); + } + return s; + #elif defined(LOCALE_IS_UTF8) + return "utf8"; + #else + return "C"; + #endif +} + +#ifdef _WIN32 + static void Set_ForceToUTF8(bool) {} +#else + static void Set_ForceToUTF8(bool val) { g_ForceToUTF8 = val; } +#endif + +static bool Is_Default_Basic_Locale(const char *locale) +{ + const AString a (locale); + if (a.IsEqualTo_Ascii_NoCase("") + || a.IsEqualTo_Ascii_NoCase("C") + || a.IsEqualTo_Ascii_NoCase("POSIX")) + return true; + return false; +} + +static bool Is_Default_Basic_Locale() +{ + return Is_Default_Basic_Locale(GetLocale()); +} + + +void MY_SetLocale() +{ + #ifdef ENV_HAVE_LOCALE + /* + { + const char *s = GetLocale(); + printf("\nGetLocale() : returned : \"%s\"\n", s); + } + */ + + unsigned start = 0; + // unsigned lim = 0; + unsigned lim = 3; + + /* + #define MY_SET_LOCALE_FLAGS__FROM_ENV 1 + #define MY_SET_LOCALE_FLAGS__TRY_UTF8 2 + + unsigned flags = + MY_SET_LOCALE_FLAGS__FROM_ENV | + MY_SET_LOCALE_FLAGS__TRY_UTF8 + + if (flags != 0) + { + if (flags & MY_SET_LOCALE_FLAGS__FROM_ENV) + lim = (flags & MY_SET_LOCALE_FLAGS__TRY_UTF8) ? 3 : 1; + else + { + start = 1; + lim = 2; + } + } + */ + + for (unsigned i = start; i < lim; i++) + { + /* + man7: "If locale is an empty string, "", each part of the locale that + should be modified is set according to the environment variables. + for glibc: glibc, first from the user's environment variables: + 1) the environment variable LC_ALL, + 2) environment variable with the same name as the category (see the + 3) the environment variable LANG + The locale "C" or "POSIX" is a portable locale; it exists on all conforming systems. + + for WIN32 : MSDN : + Sets the locale to the default, which is the user-default + ANSI code page obtained from the operating system. + The locale name is set to the value returned by GetUserDefaultLocaleName. + The code page is set to the value returned by GetACP + */ + const char *newLocale = ""; + + #ifdef __APPLE__ + + /* look also CFLocale + there is no C.UTF-8 in macos + macos has UTF-8 locale only with some language like en_US.UTF-8 + what is best way to set UTF-8 locale in macos? */ + if (i == 1) + newLocale = "en_US.UTF-8"; + + /* file open with non-utf8 sequencies return + #define EILSEQ 92 // "Illegal byte sequence" + */ +#else + // newLocale = "C"; + if (i == 1) + { + newLocale = "C.UTF-8"; // main UTF-8 locale in ubuntu + // newLocale = ".utf8"; // supported in new Windows 10 build 17134 (April 2018 Update), the Universal C Runtime + // newLocale = "en_US.utf8"; // supported by ubuntu ? + // newLocale = "en_US.UTF-8"; + /* setlocale() in ubuntu allows locales with minor chracter changes in strings + "en_US.UTF-8" / "en_US.utf8" */ + } + +#endif + + // printf("\nsetlocale(LC_ALL, \"%s\") : returned: ", newLocale); + + // const char *s = + setlocale(LC_ALL, newLocale); + + /* + if (!s) + printf("NULL: can't set locale"); + else + printf("\"%s\"\n", s); + */ + + // request curent locale of program + const char *locale = GetLocale(); + if (locale) + { + AString a (locale); + a.MakeLower_Ascii(); + // if (a.Find("utf") >= 0) + { + if (IsNativeUTF8()) + { + Set_ForceToUTF8(true); + return; + } + } + if (!Is_Default_Basic_Locale(locale)) + { + // if there is some non-default and non-utf locale, we want to use it + break; // comment it for debug + } + } + } + + if (IsNativeUTF8()) + { + Set_ForceToUTF8(true); + return; + } + + if (Is_Default_Basic_Locale()) + { + Set_ForceToUTF8(true); + return; + } + + Set_ForceToUTF8(false); + + #elif defined(LOCALE_IS_UTF8) + // assume LC_CTYPE="utf8" + #else + // assume LC_CTYPE="C" + #endif +} +#endif |