From 64ccc49c219848535906743179768cebec37cbdd Mon Sep 17 00:00:00 2001 From: Vas Crabb Date: Wed, 15 Mar 2023 06:50:01 +1100 Subject: tools/srcclean.cpp: Escape 'high' Unicode characters found in strings rather than nuking them. --- src/tools/srcclean.cpp | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/tools/srcclean.cpp b/src/tools/srcclean.cpp index 77a6381936f..adcd52541b6 100644 --- a/src/tools/srcclean.cpp +++ b/src/tools/srcclean.cpp @@ -119,6 +119,7 @@ protected: static constexpr char32_t CYRILLIC_SUPPLEMENT_LAST = 0x0000'052fU; static constexpr char32_t LINE_SEPARATOR = 0x0000'2028U; static constexpr char32_t PARAGRAPH_SEPARATOR = 0x0000'2029U; + static constexpr char32_t SUPPLEMENTARY_FIRST = 0x0001'0000U; template cleaner_base(OutputIt &&output, newline newline_mode, unsigned tab_width); @@ -138,7 +139,6 @@ private: static constexpr char32_t NONCHARACTER_LAST = 0x0000'fdefU; static constexpr char32_t ZERO_WIDTH_NB_SPACE = 0x0000'feffU; static constexpr char32_t REPLACEMENT_CHARACTER = 0x0000'fffdU; - static constexpr char32_t SUPPLEMENTARY_FIRST = 0x0001'0000U; static constexpr char32_t SUPPLEMENTARY_LAST = 0x0010'ffffU; static constexpr char32_t CODE_LENGTH_THRESHOLDS[6]{ @@ -1016,14 +1016,8 @@ void cpp_cleaner::output_character(char32_t ch) break; case parse_state::COMMENT: case parse_state::LINE_COMMENT: - break; case parse_state::STRING_CONSTANT: case parse_state::CHARACTER_CONSTANT: - if (CYRILLIC_SUPPLEMENT_LAST < ch) - { - ++m_non_ascii; - ch = QUESTION_MARK; - } break; } @@ -1323,9 +1317,25 @@ void cpp_cleaner::process_text(char32_t ch) output_character(char32_t(std::uint8_t('v'))); break; default: - output_character(ch); - if (!m_escape && (((parse_state::STRING_CONSTANT == m_parse_state) ? DOUBLE_QUOTE : SINGLE_QUOTE) == ch)) - m_parse_state = parse_state::DEFAULT; + if (CYRILLIC_SUPPLEMENT_LAST >= ch) + { + output_character(ch); + if (!m_escape && (((parse_state::STRING_CONSTANT == m_parse_state) ? DOUBLE_QUOTE : SINGLE_QUOTE) == ch)) + m_parse_state = parse_state::DEFAULT; + } + else + { + ++m_non_ascii; + bool const supplementary(SUPPLEMENTARY_FIRST <= ch); + if (!m_escape) + output_character(BACKSLASH); + output_character(char32_t(std::uint8_t(supplementary ? 'U' : 'u'))); + for (int shift = supplementary ? 28 : 12; 0 <= shift; shift -= 4) + { + std::uint8_t const digit((ch >> shift) & 0x0fU); + output_character(std::uint8_t((10U > digit) ? '0' : ('a' - 10)) + digit); + } + } } m_escape = (BACKSLASH == ch) && !m_escape; } -- cgit v1.2.3