summaryrefslogtreecommitdiffstatshomepage
path: root/src/tools/srcclean.cpp
diff options
context:
space:
mode:
author Vas Crabb <vas@vastheman.com>2023-03-15 06:50:01 +1100
committer Vas Crabb <vas@vastheman.com>2023-03-15 06:50:01 +1100
commit64ccc49c219848535906743179768cebec37cbdd (patch)
tree7635a0e8d3079f5e43368456d3b2635f735a484e /src/tools/srcclean.cpp
parent57ec3ee4a166f1651dff7a44867a0d5d1582b295 (diff)
tools/srcclean.cpp: Escape 'high' Unicode characters found in strings rather than nuking them.
Diffstat (limited to 'src/tools/srcclean.cpp')
-rw-r--r--src/tools/srcclean.cpp30
1 files changed, 20 insertions, 10 deletions
diff --git a/src/tools/srcclean.cpp b/src/tools/srcclean.cpp
index 77a6381936f..adcd52541b6 100644
--- a/src/tools/srcclean.cpp
+++ b/src/tools/srcclean.cpp
@@ -119,6 +119,7 @@ protected:
static constexpr char32_t CYRILLIC_SUPPLEMENT_LAST = 0x0000'052fU;
static constexpr char32_t LINE_SEPARATOR = 0x0000'2028U;
static constexpr char32_t PARAGRAPH_SEPARATOR = 0x0000'2029U;
+ static constexpr char32_t SUPPLEMENTARY_FIRST = 0x0001'0000U;
template <typename OutputIt>
cleaner_base(OutputIt &&output, newline newline_mode, unsigned tab_width);
@@ -138,7 +139,6 @@ private:
static constexpr char32_t NONCHARACTER_LAST = 0x0000'fdefU;
static constexpr char32_t ZERO_WIDTH_NB_SPACE = 0x0000'feffU;
static constexpr char32_t REPLACEMENT_CHARACTER = 0x0000'fffdU;
- static constexpr char32_t SUPPLEMENTARY_FIRST = 0x0001'0000U;
static constexpr char32_t SUPPLEMENTARY_LAST = 0x0010'ffffU;
static constexpr char32_t CODE_LENGTH_THRESHOLDS[6]{
@@ -1016,14 +1016,8 @@ void cpp_cleaner::output_character(char32_t ch)
break;
case parse_state::COMMENT:
case parse_state::LINE_COMMENT:
- break;
case parse_state::STRING_CONSTANT:
case parse_state::CHARACTER_CONSTANT:
- if (CYRILLIC_SUPPLEMENT_LAST < ch)
- {
- ++m_non_ascii;
- ch = QUESTION_MARK;
- }
break;
}
@@ -1323,9 +1317,25 @@ void cpp_cleaner::process_text(char32_t ch)
output_character(char32_t(std::uint8_t('v')));
break;
default:
- output_character(ch);
- if (!m_escape && (((parse_state::STRING_CONSTANT == m_parse_state) ? DOUBLE_QUOTE : SINGLE_QUOTE) == ch))
- m_parse_state = parse_state::DEFAULT;
+ if (CYRILLIC_SUPPLEMENT_LAST >= ch)
+ {
+ output_character(ch);
+ if (!m_escape && (((parse_state::STRING_CONSTANT == m_parse_state) ? DOUBLE_QUOTE : SINGLE_QUOTE) == ch))
+ m_parse_state = parse_state::DEFAULT;
+ }
+ else
+ {
+ ++m_non_ascii;
+ bool const supplementary(SUPPLEMENTARY_FIRST <= ch);
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t(supplementary ? 'U' : 'u')));
+ for (int shift = supplementary ? 28 : 12; 0 <= shift; shift -= 4)
+ {
+ std::uint8_t const digit((ch >> shift) & 0x0fU);
+ output_character(std::uint8_t((10U > digit) ? '0' : ('a' - 10)) + digit);
+ }
+ }
}
m_escape = (BACKSLASH == ch) && !m_escape;
}