summaryrefslogtreecommitdiffstats
path: root/src/tools/srcclean.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/tools/srcclean.cpp')
-rw-r--r--src/tools/srcclean.cpp304
1 files changed, 302 insertions, 2 deletions
diff --git a/src/tools/srcclean.cpp b/src/tools/srcclean.cpp
index 7f52c3c6feb..052d2a37334 100644
--- a/src/tools/srcclean.cpp
+++ b/src/tools/srcclean.cpp
@@ -38,6 +38,9 @@
may have spacing adjusted in a way that affects behaviour when
uncommented
+ Known JSON limitations:
+ * Doesn't detect invalid numbers or literals
+
Known XML limitations:
* No special handling for CDATA
* No special handling for processing instructions
@@ -102,14 +105,20 @@ protected:
static constexpr char32_t HORIZONTAL_TAB = 0x0000'0009U;
static constexpr char32_t LINE_FEED = 0x0000'000aU;
static constexpr char32_t VERTICAL_TAB = 0x0000'000bU;
+ static constexpr char32_t FORM_FEED = 0x0000'000cU;
+ static constexpr char32_t C0_CONTROL_LAST = 0x0000'001fU;
static constexpr char32_t SPACE = 0x0000'0020U;
static constexpr char32_t DOUBLE_QUOTE = 0x0000'0022U;
static constexpr char32_t SINGLE_QUOTE = 0x0000'0027U;
+ static constexpr char32_t ASTERISK = 0x0000'002aU;
static constexpr char32_t HYPHEN_MINUS = 0x0000'002dU;
+ static constexpr char32_t SLASH = 0x0000'002fU;
static constexpr char32_t QUESTION_MARK = 0x0000'003fU;
static constexpr char32_t BACKSLASH = 0x0000'005cU;
static constexpr char32_t BASIC_LATIN_LAST = 0x0000'007fU;
static constexpr char32_t CYRILLIC_SUPPLEMENT_LAST = 0x0000'052fU;
+ static constexpr char32_t LINE_SEPARATOR = 0x0000'2028U;
+ static constexpr char32_t PARAGRAPH_SEPARATOR = 0x0000'2029U;
template <typename OutputIt>
cleaner_base(OutputIt &&output, newline newline_mode, unsigned tab_width);
@@ -825,8 +834,6 @@ protected:
void output_character(char32_t ch);
private:
- static constexpr char32_t ASTERISK = 0x0000'002aU;
- static constexpr char32_t SLASH = 0x0000'002fU;
static constexpr char32_t UPPERCASE_FIRST = 0x0000'0041U;
static constexpr char32_t UPPERCASE_B = 0x0000'0042U;
static constexpr char32_t UPPERCASE_X = 0x0000'0058U;
@@ -1741,6 +1748,289 @@ void lua_cleaner::process_long_string_constant(char32_t ch)
/***************************************************************************
+ JSON DATA CLEANER CLASS
+***************************************************************************/
+
+class json_cleaner : public cleaner_base
+{
+public:
+ template <typename OutputIt>
+ json_cleaner(OutputIt &&output, newline newline_mode, unsigned tab_width);
+
+ virtual bool affected() const override;
+ virtual void summarise(std::ostream &os) const override;
+
+protected:
+ void output_character(char32_t ch);
+
+private:
+ enum class parse_state
+ {
+ DEFAULT,
+ COMMENT,
+ LINE_COMMENT,
+ STRING_CONSTANT
+ };
+
+ virtual void process_characters(char32_t const *begin, char32_t const *end) override;
+ virtual void input_complete() override;
+
+ void process_default(char32_t ch);
+ void process_comment(char32_t ch);
+ void process_line_comment(char32_t ch);
+ void process_text(char32_t ch);
+
+ parse_state m_parse_state = parse_state::DEFAULT;
+ std::uint64_t m_input_line = 1U;
+ bool m_escape = false;
+ std::uint64_t m_comment_line = 0U;
+
+ std::uint64_t m_tabs_escaped = 0U;
+ std::uint64_t m_newlines_escaped = 0U;
+ std::uint64_t m_form_feeds_escaped = 0U;
+ std::uint64_t m_line_separators_escaped = 0U;
+ std::uint64_t m_paragraph_separators_escaped = 0U;
+ std::uint64_t m_c0_control_escaped = 0U;
+ std::uint64_t m_non_ascii = 0U;
+};
+
+
+template <typename OutputIt>
+json_cleaner::json_cleaner(
+ OutputIt &&output,
+ newline newline_mode,
+ unsigned tab_width)
+ : cleaner_base(std::forward<OutputIt>(output), newline_mode, tab_width)
+{
+}
+
+
+bool json_cleaner::affected() const
+{
+ return
+ cleaner_base::affected() ||
+ m_tabs_escaped ||
+ m_newlines_escaped ||
+ m_form_feeds_escaped ||
+ m_line_separators_escaped ||
+ m_paragraph_separators_escaped ||
+ m_c0_control_escaped ||
+ m_non_ascii;
+}
+
+
+void json_cleaner::summarise(std::ostream &os) const
+{
+ cleaner_base::summarise(os);
+ if (m_tabs_escaped)
+ util::stream_format(os, "%1$u tab(s) escaped\n", m_tabs_escaped);
+ if (m_newlines_escaped)
+ util::stream_format(os, "%1$u line feed(s) escaped\n", m_newlines_escaped);
+ if (m_form_feeds_escaped)
+ util::stream_format(os, "%1$u form feed(s) escaped\n", m_form_feeds_escaped);
+ if (m_line_separators_escaped)
+ util::stream_format(os, "%1$u line separator(s) escaped\n", m_line_separators_escaped);
+ if (m_paragraph_separators_escaped)
+ util::stream_format(os, "%1$u paragraph separator(s) escaped\n", m_paragraph_separators_escaped);
+ if (m_c0_control_escaped)
+ util::stream_format(os, "%1$u C0 control character(s) escaped\n", m_c0_control_escaped);
+ if (m_non_ascii)
+ util::stream_format(os, "%1$u non-ASCII character(s) replaced\n", m_non_ascii);
+}
+
+
+void json_cleaner::output_character(char32_t ch)
+{
+ switch (m_parse_state)
+ {
+ case parse_state::DEFAULT:
+ if (BASIC_LATIN_LAST < ch)
+ {
+ ++m_non_ascii;
+ ch = QUESTION_MARK;
+ }
+ break;
+ case parse_state::COMMENT:
+ case parse_state::LINE_COMMENT:
+ case parse_state::STRING_CONSTANT:
+ break;
+ }
+
+ cleaner_base::output_character(ch);
+}
+
+
+void json_cleaner::process_characters(char32_t const *begin, char32_t const *end)
+{
+ while (begin != end)
+ {
+ char32_t const ch(*begin++);
+ switch (m_parse_state)
+ {
+ case parse_state::DEFAULT:
+ process_default(ch);
+ break;
+ case parse_state::COMMENT:
+ process_comment(ch);
+ break;
+ case parse_state::LINE_COMMENT:
+ process_line_comment(ch);
+ break;
+ case parse_state::STRING_CONSTANT:
+ process_text(ch);
+ break;
+ }
+
+ if (LINE_FEED == ch)
+ ++m_input_line;
+ }
+}
+
+
+void json_cleaner::input_complete()
+{
+ switch (m_parse_state)
+ {
+ case parse_state::COMMENT:
+ throw std::runtime_error(util::string_format("unterminated multi-line comment beginning on line %1$u", m_comment_line));
+ case parse_state::STRING_CONSTANT:
+ throw std::runtime_error(util::string_format("unterminated string literal on line %1$u", m_input_line));
+ default:
+ break;
+ }
+}
+
+
+void json_cleaner::process_default(char32_t ch)
+{
+ switch (ch)
+ {
+ case DOUBLE_QUOTE:
+ m_parse_state = parse_state::STRING_CONSTANT;
+ break;
+ case ASTERISK:
+ if (m_escape)
+ {
+ m_parse_state = parse_state::COMMENT;
+ m_comment_line = m_input_line;
+ set_tab_limit();
+ }
+ break;
+ case SLASH:
+ if (m_escape)
+ m_parse_state = parse_state::LINE_COMMENT;
+ break;
+ }
+ m_escape = (SLASH == ch) && !m_escape;
+ output_character(ch);
+}
+
+
+void json_cleaner::process_comment(char32_t ch)
+{
+ switch (ch)
+ {
+ case SLASH:
+ if (m_escape)
+ {
+ m_parse_state = parse_state::DEFAULT;
+ m_comment_line = 0U;
+ output_character(ch);
+ reset_tab_limit();
+ }
+ else
+ {
+ output_character(ch);
+ }
+ m_escape = false;
+ break;
+ default:
+ m_escape = ASTERISK == ch;
+ output_character(ch);
+ }
+}
+
+
+void json_cleaner::process_line_comment(char32_t ch)
+{
+ switch (ch)
+ {
+ case LINE_FEED:
+ m_parse_state = parse_state::DEFAULT;
+ [[fallthrough]];
+ default:
+ output_character(ch);
+ }
+}
+
+
+void json_cleaner::process_text(char32_t ch)
+{
+ switch (ch)
+ {
+ case HORIZONTAL_TAB:
+ ++m_tabs_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('t')));
+ break;
+ case LINE_FEED:
+ ++m_newlines_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('n')));
+ break;
+ case FORM_FEED:
+ ++m_form_feeds_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('f')));
+ break;
+ case LINE_SEPARATOR:
+ ++m_line_separators_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('u')));
+ output_character(char32_t(std::uint8_t('2')));
+ output_character(char32_t(std::uint8_t('0')));
+ output_character(char32_t(std::uint8_t('2')));
+ output_character(char32_t(std::uint8_t('8')));
+ break;
+ case PARAGRAPH_SEPARATOR:
+ ++m_paragraph_separators_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('u')));
+ output_character(char32_t(std::uint8_t('2')));
+ output_character(char32_t(std::uint8_t('0')));
+ output_character(char32_t(std::uint8_t('2')));
+ output_character(char32_t(std::uint8_t('9')));
+ break;
+ default:
+ if (C0_CONTROL_LAST >= ch)
+ {
+ ++m_c0_control_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('u')));
+ output_character(char32_t(std::uint8_t('0')));
+ output_character(char32_t(std::uint8_t('0')));
+ output_character(char32_t(std::uint8_t('0') + (ch / 0x10U)));
+ output_character(char32_t(std::uint8_t('0') + (ch % 0x10U)));
+ }
+ else
+ {
+ output_character(ch);
+ if (!m_escape && (DOUBLE_QUOTE == ch))
+ m_parse_state = parse_state::DEFAULT;
+ }
+ }
+ m_escape = (BACKSLASH == ch) && !m_escape;
+}
+
+
+
+/***************************************************************************
XML DATA CLEANER CLASS
***************************************************************************/
@@ -1894,6 +2184,13 @@ bool is_lua_source_extension(char const *ext)
}
+bool is_json_extension(char const *ext)
+{
+ return
+ !core_stricmp(ext, ".json");
+}
+
+
bool is_xml_extension(char const *ext)
{
return
@@ -1974,12 +2271,15 @@ int main(int argc, char *argv[])
char const *const ext(std::strrchr(argv[i], '.'));
bool const is_c_file(ext && is_c_source_extension(ext));
bool const is_lua_file(ext && is_lua_source_extension(ext));
+ bool const is_json_file(ext && is_json_extension(ext));
bool const is_xml_file(ext && is_xml_extension(ext));
std::unique_ptr<cleaner_base> cleaner;
if (is_c_file)
cleaner = std::make_unique<cpp_cleaner>(std::back_inserter(output), newline_mode, 4U);
else if (is_lua_file)
cleaner = std::make_unique<lua_cleaner>(std::back_inserter(output), newline_mode, 4U);
+ else if (is_json_file)
+ cleaner = std::make_unique<json_cleaner>(std::back_inserter(output), newline_mode, 4U);
else if (is_xml_file)
cleaner = std::make_unique<xml_cleaner>(std::back_inserter(output), newline_mode, 4U);
else