diff options
Diffstat (limited to 'src/tools/srcclean.cpp')
-rw-r--r-- | src/tools/srcclean.cpp | 304 |
1 files changed, 302 insertions, 2 deletions
diff --git a/src/tools/srcclean.cpp b/src/tools/srcclean.cpp index 7f52c3c6feb..052d2a37334 100644 --- a/src/tools/srcclean.cpp +++ b/src/tools/srcclean.cpp @@ -38,6 +38,9 @@ may have spacing adjusted in a way that affects behaviour when uncommented + Known JSON limitations: + * Doesn't detect invalid numbers or literals + Known XML limitations: * No special handling for CDATA * No special handling for processing instructions @@ -102,14 +105,20 @@ protected: static constexpr char32_t HORIZONTAL_TAB = 0x0000'0009U; static constexpr char32_t LINE_FEED = 0x0000'000aU; static constexpr char32_t VERTICAL_TAB = 0x0000'000bU; + static constexpr char32_t FORM_FEED = 0x0000'000cU; + static constexpr char32_t C0_CONTROL_LAST = 0x0000'001fU; static constexpr char32_t SPACE = 0x0000'0020U; static constexpr char32_t DOUBLE_QUOTE = 0x0000'0022U; static constexpr char32_t SINGLE_QUOTE = 0x0000'0027U; + static constexpr char32_t ASTERISK = 0x0000'002aU; static constexpr char32_t HYPHEN_MINUS = 0x0000'002dU; + static constexpr char32_t SLASH = 0x0000'002fU; static constexpr char32_t QUESTION_MARK = 0x0000'003fU; static constexpr char32_t BACKSLASH = 0x0000'005cU; static constexpr char32_t BASIC_LATIN_LAST = 0x0000'007fU; static constexpr char32_t CYRILLIC_SUPPLEMENT_LAST = 0x0000'052fU; + static constexpr char32_t LINE_SEPARATOR = 0x0000'2028U; + static constexpr char32_t PARAGRAPH_SEPARATOR = 0x0000'2029U; template <typename OutputIt> cleaner_base(OutputIt &&output, newline newline_mode, unsigned tab_width); @@ -825,8 +834,6 @@ protected: void output_character(char32_t ch); private: - static constexpr char32_t ASTERISK = 0x0000'002aU; - static constexpr char32_t SLASH = 0x0000'002fU; static constexpr char32_t UPPERCASE_FIRST = 0x0000'0041U; static constexpr char32_t UPPERCASE_B = 0x0000'0042U; static constexpr char32_t UPPERCASE_X = 0x0000'0058U; @@ -1741,6 +1748,289 @@ void lua_cleaner::process_long_string_constant(char32_t ch) /*************************************************************************** + JSON DATA CLEANER CLASS +***************************************************************************/ + +class json_cleaner : public cleaner_base +{ +public: + template <typename OutputIt> + json_cleaner(OutputIt &&output, newline newline_mode, unsigned tab_width); + + virtual bool affected() const override; + virtual void summarise(std::ostream &os) const override; + +protected: + void output_character(char32_t ch); + +private: + enum class parse_state + { + DEFAULT, + COMMENT, + LINE_COMMENT, + STRING_CONSTANT + }; + + virtual void process_characters(char32_t const *begin, char32_t const *end) override; + virtual void input_complete() override; + + void process_default(char32_t ch); + void process_comment(char32_t ch); + void process_line_comment(char32_t ch); + void process_text(char32_t ch); + + parse_state m_parse_state = parse_state::DEFAULT; + std::uint64_t m_input_line = 1U; + bool m_escape = false; + std::uint64_t m_comment_line = 0U; + + std::uint64_t m_tabs_escaped = 0U; + std::uint64_t m_newlines_escaped = 0U; + std::uint64_t m_form_feeds_escaped = 0U; + std::uint64_t m_line_separators_escaped = 0U; + std::uint64_t m_paragraph_separators_escaped = 0U; + std::uint64_t m_c0_control_escaped = 0U; + std::uint64_t m_non_ascii = 0U; +}; + + +template <typename OutputIt> +json_cleaner::json_cleaner( + OutputIt &&output, + newline newline_mode, + unsigned tab_width) + : cleaner_base(std::forward<OutputIt>(output), newline_mode, tab_width) +{ +} + + +bool json_cleaner::affected() const +{ + return + cleaner_base::affected() || + m_tabs_escaped || + m_newlines_escaped || + m_form_feeds_escaped || + m_line_separators_escaped || + m_paragraph_separators_escaped || + m_c0_control_escaped || + m_non_ascii; +} + + +void json_cleaner::summarise(std::ostream &os) const +{ + cleaner_base::summarise(os); + if (m_tabs_escaped) + util::stream_format(os, "%1$u tab(s) escaped\n", m_tabs_escaped); + if (m_newlines_escaped) + util::stream_format(os, "%1$u line feed(s) escaped\n", m_newlines_escaped); + if (m_form_feeds_escaped) + util::stream_format(os, "%1$u form feed(s) escaped\n", m_form_feeds_escaped); + if (m_line_separators_escaped) + util::stream_format(os, "%1$u line separator(s) escaped\n", m_line_separators_escaped); + if (m_paragraph_separators_escaped) + util::stream_format(os, "%1$u paragraph separator(s) escaped\n", m_paragraph_separators_escaped); + if (m_c0_control_escaped) + util::stream_format(os, "%1$u C0 control character(s) escaped\n", m_c0_control_escaped); + if (m_non_ascii) + util::stream_format(os, "%1$u non-ASCII character(s) replaced\n", m_non_ascii); +} + + +void json_cleaner::output_character(char32_t ch) +{ + switch (m_parse_state) + { + case parse_state::DEFAULT: + if (BASIC_LATIN_LAST < ch) + { + ++m_non_ascii; + ch = QUESTION_MARK; + } + break; + case parse_state::COMMENT: + case parse_state::LINE_COMMENT: + case parse_state::STRING_CONSTANT: + break; + } + + cleaner_base::output_character(ch); +} + + +void json_cleaner::process_characters(char32_t const *begin, char32_t const *end) +{ + while (begin != end) + { + char32_t const ch(*begin++); + switch (m_parse_state) + { + case parse_state::DEFAULT: + process_default(ch); + break; + case parse_state::COMMENT: + process_comment(ch); + break; + case parse_state::LINE_COMMENT: + process_line_comment(ch); + break; + case parse_state::STRING_CONSTANT: + process_text(ch); + break; + } + + if (LINE_FEED == ch) + ++m_input_line; + } +} + + +void json_cleaner::input_complete() +{ + switch (m_parse_state) + { + case parse_state::COMMENT: + throw std::runtime_error(util::string_format("unterminated multi-line comment beginning on line %1$u", m_comment_line)); + case parse_state::STRING_CONSTANT: + throw std::runtime_error(util::string_format("unterminated string literal on line %1$u", m_input_line)); + default: + break; + } +} + + +void json_cleaner::process_default(char32_t ch) +{ + switch (ch) + { + case DOUBLE_QUOTE: + m_parse_state = parse_state::STRING_CONSTANT; + break; + case ASTERISK: + if (m_escape) + { + m_parse_state = parse_state::COMMENT; + m_comment_line = m_input_line; + set_tab_limit(); + } + break; + case SLASH: + if (m_escape) + m_parse_state = parse_state::LINE_COMMENT; + break; + } + m_escape = (SLASH == ch) && !m_escape; + output_character(ch); +} + + +void json_cleaner::process_comment(char32_t ch) +{ + switch (ch) + { + case SLASH: + if (m_escape) + { + m_parse_state = parse_state::DEFAULT; + m_comment_line = 0U; + output_character(ch); + reset_tab_limit(); + } + else + { + output_character(ch); + } + m_escape = false; + break; + default: + m_escape = ASTERISK == ch; + output_character(ch); + } +} + + +void json_cleaner::process_line_comment(char32_t ch) +{ + switch (ch) + { + case LINE_FEED: + m_parse_state = parse_state::DEFAULT; + [[fallthrough]]; + default: + output_character(ch); + } +} + + +void json_cleaner::process_text(char32_t ch) +{ + switch (ch) + { + case HORIZONTAL_TAB: + ++m_tabs_escaped; + if (!m_escape) + output_character(BACKSLASH); + output_character(char32_t(std::uint8_t('t'))); + break; + case LINE_FEED: + ++m_newlines_escaped; + if (!m_escape) + output_character(BACKSLASH); + output_character(char32_t(std::uint8_t('n'))); + break; + case FORM_FEED: + ++m_form_feeds_escaped; + if (!m_escape) + output_character(BACKSLASH); + output_character(char32_t(std::uint8_t('f'))); + break; + case LINE_SEPARATOR: + ++m_line_separators_escaped; + if (!m_escape) + output_character(BACKSLASH); + output_character(char32_t(std::uint8_t('u'))); + output_character(char32_t(std::uint8_t('2'))); + output_character(char32_t(std::uint8_t('0'))); + output_character(char32_t(std::uint8_t('2'))); + output_character(char32_t(std::uint8_t('8'))); + break; + case PARAGRAPH_SEPARATOR: + ++m_paragraph_separators_escaped; + if (!m_escape) + output_character(BACKSLASH); + output_character(char32_t(std::uint8_t('u'))); + output_character(char32_t(std::uint8_t('2'))); + output_character(char32_t(std::uint8_t('0'))); + output_character(char32_t(std::uint8_t('2'))); + output_character(char32_t(std::uint8_t('9'))); + break; + default: + if (C0_CONTROL_LAST >= ch) + { + ++m_c0_control_escaped; + if (!m_escape) + output_character(BACKSLASH); + output_character(char32_t(std::uint8_t('u'))); + output_character(char32_t(std::uint8_t('0'))); + output_character(char32_t(std::uint8_t('0'))); + output_character(char32_t(std::uint8_t('0') + (ch / 0x10U))); + output_character(char32_t(std::uint8_t('0') + (ch % 0x10U))); + } + else + { + output_character(ch); + if (!m_escape && (DOUBLE_QUOTE == ch)) + m_parse_state = parse_state::DEFAULT; + } + } + m_escape = (BACKSLASH == ch) && !m_escape; +} + + + +/*************************************************************************** XML DATA CLEANER CLASS ***************************************************************************/ @@ -1894,6 +2184,13 @@ bool is_lua_source_extension(char const *ext) } +bool is_json_extension(char const *ext) +{ + return + !core_stricmp(ext, ".json"); +} + + bool is_xml_extension(char const *ext) { return @@ -1974,12 +2271,15 @@ int main(int argc, char *argv[]) char const *const ext(std::strrchr(argv[i], '.')); bool const is_c_file(ext && is_c_source_extension(ext)); bool const is_lua_file(ext && is_lua_source_extension(ext)); + bool const is_json_file(ext && is_json_extension(ext)); bool const is_xml_file(ext && is_xml_extension(ext)); std::unique_ptr<cleaner_base> cleaner; if (is_c_file) cleaner = std::make_unique<cpp_cleaner>(std::back_inserter(output), newline_mode, 4U); else if (is_lua_file) cleaner = std::make_unique<lua_cleaner>(std::back_inserter(output), newline_mode, 4U); + else if (is_json_file) + cleaner = std::make_unique<json_cleaner>(std::back_inserter(output), newline_mode, 4U); else if (is_xml_file) cleaner = std::make_unique<xml_cleaner>(std::back_inserter(output), newline_mode, 4U); else |