summaryrefslogtreecommitdiffstatshomepage
path: root/src/tools
diff options
context:
space:
mode:
author Vas Crabb <vas@vastheman.com>2021-08-20 05:47:40 +1000
committer Vas Crabb <vas@vastheman.com>2021-08-20 05:52:27 +1000
commit0c49c74ada4fee63f0bfd14710a437b23fe57991 (patch)
treef24d8b14d52ce0efac60e744651e3e0ec3d8f9d1 /src/tools
parent59c4e5fed6f40e9234a46451dc255266101757cb (diff)
srcclean: Added JSON cleaning support, and some cleanup.
Made pbobble parent of bublbust, as it seems to be more widespread and more complete. Also fixed some ROM labels for bublbust. Made tbyahhoo parent of mtwinbee as the latter has substantial content removed rather than being localised, making it less complete. Applied srcclean to JSON files in bgfx subtree.
Diffstat (limited to 'src/tools')
-rw-r--r--src/tools/srcclean.cpp304
1 files changed, 302 insertions, 2 deletions
diff --git a/src/tools/srcclean.cpp b/src/tools/srcclean.cpp
index 7f52c3c6feb..052d2a37334 100644
--- a/src/tools/srcclean.cpp
+++ b/src/tools/srcclean.cpp
@@ -38,6 +38,9 @@
may have spacing adjusted in a way that affects behaviour when
uncommented
+ Known JSON limitations:
+ * Doesn't detect invalid numbers or literals
+
Known XML limitations:
* No special handling for CDATA
* No special handling for processing instructions
@@ -102,14 +105,20 @@ protected:
static constexpr char32_t HORIZONTAL_TAB = 0x0000'0009U;
static constexpr char32_t LINE_FEED = 0x0000'000aU;
static constexpr char32_t VERTICAL_TAB = 0x0000'000bU;
+ static constexpr char32_t FORM_FEED = 0x0000'000cU;
+ static constexpr char32_t C0_CONTROL_LAST = 0x0000'001fU;
static constexpr char32_t SPACE = 0x0000'0020U;
static constexpr char32_t DOUBLE_QUOTE = 0x0000'0022U;
static constexpr char32_t SINGLE_QUOTE = 0x0000'0027U;
+ static constexpr char32_t ASTERISK = 0x0000'002aU;
static constexpr char32_t HYPHEN_MINUS = 0x0000'002dU;
+ static constexpr char32_t SLASH = 0x0000'002fU;
static constexpr char32_t QUESTION_MARK = 0x0000'003fU;
static constexpr char32_t BACKSLASH = 0x0000'005cU;
static constexpr char32_t BASIC_LATIN_LAST = 0x0000'007fU;
static constexpr char32_t CYRILLIC_SUPPLEMENT_LAST = 0x0000'052fU;
+ static constexpr char32_t LINE_SEPARATOR = 0x0000'2028U;
+ static constexpr char32_t PARAGRAPH_SEPARATOR = 0x0000'2029U;
template <typename OutputIt>
cleaner_base(OutputIt &&output, newline newline_mode, unsigned tab_width);
@@ -825,8 +834,6 @@ protected:
void output_character(char32_t ch);
private:
- static constexpr char32_t ASTERISK = 0x0000'002aU;
- static constexpr char32_t SLASH = 0x0000'002fU;
static constexpr char32_t UPPERCASE_FIRST = 0x0000'0041U;
static constexpr char32_t UPPERCASE_B = 0x0000'0042U;
static constexpr char32_t UPPERCASE_X = 0x0000'0058U;
@@ -1741,6 +1748,289 @@ void lua_cleaner::process_long_string_constant(char32_t ch)
/***************************************************************************
+ JSON DATA CLEANER CLASS
+***************************************************************************/
+
+class json_cleaner : public cleaner_base
+{
+public:
+ template <typename OutputIt>
+ json_cleaner(OutputIt &&output, newline newline_mode, unsigned tab_width);
+
+ virtual bool affected() const override;
+ virtual void summarise(std::ostream &os) const override;
+
+protected:
+ void output_character(char32_t ch);
+
+private:
+ enum class parse_state
+ {
+ DEFAULT,
+ COMMENT,
+ LINE_COMMENT,
+ STRING_CONSTANT
+ };
+
+ virtual void process_characters(char32_t const *begin, char32_t const *end) override;
+ virtual void input_complete() override;
+
+ void process_default(char32_t ch);
+ void process_comment(char32_t ch);
+ void process_line_comment(char32_t ch);
+ void process_text(char32_t ch);
+
+ parse_state m_parse_state = parse_state::DEFAULT;
+ std::uint64_t m_input_line = 1U;
+ bool m_escape = false;
+ std::uint64_t m_comment_line = 0U;
+
+ std::uint64_t m_tabs_escaped = 0U;
+ std::uint64_t m_newlines_escaped = 0U;
+ std::uint64_t m_form_feeds_escaped = 0U;
+ std::uint64_t m_line_separators_escaped = 0U;
+ std::uint64_t m_paragraph_separators_escaped = 0U;
+ std::uint64_t m_c0_control_escaped = 0U;
+ std::uint64_t m_non_ascii = 0U;
+};
+
+
+template <typename OutputIt>
+json_cleaner::json_cleaner(
+ OutputIt &&output,
+ newline newline_mode,
+ unsigned tab_width)
+ : cleaner_base(std::forward<OutputIt>(output), newline_mode, tab_width)
+{
+}
+
+
+bool json_cleaner::affected() const
+{
+ return
+ cleaner_base::affected() ||
+ m_tabs_escaped ||
+ m_newlines_escaped ||
+ m_form_feeds_escaped ||
+ m_line_separators_escaped ||
+ m_paragraph_separators_escaped ||
+ m_c0_control_escaped ||
+ m_non_ascii;
+}
+
+
+void json_cleaner::summarise(std::ostream &os) const
+{
+ cleaner_base::summarise(os);
+ if (m_tabs_escaped)
+ util::stream_format(os, "%1$u tab(s) escaped\n", m_tabs_escaped);
+ if (m_newlines_escaped)
+ util::stream_format(os, "%1$u line feed(s) escaped\n", m_newlines_escaped);
+ if (m_form_feeds_escaped)
+ util::stream_format(os, "%1$u form feed(s) escaped\n", m_form_feeds_escaped);
+ if (m_line_separators_escaped)
+ util::stream_format(os, "%1$u line separator(s) escaped\n", m_line_separators_escaped);
+ if (m_paragraph_separators_escaped)
+ util::stream_format(os, "%1$u paragraph separator(s) escaped\n", m_paragraph_separators_escaped);
+ if (m_c0_control_escaped)
+ util::stream_format(os, "%1$u C0 control character(s) escaped\n", m_c0_control_escaped);
+ if (m_non_ascii)
+ util::stream_format(os, "%1$u non-ASCII character(s) replaced\n", m_non_ascii);
+}
+
+
+void json_cleaner::output_character(char32_t ch)
+{
+ switch (m_parse_state)
+ {
+ case parse_state::DEFAULT:
+ if (BASIC_LATIN_LAST < ch)
+ {
+ ++m_non_ascii;
+ ch = QUESTION_MARK;
+ }
+ break;
+ case parse_state::COMMENT:
+ case parse_state::LINE_COMMENT:
+ case parse_state::STRING_CONSTANT:
+ break;
+ }
+
+ cleaner_base::output_character(ch);
+}
+
+
+void json_cleaner::process_characters(char32_t const *begin, char32_t const *end)
+{
+ while (begin != end)
+ {
+ char32_t const ch(*begin++);
+ switch (m_parse_state)
+ {
+ case parse_state::DEFAULT:
+ process_default(ch);
+ break;
+ case parse_state::COMMENT:
+ process_comment(ch);
+ break;
+ case parse_state::LINE_COMMENT:
+ process_line_comment(ch);
+ break;
+ case parse_state::STRING_CONSTANT:
+ process_text(ch);
+ break;
+ }
+
+ if (LINE_FEED == ch)
+ ++m_input_line;
+ }
+}
+
+
+void json_cleaner::input_complete()
+{
+ switch (m_parse_state)
+ {
+ case parse_state::COMMENT:
+ throw std::runtime_error(util::string_format("unterminated multi-line comment beginning on line %1$u", m_comment_line));
+ case parse_state::STRING_CONSTANT:
+ throw std::runtime_error(util::string_format("unterminated string literal on line %1$u", m_input_line));
+ default:
+ break;
+ }
+}
+
+
+void json_cleaner::process_default(char32_t ch)
+{
+ switch (ch)
+ {
+ case DOUBLE_QUOTE:
+ m_parse_state = parse_state::STRING_CONSTANT;
+ break;
+ case ASTERISK:
+ if (m_escape)
+ {
+ m_parse_state = parse_state::COMMENT;
+ m_comment_line = m_input_line;
+ set_tab_limit();
+ }
+ break;
+ case SLASH:
+ if (m_escape)
+ m_parse_state = parse_state::LINE_COMMENT;
+ break;
+ }
+ m_escape = (SLASH == ch) && !m_escape;
+ output_character(ch);
+}
+
+
+void json_cleaner::process_comment(char32_t ch)
+{
+ switch (ch)
+ {
+ case SLASH:
+ if (m_escape)
+ {
+ m_parse_state = parse_state::DEFAULT;
+ m_comment_line = 0U;
+ output_character(ch);
+ reset_tab_limit();
+ }
+ else
+ {
+ output_character(ch);
+ }
+ m_escape = false;
+ break;
+ default:
+ m_escape = ASTERISK == ch;
+ output_character(ch);
+ }
+}
+
+
+void json_cleaner::process_line_comment(char32_t ch)
+{
+ switch (ch)
+ {
+ case LINE_FEED:
+ m_parse_state = parse_state::DEFAULT;
+ [[fallthrough]];
+ default:
+ output_character(ch);
+ }
+}
+
+
+void json_cleaner::process_text(char32_t ch)
+{
+ switch (ch)
+ {
+ case HORIZONTAL_TAB:
+ ++m_tabs_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('t')));
+ break;
+ case LINE_FEED:
+ ++m_newlines_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('n')));
+ break;
+ case FORM_FEED:
+ ++m_form_feeds_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('f')));
+ break;
+ case LINE_SEPARATOR:
+ ++m_line_separators_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('u')));
+ output_character(char32_t(std::uint8_t('2')));
+ output_character(char32_t(std::uint8_t('0')));
+ output_character(char32_t(std::uint8_t('2')));
+ output_character(char32_t(std::uint8_t('8')));
+ break;
+ case PARAGRAPH_SEPARATOR:
+ ++m_paragraph_separators_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('u')));
+ output_character(char32_t(std::uint8_t('2')));
+ output_character(char32_t(std::uint8_t('0')));
+ output_character(char32_t(std::uint8_t('2')));
+ output_character(char32_t(std::uint8_t('9')));
+ break;
+ default:
+ if (C0_CONTROL_LAST >= ch)
+ {
+ ++m_c0_control_escaped;
+ if (!m_escape)
+ output_character(BACKSLASH);
+ output_character(char32_t(std::uint8_t('u')));
+ output_character(char32_t(std::uint8_t('0')));
+ output_character(char32_t(std::uint8_t('0')));
+ output_character(char32_t(std::uint8_t('0') + (ch / 0x10U)));
+ output_character(char32_t(std::uint8_t('0') + (ch % 0x10U)));
+ }
+ else
+ {
+ output_character(ch);
+ if (!m_escape && (DOUBLE_QUOTE == ch))
+ m_parse_state = parse_state::DEFAULT;
+ }
+ }
+ m_escape = (BACKSLASH == ch) && !m_escape;
+}
+
+
+
+/***************************************************************************
XML DATA CLEANER CLASS
***************************************************************************/
@@ -1894,6 +2184,13 @@ bool is_lua_source_extension(char const *ext)
}
+bool is_json_extension(char const *ext)
+{
+ return
+ !core_stricmp(ext, ".json");
+}
+
+
bool is_xml_extension(char const *ext)
{
return
@@ -1974,12 +2271,15 @@ int main(int argc, char *argv[])
char const *const ext(std::strrchr(argv[i], '.'));
bool const is_c_file(ext && is_c_source_extension(ext));
bool const is_lua_file(ext && is_lua_source_extension(ext));
+ bool const is_json_file(ext && is_json_extension(ext));
bool const is_xml_file(ext && is_xml_extension(ext));
std::unique_ptr<cleaner_base> cleaner;
if (is_c_file)
cleaner = std::make_unique<cpp_cleaner>(std::back_inserter(output), newline_mode, 4U);
else if (is_lua_file)
cleaner = std::make_unique<lua_cleaner>(std::back_inserter(output), newline_mode, 4U);
+ else if (is_json_file)
+ cleaner = std::make_unique<json_cleaner>(std::back_inserter(output), newline_mode, 4U);
else if (is_xml_file)
cleaner = std::make_unique<xml_cleaner>(std::back_inserter(output), newline_mode, 4U);
else