summaryrefslogtreecommitdiffstatshomepage
path: root/src/tools
diff options
context:
space:
mode:
authorGravatarGravatar Vas Crabb <vas@vastheman.com>2019-10-04 22:55:01 +1000
committerGravatarGravatar Vas Crabb <vas@vastheman.com>2019-10-04 22:55:01 +1000
commitbcfa6047c37c95fdb5513ebe633ec0470c400fb6 (patch)
tree9dbb0c99c68c40deda2914a0fda5dbd8e4b82524 /src/tools
parent3bbbe5080b30e41e379644033b7dbfc4419ca576 (diff)
Build system maintenance:
* Re-write makedep.py for better performance and better parsing front-end * Make srcclean deal with kinds of preprocessor abuse I never want to see in real life (nw) The new parser front-end is better at recognising C++ syntax and also substantially faster - bootstrapping a single-driver build should be noticeably quicker. Having a single parser for C++, .lst and .flt files also gets us a bit closer to making it simpler to create custom subtargets.
Diffstat (limited to 'src/tools')
-rw-r--r--src/tools/srcclean.cpp145
1 files changed, 125 insertions, 20 deletions
diff --git a/src/tools/srcclean.cpp b/src/tools/srcclean.cpp
index 431bc00434c..331eab21bae 100644
--- a/src/tools/srcclean.cpp
+++ b/src/tools/srcclean.cpp
@@ -24,10 +24,12 @@
#define MY_MACRO \
"string that \
continues"
- * Will not produce expected output for a string continuation that
- breaks an escape sequence, e.g. this:
- "bad\\
- tbehaviour"
+ * Numeric literals broken by line continuations are not recognised
+ * Will not recognise a comment delimiter broken by multiple line
+ continuations. e.g. this:
+ /\
+ \
+ / preprocessor abuse
Known Lua limitations:
* Whitespace normalisation is applied inside long string literals
@@ -868,19 +870,19 @@ private:
bool tail_is(char32_t ch) const
{
- return !m_tail.empty() && (m_tail.front() == ch);
+ return !m_tail.empty() && (m_tail.back() == ch);
}
void pop_tail()
{
if (!m_tail.empty())
- m_tail.pop_front();
+ m_tail.pop_back();
}
void replace_tail(char32_t ch)
{
assert(!m_tail.empty());
- *m_tail.begin() = ch;
+ m_tail.back() = ch;
}
void flush_tail()
@@ -938,10 +940,12 @@ private:
bool m_escape = false;
std::deque<char32_t> m_tail;
std::uint64_t m_comment_line = 0U;
+ bool m_broken_escape = false;
char32_t m_lead_digit = 0U;
unsigned m_radix = 0U;
std::uint64_t m_tabs_escaped = 0U;
+ std::uint64_t m_broken_comment_delimiters = 0U;
std::uint64_t m_line_comment_continuations = 0U;
std::uint64_t m_string_continuations = 0U;
std::uint64_t m_uppercase_radix = 0U;
@@ -965,6 +969,7 @@ bool cpp_cleaner::affected() const
return
cleaner_base::affected() ||
m_tabs_escaped ||
+ m_broken_comment_delimiters ||
m_line_comment_continuations ||
m_string_continuations ||
m_uppercase_radix ||
@@ -977,6 +982,8 @@ void cpp_cleaner::summarise(std::ostream &os) const
cleaner_base::summarise(os);
if (m_tabs_escaped)
util::stream_format(os, "%1$u tab(s) escaped\n", m_tabs_escaped);
+ if (m_broken_comment_delimiters)
+ util::stream_format(os, "%1$u broken comment delimiter(s) replaced\n", m_broken_comment_delimiters);
if (m_line_comment_continuations)
util::stream_format(os, "%1$u line comment continuation(s) replaced\n", m_line_comment_continuations);
if (m_string_continuations)
@@ -1016,16 +1023,17 @@ void cpp_cleaner::output_character(char32_t ch)
switch (ch)
{
+ case HORIZONTAL_TAB:
+ case SPACE:
+ case BACKSLASH:
+ m_tail.emplace_back(ch);
+ break;
default:
flush_tail();
if (LINE_FEED == ch)
- {
cleaner_base::output_character(ch);
- break;
- }
- case HORIZONTAL_TAB:
- case SPACE:
- m_tail.emplace_back(ch);
+ else
+ m_tail.emplace_back(ch);
}
}
@@ -1085,6 +1093,13 @@ void cpp_cleaner::process_default(char32_t ch)
{
switch (ch)
{
+ case LINE_FEED:
+ if (m_escape && tail_is(BACKSLASH))
+ {
+ m_broken_escape = true;
+ return;
+ }
+ break;
case DOUBLE_QUOTE:
m_parse_state = parse_state::STRING_CONSTANT;
break;
@@ -1097,11 +1112,35 @@ void cpp_cleaner::process_default(char32_t ch)
m_parse_state = parse_state::COMMENT;
m_comment_line = m_input_line;
set_tab_limit();
+ if (m_broken_escape)
+ {
+ ++m_broken_comment_delimiters;
+ assert(tail_is(BACKSLASH));
+ pop_tail();
+ output_character(ch);
+ output_character(LINE_FEED);
+ m_escape = false;
+ m_broken_escape = false;
+ return;
+ }
}
break;
case SLASH:
if (m_escape)
+ {
m_parse_state = parse_state::LINE_COMMENT;
+ if (m_broken_escape)
+ {
+ ++m_broken_comment_delimiters;
+ assert(tail_is(BACKSLASH));
+ pop_tail();
+ assert(tail_is(SLASH));
+ pop_tail();
+ output_character(LINE_FEED);
+ output_character(SLASH);
+ m_broken_escape = false;
+ }
+ }
break;
default:
if (is_token_lead(ch))
@@ -1112,11 +1151,15 @@ void cpp_cleaner::process_default(char32_t ch)
{
m_parse_state = parse_state::NUMERIC_CONSTANT;
m_escape = false;
+ m_broken_escape = false;
process_numeric(ch);
return;
}
}
- m_escape = (SLASH == ch) ? !m_escape : false;
+ if (m_broken_escape)
+ output_character(LINE_FEED);
+ m_escape = m_escape ? ((BACKSLASH == ch) && tail_is(SLASH)) : (SLASH == ch);
+ m_broken_escape = false;
output_character(ch);
}
@@ -1125,18 +1168,65 @@ void cpp_cleaner::process_comment(char32_t ch)
{
switch (ch)
{
- case SLASH:
- if (m_escape)
+ case LINE_FEED:
+ if (m_escape && tail_is(BACKSLASH))
+ {
+ m_broken_escape = true;
+ }
+ else
{
m_escape = false;
+ m_broken_escape = false;
+ output_character(ch);
+ }
+ break;
+ case SLASH:
+ if (m_broken_escape)
+ {
m_parse_state = parse_state::DEFAULT;
m_comment_line = 0U;
+ ++m_broken_comment_delimiters;
+ assert(tail_is(BACKSLASH));
+ pop_tail();
+ assert(tail_is(ASTERISK));
+ pop_tail();
+ output_character(LINE_FEED);
+ output_character(ASTERISK);
output_character(ch);
reset_tab_limit();
- break;
}
+ else if (m_escape)
+ {
+ m_parse_state = parse_state::DEFAULT;
+ m_comment_line = 0U;
+ output_character(ch);
+ reset_tab_limit();
+ }
+ else
+ {
+ output_character(ch);
+ }
+ m_escape = false;
+ m_broken_escape = false;
+ break;
+ case BACKSLASH:
+ if (m_broken_escape)
+ {
+ m_escape = false;
+ m_broken_escape = false;
+ output_character(LINE_FEED);
+ }
+ else if (m_escape)
+ {
+ m_escape = tail_is(ASTERISK);
+ }
+ output_character(ch);
+ break;
default:
+ if (m_broken_escape)
+ output_character(LINE_FEED);
m_escape = ASTERISK == ch;
+ m_broken_escape = false;
output_character(ch);
}
}
@@ -1195,9 +1285,24 @@ void cpp_cleaner::process_text(char32_t ch)
else if (tail_is(BACKSLASH))
{
++m_string_continuations;
- replace_tail(DOUBLE_QUOTE);
- output_character(ch);
- output_character(DOUBLE_QUOTE);
+ if (m_escape)
+ {
+ replace_tail(DOUBLE_QUOTE);
+ output_character(ch);
+ output_character(DOUBLE_QUOTE);
+ }
+ else
+ {
+ pop_tail();
+ assert(tail_is(BACKSLASH));
+ pop_tail();
+ output_character(DOUBLE_QUOTE);
+ output_character(ch);
+ output_character(DOUBLE_QUOTE);
+ output_character(BACKSLASH);
+ m_escape = true;
+ return;
+ }
}
else
{