summaryrefslogtreecommitdiffstatshomepage
path: root/src/lib/netlist/plib/pparser.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/netlist/plib/pparser.cpp')
-rw-r--r--src/lib/netlist/plib/pparser.cpp464
1 files changed, 373 insertions, 91 deletions
diff --git a/src/lib/netlist/plib/pparser.cpp b/src/lib/netlist/plib/pparser.cpp
index abfa524d332..3480a8e3bea 100644
--- a/src/lib/netlist/plib/pparser.cpp
+++ b/src/lib/netlist/plib/pparser.cpp
@@ -9,7 +9,6 @@
#include "palloc.h"
#include "putil.h"
-//#include <cstdarg>
namespace plib {
// ----------------------------------------------------------------------------------------
@@ -21,7 +20,6 @@ pstring ptokenizer::currentline_str()
return m_cur_line;
}
-
void ptokenizer::skipeol()
{
pstring::value_type c = getc();
@@ -38,7 +36,6 @@ void ptokenizer::skipeol()
}
}
-
pstring::value_type ptokenizer::getc()
{
if (m_unget != 0)
@@ -49,7 +46,7 @@ pstring::value_type ptokenizer::getc()
}
if (m_px == m_cur_line.end())
{
- m_lineno++;
+ ++m_source_location.back();
if (m_strm.readline(m_cur_line))
m_px = m_cur_line.begin();
else
@@ -143,45 +140,54 @@ long ptokenizer::get_number_long()
ptokenizer::token_t ptokenizer::get_token()
{
+ token_t ret = get_token_internal();
while (true)
{
- token_t ret = get_token_internal();
if (ret.is_type(ENDOFFILE))
return ret;
-
- if (m_support_line_markers && ret.is_type(LINEMARKER))
+ else if (m_support_line_markers && ret.is_type(LINEMARKER))
{
- // FIXME: do something with information gathered
+ bool benter(false);
+ bool bexit(false);
+ pstring file;
+ unsigned lineno;
+
ret = get_token_internal();
if (!ret.is_type(NUMBER))
error(pfmt("Expected line number after line marker but got <{1}>")(ret.str()) );
+ lineno = pstonum<unsigned>(ret.str());
ret = get_token_internal();
if (!ret.is_type(STRING))
error(pfmt("Expected file name after line marker but got <{1}>")(ret.str()) );
+ file = ret.str();
ret = get_token_internal();
while (ret.is_type(NUMBER))
{
+ if (ret.str() == "1")
+ benter = true;
+ if (ret.str() == "2")
+ bexit = false;
// FIXME: process flags; actually only 1 (file enter) and 2 (after file exit)
ret = get_token_internal();
}
+ if (bexit) // pop the last location
+ m_source_location.pop_back();
+ if (!benter) // new location!
+ m_source_location.pop_back();
+ m_source_location.emplace_back(plib::source_location(file, lineno));
}
-
- if (ret.is(m_tok_comment_start))
+ else if (ret.is(m_tok_comment_start))
{
do {
ret = get_token_internal();
} while (ret.is_not(m_tok_comment_end));
+ ret = get_token_internal();
}
else if (ret.is(m_tok_line_comment))
{
skipeol();
+ ret = get_token_internal();
}
-#if 0
- else if (ret.str() == "#")
- {
- skipeol();
- }
-#endif
else
{
return ret;
@@ -202,7 +208,7 @@ ptokenizer::token_t ptokenizer::get_token_internal()
}
}
if (m_support_line_markers && c == '#')
- return token_t(LINEMARKER);
+ return token_t(LINEMARKER, "#");
else if (m_number_chars_start.find(c) != pstring::npos)
{
/* read number while we receive number or identifier chars
@@ -274,6 +280,25 @@ ptokenizer::token_t ptokenizer::get_token_internal()
}
}
+void ptokenizer::error(const pstring &errs)
+{
+ pstring s("");
+ pstring trail (" from ");
+ pstring trail_first("In file included from ");
+ pstring e = plib::pfmt("{1}:{2}:0: error: {3}\n")
+ (m_source_location.back().file_name(), m_source_location.back().line(), errs);
+ m_source_location.pop_back();
+ while (m_source_location.size() > 0)
+ {
+ if (m_source_location.size() == 1)
+ trail = trail_first;
+ s = trail + plib::pfmt("{1}:{2}:0\n")(m_source_location.back().file_name(), m_source_location.back().line()) + s;
+ m_source_location.pop_back();
+ }
+ verror("\n" + s + e + " " + currentline_str() + "\n");
+}
+
+
// ----------------------------------------------------------------------------------------
// A simple preprocessor
// ----------------------------------------------------------------------------------------
@@ -298,12 +323,20 @@ ppreprocessor::ppreprocessor(psource_collection_t<> &sources, defines_map_type *
m_expr_sep.emplace_back("||");
m_expr_sep.emplace_back("==");
m_expr_sep.emplace_back(",");
+ m_expr_sep.emplace_back(";");
+ m_expr_sep.emplace_back(".");
+ m_expr_sep.emplace_back("##");
+ m_expr_sep.emplace_back("#");
m_expr_sep.emplace_back(" ");
m_expr_sep.emplace_back("\t");
+ m_expr_sep.emplace_back("\"");
if (defines != nullptr)
m_defines = *defines;
m_defines.insert({"__PLIB_PREPROCESSOR__", define_t("__PLIB_PREPROCESSOR__", "1")});
+ auto idx = m_defines.find("__PREPROCESSOR_DEBUG__");
+ m_debug_out = idx != m_defines.end();
+
}
void ppreprocessor::error(const pstring &err)
@@ -311,58 +344,127 @@ void ppreprocessor::error(const pstring &err)
pstring s("");
pstring trail (" from ");
pstring trail_first("In file included from ");
- pstring e = plib::pfmt("PREPRO ERROR: {1}:{2}:0: error: {3}\n")
- (m_stack.top().m_name, m_stack.top().m_lineno, err);
- m_stack.pop();
+ pstring e = plib::pfmt("{1}:{2}:0: error: {3}\n")
+ (m_stack.back().m_name, m_stack.back().m_lineno, err);
+ m_stack.pop_back();
while (m_stack.size() > 0)
{
if (m_stack.size() == 1)
trail = trail_first;
- s = trail + plib::pfmt("{1}:{2}:0\n")(m_stack.top().m_name, m_stack.top().m_lineno) + s;
- m_stack.pop();
+ s = trail + plib::pfmt("{1}:{2}:0\n")(m_stack.back().m_name, m_stack.back().m_lineno) + s;
+ m_stack.pop_back();
}
- throw pexception("\n" + s + e);
+ throw pexception("\n" + s + e + " " + m_line + "\n");
}
+
+
+template <typename PP, typename L = ppreprocessor::string_list>
+struct simple_iter
+{
+ simple_iter(PP *parent, const L &tokens)
+ : m_tokens(tokens), m_parent(parent), m_pos(0)
+ {}
+
+ /**! skip white space in token list
+ *
+ */
+ void skip_ws()
+ {
+ while (m_pos < m_tokens.size() && (m_tokens[m_pos] == " " || m_tokens[m_pos] == "\t"))
+ m_pos++;
+ }
+
+ /**! return next token skipping white space
+ *
+ * @return next token
+ */
+ pstring next()
+ {
+ skip_ws();
+ if (m_pos >= m_tokens.size())
+ error("unexpected end of line");
+ return m_tokens[m_pos++];
+ }
+
+ /**! return next token including white space
+ *
+ * @return next token
+ */
+ pstring next_ws()
+ {
+ if (m_pos >= m_tokens.size())
+ error("unexpected end of line");
+ return m_tokens[m_pos++];
+ }
+
+ pstring peek_ws()
+ {
+ if (m_pos >= m_tokens.size())
+ error("unexpected end of line");
+ return m_tokens[m_pos];
+ }
+
+ pstring last()
+ {
+ if (m_pos == 0)
+ error("no last token at beginning of line");
+ if (m_pos > m_tokens.size())
+ error("unexpected end of line");
+ return m_tokens[m_pos-1];
+ }
+
+ bool eod()
+ {
+ return (m_pos >= m_tokens.size());
+ }
+
+ void error(pstring err)
+ {
+ m_parent->error(err);
+ }
+private:
+ L m_tokens;
+ PP *m_parent;
+ std::size_t m_pos;
+};
+
#define CHECKTOK2(p_op, p_prio) \
else if (tok == # p_op) \
{ \
- if (!has_val) \
- { error("parsing error!"); return 1;} \
+ if (!has_val) \
+ { sexpr.error("parsing error!"); return 1;} \
if (prio < (p_prio)) \
return val; \
- start++; \
- const auto v2 = expr(sexpr, start, (p_prio)); \
+ sexpr.next(); \
+ const auto v2 = prepro_expr(sexpr, (p_prio)); \
val = (val p_op v2); \
} \
// Operator precedence see https://en.cppreference.com/w/cpp/language/operator_precedence
-int ppreprocessor::expr(const std::vector<pstring> &sexpr, std::size_t &start, int prio)
+template <typename PP>
+static int prepro_expr(simple_iter<PP> &sexpr, int prio)
{
int val(0);
bool has_val(false);
- pstring tok=sexpr[start];
+ pstring tok=sexpr.peek_ws();
if (tok == "(")
{
- start++;
- val = expr(sexpr, start, /*prio*/ 255);
- if (sexpr[start] != ")")
- error("parsing error!");
+ sexpr.next();
+ val = prepro_expr(sexpr, 255);
+ if (sexpr.next() != ")")
+ sexpr.error("expected ')'");
has_val = true;
- start++;
}
- while (start < sexpr.size())
+ while (!sexpr.eod())
{
- tok=sexpr[start];
+ tok = sexpr.peek_ws();
if (tok == ")")
{
if (!has_val)
- {
- error("parsing error!");
- return 1; // tease compiler
- }
+ sexpr.error("Found ')' but have no value computed");
else
return val;
}
@@ -371,15 +473,12 @@ int ppreprocessor::expr(const std::vector<pstring> &sexpr, std::size_t &start, i
if (prio < 3)
{
if (!has_val)
- {
- error("parsing error!");
- return 1; // tease compiler
- }
+ sexpr.error("parsing error!");
else
return val;
}
- start++;
- val = !expr(sexpr, start, 3);
+ sexpr.next();
+ val = !prepro_expr(sexpr, 3);
has_val = true;
}
CHECKTOK2(*, 5)
@@ -393,16 +492,12 @@ int ppreprocessor::expr(const std::vector<pstring> &sexpr, std::size_t &start, i
{
val = plib::pstonum<decltype(val)>(tok);
has_val = true;
- start++;
+ sexpr.next();
}
}
if (!has_val)
- {
- error("parsing error!");
- return 1; // tease compiler
- }
- else
- return val;
+ sexpr.error("No value computed. Empty expression ?");
+ return val;
}
ppreprocessor::define_t *ppreprocessor::get_define(const pstring &name)
@@ -411,16 +506,158 @@ ppreprocessor::define_t *ppreprocessor::get_define(const pstring &name)
return (idx != m_defines.end()) ? &idx->second : nullptr;
}
-pstring ppreprocessor::replace_macros(const pstring &line)
+ppreprocessor::string_list ppreprocessor::tokenize(const pstring &str,
+ const string_list &sep, bool remove_ws, bool concat)
{
- std::vector<pstring> elems(psplit(line, m_expr_sep));
- pstring ret("");
- for (auto & elem : elems)
+ const pstring STR = "\"";
+ string_list tmpret;
+ string_list tmp(psplit(str, sep));
+ std::size_t pi(0);
+
+ while (pi < tmp.size())
{
- define_t *def = get_define(elem);
- ret += (def != nullptr) ? def->m_replace : elem;
+ if (tmp[pi] == STR)
+ {
+ pstring s(STR);
+ pi++;
+ // FIXME : \"
+ while (pi < tmp.size() && tmp[pi] != STR)
+ {
+ s += tmp[pi];
+ pi++;
+ }
+ s += STR;
+ tmpret.push_back(s);
+ }
+ else
+ if (!remove_ws || (tmp[pi] != " " && tmp[pi] != "\t"))
+ tmpret.push_back(tmp[pi]);
+ pi++;
}
- return ret;
+ if (!concat)
+ return tmpret;
+ else
+ {
+ // FIXME: error if concat at beginning or end
+ string_list ret;
+ pi = 0;
+ while (pi<tmpret.size())
+ {
+ if (tmpret[pi] == "##")
+ {
+ while (ret.back() == " " || ret.back() == "\t")
+ ret.pop_back();
+ pstring cc = ret.back();
+ ret.pop_back();
+ pi++;
+ while (pi < tmpret.size() && (tmpret[pi] == " " || tmpret[pi] == "\t"))
+ pi++;
+ if (pi == tmpret.size())
+ error("## found at end of sequence");
+ ret.push_back(cc + tmpret[pi]);
+ }
+ else
+ ret.push_back(tmpret[pi]);
+ pi++;
+ }
+ return ret;
+ }
+}
+
+bool ppreprocessor::is_valid_token(const pstring &str)
+{
+ if (str.length() == 0)
+ return false;
+ char c = str.at(0);
+ return ((c>='a' && c<='z') || (c>='A' && c<='Z') || c == '_');
+}
+
+pstring ppreprocessor::replace_macros(const pstring &line)
+{
+ //std::vector<pstring> elems(psplit(line, m_expr_sep));
+ bool repeat(false);
+ pstring tmpret(line);
+ do
+ {
+ repeat = false;
+ auto elems(simple_iter<ppreprocessor>(this, tokenize(tmpret, m_expr_sep, false, true)));
+ tmpret = "";
+ while (!elems.eod())
+ {
+ auto token(elems.next_ws());
+ define_t *def = get_define(token);
+ if (def == nullptr)
+ tmpret += token;
+ else if (!def->m_has_params)
+ {
+ tmpret += def->m_replace;
+ repeat = true;
+ }
+ else
+ {
+ token = elems.next();
+ if (token != "(")
+ error("expected '(' in macro expansion of " + def->m_name);
+ string_list rep;
+ token = elems.next();
+ while (token != ")")
+ {
+ pstring par("");
+ int pcnt(1);
+ while (true)
+ {
+ if (pcnt==1 && token == ",")
+ {
+ token = elems.next();
+ break;
+ }
+ if (token == "(")
+ pcnt++;
+ if (token == ")")
+ if (--pcnt == 0)
+ break;
+ par += token;
+ token = elems.next();
+ }
+ rep.push_back(par);
+ }
+ repeat = true;
+ if (def->m_params.size() != rep.size())
+ error(pfmt("Expected {1} parameters, got {2}")(def->m_params.size(), rep.size()));
+ auto r(simple_iter<ppreprocessor>(this, tokenize(def->m_replace, m_expr_sep, false, false)));
+ bool stringify_next = false;
+ while (!r.eod())
+ {
+ pstring token(r.next());
+ if (token == "#")
+ stringify_next = true;
+ else if (token != " " && token != "\t")
+ {
+ for (std::size_t i=0; i<def->m_params.size(); i++)
+ if (def->m_params[i] == token)
+ {
+ if (stringify_next)
+ {
+ stringify_next = false;
+ token = "\"" + rep[i] + "\"";
+ }
+ else
+ token = rep[i];
+ break;
+ }
+ if (stringify_next)
+ error("'#' is not followed by a macro parameter");
+ tmpret += token;
+ tmpret += " "; // make sure this is not concatenated with next token
+ }
+ else
+ tmpret += token;
+ }
+ }
+ }
+ } while (repeat);
+
+ return tmpret;
}
static pstring catremainder(const std::vector<pstring> &elems, std::size_t start, const pstring &sep)
@@ -476,7 +713,7 @@ pstring ppreprocessor::process_comments(pstring line)
return ret;
}
-pstring ppreprocessor::process_line(pstring line)
+std::pair<pstring,bool> ppreprocessor::process_line(pstring line)
{
bool line_cont = plib::right(line, 1) == "\\";
if (line_cont)
@@ -490,7 +727,7 @@ pstring ppreprocessor::process_line(pstring line)
if (line_cont)
{
m_state = LINE_CONTINUATION;
- return "";
+ return {"", false};
}
else
m_state = PROCESS;
@@ -498,18 +735,20 @@ pstring ppreprocessor::process_line(pstring line)
line = process_comments(m_line);
pstring lt = plib::trim(plib::replace_all(line, "\t", " "));
- pstring ret;
// FIXME ... revise and extend macro handling
if (plib::startsWith(lt, "#"))
{
- std::vector<pstring> lti(psplit(lt, " ", true));
+ string_list lti(psplit(lt, " ", true));
if (lti[0] == "#if")
{
m_if_level++;
- std::size_t start = 0;
lt = replace_macros(lt);
- std::vector<pstring> t(psplit(replace_all(lt.substr(3), " ", ""), m_expr_sep));
- auto val = static_cast<int>(expr(t, start, 255));
+ //std::vector<pstring> t(psplit(replace_all(lt.substr(3), " ", ""), m_expr_sep));
+ auto t(simple_iter<ppreprocessor>(this, tokenize(lt.substr(3), m_expr_sep, true, true)));
+ auto val = static_cast<int>(prepro_expr(t, 255));
+ t.skip_ws();
+ if (!t.eod())
+ error("found unprocessed content at end of line");
if (val == 0)
m_if_flag |= (1 << m_if_level);
}
@@ -548,18 +787,18 @@ pstring ppreprocessor::process_line(pstring line)
{
arg = arg.substr(1, arg.length() - 2);
/* first try local context */
- auto l(plib::util::buildpath({m_stack.top().m_local_path, arg}));
+ auto l(plib::util::buildpath({m_stack.back().m_local_path, arg}));
auto lstrm(m_sources.get_stream<>(l));
if (lstrm)
{
- m_stack.emplace(input_context(std::move(lstrm), plib::util::path(l), l));
+ m_stack.emplace_back(input_context(std::move(lstrm), plib::util::path(l), l));
}
else
{
auto strm(m_sources.get_stream<>(arg));
if (strm)
{
- m_stack.emplace(input_context(std::move(strm), plib::util::path(arg), arg));
+ m_stack.emplace_back(input_context(std::move(strm), plib::util::path(arg), arg));
}
else
error("include not found:" + arg);
@@ -567,6 +806,8 @@ pstring ppreprocessor::process_line(pstring line)
}
else
error("include misspelled:" + arg);
+ pstring linemarker = pfmt("# {1} \"{2}\" 1\n")(m_stack.back().m_lineno, m_stack.back().m_name);
+ push_out(linemarker);
}
}
else if (lti[0] == "#pragma")
@@ -582,52 +823,93 @@ pstring ppreprocessor::process_line(pstring line)
if (m_if_flag == 0)
{
if (lti.size() < 2)
- error("define needs at least one argument: " + line);
- else if (lti.size() == 2)
- m_defines.insert({lti[1], define_t(lti[1], "")});
+ error("define needs at least one argument");
+ auto args(simple_iter<ppreprocessor>(this, tokenize(lt.substr(8), m_expr_sep, false, false)));
+ pstring n = args.next();
+ if (!is_valid_token(n))
+ error("define expected identifier");
+ if (args.next_ws() == "(")
+ {
+ define_t def(n);
+ def.m_has_params = true;
+ auto token(args.next());
+ while (true)
+ {
+ if (token == ")")
+ break;
+ def.m_params.push_back(token);
+ token = args.next();
+ if (token != "," && token != ")")
+ error(pfmt("expected , or ), found <{1}>")(token));
+ if (token == ",")
+ token = args.next();
+ }
+ pstring r;
+ while (!args.eod())
+ r += args.next_ws();
+ def.m_replace = r;
+ m_defines.insert({n, def});
+ }
else
{
- pstring arg("");
- for (std::size_t i=2; i<lti.size() - 1; i++)
- arg += lti[i] + " ";
- arg += lti[lti.size()-1];
- m_defines.insert({lti[1], define_t(lti[1], arg)});
+ pstring r;
+ while (!args.eod())
+ r += args.next_ws();
+ m_defines.insert({n, define_t(n, r)});
}
}
}
else
{
if (m_if_flag == 0)
- error(pfmt("unknown directive on line {1}: {2}")(m_stack.top().m_lineno)(replace_macros(line)));
+ error("unknown directive");
}
+ return { "", false };
}
else
{
- lt = replace_macros(lt);
if (m_if_flag == 0)
- ret += lt;
+ return { replace_macros(lt), true };
+ else
+ return { "", false };
}
- return ret;
}
+void ppreprocessor::push_out(pstring s)
+{
+ m_outbuf += decltype(m_outbuf)(s.c_str());
+ if (m_debug_out)
+ std::cerr << s;
+}
+
+
void ppreprocessor::process_stack()
{
while (m_stack.size() > 0)
{
pstring line;
- pstring linemarker = pfmt("# {1} \"{2}\" 1\n")(m_stack.top().m_lineno, m_stack.top().m_name);
- m_outbuf += decltype(m_outbuf)(linemarker.c_str());
- while (m_stack.top().m_reader.readline(line))
+ pstring linemarker = pfmt("# {1} \"{2}\"\n")(m_stack.back().m_lineno, m_stack.back().m_name);
+ push_out(linemarker);
+ bool last_skipped=false;
+ while (m_stack.back().m_reader.readline(line))
{
- m_stack.top().m_lineno++;
- line = process_line(line);
- m_outbuf += decltype(m_outbuf)(line.c_str()) + static_cast<char>(10);
+ m_stack.back().m_lineno++;
+ auto r(process_line(line));
+ if (r.second)
+ {
+ if (last_skipped)
+ push_out(pfmt("# {1} \"{2}\"\n")(m_stack.back().m_lineno, m_stack.back().m_name));
+ push_out(r.first + "\n");
+ last_skipped = false;
+ }
+ else
+ last_skipped = true;
}
- m_stack.pop();
+ m_stack.pop_back();
if (m_stack.size() > 0)
{
- linemarker = pfmt("# {1} \"{2}\" 2\n")(m_stack.top().m_lineno, m_stack.top().m_name);
- m_outbuf += decltype(m_outbuf)(linemarker.c_str());
+ linemarker = pfmt("# {1} \"{2}\" 2\n")(m_stack.back().m_lineno, m_stack.back().m_name);
+ push_out(linemarker);
}
}
}