// license:GPL-2.0+ // copyright-holders:Couriersud /* * pparser.c * */ #include "pparser.h" #include "palloc.h" #include "putil.h" namespace plib { // ---------------------------------------------------------------------------------------- // A simple tokenizer // ---------------------------------------------------------------------------------------- pstring ptokenizer::currentline_str() { return m_cur_line; } void ptokenizer::skipeol() { pstring::value_type c = getc(); while (c) { if (c == 10) { c = getc(); if (c != 13) ungetc(c); return; } c = getc(); } } pstring::value_type ptokenizer::getc() { if (m_unget != 0) { pstring::value_type c = m_unget; m_unget = 0; return c; } if (m_px == m_cur_line.end()) { ++m_source_location.back(); if (m_strm.readline(m_cur_line)) m_px = m_cur_line.begin(); else return 0; return '\n'; } pstring::value_type c = *(m_px++); return c; } void ptokenizer::ungetc(pstring::value_type c) { m_unget = c; } void ptokenizer::require_token(const token_id_t &token_num) { require_token(get_token(), token_num); } void ptokenizer::require_token(const token_t &tok, const token_id_t &token_num) { if (!tok.is(token_num)) { pstring val(""); for (auto &i : m_tokens) if (i.second.id() == token_num.id()) val = i.first; error(pfmt("Expected token <{1}> got <{2}>")(val)(tok.str()) ); } } pstring ptokenizer::get_string() { token_t tok = get_token(); if (!tok.is_type(STRING)) { error(pfmt("Expected a string, got <{1}>")(tok.str()) ); } return tok.str(); } pstring ptokenizer::get_identifier() { token_t tok = get_token(); if (!tok.is_type(IDENTIFIER)) { error(pfmt("Expected an identifier, got <{1}>")(tok.str()) ); } return tok.str(); } pstring ptokenizer::get_identifier_or_number() { token_t tok = get_token(); if (!(tok.is_type(IDENTIFIER) || tok.is_type(NUMBER))) { error(pfmt("Expected an identifier or number, got <{1}>")(tok.str()) ); } return tok.str(); } // FIXME: combine into template double ptokenizer::get_number_double() { token_t tok = get_token(); if (!tok.is_type(NUMBER)) { error(pfmt("Expected a number, got <{1}>")(tok.str()) ); } bool err(false); auto ret = plib::pstonum_ne(tok.str(), err); if (err) error(pfmt("Expected a number, got <{1}>")(tok.str()) ); return ret; } long ptokenizer::get_number_long() { token_t tok = get_token(); if (!tok.is_type(NUMBER)) { error(pfmt("Expected a long int, got <{1}>")(tok.str()) ); } bool err(false); auto ret = plib::pstonum_ne(tok.str(), err); if (err) error(pfmt("Expected a long int, got <{1}>")(tok.str()) ); return ret; } ptokenizer::token_t ptokenizer::get_token() { token_t ret = get_token_internal(); while (true) { if (ret.is_type(ENDOFFILE)) return ret; else if (m_support_line_markers && ret.is_type(LINEMARKER)) { bool benter(false); bool bexit(false); pstring file; unsigned lineno; ret = get_token_internal(); if (!ret.is_type(NUMBER)) error(pfmt("Expected line number after line marker but got <{1}>")(ret.str()) ); lineno = pstonum(ret.str()); ret = get_token_internal(); if (!ret.is_type(STRING)) error(pfmt("Expected file name after line marker but got <{1}>")(ret.str()) ); file = ret.str(); ret = get_token_internal(); while (ret.is_type(NUMBER)) { if (ret.str() == "1") benter = true; if (ret.str() == "2") bexit = false; // FIXME: process flags; actually only 1 (file enter) and 2 (after file exit) ret = get_token_internal(); } if (bexit) // pop the last location m_source_location.pop_back(); if (!benter) // new location! m_source_location.pop_back(); m_source_location.emplace_back(plib::source_location(file, lineno)); } else if (ret.is(m_tok_comment_start)) { do { ret = get_token_internal(); } while (ret.is_not(m_tok_comment_end)); ret = get_token_internal(); } else if (ret.is(m_tok_line_comment)) { skipeol(); ret = get_token_internal(); } else { return ret; } } } ptokenizer::token_t ptokenizer::get_token_internal() { /* skip ws */ pstring::value_type c = getc(); while (m_whitespace.find(c) != pstring::npos) { c = getc(); if (eof()) { return token_t(ENDOFFILE); } } if (m_support_line_markers && c == '#') return token_t(LINEMARKER, "#"); else if (m_number_chars_start.find(c) != pstring::npos) { /* read number while we receive number or identifier chars * treat it as an identifier when there are identifier chars in it * */ token_type ret = NUMBER; pstring tokstr = ""; while (true) { if (m_identifier_chars.find(c) != pstring::npos && m_number_chars.find(c) == pstring::npos) ret = IDENTIFIER; else if (m_number_chars.find(c) == pstring::npos) break; tokstr += c; c = getc(); } ungetc(c); return token_t(ret, tokstr); } else if (m_identifier_chars.find(c) != pstring::npos) { /* read identifier till non identifier char */ pstring tokstr = ""; while (m_identifier_chars.find(c) != pstring::npos) { tokstr += c; c = getc(); } ungetc(c); auto id = m_tokens.find(tokstr); if (id != m_tokens.end()) return token_t(id->second, tokstr); else return token_t(IDENTIFIER, tokstr); } else if (c == m_string) { pstring tokstr = ""; c = getc(); while (c != m_string) { tokstr += c; c = getc(); } return token_t(STRING, tokstr); } else { /* read identifier till first identifier char or ws */ pstring tokstr = ""; while ((m_identifier_chars.find(c) == pstring::npos) && (m_whitespace.find(c) == pstring::npos)) { tokstr += c; /* expensive, check for single char tokens */ if (tokstr.length() == 1) { auto id = m_tokens.find(tokstr); if (id != m_tokens.end()) return token_t(id->second, tokstr); } c = getc(); } ungetc(c); auto id = m_tokens.find(tokstr); if (id != m_tokens.end()) return token_t(id->second, tokstr); else return token_t(UNKNOWN, tokstr); } } void ptokenizer::error(const pstring &errs) { pstring s(""); pstring trail (" from "); pstring trail_first("In file included from "); pstring e = plib::pfmt("{1}:{2}:0: error: {3}\n") (m_source_location.back().file_name(), m_source_location.back().line(), errs); m_source_location.pop_back(); while (m_source_location.size() > 0) { if (m_source_location.size() == 1) trail = trail_first; s = trail + plib::pfmt("{1}:{2}:0\n")(m_source_location.back().file_name(), m_source_location.back().line()) + s; m_source_location.pop_back(); } verror("\n" + s + e + " " + currentline_str() + "\n"); } // ---------------------------------------------------------------------------------------- // A simple preprocessor // ---------------------------------------------------------------------------------------- ppreprocessor::ppreprocessor(psource_collection_t<> &sources, defines_map_type *defines) : std::istream(new readbuffer(this)) , m_sources(sources) , m_if_flag(0) , m_if_level(0) , m_pos(0) , m_state(PROCESS) , m_comment(false) { m_expr_sep.emplace_back("!"); m_expr_sep.emplace_back("("); m_expr_sep.emplace_back(")"); m_expr_sep.emplace_back("+"); m_expr_sep.emplace_back("-"); m_expr_sep.emplace_back("*"); m_expr_sep.emplace_back("/"); m_expr_sep.emplace_back("&&"); m_expr_sep.emplace_back("||"); m_expr_sep.emplace_back("=="); m_expr_sep.emplace_back(","); m_expr_sep.emplace_back(";"); m_expr_sep.emplace_back("."); m_expr_sep.emplace_back("##"); m_expr_sep.emplace_back("#"); m_expr_sep.emplace_back(" "); m_expr_sep.emplace_back("\t"); m_expr_sep.emplace_back("\""); if (defines != nullptr) m_defines = *defines; m_defines.insert({"__PLIB_PREPROCESSOR__", define_t("__PLIB_PREPROCESSOR__", "1")}); auto idx = m_defines.find("__PREPROCESSOR_DEBUG__"); m_debug_out = idx != m_defines.end(); } void ppreprocessor::error(const pstring &err) { pstring s(""); pstring trail (" from "); pstring trail_first("In file included from "); pstring e = plib::pfmt("{1}:{2}:0: error: {3}\n") (m_stack.back().m_name, m_stack.back().m_lineno, err); m_stack.pop_back(); while (m_stack.size() > 0) { if (m_stack.size() == 1) trail = trail_first; s = trail + plib::pfmt("{1}:{2}:0\n")(m_stack.back().m_name, m_stack.back().m_lineno) + s; m_stack.pop_back(); } throw pexception("\n" + s + e + " " + m_line + "\n"); } template struct simple_iter { simple_iter(PP *parent, const L &tokens) : m_tokens(tokens), m_parent(parent), m_pos(0) {} /**! skip white space in token list * */ void skip_ws() { while (m_pos < m_tokens.size() && (m_tokens[m_pos] == " " || m_tokens[m_pos] == "\t")) m_pos++; } /**! return next token skipping white space * * @return next token */ pstring next() { skip_ws(); if (m_pos >= m_tokens.size()) error("unexpected end of line"); return m_tokens[m_pos++]; } /**! return next token including white space * * @return next token */ pstring next_ws() { if (m_pos >= m_tokens.size()) error("unexpected end of line"); return m_tokens[m_pos++]; } pstring peek_ws() { if (m_pos >= m_tokens.size()) error("unexpected end of line"); return m_tokens[m_pos]; } pstring last() { if (m_pos == 0) error("no last token at beginning of line"); if (m_pos > m_tokens.size()) error("unexpected end of line"); return m_tokens[m_pos-1]; } bool eod() { return (m_pos >= m_tokens.size()); } void error(pstring err) { m_parent->error(err); } private: L m_tokens; PP *m_parent; std::size_t m_pos; }; #define CHECKTOK2(p_op, p_prio) \ else if (tok == # p_op) \ { \ if (!has_val) \ { sexpr.error("parsing error!"); return 1;} \ if (prio < (p_prio)) \ return val; \ sexpr.next(); \ const auto v2 = prepro_expr(sexpr, (p_prio)); \ val = (val p_op v2); \ } \ // Operator precedence see https://en.cppreference.com/w/cpp/language/operator_precedence template static int prepro_expr(simple_iter &sexpr, int prio) { int val(0); bool has_val(false); pstring tok=sexpr.peek_ws(); if (tok == "(") { sexpr.next(); val = prepro_expr(sexpr, 255); if (sexpr.next() != ")") sexpr.error("expected ')'"); has_val = true; } while (!sexpr.eod()) { tok = sexpr.peek_ws(); if (tok == ")") { if (!has_val) sexpr.error("Found ')' but have no value computed"); else return val; } else if (tok == "!") { if (prio < 3) { if (!has_val) sexpr.error("parsing error!"); else return val; } sexpr.next(); val = !prepro_expr(sexpr, 3); has_val = true; } CHECKTOK2(*, 5) CHECKTOK2(/, 5) // NOLINT(clang-analyzer-core.DivideZero) CHECKTOK2(+, 6) CHECKTOK2(-, 6) CHECKTOK2(==, 10) CHECKTOK2(&&, 14) CHECKTOK2(||, 15) else { val = plib::pstonum(tok); has_val = true; sexpr.next(); } } if (!has_val) sexpr.error("No value computed. Empty expression ?"); return val; } ppreprocessor::define_t *ppreprocessor::get_define(const pstring &name) { auto idx = m_defines.find(name); return (idx != m_defines.end()) ? &idx->second : nullptr; } ppreprocessor::string_list ppreprocessor::tokenize(const pstring &str, const string_list &sep, bool remove_ws, bool concat) { const pstring STR = "\""; string_list tmpret; string_list tmp(psplit(str, sep)); std::size_t pi(0); while (pi < tmp.size()) { if (tmp[pi] == STR) { pstring s(STR); pi++; // FIXME : \" while (pi < tmp.size() && tmp[pi] != STR) { s += tmp[pi]; pi++; } s += STR; tmpret.push_back(s); } else if (!remove_ws || (tmp[pi] != " " && tmp[pi] != "\t")) tmpret.push_back(tmp[pi]); pi++; } if (!concat) return tmpret; else { // FIXME: error if concat at beginning or end string_list ret; pi = 0; while (pi='a' && c<='z') || (c>='A' && c<='Z') || c == '_'); } pstring ppreprocessor::replace_macros(const pstring &line) { //std::vector elems(psplit(line, m_expr_sep)); bool repeat(false); pstring tmpret(line); do { repeat = false; auto elems(simple_iter(this, tokenize(tmpret, m_expr_sep, false, true))); tmpret = ""; while (!elems.eod()) { auto token(elems.next_ws()); define_t *def = get_define(token); if (def == nullptr) tmpret += token; else if (!def->m_has_params) { tmpret += def->m_replace; repeat = true; } else { token = elems.next(); if (token != "(") error("expected '(' in macro expansion of " + def->m_name); string_list rep; token = elems.next(); while (token != ")") { pstring par(""); int pcnt(1); while (true) { if (pcnt==1 && token == ",") { token = elems.next(); break; } if (token == "(") pcnt++; if (token == ")") if (--pcnt == 0) break; par += token; token = elems.next(); } rep.push_back(par); } repeat = true; if (def->m_params.size() != rep.size()) error(pfmt("Expected {1} parameters, got {2}")(def->m_params.size(), rep.size())); auto r(simple_iter(this, tokenize(def->m_replace, m_expr_sep, false, false))); bool stringify_next = false; while (!r.eod()) { pstring token(r.next()); if (token == "#") stringify_next = true; else if (token != " " && token != "\t") { for (std::size_t i=0; im_params.size(); i++) if (def->m_params[i] == token) { if (stringify_next) { stringify_next = false; token = "\"" + rep[i] + "\""; } else token = rep[i]; break; } if (stringify_next) error("'#' is not followed by a macro parameter"); tmpret += token; tmpret += " "; // make sure this is not concatenated with next token } else tmpret += token; } } } } while (repeat); return tmpret; } static pstring catremainder(const std::vector &elems, std::size_t start, const pstring &sep) { pstring ret(""); for (std::size_t i = start; i < elems.size(); i++) { ret += elems[i]; ret += sep; } return ret; } pstring ppreprocessor::process_comments(pstring line) { bool in_string = false; std::size_t e = line.size(); pstring ret = ""; for (std::size_t i=0; i < e; ) { pstring c = plib::left(line, 1); line = line.substr(1); if (!m_comment) { if (c=="\"") { in_string = !in_string; ret += c; } else if (in_string && c=="\\") { i++; ret += (c + plib::left(line, 1)); line = line.substr(1); } else if (!in_string && c=="/" && plib::left(line,1) == "*") m_comment = true; else if (!in_string && c=="/" && plib::left(line,1) == "/") break; else ret += c; } else if (c=="*" && plib::left(line,1) == "/") { i++; line = line.substr(1); m_comment = false; } i++; } return ret; } std::pair ppreprocessor::process_line(pstring line) { bool line_cont = plib::right(line, 1) == "\\"; if (line_cont) line = plib::left(line, line.size() - 1); if (m_state == LINE_CONTINUATION) m_line += line; else m_line = line; if (line_cont) { m_state = LINE_CONTINUATION; return {"", false}; } else m_state = PROCESS; line = process_comments(m_line); pstring lt = plib::trim(plib::replace_all(line, "\t", " ")); // FIXME ... revise and extend macro handling if (plib::startsWith(lt, "#")) { string_list lti(psplit(lt, " ", true)); if (lti[0] == "#if") { m_if_level++; lt = replace_macros(lt); //std::vector t(psplit(replace_all(lt.substr(3), " ", ""), m_expr_sep)); auto t(simple_iter(this, tokenize(lt.substr(3), m_expr_sep, true, true))); auto val = static_cast(prepro_expr(t, 255)); t.skip_ws(); if (!t.eod()) error("found unprocessed content at end of line"); if (val == 0) m_if_flag |= (1 << m_if_level); } else if (lti[0] == "#ifdef") { m_if_level++; if (get_define(lti[1]) == nullptr) m_if_flag |= (1 << m_if_level); } else if (lti[0] == "#ifndef") { m_if_level++; if (get_define(lti[1]) != nullptr) m_if_flag |= (1 << m_if_level); } else if (lti[0] == "#else") { m_if_flag ^= (1 << m_if_level); } else if (lti[0] == "#endif") { m_if_flag &= ~(1 << m_if_level); m_if_level--; } else if (lti[0] == "#include") { if (m_if_flag == 0) { pstring arg(""); for (std::size_t i=1; i(l)); if (lstrm) { m_stack.emplace_back(input_context(std::move(lstrm), plib::util::path(l), l)); } else { auto strm(m_sources.get_stream<>(arg)); if (strm) { m_stack.emplace_back(input_context(std::move(strm), plib::util::path(arg), arg)); } else error("include not found:" + arg); } } else error("include misspelled:" + arg); pstring linemarker = pfmt("# {1} \"{2}\" 1\n")(m_stack.back().m_lineno, m_stack.back().m_name); push_out(linemarker); } } else if (lti[0] == "#pragma") { if (m_if_flag == 0 && lti.size() > 3 && lti[1] == "NETLIST") { if (lti[2] == "warning") error("NETLIST: " + catremainder(lti, 3, " ")); } } else if (lti[0] == "#define") { if (m_if_flag == 0) { if (lti.size() < 2) error("define needs at least one argument"); auto args(simple_iter(this, tokenize(lt.substr(8), m_expr_sep, false, false))); pstring n = args.next(); if (!is_valid_token(n)) error("define expected identifier"); if (args.next_ws() == "(") { define_t def(n); def.m_has_params = true; auto token(args.next()); while (true) { if (token == ")") break; def.m_params.push_back(token); token = args.next(); if (token != "," && token != ")") error(pfmt("expected , or ), found <{1}>")(token)); if (token == ",") token = args.next(); } pstring r; while (!args.eod()) r += args.next_ws(); def.m_replace = r; m_defines.insert({n, def}); } else { pstring r; while (!args.eod()) r += args.next_ws(); m_defines.insert({n, define_t(n, r)}); } } } else { if (m_if_flag == 0) error("unknown directive"); } return { "", false }; } else { if (m_if_flag == 0) return { replace_macros(lt), true }; else return { "", false }; } } void ppreprocessor::push_out(pstring s) { m_outbuf += decltype(m_outbuf)(s.c_str()); if (m_debug_out) std::cerr << s; } void ppreprocessor::process_stack() { while (m_stack.size() > 0) { pstring line; pstring linemarker = pfmt("# {1} \"{2}\"\n")(m_stack.back().m_lineno, m_stack.back().m_name); push_out(linemarker); bool last_skipped=false; while (m_stack.back().m_reader.readline(line)) { m_stack.back().m_lineno++; auto r(process_line(line)); if (r.second) { if (last_skipped) push_out(pfmt("# {1} \"{2}\"\n")(m_stack.back().m_lineno, m_stack.back().m_name)); push_out(r.first + "\n"); last_skipped = false; } else last_skipped = true; } m_stack.pop_back(); if (m_stack.size() > 0) { linemarker = pfmt("# {1} \"{2}\" 2\n")(m_stack.back().m_lineno, m_stack.back().m_name); push_out(linemarker); } } } } // namespace plib