diff options
author | 2020-08-05 21:43:54 +0200 | |
---|---|---|
committer | 2020-08-06 09:44:59 +0200 | |
commit | d524688df1535b2825773cbdf430bf5377aff189 (patch) | |
tree | e7494fd21962fc50ee27e296c9d9f3104301b278 /src/lib/netlist/plib/ptokenizer.cpp | |
parent | bca7016758229428934398a13406847604d77152 (diff) |
netlist: Refactored pokenizer.
* Separated tokenizing and reading of tokens.
* This enables caching of parsing results on the token level.
* Implemented caching of token stream.
* Overall this significantly improves parsing performance (~10x)
* Next step towards a bare-bone nltool which does not depend on macro
devices.
Diffstat (limited to 'src/lib/netlist/plib/ptokenizer.cpp')
-rw-r--r-- | src/lib/netlist/plib/ptokenizer.cpp | 98 |
1 files changed, 58 insertions, 40 deletions
diff --git a/src/lib/netlist/plib/ptokenizer.cpp b/src/lib/netlist/plib/ptokenizer.cpp index a0542b61a86..cc3d6538f8d 100644 --- a/src/lib/netlist/plib/ptokenizer.cpp +++ b/src/lib/netlist/plib/ptokenizer.cpp @@ -1,9 +1,9 @@ // license:GPL-2.0+ // copyright-holders:Couriersud -#include "ptokenizer.h" #include "palloc.h" #include "pstonum.h" +#include "ptokenizer.h" #include "putil.h" namespace plib { @@ -21,11 +21,6 @@ namespace plib { // A simple tokenizer // ---------------------------------------------------------------------------------------- - pstring ptokenizer::currentline_str() const - { - return m_cur_line; - } - void ptokenizer::skipeol() { pstring::value_type c = getc(); @@ -52,9 +47,13 @@ namespace plib { } if (m_px == m_cur_line.end()) { - ++m_source_location.back(); - if (m_strm.readline(m_cur_line)) + //++m_source_location.back(); + if (m_strm->readline(m_cur_line)) + { m_px = m_cur_line.begin(); + if (*m_px != '#') + m_token_queue->push_back(token_t(token_type::SOURCELINE, m_cur_line)); + } else return 0; return '\n'; @@ -68,23 +67,19 @@ namespace plib { m_unget = c; } - void ptokenizer::require_token(const token_id_t &token_num) + void ptoken_reader::require_token(const token_id_t &token_num) { require_token(get_token(), token_num); } - void ptokenizer::require_token(const token_t &tok, const token_id_t &token_num) + void ptoken_reader::require_token(const token_t &tok, const token_id_t &token_num) { if (!tok.is(token_num)) { - pstring val(""); - for (auto &i : m_tokens) - if (i.second.id() == token_num.id()) - val = i.first; - error(MF_EXPECTED_TOKEN_1_GOT_2(val, tok.str())); + error(MF_EXPECTED_TOKEN_1_GOT_2(token_num.name(), tok.str())); } } - pstring ptokenizer::get_string() + pstring ptoken_reader::get_string() { token_t tok = get_token(); if (!tok.is_type(token_type::STRING)) @@ -95,7 +90,7 @@ namespace plib { } - pstring ptokenizer::get_identifier() + pstring ptoken_reader::get_identifier() { token_t tok = get_token(); if (!tok.is_type(token_type::IDENTIFIER)) @@ -105,7 +100,7 @@ namespace plib { return tok.str(); } - pstring ptokenizer::get_identifier_or_number() + pstring ptoken_reader::get_identifier_or_number() { token_t tok = get_token(); if (!(tok.is_type(token_type::IDENTIFIER) || tok.is_type(token_type::NUMBER))) @@ -116,7 +111,7 @@ namespace plib { } // FIXME: combine into template - double ptokenizer::get_number_double() + double ptoken_reader::get_number_double() { token_t tok = get_token(); if (!tok.is_type(token_type::NUMBER)) @@ -130,7 +125,7 @@ namespace plib { return ret; } - long ptokenizer::get_number_long() + long ptoken_reader::get_number_long() { token_t tok = get_token(); if (!tok.is_type(token_type::NUMBER)) @@ -144,30 +139,31 @@ namespace plib { return ret; } - ptokenizer::token_t ptokenizer::get_token() + ptoken_reader::token_t ptoken_reader::get_token() { - token_t ret = get_token_internal(); + token_t ret = get_token_queue(); while (true) { if (ret.is_type(token_type::token_type::ENDOFFILE)) return ret; - if (m_support_line_markers && ret.is_type(token_type::LINEMARKER)) + //printf("%s\n", ret.str().c_str()); + if (ret.is_type(token_type::LINEMARKER)) { bool benter(false); bool bexit(false); pstring file; unsigned lineno(0); - ret = get_token_internal(); + ret = get_token_queue(); if (!ret.is_type(token_type::NUMBER)) error(MF_EXPECTED_LINENUM_GOT_1(ret.str())); lineno = pstonum<unsigned>(ret.str()); - ret = get_token_internal(); + ret = get_token_queue(); if (!ret.is_type(token_type::STRING)) error(MF_EXPECTED_FILENAME_GOT_1(ret.str())); file = ret.str(); - ret = get_token_internal(); + ret = get_token_queue(); while (ret.is_type(token_type::NUMBER)) { if (ret.str() == "1") @@ -175,7 +171,7 @@ namespace plib { if (ret.str() == "2") bexit = false; // FIXME: process flags; actually only 1 (file enter) and 2 (after file exit) - ret = get_token_internal(); + ret = get_token_queue(); } if (bexit) // pop the last location m_source_location.pop_back(); @@ -183,17 +179,11 @@ namespace plib { m_source_location.pop_back(); m_source_location.emplace_back(plib::source_location(file, lineno)); } - else if (ret.is(m_tok_comment_start)) + else if (ret.is_type(token_type::SOURCELINE)) { - do { - ret = get_token_internal(); - } while (ret.is_not(m_tok_comment_end)); - ret = get_token_internal(); - } - else if (ret.is(m_tok_line_comment)) - { - skipeol(); - ret = get_token_internal(); + m_line = ret.str(); + ++m_source_location.back(); + ret = get_token_queue(); } else { @@ -202,7 +192,7 @@ namespace plib { } } - ptokenizer::token_t ptokenizer::get_token_internal() + ptoken_reader::token_t ptokenizer::get_token_internal() { // skip ws pstring::value_type c = getc(); @@ -287,7 +277,35 @@ namespace plib { } } - void ptokenizer::error(const perrmsg &errs) + ptoken_reader::token_t ptokenizer::get_token_comment() + { + token_t ret = get_token_internal(); + while (true) + { + if (ret.is_type(token_type::token_type::ENDOFFILE)) + return ret; + + if (ret.is(m_tok_comment_start)) + { + do { + ret = get_token_internal(); + } while (ret.is_not(m_tok_comment_end)); + ret = get_token_internal(); + } + else if (ret.is(m_tok_line_comment)) + { + skipeol(); + ret = get_token_internal(); + } + else + { + return ret; + } + } + } + + + void ptoken_reader::error(const perrmsg &errs) { pstring s(""); pstring trail (" from "); @@ -302,7 +320,7 @@ namespace plib { s = plib::pfmt("{1}{2}:{3}:0\n{4}")(trail, m_source_location.back().file_name(), m_source_location.back().line(), s); m_source_location.pop_back(); } - verror("\n" + s + e + " " + currentline_str() + "\n"); + verror("\n" + s + e + " " + m_line + "\n"); } } // namespace plib |