summaryrefslogtreecommitdiffstatshomepage
path: root/src/lib/netlist/plib/ptokenizer.cpp
diff options
context:
space:
mode:
author couriersud <couriersud@gmx.org>2020-08-05 21:43:54 +0200
committer couriersud <couriersud@gmx.org>2020-08-06 09:44:59 +0200
commitd524688df1535b2825773cbdf430bf5377aff189 (patch)
treee7494fd21962fc50ee27e296c9d9f3104301b278 /src/lib/netlist/plib/ptokenizer.cpp
parentbca7016758229428934398a13406847604d77152 (diff)
netlist: Refactored pokenizer.
* Separated tokenizing and reading of tokens. * This enables caching of parsing results on the token level. * Implemented caching of token stream. * Overall this significantly improves parsing performance (~10x) * Next step towards a bare-bone nltool which does not depend on macro devices.
Diffstat (limited to 'src/lib/netlist/plib/ptokenizer.cpp')
-rw-r--r--src/lib/netlist/plib/ptokenizer.cpp98
1 files changed, 58 insertions, 40 deletions
diff --git a/src/lib/netlist/plib/ptokenizer.cpp b/src/lib/netlist/plib/ptokenizer.cpp
index a0542b61a86..cc3d6538f8d 100644
--- a/src/lib/netlist/plib/ptokenizer.cpp
+++ b/src/lib/netlist/plib/ptokenizer.cpp
@@ -1,9 +1,9 @@
// license:GPL-2.0+
// copyright-holders:Couriersud
-#include "ptokenizer.h"
#include "palloc.h"
#include "pstonum.h"
+#include "ptokenizer.h"
#include "putil.h"
namespace plib {
@@ -21,11 +21,6 @@ namespace plib {
// A simple tokenizer
// ----------------------------------------------------------------------------------------
- pstring ptokenizer::currentline_str() const
- {
- return m_cur_line;
- }
-
void ptokenizer::skipeol()
{
pstring::value_type c = getc();
@@ -52,9 +47,13 @@ namespace plib {
}
if (m_px == m_cur_line.end())
{
- ++m_source_location.back();
- if (m_strm.readline(m_cur_line))
+ //++m_source_location.back();
+ if (m_strm->readline(m_cur_line))
+ {
m_px = m_cur_line.begin();
+ if (*m_px != '#')
+ m_token_queue->push_back(token_t(token_type::SOURCELINE, m_cur_line));
+ }
else
return 0;
return '\n';
@@ -68,23 +67,19 @@ namespace plib {
m_unget = c;
}
- void ptokenizer::require_token(const token_id_t &token_num)
+ void ptoken_reader::require_token(const token_id_t &token_num)
{
require_token(get_token(), token_num);
}
- void ptokenizer::require_token(const token_t &tok, const token_id_t &token_num)
+ void ptoken_reader::require_token(const token_t &tok, const token_id_t &token_num)
{
if (!tok.is(token_num))
{
- pstring val("");
- for (auto &i : m_tokens)
- if (i.second.id() == token_num.id())
- val = i.first;
- error(MF_EXPECTED_TOKEN_1_GOT_2(val, tok.str()));
+ error(MF_EXPECTED_TOKEN_1_GOT_2(token_num.name(), tok.str()));
}
}
- pstring ptokenizer::get_string()
+ pstring ptoken_reader::get_string()
{
token_t tok = get_token();
if (!tok.is_type(token_type::STRING))
@@ -95,7 +90,7 @@ namespace plib {
}
- pstring ptokenizer::get_identifier()
+ pstring ptoken_reader::get_identifier()
{
token_t tok = get_token();
if (!tok.is_type(token_type::IDENTIFIER))
@@ -105,7 +100,7 @@ namespace plib {
return tok.str();
}
- pstring ptokenizer::get_identifier_or_number()
+ pstring ptoken_reader::get_identifier_or_number()
{
token_t tok = get_token();
if (!(tok.is_type(token_type::IDENTIFIER) || tok.is_type(token_type::NUMBER)))
@@ -116,7 +111,7 @@ namespace plib {
}
// FIXME: combine into template
- double ptokenizer::get_number_double()
+ double ptoken_reader::get_number_double()
{
token_t tok = get_token();
if (!tok.is_type(token_type::NUMBER))
@@ -130,7 +125,7 @@ namespace plib {
return ret;
}
- long ptokenizer::get_number_long()
+ long ptoken_reader::get_number_long()
{
token_t tok = get_token();
if (!tok.is_type(token_type::NUMBER))
@@ -144,30 +139,31 @@ namespace plib {
return ret;
}
- ptokenizer::token_t ptokenizer::get_token()
+ ptoken_reader::token_t ptoken_reader::get_token()
{
- token_t ret = get_token_internal();
+ token_t ret = get_token_queue();
while (true)
{
if (ret.is_type(token_type::token_type::ENDOFFILE))
return ret;
- if (m_support_line_markers && ret.is_type(token_type::LINEMARKER))
+ //printf("%s\n", ret.str().c_str());
+ if (ret.is_type(token_type::LINEMARKER))
{
bool benter(false);
bool bexit(false);
pstring file;
unsigned lineno(0);
- ret = get_token_internal();
+ ret = get_token_queue();
if (!ret.is_type(token_type::NUMBER))
error(MF_EXPECTED_LINENUM_GOT_1(ret.str()));
lineno = pstonum<unsigned>(ret.str());
- ret = get_token_internal();
+ ret = get_token_queue();
if (!ret.is_type(token_type::STRING))
error(MF_EXPECTED_FILENAME_GOT_1(ret.str()));
file = ret.str();
- ret = get_token_internal();
+ ret = get_token_queue();
while (ret.is_type(token_type::NUMBER))
{
if (ret.str() == "1")
@@ -175,7 +171,7 @@ namespace plib {
if (ret.str() == "2")
bexit = false;
// FIXME: process flags; actually only 1 (file enter) and 2 (after file exit)
- ret = get_token_internal();
+ ret = get_token_queue();
}
if (bexit) // pop the last location
m_source_location.pop_back();
@@ -183,17 +179,11 @@ namespace plib {
m_source_location.pop_back();
m_source_location.emplace_back(plib::source_location(file, lineno));
}
- else if (ret.is(m_tok_comment_start))
+ else if (ret.is_type(token_type::SOURCELINE))
{
- do {
- ret = get_token_internal();
- } while (ret.is_not(m_tok_comment_end));
- ret = get_token_internal();
- }
- else if (ret.is(m_tok_line_comment))
- {
- skipeol();
- ret = get_token_internal();
+ m_line = ret.str();
+ ++m_source_location.back();
+ ret = get_token_queue();
}
else
{
@@ -202,7 +192,7 @@ namespace plib {
}
}
- ptokenizer::token_t ptokenizer::get_token_internal()
+ ptoken_reader::token_t ptokenizer::get_token_internal()
{
// skip ws
pstring::value_type c = getc();
@@ -287,7 +277,35 @@ namespace plib {
}
}
- void ptokenizer::error(const perrmsg &errs)
+ ptoken_reader::token_t ptokenizer::get_token_comment()
+ {
+ token_t ret = get_token_internal();
+ while (true)
+ {
+ if (ret.is_type(token_type::token_type::ENDOFFILE))
+ return ret;
+
+ if (ret.is(m_tok_comment_start))
+ {
+ do {
+ ret = get_token_internal();
+ } while (ret.is_not(m_tok_comment_end));
+ ret = get_token_internal();
+ }
+ else if (ret.is(m_tok_line_comment))
+ {
+ skipeol();
+ ret = get_token_internal();
+ }
+ else
+ {
+ return ret;
+ }
+ }
+ }
+
+
+ void ptoken_reader::error(const perrmsg &errs)
{
pstring s("");
pstring trail (" from ");
@@ -302,7 +320,7 @@ namespace plib {
s = plib::pfmt("{1}{2}:{3}:0\n{4}")(trail, m_source_location.back().file_name(), m_source_location.back().line(), s);
m_source_location.pop_back();
}
- verror("\n" + s + e + " " + currentline_str() + "\n");
+ verror("\n" + s + e + " " + m_line + "\n");
}
} // namespace plib