diff options
Diffstat (limited to 'src/lib/netlist/plib/ptokenizer.h')
-rw-r--r-- | src/lib/netlist/plib/ptokenizer.h | 169 |
1 files changed, 169 insertions, 0 deletions
diff --git a/src/lib/netlist/plib/ptokenizer.h b/src/lib/netlist/plib/ptokenizer.h new file mode 100644 index 00000000000..5a6b886a2a2 --- /dev/null +++ b/src/lib/netlist/plib/ptokenizer.h @@ -0,0 +1,169 @@ +// license:GPL-2.0+ +// copyright-holders:Couriersud +/* + * ptokenizer.h + * + */ + +#ifndef PTOKENIZER_H_ +#define PTOKENIZER_H_ + +#include "plists.h" +#include "pstream.h" +#include "pstring.h" + +#include "putil.h" // psource_t + +#include <unordered_map> +#include <vector> + +namespace plib { + + class ptokenizer + { + public: + template <typename T> + ptokenizer(T &&strm) // NOLINT(misc-forwarding-reference-overload, bugprone-forwarding-reference-overload) + : m_strm(std::forward<T>(strm)) + , m_cur_line("") + , m_px(m_cur_line.begin()) + , m_unget(0) + , m_string('"') + , m_support_line_markers(true) // FIXME + { + // add a first entry to the stack + m_source_location.emplace_back(plib::source_location("Unknown", 0)); + } + + COPYASSIGNMOVE(ptokenizer, delete) + + virtual ~ptokenizer() = default; + + enum token_type + { + IDENTIFIER, + NUMBER, + TOKEN, + STRING, + COMMENT, + LINEMARKER, + UNKNOWN, + ENDOFFILE + }; + + struct token_id_t + { + public: + + static constexpr std::size_t npos = static_cast<std::size_t>(-1); + + token_id_t() : m_id(npos) {} + explicit token_id_t(const std::size_t id) : m_id(id) {} + std::size_t id() const { return m_id; } + private: + std::size_t m_id; + }; + + struct token_t + { + explicit token_t(token_type type) + : m_type(type), m_id(), m_token("") + { + } + token_t(token_type type, const pstring &str) + : m_type(type), m_id(), m_token(str) + { + } + token_t(const token_id_t &id, const pstring &str) + : m_type(TOKEN), m_id(id), m_token(str) + { + } + + bool is(const token_id_t &tok_id) const { return m_id.id() == tok_id.id(); } + bool is_not(const token_id_t &tok_id) const { return !is(tok_id); } + + bool is_type(const token_type type) const { return m_type == type; } + + pstring str() const { return m_token; } + + private: + token_type m_type; + token_id_t m_id; + pstring m_token; + }; + + pstring currentline_str(); + + /* tokenizer stuff follows ... */ + + token_t get_token(); + pstring get_string(); + pstring get_identifier(); + pstring get_identifier_or_number(); + double get_number_double(); + long get_number_long(); + + void require_token(const token_id_t &token_num); + void require_token(const token_t &tok, const token_id_t &token_num); + + token_id_t register_token(const pstring &token) + { + token_id_t ret(m_tokens.size()); + m_tokens.emplace(token, ret); + return ret; + } + + ptokenizer & identifier_chars(const pstring &s) { m_identifier_chars = s; return *this; } + ptokenizer & number_chars(const pstring &st, const pstring & rem) { m_number_chars_start = st; m_number_chars = rem; return *this; } + ptokenizer & string_char(pstring::value_type c) { m_string = c; return *this; } + ptokenizer & whitespace(const pstring & s) { m_whitespace = s; return *this; } + ptokenizer & comment(const pstring &start, const pstring &end, const pstring &line) + { + m_tok_comment_start = register_token(start); + m_tok_comment_end = register_token(end); + m_tok_line_comment = register_token(line); + return *this; + } + + token_t get_token_internal(); + void error(const pstring &errs); + + putf8_reader &stream() { return m_strm; } + protected: + virtual void verror(const pstring &msg) = 0; + + private: + void skipeol(); + + pstring::value_type getc(); + void ungetc(pstring::value_type c); + + bool eof() { return m_strm.eof(); } + + putf8_reader m_strm; + + pstring m_cur_line; + pstring::const_iterator m_px; + pstring::value_type m_unget; + + /* tokenizer stuff follows ... */ + + pstring m_identifier_chars; + pstring m_number_chars; + pstring m_number_chars_start; + std::unordered_map<pstring, token_id_t> m_tokens; + pstring m_whitespace; + pstring::value_type m_string; + + token_id_t m_tok_comment_start; + token_id_t m_tok_comment_end; + token_id_t m_tok_line_comment; + + /* source locations, vector used as stack because we need to loop through stack */ + bool m_support_line_markers; + std::vector<plib::source_location> m_source_location; + }; + +} // namespace plib + +#endif /* PTOKENIZER_H_ */ |