1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
|
// license:BSD-3-Clause
// copyright-holders:Couriersud
#ifndef PTOKENIZER_H_
#define PTOKENIZER_H_
///
/// \file ptokenizer.h
///
#include "pstream.h"
#include "pstring.h"
#include "penum.h"
#include "psource.h"
#include <unordered_map>
#include <vector>
namespace plib {
namespace detail {
PENUM(token_type,
IDENTIFIER,
NUMBER,
TOKEN,
STRING,
COMMENT,
LINEMARKER,
SOURCELINE,
UNKNOWN,
ENDOFFILE
)
struct token_id_t
{
public:
static constexpr std::size_t npos = static_cast<std::size_t>(-1);
token_id_t()
: m_id(npos)
{}
explicit token_id_t(std::size_t id, const pstring &name)
: m_id(id)
, m_name(name)
{}
token_id_t(const token_id_t &) = default;
token_id_t &operator=(const token_id_t &) = default;
token_id_t(token_id_t &&) noexcept = default;
token_id_t &operator=(token_id_t &&) noexcept = default;
~token_id_t() = default;
constexpr std::size_t id() const { return m_id; }
constexpr const pstring & name() const { return m_name; }
private:
std::size_t m_id;
pstring m_name;
};
struct token_t
{
explicit token_t(token_type type)
: m_type(type), m_id(token_id_t::npos), m_token("")
{
}
token_t(token_type type, const pstring &str)
: m_type(type), m_id(token_id_t::npos), m_token(str)
{
}
token_t(const token_id_t &id)
: m_type(token_type::TOKEN), m_id(id.id()), m_token(id.name())
{
}
token_t(const token_id_t &id, const pstring &str)
: m_type(token_type::TOKEN), m_id(id.id()), m_token(str)
{
}
token_t(const token_t &) = default;
token_t &operator=(const token_t &) = default;
token_t(token_t &&) noexcept = default;
token_t &operator=(token_t &&) noexcept = default;
~token_t() = default;
constexpr bool is(const token_id_t &tok_id) const noexcept { return m_id == tok_id.id(); }
constexpr bool is_not(const token_id_t &tok_id) const noexcept { return !is(tok_id); }
constexpr bool is_type(const token_type type) const noexcept { return m_type == type; }
constexpr token_type type() const noexcept { return m_type; }
constexpr const pstring &str() const noexcept { return m_token; }
private:
token_type m_type;
std::size_t m_id;
pstring m_token;
};
class token_store_t : public std::vector<token_t>
{
using std::vector<token_t>::vector;
};
} // namespace detail
class tokenizer_t
{
public:
using token_type = detail::token_type;
using token_id_t = detail::token_id_t;
using token_t = detail::token_t;
using token_store_t = detail::token_store_t;
explicit tokenizer_t() // NOLINT(misc-forwarding-reference-overload, bugprone-forwarding-reference-overload)
: m_strm(nullptr)
, m_unget(0)
, m_string('"')
, m_support_line_markers(true) // FIXME
, m_token_queue(nullptr)
{
clear();
}
tokenizer_t(const tokenizer_t &) = delete;
tokenizer_t &operator=(const tokenizer_t &) = delete;
tokenizer_t(tokenizer_t &&) noexcept = delete;
tokenizer_t &operator=(tokenizer_t &&) noexcept = delete;
virtual ~tokenizer_t() = default;
// tokenizer stuff follows ...
token_id_t register_token(const pstring &token)
{
token_id_t ret(m_tokens.size(), token);
m_tokens.emplace(token, ret);
return ret;
}
tokenizer_t & identifier_chars(const pstring &s) { m_identifier_chars = s; return *this; }
tokenizer_t & number_chars(const pstring &st, const pstring & rem) { m_number_chars_start = st; m_number_chars = rem; return *this; }
tokenizer_t & string_char(pstring::value_type c) { m_string = c; return *this; }
tokenizer_t & whitespace(const pstring & s) { m_whitespace = s; return *this; }
tokenizer_t & comment(const pstring &start, const pstring &end, const pstring &line)
{
m_tok_comment_start = register_token(start);
m_tok_comment_end = register_token(end);
m_tok_line_comment = register_token(line);
return *this;
}
void append_to_store(putf8_reader *reader, token_store_t &store);
private:
void clear()
{
m_cur_line = "";
m_px = m_cur_line.begin();
m_unget = 0;
}
token_t get_token_internal();
// get internal token with comment processing
token_t get_token_comment();
void skip_eol();
pstring::value_type getc();
void ungetc(pstring::value_type c);
bool eof() const { return m_strm->eof(); }
putf8_reader *m_strm;
pstring m_cur_line;
pstring::const_iterator m_px;
pstring::value_type m_unget;
// tokenizer stuff follows ...
pstring m_identifier_chars;
pstring m_number_chars;
pstring m_number_chars_start;
std::unordered_map<pstring, token_id_t> m_tokens;
pstring m_whitespace;
pstring::value_type m_string;
token_id_t m_tok_comment_start;
token_id_t m_tok_comment_end;
token_id_t m_tok_line_comment;
protected:
bool m_support_line_markers;
token_store_t *m_token_queue;
};
class token_reader_t
{
public:
using token_t = tokenizer_t::token_t;
using token_type = tokenizer_t::token_type;
using token_id_t = tokenizer_t::token_id_t;
using token_store = tokenizer_t::token_store_t;
explicit token_reader_t()
: m_idx(0)
, m_token_store(nullptr)
{
// add a first entry to the stack
m_source_location.emplace_back(plib::source_location("Unknown", 0));
}
token_reader_t(const token_reader_t &) = delete;
token_reader_t &operator=(const token_reader_t &) = delete;
token_reader_t(token_reader_t &&) noexcept = delete;
token_reader_t &operator=(token_reader_t &&) noexcept = delete;
virtual ~token_reader_t() = default;
void set_token_source(const token_store *store)
{
m_token_store = store;
}
pstring current_line_str() const;
// tokenizer stuff follows ...
token_t get_token();
token_t get_token_raw(); // includes line information
pstring get_string();
pstring get_identifier();
pstring get_identifier_or_number();
double get_number_double();
long get_number_long();
void require_token(const token_id_t &token_num);
void require_token(const token_t &tok, const token_id_t &token_num);
void error(const perrmsg &errs);
plib::source_location location() { return m_source_location.back(); }
pstring current_line() const { return m_line; }
protected:
virtual void verror(const pstring &msg) = 0;
private:
bool process_line_token(const token_t &tok);
token_t get_token_queue()
{
if (m_idx < m_token_store->size())
return (*m_token_store)[m_idx++];
return token_t(token_type::ENDOFFILE);
}
// source locations, vector used as stack because we need to loop through stack
std::vector<plib::source_location> m_source_location;
pstring m_line;
std::size_t m_idx;
const token_store * m_token_store;
};
} // namespace plib
#endif // PTOKENIZER_H_
|