YSTest: Lexical.cpp 源文件

YSTest PreAlpha_b800_20170810 The YSLib Test Project

- 首页
- 相关页面
- 模块
- 命名空间
- 类
- 文件

- 文件列表
- 文件成员

Lexical.cpp
浏览该文件的文档.
1 /*
2 © 2012-2017 FrankHB.
3
4 This file is part of the YSLib project, and may only be used,
5 modified, and distributed under the terms of the YSLib project
6 license, LICENSE.TXT. By continuing to use, modify, or distribute
7 this file you indicate that you have read the license and
8 understand and accept it fully.
9 */
10
28 #include "NPL/YModules.h"
29 #include YFM_NPL_Lexical
30 #include <ystdex/string.hpp> // for ystdex::get_mid;
31
32 namespace NPL
33 {
34
35 string
36 UnescapeContext::Done()
37 {
38 auto res(std::move(Prefix));
39
40 res += std::move(sequence);
41 Clear();
42 return res;
43 }
44
45
46 bool
47 HandleBackslashPrefix(char c, string& pfx)
48 {
49 if(c == '\\')
50 {
51 pfx = "\\";
52 return true;
53 }
54 return {};
55 }
56
57 bool
58 NPLUnescape(string& buf, const UnescapeContext& uctx, char ld)
59 {
60 const auto& escs(uctx.GetSequence());
61
62 if(uctx.IsHandling() && escs.length() == 1)
63 {
64 switch(escs[0])
65 {
66 case '\\':
67 buf += '\\';
68 break;
69 case 'a':
70 buf += '\a';
71 break;
72 case 'b':
73 if(!buf.empty())
74 buf.pop_back();
75 break;
76 case 'f':
77 buf += '\f';
78 break;
79 case 'n':
80 buf += '\n';
81 break;
82 case 'r':
83 buf += '\r';
84 break;
85 case 't':
86 buf += '\t';
87 break;
88 case 'v':
89 buf += '\v';
90 break;
91 case '\'':
92 case '"':
93 if(escs[0] == ld)
94 {
95 buf += ld;
96 break;
97 }
98 YB_ATTR_fallthrough;
99 default:
100 return {};
101 }
102 return true;
103 }
104 return {};
105 }
106
107
108 LexicalAnalyzer::LexicalAnalyzer()
109 : unescape_context(), ld(), cbuf(), qlist()
110 {}
111
112 bool
113 LexicalAnalyzer::CheckEscape(byte b, Unescaper unescape)
114 {
115 if(!(b < 0x80))
116 {
117 // NOTE: Stop unescaping. The escaped sequence should have no
118 // multibyte characters.
119 if(unescape_context.IsHandling())
120 cbuf += unescape_context.Done();
121 cbuf += char(b);
122 }
123 else if(unescape_context.IsHandling())
124 {
125 unescape_context.Push(b);
126 if(unescape(cbuf, unescape_context, ld))
127 unescape_context.Clear();
128 else
129 cbuf += unescape_context.Done();
130 }
131 else
132 return {};
133 return true;
134 }
135
136 bool
137 LexicalAnalyzer::CheckLineConcatnater(char c, char concat, char newline)
138 {
139 if(line_concat == concat && c == newline)
140 {
141 if(!unescape_context.PopIf(concat))
142 {
143 auto& pfx(unescape_context.Prefix);
144
145 if(!pfx.empty() && pfx.back() == concat)
146 pfx.pop_back();
147 else if(!cbuf.empty() && cbuf.back() == line_concat)
148 cbuf.pop_back();
149 }
150 return true;
151 }
152 else if(c == concat)
153 line_concat = concat;
154 else
155 line_concat = {};
156 return {};
157 }
158
159 bool
160 LexicalAnalyzer::FilterForParse(char c, Unescaper unescape,
161 PrefixHandler prefix_handler)
162 {
163 return !(CheckLineConcatnater(c) || CheckEscape(byte(c), unescape)
164 || prefix_handler(c, unescape_context.Prefix));
165 }
166
167 void
168 LexicalAnalyzer::ParseByte(char c, Unescaper unescape,
169 PrefixHandler prefix_handler)
170 {
171 if(FilterForParse(c, unescape, prefix_handler))
172 switch(c)
173 {
174 case '\'':
175 case '"':
176 if(ld == char())
177 {
178 ld = c;
179 qlist.push_back(cbuf.size());
180 cbuf += c;
181 }
182 else if(ld == c)
183 {
184 ld = char();
185 cbuf += c;
186 qlist.push_back(cbuf.size());
187 }
188 else
189 cbuf += c;
190 case char():
191 break;
192 case ' ':
193 case '\f':
194 case '\n':
195 // case '\r':
196 case '\t':
197 case '\v':
198 if(ld == char())
199 {
200 cbuf += ' ';
201 break;
202 }
203 YB_ATTR_fallthrough;
204 default:
205 cbuf += c;
206 }
207 }
208
209 void
210 LexicalAnalyzer::ParseQuoted(char c, Unescaper unescape,
211 PrefixHandler prefix_handler)
212 {
213 if(FilterForParse(c, unescape, prefix_handler))
214 cbuf += c;
215 }
216
217 list<string>
218 LexicalAnalyzer::Literalize() const
219 {
220 size_t i(0);
221 list<string> result;
222
223 std::for_each(qlist.cbegin(), qlist.cend(), [&](size_t s){
224 if(s != i)
225 {
226 result.push_back(cbuf.substr(i, s - i));
227 i = s;
228 }
229 });
230 result.push_back(cbuf.substr(i));
231 return result;
232 }
233
234
235 char
236 CheckLiteral(string_view sv) ynothrowv
237 {
238 YAssertNonnull(sv.data());
239 if(sv.length() > 1)
240 if(const char c = ystdex::get_quote_mark_nonstrict(sv))
241 {
242 if(c == '\'' || c == '"')
243 return c;
244 }
245 return {};
246 }
247
248 string_view
249 Deliteralize(string_view sv) ynothrowv
250 {
251 return CheckLiteral(sv) != char() ? DeliteralizeUnchecked(sv) : sv;
252 }
253
254 string
255 Escape(string_view sv)
256 {
257 YAssertNonnull(sv.data());
258
259 char last{};
260 string res;
261
262 res.reserve(sv.length());
263 for(char c : sv)
264 {
265 char unescaped{};
266
267 switch(c)
268 {
269 case '\a':
270 unescaped = 'a';
271 break;
272 case '\b':
273 unescaped = 'b';
274 break;
275 case '\f':
276 unescaped = 'f';
277 break;
278 case '\n':
279 unescaped = 'n';
280 break;
281 case '\r':
282 unescaped = 'r';
283 break;
284 case '\t':
285 unescaped = 't';
286 break;
287 case '\v':
288 unescaped = 'v';
289 break;
290 case '\'':
291 case '"':
292 unescaped = c;
293 }
294 if(last == '\\')
295 {
296 if(c == '\\')
297 {
298 yunseq(last = char(), res += '\\');
299 continue;
300 }
301 switch(c)
302 {
303 case 'a':
304 case 'b':
305 case 'f':
306 case 'n':
307 case 'r':
308 case 't':
309 case 'v':
310 case '\'':
311 case '"':
312 res += '\\';
313 }
314 }
315 if(unescaped == char())
316 res += c;
317 else
318 {
319 res += '\\';
320 res += unescaped;
321 unescaped = char();
322 }
323 last = c;
324 }
325 return res;
326 }
327
328 string
329 EscapeLiteral(string_view sv)
330 {
331 const char c(CheckLiteral(sv));
332 // TODO: Use %get_mid for %string_view.
333 auto content(Escape(c == char() ? sv : ystdex::get_mid(string(sv))));
334
335 if(!content.empty() && content.back() == '\\')
336 content += '\\';
337 return c == char() ? std::move(content) : ystdex::quote(content, c);
338 }
339
340 string
341 EscapeXML(string_view sv)
342 {
343 YAssertNonnull(sv.data());
344
345 string res;
346
347 res.reserve(sv.length());
348 for(char c : sv)
349 switch(c)
350 {
351 case '\0':
352 YTraceDe(YSLib::Warning, "Invalid character '#x%X' found, ignored.",
353 unsigned(c));
354 break;
355 case '&':
356 res += "&amp;";
357 break;
358 case '<':
359 res += "&lt;";
360 break;
361 case '>':
362 res += "&gt;";
363 break;
364 default:
365 res += c;
366 }
367 return res;
368 }
369
370 string
371 Literalize(string_view sv, char c)
372 {
373 return CheckLiteral(sv) == char() && c != char()
374 ? ystdex::quote(string(sv), c) : string(sv);
375 }
376
377
378 list<string>
379 Decompose(string_view src)
380 {
381 YAssertNonnull(src.data());
382
383 list<string> dst;
384 using iter_type = typename string_view::const_iterator;
385
386 ystdex::split_l(src.cbegin(), src.cend(), IsDelimeter,
387 // TODO: Blocked. Use C++14 generic lambda expressions.
388 [&](iter_type b, iter_type e){
389 YAssert(e >= b, "Invalid split result found.");
390
391 string_view sv(b, size_t(e - b));
392
393 YAssert(!sv.empty(), "Null token found.");
394 if(IsGraphicalDelimeter(*b))
395 {
396 dst.push_back({sv.front()});
397 sv.remove_prefix(1);
398 }
399 ystdex::trim(sv);
400 if(!sv.empty())
401 dst.push_back(string(sv));
402 });
403 return dst;
404 }
405
406 list<string>
407 Tokenize(const list<string>& src)
408 {
409 list<string> dst;
410
411 for(const auto& str : src)
412 if(!str.empty())
413 {
414 if(str[0] != '\'' && str[0] != '"')
415 dst.splice(dst.end(), Decompose(str));
416 else
417 dst.push_back(str);
418 }
419 return dst;
420 }
421
422 } // namespace NPL;
423
YTraceDe
#define YTraceDe(...)
YCLib 默认调试跟踪。
Definition: Debug.h:313
NPL::HandleBackslashPrefix
YF_API bool HandleBackslashPrefix(char, string &)
设置反斜杠转义前缀：当输入 '\' 时设置前缀为 "\\" 。
Definition: Lexical.cpp:47
ystdex::get_mid
_tString get_mid(const _tString &str, typename _tString::size_type l=1)
取删除前缀和后缀的子字符串。
Definition: string.hpp:829
NPL::LexicalAnalyzer::Unescaper
std::function< bool(string &, const UnescapeContext &, char)> Unescaper
指定匹配转义序列的反转义算法：解析转义序列并按需修改指定缓存。
Definition: Lexical.h:173
NPL::LexicalAnalyzer::unescape_context
UnescapeContext unescape_context
反转移上下文：储存反转义中间结果。
Definition: Lexical.h:186
NPL::dst
YF_API const char src const string const char * dst
Definition: Dependency.h:95
ystdex::get_quote_mark_nonstrict
string_traits< _tString >::value_type get_quote_mark_nonstrict(const _tString &str)
取相同的前缀和后缀元素。
Definition: string.hpp:786
NPL::Decompose
list< string > Decompose(string_view src)
Definition: Lexical.cpp:379
YModules.h
YAssertNonnull
#define YAssertNonnull(_expr)
Definition: cassert.h:120
NPL::LexicalAnalyzer::FilterForParse
bool FilterForParse(char, Unescaper, PrefixHandler)
Definition: Lexical.cpp:160
NPL::UnescapeContext::Push
char c Push
Definition: Lexical.h:97
platform::containers::string
basic_string< char > string
Definition: Container.h:101
NPL::UnescapeContext::sequence
string sequence
有效转义序列。
Definition: Lexical.h:71
YSLib::Text::buf
const char * buf
Definition: TextFile.h:81
NPL::NPLUnescape
YF_API bool NPLUnescape(string &, const UnescapeContext &, char)
NPL 转义匹配算法。
Definition: Lexical.cpp:58
yunseq
#define yunseq
无序列依赖表达式组求值。
Definition: ydef.h:1022
NPL::LexicalAnalyzer::ld
char ld
字面分隔符状态：表示正在处理字面量中的有效字符。
Definition: Lexical.h:191
NPL::UnescapeContext
反转义上下文。
Definition: Lexical.h:59
NPL::LexicalAnalyzer::ParseQuoted
void ParseQuoted(char, Unescaper=NPLUnescape, PrefixHandler=HandleBackslashPrefix)
解析单个字面量字符并添加至字符解析结果：反转义以外无视边界字符。
Definition: Lexical.cpp:210
ystdex::quote
_tString quote(const _tString &str, typename string_traits< _tString >::value_type c=typename string_traits< _tString >::value_type('"'))
取添加前缀和后缀的字符串。
Definition: string.hpp:801
YAssert
#define YAssert(_expr, _msg)
Definition: cassert.h:111
NPL::Tokenize
YF_API list< string > Tokenize(const list< string > &)
记号化：提取字符串列表中的记号。
Definition: Lexical.cpp:407
ystdex::split_l
_tIn split_l(_tIn b, _tIn e, _fPred is_delim, _fInsert insert)
以指定分隔符分割保留起始分隔符的序列。
Definition: algorithm.hpp:263
NPL::UnescapeContext::IsHandling
!Prefix IsHandling
Definition: Lexical.h:82
NPL::LexicalAnalyzer::qlist
vector< size_t > qlist
字符解析中间结果中非转义的引号出现的位置的有序列表。
Definition: Lexical.h:200
CHRLib::s
_tDst const _tSrc * s
Definition: CharacterProcessing.h:140
YSLib::i
return i
Definition: YCoreUtilities.h:176
NPL::EscapeLiteral
YF_API string EscapeLiteral(string_view)
编码转义字符串字面量。
Definition: Lexical.cpp:329
NPL::LexicalAnalyzer::PrefixHandler
std::function< bool(char, string &)> PrefixHandler
转移序列前缀处理器。
Definition: Lexical.h:165
ystdex::concat
void concat(_tCon &con, _tIn first, _tIn last)
插入元素到容器末尾。
Definition: container.hpp:436
NPL::UnescapeContext::Prefix
string Prefix
转义序列前缀。
Definition: Lexical.h:67
NPL::LexicalAnalyzer::line_concat
char line_concat
指示当前续行符状态。
Definition: Lexical.h:180
NPL::LexicalAnalyzer::CheckLineConcatnater
const string cbuf DefGetter(const ynothrow, const vector< size_t > &, Quotes, qlist) private bool CheckLineConcatnater(char, char= '\\', char= '\n')
Definition: Lexical.h:217
ynothrowv
#define ynothrowv
YSLib 无异常抛出保证验证：有条件地使用无异常抛出规范。
Definition: ydef.h:786
string.hpp
ISO C++ 标准字符串扩展。
CHRLib::e
const char const char * e
Definition: CharacterProcessing.h:109
NPL::LexicalAnalyzer::cbuf
string cbuf
字符解析中间结果。
Definition: Lexical.h:195
NPL::LexicalAnalyzer::Literalize
list< string > Literalize() const
根据中间结果取字符串列表。
Definition: Lexical.cpp:218
NPL::LexicalAnalyzer::LexicalAnalyzer
LexicalAnalyzer()
Definition: Lexical.cpp:108
ystdex::byte
unsigned char byte
字节类型。
Definition: ydef.h:834
NPL
Definition: NPL/Configuration.h:34
platform::Descriptions::Warning
Definition: YCommon.h:167
NPL::Escape
string Escape(string_view sv)
Definition: Lexical.cpp:255
NPL::Deliteralize
YF_API string_view Deliteralize(string_view) ynothrowv
去除字面量边界分隔符。
Definition: Lexical.cpp:249
YSLib::Text::str
const char const char * str
Definition: TextFile.h:81
CHRLib::c
c
Definition: CharacterProcessing.h:81
NPL::UnescapeContext::PopIf
PopIf
Definition: Lexical.h:95
NPL::CheckLiteral
YF_API char CheckLiteral(string_view) ynothrowv
检查指定字符串是否为字面量。
Definition: Lexical.cpp:236
NPL::Literalize
YF_API string Literalize(string_view, char= '"')
修饰字符串为字面量。
Definition: Lexical.cpp:371
YB_ATTR_fallthrough
#define YB_ATTR_fallthrough
用于 switch 语句的 fallthrough 属性。
Definition: ydef.h:383
ystdex::trim
_tString && trim(_tString &&str, typename string_traits< _tString >::value_type c)
删除字符串中指定的连续前缀与后缀字符。
Definition: string.hpp:707
NPL::LexicalAnalyzer::ParseByte
void ParseByte(char, Unescaper=NPLUnescape, PrefixHandler=HandleBackslashPrefix)
解析单个字符并添加至字符解析结果。
Definition: Lexical.cpp:168
ystdex::string_view
basic_string_view< char > string_view
Definition: string_view.hpp:415
NPL::EscapeXML
YF_API string EscapeXML(string_view)
编码 XML 字符串。
Definition: Lexical.cpp:341