YSTest: TextManager.cpp 源文件

YSTest PreAlpha_b800_20170810 The YSLib Test Project

- 首页
- 相关页面
- 模块
- 命名空间
- 类
- 文件

- 文件列表
- 文件成员

TextManager.cpp
浏览该文件的文档.
1 /*
2 © 2010-2016 FrankHB.
3
4 This file is part of the YSLib project, and may only be used,
5 modified, and distributed under the terms of the YSLib project
6 license, LICENSE.TXT. By continuing to use, modify, or distribute
7 this file you indicate that you have read the license and
8 understand and accept it fully.
9 */
10
28 #include "YSLib/Service/YModules.h"
29 #include YFM_YSLib_Service_TextManager
30 #include YFM_CHRLib_MappingEx
31 #include <ystdex/any_iterator.hpp> // for ystdex::input_monomorphic_iterator,
32 // ystdex::make_transform, std::istreambuf_iterator;
33 #include YFM_CHRLib_Convert
34
35 namespace YSLib
36 {
37
38 namespace Text
39 {
40
41 namespace
42 {
43
45
46 using MonoIter = ystdex::input_monomorphic_iterator;
48 yconstexpr const auto& FetchMapperFunc(FetchMapperPtr<ConversionResult,
49 char16_t&, GuardPair<MonoIter>&&, ConversionState&&>);
50 yconstexpr const auto& FetchSkipMapperFunc(FetchMapperPtr<ConversionResult,
51 ystdex::pseudo_output&&, GuardPair<MonoIter>&&, ConversionState&&>);
52
53
54 template<typename _func, typename _vPFun, typename _tIn, typename... _tParams>
55 size_t
56 ConvertChar(_func f, _vPFun pfun, _tIn&& i, _tParams&&... args)
57 {
58 using InIter = ystdex::decay_t<_tIn>;
59 const auto trans([](InIter iter){
60 return ystdex::make_transform(iter, [](InIter x){
61 return byte(*x);
62 });
63 });
64 using RetainedIter
65 = ystdex::pair_iterator<ystdex::decay_t<decltype(trans(i))>, size_t>;
66 ConversionState st;
67 RetainedIter it(trans(i));
68 GuardPair<RetainedIter> gpr(it, RetainedIter(trans(InIter()), 0));
69 const auto
70 res(ConvertCharacter(pfun, yforward(args)..., gpr, std::move(st)));
71
72 switch(ConversionResult(YB_EXPECT(long(res), long(ConversionResult::OK))))
73 {
74 case ConversionResult::OK:
75 f(yforward(args)...);
76 break;
77 // case ConversionResult::Invalid:
78 // TODO: Insert placeholders for %ConversionResult::Invalid.
79 default:
80 YTraceDe(Warning, "Encoding conversion failed with state = %zu.",
81 size_t(res));
82 }
83 i = get<0>(it.base()).get();
84 return get<1>(it.base());
85 }
87
89 class Sentry
90 {
91 private:
93 tidy_ptr<std::streambuf> fp{};
94
95 public:
97 std::istreambuf_iterator<char> Iterator{};
98
99 DefDeCtor(Sentry)
101 Sentry(std::streambuf& sb)
102 : fp(make_observer(&sb)), Iterator(&sb)
103 {}
104 DefDeMoveCtor(Sentry)
105 ~Sentry()
106 {
107 if(fp)
108 TryExpr(fp->sungetc())
109 CatchIgnore(std::ios_base::failure&)
110 CatchExpr(..., yassume(false))
111 }
112
113 DefDeMoveAssignment(Sentry)
114 };
115
116 } // unnamed namespace;
117
118
119 TextFileBuffer::iterator::iterator(TextFileBuffer* p_buf, size_t b, size_t idx)
120 ynothrow
121 : p_buffer(p_buf), block(b), index(idx)
122 {}
123
124 TextFileBuffer::iterator&
125 TextFileBuffer::iterator::operator++() ynothrow
126 {
127 YAssert(GetBlockN() < Deref(p_buffer).n_block, "End iterator found.");
128
129 auto& vec((*p_buffer)[GetBlockN()].first);
130
131 YAssert(GetIndexN() < vec.size(), "Invalid index found.");
132 if(YB_UNLIKELY(++index == vec.size()))
133 yunseq(++block, index = 0);
134 return *this;
135 }
136
137 TextFileBuffer::iterator&
138 TextFileBuffer::iterator::operator--() ynothrow
139 {
140 YAssert(block != 0 || index != 0, "Begin iterator found.");
141 YAssert(block < Deref(p_buffer).n_block || *this == p_buffer->end(),
142 "Invalid iterator found.");
143 if(index == 0)
144 {
145 index = (*p_buffer)[--block].first.size();
146
147 YAssert(index != 0, "Invalid index found.");
148 }
149 else
150 YAssert(index < (*p_buffer)[block].first.size(),
151 "Invalid index found.");
152 --index;
153 return *this;
154 }
155
156 TextFileBuffer::iterator::reference
157 TextFileBuffer::iterator::operator*() const
158 {
159 auto& vec(Deref(p_buffer)[GetBlockN()].first);
160
161 YAssert(!vec.empty(), "Empty block found.");
162
163 const auto idx(GetIndexN());
164
165 YAssert(idx < vec.size(), "Invalid index found.");
166 return vec[idx];
167 }
168
169 bool
170 operator==(const TextFileBuffer::iterator& x, const TextFileBuffer::iterator& y)
171 ynothrow
172 {
173 YAssert(x.p_buffer == y.p_buffer,
174 "Iterators of different buffers are not comparable.");
175
176 return x.block == y.block && x.index == y.index;
177 }
178
179
180 TextFileBuffer::TextFileBuffer(std::streambuf& file, Encoding enc)
181 : File(file), fsize([&, this]() -> size_t{
182 const auto
183 pos(File.pubseekoff(0, std::ios_base::end, std::ios_base::in));
184
185 if(pos != std::streampos(std::streamoff(-1)))
186 {
187 const auto
188 xpos(File.pubseekoff(0, std::ios_base::cur, std::ios_base::in));
189
190 if(xpos != std::streampos(std::streamoff(-1)))
191 return size_t(xpos);
192 }
193 throw LoggedEvent("Failed getting size of file.");
194 }()), encoding(enc), bl([this]() -> size_t{
195 if(encoding == CharSet::Null)
196 {
197 size_t blen;
198
199 tie(encoding, blen) = DetectBOM(File, fsize);
200 // TODO: More accurate encoding checking for text stream without
201 // BOM.
202 if(encoding == CharSet::Null)
203 encoding = CharSet::GBK;
204 return blen;
205 }
206 return 0;
207 }()), n_text_size(fsize - bl), n_block((n_text_size + BlockSize - 1)
208 / BlockSize), fixed_width(FetchFixedCharWidth(encoding)), max_width(
209 fixed_width == 0 ? FetchMaxVariantCharWidth(encoding) : fixed_width)
210 {
211 YAssert(max_width != 0, "Unknown encoding found.");
212
213 if(fixed_width == 0)
214 ++fixed_width;
215 // TODO: Implementation for non-fixed-width char streams.
216 }
217
218 TextFileBuffer::BlockType&
219 TextFileBuffer::operator[](size_t idx)
220 {
221 YAssert(idx < n_block, "Invalid index found.");
222
223 auto& b(buffer[idx]);
224 auto& vec(b.first);
225
226 if(YB_UNLIKELY(vec.empty()))
227 {
228 if(const auto pfun = FetchMapperFunc(encoding))
229 {
230 // XXX: Conversion to 'std::streamoff' might be
231 // implementation-defined.
232 Seek(std::streamoff(idx * BlockSize));
233
234 size_t len(idx == n_block - 1 && n_text_size % BlockSize != 0
235 ? n_text_size % BlockSize : BlockSize);
236
237 vec.reserve(len / fixed_width);
238
239 size_t n_byte(0);
240 char16_t c;
241 Sentry sentry(File);
242
243 while(n_byte < len)
244 n_byte += ConvertChar([&](char16_t uc){
245 vec.push_back(uc);
246 }, pfun, sentry.Iterator, c);
247 vec.shrink_to_fit();
248 }
249 }
250 return b;
251 }
252
253 TextFileBuffer::iterator
254 TextFileBuffer::GetIterator(size_t pos)
255 {
256 if(pos < n_text_size)
257 {
258 const size_t idx(pos / BlockSize);
259
260 pos %= BlockSize;
261 if(fixed_width == max_width)
262 return TextFileBuffer::iterator(this, idx, pos / max_width);
263
264 if(const auto pfun = FetchSkipMapperFunc(encoding))
265 {
266 // XXX: Conversion to 'std::streamoff' might be
267 // implementation-defined.
268 Seek(std::streamoff(idx * BlockSize));
269
270 size_t n_byte(0), n_char(0);
271 Sentry sentry(File);
272
273 while(n_byte < pos)
274 n_byte += ConvertChar([&](ystdex::pseudo_output){
275 ++n_char;
276 }, pfun, sentry.Iterator, ystdex::pseudo_output());
277 return TextFileBuffer::iterator(this, idx, n_char);
278 }
279 return TextFileBuffer::iterator(this, idx, 0);
280 }
281 return end();
282 }
283 size_t
284 TextFileBuffer::GetPosition(TextFileBuffer::iterator i)
285 {
286 if(i == end())
287 return n_text_size;
288
289 auto idx(i.GetBlockN());
290 const auto pos(i.GetIndexN());
291
292 if(fixed_width == max_width)
293 return idx * BlockSize + pos * max_width;
294
295 if(const auto pfun = FetchSkipMapperFunc(encoding))
296 {
297 const auto& vec((*this)[idx].first);
298
299 YAssert(!vec.empty(), "Block loading failed.");
300 // XXX: Conversion to 'std::streamoff' might be
301 // implementation-defined.
302 Seek(std::streamoff(idx *= BlockSize));
303
304 // XXX: Conversion to 'ptrdiff_t' might be implementation-defined.
305 const auto mid(vec.cbegin() + ptrdiff_t(pos));
306
307 YAssert(mid <= vec.cend(), "Wrong iterator found.");
308
309 auto it(vec.cbegin());
310
311 YAssert(it <= mid, "Wrong iterator found.");
312
313 size_t n_byte(0);
314 Sentry sentry(File);
315
316 while(it != mid)
317 n_byte += ConvertChar([&](ystdex::pseudo_output){
318 ++it;
319 }, pfun, sentry.Iterator, ystdex::pseudo_output());
320 return idx + n_byte;
321 }
322 return idx;
323 }
324
325 void
326 TextFileBuffer::Seek(std::streamoff off)
327 {
328 if(YB_UNLIKELY(File.pubseekpos(std::streamoff(bl) + off, std::ios_base::in)
329 == std::streampos(std::streamoff(-1))))
330 throw LoggedEvent("Failed setting reading position.");
331 }
332
333 TextFileBuffer::iterator
334 TextFileBuffer::begin() ynothrow
335 {
336 return TextFileBuffer::iterator(this);
337 }
338
339 TextFileBuffer::iterator
340 TextFileBuffer::end() ynothrow
341 {
342 return TextFileBuffer::iterator(this, n_block);
343 }
344
345
346 string
347 CopySliceFrom(TextFileBuffer& buf, size_t pos, size_t len)
348 {
349 const auto i_end(buf.end());
350 auto i_beg(buf.GetIterator(pos));
351
352 if(i_beg == i_end)
353 throw std::out_of_range("Wrong offset found.");
354 if(len == 0)
355 return {};
356
357 YAssert(pos < pos + len, "Unexpected unsigned integer round up.");
358
359 string str;
360
361 str.reserve(len * 2);
362 while(len != 0 && i_beg != i_end)
363 {
364 str.push_back(*i_beg);
365 yunseq(++i_beg, --len);
366 }
367 return str;
368 }
369
370 } // namespace Text;
371
372 } // namespace YSLib;
373
YTraceDe
#define YTraceDe(...)
YCLib 默认调试跟踪。
Definition: Debug.h:313
CHRLib::FetchFixedCharWidth
YF_API YB_STATELESS size_t FetchFixedCharWidth(Encoding)
取指定固定编码的固定字符宽度。
Definition: chrmap.cpp:37
YSLib::Text::TextFileBuffer::iterator::operator*
reference operator*() const
Definition: TextManager.cpp:157
YSLib::Text::TextFileBuffer::buffer
MapType buffer
缓冲映射。
Definition: TextManager.h:147
YModules.h
CHRLib::CharSet::GBK
yconstexpr const Encoding GBK(csGBK)
YSLib
YSLib 命名空间。
Definition: AndroidHost.h:45
CHRLib::FetchMaxVariantCharWidth
YF_API YB_STATELESS size_t FetchMaxVariantCharWidth(Encoding)
取指定变长编码的最大字符宽度。
Definition: chrmap.cpp:67
YSLib::Text::TextFileBuffer::n_text_size
size_t n_text_size
文本大小。
Definition: TextManager.h:143
ystdex::make_observer
observer_ptr< _type > make_observer(_type *p)
Definition: pointer.hpp:326
platform::Deref
yconstfn auto Deref(_type &&p) -> decltype(*p)
断言并解引用非空指针。
Definition: Debug.h:451
YSLib::Text::TextFileBuffer::max_width
size_t max_width
最大编码每字符的字节数。
Definition: TextManager.h:161
YSLib::UI::y
SPos SPos y
Definition: YWidget.cpp:44
NPL::x
const TermNode &term ynothrow const TermNode &term ynothrow const TermNode &term ynothrow TermNode &term const TermNode &term ValueNode &node const ValueNode &node const _type & x
Definition: SContext.h:94
YSLib::Text::TextFileBuffer::Seek
void Seek(std::streamoff)
使用相对于文本位置的（跳过 BOM ）偏移指定的参数设置流缓冲读取位置。
Definition: TextManager.cpp:326
YSLib::Text::TextFileBuffer::n_block
size_t n_block
区块数。
Definition: TextManager.h:145
CHRLib::f
f(d, s)
Text
CHRLib::FetchMapperPtr
yconstfn_relaxed ystdex::add_ptr_t< _tRet(_tParams...)> FetchMapperPtr(Encoding enc)
取指定编码映射的转换函数指针。
Definition: MappingEx.h:285
DefDeMoveCtor
#define DefDeMoveCtor(_t)
Definition: YBaseMacro.h:167
YSLib::Text::TextFileBuffer::begin
iterator begin() ynothrow
取文本缓冲区起始迭代器。
Definition: TextManager.cpp:334
YB_UNLIKELY
#define YB_UNLIKELY(_expr)
分支预测提示。
Definition: ydef.h:464
YSLib::LoggedEvent
记录日志的异常事件类。
Definition: YException.h:50
CHRLib::args
_tParams && args
Definition: CharacterProcessing.h:538
yconstexpr
#define yconstexpr
指定编译时常量表达式。
Definition: ydef.h:679
CatchIgnore
#define CatchIgnore(_ex)
指定异常处理器忽略指定异常。
Definition: YBaseMacro.h:133
YSLib::Text::buf
const char * buf
Definition: TextFile.h:81
CHRLib::CharSet::Null
空字符集。
Definition: Encoding.h:63
ystdex::cpp2011::end
_type * end(_type(&&array)[_vN])
Definition: range.hpp:54
YSLib::YAssert
YAssert(!(v< Deref(a)),"Value less than lower bound found.")
ystdex::tuple_iterator
元组迭代器。
Definition: iterator.hpp:651
Iterator
std::istreambuf_iterator< char > Iterator
Definition: TextManager.cpp:97
yunseq
#define yunseq
无序列依赖表达式组求值。
Definition: ydef.h:1022
YSLib::Text::TextFileBuffer
文本文件块缓冲区。
Definition: TextManager.h:47
YSLib::Text::TextFileBuffer::TextFileBuffer
TextFileBuffer(std::streambuf &, Encoding=CharSet::Null)
构造：使用流和指定编码。
Definition: TextManager.cpp:180
YSLib::Text::TextFileBuffer::operator[]
BlockType & operator[](size_t)
块随机访问。
Definition: TextManager.cpp:219
TryExpr
#define TryExpr(...)
指定表达式在 try 块中求值。
Definition: YBaseMacro.h:120
YSLib::Text::CopySliceFrom
YF_API string CopySliceFrom(TextFileBuffer &, size_t, size_t)
从文本文件缓冲区复制指定字节位置起始的字符串。
Definition: TextManager.cpp:347
DefDeMoveAssignment
#define DefDeMoveAssignment(_t)
Definition: YBaseMacro.h:191
DefDeCtor
#define DefDeCtor(_t)
简单通用成员函数定义。
Definition: YBaseMacro.h:157
CatchExpr
#define CatchExpr(_ex,...)
指定表达式作为异常处理器。
Definition: YBaseMacro.h:129
YSLib::Text::TextFileBuffer::iterator::operator++
iterator & operator++() ynothrow
迭代：循环向后遍历。
Definition: TextManager.cpp:125
YSLib::i
return i
Definition: YCoreUtilities.h:176
YSLib::Text::TextFileBuffer::end
iterator end() ynothrow
取文本缓冲区终止迭代器。
Definition: TextManager.cpp:340
YSLib::Text::TextFileBuffer::encoding
Encoding encoding
Definition: TextManager.h:137
YB_EXPECT
#define YB_EXPECT(_expr, _constant)
Definition: ydef.h:462
any_iterator.hpp
动态泛型迭代器。
CHRLib::enc
enc
Definition: CharacterProcessing.h:81
YSLib::ynothrow
YB_PURE const _type size_t n ynothrow
Definition: YCoreUtilities.h:168
YSLib::Text::DetectBOM
YF_API size_t DetectBOM(const char *)
Definition: TextFile.cpp:103
YSLib::Text::TextFileBuffer::GetPosition
size_t GetPosition(iterator)
取迭代器对应的文本字节位置。
Definition: TextManager.cpp:284
CHRLib::ConvertCharacter
ConversionResult ConvertCharacter(_fConv f, _tOut &&obj, _tIn &&i, ConversionState &&st)
Definition: Convert.hpp:41
YSLib::Text::operator==
bool operator==(const TextFileBuffer::iterator &x, const TextFileBuffer::iterator &y) ynothrow
Definition: TextManager.cpp:170
ystdex::pseudo_output
伪输出对象。
Definition: ref.hpp:307
std
Definition: memory.hpp:1183
ystdex::byte
unsigned char byte
字节类型。
Definition: ydef.h:834
YSLib::Text::TextFileBuffer::iterator
目标编码迭代器类型。
Definition: TextManager.h:69
YSLib::Text::TextFileBuffer::BlockType
pair< vector< char16_t >, size_t > BlockType
缓冲区块类型。
Definition: TextManager.h:56
platform::Descriptions::Warning
Definition: YCommon.h:167
ystdex::input_monomorphic_iterator
any_input_iterator< void_ref, ptrdiff_t, void *, void_ref_any > input_monomorphic_iterator
Definition: any_iterator.hpp:361
CHRLib::CharSet::Encoding
Encoding
字符流编码标识。
Definition: Encoding.h:58
YSLib::Text::str
const char const char * str
Definition: TextFile.h:81
CHRLib::c
c
Definition: CharacterProcessing.h:81
yforward
#define yforward(_expr)
根据参数类型使用 std::forward 传递对应参数。
Definition: ydef.h:995
yassume
#define yassume(_expr)
假定：环境语义。
Definition: cassert.h:105
YSLib::Text::TextFileBuffer::BlockSize
static yconstexpr const size_t BlockSize
默认文本区块大小。
Definition: TextManager.h:124
CHRLib::char16_t
char16_t((*c_ptr<< CHAR_BIT)|c_ptr[1])) inline PDefH(char16_t
取 c_ptr 指向的小端序双字节字符。
YSLib::Text::TextFileBuffer::iterator::operator--
iterator & operator--() ynothrow
迭代：循环向前遍历。
Definition: TextManager.cpp:138
YSLib::Text::TextFileBuffer::GetIterator
encoding n_text_size iterator GetIterator(size_t)
取文本字节位置对应的迭代器。
Definition: TextManager.cpp:254
YSLib::Text::TextFileBuffer::File
std::streambuf & File
流缓冲引用。
Definition: TextManager.h:131
YSLib::Text::TextFileBuffer::bl
size_t bl
Definition: TextManager.h:138
fp
tidy_ptr< std::streambuf > fp
Definition: TextManager.cpp:93