YSTest: TextManager.cpp 源文件

YSTest PreAlpha_b700_20160611 The YSLib Test Project

- 首页
- 相关页面
- 模块
- 命名空间
- 类
- 文件

- 文件列表
- 文件成员

TextManager.cpp
浏览该文件的文档.
1 /*
2 © 2010-2016 FrankHB.
3
4 This file is part of the YSLib project, and may only be used,
5 modified, and distributed under the terms of the YSLib project
6 license, LICENSE.TXT. By continuing to use, modify, or distribute
7 this file you indicate that you have read the license and
8 understand and accept it fully.
9 */
10
28 #include "YSLib/Service/YModules.h"
29 #include YFM_YSLib_Service_TextManager
30 #include YFM_CHRLib_MappingEx
31 #include <ystdex/any_iterator.hpp>
32 #include YFM_CHRLib_Convert
33
34 namespace YSLib
35 {
36
37 namespace Text
38 {
39
40 namespace
41 {
42
44
45 using MonoIter = ystdex::input_monomorphic_iterator;
47 yconstexpr const auto& FetchMapperFunc(FetchMapperPtr<ConversionResult, char16_t&,
48 GuardPair<MonoIter>&&, ConversionState&&>);
49 yconstexpr const auto& FetchSkipMapperFunc(FetchMapperPtr<ConversionResult,
50 ystdex::pseudo_output&&, GuardPair<MonoIter>&&, ConversionState&&>);
51
52
53 template<typename _func, typename _vPFun, typename _tIn, typename... _tParams>
54 size_t
55 ConvertChar(_func f, _vPFun pfun, _tIn&& i, _tParams&&... args)
56 {
57 using InIter = ystdex::decay_t<_tIn>;
58 using RetainedIter = ystdex::pair_iterator<InIter, size_t>;
59 ConversionState st;
60 RetainedIter it(i);
61 GuardPair<RetainedIter> gpr(it, RetainedIter(InIter(), 0));
62 const auto
63 res(ConvertCharacter(pfun, yforward(args)..., gpr, std::move(st)));
64
65 switch(ConversionResult(YB_EXPECT(long(res), long(ConversionResult::OK))))
66 {
67 case ConversionResult::OK:
68 f(yforward(args)...);
69 break;
70 // case ConversionResult::Invalid:
71 // TODO: Insert placeholders for %ConversionResult::Invalid.
72 default:
73 YTraceDe(Warning, "Encoding conversion failed with state = %zu.",
74 size_t(res));
75 }
76 i = get<0>(it.base());
77 return get<1>(it.base());
78 }
80
82 class Sentry
83 {
84 private:
86 tidy_ptr<std::istream> fp{};
87
88 public:
90 std::istream_iterator<char> Iterator{};
91
92 DefDeCtor(Sentry)
93 Sentry(std::istream& is)
94 : fp(make_observer(&is)), Iterator(is)
95 {}
96 DefDeMoveCtor(Sentry)
97 ~Sentry()
98 {
99 if(fp)
100 TryExpr(fp->clear(), fp->unget(), fp->clear())
101 CatchIgnore(std::ios_base::failure&)
102 CatchExpr(..., yassume(false))
103 }
104
105 DefDeMoveAssignment(Sentry)
106 };
107
108 } // unnamed namespace;
109
110
111 TextFileBuffer::iterator::iterator(TextFileBuffer* p_buf, size_t b, size_t idx)
112 ynothrow
113 : p_buffer(p_buf), block(b), index(idx)
114 {}
115
116 TextFileBuffer::iterator&
117 TextFileBuffer::iterator::operator++() ynothrow
118 {
119 YAssert(GetBlockN() < Deref(p_buffer).n_block, "End iterator found.");
120
121 auto& vec((*p_buffer)[GetBlockN()].first);
122
123 YAssert(GetIndexN() < vec.size(), "Invalid index found.");
124 if(YB_UNLIKELY(++index == vec.size()))
125 yunseq(++block, index = 0);
126 return *this;
127 }
128
129 TextFileBuffer::iterator&
130 TextFileBuffer::iterator::operator--() ynothrow
131 {
132 YAssert(block != 0 || index != 0, "Begin iterator found.");
133 YAssert(block < Deref(p_buffer).n_block || *this == p_buffer->end(),
134 "Invalid iterator found.");
135 if(index == 0)
136 {
137 index = (*p_buffer)[--block].first.size();
138
139 YAssert(index != 0, "Invalid index found.");
140 }
141 else
142 YAssert(index < (*p_buffer)[block].first.size(),
143 "Invalid index found.");
144 --index;
145 return *this;
146 }
147
148 TextFileBuffer::iterator::reference
149 TextFileBuffer::iterator::operator*() const
150 {
151 auto& vec(Deref(p_buffer)[GetBlockN()].first);
152
153 YAssert(!vec.empty(), "Empty block found.");
154
155 const auto idx(GetIndexN());
156
157 YAssert(idx < vec.size(), "Invalid index found.");
158 return vec[idx];
159 }
160
161 bool
162 operator==(const TextFileBuffer::iterator& x, const TextFileBuffer::iterator& y)
163 ynothrow
164 {
165 YAssert(x.p_buffer == y.p_buffer,
166 "Iterators of different buffers are not comparable.");
167
168 return x.block == y.block && x.index == y.index;
169 }
170
171
172 TextFileBuffer::TextFileBuffer(std::istream& file, Encoding enc)
173 : File(file), fsize([&, this]() -> size_t{
174 File.seekg(0, std::ios_base::end);
175 if(!File)
176 throw LoggedEvent("Failed getting size of file.");
177 return size_t(File.tellg());
178 }()), encoding(enc), bl([this]() -> size_t{
179 if(encoding == CharSet::Null)
180 {
181 size_t blen;
182
183 tie(encoding, blen) = DetectBOM(File, fsize);
184 // TODO: More accurate encoding checking for text stream without
185 // BOM.
186 if(encoding == CharSet::Null)
187 encoding = CharSet::GBK;
188 return blen;
189 }
190 return 0;
191 }()), n_text_size(fsize - bl), n_block((n_text_size + BlockSize - 1)
192 / BlockSize), fixed_width(FetchFixedCharWidth(encoding)), max_width(
193 fixed_width == 0 ? FetchMaxVariantCharWidth(encoding) : fixed_width)
194 {
195 YAssert(max_width != 0, "Unknown encoding found.");
196
197 if(fixed_width == 0)
198 ++fixed_width;
199 // TODO: Implementation for non-fixed-width char streams.
200 }
201
202 TextFileBuffer::BlockType&
203 TextFileBuffer::operator[](size_t idx)
204 {
205 YAssert(idx < n_block, "Invalid index found.");
206
207 auto& b(buffer[idx]);
208 auto& vec(b.first);
209
210 if(YB_UNLIKELY(vec.empty()))
211 {
212 YAssert(bool(File), "Invalid file found.");
213 if(const auto pfun = FetchMapperFunc(encoding))
214 {
215 // XXX: Conversion to 'fstream::off_type' might be
216 // implementation-defined.
217 File.seekg(fstream::off_type(bl + idx * BlockSize));
218
219 size_t len(idx == n_block - 1 && n_text_size % BlockSize != 0
220 ? n_text_size % BlockSize : BlockSize);
221
222 vec.reserve(len / fixed_width);
223
224 size_t n_byte(0);
225 char16_t c;
226 Sentry sentry(File);
227
228 while(n_byte < len)
229 n_byte += ConvertChar([&](char16_t uc){
230 vec.push_back(uc);
231 }, pfun, sentry.Iterator, c);
232 vec.shrink_to_fit();
233 }
234 }
235 return b;
236 }
237
238 TextFileBuffer::iterator
239 TextFileBuffer::begin() ynothrow
240 {
241 return TextFileBuffer::iterator(this);
242 }
243 TextFileBuffer::iterator
244 TextFileBuffer::end() ynothrow
245 {
246 return TextFileBuffer::iterator(this, n_block);
247 }
248 TextFileBuffer::iterator
249 TextFileBuffer::GetIterator(size_t pos)
250 {
251 if(pos < n_text_size)
252 {
253 const size_t idx(pos / BlockSize);
254
255 pos %= BlockSize;
256 if(fixed_width == max_width)
257 return TextFileBuffer::iterator(this, idx, pos / max_width);
258
259 YAssert(bool(File), "Invalid file found.");
260
261 if(const auto pfun = FetchSkipMapperFunc(encoding))
262 {
263 // XXX: Conversion to 'fstream::off_type' might be
264 // implementation-defined.
265 File.seekg(fstream::off_type(bl + idx * BlockSize));
266
267 size_t n_byte(0), n_char(0);
268 Sentry sentry(File);
269
270 while(n_byte < pos)
271 n_byte += ConvertChar([&](ystdex::pseudo_output){
272 ++n_char;
273 }, pfun, sentry.Iterator, ystdex::pseudo_output());
274 return TextFileBuffer::iterator(this, idx, n_char);
275 }
276 return TextFileBuffer::iterator(this, idx, 0);
277 }
278 return end();
279 }
280 size_t
281 TextFileBuffer::GetPosition(TextFileBuffer::iterator i)
282 {
283 if(i == end())
284 return n_text_size;
285
286 auto idx(i.GetBlockN());
287 const auto pos(i.GetIndexN());
288
289 if(fixed_width == max_width)
290 return idx * BlockSize + pos * max_width;
291
292 if(const auto pfun = FetchSkipMapperFunc(encoding))
293 {
294 const auto& vec((*this)[idx].first);
295
296 YAssert(!vec.empty() && bool(File), "Block loading failed.");
297 // XXX: Conversion to 'fstream::off_type' might be
298 // implementation-defined.
299 File.seekg(fstream::off_type(bl + (idx *= BlockSize)));
300
301 // XXX: Conversion to 'ptrdiff_t' might be implementation-defined.
302 const auto mid(vec.cbegin() + ptrdiff_t(pos));
303
304 YAssert(mid <= vec.cend(), "Wrong iterator found.");
305
306 auto it(vec.cbegin());
307
308 YAssert(it <= mid, "Wrong iterator found.");
309
310 size_t n_byte(0);
311 Sentry sentry(File);
312
313 while(it != mid)
314 n_byte += ConvertChar([&](ystdex::pseudo_output){
315 ++it;
316 }, pfun, sentry.Iterator, ystdex::pseudo_output());
317 return idx + n_byte;
318 }
319 return idx;
320 }
321
322
323 string
324 CopySliceFrom(TextFileBuffer& buf, size_t pos, size_t len)
325 {
326 const auto i_end(buf.end());
327 auto i_beg(buf.GetIterator(pos));
328
329 if(i_beg == i_end)
330 throw std::out_of_range("Wrong offset found.");
331 if(len == 0)
332 return {};
333
334 YAssert(pos < pos + len, "Unexpected unsigned integer round up.");
335
336 string str;
337
338 str.reserve(len * 2);
339 while(len != 0 && i_beg != i_end)
340 {
341 str.push_back(*i_beg);
342 yunseq(++i_beg, --len);
343 }
344 return str;
345 }
346
347 } // namespace Text;
348
349 } // namespace YSLib;
350
yassume
#define yassume(_expr)
假定：环境语义。
Definition: cassert.h:99
CHRLib::enc
enc
Definition: CharacterProcessing.h:81
YSLib::Text::TextFileBuffer::GetIterator
encoding n_text_size iterator GetIterator(size_t)
取文本字节位置对应的迭代器。
Definition: TextManager.cpp:249
YSLib::Drawing::x
The boundary is out of the const Rect SDst SPos x
Definition: YDraw.h:127
any_iterator.hpp
动态泛型迭代器。
yconstexpr
#define yconstexpr
指定编译时常量表达式。
Definition: ydef.h:653
CHRLib::ConvertCharacter
ConversionResult ConvertCharacter(_fConv f, _tOut &&obj, _tIn &&i, ConversionState &&st)
Definition: Convert.hpp:41
platform::Descriptions::Warning
Definition: YCommon.h:157
YSLib::Text::TextFileBuffer::BlockSize
static yconstexpr const size_t BlockSize
默认文本区块大小。
Definition: TextManager.h:123
YSLib::Text::buf
const char * buf
Definition: TextFile.h:77
YSLib::Text::TextFileBuffer::bl
size_t bl
Definition: TextManager.h:137
YSLib
YSLib 命名空间。
Definition: AndroidHost.h:45
CHRLib::CharSet::Null
空字符集。
Definition: Encoding.h:63
YSLib::Text::TextFileBuffer::n_block
size_t n_block
区块数。
Definition: TextManager.h:144
YSLib::Text::TextFileBuffer::iterator
目标编码迭代器类型。
Definition: TextManager.h:68
CatchExpr
#define CatchExpr(_ex,...)
指定表达式作为异常处理器。
Definition: YBaseMacro.h:129
YSLib::Text::TextFileBuffer::GetPosition
size_t GetPosition(iterator)
取迭代器对应的文本字节位置。
Definition: TextManager.cpp:281
YSLib::LoggedEvent
记录日志的异常事件类。
Definition: YException.h:49
YSLib::Text::TextFileBuffer::begin
iterator begin() ynothrow
取文本缓冲区起始迭代器。
Definition: TextManager.cpp:239
DefDeCtor
#define DefDeCtor(_t)
简单通用成员函数定义。
Definition: YBaseMacro.h:156
YSLib::Text::TextFileBuffer::n_text_size
size_t n_text_size
文本大小。
Definition: TextManager.h:142
CHRLib::c
c
Definition: CharacterProcessing.h:81
CHRLib::CharSet::Encoding
Encoding
字符流编码标识。
Definition: Encoding.h:58
CHRLib::char16_t
char16_t((*c_ptr<< CHAR_BIT)|c_ptr[1])) inline PDefH(char16_t
取 c_ptr 指向的小端序双字节字符。
YSLib::size_t
name const ValueNode::Container const string &name ynothrow observer_ptr< ValueNode::Container > const string &name ynothrow observer_ptr< const ValueNode::Container > const string &name ynothrow size_t
Definition: ValueNode.h:639
YTraceDe
#define YTraceDe(...)
YCLib 默认调试跟踪。
Definition: Debug.h:305
CatchIgnore
#define CatchIgnore(_ex)
指定异常处理器忽略指定异常。
Definition: YBaseMacro.h:133
YSLib::ynothrow
YB_PURE const _type size_t n ynothrow
Definition: YCoreUtilities.h:168
YSLib::Text::TextFileBuffer::size_t
encoding size_t
Definition: TextManager.h:190
Iterator
std::istream_iterator< char > Iterator
Definition: TextManager.cpp:90
CHRLib::FetchFixedCharWidth
YF_API YB_STATELESS size_t FetchFixedCharWidth(Encoding)
取指定固定编码的固定字符宽度。
Definition: chrmap.cpp:37
CHRLib::FetchMapperPtr
yconstfn_relaxed ystdex::add_pointer_t< _tRet(_tParams...)> FetchMapperPtr(Encoding enc)
取指定编码映射的转换函数指针。
Definition: MappingEx.h:285
YB_UNLIKELY
#define YB_UNLIKELY(_expr)
分支预测提示。
Definition: ydef.h:438
YSLib::Text::TextFileBuffer::buffer
MapType buffer
缓冲映射。
Definition: TextManager.h:146
YModules.h
std
Definition: memory.hpp:1100
platform::Deref
yconstfn auto Deref(_type &&p) -> decltype(*p)
断言并解引用非空指针。
Definition: Debug.h:440
YSLib::Text::TextFileBuffer::max_width
size_t max_width
最大编码每字符的字节数。
Definition: TextManager.h:160
YSLib::Text::TextFileBuffer::iterator::operator--
iterator & operator--() ynothrow
迭代：循环向前遍历。
Definition: TextManager.cpp:130
YSLib::Text::TextFileBuffer::File
std::istream & File
流引用。
Definition: TextManager.h:130
TryExpr
#define TryExpr(...)
指定表达式在 try 块中求值。
Definition: YBaseMacro.h:120
YSLib::Text::DetectBOM
YF_API size_t DetectBOM(const char *)
Definition: TextFile.cpp:72
DefDeMoveAssignment
#define DefDeMoveAssignment(_t)
Definition: YBaseMacro.h:190
YSLib::YAssert
YAssert(!(v< Deref(a)),"Value less than lower bound found.")
YSLib::Text::TextFileBuffer::iterator::operator++
iterator & operator++() ynothrow
迭代：循环向后遍历。
Definition: TextManager.cpp:117
yunseq
#define yunseq
无序列依赖表达式组求值。
Definition: ydef.h:996
YSLib::Text::TextFileBuffer::iterator::operator*
reference operator*() const
Definition: TextManager.cpp:149
YSLib::Text::str
const char const char * str
Definition: TextFile.h:77
DefDeMoveCtor
#define DefDeMoveCtor(_t)
Definition: YBaseMacro.h:166
ystdex::pseudo_output
伪输出对象。
Definition: ref.hpp:254
YSLib::UI::y
SPos SPos y
Definition: YWidget.cpp:44
fp
tidy_ptr< std::istream > fp
Definition: TextManager.cpp:86
YSLib::i
return i
Definition: YCoreUtilities.h:176
YSLib::Text::TextFileBuffer::operator[]
BlockType & operator[](size_t)
块随机访问。
Definition: TextManager.cpp:203
CHRLib::FetchMaxVariantCharWidth
YF_API YB_STATELESS size_t FetchMaxVariantCharWidth(Encoding)
取指定变长编码的最大字符宽度。
Definition: chrmap.cpp:67
ystdex::input_monomorphic_iterator
any_input_iterator< void_ref, ptrdiff_t, void *, void_ref > input_monomorphic_iterator
Definition: any_iterator.hpp:331
YSLib::Text::TextFileBuffer::TextFileBuffer
TextFileBuffer(std::istream &, Encoding=CharSet::Null)
构造：使用流和指定编码。
Definition: TextManager.cpp:172
Text
YSLib::Text::TextFileBuffer
文本文件块缓冲区。
Definition: TextManager.h:46
YSLib::Text::CopySliceFrom
YF_API string CopySliceFrom(TextFileBuffer &, size_t, size_t)
从文本文件缓冲区复制指定字节位置起始的字符串。
Definition: TextManager.cpp:324
YSLib::Text::operator==
bool operator==(const TextFileBuffer::iterator &x, const TextFileBuffer::iterator &y) ynothrow
Definition: TextManager.cpp:162
yforward
#define yforward(_expr)
根据参数类型使用 std::forward 传递对应参数。
Definition: ydef.h:969
YSLib::Text::TextFileBuffer::end
iterator end() ynothrow
取文本缓冲区终止迭代器。
Definition: TextManager.cpp:244
ystdex::make_observer
observer_ptr< _type > make_observer(_type *p)
Definition: memory.hpp:1042
YB_EXPECT
#define YB_EXPECT(_expr, _constant)
Definition: ydef.h:436
YSLib::IO::f
const string &path const String &path _func f
Definition: YSLib/Service/FileSystem.h:430
CHRLib::args
_tParams && args
Definition: CharacterProcessing.h:538
YSLib::Text::TextFileBuffer::encoding
Encoding encoding
Definition: TextManager.h:136
CHRLib::CharSet::GBK
yconstexpr const Encoding GBK(csGBK)
YSLib::Text::TextFileBuffer::BlockType
pair< vector< char16_t >, size_t > BlockType
缓冲区块类型。
Definition: TextManager.h:55
ystdex::cpp2011::end
_type * end(_type(&&array)[_vN])
Definition: range.hpp:54
ystdex::tuple_iterator
元组迭代器。
Definition: iterator.hpp:647
platform::bool
std::FILE *fp bool
Definition: FileIO.h:1187