• Skip to content
  • Skip to link menu
KDE API Reference
  • KDE API Reference
  • kdelibs API Reference
  • KDE Home
  • Contact Us
 

KDECore

  • sources
  • kde-4.14
  • kdelibs
  • kdecore
  • localization
  • probers
nsMBCSSM.cpp
Go to the documentation of this file.
1 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* -*- C++ -*-
3 * Copyright (C) 1998 <developer@mozilla.org>
4 *
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included
15 * in all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 */
25 
26 #include "nsCodingStateMachine.h"
27 
28 /*
29 Modification from frank tang's original work:
30 . 0x00 is allowed as a legal character. Since some web pages contains this char in
31  text stream.
32 */
33 
34 // BIG5
35 
36 namespace kencodingprober {
37 static unsigned int BIG5_cls [ 256 / 8 ] = {
38 //PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07
39 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 //allow 0x00 as legal value
40 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f
41 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17
42 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f
43 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27
44 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f
45 PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37
46 PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f
47 PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47
48 PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f
49 PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57
50 PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f
51 PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67
52 PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f
53 PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77
54 PCK4BITS(2,2,2,2,2,2,2,1), // 78 - 7f
55 PCK4BITS(4,4,4,4,4,4,4,4), // 80 - 87
56 PCK4BITS(4,4,4,4,4,4,4,4), // 88 - 8f
57 PCK4BITS(4,4,4,4,4,4,4,4), // 90 - 97
58 PCK4BITS(4,4,4,4,4,4,4,4), // 98 - 9f
59 PCK4BITS(4,3,3,3,3,3,3,3), // a0 - a7
60 PCK4BITS(3,3,3,3,3,3,3,3), // a8 - af
61 PCK4BITS(3,3,3,3,3,3,3,3), // b0 - b7
62 PCK4BITS(3,3,3,3,3,3,3,3), // b8 - bf
63 PCK4BITS(3,3,3,3,3,3,3,3), // c0 - c7
64 PCK4BITS(3,3,3,3,3,3,3,3), // c8 - cf
65 PCK4BITS(3,3,3,3,3,3,3,3), // d0 - d7
66 PCK4BITS(3,3,3,3,3,3,3,3), // d8 - df
67 PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7
68 PCK4BITS(3,3,3,3,3,3,3,3), // e8 - ef
69 PCK4BITS(3,3,3,3,3,3,3,3), // f0 - f7
70 PCK4BITS(3,3,3,3,3,3,3,0) // f8 - ff
71 };
72 
73 
74 static unsigned int BIG5_st [ 3] = {
75 PCK4BITS(eError,eStart,eStart, 3,eError,eError,eError,eError),//00-07
76 PCK4BITS(eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError),//08-0f
77 PCK4BITS(eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart) //10-17
78 };
79 
80 static const unsigned int Big5CharLenTable[] = {0, 1, 1, 2, 0};
81 
82 SMModel Big5SMModel = {
83  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, BIG5_cls },
84  5,
85  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, BIG5_st },
86  Big5CharLenTable,
87  "Big5",
88 };
89 
90 static unsigned int EUCJP_cls [ 256 / 8 ] = {
91 //PCK4BITS(5,4,4,4,4,4,4,4), // 00 - 07
92 PCK4BITS(4,4,4,4,4,4,4,4), // 00 - 07
93 PCK4BITS(4,4,4,4,4,4,5,5), // 08 - 0f
94 PCK4BITS(4,4,4,4,4,4,4,4), // 10 - 17
95 PCK4BITS(4,4,4,5,4,4,4,4), // 18 - 1f
96 PCK4BITS(4,4,4,4,4,4,4,4), // 20 - 27
97 PCK4BITS(4,4,4,4,4,4,4,4), // 28 - 2f
98 PCK4BITS(4,4,4,4,4,4,4,4), // 30 - 37
99 PCK4BITS(4,4,4,4,4,4,4,4), // 38 - 3f
100 PCK4BITS(4,4,4,4,4,4,4,4), // 40 - 47
101 PCK4BITS(4,4,4,4,4,4,4,4), // 48 - 4f
102 PCK4BITS(4,4,4,4,4,4,4,4), // 50 - 57
103 PCK4BITS(4,4,4,4,4,4,4,4), // 58 - 5f
104 PCK4BITS(4,4,4,4,4,4,4,4), // 60 - 67
105 PCK4BITS(4,4,4,4,4,4,4,4), // 68 - 6f
106 PCK4BITS(4,4,4,4,4,4,4,4), // 70 - 77
107 PCK4BITS(4,4,4,4,4,4,4,4), // 78 - 7f
108 PCK4BITS(5,5,5,5,5,5,5,5), // 80 - 87
109 PCK4BITS(5,5,5,5,5,5,1,3), // 88 - 8f
110 PCK4BITS(5,5,5,5,5,5,5,5), // 90 - 97
111 PCK4BITS(5,5,5,5,5,5,5,5), // 98 - 9f
112 PCK4BITS(5,2,2,2,2,2,2,2), // a0 - a7
113 PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af
114 PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7
115 PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf
116 PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7
117 PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf
118 PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7
119 PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df
120 PCK4BITS(0,0,0,0,0,0,0,0), // e0 - e7
121 PCK4BITS(0,0,0,0,0,0,0,0), // e8 - ef
122 PCK4BITS(0,0,0,0,0,0,0,0), // f0 - f7
123 PCK4BITS(0,0,0,0,0,0,0,5) // f8 - ff
124 };
125 
126 
127 static unsigned int EUCJP_st [ 5] = {
128 PCK4BITS( 3, 4, 3, 5,eStart,eError,eError,eError),//00-07
129 PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f
130 PCK4BITS(eItsMe,eItsMe,eStart,eError,eStart,eError,eError,eError),//10-17
131 PCK4BITS(eError,eError,eStart,eError,eError,eError, 3,eError),//18-1f
132 PCK4BITS( 3,eError,eError,eError,eStart,eStart,eStart,eStart) //20-27
133 };
134 
135 static const unsigned int EUCJPCharLenTable[] = {2, 2, 2, 3, 1, 0};
136 
137 SMModel EUCJPSMModel = {
138  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCJP_cls },
139  6,
140  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCJP_st },
141  EUCJPCharLenTable,
142  "EUC-JP",
143 };
144 
145 static unsigned int EUCKR_cls [ 256 / 8 ] = {
146 //PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07
147 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07
148 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f
149 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17
150 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f
151 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27
152 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f
153 PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37
154 PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f
155 PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47
156 PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f
157 PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57
158 PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f
159 PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67
160 PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f
161 PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77
162 PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f
163 PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87
164 PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f
165 PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97
166 PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f
167 PCK4BITS(0,2,2,2,2,2,2,2), // a0 - a7
168 PCK4BITS(2,2,2,2,2,3,3,3), // a8 - af
169 PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7
170 PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf
171 PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7
172 PCK4BITS(2,3,2,2,2,2,2,2), // c8 - cf
173 PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7
174 PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df
175 PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7
176 PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef
177 PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7
178 PCK4BITS(2,2,2,2,2,2,2,0) // f8 - ff
179 };
180 
181 
182 static unsigned int EUCKR_st [ 2] = {
183 PCK4BITS(eError,eStart, 3,eError,eError,eError,eError,eError),//00-07
184 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart) //08-0f
185 };
186 
187 static const unsigned int EUCKRCharLenTable[] = {0, 1, 2, 0};
188 
189 SMModel EUCKRSMModel = {
190  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCKR_cls },
191  4,
192  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCKR_st },
193  EUCKRCharLenTable,
194  "EUC-KR",
195 };
196 
197 /* obsolete GB2312 by gb18030
198 static unsigned int GB2312_cls [ 256 / 8 ] = {
199 //PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07
200 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07
201 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f
202 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17
203 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f
204 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27
205 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f
206 PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37
207 PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f
208 PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47
209 PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f
210 PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57
211 PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f
212 PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67
213 PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f
214 PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77
215 PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f
216 PCK4BITS(1,0,0,0,0,0,0,0), // 80 - 87
217 PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f
218 PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97
219 PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f
220 PCK4BITS(0,2,2,2,2,2,2,2), // a0 - a7
221 PCK4BITS(2,2,3,3,3,3,3,3), // a8 - af
222 PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7
223 PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf
224 PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7
225 PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf
226 PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7
227 PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df
228 PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7
229 PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef
230 PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7
231 PCK4BITS(2,2,2,2,2,2,2,0) // f8 - ff
232 };
233 
234 
235 static unsigned int GB2312_st [ 2] = {
236 PCK4BITS(eError,eStart, 3,eError,eError,eError,eError,eError),//00-07
237 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart) //08-0f
238 };
239 
240 static const unsigned int GB2312CharLenTable[] = {0, 1, 2, 0};
241 
242 SMModel GB2312SMModel = {
243  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB2312_cls },
244  4,
245  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB2312_st },
246  GB2312CharLenTable,
247  "GB2312",
248 };
249 */
250 
251 // the following state machine data was created by perl script in
252 // intl/chardet/tools. It should be the same as in PSM detector.
253 static unsigned int GB18030_cls [ 256 / 8 ] = {
254 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07
255 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f
256 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17
257 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f
258 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27
259 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f
260 PCK4BITS(3,3,3,3,3,3,3,3), // 30 - 37
261 PCK4BITS(3,3,1,1,1,1,1,1), // 38 - 3f
262 PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47
263 PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f
264 PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57
265 PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f
266 PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67
267 PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f
268 PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77
269 PCK4BITS(2,2,2,2,2,2,2,4), // 78 - 7f
270 PCK4BITS(5,6,6,6,6,6,6,6), // 80 - 87
271 PCK4BITS(6,6,6,6,6,6,6,6), // 88 - 8f
272 PCK4BITS(6,6,6,6,6,6,6,6), // 90 - 97
273 PCK4BITS(6,6,6,6,6,6,6,6), // 98 - 9f
274 PCK4BITS(6,6,6,6,6,6,6,6), // a0 - a7
275 PCK4BITS(6,6,6,6,6,6,6,6), // a8 - af
276 PCK4BITS(6,6,6,6,6,6,6,6), // b0 - b7
277 PCK4BITS(6,6,6,6,6,6,6,6), // b8 - bf
278 PCK4BITS(6,6,6,6,6,6,6,6), // c0 - c7
279 PCK4BITS(6,6,6,6,6,6,6,6), // c8 - cf
280 PCK4BITS(6,6,6,6,6,6,6,6), // d0 - d7
281 PCK4BITS(6,6,6,6,6,6,6,6), // d8 - df
282 PCK4BITS(6,6,6,6,6,6,6,6), // e0 - e7
283 PCK4BITS(6,6,6,6,6,6,6,6), // e8 - ef
284 PCK4BITS(6,6,6,6,6,6,6,6), // f0 - f7
285 PCK4BITS(6,6,6,6,6,6,6,0) // f8 - ff
286 };
287 
288 
289 static unsigned int GB18030_st [ 6] = {
290 PCK4BITS(eError,eStart,eStart,eStart,eStart,eStart, 3,eError),//00-07
291 PCK4BITS(eError,eError,eError,eError,eError,eError,eItsMe,eItsMe),//08-0f
292 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart),//10-17
293 PCK4BITS( 4,eError,eStart,eStart,eError,eError,eError,eError),//18-1f
294 PCK4BITS(eError,eError, 5,eError,eError,eError,eItsMe,eError),//20-27
295 PCK4BITS(eError,eError,eStart,eStart,eStart,eStart,eStart,eStart) //28-2f
296 };
297 
298 // To be accurate, the length of class 6 can be either 2 or 4.
299 // But it is not necessary to discriminate between the two since
300 // it is used for frequency analysis only, and we are validing
301 // each code range there as well. So it is safe to set it to be
302 // 2 here.
303 static const unsigned int GB18030CharLenTable[] = {0, 1, 1, 1, 1, 1, 2};
304 
305 SMModel GB18030SMModel = {
306  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB18030_cls },
307  7,
308  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB18030_st },
309  GB18030CharLenTable,
310  "GB18030",
311 };
312 
313 // sjis
314 
315 static unsigned int SJIS_cls [ 256 / 8 ] = {
316 //PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07
317 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07
318 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f
319 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17
320 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f
321 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27
322 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f
323 PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37
324 PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f
325 PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47
326 PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f
327 PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57
328 PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f
329 PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67
330 PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f
331 PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77
332 PCK4BITS(2,2,2,2,2,2,2,1), // 78 - 7f
333 PCK4BITS(3,3,3,3,3,3,3,3), // 80 - 87
334 PCK4BITS(3,3,3,3,3,3,3,3), // 88 - 8f
335 PCK4BITS(3,3,3,3,3,3,3,3), // 90 - 97
336 PCK4BITS(3,3,3,3,3,3,3,3), // 98 - 9f
337 //0xa0 is illegal in sjis encoding, but some pages does
338 //contain such byte. We need to be more error forgiven.
339 PCK4BITS(2,2,2,2,2,2,2,2), // a0 - a7
340 PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af
341 PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7
342 PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf
343 PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7
344 PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf
345 PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7
346 PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df
347 PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7
348 PCK4BITS(3,3,3,3,3,4,4,4), // e8 - ef
349 PCK4BITS(4,4,4,4,4,4,4,4), // f0 - f7
350 PCK4BITS(4,4,4,4,4,0,0,0) // f8 - ff
351 };
352 
353 
354 static unsigned int SJIS_st [ 3] = {
355 PCK4BITS(eError,eStart,eStart, 3,eError,eError,eError,eError),//00-07
356 PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f
357 PCK4BITS(eItsMe,eItsMe,eError,eError,eStart,eStart,eStart,eStart) //10-17
358 };
359 
360 static const unsigned int SJISCharLenTable[] = {0, 1, 1, 2, 0, 0};
361 
362 SMModel SJISSMModel = {
363  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, SJIS_cls },
364  6,
365  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, SJIS_st },
366  SJISCharLenTable,
367  "Shift_JIS",
368 };
369 
370 
371 static unsigned int UCS2BE_cls [ 256 / 8 ] = {
372 PCK4BITS(0,0,0,0,0,0,0,0), // 00 - 07
373 PCK4BITS(0,0,1,0,0,2,0,0), // 08 - 0f
374 PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17
375 PCK4BITS(0,0,0,3,0,0,0,0), // 18 - 1f
376 PCK4BITS(0,0,0,0,0,0,0,0), // 20 - 27
377 PCK4BITS(0,3,3,3,3,3,0,0), // 28 - 2f
378 PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37
379 PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f
380 PCK4BITS(0,0,0,0,0,0,0,0), // 40 - 47
381 PCK4BITS(0,0,0,0,0,0,0,0), // 48 - 4f
382 PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57
383 PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f
384 PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67
385 PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f
386 PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77
387 PCK4BITS(0,0,0,0,0,0,0,0), // 78 - 7f
388 PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87
389 PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f
390 PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97
391 PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f
392 PCK4BITS(0,0,0,0,0,0,0,0), // a0 - a7
393 PCK4BITS(0,0,0,0,0,0,0,0), // a8 - af
394 PCK4BITS(0,0,0,0,0,0,0,0), // b0 - b7
395 PCK4BITS(0,0,0,0,0,0,0,0), // b8 - bf
396 PCK4BITS(0,0,0,0,0,0,0,0), // c0 - c7
397 PCK4BITS(0,0,0,0,0,0,0,0), // c8 - cf
398 PCK4BITS(0,0,0,0,0,0,0,0), // d0 - d7
399 PCK4BITS(0,0,0,0,0,0,0,0), // d8 - df
400 PCK4BITS(0,0,0,0,0,0,0,0), // e0 - e7
401 PCK4BITS(0,0,0,0,0,0,0,0), // e8 - ef
402 PCK4BITS(0,0,0,0,0,0,0,0), // f0 - f7
403 PCK4BITS(0,0,0,0,0,0,4,5) // f8 - ff
404 };
405 
406 
407 static unsigned int UCS2BE_st [ 7] = {
408 PCK4BITS( 5, 7, 7,eError, 4, 3,eError,eError),//00-07
409 PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f
410 PCK4BITS(eItsMe,eItsMe, 6, 6, 6, 6,eError,eError),//10-17
411 PCK4BITS( 6, 6, 6, 6, 6,eItsMe, 6, 6),//18-1f
412 PCK4BITS( 6, 6, 6, 6, 5, 7, 7,eError),//20-27
413 PCK4BITS( 5, 8, 6, 6,eError, 6, 6, 6),//28-2f
414 PCK4BITS( 6, 6, 6, 6,eError,eError,eStart,eStart) //30-37
415 };
416 
417 static const unsigned int UCS2BECharLenTable[] = {2, 2, 2, 0, 2, 2};
418 
419 SMModel UCS2BESMModel = {
420  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, UCS2BE_cls },
421  6,
422  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, UCS2BE_st },
423  UCS2BECharLenTable,
424  "UTF-16BE",
425 };
426 
427 static unsigned int UCS2LE_cls [ 256 / 8 ] = {
428 PCK4BITS(0,0,0,0,0,0,0,0), // 00 - 07
429 PCK4BITS(0,0,1,0,0,2,0,0), // 08 - 0f
430 PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17
431 PCK4BITS(0,0,0,3,0,0,0,0), // 18 - 1f
432 PCK4BITS(0,0,0,0,0,0,0,0), // 20 - 27
433 PCK4BITS(0,3,3,3,3,3,0,0), // 28 - 2f
434 PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37
435 PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f
436 PCK4BITS(0,0,0,0,0,0,0,0), // 40 - 47
437 PCK4BITS(0,0,0,0,0,0,0,0), // 48 - 4f
438 PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57
439 PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f
440 PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67
441 PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f
442 PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77
443 PCK4BITS(0,0,0,0,0,0,0,0), // 78 - 7f
444 PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87
445 PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f
446 PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97
447 PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f
448 PCK4BITS(0,0,0,0,0,0,0,0), // a0 - a7
449 PCK4BITS(0,0,0,0,0,0,0,0), // a8 - af
450 PCK4BITS(0,0,0,0,0,0,0,0), // b0 - b7
451 PCK4BITS(0,0,0,0,0,0,0,0), // b8 - bf
452 PCK4BITS(0,0,0,0,0,0,0,0), // c0 - c7
453 PCK4BITS(0,0,0,0,0,0,0,0), // c8 - cf
454 PCK4BITS(0,0,0,0,0,0,0,0), // d0 - d7
455 PCK4BITS(0,0,0,0,0,0,0,0), // d8 - df
456 PCK4BITS(0,0,0,0,0,0,0,0), // e0 - e7
457 PCK4BITS(0,0,0,0,0,0,0,0), // e8 - ef
458 PCK4BITS(0,0,0,0,0,0,0,0), // f0 - f7
459 PCK4BITS(0,0,0,0,0,0,4,5) // f8 - ff
460 };
461 
462 
463 static unsigned int UCS2LE_st [ 7] = {
464 PCK4BITS( 6, 6, 7, 6, 4, 3,eError,eError),//00-07
465 PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f
466 PCK4BITS(eItsMe,eItsMe, 5, 5, 5,eError,eItsMe,eError),//10-17
467 PCK4BITS( 5, 5, 5,eError, 5,eError, 6, 6),//18-1f
468 PCK4BITS( 7, 6, 8, 8, 5, 5, 5,eError),//20-27
469 PCK4BITS( 5, 5, 5,eError,eError,eError, 5, 5),//28-2f
470 PCK4BITS( 5, 5, 5,eError, 5,eError,eStart,eStart) //30-37
471 };
472 
473 static const unsigned int UCS2LECharLenTable[] = {2, 2, 2, 2, 2, 2};
474 
475 SMModel UCS2LESMModel = {
476  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, UCS2LE_cls },
477  6,
478  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, UCS2LE_st },
479  UCS2LECharLenTable,
480  "UTF-16LE",
481 };
482 
483 
484 static unsigned int UTF8_cls [ 256 / 8 ] = {
485 //PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07
486 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 //allow 0x00 as a legal value
487 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f
488 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17
489 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f
490 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27
491 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f
492 PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37
493 PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f
494 PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47
495 PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f
496 PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57
497 PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f
498 PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67
499 PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f
500 PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77
501 PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f
502 PCK4BITS(2,2,2,2,3,3,3,3), // 80 - 87
503 PCK4BITS(4,4,4,4,4,4,4,4), // 88 - 8f
504 PCK4BITS(4,4,4,4,4,4,4,4), // 90 - 97
505 PCK4BITS(4,4,4,4,4,4,4,4), // 98 - 9f
506 PCK4BITS(5,5,5,5,5,5,5,5), // a0 - a7
507 PCK4BITS(5,5,5,5,5,5,5,5), // a8 - af
508 PCK4BITS(5,5,5,5,5,5,5,5), // b0 - b7
509 PCK4BITS(5,5,5,5,5,5,5,5), // b8 - bf
510 PCK4BITS(0,0,6,6,6,6,6,6), // c0 - c7
511 PCK4BITS(6,6,6,6,6,6,6,6), // c8 - cf
512 PCK4BITS(6,6,6,6,6,6,6,6), // d0 - d7
513 PCK4BITS(6,6,6,6,6,6,6,6), // d8 - df
514 PCK4BITS(7,8,8,8,8,8,8,8), // e0 - e7
515 PCK4BITS(8,8,8,8,8,9,8,8), // e8 - ef
516 PCK4BITS(10,11,11,11,11,11,11,11), // f0 - f7
517 PCK4BITS(12,13,13,13,14,15,0,0) // f8 - ff
518 };
519 
520 
521 static unsigned int UTF8_st [ 26] = {
522 PCK4BITS(eError,eStart,eError,eError,eError,eError, 12, 10),//00-07
523 PCK4BITS( 9, 11, 8, 7, 6, 5, 4, 3),//08-0f
524 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//10-17
525 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//18-1f
526 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe),//20-27
527 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe),//28-2f
528 PCK4BITS(eError,eError, 5, 5, 5, 5,eError,eError),//30-37
529 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//38-3f
530 PCK4BITS(eError,eError,eError, 5, 5, 5,eError,eError),//40-47
531 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//48-4f
532 PCK4BITS(eError,eError, 7, 7, 7, 7,eError,eError),//50-57
533 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//58-5f
534 PCK4BITS(eError,eError,eError,eError, 7, 7,eError,eError),//60-67
535 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//68-6f
536 PCK4BITS(eError,eError, 9, 9, 9, 9,eError,eError),//70-77
537 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//78-7f
538 PCK4BITS(eError,eError,eError,eError,eError, 9,eError,eError),//80-87
539 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//88-8f
540 PCK4BITS(eError,eError, 12, 12, 12, 12,eError,eError),//90-97
541 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//98-9f
542 PCK4BITS(eError,eError,eError,eError,eError, 12,eError,eError),//a0-a7
543 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//a8-af
544 PCK4BITS(eError,eError, 12, 12, 12,eError,eError,eError),//b0-b7
545 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//b8-bf
546 PCK4BITS(eError,eError,eStart,eStart,eStart,eStart,eError,eError),//c0-c7
547 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError) //c8-cf
548 };
549 
550 static const unsigned int UTF8CharLenTable[] = {0, 1, 0, 0, 0, 0, 2, 3,
551  3, 3, 4, 4, 5, 5, 6, 6 };
552 
553 SMModel UTF8SMModel = {
554  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, UTF8_cls },
555  16,
556  {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, UTF8_st },
557  UTF8CharLenTable,
558  "UTF-8",
559 };
560 }
561 
562 
kencodingprober::UTF8_st
static unsigned int UTF8_st[26]
Definition: nsMBCSSM.cpp:521
kencodingprober::SMModel
Definition: nsCodingStateMachine.h:44
kencodingprober::UCS2LE_cls
static unsigned int UCS2LE_cls[256/8]
Definition: nsMBCSSM.cpp:427
kencodingprober::EUCKRSMModel
KDE_NO_EXPORT SMModel EUCKRSMModel
Definition: nsMBCSSM.cpp:189
kencodingprober::EUCJP_cls
static unsigned int EUCJP_cls[256/8]
Definition: nsMBCSSM.cpp:90
kencodingprober::EUCJPSMModel
KDE_NO_EXPORT SMModel EUCJPSMModel
Definition: nsMBCSSM.cpp:137
kencodingprober::GB18030_st
static unsigned int GB18030_st[6]
Definition: nsMBCSSM.cpp:289
kencodingprober::UCS2LESMModel
KDE_NO_EXPORT SMModel UCS2LESMModel
Definition: nsMBCSSM.cpp:475
kencodingprober::eUnitMsk4bits
Definition: nsPkgInt.h:49
kencodingprober::UCS2BESMModel
KDE_NO_EXPORT SMModel UCS2BESMModel
Definition: nsMBCSSM.cpp:419
kencodingprober::UTF8_cls
static unsigned int UTF8_cls[256/8]
Definition: nsMBCSSM.cpp:484
kencodingprober::Big5CharLenTable
static const unsigned int Big5CharLenTable[]
Definition: nsMBCSSM.cpp:80
kencodingprober::SJISCharLenTable
static const unsigned int SJISCharLenTable[]
Definition: nsMBCSSM.cpp:360
kencodingprober::UCS2BE_st
static unsigned int UCS2BE_st[7]
Definition: nsMBCSSM.cpp:407
kencodingprober::SJIS_cls
static unsigned int SJIS_cls[256/8]
Definition: nsMBCSSM.cpp:315
kencodingprober::eError
Definition: nsCodingStateMachine.h:37
nsCodingStateMachine.h
kencodingprober::EUCJPCharLenTable
static const unsigned int EUCJPCharLenTable[]
Definition: nsMBCSSM.cpp:135
kencodingprober::GB18030CharLenTable
static const unsigned int GB18030CharLenTable[]
Definition: nsMBCSSM.cpp:303
kencodingprober::EUCKRCharLenTable
static const unsigned int EUCKRCharLenTable[]
Definition: nsMBCSSM.cpp:187
kencodingprober::eIdxSft4bits
Definition: nsPkgInt.h:31
PCK4BITS
#define PCK4BITS(a, b, c, d, e, f, g, h)
Definition: nsPkgInt.h:68
kencodingprober::BIG5_cls
static unsigned int BIG5_cls[256/8]
Definition: nsMBCSSM.cpp:37
kencodingprober::EUCKR_st
static unsigned int EUCKR_st[2]
Definition: nsMBCSSM.cpp:182
kencodingprober::BIG5_st
static unsigned int BIG5_st[3]
Definition: nsMBCSSM.cpp:74
kencodingprober::Big5SMModel
KDE_NO_EXPORT SMModel Big5SMModel
Definition: nsMBCSSM.cpp:82
kencodingprober::EUCJP_st
static unsigned int EUCJP_st[5]
Definition: nsMBCSSM.cpp:127
kencodingprober::eSftMsk4bits
Definition: nsPkgInt.h:37
kencodingprober::UTF8CharLenTable
static const unsigned int UTF8CharLenTable[]
Definition: nsMBCSSM.cpp:550
kencodingprober::GB18030_cls
static unsigned int GB18030_cls[256/8]
Definition: nsMBCSSM.cpp:253
kencodingprober::UCS2LECharLenTable
static const unsigned int UCS2LECharLenTable[]
Definition: nsMBCSSM.cpp:473
kencodingprober::eItsMe
Definition: nsCodingStateMachine.h:38
kencodingprober::SJISSMModel
KDE_NO_EXPORT SMModel SJISSMModel
Definition: nsMBCSSM.cpp:362
kencodingprober::eStart
Definition: nsCodingStateMachine.h:36
kencodingprober::UTF8SMModel
KDE_NO_EXPORT SMModel UTF8SMModel
Definition: nsMBCSSM.cpp:553
kencodingprober::UCS2BECharLenTable
static const unsigned int UCS2BECharLenTable[]
Definition: nsMBCSSM.cpp:417
kencodingprober::eBitSft4bits
Definition: nsPkgInt.h:43
kencodingprober::SJIS_st
static unsigned int SJIS_st[3]
Definition: nsMBCSSM.cpp:354
kencodingprober::EUCKR_cls
static unsigned int EUCKR_cls[256/8]
Definition: nsMBCSSM.cpp:145
kencodingprober::GB18030SMModel
KDE_NO_EXPORT SMModel GB18030SMModel
Definition: nsMBCSSM.cpp:305
kencodingprober::UCS2LE_st
static unsigned int UCS2LE_st[7]
Definition: nsMBCSSM.cpp:463
kencodingprober::UCS2BE_cls
static unsigned int UCS2BE_cls[256/8]
Definition: nsMBCSSM.cpp:371
This file is part of the KDE documentation.
Documentation copyright © 1996-2020 The KDE developers.
Generated on Mon Jun 22 2020 13:22:12 by doxygen 1.8.7 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.

KDECore

Skip menu "KDECore"
  • Main Page
  • Namespace List
  • Namespace Members
  • Alphabetical List
  • Class List
  • Class Hierarchy
  • Class Members
  • File List
  • File Members
  • Modules
  • Related Pages

kdelibs API Reference

Skip menu "kdelibs API Reference"
  • DNSSD
  • Interfaces
  •   KHexEdit
  •   KMediaPlayer
  •   KSpeech
  •   KTextEditor
  • kconf_update
  • KDE3Support
  •   KUnitTest
  • KDECore
  • KDED
  • KDEsu
  • KDEUI
  • KDEWebKit
  • KDocTools
  • KFile
  • KHTML
  • KImgIO
  • KInit
  • kio
  • KIOSlave
  • KJS
  •   KJS-API
  •   WTF
  • kjsembed
  • KNewStuff
  • KParts
  • KPty
  • Kross
  • KUnitConversion
  • KUtils
  • Nepomuk
  • Plasma
  • Solid
  • Sonnet
  • ThreadWeaver

Search



Report problems with this website to our bug tracking system.
Contact the specific authors with questions and comments about the page contents.

KDE® and the K Desktop Environment® logo are registered trademarks of KDE e.V. | Legal