The Tor Browser / file comparison
comparison: extensions/universalchardet/src/base/nsMBCSSM.cpp
extensions/universalchardet/src/base/nsMBCSSM.cpp
- changeset 0
- 6474c204b198
equal
deleted
inserted
replaced
| -1:000000000000 | 0:0d4d02976929 |
|---|---|
| 1 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ | |
| 2 /* This Source Code Form is subject to the terms of the Mozilla Public | |
| 3 * License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ | |
| 5 #include "nsCodingStateMachine.h" | |
| 6 | |
| 7 /* | |
| 8 Modification from frank tang's original work: | |
| 9 . 0x00 is allowed as a legal character. Since some web pages contains this char in | |
| 10 text stream. | |
| 11 */ | |
| 12 | |
| 13 // BIG5 | |
| 14 | |
| 15 static const uint32_t BIG5_cls [ 256 / 8 ] = { | |
| 16 //PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07 | |
| 17 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 //allow 0x00 as legal value | |
| 18 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f | |
| 19 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 | |
| 20 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f | |
| 21 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 | |
| 22 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f | |
| 23 PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 | |
| 24 PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f | |
| 25 PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 | |
| 26 PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f | |
| 27 PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 | |
| 28 PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f | |
| 29 PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 | |
| 30 PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f | |
| 31 PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 | |
| 32 PCK4BITS(2,2,2,2,2,2,2,1), // 78 - 7f | |
| 33 PCK4BITS(4,4,4,4,4,4,4,4), // 80 - 87 | |
| 34 PCK4BITS(4,4,4,4,4,4,4,4), // 88 - 8f | |
| 35 PCK4BITS(4,4,4,4,4,4,4,4), // 90 - 97 | |
| 36 PCK4BITS(4,4,4,4,4,4,4,4), // 98 - 9f | |
| 37 PCK4BITS(4,3,3,3,3,3,3,3), // a0 - a7 | |
| 38 PCK4BITS(3,3,3,3,3,3,3,3), // a8 - af | |
| 39 PCK4BITS(3,3,3,3,3,3,3,3), // b0 - b7 | |
| 40 PCK4BITS(3,3,3,3,3,3,3,3), // b8 - bf | |
| 41 PCK4BITS(3,3,3,3,3,3,3,3), // c0 - c7 | |
| 42 PCK4BITS(3,3,3,3,3,3,3,3), // c8 - cf | |
| 43 PCK4BITS(3,3,3,3,3,3,3,3), // d0 - d7 | |
| 44 PCK4BITS(3,3,3,3,3,3,3,3), // d8 - df | |
| 45 PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7 | |
| 46 PCK4BITS(3,3,3,3,3,3,3,3), // e8 - ef | |
| 47 PCK4BITS(3,3,3,3,3,3,3,3), // f0 - f7 | |
| 48 PCK4BITS(3,3,3,3,3,3,3,0) // f8 - ff | |
| 49 }; | |
| 50 | |
| 51 | |
| 52 static const uint32_t BIG5_st [ 3] = { | |
| 53 PCK4BITS(eError,eStart,eStart, 3,eError,eError,eError,eError),//00-07 | |
| 54 PCK4BITS(eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError),//08-0f | |
| 55 PCK4BITS(eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart) //10-17 | |
| 56 }; | |
| 57 | |
| 58 static const uint32_t Big5CharLenTable[] = {0, 1, 1, 2, 0}; | |
| 59 | |
| 60 SMModel const Big5SMModel = { | |
| 61 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, BIG5_cls }, | |
| 62 5, | |
| 63 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, BIG5_st }, | |
| 64 CHAR_LEN_TABLE(Big5CharLenTable), | |
| 65 "Big5", | |
| 66 }; | |
| 67 | |
| 68 static const uint32_t EUCJP_cls [ 256 / 8 ] = { | |
| 69 //PCK4BITS(5,4,4,4,4,4,4,4), // 00 - 07 | |
| 70 PCK4BITS(4,4,4,4,4,4,4,4), // 00 - 07 | |
| 71 PCK4BITS(4,4,4,4,4,4,5,5), // 08 - 0f | |
| 72 PCK4BITS(4,4,4,4,4,4,4,4), // 10 - 17 | |
| 73 PCK4BITS(4,4,4,5,4,4,4,4), // 18 - 1f | |
| 74 PCK4BITS(4,4,4,4,4,4,4,4), // 20 - 27 | |
| 75 PCK4BITS(4,4,4,4,4,4,4,4), // 28 - 2f | |
| 76 PCK4BITS(4,4,4,4,4,4,4,4), // 30 - 37 | |
| 77 PCK4BITS(4,4,4,4,4,4,4,4), // 38 - 3f | |
| 78 PCK4BITS(4,4,4,4,4,4,4,4), // 40 - 47 | |
| 79 PCK4BITS(4,4,4,4,4,4,4,4), // 48 - 4f | |
| 80 PCK4BITS(4,4,4,4,4,4,4,4), // 50 - 57 | |
| 81 PCK4BITS(4,4,4,4,4,4,4,4), // 58 - 5f | |
| 82 PCK4BITS(4,4,4,4,4,4,4,4), // 60 - 67 | |
| 83 PCK4BITS(4,4,4,4,4,4,4,4), // 68 - 6f | |
| 84 PCK4BITS(4,4,4,4,4,4,4,4), // 70 - 77 | |
| 85 PCK4BITS(4,4,4,4,4,4,4,4), // 78 - 7f | |
| 86 PCK4BITS(5,5,5,5,5,5,5,5), // 80 - 87 | |
| 87 PCK4BITS(5,5,5,5,5,5,1,3), // 88 - 8f | |
| 88 PCK4BITS(5,5,5,5,5,5,5,5), // 90 - 97 | |
| 89 PCK4BITS(5,5,5,5,5,5,5,5), // 98 - 9f | |
| 90 PCK4BITS(5,2,2,2,2,2,2,2), // a0 - a7 | |
| 91 PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af | |
| 92 PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 | |
| 93 PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf | |
| 94 PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 | |
| 95 PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf | |
| 96 PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 | |
| 97 PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df | |
| 98 PCK4BITS(0,0,0,0,0,0,0,0), // e0 - e7 | |
| 99 PCK4BITS(0,0,0,0,0,0,0,0), // e8 - ef | |
| 100 PCK4BITS(0,0,0,0,0,0,0,0), // f0 - f7 | |
| 101 PCK4BITS(0,0,0,0,0,0,0,5) // f8 - ff | |
| 102 }; | |
| 103 | |
| 104 | |
| 105 static const uint32_t EUCJP_st [ 5] = { | |
| 106 PCK4BITS( 3, 4, 3, 5,eStart,eError,eError,eError),//00-07 | |
| 107 PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f | |
| 108 PCK4BITS(eItsMe,eItsMe,eStart,eError,eStart,eError,eError,eError),//10-17 | |
| 109 PCK4BITS(eError,eError,eStart,eError,eError,eError, 3,eError),//18-1f | |
| 110 PCK4BITS( 3,eError,eError,eError,eStart,eStart,eStart,eStart) //20-27 | |
| 111 }; | |
| 112 | |
| 113 static const uint32_t EUCJPCharLenTable[] = {2, 2, 2, 3, 1, 0}; | |
| 114 | |
| 115 const SMModel EUCJPSMModel = { | |
| 116 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCJP_cls }, | |
| 117 6, | |
| 118 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCJP_st }, | |
| 119 CHAR_LEN_TABLE(EUCJPCharLenTable), | |
| 120 "EUC-JP", | |
| 121 }; | |
| 122 | |
| 123 static const uint32_t EUCKR_cls [ 256 / 8 ] = { | |
| 124 //PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07 | |
| 125 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 | |
| 126 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f | |
| 127 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 | |
| 128 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f | |
| 129 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 | |
| 130 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f | |
| 131 PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 | |
| 132 PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f | |
| 133 PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47 | |
| 134 PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f | |
| 135 PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57 | |
| 136 PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f | |
| 137 PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67 | |
| 138 PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f | |
| 139 PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77 | |
| 140 PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f | |
| 141 PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87 | |
| 142 PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f | |
| 143 PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 | |
| 144 PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f | |
| 145 PCK4BITS(0,2,2,2,2,2,2,2), // a0 - a7 | |
| 146 PCK4BITS(2,2,2,2,2,3,3,3), // a8 - af | |
| 147 PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 | |
| 148 PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf | |
| 149 PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 | |
| 150 PCK4BITS(2,3,2,2,2,2,2,2), // c8 - cf | |
| 151 PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 | |
| 152 PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df | |
| 153 PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 | |
| 154 PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef | |
| 155 PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 | |
| 156 PCK4BITS(2,2,2,2,2,2,2,0) // f8 - ff | |
| 157 }; | |
| 158 | |
| 159 | |
| 160 static const uint32_t EUCKR_st [ 2] = { | |
| 161 PCK4BITS(eError,eStart, 3,eError,eError,eError,eError,eError),//00-07 | |
| 162 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart) //08-0f | |
| 163 }; | |
| 164 | |
| 165 static const uint32_t EUCKRCharLenTable[] = {0, 1, 2, 0}; | |
| 166 | |
| 167 const SMModel EUCKRSMModel = { | |
| 168 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCKR_cls }, | |
| 169 4, | |
| 170 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCKR_st }, | |
| 171 CHAR_LEN_TABLE(EUCKRCharLenTable), | |
| 172 "EUC-KR", | |
| 173 }; | |
| 174 | |
| 175 static const uint32_t EUCTW_cls [ 256 / 8 ] = { | |
| 176 //PCK4BITS(0,2,2,2,2,2,2,2), // 00 - 07 | |
| 177 PCK4BITS(2,2,2,2,2,2,2,2), // 00 - 07 | |
| 178 PCK4BITS(2,2,2,2,2,2,0,0), // 08 - 0f | |
| 179 PCK4BITS(2,2,2,2,2,2,2,2), // 10 - 17 | |
| 180 PCK4BITS(2,2,2,0,2,2,2,2), // 18 - 1f | |
| 181 PCK4BITS(2,2,2,2,2,2,2,2), // 20 - 27 | |
| 182 PCK4BITS(2,2,2,2,2,2,2,2), // 28 - 2f | |
| 183 PCK4BITS(2,2,2,2,2,2,2,2), // 30 - 37 | |
| 184 PCK4BITS(2,2,2,2,2,2,2,2), // 38 - 3f | |
| 185 PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 | |
| 186 PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f | |
| 187 PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 | |
| 188 PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f | |
| 189 PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 | |
| 190 PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f | |
| 191 PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 | |
| 192 PCK4BITS(2,2,2,2,2,2,2,2), // 78 - 7f | |
| 193 PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87 | |
| 194 PCK4BITS(0,0,0,0,0,0,6,0), // 88 - 8f | |
| 195 PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 | |
| 196 PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f | |
| 197 PCK4BITS(0,3,4,4,4,4,4,4), // a0 - a7 | |
| 198 PCK4BITS(5,5,1,1,1,1,1,1), // a8 - af | |
| 199 PCK4BITS(1,1,1,1,1,1,1,1), // b0 - b7 | |
| 200 PCK4BITS(1,1,1,1,1,1,1,1), // b8 - bf | |
| 201 PCK4BITS(1,1,3,1,3,3,3,3), // c0 - c7 | |
| 202 PCK4BITS(3,3,3,3,3,3,3,3), // c8 - cf | |
| 203 PCK4BITS(3,3,3,3,3,3,3,3), // d0 - d7 | |
| 204 PCK4BITS(3,3,3,3,3,3,3,3), // d8 - df | |
| 205 PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7 | |
| 206 PCK4BITS(3,3,3,3,3,3,3,3), // e8 - ef | |
| 207 PCK4BITS(3,3,3,3,3,3,3,3), // f0 - f7 | |
| 208 PCK4BITS(3,3,3,3,3,3,3,0) // f8 - ff | |
| 209 }; | |
| 210 | |
| 211 | |
| 212 static const uint32_t EUCTW_st [ 6] = { | |
| 213 PCK4BITS(eError,eError,eStart, 3, 3, 3, 4,eError),//00-07 | |
| 214 PCK4BITS(eError,eError,eError,eError,eError,eError,eItsMe,eItsMe),//08-0f | |
| 215 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eStart,eError),//10-17 | |
| 216 PCK4BITS(eStart,eStart,eStart,eError,eError,eError,eError,eError),//18-1f | |
| 217 PCK4BITS( 5,eError,eError,eError,eStart,eError,eStart,eStart),//20-27 | |
| 218 PCK4BITS(eStart,eError,eStart,eStart,eStart,eStart,eStart,eStart) //28-2f | |
| 219 }; | |
| 220 | |
| 221 static const uint32_t EUCTWCharLenTable[] = {0, 0, 1, 2, 2, 2, 3}; | |
| 222 | |
| 223 const SMModel EUCTWSMModel = { | |
| 224 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCTW_cls }, | |
| 225 7, | |
| 226 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, EUCTW_st }, | |
| 227 CHAR_LEN_TABLE(EUCTWCharLenTable), | |
| 228 "x-euc-tw", | |
| 229 }; | |
| 230 | |
| 231 /* obsolete GB2312 by gb18030 | |
| 232 static uint32_t GB2312_cls [ 256 / 8 ] = { | |
| 233 //PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07 | |
| 234 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 | |
| 235 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f | |
| 236 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 | |
| 237 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f | |
| 238 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 | |
| 239 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f | |
| 240 PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 | |
| 241 PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f | |
| 242 PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47 | |
| 243 PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f | |
| 244 PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57 | |
| 245 PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f | |
| 246 PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67 | |
| 247 PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f | |
| 248 PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77 | |
| 249 PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f | |
| 250 PCK4BITS(1,0,0,0,0,0,0,0), // 80 - 87 | |
| 251 PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f | |
| 252 PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 | |
| 253 PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f | |
| 254 PCK4BITS(0,2,2,2,2,2,2,2), // a0 - a7 | |
| 255 PCK4BITS(2,2,3,3,3,3,3,3), // a8 - af | |
| 256 PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 | |
| 257 PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf | |
| 258 PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 | |
| 259 PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf | |
| 260 PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 | |
| 261 PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df | |
| 262 PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 | |
| 263 PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef | |
| 264 PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 | |
| 265 PCK4BITS(2,2,2,2,2,2,2,0) // f8 - ff | |
| 266 }; | |
| 267 | |
| 268 | |
| 269 static uint32_t GB2312_st [ 2] = { | |
| 270 PCK4BITS(eError,eStart, 3,eError,eError,eError,eError,eError),//00-07 | |
| 271 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart) //08-0f | |
| 272 }; | |
| 273 | |
| 274 static const uint32_t GB2312CharLenTable[] = {0, 1, 2, 0}; | |
| 275 | |
| 276 SMModel GB2312SMModel = { | |
| 277 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB2312_cls }, | |
| 278 4, | |
| 279 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB2312_st }, | |
| 280 CHAR_LEN_TABLE(GB2312CharLenTable), | |
| 281 "GB2312", | |
| 282 }; | |
| 283 */ | |
| 284 | |
| 285 // the following state machine data was created by perl script in | |
| 286 // intl/chardet/tools. It should be the same as in PSM detector. | |
| 287 static const uint32_t GB18030_cls [ 256 / 8 ] = { | |
| 288 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 | |
| 289 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f | |
| 290 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 | |
| 291 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f | |
| 292 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 | |
| 293 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f | |
| 294 PCK4BITS(3,3,3,3,3,3,3,3), // 30 - 37 | |
| 295 PCK4BITS(3,3,1,1,1,1,1,1), // 38 - 3f | |
| 296 PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 | |
| 297 PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f | |
| 298 PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 | |
| 299 PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f | |
| 300 PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 | |
| 301 PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f | |
| 302 PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 | |
| 303 PCK4BITS(2,2,2,2,2,2,2,4), // 78 - 7f | |
| 304 PCK4BITS(5,6,6,6,6,6,6,6), // 80 - 87 | |
| 305 PCK4BITS(6,6,6,6,6,6,6,6), // 88 - 8f | |
| 306 PCK4BITS(6,6,6,6,6,6,6,6), // 90 - 97 | |
| 307 PCK4BITS(6,6,6,6,6,6,6,6), // 98 - 9f | |
| 308 PCK4BITS(6,6,6,6,6,6,6,6), // a0 - a7 | |
| 309 PCK4BITS(6,6,6,6,6,6,6,6), // a8 - af | |
| 310 PCK4BITS(6,6,6,6,6,6,6,6), // b0 - b7 | |
| 311 PCK4BITS(6,6,6,6,6,6,6,6), // b8 - bf | |
| 312 PCK4BITS(6,6,6,6,6,6,6,6), // c0 - c7 | |
| 313 PCK4BITS(6,6,6,6,6,6,6,6), // c8 - cf | |
| 314 PCK4BITS(6,6,6,6,6,6,6,6), // d0 - d7 | |
| 315 PCK4BITS(6,6,6,6,6,6,6,6), // d8 - df | |
| 316 PCK4BITS(6,6,6,6,6,6,6,6), // e0 - e7 | |
| 317 PCK4BITS(6,6,6,6,6,6,6,6), // e8 - ef | |
| 318 PCK4BITS(6,6,6,6,6,6,6,6), // f0 - f7 | |
| 319 PCK4BITS(6,6,6,6,6,6,6,0) // f8 - ff | |
| 320 }; | |
| 321 | |
| 322 | |
| 323 static const uint32_t GB18030_st [ 6] = { | |
| 324 PCK4BITS(eError,eStart,eStart,eStart,eStart,eStart, 3,eError),//00-07 | |
| 325 PCK4BITS(eError,eError,eError,eError,eError,eError,eItsMe,eItsMe),//08-0f | |
| 326 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart),//10-17 | |
| 327 PCK4BITS( 4,eError,eStart,eStart,eError,eError,eError,eError),//18-1f | |
| 328 PCK4BITS(eError,eError, 5,eError,eError,eError,eItsMe,eError),//20-27 | |
| 329 PCK4BITS(eError,eError,eStart,eStart,eStart,eStart,eStart,eStart) //28-2f | |
| 330 }; | |
| 331 | |
| 332 // To be accurate, the length of class 6 can be either 2 or 4. | |
| 333 // But it is not necessary to discriminate between the two since | |
| 334 // it is used for frequency analysis only, and we are validing | |
| 335 // each code range there as well. So it is safe to set it to be | |
| 336 // 2 here. | |
| 337 static const uint32_t GB18030CharLenTable[] = {0, 1, 1, 1, 1, 1, 2}; | |
| 338 | |
| 339 const SMModel GB18030SMModel = { | |
| 340 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB18030_cls }, | |
| 341 7, | |
| 342 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, GB18030_st }, | |
| 343 CHAR_LEN_TABLE(GB18030CharLenTable), | |
| 344 "GB18030", | |
| 345 }; | |
| 346 | |
| 347 // sjis | |
| 348 | |
| 349 static const uint32_t SJIS_cls [ 256 / 8 ] = { | |
| 350 //PCK4BITS(0,1,1,1,1,1,1,1), // 00 - 07 | |
| 351 PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 | |
| 352 PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f | |
| 353 PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 | |
| 354 PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f | |
| 355 PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 | |
| 356 PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f | |
| 357 PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 | |
| 358 PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f | |
| 359 PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 | |
| 360 PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f | |
| 361 PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 | |
| 362 PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f | |
| 363 PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 | |
| 364 PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f | |
| 365 PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 | |
| 366 PCK4BITS(2,2,2,2,2,2,2,1), // 78 - 7f | |
| 367 PCK4BITS(3,3,3,3,3,3,3,3), // 80 - 87 | |
| 368 PCK4BITS(3,3,3,3,3,3,3,3), // 88 - 8f | |
| 369 PCK4BITS(3,3,3,3,3,3,3,3), // 90 - 97 | |
| 370 PCK4BITS(3,3,3,3,3,3,3,3), // 98 - 9f | |
| 371 //0xa0 is illegal in sjis encoding, but some pages does | |
| 372 //contain such byte. We need to be more error forgiven. | |
| 373 PCK4BITS(2,2,2,2,2,2,2,2), // a0 - a7 | |
| 374 PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af | |
| 375 PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 | |
| 376 PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf | |
| 377 PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 | |
| 378 PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf | |
| 379 PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 | |
| 380 PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df | |
| 381 PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7 | |
| 382 PCK4BITS(3,3,3,3,3,4,4,4), // e8 - ef | |
| 383 PCK4BITS(4,4,4,4,4,4,4,4), // f0 - f7 | |
| 384 PCK4BITS(4,4,4,4,4,0,0,0) // f8 - ff | |
| 385 }; | |
| 386 | |
| 387 | |
| 388 static const uint32_t SJIS_st [ 3] = { | |
| 389 PCK4BITS(eError,eStart,eStart, 3,eError,eError,eError,eError),//00-07 | |
| 390 PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f | |
| 391 PCK4BITS(eItsMe,eItsMe,eError,eError,eStart,eStart,eStart,eStart) //10-17 | |
| 392 }; | |
| 393 | |
| 394 static const uint32_t SJISCharLenTable[] = {0, 1, 1, 2, 0, 0}; | |
| 395 | |
| 396 const SMModel SJISSMModel = { | |
| 397 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, SJIS_cls }, | |
| 398 6, | |
| 399 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, SJIS_st }, | |
| 400 CHAR_LEN_TABLE(SJISCharLenTable), | |
| 401 "Shift_JIS", | |
| 402 }; | |
| 403 | |
| 404 | |
| 405 static const uint32_t UTF8_cls [ 256 / 8 ] = { | |
| 406 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 00 - 07 | |
| 407 PCK4BITS( 1, 1, 1, 1, 1, 1, 0, 0), // 08 - 0f | |
| 408 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 10 - 17 | |
| 409 PCK4BITS( 1, 1, 1, 0, 1, 1, 1, 1), // 18 - 1f | |
| 410 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 20 - 27 | |
| 411 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 28 - 2f | |
| 412 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 30 - 37 | |
| 413 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 38 - 3f | |
| 414 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 40 - 47 | |
| 415 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 48 - 4f | |
| 416 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 50 - 57 | |
| 417 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 58 - 5f | |
| 418 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 60 - 67 | |
| 419 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 68 - 6f | |
| 420 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 70 - 77 | |
| 421 PCK4BITS( 1, 1, 1, 1, 1, 1, 1, 1), // 78 - 7f | |
| 422 PCK4BITS( 2, 2, 2, 2, 2, 2, 2, 2), // 80 - 87 | |
| 423 PCK4BITS( 2, 2, 2, 2, 2, 2, 2, 2), // 88 - 8f | |
| 424 PCK4BITS( 3, 3, 3, 3, 3, 3, 3, 3), // 90 - 97 | |
| 425 PCK4BITS( 3, 3, 3, 3, 3, 3, 3, 3), // 98 - 9f | |
| 426 PCK4BITS( 4, 4, 4, 4, 4, 4, 4, 4), // a0 - a7 | |
| 427 PCK4BITS( 4, 4, 4, 4, 4, 4, 4, 4), // a8 - af | |
| 428 PCK4BITS( 4, 4, 4, 4, 4, 4, 4, 4), // b0 - b7 | |
| 429 PCK4BITS( 4, 4, 4, 4, 4, 4, 4, 4), // b8 - bf | |
| 430 PCK4BITS( 0, 0, 5, 5, 5, 5, 5, 5), // c0 - c7 | |
| 431 PCK4BITS( 5, 5, 5, 5, 5, 5, 5, 5), // c8 - cf | |
| 432 PCK4BITS( 5, 5, 5, 5, 5, 5, 5, 5), // d0 - d7 | |
| 433 PCK4BITS( 5, 5, 5, 5, 5, 5, 5, 5), // d8 - df | |
| 434 PCK4BITS( 6, 7, 7, 7, 7, 7, 7, 7), // e0 - e7 | |
| 435 PCK4BITS( 7, 7, 7, 7, 7, 8, 7, 7), // e8 - ef | |
| 436 PCK4BITS( 9,10,10,10,11, 0, 0, 0), // f0 - f7 | |
| 437 PCK4BITS( 0, 0, 0, 0, 0, 0, 0, 0) // f8 - ff | |
| 438 }; | |
| 439 | |
| 440 | |
| 441 static const uint32_t UTF8_st [ 15] = { | |
| 442 PCK4BITS(eError,eStart,eError,eError,eError, 3, 4, 5), // 00 - 07 | |
| 443 PCK4BITS( 6, 7, 8, 9,eError,eError,eError,eError), // 08 - 0f | |
| 444 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError), // 10 - 17 | |
| 445 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe), // 18 - 1f | |
| 446 PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart), // 20 - 27 | |
| 447 PCK4BITS(eStart,eError,eError,eError,eError,eError,eError,eError), // 28 - 2f | |
| 448 PCK4BITS(eError,eError,eError,eError, 3,eError,eError,eError), // 30 - 37 | |
| 449 PCK4BITS(eError,eError,eError,eError,eError,eError, 3, 3), // 38 - 3f | |
| 450 PCK4BITS( 3,eError,eError,eError,eError,eError,eError,eError), // 40 - 47 | |
| 451 PCK4BITS(eError,eError, 3, 3,eError,eError,eError,eError), // 48 - 4f | |
| 452 PCK4BITS(eError,eError,eError,eError,eError,eError, 5, 5), // 50 - 57 | |
| 453 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError), // 58 - 5f | |
| 454 PCK4BITS(eError,eError, 5, 5, 5,eError,eError,eError), // 60 - 67 | |
| 455 PCK4BITS(eError,eError,eError,eError,eError,eError, 5,eError), // 68 - 6f | |
| 456 PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError) // 70 - 77 | |
| 457 }; | |
| 458 | |
| 459 static const uint32_t UTF8CharLenTable[] = {0, 1, 0, 0, 0, 2, 3, 3, 3, 4, 4, 4}; | |
| 460 | |
| 461 const SMModel UTF8SMModel = { | |
| 462 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, UTF8_cls }, | |
| 463 12, | |
| 464 {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, UTF8_st }, | |
| 465 CHAR_LEN_TABLE(UTF8CharLenTable), | |
| 466 "UTF-8", | |
| 467 }; |
