The Tor Browser / file comparison
comparison: intl/icu/source/tools/gennorm2/gennorm2.cpp
intl/icu/source/tools/gennorm2/gennorm2.cpp
- branch
- TOR_BUG_3246
- changeset 4
- fc2d59ddac77
equal
deleted
inserted
replaced
| -1:000000000000 | 0:35a788e39df5 |
|---|---|
| 1 /* | |
| 2 ******************************************************************************* | |
| 3 * | |
| 4 * Copyright (C) 2009-2012, International Business Machines | |
| 5 * Corporation and others. All Rights Reserved. | |
| 6 * | |
| 7 ******************************************************************************* | |
| 8 * file name: gennorm2.cpp | |
| 9 * encoding: US-ASCII | |
| 10 * tab size: 8 (not used) | |
| 11 * indentation:4 | |
| 12 * | |
| 13 * created on: 2009nov25 | |
| 14 * created by: Markus W. Scherer | |
| 15 * | |
| 16 * This program reads text files that define Unicode normalization, | |
| 17 * parses them, and builds a binary data file. | |
| 18 */ | |
| 19 | |
| 20 #include "unicode/utypes.h" | |
| 21 #include "n2builder.h" | |
| 22 | |
| 23 #include <stdio.h> | |
| 24 #include <stdlib.h> | |
| 25 #include <string.h> | |
| 26 #include "unicode/errorcode.h" | |
| 27 #include "unicode/localpointer.h" | |
| 28 #include "unicode/putil.h" | |
| 29 #include "unicode/uchar.h" | |
| 30 #include "unicode/unistr.h" | |
| 31 #include "charstr.h" | |
| 32 #include "normalizer2impl.h" | |
| 33 #include "toolutil.h" | |
| 34 #include "uoptions.h" | |
| 35 #include "uparse.h" | |
| 36 | |
| 37 #if UCONFIG_NO_NORMALIZATION | |
| 38 #include "unewdata.h" | |
| 39 #endif | |
| 40 | |
| 41 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) | |
| 42 | |
| 43 U_NAMESPACE_BEGIN | |
| 44 | |
| 45 UBool beVerbose=FALSE, haveCopyright=TRUE; | |
| 46 | |
| 47 U_DEFINE_LOCAL_OPEN_POINTER(LocalStdioFilePointer, FILE, fclose); | |
| 48 | |
| 49 #if !UCONFIG_NO_NORMALIZATION | |
| 50 void parseFile(FILE *f, Normalizer2DataBuilder &builder); | |
| 51 #endif | |
| 52 | |
| 53 /* -------------------------------------------------------------------------- */ | |
| 54 | |
| 55 enum { | |
| 56 HELP_H, | |
| 57 HELP_QUESTION_MARK, | |
| 58 VERBOSE, | |
| 59 COPYRIGHT, | |
| 60 SOURCEDIR, | |
| 61 OUTPUT_FILENAME, | |
| 62 UNICODE_VERSION, | |
| 63 OPT_FAST | |
| 64 }; | |
| 65 | |
| 66 static UOption options[]={ | |
| 67 UOPTION_HELP_H, | |
| 68 UOPTION_HELP_QUESTION_MARK, | |
| 69 UOPTION_VERBOSE, | |
| 70 UOPTION_COPYRIGHT, | |
| 71 UOPTION_SOURCEDIR, | |
| 72 UOPTION_DEF("output", 'o', UOPT_REQUIRES_ARG), | |
| 73 UOPTION_DEF("unicode", 'u', UOPT_REQUIRES_ARG), | |
| 74 UOPTION_DEF("fast", '\1', UOPT_NO_ARG) | |
| 75 }; | |
| 76 | |
| 77 extern "C" int | |
| 78 main(int argc, char* argv[]) { | |
| 79 U_MAIN_INIT_ARGS(argc, argv); | |
| 80 | |
| 81 /* preset then read command line options */ | |
| 82 options[SOURCEDIR].value=""; | |
| 83 argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[HELP_H]), options); | |
| 84 | |
| 85 /* error handling, printing usage message */ | |
| 86 if(argc<0) { | |
| 87 fprintf(stderr, | |
| 88 "error in command line argument \"%s\"\n", | |
| 89 argv[-argc]); | |
| 90 } | |
| 91 if(!options[OUTPUT_FILENAME].doesOccur) { | |
| 92 argc=-1; | |
| 93 } | |
| 94 if( argc<2 || | |
| 95 options[HELP_H].doesOccur || options[HELP_QUESTION_MARK].doesOccur | |
| 96 ) { | |
| 97 /* | |
| 98 * Broken into chunks because the C89 standard says the minimum | |
| 99 * required supported string length is 509 bytes. | |
| 100 */ | |
| 101 fprintf(stderr, | |
| 102 "Usage: %s [-options] infiles+ -o outputfilename\n" | |
| 103 "\n" | |
| 104 "Reads the infiles with normalization data and\n" | |
| 105 "creates a binary file (outputfilename) with the data.\n" | |
| 106 "\n", | |
| 107 argv[0]); | |
| 108 fprintf(stderr, | |
| 109 "Options:\n" | |
| 110 "\t-h or -? or --help this usage text\n" | |
| 111 "\t-v or --verbose verbose output\n" | |
| 112 "\t-c or --copyright include a copyright notice\n" | |
| 113 "\t-u or --unicode Unicode version, followed by the version like 5.2.0\n"); | |
| 114 fprintf(stderr, | |
| 115 "\t-s or --sourcedir source directory, followed by the path\n" | |
| 116 "\t-o or --output output filename\n"); | |
| 117 fprintf(stderr, | |
| 118 "\t --fast optimize the .nrm file for fast normalization,\n" | |
| 119 "\t which might increase its size (Writes fully decomposed\n" | |
| 120 "\t regular mappings instead of delta mappings.\n" | |
| 121 "\t You should measure the runtime speed to make sure that\n" | |
| 122 "\t this is a good trade-off.)\n"); | |
| 123 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; | |
| 124 } | |
| 125 | |
| 126 beVerbose=options[VERBOSE].doesOccur; | |
| 127 haveCopyright=options[COPYRIGHT].doesOccur; | |
| 128 | |
| 129 IcuToolErrorCode errorCode("gennorm2/main()"); | |
| 130 | |
| 131 #if UCONFIG_NO_NORMALIZATION | |
| 132 | |
| 133 fprintf(stderr, | |
| 134 "gennorm2 writes a dummy binary data file " | |
| 135 "because UCONFIG_NO_NORMALIZATION is set, \n" | |
| 136 "see icu/source/common/unicode/uconfig.h\n"); | |
| 137 udata_createDummy(NULL, NULL, options[OUTPUT_FILENAME].value, errorCode); | |
| 138 // Should not return an error since this is the expected behaviour if UCONFIG_NO_NORMALIZATION is on. | |
| 139 // return U_UNSUPPORTED_ERROR; | |
| 140 return 0; | |
| 141 | |
| 142 #else | |
| 143 | |
| 144 LocalPointer<Normalizer2DataBuilder> builder(new Normalizer2DataBuilder(errorCode)); | |
| 145 errorCode.assertSuccess(); | |
| 146 | |
| 147 if(options[UNICODE_VERSION].doesOccur) { | |
| 148 builder->setUnicodeVersion(options[UNICODE_VERSION].value); | |
| 149 } | |
| 150 | |
| 151 if(options[OPT_FAST].doesOccur) { | |
| 152 builder->setOptimization(Normalizer2DataBuilder::OPTIMIZE_FAST); | |
| 153 } | |
| 154 | |
| 155 // prepare the filename beginning with the source dir | |
| 156 CharString filename(options[SOURCEDIR].value, errorCode); | |
| 157 int32_t pathLength=filename.length(); | |
| 158 if( pathLength>0 && | |
| 159 filename[pathLength-1]!=U_FILE_SEP_CHAR && | |
| 160 filename[pathLength-1]!=U_FILE_ALT_SEP_CHAR | |
| 161 ) { | |
| 162 filename.append(U_FILE_SEP_CHAR, errorCode); | |
| 163 pathLength=filename.length(); | |
| 164 } | |
| 165 | |
| 166 for(int i=1; i<argc; ++i) { | |
| 167 printf("gennorm2: processing %s\n", argv[i]); | |
| 168 filename.append(argv[i], errorCode); | |
| 169 LocalStdioFilePointer f(fopen(filename.data(), "r")); | |
| 170 if(f==NULL) { | |
| 171 fprintf(stderr, "gennorm2 error: unable to open %s\n", filename.data()); | |
| 172 exit(U_FILE_ACCESS_ERROR); | |
| 173 } | |
| 174 builder->setOverrideHandling(Normalizer2DataBuilder::OVERRIDE_PREVIOUS); | |
| 175 parseFile(f.getAlias(), *builder); | |
| 176 filename.truncate(pathLength); | |
| 177 } | |
| 178 | |
| 179 builder->writeBinaryFile(options[OUTPUT_FILENAME].value); | |
| 180 | |
| 181 return errorCode.get(); | |
| 182 | |
| 183 #endif | |
| 184 } | |
| 185 | |
| 186 #if !UCONFIG_NO_NORMALIZATION | |
| 187 | |
| 188 void parseFile(FILE *f, Normalizer2DataBuilder &builder) { | |
| 189 IcuToolErrorCode errorCode("gennorm2/parseFile()"); | |
| 190 char line[300]; | |
| 191 uint32_t startCP, endCP; | |
| 192 while(NULL!=fgets(line, (int)sizeof(line), f)) { | |
| 193 char *comment=(char *)strchr(line, '#'); | |
| 194 if(comment!=NULL) { | |
| 195 *comment=0; | |
| 196 } | |
| 197 u_rtrim(line); | |
| 198 if(line[0]==0) { | |
| 199 continue; // skip empty and comment-only lines | |
| 200 } | |
| 201 if(line[0]=='*') { | |
| 202 const char *s=u_skipWhitespace(line+1); | |
| 203 if(0==strncmp(s, "Unicode", 7)) { | |
| 204 s=u_skipWhitespace(s+7); | |
| 205 builder.setUnicodeVersion(s); | |
| 206 } | |
| 207 continue; // reserved syntax | |
| 208 } | |
| 209 const char *delimiter; | |
| 210 int32_t rangeLength= | |
| 211 u_parseCodePointRangeAnyTerminator(line, &startCP, &endCP, &delimiter, errorCode); | |
| 212 if(errorCode.isFailure()) { | |
| 213 fprintf(stderr, "gennorm2 error: parsing code point range from %s\n", line); | |
| 214 exit(errorCode.reset()); | |
| 215 } | |
| 216 delimiter=u_skipWhitespace(delimiter); | |
| 217 if(*delimiter==':') { | |
| 218 const char *s=u_skipWhitespace(delimiter+1); | |
| 219 char *end; | |
| 220 unsigned long value=strtoul(s, &end, 10); | |
| 221 if(end<=s || *u_skipWhitespace(end)!=0 || value>=0xff) { | |
| 222 fprintf(stderr, "gennorm2 error: parsing ccc from %s\n", line); | |
| 223 exit(U_PARSE_ERROR); | |
| 224 } | |
| 225 for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) { | |
| 226 builder.setCC(c, (uint8_t)value); | |
| 227 } | |
| 228 continue; | |
| 229 } | |
| 230 if(*delimiter=='-') { | |
| 231 if(*u_skipWhitespace(delimiter+1)!=0) { | |
| 232 fprintf(stderr, "gennorm2 error: parsing remove-mapping %s\n", line); | |
| 233 exit(U_PARSE_ERROR); | |
| 234 } | |
| 235 for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) { | |
| 236 builder.removeMapping(c); | |
| 237 } | |
| 238 continue; | |
| 239 } | |
| 240 if(*delimiter=='=' || *delimiter=='>') { | |
| 241 UChar uchars[Normalizer2Impl::MAPPING_LENGTH_MASK]; | |
| 242 int32_t length=u_parseString(delimiter+1, uchars, LENGTHOF(uchars), NULL, errorCode); | |
| 243 if(errorCode.isFailure()) { | |
| 244 fprintf(stderr, "gennorm2 error: parsing mapping string from %s\n", line); | |
| 245 exit(errorCode.reset()); | |
| 246 } | |
| 247 UnicodeString mapping(FALSE, uchars, length); | |
| 248 if(*delimiter=='=') { | |
| 249 if(rangeLength!=1) { | |
| 250 fprintf(stderr, | |
| 251 "gennorm2 error: round-trip mapping for more than 1 code point on %s\n", | |
| 252 line); | |
| 253 exit(U_PARSE_ERROR); | |
| 254 } | |
| 255 builder.setRoundTripMapping((UChar32)startCP, mapping); | |
| 256 } else { | |
| 257 for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) { | |
| 258 builder.setOneWayMapping(c, mapping); | |
| 259 } | |
| 260 } | |
| 261 continue; | |
| 262 } | |
| 263 fprintf(stderr, "gennorm2 error: unrecognized data line %s\n", line); | |
| 264 exit(U_PARSE_ERROR); | |
| 265 } | |
| 266 } | |
| 267 | |
| 268 #endif // !UCONFIG_NO_NORMALIZATION | |
| 269 | |
| 270 U_NAMESPACE_END | |
| 271 | |
| 272 /* | |
| 273 * Hey, Emacs, please set the following: | |
| 274 * | |
| 275 * Local Variables: | |
| 276 * indent-tabs-mode: nil | |
| 277 * End: | |
| 278 * | |
| 279 */ |
