The Tor Browser / file comparison

comparison: intl/icu/source/tools/makeconv/makeconv.c

intl/icu/source/tools/makeconv/makeconv.c

branch
TOR_BUG_9701
changeset 15
b8a032363ba2
equal deleted inserted replaced
-1:000000000000 0:3081705bed20
1 /*
2 ********************************************************************************
3 *
4 * Copyright (C) 1998-2012, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 ********************************************************************************
8 *
9 *
10 * makeconv.c:
11 * tool creating a binary (compressed) representation of the conversion mapping
12 * table (IBM NLTC ucmap format).
13 *
14 * 05/04/2000 helena Added fallback mapping into the picture...
15 * 06/29/2000 helena Major rewrite of the callback APIs.
16 */
17
18 #include <stdio.h>
19 #include "unicode/putil.h"
20 #include "unicode/ucnv_err.h"
21 #include "ucnv_bld.h"
22 #include "ucnv_imp.h"
23 #include "ucnv_cnv.h"
24 #include "cstring.h"
25 #include "cmemory.h"
26 #include "uinvchar.h"
27 #include "filestrm.h"
28 #include "toolutil.h"
29 #include "uoptions.h"
30 #include "unicode/udata.h"
31 #include "unewdata.h"
32 #include "uparse.h"
33 #include "ucm.h"
34 #include "makeconv.h"
35 #include "genmbcs.h"
36
37 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
38
39 #define DEBUG 0
40
41 typedef struct ConvData {
42 UCMFile *ucm;
43 NewConverter *cnvData, *extData;
44 UConverterSharedData sharedData;
45 UConverterStaticData staticData;
46 } ConvData;
47
48 static void
49 initConvData(ConvData *data) {
50 uprv_memset(data, 0, sizeof(ConvData));
51 data->sharedData.structSize=sizeof(UConverterSharedData);
52 data->staticData.structSize=sizeof(UConverterStaticData);
53 data->sharedData.staticData=&data->staticData;
54 }
55
56 static void
57 cleanupConvData(ConvData *data) {
58 if(data!=NULL) {
59 if(data->cnvData!=NULL) {
60 data->cnvData->close(data->cnvData);
61 data->cnvData=NULL;
62 }
63 if(data->extData!=NULL) {
64 data->extData->close(data->extData);
65 data->extData=NULL;
66 }
67 ucm_close(data->ucm);
68 data->ucm=NULL;
69 }
70 }
71
72 /*
73 * from ucnvstat.c - static prototypes of data-based converters
74 */
75 extern const UConverterStaticData * ucnv_converterStaticData[UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES];
76
77 /*
78 * Global - verbosity
79 */
80 UBool VERBOSE = FALSE;
81 UBool SMALL = FALSE;
82 UBool IGNORE_SISO_CHECK = FALSE;
83
84 static void
85 createConverter(ConvData *data, const char* converterName, UErrorCode *pErrorCode);
86
87 /*
88 * Set up the UNewData and write the converter..
89 */
90 static void
91 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status);
92
93 UBool haveCopyright=TRUE;
94
95 static UDataInfo dataInfo={
96 sizeof(UDataInfo),
97 0,
98
99 U_IS_BIG_ENDIAN,
100 U_CHARSET_FAMILY,
101 sizeof(UChar),
102 0,
103
104 {0x63, 0x6e, 0x76, 0x74}, /* dataFormat="cnvt" */
105 {6, 2, 0, 0}, /* formatVersion */
106 {0, 0, 0, 0} /* dataVersion (calculated at runtime) */
107 };
108
109 static void
110 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status)
111 {
112 UNewDataMemory *mem = NULL;
113 uint32_t sz2;
114 uint32_t size = 0;
115 int32_t tableType;
116
117 if(U_FAILURE(*status))
118 {
119 return;
120 }
121
122 tableType=TABLE_NONE;
123 if(data->cnvData!=NULL) {
124 tableType|=TABLE_BASE;
125 }
126 if(data->extData!=NULL) {
127 tableType|=TABLE_EXT;
128 }
129
130 mem = udata_create(cnvDir, "cnv", cnvName, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : NULL, status);
131
132 if(U_FAILURE(*status))
133 {
134 fprintf(stderr, "Couldn't create the udata %s.%s: %s\n",
135 cnvName,
136 "cnv",
137 u_errorName(*status));
138 return;
139 }
140
141 if(VERBOSE)
142 {
143 printf("- Opened udata %s.%s\n", cnvName, "cnv");
144 }
145
146
147 /* all read only, clean, platform independent data. Mmmm. :) */
148 udata_writeBlock(mem, &data->staticData, sizeof(UConverterStaticData));
149 size += sizeof(UConverterStaticData); /* Is 4-aligned - by size */
150 /* Now, write the table */
151 if(tableType&TABLE_BASE) {
152 size += data->cnvData->write(data->cnvData, &data->staticData, mem, tableType);
153 }
154 if(tableType&TABLE_EXT) {
155 size += data->extData->write(data->extData, &data->staticData, mem, tableType);
156 }
157
158 sz2 = udata_finish(mem, status);
159 if(size != sz2)
160 {
161 fprintf(stderr, "error: wrote %u bytes to the .cnv file but counted %u bytes\n", (int)sz2, (int)size);
162 *status=U_INTERNAL_PROGRAM_ERROR;
163 }
164 if(VERBOSE)
165 {
166 printf("- Wrote %u bytes to the udata.\n", (int)sz2);
167 }
168 }
169
170 enum {
171 OPT_HELP_H,
172 OPT_HELP_QUESTION_MARK,
173 OPT_COPYRIGHT,
174 OPT_VERSION,
175 OPT_DESTDIR,
176 OPT_VERBOSE,
177 OPT_SMALL,
178 OPT_IGNORE_SISO_CHECK,
179 OPT_COUNT
180 };
181
182 static UOption options[]={
183 UOPTION_HELP_H,
184 UOPTION_HELP_QUESTION_MARK,
185 UOPTION_COPYRIGHT,
186 UOPTION_VERSION,
187 UOPTION_DESTDIR,
188 UOPTION_VERBOSE,
189 { "small", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 },
190 { "ignore-siso-check", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 }
191 };
192
193 int main(int argc, char* argv[])
194 {
195 ConvData data;
196 UErrorCode err = U_ZERO_ERROR, localError;
197 char outFileName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
198 const char* destdir, *arg;
199 size_t destdirlen;
200 char* dot = NULL, *outBasename;
201 char cnvName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
202 char cnvNameWithPkg[UCNV_MAX_FULL_FILE_NAME_LENGTH];
203 UVersionInfo icuVersion;
204 UBool printFilename;
205
206 err = U_ZERO_ERROR;
207
208 U_MAIN_INIT_ARGS(argc, argv);
209
210 /* Set up the ICU version number */
211 u_getVersion(icuVersion);
212 uprv_memcpy(&dataInfo.dataVersion, &icuVersion, sizeof(UVersionInfo));
213
214 /* preset then read command line options */
215 options[OPT_DESTDIR].value=u_getDataDirectory();
216 argc=u_parseArgs(argc, argv, LENGTHOF(options), options);
217
218 /* error handling, printing usage message */
219 if(argc<0) {
220 fprintf(stderr,
221 "error in command line argument \"%s\"\n",
222 argv[-argc]);
223 } else if(argc<2) {
224 argc=-1;
225 }
226 if(argc<0 || options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur) {
227 FILE *stdfile=argc<0 ? stderr : stdout;
228 fprintf(stdfile,
229 "usage: %s [-options] files...\n"
230 "\tread .ucm codepage mapping files and write .cnv files\n"
231 "options:\n"
232 "\t-h or -? or --help this usage text\n"
233 "\t-V or --version show a version message\n"
234 "\t-c or --copyright include a copyright notice\n"
235 "\t-d or --destdir destination directory, followed by the path\n"
236 "\t-v or --verbose Turn on verbose output\n",
237 argv[0]);
238 fprintf(stdfile,
239 "\t --small Generate smaller .cnv files. They will be\n"
240 "\t significantly smaller but may not be compatible with\n"
241 "\t older versions of ICU and will require heap memory\n"
242 "\t allocation when loaded.\n"
243 "\t --ignore-siso-check Use SI/SO other than 0xf/0xe.\n");
244 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
245 }
246
247 if(options[OPT_VERSION].doesOccur) {
248 printf("makeconv version %u.%u, ICU tool to read .ucm codepage mapping files and write .cnv files\n",
249 dataInfo.formatVersion[0], dataInfo.formatVersion[1]);
250 printf("%s\n", U_COPYRIGHT_STRING);
251 exit(0);
252 }
253
254 /* get the options values */
255 haveCopyright = options[OPT_COPYRIGHT].doesOccur;
256 destdir = options[OPT_DESTDIR].value;
257 VERBOSE = options[OPT_VERBOSE].doesOccur;
258 SMALL = options[OPT_SMALL].doesOccur;
259
260 if (options[OPT_IGNORE_SISO_CHECK].doesOccur) {
261 IGNORE_SISO_CHECK = TRUE;
262 }
263
264 if (destdir != NULL && *destdir != 0) {
265 uprv_strcpy(outFileName, destdir);
266 destdirlen = uprv_strlen(destdir);
267 outBasename = outFileName + destdirlen;
268 if (*(outBasename - 1) != U_FILE_SEP_CHAR) {
269 *outBasename++ = U_FILE_SEP_CHAR;
270 ++destdirlen;
271 }
272 } else {
273 destdirlen = 0;
274 outBasename = outFileName;
275 }
276
277 #if DEBUG
278 {
279 int i;
280 printf("makeconv: processing %d files...\n", argc - 1);
281 for(i=1; i<argc; ++i) {
282 printf("%s ", argv[i]);
283 }
284 printf("\n");
285 fflush(stdout);
286 }
287 #endif
288
289 err = U_ZERO_ERROR;
290 printFilename = (UBool) (argc > 2 || VERBOSE);
291 for (++argv; --argc; ++argv)
292 {
293 arg = getLongPathname(*argv);
294
295 /* Check for potential buffer overflow */
296 if(strlen(arg) >= UCNV_MAX_FULL_FILE_NAME_LENGTH)
297 {
298 fprintf(stderr, "%s\n", u_errorName(U_BUFFER_OVERFLOW_ERROR));
299 return U_BUFFER_OVERFLOW_ERROR;
300 }
301
302 /*produces the right destination path for display*/
303 if (destdirlen != 0)
304 {
305 const char *basename;
306
307 /* find the last file sepator */
308 basename = findBasename(arg);
309 uprv_strcpy(outBasename, basename);
310 }
311 else
312 {
313 uprv_strcpy(outFileName, arg);
314 }
315
316 /*removes the extension if any is found*/
317 dot = uprv_strrchr(outBasename, '.');
318 if (dot)
319 {
320 *dot = '\0';
321 }
322
323 /* the basename without extension is the converter name */
324 uprv_strcpy(cnvName, outBasename);
325
326 /*Adds the target extension*/
327 uprv_strcat(outBasename, CONVERTER_FILE_EXTENSION);
328
329 #if DEBUG
330 printf("makeconv: processing %s ...\n", arg);
331 fflush(stdout);
332 #endif
333 localError = U_ZERO_ERROR;
334 initConvData(&data);
335 createConverter(&data, arg, &localError);
336
337 if (U_FAILURE(localError))
338 {
339 /* if an error is found, print out an error msg and keep going */
340 fprintf(stderr, "Error creating converter for \"%s\" file for \"%s\" (%s)\n", outFileName, arg,
341 u_errorName(localError));
342 if(U_SUCCESS(err)) {
343 err = localError;
344 }
345 }
346 else
347 {
348 /* Insure the static data name matches the file name */
349 /* Changed to ignore directory and only compare base name
350 LDH 1/2/08*/
351 char *p;
352 p = strrchr(cnvName, U_FILE_SEP_CHAR); /* Find last file separator */
353
354 if(p == NULL) /* OK, try alternate */
355 {
356 p = strrchr(cnvName, U_FILE_ALT_SEP_CHAR);
357 if(p == NULL)
358 {
359 p=cnvName; /* If no separators, no problem */
360 }
361 }
362 else
363 {
364 p++; /* If found separtor, don't include it in compare */
365 }
366 if(uprv_stricmp(p,data.staticData.name))
367 {
368 fprintf(stderr, "Warning: %s%s claims to be '%s'\n",
369 cnvName, CONVERTER_FILE_EXTENSION,
370 data.staticData.name);
371 }
372
373 uprv_strcpy((char*)data.staticData.name, cnvName);
374
375 if(!uprv_isInvariantString((char*)data.staticData.name, -1)) {
376 fprintf(stderr,
377 "Error: A converter name must contain only invariant characters.\n"
378 "%s is not a valid converter name.\n",
379 data.staticData.name);
380 if(U_SUCCESS(err)) {
381 err = U_INVALID_TABLE_FORMAT;
382 }
383 }
384
385 uprv_strcpy(cnvNameWithPkg, cnvName);
386
387 localError = U_ZERO_ERROR;
388 writeConverterData(&data, cnvNameWithPkg, destdir, &localError);
389
390 if(U_FAILURE(localError))
391 {
392 /* if an error is found, print out an error msg and keep going*/
393 fprintf(stderr, "Error writing \"%s\" file for \"%s\" (%s)\n", outFileName, arg,
394 u_errorName(localError));
395 if(U_SUCCESS(err)) {
396 err = localError;
397 }
398 }
399 else if (printFilename)
400 {
401 puts(outBasename);
402 }
403 }
404 fflush(stdout);
405 fflush(stderr);
406
407 cleanupConvData(&data);
408 }
409
410 return err;
411 }
412
413 static void
414 getPlatformAndCCSIDFromName(const char *name, int8_t *pPlatform, int32_t *pCCSID) {
415 if( (name[0]=='i' || name[0]=='I') &&
416 (name[1]=='b' || name[1]=='B') &&
417 (name[2]=='m' || name[2]=='M')
418 ) {
419 name+=3;
420 if(*name=='-') {
421 ++name;
422 }
423 *pPlatform=UCNV_IBM;
424 *pCCSID=(int32_t)uprv_strtoul(name, NULL, 10);
425 } else {
426 *pPlatform=UCNV_UNKNOWN;
427 *pCCSID=0;
428 }
429 }
430
431 static void
432 readHeader(ConvData *data,
433 FileStream* convFile,
434 const char* converterName,
435 UErrorCode *pErrorCode) {
436 char line[1024];
437 char *s, *key, *value;
438 const UConverterStaticData *prototype;
439 UConverterStaticData *staticData;
440
441 if(U_FAILURE(*pErrorCode)) {
442 return;
443 }
444
445 staticData=&data->staticData;
446 staticData->platform=UCNV_IBM;
447 staticData->subCharLen=0;
448
449 while(T_FileStream_readLine(convFile, line, sizeof(line))) {
450 /* basic parsing and handling of state-related items */
451 if(ucm_parseHeaderLine(data->ucm, line, &key, &value)) {
452 continue;
453 }
454
455 /* stop at the beginning of the mapping section */
456 if(uprv_strcmp(line, "CHARMAP")==0) {
457 break;
458 }
459
460 /* collect the information from the header field, ignore unknown keys */
461 if(uprv_strcmp(key, "code_set_name")==0) {
462 if(*value!=0) {
463 uprv_strcpy((char *)staticData->name, value);
464 getPlatformAndCCSIDFromName(value, &staticData->platform, &staticData->codepage);
465 }
466 } else if(uprv_strcmp(key, "subchar")==0) {
467 uint8_t bytes[UCNV_EXT_MAX_BYTES];
468 int8_t length;
469
470 s=value;
471 length=ucm_parseBytes(bytes, line, (const char **)&s);
472 if(1<=length && length<=4 && *s==0) {
473 staticData->subCharLen=length;
474 uprv_memcpy(staticData->subChar, bytes, length);
475 } else {
476 fprintf(stderr, "error: illegal <subchar> %s\n", value);
477 *pErrorCode=U_INVALID_TABLE_FORMAT;
478 return;
479 }
480 } else if(uprv_strcmp(key, "subchar1")==0) {
481 uint8_t bytes[UCNV_EXT_MAX_BYTES];
482
483 s=value;
484 if(1==ucm_parseBytes(bytes, line, (const char **)&s) && *s==0) {
485 staticData->subChar1=bytes[0];
486 } else {
487 fprintf(stderr, "error: illegal <subchar1> %s\n", value);
488 *pErrorCode=U_INVALID_TABLE_FORMAT;
489 return;
490 }
491 }
492 }
493
494 /* copy values from the UCMFile to the static data */
495 staticData->maxBytesPerChar=(int8_t)data->ucm->states.maxCharLength;
496 staticData->minBytesPerChar=(int8_t)data->ucm->states.minCharLength;
497 staticData->conversionType=data->ucm->states.conversionType;
498
499 if(staticData->conversionType==UCNV_UNSUPPORTED_CONVERTER) {
500 fprintf(stderr, "ucm error: missing conversion type (<uconv_class>)\n");
501 *pErrorCode=U_INVALID_TABLE_FORMAT;
502 return;
503 }
504
505 /*
506 * Now that we know the type, copy any 'default' values from the table.
507 * We need not check the type any further because the parser only
508 * recognizes what we have prototypes for.
509 *
510 * For delta (extension-only) tables, copy values from the base file
511 * instead, see createConverter().
512 */
513 if(data->ucm->baseName[0]==0) {
514 prototype=ucnv_converterStaticData[staticData->conversionType];
515 if(prototype!=NULL) {
516 if(staticData->name[0]==0) {
517 uprv_strcpy((char *)staticData->name, prototype->name);
518 }
519
520 if(staticData->codepage==0) {
521 staticData->codepage=prototype->codepage;
522 }
523
524 if(staticData->platform==0) {
525 staticData->platform=prototype->platform;
526 }
527
528 if(staticData->minBytesPerChar==0) {
529 staticData->minBytesPerChar=prototype->minBytesPerChar;
530 }
531
532 if(staticData->maxBytesPerChar==0) {
533 staticData->maxBytesPerChar=prototype->maxBytesPerChar;
534 }
535
536 if(staticData->subCharLen==0) {
537 staticData->subCharLen=prototype->subCharLen;
538 if(prototype->subCharLen>0) {
539 uprv_memcpy(staticData->subChar, prototype->subChar, prototype->subCharLen);
540 }
541 }
542 }
543 }
544
545 if(data->ucm->states.outputType<0) {
546 data->ucm->states.outputType=(int8_t)data->ucm->states.maxCharLength-1;
547 }
548
549 if( staticData->subChar1!=0 &&
550 (staticData->minBytesPerChar>1 ||
551 (staticData->conversionType!=UCNV_MBCS &&
552 staticData->conversionType!=UCNV_EBCDIC_STATEFUL))
553 ) {
554 fprintf(stderr, "error: <subchar1> defined for a type other than MBCS or EBCDIC_STATEFUL\n");
555 *pErrorCode=U_INVALID_TABLE_FORMAT;
556 }
557 }
558
559 /* return TRUE if a base table was read, FALSE for an extension table */
560 static UBool
561 readFile(ConvData *data, const char* converterName,
562 UErrorCode *pErrorCode) {
563 char line[1024];
564 char *end;
565 FileStream *convFile;
566
567 UCMStates *baseStates;
568 UBool dataIsBase;
569
570 if(U_FAILURE(*pErrorCode)) {
571 return FALSE;
572 }
573
574 data->ucm=ucm_open();
575
576 convFile=T_FileStream_open(converterName, "r");
577 if(convFile==NULL) {
578 *pErrorCode=U_FILE_ACCESS_ERROR;
579 return FALSE;
580 }
581
582 readHeader(data, convFile, converterName, pErrorCode);
583 if(U_FAILURE(*pErrorCode)) {
584 return FALSE;
585 }
586
587 if(data->ucm->baseName[0]==0) {
588 dataIsBase=TRUE;
589 baseStates=&data->ucm->states;
590 ucm_processStates(baseStates, IGNORE_SISO_CHECK);
591 } else {
592 dataIsBase=FALSE;
593 baseStates=NULL;
594 }
595
596 /* read the base table */
597 ucm_readTable(data->ucm, convFile, dataIsBase, baseStates, pErrorCode);
598 if(U_FAILURE(*pErrorCode)) {
599 return FALSE;
600 }
601
602 /* read an extension table if there is one */
603 while(T_FileStream_readLine(convFile, line, sizeof(line))) {
604 end=uprv_strchr(line, 0);
605 while(line<end &&
606 (*(end-1)=='\n' || *(end-1)=='\r' || *(end-1)==' ' || *(end-1)=='\t')) {
607 --end;
608 }
609 *end=0;
610
611 if(line[0]=='#' || u_skipWhitespace(line)==end) {
612 continue; /* ignore empty and comment lines */
613 }
614
615 if(0==uprv_strcmp(line, "CHARMAP")) {
616 /* read the extension table */
617 ucm_readTable(data->ucm, convFile, FALSE, baseStates, pErrorCode);
618 } else {
619 fprintf(stderr, "unexpected text after the base mapping table\n");
620 }
621 break;
622 }
623
624 T_FileStream_close(convFile);
625
626 if(data->ucm->base->flagsType==UCM_FLAGS_MIXED || data->ucm->ext->flagsType==UCM_FLAGS_MIXED) {
627 fprintf(stderr, "error: some entries have the mapping precision (with '|'), some do not\n");
628 *pErrorCode=U_INVALID_TABLE_FORMAT;
629 }
630
631 return dataIsBase;
632 }
633
634 static void
635 createConverter(ConvData *data, const char *converterName, UErrorCode *pErrorCode) {
636 ConvData baseData;
637 UBool dataIsBase;
638
639 UConverterStaticData *staticData;
640 UCMStates *states, *baseStates;
641
642 if(U_FAILURE(*pErrorCode)) {
643 return;
644 }
645
646 initConvData(data);
647
648 dataIsBase=readFile(data, converterName, pErrorCode);
649 if(U_FAILURE(*pErrorCode)) {
650 return;
651 }
652
653 staticData=&data->staticData;
654 states=&data->ucm->states;
655
656 if(dataIsBase) {
657 /*
658 * Build a normal .cnv file with a base table
659 * and an optional extension table.
660 */
661 data->cnvData=MBCSOpen(data->ucm);
662 if(data->cnvData==NULL) {
663 *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
664
665 } else if(!data->cnvData->isValid(data->cnvData,
666 staticData->subChar, staticData->subCharLen)
667 ) {
668 fprintf(stderr, " the substitution character byte sequence is illegal in this codepage structure!\n");
669 *pErrorCode=U_INVALID_TABLE_FORMAT;
670
671 } else if(staticData->subChar1!=0 &&
672 !data->cnvData->isValid(data->cnvData, &staticData->subChar1, 1)
673 ) {
674 fprintf(stderr, " the subchar1 byte is illegal in this codepage structure!\n");
675 *pErrorCode=U_INVALID_TABLE_FORMAT;
676
677 } else if(
678 data->ucm->ext->mappingsLength>0 &&
679 !ucm_checkBaseExt(states, data->ucm->base, data->ucm->ext, data->ucm->ext, FALSE)
680 ) {
681 *pErrorCode=U_INVALID_TABLE_FORMAT;
682 } else if(data->ucm->base->flagsType&UCM_FLAGS_EXPLICIT) {
683 /* sort the table so that it can be turned into UTF-8-friendly data */
684 ucm_sortTable(data->ucm->base);
685 }
686
687 if(U_SUCCESS(*pErrorCode)) {
688 if(
689 /* add the base table after ucm_checkBaseExt()! */
690 !data->cnvData->addTable(data->cnvData, data->ucm->base, &data->staticData)
691 ) {
692 *pErrorCode=U_INVALID_TABLE_FORMAT;
693 } else {
694 /*
695 * addTable() may have requested moving more mappings to the extension table
696 * if they fit into the base toUnicode table but not into the
697 * base fromUnicode table.
698 * (Especially for UTF-8-friendly fromUnicode tables.)
699 * Such mappings will have the MBCS_FROM_U_EXT_FLAG set, which causes them
700 * to be excluded from the extension toUnicode data.
701 * See MBCSOkForBaseFromUnicode() for which mappings do not fit into
702 * the base fromUnicode table.
703 */
704 ucm_moveMappings(data->ucm->base, data->ucm->ext);
705 ucm_sortTable(data->ucm->ext);
706 if(data->ucm->ext->mappingsLength>0) {
707 /* prepare the extension table, if there is one */
708 data->extData=CnvExtOpen(data->ucm);
709 if(data->extData==NULL) {
710 *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
711 } else if(
712 !data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)
713 ) {
714 *pErrorCode=U_INVALID_TABLE_FORMAT;
715 }
716 }
717 }
718 }
719 } else {
720 /* Build an extension-only .cnv file. */
721 char baseFilename[500];
722 char *basename;
723
724 initConvData(&baseData);
725
726 /* assemble a path/filename for data->ucm->baseName */
727 uprv_strcpy(baseFilename, converterName);
728 basename=(char *)findBasename(baseFilename);
729 uprv_strcpy(basename, data->ucm->baseName);
730 uprv_strcat(basename, ".ucm");
731
732 /* read the base table */
733 dataIsBase=readFile(&baseData, baseFilename, pErrorCode);
734 if(U_FAILURE(*pErrorCode)) {
735 return;
736 } else if(!dataIsBase) {
737 fprintf(stderr, "error: the <icu:base> file \"%s\" is not a base table file\n", baseFilename);
738 *pErrorCode=U_INVALID_TABLE_FORMAT;
739 } else {
740 /* prepare the extension table */
741 data->extData=CnvExtOpen(data->ucm);
742 if(data->extData==NULL) {
743 *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
744 } else {
745 /* fill in gaps in extension file header fields */
746 UCMapping *m, *mLimit;
747 uint8_t fallbackFlags;
748
749 baseStates=&baseData.ucm->states;
750 if(states->conversionType==UCNV_DBCS) {
751 staticData->minBytesPerChar=(int8_t)(states->minCharLength=2);
752 } else if(states->minCharLength==0) {
753 staticData->minBytesPerChar=(int8_t)(states->minCharLength=baseStates->minCharLength);
754 }
755 if(states->maxCharLength<states->minCharLength) {
756 staticData->maxBytesPerChar=(int8_t)(states->maxCharLength=baseStates->maxCharLength);
757 }
758
759 if(staticData->subCharLen==0) {
760 uprv_memcpy(staticData->subChar, baseData.staticData.subChar, 4);
761 staticData->subCharLen=baseData.staticData.subCharLen;
762 }
763 /*
764 * do not copy subChar1 -
765 * only use what is explicitly specified
766 * because it cannot be unset in the extension file header
767 */
768
769 /* get the fallback flags */
770 fallbackFlags=0;
771 for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength;
772 m<mLimit && fallbackFlags!=3;
773 ++m
774 ) {
775 if(m->f==1) {
776 fallbackFlags|=1;
777 } else if(m->f==3) {
778 fallbackFlags|=2;
779 }
780 }
781
782 if(fallbackFlags&1) {
783 staticData->hasFromUnicodeFallback=TRUE;
784 }
785 if(fallbackFlags&2) {
786 staticData->hasToUnicodeFallback=TRUE;
787 }
788
789 if(1!=ucm_countChars(baseStates, staticData->subChar, staticData->subCharLen)) {
790 fprintf(stderr, " the substitution character byte sequence is illegal in this codepage structure!\n");
791 *pErrorCode=U_INVALID_TABLE_FORMAT;
792
793 } else if(staticData->subChar1!=0 && 1!=ucm_countChars(baseStates, &staticData->subChar1, 1)) {
794 fprintf(stderr, " the subchar1 byte is illegal in this codepage structure!\n");
795 *pErrorCode=U_INVALID_TABLE_FORMAT;
796
797 } else if(
798 !ucm_checkValidity(data->ucm->ext, baseStates) ||
799 !ucm_checkBaseExt(baseStates, baseData.ucm->base, data->ucm->ext, data->ucm->ext, FALSE)
800 ) {
801 *pErrorCode=U_INVALID_TABLE_FORMAT;
802 } else {
803 if(states->maxCharLength>1) {
804 /*
805 * When building a normal .cnv file with a base table
806 * for an MBCS (not SBCS) table with explicit precision flags,
807 * the MBCSAddTable() function marks some mappings for moving
808 * to the extension table.
809 * They fit into the base toUnicode table but not into the
810 * base fromUnicode table.
811 * (Note: We do have explicit precision flags because they are
812 * required for extension table generation, and
813 * ucm_checkBaseExt() verified it.)
814 *
815 * We do not call MBCSAddTable() here (we probably could)
816 * so we need to do the analysis before building the extension table.
817 * We assume that MBCSAddTable() will build a UTF-8-friendly table.
818 * Redundant mappings in the extension table are ok except they cost some size.
819 *
820 * Do this after ucm_checkBaseExt().
821 */
822 const MBCSData *mbcsData=MBCSGetDummy();
823 int32_t needsMove=0;
824 for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength;
825 m<mLimit;
826 ++m
827 ) {
828 if(!MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, m->u, m->f)) {
829 m->f|=MBCS_FROM_U_EXT_FLAG;
830 m->moveFlag=UCM_MOVE_TO_EXT;
831 ++needsMove;
832 }
833 }
834
835 if(needsMove!=0) {
836 ucm_moveMappings(baseData.ucm->base, data->ucm->ext);
837 ucm_sortTable(data->ucm->ext);
838 }
839 }
840 if(!data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)) {
841 *pErrorCode=U_INVALID_TABLE_FORMAT;
842 }
843 }
844 }
845 }
846
847 cleanupConvData(&baseData);
848 }
849 }
850
851 /*
852 * Hey, Emacs, please set the following:
853 *
854 * Local Variables:
855 * indent-tabs-mode: nil
856 * End:
857 *
858 */

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial