The Tor Browser / file comparison

comparison: intl/icu/source/tools/gencnval/gencnval.c

intl/icu/source/tools/gencnval/gencnval.c

branch
TOR_BUG_9701
changeset 15
b8a032363ba2
equal deleted inserted replaced
-1:000000000000 0:19d5f23d8227
1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 1999-2012, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: gencnval.c
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created on: 1999nov05
14 * created by: Markus W. Scherer
15 *
16 * This program reads convrtrs.txt and writes a memory-mappable
17 * converter name alias table to cnvalias.dat .
18 *
19 * This program currently writes version 2.1 of the data format. See
20 * ucnv_io.c for more details on the format. Note that version 2.1
21 * is written in such a way that a 2.0 reader will be able to use it,
22 * and a 2.1 reader will be able to read 2.0.
23 */
24
25 #include "unicode/utypes.h"
26 #include "unicode/putil.h"
27 #include "unicode/ucnv.h" /* ucnv_compareNames() */
28 #include "ucnv_io.h"
29 #include "cmemory.h"
30 #include "cstring.h"
31 #include "uinvchar.h"
32 #include "filestrm.h"
33 #include "unicode/uclean.h"
34 #include "unewdata.h"
35 #include "uoptions.h"
36
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <ctype.h>
40
41 /* TODO: Need to check alias name length is less than UCNV_MAX_CONVERTER_NAME_LENGTH */
42
43 /* STRING_STORE_SIZE + TAG_STORE_SIZE <= ((2^16 - 1) * 2)
44 That is the maximum size for the string stores combined
45 because the strings are index at 16-bit boundries by a
46 16-bit index, and there is only one section for the
47 strings.
48 */
49 #define STRING_STORE_SIZE 0x1FBFE /* 130046 */
50 #define TAG_STORE_SIZE 0x400 /* 1024 */
51
52 /* The combined tag and converter count can affect the number of lists
53 created. The size of all lists must be less than (2^17 - 1)
54 because the lists are indexed as a 16-bit array with a 16-bit index.
55 */
56 #define MAX_TAG_COUNT 0x3F /* 63 */
57 #define MAX_CONV_COUNT UCNV_CONVERTER_INDEX_MASK
58 #define MAX_ALIAS_COUNT 0xFFFF /* 65535 */
59
60 /* The maximum number of aliases that a standard tag/converter combination can have.
61 At this moment 6/18/2002, IANA has 12 names for ASCII. Don't go below 15 for
62 this value. I don't recommend more than 31 for this value.
63 */
64 #define MAX_TC_ALIAS_COUNT 0x1F /* 31 */
65
66 #define MAX_LINE_SIZE 0x7FFF /* 32767 */
67 #define MAX_LIST_SIZE 0xFFFF /* 65535 */
68
69 #define DATA_NAME "cnvalias"
70 #define DATA_TYPE "icu" /* ICU alias table */
71
72 #define ALL_TAG_STR "ALL"
73 #define ALL_TAG_NUM 1
74 #define EMPTY_TAG_NUM 0
75
76 /* UDataInfo cf. udata.h */
77 static const UDataInfo dataInfo={
78 sizeof(UDataInfo),
79 0,
80
81 U_IS_BIG_ENDIAN,
82 U_CHARSET_FAMILY,
83 sizeof(UChar),
84 0,
85
86 {0x43, 0x76, 0x41, 0x6c}, /* dataFormat="CvAl" */
87 {3, 0, 1, 0}, /* formatVersion */
88 {1, 4, 2, 0} /* dataVersion */
89 };
90
91 typedef struct {
92 char *store;
93 uint32_t top;
94 uint32_t max;
95 } StringBlock;
96
97 static char stringStore[STRING_STORE_SIZE];
98 static StringBlock stringBlock = { stringStore, 0, STRING_STORE_SIZE };
99
100 typedef struct {
101 uint16_t aliasCount;
102 uint16_t *aliases; /* Index into stringStore */
103 } AliasList;
104
105 typedef struct {
106 uint16_t converter; /* Index into stringStore */
107 uint16_t totalAliasCount; /* Total aliases in this column */
108 } Converter;
109
110 static Converter converters[MAX_CONV_COUNT];
111 static uint16_t converterCount=0;
112
113 static char tagStore[TAG_STORE_SIZE];
114 static StringBlock tagBlock = { tagStore, 0, TAG_STORE_SIZE };
115
116 typedef struct {
117 uint16_t tag; /* Index into tagStore */
118 uint16_t totalAliasCount; /* Total aliases in this row */
119 AliasList aliasList[MAX_CONV_COUNT];
120 } Tag;
121
122 /* Think of this as a 3D array. It's tagCount by converterCount by aliasCount */
123 static Tag tags[MAX_TAG_COUNT];
124 static uint16_t tagCount = 0;
125
126 /* Used for storing all aliases */
127 static uint16_t knownAliases[MAX_ALIAS_COUNT];
128 static uint16_t knownAliasesCount = 0;
129 /*static uint16_t duplicateKnownAliasesCount = 0;*/
130
131 /* Used for storing the lists section that point to aliases */
132 static uint16_t aliasLists[MAX_LIST_SIZE];
133 static uint16_t aliasListsSize = 0;
134
135 /* Were the standard tags declared before the aliases. */
136 static UBool standardTagsUsed = FALSE;
137 static UBool verbose = FALSE;
138 static int lineNum = 1;
139
140 static UConverterAliasOptions tableOptions = {
141 UCNV_IO_STD_NORMALIZED,
142 1 /* containsCnvOptionInfo */
143 };
144
145
146 /**
147 * path to convrtrs.txt
148 */
149 const char *path;
150
151 /* prototypes --------------------------------------------------------------- */
152
153 static void
154 parseLine(const char *line);
155
156 static void
157 parseFile(FileStream *in);
158
159 static int32_t
160 chomp(char *line);
161
162 static void
163 addOfficialTaggedStandards(char *line, int32_t lineLen);
164
165 static uint16_t
166 addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool defaultName);
167
168 static uint16_t
169 addConverter(const char *converter);
170
171 static char *
172 allocString(StringBlock *block, const char *s, int32_t length);
173
174 static uint16_t
175 addToKnownAliases(const char *alias);
176
177 static int
178 compareAliases(const void *alias1, const void *alias2);
179
180 static uint16_t
181 getTagNumber(const char *tag, uint16_t tagLen);
182
183 /*static void
184 addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter);*/
185
186 static void
187 writeAliasTable(UNewDataMemory *out);
188
189 /* -------------------------------------------------------------------------- */
190
191 /* Presumes that you used allocString() */
192 #define GET_ALIAS_STR(index) (stringStore + ((size_t)(index) << 1))
193 #define GET_TAG_STR(index) (tagStore + ((size_t)(index) << 1))
194
195 /* Presumes that you used allocString() */
196 #define GET_ALIAS_NUM(str) ((uint16_t)((str - stringStore) >> 1))
197 #define GET_TAG_NUM(str) ((uint16_t)((str - tagStore) >> 1))
198
199 enum
200 {
201 HELP1,
202 HELP2,
203 VERBOSE,
204 COPYRIGHT,
205 DESTDIR,
206 SOURCEDIR
207 };
208
209 static UOption options[]={
210 UOPTION_HELP_H,
211 UOPTION_HELP_QUESTION_MARK,
212 UOPTION_VERBOSE,
213 UOPTION_COPYRIGHT,
214 UOPTION_DESTDIR,
215 UOPTION_SOURCEDIR
216 };
217
218 extern int
219 main(int argc, char* argv[]) {
220 int i, n;
221 char pathBuf[512];
222 FileStream *in;
223 UNewDataMemory *out;
224 UErrorCode errorCode=U_ZERO_ERROR;
225
226 U_MAIN_INIT_ARGS(argc, argv);
227
228 /* preset then read command line options */
229 options[DESTDIR].value=options[SOURCEDIR].value=u_getDataDirectory();
230 argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
231
232 /* error handling, printing usage message */
233 if(argc<0) {
234 fprintf(stderr,
235 "error in command line argument \"%s\"\n",
236 argv[-argc]);
237 }
238 if(argc<0 || options[HELP1].doesOccur || options[HELP2].doesOccur) {
239 fprintf(stderr,
240 "usage: %s [-options] [convrtrs.txt]\n"
241 "\tread convrtrs.txt and create " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE "\n"
242 "options:\n"
243 "\t-h or -? or --help this usage text\n"
244 "\t-v or --verbose prints out extra information about the alias table\n"
245 "\t-c or --copyright include a copyright notice\n"
246 "\t-d or --destdir destination directory, followed by the path\n"
247 "\t-s or --sourcedir source directory, followed by the path\n",
248 argv[0]);
249 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
250 }
251
252 if(options[VERBOSE].doesOccur) {
253 verbose = TRUE;
254 }
255
256 if(argc>=2) {
257 path=argv[1];
258 } else {
259 path=options[SOURCEDIR].value;
260 if(path!=NULL && *path!=0) {
261 char *end;
262
263 uprv_strcpy(pathBuf, path);
264 end = uprv_strchr(pathBuf, 0);
265 if(*(end-1)!=U_FILE_SEP_CHAR) {
266 *(end++)=U_FILE_SEP_CHAR;
267 }
268 uprv_strcpy(end, "convrtrs.txt");
269 path=pathBuf;
270 } else {
271 path = "convrtrs.txt";
272 }
273 }
274
275 uprv_memset(stringStore, 0, sizeof(stringStore));
276 uprv_memset(tagStore, 0, sizeof(tagStore));
277 uprv_memset(converters, 0, sizeof(converters));
278 uprv_memset(tags, 0, sizeof(tags));
279 uprv_memset(aliasLists, 0, sizeof(aliasLists));
280 uprv_memset(knownAliases, 0, sizeof(aliasLists));
281
282
283 in=T_FileStream_open(path, "r");
284 if(in==NULL) {
285 fprintf(stderr, "gencnval: unable to open input file %s\n", path);
286 exit(U_FILE_ACCESS_ERROR);
287 }
288 parseFile(in);
289 T_FileStream_close(in);
290
291 /* create the output file */
292 out=udata_create(options[DESTDIR].value, DATA_TYPE, DATA_NAME, &dataInfo,
293 options[COPYRIGHT].doesOccur ? U_COPYRIGHT_STRING : NULL, &errorCode);
294 if(U_FAILURE(errorCode)) {
295 fprintf(stderr, "gencnval: unable to open output file - error %s\n", u_errorName(errorCode));
296 exit(errorCode);
297 }
298
299 /* write the table of aliases based on a tag/converter name combination */
300 writeAliasTable(out);
301
302 /* finish */
303 udata_finish(out, &errorCode);
304 if(U_FAILURE(errorCode)) {
305 fprintf(stderr, "gencnval: error finishing output file - %s\n", u_errorName(errorCode));
306 exit(errorCode);
307 }
308
309 /* clean up tags */
310 for (i = 0; i < MAX_TAG_COUNT; i++) {
311 for (n = 0; n < MAX_CONV_COUNT; n++) {
312 if (tags[i].aliasList[n].aliases!=NULL) {
313 uprv_free(tags[i].aliasList[n].aliases);
314 }
315 }
316 }
317
318 return 0;
319 }
320
321 static void
322 parseFile(FileStream *in) {
323 char line[MAX_LINE_SIZE];
324 char lastLine[MAX_LINE_SIZE];
325 int32_t lineSize = 0;
326 int32_t lastLineSize = 0;
327 UBool validParse = TRUE;
328
329 lineNum = 0;
330
331 /* Add the empty tag, which is for untagged aliases */
332 getTagNumber("", 0);
333 getTagNumber(ALL_TAG_STR, 3);
334 allocString(&stringBlock, "", 0);
335
336 /* read the list of aliases */
337 while (validParse) {
338 validParse = FALSE;
339
340 /* Read non-empty lines that don't start with a space character. */
341 while (T_FileStream_readLine(in, lastLine, MAX_LINE_SIZE) != NULL) {
342 lastLineSize = chomp(lastLine);
343 if (lineSize == 0 || (lastLineSize > 0 && isspace((int)*lastLine))) {
344 uprv_strcpy(line + lineSize, lastLine);
345 lineSize += lastLineSize;
346 } else if (lineSize > 0) {
347 validParse = TRUE;
348 break;
349 }
350 lineNum++;
351 }
352
353 if (validParse || lineSize > 0) {
354 if (isspace((int)*line)) {
355 fprintf(stderr, "%s:%d: error: cannot start an alias with a space\n", path, lineNum-1);
356 exit(U_PARSE_ERROR);
357 } else if (line[0] == '{') {
358 if (!standardTagsUsed && line[lineSize - 1] != '}') {
359 fprintf(stderr, "%s:%d: error: alias needs to start with a converter name\n", path, lineNum);
360 exit(U_PARSE_ERROR);
361 }
362 addOfficialTaggedStandards(line, lineSize);
363 standardTagsUsed = TRUE;
364 } else {
365 if (standardTagsUsed) {
366 parseLine(line);
367 }
368 else {
369 fprintf(stderr, "%s:%d: error: alias table needs to start a list of standard tags\n", path, lineNum);
370 exit(U_PARSE_ERROR);
371 }
372 }
373 /* Was the last line consumed */
374 if (lastLineSize > 0) {
375 uprv_strcpy(line, lastLine);
376 lineSize = lastLineSize;
377 }
378 else {
379 lineSize = 0;
380 }
381 }
382 lineNum++;
383 }
384 }
385
386 /* This works almost like the Perl chomp.
387 It removes the newlines, comments and trailing whitespace (not preceding whitespace).
388 */
389 static int32_t
390 chomp(char *line) {
391 char *s = line;
392 char *lastNonSpace = line;
393 while(*s!=0) {
394 /* truncate at a newline or a comment */
395 if(*s == '\r' || *s == '\n' || *s == '#') {
396 *s = 0;
397 break;
398 }
399 if (!isspace((int)*s)) {
400 lastNonSpace = s;
401 }
402 ++s;
403 }
404 if (lastNonSpace++ > line) {
405 *lastNonSpace = 0;
406 s = lastNonSpace;
407 }
408 return (int32_t)(s - line);
409 }
410
411 static void
412 parseLine(const char *line) {
413 uint16_t pos=0, start, limit, length, cnv;
414 char *converter, *alias;
415
416 /* skip leading white space */
417 /* There is no whitespace at the beginning anymore */
418 /* while(line[pos]!=0 && isspace(line[pos])) {
419 ++pos;
420 }
421 */
422
423 /* is there nothing on this line? */
424 if(line[pos]==0) {
425 return;
426 }
427
428 /* get the converter name */
429 start=pos;
430 while(line[pos]!=0 && !isspace((int)line[pos])) {
431 ++pos;
432 }
433 limit=pos;
434
435 /* store the converter name */
436 length=(uint16_t)(limit-start);
437 converter=allocString(&stringBlock, line+start, length);
438
439 /* add the converter to the converter table */
440 cnv=addConverter(converter);
441
442 /* The name itself may be tagged, so let's added it to the aliases list properly */
443 pos = start;
444
445 /* get all the real aliases */
446 for(;;) {
447
448 /* skip white space */
449 while(line[pos]!=0 && isspace((int)line[pos])) {
450 ++pos;
451 }
452
453 /* is there no more alias name on this line? */
454 if(line[pos]==0) {
455 break;
456 }
457
458 /* get an alias name */
459 start=pos;
460 while(line[pos]!=0 && line[pos]!='{' && !isspace((int)line[pos])) {
461 ++pos;
462 }
463 limit=pos;
464
465 /* store the alias name */
466 length=(uint16_t)(limit-start);
467 if (start == 0) {
468 /* add the converter as its own alias to the alias table */
469 alias = converter;
470 addAlias(alias, ALL_TAG_NUM, cnv, TRUE);
471 }
472 else {
473 alias=allocString(&stringBlock, line+start, length);
474 addAlias(alias, ALL_TAG_NUM, cnv, FALSE);
475 }
476 addToKnownAliases(alias);
477
478 /* add the alias/converter pair to the alias table */
479 /* addAlias(alias, 0, cnv, FALSE);*/
480
481 /* skip whitespace */
482 while (line[pos] && isspace((int)line[pos])) {
483 ++pos;
484 }
485
486 /* handle tags if they are present */
487 if (line[pos] == '{') {
488 ++pos;
489 do {
490 start = pos;
491 while (line[pos] && line[pos] != '}' && !isspace((int)line[pos])) {
492 ++pos;
493 }
494 limit = pos;
495
496 if (start != limit) {
497 /* add the tag to the tag table */
498 uint16_t tag = getTagNumber(line + start, (uint16_t)(limit - start));
499 addAlias(alias, tag, cnv, (UBool)(line[limit-1] == '*'));
500 }
501
502 while (line[pos] && isspace((int)line[pos])) {
503 ++pos;
504 }
505 } while (line[pos] && line[pos] != '}');
506
507 if (line[pos] == '}') {
508 ++pos;
509 } else {
510 fprintf(stderr, "%s:%d: Unterminated tag list\n", path, lineNum);
511 exit(U_UNMATCHED_BRACES);
512 }
513 } else {
514 addAlias(alias, EMPTY_TAG_NUM, cnv, (UBool)(tags[0].aliasList[cnv].aliasCount == 0));
515 }
516 }
517 }
518
519 static uint16_t
520 getTagNumber(const char *tag, uint16_t tagLen) {
521 char *atag;
522 uint16_t t;
523 UBool preferredName = ((tagLen > 0) ? (tag[tagLen - 1] == '*') : (FALSE));
524
525 if (tagCount >= MAX_TAG_COUNT) {
526 fprintf(stderr, "%s:%d: too many tags\n", path, lineNum);
527 exit(U_BUFFER_OVERFLOW_ERROR);
528 }
529
530 if (preferredName) {
531 /* puts(tag);*/
532 tagLen--;
533 }
534
535 for (t = 0; t < tagCount; ++t) {
536 const char *currTag = GET_TAG_STR(tags[t].tag);
537 if (uprv_strlen(currTag) == tagLen && !uprv_strnicmp(currTag, tag, tagLen)) {
538 return t;
539 }
540 }
541
542 /* we need to add this tag */
543 if (tagCount >= MAX_TAG_COUNT) {
544 fprintf(stderr, "%s:%d: error: too many tags\n", path, lineNum);
545 exit(U_BUFFER_OVERFLOW_ERROR);
546 }
547
548 /* allocate a new entry in the tag table */
549 atag = allocString(&tagBlock, tag, tagLen);
550
551 if (standardTagsUsed) {
552 fprintf(stderr, "%s:%d: error: Tag \"%s\" is not declared at the beginning of the alias table.\n",
553 path, lineNum, atag);
554 exit(1);
555 }
556 else if (tagLen > 0 && strcmp(tag, ALL_TAG_STR) != 0) {
557 fprintf(stderr, "%s:%d: warning: Tag \"%s\" was added to the list of standards because it was not declared at beginning of the alias table.\n",
558 path, lineNum, atag);
559 }
560
561 /* add the tag to the tag table */
562 tags[tagCount].tag = GET_TAG_NUM(atag);
563 /* The aliasList should be set to 0's already */
564
565 return tagCount++;
566 }
567
568 /*static void
569 addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter) {
570 tags[tag].aliases[converter] = alias;
571 }
572 */
573
574 static void
575 addOfficialTaggedStandards(char *line, int32_t lineLen) {
576 char *atag;
577 char *endTagExp;
578 char *tag;
579 static const char WHITESPACE[] = " \t";
580
581 if (tagCount > UCNV_NUM_RESERVED_TAGS) {
582 fprintf(stderr, "%s:%d: error: official tags already added\n", path, lineNum);
583 exit(U_BUFFER_OVERFLOW_ERROR);
584 }
585 tag = strchr(line, '{');
586 if (tag == NULL) {
587 /* Why were we called? */
588 fprintf(stderr, "%s:%d: error: Missing start of tag group\n", path, lineNum);
589 exit(U_PARSE_ERROR);
590 }
591 tag++;
592 endTagExp = strchr(tag, '}');
593 if (endTagExp == NULL) {
594 fprintf(stderr, "%s:%d: error: Missing end of tag group\n", path, lineNum);
595 exit(U_PARSE_ERROR);
596 }
597 endTagExp[0] = 0;
598
599 tag = strtok(tag, WHITESPACE);
600 while (tag != NULL) {
601 /* printf("Adding original tag \"%s\"\n", tag);*/
602
603 /* allocate a new entry in the tag table */
604 atag = allocString(&tagBlock, tag, -1);
605
606 /* add the tag to the tag table */
607 tags[tagCount++].tag = (uint16_t)((atag - tagStore) >> 1);
608
609 /* The aliasList should already be set to 0's */
610
611 /* Get next tag */
612 tag = strtok(NULL, WHITESPACE);
613 }
614 }
615
616 static uint16_t
617 addToKnownAliases(const char *alias) {
618 /* uint32_t idx; */
619 /* strict matching */
620 /* for (idx = 0; idx < knownAliasesCount; idx++) {
621 uint16_t num = GET_ALIAS_NUM(alias);
622 if (knownAliases[idx] != num
623 && uprv_strcmp(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
624 {
625 fprintf(stderr, "%s:%d: warning: duplicate alias %s and %s found\n", path,
626 lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
627 duplicateKnownAliasesCount++;
628 break;
629 }
630 else if (knownAliases[idx] != num
631 && ucnv_compareNames(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
632 {
633 if (verbose) {
634 fprintf(stderr, "%s:%d: information: duplicate alias %s and %s found\n", path,
635 lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
636 }
637 duplicateKnownAliasesCount++;
638 break;
639 }
640 }
641 */
642 if (knownAliasesCount >= MAX_ALIAS_COUNT) {
643 fprintf(stderr, "%s:%d: warning: Too many aliases defined for all converters\n",
644 path, lineNum);
645 exit(U_BUFFER_OVERFLOW_ERROR);
646 }
647 /* TODO: We could try to unlist exact duplicates. */
648 return knownAliases[knownAliasesCount++] = GET_ALIAS_NUM(alias);
649 }
650
651 /*
652 @param standard When standard is 0, then it's the "empty" tag.
653 */
654 static uint16_t
655 addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool defaultName) {
656 uint32_t idx, idx2;
657 UBool startEmptyWithoutDefault = FALSE;
658 AliasList *aliasList;
659
660 if(standard>=MAX_TAG_COUNT) {
661 fprintf(stderr, "%s:%d: error: too many standard tags\n", path, lineNum);
662 exit(U_BUFFER_OVERFLOW_ERROR);
663 }
664 if(converter>=MAX_CONV_COUNT) {
665 fprintf(stderr, "%s:%d: error: too many converter names\n", path, lineNum);
666 exit(U_BUFFER_OVERFLOW_ERROR);
667 }
668 aliasList = &tags[standard].aliasList[converter];
669
670 if (strchr(alias, '}')) {
671 fprintf(stderr, "%s:%d: error: unmatched } found\n", path,
672 lineNum);
673 }
674
675 if(aliasList->aliasCount + 1 >= MAX_TC_ALIAS_COUNT) {
676 fprintf(stderr, "%s:%d: error: too many aliases for alias %s and converter %s\n", path,
677 lineNum, alias, GET_ALIAS_STR(converters[converter].converter));
678 exit(U_BUFFER_OVERFLOW_ERROR);
679 }
680
681 /* Show this warning only once. All aliases are added to the "ALL" tag. */
682 if (standard == ALL_TAG_NUM && GET_ALIAS_STR(converters[converter].converter) != alias) {
683 /* Normally these option values are parsed at runtime, and they can
684 be discarded when the alias is a default converter. Options should
685 only be on a converter and not an alias. */
686 if (uprv_strchr(alias, UCNV_OPTION_SEP_CHAR) != 0)
687 {
688 fprintf(stderr, "warning(line %d): alias %s contains a \""UCNV_OPTION_SEP_STRING"\". Options are parsed at run-time and do not need to be in the alias table.\n",
689 lineNum, alias);
690 }
691 if (uprv_strchr(alias, UCNV_VALUE_SEP_CHAR) != 0)
692 {
693 fprintf(stderr, "warning(line %d): alias %s contains an \""UCNV_VALUE_SEP_STRING"\". Options are parsed at run-time and do not need to be in the alias table.\n",
694 lineNum, alias);
695 }
696 }
697
698 if (standard != ALL_TAG_NUM) {
699 /* Check for duplicate aliases for this tag on all converters */
700 for (idx = 0; idx < converterCount; idx++) {
701 for (idx2 = 0; idx2 < tags[standard].aliasList[idx].aliasCount; idx2++) {
702 uint16_t aliasNum = tags[standard].aliasList[idx].aliases[idx2];
703 if (aliasNum
704 && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
705 {
706 if (idx == converter) {
707 /*
708 * (alias, standard) duplicates are harmless if they map to the same converter.
709 * Only print a warning in verbose mode, or if the alias is a precise duplicate,
710 * not just a lenient-match duplicate.
711 */
712 if (verbose || 0 == uprv_strcmp(alias, GET_ALIAS_STR(aliasNum))) {
713 fprintf(stderr, "%s:%d: warning: duplicate aliases %s and %s found for standard %s and converter %s\n", path,
714 lineNum, alias, GET_ALIAS_STR(aliasNum),
715 GET_TAG_STR(tags[standard].tag),
716 GET_ALIAS_STR(converters[converter].converter));
717 }
718 } else {
719 fprintf(stderr, "%s:%d: warning: duplicate aliases %s and %s found for standard tag %s between converter %s and converter %s\n", path,
720 lineNum, alias, GET_ALIAS_STR(aliasNum),
721 GET_TAG_STR(tags[standard].tag),
722 GET_ALIAS_STR(converters[converter].converter),
723 GET_ALIAS_STR(converters[idx].converter));
724 }
725 break;
726 }
727 }
728 }
729
730 /* Check for duplicate default aliases for this converter on all tags */
731 /* It's okay to have multiple standards prefer the same name */
732 /* if (verbose && !dupFound) {
733 for (idx = 0; idx < tagCount; idx++) {
734 if (tags[idx].aliasList[converter].aliases) {
735 uint16_t aliasNum = tags[idx].aliasList[converter].aliases[0];
736 if (aliasNum
737 && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
738 {
739 fprintf(stderr, "%s:%d: warning: duplicate alias %s found for converter %s and standard tag %s\n", path,
740 lineNum, alias, GET_ALIAS_STR(converters[converter].converter), GET_TAG_STR(tags[standard].tag));
741 break;
742 }
743 }
744 }
745 }*/
746 }
747
748 if (aliasList->aliasCount <= 0) {
749 aliasList->aliasCount++;
750 startEmptyWithoutDefault = TRUE;
751 }
752 aliasList->aliases = (uint16_t *)uprv_realloc(aliasList->aliases, (aliasList->aliasCount + 1) * sizeof(aliasList->aliases[0]));
753 if (startEmptyWithoutDefault) {
754 aliasList->aliases[0] = 0;
755 }
756 if (defaultName) {
757 if (aliasList->aliases[0] != 0) {
758 fprintf(stderr, "%s:%d: error: Alias %s and %s cannot both be the default alias for standard tag %s and converter %s\n", path,
759 lineNum,
760 alias,
761 GET_ALIAS_STR(aliasList->aliases[0]),
762 GET_TAG_STR(tags[standard].tag),
763 GET_ALIAS_STR(converters[converter].converter));
764 exit(U_PARSE_ERROR);
765 }
766 aliasList->aliases[0] = GET_ALIAS_NUM(alias);
767 } else {
768 aliasList->aliases[aliasList->aliasCount++] = GET_ALIAS_NUM(alias);
769 }
770 /* aliasList->converter = converter;*/
771
772 converters[converter].totalAliasCount++; /* One more to the column */
773 tags[standard].totalAliasCount++; /* One more to the row */
774
775 return aliasList->aliasCount;
776 }
777
778 static uint16_t
779 addConverter(const char *converter) {
780 uint32_t idx;
781 if(converterCount>=MAX_CONV_COUNT) {
782 fprintf(stderr, "%s:%d: error: too many converters\n", path, lineNum);
783 exit(U_BUFFER_OVERFLOW_ERROR);
784 }
785
786 for (idx = 0; idx < converterCount; idx++) {
787 if (ucnv_compareNames(converter, GET_ALIAS_STR(converters[idx].converter)) == 0) {
788 fprintf(stderr, "%s:%d: error: duplicate converter %s found!\n", path, lineNum, converter);
789 exit(U_PARSE_ERROR);
790 break;
791 }
792 }
793
794 converters[converterCount].converter = GET_ALIAS_NUM(converter);
795 converters[converterCount].totalAliasCount = 0;
796
797 return converterCount++;
798 }
799
800 /* resolve this alias based on the prioritization of the standard tags. */
801 static void
802 resolveAliasToConverter(uint16_t alias, uint16_t *tagNum, uint16_t *converterNum) {
803 uint16_t idx, idx2, idx3;
804
805 for (idx = UCNV_NUM_RESERVED_TAGS; idx < tagCount; idx++) {
806 for (idx2 = 0; idx2 < converterCount; idx2++) {
807 for (idx3 = 0; idx3 < tags[idx].aliasList[idx2].aliasCount; idx3++) {
808 uint16_t aliasNum = tags[idx].aliasList[idx2].aliases[idx3];
809 if (aliasNum == alias) {
810 *tagNum = idx;
811 *converterNum = idx2;
812 return;
813 }
814 }
815 }
816 }
817 /* Do the leftovers last, just in case */
818 /* There is no need to do the ALL tag */
819 idx = 0;
820 for (idx2 = 0; idx2 < converterCount; idx2++) {
821 for (idx3 = 0; idx3 < tags[idx].aliasList[idx2].aliasCount; idx3++) {
822 uint16_t aliasNum = tags[idx].aliasList[idx2].aliases[idx3];
823 if (aliasNum == alias) {
824 *tagNum = idx;
825 *converterNum = idx2;
826 return;
827 }
828 }
829 }
830 *tagNum = UINT16_MAX;
831 *converterNum = UINT16_MAX;
832 fprintf(stderr, "%s: warning: alias %s not found\n",
833 path,
834 GET_ALIAS_STR(alias));
835 return;
836 }
837
838 /* The knownAliases should be sorted before calling this function */
839 static uint32_t
840 resolveAliases(uint16_t *uniqueAliasArr, uint16_t *uniqueAliasToConverterArr, uint16_t aliasOffset) {
841 uint32_t uniqueAliasIdx = 0;
842 uint32_t idx;
843 uint16_t currTagNum, oldTagNum;
844 uint16_t currConvNum, oldConvNum;
845 const char *lastName;
846
847 resolveAliasToConverter(knownAliases[0], &oldTagNum, &currConvNum);
848 uniqueAliasToConverterArr[uniqueAliasIdx] = currConvNum;
849 oldConvNum = currConvNum;
850 uniqueAliasArr[uniqueAliasIdx] = knownAliases[0] + aliasOffset;
851 uniqueAliasIdx++;
852 lastName = GET_ALIAS_STR(knownAliases[0]);
853
854 for (idx = 1; idx < knownAliasesCount; idx++) {
855 resolveAliasToConverter(knownAliases[idx], &currTagNum, &currConvNum);
856 if (ucnv_compareNames(lastName, GET_ALIAS_STR(knownAliases[idx])) == 0) {
857 /* duplicate found */
858 if ((currTagNum < oldTagNum && currTagNum >= UCNV_NUM_RESERVED_TAGS)
859 || oldTagNum == 0) {
860 oldTagNum = currTagNum;
861 uniqueAliasToConverterArr[uniqueAliasIdx - 1] = currConvNum;
862 uniqueAliasArr[uniqueAliasIdx - 1] = knownAliases[idx] + aliasOffset;
863 if (verbose) {
864 printf("using %s instead of %s -> %s",
865 GET_ALIAS_STR(knownAliases[idx]),
866 lastName,
867 GET_ALIAS_STR(converters[currConvNum].converter));
868 if (oldConvNum != currConvNum) {
869 printf(" (alias conflict)");
870 }
871 puts("");
872 }
873 }
874 else {
875 /* else ignore it */
876 if (verbose) {
877 printf("folding %s into %s -> %s",
878 GET_ALIAS_STR(knownAliases[idx]),
879 lastName,
880 GET_ALIAS_STR(converters[oldConvNum].converter));
881 if (oldConvNum != currConvNum) {
882 printf(" (alias conflict)");
883 }
884 puts("");
885 }
886 }
887 if (oldConvNum != currConvNum) {
888 uniqueAliasToConverterArr[uniqueAliasIdx - 1] |= UCNV_AMBIGUOUS_ALIAS_MAP_BIT;
889 }
890 }
891 else {
892 uniqueAliasToConverterArr[uniqueAliasIdx] = currConvNum;
893 oldConvNum = currConvNum;
894 uniqueAliasArr[uniqueAliasIdx] = knownAliases[idx] + aliasOffset;
895 uniqueAliasIdx++;
896 lastName = GET_ALIAS_STR(knownAliases[idx]);
897 oldTagNum = currTagNum;
898 /*printf("%s -> %s\n", GET_ALIAS_STR(knownAliases[idx]), GET_ALIAS_STR(converters[currConvNum].converter));*/
899 }
900 if (uprv_strchr(GET_ALIAS_STR(converters[currConvNum].converter), UCNV_OPTION_SEP_CHAR) != NULL) {
901 uniqueAliasToConverterArr[uniqueAliasIdx-1] |= UCNV_CONTAINS_OPTION_BIT;
902 }
903 }
904 return uniqueAliasIdx;
905 }
906
907 static void
908 createOneAliasList(uint16_t *aliasArrLists, uint32_t tag, uint32_t converter, uint16_t offset) {
909 uint32_t aliasNum;
910 AliasList *aliasList = &tags[tag].aliasList[converter];
911
912 if (aliasList->aliasCount == 0) {
913 aliasArrLists[tag*converterCount + converter] = 0;
914 }
915 else {
916 aliasLists[aliasListsSize++] = aliasList->aliasCount;
917
918 /* write into the array area a 1's based index. */
919 aliasArrLists[tag*converterCount + converter] = aliasListsSize;
920
921 /* printf("tag %s converter %s\n",
922 GET_TAG_STR(tags[tag].tag),
923 GET_ALIAS_STR(converters[converter].converter));*/
924 for (aliasNum = 0; aliasNum < aliasList->aliasCount; aliasNum++) {
925 uint16_t value;
926 /* printf(" %s\n",
927 GET_ALIAS_STR(aliasList->aliases[aliasNum]));*/
928 if (aliasList->aliases[aliasNum]) {
929 value = aliasList->aliases[aliasNum] + offset;
930 } else {
931 value = 0;
932 if (tag != 0) { /* Only show the warning when it's not the leftover tag. */
933 fprintf(stderr, "%s: warning: tag %s does not have a default alias for %s\n",
934 path,
935 GET_TAG_STR(tags[tag].tag),
936 GET_ALIAS_STR(converters[converter].converter));
937 }
938 }
939 aliasLists[aliasListsSize++] = value;
940 if (aliasListsSize >= MAX_LIST_SIZE) {
941 fprintf(stderr, "%s: error: Too many alias lists\n", path);
942 exit(U_BUFFER_OVERFLOW_ERROR);
943 }
944
945 }
946 }
947 }
948
949 static void
950 createNormalizedAliasStrings(char *normalizedStrings, const char *origStringBlock, int32_t stringBlockLength) {
951 int32_t currStrLen;
952 uprv_memcpy(normalizedStrings, origStringBlock, stringBlockLength);
953 while ((currStrLen = (int32_t)uprv_strlen(origStringBlock)) < stringBlockLength) {
954 int32_t currStrSize = currStrLen + 1;
955 if (currStrLen > 0) {
956 int32_t normStrLen;
957 ucnv_io_stripForCompare(normalizedStrings, origStringBlock);
958 normStrLen = uprv_strlen(normalizedStrings);
959 if (normStrLen > 0) {
960 uprv_memset(normalizedStrings + normStrLen, 0, currStrSize - normStrLen);
961 }
962 }
963 stringBlockLength -= currStrSize;
964 normalizedStrings += currStrSize;
965 origStringBlock += currStrSize;
966 }
967 }
968
969 static void
970 writeAliasTable(UNewDataMemory *out) {
971 uint32_t i, j;
972 uint32_t uniqueAliasesSize;
973 uint16_t aliasOffset = (uint16_t)(tagBlock.top/sizeof(uint16_t));
974 uint16_t *aliasArrLists = (uint16_t *)uprv_malloc(tagCount * converterCount * sizeof(uint16_t));
975 uint16_t *uniqueAliases = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t));
976 uint16_t *uniqueAliasesToConverter = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t));
977
978 qsort(knownAliases, knownAliasesCount, sizeof(knownAliases[0]), compareAliases);
979 uniqueAliasesSize = resolveAliases(uniqueAliases, uniqueAliasesToConverter, aliasOffset);
980
981 /* Array index starts at 1. aliasLists[0] is the size of the lists section. */
982 aliasListsSize = 0;
983
984 /* write the offsets of all the aliases lists in a 2D array, and create the lists. */
985 for (i = 0; i < tagCount; ++i) {
986 for (j = 0; j < converterCount; ++j) {
987 createOneAliasList(aliasArrLists, i, j, aliasOffset);
988 }
989 }
990
991 /* Write the size of the TOC */
992 if (tableOptions.stringNormalizationType == UCNV_IO_UNNORMALIZED) {
993 udata_write32(out, 8);
994 }
995 else {
996 udata_write32(out, 9);
997 }
998
999 /* Write the sizes of each section */
1000 /* All sizes are the number of uint16_t units, not bytes */
1001 udata_write32(out, converterCount);
1002 udata_write32(out, tagCount);
1003 udata_write32(out, uniqueAliasesSize); /* list of aliases */
1004 udata_write32(out, uniqueAliasesSize); /* The preresolved form of mapping an untagged the alias to a converter */
1005 udata_write32(out, tagCount * converterCount);
1006 udata_write32(out, aliasListsSize + 1);
1007 udata_write32(out, sizeof(tableOptions) / sizeof(uint16_t));
1008 udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
1009 if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) {
1010 udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
1011 }
1012
1013 /* write the table of converters */
1014 /* Think of this as the column headers */
1015 for(i=0; i<converterCount; ++i) {
1016 udata_write16(out, (uint16_t)(converters[i].converter + aliasOffset));
1017 }
1018
1019 /* write the table of tags */
1020 /* Think of this as the row headers */
1021 for(i=UCNV_NUM_RESERVED_TAGS; i<tagCount; ++i) {
1022 udata_write16(out, tags[i].tag);
1023 }
1024 /* The empty tag is considered the leftover list, and put that at the end of the priority list. */
1025 udata_write16(out, tags[EMPTY_TAG_NUM].tag);
1026 udata_write16(out, tags[ALL_TAG_NUM].tag);
1027
1028 /* Write the unique list of aliases */
1029 udata_writeBlock(out, uniqueAliases, uniqueAliasesSize * sizeof(uint16_t));
1030
1031 /* Write the unique list of aliases */
1032 udata_writeBlock(out, uniqueAliasesToConverter, uniqueAliasesSize * sizeof(uint16_t));
1033
1034 /* Write the array to the lists */
1035 udata_writeBlock(out, (const void *)(aliasArrLists + (2*converterCount)), (((tagCount - 2) * converterCount) * sizeof(uint16_t)));
1036 /* Now write the leftover part of the array for the EMPTY and ALL lists */
1037 udata_writeBlock(out, (const void *)aliasArrLists, (2 * converterCount * sizeof(uint16_t)));
1038
1039 /* Offset the next array to make the index start at 1. */
1040 udata_write16(out, 0xDEAD);
1041
1042 /* Write the lists */
1043 udata_writeBlock(out, (const void *)aliasLists, aliasListsSize * sizeof(uint16_t));
1044
1045 /* Write any options for the alias table. */
1046 udata_writeBlock(out, (const void *)&tableOptions, sizeof(tableOptions));
1047
1048 /* write the tags strings */
1049 udata_writeString(out, tagBlock.store, tagBlock.top);
1050
1051 /* write the aliases strings */
1052 udata_writeString(out, stringBlock.store, stringBlock.top);
1053
1054 /* write the normalized aliases strings */
1055 if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) {
1056 char *normalizedStrings = (char *)uprv_malloc(tagBlock.top + stringBlock.top);
1057 createNormalizedAliasStrings(normalizedStrings, tagBlock.store, tagBlock.top);
1058 createNormalizedAliasStrings(normalizedStrings + tagBlock.top, stringBlock.store, stringBlock.top);
1059
1060 /* Write out the complete normalized array. */
1061 udata_writeString(out, normalizedStrings, tagBlock.top + stringBlock.top);
1062 uprv_free(normalizedStrings);
1063 }
1064
1065 uprv_free(uniqueAliasesToConverter);
1066 uprv_free(uniqueAliases);
1067 uprv_free(aliasArrLists);
1068 }
1069
1070 static char *
1071 allocString(StringBlock *block, const char *s, int32_t length) {
1072 uint32_t top;
1073 char *p;
1074
1075 if(length<0) {
1076 length=(int32_t)uprv_strlen(s);
1077 }
1078
1079 /*
1080 * add 1 for the terminating NUL
1081 * and round up (+1 &~1)
1082 * to keep the addresses on a 16-bit boundary
1083 */
1084 top=block->top + (uint32_t)((length + 1 + 1) & ~1);
1085
1086 if(top >= block->max) {
1087 fprintf(stderr, "%s:%d: error: out of memory\n", path, lineNum);
1088 exit(U_MEMORY_ALLOCATION_ERROR);
1089 }
1090
1091 /* get the pointer and copy the string */
1092 p = block->store + block->top;
1093 uprv_memcpy(p, s, length);
1094 p[length] = 0; /* NUL-terminate it */
1095 if((length & 1) == 0) {
1096 p[length + 1] = 0; /* set the padding byte */
1097 }
1098
1099 /* check for invariant characters now that we have a NUL-terminated string for easy output */
1100 if(!uprv_isInvariantString(p, length)) {
1101 fprintf(stderr, "%s:%d: error: the name %s contains not just invariant characters\n", path, lineNum, p);
1102 exit(U_INVALID_TABLE_FORMAT);
1103 }
1104
1105 block->top = top;
1106 return p;
1107 }
1108
1109 static int
1110 compareAliases(const void *alias1, const void *alias2) {
1111 /* Names like IBM850 and ibm-850 need to be sorted together */
1112 int result = ucnv_compareNames(GET_ALIAS_STR(*(uint16_t*)alias1), GET_ALIAS_STR(*(uint16_t*)alias2));
1113 if (!result) {
1114 /* Sort the shortest first */
1115 return (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias1)) - (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias2));
1116 }
1117 return result;
1118 }
1119
1120 /*
1121 * Hey, Emacs, please set the following:
1122 *
1123 * Local Variables:
1124 * indent-tabs-mode: nil
1125 * End:
1126 *
1127 */
1128

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial