The Tor Browser / file comparison

comparison: intl/icu/source/tools/toolutil/pkg_gencmn.c

intl/icu/source/tools/toolutil/pkg_gencmn.c

branch
TOR_BUG_3246
changeset 4
fc2d59ddac77
equal deleted inserted replaced
-1:000000000000 0:387725ad0798
1 /******************************************************************************
2 * Copyright (C) 2008-2012, International Business Machines
3 * Corporation and others. All Rights Reserved.
4 *******************************************************************************
5 */
6 #include "unicode/utypes.h"
7
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include "unicode/utypes.h"
11 #include "unicode/putil.h"
12 #include "cmemory.h"
13 #include "cstring.h"
14 #include "filestrm.h"
15 #include "toolutil.h"
16 #include "unicode/uclean.h"
17 #include "unewdata.h"
18 #include "putilimp.h"
19 #include "pkg_gencmn.h"
20
21 #define STRING_STORE_SIZE 200000
22
23 #define COMMON_DATA_NAME U_ICUDATA_NAME
24 #define DATA_TYPE "dat"
25
26 /* ICU package data file format (.dat files) ------------------------------- ***
27
28 Description of the data format after the usual ICU data file header
29 (UDataInfo etc.).
30
31 Format version 1
32
33 A .dat package file contains a simple Table of Contents of item names,
34 followed by the items themselves:
35
36 1. ToC table
37
38 uint32_t count; - number of items
39 UDataOffsetTOCEntry entry[count]; - pair of uint32_t values per item:
40 uint32_t nameOffset; - offset of the item name
41 uint32_t dataOffset; - offset of the item data
42 both are byte offsets from the beginning of the data
43
44 2. item name strings
45
46 All item names are stored as char * strings in one block between the ToC table
47 and the data items.
48
49 3. data items
50
51 The data items are stored following the item names block.
52 Each data item is 16-aligned.
53 The data items are stored in the sorted order of their names.
54
55 Therefore, the top of the name strings block is the offset of the first item,
56 the length of the last item is the difference between its offset and
57 the .dat file length, and the length of all previous items is the difference
58 between its offset and the next one.
59
60 ----------------------------------------------------------------------------- */
61
62 /* UDataInfo cf. udata.h */
63 static const UDataInfo dataInfo={
64 sizeof(UDataInfo),
65 0,
66
67 U_IS_BIG_ENDIAN,
68 U_CHARSET_FAMILY,
69 sizeof(UChar),
70 0,
71
72 {0x43, 0x6d, 0x6e, 0x44}, /* dataFormat="CmnD" */
73 {1, 0, 0, 0}, /* formatVersion */
74 {3, 0, 0, 0} /* dataVersion */
75 };
76
77 static uint32_t maxSize;
78
79 static char stringStore[STRING_STORE_SIZE];
80 static uint32_t stringTop=0, basenameTotal=0;
81
82 typedef struct {
83 char *pathname, *basename;
84 uint32_t basenameLength, basenameOffset, fileSize, fileOffset;
85 } File;
86
87 #define CHUNK_FILE_COUNT 256
88 static File *files = NULL;
89 static uint32_t fileCount=0;
90 static uint32_t fileMax = 0;
91
92
93 static char *symPrefix = NULL;
94
95 #define LINE_BUFFER_SIZE 512
96 /* prototypes --------------------------------------------------------------- */
97
98 static void
99 addFile(const char *filename, const char *name, const char *source, UBool sourceTOC, UBool verbose);
100
101 static char *
102 allocString(uint32_t length);
103
104 static int
105 compareFiles(const void *file1, const void *file2);
106
107 static char *
108 pathToFullPath(const char *path, const char *source);
109
110 /* map non-tree separator (such as '\') to tree separator ('/') inplace. */
111 static void
112 fixDirToTreePath(char *s);
113 /* -------------------------------------------------------------------------- */
114
115 U_CAPI void U_EXPORT2
116 createCommonDataFile(const char *destDir, const char *name, const char *entrypointName, const char *type, const char *source, const char *copyRight,
117 const char *dataFile, uint32_t max_size, UBool sourceTOC, UBool verbose, char *gencmnFileName) {
118 static char buffer[4096];
119 char *line;
120 char *linePtr;
121 char *s = NULL;
122 UErrorCode errorCode=U_ZERO_ERROR;
123 uint32_t i, fileOffset, basenameOffset, length, nread;
124 FileStream *in, *file;
125
126 line = (char *)uprv_malloc(sizeof(char) * LINE_BUFFER_SIZE);
127 if (line == NULL) {
128 fprintf(stderr, "gencmn: unable to allocate memory for line buffer of size %d\n", LINE_BUFFER_SIZE);
129 exit(U_MEMORY_ALLOCATION_ERROR);
130 }
131
132 linePtr = line;
133
134 maxSize = max_size;
135
136 if (destDir == NULL) {
137 destDir = u_getDataDirectory();
138 }
139 if (name == NULL) {
140 name = COMMON_DATA_NAME;
141 }
142 if (type == NULL) {
143 type = DATA_TYPE;
144 }
145 if (source == NULL) {
146 source = ".";
147 }
148
149 if (dataFile == NULL) {
150 in = T_FileStream_stdin();
151 } else {
152 in = T_FileStream_open(dataFile, "r");
153 if(in == NULL) {
154 fprintf(stderr, "gencmn: unable to open input file %s\n", dataFile);
155 exit(U_FILE_ACCESS_ERROR);
156 }
157 }
158
159 if (verbose) {
160 if(sourceTOC) {
161 printf("generating %s_%s.c (table of contents source file)\n", name, type);
162 } else {
163 printf("generating %s.%s (common data file with table of contents)\n", name, type);
164 }
165 }
166
167 /* read the list of files and get their lengths */
168 while((s != NULL && *s != 0) || (s=T_FileStream_readLine(in, (line=linePtr),
169 LINE_BUFFER_SIZE))!=NULL) {
170 /* remove trailing newline characters and parse space separated items */
171 if (s != NULL && *s != 0) {
172 line=s;
173 } else {
174 s=line;
175 }
176 while(*s!=0) {
177 if(*s==' ') {
178 *s=0;
179 ++s;
180 break;
181 } else if(*s=='\r' || *s=='\n') {
182 *s=0;
183 break;
184 }
185 ++s;
186 }
187
188 /* check for comment */
189
190 if (*line == '#') {
191 continue;
192 }
193
194 /* add the file */
195 #if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR)
196 {
197 char *t;
198 while((t = uprv_strchr(line,U_FILE_ALT_SEP_CHAR))) {
199 *t = U_FILE_SEP_CHAR;
200 }
201 }
202 #endif
203 addFile(getLongPathname(line), name, source, sourceTOC, verbose);
204 }
205
206 uprv_free(linePtr);
207
208 if(in!=T_FileStream_stdin()) {
209 T_FileStream_close(in);
210 }
211
212 if(fileCount==0) {
213 fprintf(stderr, "gencmn: no files listed in %s\n", dataFile == NULL ? "<stdin>" : dataFile);
214 return;
215 }
216
217 /* sort the files by basename */
218 qsort(files, fileCount, sizeof(File), compareFiles);
219
220 if(!sourceTOC) {
221 UNewDataMemory *out;
222
223 /* determine the offsets of all basenames and files in this common one */
224 basenameOffset=4+8*fileCount;
225 fileOffset=(basenameOffset+(basenameTotal+15))&~0xf;
226 for(i=0; i<fileCount; ++i) {
227 files[i].fileOffset=fileOffset;
228 fileOffset+=(files[i].fileSize+15)&~0xf;
229 files[i].basenameOffset=basenameOffset;
230 basenameOffset+=files[i].basenameLength;
231 }
232
233 /* create the output file */
234 out=udata_create(destDir, type, name,
235 &dataInfo,
236 copyRight == NULL ? U_COPYRIGHT_STRING : copyRight,
237 &errorCode);
238 if(U_FAILURE(errorCode)) {
239 fprintf(stderr, "gencmn: udata_create(-d %s -n %s -t %s) failed - %s\n",
240 destDir, name, type,
241 u_errorName(errorCode));
242 exit(errorCode);
243 }
244
245 /* write the table of contents */
246 udata_write32(out, fileCount);
247 for(i=0; i<fileCount; ++i) {
248 udata_write32(out, files[i].basenameOffset);
249 udata_write32(out, files[i].fileOffset);
250 }
251
252 /* write the basenames */
253 for(i=0; i<fileCount; ++i) {
254 udata_writeString(out, files[i].basename, files[i].basenameLength);
255 }
256 length=4+8*fileCount+basenameTotal;
257
258 /* copy the files */
259 for(i=0; i<fileCount; ++i) {
260 /* pad to 16-align the next file */
261 length&=0xf;
262 if(length!=0) {
263 udata_writePadding(out, 16-length);
264 }
265
266 if (verbose) {
267 printf("adding %s (%ld byte%s)\n", files[i].pathname, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
268 }
269
270 /* copy the next file */
271 file=T_FileStream_open(files[i].pathname, "rb");
272 if(file==NULL) {
273 fprintf(stderr, "gencmn: unable to open listed file %s\n", files[i].pathname);
274 exit(U_FILE_ACCESS_ERROR);
275 }
276 for(nread = 0;;) {
277 length=T_FileStream_read(file, buffer, sizeof(buffer));
278 if(length <= 0) {
279 break;
280 }
281 nread += length;
282 udata_writeBlock(out, buffer, length);
283 }
284 T_FileStream_close(file);
285 length=files[i].fileSize;
286
287 if (nread != files[i].fileSize) {
288 fprintf(stderr, "gencmn: unable to read %s properly (got %ld/%ld byte%s)\n", files[i].pathname, (long)nread, (long)files[i].fileSize, files[i].fileSize == 1 ? "" : "s");
289 exit(U_FILE_ACCESS_ERROR);
290 }
291 }
292
293 /* pad to 16-align the last file (cleaner, avoids growing .dat files in icuswap) */
294 length&=0xf;
295 if(length!=0) {
296 udata_writePadding(out, 16-length);
297 }
298
299 /* finish */
300 udata_finish(out, &errorCode);
301 if(U_FAILURE(errorCode)) {
302 fprintf(stderr, "gencmn: udata_finish() failed - %s\n", u_errorName(errorCode));
303 exit(errorCode);
304 }
305 } else {
306 /* write a .c source file with the table of contents */
307 char *filename;
308 FileStream *out;
309
310 /* create the output filename */
311 filename=s=buffer;
312 uprv_strcpy(filename, destDir);
313 s=filename+uprv_strlen(filename);
314 if(s>filename && *(s-1)!=U_FILE_SEP_CHAR) {
315 *s++=U_FILE_SEP_CHAR;
316 }
317 uprv_strcpy(s, name);
318 if(*(type)!=0) {
319 s+=uprv_strlen(s);
320 *s++='_';
321 uprv_strcpy(s, type);
322 }
323 s+=uprv_strlen(s);
324 uprv_strcpy(s, ".c");
325
326 /* open the output file */
327 out=T_FileStream_open(filename, "w");
328 if (gencmnFileName != NULL) {
329 uprv_strcpy(gencmnFileName, filename);
330 }
331 if(out==NULL) {
332 fprintf(stderr, "gencmn: unable to open .c output file %s\n", filename);
333 exit(U_FILE_ACCESS_ERROR);
334 }
335
336 /* write the source file */
337 sprintf(buffer,
338 "/*\n"
339 " * ICU common data table of contents for %s.%s\n"
340 " * Automatically generated by icu/source/tools/gencmn/gencmn .\n"
341 " */\n\n"
342 "#include \"unicode/utypes.h\"\n"
343 "#include \"unicode/udata.h\"\n"
344 "\n"
345 "/* external symbol declarations for data (%d files) */\n",
346 name, type, fileCount);
347 T_FileStream_writeLine(out, buffer);
348
349 sprintf(buffer, "extern const char\n %s%s[]", symPrefix?symPrefix:"", files[0].pathname);
350 T_FileStream_writeLine(out, buffer);
351 for(i=1; i<fileCount; ++i) {
352 sprintf(buffer, ",\n %s%s[]", symPrefix?symPrefix:"", files[i].pathname);
353 T_FileStream_writeLine(out, buffer);
354 }
355 T_FileStream_writeLine(out, ";\n\n");
356
357 sprintf(
358 buffer,
359 "U_EXPORT struct {\n"
360 " uint16_t headerSize;\n"
361 " uint8_t magic1, magic2;\n"
362 " UDataInfo info;\n"
363 " char padding[%lu];\n"
364 " uint32_t count, reserved;\n"
365 " struct {\n"
366 " const char *name;\n"
367 " const void *data;\n"
368 " } toc[%lu];\n"
369 "} U_EXPORT2 %s_dat = {\n"
370 " 32, 0xda, 0x27, {\n"
371 " %lu, 0,\n"
372 " %u, %u, %u, 0,\n"
373 " {0x54, 0x6f, 0x43, 0x50},\n"
374 " {1, 0, 0, 0},\n"
375 " {0, 0, 0, 0}\n"
376 " },\n"
377 " \"\", %lu, 0, {\n",
378 (unsigned long)32-4-sizeof(UDataInfo),
379 (unsigned long)fileCount,
380 entrypointName,
381 (unsigned long)sizeof(UDataInfo),
382 U_IS_BIG_ENDIAN,
383 U_CHARSET_FAMILY,
384 U_SIZEOF_UCHAR,
385 (unsigned long)fileCount
386 );
387 T_FileStream_writeLine(out, buffer);
388
389 sprintf(buffer, " { \"%s\", %s%s }", files[0].basename, symPrefix?symPrefix:"", files[0].pathname);
390 T_FileStream_writeLine(out, buffer);
391 for(i=1; i<fileCount; ++i) {
392 sprintf(buffer, ",\n { \"%s\", %s%s }", files[i].basename, symPrefix?symPrefix:"", files[i].pathname);
393 T_FileStream_writeLine(out, buffer);
394 }
395
396 T_FileStream_writeLine(out, "\n }\n};\n");
397 T_FileStream_close(out);
398
399 uprv_free(symPrefix);
400 }
401 }
402
403 static void
404 addFile(const char *filename, const char *name, const char *source, UBool sourceTOC, UBool verbose) {
405 char *s;
406 uint32_t length;
407 char *fullPath = NULL;
408
409 if(fileCount==fileMax) {
410 fileMax += CHUNK_FILE_COUNT;
411 files = uprv_realloc(files, fileMax*sizeof(files[0])); /* note: never freed. */
412 if(files==NULL) {
413 fprintf(stderr, "pkgdata/gencmn: Could not allocate %u bytes for %d files\n", (unsigned int)(fileMax*sizeof(files[0])), fileCount);
414 exit(U_MEMORY_ALLOCATION_ERROR);
415 }
416 }
417
418 if(!sourceTOC) {
419 FileStream *file;
420
421 if(uprv_pathIsAbsolute(filename)) {
422 fprintf(stderr, "gencmn: Error: absolute path encountered. Old style paths are not supported. Use relative paths such as 'fur.res' or 'translit%cfur.res'.\n\tBad path: '%s'\n", U_FILE_SEP_CHAR, filename);
423 exit(U_ILLEGAL_ARGUMENT_ERROR);
424 }
425 fullPath = pathToFullPath(filename, source);
426 /* store the pathname */
427 length = (uint32_t)(uprv_strlen(filename) + 1 + uprv_strlen(name) + 1);
428 s=allocString(length);
429 uprv_strcpy(s, name);
430 uprv_strcat(s, U_TREE_ENTRY_SEP_STRING);
431 uprv_strcat(s, filename);
432
433 /* get the basename */
434 fixDirToTreePath(s);
435 files[fileCount].basename=s;
436 files[fileCount].basenameLength=length;
437
438 files[fileCount].pathname=fullPath;
439
440 basenameTotal+=length;
441
442 /* try to open the file */
443 file=T_FileStream_open(fullPath, "rb");
444 if(file==NULL) {
445 fprintf(stderr, "gencmn: unable to open listed file %s\n", fullPath);
446 exit(U_FILE_ACCESS_ERROR);
447 }
448
449 /* get the file length */
450 length=T_FileStream_size(file);
451 if(T_FileStream_error(file) || length<=20) {
452 fprintf(stderr, "gencmn: unable to get length of listed file %s\n", fullPath);
453 exit(U_FILE_ACCESS_ERROR);
454 }
455
456 T_FileStream_close(file);
457
458 /* do not add files that are longer than maxSize */
459 if(maxSize && length>maxSize) {
460 if (verbose) {
461 printf("%s ignored (size %ld > %ld)\n", fullPath, (long)length, (long)maxSize);
462 }
463 return;
464 }
465 files[fileCount].fileSize=length;
466 } else {
467 char *t;
468 /* get and store the basename */
469 /* need to include the package name */
470 length = (uint32_t)(uprv_strlen(filename) + 1 + uprv_strlen(name) + 1);
471 s=allocString(length);
472 uprv_strcpy(s, name);
473 uprv_strcat(s, U_TREE_ENTRY_SEP_STRING);
474 uprv_strcat(s, filename);
475 fixDirToTreePath(s);
476 files[fileCount].basename=s;
477 /* turn the basename into an entry point name and store in the pathname field */
478 t=files[fileCount].pathname=allocString(length);
479 while(--length>0) {
480 if(*s=='.' || *s=='-' || *s=='/') {
481 *t='_';
482 } else {
483 *t=*s;
484 }
485 ++s;
486 ++t;
487 }
488 *t=0;
489 }
490 ++fileCount;
491 }
492
493 static char *
494 allocString(uint32_t length) {
495 uint32_t top=stringTop+length;
496 char *p;
497
498 if(top>STRING_STORE_SIZE) {
499 fprintf(stderr, "gencmn: out of memory\n");
500 exit(U_MEMORY_ALLOCATION_ERROR);
501 }
502 p=stringStore+stringTop;
503 stringTop=top;
504 return p;
505 }
506
507 static char *
508 pathToFullPath(const char *path, const char *source) {
509 int32_t length;
510 int32_t newLength;
511 char *fullPath;
512 int32_t n;
513
514 length = (uint32_t)(uprv_strlen(path) + 1);
515 newLength = (length + 1 + (int32_t)uprv_strlen(source));
516 fullPath = uprv_malloc(newLength);
517 if(source != NULL) {
518 uprv_strcpy(fullPath, source);
519 uprv_strcat(fullPath, U_FILE_SEP_STRING);
520 } else {
521 fullPath[0] = 0;
522 }
523 n = (int32_t)uprv_strlen(fullPath);
524 fullPath[n] = 0; /* Suppress compiler warning for unused variable n */
525 /* when conditional code below is not compiled. */
526 uprv_strcat(fullPath, path);
527
528 #if (U_FILE_ALT_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR)
529 #if (U_FILE_ALT_SEP_CHAR != U_FILE_SEP_CHAR)
530 /* replace tree separator (such as '/') with file sep char (such as ':' or '\\') */
531 for(;fullPath[n];n++) {
532 if(fullPath[n] == U_FILE_ALT_SEP_CHAR) {
533 fullPath[n] = U_FILE_SEP_CHAR;
534 }
535 }
536 #endif
537 #endif
538 #if (U_FILE_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR)
539 /* replace tree separator (such as '/') with file sep char (such as ':' or '\\') */
540 for(;fullPath[n];n++) {
541 if(fullPath[n] == U_TREE_ENTRY_SEP_CHAR) {
542 fullPath[n] = U_FILE_SEP_CHAR;
543 }
544 }
545 #endif
546 return fullPath;
547 }
548
549 static int
550 compareFiles(const void *file1, const void *file2) {
551 /* sort by basename */
552 return uprv_strcmp(((File *)file1)->basename, ((File *)file2)->basename);
553 }
554
555 static void
556 fixDirToTreePath(char *s)
557 {
558 #if (U_FILE_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR) || ((U_FILE_ALT_SEP_CHAR != U_FILE_SEP_CHAR) && (U_FILE_ALT_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR))
559 char *t;
560 #endif
561 #if (U_FILE_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR)
562 for(t=s;t=uprv_strchr(t,U_FILE_SEP_CHAR);) {
563 *t = U_TREE_ENTRY_SEP_CHAR;
564 }
565 #endif
566 #if (U_FILE_ALT_SEP_CHAR != U_FILE_SEP_CHAR) && (U_FILE_ALT_SEP_CHAR != U_TREE_ENTRY_SEP_CHAR)
567 for(t=s;t=uprv_strchr(t,U_FILE_ALT_SEP_CHAR);) {
568 *t = U_TREE_ENTRY_SEP_CHAR;
569 }
570 #endif
571 }

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial