michael@0: /*
michael@0: **********************************************************************
michael@0: *   Copyright (C) 2002-2013, International Business Machines
michael@0: *   Corporation and others.  All Rights Reserved.
michael@0: **********************************************************************
michael@0: *
michael@0: * File gendict.cpp
michael@0: */
michael@0: 
michael@0: #include "unicode/utypes.h"
michael@0: #include "unicode/uchar.h"
michael@0: #include "unicode/ucnv.h"
michael@0: #include "unicode/uniset.h"
michael@0: #include "unicode/unistr.h"
michael@0: #include "unicode/uclean.h"
michael@0: #include "unicode/udata.h"
michael@0: #include "unicode/putil.h"
michael@0: #include "unicode/ucharstriebuilder.h"
michael@0: #include "unicode/bytestriebuilder.h"
michael@0: #include "unicode/ucharstrie.h"
michael@0: #include "unicode/bytestrie.h"
michael@0: #include "unicode/ucnv.h"
michael@0: #include "unicode/utf16.h"
michael@0: 
michael@0: #include "charstr.h"
michael@0: #include "dictionarydata.h"
michael@0: #include "uoptions.h"
michael@0: #include "unewdata.h"
michael@0: #include "cmemory.h"
michael@0: #include "uassert.h"
michael@0: #include "ucbuf.h"
michael@0: #include "toolutil.h"
michael@0: #include "cstring.h"
michael@0: 
michael@0: #include <stdio.h>
michael@0: #include <stdlib.h>
michael@0: #include <string.h>
michael@0: 
michael@0: #include "putilimp.h"
michael@0: UDate startTime;
michael@0: 
michael@0: static int elapsedTime() {
michael@0:   return (int)uprv_floor((uprv_getRawUTCtime()-startTime)/1000.0);
michael@0: }
michael@0: 
michael@0: #if U_PLATFORM_IMPLEMENTS_POSIX && !U_PLATFORM_HAS_WIN32_API
michael@0: 
michael@0: #include <signal.h>
michael@0: #include <unistd.h>
michael@0: 
michael@0: const char *wToolname="gendict";
michael@0: const char *wOutname="(some file)";
michael@0: 
michael@0: const int firstSeconds = 5; /* seconds between notices*/
michael@0: const int nextSeconds = 15; /* seconds between notices*/
michael@0: 
michael@0: static void alarm_fn(int /*n*/) {
michael@0:   printf("%s: still writing\t%s (%ds)\t...\n",    wToolname, wOutname, elapsedTime());
michael@0:   
michael@0:   signal(SIGALRM, &alarm_fn);
michael@0:   alarm(nextSeconds); // reset the alarm
michael@0: }
michael@0: 
michael@0: static void install_watchdog(const char *toolName, const char *outFileName) {
michael@0:   wToolname=toolName;
michael@0:   wOutname=outFileName;
michael@0: 
michael@0:   signal(SIGALRM, &alarm_fn);
michael@0: 
michael@0:   alarm(firstSeconds); // set the alarm
michael@0: }
michael@0: 
michael@0: #else
michael@0: static void install_watchdog(const char*, const char*) {
michael@0:   // not implemented
michael@0: }
michael@0: #endif
michael@0: 
michael@0: 
michael@0: 
michael@0: 
michael@0: U_NAMESPACE_USE
michael@0: 
michael@0: static char *progName;
michael@0: static UOption options[]={
michael@0:     UOPTION_HELP_H,             /* 0 */
michael@0:     UOPTION_HELP_QUESTION_MARK, /* 1 */
michael@0:     UOPTION_VERBOSE,            /* 2 */
michael@0:     UOPTION_ICUDATADIR,         /* 4 */
michael@0:     UOPTION_COPYRIGHT,          /* 5 */
michael@0:     { "uchars", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 6 */
michael@0:     { "bytes", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 7 */
michael@0:     { "transform", NULL, NULL, NULL, '\1', UOPT_REQUIRES_ARG, 0}, /* 8 */
michael@0: };
michael@0: 
michael@0: enum arguments {
michael@0:     ARG_HELP = 0,
michael@0:     ARG_QMARK,
michael@0:     ARG_VERBOSE,
michael@0:     ARG_ICUDATADIR,
michael@0:     ARG_COPYRIGHT,
michael@0:     ARG_UCHARS,
michael@0:     ARG_BYTES,
michael@0:     ARG_TRANSFORM
michael@0: };
michael@0: 
michael@0: // prints out the standard usage method describing command line arguments, 
michael@0: // then bails out with the desired exit code
michael@0: static void usageAndDie(UErrorCode retCode) {
michael@0:     fprintf((U_SUCCESS(retCode) ? stdout : stderr), "Usage: %s -trietype [-options] input-dictionary-file output-file\n", progName);
michael@0:     fprintf((U_SUCCESS(retCode) ? stdout : stderr),
michael@0:            "\tRead in a word list and write out a string trie dictionary\n"
michael@0:            "options:\n"
michael@0:            "\t-h or -? or --help  this usage text\n"
michael@0:            "\t-V or --version     show a version message\n"
michael@0:            "\t-c or --copyright   include a copyright notice\n"
michael@0:            "\t-v or --verbose     turn on verbose output\n"
michael@0:            "\t-i or --icudatadir  directory for locating any needed intermediate data files,\n" // TODO: figure out if we need this option
michael@0:            "\t                    followed by path, defaults to %s\n"
michael@0:            "\t--uchars            output a UCharsTrie (mutually exclusive with -b!)\n"
michael@0:            "\t--bytes             output a BytesTrie (mutually exclusive with -u!)\n"
michael@0:            "\t--transform         the kind of transform to use (eg --transform offset-40A3,\n"
michael@0:            "\t                    which specifies an offset transform with constant 0x40A3)\n",
michael@0:             u_getDataDirectory());
michael@0:     exit(retCode);
michael@0: }
michael@0: 
michael@0: 
michael@0: /* UDataInfo cf. udata.h */
michael@0: static UDataInfo dataInfo = {
michael@0:     sizeof(UDataInfo),
michael@0:     0,
michael@0: 
michael@0:     U_IS_BIG_ENDIAN,
michael@0:     U_CHARSET_FAMILY,
michael@0:     U_SIZEOF_UCHAR,
michael@0:     0,
michael@0: 
michael@0:     { 0x44, 0x69, 0x63, 0x74 },     /* "Dict" */
michael@0:     { 1, 0, 0, 0 },                 /* format version */
michael@0:     { 0, 0, 0, 0 }                  /* data version */
michael@0: };
michael@0: 
michael@0: #if !UCONFIG_NO_BREAK_ITERATION
michael@0: 
michael@0: // A wrapper for both BytesTrieBuilder and UCharsTrieBuilder.
michael@0: // may want to put this somewhere in ICU, as it could be useful outside
michael@0: // of this tool?
michael@0: class DataDict {
michael@0: private:
michael@0:     BytesTrieBuilder *bt;
michael@0:     UCharsTrieBuilder *ut;
michael@0:     UChar32 transformConstant;
michael@0:     int32_t transformType;
michael@0: public:
michael@0:     // constructs a new data dictionary. if there is an error, 
michael@0:     // it will be returned in status
michael@0:     // isBytesTrie != 0 will produce a BytesTrieBuilder,
michael@0:     // isBytesTrie == 0 will produce a UCharsTrieBuilder
michael@0:     DataDict(UBool isBytesTrie, UErrorCode &status) : bt(NULL), ut(NULL), 
michael@0:         transformConstant(0), transformType(DictionaryData::TRANSFORM_NONE) {
michael@0:         if (isBytesTrie) {
michael@0:             bt = new BytesTrieBuilder(status);
michael@0:         } else {
michael@0:             ut = new UCharsTrieBuilder(status);
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     ~DataDict() {
michael@0:         delete bt;
michael@0:         delete ut;
michael@0:     }
michael@0: 
michael@0: private:
michael@0:     char transform(UChar32 c, UErrorCode &status) {
michael@0:         if (transformType == DictionaryData::TRANSFORM_TYPE_OFFSET) {
michael@0:             if (c == 0x200D) { return (char)0xFF; }
michael@0:             else if (c == 0x200C) { return (char)0xFE; }
michael@0:             int32_t delta = c - transformConstant;
michael@0:             if (delta < 0 || 0xFD < delta) {
michael@0:                 fprintf(stderr, "Codepoint U+%04lx out of range for --transform offset-%04lx!\n",
michael@0:                         (long)c, (long)transformConstant);
michael@0:                 exit(U_ILLEGAL_ARGUMENT_ERROR); // TODO: should return and print the line number
michael@0:             }
michael@0:             return (char)delta;
michael@0:         } else { // no such transform type 
michael@0:             status = U_INTERNAL_PROGRAM_ERROR;
michael@0:             return (char)c; // it should be noted this transform type will not generally work
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     void transform(const UnicodeString &word, CharString &buf, UErrorCode &errorCode) {
michael@0:         UChar32 c = 0;
michael@0:         int32_t len = word.length();
michael@0:         for (int32_t i = 0; i < len; i += U16_LENGTH(c)) {
michael@0:             c = word.char32At(i);
michael@0:             buf.append(transform(c, errorCode), errorCode);
michael@0:         }
michael@0:     }
michael@0: 
michael@0: public:
michael@0:     // sets the desired transformation data.
michael@0:     // should be populated from a command line argument
michael@0:     // so far the only acceptable format is offset-<hex constant>
michael@0:     // eventually others (mask-<hex constant>?) may be enabled
michael@0:     // more complex functions may be more difficult
michael@0:     void setTransform(const char *t) {
michael@0:         if (strncmp(t, "offset-", 7) == 0) {
michael@0:             char *end;
michael@0:             unsigned long base = uprv_strtoul(t + 7, &end, 16);
michael@0:             if (end == (t + 7) || *end != 0 || base > 0x10FF80) {
michael@0:                 fprintf(stderr, "Syntax for offset value in --transform offset-%s invalid!\n", t + 7);
michael@0:                 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
michael@0:             }
michael@0:             transformType = DictionaryData::TRANSFORM_TYPE_OFFSET;
michael@0:             transformConstant = (UChar32)base;
michael@0:         }
michael@0:         else {
michael@0:             fprintf(stderr, "Invalid transform specified: %s\n", t);
michael@0:             usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     // add a word to the trie
michael@0:     void addWord(const UnicodeString &word, int32_t value, UErrorCode &status) {
michael@0:         if (bt) {
michael@0:             CharString buf;
michael@0:             transform(word, buf, status);
michael@0:             bt->add(buf.toStringPiece(), value, status);
michael@0:         }
michael@0:         if (ut) { ut->add(word, value, status); }
michael@0:     }
michael@0: 
michael@0:     // if we are a bytestrie, give back the StringPiece representing the serialized version of us
michael@0:     StringPiece serializeBytes(UErrorCode &status) {
michael@0:         return bt->buildStringPiece(USTRINGTRIE_BUILD_SMALL, status);
michael@0:     }
michael@0: 
michael@0:     // if we are a ucharstrie, produce the UnicodeString representing the serialized version of us
michael@0:     void serializeUChars(UnicodeString &s, UErrorCode &status) {
michael@0:         ut->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, s, status);
michael@0:     }
michael@0: 
michael@0:     int32_t getTransform() {
michael@0:         return (int32_t)(transformType | transformConstant); 
michael@0:     }
michael@0: };
michael@0: #endif
michael@0: 
michael@0: static const UChar LINEFEED_CHARACTER = 0x000A;
michael@0: static const UChar CARRIAGE_RETURN_CHARACTER = 0x000D;
michael@0: 
michael@0: static UBool readLine(UCHARBUF *f, UnicodeString &fileLine, IcuToolErrorCode &errorCode) {
michael@0:     int32_t lineLength;
michael@0:     const UChar *line = ucbuf_readline(f, &lineLength, errorCode);
michael@0:     if(line == NULL || errorCode.isFailure()) { return FALSE; }
michael@0:     // Strip trailing CR/LF, comments, and spaces.
michael@0:     const UChar *comment = u_memchr(line, 0x23, lineLength);  // '#'
michael@0:     if(comment != NULL) {
michael@0:         lineLength = (int32_t)(comment - line);
michael@0:     } else {
michael@0:         while(lineLength > 0 && (line[lineLength - 1] == CARRIAGE_RETURN_CHARACTER || line[lineLength - 1] == LINEFEED_CHARACTER)) { --lineLength; }
michael@0:     }
michael@0:     while(lineLength > 0 && u_isspace(line[lineLength - 1])) { --lineLength; }
michael@0:     fileLine.setTo(FALSE, line, lineLength);
michael@0:     return TRUE;
michael@0: }
michael@0: 
michael@0: //----------------------------------------------------------------------------
michael@0: //
michael@0: //  main      for gendict
michael@0: //
michael@0: //----------------------------------------------------------------------------
michael@0: int  main(int argc, char **argv) {
michael@0:     //
michael@0:     // Pick up and check the command line arguments,
michael@0:     //    using the standard ICU tool utils option handling.
michael@0:     //
michael@0:     U_MAIN_INIT_ARGS(argc, argv);
michael@0:     progName = argv[0];
michael@0:     argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
michael@0:     if(argc<0) {
michael@0:         // Unrecognized option
michael@0:         fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
michael@0:         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
michael@0:     }
michael@0: 
michael@0:     if(options[ARG_HELP].doesOccur || options[ARG_QMARK].doesOccur) {
michael@0:         //  -? or -h for help.
michael@0:         usageAndDie(U_ZERO_ERROR);
michael@0:     }
michael@0: 
michael@0:     UBool verbose = options[ARG_VERBOSE].doesOccur;
michael@0: 
michael@0:     if (argc < 3) {
michael@0:         fprintf(stderr, "input and output file must both be specified.\n");
michael@0:         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
michael@0:     }
michael@0:     const char *outFileName  = argv[2];
michael@0:     const char *wordFileName = argv[1];
michael@0: 
michael@0:     startTime = uprv_getRawUTCtime(); // initialize start timer
michael@0:     // set up the watchdog
michael@0:     install_watchdog(progName, outFileName);
michael@0: 
michael@0:     if (options[ARG_ICUDATADIR].doesOccur) {
michael@0:         u_setDataDirectory(options[ARG_ICUDATADIR].value);
michael@0:     }
michael@0: 
michael@0:     const char *copyright = NULL;
michael@0:     if (options[ARG_COPYRIGHT].doesOccur) {
michael@0:         copyright = U_COPYRIGHT_STRING;
michael@0:     }
michael@0: 
michael@0:     if (options[ARG_UCHARS].doesOccur == options[ARG_BYTES].doesOccur) {
michael@0:         fprintf(stderr, "you must specify exactly one type of trie to output!\n");
michael@0:         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
michael@0:     }
michael@0:     UBool isBytesTrie = options[ARG_BYTES].doesOccur;
michael@0:     if (isBytesTrie != options[ARG_TRANSFORM].doesOccur) {
michael@0:         fprintf(stderr, "you must provide a transformation for a bytes trie, and must not provide one for a uchars trie!\n");
michael@0:         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
michael@0:     }
michael@0: 
michael@0:     IcuToolErrorCode status("gendict/main()");
michael@0: 
michael@0: #if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO
michael@0:     const char* outDir=NULL;
michael@0: 
michael@0:     UNewDataMemory *pData;
michael@0:     char msg[1024];
michael@0:     UErrorCode tempstatus = U_ZERO_ERROR;
michael@0: 
michael@0:     /* write message with just the name */ // potential for a buffer overflow here...
michael@0:     sprintf(msg, "gendict writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName);
michael@0:     fprintf(stderr, "%s\n", msg);
michael@0: 
michael@0:     /* write the dummy data file */
michael@0:     pData = udata_create(outDir, NULL, outFileName, &dataInfo, NULL, &tempstatus);
michael@0:     udata_writeBlock(pData, msg, strlen(msg));
michael@0:     udata_finish(pData, &tempstatus);
michael@0:     return (int)tempstatus;
michael@0: 
michael@0: #else
michael@0:     //  Read in the dictionary source file
michael@0:     if (verbose) { printf("Opening file %s...\n", wordFileName); }
michael@0:     const char *codepage = "UTF-8";
michael@0:     UCHARBUF *f = ucbuf_open(wordFileName, &codepage, TRUE, FALSE, status);
michael@0:     if (status.isFailure()) {
michael@0:         fprintf(stderr, "error opening input file: ICU Error \"%s\"\n", status.errorName());
michael@0:         exit(status.reset());
michael@0:     }
michael@0:     if (verbose) { printf("Initializing dictionary builder of type %s...\n", (isBytesTrie ? "BytesTrie" : "UCharsTrie")); }
michael@0:     DataDict dict(isBytesTrie, status);
michael@0:     if (status.isFailure()) {
michael@0:         fprintf(stderr, "new DataDict: ICU Error \"%s\"\n", status.errorName());
michael@0:         exit(status.reset());
michael@0:     }
michael@0:     if (options[ARG_TRANSFORM].doesOccur) {
michael@0:         dict.setTransform(options[ARG_TRANSFORM].value);
michael@0:     }
michael@0: 
michael@0:     UnicodeString fileLine;
michael@0:     if (verbose) { puts("Adding words to dictionary..."); }
michael@0:     UBool hasValues = FALSE;
michael@0:     UBool hasValuelessContents = FALSE;
michael@0:     int lineCount = 0;
michael@0:     int wordCount = 0;
michael@0:     int minlen = 255;
michael@0:     int maxlen = 0;
michael@0:     UBool isOk = TRUE;
michael@0:     while (readLine(f, fileLine, status)) {
michael@0:         lineCount++;
michael@0:         if (fileLine.isEmpty()) continue;
michael@0:         
michael@0:         // Parse word [spaces value].
michael@0:         int32_t keyLen;
michael@0:         for (keyLen = 0; keyLen < fileLine.length() && !u_isspace(fileLine[keyLen]); ++keyLen) {}
michael@0:         if (keyLen == 0) {
michael@0:             fprintf(stderr, "Error: no word on line %i!\n", lineCount);
michael@0:             isOk = FALSE;
michael@0:             continue;
michael@0:         }
michael@0:         int32_t valueStart;
michael@0:         for (valueStart = keyLen;
michael@0:             valueStart < fileLine.length() && u_isspace(fileLine[valueStart]);
michael@0:             ++valueStart) {}
michael@0: 
michael@0:         if (keyLen < valueStart) {
michael@0:             int32_t valueLength = fileLine.length() - valueStart;
michael@0:             if (valueLength > 15) {
michael@0:                 fprintf(stderr, "Error: value too long on line %i!\n", lineCount);
michael@0:                 isOk = FALSE;
michael@0:                 continue;
michael@0:             }
michael@0:             char s[16];
michael@0:             fileLine.extract(valueStart, valueLength, s, 16, US_INV);
michael@0:             char *end;
michael@0:             unsigned long value = uprv_strtoul(s, &end, 0);
michael@0:             if (end == s || *end != 0 || (int32_t)uprv_strlen(s) != valueLength || value > 0xffffffff) {
michael@0:                 fprintf(stderr, "Error: value syntax error or value too large on line %i!\n", lineCount);
michael@0:                 isOk = FALSE;
michael@0:                 continue;
michael@0:             }
michael@0:             dict.addWord(fileLine.tempSubString(0, keyLen), (int32_t)value, status);
michael@0:             hasValues = TRUE;
michael@0:             wordCount++;
michael@0:             if (keyLen < minlen) minlen = keyLen;
michael@0:             if (keyLen > maxlen) maxlen = keyLen;
michael@0:         } else {
michael@0:             dict.addWord(fileLine.tempSubString(0, keyLen), 0, status);
michael@0:             hasValuelessContents = TRUE;
michael@0:             wordCount++;
michael@0:             if (keyLen < minlen) minlen = keyLen;
michael@0:             if (keyLen > maxlen) maxlen = keyLen;
michael@0:         }
michael@0: 
michael@0:         if (status.isFailure()) {
michael@0:             fprintf(stderr, "ICU Error \"%s\": Failed to add word to trie at input line %d in input file\n",
michael@0:                 status.errorName(), lineCount);
michael@0:             exit(status.reset());
michael@0:         }
michael@0:     }
michael@0:     if (verbose) { printf("Processed %d lines, added %d words, minlen %d, maxlen %d\n", lineCount, wordCount, minlen, maxlen); }
michael@0: 
michael@0:     if (!isOk && status.isSuccess()) {
michael@0:         status.set(U_ILLEGAL_ARGUMENT_ERROR);
michael@0:     }
michael@0:     if (hasValues && hasValuelessContents) {
michael@0:         fprintf(stderr, "warning: file contained both valued and unvalued strings!\n");
michael@0:     }
michael@0: 
michael@0:     if (verbose) { printf("Serializing data...isBytesTrie? %d\n", isBytesTrie); }
michael@0:     int32_t outDataSize;
michael@0:     const void *outData;
michael@0:     UnicodeString usp;
michael@0:     if (isBytesTrie) {
michael@0:         StringPiece sp = dict.serializeBytes(status);
michael@0:         outDataSize = sp.size();
michael@0:         outData = sp.data();
michael@0:     } else {
michael@0:         dict.serializeUChars(usp, status);
michael@0:         outDataSize = usp.length() * U_SIZEOF_UCHAR;
michael@0:         outData = usp.getBuffer();
michael@0:     }
michael@0:     if (status.isFailure()) {
michael@0:         fprintf(stderr, "gendict: got failure of type %s while serializing, if U_ILLEGAL_ARGUMENT_ERROR possibly due to duplicate dictionary entries\n", status.errorName());
michael@0:         exit(status.reset());
michael@0:     }
michael@0:     if (verbose) { puts("Opening output file..."); }
michael@0:     UNewDataMemory *pData = udata_create(NULL, NULL, outFileName, &dataInfo, copyright, status);
michael@0:     if (status.isFailure()) {
michael@0:         fprintf(stderr, "gendict: could not open output file \"%s\", \"%s\"\n", outFileName, status.errorName());
michael@0:         exit(status.reset());
michael@0:     }
michael@0: 
michael@0:     if (verbose) { puts("Writing to output file..."); }
michael@0:     int32_t indexes[DictionaryData::IX_COUNT] = {
michael@0:         DictionaryData::IX_COUNT * sizeof(int32_t), 0, 0, 0, 0, 0, 0, 0
michael@0:     };
michael@0:     int32_t size = outDataSize + indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
michael@0:     indexes[DictionaryData::IX_RESERVED1_OFFSET] = size;
michael@0:     indexes[DictionaryData::IX_RESERVED2_OFFSET] = size;
michael@0:     indexes[DictionaryData::IX_TOTAL_SIZE] = size;
michael@0: 
michael@0:     indexes[DictionaryData::IX_TRIE_TYPE] = isBytesTrie ? DictionaryData::TRIE_TYPE_BYTES : DictionaryData::TRIE_TYPE_UCHARS;
michael@0:     if (hasValues) {
michael@0:         indexes[DictionaryData::IX_TRIE_TYPE] |= DictionaryData::TRIE_HAS_VALUES;
michael@0:     }
michael@0: 
michael@0:     indexes[DictionaryData::IX_TRANSFORM] = dict.getTransform();
michael@0:     udata_writeBlock(pData, indexes, sizeof(indexes));
michael@0:     udata_writeBlock(pData, outData, outDataSize);
michael@0:     size_t bytesWritten = udata_finish(pData, status);
michael@0:     if (status.isFailure()) {
michael@0:         fprintf(stderr, "gendict: error \"%s\" writing the output file\n", status.errorName());
michael@0:         exit(status.reset());
michael@0:     }
michael@0: 
michael@0:     if (bytesWritten != (size_t)size) {
michael@0:         fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName);
michael@0:         exit(U_INTERNAL_PROGRAM_ERROR);
michael@0:     }
michael@0: 
michael@0:     printf("%s: done writing\t%s (%ds).\n", progName, outFileName, elapsedTime());
michael@0: 
michael@0: #ifdef TEST_GENDICT
michael@0:     if (isBytesTrie) {
michael@0:         BytesTrie::Iterator it(outData, outDataSize, status);
michael@0:         while (it.hasNext()) {
michael@0:             it.next(status);
michael@0:             const StringPiece s = it.getString();
michael@0:             int32_t val = it.getValue();
michael@0:             printf("%s -> %i\n", s.data(), val);
michael@0:         }
michael@0:     } else {
michael@0:         UCharsTrie::Iterator it((const UChar *)outData, outDataSize, status);
michael@0:         while (it.hasNext()) {
michael@0:             it.next(status);
michael@0:             const UnicodeString s = it.getString();
michael@0:             int32_t val = it.getValue();
michael@0:             char tmp[1024];
michael@0:             s.extract(0, s.length(), tmp, 1024);
michael@0:             printf("%s -> %i\n", tmp, val);
michael@0:         }
michael@0:     }
michael@0: #endif
michael@0: 
michael@0:     return 0;
michael@0: #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
michael@0: }