michael@0: /*
michael@0: **********************************************************************
michael@0: *   Copyright (C) 2002-2009, International Business Machines
michael@0: *   Corporation and others.  All Rights Reserved.
michael@0: **********************************************************************
michael@0: *
michael@0: * File genbrk.c
michael@0: */
michael@0: 
michael@0: //--------------------------------------------------------------------
michael@0: //
michael@0: //   Tool for generating RuleBasedBreakIterator data files (.brk files).
michael@0: //   .brk files contain the precompiled rules for standard types
michael@0: //   of iterators - word, line, sentence, etc.
michael@0: //
michael@0: //   Usage:  genbrk [options] -r rule-file.txt  -o output-file.brk
michael@0: //
michael@0: //       options:   -v         verbose
michael@0: //                  -? or -h   help
michael@0: //
michael@0: //   The input rule file is a plain text file containing break rules
michael@0: //    in the input format accepted by RuleBasedBreakIterators.  The
michael@0: //    file can be encoded as utf-8, or utf-16 (either endian), or
michael@0: //    in the default code page (platform dependent.).  utf encoded
michael@0: //    files must include a BOM.
michael@0: //
michael@0: //--------------------------------------------------------------------
michael@0: 
michael@0: #include "unicode/utypes.h"
michael@0: #include "unicode/ucnv.h"
michael@0: #include "unicode/unistr.h"
michael@0: #include "unicode/rbbi.h"
michael@0: #include "unicode/uclean.h"
michael@0: #include "unicode/udata.h"
michael@0: #include "unicode/putil.h"
michael@0: 
michael@0: #include "uoptions.h"
michael@0: #include "unewdata.h"
michael@0: #include "ucmndata.h"
michael@0: #include "rbbidata.h"
michael@0: #include "cmemory.h"
michael@0: 
michael@0: #include <stdio.h>
michael@0: #include <stdlib.h>
michael@0: #include <string.h>
michael@0: 
michael@0: U_NAMESPACE_USE
michael@0: 
michael@0: static char *progName;
michael@0: static UOption options[]={
michael@0:     UOPTION_HELP_H,             /* 0 */
michael@0:     UOPTION_HELP_QUESTION_MARK, /* 1 */
michael@0:     UOPTION_VERBOSE,            /* 2 */
michael@0:     { "rules", NULL, NULL, NULL, 'r', UOPT_REQUIRES_ARG, 0 },   /* 3 */
michael@0:     { "out",   NULL, NULL, NULL, 'o', UOPT_REQUIRES_ARG, 0 },   /* 4 */
michael@0:     UOPTION_ICUDATADIR,         /* 5 */
michael@0:     UOPTION_DESTDIR,            /* 6 */
michael@0:     UOPTION_COPYRIGHT,          /* 7 */
michael@0: };
michael@0: 
michael@0: void usageAndDie(int retCode) {
michael@0:         printf("Usage: %s [-v] [-options] -r rule-file -o output-file\n", progName);
michael@0:         printf("\tRead in break iteration rules text and write out the binary data\n"
michael@0:             "options:\n"
michael@0:             "\t-h or -? or --help  this usage text\n"
michael@0:             "\t-V or --version     show a version message\n"
michael@0:             "\t-c or --copyright   include a copyright notice\n"
michael@0:             "\t-v or --verbose     turn on verbose output\n"
michael@0:             "\t-i or --icudatadir  directory for locating any needed intermediate data files,\n"
michael@0:             "\t                    followed by path, defaults to %s\n"
michael@0:             "\t-d or --destdir     destination directory, followed by the path\n",
michael@0:             u_getDataDirectory());
michael@0:         exit (retCode);
michael@0: }
michael@0: 
michael@0: 
michael@0: #if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO
michael@0: 
michael@0: /* dummy UDataInfo cf. udata.h */
michael@0: static UDataInfo dummyDataInfo = {
michael@0:     sizeof(UDataInfo),
michael@0:     0,
michael@0: 
michael@0:     U_IS_BIG_ENDIAN,
michael@0:     U_CHARSET_FAMILY,
michael@0:     U_SIZEOF_UCHAR,
michael@0:     0,
michael@0: 
michael@0:     { 0, 0, 0, 0 },                 /* dummy dataFormat */
michael@0:     { 0, 0, 0, 0 },                 /* dummy formatVersion */
michael@0:     { 0, 0, 0, 0 }                  /* dummy dataVersion */
michael@0: };
michael@0: 
michael@0: #else
michael@0: 
michael@0: //
michael@0: //  Set up the ICU data header, defined in ucmndata.h
michael@0: //
michael@0: DataHeader dh ={
michael@0:     {sizeof(DataHeader),           // Struct MappedData
michael@0:         0xda,
michael@0:         0x27},
michael@0: 
michael@0:     {                               // struct UDataInfo
michael@0:         sizeof(UDataInfo),          //     size
michael@0:         0,                          //     reserved
michael@0:         U_IS_BIG_ENDIAN,
michael@0:         U_CHARSET_FAMILY,
michael@0:         U_SIZEOF_UCHAR,
michael@0:         0,                          //     reserved
michael@0: 
michael@0:     { 0x42, 0x72, 0x6b, 0x20 },     //     dataFormat="Brk "
michael@0:     { 0xff, 0, 0, 0 },              //     formatVersion.  Filled in later with values
michael@0:                                     //      from the RBBI rule builder.  The  values declared
michael@0:                                     //      here should never appear in any real RBBI data.
michael@0:         { 4, 1, 0, 0 }              //   dataVersion (Unicode version)
michael@0:     }};
michael@0: 
michael@0: #endif
michael@0: 
michael@0: //----------------------------------------------------------------------------
michael@0: //
michael@0: //  main      for genbrk
michael@0: //
michael@0: //----------------------------------------------------------------------------
michael@0: int  main(int argc, char **argv) {
michael@0:     UErrorCode  status = U_ZERO_ERROR;
michael@0:     const char *ruleFileName;
michael@0:     const char *outFileName;
michael@0:     const char *outDir = NULL;
michael@0:     const char *copyright = NULL;
michael@0: 
michael@0:     //
michael@0:     // Pick up and check the command line arguments,
michael@0:     //    using the standard ICU tool utils option handling.
michael@0:     //
michael@0:     U_MAIN_INIT_ARGS(argc, argv);
michael@0:     progName = argv[0];
michael@0:     argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
michael@0:     if(argc<0) {
michael@0:         // Unrecognized option
michael@0:         fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
michael@0:         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
michael@0:     }
michael@0: 
michael@0:     if(options[0].doesOccur || options[1].doesOccur) {
michael@0:         //  -? or -h for help.
michael@0:         usageAndDie(0);
michael@0:     }
michael@0: 
michael@0:     if (!(options[3].doesOccur && options[4].doesOccur)) {
michael@0:         fprintf(stderr, "rule file and output file must both be specified.\n");
michael@0:         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
michael@0:     }
michael@0:     ruleFileName = options[3].value;
michael@0:     outFileName  = options[4].value;
michael@0: 
michael@0:     if (options[5].doesOccur) {
michael@0:         u_setDataDirectory(options[5].value);
michael@0:     }
michael@0: 
michael@0:     status = U_ZERO_ERROR;
michael@0: 
michael@0:     /* Combine the directory with the file name */
michael@0:     if(options[6].doesOccur) {
michael@0:         outDir = options[6].value;
michael@0:     }
michael@0:     if (options[7].doesOccur) {
michael@0:         copyright = U_COPYRIGHT_STRING;
michael@0:     }
michael@0: 
michael@0: #if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO
michael@0: 
michael@0:     UNewDataMemory *pData;
michael@0:     char msg[1024];
michael@0: 
michael@0:     /* write message with just the name */
michael@0:     sprintf(msg, "genbrk writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName);
michael@0:     fprintf(stderr, "%s\n", msg);
michael@0: 
michael@0:     /* write the dummy data file */
michael@0:     pData = udata_create(outDir, NULL, outFileName, &dummyDataInfo, NULL, &status);
michael@0:     udata_writeBlock(pData, msg, strlen(msg));
michael@0:     udata_finish(pData, &status);
michael@0:     return (int)status;
michael@0: 
michael@0: #else
michael@0:     /* Initialize ICU */
michael@0:     u_init(&status);
michael@0:     if (U_FAILURE(status)) {
michael@0:         fprintf(stderr, "%s: can not initialize ICU.  status = %s\n",
michael@0:             argv[0], u_errorName(status));
michael@0:         exit(1);
michael@0:     }
michael@0:     status = U_ZERO_ERROR;
michael@0: 
michael@0:     //
michael@0:     //  Read in the rule source file
michael@0:     //
michael@0:     long        result;
michael@0:     long        ruleFileSize;
michael@0:     FILE        *file;
michael@0:     char        *ruleBufferC;
michael@0: 
michael@0:     file = fopen(ruleFileName, "rb");
michael@0:     if( file == 0 ) {
michael@0:         fprintf(stderr, "Could not open file \"%s\"\n", ruleFileName);
michael@0:         exit(-1);
michael@0:     }
michael@0:     fseek(file, 0, SEEK_END);
michael@0:     ruleFileSize = ftell(file);
michael@0:     fseek(file, 0, SEEK_SET);
michael@0:     ruleBufferC = new char[ruleFileSize+10];
michael@0: 
michael@0:     result = (long)fread(ruleBufferC, 1, ruleFileSize, file);
michael@0:     if (result != ruleFileSize)  {
michael@0:         fprintf(stderr, "Error reading file \"%s\"\n", ruleFileName);
michael@0:         exit (-1);
michael@0:     }
michael@0:     ruleBufferC[ruleFileSize]=0;
michael@0:     fclose(file);
michael@0: 
michael@0:     //
michael@0:     // Look for a Unicode Signature (BOM) on the rule file
michael@0:     //
michael@0:     int32_t        signatureLength;
michael@0:     const char *   ruleSourceC = ruleBufferC;
michael@0:     const char*    encoding = ucnv_detectUnicodeSignature(
michael@0:                            ruleSourceC, ruleFileSize, &signatureLength, &status);
michael@0:     if (U_FAILURE(status)) {
michael@0:         exit(status);
michael@0:     }
michael@0:     if(encoding!=NULL ){
michael@0:         ruleSourceC  += signatureLength;
michael@0:         ruleFileSize -= signatureLength;
michael@0:     }
michael@0: 
michael@0:     //
michael@0:     // Open a converter to take the rule file to UTF-16
michael@0:     //
michael@0:     UConverter* conv;
michael@0:     conv = ucnv_open(encoding, &status);
michael@0:     if (U_FAILURE(status)) {
michael@0:         fprintf(stderr, "ucnv_open: ICU Error \"%s\"\n", u_errorName(status));
michael@0:         exit(status);
michael@0:     }
michael@0: 
michael@0:     //
michael@0:     // Convert the rules to UChar.
michael@0:     //  Preflight first to determine required buffer size.
michael@0:     //
michael@0:     uint32_t destCap = ucnv_toUChars(conv,
michael@0:                        NULL,           //  dest,
michael@0:                        0,              //  destCapacity,
michael@0:                        ruleSourceC,
michael@0:                        ruleFileSize,
michael@0:                        &status);
michael@0:     if (status != U_BUFFER_OVERFLOW_ERROR) {
michael@0:         fprintf(stderr, "ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
michael@0:         exit(status);
michael@0:     };
michael@0: 
michael@0:     status = U_ZERO_ERROR;
michael@0:     UChar *ruleSourceU = new UChar[destCap+1];
michael@0:     ucnv_toUChars(conv,
michael@0:                   ruleSourceU,     //  dest,
michael@0:                   destCap+1,
michael@0:                   ruleSourceC,
michael@0:                   ruleFileSize,
michael@0:                   &status);
michael@0:     if (U_FAILURE(status)) {
michael@0:         fprintf(stderr, "ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
michael@0:         exit(status);
michael@0:     };
michael@0:     ucnv_close(conv);
michael@0: 
michael@0: 
michael@0:     //
michael@0:     //  Put the source rules into a UnicodeString
michael@0:     //
michael@0:     UnicodeString ruleSourceS(FALSE, ruleSourceU, destCap);
michael@0: 
michael@0:     //
michael@0:     //  Create the break iterator from the rules
michael@0:     //     This will compile the rules.
michael@0:     //
michael@0:     UParseError parseError;
michael@0:     parseError.line = 0;
michael@0:     parseError.offset = 0;
michael@0:     RuleBasedBreakIterator *bi = new RuleBasedBreakIterator(ruleSourceS, parseError, status);
michael@0:     if (U_FAILURE(status)) {
michael@0:         fprintf(stderr, "createRuleBasedBreakIterator: ICU Error \"%s\"  at line %d, column %d\n",
michael@0:                 u_errorName(status), (int)parseError.line, (int)parseError.offset);
michael@0:         exit(status);
michael@0:     };
michael@0: 
michael@0: 
michael@0:     //
michael@0:     //  Get the compiled rule data from the break iterator.
michael@0:     //
michael@0:     uint32_t        outDataSize;
michael@0:     const uint8_t  *outData;
michael@0:     outData = bi->getBinaryRules(outDataSize);
michael@0: 
michael@0:     // Copy the data format version numbers from the RBBI data header into the UDataMemory header.
michael@0:     uprv_memcpy(dh.info.formatVersion, ((RBBIDataHeader *)outData)->fFormatVersion, sizeof(dh.info.formatVersion));
michael@0: 
michael@0:     //
michael@0:     //  Create the output file
michael@0:     //
michael@0:     size_t bytesWritten;
michael@0:     UNewDataMemory *pData;
michael@0:     pData = udata_create(outDir, NULL, outFileName, &(dh.info), copyright, &status);
michael@0:     if(U_FAILURE(status)) {
michael@0:         fprintf(stderr, "genbrk: Could not open output file \"%s\", \"%s\"\n", 
michael@0:                          outFileName, u_errorName(status));
michael@0:         exit(status);
michael@0:     }
michael@0: 
michael@0: 
michael@0:     //  Write the data itself.
michael@0:     udata_writeBlock(pData, outData, outDataSize);
michael@0:     // finish up 
michael@0:     bytesWritten = udata_finish(pData, &status);
michael@0:     if(U_FAILURE(status)) {
michael@0:         fprintf(stderr, "genbrk: error %d writing the output file\n", status);
michael@0:         exit(status);
michael@0:     }
michael@0:     
michael@0:     if (bytesWritten != outDataSize) {
michael@0:         fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName);
michael@0:         exit(-1);
michael@0:     }
michael@0: 
michael@0:     delete bi;
michael@0:     delete[] ruleSourceU;
michael@0:     delete[] ruleBufferC;
michael@0:     u_cleanup();
michael@0: 
michael@0: 
michael@0:     printf("genbrk: tool completed successfully.\n");
michael@0:     return 0;
michael@0: 
michael@0: #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
michael@0: }
michael@0: