The Tor Browser: intl/icu/source/common/rbbidata.cpp@b8a032363ba2 (annotated)

intl/icu/source/common/rbbidata.cpp@b8a032363ba2 (annotated)

intl/icu/source/common/rbbidata.cpp

Thu, 22 Jan 2015 13:21:57 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Thu, 22 Jan 2015 13:21:57 +0100
branch: TOR_BUG_9701
changeset 15: b8a032363ba2
permissions: -rw-r--r--

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

 /*
 ***************************************************************************
 *   Copyright (C) 1999-2010 International Business Machines Corporation   *
 *   and others. All rights reserved.                                      *
 ***************************************************************************
 */
 #include "unicode/utypes.h"
 #if !UCONFIG_NO_BREAK_ITERATION
 #include "unicode/utypes.h"
 #include "rbbidata.h"
 #include "rbbirb.h"
 #include "utrie.h"
 #include "udatamem.h"
 #include "cmemory.h"
 #include "cstring.h"
 #include "umutex.h"
 #include "uassert.h"
 //-----------------------------------------------------------------------------------
 //
 //   Trie access folding function.  Copied as-is from properties code in uchar.c
 //
 //-----------------------------------------------------------------------------------
 U_CDECL_BEGIN
 static int32_t U_CALLCONV
 getFoldingOffset(uint32_t data) {
     /* if bit 15 is set, then the folding offset is in bits 14..0 of the 16-bit trie result */
     if(data&0x8000) {
         return (int32_t)(data&0x7fff);
     } else {
         return 0;
     }
 }
 U_CDECL_END
 U_NAMESPACE_BEGIN
 //-----------------------------------------------------------------------------
 //
 //    Constructors.
 //
 //-----------------------------------------------------------------------------
 RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, UErrorCode &status) {
     init(data, status);
 }
 RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, enum EDontAdopt, UErrorCode &status) {
     init(data, status);
     fDontFreeData = TRUE;
 }
 RBBIDataWrapper::RBBIDataWrapper(UDataMemory* udm, UErrorCode &status) {
     const RBBIDataHeader *d = (const RBBIDataHeader *)
         // ((char *)&(udm->pHeader->info) + udm->pHeader->info.size);
         // taking into consideration the padding added in by udata_write
         ((char *)(udm->pHeader) + udm->pHeader->dataHeader.headerSize);
     init(d, status);
     fUDataMem = udm;
 }
 //-----------------------------------------------------------------------------
 //
 //    init().   Does most of the work of construction, shared between the
 //              constructors.
 //
 //-----------------------------------------------------------------------------
 void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) {
     if (U_FAILURE(status)) {
         return;
     }
     fHeader = data;
     if (fHeader->fMagic != 0xb1a0 || fHeader->fFormatVersion[0] != 3)
     {
         status = U_INVALID_FORMAT_ERROR;
         return;
     }
     // Note: in ICU version 3.2 and earlier, there was a formatVersion 1
     //       that is no longer supported.  At that time fFormatVersion was
     //       an int32_t field, rather than an array of 4 bytes.
     fDontFreeData = FALSE;
     fUDataMem     = NULL;
     fReverseTable = NULL;
     fSafeFwdTable = NULL;
     fSafeRevTable = NULL;
     if (data->fFTableLen != 0) {
         fForwardTable = (RBBIStateTable *)((char *)data + fHeader->fFTable);
     }
     if (data->fRTableLen != 0) {
         fReverseTable = (RBBIStateTable *)((char *)data + fHeader->fRTable);
     }
     if (data->fSFTableLen != 0) {
         fSafeFwdTable = (RBBIStateTable *)((char *)data + fHeader->fSFTable);
     }
     if (data->fSRTableLen != 0) {
         fSafeRevTable = (RBBIStateTable *)((char *)data + fHeader->fSRTable);
     }
     utrie_unserialize(&fTrie,
                        (uint8_t *)data + fHeader->fTrie,
                        fHeader->fTrieLen,
                        &status);
     if (U_FAILURE(status)) {
         return;
     }
     fTrie.getFoldingOffset=getFoldingOffset;
     fRuleSource   = (UChar *)((char *)data + fHeader->fRuleSource);
     fRuleString.setTo(TRUE, fRuleSource, -1);
     U_ASSERT(data->fRuleSourceLen > 0);
     fRuleStatusTable = (int32_t *)((char *)data + fHeader->fStatusTable);
     fStatusMaxIdx    = data->fStatusTableLen / sizeof(int32_t);
     fRefCount = 1;
 #ifdef RBBI_DEBUG
     char *debugEnv = getenv("U_RBBIDEBUG");
     if (debugEnv && uprv_strstr(debugEnv, "data")) {this->printData();}
 #endif
 }
 //-----------------------------------------------------------------------------
 //
 //    Destructor.     Don't call this - use removeReference() instead.
 //
 //-----------------------------------------------------------------------------
 RBBIDataWrapper::~RBBIDataWrapper() {
     U_ASSERT(fRefCount == 0);
     if (fUDataMem) {
         udata_close(fUDataMem);
     } else if (!fDontFreeData) {
         uprv_free((void *)fHeader);
     }
 }
 //-----------------------------------------------------------------------------
 //
 //   Operator ==    Consider two RBBIDataWrappers to be equal if they
 //                  refer to the same underlying data.  Although
 //                  the data wrappers are normally shared between
 //                  iterator instances, it's possible to independently
 //                  open the same data twice, and get two instances, which
 //                  should still be ==.
 //
 //-----------------------------------------------------------------------------
 UBool RBBIDataWrapper::operator ==(const RBBIDataWrapper &other) const {
     if (fHeader == other.fHeader) {
         return TRUE;
     }
     if (fHeader->fLength != other.fHeader->fLength) {
         return FALSE;
     }
     if (uprv_memcmp(fHeader, other.fHeader, fHeader->fLength) == 0) {
         return TRUE;
     }
     return FALSE;
 }
 int32_t  RBBIDataWrapper::hashCode() {
     return fHeader->fFTableLen;
 }
 //-----------------------------------------------------------------------------
 //
 //    Reference Counting.   A single RBBIDataWrapper object is shared among
 //                          however many RulesBasedBreakIterator instances are
 //                          referencing the same data.
 //
 //-----------------------------------------------------------------------------
 void RBBIDataWrapper::removeReference() {
     if (umtx_atomic_dec(&fRefCount) == 0) {
         delete this;
     }
 }
 RBBIDataWrapper *RBBIDataWrapper::addReference() {
    umtx_atomic_inc(&fRefCount);
    return this;
 }
 //-----------------------------------------------------------------------------
 //
 //  getRuleSourceString
 //
 //-----------------------------------------------------------------------------
 const UnicodeString &RBBIDataWrapper::getRuleSourceString() const {
     return fRuleString;
 }
 //-----------------------------------------------------------------------------
 //
 //  print   -  debugging function to dump the runtime data tables.
 //
 //-----------------------------------------------------------------------------
 #ifdef RBBI_DEBUG
 void  RBBIDataWrapper::printTable(const char *heading, const RBBIStateTable *table) {
     uint32_t   c;
     uint32_t   s;
     RBBIDebugPrintf("   %s\n", heading);
     RBBIDebugPrintf("State |  Acc  LA TagIx");
     for (c=0; c<fHeader->fCatCount; c++) {RBBIDebugPrintf("%3d ", c);}
     RBBIDebugPrintf("\n------|---------------"); for (c=0;c<fHeader->fCatCount; c++) {
         RBBIDebugPrintf("----");
     }
     RBBIDebugPrintf("\n");
     if (table == NULL) {
         RBBIDebugPrintf("         N U L L   T A B L E\n\n");
         return;
     }
     for (s=0; s<table->fNumStates; s++) {
         RBBIStateTableRow *row = (RBBIStateTableRow *)
                                   (table->fTableData + (table->fRowLen * s));
         RBBIDebugPrintf("%4d  |  %3d %3d %3d ", s, row->fAccepting, row->fLookAhead, row->fTagIdx);
         for (c=0; c<fHeader->fCatCount; c++)  {
             RBBIDebugPrintf("%3d ", row->fNextState[c]);
         }
         RBBIDebugPrintf("\n");
     }
     RBBIDebugPrintf("\n");
 }
 #endif
 #ifdef RBBI_DEBUG
 void  RBBIDataWrapper::printData() {
     RBBIDebugPrintf("RBBI Data at %p\n", (void *)fHeader);
     RBBIDebugPrintf("   Version = {%d %d %d %d}\n", fHeader->fFormatVersion[0], fHeader->fFormatVersion[1],
                                                     fHeader->fFormatVersion[2], fHeader->fFormatVersion[3]);
     RBBIDebugPrintf("   total length of data  = %d\n", fHeader->fLength);
     RBBIDebugPrintf("   number of character categories = %d\n\n", fHeader->fCatCount);
     printTable("Forward State Transition Table", fForwardTable);
     printTable("Reverse State Transition Table", fReverseTable);
     printTable("Safe Forward State Transition Table", fSafeFwdTable);
     printTable("Safe Reverse State Transition Table", fSafeRevTable);
     RBBIDebugPrintf("\nOrignal Rules source:\n");
     for (int32_t c=0; fRuleSource[c] != 0; c++) {
         RBBIDebugPrintf("%c", fRuleSource[c]);
     }
     RBBIDebugPrintf("\n\n");
 }
 #endif
 U_NAMESPACE_END
 U_NAMESPACE_USE
 //-----------------------------------------------------------------------------
 //
 //  ubrk_swap   -  byte swap and char encoding swap of RBBI data
 //
 //-----------------------------------------------------------------------------
 U_CAPI int32_t U_EXPORT2
 ubrk_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
            UErrorCode *status) {
     if (status == NULL || U_FAILURE(*status)) {
         return 0;
     }
     if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) {
         *status=U_ILLEGAL_ARGUMENT_ERROR;
         return 0;
     }
     //
     //  Check that the data header is for for break data.
     //    (Header contents are defined in genbrk.cpp)
     //
     const UDataInfo *pInfo = (const UDataInfo *)((const char *)inData+4);
     if(!(  pInfo->dataFormat[0]==0x42 &&   /* dataFormat="Brk " */
            pInfo->dataFormat[1]==0x72 &&
            pInfo->dataFormat[2]==0x6b &&
            pInfo->dataFormat[3]==0x20 &&
            pInfo->formatVersion[0]==3  )) {
         udata_printError(ds, "ubrk_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized\n",
                          pInfo->dataFormat[0], pInfo->dataFormat[1],
                          pInfo->dataFormat[2], pInfo->dataFormat[3],
                          pInfo->formatVersion[0]);
         *status=U_UNSUPPORTED_ERROR;
         return 0;
     }
     //
     // Swap the data header.  (This is the generic ICU Data Header, not the RBBI Specific
     //                         RBBIDataHeader).  This swap also conveniently gets us
     //                         the size of the ICU d.h., which lets us locate the start
     //                         of the RBBI specific data.
     //
     int32_t headerSize=udata_swapDataHeader(ds, inData, length, outData, status);
     //
     // Get the RRBI Data Header, and check that it appears to be OK.
     //
     //    Note:  ICU 3.2 and earlier, RBBIDataHeader::fDataFormat was actually
     //           an int32_t with a value of 1.  Starting with ICU 3.4,
     //           RBBI's fDataFormat matches the dataFormat field from the
     //           UDataInfo header, four int8_t bytes.  The value is {3,1,0,0}
     //
     const uint8_t  *inBytes =(const uint8_t *)inData+headerSize;
     RBBIDataHeader *rbbiDH = (RBBIDataHeader *)inBytes;
     if (ds->readUInt32(rbbiDH->fMagic) != 0xb1a0 ||
         rbbiDH->fFormatVersion[0] != 3 ||
         ds->readUInt32(rbbiDH->fLength)  <  sizeof(RBBIDataHeader))
     {
         udata_printError(ds, "ubrk_swap(): RBBI Data header is invalid.\n");
         *status=U_UNSUPPORTED_ERROR;
         return 0;
     }
     //
     // Prefight operation?  Just return the size
     //
     int32_t breakDataLength = ds->readUInt32(rbbiDH->fLength);
     int32_t totalSize = headerSize + breakDataLength;
     if (length < 0) {
         return totalSize;
     }
     //
     // Check that length passed in is consistent with length from RBBI data header.
     //
     if (length < totalSize) {
         udata_printError(ds, "ubrk_swap(): too few bytes (%d after ICU Data header) for break data.\n",
                             breakDataLength);
         *status=U_INDEX_OUTOFBOUNDS_ERROR;
         return 0;
         }
     //
     // Swap the Data.  Do the data itself first, then the RBBI Data Header, because
     //                 we need to reference the header to locate the data, and an
     //                 inplace swap of the header leaves it unusable.
     //
     uint8_t         *outBytes = (uint8_t *)outData + headerSize;
     RBBIDataHeader  *outputDH = (RBBIDataHeader *)outBytes;
     int32_t   tableStartOffset;
     int32_t   tableLength;
     //
     // If not swapping in place, zero out the output buffer before starting.
     //    Individual tables and other data items within are aligned to 8 byte boundaries
     //    when originally created.  Any unused space between items needs to be zero.
     //
     if (inBytes != outBytes) {
         uprv_memset(outBytes, 0, breakDataLength);
     }
     //
     // Each state table begins with several 32 bit fields.  Calculate the size
     //   in bytes of these.
     //
     int32_t         topSize = offsetof(RBBIStateTable, fTableData);
     // Forward state table.
     tableStartOffset = ds->readUInt32(rbbiDH->fFTable);
     tableLength      = ds->readUInt32(rbbiDH->fFTableLen);
     if (tableLength > 0) {
         ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
                             outBytes+tableStartOffset, status);
         ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
                             outBytes+tableStartOffset+topSize, status);
     }
     // Reverse state table.  Same layout as forward table, above.
     tableStartOffset = ds->readUInt32(rbbiDH->fRTable);
     tableLength      = ds->readUInt32(rbbiDH->fRTableLen);
     if (tableLength > 0) {
         ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
                             outBytes+tableStartOffset, status);
         ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
                             outBytes+tableStartOffset+topSize, status);
     }
     // Safe Forward state table.  Same layout as forward table, above.
     tableStartOffset = ds->readUInt32(rbbiDH->fSFTable);
     tableLength      = ds->readUInt32(rbbiDH->fSFTableLen);
     if (tableLength > 0) {
         ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
                             outBytes+tableStartOffset, status);
         ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
                             outBytes+tableStartOffset+topSize, status);
     }
     // Safe Reverse state table.  Same layout as forward table, above.
     tableStartOffset = ds->readUInt32(rbbiDH->fSRTable);
     tableLength      = ds->readUInt32(rbbiDH->fSRTableLen);
     if (tableLength > 0) {
         ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
                             outBytes+tableStartOffset, status);
         ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
                             outBytes+tableStartOffset+topSize, status);
     }
     // Trie table for character categories
     utrie_swap(ds, inBytes+ds->readUInt32(rbbiDH->fTrie), ds->readUInt32(rbbiDH->fTrieLen),
                             outBytes+ds->readUInt32(rbbiDH->fTrie), status);
     // Source Rules Text.  It's UChar data
     ds->swapArray16(ds, inBytes+ds->readUInt32(rbbiDH->fRuleSource), ds->readUInt32(rbbiDH->fRuleSourceLen),
                         outBytes+ds->readUInt32(rbbiDH->fRuleSource), status);
     // Table of rule status values.  It's all int_32 values
     ds->swapArray32(ds, inBytes+ds->readUInt32(rbbiDH->fStatusTable), ds->readUInt32(rbbiDH->fStatusTableLen),
                         outBytes+ds->readUInt32(rbbiDH->fStatusTable), status);
     // And, last, the header.
     //   It is all int32_t values except for fFormataVersion, which is an array of four bytes.
     //   Swap the whole thing as int32_t, then re-swap the one field.
     //
     ds->swapArray32(ds, inBytes, sizeof(RBBIDataHeader), outBytes, status);
     ds->swapArray32(ds, outputDH->fFormatVersion, 4, outputDH->fFormatVersion, status);
     return totalSize;
 }
 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */

The Tor Browser / annotate

intl/icu/source/common/rbbidata.cpp@b8a032363ba2 (annotated)

intl/icu/source/common/rbbidata.cpp