The Tor Browser: intl/icu/source/tools/toolutil/ucm.h@6474c204b198 (annotated)

intl/icu/source/tools/toolutil/ucm.h@6474c204b198 (annotated)

intl/icu/source/tools/toolutil/ucm.h

Wed, 31 Dec 2014 06:09:35 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 06:09:35 +0100
changeset 0: 6474c204b198
permissions: -rw-r--r--

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

 /*
  *******************************************************************************
  *   Copyright (C) 2003-2013, International Business Machines
  *   Corporation and others.  All Rights Reserved.
  *******************************************************************************
  *   file name:  ucm.h
  *   encoding:   US-ASCII
  *   tab size:   8 (not used)
  *   indentation:4
  *
  *   created on: 2003jun20
  *   created by: Markus W. Scherer
  *
  *   Definitions for the .ucm file parser and handler module ucm.c.
  */
 #ifndef __UCM_H__
 #define __UCM_H__
 #include "unicode/utypes.h"
 #include "ucnvmbcs.h"
 #include "ucnv_ext.h"
 #include "filestrm.h"
 #include <stdio.h>
 #if !UCONFIG_NO_CONVERSION
 U_CDECL_BEGIN
 /* constants for UCMapping.moveFlag */
 enum {
     UCM_MOVE_TO_EXT=1,
     UCM_REMOVE_MAPPING=2
 };
 /*
  * Per-mapping data structure
  *
  * u if uLen==1: Unicode code point
  *   else index to uLen code points
  * b if bLen<=4: up to 4 bytes
  *   else index to bLen bytes
  * uLen number of code points
  * bLen number of words containing left-justified bytes
  * bIsMultipleChars indicates that the bytes contain more than one sequence
  *                  according to the state table
  * f flag for roundtrip (0), fallback (1), sub mapping (2), reverse fallback (3)
  *   or "good one-way" mapping (4).
  *   Same values as in the source file after |
  */
 typedef struct UCMapping {
     UChar32 u;
     union {
         uint32_t idx;
         uint8_t bytes[4];
     } b;
     int8_t uLen, bLen, f, moveFlag;
 } UCMapping;
 /* constants for UCMTable.flagsType */
 enum {
     UCM_FLAGS_INITIAL,  /* no mappings parsed yet */
     UCM_FLAGS_EXPLICIT, /* .ucm file has mappings with | fallback indicators */
     UCM_FLAGS_IMPLICIT, /* .ucm file has mappings without | fallback indicators, later wins */
     UCM_FLAGS_MIXED     /* both implicit and explicit */
 };
 typedef struct UCMTable {
     UCMapping *mappings;
     int32_t mappingsCapacity, mappingsLength;
     UChar32 *codePoints;
     int32_t codePointsCapacity, codePointsLength;
     uint8_t *bytes;
     int32_t bytesCapacity, bytesLength;
     /* index map for mapping by bytes first */
     int32_t *reverseMap;
     uint8_t unicodeMask;
     int8_t flagsType; /* UCM_FLAGS_INITIAL etc. */
     UBool isSorted;
 } UCMTable;
 enum {
     MBCS_STATE_FLAG_DIRECT=1,
     MBCS_STATE_FLAG_SURROGATES,
     MBCS_STATE_FLAG_READY=16
 };
 typedef struct UCMStates {
     int32_t stateTable[MBCS_MAX_STATE_COUNT][256];
     uint32_t stateFlags[MBCS_MAX_STATE_COUNT],
              stateOffsetSum[MBCS_MAX_STATE_COUNT];
     int32_t countStates, minCharLength, maxCharLength, countToUCodeUnits;
     int8_t conversionType, outputType;
 } UCMStates;
 typedef struct UCMFile {
     UCMTable *base, *ext;
     UCMStates states;
     char baseName[UCNV_MAX_CONVERTER_NAME_LENGTH];
 } UCMFile;
 /* simple accesses ---------------------------------------------------------- */
 #define UCM_GET_CODE_POINTS(t, m) \
     (((m)->uLen==1) ? &(m)->u : (t)->codePoints+(m)->u)
 #define UCM_GET_BYTES(t, m) \
     (((m)->bLen<=4) ? (m)->b.bytes : (t)->bytes+(m)->b.idx)
 /* APIs --------------------------------------------------------------------- */
 U_CAPI UCMFile * U_EXPORT2
 ucm_open(void);
 U_CAPI void U_EXPORT2
 ucm_close(UCMFile *ucm);
 U_CAPI UBool U_EXPORT2
 ucm_parseHeaderLine(UCMFile *ucm,
                     char *line, char **pKey, char **pValue);
 /* @return -1 illegal bytes  0 suitable for base table  1 needs to go into extension table */
 U_CAPI int32_t U_EXPORT2
 ucm_mappingType(UCMStates *baseStates,
                 UCMapping *m,
                 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
                 uint8_t bytes[UCNV_EXT_MAX_BYTES]);
 /* add a mapping to the base or extension table as appropriate */
 U_CAPI UBool U_EXPORT2
 ucm_addMappingAuto(UCMFile *ucm, UBool forBase, UCMStates *baseStates,
                    UCMapping *m,
                    UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
                    uint8_t bytes[UCNV_EXT_MAX_BYTES]);
 U_CAPI UBool U_EXPORT2
 ucm_addMappingFromLine(UCMFile *ucm, const char *line, UBool forBase, UCMStates *baseStates);
 U_CAPI UCMTable * U_EXPORT2
 ucm_openTable(void);
 U_CAPI void U_EXPORT2
 ucm_closeTable(UCMTable *table);
 U_CAPI void U_EXPORT2
 ucm_resetTable(UCMTable *table);
 U_CAPI void U_EXPORT2
 ucm_sortTable(UCMTable *t);
 /*
  * Remove mappings with their move flag set from the base table
  * and move some of them (with UCM_MOVE_TO_EXT) to the extension table.
  */
 U_CAPI void U_EXPORT2
 ucm_moveMappings(UCMTable *base, UCMTable *ext);
 /**
  * Read a table from a .ucm file, from after the CHARMAP line to
  * including the END CHARMAP line.
  */
 U_CAPI void U_EXPORT2
 ucm_readTable(UCMFile *ucm, FileStream* convFile,
               UBool forBase, UCMStates *baseStates,
               UErrorCode *pErrorCode);
 /**
  * Check the validity of mappings against a base table's states;
  * necessary for extension-only tables that were read before their base tables.
  */
 U_CAPI UBool U_EXPORT2
 ucm_checkValidity(UCMTable *ext, UCMStates *baseStates);
 /**
  * Check a base table against an extension table.
  * Set the moveTarget!=NULL if it is possible to move mappings from the base.
  * This is the case where base and extension tables are parsed from a single file
  * (moveTarget==ext)
  * or when delta file mappings are subtracted from a base table.
  *
  * When a base table cannot be modified because a delta file is parsed in makeconv,
  * then set moveTarget=NULL.
  *
  * if(intersectBase) then mappings that exist in the base table but not in
  * the extension table are moved to moveTarget instead of showing an error.
  *
  * Special mode:
  * If intersectBase==2 for a DBCS extension table, then SBCS mappings are
  * not moved out of the base unless their Unicode input requires it.
  * This helps ucmkbase generate base tables for DBCS-only extension .cnv files.
  *
  * For both tables in the same file, the extension table is automatically
  * built.
  * For separate files, the extension file can use a complete mapping table (.ucm file),
  * so that common mappings need not be stripped out manually.
  *
  *
  * Sort both tables, and then for each mapping direction:
  *
  * If intersectBase is TRUE and the base table contains a mapping
  * that does not exist in the extension table, then this mapping is moved
  * to moveTarget.
  *
  * - otherwise -
  *
  * If the base table contains a mapping for which the input sequence is
  * the same as the extension input, then
  * - if the output is the same: remove the extension mapping
  * - else: error
  *
  * If the base table contains a mapping for which the input sequence is
  * a prefix of the extension input, then
  * - if moveTarget!=NULL: move the base mapping to the moveTarget table
  * - else: error
  *
  * @return FALSE in case of an irreparable error
  */
 U_CAPI UBool U_EXPORT2
 ucm_checkBaseExt(UCMStates *baseStates, UCMTable *base, UCMTable *ext,
                  UCMTable *moveTarget, UBool intersectBase);
 U_CAPI void U_EXPORT2
 ucm_printTable(UCMTable *table, FILE *f, UBool byUnicode);
 U_CAPI void U_EXPORT2
 ucm_printMapping(UCMTable *table, UCMapping *m, FILE *f);
 U_CAPI void U_EXPORT2
 ucm_addState(UCMStates *states, const char *s);
 U_CAPI void U_EXPORT2
 ucm_processStates(UCMStates *states, UBool ignoreSISOCheck);
 U_CAPI int32_t U_EXPORT2
 ucm_countChars(UCMStates *states,
                const uint8_t *bytes, int32_t length);
 U_CAPI int8_t U_EXPORT2
 ucm_parseBytes(uint8_t bytes[UCNV_EXT_MAX_BYTES], const char *line, const char **ps);
 U_CAPI UBool U_EXPORT2
 ucm_parseMappingLine(UCMapping *m,
                      UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
                      uint8_t bytes[UCNV_EXT_MAX_BYTES],
                      const char *line);
 U_CAPI void U_EXPORT2
 ucm_addMapping(UCMTable *table,
                UCMapping *m,
                UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
                uint8_t bytes[UCNV_EXT_MAX_BYTES]);
 /* very makeconv-specific functions ----------------------------------------- */
 /* finalize and optimize states after the toUnicode mappings are processed */
 U_CAPI void U_EXPORT2
 ucm_optimizeStates(UCMStates *states,
                    uint16_t **pUnicodeCodeUnits,
                    _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks,
                    UBool verbose);
 /* moved here because it is used inside ucmstate.c */
 U_CAPI int32_t U_EXPORT2
 ucm_findFallback(_MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks,
                  uint32_t offset);
 /* very rptp2ucm-specific functions ----------------------------------------- */
 /*
  * Input: Separate tables with mappings from/to Unicode,
  * subchar and subchar1 (0 if none).
  * All mappings must have flag 0.
  *
  * Output: fromUTable will contain the union of mappings with the correct
  * precision flags, and be sorted.
  */
 U_CAPI void U_EXPORT2
 ucm_mergeTables(UCMTable *fromUTable, UCMTable *toUTable,
                 const uint8_t *subchar, int32_t subcharLength,
                 uint8_t subchar1);
 U_CAPI UBool U_EXPORT2
 ucm_separateMappings(UCMFile *ucm, UBool isSISO);
 U_CDECL_END
 #endif
 #endif

The Tor Browser / annotate

intl/icu/source/tools/toolutil/ucm.h@6474c204b198 (annotated)

intl/icu/source/tools/toolutil/ucm.h