The Tor Browser / file revision

 /*

 **********************************************************************

 *   Copyright (C) 1999-2011, International Business Machines

 *   Corporation and others.  All Rights Reserved.

 **********************************************************************

 *

 *   ucnv_cnv.h:

 *   Definitions for converter implementations.

 *

 * Modification History:

 *

 *   Date        Name        Description

 *   05/09/00    helena      Added implementation to handle fallback mappings.

 *   06/29/2000  helena      Major rewrite of the callback APIs.

 */

 #ifndef UCNV_CNV_H

 #define UCNV_CNV_H

 #include "unicode/utypes.h"

 #if !UCONFIG_NO_CONVERSION

 #include "unicode/ucnv.h"

 #include "unicode/ucnv_err.h"

 #include "unicode/uset.h"

 #include "uset_imp.h"

 U_CDECL_BEGIN

 /* this is used in fromUnicode DBCS tables as an "unassigned" marker */

 #define missingCharMarker 0xFFFF

 /*

  * #define missingUCharMarker 0xfffe

  *

  * commented out because there are actually two values used in toUnicode tables:

  * U+fffe "unassigned"

  * U+ffff "illegal"

  */

 /** Forward declaration, see ucnv_bld.h */

 struct UConverterSharedData;

 typedef struct UConverterSharedData UConverterSharedData;

 /* function types for UConverterImpl ---------------------------------------- */

 /* struct with arguments for UConverterLoad and ucnv_load() */

 typedef struct {

     int32_t size;               /* sizeof(UConverterLoadArgs) */

     int32_t nestedLoads;        /* count nested ucnv_load() calls */

     UBool onlyTestIsLoadable;   /* input: don't actually load */

     UBool reserved0;            /* reserved - for good alignment of the pointers */

     int16_t reserved;           /* reserved - for good alignment of the pointers */

     uint32_t options;

     const char *pkg, *name, *locale;

 } UConverterLoadArgs;

 #define UCNV_LOAD_ARGS_INITIALIZER \

     { (int32_t)sizeof(UConverterLoadArgs), 0, FALSE, FALSE, 0, 0, NULL, NULL, NULL }

 typedef void (*UConverterLoad) (UConverterSharedData *sharedData,

                                 UConverterLoadArgs *pArgs,

                                 const uint8_t *raw, UErrorCode *pErrorCode);

 typedef void (*UConverterUnload) (UConverterSharedData *sharedData);

 typedef void (*UConverterOpen) (UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *pErrorCode);

 typedef void (*UConverterClose) (UConverter *cnv);

 typedef enum UConverterResetChoice {

     UCNV_RESET_BOTH,

     UCNV_RESET_TO_UNICODE,

     UCNV_RESET_FROM_UNICODE

 } UConverterResetChoice;

 typedef void (*UConverterReset) (UConverter *cnv, UConverterResetChoice choice);

 /*

  * Converter implementation function(s) for ucnv_toUnicode().

  * If the toUnicodeWithOffsets function pointer is NULL,

  * then the toUnicode function will be used and the offsets will be set to -1.

  *

  * Must maintain state across buffers. Use toUBytes[toULength] for partial input

  * sequences; it will be checked in ucnv.c at the end of the input stream

  * to detect truncated input.

  * Some converters may need additional detection and may then set U_TRUNCATED_CHAR_FOUND.

  *

  * The toUnicodeWithOffsets must write exactly as many offset values as target

  * units. Write offset values of -1 for when the source index corresponding to

  * the output unit is not known (e.g., the character started in an earlier buffer).

  * The pArgs->offsets pointer need not be moved forward.

  *

  * At function return, either one of the following conditions must be true:

  * - U_BUFFER_OVERFLOW_ERROR and the target is full: target==targetLimit

  * - another error code with toUBytes[toULength] set to the offending input

  * - no error, and the source is consumed: source==sourceLimit

  *

  * The ucnv.c code will handle the end of the input (reset)

  * (reset, and truncation detection) and callbacks.

  */

 typedef void (*UConverterToUnicode) (UConverterToUnicodeArgs *, UErrorCode *);

 /*

  * Same rules as for UConverterToUnicode.

  * A lead surrogate is kept in fromUChar32 across buffers, and if an error

  * occurs, then the offending input code point must be put into fromUChar32

  * as well.

  */

 typedef void (*UConverterFromUnicode) (UConverterFromUnicodeArgs *, UErrorCode *);

 /*

  * Converter implementation function for ucnv_convertEx(), for direct conversion

  * between two charsets without pivoting through UTF-16.

  * The rules are the same as for UConverterToUnicode and UConverterFromUnicode.

  * In addition,

  * - The toUnicode side must behave and keep state exactly like the

  *   UConverterToUnicode implementation for the same source charset.

  * - A U_USING_DEFAULT_WARNING can be set to request to temporarily fall back

  *   to pivoting. When this function is called, the conversion framework makes

  *   sure that this warning is not set on input.

  * - Continuing a partial match and flushing the toUnicode replay buffer

  *   are handled by pivoting, using the toUnicode and fromUnicode functions.

  */

 typedef void (*UConverterConvert) (UConverterFromUnicodeArgs *pFromUArgs,

                                    UConverterToUnicodeArgs *pToUArgs,

                                    UErrorCode *pErrorCode);

 /*

  * Converter implementation function for ucnv_getNextUChar().

  * If the function pointer is NULL, then the toUnicode function will be used.

  *

  * Will be called at a character boundary (toULength==0).

  * May return with

  * - U_INDEX_OUTOFBOUNDS_ERROR if there was no output for the input

  *   (the return value will be ignored)

  * - U_TRUNCATED_CHAR_FOUND or another error code (never U_BUFFER_OVERFLOW_ERROR!)

  *   with toUBytes[toULength] set to the offending input

  *   (the return value will be ignored)

  * - return UCNV_GET_NEXT_UCHAR_USE_TO_U, without moving the source pointer,

  *   to indicate that the ucnv.c code shall call the toUnicode function instead

  * - return a real code point result

  *

  * Unless UCNV_GET_NEXT_UCHAR_USE_TO_U is returned, the source bytes must be consumed.

  *

  * The ucnv.c code will handle the end of the input (reset)

  * (except for truncation detection!) and callbacks.

  */

 typedef UChar32 (*UConverterGetNextUChar) (UConverterToUnicodeArgs *, UErrorCode *);

 typedef void (*UConverterGetStarters)(const UConverter* converter,

                                       UBool starters[256],

                                       UErrorCode *pErrorCode);

 /* If this function pointer is null or if the function returns null

  * the name field in static data struct should be returned by 

  * ucnv_getName() API function

  */

 typedef const char * (*UConverterGetName) (const UConverter *cnv);

 /**

  * Write the codepage substitution character.

  * If this function is not set, then ucnv_cbFromUWriteSub() writes

  * the substitution character from UConverter.

  * For stateful converters, it is typically necessary to handle this

  * specificially for the converter in order to properly maintain the state.

  */

 typedef void (*UConverterWriteSub) (UConverterFromUnicodeArgs *pArgs, int32_t offsetIndex, UErrorCode *pErrorCode);

 /**

  * For converter-specific safeClone processing

  * If this function is not set, then ucnv_safeClone assumes that the converter has no private data that changes

  * after the converter is done opening.

  * If this function is set, then it is called just after a memcpy() of

  * converter data to the new, empty converter, and is expected to set up

  * the initial state of the converter.  It is not expected to increment the

  * reference counts of the standard data types such as the shared data.

  */

 typedef UConverter * (*UConverterSafeClone) (const UConverter   *cnv, 

                                              void               *stackBuffer,

                                              int32_t            *pBufferSize, 

                                              UErrorCode         *status);

 /**

  * Filters for some ucnv_getUnicodeSet() implementation code.

  */

 typedef enum UConverterSetFilter {

     UCNV_SET_FILTER_NONE,

     UCNV_SET_FILTER_DBCS_ONLY,

     UCNV_SET_FILTER_2022_CN,

     UCNV_SET_FILTER_SJIS,

     UCNV_SET_FILTER_GR94DBCS,

     UCNV_SET_FILTER_HZ,

     UCNV_SET_FILTER_COUNT

 } UConverterSetFilter;

 /**

  * Fills the set of Unicode code points that can be converted by an ICU converter.

  * The API function ucnv_getUnicodeSet() clears the USet before calling

  * the converter's getUnicodeSet() implementation; the converter should only

  * add the appropriate code points to allow recursive use.

  * For example, the ISO-2022-JP converter will call each subconverter's

  * getUnicodeSet() implementation to consecutively add code points to

  * the same USet, which will result in a union of the sets of all subconverters.

  *

  * For more documentation, see ucnv_getUnicodeSet() in ucnv.h.

  */

 typedef void (*UConverterGetUnicodeSet) (const UConverter *cnv,

                                          const USetAdder *sa,

                                          UConverterUnicodeSet which,

                                          UErrorCode *pErrorCode);

 UBool CONVERSION_U_SUCCESS (UErrorCode err);

 /**

  * UConverterImpl contains all the data and functions for a converter type.

  * Its function pointers work much like a C++ vtable.

  * Many converter types need to define only a subset of the functions;

  * when a function pointer is NULL, then a default action will be performed.

  *

  * Every converter type must implement toUnicode, fromUnicode, and getNextUChar,

  * otherwise the converter may crash.

  * Every converter type that has variable-length codepage sequences should

  * also implement toUnicodeWithOffsets and fromUnicodeWithOffsets for

  * correct offset handling.

  * All other functions may or may not be implemented - it depends only on

  * whether the converter type needs them.

  *

  * When open() fails, then close() will be called, if present.

  */

 struct UConverterImpl {

     UConverterType type;

     UConverterLoad load;

     UConverterUnload unload;

     UConverterOpen open;

     UConverterClose close;

     UConverterReset reset;

     UConverterToUnicode toUnicode;

     UConverterToUnicode toUnicodeWithOffsets;

     UConverterFromUnicode fromUnicode;

     UConverterFromUnicode fromUnicodeWithOffsets;

     UConverterGetNextUChar getNextUChar;

     UConverterGetStarters getStarters;

     UConverterGetName getName;

     UConverterWriteSub writeSub;

     UConverterSafeClone safeClone;

     UConverterGetUnicodeSet getUnicodeSet;

     UConverterConvert toUTF8;

     UConverterConvert fromUTF8;

 };

 extern const UConverterSharedData

     _MBCSData, _Latin1Data,

     _UTF8Data, _UTF16BEData, _UTF16LEData, _UTF32BEData, _UTF32LEData,

     _ISO2022Data, 

     _LMBCSData1,_LMBCSData2, _LMBCSData3, _LMBCSData4, _LMBCSData5, _LMBCSData6,

     _LMBCSData8,_LMBCSData11,_LMBCSData16,_LMBCSData17,_LMBCSData18,_LMBCSData19,

     _HZData,_ISCIIData, _SCSUData, _ASCIIData,

     _UTF7Data, _Bocu1Data, _UTF16Data, _UTF32Data, _CESU8Data, _IMAPData, _CompoundTextData;

 U_CDECL_END

 /** Always use fallbacks from codepage to Unicode */

 #define TO_U_USE_FALLBACK(useFallback) TRUE

 #define UCNV_TO_U_USE_FALLBACK(cnv) TRUE

 /** Use fallbacks from Unicode to codepage when cnv->useFallback or for private-use code points */

 #define IS_PRIVATE_USE(c) ((uint32_t)((c)-0xe000)<0x1900 || (uint32_t)((c)-0xf0000)<0x20000)

 #define FROM_U_USE_FALLBACK(useFallback, c) ((useFallback) || IS_PRIVATE_USE(c))

 #define UCNV_FROM_U_USE_FALLBACK(cnv, c) FROM_U_USE_FALLBACK((cnv)->useFallback, c)

 /**

  * Magic number for ucnv_getNextUChar(), returned by a

  * getNextUChar() implementation to indicate to use the converter's toUnicode()

  * instead of the native function.

  * @internal

  */

 #define UCNV_GET_NEXT_UCHAR_USE_TO_U -9

 U_CFUNC void

 ucnv_getCompleteUnicodeSet(const UConverter *cnv,

                    const USetAdder *sa,

                    UConverterUnicodeSet which,

                    UErrorCode *pErrorCode);

 U_CFUNC void

 ucnv_getNonSurrogateUnicodeSet(const UConverter *cnv,

                                const USetAdder *sa,

                                UConverterUnicodeSet which,

                                UErrorCode *pErrorCode);

 U_CFUNC void

 ucnv_fromUWriteBytes(UConverter *cnv,

                      const char *bytes, int32_t length,

                      char **target, const char *targetLimit,

                      int32_t **offsets,

                      int32_t sourceIndex,

                      UErrorCode *pErrorCode);

 U_CFUNC void

 ucnv_toUWriteUChars(UConverter *cnv,

                     const UChar *uchars, int32_t length,

                     UChar **target, const UChar *targetLimit,

                     int32_t **offsets,

                     int32_t sourceIndex,

                     UErrorCode *pErrorCode);

 U_CFUNC void

 ucnv_toUWriteCodePoint(UConverter *cnv,

                        UChar32 c,

                        UChar **target, const UChar *targetLimit,

                        int32_t **offsets,

                        int32_t sourceIndex,

                        UErrorCode *pErrorCode);

 #endif

 #endif /* UCNV_CNV */