michael@0: /* michael@0: ********************************************************************** michael@0: * Copyright (C) 1999-2011, International Business Machines michael@0: * Corporation and others. All Rights Reserved. michael@0: ********************************************************************** michael@0: * michael@0: * ucnv_cnv.h: michael@0: * Definitions for converter implementations. michael@0: * michael@0: * Modification History: michael@0: * michael@0: * Date Name Description michael@0: * 05/09/00 helena Added implementation to handle fallback mappings. michael@0: * 06/29/2000 helena Major rewrite of the callback APIs. michael@0: */ michael@0: michael@0: #ifndef UCNV_CNV_H michael@0: #define UCNV_CNV_H michael@0: michael@0: #include "unicode/utypes.h" michael@0: michael@0: #if !UCONFIG_NO_CONVERSION michael@0: michael@0: #include "unicode/ucnv.h" michael@0: #include "unicode/ucnv_err.h" michael@0: #include "unicode/uset.h" michael@0: #include "uset_imp.h" michael@0: michael@0: U_CDECL_BEGIN michael@0: michael@0: /* this is used in fromUnicode DBCS tables as an "unassigned" marker */ michael@0: #define missingCharMarker 0xFFFF michael@0: michael@0: /* michael@0: * #define missingUCharMarker 0xfffe michael@0: * michael@0: * commented out because there are actually two values used in toUnicode tables: michael@0: * U+fffe "unassigned" michael@0: * U+ffff "illegal" michael@0: */ michael@0: michael@0: /** Forward declaration, see ucnv_bld.h */ michael@0: struct UConverterSharedData; michael@0: typedef struct UConverterSharedData UConverterSharedData; michael@0: michael@0: /* function types for UConverterImpl ---------------------------------------- */ michael@0: michael@0: /* struct with arguments for UConverterLoad and ucnv_load() */ michael@0: typedef struct { michael@0: int32_t size; /* sizeof(UConverterLoadArgs) */ michael@0: int32_t nestedLoads; /* count nested ucnv_load() calls */ michael@0: UBool onlyTestIsLoadable; /* input: don't actually load */ michael@0: UBool reserved0; /* reserved - for good alignment of the pointers */ michael@0: int16_t reserved; /* reserved - for good alignment of the pointers */ michael@0: uint32_t options; michael@0: const char *pkg, *name, *locale; michael@0: } UConverterLoadArgs; michael@0: michael@0: #define UCNV_LOAD_ARGS_INITIALIZER \ michael@0: { (int32_t)sizeof(UConverterLoadArgs), 0, FALSE, FALSE, 0, 0, NULL, NULL, NULL } michael@0: michael@0: typedef void (*UConverterLoad) (UConverterSharedData *sharedData, michael@0: UConverterLoadArgs *pArgs, michael@0: const uint8_t *raw, UErrorCode *pErrorCode); michael@0: typedef void (*UConverterUnload) (UConverterSharedData *sharedData); michael@0: michael@0: typedef void (*UConverterOpen) (UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *pErrorCode); michael@0: typedef void (*UConverterClose) (UConverter *cnv); michael@0: michael@0: typedef enum UConverterResetChoice { michael@0: UCNV_RESET_BOTH, michael@0: UCNV_RESET_TO_UNICODE, michael@0: UCNV_RESET_FROM_UNICODE michael@0: } UConverterResetChoice; michael@0: michael@0: typedef void (*UConverterReset) (UConverter *cnv, UConverterResetChoice choice); michael@0: michael@0: /* michael@0: * Converter implementation function(s) for ucnv_toUnicode(). michael@0: * If the toUnicodeWithOffsets function pointer is NULL, michael@0: * then the toUnicode function will be used and the offsets will be set to -1. michael@0: * michael@0: * Must maintain state across buffers. Use toUBytes[toULength] for partial input michael@0: * sequences; it will be checked in ucnv.c at the end of the input stream michael@0: * to detect truncated input. michael@0: * Some converters may need additional detection and may then set U_TRUNCATED_CHAR_FOUND. michael@0: * michael@0: * The toUnicodeWithOffsets must write exactly as many offset values as target michael@0: * units. Write offset values of -1 for when the source index corresponding to michael@0: * the output unit is not known (e.g., the character started in an earlier buffer). michael@0: * The pArgs->offsets pointer need not be moved forward. michael@0: * michael@0: * At function return, either one of the following conditions must be true: michael@0: * - U_BUFFER_OVERFLOW_ERROR and the target is full: target==targetLimit michael@0: * - another error code with toUBytes[toULength] set to the offending input michael@0: * - no error, and the source is consumed: source==sourceLimit michael@0: * michael@0: * The ucnv.c code will handle the end of the input (reset) michael@0: * (reset, and truncation detection) and callbacks. michael@0: */ michael@0: typedef void (*UConverterToUnicode) (UConverterToUnicodeArgs *, UErrorCode *); michael@0: michael@0: /* michael@0: * Same rules as for UConverterToUnicode. michael@0: * A lead surrogate is kept in fromUChar32 across buffers, and if an error michael@0: * occurs, then the offending input code point must be put into fromUChar32 michael@0: * as well. michael@0: */ michael@0: typedef void (*UConverterFromUnicode) (UConverterFromUnicodeArgs *, UErrorCode *); michael@0: michael@0: /* michael@0: * Converter implementation function for ucnv_convertEx(), for direct conversion michael@0: * between two charsets without pivoting through UTF-16. michael@0: * The rules are the same as for UConverterToUnicode and UConverterFromUnicode. michael@0: * In addition, michael@0: * - The toUnicode side must behave and keep state exactly like the michael@0: * UConverterToUnicode implementation for the same source charset. michael@0: * - A U_USING_DEFAULT_WARNING can be set to request to temporarily fall back michael@0: * to pivoting. When this function is called, the conversion framework makes michael@0: * sure that this warning is not set on input. michael@0: * - Continuing a partial match and flushing the toUnicode replay buffer michael@0: * are handled by pivoting, using the toUnicode and fromUnicode functions. michael@0: */ michael@0: typedef void (*UConverterConvert) (UConverterFromUnicodeArgs *pFromUArgs, michael@0: UConverterToUnicodeArgs *pToUArgs, michael@0: UErrorCode *pErrorCode); michael@0: michael@0: /* michael@0: * Converter implementation function for ucnv_getNextUChar(). michael@0: * If the function pointer is NULL, then the toUnicode function will be used. michael@0: * michael@0: * Will be called at a character boundary (toULength==0). michael@0: * May return with michael@0: * - U_INDEX_OUTOFBOUNDS_ERROR if there was no output for the input michael@0: * (the return value will be ignored) michael@0: * - U_TRUNCATED_CHAR_FOUND or another error code (never U_BUFFER_OVERFLOW_ERROR!) michael@0: * with toUBytes[toULength] set to the offending input michael@0: * (the return value will be ignored) michael@0: * - return UCNV_GET_NEXT_UCHAR_USE_TO_U, without moving the source pointer, michael@0: * to indicate that the ucnv.c code shall call the toUnicode function instead michael@0: * - return a real code point result michael@0: * michael@0: * Unless UCNV_GET_NEXT_UCHAR_USE_TO_U is returned, the source bytes must be consumed. michael@0: * michael@0: * The ucnv.c code will handle the end of the input (reset) michael@0: * (except for truncation detection!) and callbacks. michael@0: */ michael@0: typedef UChar32 (*UConverterGetNextUChar) (UConverterToUnicodeArgs *, UErrorCode *); michael@0: michael@0: typedef void (*UConverterGetStarters)(const UConverter* converter, michael@0: UBool starters[256], michael@0: UErrorCode *pErrorCode); michael@0: michael@0: /* If this function pointer is null or if the function returns null michael@0: * the name field in static data struct should be returned by michael@0: * ucnv_getName() API function michael@0: */ michael@0: typedef const char * (*UConverterGetName) (const UConverter *cnv); michael@0: michael@0: /** michael@0: * Write the codepage substitution character. michael@0: * If this function is not set, then ucnv_cbFromUWriteSub() writes michael@0: * the substitution character from UConverter. michael@0: * For stateful converters, it is typically necessary to handle this michael@0: * specificially for the converter in order to properly maintain the state. michael@0: */ michael@0: typedef void (*UConverterWriteSub) (UConverterFromUnicodeArgs *pArgs, int32_t offsetIndex, UErrorCode *pErrorCode); michael@0: michael@0: /** michael@0: * For converter-specific safeClone processing michael@0: * If this function is not set, then ucnv_safeClone assumes that the converter has no private data that changes michael@0: * after the converter is done opening. michael@0: * If this function is set, then it is called just after a memcpy() of michael@0: * converter data to the new, empty converter, and is expected to set up michael@0: * the initial state of the converter. It is not expected to increment the michael@0: * reference counts of the standard data types such as the shared data. michael@0: */ michael@0: typedef UConverter * (*UConverterSafeClone) (const UConverter *cnv, michael@0: void *stackBuffer, michael@0: int32_t *pBufferSize, michael@0: UErrorCode *status); michael@0: michael@0: /** michael@0: * Filters for some ucnv_getUnicodeSet() implementation code. michael@0: */ michael@0: typedef enum UConverterSetFilter { michael@0: UCNV_SET_FILTER_NONE, michael@0: UCNV_SET_FILTER_DBCS_ONLY, michael@0: UCNV_SET_FILTER_2022_CN, michael@0: UCNV_SET_FILTER_SJIS, michael@0: UCNV_SET_FILTER_GR94DBCS, michael@0: UCNV_SET_FILTER_HZ, michael@0: UCNV_SET_FILTER_COUNT michael@0: } UConverterSetFilter; michael@0: michael@0: /** michael@0: * Fills the set of Unicode code points that can be converted by an ICU converter. michael@0: * The API function ucnv_getUnicodeSet() clears the USet before calling michael@0: * the converter's getUnicodeSet() implementation; the converter should only michael@0: * add the appropriate code points to allow recursive use. michael@0: * For example, the ISO-2022-JP converter will call each subconverter's michael@0: * getUnicodeSet() implementation to consecutively add code points to michael@0: * the same USet, which will result in a union of the sets of all subconverters. michael@0: * michael@0: * For more documentation, see ucnv_getUnicodeSet() in ucnv.h. michael@0: */ michael@0: typedef void (*UConverterGetUnicodeSet) (const UConverter *cnv, michael@0: const USetAdder *sa, michael@0: UConverterUnicodeSet which, michael@0: UErrorCode *pErrorCode); michael@0: michael@0: UBool CONVERSION_U_SUCCESS (UErrorCode err); michael@0: michael@0: /** michael@0: * UConverterImpl contains all the data and functions for a converter type. michael@0: * Its function pointers work much like a C++ vtable. michael@0: * Many converter types need to define only a subset of the functions; michael@0: * when a function pointer is NULL, then a default action will be performed. michael@0: * michael@0: * Every converter type must implement toUnicode, fromUnicode, and getNextUChar, michael@0: * otherwise the converter may crash. michael@0: * Every converter type that has variable-length codepage sequences should michael@0: * also implement toUnicodeWithOffsets and fromUnicodeWithOffsets for michael@0: * correct offset handling. michael@0: * All other functions may or may not be implemented - it depends only on michael@0: * whether the converter type needs them. michael@0: * michael@0: * When open() fails, then close() will be called, if present. michael@0: */ michael@0: struct UConverterImpl { michael@0: UConverterType type; michael@0: michael@0: UConverterLoad load; michael@0: UConverterUnload unload; michael@0: michael@0: UConverterOpen open; michael@0: UConverterClose close; michael@0: UConverterReset reset; michael@0: michael@0: UConverterToUnicode toUnicode; michael@0: UConverterToUnicode toUnicodeWithOffsets; michael@0: UConverterFromUnicode fromUnicode; michael@0: UConverterFromUnicode fromUnicodeWithOffsets; michael@0: UConverterGetNextUChar getNextUChar; michael@0: michael@0: UConverterGetStarters getStarters; michael@0: UConverterGetName getName; michael@0: UConverterWriteSub writeSub; michael@0: UConverterSafeClone safeClone; michael@0: UConverterGetUnicodeSet getUnicodeSet; michael@0: michael@0: UConverterConvert toUTF8; michael@0: UConverterConvert fromUTF8; michael@0: }; michael@0: michael@0: extern const UConverterSharedData michael@0: _MBCSData, _Latin1Data, michael@0: _UTF8Data, _UTF16BEData, _UTF16LEData, _UTF32BEData, _UTF32LEData, michael@0: _ISO2022Data, michael@0: _LMBCSData1,_LMBCSData2, _LMBCSData3, _LMBCSData4, _LMBCSData5, _LMBCSData6, michael@0: _LMBCSData8,_LMBCSData11,_LMBCSData16,_LMBCSData17,_LMBCSData18,_LMBCSData19, michael@0: _HZData,_ISCIIData, _SCSUData, _ASCIIData, michael@0: _UTF7Data, _Bocu1Data, _UTF16Data, _UTF32Data, _CESU8Data, _IMAPData, _CompoundTextData; michael@0: michael@0: U_CDECL_END michael@0: michael@0: /** Always use fallbacks from codepage to Unicode */ michael@0: #define TO_U_USE_FALLBACK(useFallback) TRUE michael@0: #define UCNV_TO_U_USE_FALLBACK(cnv) TRUE michael@0: michael@0: /** Use fallbacks from Unicode to codepage when cnv->useFallback or for private-use code points */ michael@0: #define IS_PRIVATE_USE(c) ((uint32_t)((c)-0xe000)<0x1900 || (uint32_t)((c)-0xf0000)<0x20000) michael@0: #define FROM_U_USE_FALLBACK(useFallback, c) ((useFallback) || IS_PRIVATE_USE(c)) michael@0: #define UCNV_FROM_U_USE_FALLBACK(cnv, c) FROM_U_USE_FALLBACK((cnv)->useFallback, c) michael@0: michael@0: /** michael@0: * Magic number for ucnv_getNextUChar(), returned by a michael@0: * getNextUChar() implementation to indicate to use the converter's toUnicode() michael@0: * instead of the native function. michael@0: * @internal michael@0: */ michael@0: #define UCNV_GET_NEXT_UCHAR_USE_TO_U -9 michael@0: michael@0: U_CFUNC void michael@0: ucnv_getCompleteUnicodeSet(const UConverter *cnv, michael@0: const USetAdder *sa, michael@0: UConverterUnicodeSet which, michael@0: UErrorCode *pErrorCode); michael@0: michael@0: U_CFUNC void michael@0: ucnv_getNonSurrogateUnicodeSet(const UConverter *cnv, michael@0: const USetAdder *sa, michael@0: UConverterUnicodeSet which, michael@0: UErrorCode *pErrorCode); michael@0: michael@0: U_CFUNC void michael@0: ucnv_fromUWriteBytes(UConverter *cnv, michael@0: const char *bytes, int32_t length, michael@0: char **target, const char *targetLimit, michael@0: int32_t **offsets, michael@0: int32_t sourceIndex, michael@0: UErrorCode *pErrorCode); michael@0: U_CFUNC void michael@0: ucnv_toUWriteUChars(UConverter *cnv, michael@0: const UChar *uchars, int32_t length, michael@0: UChar **target, const UChar *targetLimit, michael@0: int32_t **offsets, michael@0: int32_t sourceIndex, michael@0: UErrorCode *pErrorCode); michael@0: michael@0: U_CFUNC void michael@0: ucnv_toUWriteCodePoint(UConverter *cnv, michael@0: UChar32 c, michael@0: UChar **target, const UChar *targetLimit, michael@0: int32_t **offsets, michael@0: int32_t sourceIndex, michael@0: UErrorCode *pErrorCode); michael@0: michael@0: #endif michael@0: michael@0: #endif /* UCNV_CNV */