michael@0: /* michael@0: *************************************************************************** michael@0: * Copyright (C) 2008-2013, International Business Machines Corporation michael@0: * and others. All Rights Reserved. michael@0: *************************************************************************** michael@0: * michael@0: * uspoof_impl.h michael@0: * michael@0: * Implemenation header for spoof detection michael@0: * michael@0: */ michael@0: michael@0: #ifndef USPOOFIM_H michael@0: #define USPOOFIM_H michael@0: michael@0: #include "unicode/utypes.h" michael@0: #include "unicode/uspoof.h" michael@0: #include "unicode/uscript.h" michael@0: #include "unicode/udata.h" michael@0: michael@0: #include "utrie2.h" michael@0: michael@0: #if !UCONFIG_NO_NORMALIZATION michael@0: michael@0: #ifdef __cplusplus michael@0: michael@0: U_NAMESPACE_BEGIN michael@0: michael@0: // The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from michael@0: // a single input code point. This is function of the unicode.org data. michael@0: #define USPOOF_MAX_SKELETON_EXPANSION 20 michael@0: michael@0: // The default stack buffer size for copies or conversions or normalizations michael@0: // of input strings being checked. (Used in multiple places.) michael@0: #define USPOOF_STACK_BUFFER_SIZE 100 michael@0: michael@0: // Magic number for sanity checking spoof data. michael@0: #define USPOOF_MAGIC 0x3845fdef michael@0: michael@0: class IdentifierInfo; michael@0: class ScriptSet; michael@0: class SpoofData; michael@0: struct SpoofDataHeader; michael@0: struct SpoofStringLengthsElement; michael@0: michael@0: /** michael@0: * Class SpoofImpl corresponds directly to the plain C API opaque type michael@0: * USpoofChecker. One can be cast to the other. michael@0: */ michael@0: class SpoofImpl : public UObject { michael@0: public: michael@0: SpoofImpl(SpoofData *data, UErrorCode &status); michael@0: SpoofImpl(); michael@0: virtual ~SpoofImpl(); michael@0: michael@0: /** Copy constructor, used by the user level uspoof_clone() function. michael@0: */ michael@0: SpoofImpl(const SpoofImpl &src, UErrorCode &status); michael@0: michael@0: static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status); michael@0: static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status); michael@0: michael@0: /** Get the confusable skeleton transform for a single code point. michael@0: * The result is a string with a length between 1 and 18. michael@0: * @param tableMask bit flag specifying which confusable table to use. michael@0: * One of USPOOF_SL_TABLE_FLAG, USPOOF_MA_TABLE_FLAG, etc. michael@0: * @return The length in UTF-16 code units of the substition string. michael@0: */ michael@0: int32_t confusableLookup(UChar32 inChar, int32_t tableMask, UnicodeString &destBuf) const; michael@0: michael@0: /** Set and Get AllowedLocales, implementations of the corresponding API */ michael@0: void setAllowedLocales(const char *localesList, UErrorCode &status); michael@0: const char * getAllowedLocales(UErrorCode &status); michael@0: michael@0: // Add (union) to the UnicodeSet all of the characters for the scripts used for michael@0: // the specified locale. Part of the implementation of setAllowedLocales. michael@0: void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status); michael@0: michael@0: michael@0: /** parse a hex number. Untility used by the builders. */ michael@0: static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status); michael@0: michael@0: // Implementation for Whole Script tests. michael@0: // Return the test bit flag to be ORed into the eventual user return value michael@0: // if a Spoof opportunity is detected. michael@0: void wholeScriptCheck( michael@0: const UnicodeString &text, ScriptSet *result, UErrorCode &status) const; michael@0: michael@0: static UClassID U_EXPORT2 getStaticClassID(void); michael@0: virtual UClassID getDynamicClassID(void) const; michael@0: michael@0: // IdentifierInfo Cache. IdentifierInfo objects are somewhat expensive to create. michael@0: // Maintain a one-element cache, which is sufficient to avoid repeatedly michael@0: // creating new ones unless we get multi-thread concurrency in spoof michael@0: // check operations, which should be statistically uncommon. michael@0: IdentifierInfo *getIdentifierInfo(UErrorCode &status) const; michael@0: void releaseIdentifierInfo(IdentifierInfo *idInfo) const; michael@0: michael@0: // michael@0: // Data Members michael@0: // michael@0: michael@0: int32_t fMagic; // Internal sanity check. michael@0: int32_t fChecks; // Bit vector of checks to perform. michael@0: michael@0: SpoofData *fSpoofData; michael@0: michael@0: const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters. michael@0: // for this Spoof Checker. Defaults to all chars. michael@0: michael@0: const char *fAllowedLocales; // The list of allowed locales. michael@0: URestrictionLevel fRestrictionLevel; // The maximum restriction level for an acceptable identifier. michael@0: michael@0: IdentifierInfo *fCachedIdentifierInfo; // Do not use directly. See getIdentifierInfo().:w michael@0: }; michael@0: michael@0: michael@0: michael@0: // michael@0: // Confusable Mappings Data Structures michael@0: // michael@0: // For the confusable data, we are essentially implementing a map, michael@0: // key: a code point michael@0: // value: a string. Most commonly one char in length, but can be more. michael@0: // michael@0: // The keys are stored as a sorted array of 32 bit ints. michael@0: // bits 0-23 a code point value michael@0: // bits 24-31 flags michael@0: // 24: 1 if entry applies to SL table michael@0: // 25: 1 if entry applies to SA table michael@0: // 26: 1 if entry applies to ML table michael@0: // 27: 1 if entry applies to MA table michael@0: // 28: 1 if there are multiple entries for this code point. michael@0: // 29-30: length of value string, in UChars. michael@0: // values are (1, 2, 3, other) michael@0: // The key table is sorted in ascending code point order. (not on the michael@0: // 32 bit int value, the flag bits do not participate in the sorting.) michael@0: // michael@0: // Lookup is done by means of a binary search in the key table. michael@0: // michael@0: // The corresponding values are kept in a parallel array of 16 bit ints. michael@0: // If the value string is of length 1, it is literally in the value array. michael@0: // For longer strings, the value array contains an index into the strings table. michael@0: // michael@0: // String Table: michael@0: // The strings table contains all of the value strings (those of length two or greater) michael@0: // concatentated together into one long UChar (UTF-16) array. michael@0: // michael@0: // The array is arranged by length of the strings - all strings of the same length michael@0: // are stored together. The sections are ordered by length of the strings - michael@0: // all two char strings first, followed by all of the three Char strings, etc. michael@0: // michael@0: // There is no nul character or other mark between adjacent strings. michael@0: // michael@0: // String Lengths table michael@0: // The length of strings from 1 to 3 is flagged in the key table. michael@0: // For strings of length 4 or longer, the string length table provides a michael@0: // mapping between an index into the string table and the corresponding length. michael@0: // Strings of these lengths are rare, so lookup time is not an issue. michael@0: // Each entry consists of michael@0: // uint16_t index of the _last_ string with this length michael@0: // uint16_t the length michael@0: // michael@0: michael@0: // Flag bits in the Key entries michael@0: #define USPOOF_SL_TABLE_FLAG (1<<24) michael@0: #define USPOOF_SA_TABLE_FLAG (1<<25) michael@0: #define USPOOF_ML_TABLE_FLAG (1<<26) michael@0: #define USPOOF_MA_TABLE_FLAG (1<<27) michael@0: #define USPOOF_KEY_MULTIPLE_VALUES (1<<28) michael@0: #define USPOOF_KEY_LENGTH_SHIFT 29 michael@0: #define USPOOF_KEY_LENGTH_FIELD(x) (((x)>>29) & 3) michael@0: michael@0: michael@0: struct SpoofStringLengthsElement { michael@0: uint16_t fLastString; // index in string table of last string with this length michael@0: uint16_t fStrLength; // Length of strings michael@0: }; michael@0: michael@0: michael@0: michael@0: //------------------------------------------------------------------------------------- michael@0: // michael@0: // SpoofData michael@0: // michael@0: // A small class that wraps the raw (usually memory mapped) spoof data. michael@0: // Serves two primary functions: michael@0: // 1. Convenience. Contains real pointers to the data, to avoid dealing with michael@0: // the offsets in the raw data. michael@0: // 2. Reference counting. When a spoof checker is cloned, the raw data is shared michael@0: // and must be retained until all checkers using the data are closed. michael@0: // Nothing in this struct includes state that is specific to any particular michael@0: // USpoofDetector object. michael@0: // michael@0: //--------------------------------------------------------------------------------------- michael@0: class SpoofData: public UMemory { michael@0: public: michael@0: static SpoofData *getDefault(UErrorCode &status); // Load standard ICU spoof data. michael@0: SpoofData(UErrorCode &status); // Create new spoof data wrapper. michael@0: // Only used when building new data from rules. michael@0: michael@0: // Constructor for use when creating from prebuilt default data. michael@0: // A UDataMemory is what the ICU internal data loading functions provide. michael@0: // The udm is adopted by the SpoofData. michael@0: SpoofData(UDataMemory *udm, UErrorCode &status); michael@0: michael@0: // Constructor for use when creating from serialized data. michael@0: // michael@0: SpoofData(const void *serializedData, int32_t length, UErrorCode &status); michael@0: michael@0: // Check raw Spoof Data Version compatibility. michael@0: // Return TRUE it looks good. michael@0: static UBool validateDataVersion(const SpoofDataHeader *rawData, UErrorCode &status); michael@0: ~SpoofData(); // Destructor not normally used. michael@0: // Use removeReference() instead. michael@0: // Reference Counting functions. michael@0: // Clone of a user-level spoof detector increments the ref count on the data. michael@0: // Close of a user-level spoof detector decrements the ref count. michael@0: // If the data is owned by us, it will be deleted when count goes to zero. michael@0: SpoofData *addReference(); michael@0: void removeReference(); michael@0: michael@0: // Reserve space in the raw data. For use by builder when putting together a michael@0: // new set of data. Init the new storage to zero, to prevent inconsistent michael@0: // results if it is not all otherwise set by the requester. michael@0: // Return: michael@0: // pointer to the new space that was added by this function. michael@0: void *reserveSpace(int32_t numBytes, UErrorCode &status); michael@0: michael@0: // initialize the pointers from this object to the raw data. michael@0: void initPtrs(UErrorCode &status); michael@0: michael@0: // Reset all fields to an initial state. michael@0: // Called from the top of all constructors. michael@0: void reset(); michael@0: michael@0: SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data michael@0: UBool fDataOwned; // True if the raw data is owned, and needs michael@0: // to be deleted when refcount goes to zero. michael@0: UDataMemory *fUDM; // If not NULL, our data came from a michael@0: // UDataMemory, which we must close when michael@0: // we are done. michael@0: michael@0: uint32_t fMemLimit; // Limit of available raw data space michael@0: u_atomic_int32_t fRefCount; michael@0: michael@0: // Confusable data michael@0: int32_t *fCFUKeys; michael@0: uint16_t *fCFUValues; michael@0: SpoofStringLengthsElement *fCFUStringLengths; michael@0: UChar *fCFUStrings; michael@0: michael@0: // Whole Script Confusable Data michael@0: UTrie2 *fAnyCaseTrie; michael@0: UTrie2 *fLowerCaseTrie; michael@0: ScriptSet *fScriptSets; michael@0: }; michael@0: michael@0: michael@0: //--------------------------------------------------------------------------------------- michael@0: // michael@0: // Raw Binary Data Formats, as loaded from the ICU data file, michael@0: // or as built by the builder. michael@0: // michael@0: //--------------------------------------------------------------------------------------- michael@0: struct SpoofDataHeader { michael@0: int32_t fMagic; // (0x3845fdef) michael@0: uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo michael@0: // if there is one associated with this data. michael@0: int32_t fLength; // Total lenght in bytes of this spoof data, michael@0: // including all sections, not just the header. michael@0: michael@0: // The following four sections refer to data representing the confusable data michael@0: // from the Unicode.org data from "confusables.txt" michael@0: michael@0: int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *) michael@0: int32_t fCFUKeysSize; // number of entries in keys table (32 bits each) michael@0: michael@0: // TODO: change name to fCFUValues, for consistency. michael@0: int32_t fCFUStringIndex; // byte offset to String Indexes table michael@0: int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each) michael@0: // (number of entries must be same as in Keys table michael@0: michael@0: int32_t fCFUStringTable; // byte offset of String table michael@0: int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars) michael@0: michael@0: int32_t fCFUStringLengths; // byte offset to String Lengths table michael@0: int32_t fCFUStringLengthsSize; // number of entries in lengths table. (2 x 16 bits each) michael@0: michael@0: michael@0: // The following sections are for data from confusablesWholeScript.txt michael@0: michael@0: int32_t fAnyCaseTrie; // byte offset to the serialized Any Case Trie michael@0: int32_t fAnyCaseTrieLength; // Length (bytes) of the serialized Any Case Trie michael@0: michael@0: int32_t fLowerCaseTrie; // byte offset to the serialized Lower Case Trie michael@0: int32_t fLowerCaseTrieLength; // Length (bytes) of the serialized Lower Case Trie michael@0: michael@0: int32_t fScriptSets; // byte offset to array of ScriptSets michael@0: int32_t fScriptSetsLength; // Number of ScriptSets (24 bytes each) michael@0: michael@0: michael@0: // The following sections are for data from xidmodifications.txt michael@0: michael@0: michael@0: int32_t unused[15]; // Padding, Room for Expansion michael@0: michael@0: }; michael@0: michael@0: michael@0: michael@0: michael@0: // michael@0: // Structure for the Whole Script Confusable Data michael@0: // See Unicode UAX-39, Unicode Security Mechanisms, for a description of the michael@0: // Whole Script confusable data michael@0: // michael@0: // The data provides mappings from code points to a set of scripts michael@0: // that contain characters that might be confused with the code point. michael@0: // There are two mappings, one for lower case only, and one for characters michael@0: // of any case. michael@0: // michael@0: // The actual data consists of a utrie2 to map from a code point to an offset, michael@0: // and an array of UScriptSets (essentially bit maps) that is indexed michael@0: // by the offsets obtained from the Trie. michael@0: // michael@0: // michael@0: michael@0: michael@0: U_NAMESPACE_END michael@0: #endif /* __cplusplus */ michael@0: michael@0: /** michael@0: * Endianness swap function for binary spoof data. michael@0: * @internal michael@0: */ michael@0: U_CAPI int32_t U_EXPORT2 michael@0: uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, michael@0: UErrorCode *status); michael@0: michael@0: michael@0: #endif michael@0: michael@0: #endif /* USPOOFIM_H */ michael@0: