1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/intl/icu/source/i18n/uspoof_impl.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,345 @@
1.4 +/*
1.5 +***************************************************************************
1.6 +* Copyright (C) 2008-2013, International Business Machines Corporation
1.7 +* and others. All Rights Reserved.
1.8 +***************************************************************************
1.9 +*
1.10 +* uspoof_impl.h
1.11 +*
1.12 +* Implemenation header for spoof detection
1.13 +*
1.14 +*/
1.15 +
1.16 +#ifndef USPOOFIM_H
1.17 +#define USPOOFIM_H
1.18 +
1.19 +#include "unicode/utypes.h"
1.20 +#include "unicode/uspoof.h"
1.21 +#include "unicode/uscript.h"
1.22 +#include "unicode/udata.h"
1.23 +
1.24 +#include "utrie2.h"
1.25 +
1.26 +#if !UCONFIG_NO_NORMALIZATION
1.27 +
1.28 +#ifdef __cplusplus
1.29 +
1.30 +U_NAMESPACE_BEGIN
1.31 +
1.32 +// The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from
1.33 +// a single input code point. This is function of the unicode.org data.
1.34 +#define USPOOF_MAX_SKELETON_EXPANSION 20
1.35 +
1.36 +// The default stack buffer size for copies or conversions or normalizations
1.37 +// of input strings being checked. (Used in multiple places.)
1.38 +#define USPOOF_STACK_BUFFER_SIZE 100
1.39 +
1.40 +// Magic number for sanity checking spoof data.
1.41 +#define USPOOF_MAGIC 0x3845fdef
1.42 +
1.43 +class IdentifierInfo;
1.44 +class ScriptSet;
1.45 +class SpoofData;
1.46 +struct SpoofDataHeader;
1.47 +struct SpoofStringLengthsElement;
1.48 +
1.49 +/**
1.50 + * Class SpoofImpl corresponds directly to the plain C API opaque type
1.51 + * USpoofChecker. One can be cast to the other.
1.52 + */
1.53 +class SpoofImpl : public UObject {
1.54 +public:
1.55 + SpoofImpl(SpoofData *data, UErrorCode &status);
1.56 + SpoofImpl();
1.57 + virtual ~SpoofImpl();
1.58 +
1.59 + /** Copy constructor, used by the user level uspoof_clone() function.
1.60 + */
1.61 + SpoofImpl(const SpoofImpl &src, UErrorCode &status);
1.62 +
1.63 + static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status);
1.64 + static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status);
1.65 +
1.66 + /** Get the confusable skeleton transform for a single code point.
1.67 + * The result is a string with a length between 1 and 18.
1.68 + * @param tableMask bit flag specifying which confusable table to use.
1.69 + * One of USPOOF_SL_TABLE_FLAG, USPOOF_MA_TABLE_FLAG, etc.
1.70 + * @return The length in UTF-16 code units of the substition string.
1.71 + */
1.72 + int32_t confusableLookup(UChar32 inChar, int32_t tableMask, UnicodeString &destBuf) const;
1.73 +
1.74 + /** Set and Get AllowedLocales, implementations of the corresponding API */
1.75 + void setAllowedLocales(const char *localesList, UErrorCode &status);
1.76 + const char * getAllowedLocales(UErrorCode &status);
1.77 +
1.78 + // Add (union) to the UnicodeSet all of the characters for the scripts used for
1.79 + // the specified locale. Part of the implementation of setAllowedLocales.
1.80 + void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status);
1.81 +
1.82 +
1.83 + /** parse a hex number. Untility used by the builders. */
1.84 + static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status);
1.85 +
1.86 + // Implementation for Whole Script tests.
1.87 + // Return the test bit flag to be ORed into the eventual user return value
1.88 + // if a Spoof opportunity is detected.
1.89 + void wholeScriptCheck(
1.90 + const UnicodeString &text, ScriptSet *result, UErrorCode &status) const;
1.91 +
1.92 + static UClassID U_EXPORT2 getStaticClassID(void);
1.93 + virtual UClassID getDynamicClassID(void) const;
1.94 +
1.95 + // IdentifierInfo Cache. IdentifierInfo objects are somewhat expensive to create.
1.96 + // Maintain a one-element cache, which is sufficient to avoid repeatedly
1.97 + // creating new ones unless we get multi-thread concurrency in spoof
1.98 + // check operations, which should be statistically uncommon.
1.99 + IdentifierInfo *getIdentifierInfo(UErrorCode &status) const;
1.100 + void releaseIdentifierInfo(IdentifierInfo *idInfo) const;
1.101 +
1.102 + //
1.103 + // Data Members
1.104 + //
1.105 +
1.106 + int32_t fMagic; // Internal sanity check.
1.107 + int32_t fChecks; // Bit vector of checks to perform.
1.108 +
1.109 + SpoofData *fSpoofData;
1.110 +
1.111 + const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters.
1.112 + // for this Spoof Checker. Defaults to all chars.
1.113 +
1.114 + const char *fAllowedLocales; // The list of allowed locales.
1.115 + URestrictionLevel fRestrictionLevel; // The maximum restriction level for an acceptable identifier.
1.116 +
1.117 + IdentifierInfo *fCachedIdentifierInfo; // Do not use directly. See getIdentifierInfo().:w
1.118 +};
1.119 +
1.120 +
1.121 +
1.122 +//
1.123 +// Confusable Mappings Data Structures
1.124 +//
1.125 +// For the confusable data, we are essentially implementing a map,
1.126 +// key: a code point
1.127 +// value: a string. Most commonly one char in length, but can be more.
1.128 +//
1.129 +// The keys are stored as a sorted array of 32 bit ints.
1.130 +// bits 0-23 a code point value
1.131 +// bits 24-31 flags
1.132 +// 24: 1 if entry applies to SL table
1.133 +// 25: 1 if entry applies to SA table
1.134 +// 26: 1 if entry applies to ML table
1.135 +// 27: 1 if entry applies to MA table
1.136 +// 28: 1 if there are multiple entries for this code point.
1.137 +// 29-30: length of value string, in UChars.
1.138 +// values are (1, 2, 3, other)
1.139 +// The key table is sorted in ascending code point order. (not on the
1.140 +// 32 bit int value, the flag bits do not participate in the sorting.)
1.141 +//
1.142 +// Lookup is done by means of a binary search in the key table.
1.143 +//
1.144 +// The corresponding values are kept in a parallel array of 16 bit ints.
1.145 +// If the value string is of length 1, it is literally in the value array.
1.146 +// For longer strings, the value array contains an index into the strings table.
1.147 +//
1.148 +// String Table:
1.149 +// The strings table contains all of the value strings (those of length two or greater)
1.150 +// concatentated together into one long UChar (UTF-16) array.
1.151 +//
1.152 +// The array is arranged by length of the strings - all strings of the same length
1.153 +// are stored together. The sections are ordered by length of the strings -
1.154 +// all two char strings first, followed by all of the three Char strings, etc.
1.155 +//
1.156 +// There is no nul character or other mark between adjacent strings.
1.157 +//
1.158 +// String Lengths table
1.159 +// The length of strings from 1 to 3 is flagged in the key table.
1.160 +// For strings of length 4 or longer, the string length table provides a
1.161 +// mapping between an index into the string table and the corresponding length.
1.162 +// Strings of these lengths are rare, so lookup time is not an issue.
1.163 +// Each entry consists of
1.164 +// uint16_t index of the _last_ string with this length
1.165 +// uint16_t the length
1.166 +//
1.167 +
1.168 +// Flag bits in the Key entries
1.169 +#define USPOOF_SL_TABLE_FLAG (1<<24)
1.170 +#define USPOOF_SA_TABLE_FLAG (1<<25)
1.171 +#define USPOOF_ML_TABLE_FLAG (1<<26)
1.172 +#define USPOOF_MA_TABLE_FLAG (1<<27)
1.173 +#define USPOOF_KEY_MULTIPLE_VALUES (1<<28)
1.174 +#define USPOOF_KEY_LENGTH_SHIFT 29
1.175 +#define USPOOF_KEY_LENGTH_FIELD(x) (((x)>>29) & 3)
1.176 +
1.177 +
1.178 +struct SpoofStringLengthsElement {
1.179 + uint16_t fLastString; // index in string table of last string with this length
1.180 + uint16_t fStrLength; // Length of strings
1.181 +};
1.182 +
1.183 +
1.184 +
1.185 +//-------------------------------------------------------------------------------------
1.186 +//
1.187 +// SpoofData
1.188 +//
1.189 +// A small class that wraps the raw (usually memory mapped) spoof data.
1.190 +// Serves two primary functions:
1.191 +// 1. Convenience. Contains real pointers to the data, to avoid dealing with
1.192 +// the offsets in the raw data.
1.193 +// 2. Reference counting. When a spoof checker is cloned, the raw data is shared
1.194 +// and must be retained until all checkers using the data are closed.
1.195 +// Nothing in this struct includes state that is specific to any particular
1.196 +// USpoofDetector object.
1.197 +//
1.198 +//---------------------------------------------------------------------------------------
1.199 +class SpoofData: public UMemory {
1.200 + public:
1.201 + static SpoofData *getDefault(UErrorCode &status); // Load standard ICU spoof data.
1.202 + SpoofData(UErrorCode &status); // Create new spoof data wrapper.
1.203 + // Only used when building new data from rules.
1.204 +
1.205 + // Constructor for use when creating from prebuilt default data.
1.206 + // A UDataMemory is what the ICU internal data loading functions provide.
1.207 + // The udm is adopted by the SpoofData.
1.208 + SpoofData(UDataMemory *udm, UErrorCode &status);
1.209 +
1.210 + // Constructor for use when creating from serialized data.
1.211 + //
1.212 + SpoofData(const void *serializedData, int32_t length, UErrorCode &status);
1.213 +
1.214 + // Check raw Spoof Data Version compatibility.
1.215 + // Return TRUE it looks good.
1.216 + static UBool validateDataVersion(const SpoofDataHeader *rawData, UErrorCode &status);
1.217 + ~SpoofData(); // Destructor not normally used.
1.218 + // Use removeReference() instead.
1.219 + // Reference Counting functions.
1.220 + // Clone of a user-level spoof detector increments the ref count on the data.
1.221 + // Close of a user-level spoof detector decrements the ref count.
1.222 + // If the data is owned by us, it will be deleted when count goes to zero.
1.223 + SpoofData *addReference();
1.224 + void removeReference();
1.225 +
1.226 + // Reserve space in the raw data. For use by builder when putting together a
1.227 + // new set of data. Init the new storage to zero, to prevent inconsistent
1.228 + // results if it is not all otherwise set by the requester.
1.229 + // Return:
1.230 + // pointer to the new space that was added by this function.
1.231 + void *reserveSpace(int32_t numBytes, UErrorCode &status);
1.232 +
1.233 + // initialize the pointers from this object to the raw data.
1.234 + void initPtrs(UErrorCode &status);
1.235 +
1.236 + // Reset all fields to an initial state.
1.237 + // Called from the top of all constructors.
1.238 + void reset();
1.239 +
1.240 + SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data
1.241 + UBool fDataOwned; // True if the raw data is owned, and needs
1.242 + // to be deleted when refcount goes to zero.
1.243 + UDataMemory *fUDM; // If not NULL, our data came from a
1.244 + // UDataMemory, which we must close when
1.245 + // we are done.
1.246 +
1.247 + uint32_t fMemLimit; // Limit of available raw data space
1.248 + u_atomic_int32_t fRefCount;
1.249 +
1.250 + // Confusable data
1.251 + int32_t *fCFUKeys;
1.252 + uint16_t *fCFUValues;
1.253 + SpoofStringLengthsElement *fCFUStringLengths;
1.254 + UChar *fCFUStrings;
1.255 +
1.256 + // Whole Script Confusable Data
1.257 + UTrie2 *fAnyCaseTrie;
1.258 + UTrie2 *fLowerCaseTrie;
1.259 + ScriptSet *fScriptSets;
1.260 + };
1.261 +
1.262 +
1.263 +//---------------------------------------------------------------------------------------
1.264 +//
1.265 +// Raw Binary Data Formats, as loaded from the ICU data file,
1.266 +// or as built by the builder.
1.267 +//
1.268 +//---------------------------------------------------------------------------------------
1.269 +struct SpoofDataHeader {
1.270 + int32_t fMagic; // (0x3845fdef)
1.271 + uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo
1.272 + // if there is one associated with this data.
1.273 + int32_t fLength; // Total lenght in bytes of this spoof data,
1.274 + // including all sections, not just the header.
1.275 +
1.276 + // The following four sections refer to data representing the confusable data
1.277 + // from the Unicode.org data from "confusables.txt"
1.278 +
1.279 + int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *)
1.280 + int32_t fCFUKeysSize; // number of entries in keys table (32 bits each)
1.281 +
1.282 + // TODO: change name to fCFUValues, for consistency.
1.283 + int32_t fCFUStringIndex; // byte offset to String Indexes table
1.284 + int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each)
1.285 + // (number of entries must be same as in Keys table
1.286 +
1.287 + int32_t fCFUStringTable; // byte offset of String table
1.288 + int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars)
1.289 +
1.290 + int32_t fCFUStringLengths; // byte offset to String Lengths table
1.291 + int32_t fCFUStringLengthsSize; // number of entries in lengths table. (2 x 16 bits each)
1.292 +
1.293 +
1.294 + // The following sections are for data from confusablesWholeScript.txt
1.295 +
1.296 + int32_t fAnyCaseTrie; // byte offset to the serialized Any Case Trie
1.297 + int32_t fAnyCaseTrieLength; // Length (bytes) of the serialized Any Case Trie
1.298 +
1.299 + int32_t fLowerCaseTrie; // byte offset to the serialized Lower Case Trie
1.300 + int32_t fLowerCaseTrieLength; // Length (bytes) of the serialized Lower Case Trie
1.301 +
1.302 + int32_t fScriptSets; // byte offset to array of ScriptSets
1.303 + int32_t fScriptSetsLength; // Number of ScriptSets (24 bytes each)
1.304 +
1.305 +
1.306 + // The following sections are for data from xidmodifications.txt
1.307 +
1.308 +
1.309 + int32_t unused[15]; // Padding, Room for Expansion
1.310 +
1.311 + };
1.312 +
1.313 +
1.314 +
1.315 +
1.316 +//
1.317 +// Structure for the Whole Script Confusable Data
1.318 +// See Unicode UAX-39, Unicode Security Mechanisms, for a description of the
1.319 +// Whole Script confusable data
1.320 +//
1.321 +// The data provides mappings from code points to a set of scripts
1.322 +// that contain characters that might be confused with the code point.
1.323 +// There are two mappings, one for lower case only, and one for characters
1.324 +// of any case.
1.325 +//
1.326 +// The actual data consists of a utrie2 to map from a code point to an offset,
1.327 +// and an array of UScriptSets (essentially bit maps) that is indexed
1.328 +// by the offsets obtained from the Trie.
1.329 +//
1.330 +//
1.331 +
1.332 +
1.333 +U_NAMESPACE_END
1.334 +#endif /* __cplusplus */
1.335 +
1.336 +/**
1.337 + * Endianness swap function for binary spoof data.
1.338 + * @internal
1.339 + */
1.340 +U_CAPI int32_t U_EXPORT2
1.341 +uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
1.342 + UErrorCode *status);
1.343 +
1.344 +
1.345 +#endif
1.346 +
1.347 +#endif /* USPOOFIM_H */
1.348 +
