1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/intl/icu/source/common/unicode/utf.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,223 @@
1.4 +/*
1.5 +*******************************************************************************
1.6 +*
1.7 +* Copyright (C) 1999-2011, International Business Machines
1.8 +* Corporation and others. All Rights Reserved.
1.9 +*
1.10 +*******************************************************************************
1.11 +* file name: utf.h
1.12 +* encoding: US-ASCII
1.13 +* tab size: 8 (not used)
1.14 +* indentation:4
1.15 +*
1.16 +* created on: 1999sep09
1.17 +* created by: Markus W. Scherer
1.18 +*/
1.19 +
1.20 +/**
1.21 + * \file
1.22 + * \brief C API: Code point macros
1.23 + *
1.24 + * This file defines macros for checking whether a code point is
1.25 + * a surrogate or a non-character etc.
1.26 + *
1.27 + * The UChar and UChar32 data types for Unicode code units and code points
1.28 + * are defined in umachine.h because they can be machine-dependent.
1.29 + *
1.30 + * If U_NO_DEFAULT_INCLUDE_UTF_HEADERS is 0 then utf.h is included by utypes.h
1.31 + * and itself includes utf8.h and utf16.h after some
1.32 + * common definitions.
1.33 + * If U_NO_DEFAULT_INCLUDE_UTF_HEADERS is 1 then each of these headers must be
1.34 + * included explicitly if their definitions are used.
1.35 + *
1.36 + * utf8.h and utf16.h define macros for efficiently getting code points
1.37 + * in and out of UTF-8/16 strings.
1.38 + * utf16.h macros have "U16_" prefixes.
1.39 + * utf8.h defines similar macros with "U8_" prefixes for UTF-8 string handling.
1.40 + *
1.41 + * ICU mostly processes 16-bit Unicode strings.
1.42 + * Most of the time, such strings are well-formed UTF-16.
1.43 + * Single, unpaired surrogates must be handled as well, and are treated in ICU
1.44 + * like regular code points where possible.
1.45 + * (Pairs of surrogate code points are indistinguishable from supplementary
1.46 + * code points encoded as pairs of supplementary code units.)
1.47 + *
1.48 + * In fact, almost all Unicode code points in normal text (>99%)
1.49 + * are on the BMP (<=U+ffff) and even <=U+d7ff.
1.50 + * ICU functions handle supplementary code points (U+10000..U+10ffff)
1.51 + * but are optimized for the much more frequently occurring BMP code points.
1.52 + *
1.53 + * umachine.h defines UChar to be an unsigned 16-bit integer.
1.54 + * Where available, UChar is defined to be a char16_t
1.55 + * or a wchar_t (if that is an unsigned 16-bit type), otherwise uint16_t.
1.56 + *
1.57 + * UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit
1.58 + * Unicode code point (Unicode scalar value, 0..0x10ffff).
1.59 + * Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as
1.60 + * the definition of UChar. For details see the documentation for UChar32 itself.
1.61 + *
1.62 + * utf.h defines a small number of C macros for single Unicode code points.
1.63 + * These are simple checks for surrogates and non-characters.
1.64 + * For actual Unicode character properties see uchar.h.
1.65 + *
1.66 + * By default, string operations must be done with error checking in case
1.67 + * a string is not well-formed UTF-16.
1.68 + * The macros will detect if a surrogate code unit is unpaired
1.69 + * (lead unit without trail unit or vice versa) and just return the unit itself
1.70 + * as the code point.
1.71 + *
1.72 + * The regular "safe" macros require that the initial, passed-in string index
1.73 + * is within bounds. They only check the index when they read more than one
1.74 + * code unit. This is usually done with code similar to the following loop:
1.75 + * <pre>while(i<length) {
1.76 + * U16_NEXT(s, i, length, c);
1.77 + * // use c
1.78 + * }</pre>
1.79 + *
1.80 + * When it is safe to assume that text is well-formed UTF-16
1.81 + * (does not contain single, unpaired surrogates), then one can use
1.82 + * U16_..._UNSAFE macros.
1.83 + * These do not check for proper code unit sequences or truncated text and may
1.84 + * yield wrong results or even cause a crash if they are used with "malformed"
1.85 + * text.
1.86 + * In practice, U16_..._UNSAFE macros will produce slightly less code but
1.87 + * should not be faster because the processing is only different when a
1.88 + * surrogate code unit is detected, which will be rare.
1.89 + *
1.90 + * Similarly for UTF-8, there are "safe" macros without a suffix,
1.91 + * and U8_..._UNSAFE versions.
1.92 + * The performance differences are much larger here because UTF-8 provides so
1.93 + * many opportunities for malformed sequences.
1.94 + * The unsafe UTF-8 macros are entirely implemented inside the macro definitions
1.95 + * and are fast, while the safe UTF-8 macros call functions for all but the
1.96 + * trivial (ASCII) cases.
1.97 + * (ICU 3.6 optimizes U8_NEXT() and U8_APPEND() to handle most other common
1.98 + * characters inline as well.)
1.99 + *
1.100 + * Unlike with UTF-16, malformed sequences cannot be expressed with distinct
1.101 + * code point values (0..U+10ffff). They are indicated with negative values instead.
1.102 + *
1.103 + * For more information see the ICU User Guide Strings chapter
1.104 + * (http://userguide.icu-project.org/strings).
1.105 + *
1.106 + * <em>Usage:</em>
1.107 + * ICU coding guidelines for if() statements should be followed when using these macros.
1.108 + * Compound statements (curly braces {}) must be used for if-else-while...
1.109 + * bodies and all macro statements should be terminated with semicolon.
1.110 + *
1.111 + * @stable ICU 2.4
1.112 + */
1.113 +
1.114 +#ifndef __UTF_H__
1.115 +#define __UTF_H__
1.116 +
1.117 +#include "unicode/umachine.h"
1.118 +/* include the utfXX.h after the following definitions */
1.119 +
1.120 +/* single-code point definitions -------------------------------------------- */
1.121 +
1.122 +/**
1.123 + * Is this code point a Unicode noncharacter?
1.124 + * @param c 32-bit code point
1.125 + * @return TRUE or FALSE
1.126 + * @stable ICU 2.4
1.127 + */
1.128 +#define U_IS_UNICODE_NONCHAR(c) \
1.129 + ((c)>=0xfdd0 && \
1.130 + ((uint32_t)(c)<=0xfdef || ((c)&0xfffe)==0xfffe) && \
1.131 + (uint32_t)(c)<=0x10ffff)
1.132 +
1.133 +/**
1.134 + * Is c a Unicode code point value (0..U+10ffff)
1.135 + * that can be assigned a character?
1.136 + *
1.137 + * Code points that are not characters include:
1.138 + * - single surrogate code points (U+d800..U+dfff, 2048 code points)
1.139 + * - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points)
1.140 + * - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points)
1.141 + * - the highest Unicode code point value is U+10ffff
1.142 + *
1.143 + * This means that all code points below U+d800 are character code points,
1.144 + * and that boundary is tested first for performance.
1.145 + *
1.146 + * @param c 32-bit code point
1.147 + * @return TRUE or FALSE
1.148 + * @stable ICU 2.4
1.149 + */
1.150 +#define U_IS_UNICODE_CHAR(c) \
1.151 + ((uint32_t)(c)<0xd800 || \
1.152 + ((uint32_t)(c)>0xdfff && \
1.153 + (uint32_t)(c)<=0x10ffff && \
1.154 + !U_IS_UNICODE_NONCHAR(c)))
1.155 +
1.156 +/**
1.157 + * Is this code point a BMP code point (U+0000..U+ffff)?
1.158 + * @param c 32-bit code point
1.159 + * @return TRUE or FALSE
1.160 + * @stable ICU 2.8
1.161 + */
1.162 +#define U_IS_BMP(c) ((uint32_t)(c)<=0xffff)
1.163 +
1.164 +/**
1.165 + * Is this code point a supplementary code point (U+10000..U+10ffff)?
1.166 + * @param c 32-bit code point
1.167 + * @return TRUE or FALSE
1.168 + * @stable ICU 2.8
1.169 + */
1.170 +#define U_IS_SUPPLEMENTARY(c) ((uint32_t)((c)-0x10000)<=0xfffff)
1.171 +
1.172 +/**
1.173 + * Is this code point a lead surrogate (U+d800..U+dbff)?
1.174 + * @param c 32-bit code point
1.175 + * @return TRUE or FALSE
1.176 + * @stable ICU 2.4
1.177 + */
1.178 +#define U_IS_LEAD(c) (((c)&0xfffffc00)==0xd800)
1.179 +
1.180 +/**
1.181 + * Is this code point a trail surrogate (U+dc00..U+dfff)?
1.182 + * @param c 32-bit code point
1.183 + * @return TRUE or FALSE
1.184 + * @stable ICU 2.4
1.185 + */
1.186 +#define U_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00)
1.187 +
1.188 +/**
1.189 + * Is this code point a surrogate (U+d800..U+dfff)?
1.190 + * @param c 32-bit code point
1.191 + * @return TRUE or FALSE
1.192 + * @stable ICU 2.4
1.193 + */
1.194 +#define U_IS_SURROGATE(c) (((c)&0xfffff800)==0xd800)
1.195 +
1.196 +/**
1.197 + * Assuming c is a surrogate code point (U_IS_SURROGATE(c)),
1.198 + * is it a lead surrogate?
1.199 + * @param c 32-bit code point
1.200 + * @return TRUE or FALSE
1.201 + * @stable ICU 2.4
1.202 + */
1.203 +#define U_IS_SURROGATE_LEAD(c) (((c)&0x400)==0)
1.204 +
1.205 +/**
1.206 + * Assuming c is a surrogate code point (U_IS_SURROGATE(c)),
1.207 + * is it a trail surrogate?
1.208 + * @param c 32-bit code point
1.209 + * @return TRUE or FALSE
1.210 + * @stable ICU 4.2
1.211 + */
1.212 +#define U_IS_SURROGATE_TRAIL(c) (((c)&0x400)!=0)
1.213 +
1.214 +/* include the utfXX.h ------------------------------------------------------ */
1.215 +
1.216 +#if !U_NO_DEFAULT_INCLUDE_UTF_HEADERS
1.217 +
1.218 +#include "unicode/utf8.h"
1.219 +#include "unicode/utf16.h"
1.220 +
1.221 +/* utf_old.h contains deprecated, pre-ICU 2.4 definitions */
1.222 +#include "unicode/utf_old.h"
1.223 +
1.224 +#endif /* !U_NO_DEFAULT_INCLUDE_UTF_HEADERS */
1.225 +
1.226 +#endif /* __UTF_H__ */
