The Tor Browser / file comparison

comparison: intl/icu/source/i18n/unicode/ucsdet.h

intl/icu/source/i18n/unicode/ucsdet.h

branch
TOR_BUG_3246
changeset 4
fc2d59ddac77
equal deleted inserted replaced
-1:000000000000 0:5eb58a877171
1 /*
2 **********************************************************************
3 * Copyright (C) 2005-2013, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 **********************************************************************
6 * file name: ucsdet.h
7 * encoding: US-ASCII
8 * indentation:4
9 *
10 * created on: 2005Aug04
11 * created by: Andy Heninger
12 *
13 * ICU Character Set Detection, API for C
14 *
15 * Draft version 18 Oct 2005
16 *
17 */
18
19 #ifndef __UCSDET_H
20 #define __UCSDET_H
21
22 #include "unicode/utypes.h"
23
24 #if !UCONFIG_NO_CONVERSION
25
26 #include "unicode/localpointer.h"
27 #include "unicode/uenum.h"
28
29 /**
30 * \file
31 * \brief C API: Charset Detection API
32 *
33 * This API provides a facility for detecting the
34 * charset or encoding of character data in an unknown text format.
35 * The input data can be from an array of bytes.
36 * <p>
37 * Character set detection is at best an imprecise operation. The detection
38 * process will attempt to identify the charset that best matches the characteristics
39 * of the byte data, but the process is partly statistical in nature, and
40 * the results can not be guaranteed to always be correct.
41 * <p>
42 * For best accuracy in charset detection, the input data should be primarily
43 * in a single language, and a minimum of a few hundred bytes worth of plain text
44 * in the language are needed. The detection process will attempt to
45 * ignore html or xml style markup that could otherwise obscure the content.
46 */
47
48
49 struct UCharsetDetector;
50 /**
51 * Structure representing a charset detector
52 * @stable ICU 3.6
53 */
54 typedef struct UCharsetDetector UCharsetDetector;
55
56 struct UCharsetMatch;
57 /**
58 * Opaque structure representing a match that was identified
59 * from a charset detection operation.
60 * @stable ICU 3.6
61 */
62 typedef struct UCharsetMatch UCharsetMatch;
63
64 /**
65 * Open a charset detector.
66 *
67 * @param status Any error conditions occurring during the open
68 * operation are reported back in this variable.
69 * @return the newly opened charset detector.
70 * @stable ICU 3.6
71 */
72 U_STABLE UCharsetDetector * U_EXPORT2
73 ucsdet_open(UErrorCode *status);
74
75 /**
76 * Close a charset detector. All storage and any other resources
77 * owned by this charset detector will be released. Failure to
78 * close a charset detector when finished with it can result in
79 * memory leaks in the application.
80 *
81 * @param ucsd The charset detector to be closed.
82 * @stable ICU 3.6
83 */
84 U_STABLE void U_EXPORT2
85 ucsdet_close(UCharsetDetector *ucsd);
86
87 #if U_SHOW_CPLUSPLUS_API
88
89 U_NAMESPACE_BEGIN
90
91 /**
92 * \class LocalUCharsetDetectorPointer
93 * "Smart pointer" class, closes a UCharsetDetector via ucsdet_close().
94 * For most methods see the LocalPointerBase base class.
95 *
96 * @see LocalPointerBase
97 * @see LocalPointer
98 * @stable ICU 4.4
99 */
100 U_DEFINE_LOCAL_OPEN_POINTER(LocalUCharsetDetectorPointer, UCharsetDetector, ucsdet_close);
101
102 U_NAMESPACE_END
103
104 #endif
105
106 /**
107 * Set the input byte data whose charset is to detected.
108 *
109 * Ownership of the input text byte array remains with the caller.
110 * The input string must not be altered or deleted until the charset
111 * detector is either closed or reset to refer to different input text.
112 *
113 * @param ucsd the charset detector to be used.
114 * @param textIn the input text of unknown encoding. .
115 * @param len the length of the input text, or -1 if the text
116 * is NUL terminated.
117 * @param status any error conditions are reported back in this variable.
118 *
119 * @stable ICU 3.6
120 */
121 U_STABLE void U_EXPORT2
122 ucsdet_setText(UCharsetDetector *ucsd, const char *textIn, int32_t len, UErrorCode *status);
123
124
125 /** Set the declared encoding for charset detection.
126 * The declared encoding of an input text is an encoding obtained
127 * by the user from an http header or xml declaration or similar source that
128 * can be provided as an additional hint to the charset detector.
129 *
130 * How and whether the declared encoding will be used during the
131 * detection process is TBD.
132 *
133 * @param ucsd the charset detector to be used.
134 * @param encoding an encoding for the current data obtained from
135 * a header or declaration or other source outside
136 * of the byte data itself.
137 * @param length the length of the encoding name, or -1 if the name string
138 * is NUL terminated.
139 * @param status any error conditions are reported back in this variable.
140 *
141 * @stable ICU 3.6
142 */
143 U_STABLE void U_EXPORT2
144 ucsdet_setDeclaredEncoding(UCharsetDetector *ucsd, const char *encoding, int32_t length, UErrorCode *status);
145
146
147 /**
148 * Return the charset that best matches the supplied input data.
149 *
150 * Note though, that because the detection
151 * only looks at the start of the input data,
152 * there is a possibility that the returned charset will fail to handle
153 * the full set of input data.
154 * <p>
155 * The returned UCharsetMatch object is owned by the UCharsetDetector.
156 * It will remain valid until the detector input is reset, or until
157 * the detector is closed.
158 * <p>
159 * The function will fail if
160 * <ul>
161 * <li>no charset appears to match the data.</li>
162 * <li>no input text has been provided</li>
163 * </ul>
164 *
165 * @param ucsd the charset detector to be used.
166 * @param status any error conditions are reported back in this variable.
167 * @return a UCharsetMatch representing the best matching charset,
168 * or NULL if no charset matches the byte data.
169 *
170 * @stable ICU 3.6
171 */
172 U_STABLE const UCharsetMatch * U_EXPORT2
173 ucsdet_detect(UCharsetDetector *ucsd, UErrorCode *status);
174
175
176 /**
177 * Find all charset matches that appear to be consistent with the input,
178 * returning an array of results. The results are ordered with the
179 * best quality match first.
180 *
181 * Because the detection only looks at a limited amount of the
182 * input byte data, some of the returned charsets may fail to handle
183 * the all of input data.
184 * <p>
185 * The returned UCharsetMatch objects are owned by the UCharsetDetector.
186 * They will remain valid until the detector is closed or modified
187 *
188 * <p>
189 * Return an error if
190 * <ul>
191 * <li>no charsets appear to match the input data.</li>
192 * <li>no input text has been provided</li>
193 * </ul>
194 *
195 * @param ucsd the charset detector to be used.
196 * @param matchesFound pointer to a variable that will be set to the
197 * number of charsets identified that are consistent with
198 * the input data. Output only.
199 * @param status any error conditions are reported back in this variable.
200 * @return A pointer to an array of pointers to UCharSetMatch objects.
201 * This array, and the UCharSetMatch instances to which it refers,
202 * are owned by the UCharsetDetector, and will remain valid until
203 * the detector is closed or modified.
204 * @stable ICU 3.6
205 */
206 U_STABLE const UCharsetMatch ** U_EXPORT2
207 ucsdet_detectAll(UCharsetDetector *ucsd, int32_t *matchesFound, UErrorCode *status);
208
209
210
211 /**
212 * Get the name of the charset represented by a UCharsetMatch.
213 *
214 * The storage for the returned name string is owned by the
215 * UCharsetMatch, and will remain valid while the UCharsetMatch
216 * is valid.
217 *
218 * The name returned is suitable for use with the ICU conversion APIs.
219 *
220 * @param ucsm The charset match object.
221 * @param status Any error conditions are reported back in this variable.
222 * @return The name of the matching charset.
223 *
224 * @stable ICU 3.6
225 */
226 U_STABLE const char * U_EXPORT2
227 ucsdet_getName(const UCharsetMatch *ucsm, UErrorCode *status);
228
229 /**
230 * Get a confidence number for the quality of the match of the byte
231 * data with the charset. Confidence numbers range from zero to 100,
232 * with 100 representing complete confidence and zero representing
233 * no confidence.
234 *
235 * The confidence values are somewhat arbitrary. They define an
236 * an ordering within the results for any single detection operation
237 * but are not generally comparable between the results for different input.
238 *
239 * A confidence value of ten does have a general meaning - it is used
240 * for charsets that can represent the input data, but for which there
241 * is no other indication that suggests that the charset is the correct one.
242 * Pure 7 bit ASCII data, for example, is compatible with a
243 * great many charsets, most of which will appear as possible matches
244 * with a confidence of 10.
245 *
246 * @param ucsm The charset match object.
247 * @param status Any error conditions are reported back in this variable.
248 * @return A confidence number for the charset match.
249 *
250 * @stable ICU 3.6
251 */
252 U_STABLE int32_t U_EXPORT2
253 ucsdet_getConfidence(const UCharsetMatch *ucsm, UErrorCode *status);
254
255 /**
256 * Get the RFC 3066 code for the language of the input data.
257 *
258 * The Charset Detection service is intended primarily for detecting
259 * charsets, not language. For some, but not all, charsets, a language is
260 * identified as a byproduct of the detection process, and that is what
261 * is returned by this function.
262 *
263 * CAUTION:
264 * 1. Language information is not available for input data encoded in
265 * all charsets. In particular, no language is identified
266 * for UTF-8 input data.
267 *
268 * 2. Closely related languages may sometimes be confused.
269 *
270 * If more accurate language detection is required, a linguistic
271 * analysis package should be used.
272 *
273 * The storage for the returned name string is owned by the
274 * UCharsetMatch, and will remain valid while the UCharsetMatch
275 * is valid.
276 *
277 * @param ucsm The charset match object.
278 * @param status Any error conditions are reported back in this variable.
279 * @return The RFC 3066 code for the language of the input data, or
280 * an empty string if the language could not be determined.
281 *
282 * @stable ICU 3.6
283 */
284 U_STABLE const char * U_EXPORT2
285 ucsdet_getLanguage(const UCharsetMatch *ucsm, UErrorCode *status);
286
287
288 /**
289 * Get the entire input text as a UChar string, placing it into
290 * a caller-supplied buffer. A terminating
291 * NUL character will be appended to the buffer if space is available.
292 *
293 * The number of UChars in the output string, not including the terminating
294 * NUL, is returned.
295 *
296 * If the supplied buffer is smaller than required to hold the output,
297 * the contents of the buffer are undefined. The full output string length
298 * (in UChars) is returned as always, and can be used to allocate a buffer
299 * of the correct size.
300 *
301 *
302 * @param ucsm The charset match object.
303 * @param buf A UChar buffer to be filled with the converted text data.
304 * @param cap The capacity of the buffer in UChars.
305 * @param status Any error conditions are reported back in this variable.
306 * @return The number of UChars in the output string.
307 *
308 * @stable ICU 3.6
309 */
310 U_STABLE int32_t U_EXPORT2
311 ucsdet_getUChars(const UCharsetMatch *ucsm,
312 UChar *buf, int32_t cap, UErrorCode *status);
313
314
315
316 /**
317 * Get an iterator over the set of all detectable charsets -
318 * over the charsets that are known to the charset detection
319 * service.
320 *
321 * The returned UEnumeration provides access to the names of
322 * the charsets.
323 *
324 * <p>
325 * The state of the Charset detector that is passed in does not
326 * affect the result of this function, but requiring a valid, open
327 * charset detector as a parameter insures that the charset detection
328 * service has been safely initialized and that the required detection
329 * data is available.
330 *
331 * <p>
332 * <b>Note:</b> Multiple different charset encodings in a same family may use
333 * a single shared name in this implementation. For example, this method returns
334 * an array including "ISO-8859-1" (ISO Latin 1), but not including "windows-1252"
335 * (Windows Latin 1). However, actual detection result could be "windows-1252"
336 * when the input data matches Latin 1 code points with any points only available
337 * in "windows-1252".
338 *
339 * @param ucsd a Charset detector.
340 * @param status Any error conditions are reported back in this variable.
341 * @return an iterator providing access to the detectable charset names.
342 * @stable ICU 3.6
343 */
344 U_STABLE UEnumeration * U_EXPORT2
345 ucsdet_getAllDetectableCharsets(const UCharsetDetector *ucsd, UErrorCode *status);
346
347 /**
348 * Test whether input filtering is enabled for this charset detector.
349 * Input filtering removes text that appears to be HTML or xml
350 * markup from the input before applying the code page detection
351 * heuristics.
352 *
353 * @param ucsd The charset detector to check.
354 * @return TRUE if filtering is enabled.
355 * @stable ICU 3.6
356 */
357
358 U_STABLE UBool U_EXPORT2
359 ucsdet_isInputFilterEnabled(const UCharsetDetector *ucsd);
360
361
362 /**
363 * Enable filtering of input text. If filtering is enabled,
364 * text within angle brackets ("<" and ">") will be removed
365 * before detection, which will remove most HTML or xml markup.
366 *
367 * @param ucsd the charset detector to be modified.
368 * @param filter <code>true</code> to enable input text filtering.
369 * @return The previous setting.
370 *
371 * @stable ICU 3.6
372 */
373 U_STABLE UBool U_EXPORT2
374 ucsdet_enableInputFilter(UCharsetDetector *ucsd, UBool filter);
375
376 #ifndef U_HIDE_INTERNAL_API
377 /**
378 * Get an iterator over the set of detectable charsets -
379 * over the charsets that are enabled by the specified charset detector.
380 *
381 * The returned UEnumeration provides access to the names of
382 * the charsets.
383 *
384 * @param ucsd a Charset detector.
385 * @param status Any error conditions are reported back in this variable.
386 * @return an iterator providing access to the detectable charset names by
387 * the specified charset detector.
388 * @internal
389 */
390 U_INTERNAL UEnumeration * U_EXPORT2
391 ucsdet_getDetectableCharsets(const UCharsetDetector *ucsd, UErrorCode *status);
392
393 /**
394 * Enable or disable individual charset encoding.
395 * A name of charset encoding must be included in the names returned by
396 * {@link #getAllDetectableCharsets()}.
397 *
398 * @param ucsd a Charset detector.
399 * @param encoding encoding the name of charset encoding.
400 * @param enabled <code>TRUE</code> to enable, or <code>FALSE</code> to disable the
401 * charset encoding.
402 * @param status receives the return status. When the name of charset encoding
403 * is not supported, U_ILLEGAL_ARGUMENT_ERROR is set.
404 * @internal
405 */
406 U_INTERNAL void U_EXPORT2
407 ucsdet_setDetectableCharset(UCharsetDetector *ucsd, const char *encoding, UBool enabled, UErrorCode *status);
408 #endif /* U_HIDE_INTERNAL_API */
409
410 #endif
411 #endif /* __UCSDET_H */
412
413

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial