michael@0: /*
michael@0: **********************************************************************
michael@0: * Copyright (C) 2005-2012, International Business Machines
michael@0: * Corporation and others. All Rights Reserved.
michael@0: **********************************************************************
michael@0: */
michael@0:
michael@0: #ifndef __CSRMBCS_H
michael@0: #define __CSRMBCS_H
michael@0:
michael@0: #include "unicode/utypes.h"
michael@0:
michael@0: #if !UCONFIG_NO_CONVERSION
michael@0:
michael@0: #include "csrecog.h"
michael@0:
michael@0: U_NAMESPACE_BEGIN
michael@0:
michael@0: // "Character" iterated character class.
michael@0: // Recognizers for specific mbcs encodings make their "characters" available
michael@0: // by providing a nextChar() function that fills in an instance of IteratedChar
michael@0: // with the next char from the input.
michael@0: // The returned characters are not converted to Unicode, but remain as the raw
michael@0: // bytes (concatenated into an int) from the codepage data.
michael@0: //
michael@0: // For Asian charsets, use the raw input rather than the input that has been
michael@0: // stripped of markup. Detection only considers multi-byte chars, effectively
michael@0: // stripping markup anyway, and double byte chars do occur in markup too.
michael@0: //
michael@0: class IteratedChar : public UMemory
michael@0: {
michael@0: public:
michael@0: uint32_t charValue; // 1-4 bytes from the raw input data
michael@0: int32_t index;
michael@0: int32_t nextIndex;
michael@0: UBool error;
michael@0: UBool done;
michael@0:
michael@0: public:
michael@0: IteratedChar();
michael@0: //void reset();
michael@0: int32_t nextByte(InputText* det);
michael@0: };
michael@0:
michael@0:
michael@0: class CharsetRecog_mbcs : public CharsetRecognizer {
michael@0:
michael@0: protected:
michael@0: /**
michael@0: * Test the match of this charset with the input text data
michael@0: * which is obtained via the CharsetDetector object.
michael@0: *
michael@0: * @param det The CharsetDetector, which contains the input text
michael@0: * to be checked for being in this charset.
michael@0: * @return Two values packed into one int (Damn java, anyhow)
michael@0: *
michael@0: * bits 0-7: the match confidence, ranging from 0-100
michael@0: *
michael@0: * bits 8-15: The match reason, an enum-like value.
michael@0: */
michael@0: int32_t match_mbcs(InputText* det, const uint16_t commonChars[], int32_t commonCharsLen) const;
michael@0:
michael@0: public:
michael@0:
michael@0: virtual ~CharsetRecog_mbcs();
michael@0:
michael@0: /**
michael@0: * Get the IANA name of this charset.
michael@0: * @return the charset name.
michael@0: */
michael@0:
michael@0: const char *getName() const = 0;
michael@0: const char *getLanguage() const = 0;
michael@0: UBool match(InputText* input, CharsetMatch *results) const = 0;
michael@0:
michael@0: /**
michael@0: * Get the next character (however many bytes it is) from the input data
michael@0: * Subclasses for specific charset encodings must implement this function
michael@0: * to get characters according to the rules of their encoding scheme.
michael@0: *
michael@0: * This function is not a method of class IteratedChar only because
michael@0: * that would require a lot of extra derived classes, which is awkward.
michael@0: * @param it The IteratedChar "struct" into which the returned char is placed.
michael@0: * @param det The charset detector, which is needed to get at the input byte data
michael@0: * being iterated over.
michael@0: * @return True if a character was returned, false at end of input.
michael@0: */
michael@0: virtual UBool nextChar(IteratedChar *it, InputText *textIn) const = 0;
michael@0:
michael@0: };
michael@0:
michael@0:
michael@0: /**
michael@0: * Shift-JIS charset recognizer.
michael@0: *
michael@0: */
michael@0: class CharsetRecog_sjis : public CharsetRecog_mbcs {
michael@0: public:
michael@0: virtual ~CharsetRecog_sjis();
michael@0:
michael@0: UBool nextChar(IteratedChar *it, InputText *det) const;
michael@0:
michael@0: UBool match(InputText* input, CharsetMatch *results) const;
michael@0:
michael@0: const char *getName() const;
michael@0: const char *getLanguage() const;
michael@0:
michael@0: };
michael@0:
michael@0:
michael@0: /**
michael@0: * EUC charset recognizers. One abstract class that provides the common function
michael@0: * for getting the next character according to the EUC encoding scheme,
michael@0: * and nested derived classes for EUC_KR, EUC_JP, EUC_CN.
michael@0: *
michael@0: */
michael@0: class CharsetRecog_euc : public CharsetRecog_mbcs
michael@0: {
michael@0: public:
michael@0: virtual ~CharsetRecog_euc();
michael@0:
michael@0: const char *getName() const = 0;
michael@0: const char *getLanguage() const = 0;
michael@0:
michael@0: UBool match(InputText* input, CharsetMatch *results) const = 0;
michael@0: /*
michael@0: * (non-Javadoc)
michael@0: * Get the next character value for EUC based encodings.
michael@0: * Character "value" is simply the raw bytes that make up the character
michael@0: * packed into an int.
michael@0: */
michael@0: UBool nextChar(IteratedChar *it, InputText *det) const;
michael@0: };
michael@0:
michael@0: /**
michael@0: * The charset recognize for EUC-JP. A singleton instance of this class
michael@0: * is created and kept by the public CharsetDetector class
michael@0: */
michael@0: class CharsetRecog_euc_jp : public CharsetRecog_euc
michael@0: {
michael@0: public:
michael@0: virtual ~CharsetRecog_euc_jp();
michael@0:
michael@0: const char *getName() const;
michael@0: const char *getLanguage() const;
michael@0:
michael@0: UBool match(InputText* input, CharsetMatch *results) const;
michael@0: };
michael@0:
michael@0: /**
michael@0: * The charset recognize for EUC-KR. A singleton instance of this class
michael@0: * is created and kept by the public CharsetDetector class
michael@0: */
michael@0: class CharsetRecog_euc_kr : public CharsetRecog_euc
michael@0: {
michael@0: public:
michael@0: virtual ~CharsetRecog_euc_kr();
michael@0:
michael@0: const char *getName() const;
michael@0: const char *getLanguage() const;
michael@0:
michael@0: UBool match(InputText* input, CharsetMatch *results) const;
michael@0: };
michael@0:
michael@0: /**
michael@0: *
michael@0: * Big5 charset recognizer.
michael@0: *
michael@0: */
michael@0: class CharsetRecog_big5 : public CharsetRecog_mbcs
michael@0: {
michael@0: public:
michael@0: virtual ~CharsetRecog_big5();
michael@0:
michael@0: UBool nextChar(IteratedChar* it, InputText* det) const;
michael@0:
michael@0: const char *getName() const;
michael@0: const char *getLanguage() const;
michael@0:
michael@0: UBool match(InputText* input, CharsetMatch *results) const;
michael@0: };
michael@0:
michael@0:
michael@0: /**
michael@0: *
michael@0: * GB-18030 recognizer. Uses simplified Chinese statistics.
michael@0: *
michael@0: */
michael@0: class CharsetRecog_gb_18030 : public CharsetRecog_mbcs
michael@0: {
michael@0: public:
michael@0: virtual ~CharsetRecog_gb_18030();
michael@0:
michael@0: UBool nextChar(IteratedChar* it, InputText* det) const;
michael@0:
michael@0: const char *getName() const;
michael@0: const char *getLanguage() const;
michael@0:
michael@0: UBool match(InputText* input, CharsetMatch *results) const;
michael@0: };
michael@0:
michael@0: U_NAMESPACE_END
michael@0:
michael@0: #endif
michael@0: #endif /* __CSRMBCS_H */