michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* vim: set ts=2 et sw=2 tw=80: */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include "nsUCSupport.h"
michael@0: #include "nsUTF8ToUnicode.h"
michael@0: #include "mozilla/SSE.h"
michael@0: #include "nsCharTraits.h"
michael@0: #include <algorithm>
michael@0: 
michael@0: #define UNICODE_BYTE_ORDER_MARK    0xFEFF
michael@0: 
michael@0: static char16_t* EmitSurrogatePair(uint32_t ucs4, char16_t* aDest)
michael@0: {
michael@0:   NS_ASSERTION(ucs4 > 0xFFFF, "Should be a supplementary character");
michael@0:   ucs4 -= 0x00010000;
michael@0:   *aDest++ = 0xD800 | (0x000003FF & (ucs4 >> 10));
michael@0:   *aDest++ = 0xDC00 | (0x000003FF & ucs4);
michael@0:   return aDest;
michael@0: }
michael@0: 
michael@0: //----------------------------------------------------------------------
michael@0: // Class nsUTF8ToUnicode [implementation]
michael@0: 
michael@0: nsUTF8ToUnicode::nsUTF8ToUnicode()
michael@0: : nsBasicDecoderSupport()
michael@0: {
michael@0:   Reset();
michael@0: }
michael@0: 
michael@0: //----------------------------------------------------------------------
michael@0: // Subclassing of nsTableDecoderSupport class [implementation]
michael@0: 
michael@0: /**
michael@0:  * Normally the maximum length of the output of the UTF8 decoder in UTF16
michael@0:  *  code units is the same as the length of the input in UTF8 code units,
michael@0:  *  since 1-byte, 2-byte and 3-byte UTF-8 sequences decode to a single
michael@0:  *  UTF-16 character, and 4-byte UTF-8 sequences decode to a surrogate pair.
michael@0:  *
michael@0:  * However, there is an edge case where the output can be longer than the
michael@0:  *  input: if the previous buffer ended with an incomplete multi-byte
michael@0:  *  sequence and this buffer does not begin with a valid continuation
michael@0:  *  byte, we will return NS_ERROR_ILLEGAL_INPUT and the caller may insert a
michael@0:  *  replacement character in the output buffer which corresponds to no
michael@0:  *  character in the input buffer. So in the worst case the destination
michael@0:  *  will need to be one code unit longer than the source.
michael@0:  *  See bug 301797.
michael@0:  */
michael@0: NS_IMETHODIMP nsUTF8ToUnicode::GetMaxLength(const char * aSrc,
michael@0:                                             int32_t aSrcLength,
michael@0:                                             int32_t * aDestLength)
michael@0: {
michael@0:   *aDestLength = aSrcLength + 1;
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: 
michael@0: //----------------------------------------------------------------------
michael@0: // Subclassing of nsBasicDecoderSupport class [implementation]
michael@0: 
michael@0: NS_IMETHODIMP nsUTF8ToUnicode::Reset()
michael@0: {
michael@0: 
michael@0:   mUcs4  = 0;     // cached Unicode character
michael@0:   mState = 0;     // cached expected number of octets after the current octet
michael@0:                   // until the beginning of the next UTF8 character sequence
michael@0:   mBytes = 1;     // cached expected number of octets in the current sequence
michael@0:   mFirst = true;
michael@0: 
michael@0:   return NS_OK;
michael@0: 
michael@0: }
michael@0: 
michael@0: //----------------------------------------------------------------------
michael@0: // Subclassing of nsBasicDecoderSupport class [implementation]
michael@0: 
michael@0: // Fast ASCII -> UTF16 inner loop implementations
michael@0: //
michael@0: // Convert_ascii_run will update src and dst to the new values, and
michael@0: // len must be the maximum number ascii chars that it would be valid
michael@0: // to take from src and place into dst.  (That is, the minimum of the
michael@0: // number of bytes left in src and the number of unichars available in
michael@0: // dst.)
michael@0: 
michael@0: #if defined(__arm__) || defined(_M_ARM)
michael@0: 
michael@0: // on ARM, do extra work to avoid byte/halfword reads/writes by
michael@0: // reading/writing a word at a time for as long as we can
michael@0: static inline void
michael@0: Convert_ascii_run (const char *&src,
michael@0:                    char16_t *&dst,
michael@0:                    int32_t len)
michael@0: {
michael@0:   const uint32_t *src32;
michael@0:   uint32_t *dst32;
michael@0: 
michael@0:   // with some alignments, we'd never actually break out of the slow loop, so
michael@0:   // check and do the faster slow loop
michael@0:   if ((((NS_PTR_TO_UINT32(dst) & 3) == 0) && ((NS_PTR_TO_UINT32(src) & 1) == 0)) ||
michael@0:       (((NS_PTR_TO_UINT32(dst) & 3) == 2) && ((NS_PTR_TO_UINT32(src) & 1) == 1)))
michael@0:   {
michael@0:     while (((NS_PTR_TO_UINT32(src) & 3) ||
michael@0:             (NS_PTR_TO_UINT32(dst) & 3)) &&
michael@0:            len > 0)
michael@0:     {
michael@0:       if (*src & 0x80U)
michael@0:         return;
michael@0:       *dst++ = (char16_t) *src++;
michael@0:       len--;
michael@0:     }
michael@0:   } else {
michael@0:     goto finish;
michael@0:   }
michael@0: 
michael@0:   // then go 4 bytes at a time
michael@0:   src32 = (const uint32_t*) src;
michael@0:   dst32 = (uint32_t*) dst;
michael@0: 
michael@0:   while (len > 4) {
michael@0:     uint32_t in = *src32++;
michael@0: 
michael@0:     if (in & 0x80808080U) {
michael@0:       src32--;
michael@0:       break;
michael@0:     }
michael@0: 
michael@0:     *dst32++ = ((in & 0x000000ff) >>  0) | ((in & 0x0000ff00) << 8);
michael@0:     *dst32++ = ((in & 0x00ff0000) >> 16) | ((in & 0xff000000) >> 8);
michael@0: 
michael@0:     len -= 4;
michael@0:   }
michael@0: 
michael@0:   src = (const char *) src32;
michael@0:   dst = (char16_t *) dst32;
michael@0: 
michael@0: finish:
michael@0:   while (len-- > 0 && (*src & 0x80U) == 0) {
michael@0:     *dst++ = (char16_t) *src++;
michael@0:   }
michael@0: }
michael@0: 
michael@0: #else
michael@0: 
michael@0: #ifdef MOZILLA_MAY_SUPPORT_SSE2
michael@0: namespace mozilla {
michael@0: namespace SSE2 {
michael@0: 
michael@0: void Convert_ascii_run(const char *&src, char16_t *&dst, int32_t len);
michael@0: 
michael@0: }
michael@0: }
michael@0: #endif
michael@0: 
michael@0: static inline void
michael@0: Convert_ascii_run (const char *&src,
michael@0:                    char16_t *&dst,
michael@0:                    int32_t len)
michael@0: {
michael@0: #ifdef MOZILLA_MAY_SUPPORT_SSE2
michael@0:   if (mozilla::supports_sse2()) {
michael@0:     mozilla::SSE2::Convert_ascii_run(src, dst, len);
michael@0:     return;
michael@0:   }
michael@0: #endif
michael@0: 
michael@0:   while (len-- > 0 && (*src & 0x80U) == 0) {
michael@0:     *dst++ = (char16_t) *src++;
michael@0:   }
michael@0: }
michael@0: 
michael@0: #endif
michael@0: 
michael@0: NS_IMETHODIMP nsUTF8ToUnicode::Convert(const char * aSrc,
michael@0:                                        int32_t * aSrcLength,
michael@0:                                        char16_t * aDest,
michael@0:                                        int32_t * aDestLength)
michael@0: {
michael@0:   uint32_t aSrcLen   = (uint32_t) (*aSrcLength);
michael@0:   uint32_t aDestLen = (uint32_t) (*aDestLength);
michael@0: 
michael@0:   const char *in, *inend;
michael@0:   inend = aSrc + aSrcLen;
michael@0: 
michael@0:   char16_t *out, *outend;
michael@0:   outend = aDest + aDestLen;
michael@0: 
michael@0:   nsresult res = NS_OK; // conversion result
michael@0: 
michael@0:   out = aDest;
michael@0:   if (mState == 0xFF) {
michael@0:     // Emit supplementary character left over from previous iteration. It is
michael@0:     // caller's responsibility to keep a sufficient buffer.
michael@0:     if (aDestLen < 2) {
michael@0:       *aSrcLength = *aDestLength = 0;
michael@0:       return NS_OK_UDEC_MOREOUTPUT;
michael@0:     }
michael@0:     out = EmitSurrogatePair(mUcs4, out);
michael@0:     mUcs4 = 0;
michael@0:     mState = 0;
michael@0:     mBytes = 1;
michael@0:     mFirst = false;
michael@0:   }
michael@0: 
michael@0:   // alias these locally for speed
michael@0:   int32_t mUcs4 = this->mUcs4;
michael@0:   uint8_t mState = this->mState;
michael@0:   uint8_t mBytes = this->mBytes;
michael@0:   bool mFirst = this->mFirst;
michael@0: 
michael@0:   // Set mFirst to false now so we don't have to every time through the ASCII
michael@0:   // branch within the loop.
michael@0:   if (mFirst && aSrcLen && (0 == (0x80 & (*aSrc))))
michael@0:     mFirst = false;
michael@0: 
michael@0:   for (in = aSrc; ((in < inend) && (out < outend)); ++in) {
michael@0:     uint8_t c = *in;
michael@0:     if (0 == mState) {
michael@0:       // When mState is zero we expect either a US-ASCII character or a
michael@0:       // multi-octet sequence.
michael@0:       if (c < 0x80) {  // 00..7F
michael@0:         int32_t max_loops = std::min(inend - in, outend - out);
michael@0:         Convert_ascii_run(in, out, max_loops);
michael@0:         --in; // match the rest of the cases
michael@0:         mBytes = 1;
michael@0:       } else if (c < 0xC2) {  // C0/C1
michael@0:         // Overlong 2 octet sequence
michael@0:         if (mErrBehavior == kOnError_Signal) {
michael@0:           res = NS_ERROR_ILLEGAL_INPUT;
michael@0:           break;
michael@0:         }
michael@0:         *out++ = UCS2_REPLACEMENT_CHAR;
michael@0:         mFirst = false;
michael@0:       } else if (c < 0xE0) {  // C2..DF
michael@0:         // First octet of 2 octet sequence
michael@0:         mUcs4 = c;
michael@0:         mUcs4 = (mUcs4 & 0x1F) << 6;
michael@0:         mState = 1;
michael@0:         mBytes = 2;
michael@0:       } else if (c < 0xF0) {  // E0..EF
michael@0:         // First octet of 3 octet sequence
michael@0:         mUcs4 = c;
michael@0:         mUcs4 = (mUcs4 & 0x0F) << 12;
michael@0:         mState = 2;
michael@0:         mBytes = 3;
michael@0:       } else if (c < 0xF5) {  // F0..F4
michael@0:         // First octet of 4 octet sequence
michael@0:         mUcs4 = c;
michael@0:         mUcs4 = (mUcs4 & 0x07) << 18;
michael@0:         mState = 3;
michael@0:         mBytes = 4;
michael@0:       } else {  // F5..FF
michael@0:         /* Current octet is neither in the US-ASCII range nor a legal first
michael@0:          * octet of a multi-octet sequence.
michael@0:          */
michael@0:         if (mErrBehavior == kOnError_Signal) {
michael@0:           /* Return an error condition. Caller is responsible for flushing and
michael@0:            * refilling the buffer and resetting state.
michael@0:            */
michael@0:           res = NS_ERROR_ILLEGAL_INPUT;
michael@0:           break;
michael@0:         }
michael@0:         *out++ = UCS2_REPLACEMENT_CHAR;
michael@0:         mFirst = false;
michael@0:       }
michael@0:     } else {
michael@0:       // When mState is non-zero, we expect a continuation of the multi-octet
michael@0:       // sequence
michael@0:       if (0x80 == (0xC0 & c)) {
michael@0:         if (mState > 1) {
michael@0:           // If we are here, all possibilities are:
michael@0:           // mState == 2 && mBytes == 3 ||
michael@0:           // mState == 2 && mBytes == 4 ||
michael@0:           // mState == 3 && mBytes == 4
michael@0:           if ((mBytes == 3 && ((!mUcs4 && c < 0xA0) ||             // E0 80..9F
michael@0:                                (mUcs4 == 0xD000 && c > 0x9F))) ||  // ED A0..BF
michael@0:               (mState == 3 && ((!mUcs4 && c < 0x90) ||             // F0 80..8F
michael@0:                                (mUcs4 == 0x100000 && c > 0x8F)))) {// F4 90..BF
michael@0:             // illegal sequences or sequences converted into illegal ranges.
michael@0:             in--;
michael@0:             if (mErrBehavior == kOnError_Signal) {
michael@0:               res = NS_ERROR_ILLEGAL_INPUT;
michael@0:               break;
michael@0:             }
michael@0:             *out++ = UCS2_REPLACEMENT_CHAR;
michael@0:             mState = 0;
michael@0:             mFirst = false;
michael@0:             continue;
michael@0:           }
michael@0:         }
michael@0: 
michael@0:         // Legal continuation.
michael@0:         uint32_t shift = (mState - 1) * 6;
michael@0:         uint32_t tmp = c;
michael@0:         tmp = (tmp & 0x0000003FL) << shift;
michael@0:         mUcs4 |= tmp;
michael@0: 
michael@0:         if (0 == --mState) {
michael@0:           /* End of the multi-octet sequence. mUcs4 now contains the final
michael@0:            * Unicode codepoint to be output
michael@0:            */
michael@0: 
michael@0:           if (mUcs4 > 0xFFFF) {
michael@0:             // mUcs4 is in the range 0x10000 - 0x10FFFF. Output a UTF-16 pair
michael@0:             if (out + 2 > outend) {
michael@0:               // insufficient space left in the buffer. Keep mUcs4 for the
michael@0:               // next iteration.
michael@0:               mState = 0xFF;
michael@0:               ++in;
michael@0:               res = NS_OK_UDEC_MOREOUTPUT;
michael@0:               break;
michael@0:             }
michael@0:             out = EmitSurrogatePair(mUcs4, out);
michael@0:           } else if (UNICODE_BYTE_ORDER_MARK != mUcs4 || !mFirst) {
michael@0:             // Don't output the BOM only if it is the first character
michael@0:             *out++ = mUcs4;
michael@0:           }
michael@0:           //initialize UTF8 cache
michael@0:           mUcs4  = 0;
michael@0:           mState = 0;
michael@0:           mBytes = 1;
michael@0:           mFirst = false;
michael@0:         }
michael@0:       } else {
michael@0:         /* ((0xC0 & c != 0x80) && (mState != 0))
michael@0:          *
michael@0:          * Incomplete multi-octet sequence. Unconsume this
michael@0:          * octet and return an error condition. Caller is responsible
michael@0:          * for flushing and refilling the buffer and resetting state.
michael@0:          */
michael@0:         in--;
michael@0:         if (mErrBehavior == kOnError_Signal) {
michael@0:           res = NS_ERROR_ILLEGAL_INPUT;
michael@0:           break;
michael@0:         }
michael@0:         *out++ = UCS2_REPLACEMENT_CHAR;
michael@0:         mState = 0;
michael@0:         mFirst = false;
michael@0:       }
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // output not finished, output buffer too short
michael@0:   if ((NS_OK == res) && (in < inend) && (out >= outend))
michael@0:     res = NS_OK_UDEC_MOREOUTPUT;
michael@0: 
michael@0:   // last UCS4 is incomplete, make sure the caller
michael@0:   // returns with properly aligned continuation of the buffer
michael@0:   if ((NS_OK == res) && (mState != 0))
michael@0:     res = NS_OK_UDEC_MOREINPUT;
michael@0: 
michael@0:   *aSrcLength = in - aSrc;
michael@0:   *aDestLength = out - aDest;
michael@0: 
michael@0:   this->mUcs4 = mUcs4;
michael@0:   this->mState = mState;
michael@0:   this->mBytes = mBytes;
michael@0:   this->mFirst = mFirst;
michael@0: 
michael@0:   return(res);
michael@0: }