michael@0: /*  GRAPHITE2 LICENSING
michael@0: 
michael@0:     Copyright 2011, SIL International
michael@0:     All rights reserved.
michael@0: 
michael@0:     This library is free software; you can redistribute it and/or modify
michael@0:     it under the terms of the GNU Lesser General Public License as published
michael@0:     by the Free Software Foundation; either version 2.1 of License, or
michael@0:     (at your option) any later version.
michael@0: 
michael@0:     This program is distributed in the hope that it will be useful,
michael@0:     but WITHOUT ANY WARRANTY; without even the implied warranty of
michael@0:     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
michael@0:     Lesser General Public License for more details.
michael@0: 
michael@0:     You should also have received a copy of the GNU Lesser General Public
michael@0:     License along with this library in the file named "LICENSE".
michael@0:     If not, write to the Free Software Foundation, 51 Franklin Street,
michael@0:     Suite 500, Boston, MA 02110-1335, USA or visit their web page on the
michael@0:     internet at http://www.fsf.org/licenses/lgpl.html.
michael@0: 
michael@0: Alternatively, the contents of this file may be used under the terms of the
michael@0: Mozilla Public License (http://mozilla.org/MPL) or the GNU General Public
michael@0: License, as published by the Free Software Foundation, either version 2
michael@0: of the License or (at your option) any later version.
michael@0: */
michael@0: #pragma once
michael@0: 
michael@0: #include <cstdlib>
michael@0: #include "inc/Main.h"
michael@0: 
michael@0: namespace graphite2 {
michael@0: 
michael@0: typedef uint32  uchar_t;
michael@0: 
michael@0: template <int N>
michael@0: struct _utf_codec
michael@0: {
michael@0:     typedef uchar_t codeunit_t;
michael@0: 
michael@0:     static void     put(codeunit_t * cp, const uchar_t , int8 & len) throw();
michael@0:     static uchar_t  get(const codeunit_t * cp, int8 & len) throw();
michael@0: };
michael@0: 
michael@0: 
michael@0: template <>
michael@0: struct _utf_codec<32>
michael@0: {
michael@0: private:
michael@0:     static const uchar_t    limit = 0x110000;
michael@0: public:
michael@0:     typedef uint32  codeunit_t;
michael@0: 
michael@0:     inline
michael@0:     static void put(codeunit_t * cp, const uchar_t usv, int8 & l) throw()
michael@0:     {
michael@0:         *cp = usv; l = 1;
michael@0:     }
michael@0: 
michael@0:     inline
michael@0:     static uchar_t get(const codeunit_t * cp, int8 & l) throw()
michael@0:     {
michael@0:         if (cp[0] < limit)  { l = 1;  return cp[0]; }
michael@0:         else                { l = -1; return 0xFFFD; }
michael@0:     }
michael@0: };
michael@0: 
michael@0: 
michael@0: template <>
michael@0: struct _utf_codec<16>
michael@0: {
michael@0: private:
michael@0:     static const int32  lead_offset      = 0xD800 - (0x10000 >> 10);
michael@0:     static const int32  surrogate_offset = 0x10000 - (0xD800 << 10) - 0xDC00;
michael@0: public:
michael@0:     typedef uint16  codeunit_t;
michael@0: 
michael@0:     inline
michael@0:     static void put(codeunit_t * cp, const uchar_t usv, int8 & l) throw()
michael@0:     {
michael@0:         if (usv < 0x10000)  { l = 1; cp[0] = codeunit_t(usv); }
michael@0:         else
michael@0:         {
michael@0:             cp[0] = codeunit_t(lead_offset + (usv >> 10));
michael@0:             cp[1] = codeunit_t(0xDC00 + (usv & 0x3FF));
michael@0:             l = 2;
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     inline
michael@0:     static uchar_t get(const codeunit_t * cp, int8 & l) throw()
michael@0:     {
michael@0:         const uint32    uh = cp[0];
michael@0:         l = 1;
michael@0: 
michael@0:         if (0xD800 > uh || uh > 0xDFFF) { return uh; }
michael@0:         const uint32 ul = cp[1];
michael@0:         if (uh > 0xDBFF || 0xDC00 > ul || ul > 0xDFFF) { l = -1; return 0xFFFD; }
michael@0:         ++l;
michael@0:         return (uh<<10) + ul + surrogate_offset;
michael@0:     }
michael@0: };
michael@0: 
michael@0: 
michael@0: template <>
michael@0: struct _utf_codec<8>
michael@0: {
michael@0: private:
michael@0:     static const int8 sz_lut[16];
michael@0:     static const byte mask_lut[5];
michael@0: 
michael@0: 
michael@0: public:
michael@0:     typedef uint8   codeunit_t;
michael@0: 
michael@0:     inline
michael@0:     static void put(codeunit_t * cp, const uchar_t usv, int8 & l) throw()
michael@0:     {
michael@0:         if (usv < 0x80)     {l = 1; cp[0] = usv; return; }
michael@0:         if (usv < 0x0800)   {l = 2; cp[0] = 0xC0 + (usv >> 6);  cp[1] = 0x80 + (usv & 0x3F); return; }
michael@0:         if (usv < 0x10000)  {l = 3; cp[0] = 0xE0 + (usv >> 12); cp[1] = 0x80 + ((usv >> 6) & 0x3F);  cp[2] = 0x80 + (usv & 0x3F); return; }
michael@0:         else                {l = 4; cp[0] = 0xF0 + (usv >> 18); cp[1] = 0x80 + ((usv >> 12) & 0x3F); cp[2] = 0x80 + ((usv >> 6) & 0x3F); cp[3] = 0x80 + (usv & 0x3F); return; }
michael@0:     }
michael@0: 
michael@0:     inline
michael@0:     static uchar_t get(const codeunit_t * cp, int8 & l) throw()
michael@0:     {
michael@0:         const int8 seq_sz = sz_lut[*cp >> 4];
michael@0:         uchar_t u = *cp & mask_lut[seq_sz];
michael@0:         l = 1;
michael@0:         bool toolong = false;
michael@0: 
michael@0:         switch(seq_sz) {
michael@0:             case 4:     u <<= 6; u |= *++cp & 0x3F; if (*cp >> 6 != 2) break; ++l; toolong  = (u < 0x10); // no break
michael@0:             case 3:     u <<= 6; u |= *++cp & 0x3F; if (*cp >> 6 != 2) break; ++l; toolong |= (u < 0x20); // no break
michael@0:             case 2:     u <<= 6; u |= *++cp & 0x3F; if (*cp >> 6 != 2) break; ++l; toolong |= (u < 0x80); // no break
michael@0:             case 1:     break;
michael@0:             case 0:     l = -1; return 0xFFFD;
michael@0:         }
michael@0: 
michael@0:         if (l != seq_sz || toolong)
michael@0:         {
michael@0:             l = -l;
michael@0:             return 0xFFFD;
michael@0:         }
michael@0:         return u;
michael@0:     }
michael@0: };
michael@0: 
michael@0: 
michael@0: template <typename C>
michael@0: class _utf_iterator
michael@0: {
michael@0:     typedef _utf_codec<sizeof(C)*8> codec;
michael@0: 
michael@0:     C             * cp;
michael@0:     mutable int8    sl;
michael@0: 
michael@0: public:
michael@0:     typedef C           codeunit_type;
michael@0:     typedef uchar_t     value_type;
michael@0:     typedef uchar_t   * pointer;
michael@0: 
michael@0:     class reference
michael@0:     {
michael@0:         const _utf_iterator & _i;
michael@0: 
michael@0:         reference(const _utf_iterator & i): _i(i) {}
michael@0:     public:
michael@0:         operator value_type () const throw ()                   { return codec::get(_i.cp, _i.sl); }
michael@0:         reference & operator = (const value_type usv) throw()   { codec::put(_i.cp, usv, _i.sl); return *this; }
michael@0: 
michael@0:         friend class _utf_iterator;
michael@0:     };
michael@0: 
michael@0: 
michael@0:     _utf_iterator(const void * us=0)    : cp(reinterpret_cast<C *>(const_cast<void *>(us))), sl(1) { }
michael@0: 
michael@0:     _utf_iterator   & operator ++ ()    { cp += abs(sl); return *this; }
michael@0:     _utf_iterator   operator ++ (int)   { _utf_iterator tmp(*this); operator++(); return tmp; }
michael@0: 
michael@0:     bool operator == (const _utf_iterator & rhs) const throw() { return cp >= rhs.cp; }
michael@0:     bool operator != (const _utf_iterator & rhs) const throw() { return !operator==(rhs); }
michael@0: 
michael@0:     reference   operator * () const throw() { return *this; }
michael@0:     pointer     operator ->() const throw() { return &operator *(); }
michael@0: 
michael@0:     operator codeunit_type * () const throw() { return cp; }
michael@0: 
michael@0:     bool error() const throw()  { return sl < 1; }
michael@0: };
michael@0: 
michael@0: template <typename C>
michael@0: struct utf
michael@0: {
michael@0:     typedef typename _utf_codec<sizeof(C)*8>::codeunit_t codeunit_t;
michael@0: 
michael@0:     typedef _utf_iterator<C>        iterator;
michael@0:     typedef _utf_iterator<const C>  const_iterator;
michael@0: };
michael@0: 
michael@0: 
michael@0: typedef utf<uint32> utf32;
michael@0: typedef utf<uint16> utf16;
michael@0: typedef utf<uint8>  utf8;
michael@0: 
michael@0: } // namespace graphite2