michael@0: /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0:  
michael@0: #include "nsCharSetProber.h"
michael@0: #include "prmem.h"
michael@0: 
michael@0: //This filter applies to all scripts which do not use English characters
michael@0: bool nsCharSetProber::FilterWithoutEnglishLetters(const char* aBuf, uint32_t aLen, char** newBuf, uint32_t& newLen)
michael@0: {
michael@0:   char *newptr;
michael@0:   char *prevPtr, *curPtr;
michael@0:   
michael@0:   bool meetMSB = false;   
michael@0:   newptr = *newBuf = (char*)PR_Malloc(aLen);
michael@0:   if (!newptr)
michael@0:     return false;
michael@0: 
michael@0:   for (curPtr = prevPtr = (char*)aBuf; curPtr < aBuf+aLen; curPtr++)
michael@0:   {
michael@0:     if (*curPtr & 0x80)
michael@0:     {
michael@0:       meetMSB = true;
michael@0:     }
michael@0:     else if (*curPtr < 'A' || (*curPtr > 'Z' && *curPtr < 'a') || *curPtr > 'z') 
michael@0:     {
michael@0:       //current char is a symbol, most likely a punctuation. we treat it as segment delimiter
michael@0:       if (meetMSB && curPtr > prevPtr) 
michael@0:       //this segment contains more than single symbol, and it has upper ASCII, we need to keep it
michael@0:       {
michael@0:         while (prevPtr < curPtr) *newptr++ = *prevPtr++;  
michael@0:         prevPtr++;
michael@0:         *newptr++ = ' ';
michael@0:         meetMSB = false;
michael@0:       }
michael@0:       else //ignore current segment. (either because it is just a symbol or just an English word)
michael@0:         prevPtr = curPtr+1;
michael@0:     }
michael@0:   }
michael@0:   if (meetMSB && curPtr > prevPtr) 
michael@0:     while (prevPtr < curPtr) *newptr++ = *prevPtr++;  
michael@0: 
michael@0:   newLen = newptr - *newBuf;
michael@0: 
michael@0:   return true;
michael@0: }
michael@0: 
michael@0: //This filter applies to all scripts which contain both English characters and upper ASCII characters.
michael@0: bool nsCharSetProber::FilterWithEnglishLetters(const char* aBuf, uint32_t aLen, char** newBuf, uint32_t& newLen)
michael@0: {
michael@0:   //do filtering to reduce load to probers
michael@0:   char *newptr;
michael@0:   char *prevPtr, *curPtr;
michael@0:   bool isInTag = false;
michael@0: 
michael@0:   newptr = *newBuf = (char*)PR_Malloc(aLen);
michael@0:   if (!newptr)
michael@0:     return false;
michael@0: 
michael@0:   for (curPtr = prevPtr = (char*)aBuf; curPtr < aBuf+aLen; curPtr++)
michael@0:   {
michael@0:     if (*curPtr == '>')
michael@0:       isInTag = false;
michael@0:     else if (*curPtr == '<')
michael@0:       isInTag = true;
michael@0: 
michael@0:     if (!(*curPtr & 0x80) &&
michael@0:         (*curPtr < 'A' || (*curPtr > 'Z' && *curPtr < 'a') || *curPtr > 'z') )
michael@0:     {
michael@0:       if (curPtr > prevPtr && !isInTag) // Current segment contains more than just a symbol 
michael@0:                                         // and it is not inside a tag, keep it.
michael@0:       {
michael@0:         while (prevPtr < curPtr) *newptr++ = *prevPtr++;  
michael@0:         prevPtr++;
michael@0:         *newptr++ = ' ';
michael@0:       }
michael@0:       else
michael@0:         prevPtr = curPtr+1;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // If the current segment contains more than just a symbol 
michael@0:   // and it is not inside a tag then keep it.
michael@0:   if (!isInTag)
michael@0:     while (prevPtr < curPtr)
michael@0:       *newptr++ = *prevPtr++;  
michael@0: 
michael@0:   newLen = newptr - *newBuf;
michael@0: 
michael@0:   return true;
michael@0: }