michael@0: // Copyright 2013 Google Inc. All Rights Reserved.
michael@0: //
michael@0: // Licensed under the Apache License, Version 2.0 (the "License");
michael@0: // you may not use this file except in compliance with the License.
michael@0: // You may obtain a copy of the License at
michael@0: //
michael@0: //     http://www.apache.org/licenses/LICENSE-2.0
michael@0: //
michael@0: // Unless required by applicable law or agreed to in writing, software
michael@0: // distributed under the License is distributed on an "AS IS" BASIS,
michael@0: // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
michael@0: // See the License for the specific language governing permissions and
michael@0: // limitations under the License.
michael@0: 
michael@0: //
michael@0: // Author: dsites@google.com (Dick Sites)
michael@0: //
michael@0: 
michael@0: #ifndef I18N_ENCODINGS_CLD2_INTERNAL_TOTE_H_
michael@0: #define I18N_ENCODINGS_CLD2_INTERNAL_TOTE_H_
michael@0: 
michael@0: #include <stdio.h>
michael@0: #include "integral_types.h"        // for uint8 etc
michael@0: 
michael@0: namespace CLD2 {
michael@0: 
michael@0: 
michael@0: // Take a set of <key, score> pairs and tote them up.
michael@0: // Key is an 8-bit per-script language
michael@0: // After explicitly sorting, retrieve top key, score pairs
michael@0: // Normal use is key=per-script language
michael@0: // The main data structure is an array of 256 uint16 counts. We normally
michael@0: // expect this to be initialized, added-to about 60 times, then the top three
michael@0: // items found. The reduce the initial and final time, we also keep a bit vector
michael@0: // of unused (and uninitialized) parts, each of 64 bits covering four keys.
michael@0: class Tote {
michael@0:  public:
michael@0:   Tote();
michael@0:   ~Tote();
michael@0:   void Reinit();
michael@0:   void AddScoreCount();
michael@0:   void Add(uint8 ikey, int idelta);
michael@0:   void AddBytes(int ibytes) {byte_count_ += ibytes;}
michael@0:   void CurrentTopThreeKeys(int* key3) const;
michael@0:   int GetScoreCount() const {return score_count_;}
michael@0:   int GetByteCount() const {return byte_count_;}
michael@0:   int GetScore(int i) const {return score_[i];}
michael@0:   void SetScoreCount(uint16 v) {score_count_ = v;}
michael@0:   void SetScore(int i, int v) {score_[i] = v;}
michael@0: 
michael@0:  private:
michael@0:   uint64 in_use_mask_;      // 64 bits, one for each group of 4 scores.
michael@0:                             //    0 = not initialized,not used
michael@0:   int byte_count_;          // Bytes of text scored
michael@0:   int score_count_;         // Number of quadgrams/etc. scored
michael@0:   union {
michael@0:     uint64 gscore_[64];     // For alignment and clearing quickly
michael@0:     uint16 score_[256];     // Probability score sum
michael@0:   };
michael@0: 
michael@0: };
michael@0: 
michael@0: 
michael@0: // Take a set of <key, score, reliability> triples and tote them up.
michael@0: // Key is a 16-bit full language
michael@0: // After explicitly sorting, retrieve top key, score, reliability triples
michael@0: class DocTote {
michael@0:  public:
michael@0:   DocTote();
michael@0:   ~DocTote();
michael@0:   void Reinit();
michael@0:   void Add(uint16 ikey, int ibytes, int score, int ireliability);
michael@0:   int Find(uint16 ikey);
michael@0:   void AddClosePair(int subscr, int val) {closepair_[subscr] += val;}
michael@0:   int CurrentTopKey();
michael@0:   Tote* RunningScore() {return &runningscore_;}
michael@0:   void Sort(int n);
michael@0:   void Dump(FILE* f);
michael@0: 
michael@0:   int GetIncrCount() const {return incr_count_;}
michael@0:   int GetClosePair(int subscr) const {return closepair_[subscr];}
michael@0:   int MaxSize() const {return kMaxSize_;}
michael@0:   uint16 Key(int i) const {return key_[i];}
michael@0:   int Value(int i) const {return value_[i];}      // byte count
michael@0:   int Score(int i) const {return score_[i];}      // sum lg prob
michael@0:   int Reliability(int i) const {return reliability_[i];}
michael@0:   void SetKey(int i, int v) {key_[i] = v;}
michael@0:   void SetValue(int i, int v) {value_[i] = v;}
michael@0:   void SetScore(int i, int v) {score_[i] = v;}
michael@0:   void SetReliability(int i, int v) {reliability_[i] = v;}
michael@0: 
michael@0:   static const uint16 kUnusedKey = 0xFFFF;
michael@0: 
michael@0:  private:
michael@0:   static const int kMaxSize_ = 24;
michael@0:   static const int kMaxClosePairSize_ = 8;
michael@0: 
michael@0:   int incr_count_;         // Number of Add calls
michael@0:   int sorted_;             // Contents have been sorted, cannot Add
michael@0:   Tote runningscore_;      // Top lang scores across entire doc, for
michael@0:                            // helping resolve close pairs
michael@0:   // Align at multiple of 8 bytes
michael@0:   int closepair_[kMaxClosePairSize_];
michael@0:   uint16 key_[kMaxSize_];   // Lang unassigned = 0xFFFF, valid = 1..1023
michael@0:   int value_[kMaxSize_];    // Bytecount this lang
michael@0:   int score_[kMaxSize_];    // Probability score sum
michael@0:   int reliability_[kMaxSize_];  // Percentage 0..100
michael@0: };
michael@0: 
michael@0: }       // End namespace CLD2
michael@0: 
michael@0: #endif  // I18N_ENCODINGS_CLD2_INTERNAL_TOTE_H_