The Tor Browser / file comparison
comparison: browser/components/translation/cld2/internal/tote.h
browser/components/translation/cld2/internal/tote.h
- changeset 0
- 6474c204b198
equal
deleted
inserted
replaced
| -1:000000000000 | 0:39d1588e7ee9 |
|---|---|
| 1 // Copyright 2013 Google Inc. All Rights Reserved. | |
| 2 // | |
| 3 // Licensed under the Apache License, Version 2.0 (the "License"); | |
| 4 // you may not use this file except in compliance with the License. | |
| 5 // You may obtain a copy of the License at | |
| 6 // | |
| 7 // http://www.apache.org/licenses/LICENSE-2.0 | |
| 8 // | |
| 9 // Unless required by applicable law or agreed to in writing, software | |
| 10 // distributed under the License is distributed on an "AS IS" BASIS, | |
| 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 12 // See the License for the specific language governing permissions and | |
| 13 // limitations under the License. | |
| 14 | |
| 15 // | |
| 16 // Author: dsites@google.com (Dick Sites) | |
| 17 // | |
| 18 | |
| 19 #ifndef I18N_ENCODINGS_CLD2_INTERNAL_TOTE_H_ | |
| 20 #define I18N_ENCODINGS_CLD2_INTERNAL_TOTE_H_ | |
| 21 | |
| 22 #include <stdio.h> | |
| 23 #include "integral_types.h" // for uint8 etc | |
| 24 | |
| 25 namespace CLD2 { | |
| 26 | |
| 27 | |
| 28 // Take a set of <key, score> pairs and tote them up. | |
| 29 // Key is an 8-bit per-script language | |
| 30 // After explicitly sorting, retrieve top key, score pairs | |
| 31 // Normal use is key=per-script language | |
| 32 // The main data structure is an array of 256 uint16 counts. We normally | |
| 33 // expect this to be initialized, added-to about 60 times, then the top three | |
| 34 // items found. The reduce the initial and final time, we also keep a bit vector | |
| 35 // of unused (and uninitialized) parts, each of 64 bits covering four keys. | |
| 36 class Tote { | |
| 37 public: | |
| 38 Tote(); | |
| 39 ~Tote(); | |
| 40 void Reinit(); | |
| 41 void AddScoreCount(); | |
| 42 void Add(uint8 ikey, int idelta); | |
| 43 void AddBytes(int ibytes) {byte_count_ += ibytes;} | |
| 44 void CurrentTopThreeKeys(int* key3) const; | |
| 45 int GetScoreCount() const {return score_count_;} | |
| 46 int GetByteCount() const {return byte_count_;} | |
| 47 int GetScore(int i) const {return score_[i];} | |
| 48 void SetScoreCount(uint16 v) {score_count_ = v;} | |
| 49 void SetScore(int i, int v) {score_[i] = v;} | |
| 50 | |
| 51 private: | |
| 52 uint64 in_use_mask_; // 64 bits, one for each group of 4 scores. | |
| 53 // 0 = not initialized,not used | |
| 54 int byte_count_; // Bytes of text scored | |
| 55 int score_count_; // Number of quadgrams/etc. scored | |
| 56 union { | |
| 57 uint64 gscore_[64]; // For alignment and clearing quickly | |
| 58 uint16 score_[256]; // Probability score sum | |
| 59 }; | |
| 60 | |
| 61 }; | |
| 62 | |
| 63 | |
| 64 // Take a set of <key, score, reliability> triples and tote them up. | |
| 65 // Key is a 16-bit full language | |
| 66 // After explicitly sorting, retrieve top key, score, reliability triples | |
| 67 class DocTote { | |
| 68 public: | |
| 69 DocTote(); | |
| 70 ~DocTote(); | |
| 71 void Reinit(); | |
| 72 void Add(uint16 ikey, int ibytes, int score, int ireliability); | |
| 73 int Find(uint16 ikey); | |
| 74 void AddClosePair(int subscr, int val) {closepair_[subscr] += val;} | |
| 75 int CurrentTopKey(); | |
| 76 Tote* RunningScore() {return &runningscore_;} | |
| 77 void Sort(int n); | |
| 78 void Dump(FILE* f); | |
| 79 | |
| 80 int GetIncrCount() const {return incr_count_;} | |
| 81 int GetClosePair(int subscr) const {return closepair_[subscr];} | |
| 82 int MaxSize() const {return kMaxSize_;} | |
| 83 uint16 Key(int i) const {return key_[i];} | |
| 84 int Value(int i) const {return value_[i];} // byte count | |
| 85 int Score(int i) const {return score_[i];} // sum lg prob | |
| 86 int Reliability(int i) const {return reliability_[i];} | |
| 87 void SetKey(int i, int v) {key_[i] = v;} | |
| 88 void SetValue(int i, int v) {value_[i] = v;} | |
| 89 void SetScore(int i, int v) {score_[i] = v;} | |
| 90 void SetReliability(int i, int v) {reliability_[i] = v;} | |
| 91 | |
| 92 static const uint16 kUnusedKey = 0xFFFF; | |
| 93 | |
| 94 private: | |
| 95 static const int kMaxSize_ = 24; | |
| 96 static const int kMaxClosePairSize_ = 8; | |
| 97 | |
| 98 int incr_count_; // Number of Add calls | |
| 99 int sorted_; // Contents have been sorted, cannot Add | |
| 100 Tote runningscore_; // Top lang scores across entire doc, for | |
| 101 // helping resolve close pairs | |
| 102 // Align at multiple of 8 bytes | |
| 103 int closepair_[kMaxClosePairSize_]; | |
| 104 uint16 key_[kMaxSize_]; // Lang unassigned = 0xFFFF, valid = 1..1023 | |
| 105 int value_[kMaxSize_]; // Bytecount this lang | |
| 106 int score_[kMaxSize_]; // Probability score sum | |
| 107 int reliability_[kMaxSize_]; // Percentage 0..100 | |
| 108 }; | |
| 109 | |
| 110 } // End namespace CLD2 | |
| 111 | |
| 112 #endif // I18N_ENCODINGS_CLD2_INTERNAL_TOTE_H_ |
