The Tor Browser / file comparison

comparison: extensions/spellcheck/hunspell/src/suggestmgr.cpp

extensions/spellcheck/hunspell/src/suggestmgr.cpp

branch
TOR_BUG_3246
changeset 6
8bccb770b82d
equal deleted inserted replaced
-1:000000000000 0:4aeae4adeafe
1 /******* BEGIN LICENSE BLOCK *******
2 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
3 *
4 * The contents of this file are subject to the Mozilla Public License Version
5 * 1.1 (the "License"); you may not use this file except in compliance with
6 * the License. You may obtain a copy of the License at
7 * http://www.mozilla.org/MPL/
8 *
9 * Software distributed under the License is distributed on an "AS IS" basis,
10 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
11 * for the specific language governing rights and limitations under the
12 * License.
13 *
14 * The Initial Developers of the Original Code are Kevin Hendricks (MySpell)
15 * and László Németh (Hunspell). Portions created by the Initial Developers
16 * are Copyright (C) 2002-2005 the Initial Developers. All Rights Reserved.
17 *
18 * Contributor(s): Kevin Hendricks (kevin.hendricks@sympatico.ca)
19 * David Einstein (deinst@world.std.com)
20 * László Németh (nemethl@gyorsposta.hu)
21 * Caolan McNamara (caolanm@redhat.com)
22 * Davide Prina
23 * Giuseppe Modugno
24 * Gianluca Turconi
25 * Simon Brouwer
26 * Noll Janos
27 * Biro Arpad
28 * Goldman Eleonora
29 * Sarlos Tamas
30 * Bencsath Boldizsar
31 * Halacsy Peter
32 * Dvornik Laszlo
33 * Gefferth Andras
34 * Nagy Viktor
35 * Varga Daniel
36 * Chris Halls
37 * Rene Engelhard
38 * Bram Moolenaar
39 * Dafydd Jones
40 * Harri Pitkanen
41 * Andras Timar
42 * Tor Lillqvist
43 *
44 * Alternatively, the contents of this file may be used under the terms of
45 * either the GNU General Public License Version 2 or later (the "GPL"), or
46 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
47 * in which case the provisions of the GPL or the LGPL are applicable instead
48 * of those above. If you wish to allow use of your version of this file only
49 * under the terms of either the GPL or the LGPL, and not to allow others to
50 * use your version of this file under the terms of the MPL, indicate your
51 * decision by deleting the provisions above and replace them with the notice
52 * and other provisions required by the GPL or the LGPL. If you do not delete
53 * the provisions above, a recipient may use your version of this file under
54 * the terms of any one of the MPL, the GPL or the LGPL.
55 *
56 ******* END LICENSE BLOCK *******/
57
58 #include <stdlib.h>
59 #include <string.h>
60 #include <stdio.h>
61 #include <ctype.h>
62
63 #include "suggestmgr.hxx"
64 #include "htypes.hxx"
65 #include "csutil.hxx"
66
67 const w_char W_VLINE = { '\0', '|' };
68
69 SuggestMgr::SuggestMgr(const char * tryme, int maxn,
70 AffixMgr * aptr)
71 {
72
73 // register affix manager and check in string of chars to
74 // try when building candidate suggestions
75 pAMgr = aptr;
76
77 csconv = NULL;
78
79 ckeyl = 0;
80 ckey = NULL;
81 ckey_utf = NULL;
82
83 ctryl = 0;
84 ctry = NULL;
85 ctry_utf = NULL;
86
87 utf8 = 0;
88 langnum = 0;
89 complexprefixes = 0;
90
91 maxSug = maxn;
92 nosplitsugs = 0;
93 maxngramsugs = MAXNGRAMSUGS;
94 maxcpdsugs = MAXCOMPOUNDSUGS;
95
96 if (pAMgr) {
97 langnum = pAMgr->get_langnum();
98 ckey = pAMgr->get_key_string();
99 nosplitsugs = pAMgr->get_nosplitsugs();
100 if (pAMgr->get_maxngramsugs() >= 0)
101 maxngramsugs = pAMgr->get_maxngramsugs();
102 utf8 = pAMgr->get_utf8();
103 if (pAMgr->get_maxcpdsugs() >= 0)
104 maxcpdsugs = pAMgr->get_maxcpdsugs();
105 if (!utf8)
106 {
107 char * enc = pAMgr->get_encoding();
108 csconv = get_current_cs(enc);
109 free(enc);
110 }
111 complexprefixes = pAMgr->get_complexprefixes();
112 }
113
114 if (ckey) {
115 if (utf8) {
116 w_char t[MAXSWL];
117 ckeyl = u8_u16(t, MAXSWL, ckey);
118 ckey_utf = (w_char *) malloc(ckeyl * sizeof(w_char));
119 if (ckey_utf) memcpy(ckey_utf, t, ckeyl * sizeof(w_char));
120 else ckeyl = 0;
121 } else {
122 ckeyl = strlen(ckey);
123 }
124 }
125
126 if (tryme) {
127 ctry = mystrdup(tryme);
128 if (ctry) ctryl = strlen(ctry);
129 if (ctry && utf8) {
130 w_char t[MAXSWL];
131 ctryl = u8_u16(t, MAXSWL, tryme);
132 ctry_utf = (w_char *) malloc(ctryl * sizeof(w_char));
133 if (ctry_utf) memcpy(ctry_utf, t, ctryl * sizeof(w_char));
134 else ctryl = 0;
135 }
136 }
137 }
138
139
140 SuggestMgr::~SuggestMgr()
141 {
142 pAMgr = NULL;
143 if (ckey) free(ckey);
144 ckey = NULL;
145 if (ckey_utf) free(ckey_utf);
146 ckey_utf = NULL;
147 ckeyl = 0;
148 if (ctry) free(ctry);
149 ctry = NULL;
150 if (ctry_utf) free(ctry_utf);
151 ctry_utf = NULL;
152 ctryl = 0;
153 maxSug = 0;
154 #ifdef MOZILLA_CLIENT
155 delete [] csconv;
156 #endif
157 }
158
159 int SuggestMgr::testsug(char** wlst, const char * candidate, int wl, int ns, int cpdsuggest,
160 int * timer, clock_t * timelimit) {
161 int cwrd = 1;
162 if (ns == maxSug) return maxSug;
163 for (int k=0; k < ns; k++) {
164 if (strcmp(candidate,wlst[k]) == 0) cwrd = 0;
165 }
166 if ((cwrd) && checkword(candidate, wl, cpdsuggest, timer, timelimit)) {
167 wlst[ns] = mystrdup(candidate);
168 if (wlst[ns] == NULL) {
169 for (int j=0; j<ns; j++) free(wlst[j]);
170 return -1;
171 }
172 ns++;
173 }
174 return ns;
175 }
176
177 // generate suggestions for a misspelled word
178 // pass in address of array of char * pointers
179 // onlycompoundsug: probably bad suggestions (need for ngram sugs, too)
180
181 int SuggestMgr::suggest(char*** slst, const char * w, int nsug,
182 int * onlycompoundsug)
183 {
184 int nocompoundtwowords = 0;
185 char ** wlst;
186 w_char word_utf[MAXSWL];
187 int wl = 0;
188 int nsugorig = nsug;
189 char w2[MAXWORDUTF8LEN];
190 const char * word = w;
191 int oldSug = 0;
192
193 // word reversing wrapper for complex prefixes
194 if (complexprefixes) {
195 strcpy(w2, w);
196 if (utf8) reverseword_utf(w2); else reverseword(w2);
197 word = w2;
198 }
199
200 if (*slst) {
201 wlst = *slst;
202 } else {
203 wlst = (char **) malloc(maxSug * sizeof(char *));
204 if (wlst == NULL) return -1;
205 for (int i = 0; i < maxSug; i++) {
206 wlst[i] = NULL;
207 }
208 }
209
210 if (utf8) {
211 wl = u8_u16(word_utf, MAXSWL, word);
212 if (wl == -1) {
213 *slst = wlst;
214 return nsug;
215 }
216 }
217
218 for (int cpdsuggest=0; (cpdsuggest<2) && (nocompoundtwowords==0); cpdsuggest++) {
219
220 // limit compound suggestion
221 if (cpdsuggest > 0) oldSug = nsug;
222
223 // suggestions for an uppercase word (html -> HTML)
224 if ((nsug < maxSug) && (nsug > -1)) {
225 nsug = (utf8) ? capchars_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
226 capchars(wlst, word, nsug, cpdsuggest);
227 }
228
229 // perhaps we made a typical fault of spelling
230 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
231 nsug = replchars(wlst, word, nsug, cpdsuggest);
232 }
233
234 // perhaps we made chose the wrong char from a related set
235 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
236 nsug = mapchars(wlst, word, nsug, cpdsuggest);
237 }
238
239 // only suggest compound words when no other suggestion
240 if ((cpdsuggest == 0) && (nsug > nsugorig)) nocompoundtwowords=1;
241
242 // did we swap the order of chars by mistake
243 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
244 nsug = (utf8) ? swapchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
245 swapchar(wlst, word, nsug, cpdsuggest);
246 }
247
248 // did we swap the order of non adjacent chars by mistake
249 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
250 nsug = (utf8) ? longswapchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
251 longswapchar(wlst, word, nsug, cpdsuggest);
252 }
253
254 // did we just hit the wrong key in place of a good char (case and keyboard)
255 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
256 nsug = (utf8) ? badcharkey_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
257 badcharkey(wlst, word, nsug, cpdsuggest);
258 }
259
260 // did we add a char that should not be there
261 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
262 nsug = (utf8) ? extrachar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
263 extrachar(wlst, word, nsug, cpdsuggest);
264 }
265
266
267 // did we forgot a char
268 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
269 nsug = (utf8) ? forgotchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
270 forgotchar(wlst, word, nsug, cpdsuggest);
271 }
272
273 // did we move a char
274 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
275 nsug = (utf8) ? movechar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
276 movechar(wlst, word, nsug, cpdsuggest);
277 }
278
279 // did we just hit the wrong key in place of a good char
280 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
281 nsug = (utf8) ? badchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
282 badchar(wlst, word, nsug, cpdsuggest);
283 }
284
285 // did we double two characters
286 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
287 nsug = (utf8) ? doubletwochars_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
288 doubletwochars(wlst, word, nsug, cpdsuggest);
289 }
290
291 // perhaps we forgot to hit space and two words ran together
292 if (!nosplitsugs && (nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
293 nsug = twowords(wlst, word, nsug, cpdsuggest);
294 }
295
296 } // repeating ``for'' statement compounding support
297
298 if (nsug < 0) {
299 // we ran out of memory - we should free up as much as possible
300 for (int i = 0; i < maxSug; i++)
301 if (wlst[i] != NULL) free(wlst[i]);
302 free(wlst);
303 wlst = NULL;
304 }
305
306 if (!nocompoundtwowords && (nsug > 0) && onlycompoundsug) *onlycompoundsug = 1;
307
308 *slst = wlst;
309 return nsug;
310 }
311
312 // generate suggestions for a word with typical mistake
313 // pass in address of array of char * pointers
314 #ifdef HUNSPELL_EXPERIMENTAL
315 int SuggestMgr::suggest_auto(char*** slst, const char * w, int nsug)
316 {
317 int nocompoundtwowords = 0;
318 char ** wlst;
319 int oldSug;
320
321 char w2[MAXWORDUTF8LEN];
322 const char * word = w;
323
324 // word reversing wrapper for complex prefixes
325 if (complexprefixes) {
326 strcpy(w2, w);
327 if (utf8) reverseword_utf(w2); else reverseword(w2);
328 word = w2;
329 }
330
331 if (*slst) {
332 wlst = *slst;
333 } else {
334 wlst = (char **) malloc(maxSug * sizeof(char *));
335 if (wlst == NULL) return -1;
336 }
337
338 for (int cpdsuggest=0; (cpdsuggest<2) && (nocompoundtwowords==0); cpdsuggest++) {
339
340 // limit compound suggestion
341 if (cpdsuggest > 0) oldSug = nsug;
342
343 // perhaps we made a typical fault of spelling
344 if ((nsug < maxSug) && (nsug > -1))
345 nsug = replchars(wlst, word, nsug, cpdsuggest);
346
347 // perhaps we made chose the wrong char from a related set
348 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs)))
349 nsug = mapchars(wlst, word, nsug, cpdsuggest);
350
351 if ((cpdsuggest==0) && (nsug>0)) nocompoundtwowords=1;
352
353 // perhaps we forgot to hit space and two words ran together
354
355 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs)) && check_forbidden(word, strlen(word))) {
356 nsug = twowords(wlst, word, nsug, cpdsuggest);
357 }
358
359 } // repeating ``for'' statement compounding support
360
361 if (nsug < 0) {
362 for (int i=0;i<maxSug; i++)
363 if (wlst[i] != NULL) free(wlst[i]);
364 free(wlst);
365 return -1;
366 }
367
368 *slst = wlst;
369 return nsug;
370 }
371 #endif // END OF HUNSPELL_EXPERIMENTAL CODE
372
373 // suggestions for an uppercase word (html -> HTML)
374 int SuggestMgr::capchars_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
375 {
376 char candidate[MAXSWUTF8L];
377 w_char candidate_utf[MAXSWL];
378 memcpy(candidate_utf, word, wl * sizeof(w_char));
379 mkallcap_utf(candidate_utf, wl, langnum);
380 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
381 return testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
382 }
383
384 // suggestions for an uppercase word (html -> HTML)
385 int SuggestMgr::capchars(char** wlst, const char * word, int ns, int cpdsuggest)
386 {
387 char candidate[MAXSWUTF8L];
388 strcpy(candidate, word);
389 mkallcap(candidate, csconv);
390 return testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
391 }
392
393 // suggestions for when chose the wrong char out of a related set
394 int SuggestMgr::mapchars(char** wlst, const char * word, int ns, int cpdsuggest)
395 {
396 char candidate[MAXSWUTF8L];
397 clock_t timelimit;
398 int timer;
399 candidate[0] = '\0';
400
401 int wl = strlen(word);
402 if (wl < 2 || ! pAMgr) return ns;
403
404 int nummap = pAMgr->get_nummap();
405 struct mapentry* maptable = pAMgr->get_maptable();
406 if (maptable==NULL) return ns;
407
408 timelimit = clock();
409 timer = MINTIMER;
410 return map_related(word, (char *) &candidate, 0, 0, wlst, cpdsuggest, ns, maptable, nummap, &timer, &timelimit);
411 }
412
413 int SuggestMgr::map_related(const char * word, char * candidate, int wn, int cn,
414 char** wlst, int cpdsuggest, int ns,
415 const mapentry* maptable, int nummap, int * timer, clock_t * timelimit)
416 {
417 if (*(word + wn) == '\0') {
418 int cwrd = 1;
419 *(candidate + cn) = '\0';
420 int wl = strlen(candidate);
421 for (int m=0; m < ns; m++)
422 if (strcmp(candidate, wlst[m]) == 0) cwrd = 0;
423 if ((cwrd) && checkword(candidate, wl, cpdsuggest, timer, timelimit)) {
424 if (ns < maxSug) {
425 wlst[ns] = mystrdup(candidate);
426 if (wlst[ns] == NULL) return -1;
427 ns++;
428 }
429 }
430 return ns;
431 }
432 int in_map = 0;
433 for (int j = 0; j < nummap; j++) {
434 for (int k = 0; k < maptable[j].len; k++) {
435 int len = strlen(maptable[j].set[k]);
436 if (strncmp(maptable[j].set[k], word + wn, len) == 0) {
437 in_map = 1;
438 for (int l = 0; l < maptable[j].len; l++) {
439 strcpy(candidate + cn, maptable[j].set[l]);
440 ns = map_related(word, candidate, wn + len, strlen(candidate), wlst,
441 cpdsuggest, ns, maptable, nummap, timer, timelimit);
442 if (!(*timer)) return ns;
443 }
444 }
445 }
446 }
447 if (!in_map) {
448 *(candidate + cn) = *(word + wn);
449 ns = map_related(word, candidate, wn + 1, cn + 1, wlst, cpdsuggest,
450 ns, maptable, nummap, timer, timelimit);
451 }
452 return ns;
453 }
454
455 // suggestions for a typical fault of spelling, that
456 // differs with more, than 1 letter from the right form.
457 int SuggestMgr::replchars(char** wlst, const char * word, int ns, int cpdsuggest)
458 {
459 char candidate[MAXSWUTF8L];
460 const char * r;
461 int lenr, lenp;
462 int wl = strlen(word);
463 if (wl < 2 || ! pAMgr) return ns;
464 int numrep = pAMgr->get_numrep();
465 struct replentry* reptable = pAMgr->get_reptable();
466 if (reptable==NULL) return ns;
467 for (int i=0; i < numrep; i++ ) {
468 r = word;
469 lenr = strlen(reptable[i].pattern2);
470 lenp = strlen(reptable[i].pattern);
471 // search every occurence of the pattern in the word
472 while ((r=strstr(r, reptable[i].pattern)) != NULL && (!reptable[i].end || strlen(r) == strlen(reptable[i].pattern)) &&
473 (!reptable[i].start || r == word)) {
474 strcpy(candidate, word);
475 if (r-word + lenr + strlen(r+lenp) >= MAXSWUTF8L) break;
476 strcpy(candidate+(r-word),reptable[i].pattern2);
477 strcpy(candidate+(r-word)+lenr, r+lenp);
478 ns = testsug(wlst, candidate, wl-lenp+lenr, ns, cpdsuggest, NULL, NULL);
479 if (ns == -1) return -1;
480 // check REP suggestions with space
481 char * sp = strchr(candidate, ' ');
482 if (sp) {
483 char * prev = candidate;
484 while (sp) {
485 *sp = '\0';
486 if (checkword(prev, strlen(prev), 0, NULL, NULL)) {
487 int oldns = ns;
488 *sp = ' ';
489 ns = testsug(wlst, sp + 1, strlen(sp + 1), ns, cpdsuggest, NULL, NULL);
490 if (ns == -1) return -1;
491 if (oldns < ns) {
492 free(wlst[ns - 1]);
493 wlst[ns - 1] = mystrdup(candidate);
494 if (!wlst[ns - 1]) return -1;
495 }
496 }
497 *sp = ' ';
498 prev = sp + 1;
499 sp = strchr(prev, ' ');
500 }
501 }
502 r++; // search for the next letter
503 }
504 }
505 return ns;
506 }
507
508 // perhaps we doubled two characters (pattern aba -> ababa, for example vacation -> vacacation)
509 int SuggestMgr::doubletwochars(char** wlst, const char * word, int ns, int cpdsuggest)
510 {
511 char candidate[MAXSWUTF8L];
512 int state=0;
513 int wl = strlen(word);
514 if (wl < 5 || ! pAMgr) return ns;
515 for (int i=2; i < wl; i++ ) {
516 if (word[i]==word[i-2]) {
517 state++;
518 if (state==3) {
519 strcpy(candidate,word);
520 strcpy(candidate+i-1,word+i+1);
521 ns = testsug(wlst, candidate, wl-2, ns, cpdsuggest, NULL, NULL);
522 if (ns == -1) return -1;
523 state=0;
524 }
525 } else {
526 state=0;
527 }
528 }
529 return ns;
530 }
531
532 // perhaps we doubled two characters (pattern aba -> ababa, for example vacation -> vacacation)
533 int SuggestMgr::doubletwochars_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
534 {
535 w_char candidate_utf[MAXSWL];
536 char candidate[MAXSWUTF8L];
537 int state=0;
538 if (wl < 5 || ! pAMgr) return ns;
539 for (int i=2; i < wl; i++) {
540 if (w_char_eq(word[i], word[i-2])) {
541 state++;
542 if (state==3) {
543 memcpy(candidate_utf, word, (i - 1) * sizeof(w_char));
544 memcpy(candidate_utf+i-1, word+i+1, (wl-i-1) * sizeof(w_char));
545 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl-2);
546 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
547 if (ns == -1) return -1;
548 state=0;
549 }
550 } else {
551 state=0;
552 }
553 }
554 return ns;
555 }
556
557 // error is wrong char in place of correct one (case and keyboard related version)
558 int SuggestMgr::badcharkey(char ** wlst, const char * word, int ns, int cpdsuggest)
559 {
560 char tmpc;
561 char candidate[MAXSWUTF8L];
562 int wl = strlen(word);
563 strcpy(candidate, word);
564 // swap out each char one by one and try uppercase and neighbor
565 // keyboard chars in its place to see if that makes a good word
566
567 for (int i=0; i < wl; i++) {
568 tmpc = candidate[i];
569 // check with uppercase letters
570 candidate[i] = csconv[((unsigned char)tmpc)].cupper;
571 if (tmpc != candidate[i]) {
572 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
573 if (ns == -1) return -1;
574 candidate[i] = tmpc;
575 }
576 // check neighbor characters in keyboard string
577 if (!ckey) continue;
578 char * loc = strchr(ckey, tmpc);
579 while (loc) {
580 if ((loc > ckey) && (*(loc - 1) != '|')) {
581 candidate[i] = *(loc - 1);
582 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
583 if (ns == -1) return -1;
584 }
585 if ((*(loc + 1) != '|') && (*(loc + 1) != '\0')) {
586 candidate[i] = *(loc + 1);
587 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
588 if (ns == -1) return -1;
589 }
590 loc = strchr(loc + 1, tmpc);
591 }
592 candidate[i] = tmpc;
593 }
594 return ns;
595 }
596
597 // error is wrong char in place of correct one (case and keyboard related version)
598 int SuggestMgr::badcharkey_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
599 {
600 w_char tmpc;
601 w_char candidate_utf[MAXSWL];
602 char candidate[MAXSWUTF8L];
603 memcpy(candidate_utf, word, wl * sizeof(w_char));
604 // swap out each char one by one and try all the tryme
605 // chars in its place to see if that makes a good word
606 for (int i=0; i < wl; i++) {
607 tmpc = candidate_utf[i];
608 // check with uppercase letters
609 mkallcap_utf(candidate_utf + i, 1, langnum);
610 if (!w_char_eq(tmpc, candidate_utf[i])) {
611 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
612 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
613 if (ns == -1) return -1;
614 candidate_utf[i] = tmpc;
615 }
616 // check neighbor characters in keyboard string
617 if (!ckey) continue;
618 w_char * loc = ckey_utf;
619 while ((loc < (ckey_utf + ckeyl)) && !w_char_eq(*loc, tmpc)) loc++;
620 while (loc < (ckey_utf + ckeyl)) {
621 if ((loc > ckey_utf) && !w_char_eq(*(loc - 1), W_VLINE)) {
622 candidate_utf[i] = *(loc - 1);
623 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
624 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
625 if (ns == -1) return -1;
626 }
627 if (((loc + 1) < (ckey_utf + ckeyl)) && !w_char_eq(*(loc + 1), W_VLINE)) {
628 candidate_utf[i] = *(loc + 1);
629 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
630 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
631 if (ns == -1) return -1;
632 }
633 do { loc++; } while ((loc < (ckey_utf + ckeyl)) && !w_char_eq(*loc, tmpc));
634 }
635 candidate_utf[i] = tmpc;
636 }
637 return ns;
638 }
639
640 // error is wrong char in place of correct one
641 int SuggestMgr::badchar(char ** wlst, const char * word, int ns, int cpdsuggest)
642 {
643 char tmpc;
644 char candidate[MAXSWUTF8L];
645 clock_t timelimit = clock();
646 int timer = MINTIMER;
647 int wl = strlen(word);
648 strcpy(candidate, word);
649 // swap out each char one by one and try all the tryme
650 // chars in its place to see if that makes a good word
651 for (int j=0; j < ctryl; j++) {
652 for (int i=wl-1; i >= 0; i--) {
653 tmpc = candidate[i];
654 if (ctry[j] == tmpc) continue;
655 candidate[i] = ctry[j];
656 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, &timer, &timelimit);
657 if (ns == -1) return -1;
658 if (!timer) return ns;
659 candidate[i] = tmpc;
660 }
661 }
662 return ns;
663 }
664
665 // error is wrong char in place of correct one
666 int SuggestMgr::badchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
667 {
668 w_char tmpc;
669 w_char candidate_utf[MAXSWL];
670 char candidate[MAXSWUTF8L];
671 clock_t timelimit = clock();
672 int timer = MINTIMER;
673 memcpy(candidate_utf, word, wl * sizeof(w_char));
674 // swap out each char one by one and try all the tryme
675 // chars in its place to see if that makes a good word
676 for (int j=0; j < ctryl; j++) {
677 for (int i=wl-1; i >= 0; i--) {
678 tmpc = candidate_utf[i];
679 if (w_char_eq(tmpc, ctry_utf[j])) continue;
680 candidate_utf[i] = ctry_utf[j];
681 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
682 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, &timer, &timelimit);
683 if (ns == -1) return -1;
684 if (!timer) return ns;
685 candidate_utf[i] = tmpc;
686 }
687 }
688 return ns;
689 }
690
691 // error is word has an extra letter it does not need
692 int SuggestMgr::extrachar_utf(char** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
693 {
694 char candidate[MAXSWUTF8L];
695 w_char candidate_utf[MAXSWL];
696 w_char * p;
697 w_char tmpc = W_VLINE; // not used value, only for VCC warning message
698 if (wl < 2) return ns;
699 // try omitting one char of word at a time
700 memcpy(candidate_utf, word, wl * sizeof(w_char));
701 for (p = candidate_utf + wl - 1; p >= candidate_utf; p--) {
702 w_char tmpc2 = *p;
703 if (p < candidate_utf + wl - 1) *p = tmpc;
704 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl - 1);
705 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
706 if (ns == -1) return -1;
707 tmpc = tmpc2;
708 }
709 return ns;
710 }
711
712 // error is word has an extra letter it does not need
713 int SuggestMgr::extrachar(char** wlst, const char * word, int ns, int cpdsuggest)
714 {
715 char tmpc = '\0';
716 char candidate[MAXSWUTF8L];
717 char * p;
718 int wl = strlen(word);
719 if (wl < 2) return ns;
720 // try omitting one char of word at a time
721 strcpy (candidate, word);
722 for (p = candidate + wl - 1; p >=candidate; p--) {
723 char tmpc2 = *p;
724 *p = tmpc;
725 ns = testsug(wlst, candidate, wl-1, ns, cpdsuggest, NULL, NULL);
726 if (ns == -1) return -1;
727 tmpc = tmpc2;
728 }
729 return ns;
730 }
731
732 // error is missing a letter it needs
733 int SuggestMgr::forgotchar(char ** wlst, const char * word, int ns, int cpdsuggest)
734 {
735 char candidate[MAXSWUTF8L];
736 char * p;
737 clock_t timelimit = clock();
738 int timer = MINTIMER;
739 int wl = strlen(word);
740 // try inserting a tryme character before every letter (and the null terminator)
741 for (int i = 0; i < ctryl; i++) {
742 strcpy(candidate, word);
743 for (p = candidate + wl; p >= candidate; p--) {
744 *(p+1) = *p;
745 *p = ctry[i];
746 ns = testsug(wlst, candidate, wl+1, ns, cpdsuggest, &timer, &timelimit);
747 if (ns == -1) return -1;
748 if (!timer) return ns;
749 }
750 }
751 return ns;
752 }
753
754 // error is missing a letter it needs
755 int SuggestMgr::forgotchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
756 {
757 w_char candidate_utf[MAXSWL];
758 char candidate[MAXSWUTF8L];
759 w_char * p;
760 clock_t timelimit = clock();
761 int timer = MINTIMER;
762 // try inserting a tryme character at the end of the word and before every letter
763 for (int i = 0; i < ctryl; i++) {
764 memcpy (candidate_utf, word, wl * sizeof(w_char));
765 for (p = candidate_utf + wl; p >= candidate_utf; p--) {
766 *(p + 1) = *p;
767 *p = ctry_utf[i];
768 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl + 1);
769 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, &timer, &timelimit);
770 if (ns == -1) return -1;
771 if (!timer) return ns;
772 }
773 }
774 return ns;
775 }
776
777
778 /* error is should have been two words */
779 int SuggestMgr::twowords(char ** wlst, const char * word, int ns, int cpdsuggest)
780 {
781 char candidate[MAXSWUTF8L];
782 char * p;
783 int c1, c2;
784 int forbidden = 0;
785 int cwrd;
786
787 int wl=strlen(word);
788 if (wl < 3) return ns;
789
790 if (langnum == LANG_hu) forbidden = check_forbidden(word, wl);
791
792 strcpy(candidate + 1, word);
793 // split the string into two pieces after every char
794 // if both pieces are good words make them a suggestion
795 for (p = candidate + 1; p[1] != '\0'; p++) {
796 p[-1] = *p;
797 // go to end of the UTF-8 character
798 while (utf8 && ((p[1] & 0xc0) == 0x80)) {
799 *p = p[1];
800 p++;
801 }
802 if (utf8 && p[1] == '\0') break; // last UTF-8 character
803 *p = '\0';
804 c1 = checkword(candidate,strlen(candidate), cpdsuggest, NULL, NULL);
805 if (c1) {
806 c2 = checkword((p+1),strlen(p+1), cpdsuggest, NULL, NULL);
807 if (c2) {
808 *p = ' ';
809
810 // spec. Hungarian code (need a better compound word support)
811 if ((langnum == LANG_hu) && !forbidden &&
812 // if 3 repeating letter, use - instead of space
813 (((p[-1] == p[1]) && (((p>candidate+1) && (p[-1] == p[-2])) || (p[-1] == p[2]))) ||
814 // or multiple compounding, with more, than 6 syllables
815 ((c1 == 3) && (c2 >= 2)))) *p = '-';
816
817 cwrd = 1;
818 for (int k=0; k < ns; k++)
819 if (strcmp(candidate,wlst[k]) == 0) cwrd = 0;
820 if (ns < maxSug) {
821 if (cwrd) {
822 wlst[ns] = mystrdup(candidate);
823 if (wlst[ns] == NULL) return -1;
824 ns++;
825 }
826 } else return ns;
827 // add two word suggestion with dash, if TRY string contains
828 // "a" or "-"
829 // NOTE: cwrd doesn't modified for REP twoword sugg.
830 if (ctry && (strchr(ctry, 'a') || strchr(ctry, '-')) &&
831 mystrlen(p + 1) > 1 &&
832 mystrlen(candidate) - mystrlen(p) > 1) {
833 *p = '-';
834 for (int k=0; k < ns; k++)
835 if (strcmp(candidate,wlst[k]) == 0) cwrd = 0;
836 if (ns < maxSug) {
837 if (cwrd) {
838 wlst[ns] = mystrdup(candidate);
839 if (wlst[ns] == NULL) return -1;
840 ns++;
841 }
842 } else return ns;
843 }
844 }
845 }
846 }
847 return ns;
848 }
849
850
851 // error is adjacent letter were swapped
852 int SuggestMgr::swapchar(char ** wlst, const char * word, int ns, int cpdsuggest)
853 {
854 char candidate[MAXSWUTF8L];
855 char * p;
856 char tmpc;
857 int wl=strlen(word);
858 // try swapping adjacent chars one by one
859 strcpy(candidate, word);
860 for (p = candidate; p[1] != 0; p++) {
861 tmpc = *p;
862 *p = p[1];
863 p[1] = tmpc;
864 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
865 if (ns == -1) return -1;
866 p[1] = *p;
867 *p = tmpc;
868 }
869 // try double swaps for short words
870 // ahev -> have, owudl -> would
871 if (wl == 4 || wl == 5) {
872 candidate[0] = word[1];
873 candidate[1] = word[0];
874 candidate[2] = word[2];
875 candidate[wl - 2] = word[wl - 1];
876 candidate[wl - 1] = word[wl - 2];
877 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
878 if (ns == -1) return -1;
879 if (wl == 5) {
880 candidate[0] = word[0];
881 candidate[1] = word[2];
882 candidate[2] = word[1];
883 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
884 if (ns == -1) return -1;
885 }
886 }
887 return ns;
888 }
889
890 // error is adjacent letter were swapped
891 int SuggestMgr::swapchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
892 {
893 w_char candidate_utf[MAXSWL];
894 char candidate[MAXSWUTF8L];
895 w_char * p;
896 w_char tmpc;
897 int len = 0;
898 // try swapping adjacent chars one by one
899 memcpy (candidate_utf, word, wl * sizeof(w_char));
900 for (p = candidate_utf; p < (candidate_utf + wl - 1); p++) {
901 tmpc = *p;
902 *p = p[1];
903 p[1] = tmpc;
904 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
905 if (len == 0) len = strlen(candidate);
906 ns = testsug(wlst, candidate, len, ns, cpdsuggest, NULL, NULL);
907 if (ns == -1) return -1;
908 p[1] = *p;
909 *p = tmpc;
910 }
911 // try double swaps for short words
912 // ahev -> have, owudl -> would, suodn -> sound
913 if (wl == 4 || wl == 5) {
914 candidate_utf[0] = word[1];
915 candidate_utf[1] = word[0];
916 candidate_utf[2] = word[2];
917 candidate_utf[wl - 2] = word[wl - 1];
918 candidate_utf[wl - 1] = word[wl - 2];
919 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
920 ns = testsug(wlst, candidate, len, ns, cpdsuggest, NULL, NULL);
921 if (ns == -1) return -1;
922 if (wl == 5) {
923 candidate_utf[0] = word[0];
924 candidate_utf[1] = word[2];
925 candidate_utf[2] = word[1];
926 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
927 ns = testsug(wlst, candidate, len, ns, cpdsuggest, NULL, NULL);
928 if (ns == -1) return -1;
929 }
930 }
931 return ns;
932 }
933
934 // error is not adjacent letter were swapped
935 int SuggestMgr::longswapchar(char ** wlst, const char * word, int ns, int cpdsuggest)
936 {
937 char candidate[MAXSWUTF8L];
938 char * p;
939 char * q;
940 char tmpc;
941 int wl=strlen(word);
942 // try swapping not adjacent chars one by one
943 strcpy(candidate, word);
944 for (p = candidate; *p != 0; p++) {
945 for (q = candidate; *q != 0; q++) {
946 if (abs((int)(p-q)) > 1) {
947 tmpc = *p;
948 *p = *q;
949 *q = tmpc;
950 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
951 if (ns == -1) return -1;
952 *q = *p;
953 *p = tmpc;
954 }
955 }
956 }
957 return ns;
958 }
959
960
961 // error is adjacent letter were swapped
962 int SuggestMgr::longswapchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
963 {
964 w_char candidate_utf[MAXSWL];
965 char candidate[MAXSWUTF8L];
966 w_char * p;
967 w_char * q;
968 w_char tmpc;
969 // try swapping not adjacent chars
970 memcpy (candidate_utf, word, wl * sizeof(w_char));
971 for (p = candidate_utf; p < (candidate_utf + wl); p++) {
972 for (q = candidate_utf; q < (candidate_utf + wl); q++) {
973 if (abs((int)(p-q)) > 1) {
974 tmpc = *p;
975 *p = *q;
976 *q = tmpc;
977 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
978 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
979 if (ns == -1) return -1;
980 *q = *p;
981 *p = tmpc;
982 }
983 }
984 }
985 return ns;
986 }
987
988 // error is a letter was moved
989 int SuggestMgr::movechar(char ** wlst, const char * word, int ns, int cpdsuggest)
990 {
991 char candidate[MAXSWUTF8L];
992 char * p;
993 char * q;
994 char tmpc;
995
996 int wl=strlen(word);
997 // try moving a char
998 strcpy(candidate, word);
999 for (p = candidate; *p != 0; p++) {
1000 for (q = p + 1; (*q != 0) && ((q - p) < 10); q++) {
1001 tmpc = *(q-1);
1002 *(q-1) = *q;
1003 *q = tmpc;
1004 if ((q-p) < 2) continue; // omit swap char
1005 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
1006 if (ns == -1) return -1;
1007 }
1008 strcpy(candidate, word);
1009 }
1010 for (p = candidate + wl - 1; p > candidate; p--) {
1011 for (q = p - 1; (q >= candidate) && ((p - q) < 10); q--) {
1012 tmpc = *(q+1);
1013 *(q+1) = *q;
1014 *q = tmpc;
1015 if ((p-q) < 2) continue; // omit swap char
1016 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
1017 if (ns == -1) return -1;
1018 }
1019 strcpy(candidate, word);
1020 }
1021 return ns;
1022 }
1023
1024 // error is a letter was moved
1025 int SuggestMgr::movechar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
1026 {
1027 w_char candidate_utf[MAXSWL];
1028 char candidate[MAXSWUTF8L];
1029 w_char * p;
1030 w_char * q;
1031 w_char tmpc;
1032 // try moving a char
1033 memcpy (candidate_utf, word, wl * sizeof(w_char));
1034 for (p = candidate_utf; p < (candidate_utf + wl); p++) {
1035 for (q = p + 1; (q < (candidate_utf + wl)) && ((q - p) < 10); q++) {
1036 tmpc = *(q-1);
1037 *(q-1) = *q;
1038 *q = tmpc;
1039 if ((q-p) < 2) continue; // omit swap char
1040 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
1041 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
1042 if (ns == -1) return -1;
1043 }
1044 memcpy (candidate_utf, word, wl * sizeof(w_char));
1045 }
1046 for (p = candidate_utf + wl - 1; p > candidate_utf; p--) {
1047 for (q = p - 1; (q >= candidate_utf) && ((p - q) < 10); q--) {
1048 tmpc = *(q+1);
1049 *(q+1) = *q;
1050 *q = tmpc;
1051 if ((p-q) < 2) continue; // omit swap char
1052 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
1053 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
1054 if (ns == -1) return -1;
1055 }
1056 memcpy (candidate_utf, word, wl * sizeof(w_char));
1057 }
1058 return ns;
1059 }
1060
1061 // generate a set of suggestions for very poorly spelled words
1062 int SuggestMgr::ngsuggest(char** wlst, char * w, int ns, HashMgr** pHMgr, int md)
1063 {
1064
1065 int i, j;
1066 int lval;
1067 int sc, scphon;
1068 int lp, lpphon;
1069 int nonbmp = 0;
1070
1071 // exhaustively search through all root words
1072 // keeping track of the MAX_ROOTS most similar root words
1073 struct hentry * roots[MAX_ROOTS];
1074 char * rootsphon[MAX_ROOTS];
1075 int scores[MAX_ROOTS];
1076 int scoresphon[MAX_ROOTS];
1077 for (i = 0; i < MAX_ROOTS; i++) {
1078 roots[i] = NULL;
1079 scores[i] = -100 * i;
1080 rootsphon[i] = NULL;
1081 scoresphon[i] = -100 * i;
1082 }
1083 lp = MAX_ROOTS - 1;
1084 lpphon = MAX_ROOTS - 1;
1085 scphon = -20000;
1086 int low = NGRAM_LOWERING;
1087
1088 char w2[MAXWORDUTF8LEN];
1089 char f[MAXSWUTF8L];
1090 char * word = w;
1091
1092 // word reversing wrapper for complex prefixes
1093 if (complexprefixes) {
1094 strcpy(w2, w);
1095 if (utf8) reverseword_utf(w2); else reverseword(w2);
1096 word = w2;
1097 }
1098
1099 char mw[MAXSWUTF8L];
1100 w_char u8[MAXSWL];
1101 int nc = strlen(word);
1102 int n = (utf8) ? u8_u16(u8, MAXSWL, word) : nc;
1103
1104 // set character based ngram suggestion for words with non-BMP Unicode characters
1105 if (n == -1) {
1106 utf8 = 0; // XXX not state-free
1107 n = nc;
1108 nonbmp = 1;
1109 low = 0;
1110 }
1111
1112 struct hentry* hp = NULL;
1113 int col = -1;
1114 phonetable * ph = (pAMgr) ? pAMgr->get_phonetable() : NULL;
1115 char target[MAXSWUTF8L];
1116 char candidate[MAXSWUTF8L];
1117 if (ph) {
1118 if (utf8) {
1119 w_char _w[MAXSWL];
1120 int _wl = u8_u16(_w, MAXSWL, word);
1121 mkallcap_utf(_w, _wl, langnum);
1122 u16_u8(candidate, MAXSWUTF8L, _w, _wl);
1123 } else {
1124 strcpy(candidate, word);
1125 if (!nonbmp) mkallcap(candidate, csconv);
1126 }
1127 phonet(candidate, target, nc, *ph); // XXX phonet() is 8-bit (nc, not n)
1128 }
1129
1130 FLAG forbiddenword = pAMgr ? pAMgr->get_forbiddenword() : FLAG_NULL;
1131 FLAG nosuggest = pAMgr ? pAMgr->get_nosuggest() : FLAG_NULL;
1132 FLAG nongramsuggest = pAMgr ? pAMgr->get_nongramsuggest() : FLAG_NULL;
1133 FLAG onlyincompound = pAMgr ? pAMgr->get_onlyincompound() : FLAG_NULL;
1134
1135 for (i = 0; i < md; i++) {
1136 while (0 != (hp = (pHMgr[i])->walk_hashtable(col, hp))) {
1137 if ((hp->astr) && (pAMgr) &&
1138 (TESTAFF(hp->astr, forbiddenword, hp->alen) ||
1139 TESTAFF(hp->astr, ONLYUPCASEFLAG, hp->alen) ||
1140 TESTAFF(hp->astr, nosuggest, hp->alen) ||
1141 TESTAFF(hp->astr, nongramsuggest, hp->alen) ||
1142 TESTAFF(hp->astr, onlyincompound, hp->alen))) continue;
1143
1144 sc = ngram(3, word, HENTRY_WORD(hp), NGRAM_LONGER_WORSE + low) +
1145 leftcommonsubstring(word, HENTRY_WORD(hp));
1146
1147 // check special pronounciation
1148 if ((hp->var & H_OPT_PHON) && copy_field(f, HENTRY_DATA(hp), MORPH_PHON)) {
1149 int sc2 = ngram(3, word, f, NGRAM_LONGER_WORSE + low) +
1150 + leftcommonsubstring(word, f);
1151 if (sc2 > sc) sc = sc2;
1152 }
1153
1154 scphon = -20000;
1155 if (ph && (sc > 2) && (abs(n - (int) hp->clen) <= 3)) {
1156 char target2[MAXSWUTF8L];
1157 if (utf8) {
1158 w_char _w[MAXSWL];
1159 int _wl = u8_u16(_w, MAXSWL, HENTRY_WORD(hp));
1160 mkallcap_utf(_w, _wl, langnum);
1161 u16_u8(candidate, MAXSWUTF8L, _w, _wl);
1162 } else {
1163 strcpy(candidate, HENTRY_WORD(hp));
1164 mkallcap(candidate, csconv);
1165 }
1166 phonet(candidate, target2, -1, *ph);
1167 scphon = 2 * ngram(3, target, target2, NGRAM_LONGER_WORSE);
1168 }
1169
1170 if (sc > scores[lp]) {
1171 scores[lp] = sc;
1172 roots[lp] = hp;
1173 lval = sc;
1174 for (j=0; j < MAX_ROOTS; j++)
1175 if (scores[j] < lval) {
1176 lp = j;
1177 lval = scores[j];
1178 }
1179 }
1180
1181
1182 if (scphon > scoresphon[lpphon]) {
1183 scoresphon[lpphon] = scphon;
1184 rootsphon[lpphon] = HENTRY_WORD(hp);
1185 lval = scphon;
1186 for (j=0; j < MAX_ROOTS; j++)
1187 if (scoresphon[j] < lval) {
1188 lpphon = j;
1189 lval = scoresphon[j];
1190 }
1191 }
1192 }}
1193
1194 // find minimum threshold for a passable suggestion
1195 // mangle original word three differnt ways
1196 // and score them to generate a minimum acceptable score
1197 int thresh = 0;
1198 for (int sp = 1; sp < 4; sp++) {
1199 if (utf8) {
1200 for (int k=sp; k < n; k+=4) *((unsigned short *) u8 + k) = '*';
1201 u16_u8(mw, MAXSWUTF8L, u8, n);
1202 thresh = thresh + ngram(n, word, mw, NGRAM_ANY_MISMATCH + low);
1203 } else {
1204 strcpy(mw, word);
1205 for (int k=sp; k < n; k+=4) *(mw + k) = '*';
1206 thresh = thresh + ngram(n, word, mw, NGRAM_ANY_MISMATCH + low);
1207 }
1208 }
1209 thresh = thresh / 3;
1210 thresh--;
1211
1212 // now expand affixes on each of these root words and
1213 // and use length adjusted ngram scores to select
1214 // possible suggestions
1215 char * guess[MAX_GUESS];
1216 char * guessorig[MAX_GUESS];
1217 int gscore[MAX_GUESS];
1218 for(i=0;i<MAX_GUESS;i++) {
1219 guess[i] = NULL;
1220 guessorig[i] = NULL;
1221 gscore[i] = -100 * i;
1222 }
1223
1224 lp = MAX_GUESS - 1;
1225
1226 struct guessword * glst;
1227 glst = (struct guessword *) calloc(MAX_WORDS,sizeof(struct guessword));
1228 if (! glst) {
1229 if (nonbmp) utf8 = 1;
1230 return ns;
1231 }
1232
1233 for (i = 0; i < MAX_ROOTS; i++) {
1234 if (roots[i]) {
1235 struct hentry * rp = roots[i];
1236 int nw = pAMgr->expand_rootword(glst, MAX_WORDS, HENTRY_WORD(rp), rp->blen,
1237 rp->astr, rp->alen, word, nc,
1238 ((rp->var & H_OPT_PHON) ? copy_field(f, HENTRY_DATA(rp), MORPH_PHON) : NULL));
1239
1240 for (int k = 0; k < nw ; k++) {
1241 sc = ngram(n, word, glst[k].word, NGRAM_ANY_MISMATCH + low) +
1242 leftcommonsubstring(word, glst[k].word);
1243
1244 if (sc > thresh) {
1245 if (sc > gscore[lp]) {
1246 if (guess[lp]) {
1247 free (guess[lp]);
1248 if (guessorig[lp]) {
1249 free(guessorig[lp]);
1250 guessorig[lp] = NULL;
1251 }
1252 }
1253 gscore[lp] = sc;
1254 guess[lp] = glst[k].word;
1255 guessorig[lp] = glst[k].orig;
1256 lval = sc;
1257 for (j=0; j < MAX_GUESS; j++)
1258 if (gscore[j] < lval) {
1259 lp = j;
1260 lval = gscore[j];
1261 }
1262 } else {
1263 free(glst[k].word);
1264 if (glst[k].orig) free(glst[k].orig);
1265 }
1266 } else {
1267 free(glst[k].word);
1268 if (glst[k].orig) free(glst[k].orig);
1269 }
1270 }
1271 }
1272 }
1273 free(glst);
1274
1275 // now we are done generating guesses
1276 // sort in order of decreasing score
1277
1278
1279 bubblesort(&guess[0], &guessorig[0], &gscore[0], MAX_GUESS);
1280 if (ph) bubblesort(&rootsphon[0], NULL, &scoresphon[0], MAX_ROOTS);
1281
1282 // weight suggestions with a similarity index, based on
1283 // the longest common subsequent algorithm and resort
1284
1285 int is_swap = 0;
1286 int re = 0;
1287 double fact = 1.0;
1288 if (pAMgr) {
1289 int maxd = pAMgr->get_maxdiff();
1290 if (maxd >= 0) fact = (10.0 - maxd)/5.0;
1291 }
1292
1293 for (i=0; i < MAX_GUESS; i++) {
1294 if (guess[i]) {
1295 // lowering guess[i]
1296 char gl[MAXSWUTF8L];
1297 int len;
1298 if (utf8) {
1299 w_char _w[MAXSWL];
1300 len = u8_u16(_w, MAXSWL, guess[i]);
1301 mkallsmall_utf(_w, len, langnum);
1302 u16_u8(gl, MAXSWUTF8L, _w, len);
1303 } else {
1304 strcpy(gl, guess[i]);
1305 if (!nonbmp) mkallsmall(gl, csconv);
1306 len = strlen(guess[i]);
1307 }
1308
1309 int _lcs = lcslen(word, gl);
1310
1311 // same characters with different casing
1312 if ((n == len) && (n == _lcs)) {
1313 gscore[i] += 2000;
1314 break;
1315 }
1316 // using 2-gram instead of 3, and other weightening
1317
1318 re = ngram(2, word, gl, NGRAM_ANY_MISMATCH + low + NGRAM_WEIGHTED) +
1319 ngram(2, gl, word, NGRAM_ANY_MISMATCH + low + NGRAM_WEIGHTED);
1320
1321 gscore[i] =
1322 // length of longest common subsequent minus length difference
1323 2 * _lcs - abs((int) (n - len)) +
1324 // weight length of the left common substring
1325 leftcommonsubstring(word, gl) +
1326 // weight equal character positions
1327 (!nonbmp && commoncharacterpositions(word, gl, &is_swap) ? 1: 0) +
1328 // swap character (not neighboring)
1329 ((is_swap) ? 10 : 0) +
1330 // ngram
1331 ngram(4, word, gl, NGRAM_ANY_MISMATCH + low) +
1332 // weighted ngrams
1333 re +
1334 // different limit for dictionaries with PHONE rules
1335 (ph ? (re < len * fact ? -1000 : 0) : (re < (n + len)*fact? -1000 : 0));
1336 }
1337 }
1338
1339 bubblesort(&guess[0], &guessorig[0], &gscore[0], MAX_GUESS);
1340
1341 // phonetic version
1342 if (ph) for (i=0; i < MAX_ROOTS; i++) {
1343 if (rootsphon[i]) {
1344 // lowering rootphon[i]
1345 char gl[MAXSWUTF8L];
1346 int len;
1347 if (utf8) {
1348 w_char _w[MAXSWL];
1349 len = u8_u16(_w, MAXSWL, rootsphon[i]);
1350 mkallsmall_utf(_w, len, langnum);
1351 u16_u8(gl, MAXSWUTF8L, _w, len);
1352 } else {
1353 strcpy(gl, rootsphon[i]);
1354 if (!nonbmp) mkallsmall(gl, csconv);
1355 len = strlen(rootsphon[i]);
1356 }
1357
1358 // heuristic weigthing of ngram scores
1359 scoresphon[i] += 2 * lcslen(word, gl) - abs((int) (n - len)) +
1360 // weight length of the left common substring
1361 leftcommonsubstring(word, gl);
1362 }
1363 }
1364
1365 if (ph) bubblesort(&rootsphon[0], NULL, &scoresphon[0], MAX_ROOTS);
1366
1367 // copy over
1368 int oldns = ns;
1369
1370 int same = 0;
1371 for (i=0; i < MAX_GUESS; i++) {
1372 if (guess[i]) {
1373 if ((ns < oldns + maxngramsugs) && (ns < maxSug) && (!same || (gscore[i] > 1000))) {
1374 int unique = 1;
1375 // leave only excellent suggestions, if exists
1376 if (gscore[i] > 1000) same = 1; else if (gscore[i] < -100) {
1377 same = 1;
1378 // keep the best ngram suggestions, unless in ONLYMAXDIFF mode
1379 if (ns > oldns || (pAMgr && pAMgr->get_onlymaxdiff())) {
1380 free(guess[i]);
1381 if (guessorig[i]) free(guessorig[i]);
1382 continue;
1383 }
1384 }
1385 for (j = 0; j < ns; j++) {
1386 // don't suggest previous suggestions or a previous suggestion with prefixes or affixes
1387 if ((!guessorig[i] && strstr(guess[i], wlst[j])) ||
1388 (guessorig[i] && strstr(guessorig[i], wlst[j])) ||
1389 // check forbidden words
1390 !checkword(guess[i], strlen(guess[i]), 0, NULL, NULL)) unique = 0;
1391 }
1392 if (unique) {
1393 wlst[ns++] = guess[i];
1394 if (guessorig[i]) {
1395 free(guess[i]);
1396 wlst[ns-1] = guessorig[i];
1397 }
1398 } else {
1399 free(guess[i]);
1400 if (guessorig[i]) free(guessorig[i]);
1401 }
1402 } else {
1403 free(guess[i]);
1404 if (guessorig[i]) free(guessorig[i]);
1405 }
1406 }
1407 }
1408
1409 oldns = ns;
1410 if (ph) for (i=0; i < MAX_ROOTS; i++) {
1411 if (rootsphon[i]) {
1412 if ((ns < oldns + MAXPHONSUGS) && (ns < maxSug)) {
1413 int unique = 1;
1414 for (j = 0; j < ns; j++) {
1415 // don't suggest previous suggestions or a previous suggestion with prefixes or affixes
1416 if (strstr(rootsphon[i], wlst[j]) ||
1417 // check forbidden words
1418 !checkword(rootsphon[i], strlen(rootsphon[i]), 0, NULL, NULL)) unique = 0;
1419 }
1420 if (unique) {
1421 wlst[ns++] = mystrdup(rootsphon[i]);
1422 if (!wlst[ns - 1]) return ns - 1;
1423 }
1424 }
1425 }
1426 }
1427
1428 if (nonbmp) utf8 = 1;
1429 return ns;
1430 }
1431
1432
1433 // see if a candidate suggestion is spelled correctly
1434 // needs to check both root words and words with affixes
1435
1436 // obsolote MySpell-HU modifications:
1437 // return value 2 and 3 marks compounding with hyphen (-)
1438 // `3' marks roots without suffix
1439 int SuggestMgr::checkword(const char * word, int len, int cpdsuggest, int * timer, clock_t * timelimit)
1440 {
1441 struct hentry * rv=NULL;
1442 struct hentry * rv2=NULL;
1443 int nosuffix = 0;
1444
1445 // check time limit
1446 if (timer) {
1447 (*timer)--;
1448 if (!(*timer) && timelimit) {
1449 if ((clock() - *timelimit) > TIMELIMIT) return 0;
1450 *timer = MAXPLUSTIMER;
1451 }
1452 }
1453
1454 if (pAMgr) {
1455 if (cpdsuggest==1) {
1456 if (pAMgr->get_compound()) {
1457 rv = pAMgr->compound_check(word, len, 0, 0, 100, 0, NULL, 0, 1, 0); //EXT
1458 if (rv && (!(rv2 = pAMgr->lookup(word)) || !rv2->astr ||
1459 !(TESTAFF(rv2->astr,pAMgr->get_forbiddenword(),rv2->alen) ||
1460 TESTAFF(rv2->astr,pAMgr->get_nosuggest(),rv2->alen)))) return 3; // XXX obsolote categorisation + only ICONV needs affix flag check?
1461 }
1462 return 0;
1463 }
1464
1465 rv = pAMgr->lookup(word);
1466
1467 if (rv) {
1468 if ((rv->astr) && (TESTAFF(rv->astr,pAMgr->get_forbiddenword(),rv->alen)
1469 || TESTAFF(rv->astr,pAMgr->get_nosuggest(),rv->alen))) return 0;
1470 while (rv) {
1471 if (rv->astr && (TESTAFF(rv->astr,pAMgr->get_needaffix(),rv->alen) ||
1472 TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
1473 TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) {
1474 rv = rv->next_homonym;
1475 } else break;
1476 }
1477 } else rv = pAMgr->prefix_check(word, len, 0); // only prefix, and prefix + suffix XXX
1478
1479 if (rv) {
1480 nosuffix=1;
1481 } else {
1482 rv = pAMgr->suffix_check(word, len, 0, NULL, NULL, 0, NULL); // only suffix
1483 }
1484
1485 if (!rv && pAMgr->have_contclass()) {
1486 rv = pAMgr->suffix_check_twosfx(word, len, 0, NULL, FLAG_NULL);
1487 if (!rv) rv = pAMgr->prefix_check_twosfx(word, len, 1, FLAG_NULL);
1488 }
1489
1490 // check forbidden words
1491 if ((rv) && (rv->astr) && (TESTAFF(rv->astr,pAMgr->get_forbiddenword(),rv->alen) ||
1492 TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
1493 TESTAFF(rv->astr,pAMgr->get_nosuggest(),rv->alen) ||
1494 TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) return 0;
1495
1496 if (rv) { // XXX obsolote
1497 if ((pAMgr->get_compoundflag()) &&
1498 TESTAFF(rv->astr, pAMgr->get_compoundflag(), rv->alen)) return 2 + nosuffix;
1499 return 1;
1500 }
1501 }
1502 return 0;
1503 }
1504
1505 int SuggestMgr::check_forbidden(const char * word, int len)
1506 {
1507 struct hentry * rv = NULL;
1508
1509 if (pAMgr) {
1510 rv = pAMgr->lookup(word);
1511 if (rv && rv->astr && (TESTAFF(rv->astr,pAMgr->get_needaffix(),rv->alen) ||
1512 TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) rv = NULL;
1513 if (!(pAMgr->prefix_check(word,len,1)))
1514 rv = pAMgr->suffix_check(word,len, 0, NULL, NULL, 0, NULL); // prefix+suffix, suffix
1515 // check forbidden words
1516 if ((rv) && (rv->astr) && TESTAFF(rv->astr,pAMgr->get_forbiddenword(),rv->alen)) return 1;
1517 }
1518 return 0;
1519 }
1520
1521 #ifdef HUNSPELL_EXPERIMENTAL
1522 // suggest possible stems
1523 int SuggestMgr::suggest_pos_stems(char*** slst, const char * w, int nsug)
1524 {
1525 char ** wlst;
1526
1527 struct hentry * rv = NULL;
1528
1529 char w2[MAXSWUTF8L];
1530 const char * word = w;
1531
1532 // word reversing wrapper for complex prefixes
1533 if (complexprefixes) {
1534 strcpy(w2, w);
1535 if (utf8) reverseword_utf(w2); else reverseword(w2);
1536 word = w2;
1537 }
1538
1539 int wl = strlen(word);
1540
1541
1542 if (*slst) {
1543 wlst = *slst;
1544 } else {
1545 wlst = (char **) calloc(maxSug, sizeof(char *));
1546 if (wlst == NULL) return -1;
1547 }
1548
1549 rv = pAMgr->suffix_check(word, wl, 0, NULL, wlst, maxSug, &nsug);
1550
1551 // delete dash from end of word
1552 if (nsug > 0) {
1553 for (int j=0; j < nsug; j++) {
1554 if (wlst[j][strlen(wlst[j]) - 1] == '-') wlst[j][strlen(wlst[j]) - 1] = '\0';
1555 }
1556 }
1557
1558 *slst = wlst;
1559 return nsug;
1560 }
1561 #endif // END OF HUNSPELL_EXPERIMENTAL CODE
1562
1563
1564 char * SuggestMgr::suggest_morph(const char * w)
1565 {
1566 char result[MAXLNLEN];
1567 char * r = (char *) result;
1568 char * st;
1569
1570 struct hentry * rv = NULL;
1571
1572 *result = '\0';
1573
1574 if (! pAMgr) return NULL;
1575
1576 char w2[MAXSWUTF8L];
1577 const char * word = w;
1578
1579 // word reversing wrapper for complex prefixes
1580 if (complexprefixes) {
1581 strcpy(w2, w);
1582 if (utf8) reverseword_utf(w2); else reverseword(w2);
1583 word = w2;
1584 }
1585
1586 rv = pAMgr->lookup(word);
1587
1588 while (rv) {
1589 if ((!rv->astr) || !(TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen) ||
1590 TESTAFF(rv->astr, pAMgr->get_needaffix(), rv->alen) ||
1591 TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) {
1592 if (!HENTRY_FIND(rv, MORPH_STEM)) {
1593 mystrcat(result, " ", MAXLNLEN);
1594 mystrcat(result, MORPH_STEM, MAXLNLEN);
1595 mystrcat(result, word, MAXLNLEN);
1596 }
1597 if (HENTRY_DATA(rv)) {
1598 mystrcat(result, " ", MAXLNLEN);
1599 mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
1600 }
1601 mystrcat(result, "\n", MAXLNLEN);
1602 }
1603 rv = rv->next_homonym;
1604 }
1605
1606 st = pAMgr->affix_check_morph(word,strlen(word));
1607 if (st) {
1608 mystrcat(result, st, MAXLNLEN);
1609 free(st);
1610 }
1611
1612 if (pAMgr->get_compound() && (*result == '\0'))
1613 pAMgr->compound_check_morph(word, strlen(word),
1614 0, 0, 100, 0,NULL, 0, &r, NULL);
1615
1616 return (*result) ? mystrdup(line_uniq(result, MSEP_REC)) : NULL;
1617 }
1618
1619 #ifdef HUNSPELL_EXPERIMENTAL
1620 char * SuggestMgr::suggest_morph_for_spelling_error(const char * word)
1621 {
1622 char * p = NULL;
1623 char ** wlst = (char **) calloc(maxSug, sizeof(char *));
1624 if (!**wlst) return NULL;
1625 // we will use only the first suggestion
1626 for (int i = 0; i < maxSug - 1; i++) wlst[i] = "";
1627 int ns = suggest(&wlst, word, maxSug - 1, NULL);
1628 if (ns == maxSug) {
1629 p = suggest_morph(wlst[maxSug - 1]);
1630 free(wlst[maxSug - 1]);
1631 }
1632 if (wlst) free(wlst);
1633 return p;
1634 }
1635 #endif // END OF HUNSPELL_EXPERIMENTAL CODE
1636
1637 /* affixation */
1638 char * SuggestMgr::suggest_hentry_gen(hentry * rv, char * pattern)
1639 {
1640 char result[MAXLNLEN];
1641 *result = '\0';
1642 int sfxcount = get_sfxcount(pattern);
1643
1644 if (get_sfxcount(HENTRY_DATA(rv)) > sfxcount) return NULL;
1645
1646 if (HENTRY_DATA(rv)) {
1647 char * aff = pAMgr->morphgen(HENTRY_WORD(rv), rv->blen, rv->astr, rv->alen,
1648 HENTRY_DATA(rv), pattern, 0);
1649 if (aff) {
1650 mystrcat(result, aff, MAXLNLEN);
1651 mystrcat(result, "\n", MAXLNLEN);
1652 free(aff);
1653 }
1654 }
1655
1656 // check all allomorphs
1657 char allomorph[MAXLNLEN];
1658 char * p = NULL;
1659 if (HENTRY_DATA(rv)) p = (char *) strstr(HENTRY_DATA2(rv), MORPH_ALLOMORPH);
1660 while (p) {
1661 struct hentry * rv2 = NULL;
1662 p += MORPH_TAG_LEN;
1663 int plen = fieldlen(p);
1664 strncpy(allomorph, p, plen);
1665 allomorph[plen] = '\0';
1666 rv2 = pAMgr->lookup(allomorph);
1667 while (rv2) {
1668 // if (HENTRY_DATA(rv2) && get_sfxcount(HENTRY_DATA(rv2)) <= sfxcount) {
1669 if (HENTRY_DATA(rv2)) {
1670 char * st = (char *) strstr(HENTRY_DATA2(rv2), MORPH_STEM);
1671 if (st && (strncmp(st + MORPH_TAG_LEN,
1672 HENTRY_WORD(rv), fieldlen(st + MORPH_TAG_LEN)) == 0)) {
1673 char * aff = pAMgr->morphgen(HENTRY_WORD(rv2), rv2->blen, rv2->astr, rv2->alen,
1674 HENTRY_DATA(rv2), pattern, 0);
1675 if (aff) {
1676 mystrcat(result, aff, MAXLNLEN);
1677 mystrcat(result, "\n", MAXLNLEN);
1678 free(aff);
1679 }
1680 }
1681 }
1682 rv2 = rv2->next_homonym;
1683 }
1684 p = strstr(p + plen, MORPH_ALLOMORPH);
1685 }
1686
1687 return (*result) ? mystrdup(result) : NULL;
1688 }
1689
1690 char * SuggestMgr::suggest_gen(char ** desc, int n, char * pattern) {
1691 char result[MAXLNLEN];
1692 char result2[MAXLNLEN];
1693 char newpattern[MAXLNLEN];
1694 *newpattern = '\0';
1695 if (n == 0) return 0;
1696 *result2 = '\0';
1697 struct hentry * rv = NULL;
1698 if (!pAMgr) return NULL;
1699
1700 // search affixed forms with and without derivational suffixes
1701 while(1) {
1702
1703 for (int k = 0; k < n; k++) {
1704 *result = '\0';
1705 // add compound word parts (except the last one)
1706 char * s = (char *) desc[k];
1707 char * part = strstr(s, MORPH_PART);
1708 if (part) {
1709 char * nextpart = strstr(part + 1, MORPH_PART);
1710 while (nextpart) {
1711 copy_field(result + strlen(result), part, MORPH_PART);
1712 part = nextpart;
1713 nextpart = strstr(part + 1, MORPH_PART);
1714 }
1715 s = part;
1716 }
1717
1718 char **pl;
1719 char tok[MAXLNLEN];
1720 strcpy(tok, s);
1721 char * alt = strstr(tok, " | ");
1722 while (alt) {
1723 alt[1] = MSEP_ALT;
1724 alt = strstr(alt, " | ");
1725 }
1726 int pln = line_tok(tok, &pl, MSEP_ALT);
1727 for (int i = 0; i < pln; i++) {
1728 // remove inflectional and terminal suffixes
1729 char * is = strstr(pl[i], MORPH_INFL_SFX);
1730 if (is) *is = '\0';
1731 char * ts = strstr(pl[i], MORPH_TERM_SFX);
1732 while (ts) {
1733 *ts = '_';
1734 ts = strstr(pl[i], MORPH_TERM_SFX);
1735 }
1736 char * st = strstr(s, MORPH_STEM);
1737 if (st) {
1738 copy_field(tok, st, MORPH_STEM);
1739 rv = pAMgr->lookup(tok);
1740 while (rv) {
1741 char newpat[MAXLNLEN];
1742 strcpy(newpat, pl[i]);
1743 strcat(newpat, pattern);
1744 char * sg = suggest_hentry_gen(rv, newpat);
1745 if (!sg) sg = suggest_hentry_gen(rv, pattern);
1746 if (sg) {
1747 char ** gen;
1748 int genl = line_tok(sg, &gen, MSEP_REC);
1749 free(sg);
1750 sg = NULL;
1751 for (int j = 0; j < genl; j++) {
1752 if (strstr(pl[i], MORPH_SURF_PFX)) {
1753 int r2l = strlen(result2);
1754 result2[r2l] = MSEP_REC;
1755 strcpy(result2 + r2l + 1, result);
1756 copy_field(result2 + strlen(result2), pl[i], MORPH_SURF_PFX);
1757 mystrcat(result2, gen[j], MAXLNLEN);
1758 } else {
1759 sprintf(result2 + strlen(result2), "%c%s%s",
1760 MSEP_REC, result, gen[j]);
1761 }
1762 }
1763 freelist(&gen, genl);
1764 }
1765 rv = rv->next_homonym;
1766 }
1767 }
1768 }
1769 freelist(&pl, pln);
1770 }
1771
1772 if (*result2 || !strstr(pattern, MORPH_DERI_SFX)) break;
1773 strcpy(newpattern, pattern);
1774 pattern = newpattern;
1775 char * ds = strstr(pattern, MORPH_DERI_SFX);
1776 while (ds) {
1777 strncpy(ds, MORPH_TERM_SFX, MORPH_TAG_LEN);
1778 ds = strstr(pattern, MORPH_DERI_SFX);
1779 }
1780 }
1781 return (*result2 ? mystrdup(result2) : NULL);
1782 }
1783
1784
1785 // generate an n-gram score comparing s1 and s2
1786 int SuggestMgr::ngram(int n, char * s1, const char * s2, int opt)
1787 {
1788 int nscore = 0;
1789 int ns;
1790 int l1;
1791 int l2;
1792 int test = 0;
1793
1794 if (utf8) {
1795 w_char su1[MAXSWL];
1796 w_char su2[MAXSWL];
1797 l1 = u8_u16(su1, MAXSWL, s1);
1798 l2 = u8_u16(su2, MAXSWL, s2);
1799 if ((l2 <= 0) || (l1 == -1)) return 0;
1800 // lowering dictionary word
1801 if (opt & NGRAM_LOWERING) mkallsmall_utf(su2, l2, langnum);
1802 for (int j = 1; j <= n; j++) {
1803 ns = 0;
1804 for (int i = 0; i <= (l1-j); i++) {
1805 int k = 0;
1806 for (int l = 0; l <= (l2-j); l++) {
1807 for (k = 0; k < j; k++) {
1808 w_char * c1 = su1 + i + k;
1809 w_char * c2 = su2 + l + k;
1810 if ((c1->l != c2->l) || (c1->h != c2->h)) break;
1811 }
1812 if (k == j) {
1813 ns++;
1814 break;
1815 }
1816 }
1817 if (k != j && opt & NGRAM_WEIGHTED) {
1818 ns--;
1819 test++;
1820 if (i == 0 || i == l1-j) ns--; // side weight
1821 }
1822 }
1823 nscore = nscore + ns;
1824 if (ns < 2 && !(opt & NGRAM_WEIGHTED)) break;
1825 }
1826 } else {
1827 l2 = strlen(s2);
1828 if (l2 == 0) return 0;
1829 l1 = strlen(s1);
1830 char *t = mystrdup(s2);
1831 if (opt & NGRAM_LOWERING) mkallsmall(t, csconv);
1832 for (int j = 1; j <= n; j++) {
1833 ns = 0;
1834 for (int i = 0; i <= (l1-j); i++) {
1835 char c = *(s1 + i + j);
1836 *(s1 + i + j) = '\0';
1837 if (strstr(t,(s1+i))) {
1838 ns++;
1839 } else if (opt & NGRAM_WEIGHTED) {
1840 ns--;
1841 test++;
1842 if (i == 0 || i == l1-j) ns--; // side weight
1843 }
1844 *(s1 + i + j ) = c;
1845 }
1846 nscore = nscore + ns;
1847 if (ns < 2 && !(opt & NGRAM_WEIGHTED)) break;
1848 }
1849 free(t);
1850 }
1851
1852 ns = 0;
1853 if (opt & NGRAM_LONGER_WORSE) ns = (l2-l1)-2;
1854 if (opt & NGRAM_ANY_MISMATCH) ns = abs(l2-l1)-2;
1855 ns = (nscore - ((ns > 0) ? ns : 0));
1856 return ns;
1857 }
1858
1859 // length of the left common substring of s1 and (decapitalised) s2
1860 int SuggestMgr::leftcommonsubstring(char * s1, const char * s2) {
1861 if (utf8) {
1862 w_char su1[MAXSWL];
1863 w_char su2[MAXSWL];
1864 su1[0].l = su2[0].l = su1[0].h = su2[0].h = 0;
1865 // decapitalize dictionary word
1866 if (complexprefixes) {
1867 int l1 = u8_u16(su1, MAXSWL, s1);
1868 int l2 = u8_u16(su2, MAXSWL, s2);
1869 if (*((short *)su1+l1-1) == *((short *)su2+l2-1)) return 1;
1870 } else {
1871 int i;
1872 u8_u16(su1, 1, s1);
1873 u8_u16(su2, 1, s2);
1874 unsigned short idx = (su2->h << 8) + su2->l;
1875 unsigned short otheridx = (su1->h << 8) + su1->l;
1876 if (otheridx != idx &&
1877 (otheridx != unicodetolower(idx, langnum))) return 0;
1878 int l1 = u8_u16(su1, MAXSWL, s1);
1879 int l2 = u8_u16(su2, MAXSWL, s2);
1880 for(i = 1; (i < l1) && (i < l2) &&
1881 (su1[i].l == su2[i].l) && (su1[i].h == su2[i].h); i++);
1882 return i;
1883 }
1884 } else {
1885 if (complexprefixes) {
1886 int l1 = strlen(s1);
1887 int l2 = strlen(s2);
1888 if (*(s2+l1-1) == *(s2+l2-1)) return 1;
1889 } else {
1890 char * olds = s1;
1891 // decapitalise dictionary word
1892 if ((*s1 != *s2) && (*s1 != csconv[((unsigned char)*s2)].clower)) return 0;
1893 do {
1894 s1++; s2++;
1895 } while ((*s1 == *s2) && (*s1 != '\0'));
1896 return (int)(s1 - olds);
1897 }
1898 }
1899 return 0;
1900 }
1901
1902 int SuggestMgr::commoncharacterpositions(char * s1, const char * s2, int * is_swap) {
1903 int num = 0;
1904 int diff = 0;
1905 int diffpos[2];
1906 *is_swap = 0;
1907 if (utf8) {
1908 w_char su1[MAXSWL];
1909 w_char su2[MAXSWL];
1910 int l1 = u8_u16(su1, MAXSWL, s1);
1911 int l2 = u8_u16(su2, MAXSWL, s2);
1912 // decapitalize dictionary word
1913 if (complexprefixes) {
1914 mkallsmall_utf(su2+l2-1, 1, langnum);
1915 } else {
1916 mkallsmall_utf(su2, 1, langnum);
1917 }
1918 for (int i = 0; (i < l1) && (i < l2); i++) {
1919 if (((short *) su1)[i] == ((short *) su2)[i]) {
1920 num++;
1921 } else {
1922 if (diff < 2) diffpos[diff] = i;
1923 diff++;
1924 }
1925 }
1926 if ((diff == 2) && (l1 == l2) &&
1927 (((short *) su1)[diffpos[0]] == ((short *) su2)[diffpos[1]]) &&
1928 (((short *) su1)[diffpos[1]] == ((short *) su2)[diffpos[0]])) *is_swap = 1;
1929 } else {
1930 int i;
1931 char t[MAXSWUTF8L];
1932 strcpy(t, s2);
1933 // decapitalize dictionary word
1934 if (complexprefixes) {
1935 int l2 = strlen(t);
1936 *(t+l2-1) = csconv[((unsigned char)*(t+l2-1))].clower;
1937 } else {
1938 mkallsmall(t, csconv);
1939 }
1940 for (i = 0; (*(s1+i) != 0) && (*(t+i) != 0); i++) {
1941 if (*(s1+i) == *(t+i)) {
1942 num++;
1943 } else {
1944 if (diff < 2) diffpos[diff] = i;
1945 diff++;
1946 }
1947 }
1948 if ((diff == 2) && (*(s1+i) == 0) && (*(t+i) == 0) &&
1949 (*(s1+diffpos[0]) == *(t+diffpos[1])) &&
1950 (*(s1+diffpos[1]) == *(t+diffpos[0]))) *is_swap = 1;
1951 }
1952 return num;
1953 }
1954
1955 int SuggestMgr::mystrlen(const char * word) {
1956 if (utf8) {
1957 w_char w[MAXSWL];
1958 return u8_u16(w, MAXSWL, word);
1959 } else return strlen(word);
1960 }
1961
1962 // sort in decreasing order of score
1963 void SuggestMgr::bubblesort(char** rword, char** rword2, int* rsc, int n )
1964 {
1965 int m = 1;
1966 while (m < n) {
1967 int j = m;
1968 while (j > 0) {
1969 if (rsc[j-1] < rsc[j]) {
1970 int sctmp = rsc[j-1];
1971 char * wdtmp = rword[j-1];
1972 rsc[j-1] = rsc[j];
1973 rword[j-1] = rword[j];
1974 rsc[j] = sctmp;
1975 rword[j] = wdtmp;
1976 if (rword2) {
1977 wdtmp = rword2[j-1];
1978 rword2[j-1] = rword2[j];
1979 rword2[j] = wdtmp;
1980 }
1981 j--;
1982 } else break;
1983 }
1984 m++;
1985 }
1986 return;
1987 }
1988
1989 // longest common subsequence
1990 void SuggestMgr::lcs(const char * s, const char * s2, int * l1, int * l2, char ** result) {
1991 int n, m;
1992 w_char su[MAXSWL];
1993 w_char su2[MAXSWL];
1994 char * b;
1995 char * c;
1996 int i;
1997 int j;
1998 if (utf8) {
1999 m = u8_u16(su, MAXSWL, s);
2000 n = u8_u16(su2, MAXSWL, s2);
2001 } else {
2002 m = strlen(s);
2003 n = strlen(s2);
2004 }
2005 c = (char *) malloc((m + 1) * (n + 1));
2006 b = (char *) malloc((m + 1) * (n + 1));
2007 if (!c || !b) {
2008 if (c) free(c);
2009 if (b) free(b);
2010 *result = NULL;
2011 return;
2012 }
2013 for (i = 1; i <= m; i++) c[i*(n+1)] = 0;
2014 for (j = 0; j <= n; j++) c[j] = 0;
2015 for (i = 1; i <= m; i++) {
2016 for (j = 1; j <= n; j++) {
2017 if ( ((utf8) && (*((short *) su+i-1) == *((short *)su2+j-1)))
2018 || ((!utf8) && ((*(s+i-1)) == (*(s2+j-1))))) {
2019 c[i*(n+1) + j] = c[(i-1)*(n+1) + j-1]+1;
2020 b[i*(n+1) + j] = LCS_UPLEFT;
2021 } else if (c[(i-1)*(n+1) + j] >= c[i*(n+1) + j-1]) {
2022 c[i*(n+1) + j] = c[(i-1)*(n+1) + j];
2023 b[i*(n+1) + j] = LCS_UP;
2024 } else {
2025 c[i*(n+1) + j] = c[i*(n+1) + j-1];
2026 b[i*(n+1) + j] = LCS_LEFT;
2027 }
2028 }
2029 }
2030 *result = b;
2031 free(c);
2032 *l1 = m;
2033 *l2 = n;
2034 }
2035
2036 int SuggestMgr::lcslen(const char * s, const char* s2) {
2037 int m;
2038 int n;
2039 int i;
2040 int j;
2041 char * result;
2042 int len = 0;
2043 lcs(s, s2, &m, &n, &result);
2044 if (!result) return 0;
2045 i = m;
2046 j = n;
2047 while ((i != 0) && (j != 0)) {
2048 if (result[i*(n+1) + j] == LCS_UPLEFT) {
2049 len++;
2050 i--;
2051 j--;
2052 } else if (result[i*(n+1) + j] == LCS_UP) {
2053 i--;
2054 } else j--;
2055 }
2056 free(result);
2057 return len;
2058 }

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial