The Tor Browser / file comparison
comparison: intl/chardet/tools/charfreqtostat.pl
intl/chardet/tools/charfreqtostat.pl
- branch
- TOR_BUG_9701
- changeset 8
- 97036ab72558
equal
deleted
inserted
replaced
| -1:000000000000 | 0:19efdd65c053 |
|---|---|
| 1 #!/usr/bin/perl | |
| 2 # | |
| 3 # This Source Code Form is subject to the terms of the Mozilla Public | |
| 4 # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
| 6 sub GenNPL { | |
| 7 my($ret) = << "END_NPL"; | |
| 8 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ | |
| 9 /* This Source Code Form is subject to the terms of the Mozilla Public | |
| 10 * License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 11 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ | |
| 12 END_NPL | |
| 13 | |
| 14 return $ret; | |
| 15 } | |
| 16 | |
| 17 print GenNPL(); | |
| 18 $total=0; | |
| 19 @h; | |
| 20 @l; | |
| 21 | |
| 22 while(<STDIN>) | |
| 23 { | |
| 24 @k = split(/\s+/, $_); | |
| 25 @i = unpack("CCCC", $k[0]); | |
| 26 # printf("%x %x %s",$i[0] , $i[1] , "[" . $k[0] . "] " . $i . " " . $j . " " . $k[1] ."\n"); | |
| 27 if((0xA1 <= $i[0]) && (0xA1 <= $i[1])){ | |
| 28 $total += $k[1]; | |
| 29 $v = $i[0] - 0x00A1; | |
| 30 $h[$v] += $k[1]; | |
| 31 $u = $i[1] - 0x00A1; | |
| 32 $l[$u] += $k[1]; | |
| 33 # print "hello $v $h[$v] $u $l[$u]\n"; | |
| 34 } | |
| 35 } | |
| 36 | |
| 37 | |
| 38 $ffh = 0.0; | |
| 39 $ffl = 0.0; | |
| 40 for($i=0x00A1;$i< 0x00FF ; $i++) | |
| 41 { | |
| 42 $fh[$i - 0x00a1] = $h[$i- 0x00a1] / $total; | |
| 43 $ffh += $fh[$i - 0x00a1]; | |
| 44 | |
| 45 $fl[$i - 0x00a1] = $l[$i- 0x00a1] / $total; | |
| 46 $ffl += $fl[$i - 0x00a1]; | |
| 47 } | |
| 48 $mh = $ffh / 94.0; | |
| 49 $ml = $ffl / 94.0; | |
| 50 | |
| 51 $sumh=0.0; | |
| 52 $suml=0.0; | |
| 53 for($i=0x00A1;$i< 0x00FF ; $i++) | |
| 54 { | |
| 55 $sh = $fh[$i - 0x00a1] - $mh; | |
| 56 $sh *= $sh; | |
| 57 $sumh += $sh; | |
| 58 | |
| 59 $sl = $fl[$i - 0x00a1] - $ml; | |
| 60 $sl *= $sl; | |
| 61 $suml += $sl; | |
| 62 } | |
| 63 $sumh /= 94.0; | |
| 64 $suml /= 94.0; | |
| 65 $stdh = sqrt($sumh); | |
| 66 $stdl = sqrt($suml); | |
| 67 | |
| 68 print "{\n"; | |
| 69 print " {\n"; | |
| 70 for($i=0x00A1;$i< 0x00FF ; $i++) | |
| 71 { | |
| 72 if($i eq 0xfe) { | |
| 73 printf(" %.6ff \/\/ FreqH[%2x]\n", $fh[$i - 0x00a1] , $i); | |
| 74 } else { | |
| 75 printf(" %.6ff, \/\/ FreqH[%2x]\n", $fh[$i - 0x00a1] , $i); | |
| 76 } | |
| 77 } | |
| 78 print " },\n"; | |
| 79 printf ("%.6ff, \/\/ Lead Byte StdDev\n", $stdh); | |
| 80 printf ("%.6ff, \/\/ Lead Byte Mean\n", $mh); | |
| 81 printf ("%.6ff, \/\/ Lead Byte Weight\n", $stdh / ($stdh + $stdl)); | |
| 82 print " {\n"; | |
| 83 for($i=0x00A1;$i< 0x00FF ; $i++) | |
| 84 { | |
| 85 if($i eq 0xfe) { | |
| 86 printf(" %.6ff \/\/ FreqL[%2x]\n", $fl[$i - 0x00a1] , $i); | |
| 87 } else { | |
| 88 printf(" %.6ff, \/\/ FreqL[%2x]\n", $fl[$i - 0x00a1] , $i); | |
| 89 } | |
| 90 } | |
| 91 print " },\n"; | |
| 92 printf ("%.6ff, \/\/ Trail Byte StdDev\n", $stdl); | |
| 93 printf ("%.6ff, \/\/ Trail Byte Mean\n", $ml); | |
| 94 printf ("%.6ff \/\/ Trial Byte Weight\n", $stdl / ($stdh + $stdl)); | |
| 95 print "};\n"; |
