######################## BEGIN LICENSE BLOCK ######################## # The Original Code is Mozilla Universal charset detector code. # # The Initial Developer of the Original Code is # Netscape Communications Corporation. # Portions created by the Initial Developer are Copyright (C) 2001 # the Initial Developer. All Rights Reserved. # # Contributor(s): # Mark Pilgrim - port to Python # Shy Shalom - original C code # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### from .charsetprober import CharSetProber from .constants import eNotMe from .compat import wrap_ord FREQ_CAT_NUM = 4 UDF = 0 # undefined OTH = 1 # other ASC = 2 # ascii capital letter ASS = 3 # ascii small letter ACV = 4 # accent capital vowel ACO = 5 # accent capital other ASV = 6 # accent small vowel ASO = 7 # accent small other CLASS_NUM = 8 # total classes Latin1_CharToClass = ( OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 10 - 17 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 18 - 1F OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 20 - 27 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 28 - 2F OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 30 - 37 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 38 - 3F OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 40 - 47 ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 48 - 4F ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 50 - 57 ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, # 58 - 5F OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 60 - 67 ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 68 - 6F ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 70 - 77 ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, # 78 - 7F OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, # 80 - 87 OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, # 88 - 8F UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 90 - 97 OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, # 98 - 9F OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A0 - A7 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A8 - AF OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B0 - B7 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B8 - BF ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, # C0 - C7 ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, # C8 - CF ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, # D0 - D7 ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, # D8 - DF ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, # E0 - E7 ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, # E8 - EF ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, # F0 - F7 ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, # F8 - FF ) # 0 : illegal # 1 : very unlikely # 2 : normal # 3 : very likely Latin1ClassModel = ( # UDF OTH ASC ASS ACV ACO ASV ASO 0, 0, 0, 0, 0, 0, 0, 0, # UDF 0, 3, 3, 3, 3, 3, 3, 3, # OTH 0, 3, 3, 3, 3, 3, 3, 3, # ASC 0, 3, 3, 3, 1, 1, 3, 3, # ASS 0, 3, 3, 3, 1, 2, 1, 2, # ACV 0, 3, 3, 3, 3, 3, 3, 3, # ACO 0, 3, 1, 3, 1, 1, 1, 3, # ASV 0, 3, 1, 3, 1, 1, 3, 3, # ASO ) class Latin1Prober(CharSetProber): def __init__(self): CharSetProber.__init__(self) self.reset() def reset(self): self._mLastCharClass = OTH self._mFreqCounter = [0] * FREQ_CAT_NUM CharSetProber.reset(self) def get_charset_name(self): return "windows-1252" def feed(self, aBuf): aBuf = self.filter_with_english_letters(aBuf) for c in aBuf: charClass = Latin1_CharToClass[wrap_ord(c)] freq = Latin1ClassModel[(self._mLastCharClass * CLASS_NUM) + charClass] if freq == 0: self._mState = eNotMe break self._mFreqCounter[freq] += 1 self._mLastCharClass = charClass return self.get_state() def get_confidence(self): if self.get_state() == eNotMe: return 0.01 total = sum(self._mFreqCounter) if total < 0.01: confidence = 0.0 else: confidence = ((self._mFreqCounter[3] / total) - (self._mFreqCounter[1] * 20.0 / total)) if confidence < 0.0: confidence = 0.0 # lower the confidence of latin1 so that other more accurate # detector can take priority. confidence = confidence * 0.5 return confidence
Name | Type | Size | Permission | Actions |
---|---|---|---|---|
.__init__.pyo.40009 | File | 797 B | 0644 |
|
.big5freq.pyo.40009 | File | 111.11 KB | 0644 |
|
.big5prober.pyo.40009 | File | 1.07 KB | 0644 |
|
.chardetect.pyo.40009 | File | 1.46 KB | 0644 |
|
.chardistribution.pyo.40009 | File | 7.35 KB | 0644 |
|
.charsetgroupprober.pyo.40009 | File | 2.48 KB | 0644 |
|
.charsetprober.pyo.40009 | File | 2.08 KB | 0644 |
|
.codingstatemachine.pyo.40009 | File | 1.74 KB | 0644 |
|
.compat.pyo.40009 | File | 559 B | 0644 |
|
.constants.pyo.40009 | File | 332 B | 0644 |
|
.cp949prober.pyo.40009 | File | 1.07 KB | 0644 |
|
.escprober.pyo.40009 | File | 2.31 KB | 0644 |
|
.escsm.pyo.40009 | File | 7.13 KB | 0644 |
|
.eucjpprober.pyo.40009 | File | 2.71 KB | 0644 |
|
.euckrfreq.pyo.40009 | File | 86.69 KB | 0644 |
|
.euckrprober.pyo.40009 | File | 1.08 KB | 0644 |
|
.euctwfreq.pyo.40009 | File | 59.74 KB | 0644 |
|
.euctwprober.pyo.40009 | File | 1.08 KB | 0644 |
|
.gb2312freq.pyo.40009 | File | 67.17 KB | 0644 |
|
.gb2312prober.pyo.40009 | File | 1.08 KB | 0644 |
|
.hebrewprober.pyo.40009 | File | 3.32 KB | 0644 |
|
.jisfreq.pyo.40009 | File | 82.03 KB | 0644 |
|
.jpcntx.pyo.40009 | File | 24.04 KB | 0644 |
|
.langbulgarianmodel.pyo.40009 | File | 24.3 KB | 0644 |
|
.langcyrillicmodel.pyo.40009 | File | 29.85 KB | 0644 |
|
.langgreekmodel.pyo.40009 | File | 24 KB | 0644 |
|
.langhebrewmodel.pyo.40009 | File | 22.84 KB | 0644 |
|
.langhungarianmodel.pyo.40009 | File | 24.29 KB | 0644 |
|
.langthaimodel.pyo.40009 | File | 22.83 KB | 0644 |
|
.latin1prober.pyo.40009 | File | 3.37 KB | 0644 |
|
.mbcharsetprober.pyo.40009 | File | 2.36 KB | 0644 |
|
.mbcsgroupprober.pyo.40009 | File | 1.22 KB | 0644 |
|
.mbcssm.pyo.40009 | File | 17.25 KB | 0644 |
|
.sbcharsetprober.pyo.40009 | File | 3.22 KB | 0644 |
|
.sbcsgroupprober.pyo.40009 | File | 1.89 KB | 0644 |
|
.sjisprober.pyo.40009 | File | 2.73 KB | 0644 |
|
.universaldetector.pyo.40009 | File | 4.02 KB | 0644 |
|
.utf8prober.pyo.40009 | File | 2.14 KB | 0644 |
|
__init__.py | File | 1.26 KB | 0644 |
|
__init__.pyc | File | 797 B | 0644 |
|
__init__.pyo | File | 797 B | 0644 |
|
big5freq.py | File | 80.66 KB | 0644 |
|
big5freq.pyc | File | 111.11 KB | 0644 |
|
big5freq.pyo | File | 111.11 KB | 0644 |
|
big5prober.py | File | 1.64 KB | 0644 |
|
big5prober.pyc | File | 1.07 KB | 0644 |
|
big5prober.pyo | File | 1.07 KB | 0644 |
|
chardetect.py | File | 1.09 KB | 0644 |
|
chardetect.pyc | File | 1.46 KB | 0644 |
|
chardetect.pyo | File | 1.46 KB | 0644 |
|
chardistribution.py | File | 9.01 KB | 0644 |
|
chardistribution.pyc | File | 7.35 KB | 0644 |
|
chardistribution.pyo | File | 7.35 KB | 0644 |
|
charsetgroupprober.py | File | 3.7 KB | 0644 |
|
charsetgroupprober.pyc | File | 2.48 KB | 0644 |
|
charsetgroupprober.pyo | File | 2.48 KB | 0644 |
|
charsetprober.py | File | 1.86 KB | 0644 |
|
charsetprober.pyc | File | 2.08 KB | 0644 |
|
charsetprober.pyo | File | 2.08 KB | 0644 |
|
codingstatemachine.py | File | 2.26 KB | 0644 |
|
codingstatemachine.pyc | File | 1.74 KB | 0644 |
|
codingstatemachine.pyo | File | 1.74 KB | 0644 |
|
compat.py | File | 1.13 KB | 0644 |
|
compat.pyc | File | 559 B | 0644 |
|
compat.pyo | File | 559 B | 0644 |
|
constants.py | File | 1.3 KB | 0644 |
|
constants.pyc | File | 332 B | 0644 |
|
constants.pyo | File | 332 B | 0644 |
|
cp949prober.py | File | 1.74 KB | 0644 |
|
cp949prober.pyc | File | 1.07 KB | 0644 |
|
cp949prober.pyo | File | 1.07 KB | 0644 |
|
escprober.py | File | 3.11 KB | 0644 |
|
escprober.pyc | File | 2.31 KB | 0644 |
|
escprober.pyo | File | 2.31 KB | 0644 |
|
escsm.py | File | 7.66 KB | 0644 |
|
escsm.pyc | File | 7.13 KB | 0644 |
|
escsm.pyo | File | 7.13 KB | 0644 |
|
eucjpprober.py | File | 3.59 KB | 0644 |
|
eucjpprober.pyc | File | 2.71 KB | 0644 |
|
eucjpprober.pyo | File | 2.71 KB | 0644 |
|
euckrfreq.py | File | 44.9 KB | 0644 |
|
euckrfreq.pyc | File | 86.69 KB | 0644 |
|
euckrfreq.pyo | File | 86.69 KB | 0644 |
|
euckrprober.py | File | 1.64 KB | 0644 |
|
euckrprober.pyc | File | 1.08 KB | 0644 |
|
euckrprober.pyo | File | 1.08 KB | 0644 |
|
euctwfreq.py | File | 34.05 KB | 0644 |
|
euctwfreq.pyc | File | 59.74 KB | 0644 |
|
euctwfreq.pyo | File | 59.74 KB | 0644 |
|
euctwprober.py | File | 1.64 KB | 0644 |
|
euctwprober.pyc | File | 1.08 KB | 0644 |
|
euctwprober.pyo | File | 1.08 KB | 0644 |
|
gb2312freq.py | File | 35.17 KB | 0644 |
|
gb2312freq.pyc | File | 67.17 KB | 0644 |
|
gb2312freq.pyo | File | 67.17 KB | 0644 |
|
gb2312prober.py | File | 1.64 KB | 0644 |
|
gb2312prober.pyc | File | 1.08 KB | 0644 |
|
gb2312prober.pyo | File | 1.08 KB | 0644 |
|
hebrewprober.py | File | 13.05 KB | 0644 |
|
hebrewprober.pyc | File | 3.32 KB | 0644 |
|
hebrewprober.pyo | File | 3.32 KB | 0644 |
|
jisfreq.py | File | 46.21 KB | 0644 |
|
jisfreq.pyc | File | 82.03 KB | 0644 |
|
jisfreq.pyo | File | 82.03 KB | 0644 |
|
jpcntx.py | File | 18.66 KB | 0644 |
|
jpcntx.pyc | File | 24.04 KB | 0644 |
|
jpcntx.pyo | File | 24.04 KB | 0644 |
|
langbulgarianmodel.py | File | 12.48 KB | 0644 |
|
langbulgarianmodel.pyc | File | 24.3 KB | 0644 |
|
langbulgarianmodel.pyo | File | 24.3 KB | 0644 |
|
langcyrillicmodel.py | File | 17.31 KB | 0644 |
|
langcyrillicmodel.pyc | File | 29.85 KB | 0644 |
|
langcyrillicmodel.pyo | File | 29.85 KB | 0644 |
|
langgreekmodel.py | File | 12.33 KB | 0644 |
|
langgreekmodel.pyc | File | 24 KB | 0644 |
|
langgreekmodel.pyo | File | 24 KB | 0644 |
|
langhebrewmodel.py | File | 11.05 KB | 0644 |
|
langhebrewmodel.pyc | File | 22.84 KB | 0644 |
|
langhebrewmodel.pyo | File | 22.84 KB | 0644 |
|
langhungarianmodel.py | File | 12.24 KB | 0644 |
|
langhungarianmodel.pyc | File | 24.29 KB | 0644 |
|
langhungarianmodel.pyo | File | 24.29 KB | 0644 |
|
langthaimodel.py | File | 11.01 KB | 0644 |
|
langthaimodel.pyc | File | 22.83 KB | 0644 |
|
langthaimodel.pyo | File | 22.83 KB | 0644 |
|
latin1prober.py | File | 5.12 KB | 0644 |
|
latin1prober.pyc | File | 3.37 KB | 0644 |
|
latin1prober.pyo | File | 3.37 KB | 0644 |
|
mbcharsetprober.py | File | 3.19 KB | 0644 |
|
mbcharsetprober.pyc | File | 2.36 KB | 0644 |
|
mbcharsetprober.pyo | File | 2.36 KB | 0644 |
|
mbcsgroupprober.py | File | 1.92 KB | 0644 |
|
mbcsgroupprober.pyc | File | 1.22 KB | 0644 |
|
mbcsgroupprober.pyo | File | 1.22 KB | 0644 |
|
mbcssm.py | File | 19.15 KB | 0644 |
|
mbcssm.pyc | File | 17.25 KB | 0644 |
|
mbcssm.pyo | File | 17.25 KB | 0644 |
|
sbcharsetprober.py | File | 4.68 KB | 0644 |
|
sbcharsetprober.pyc | File | 3.22 KB | 0644 |
|
sbcharsetprober.pyo | File | 3.22 KB | 0644 |
|
sbcsgroupprober.py | File | 3.21 KB | 0644 |
|
sbcsgroupprober.pyc | File | 1.89 KB | 0644 |
|
sbcsgroupprober.pyo | File | 1.89 KB | 0644 |
|
sjisprober.py | File | 3.65 KB | 0644 |
|
sjisprober.pyc | File | 2.73 KB | 0644 |
|
sjisprober.pyo | File | 2.73 KB | 0644 |
|
universaldetector.py | File | 6.67 KB | 0644 |
|
universaldetector.pyc | File | 4.02 KB | 0644 |
|
universaldetector.pyo | File | 4.02 KB | 0644 |
|
utf8prober.py | File | 2.59 KB | 0644 |
|
utf8prober.pyc | File | 2.14 KB | 0644 |
|
utf8prober.pyo | File | 2.14 KB | 0644 |
|