import codecs import locale import re import sys from typing import List, Tuple BOMS: List[Tuple[bytes, str]] = [ (codecs.BOM_UTF8, "utf-8"), (codecs.BOM_UTF16, "utf-16"), (codecs.BOM_UTF16_BE, "utf-16-be"), (codecs.BOM_UTF16_LE, "utf-16-le"), (codecs.BOM_UTF32, "utf-32"), (codecs.BOM_UTF32_BE, "utf-32-be"), (codecs.BOM_UTF32_LE, "utf-32-le"), ] ENCODING_RE = re.compile(rb"coding[:=]\s*([-\w.]+)") def auto_decode(data: bytes) -> str: """Check a bytes string for a BOM to correctly detect the encoding Fallback to locale.getpreferredencoding(False) like open() on Python3""" for bom, encoding in BOMS: if data.startswith(bom): return data[len(bom) :].decode(encoding) # Lets check the first two lines as in PEP263 for line in data.split(b"\n")[:2]: if line[0:1] == b"#" and ENCODING_RE.search(line): result = ENCODING_RE.search(line) assert result is not None encoding = result.groups()[0].decode("ascii") return data.decode(encoding) return data.decode( locale.getpreferredencoding(False) or sys.getdefaultencoding(), )
Name | Type | Size | Permission | Actions |
---|---|---|---|---|
__pycache__ | Folder | 0755 |
|
|
__init__.py | File | 0 B | 0644 |
|
_jaraco_text.py | File | 3.27 KB | 0644 |
|
_log.py | File | 1015 B | 0644 |
|
appdirs.py | File | 1.63 KB | 0644 |
|
compat.py | File | 2.34 KB | 0644 |
|
compatibility_tags.py | File | 6.13 KB | 0644 |
|
datetime.py | File | 242 B | 0644 |
|
deprecation.py | File | 3.62 KB | 0644 |
|
direct_url_helpers.py | File | 3.12 KB | 0644 |
|
egg_link.py | File | 2.41 KB | 0644 |
|
encoding.py | File | 1.14 KB | 0644 |
|
entrypoints.py | File | 2.99 KB | 0644 |
|
filesystem.py | File | 4.83 KB | 0644 |
|
filetypes.py | File | 716 B | 0644 |
|
glibc.py | File | 3.65 KB | 0644 |
|
hashes.py | File | 4.86 KB | 0644 |
|
logging.py | File | 11.33 KB | 0644 |
|
misc.py | File | 22.98 KB | 0644 |
|
packaging.py | File | 2.06 KB | 0644 |
|
retry.py | File | 1.36 KB | 0644 |
|
setuptools_build.py | File | 4.33 KB | 0644 |
|
subprocess.py | File | 8.78 KB | 0644 |
|
temp_dir.py | File | 9.09 KB | 0644 |
|
unpacking.py | File | 11.67 KB | 0644 |
|
urls.py | File | 1.56 KB | 0644 |
|
virtualenv.py | File | 3.38 KB | 0644 |
|
wheel.py | File | 4.39 KB | 0644 |
|