|
3 | 3 | import sys |
4 | 4 |
|
5 | 5 |
|
6 | | -RE_CHARSET = re.compile(br'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I) |
7 | | -RE_PRAGMA = re.compile(br'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I) |
8 | | -RE_XML = re.compile(br'^<\?xml.*?encoding=["\']*(.+?)["\'>]') |
| 6 | +RE_CHARSET = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I) |
| 7 | +RE_PRAGMA = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I) |
| 8 | +RE_XML = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]') |
9 | 9 |
|
10 | 10 | CHARSETS = { |
11 | 11 | 'big5': 'big5hkscs', |
@@ -34,24 +34,24 @@ def get_encoding(page): |
34 | 34 | # Try any declared encodings |
35 | 35 | for declared_encoding in declared_encodings: |
36 | 36 | try: |
| 37 | + # Python3 only |
37 | 38 | if sys.version_info[0] == 3: |
38 | 39 | # declared_encoding will actually be bytes but .decode() only |
39 | 40 | # accepts `str` type. Decode blindly with ascii because no one should |
40 | 41 | # ever use non-ascii characters in the name of an encoding. |
41 | 42 | declared_encoding = declared_encoding.decode('ascii', 'replace') |
42 | 43 |
|
43 | 44 | encoding = fix_charset(declared_encoding) |
44 | | - |
45 | 45 | # Now let's decode the page |
46 | 46 | page.decode(encoding) |
47 | 47 | # It worked! |
48 | 48 | return encoding |
49 | | - except (UnicodeDecodeError, LookupError): |
| 49 | + except UnicodeDecodeError: |
50 | 50 | pass |
51 | 51 |
|
52 | 52 | # Fallback to chardet if declared encodings fail |
53 | 53 | # Remove all HTML tags, and leave only text for chardet |
54 | | - text = re.sub(br'(\s*</?[^>]*>)+\s*', b' ', page).strip() |
| 54 | + text = re.sub(r'(\s*</?[^>]*>)+\s*', ' ', page).strip() |
55 | 55 | enc = 'utf-8' |
56 | 56 | if len(text) < 10: |
57 | 57 | return enc # can't guess |
|
0 commit comments