Util for parsing iana encodings list

jgraham · jgraham · commit d9a9980fefd5 · 2007-02-27T23:35:51.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40510
diff --git a/utils/encodings.py b/utils/encodings.py
@@ -0,0 +1,225 @@
+encodings = frozenset((
+    "ansi_x3.4-1968",
+    "iso-ir-6",
+    "ansi_x3.4-1986",
+    "iso_646.irv:1991",
+    "ascii",
+    "iso646-us",
+    "us-ascii",
+    "us",
+    "ibm367",
+    "cp367",
+    "csascii",
+    "ks_c_5601-1987",
+    "korean",
+    "iso-2022-kr",
+    "csiso2022kr",
+    "euc-kr",
+    "iso-2022-jp",
+    "csiso2022jp",
+    "iso-2022-jp-2",
+    "iso-ir-58",
+    "chinese",
+    "csiso58gb231280",
+    "iso_8859-1:1987",
+    "iso-ir-100",
+    "iso_8859-1",
+    "iso-8859-1",
+    "latin1",
+    "l1",
+    "ibm819",
+    "cp819",
+    "csisolatin1",
+    "iso_8859-2:1987",
+    "iso-ir-101",
+    "iso_8859-2",
+    "iso-8859-2",
+    "latin2",
+    "l2",
+    "csisolatin2",
+    "iso_8859-3:1988",
+    "iso-ir-109",
+    "iso_8859-3",
+    "iso-8859-3",
+    "latin3",
+    "l3",
+    "csisolatin3",
+    "iso_8859-4:1988",
+    "iso-ir-110",
+    "iso_8859-4",
+    "iso-8859-4",
+    "latin4",
+    "l4",
+    "csisolatin4",
+    "iso_8859-6:1987",
+    "iso-ir-127",
+    "iso_8859-6",
+    "iso-8859-6",
+    "ecma-114",
+    "asmo-708",
+    "arabic",
+    "csisolatinarabic",
+    "iso_8859-7:1987",
+    "iso-ir-126",
+    "iso_8859-7",
+    "iso-8859-7",
+    "elot_928",
+    "ecma-118",
+    "greek",
+    "greek8",
+    "csisolatingreek",
+    "iso_8859-8:1988",
+    "iso-ir-138",
+    "iso_8859-8",
+    "iso-8859-8",
+    "hebrew",
+    "csisolatinhebrew",
+    "iso_8859-5:1988",
+    "iso-ir-144",
+    "iso_8859-5",
+    "iso-8859-5",
+    "cyrillic",
+    "csisolatincyrillic",
+    "iso_8859-9:1989",
+    "iso-ir-148",
+    "iso_8859-9",
+    "iso-8859-9",
+    "latin5",
+    "l5",
+    "csisolatin5",
+    "iso-8859-10",
+    "iso-ir-157",
+    "l6",
+    "iso_8859-10:1992",
+    "csisolatin6",
+    "latin6",
+    "hp-roman8",
+    "roman8",
+    "r8",
+    "ibm037",
+    "cp037",
+    "ebcdic-cp-us",
+    "ebcdic-cp-ca",
+    "ebcdic-cp-wt",
+    "ebcdic-cp-nl",
+    "csibm037",
+    "ibm424",
+    "cp424",
+    "ebcdic-cp-he",
+    "csibm424",
+    "ibm437",
+    "cp437",
+    "437",
+    "cspc8codepage437",
+    "ibm500",
+    "cp500",
+    "ebcdic-cp-be",
+    "ebcdic-cp-ch",
+    "csibm500",
+    "ibm775",
+    "cp775",
+    "cspc775baltic",
+    "ibm850",
+    "cp850",
+    "850",
+    "cspc850multilingual",
+    "ibm852",
+    "cp852",
+    "852",
+    "cspcp852",
+    "ibm855",
+    "cp855",
+    "855",
+    "csibm855",
+    "ibm857",
+    "cp857",
+    "857",
+    "csibm857",
+    "ibm860",
+    "cp860",
+    "860",
+    "csibm860",
+    "ibm861",
+    "cp861",
+    "861",
+    "cp-is",
+    "csibm861",
+    "ibm862",
+    "cp862",
+    "862",
+    "cspc862latinhebrew",
+    "ibm863",
+    "cp863",
+    "863",
+    "csibm863",
+    "ibm864",
+    "cp864",
+    "csibm864",
+    "ibm865",
+    "cp865",
+    "865",
+    "csibm865",
+    "ibm866",
+    "cp866",
+    "866",
+    "csibm866",
+    "ibm869",
+    "cp869",
+    "869",
+    "cp-gr",
+    "csibm869",
+    "ibm1026",
+    "cp1026",
+    "csibm1026",
+    "koi8-r",
+    "cskoi8r",
+    "koi8-u",
+    "big5-hkscs",
+    "ptcp154",
+    "csptcp154",
+    "pt154",
+    "cp154",
+    "utf-7",
+    "utf-16be",
+    "utf-16le",
+    "utf-16",
+    "utf-8",
+    "iso-8859-13",
+    "iso-8859-14",
+    "iso-ir-199",
+    "iso_8859-14:1998",
+    "iso_8859-14",
+    "latin8",
+    "iso-celtic",
+    "l8",
+    "iso-8859-15",
+    "iso_8859-15",
+    "iso-8859-16",
+    "iso-ir-226",
+    "iso_8859-16:2001",
+    "iso_8859-16",
+    "latin10",
+    "l10",
+    "gbk",
+    "cp936",
+    "ms936",
+    "gb18030",
+    "shift_jis",
+    "ms_kanji",
+    "csshiftjis",
+    "euc-jp",
+    "gb2312",
+    "big5",
+    "csbig5",
+    "windows-1250",
+    "windows-1251",
+    "windows-1252",
+    "windows-1253",
+    "windows-1254",
+    "windows-1255",
+    "windows-1256",
+    "windows-1257",
+    "windows-1258",
+    "tis-620",
+    "hz-gb-2312",
+    ))
diff --git a/utils/iana_parse.py b/utils/iana_parse.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+import sys
+import urllib2
+import codecs
+
+def main():
+    encodings = []
+    f = urllib2.urlopen(sys.argv[1])
+    for line in f:
+        if line.startswith("Name: ") or line.startswith("Alias: "):
+            enc = line.split()[1]
+            try:
+                codecs.lookup(enc)
+                if enc.lower not in encodings:
+                    encodings.append(enc.lower())
+            except LookupError:
+                pass
+    sys.stdout.write("encodings = frozenset((\n")
+    for enc in encodings:
+        sys.stdout.write('    "%s",\n'%enc)
+    sys.stdout.write('    ))')
+
+if __name__ == "__main__":
+    main()
diff --git a/utils/package.py b/utils/package.py
