make entity cleanup a bit faster (hopefully!)

annevk · annevk · commit b916ab52bb30 · 2006-12-08T15:44:16.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%4090
diff --git a/constants.py b/constants.py
@@ -133,40 +133,40 @@
     "input"
 ))
 
-entitiesWindows1252 = {
-    128: 8364,  # 0x80  0x20AC  EURO SIGN
-    129: 65533, # 0x81          UNDEFINED
-    130: 8218,  # 0x82  0x201A  SINGLE LOW-9 QUOTATION MARK
-    131: 402,   # 0x83  0x0192  LATIN SMALL LETTER F WITH HOOK
-    132: 8222,  # 0x84  0x201E  DOUBLE LOW-9 QUOTATION MARK
-    133: 8230,  # 0x85  0x2026  HORIZONTAL ELLIPSIS
-    134: 8224,  # 0x86  0x2020  DAGGER
-    135: 8225,  # 0x87  0x2021  DOUBLE DAGGER
-    136: 710,   # 0x88  0x02C6  MODIFIER LETTER CIRCUMFLEX ACCENT
-    137: 8240,  # 0x89  0x2030  PER MILLE SIGN
-    138: 352,   # 0x8A  0x0160  LATIN CAPITAL LETTER S WITH CARON
-    139: 8249,  # 0x8B  0x2039  SINGLE LEFT-POINTING ANGLE QUOTATION MARK
-    140: 338,   # 0x8C  0x0152  LATIN CAPITAL LIGATURE OE
-    141: 65533, # 0x8D          UNDEFINED
-    142: 381,   # 0x8E  0x017D  LATIN CAPITAL LETTER Z WITH CARON
-    143: 65533, # 0x8F          UNDEFINED
-    144: 65533, # 0x90          UNDEFINED
-    145: 8216,  # 0x91  0x2018  LEFT SINGLE QUOTATION MARK
-    146: 8217,  # 0x92  0x2019  RIGHT SINGLE QUOTATION MARK
-    147: 8220,  # 0x93  0x201C  LEFT DOUBLE QUOTATION MARK
-    148: 8221,  # 0x94  0x201D  RIGHT DOUBLE QUOTATION MARK
-    149: 8226,  # 0x95  0x2022  BULLET
-    150: 8211,  # 0x96  0x2013  EN DASH
-    151: 8212,  # 0x97  0x2014  EM DASH
-    152: 732,   # 0x98  0x02DC  SMALL TILDE
-    153: 8482,  # 0x99  0x2122  TRADE MARK SIGN
-    154: 353,   # 0x9A  0x0161  LATIN SMALL LETTER S WITH CARON
-    155: 8250,  # 0x9B  0x203A  SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
-    156: 339,   # 0x9C  0x0153  LATIN SMALL LIGATURE OE
-    157: 65533, # 0x9D          UNDEFINED
-    158: 382,   # 0x9E  0x017E  LATIN SMALL LETTER Z WITH CARON
-    159: 376    # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS
-}
+entitiesWindows1252 = [
+    8364,  # 0x80  0x20AC  EURO SIGN
+    65533, # 0x81          UNDEFINED
+    8218,  # 0x82  0x201A  SINGLE LOW-9 QUOTATION MARK
+    402,   # 0x83  0x0192  LATIN SMALL LETTER F WITH HOOK
+    8222,  # 0x84  0x201E  DOUBLE LOW-9 QUOTATION MARK
+    8230,  # 0x85  0x2026  HORIZONTAL ELLIPSIS
+    8224,  # 0x86  0x2020  DAGGER
+    8225,  # 0x87  0x2021  DOUBLE DAGGER
+    710,   # 0x88  0x02C6  MODIFIER LETTER CIRCUMFLEX ACCENT
+    8240,  # 0x89  0x2030  PER MILLE SIGN
+    352,   # 0x8A  0x0160  LATIN CAPITAL LETTER S WITH CARON
+    8249,  # 0x8B  0x2039  SINGLE LEFT-POINTING ANGLE QUOTATION MARK
+    338,   # 0x8C  0x0152  LATIN CAPITAL LIGATURE OE
+    65533, # 0x8D          UNDEFINED
+    381,   # 0x8E  0x017D  LATIN CAPITAL LETTER Z WITH CARON
+    65533, # 0x8F          UNDEFINED
+    65533, # 0x90          UNDEFINED
+    8216,  # 0x91  0x2018  LEFT SINGLE QUOTATION MARK
+    8217,  # 0x92  0x2019  RIGHT SINGLE QUOTATION MARK
+    8220,  # 0x93  0x201C  LEFT DOUBLE QUOTATION MARK
+    8221,  # 0x94  0x201D  RIGHT DOUBLE QUOTATION MARK
+    8226,  # 0x95  0x2022  BULLET
+    8211,  # 0x96  0x2013  EN DASH
+    8212,  # 0x97  0x2014  EM DASH
+    732,   # 0x98  0x02DC  SMALL TILDE
+    8482,  # 0x99  0x2122  TRADE MARK SIGN
+    353,   # 0x9A  0x0161  LATIN SMALL LETTER S WITH CARON
+    8250,  # 0x9B  0x203A  SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
+    339,   # 0x9C  0x0153  LATIN SMALL LIGATURE OE
+    65533, # 0x9D          UNDEFINED
+    382,   # 0x9E  0x017E  LATIN SMALL LETTER Z WITH CARON
+    376    # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS
+]
 
 entities = {
     "AElig": u"\u00C6",
diff --git a/tokenizer.py b/tokenizer.py
@@ -189,12 +189,13 @@ def consumeNumberEntity(self, isHex):
         # If the integer is between 127 and 160 (so 128 and bigger and 159 and
         # smaller) we need to do the "windows trick".
         if 127 < charAsInt < 160:
-            charAsInt = entitiesWindows1252[charAsInt]
-
+            charAsInt = entitiesWindows1252[128 - charAsInt]
+        
+        # 0 is not a good number.
+        if charAsInt == 0:
+            charAsInt = 65533
+        
         try:
-            # XXX This is wrong. This doesn't take "windows-1252 entities" into
-            # account.
-
             # XXX We should have a separate function that does "int" to
             # "unicodestring" conversion since this doesn't always work
             # according to hsivonen. Also, unichr has a limitation of 65535
