fix tokenizer tests; implement null in our simplejson

annevk · annevk · commit 987719441352 · 2007-06-18T15:44:35.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40750
diff --git a/src/tokenizer.py b/src/tokenizer.py
@@ -713,6 +713,7 @@ def doctypeNameState(self):
         elif data == EOF:
             self.tokenQueue.append({"type": "ParseError", "data":
               _("Unexpected end of file in DOCTYPE name.")})
+            self.currentToken["correct"] = False
             self.tokenQueue.append(self.currentToken)
             self.state = self.states["data"]
         else:
@@ -727,7 +728,7 @@ def afterDoctypeNameState(self):
             self.tokenQueue.append(self.currentToken)
             self.state = self.states["data"]
         elif data == EOF:
-            # XXX check spec ... self.currentToken["correct"] = False
+            self.currentToken["correct"] = False
             self.stream.queue.append(data)
             self.tokenQueue.append({"type": "ParseError", "data":
               _("Unexpected end of file in DOCTYPE.")})
diff --git a/tests/support.py b/tests/support.py
@@ -27,7 +27,8 @@
     class simplejson:
         def load(f):
             true, false = True, False
-            input=re.sub(r'(".*?(?<!\\)")',r'u\1',f.read().decode('utf-8'))
+            input = re.sub(r'(".*?(?<!\\)")',r'u\1',f.read().decode('utf-8'))
+            input = input.replace('null', 'None')
             return eval(input.replace('\r',''))
         load = staticmethod(load)
 
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -33,7 +33,7 @@ def parse(self, stream, encoding=None, innerHTML=False):
         return self.outputTokens
 
     def processDoctype(self, token):
-        self.outputTokens.append([u"DOCTYPE", token["name"], token["correct"]])
+        self.outputTokens.append([u"DOCTYPE", token["name"], token["publicId"], token["systemId"], token["correct"]])
 
     def processStartTag(self, token):
         self.outputTokens.append([u"StartTag", token["name"], token["data"]])
