fix an EOF bug in the tokenizer and add a testcase to make sure we don't regress in the future on that; also add a weird attribute testcase

annevk · annevk · commit b2d04d4cbf3d · 2006-12-22T22:44:22.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40266
diff --git a/tokenizer.py b/tokenizer.py
@@ -375,7 +375,7 @@ def emitCurrentToken(self):
         # Add token to the queue to be yielded
         self.tokenQueue.append(token)
 
-        self.changeState('data')
+        self.changeState("data")
 
     def emitToken(self, token):
         """ Used to add tokens directly to the queue to be yeilded
@@ -586,6 +586,7 @@ def attributeNameState(self):
             self.changeState("beforeAttributeName")
         elif data == u"<" or data == EOF:
             self.emitCurrentTokenWithParseError(data)
+            leavingThisState = False
         else:
             self.currentToken.data[-1][0] += data
             leavingThisState = False
