Revert breakage of test_tokenizer.py from accidential checkin of local modifications

jgraham · jgraham · commit ae0e9ea0232e · 2007-07-11T23:29:15.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40889
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -37,6 +37,8 @@ def processEmptyTag(self, token):
         self.outputTokens.append([u"StartTag", token["name"], token["data"]])
 
     def processEndTag(self, token):
+        if token["data"]:
+            self.processParseError(None)
         self.outputTokens.append([u"EndTag", token["name"]])
 
     def processComment(self, token):
@@ -53,7 +55,7 @@ def processEOF(self, token):
         pass
 
     def processParseError(self, token):
-        self.outputTokens.append([u"ParseError", token["data"]])
+        self.outputTokens.append(u"ParseError")
 
 def concatenateCharacterTokens(tokens):
     outputTokens = []
@@ -71,10 +73,9 @@ def concatenateCharacterTokens(tokens):
 def normalizeTokens(tokens):
     """ convert array of attributes to a dictionary """
     # TODO: convert tests to reflect arrays
-    for i, token in enumerate(tokens):
-        if token[0] == u'ParseError':
-            tokens[i] = token[0]
-            #token[2] = dict(token[2][::-1])
+    for token in tokens:
+        if token[0] == 'StartTag':
+            token[2] = dict(token[2][::-1])
     return tokens
 
 def tokensMatch(expectedTokens, recievedTokens):
@@ -101,14 +102,14 @@ def runTokenizerTest(self, test):
             test['lastStartTag'] = None
         parser = TokenizerTestParser(test['contentModelFlag'], 
                                      test['lastStartTag'])
-        tokens = parser.parse(test['input'])
+            
+        tokens = normalizeTokens(parser.parse(test['input']))
         tokens = concatenateCharacterTokens(tokens)
         errorMsg = "\n".join(["\n\nContent Model Flag:",
                               test['contentModelFlag'] ,
                               "\nInput:", str(test['input']),
                               "\nExpected:", str(output),
                               "\nRecieved:", str(tokens)])
-        tokens = normalizeTokens(tokens)
         self.assertEquals(tokensMatch(tokens, output), True, errorMsg)
 
 def buildTestSuite():
