Fix tokenizer test harness to not decode tests multiple times.

gsnedders · gsnedders · commit e4f6fa1d0381 · 2013-04-10T18:04:53.000+01:00
Given a double escaped test, if it had multiple initial states, the
same test object would be decoded multiple times: this led to bogus
results on every run except the first.
diff --git a/html5lib/tests/test_tokenizer.py b/html5lib/tests/test_tokenizer.py
@@ -144,10 +144,6 @@ def decode(inp):
 def runTokenizerTest(test):
     warnings.resetwarnings()
     warnings.simplefilter("error")
-    #XXX - move this out into the setup function
-    #concatenate all consecutive character tokens into a single token
-    if 'doubleEscaped' in test:
-        test = unescape(test)
 
     expected = concatenateCharacterTokens(test['output'])            
     if 'lastStartTag' not in test:
@@ -183,10 +179,10 @@ def testTokenizer():
             testName = os.path.basename(filename).replace(".test","")
             if 'tests' in tests:
                 for index,test in enumerate(tests['tests']):
-                #Skip tests with a self closing flag
-                    skip = False
                     if 'initialStates' not in test:
                         test["initialStates"] = ["Data state"]
+                    if 'doubleEscaped' in test:
+                        test = unescape(test)
                     for initialState in test["initialStates"]:
                         test["initialState"] = capitalize(initialState)
                         yield runTokenizerTest, test
