Fix several treewalker issues including the lack of end tag tokens in the elementtree treewalker

jgraham · jgraham · commit af96a7448bde · 2007-09-01T00:39:13.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40991
diff --git a/src/html5lib/treewalkers/_base.py b/src/html5lib/treewalkers/_base.py
@@ -103,6 +103,7 @@ def __iter__(self):
             details = self.getNodeDetails(currentNode)
             type, details = details[0], details[1:]
             hasChildren = False
+            endTag = None
 
             if type == DOCTYPE:
                 yield self.doctype(*details)
@@ -118,6 +119,7 @@ def __iter__(self):
                         yield token
                     hasChildren = False
                 else:
+                    endTag = name
                     yield self.startTag(name, attributes)
 
             elif type == COMMENT:
diff --git a/src/html5lib/treewalkers/etree.py b/src/html5lib/treewalkers/etree.py
@@ -78,19 +78,20 @@ def getFirstChild(self, node):
                 return (node, 0, parents)
 
         def getNextSibling(self, node):
-            assert isinstance(node, tuple), "Node is not a tuple: " + str(node)
-
-            elt, key, parents = node
-            if key == "text":
-                key = -1
-            elif key == "tail":
-                elt, key = parents.pop()
+            if isinstance(node, tuple):
+                elt, key, parents = node
+                if key == "text":
+                    key = -1
+                elif key == "tail":
+                    elt, key = parents.pop()
+                else:
+                    # Look for "tail" of the "revisited" node
+                    child = elt[key]
+                    if child.tail:
+                        parents.append((elt, key))
+                        return (child, "tail", parents)
             else:
-                # Look for "tail" of the "revisited" node
-                child = elt[key]
-                if child.tail:
-                    parents.append((elt, key))
-                    return (child, "tail", parents)
+                return None
 
             # case where key were "text" or "tail" or elt[key] had a tail
             key += 1
@@ -106,7 +107,6 @@ def getParentNode(self, node):
                 elt, key = parents.pop()
                 return elt, key, parents
             else:
-                # HACK: We could return ``elt`` but None will stop the algorithm the same way
-                return None
+                return elt
 
     return locals()
diff --git a/src/html5lib/treewalkers/genshistream.py b/src/html5lib/treewalkers/genshistream.py
@@ -1,4 +1,4 @@
-from genshi.core import START, END, XML_DECL, DOCTYPE, TEXT, \
+from genshi.core import START, END, XML_NAMESPACE, DOCTYPE, TEXT, \
     START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT
 from genshi.output import NamespaceFlattener
 
@@ -59,7 +59,7 @@ def tokens(self, event, next):
         elif kind == DOCTYPE:
             yield self.doctype(*data)
 
-        elif kind in (XML_DECL, DOCTYPE, START_NS, END_NS, \
+        elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, \
           START_CDATA, END_CDATA, PI):
             pass
 
diff --git a/tests/test_treewalkers.py b/tests/test_treewalkers.py
@@ -208,19 +208,35 @@ def runTest(self, innerHTML, input, expected, errors, treeClass):
             document = p.parse(StringIO.StringIO(input))
         document = treeClass.get("adapter", lambda x: x)(document)
         try:
-            output = convertTokens(LintFilter(treeClass["walker"](document)))
+            output = convertTokens(treeClass["walker"](document))
             output = attrlist.sub(sortattrs, output)
             expected = attrlist.sub(sortattrs, convertExpected(expected))
             self.assertEquals(expected, output, "\n".join([
                 "", "Input:", input,
                 "", "Expected:", expected,
                 "", "Recieved:", output
             ]))
-        except LintError, le:
-            self.fail(input + "\n" + le.message)
         except NotImplementedError:
             pass # Amnesty for those that confess...
 
+class TokenTestCase(unittest.TestCase):
+    def test_all_tokens(self):
+        expected = [
+            {'data': [], 'type': 'StartTag', 'name': u'html'},
+            {'data': [], 'type': 'StartTag', 'name': u'head'},
+            {'data': [], 'type': 'EndTag', 'name': u'head'},
+            {'data': [], 'type': 'StartTag', 'name': u'body'},
+            {'data': [], 'type': 'EndTag', 'name': u'body'},
+            {'data': [], 'type': 'EndTag', 'name': u'html'}]
+        for treeName, treeCls in treeTypes.iteritems():
+            p = html5parser.HTMLParser(tree = treeCls["builder"])
+            document = p.parse("<html></html>")
+            document = treeCls.get("adapter", lambda x: x)(document)
+            output = treeCls["walker"](document)
+            for expectedToken, outputToken in zip(expected, output):
+                self.assertEquals(expectedToken, outputToken)
+
+            
 def buildTestSuite():
     sys.stdout.write('Testing tree walkers '+ " ".join(treeTypes.keys()) + "\n")
 
