editorial changes

annevk · annevk · commit 7568d3140ac1 · 2007-07-22T14:18:55.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40914
diff --git a/README b/README
@@ -1,6 +1,6 @@
 html5lib is a pure-python library for parsing HTML. It is designed to
-conform to the Web Applications 1.0 specification, which has
-formalized the error handling algorithms of popular web browsers.
+conform to the HTML 5 specification, which has formalized the error handling
+algorithms of popular web browsers.
 
  = Installation =
 
@@ -36,3 +36,4 @@ http://code.google.com/p/html5lib/issues/list
 Contributions to code or documenation are actively encouraged. Submit
 patches to the issue tracker or discuss changes on irc in the #whatwg
 channel on freenode.net
+
diff --git a/src/html5lib/tokenizer.py b/src/html5lib/tokenizer.py
@@ -297,11 +297,15 @@ def emitCurrentToken(self):
 
     def dataState(self):
         data = self.stream.char()
+
+        # Keep a charbuffer to handle the escapeFlag
         if self.contentModelFlag in\
           (contentModelFlags["CDATA"], contentModelFlags["RCDATA"]):
             if len(self.lastFourChars) == 4:
                 self.lastFourChars.pop(0)
             self.lastFourChars.append(data)
+
+        # The rest of the logic
         if data == "&" and self.contentModelFlag in\
           (contentModelFlags["PCDATA"], contentModelFlags["RCDATA"]) and not\
           self.escapeFlag:
@@ -328,8 +332,6 @@ def dataState(self):
             # Directly after emitting a token you switch back to the "data
             # state". At that point spaceCharacters are important so they are
             # emitted separately.
-            # XXX need to check if we don't need a special "spaces" flag on
-            # characters.
             self.tokenQueue.append({"type": "SpaceCharacters", "data":
               data + self.stream.charsUntil(spaceCharacters, True)})
         else:
