Every place you can specify a stream, you should be able to specify an

rubys · rubys · commit f8bfb92eb845 · 2007-01-11T15:31:36.000Z
external encoding.

--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40448
diff --git a/src/html5parser.py b/src/html5parser.py
@@ -70,13 +70,18 @@ def __init__(self, strict = False, tree=simpletree.TreeBuilder):
             "trailingEnd": TrailingEndPhase(self, self.tree)
         }
 
-    def parse(self, stream, innerHTML=False):
+    def parse(self, stream, encoding=None, innerHTML=False):
         """Parse a HTML document into a well-formed tree
         
         stream - a filelike object or string containing the HTML to be parsed
         
         innerHTML - Are we parsing in innerHTML mode (note innerHTML=True 
         is not yet supported)
+
+        The optional encoding parameter must be a string that indicates
+        the encoding.  If specified, that encoding will be used,
+        regardless of any BOM or later declaration (such as in a meta
+        element)
         """
 
         self.tree.reset()
@@ -91,7 +96,7 @@ def parse(self, stream, innerHTML=False):
         # assertations
         self.innerHTML = innerHTML
 
-        self.tokenizer = tokenizer.HTMLTokenizer(stream)
+        self.tokenizer = tokenizer.HTMLTokenizer(stream, encoding)
 
         # XXX This is temporary for the moment so there isn't any other
         # changes needed for the parser to work with the iterable tokenizer
diff --git a/src/tokenizer.py b/src/tokenizer.py
@@ -32,8 +32,8 @@ class HTMLTokenizer(object):
 
     # XXX need to fix documentation
 
-    def __init__(self, stream):
-        self.stream = HTMLInputStream(stream)
+    def __init__(self, stream, encoding=None):
+        self.stream = HTMLInputStream(stream, encoding)
 
         self.states = {
             "data":self.dataState,
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -20,8 +20,8 @@ def __init__(self, contentModelFlag, lastStartTag=None):
         self._contentModelFlag = constants.contentModelFlags[contentModelFlag]
         self._lastStartTag = lastStartTag
 
-    def parse(self, stream, innerHTML=False):
-        tokenizer = self.tokenizer(stream)
+    def parse(self, stream, encoding=None, innerHTML=False):
+        tokenizer = self.tokenizer(stream, encoding)
         self.outputTokens = []
 
         tokenizer.contentModelFlag = self._contentModelFlag
