Skip to content

Commit f3c54fd

Browse files
jgrahamjgraham
committed
Attempt at merging svgmathml branch to the default branch
--HG-- branch : svgmathml rename : python/parse.py => python3/parse.py rename : python/src/html5lib/__init__.py => python3/src/html5lib/__init__.py rename : python/src/html5lib/constants.py => python3/src/html5lib/constants.py rename : python/src/html5lib/filters/optionaltags.py => python3/src/html5lib/filters/optionaltags.py rename : python/src/html5lib/html5parser.py => python3/src/html5lib/html5parser.py rename : python/src/html5lib/inputstream.py => python3/src/html5lib/inputstream.py rename : python/src/html5lib/sanitizer.py => python3/src/html5lib/sanitizer.py rename : python/src/html5lib/serializer/__init__.py => python3/src/html5lib/serializer/__init__.py rename : python/src/html5lib/tokenizer.py => python3/src/html5lib/tokenizer.py rename : python/src/html5lib/treebuilders/etree_lxml.py => python3/src/html5lib/treebuilders/etree_lxml.py rename : python/src/html5lib/treebuilders/simpletree.py => python3/src/html5lib/treebuilders/simpletree.py rename : python/tests/test_encoding.py => python3/tests/test_encoding.py rename : python/tests/test_parser.py => python3/tests/test_parser.py rename : python/tests/test_tokenizer.py => python3/tests/test_tokenizer.py
2 parents b036bb6 + d359419 commit f3c54fd

File tree

10 files changed

+76
-35
lines changed

10 files changed

+76
-35
lines changed

‎parse.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ def parse():
5252
else:
5353
tokenizer = HTMLTokenizer
5454

55-
if opts.xml:
55+
if opts.liberalxml:
5656
p = liberalxmlparser.XHTMLParser(tree=treebuilder, tokenizer=tokenizer)
5757
else:
5858
p = html5parser.HTMLParser(tree=treebuilder, tokenizer=tokenizer)
@@ -196,6 +196,9 @@ def getOptParser():
196196
parser.add_option("", "--sanitize", action="store_true", default=False,
197197
dest="sanitize", help="sanitize")
198198

199+
parser.add_option("", "--liberal-xml-parser", action="store_true", default=False,
200+
dest="liberalxml", help="parse with liberal xml parser")
201+
199202
return parser
200203

201204
if __name__ == "__main__":

‎src/html5lib/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@
1313
"""
1414
from html5parser import HTMLParser, parse
1515
from treebuilders import getTreeBuilder
16+
from serializer import serialize
1617

1718
from liberalxmlparser import XMLParser, XHTMLParser

‎src/html5lib/constants.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1091,7 +1091,6 @@
10911091
'utf16': 'utf-16',
10921092
'utf16be': 'utf-16-be',
10931093
'utf16le': 'utf-16-le',
1094-
'utf7': 'utf-7',
10951094
'utf8': 'utf-8',
10961095
'windows1250': 'cp1250',
10971096
'windows1251': 'cp1251',

‎src/html5lib/filters/optionaltags.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,11 @@ def is_optional_start(self, tagname, previous, next):
3131
elif tagname == 'head':
3232
# A head element's start tag may be omitted if the first thing
3333
# inside the head element is an element.
34-
return type == "StartTag"
34+
# XXX: we also omit the start tag if the head element is empty
35+
if type in ("StartTag", "EmptyTag"):
36+
return True
37+
elif type == "EndTag":
38+
return next["name"] == "head"
3539
elif tagname == 'body':
3640
# A body element's start tag may be omitted if the first thing
3741
# inside the body element is not a space character or a comment,
@@ -52,7 +56,7 @@ def is_optional_start(self, tagname, previous, next):
5256
# inside the colgroup element is a col element, and if the element
5357
# is not immediately preceeded by another colgroup element whose
5458
# end tag has been omitted.
55-
if type == "StartTag":
59+
if type in ("StartTag", "EmptyTag"):
5660
# XXX: we do not look at the preceding event, so instead we never
5761
# omit the colgroup element's end tag when it is immediately
5862
# followed by another colgroup element. See is_optional_end.
@@ -114,7 +118,7 @@ def is_optional_end(self, tagname, next):
114118
# footer, form, h1, h2, h3, h4, h5, h6, header, hr, menu,
115119
# nav, ol, p, pre, section, table, or ul, element, or if
116120
# there is no more content in the parent element.
117-
if type == "StartTag":
121+
if type in ("StartTag", "EmptyTag"):
118122
return next["name"] in ('address', 'article', 'aside', \
119123
'blockquote', 'datagrid', 'dialog', 'dir', 'div', \
120124
'dl', 'fieldset', 'footer', 'form', 'h1', 'h2', 'h3', \

‎src/html5lib/html5parser.py‎

Lines changed: 39 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@
1818
from constants import scopingElements, formattingElements, specialElements
1919
from constants import headingElements, tableInsertModeElements
2020
from constants import cdataElements, rcdataElements, voidElements
21+
from constants import tokenTypes, ReparseException
2122
from constants import tokenTypes, namespaces
2223

23-
def parse(doc, treebuilderName="simpletree", encoding=None):
24-
tb = treebuilders.getTreeBuilder(treebuilderName)
24+
def parse(doc, treebuilder="simpletree", encoding=None):
25+
tb = treebuilders.getTreeBuilder(treebuilder)
2526
p = HTMLParser(tb)
2627
return p.parse(doc, encoding=encoding)
2728

@@ -79,19 +80,30 @@ def __init__(self, tree = simpletree.TreeBuilder,
7980

8081
def _parse(self, stream, innerHTML=False, container="div",
8182
encoding=None, parseMeta=True, useChardet=True, **kwargs):
82-
83+
84+
self.innerHTMLMode = innerHTML
85+
self.container = container
86+
self.tokenizer = self.tokenizer_class(stream, encoding=encoding,
87+
parseMeta=parseMeta,
88+
useChardet=useChardet, **kwargs)
89+
self.reset()
90+
91+
while True:
92+
try:
93+
self.mainLoop()
94+
break
95+
except ReparseException, e:
96+
self.reset()
97+
98+
def reset(self):
8399
self.tree.reset()
84100
self.firstStartTag = False
85101
self.errors = []
86102
# "quirks" / "limited quirks" / "no quirks"
87103
self.compatMode = "no quirks"
88104

89-
self.tokenizer = self.tokenizer_class(stream, encoding=encoding,
90-
parseMeta=parseMeta,
91-
useChardet=useChardet, **kwargs)
92-
93-
if innerHTML:
94-
self.innerHTML = container.lower()
105+
if self.innerHTMLMode:
106+
self.innerHTML = self.container.lower()
95107

96108
if self.innerHTML in cdataElements:
97109
self.tokenizer.contentModelFlag = tokenizer.contentModelFlags["RCDATA"]
@@ -114,6 +126,19 @@ def _parse(self, stream, innerHTML=False, container="div",
114126
self.secondaryPhase = None
115127

116128
self.beforeRCDataPhase = None
129+
130+
def mainLoop(self):
131+
(CharactersToken,
132+
SpaceCharactersToken,
133+
StartTagToken,
134+
EndTagToken,
135+
CommentToken,
136+
DoctypeToken) = (tokenTypes["Characters"],
137+
tokenTypes["SpaceCharacters"],
138+
tokenTypes["StartTag"],
139+
tokenTypes["EndTag"],
140+
tokenTypes["Comment"],
141+
tokenTypes["Doctype"])
117142

118143
CharactersToken = tokenTypes["Characters"]
119144
SpaceCharactersToken = tokenTypes["SpaceCharacters"]
@@ -124,6 +149,8 @@ def _parse(self, stream, innerHTML=False, container="div",
124149

125150

126151
for token in self.normalizedTokens():
152+
#print self.phase.__class__.__name__
153+
#print token
127154
type = token["type"]
128155
if type == CharactersToken:
129156
self.phase.processCharacters(token)
@@ -378,18 +405,6 @@ def __init__(self, parser, tree):
378405

379406
def processEOF(self):
380407
raise NotImplementedError
381-
self.tree.generateImpliedEndTags()
382-
if len(self.tree.openElements) > 2:
383-
self.parser.parseError("expected-closing-tag-but-got-eof")
384-
elif len(self.tree.openElements) == 2 and\
385-
self.tree.openElements[1].name != "body":
386-
# This happens for framesets or something?
387-
self.parser.parseError("expected-closing-tag-but-got-eof")
388-
elif self.parser.innerHTML and len(self.tree.openElements) > 1 :
389-
# XXX This is not what the specification says. Not sure what to do
390-
# here.
391-
self.parser.parseError("eof-in-innerhtml")
392-
# Betting ends.
393408

394409
def processComment(self, token):
395410
# For most phases the following is correct. Where it's not it will be
@@ -702,10 +717,10 @@ def startTagMeta(self, token):
702717
attributes = token["data"]
703718
if self.parser.tokenizer.stream.charEncoding[1] == "tentative":
704719
if "charset" in attributes:
705-
codec = inputstream.codecName(attributes["charset"])
706-
self.parser.tokenizer.stream.changeEncoding(codec)
720+
self.parser.tokenizer.stream.changeEncoding(attributes["charset"])
707721
elif "content" in attributes:
708-
data = inputstream.EncodingBytes(attributes["content"])
722+
data = inputstream.EncodingBytes(
723+
attributes["content"].encode(self.parser.tokenizer.stream.charEncoding[0]))
709724
parser = inputstream.ContentAttrParser(data)
710725
codec = parser.parse()
711726
self.parser.tokenizer.stream.changeEncoding(codec)

‎src/html5lib/inputstream.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import codecs
22
import re
33
import types
4+
import sys
45

56
from constants import EOF, spaceCharacters, asciiLetters, asciiUppercase
67
from constants import encodings, ReparseException
@@ -10,7 +11,7 @@
1011
asciiLettersBytes = [str(item) for item in asciiLetters]
1112
asciiUppercaseBytes = [str(item) for item in asciiUppercase]
1213

13-
invalid_unicode_re = re.compile(u"[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDDF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]")
14+
invalid_unicode_re = re.compile(u"[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]")
1415

1516
non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
1617
0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
@@ -196,7 +197,8 @@ def openStream(self, source):
196197
import cStringIO
197198
stream = cStringIO.StringIO(str(source))
198199

199-
if not(hasattr(stream, "tell") and hasattr(stream, "seek")):
200+
if (not(hasattr(stream, "tell") and hasattr(stream, "seek")) or
201+
stream is sys.stdin):
200202
stream = BufferedStream(stream)
201203

202204
return stream
@@ -494,8 +496,10 @@ class EncodingBytes(str):
494496
"""String-like object with an assosiated position and various extra methods
495497
If the position is ever greater than the string length then an exception is
496498
raised"""
499+
def __new__(self, value):
500+
return str.__new__(self, value)
501+
497502
def __init__(self, value):
498-
str.__init__(self, value)
499503
self._position=-1
500504

501505
def __iter__(self):

‎src/html5lib/sanitizer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ def sanitize_token(self, token):
152152
continue
153153
val_unescaped = re.sub("[`\000-\040\177-\240\s]+", '',
154154
unescape(attrs[attr])).lower()
155-
if (re.match("^[a-z0-9][-+.a-z0-9]*:",val_unescaped) or
155+
if (re.match("^[a-z0-9][-+.a-z0-9]*:",val_unescaped) and
156156
(val_unescaped.split(':')[0] not in
157157
self.allowed_protocols)):
158158
del attrs[attr]

‎src/html5lib/serializer/__init__.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,17 @@
11

2+
from html5lib import treewalkers
3+
24
from htmlserializer import HTMLSerializer
35
from xhtmlserializer import XHTMLSerializer
6+
7+
def serialize(input, tree="simpletree", format="html", encoding=None,
8+
**serializer_opts):
9+
# XXX: Should we cache this?
10+
walker = treewalkers.getTreeWalker(tree)
11+
if format == "html":
12+
s = HTMLSerializer(**serializer_opts)
13+
elif format == "xhtml":
14+
s = XHTMLSerializer(**serializer_opts)
15+
else:
16+
raise ValueError, "type must be either html or xhtml"
17+
return s.render(walker(input), encoding)

‎src/html5lib/tokenizer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ def consumeNumberEntity(self, isHex):
151151
# Certain characters get replaced with U+FFFD
152152
if ((charAsInt <= 0x0008) or (charAsInt == 0x000B) or (0x000E <= charAsInt <= 0x001F)
153153
or (0x007F <= charAsInt <= 0x009F)
154-
or (0xD800 <= charAsInt <= 0xDFFF) or (0xFDD0 <= charAsInt <= 0xFDDF)
154+
or (0xD800 <= charAsInt <= 0xDFFF) or (0xFDD0 <= charAsInt <= 0xFDEF)
155155
or (charAsInt & 0xFFFE == 0xFFFE) # catch all U+?FFFE and U+?FFFF, where ? is 0..10
156156
or (0x10FFFF < charAsInt)):
157157
char = u"\uFFFD"

‎tests/test_parser.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,8 @@ def buildTestSuite():
136136
def testFunc(self, innerHTML=innerHTML, input=input,
137137
expected=expected, errors=errors, treeCls=treeCls):
138138
return self.runParserTest(innerHTML, input, expected, errors, treeCls)
139-
setattr(TestCase, "test_%s_%d_%s" % (testName,index+1,treeName),
139+
testFunc.__name__ = "test_%s_%d_%s" % (testName,index+1,treeName)
140+
setattr(TestCase, testFunc.__name__,
140141
testFunc)
141142

142143
return unittest.TestLoader().loadTestsFromTestCase(TestCase)

0 commit comments

Comments
 (0)