diff --git a/readability/readability.py b/readability/readability.py
index d85e7f1b..321a181a 100755
--- a/readability/readability.py
+++ b/readability/readability.py
@@ -26,7 +26,7 @@
'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter', re.I),
'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow', re.I),
'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story', re.I),
- 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget', re.I),
+ 'negativeRe': re.compile('navbar|combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget', re.I),
'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)', re.I),
#'replaceBrsRe': re.compile('(
]*>[ \n\r\t]*){2,}',re.I),
#'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I),
@@ -86,9 +86,9 @@ def __init__(self, input, positive_keywords=None, negative_keywords=None,
:param input: string of the html content.
:param positive_keywords: regex or list of patterns in classes and ids
:param negative_keywords: regex or list of patterns in classes and ids
- :param min_text_length:
+ :param min_text_length:
:param retry_length:
-
+
Example:
positive_keywords=["news-item", "block"]
negative_keywords=["mysidebar", "related", "ads"]
@@ -256,7 +256,7 @@ def select_best_candidate(self, candidates):
return None
sorted_candidates = sorted(
- candidates.values(),
+ candidates.values(),
key=lambda x: x['content_score'],
reverse=True
)
@@ -282,7 +282,7 @@ def score_paragraphs(self, ):
MIN_LEN = self.min_text_length
candidates = {}
ordered = []
- for elem in self.tags(self._html(), "p", "pre", "td"):
+ for elem in self.sorted_tags(self._html(), "p", "pre", "td"):
parent_node = elem.getparent()
if parent_node is None:
continue
@@ -414,6 +414,19 @@ def transform_misused_divs_into_paragraphs(self):
#print 'Dropped
at '+describe(elem)
child.drop_tree()
+ def sorted_tags(self, node, *tag_names):
+ root = self.html.getroottree()
+ tags = {}
+ for tag_name in tag_names:
+ for e in node.findall('.//%s' % tag_name):
+ tags[e] = root.getpath(e)
+
+ # from deep to shallow
+ tags = sorted(tags.items(), key=lambda x: len(x[1]), reverse=True)
+
+ for t in tags:
+ yield t[0]
+
def tags(self, node, *tag_names):
for tag_name in tag_names:
for e in node.findall('.//%s' % tag_name):