From edccec5d3b4cecee3fdccff7667dd81bb3ed6258 Mon Sep 17 00:00:00 2001
From: Richard Harding
]*>[ \n\r\t]*){2,}',re.I),
- #'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I),
- #'trimRe': re.compile('^\s+|\s+$/'),
- #'normalizeRe': re.compile('\s{2,}/'),
- #'killBreaksRe': re.compile('(
(\s| ?)*){1,}/'),
- #'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I),
- #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i,
+ 'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter',re.I),
+ 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow',re.I),
+ 'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story',re.I),
+ 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget',re.I),
+ 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)',re.I),
+ #'replaceBrsRe': re.compile('(
]*>[ \n\r\t]*){2,}',re.I),
+ #'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I),
+ #'trimRe': re.compile('^\s+|\s+$/'),
+ #'normalizeRe': re.compile('\s{2,}/'),
+ #'killBreaksRe': re.compile('(
(\s| ?)*){1,}/'),
+ #'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I),
+ #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i,
}
def describe(node, depth=1):
- if not hasattr(node, 'tag'):
- return "[%s]" % type(node)
- name = node.tag
- if node.get('id', ''): name += '#'+node.get('id')
- if node.get('class', ''):
- name += '.' + node.get('class').replace(' ','.')
- if name[:4] in ['div#', 'div.']:
- name = name[3:]
- if depth and node.getparent() is not None:
- return name+' - '+describe(node.getparent(), depth-1)
- return name
+ if not hasattr(node, 'tag'):
+ return "[%s]" % type(node)
+ name = node.tag
+ if node.get('id', ''): name += '#'+node.get('id')
+ if node.get('class', ''):
+ name += '.' + node.get('class').replace(' ','.')
+ if name[:4] in ['div#', 'div.']:
+ name = name[3:]
+ if depth and node.getparent() is not None:
+ return name+' - '+describe(node.getparent(), depth-1)
+ return name
def to_int(x):
- if not x: return None
- x = x.strip()
- if x.endswith('px'):
- return int(x[:-2])
- if x.endswith('em'):
- return int(x[:-2]) * 12
- return int(x)
+ if not x: return None
+ x = x.strip()
+ if x.endswith('px'):
+ return int(x[:-2])
+ if x.endswith('em'):
+ return int(x[:-2]) * 12
+ return int(x)
def clean(text):
- text = re.sub('\s*\n\s*', '\n', text)
- text = re.sub('[ \t]{2,}', ' ', text)
- return text.strip()
+ text = re.sub('\s*\n\s*', '\n', text)
+ text = re.sub('[ \t]{2,}', ' ', text)
+ return text.strip()
def text_length(i):
- return len(clean(i.text_content() or ""))
+ return len(clean(i.text_content() or ""))
class Unparseable(ValueError):
- pass
+ pass
class Document:
- TEXT_LENGTH_THRESHOLD = 25
- RETRY_LENGTH = 250
-
- def __init__(self, input, **options):
- self.input = input
- self.options = defaultdict(lambda: None)
- for k, v in options.items():
- self.options[k] = v
- self.html = None
-
- def _html(self, force=False):
- if force or self.html is None:
- self.html = self._parse(self.input)
- return self.html
-
- def _parse(self, input):
- doc = build_doc(input)
- doc = html_cleaner.clean_html(doc)
- base_href = self.options['url']
- if base_href:
- doc.make_links_absolute(base_href, resolve_base_href=True)
- else:
- doc.resolve_base_href()
- return doc
-
- def content(self):
- return get_body(self._html(True))
-
- def title(self):
- return get_title(self._html(True))
-
- def short_title(self):
- return shorten_title(self._html(True))
-
- def summary(self):
- try:
- ruthless = True
- while True:
- self._html(True)
-
- for i in self.tags(self.html, 'script', 'style'):
- i.drop_tree()
- for i in self.tags(self.html, 'body'):
- i.set('id', 'readabilityBody')
- if ruthless:
- self.remove_unlikely_candidates()
- self.transform_misused_divs_into_paragraphs()
- candidates = self.score_paragraphs()
-
- best_candidate = self.select_best_candidate(candidates)
- if best_candidate:
- article = self.get_article(candidates, best_candidate)
- else:
- if ruthless:
- logging.debug("ruthless removal did not work. ")
- ruthless = False
- self.debug("ended up stripping too much - going for a safer _parse")
- # try again
- continue
- else:
- logging.debug("Ruthless and lenient parsing did not work. Returning raw html")
- article = self.html.find('body')
- if article is None:
- article = self.html
-
- cleaned_article = self.sanitize(article, candidates)
- of_acceptable_length = len(cleaned_article or '') >= (self.options['retry_length'] or self.RETRY_LENGTH)
- if ruthless and not of_acceptable_length:
- ruthless = False
- continue # try again
- else:
- return cleaned_article
- except StandardError, e:
- #logging.exception('error getting summary: ' + str(traceback.format_exception(*sys.exc_info())))
- logging.exception('error getting summary: ' )
- raise Unparseable(str(e)), None, sys.exc_info()[2]
-
- def get_article(self, candidates, best_candidate):
- # Now that we have the top candidate, look through its siblings for content that might also be related.
- # Things like preambles, content split by ads that we removed, etc.
-
- sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2])
- output = document_fromstring('
s
- #FIXME: The current implementation ignores all descendants that are not direct children of elem
- # This results in incorrect results in case there is an buried within an for example
- if not REGEXES['divToPElementsRe'].search(unicode(''.join(map(tostring, list(elem))))):
- #self.debug("Altering %s to p" % (describe(elem)))
- elem.tag = "p"
- #print "Fixed element "+describe(elem)
-
- for elem in self.tags(self.html, 'div'):
- if elem.text and elem.text.strip():
- p = fragment_fromstring('')
- p.text = elem.text
- elem.text = None
- elem.insert(0, p)
- #print "Appended "+tounicode(p)+" to "+describe(elem)
-
- for pos, child in reversed(list(enumerate(elem))):
- if child.tail and child.tail.strip():
- p = fragment_fromstring('')
- p.text = child.tail
- child.tail = None
- elem.insert(pos + 1, p)
- #print "Inserted "+tounicode(p)+" to "+describe(elem)
- if child.tag == 'br':
- #print 'Dropped
at '+describe(elem)
- child.drop_tree()
-
- def tags(self, node, *tag_names):
- for tag_name in tag_names:
- for e in node.findall('.//%s' % tag_name):
- yield e
-
- def reverse_tags(self, node, *tag_names):
- for tag_name in tag_names:
- for e in reversed(node.findall('.//%s' % tag_name)):
- yield e
-
- def sanitize(self, node, candidates):
- MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD)
- for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"):
- if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33:
- header.drop_tree()
-
- for elem in self.tags(node, "form", "iframe", "textarea"):
- elem.drop_tree()
- allowed = {}
- # Conditionally clean