From 65d1ebb06dcc38ce3be3c551b6293d23afb3e711 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Tue, 29 Sep 2015 18:36:25 +0200 Subject: [PATCH 001/109] Fixed #70 and added xpath option --- readability/readability.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/readability/readability.py b/readability/readability.py index 6138d7f5..2aaac624 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -67,6 +67,8 @@ def text_length(i): def compile_pattern(elements): if not elements: return None + elif isinstance(elements, (list, tuple)): + return list(elements) elif isinstance(elements, regexp_type): return elements else: @@ -78,7 +80,7 @@ class Document: """Class to build a etree document out of html.""" def __init__(self, input, positive_keywords=None, negative_keywords=None, - url=None, min_text_length=25, retry_length=250, ): + url=None, min_text_length=25, retry_length=250, xpath=False): """Generate the document :param input: string of the html content. @@ -99,10 +101,16 @@ def __init__(self, input, positive_keywords=None, negative_keywords=None, self.url = url self.min_text_length = min_text_length self.retry_length = retry_length + self.xpath = xpath def _html(self, force=False): if force or self.html is None: self.html = self._parse(self.input) + if self.xpath: + root = self.html.getroottree() + for i in self.html.getiterator(): + #print root.getpath(i) + i.attrib['x'] = root.getpath(i) return self.html def _parse(self, input): From 5fc2d3684abca8860956a5d1577da7e8c9a41fd8 Mon Sep 17 00:00:00 2001 From: alphapapa Date: Sun, 3 Apr 2016 21:32:36 -0500 Subject: [PATCH 002/109] Use Mozilla User-Agent Use a "Mozilla" user-agent to avoid HTTP 403 errors. Fixes #71. --- readability/readability.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/readability/readability.py b/readability/readability.py index 2aaac624..de19a7c7 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -600,12 +600,15 @@ def main(): file = None if options.url: + headers = {'User-Agent': 'Mozilla/5.0'} if sys.version_info[0] == 3: import urllib.request, urllib.parse, urllib.error - file = urllib.request.urlopen(options.url) + request = urllib.request.Request(options.url, None, headers) + file = urllib.request.urlopen(request) else: import urllib2 - file = urllib2.urlopen(options.url) + request = urllib2.request(options.url, None, headers) + file = urllib2.urlopen(request) else: file = open(args[0], 'rt') try: From 8443a87f5ce930f4bc608c4e5289e640e3cb860b Mon Sep 17 00:00:00 2001 From: alphapapa Date: Sun, 3 Apr 2016 21:38:17 -0500 Subject: [PATCH 003/109] Update readability.py --- readability/readability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readability/readability.py b/readability/readability.py index de19a7c7..d85e7f1b 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -607,7 +607,7 @@ def main(): file = urllib.request.urlopen(request) else: import urllib2 - request = urllib2.request(options.url, None, headers) + request = urllib2.Request(options.url, None, headers) file = urllib2.urlopen(request) else: file = open(args[0], 'rt') From 75f2ea0d00c0fb0f0932752521205042d84e23cf Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Sat, 9 Apr 2016 14:34:00 +0600 Subject: [PATCH 004/109] Version bump to 0.6.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7863b1cb..371dac9b 100755 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( name="readability-lxml", - version="0.6.1", + version="0.6.2", author="Yuri Baburov", author_email="burchik@gmail.com", description="fast html to text parser (article readability tool) with python3 support", From 82837e4b5c92709e63a473bced2a4e6dc8f6ad4d Mon Sep 17 00:00:00 2001 From: Luke Murphy Date: Mon, 11 Jul 2016 16:13:27 +0200 Subject: [PATCH 005/109] makeover for the README [ci skip] --- README | 62 ------------------------------------------------------ README.rst | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 3 files changed, 62 insertions(+), 63 deletions(-) delete mode 100644 README create mode 100644 README.rst diff --git a/README b/README deleted file mode 100644 index e3d40b3d..00000000 --- a/README +++ /dev/null @@ -1,62 +0,0 @@ -This code is under the Apache License 2.0. http://www.apache.org/licenses/LICENSE-2.0 - -This is a python port of a ruby port of arc90's readability project - -http://lab.arc90.com/experiments/readability/ - -In few words, -Given a html document, it pulls out the main body text and cleans it up. -It also can clean up title based on latest readability.js code. - -Based on: - - Latest readability.js ( https://github.com/MHordecki/readability-redux/blob/master/readability/readability.js ) - - Ruby port by starrhorne and iterationlabs - - Python port by gfxmonk ( https://github.com/gfxmonk/python-readability , based on BeautifulSoup ) - - Decruft effort to move to lxml ( http://www.minvolai.com/blog/decruft-arc90s-readability-in-python/ ) - - "BR to P" fix from readability.js which improves quality for smaller texts. - - Github users contributions. - -Installation:: - - easy_install readability-lxml - or - pip install readability-lxml - -Usage:: - - from readability.readability import Document - import urllib - html = urllib.urlopen(url).read() - readable_article = Document(html).summary() - readable_title = Document(html).short_title() - -Command-line usage:: - - python -m readability.readability -u http://pypi.python.org/pypi/readability-lxml - -To open resulting page in browser:: - - python -m readability.readability -b -u http://pypi.python.org/pypi/readability-lxml - -Using positive/negative keywords example:: - - python -m readability.readability -p intro -n newsindex,homepage-box,news-section -u http://python.org - - -Document() kwarg options: - - - attributes: - - debug: output debug messages - - min_text_length: - - retry_length: - - url: will allow adjusting links to be absolute - - positive_keywords: the list of positive search patterns in classes and ids, for example: ["news-item", "block"] - - negative_keywords: the list of negative search patterns in classes and ids, for example: ["mysidebar", "related", "ads"] - - -Updates - - - 0.3 Added Document.encoding, positive_keywords and negative_keywords - - 0.4 Added Videos loading and allowed more images per paragraph - - 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and 3.4 - - 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 and 3.4 diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..a2a1c8c4 --- /dev/null +++ b/README.rst @@ -0,0 +1,61 @@ +python-readability +================== + +Given a html document, it pulls out the main body text and cleans it up. + +This is a python port of a ruby port of `arc90's readability +project `__. + +Installation +------------ + +It's easy using ``pip``, just run: + +:: + + $ pip install readability-lxml + +Usage +----- + +:: + + >> import requests + >> from readability import Document + >> + >> response = requests.get('http://example.com') + >> doc = Document(response.text) + >> doc.title() + >> 'Example Domain' + +Change Log +---------- + +- 0.3 Added Document.encoding, positive\_keywords and + negative\_keywords +- 0.4 Added Videos loading and allowed more images per paragraph +- 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and + 3.4 +- 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 + and 3.4 + +Licensing +========= + +This code is under `the Apache License +2.0 `__ license. + +Thanks to +--------- + +- Latest + `readability.js `__ +- Ruby port by starrhorne and iterationlabs +- `Python port `__ by + gfxmonk +- `Decruft + effort `__ + to move to lxml +- "BR to P" fix from readability.js which improves quality for smaller + texts +- Github users contributions. diff --git a/setup.py b/setup.py index 371dac9b..18a4faeb 100755 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ author_email="burchik@gmail.com", description="fast html to text parser (article readability tool) with python3 support", test_suite = "tests.test_article_only", - long_description=open("README").read(), + long_description=open("README.rst").read(), license="Apache License 2.0", url="http://github.com/buriy/python-readability", packages=['readability', 'readability.compat'], From a1d6bbcd3fa366400395ed12f827561ad87645ce Mon Sep 17 00:00:00 2001 From: Luke Murphy Date: Thu, 14 Jul 2016 22:00:40 +0200 Subject: [PATCH 006/109] add travis file --- .travis.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..9a668090 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,17 @@ +language: python + +python: + - "3.4" + +env: + - TOX_ENV=py26 + - TOX_ENV=py27 + - TOX_ENV=py33 + - TOX_ENV=py34 + +install: + - travis_retry pip install -U pip wheel tox + - travis_retry pip install -U -r requirements.txt -e . + +script: + - tox -e $TOX_ENV From aafcf52e58908f92efa3476849ad8b8a7902f397 Mon Sep 17 00:00:00 2001 From: Luke Murphy Date: Fri, 15 Jul 2016 16:15:06 +0200 Subject: [PATCH 007/109] add travis badge [ci skip] --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index a2a1c8c4..51eac4af 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,7 @@ +.. image:: https://travis-ci.org/buriy/python-readability.svg?branch=master + :target: https://travis-ci.org/buriy/python-readability + + python-readability ================== From b20d5c15ef7c207852f2d198581914c5f52bdd72 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Tue, 19 Jul 2016 17:25:52 +0600 Subject: [PATCH 008/109] Improved Document class documentation --- readability/readability.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/readability/readability.py b/readability/readability.py index d85e7f1b..6120bbe6 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -86,12 +86,24 @@ def __init__(self, input, positive_keywords=None, negative_keywords=None, :param input: string of the html content. :param positive_keywords: regex or list of patterns in classes and ids :param negative_keywords: regex or list of patterns in classes and ids - :param min_text_length: - :param retry_length: + :param min_text_length: Tunable. Set to a higher value for more precise detection of longer texts. + :param retry_length: Tunable. Set to a lower value for better detection of very small texts. + :param xpath: If set to True, adds x="..." attribute to each HTML node, + containing xpath path pointing to original document path (allows to + reconstruct selected summary in original document). Example: positive_keywords=["news-item", "block"] negative_keywords=["mysidebar", "related", "ads"] + + The Document class is not re-enterable. + You need to create a new Document() for each HTML file to process. + + Provides four API methods: + .get_title() + .short_title() + .get_content() + .summary() """ self.input = input self.html = None @@ -131,23 +143,33 @@ def _parse(self, input): return doc def content(self): + """Returns full document body""" return get_body(self._html(True)) def title(self): + """Returns document title""" return get_title(self._html(True)) def short_title(self): + """Returns cleaned up document title""" return shorten_title(self._html(True)) def get_clean_html(self): - return clean_attributes(tounicode(self.html)) + """ + An internal method, which can be overridden in subclasses, for example, + to disable or to improve DOM-to-text conversion in .summary() method + """ + return clean_attributes(tounicode(self.html)) def summary(self, html_partial=False): - """Generate the summary of the html docuemnt + """ + Given a HTML file, extracts the text of the article. :param html_partial: return only the div of the document, don't wrap in html and body tags. + Warning: It mangles internal DOM representation of the HTML document, + so always use other API methods before this one. """ try: ruthless = True @@ -278,7 +300,7 @@ def get_link_density(self, elem): total_length = text_length(elem) return float(link_length) / max(total_length, 1) - def score_paragraphs(self, ): + def score_paragraphs(self): MIN_LEN = self.min_text_length candidates = {} ordered = [] @@ -373,6 +395,7 @@ def score_node(self, elem): } def remove_unlikely_candidates(self): + """Utility method""" for elem in self.html.iter(): s = "%s %s" % (elem.get('class', ''), elem.get('id', '')) if len(s) < 2: @@ -382,6 +405,7 @@ def remove_unlikely_candidates(self): elem.drop_tree() def transform_misused_divs_into_paragraphs(self): + """Utility method""" for elem in self.tags(self.html, 'div'): # transform
s that do not contain other block elements into #

s From e4efc87a206e0e51c8c8a8f5759890c78a20f424 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Tue, 19 Jul 2016 17:30:23 +0600 Subject: [PATCH 009/109] Update readability.py --- readability/readability.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/readability/readability.py b/readability/readability.py index 6120bbe6..8331e279 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -97,13 +97,13 @@ def __init__(self, input, positive_keywords=None, negative_keywords=None, negative_keywords=["mysidebar", "related", "ads"] The Document class is not re-enterable. - You need to create a new Document() for each HTML file to process. + It is designed to create a new Document() for each HTML file to process it. - Provides four API methods: - .get_title() - .short_title() - .get_content() - .summary() + API methods: + .title() -- full title + .short_title() -- cleaned up title + .content() -- full content + .summary() -- cleaned up content """ self.input = input self.html = None @@ -143,7 +143,7 @@ def _parse(self, input): return doc def content(self): - """Returns full document body""" + """Returns document body""" return get_body(self._html(True)) def title(self): @@ -168,8 +168,8 @@ def summary(self, html_partial=False): :param html_partial: return only the div of the document, don't wrap in html and body tags. - Warning: It mangles internal DOM representation of the HTML document, - so always use other API methods before this one. + Warning: It mutates internal DOM representation of the HTML document, + so it is better to call other API methods before this one. """ try: ruthless = True @@ -395,7 +395,6 @@ def score_node(self, elem): } def remove_unlikely_candidates(self): - """Utility method""" for elem in self.html.iter(): s = "%s %s" % (elem.get('class', ''), elem.get('id', '')) if len(s) < 2: @@ -405,7 +404,6 @@ def remove_unlikely_candidates(self): elem.drop_tree() def transform_misused_divs_into_paragraphs(self): - """Utility method""" for elem in self.tags(self.html, 'div'): # transform

s that do not contain other block elements into #

s From 9a31587192ebe814badb8a42824704949c870009 Mon Sep 17 00:00:00 2001 From: Chris Curvey Date: Wed, 8 Feb 2017 18:00:11 -0500 Subject: [PATCH 010/109] fix encoding detection to use the encoding being tested --- readability/encoding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readability/encoding.py b/readability/encoding.py index e68f2988..cc14320d 100644 --- a/readability/encoding.py +++ b/readability/encoding.py @@ -43,7 +43,7 @@ def get_encoding(page): encoding = fix_charset(declared_encoding) # Now let's decode the page - page.decode() + page.decode(encoding) # It worked! return encoding except UnicodeDecodeError: From 27159f45b36ab180710b66e617b264a756952526 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 26 Nov 2017 22:14:25 +0200 Subject: [PATCH 011/109] Drop support for EOL Python 2.6 --- .travis.yml | 1 - setup.py | 1 - tox.ini | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9a668090..1e797655 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ python: - "3.4" env: - - TOX_ENV=py26 - TOX_ENV=py27 - TOX_ENV=py33 - TOX_ENV=py34 diff --git a/setup.py b/setup.py index 18a4faeb..0f032ded 100755 --- a/setup.py +++ b/setup.py @@ -38,7 +38,6 @@ "Topic :: Software Development :: Libraries :: Python Modules", "Programming Language :: Python", "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", diff --git a/tox.ini b/tox.ini index 50b4a74d..c0206c08 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py26, py27, py33, py34 +envlist = py27, py33, py34 [testenv] deps=pytest From f74adc6893ad203919337d6fe4ce58a9df034d13 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 26 Nov 2017 22:20:57 +0200 Subject: [PATCH 012/109] Drop support for EOL Python 3.3 --- .travis.yml | 1 - setup.py | 1 - tox.ini | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1e797655..e34557af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ python: env: - TOX_ENV=py27 - - TOX_ENV=py33 - TOX_ENV=py34 install: diff --git a/setup.py b/setup.py index 0f032ded..bcc62096 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,6 @@ "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", ], diff --git a/tox.ini b/tox.ini index c0206c08..7388ac46 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py33, py34 +envlist = py27, py34 [testenv] deps=pytest From 4172699812cd2a80f48a46fff994344e220b0be8 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 26 Nov 2017 22:23:31 +0200 Subject: [PATCH 013/109] Add Python 3.5 and 3.6 --- .travis.yml | 4 +++- setup.py | 2 ++ tox.ini | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e34557af..5bdf1d4f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,13 @@ language: python python: - - "3.4" + - "3.6" env: - TOX_ENV=py27 - TOX_ENV=py34 + - TOX_ENV=py35 + - TOX_ENV=py36 install: - travis_retry pip install -U pip wheel tox diff --git a/setup.py b/setup.py index bcc62096..c3d73626 100755 --- a/setup.py +++ b/setup.py @@ -41,6 +41,8 @@ "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", ], ) diff --git a/tox.ini b/tox.ini index 7388ac46..b29c95ed 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py34 +envlist = py27, py34, py35, py36 [testenv] deps=pytest From f4a04732fd2a3519ec4a3c1f66e669a20f1a2275 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 26 Nov 2017 22:26:41 +0200 Subject: [PATCH 014/109] Workaround for py35 --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5bdf1d4f..b0ebc5ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,10 @@ env: - TOX_ENV=py35 - TOX_ENV=py36 +before_install: + # work around https://github.com/travis-ci/travis-ci/issues/8363 + - pyenv global system 3.5 + install: - travis_retry pip install -U pip wheel tox - travis_retry pip install -U -r requirements.txt -e . From 537de2b8f6ac79b4a1d9f4d74855b0a186c021db Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Mon, 7 May 2018 12:28:25 +0700 Subject: [PATCH 015/109] Improved remove_unlikely_candidates following an advice from issue #102 --- readability/readability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readability/readability.py b/readability/readability.py index 8331e279..90fbc138 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -395,7 +395,7 @@ def score_node(self, elem): } def remove_unlikely_candidates(self): - for elem in self.html.iter(): + for elem in self.html.findall('.//*'): s = "%s %s" % (elem.get('class', ''), elem.get('id', '')) if len(s) < 2: continue From 0e50b53d056359fa1020c9279e4f876be90cb484 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Mon, 7 May 2018 12:31:39 +0700 Subject: [PATCH 016/109] Release version 0.7 . Better HTML5 support and an important bugfix. --- .travis.yml | 2 +- Makefile | 22 +++++++++++----------- README.rst | 12 +++++++----- readability/readability.py | 10 +++++----- setup.py | 3 +-- tests/test_article_only.py | 31 +++++++++++++++++++++++++++++++ 6 files changed, 56 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index b0ebc5ed..b542c481 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: before_install: # work around https://github.com/travis-ci/travis-ci/issues/8363 - - pyenv global system 3.5 + - pyenv global system 3.6 install: - travis_retry pip install -U pip wheel tox diff --git a/Makefile b/Makefile index 0a28f375..3daf2d1d 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,9 @@ # Makefile to help automate tasks WD := $(shell pwd) -PY := .env/bin/python -PIP := .env/bin/pip -PEP8 := .env/bin/pep8 -NOSE := .env/bin/nosetests - +PY := .venv/bin/python +PIP := .venv/bin/pip +PEP8 := .venv/bin/pep8 +NOSE := .venv/bin/nosetests # ########### # Tests rule! @@ -22,16 +21,17 @@ $(NOSE): .PHONY: all all: venv develop -venv: bin/python -bin/python: - virtualenv .env +venv: .venv/bin/python + +.venv/bin/python: + virtualenv .venv .PHONY: clean_venv clean_venv: - rm -rf .env + rm -rf .venv -develop: .env/lib/python*/site-packages/readability-lxml.egg-link -.env/lib/python*/site-packages/readability-lxml.egg-link: +develop: .venv/lib/python*/site-packages/readability-lxml.egg-link +.venv/lib/python*/site-packages/readability-lxml.egg-link: $(PY) setup.py develop diff --git a/README.rst b/README.rst index 51eac4af..518c7553 100644 --- a/README.rst +++ b/README.rst @@ -35,13 +35,15 @@ Usage Change Log ---------- -- 0.3 Added Document.encoding, positive\_keywords and - negative\_keywords -- 0.4 Added Videos loading and allowed more images per paragraph -- 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and - 3.4 +- 0.7 Improved HTML5 tags handling. Heuristics were changed for a lot of sites: Fixed an important +bug with stripping unwanted HTML nodes (only first matching node was removed before). - 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 and 3.4 +- 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and + 3.4 +- 0.4 Added Videos loading and allowed more images per paragraph +- 0.3 Added Document.encoding, positive\_keywords and + negative\_keywords Licensing ========= diff --git a/readability/readability.py b/readability/readability.py index 90fbc138..12f3d959 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -381,13 +381,13 @@ def class_weight(self, e): def score_node(self, elem): content_score = self.class_weight(elem) name = elem.tag.lower() - if name == "div": + if name in ["div", "article"]: content_score += 5 elif name in ["pre", "td", "blockquote"]: content_score += 3 - elif name in ["address", "ol", "ul", "dl", "dd", "dt", "li", "form"]: + elif name in ["address", "ol", "ul", "dl", "dd", "dt", "li", "form", "aside"]: content_score -= 3 - elif name in ["h1", "h2", "h3", "h4", "h5", "h6", "th"]: + elif name in ["h1", "h2", "h3", "h4", "h5", "h6", "th", "header", "footer", "nav"]: content_score -= 5 return { 'content_score': content_score, @@ -463,7 +463,7 @@ def sanitize(self, node, candidates): allowed = {} # Conditionally clean s,
    s, and
    s - for el in self.reverse_tags(node, "table", "ul", "div"): + for el in self.reverse_tags(node, "table", "ul", "div", "aside", "header", "footer", "section"): if el in allowed: continue weight = self.class_weight(el) @@ -577,7 +577,7 @@ def sanitize(self, node, candidates): if siblings and sum(siblings) > 1000: to_remove = False log.debug("Allowing %s" % describe(el)) - for desnode in self.tags(el, "table", "ul", "div"): + for desnode in self.tags(el, "table", "ul", "div", "section"): allowed[desnode] = True if to_remove: diff --git a/setup.py b/setup.py index c3d73626..09744b85 100755 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( name="readability-lxml", - version="0.6.2", + version="0.7", author="Yuri Baburov", author_email="burchik@gmail.com", description="fast html to text parser (article readability tool) with python3 support", @@ -43,6 +43,5 @@ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", - ], ) diff --git a/tests/test_article_only.py b/tests/test_article_only.py index 882d346d..87e623c7 100644 --- a/tests/test_article_only.py +++ b/tests/test_article_only.py @@ -61,3 +61,34 @@ def test_best_elem_is_root_and_passing(self): ) doc = Document(sample) doc.summary() + + def test_correct_cleanup(self): + sample = """ + + +
    test section
    +
    +

    Lot of text here.

    + +

    More text is written here, and contains punctuation and dots.

    +
    +
s,