From 747c46abce4cf4307cde626e628c47e8eb262cd3 Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Wed, 26 Sep 2018 08:26:08 +0300 Subject: [PATCH 01/90] Trim many repeated spaces to make clean() faster When Readability encounters many repeated whitespace, the cleanup regexes in clean() take forever to run, so trim the amount of whitespace to 255 characters. Additionally, test the extracting performance with "timeout_decorator". --- readability/readability.py | 3 +++ setup.py | 4 ++++ tests/test_article_only.py | 12 ++++++++++++ 3 files changed, 19 insertions(+) diff --git a/readability/readability.py b/readability/readability.py index 54874ac8..91f8a941 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -54,6 +54,9 @@ def to_int(x): def clean(text): + # Many spaces make the following regexes run forever + text = re.sub(r'\s{255,}', ' ' * 255, text) + text = re.sub('\s*\n\s*', '\n', text) text = re.sub('\t|[ \t]{2,}', ' ', text) return text.strip() diff --git a/setup.py b/setup.py index 09744b85..9672aff3 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,10 @@ lxml_requirement, "cssselect" ], + tests_require=[ + # Test timeouts + "timeout_decorator", + ], classifiers=[ "Environment :: Web Environment", "Intended Audience :: Developers", diff --git a/tests/test_article_only.py b/tests/test_article_only.py index 87e623c7..e545025c 100644 --- a/tests/test_article_only.py +++ b/tests/test_article_only.py @@ -2,6 +2,7 @@ import unittest from readability import Document +import timeout_decorator SAMPLES = os.path.join(os.path.dirname(__file__), 'samples') @@ -92,3 +93,14 @@ def test_correct_cleanup(self): assert('punctuation' in s) assert(not 'comment' in s) assert(not 'aside' in s) + + # Many spaces make some regexes run forever + @timeout_decorator.timeout(seconds=3, use_signals=False) + def test_many_repeated_spaces(self): + long_space = ' ' * 1000000 + sample = '

foo' + long_space + '

' + + doc = Document(sample) + s = doc.summary() + + assert 'foo' in s From 2bbb70b3e5957ff9d224aa84493b560e0efd802c Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Wed, 26 Sep 2018 08:42:11 +0300 Subject: [PATCH 02/90] Fix Travis build Add "test" extra and install dependencies for said extra as detailed in: https://stackoverflow.com/a/41398850/200603 --- .travis.yml | 2 +- setup.py | 14 ++++++++++---- tox.ini | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8cbcf71d..bd0fd94c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_install: install: - travis_retry pip install -U pip wheel tox - - travis_retry pip install -U -r requirements.txt -e . + - travis_retry pip install -U -r requirements.txt -e ".[test]" script: - tox -e $TOX_ENV diff --git a/setup.py b/setup.py index 9672aff3..e845c340 100755 --- a/setup.py +++ b/setup.py @@ -12,6 +12,14 @@ print("Using lxml<2.4") lxml_requirement = "lxml<2.4" +test_deps = [ + # Test timeouts + "timeout_decorator", +] +extras = { + 'test': test_deps, +} + setup( name="readability-lxml", version="0.7", @@ -28,10 +36,8 @@ lxml_requirement, "cssselect" ], - tests_require=[ - # Test timeouts - "timeout_decorator", - ], + tests_require=test_deps, + extras_require=extras, classifiers=[ "Environment :: Web Environment", "Intended Audience :: Developers", diff --git a/tox.ini b/tox.ini index 89239db6..9296cc4b 100644 --- a/tox.ini +++ b/tox.ini @@ -16,5 +16,5 @@ deps=pytest # $PYTHONDIR\Scripts\pip.exe install *.whl sitepackages=True commands = - pip install -r requirements.txt + pip install -r requirements.txt -e ".[test]" py.test From 34d198fe5a36e788d5383ca2443aa60b5e7ca2ed Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Wed, 26 Sep 2018 08:45:34 +0300 Subject: [PATCH 03/90] Add Python 3.7 classifier --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 09744b85..9d68b6e7 100755 --- a/setup.py +++ b/setup.py @@ -43,5 +43,6 @@ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", ], ) From bdb6d671d855b5f3a018a6b88be924805370afed Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Wed, 26 Sep 2018 08:45:57 +0300 Subject: [PATCH 04/90] Test with Python 3.7 on Travis --- .travis.yml | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8cbcf71d..c8cd5d86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ env: - TOX_ENV=py34 # - TOX_ENV=py35 - TOX_ENV=py36 + - TOX_ENV=py37 before_install: # work around https://github.com/travis-ci/travis-ci/issues/8363 diff --git a/tox.ini b/tox.ini index 89239db6..e0d944f2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py35, py36 +envlist = py27, py35, py36, py37 [testenv] deps=pytest From 63fbc36cb83b10e9ae30d33fcbe23a3cdb4a5469 Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Wed, 26 Sep 2018 08:48:03 +0300 Subject: [PATCH 05/90] Close sample input file after reading it Otherwise tests spit out: ResourceWarning: unclosed file <_io.TextIOWrapper name='/Users/pypt/Dropbox/etc-MediaCloud/python-readability/tests/samples/si-game.sample.html' mode='r' encoding='UTF-8'> return open(os.path.join(SAMPLES, filename)).read() --- tests/test_article_only.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_article_only.py b/tests/test_article_only.py index 87e623c7..5d5b8133 100644 --- a/tests/test_article_only.py +++ b/tests/test_article_only.py @@ -9,7 +9,9 @@ def load_sample(filename): """Helper to get the content out of the sample files""" - return open(os.path.join(SAMPLES, filename)).read() + with open(os.path.join(SAMPLES, filename)) as f: + html = f.read() + return html class TestArticleOnly(unittest.TestCase): From 0233936e72a5554060a57f3947d12fe5b4c7639c Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Wed, 26 Sep 2018 09:04:12 +0300 Subject: [PATCH 06/90] Add __version__ constant to __init__.py, read it in setup.py Users wouldn't need to install, import and use Pip ("pkg_resources") to find out which version of readability-lxml is being used. --- readability/__init__.py | 2 ++ setup.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/readability/__init__.py b/readability/__init__.py index 8822a512..e00a50a9 100644 --- a/readability/__init__.py +++ b/readability/__init__.py @@ -1 +1,3 @@ +__version__ = "0.7" + from .readability import Document diff --git a/setup.py b/setup.py index 09744b85..c1017d12 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,10 @@ #!/usr/bin/env python + from __future__ import print_function -from setuptools import setup, find_packages +import codecs +import os +import re +from setuptools import setup import sys lxml_requirement = "lxml" @@ -12,9 +16,29 @@ print("Using lxml<2.4") lxml_requirement = "lxml<2.4" + +# Adapted from https://github.com/pypa/pip/blob/master/setup.py +def find_version(*file_paths): + here = os.path.abspath(os.path.dirname(__file__)) + + # Intentionally *not* adding an encoding option to open, See: + # https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690 + with codecs.open(os.path.join(here, *file_paths), 'r') as fp: + version_file = fp.read() + version_match = re.search( + r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, + re.M, + ) + if version_match: + return version_match.group(1) + + raise RuntimeError("Unable to find version string.") + + setup( name="readability-lxml", - version="0.7", + version=find_version("readability", "__init__.py"), author="Yuri Baburov", author_email="burchik@gmail.com", description="fast html to text parser (article readability tool) with python3 support", From 34fce7664d3cddfa3414356823b37a85367c63f9 Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Wed, 26 Sep 2018 09:11:54 +0300 Subject: [PATCH 07/90] Update Python version in .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c8cd5d86..96698594 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - - "3.6" + - "3.7" env: - TOX_ENV=py27 From 68fb5ad4c09e6c9f5c932b5629f9459099a3ebdb Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Wed, 26 Sep 2018 09:16:28 +0300 Subject: [PATCH 08/90] Try a workaround to make build work on 3.7 https://github.com/travis-ci/travis-ci/issues/9815 --- .travis.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 96698594..a16e41b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,22 @@ language: python python: - - "3.7" + - "3.6" + +# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs +matrix: + include: + - name: "Python: 3.7" + python: "3.7" + dist: xenial + sudo: true + env: TOX_ENV=py37 env: - TOX_ENV=py27 - TOX_ENV=py34 # - TOX_ENV=py35 - TOX_ENV=py36 - - TOX_ENV=py37 before_install: # work around https://github.com/travis-ci/travis-ci/issues/8363 From 0b28643f0dea6df799f9734e45e402eb94ea2a07 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Fri, 5 Oct 2018 01:43:28 +0700 Subject: [PATCH 09/90] Update README.rst --- README.rst | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/README.rst b/README.rst index 518c7553..a6ca6f5f 100644 --- a/README.rst +++ b/README.rst @@ -31,19 +31,21 @@ Usage >> doc = Document(response.text) >> doc.title() >> 'Example Domain' + >> doc.summary() + >> u'
\n
\n

Example Domain

\n +

This domain is established to be used for illustrative examples in documents. You may + use this\n domain in examples without prior coordination or asking for permission.

+ \n

More information...

\n
+ \n\n
' Change Log ---------- -- 0.7 Improved HTML5 tags handling. Heuristics were changed for a lot of sites: Fixed an important -bug with stripping unwanted HTML nodes (only first matching node was removed before). -- 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 - and 3.4 -- 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and - 3.4 +- 0.7 Improved HTML5 tags handling. Fixed stripping unwanted HTML nodes (only first matching node was removed before). +- 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 and 3.4 +- 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and 3.4 - 0.4 Added Videos loading and allowed more images per paragraph -- 0.3 Added Document.encoding, positive\_keywords and - negative\_keywords +- 0.3 Added Document.encoding, positive\_keywords and negative\_keywords Licensing ========= @@ -54,14 +56,9 @@ This code is under `the Apache License Thanks to --------- -- Latest - `readability.js `__ +- Latest `readability.js `__ - Ruby port by starrhorne and iterationlabs -- `Python port `__ by - gfxmonk -- `Decruft - effort `__ - to move to lxml -- "BR to P" fix from readability.js which improves quality for smaller - texts +- `Python port `__ by gfxmonk +- `Decruft effort ` to move to lxml +- "BR to P" fix from readability.js which improves quality for smaller texts - Github users contributions. From 9aba330e68ce07e9aefb11c0abfd8798d722daab Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Fri, 5 Oct 2018 01:44:32 +0700 Subject: [PATCH 10/90] Update README.rst --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a6ca6f5f..83a50fac 100644 --- a/README.rst +++ b/README.rst @@ -41,8 +41,9 @@ Usage Change Log ---------- +- 0.7.1 Support for Python 3.7 - 0.7 Improved HTML5 tags handling. Fixed stripping unwanted HTML nodes (only first matching node was removed before). -- 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 and 3.4 +- 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 - 3.6 - 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and 3.4 - 0.4 Added Videos loading and allowed more images per paragraph - 0.3 Added Document.encoding, positive\_keywords and negative\_keywords From d40c4dd34d8d2d7caa5ea51db20ef44e5c4a8306 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Fri, 5 Oct 2018 01:48:21 +0700 Subject: [PATCH 11/90] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 83a50fac..3c1a85fb 100644 --- a/README.rst +++ b/README.rst @@ -41,7 +41,7 @@ Usage Change Log ---------- -- 0.7.1 Support for Python 3.7 +- 0.7.1 Support for Python 3.7 . Fixed a speed regression when processing of documents with lots of spaces. - 0.7 Improved HTML5 tags handling. Fixed stripping unwanted HTML nodes (only first matching node was removed before). - 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 - 3.6 - 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and 3.4 From 3cbede6be469ceb8160306ff22e01c3974a2c0e0 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Tue, 27 Nov 2018 16:51:07 +0700 Subject: [PATCH 12/90] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3c1a85fb..b71af845 100644 --- a/README.rst +++ b/README.rst @@ -41,7 +41,7 @@ Usage Change Log ---------- -- 0.7.1 Support for Python 3.7 . Fixed a speed regression when processing of documents with lots of spaces. +- 0.7.1 Support for Python 3.7 . Fixed a slowdown when processing documents with lots of spaces. - 0.7 Improved HTML5 tags handling. Fixed stripping unwanted HTML nodes (only first matching node was removed before). - 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 - 3.6 - 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and 3.4 From bac691a0a46a4d014b139aa3a9aba7019d170180 Mon Sep 17 00:00:00 2001 From: jkclee Date: Wed, 27 Mar 2019 11:35:46 +0800 Subject: [PATCH 13/90] Fix #99 --- readability/readability.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/readability/readability.py b/readability/readability.py index 91f8a941..ddd42c32 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -86,7 +86,8 @@ class Document: """Class to build a etree document out of html.""" def __init__(self, input, positive_keywords=None, negative_keywords=None, - url=None, min_text_length=25, retry_length=250, xpath=False): + url=None, min_text_length=25, retry_length=250, xpath=False, + handle_failures='discard'): """Generate the document :param input: string of the html content. @@ -97,6 +98,8 @@ def __init__(self, input, positive_keywords=None, negative_keywords=None, :param xpath: If set to True, adds x="..." attribute to each HTML node, containing xpath path pointing to original document path (allows to reconstruct selected summary in original document). + :param handle_failures: Parameter passed to `lxml` for handling failure during exception. + Support options = ["discard", "ignore", None] Examples: positive_keywords=["news-item", "block"] @@ -122,6 +125,7 @@ def __init__(self, input, positive_keywords=None, negative_keywords=None, self.min_text_length = min_text_length self.retry_length = retry_length self.xpath = xpath + self.handle_failures = handle_failures def _html(self, force=False): if force or self.html is None: @@ -141,13 +145,13 @@ def _parse(self, input): # trying to guard against bad links like try: # such support is added in lxml 3.3.0 - doc.make_links_absolute(base_href, resolve_base_href=True, handle_failures='discard') + doc.make_links_absolute(base_href, resolve_base_href=True, handle_failures=self.handle_failures) except TypeError: #make_links_absolute() got an unexpected keyword argument 'handle_failures' # then we have lxml < 3.3.0 # please upgrade to lxml >= 3.3.0 if you're failing here! - doc.make_links_absolute(base_href, resolve_base_href=True) + doc.make_links_absolute(base_href, resolve_base_href=True, handle_failures=self.handle_failures) else: - doc.resolve_base_href() + doc.resolve_base_href(handle_failures=self.handle_failures) return doc def content(self): From 0ac3c5bbc6ac7f11df51e973df3dd08d09209260 Mon Sep 17 00:00:00 2001 From: baby5 Date: Tue, 30 Apr 2019 21:41:45 +0800 Subject: [PATCH 14/90] Fix compile_pattern not support uppercase --- readability/readability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readability/readability.py b/readability/readability.py index ddd42c32..d7caa5ba 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -77,7 +77,7 @@ def compile_pattern(elements): elements = str_(elements, 'utf-8') elements = elements.split(u',') if isinstance(elements, (list, tuple)): - return re.compile(u'|'.join([re.escape(x.strip().lower()) for x in elements]), re.U) + return re.compile(u'|'.join([re.escape(x.strip()) for x in elements]), re.U) else: raise Exception("Unknown type for the pattern: {}".format(type(elements))) # assume string or string like object From de20908e571bca0d5194cfdb95a7aa20d4c4555f Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Tue, 14 May 2019 01:53:42 +0700 Subject: [PATCH 15/90] Update README.rst --- README.rst | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index b71af845..e85c786f 100644 --- a/README.rst +++ b/README.rst @@ -24,15 +24,16 @@ Usage :: - >> import requests - >> from readability import Document - >> - >> response = requests.get('http://example.com') - >> doc = Document(response.text) - >> doc.title() - >> 'Example Domain' - >> doc.summary() - >> u'
\n
\n

Example Domain

\n + >>> import requests + >>> from readability import Document + + >>> response = requests.get('http://example.com') + >>> doc = Document(response.text) + >>> doc.title() + 'Example Domain' + + >>> doc.summary() + u'
\n From 04423589420a563cc0244e2b5a282c279e48c47e Mon Sep 17 00:00:00 2001 From: Dario <4755+dariobig@users.noreply.github.com> Date: Sat, 16 Nov 2019 19:44:34 -0500 Subject: [PATCH 16/90] Catch LookupError in case of bad encoding string I've seen cases where bad encoding strings will result in errors, catching LookupError should solve the problem by falling back onto `chardet` or `utf-8` Here's one case: ``` textPayload: "Traceback (most recent call last): File "/opt/conda/lib/python3.7/site-packages/readability/readability.py", line 189, in summary self._html(True) File "/opt/conda/lib/python3.7/site-packages/readability/readability.py", line 132, in _html self.html = self._parse(self.input) File "/opt/conda/lib/python3.7/site-packages/readability/readability.py", line 141, in _parse doc, self.encoding = build_doc(input) File "/opt/conda/lib/python3.7/site-packages/readability/htmls.py", line 17, in build_doc encoding = get_encoding(page) or 'utf-8' File "/opt/conda/lib/python3.7/site-packages/readability/encoding.py", line 46, in get_encoding page.decode(encoding) LookupError: unknown encoding: utf-8, ie=edge, chrome=1 ``` --- readability/encoding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readability/encoding.py b/readability/encoding.py index cc14320d..ebacded6 100644 --- a/readability/encoding.py +++ b/readability/encoding.py @@ -46,7 +46,7 @@ def get_encoding(page): page.decode(encoding) # It worked! return encoding - except UnicodeDecodeError: + except (UnicodeDecodeError, LookupError): pass # Fallback to chardet if declared encodings fail From 326fb43b4caede42458f02987e13e8df777cd0d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89loi=20Rivard?= Date: Sun, 29 Dec 2019 15:34:21 +0100 Subject: [PATCH 17/90] Drop support for python 3.4 - Add support for python 3.8 --- .travis.yml | 29 +++++++---------------------- setup.py | 2 +- tox.ini | 2 +- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index ceccd031..2f32ecd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,30 +1,15 @@ language: python python: - - "3.6" - -# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs -matrix: - include: - - name: "Python: 3.7" - python: "3.7" - dist: xenial - sudo: true - env: TOX_ENV=py37 - -env: - - TOX_ENV=py27 - - TOX_ENV=py34 -# - TOX_ENV=py35 - - TOX_ENV=py36 - -before_install: - # work around https://github.com/travis-ci/travis-ci/issues/8363 - - pyenv global system $TRAVIS_PYTHON_VERSION + - 2.7 + - 3.5 + - 3.6 + - 3.7 + - 3.8 install: - - travis_retry pip install -U pip wheel tox + - travis_retry pip install -U pip wheel tox-travis - travis_retry pip install -U -r requirements.txt -e ".[test]" script: - - tox -e $TOX_ENV + - tox diff --git a/setup.py b/setup.py index f58dd6bb..63465c5f 100755 --- a/setup.py +++ b/setup.py @@ -73,9 +73,9 @@ def find_version(*file_paths): "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], ) diff --git a/tox.ini b/tox.ini index d3c2a7fe..f4abc6f1 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py35, py36, py37 +envlist = py27, py35, py36, py37, py38 [testenv] deps=pytest From 6c1c6391e2836c4d65c056a1b54a6ce99d2643b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89loi=20Rivard?= Date: Sun, 29 Dec 2019 15:59:33 +0100 Subject: [PATCH 18/90] Fixed a few regex warnings --- readability/debug.py | 2 +- readability/encoding.py | 2 +- readability/readability.py | 28 ++++++++++++++-------------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/readability/debug.py b/readability/debug.py index f14f6827..061014d2 100644 --- a/readability/debug.py +++ b/readability/debug.py @@ -41,7 +41,7 @@ def describe(node, depth=1): return parent + describe_node(node) -RE_COLLAPSE_WHITESPACES = re.compile('\s+', re.U) +RE_COLLAPSE_WHITESPACES = re.compile(r'\s+', re.U) def text_content(elem, length=40): diff --git a/readability/encoding.py b/readability/encoding.py index cc14320d..ff50182d 100644 --- a/readability/encoding.py +++ b/readability/encoding.py @@ -51,7 +51,7 @@ def get_encoding(page): # Fallback to chardet if declared encodings fail # Remove all HTML tags, and leave only text for chardet - text = re.sub(b'(\s*]*>)+\s*', b' ', page).strip() + text = re.sub(br'(\s*]*>)+\s*', b' ', page).strip() enc = 'utf-8' if len(text) < 10: return enc # can't guess diff --git a/readability/readability.py b/readability/readability.py index d7caa5ba..1c840055 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -23,17 +23,17 @@ log = logging.getLogger("readability.readability") REGEXES = { - 'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter', re.I), - 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow', re.I), - 'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story', re.I), - 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget', re.I), - 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)', re.I), - #'replaceBrsRe': re.compile('(]*>[ \n\r\t]*){2,}',re.I), - #'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I), - #'trimRe': re.compile('^\s+|\s+$/'), - #'normalizeRe': re.compile('\s{2,}/'), - #'killBreaksRe': re.compile('((\s| ?)*){1,}/'), - 'videoRe': re.compile('https?:\/\/(www\.)?(youtube|vimeo)\.com', re.I), + 'unlikelyCandidatesRe': re.compile(r'combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter', re.I), + 'okMaybeItsACandidateRe': re.compile(r'and|article|body|column|main|shadow', re.I), + 'positiveRe': re.compile(r'article|body|content|entry|hentry|main|page|pagination|post|text|blog|story', re.I), + 'negativeRe': re.compile(r'combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget', re.I), + 'divToPElementsRe': re.compile(r'<(a|blockquote|dl|div|img|ol|p|pre|table|ul)', re.I), + #'replaceBrsRe': re.compile(r'(]*>[ \n\r\t]*){2,}',re.I), + #'replaceFontsRe': re.compile(r'<(\/?)font[^>]*>',re.I), + #'trimRe': re.compile(r'^\s+|\s+$/'), + #'normalizeRe': re.compile(r'\s{2,}/'), + #'killBreaksRe': re.compile(r'((\s| ?)*){1,}/'), + 'videoRe': re.compile(r'https?:\/\/(www\.)?(youtube|vimeo)\.com', re.I), #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, } @@ -57,8 +57,8 @@ def clean(text): # Many spaces make the following regexes run forever text = re.sub(r'\s{255,}', ' ' * 255, text) - text = re.sub('\s*\n\s*', '\n', text) - text = re.sub('\t|[ \t]{2,}', ' ', text) + text = re.sub(r'\s*\n\s*', '\n', text) + text = re.sub(r'\t|[ \t]{2,}', ' ', text) return text.strip() @@ -271,7 +271,7 @@ def get_article(self, candidates, best_candidate, html_partial=False): append = True elif node_length <= 80 \ and link_density == 0 \ - and re.search('\.( |$)', node_content): + and re.search(r'\.( |$)', node_content): append = True if append: From 0846955dd7d7967a0cd36ecf3e4714a14ed649f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89loi=20Rivard?= Date: Sun, 29 Dec 2019 16:23:55 +0100 Subject: [PATCH 19/90] Fixed issue with self-closing tags. Fix #125 --- readability/readability.py | 2 +- tests/test_article_only.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/readability/readability.py b/readability/readability.py index d7caa5ba..645a7a9d 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -171,7 +171,7 @@ def get_clean_html(self): An internal method, which can be overridden in subclasses, for example, to disable or to improve DOM-to-text conversion in .summary() method """ - return clean_attributes(tounicode(self.html)) + return clean_attributes(tounicode(self.html, method='html')) def summary(self, html_partial=False): """ diff --git a/tests/test_article_only.py b/tests/test_article_only.py index f6595d9d..120aee26 100644 --- a/tests/test_article_only.py +++ b/tests/test_article_only.py @@ -106,3 +106,8 @@ def test_many_repeated_spaces(self): s = doc.summary() assert 'foo' in s + + def test_not_self_closing(self): + sample = '

foobar

' + doc = Document(sample) + assert '

foobar

' == doc.summary() From f9977b727d8a0103b02b316de615c3228cf3f6d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89loi=20Rivard?= Date: Sun, 29 Dec 2019 19:12:42 +0100 Subject: [PATCH 20/90] Documentation draft --- README.rst | 2 +- doc/__init__.py | 0 doc/source/__init__.py | 0 doc/source/api.rst | 30 +++++++ doc/source/conf.py | 164 +++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 13 +++ readability/readability.py | 2 +- tox.ini | 13 ++- 8 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 doc/__init__.py create mode 100644 doc/source/__init__.py create mode 100644 doc/source/api.rst create mode 100644 doc/source/conf.py create mode 100644 doc/source/index.rst diff --git a/README.rst b/README.rst index e85c786f..a060f619 100644 --- a/README.rst +++ b/README.rst @@ -50,7 +50,7 @@ Change Log - 0.3 Added Document.encoding, positive\_keywords and negative\_keywords Licensing -========= +-------- This code is under `the Apache License 2.0 `__ license. diff --git a/doc/__init__.py b/doc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/doc/source/__init__.py b/doc/source/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/doc/source/api.rst b/doc/source/api.rst new file mode 100644 index 00000000..b0e3bbbb --- /dev/null +++ b/doc/source/api.rst @@ -0,0 +1,30 @@ +Reference +========= + +.. automodule:: readability + :members: + :show-inheritance: + +.. automodule:: readability.browser + :members: + :show-inheritance: + +.. automodule:: readability.cleaners + :members: + :show-inheritance: + +.. automodule:: readability.debug + :members: + :show-inheritance: + +.. automodule:: readability.encoding + :members: + :show-inheritance: + +.. automodule:: readability.htmls + :members: + :show-inheritance: + +.. automodule:: readability.readability + :members: + :show-inheritance: diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 00000000..bb261349 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# readability documentation build configuration file, created by +# sphinx-quickstart on Thu Mar 23 16:29:38 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath("../..")) + +import readability + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "recommonmark", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = [".rst", ".md"] + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "readability" +copyright = "2020, Yuri Baburov" +author = "Yuri Baburov" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. + +# The short X.Y version. +version = readability.__version__ + +# The full version, including alpha/beta/rc tags. +release = readability.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [] #'_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "readabilitydoc" + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [(master_doc, "readability.tex", "Readability Documentation", "Yuri Baburov", "manual")] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "readability", "readability Documentation", [author], 1)] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "readability", + "Readability Documentation", + author, + "readability", + "One line description of project.", + "Miscellaneous", + ) +] + + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), +} diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 00000000..e3bce61d --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,13 @@ +.. include:: ../../README.rst + +.. toctree:: + :maxdepth: 2 + + api + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/readability/readability.py b/readability/readability.py index d7caa5ba..f5595973 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -178,7 +178,7 @@ def summary(self, html_partial=False): Given a HTML file, extracts the text of the article. :param html_partial: return only the div of the document, don't wrap - in html and body tags. + in html and body tags. Warning: It mutates internal DOM representation of the HTML document, so it is better to call other API methods before this one. diff --git a/tox.ini b/tox.ini index d3c2a7fe..2019f355 100644 --- a/tox.ini +++ b/tox.ini @@ -4,10 +4,15 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py35, py36, py37 +envlist = py27, py35, py36, py37, doc [testenv] -deps=pytest +deps = + pytest + doc: sphinx + doc: sphinx_rtd_theme + doc: recommonmark + # This creates the virtual envs with --site-packages so already packages # that are already installed will be reused. This is especially useful on # Windows. Since we use lxml instead of compiling it locally (which in turn @@ -18,3 +23,7 @@ sitepackages=True commands = pip install -r requirements.txt -e ".[test]" py.test + +[testenv:doc] +commands = + python setup.py build_sphinx From c24808fbb291540358eb415aca521f6e2506b150 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Mon, 30 Dec 2019 18:40:32 +0700 Subject: [PATCH 21/90] Update README.rst --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index a060f619..ccf40f8c 100644 --- a/README.rst +++ b/README.rst @@ -42,6 +42,7 @@ Usage Change Log ---------- +- 0.8beta Replaced XHTML output with HTML5 output in summary() call. - 0.7.1 Support for Python 3.7 . Fixed a slowdown when processing documents with lots of spaces. - 0.7 Improved HTML5 tags handling. Fixed stripping unwanted HTML nodes (only first matching node was removed before). - 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 - 3.6 From 52f767c8128e1c1815d61290c6e0e8f3585e0402 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Mon, 30 Dec 2019 18:44:49 +0700 Subject: [PATCH 22/90] Update __init__.py --- readability/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readability/__init__.py b/readability/__init__.py index e00a50a9..090cee2f 100644 --- a/readability/__init__.py +++ b/readability/__init__.py @@ -1,3 +1,3 @@ -__version__ = "0.7" +__version__ = "0.8" from .readability import Document From 0556abb794c12c17df655bce0bd6a1d3e265746e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89loi=20Rivard?= Date: Thu, 9 Jan 2020 10:27:29 +0100 Subject: [PATCH 23/90] Syntax highlight the README --- README.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index ccf40f8c..754c43cf 100644 --- a/README.rst +++ b/README.rst @@ -15,29 +15,29 @@ Installation It's easy using ``pip``, just run: -:: +.. code-block:: bash $ pip install readability-lxml Usage ----- -:: +.. code-block:: python >>> import requests >>> from readability import Document - + >>> response = requests.get('http://example.com') >>> doc = Document(response.text) >>> doc.title() 'Example Domain' - + >>> doc.summary() - u'
\n
\n

Example Domain

\n + """
\n
\n

Example Domain

\n

This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.

\n

More information...

\n
- \n\n
' + \n\n
""" Change Log ---------- From a98151e6dd374d7234f20a8ef695478775d68b78 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Tue, 28 Jan 2020 16:31:59 +0100 Subject: [PATCH 24/90] Extended travis config - Python versions added (3.9, pypy) - OS added (MacOS, 2 different versions) --- .travis.yml | 46 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f32ecd8..9e349365 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,45 @@ language: python +os: linux +cache: pip -python: - - 2.7 - - 3.5 - - 3.6 - - 3.7 - - 3.8 +matrix: + include: + - name: "Python 2.7 on Linux" + python: 2.7 + - name: "Python 3.5 on Linux" + python: 3.5 + - name: "Python 3.6 on Linux" + python: 3.6 + - name: "Python 3.7 on Linux" + python: 3.7 + - name: "Python 3.8 on Linux" + dist: xenial + python: 3.8 + - name: "Python 3.9 Nightly on Linux" + dist: bionic + python: nightly + - name: "Pypy 3 on Linux" + python: pypy3 + - name: "Python 3 on older macOS" + os: osx + osx_image: xcode9.4 + language: shell + before_install: + - sw_vers + - python3 --version + - pip3 --version + - name: "Python 3 on macOS" + os: osx + osx_image: xcode11 + language: shell + before_install: + - sw_vers + - python3 --version + - pip3 --version + allow_failures: + - python: nightly + - python: pypy3 + - os: osx install: - travis_retry pip install -U pip wheel tox-travis From 8ea6a20e01061668dbdf9287ef658019ceffa78f Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Tue, 28 Jan 2020 20:33:23 +0100 Subject: [PATCH 25/90] Skip missing interpreters in tox.ini --- tox.ini | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 168ece22..a9ec295d 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,10 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py35, py36, py37, py38, doc +envlist = + py{27,35,36,37,38,py,py3}, doc +skip_missing_interpreters = + True [testenv] deps = @@ -19,7 +22,8 @@ deps = # requires a Compiler and the build dependencies), you can download # it from http://www.lfd.uci.edu/~gohlke/pythonlibs/#lxml and install it via # $PYTHONDIR\Scripts\pip.exe install *.whl -sitepackages=True +sitepackages= + True commands = pip install -r requirements.txt -e ".[test]" py.test From 9a851025559ae06438a35819cbebe685f23ad536 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Tue, 28 Jan 2020 20:42:42 +0100 Subject: [PATCH 26/90] Set TOXENV for macOS tests --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9e349365..b7a013d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ matrix: os: osx osx_image: xcode9.4 language: shell + env: TOXENV=py37 before_install: - sw_vers - python3 --version @@ -32,6 +33,7 @@ matrix: os: osx osx_image: xcode11 language: shell + env: TOXENV=py37 before_install: - sw_vers - python3 --version From 44ee1c4a87e97a6076da733973cbbe745c2a7949 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Wed, 29 Jan 2020 17:57:54 +0700 Subject: [PATCH 27/90] Update .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b7a013d1..4e4b932b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,7 @@ matrix: - name: "Python 3 on macOS" os: osx osx_image: xcode11 + python: python3 language: shell env: TOXENV=py37 before_install: From 28db33a1adb64c4bd7139ed9333849ac6f9241c7 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Wed, 29 Jan 2020 20:12:03 +0700 Subject: [PATCH 28/90] Update .travis.yml --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e4b932b..27d9fb0f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,8 @@ matrix: - name: "Python 3.9 Nightly on Linux" dist: bionic python: nightly + - name: "Pypy on Linux" + python: pypy - name: "Pypy 3 on Linux" python: pypy3 - name: "Python 3 on older macOS" @@ -32,15 +34,15 @@ matrix: - name: "Python 3 on macOS" os: osx osx_image: xcode11 - python: python3 language: shell - env: TOXENV=py37 + env: TOXENV=py37 PYENV_VERSION=3.7 before_install: - sw_vers - python3 --version - pip3 --version allow_failures: - python: nightly + - python: pypy - python: pypy3 - os: osx From 8c122cc862de656d2526a3fe6d244a5e930329a9 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Wed, 29 Jan 2020 20:36:12 +0700 Subject: [PATCH 29/90] Update .travis.yml --- .travis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27d9fb0f..231b893d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ matrix: include: - name: "Python 2.7 on Linux" python: 2.7 + env: TOXENV=py27 - name: "Python 3.5 on Linux" python: 3.5 - name: "Python 3.6 on Linux" @@ -20,6 +21,7 @@ matrix: python: nightly - name: "Pypy on Linux" python: pypy + env: TOXENV=py27 - name: "Pypy 3 on Linux" python: pypy3 - name: "Python 3 on older macOS" @@ -35,7 +37,7 @@ matrix: os: osx osx_image: xcode11 language: shell - env: TOXENV=py37 PYENV_VERSION=3.7 + env: TOXENV=py37 before_install: - sw_vers - python3 --version @@ -47,8 +49,9 @@ matrix: - os: osx install: - - travis_retry pip install -U pip wheel tox-travis - - travis_retry pip install -U -r requirements.txt -e ".[test]" + - if echo $TOXENV | grep -q py27; then PIP=pip; else PIP=pip3; fi + - travis_retry $PIP install -U pip wheel tox-travis + - travis_retry $PIP install -U -r requirements.txt -e ".[test]" script: - tox From baf03e0d8e074bf532902d2ed6fdbbcc9d7fc91c Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Wed, 29 Jan 2020 20:44:57 +0700 Subject: [PATCH 30/90] Update .travis.yml --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 231b893d..3bb97485 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ matrix: include: - name: "Python 2.7 on Linux" python: 2.7 - env: TOXENV=py27 + env: PIP=pip - name: "Python 3.5 on Linux" python: 3.5 - name: "Python 3.6 on Linux" @@ -21,10 +21,10 @@ matrix: python: nightly - name: "Pypy on Linux" python: pypy - env: TOXENV=py27 + env: PIP=pip - name: "Pypy 3 on Linux" python: pypy3 - - name: "Python 3 on older macOS" + - name: "Python 3.7 on older macOS" os: osx osx_image: xcode9.4 language: shell @@ -33,7 +33,7 @@ matrix: - sw_vers - python3 --version - pip3 --version - - name: "Python 3 on macOS" + - name: "Python 3.7 on macOS" os: osx osx_image: xcode11 language: shell @@ -49,7 +49,7 @@ matrix: - os: osx install: - - if echo $TOXENV | grep -q py27; then PIP=pip; else PIP=pip3; fi + - if [ $PIP ]; then true; else PIP=pip3; fi - travis_retry $PIP install -U pip wheel tox-travis - travis_retry $PIP install -U -r requirements.txt -e ".[test]" From bd8293eb637eeeb47484b0e0ae1499a877c6e9be Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Thu, 30 Jan 2020 16:56:38 +0100 Subject: [PATCH 31/90] code linting --- readability/cleaners.py | 3 +++ readability/debug.py | 2 -- readability/htmls.py | 15 ++++++++++++--- readability/readability.py | 16 +++++++--------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/readability/cleaners.py b/readability/cleaners.py index 5cbab474..2a99b7ad 100644 --- a/readability/cleaners.py +++ b/readability/cleaners.py @@ -14,11 +14,13 @@ ">" # end , re.I) + def clean_attributes(html): while htmlstrip.search(html): html = htmlstrip.sub('<\\1\\2>', html) return html + def normalize_spaces(s): if not s: return '' @@ -26,6 +28,7 @@ def normalize_spaces(s): characters with a single space""" return ' '.join(s.split()) + html_cleaner = Cleaner(scripts=True, javascript=True, comments=True, style=True, links=True, meta=False, add_nofollow=False, page_structure=False, processing_instructions=True, embedded=False, diff --git a/readability/debug.py b/readability/debug.py index 061014d2..b0ffe9a9 100644 --- a/readability/debug.py +++ b/readability/debug.py @@ -49,5 +49,3 @@ def text_content(elem, length=40): if len(content) < length: return content return content[:length] + '...' - - diff --git a/readability/htmls.py b/readability/htmls.py index 843f0c0b..8088ac2d 100644 --- a/readability/htmls.py +++ b/readability/htmls.py @@ -1,7 +1,6 @@ from lxml.html import tostring -import logging import lxml.html -import re, sys +import re from .cleaners import normalize_spaces, clean_attributes from .encoding import get_encoding @@ -9,6 +8,7 @@ utf8_parser = lxml.html.HTMLParser(encoding='utf-8') + def build_doc(page): if isinstance(page, str_): encoding = None @@ -16,14 +16,16 @@ def build_doc(page): else: encoding = get_encoding(page) or 'utf-8' decoded_page = page.decode(encoding, 'replace') - + # XXX: we have to do .decode and .encode even for utf-8 pages to remove bad characters doc = lxml.html.document_fromstring(decoded_page.encode('utf-8', 'replace'), parser=utf8_parser) return doc, encoding + def js_re(src, pattern, flags, repl): return re.compile(pattern, flags).sub(src, repl.replace('$', '\\')) + def normalize_entities(cur_title): entities = { u'\u2014':'-', @@ -41,9 +43,11 @@ def normalize_entities(cur_title): return cur_title + def norm_title(title): return normalize_entities(normalize_spaces(title)) + def get_title(doc): title = doc.find('.//title') if title is None or title.text is None or len(title.text) == 0: @@ -51,16 +55,19 @@ def get_title(doc): return norm_title(title.text) + def add_match(collection, text, orig): text = norm_title(text) if len(text.split()) >= 2 and len(text) >= 15: if text.replace('"', '') in orig.replace('"', ''): collection.add(text) + TITLE_CSS_HEURISTICS = ['#title', '#head', '#heading', '.pageTitle', '.news_title', '.title', '.head', '.heading', '.contentheading', '.small_header_red'] + def shorten_title(doc): title = doc.find('.//title') if title is None or title.text is None or len(title.text) == 0: @@ -109,6 +116,8 @@ def shorten_title(doc): return title + +# is it necessary? Cleaner from LXML is initialized correctly in cleaners.py def get_body(doc): for elem in doc.xpath('.//script | .//link | .//style'): elem.drop_tree() diff --git a/readability/readability.py b/readability/readability.py index b0323b37..7116c0ee 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -4,7 +4,6 @@ import re import sys -from collections import defaultdict from lxml.etree import tostring from lxml.etree import tounicode from lxml.html import document_fromstring @@ -56,7 +55,6 @@ def to_int(x): def clean(text): # Many spaces make the following regexes run forever text = re.sub(r'\s{255,}', ' ' * 255, text) - text = re.sub(r'\s*\n\s*', '\n', text) text = re.sub(r'\t|[ \t]{2,}', ' ', text) return text.strip() @@ -65,12 +63,11 @@ def clean(text): def text_length(i): return len(clean(i.text_content() or "")) -regexp_type = type(re.compile('hello, world')) def compile_pattern(elements): if not elements: return None - elif isinstance(elements, regexp_type): + elif isinstance(elements, re._pattern_type): return elements elif isinstance(elements, (str_, bytes_)): if isinstance(elements, bytes_): @@ -82,6 +79,7 @@ def compile_pattern(elements): raise Exception("Unknown type for the pattern: {}".format(type(elements))) # assume string or string like object + class Document: """Class to build a etree document out of html.""" @@ -98,9 +96,9 @@ def __init__(self, input, positive_keywords=None, negative_keywords=None, :param xpath: If set to True, adds x="..." attribute to each HTML node, containing xpath path pointing to original document path (allows to reconstruct selected summary in original document). - :param handle_failures: Parameter passed to `lxml` for handling failure during exception. + :param handle_failures: Parameter passed to `lxml` for handling failure during exception. Support options = ["discard", "ignore", None] - + Examples: positive_keywords=["news-item", "block"] positive_keywords=["news-item, block"] @@ -290,7 +288,7 @@ def select_best_candidate(self, candidates): return None sorted_candidates = sorted( - candidates.values(), + candidates.values(), key=lambda x: x['content_score'], reverse=True ) @@ -517,10 +515,10 @@ def sanitize(self, node, candidates): #if el.tag == 'div' and counts["img"] >= 1: # continue - if counts["p"] and counts["img"] > 1+counts["p"]*1.3: + if counts["p"] and counts["img"] > 1 + counts["p"]*1.3: reason = "too many images (%s)" % counts["img"] to_remove = True - elif counts["li"] > counts["p"] and tag != "ul" and tag != "ol": + elif counts["li"] > counts["p"] and tag not in ("ol", "ul"): reason = "more
  • s than

    s" to_remove = True elif counts["input"] > (counts["p"] / 3): From e9acdd091b30549cec141d224070f9f60e97cf94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89loi=20Rivard?= Date: Thu, 30 Jan 2020 17:32:43 +0100 Subject: [PATCH 32/90] Use black to format the code --- readability/browser.py | 9 +- readability/cleaners.py | 50 ++-- readability/compat/__init__.py | 3 +- readability/debug.py | 30 +-- readability/encoding.py | 33 +-- readability/htmls.py | 71 +++--- readability/readability.py | 446 ++++++++++++++++++++------------- setup.py | 26 +- tests/test_article_only.py | 47 ++-- 9 files changed, 419 insertions(+), 296 deletions(-) diff --git a/readability/browser.py b/readability/browser.py index bcfe61c2..42117a5a 100644 --- a/readability/browser.py +++ b/readability/browser.py @@ -7,14 +7,15 @@ def open_in_browser(html): import os import webbrowser import tempfile - handle, fn = tempfile.mkstemp(suffix='.html') - f = os.fdopen(handle, 'wb') + + handle, fn = tempfile.mkstemp(suffix=".html") + f = os.fdopen(handle, "wb") try: f.write(b"") - f.write(html.encode('utf-8')) + f.write(html.encode("utf-8")) finally: # we leak the file itself here, but we should at least close it f.close() - url = 'file://' + fn.replace(os.path.sep, '/') + url = "file://" + fn.replace(os.path.sep, "/") webbrowser.open(url) return url diff --git a/readability/cleaners.py b/readability/cleaners.py index 2a99b7ad..69825c6b 100644 --- a/readability/cleaners.py +++ b/readability/cleaners.py @@ -2,35 +2,51 @@ import re from lxml.html.clean import Cleaner -bad_attrs = ['width', 'height', 'style', '[-a-z]*color', 'background[-a-z]*', 'on*'] +bad_attrs = ["width", "height", "style", "[-a-z]*color", "background[-a-z]*", "on*"] single_quoted = "'[^']+'" double_quoted = '"[^"]+"' -non_space = '[^ "\'>]+' -htmlstrip = re.compile("<" # open - "([^>]+) " # prefix - "(?:%s) *" % ('|'.join(bad_attrs),) + # undesirable attributes - '= *(?:%s|%s|%s)' % (non_space, single_quoted, double_quoted) + # value - "([^>]*)" # postfix - ">" # end -, re.I) +non_space = "[^ \"'>]+" +htmlstrip = re.compile( + "<" # open + "([^>]+) " # prefix + "(?:%s) *" % ("|".join(bad_attrs),) + + "= *(?:%s|%s|%s)" # undesirable attributes + % (non_space, single_quoted, double_quoted) + + "([^>]*)" # value # postfix + ">", # end + re.I, +) def clean_attributes(html): while htmlstrip.search(html): - html = htmlstrip.sub('<\\1\\2>', html) + html = htmlstrip.sub("<\\1\\2>", html) return html def normalize_spaces(s): if not s: - return '' + return "" """replace any sequence of whitespace characters with a single space""" - return ' '.join(s.split()) + return " ".join(s.split()) -html_cleaner = Cleaner(scripts=True, javascript=True, comments=True, - style=True, links=True, meta=False, add_nofollow=False, - page_structure=False, processing_instructions=True, embedded=False, - frames=False, forms=False, annoying_tags=False, remove_tags=None, - remove_unknown_tags=False, safe_attrs_only=False) +html_cleaner = Cleaner( + scripts=True, + javascript=True, + comments=True, + style=True, + links=True, + meta=False, + add_nofollow=False, + page_structure=False, + processing_instructions=True, + embedded=False, + frames=False, + forms=False, + annoying_tags=False, + remove_tags=None, + remove_unknown_tags=False, + safe_attrs_only=False, +) diff --git a/readability/compat/__init__.py b/readability/compat/__init__.py index d02b65cf..900f819c 100644 --- a/readability/compat/__init__.py +++ b/readability/compat/__init__.py @@ -5,10 +5,11 @@ syntax that can only be solved by conditionally importing different functions. """ import sys + if sys.version_info[0] == 2: bytes_ = str str_ = unicode - + elif sys.version_info[0] == 3: bytes_ = bytes str_ = str diff --git a/readability/debug.py b/readability/debug.py index b0ffe9a9..3bc81974 100644 --- a/readability/debug.py +++ b/readability/debug.py @@ -1,7 +1,7 @@ import re -#FIXME: use with caution, can leak memory +# FIXME: use with caution, can leak memory uids = {} uids_document = None @@ -9,17 +9,17 @@ def describe_node(node): global uids if node is None: - return '' - if not hasattr(node, 'tag'): + return "" + if not hasattr(node, "tag"): return "[%s]" % type(node) name = node.tag - if node.get('id', ''): - name += '#' + node.get('id') - if node.get('class', '').strip(): - name += '.' + '.'.join(node.get('class').split()) - if name[:4] in ['div#', 'div.']: + if node.get("id", ""): + name += "#" + node.get("id") + if node.get("class", "").strip(): + name += "." + ".".join(node.get("class").split()) + if name[:4] in ["div#", "div."]: name = name[3:] - if name in ['tr', 'td', 'div', 'p']: + if name in ["tr", "td", "div", "p"]: uid = uids.get(node) if uid is None: uid = uids[node] = len(uids) + 1 @@ -34,18 +34,18 @@ def describe(node, depth=1): uids = {} uids_document = doc - #return repr(NodeRepr(node)) - parent = '' + # return repr(NodeRepr(node)) + parent = "" if depth and node.getparent() is not None: - parent = describe(node.getparent(), depth=depth - 1) + '>' + parent = describe(node.getparent(), depth=depth - 1) + ">" return parent + describe_node(node) -RE_COLLAPSE_WHITESPACES = re.compile(r'\s+', re.U) +RE_COLLAPSE_WHITESPACES = re.compile(r"\s+", re.U) def text_content(elem, length=40): - content = RE_COLLAPSE_WHITESPACES.sub(' ', elem.text_content().replace('\r', '')) + content = RE_COLLAPSE_WHITESPACES.sub(" ", elem.text_content().replace("\r", "")) if len(content) < length: return content - return content[:length] + '...' + return content[:length] + "..." diff --git a/readability/encoding.py b/readability/encoding.py index 643d6ce8..e915866a 100644 --- a/readability/encoding.py +++ b/readability/encoding.py @@ -8,15 +8,16 @@ RE_XML = re.compile(br'^<\?xml.*?encoding=["\']*(.+?)["\'>]') CHARSETS = { - 'big5': 'big5hkscs', - 'gb2312': 'gb18030', - 'ascii': 'utf-8', - 'maccyrillic': 'cp1251', - 'win1251': 'cp1251', - 'win-1251': 'cp1251', - 'windows-1251': 'cp1251', + "big5": "big5hkscs", + "gb2312": "gb18030", + "ascii": "utf-8", + "maccyrillic": "cp1251", + "win1251": "cp1251", + "win-1251": "cp1251", + "windows-1251": "cp1251", } + def fix_charset(encoding): """Overrides encoding when charset declaration or charset determination is a subset of a larger @@ -27,9 +28,9 @@ def fix_charset(encoding): def get_encoding(page): # Regex for XML and HTML Meta charset declaration - declared_encodings = (RE_CHARSET.findall(page) + - RE_PRAGMA.findall(page) + - RE_XML.findall(page)) + declared_encodings = ( + RE_CHARSET.findall(page) + RE_PRAGMA.findall(page) + RE_XML.findall(page) + ) # Try any declared encodings for declared_encoding in declared_encodings: @@ -38,7 +39,7 @@ def get_encoding(page): # declared_encoding will actually be bytes but .decode() only # accepts `str` type. Decode blindly with ascii because no one should # ever use non-ascii characters in the name of an encoding. - declared_encoding = declared_encoding.decode('ascii', 'replace') + declared_encoding = declared_encoding.decode("ascii", "replace") encoding = fix_charset(declared_encoding) @@ -51,12 +52,12 @@ def get_encoding(page): # Fallback to chardet if declared encodings fail # Remove all HTML tags, and leave only text for chardet - text = re.sub(br'(\s*]*>)+\s*', b' ', page).strip() - enc = 'utf-8' + text = re.sub(br"(\s*]*>)+\s*", b" ", page).strip() + enc = "utf-8" if len(text) < 10: - return enc # can't guess + return enc # can't guess res = chardet.detect(text) - enc = res['encoding'] or 'utf-8' - #print '->', enc, "%.2f" % res['confidence'] + enc = res["encoding"] or "utf-8" + # print '->', enc, "%.2f" % res['confidence'] enc = fix_charset(enc) return enc diff --git a/readability/htmls.py b/readability/htmls.py index 8088ac2d..17a75c7d 100644 --- a/readability/htmls.py +++ b/readability/htmls.py @@ -6,7 +6,7 @@ from .encoding import get_encoding from .compat import str_ -utf8_parser = lxml.html.HTMLParser(encoding='utf-8') +utf8_parser = lxml.html.HTMLParser(encoding="utf-8") def build_doc(page): @@ -14,28 +14,30 @@ def build_doc(page): encoding = None decoded_page = page else: - encoding = get_encoding(page) or 'utf-8' - decoded_page = page.decode(encoding, 'replace') + encoding = get_encoding(page) or "utf-8" + decoded_page = page.decode(encoding, "replace") # XXX: we have to do .decode and .encode even for utf-8 pages to remove bad characters - doc = lxml.html.document_fromstring(decoded_page.encode('utf-8', 'replace'), parser=utf8_parser) + doc = lxml.html.document_fromstring( + decoded_page.encode("utf-8", "replace"), parser=utf8_parser + ) return doc, encoding def js_re(src, pattern, flags, repl): - return re.compile(pattern, flags).sub(src, repl.replace('$', '\\')) + return re.compile(pattern, flags).sub(src, repl.replace("$", "\\")) def normalize_entities(cur_title): entities = { - u'\u2014':'-', - u'\u2013':'-', - u'—': '-', - u'–': '-', - u'\u00A0': ' ', - u'\u00AB': '"', - u'\u00BB': '"', - u'"': '"', + u"\u2014": "-", + u"\u2013": "-", + u"—": "-", + u"–": "-", + u"\u00A0": " ", + u"\u00AB": '"', + u"\u00BB": '"', + u""": '"', } for c, r in entities.items(): if c in cur_title: @@ -49,9 +51,9 @@ def norm_title(title): def get_title(doc): - title = doc.find('.//title') + title = doc.find(".//title") if title is None or title.text is None or len(title.text) == 0: - return '[no-title]' + return "[no-title]" return norm_title(title.text) @@ -59,25 +61,34 @@ def get_title(doc): def add_match(collection, text, orig): text = norm_title(text) if len(text.split()) >= 2 and len(text) >= 15: - if text.replace('"', '') in orig.replace('"', ''): + if text.replace('"', "") in orig.replace('"', ""): collection.add(text) -TITLE_CSS_HEURISTICS = ['#title', '#head', '#heading', '.pageTitle', - '.news_title', '.title', '.head', '.heading', - '.contentheading', '.small_header_red'] +TITLE_CSS_HEURISTICS = [ + "#title", + "#head", + "#heading", + ".pageTitle", + ".news_title", + ".title", + ".head", + ".heading", + ".contentheading", + ".small_header_red", +] def shorten_title(doc): - title = doc.find('.//title') + title = doc.find(".//title") if title is None or title.text is None or len(title.text) == 0: - return '' + return "" title = orig = norm_title(title.text) candidates = set() - for item in ['.//h1', './/h2', './/h3']: + for item in [".//h1", ".//h2", ".//h3"]: for e in list(doc.iterfind(item)): if e.text: add_match(candidates, e.text, orig) @@ -94,7 +105,7 @@ def shorten_title(doc): if candidates: title = sorted(candidates, key=len)[-1] else: - for delimiter in [' | ', ' - ', ' :: ', ' / ']: + for delimiter in [" | ", " - ", " :: ", " / "]: if delimiter in title: parts = orig.split(delimiter) if len(parts[0].split()) >= 4: @@ -104,12 +115,12 @@ def shorten_title(doc): title = parts[-1] break else: - if ': ' in title: - parts = orig.split(': ') + if ": " in title: + parts = orig.split(": ") if len(parts[-1].split()) >= 4: title = parts[-1] else: - title = orig.split(': ', 1)[1] + title = orig.split(": ", 1)[1] if not 15 < len(title) < 150: return orig @@ -119,15 +130,15 @@ def shorten_title(doc): # is it necessary? Cleaner from LXML is initialized correctly in cleaners.py def get_body(doc): - for elem in doc.xpath('.//script | .//link | .//style'): + for elem in doc.xpath(".//script | .//link | .//style"): elem.drop_tree() # tostring() always return utf-8 encoded string # FIXME: isn't better to use tounicode? raw_html = str_(tostring(doc.body or doc)) cleaned = clean_attributes(raw_html) try: - #BeautifulSoup(cleaned) #FIXME do we really need to try loading it? + # BeautifulSoup(cleaned) #FIXME do we really need to try loading it? return cleaned - except Exception: #FIXME find the equivalent lxml error - #logging.error("cleansing broke html content: %s\n---------\n%s" % (raw_html, cleaned)) + except Exception: # FIXME find the equivalent lxml error + # logging.error("cleansing broke html content: %s\n---------\n%s" % (raw_html, cleaned)) return raw_html diff --git a/readability/readability.py b/readability/readability.py index 7116c0ee..fc682318 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -22,18 +22,29 @@ log = logging.getLogger("readability.readability") REGEXES = { - 'unlikelyCandidatesRe': re.compile(r'combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter', re.I), - 'okMaybeItsACandidateRe': re.compile(r'and|article|body|column|main|shadow', re.I), - 'positiveRe': re.compile(r'article|body|content|entry|hentry|main|page|pagination|post|text|blog|story', re.I), - 'negativeRe': re.compile(r'combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget', re.I), - 'divToPElementsRe': re.compile(r'<(a|blockquote|dl|div|img|ol|p|pre|table|ul)', re.I), + "unlikelyCandidatesRe": re.compile( + r"combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter", + re.I, + ), + "okMaybeItsACandidateRe": re.compile(r"and|article|body|column|main|shadow", re.I), + "positiveRe": re.compile( + r"article|body|content|entry|hentry|main|page|pagination|post|text|blog|story", + re.I, + ), + "negativeRe": re.compile( + r"combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget", + re.I, + ), + "divToPElementsRe": re.compile( + r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)", re.I + ), #'replaceBrsRe': re.compile(r'(]*>[ \n\r\t]*){2,}',re.I), #'replaceFontsRe': re.compile(r'<(\/?)font[^>]*>',re.I), #'trimRe': re.compile(r'^\s+|\s+$/'), #'normalizeRe': re.compile(r'\s{2,}/'), #'killBreaksRe': re.compile(r'((\s| ?)*){1,}/'), - 'videoRe': re.compile(r'https?:\/\/(www\.)?(youtube|vimeo)\.com', re.I), - #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, + "videoRe": re.compile(r"https?:\/\/(www\.)?(youtube|vimeo)\.com", re.I), + # skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, } @@ -45,18 +56,18 @@ def to_int(x): if not x: return None x = x.strip() - if x.endswith('px'): + if x.endswith("px"): return int(x[:-2]) - if x.endswith('em'): + if x.endswith("em"): return int(x[:-2]) * 12 return int(x) def clean(text): # Many spaces make the following regexes run forever - text = re.sub(r'\s{255,}', ' ' * 255, text) - text = re.sub(r'\s*\n\s*', '\n', text) - text = re.sub(r'\t|[ \t]{2,}', ' ', text) + text = re.sub(r"\s{255,}", " " * 255, text) + text = re.sub(r"\s*\n\s*", "\n", text) + text = re.sub(r"\t|[ \t]{2,}", " ", text) return text.strip() @@ -71,10 +82,10 @@ def compile_pattern(elements): return elements elif isinstance(elements, (str_, bytes_)): if isinstance(elements, bytes_): - elements = str_(elements, 'utf-8') - elements = elements.split(u',') + elements = str_(elements, "utf-8") + elements = elements.split(u",") if isinstance(elements, (list, tuple)): - return re.compile(u'|'.join([re.escape(x.strip()) for x in elements]), re.U) + return re.compile(u"|".join([re.escape(x.strip()) for x in elements]), re.U) else: raise Exception("Unknown type for the pattern: {}".format(type(elements))) # assume string or string like object @@ -83,9 +94,17 @@ def compile_pattern(elements): class Document: """Class to build a etree document out of html.""" - def __init__(self, input, positive_keywords=None, negative_keywords=None, - url=None, min_text_length=25, retry_length=250, xpath=False, - handle_failures='discard'): + def __init__( + self, + input, + positive_keywords=None, + negative_keywords=None, + url=None, + min_text_length=25, + retry_length=250, + xpath=False, + handle_failures="discard", + ): """Generate the document :param input: string of the html content. @@ -131,8 +150,8 @@ def _html(self, force=False): if self.xpath: root = self.html.getroottree() for i in self.html.getiterator(): - #print root.getpath(i) - i.attrib['x'] = root.getpath(i) + # print root.getpath(i) + i.attrib["x"] = root.getpath(i) return self.html def _parse(self, input): @@ -143,11 +162,19 @@ def _parse(self, input): # trying to guard against bad links like try: # such support is added in lxml 3.3.0 - doc.make_links_absolute(base_href, resolve_base_href=True, handle_failures=self.handle_failures) - except TypeError: #make_links_absolute() got an unexpected keyword argument 'handle_failures' + doc.make_links_absolute( + base_href, + resolve_base_href=True, + handle_failures=self.handle_failures, + ) + except TypeError: # make_links_absolute() got an unexpected keyword argument 'handle_failures' # then we have lxml < 3.3.0 # please upgrade to lxml >= 3.3.0 if you're failing here! - doc.make_links_absolute(base_href, resolve_base_href=True, handle_failures=self.handle_failures) + doc.make_links_absolute( + base_href, + resolve_base_href=True, + handle_failures=self.handle_failures, + ) else: doc.resolve_base_href(handle_failures=self.handle_failures) return doc @@ -169,7 +196,7 @@ def get_clean_html(self): An internal method, which can be overridden in subclasses, for example, to disable or to improve DOM-to-text conversion in .summary() method """ - return clean_attributes(tounicode(self.html, method='html')) + return clean_attributes(tounicode(self.html, method="html")) def summary(self, html_partial=False): """ @@ -185,10 +212,10 @@ def summary(self, html_partial=False): ruthless = True while True: self._html(True) - for i in self.tags(self.html, 'script', 'style'): + for i in self.tags(self.html, "script", "style"): i.drop_tree() - for i in self.tags(self.html, 'body'): - i.set('id', 'readabilityBody') + for i in self.tags(self.html, "body"): + i.set("id", "readabilityBody") if ruthless: self.remove_unlikely_candidates() self.transform_misused_divs_into_paragraphs() @@ -197,27 +224,34 @@ def summary(self, html_partial=False): best_candidate = self.select_best_candidate(candidates) if best_candidate: - article = self.get_article(candidates, best_candidate, - html_partial=html_partial) + article = self.get_article( + candidates, best_candidate, html_partial=html_partial + ) else: if ruthless: log.info("ruthless removal did not work. ") ruthless = False log.debug( - ("ended up stripping too much - " - "going for a safer _parse")) + ( + "ended up stripping too much - " + "going for a safer _parse" + ) + ) # try again continue else: log.debug( - ("Ruthless and lenient parsing did not work. " - "Returning raw html")) - article = self.html.find('body') + ( + "Ruthless and lenient parsing did not work. " + "Returning raw html" + ) + ) + article = self.html.find("body") if article is None: article = self.html cleaned_article = self.sanitize(article, candidates) - article_length = len(cleaned_article or '') + article_length = len(cleaned_article or "") retry_length = self.retry_length of_acceptable_length = article_length >= retry_length if ruthless and not of_acceptable_length: @@ -227,7 +261,7 @@ def summary(self, html_partial=False): else: return cleaned_article except Exception as e: - log.exception('error getting summary: ') + log.exception("error getting summary: ") if sys.version_info[0] == 2: from .compat.two import raise_with_traceback else: @@ -238,15 +272,13 @@ def get_article(self, candidates, best_candidate, html_partial=False): # Now that we have the top candidate, look through its siblings for # content that might also be related. # Things like preambles, content split by ads that we removed, etc. - sibling_score_threshold = max([ - 10, - best_candidate['content_score'] * 0.2]) + sibling_score_threshold = max([10, best_candidate["content_score"] * 0.2]) # create a new html document with a html->body->div if html_partial: - output = fragment_fromstring('

    ') + output = fragment_fromstring("
    ") else: - output = document_fromstring('
    ') - best_elem = best_candidate['elem'] + output = document_fromstring("
    ") + best_elem = best_candidate["elem"] parent = best_elem.getparent() siblings = parent.getchildren() if parent is not None else [best_elem] for sibling in siblings: @@ -256,8 +288,10 @@ def get_article(self, candidates, best_candidate, html_partial=False): if sibling is best_elem: append = True sibling_key = sibling # HashableElement(sibling) - if sibling_key in candidates and \ - candidates[sibling_key]['content_score'] >= sibling_score_threshold: + if ( + sibling_key in candidates + and candidates[sibling_key]["content_score"] >= sibling_score_threshold + ): append = True if sibling.tag == "p": @@ -267,9 +301,11 @@ def get_article(self, candidates, best_candidate, html_partial=False): if node_length > 80 and link_density < 0.25: append = True - elif node_length <= 80 \ - and link_density == 0 \ - and re.search(r'\.( |$)', node_content): + elif ( + node_length <= 80 + and link_density == 0 + and re.search(r"\.( |$)", node_content) + ): append = True if append: @@ -279,7 +315,7 @@ def get_article(self, candidates, best_candidate, html_partial=False): output.append(sibling) else: output.getchildren()[0].getchildren()[0].append(sibling) - #if output is not None: + # if output is not None: # output.append(best_elem) return output @@ -288,15 +324,11 @@ def select_best_candidate(self, candidates): return None sorted_candidates = sorted( - candidates.values(), - key=lambda x: x['content_score'], - reverse=True + candidates.values(), key=lambda x: x["content_score"], reverse=True ) for candidate in sorted_candidates[:5]: - elem = candidate['elem'] - log.info("Top 5 : %6.3f %s" % ( - candidate['content_score'], - describe(elem))) + elem = candidate["elem"] + log.info("Top 5 : %6.3f %s" % (candidate["content_score"], describe(elem))) best_candidate = sorted_candidates[0] return best_candidate @@ -305,7 +337,7 @@ def get_link_density(self, elem): link_length = 0 for i in elem.findall(".//a"): link_length += text_length(i) - #if len(elem.findall(".//div") or elem.findall(".//p")): + # if len(elem.findall(".//div") or elem.findall(".//p")): # link_length = link_length total_length = text_length(elem) return float(link_length) / max(total_length, 1) @@ -333,20 +365,19 @@ def score_paragraphs(self): ordered.append(parent_node) if grand_parent_node is not None and grand_parent_node not in candidates: - candidates[grand_parent_node] = self.score_node( - grand_parent_node) + candidates[grand_parent_node] = self.score_node(grand_parent_node) ordered.append(grand_parent_node) content_score = 1 - content_score += len(inner_text.split(',')) + content_score += len(inner_text.split(",")) content_score += min((inner_text_len / 100), 3) - #if elem not in candidates: + # if elem not in candidates: # candidates[elem] = self.score_node(elem) - #WTF? candidates[elem]['content_score'] += content_score - candidates[parent_node]['content_score'] += content_score + # WTF? candidates[elem]['content_score'] += content_score + candidates[parent_node]["content_score"] += content_score if grand_parent_node is not None: - candidates[grand_parent_node]['content_score'] += content_score / 2.0 + candidates[grand_parent_node]["content_score"] += content_score / 2.0 # Scale the final candidates score based on link density. Good content # should have a relatively small link density (5% or less) and be @@ -354,24 +385,23 @@ def score_paragraphs(self): for elem in ordered: candidate = candidates[elem] ld = self.get_link_density(elem) - score = candidate['content_score'] - log.debug("Branch %6.3f %s link density %.3f -> %6.3f" % ( - score, - describe(elem), - ld, - score * (1 - ld))) - candidate['content_score'] *= (1 - ld) + score = candidate["content_score"] + log.debug( + "Branch %6.3f %s link density %.3f -> %6.3f" + % (score, describe(elem), ld, score * (1 - ld)) + ) + candidate["content_score"] *= 1 - ld return candidates def class_weight(self, e): weight = 0 - for feature in [e.get('class', None), e.get('id', None)]: + for feature in [e.get("class", None), e.get("id", None)]: if feature: - if REGEXES['negativeRe'].search(feature): + if REGEXES["negativeRe"].search(feature): weight -= 25 - if REGEXES['positiveRe'].search(feature): + if REGEXES["positiveRe"].search(feature): weight += 25 if self.positive_keywords and self.positive_keywords.search(feature): @@ -380,10 +410,10 @@ def class_weight(self, e): if self.negative_keywords and self.negative_keywords.search(feature): weight -= 25 - if self.positive_keywords and self.positive_keywords.match('tag-'+e.tag): + if self.positive_keywords and self.positive_keywords.match("tag-" + e.tag): weight += 25 - if self.negative_keywords and self.negative_keywords.match('tag-'+e.tag): + if self.negative_keywords and self.negative_keywords.match("tag-" + e.tag): weight -= 25 return weight @@ -397,63 +427,76 @@ def score_node(self, elem): content_score += 3 elif name in ["address", "ol", "ul", "dl", "dd", "dt", "li", "form", "aside"]: content_score -= 3 - elif name in ["h1", "h2", "h3", "h4", "h5", "h6", "th", "header", "footer", "nav"]: + elif name in [ + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "th", + "header", + "footer", + "nav", + ]: content_score -= 5 - return { - 'content_score': content_score, - 'elem': elem - } + return {"content_score": content_score, "elem": elem} def remove_unlikely_candidates(self): - for elem in self.html.findall('.//*'): - s = "%s %s" % (elem.get('class', ''), elem.get('id', '')) + for elem in self.html.findall(".//*"): + s = "%s %s" % (elem.get("class", ""), elem.get("id", "")) if len(s) < 2: continue - if REGEXES['unlikelyCandidatesRe'].search(s) and (not REGEXES['okMaybeItsACandidateRe'].search(s)) and elem.tag not in ['html', 'body']: + if ( + REGEXES["unlikelyCandidatesRe"].search(s) + and (not REGEXES["okMaybeItsACandidateRe"].search(s)) + and elem.tag not in ["html", "body"] + ): log.debug("Removing unlikely candidate - %s" % describe(elem)) elem.drop_tree() def transform_misused_divs_into_paragraphs(self): - for elem in self.tags(self.html, 'div'): + for elem in self.tags(self.html, "div"): # transform
    s that do not contain other block elements into #

    s - #FIXME: The current implementation ignores all descendants that + # FIXME: The current implementation ignores all descendants that # are not direct children of elem # This results in incorrect results in case there is an # buried within an for example - if not REGEXES['divToPElementsRe'].search( - str_(b''.join(map(tostring, list(elem))))): - #log.debug("Altering %s to p" % (describe(elem))) + if not REGEXES["divToPElementsRe"].search( + str_(b"".join(map(tostring, list(elem)))) + ): + # log.debug("Altering %s to p" % (describe(elem))) elem.tag = "p" - #print "Fixed element "+describe(elem) + # print "Fixed element "+describe(elem) - for elem in self.tags(self.html, 'div'): + for elem in self.tags(self.html, "div"): if elem.text and elem.text.strip(): - p = fragment_fromstring('

    ') + p = fragment_fromstring("

    ") p.text = elem.text elem.text = None elem.insert(0, p) - #print "Appended "+tounicode(p)+" to "+describe(elem) + # print "Appended "+tounicode(p)+" to "+describe(elem) for pos, child in reversed(list(enumerate(elem))): if child.tail and child.tail.strip(): - p = fragment_fromstring('

    ') + p = fragment_fromstring("

    ") p.text = child.tail child.tail = None elem.insert(pos + 1, p) - #print "Inserted "+tounicode(p)+" to "+describe(elem) - if child.tag == 'br': - #print 'Dropped
    at '+describe(elem) + # print "Inserted "+tounicode(p)+" to "+describe(elem) + if child.tag == "br": + # print 'Dropped
    at '+describe(elem) child.drop_tree() def tags(self, node, *tag_names): for tag_name in tag_names: - for e in node.findall('.//%s' % tag_name): + for e in node.findall(".//%s" % tag_name): yield e def reverse_tags(self, node, *tag_names): for tag_name in tag_names: - for e in reversed(node.findall('.//%s' % tag_name)): + for e in reversed(node.findall(".//%s" % tag_name)): yield e def sanitize(self, node, candidates): @@ -467,31 +510,35 @@ def sanitize(self, node, candidates): for elem in self.tags(node, "iframe"): if "src" in elem.attrib and REGEXES["videoRe"].search(elem.attrib["src"]): - elem.text = "VIDEO" # ADD content to iframe text node to force proper output + elem.text = "VIDEO" # ADD content to iframe text node to force proper output else: elem.drop_tree() allowed = {} # Conditionally clean s,