diff --git a/.flake8 b/.flake8
new file mode 100644
index 00000000..b33811f1
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,2 @@
+[flake8]
+ignore = E501, W503
\ No newline at end of file
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
new file mode 100644
index 00000000..865b1408
--- /dev/null
+++ b/.github/workflows/python-package.yml
@@ -0,0 +1,40 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+ push:
+ branches: [ "master" ]
+ pull_request:
+ branches: [ "master" ]
+
+jobs:
+ build:
+
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install flake8 pytest
+ if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+ - name: Lint with flake8
+ run: |
+ # stop the build if there are Python syntax errors or undefined names
+ flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+ # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+ flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+ - name: Test with pytest
+ run: |
+ pytest
diff --git a/.gitignore b/.gitignore
index d8961065..b532e65e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,5 @@ nosetests.xml
.idea
.cache
/.noseids
-/.venv
\ No newline at end of file
+/.venv
+/poetry.lock
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 012e4b78..9caf08a5 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ PY := .venv/bin/python
PIP := .venv/bin/pip
PEP8 := .venv/bin/pep8
NOSE := .venv/bin/nosetests
-TWINE := twine
+TWINE := .venv/bin/twine
# ###########
# Tests rule!
@@ -24,7 +24,7 @@ all: setup develop
venv: .venv/bin/python
setup: venv
- $(PIP) install -r requirements-dev.txt
+ $(PIP) install -r requirements-dev.txt | grep -v "already satisfied" || true
.venv/bin/python:
test -d .venv || which python3 && python3 -m venv .venv || virtualenv .venv
@@ -45,11 +45,16 @@ develop: .venv/lib/python*/site-packages/readability-lxml.egg-link
.PHONY: clean_all
clean_all: clean_venv
+.PHONY: build
+build:
+ poetry build
+
# ###########
# Deploy
# ###########
.PHONY: dist
dist:
+ $(PY) -m pip install wheel
$(PY) setup.py sdist bdist_wheel
$(TWINE) check dist/*
@@ -57,6 +62,12 @@ dist:
upload:
$(TWINE) upload dist/*
-.PHONY: version_update
-version_update:
- $(EDITOR) setup.py
+.PHONY: bump
+bump:
+ $(EDITOR) readability/__init__.py
+ $(eval VERSION := $(shell grep "__version__" readability/__init__.py | cut -d'"' -f2))
+ # fix first occurrence of version in pyproject.toml
+ sed -i '0,/version = ".*"/s//version = "$(VERSION)"/' pyproject.toml
+ git commit -m "Bump version to $(VERSION)" pyproject.toml readability/__init__.py
+ git tag $(VERSION)
+ git push --tags
diff --git a/README.md b/README.md
new file mode 100644
index 00000000..e09a515a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,67 @@
+[](https://pypi.python.org/pypi/readability-lxml)
+
+# python-readability
+
+Given an HTML document, extract and clean up the main body text and title.
+
+This is a Python port of a Ruby port of [arc90's Readability project](https://web.archive.org/web/20130519040221/http://www.readability.com/).
+
+## Installation
+
+It's easy using `pip`, just run:
+
+```bash
+$ pip install readability-lxml
+```
+
+As an alternative, you may also use conda to install, just run:
+
+```bash
+$ conda install -c conda-forge readability-lxml
+```
+
+## Usage
+
+```python
+>>> import requests
+>>> from readability import Document
+
+>>> response = requests.get('http://example.com')
+>>> doc = Document(response.content)
+>>> doc.title()
+'Example Domain'
+
+>>> doc.summary()
+"""
\n
\n
Example Domain
\n
+
This domain is established to be used for illustrative examples in documents. You may
+use this\n domain in examples without prior coordination or asking for permission.
+\n
More information...
\n
+\n\n
"""
+```
+
+## Change Log
+- 0.8.4 Better CJK support, thanks @cdhigh
+- 0.8.3.1 Support for python 3.8 - 3.13
+- 0.8.3 We can now save all images via keep_all_images=True (default is to save 1 main image), thanks @botlabsDev
+- 0.8.2 Added article author(s) (thanks @mattblaha)
+- 0.8.1 Fixed processing of non-ascii HTMLs via regexps.
+- 0.8 Replaced XHTML output with HTML5 output in summary() call.
+- 0.7.1 Support for Python 3.7 . Fixed a slowdown when processing documents with lots of spaces.
+- 0.7 Improved HTML5 tags handling. Fixed stripping unwanted HTML nodes (only first matching node was removed before).
+- 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 - 3.6
+- 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and 3.4
+- 0.4 Added Videos loading and allowed more images per paragraph
+- 0.3 Added Document.encoding, positive\_keywords and negative\_keywords
+
+## Licensing
+
+This code is under [the Apache License 2.0](http://www.apache.org/licenses/LICENSE-2.0) license.
+
+## Thanks to
+
+- Latest [readability.js](https://github.com/MHordecki/readability-redux/blob/master/readability/readability.js)
+- Ruby port by starrhorne and iterationlabs
+- [Python port](https://github.com/gfxmonk/python-readability) by gfxmonk
+- [Decruft effort](https://web.archive.org/web/20110214150709/https://www.minvolai.com/blog/decruft-arc90s-readability-in-python/) to move to lxml
+- "BR to P" fix from readability.js which improves quality for smaller texts
+- Github users contributions.
diff --git a/README.rst b/README.rst
deleted file mode 100644
index 9b0a8b71..00000000
--- a/README.rst
+++ /dev/null
@@ -1,76 +0,0 @@
-.. image:: https://travis-ci.org/buriy/python-readability.svg?branch=master
- :target: https://travis-ci.org/buriy/python-readability
-.. image:: https://img.shields.io/pypi/v/readability-lxml.svg
- :target: https://pypi.python.org/pypi/readability-lxml
-
-python-readability
-==================
-
-Given an HTML document, extract and clean up the main body text and title.
-
-This is a Python port of a Ruby port of `arc90's Readability
-project `__.
-
-Installation
-------------
-
-It's easy using ``pip``, just run:
-
-.. code-block:: bash
-
- $ pip install readability-lxml
-
-As an alternative, you may also use conda to install, just run:
-
-.. code-block:: bash
-
- $ conda install -c conda-forge readability-lxml
-
-Usage
------
-
-.. code-block:: python
-
- >>> import requests
- >>> from readability import Document
-
- >>> response = requests.get('http://example.com')
- >>> doc = Document(response.content)
- >>> doc.title()
- 'Example Domain'
-
- >>> doc.summary()
- """\n
\n
Example Domain
\n
-
This domain is established to be used for illustrative examples in documents. You may
- use this\n domain in examples without prior coordination or asking for permission.
- \n
More information...
\n
- \n\n
"""
-
-Change Log
-----------
-
-- 0.8.2 Added article author(s) (thanks @mattblaha)
-- 0.8.1 Fixed processing of non-ascii HTMLs via regexps.
-- 0.8 Replaced XHTML output with HTML5 output in summary() call.
-- 0.7.1 Support for Python 3.7 . Fixed a slowdown when processing documents with lots of spaces.
-- 0.7 Improved HTML5 tags handling. Fixed stripping unwanted HTML nodes (only first matching node was removed before).
-- 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 - 3.6
-- 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and 3.4
-- 0.4 Added Videos loading and allowed more images per paragraph
-- 0.3 Added Document.encoding, positive\_keywords and negative\_keywords
-
-Licensing
----------
-
-This code is under `the Apache License
-2.0 `__ license.
-
-Thanks to
----------
-
-- Latest `readability.js `__
-- Ruby port by starrhorne and iterationlabs
-- `Python port `__ by gfxmonk
-- `Decruft effort ` to move to lxml
-- "BR to P" fix from readability.js which improves quality for smaller texts
-- Github users contributions.
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000..b377690a
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "readability-lxml"
+version = "0.8.4.1"
+description = "fast html to text parser (article readability tool) with python 3 support"
+authors = ["Yuri Baburov "]
+license = "Apache License 2.0"
+readme = "README.md"
+packages = [
+ { include = "readability" },
+]
+
+[tool.poetry.dependencies]
+python = ">=3.8.2,<3.15"
+chardet = "^5.2.0"
+cssselect = [
+ { version = "~1.2", markers = "python_version < '3.9'" },
+ { version = "~1.3", markers = "python_version >= '3.9'" }
+]
+lxml = {extras = ["html-clean"], version = "^5.4.0"}
+lxml-html-clean = {markers = "python_version < \"3.11\"", version = "^0.4.2"}
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
diff --git a/readability/__init__.py b/readability/__init__.py
index 6a263bf7..b36f021d 100644
--- a/readability/__init__.py
+++ b/readability/__init__.py
@@ -1,3 +1,3 @@
-__version__ = "0.8.2"
+__version__ = "0.8.4.1"
from .readability import Document
diff --git a/readability/cleaners.py b/readability/cleaners.py
index 69825c6b..e0b07260 100644
--- a/readability/cleaners.py
+++ b/readability/cleaners.py
@@ -1,6 +1,9 @@
# strip out a set of nuisance html attributes that can mess up rendering in RSS feeds
import re
-from lxml.html.clean import Cleaner
+try:
+ from lxml.html.clean import Cleaner
+except ImportError:
+ from lxml_html_clean import Cleaner
bad_attrs = ["width", "height", "style", "[-a-z]*color", "background[-a-z]*", "on*"]
single_quoted = "'[^']+'"
diff --git a/readability/encoding.py b/readability/encoding.py
index c95cc14d..08332df0 100644
--- a/readability/encoding.py
+++ b/readability/encoding.py
@@ -1,9 +1,8 @@
import re
try:
- import cchardet
+ import cchardet as chardet
except ImportError:
import chardet
-import sys
RE_CHARSET = re.compile(r']', flags=re.I)
diff --git a/readability/htmls.py b/readability/htmls.py
index 87299f5a..d99a9f53 100644
--- a/readability/htmls.py
+++ b/readability/htmls.py
@@ -110,27 +110,34 @@ def shorten_title(doc):
if e.text_content():
add_match(candidates, e.text_content(), orig)
+ cjk = re.compile('[\u4e00-\u9fff]+')
+
if candidates:
title = sorted(candidates, key=len)[-1]
else:
for delimiter in [" | ", " - ", " :: ", " / "]:
if delimiter in title:
parts = orig.split(delimiter)
- if len(parts[0].split()) >= 4:
- title = parts[0]
+ p0 = parts[0]
+ pl = parts[-1]
+ if (len(p0.split()) >= 4) or (len(p0) >= 4 and cjk.search(p0)):
+ title = p0
break
- elif len(parts[-1].split()) >= 4:
- title = parts[-1]
+ elif (len(pl.split()) >= 4) or (len(pl) >= 4 and cjk.search(pl)):
+ title = pl
break
else:
if ": " in title:
- parts = orig.split(": ")
- if len(parts[-1].split()) >= 4:
- title = parts[-1]
+ p1 = orig.split(": ")[-1]
+ if (len(p1.split()) >= 4) or (len(p1) >= 4 and cjk.search(p1)):
+ title = p1
else:
title = orig.split(": ", 1)[1]
- if not 15 < len(title) < 150:
+ if cjk.search(title):
+ if not (4 <= len(title) < 100): # Allow length >= 4, cap at 100
+ return orig
+ elif not 15 < len(title) < 150:
return orig
return title
diff --git a/readability/readability.py b/readability/readability.py
index c86e7d17..c5739056 100755
--- a/readability/readability.py
+++ b/readability/readability.py
@@ -42,11 +42,11 @@
"divToPElementsRe": re.compile(
r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)", re.I
),
- #'replaceBrsRe': re.compile(r'(
]*>[ \n\r\t]*){2,}',re.I),
- #'replaceFontsRe': re.compile(r'<(\/?)font[^>]*>',re.I),
- #'trimRe': re.compile(r'^\s+|\s+$/'),
- #'normalizeRe': re.compile(r'\s{2,}/'),
- #'killBreaksRe': re.compile(r'(
(\s| ?)*){1,}/'),
+ # 'replaceBrsRe': re.compile(r'(
]*>[ \n\r\t]*){2,}',re.I),
+ # 'replaceFontsRe': re.compile(r'<(\/?)font[^>]*>',re.I),
+ # 'trimRe': re.compile(r'^\s+|\s+$/'),
+ # 'normalizeRe': re.compile(r'\s{2,}/'),
+ # 'killBreaksRe': re.compile(r'(
(\s| ?)*){1,}/'),
"videoRe": re.compile(r"https?:\/\/(www\.)?(youtube|vimeo)\.com", re.I),
# skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i,
}
@@ -210,12 +210,13 @@ def get_clean_html(self):
"""
return clean_attributes(tounicode(self.html, method="html"))
- def summary(self, html_partial=False):
+ def summary(self, html_partial=False, keep_all_images=False):
"""
Given a HTML file, extracts the text of the article.
:param html_partial: return only the div of the document, don't wrap
in html and body tags.
+ :param keep_all_images: Keep all images in summary.
Warning: It mutates internal DOM representation of the HTML document,
so it is better to call other API methods before this one.
@@ -257,7 +258,7 @@ def summary(self, html_partial=False):
article = self.html.find("body")
if article is None:
article = self.html
- cleaned_article = self.sanitize(article, candidates)
+ cleaned_article = self.sanitize(article, candidates, keep_all_images)
article_length = len(cleaned_article or "")
retry_length = self.retry_length
@@ -502,7 +503,7 @@ def reverse_tags(self, node, *tag_names):
for tag_name in tag_names:
yield from reversed(node.findall(".//%s" % tag_name))
- def sanitize(self, node, candidates):
+ def sanitize(self, node, candidates, keep_all_images=False):
MIN_LEN = self.min_text_length
for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"):
if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33:
@@ -563,8 +564,8 @@ def sanitize(self, node, candidates):
to_remove = False
reason = ""
- # if el.tag == 'div' and counts["img"] >= 1:
- # continue
+ if keep_all_images and el.tag == 'div' and counts["img"] >= 1:
+ continue
if counts["p"] and counts["img"] > 1 + counts["p"] * 1.3:
reason = "too many images (%s)" % counts["img"]
to_remove = True
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 4731fa9d..996bbfc0 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,6 +1,3 @@
-lxml
-chardet
nose
-pep8
-coverage
-wrapt-timeout-decorator
+twine
+flake8
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index d6e1198b..00000000
--- a/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
--e .
diff --git a/setup.py b/setup.py
index 26894d47..dfb0846f 100755
--- a/setup.py
+++ b/setup.py
@@ -9,14 +9,8 @@
"cchardet",
]
-test_deps = [
- # Test timeouts
- "wrapt-timeout-decorator",
-]
-
extras = {
'speed': speed_deps,
- 'test': test_deps,
}
# Adapted from https://github.com/pypa/pip/blob/master/setup.py
@@ -43,8 +37,8 @@ def find_version(*file_paths):
author_email="burchik@gmail.com",
description="fast html to text parser (article readability tool) with python 3 support",
test_suite="tests.test_article_only",
- long_description=open("README.rst").read(),
- long_description_content_type='text/x-rst',
+ long_description=open("README.md").read(),
+ long_description_content_type="text/markdown",
license="Apache License 2.0",
url="http://github.com/buriy/python-readability",
packages=["readability"],
@@ -54,7 +48,6 @@ def find_version(*file_paths):
"lxml-html-clean; python_version < '3.11'",
"cssselect"
],
- tests_require=test_deps,
extras_require=extras,
classifiers=[
"Environment :: Web Environment",
@@ -72,6 +65,7 @@ def find_version(*file_paths):
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
+ "Programming Language :: Python :: 3.14",
"Programming Language :: Python :: Implementation :: PyPy",
],
)
diff --git a/tests/samples/summary-keep-all-images.sample.html b/tests/samples/summary-keep-all-images.sample.html
new file mode 100644
index 00000000..127683fc
--- /dev/null
+++ b/tests/samples/summary-keep-all-images.sample.html
@@ -0,0 +1,29 @@
+
+
+
+
+
+
+ H2 Headline H2 Headline H2 Headline H2 Headline H2 Headline H2 Headline H2 Headline H2 Headline H2 Headline H2 Headline
+
+
+
+
+ Text Text Text Text Text Text Text Text Text Text
+
+
+
+
+
+ Text Text Text Text Text Text Text Text Text Text
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_article_only.py b/tests/test_article_only.py
index c5592cfb..fe322121 100644
--- a/tests/test_article_only.py
+++ b/tests/test_article_only.py
@@ -1,8 +1,34 @@
import os
+import time
import unittest
from readability import Document
-from wrapt_timeout_decorator import *
+from functools import wraps
+
+
+class TimeoutException(Exception):
+ """Exception raised when a function exceeds its time limit."""
+ pass
+
+
+def timeout(seconds):
+ """Decorator to enforce a timeout on function execution."""
+ def decorator(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ start_time = time.perf_counter()
+ result = func(*args, **kwargs)
+ end_time = time.perf_counter()
+ elapsed_time = end_time - start_time
+ if elapsed_time > seconds:
+ raise TimeoutException(
+ f"Function '{func.__name__}' exceeded time limit of {seconds} seconds "
+ f"with an execution time of {elapsed_time:.4f} seconds"
+ )
+ return result
+ return wrapper
+ return decorator
+
SAMPLES = os.path.join(os.path.dirname(__file__), "samples")
@@ -100,7 +126,7 @@ def test_correct_cleanup(self):
assert not "aside" in s
# Many spaces make some regexes run forever
- @timeout(3, use_signals=False)
+ @timeout(3)
def test_many_repeated_spaces(self):
long_space = " " * 1000000
sample = "foo" + long_space + "
"
@@ -123,6 +149,7 @@ def test_utf8_kanji(self):
sample = load_sample("utf-8-kanji.sample.html")
doc = Document(sample)
res = doc.summary()
+ assert 0 < len(res) < 10000
def test_author_present(self):
sample = load_sample("the-hurricane-rubin-carter-denzel-washington.html")
@@ -133,3 +160,74 @@ def test_author_absent(self):
sample = load_sample("si-game.sample.html")
doc = Document(sample)
assert '[no-author]' == doc.author()
+
+ def test_keep_images_present(self):
+ sample = load_sample("summary-keep-all-images.sample.html")
+
+ doc = Document(sample)
+
+ assert "
+
+ 这是标题
+
+
+ 一些无关紧要的内容
+
+
主要文章标题
+
这是主要内容的第一段。
+
これはコンテンツの第2段落です。
+
이것은 콘텐츠의 세 번째 단락입니다.
+
This is the fourth paragraph.
+
+ More irrelevant stuff
+
+