Skip to content

Commit 5cb4b8b

Browse files
committed
Tweaks after the code reorg
1 parent f8315d0 commit 5cb4b8b

2 files changed

Lines changed: 26 additions & 3 deletions

File tree

README.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ Optional `Document` keyword argument:
6969
- attributes:
7070
- debug: output debug messages
7171
- min_text_length:
72+
- multipage: should we try to parse and combine multiple page articles?
7273
- retry_length:
7374
- url: will allow adjusting links to be absolute
7475

src/tests/test_readability.py

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
import os
12
import unittest
23

34
from helpers import load_regression_data
5+
from helpers import REGRESSION_DATA
46
from readability_lxml.readability import Document
57
from readability_lxml import readability as r
68
from readability_lxml import urlfetch
@@ -143,13 +145,28 @@ def test_short(self):
143145
self._run_urls(specs)
144146

145147

148+
class TestMultiPageHelpers(unittest.TestCase):
149+
150+
def test_find_next_page_url(self):
151+
"""Verify we can find a next page url in the html body"""
152+
html = """
153+
<html><body><a href="/?page=2">2</a></body></html>
154+
"""
155+
from lxml.html import document_fromstring
156+
doc = document_fromstring(html)
157+
158+
res = r.find_next_page_url(set(), None, doc)
159+
self.assertEqual('/?page=2', res,
160+
'Should find out page 2 url in the body.')
161+
162+
146163
class TestFindNextPageLink(unittest.TestCase):
147164

148165
def _test_page(self, url, html_path, expected):
149166
html = load_regression_data(html_path)
150167
doc = r.parse(html, url)
151168
parsed_urls = {url}
152-
actual = r.find_next_page_link(parsed_urls, url, doc)
169+
actual = r.find_next_page_url(parsed_urls, url, doc)
153170
self.assertEqual(expected, actual)
154171

155172
def test_basic(self):
@@ -178,7 +195,8 @@ class TestMultiPage(unittest.TestCase):
178195
def _make_basic_urldict(self):
179196
url_fmt = 'http://basic.com/article.html?pagewanted=%s'
180197
file_fmt = 'basic-multi-page-%s.html'
181-
pairs = [(url_fmt % i, file_fmt % i) for i in ['2', '3']]
198+
199+
pairs = [(url_fmt % i, os.path.join(REGRESSION_DATA, file_fmt % i)) for i in ['2', '3']]
182200
return dict(pairs)
183201

184202
def test_basic(self):
@@ -187,7 +205,11 @@ def test_basic(self):
187205
fetcher = urlfetch.MockUrlFetch(urldict)
188206
options = {
189207
'url': 'http://basic.com/article.html',
208+
'multipage': True,
190209
'urlfetch': fetcher
191210
}
192211
doc = Document(html, **options)
193-
doc.summary()
212+
res = doc.summary()
213+
214+
self.assertIn('Page 2', res, 'Should find the page 2 heading')
215+
self.assertIn('Page 3', res, 'Should find the page 3 heading')

0 commit comments

Comments
 (0)