From e7b777ff6b4be35c674102cc787187c9d3543126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Fri, 8 Feb 2019 17:02:50 +0100 Subject: [PATCH 01/16] Fix RobotFileParser when robot.txt is invalid --- Lib/test/test_robotparser.py | 12 ++++++++++++ Lib/urllib/robotparser.py | 6 ++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 84a267ad9567ee7..545e952b40aa09c 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -96,6 +96,18 @@ class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): bad = ['/cyberworld/map/index.html', '/', '/tmp/'] +class InvalidFileTest(BaseRobotTest, unittest.TestCase): + robots_txt = "" + + def test_request_rate(self): + agent, url = self.get_agent_and_url("/foo") + self.assertEqual(self.parser.crawl_delay(agent), None) + + def test_crawl_delay(self): + agent, url = self.get_agent_and_url("/foo") + self.assertEqual(self.parser.crawl_delay(agent), None) + + class BaseRequestRateTest(BaseRobotTest): def test_request_rate(self): diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 7089916a4f81cc0..fc06e7f50ff61a3 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -186,7 +186,8 @@ def crawl_delay(self, useragent): for entry in self.entries: if entry.applies_to(useragent): return entry.delay - return self.default_entry.delay + if self.default_entry: + return self.default_entry.delay def request_rate(self, useragent): if not self.mtime(): @@ -194,7 +195,8 @@ def request_rate(self, useragent): for entry in self.entries: if entry.applies_to(useragent): return entry.req_rate - return self.default_entry.req_rate + if self.default_entry: + return self.default_entry.req_rate def site_maps(self): if not self.sitemaps: From decb9318c90c06e22df16301149035d06d870b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Sun, 9 Jun 2019 18:30:37 +0200 Subject: [PATCH 02/16] Rename test case Co-Authored-By: Tal Einat --- Lib/test/test_robotparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 545e952b40aa09c..0f2a02ffcd8fb11 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -96,7 +96,7 @@ class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): bad = ['/cyberworld/map/index.html', '/', '/tmp/'] -class InvalidFileTest(BaseRobotTest, unittest.TestCase): +class EmptyFileTest(BaseRobotTest, unittest.TestCase): robots_txt = "" def test_request_rate(self): From b219f9dcc3b704353a9b6fa6b8391391af275aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Sun, 9 Jun 2019 18:35:40 +0200 Subject: [PATCH 03/16] Return None explicitly --- Lib/urllib/robotparser.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index fc06e7f50ff61a3..854db9f65d79aa6 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -188,6 +188,8 @@ def crawl_delay(self, useragent): return entry.delay if self.default_entry: return self.default_entry.delay + else: + return None def request_rate(self, useragent): if not self.mtime(): @@ -197,6 +199,8 @@ def request_rate(self, useragent): return entry.req_rate if self.default_entry: return self.default_entry.req_rate + else: + return None def site_maps(self): if not self.sitemaps: From f4177279899c10826dd9cbe2328e6b853ccd0189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Sun, 9 Jun 2019 18:48:17 +0200 Subject: [PATCH 04/16] Add tests for bad user-agent and missing attributes --- Lib/test/test_robotparser.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 0f2a02ffcd8fb11..7a5bbc77da4f05e 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -108,6 +108,37 @@ def test_crawl_delay(self): self.assertEqual(self.parser.crawl_delay(agent), None) +class NoDefaultUserAgentTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: figtree +Crawl-delay: 1 +Request-rate: 3/15 + """ + + def test_request_rate(self): + agent, url = self.get_agent_and_url("/foo") + self.assertEqual(self.parser.crawl_delay(agent), None) + + def test_crawl_delay(self): + agent, url = self.get_agent_and_url("/foo") + self.assertEqual(self.parser.crawl_delay(agent), None) + + +class NoRequestRateAndCrawlDelayTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /bar + """ + + def test_request_rate(self): + agent, url = self.get_agent_and_url("/foo") + self.assertEqual(self.parser.crawl_delay(agent), None) + + def test_crawl_delay(self): + agent, url = self.get_agent_and_url("/foo") + self.assertEqual(self.parser.crawl_delay(agent), None) + + class BaseRequestRateTest(BaseRobotTest): def test_request_rate(self): From 00b918ac3c60cf70e739da3dc6824ab26ef9f0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Tue, 11 Jun 2019 11:34:39 +0200 Subject: [PATCH 05/16] Always test craw_delay and request_rate --- Lib/test/test_robotparser.py | 51 +++++++----------------------------- 1 file changed, 10 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 7a5bbc77da4f05e..22bd8baadac839b 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -99,30 +99,6 @@ class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): class EmptyFileTest(BaseRobotTest, unittest.TestCase): robots_txt = "" - def test_request_rate(self): - agent, url = self.get_agent_and_url("/foo") - self.assertEqual(self.parser.crawl_delay(agent), None) - - def test_crawl_delay(self): - agent, url = self.get_agent_and_url("/foo") - self.assertEqual(self.parser.crawl_delay(agent), None) - - -class NoDefaultUserAgentTest(BaseRobotTest, unittest.TestCase): - robots_txt = """\ -User-agent: figtree -Crawl-delay: 1 -Request-rate: 3/15 - """ - - def test_request_rate(self): - agent, url = self.get_agent_and_url("/foo") - self.assertEqual(self.parser.crawl_delay(agent), None) - - def test_crawl_delay(self): - agent, url = self.get_agent_and_url("/foo") - self.assertEqual(self.parser.crawl_delay(agent), None) - class NoRequestRateAndCrawlDelayTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ @@ -130,14 +106,6 @@ class NoRequestRateAndCrawlDelayTest(BaseRobotTest, unittest.TestCase): Disallow: /bar """ - def test_request_rate(self): - agent, url = self.get_agent_and_url("/foo") - self.assertEqual(self.parser.crawl_delay(agent), None) - - def test_crawl_delay(self): - agent, url = self.get_agent_and_url("/foo") - self.assertEqual(self.parser.crawl_delay(agent), None) - class BaseRequestRateTest(BaseRobotTest): @@ -145,11 +113,14 @@ def test_request_rate(self): for url in self.good + self.bad: agent, url = self.get_agent_and_url(url) with self.subTest(url=url, agent=agent): - if self.crawl_delay: - self.assertEqual( - self.parser.crawl_delay(agent), self.crawl_delay - ) - if self.request_rate: + self.assertEqual( + self.parser.crawl_delay(agent), self.crawl_delay + ) + self.assertEqual( + self.parser.request_rate(agent), + self.request_rate + ) + if self.request_rate is not None: self.assertIsInstance( self.parser.request_rate(agent), urllib.robotparser.RequestRate @@ -184,10 +155,8 @@ class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): class DifferentAgentTest(CrawlDelayAndRequestRateTest): agent = 'FigTree Robot libwww-perl/5.04' - # these are not actually tested, but we still need to parse it - # in order to accommodate the input parameters - request_rate = None - crawl_delay = None + request_rate = urllib.robotparser.RequestRate(9, 30) + crawl_delay = 3 class InvalidRequestRateTest(BaseRobotTest, unittest.TestCase): From 8680e41c2d96f0a8b83db6ea4e1df0cfd38cc03e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Tue, 11 Jun 2019 13:11:17 +0200 Subject: [PATCH 06/16] Remove NoRequestRateAndCrawlDelayTest --- Lib/test/test_robotparser.py | 7 ------- Misc/python-embed.pc | 13 +++++++++++++ 2 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 Misc/python-embed.pc diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 22bd8baadac839b..6865963c6d15ac3 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -100,13 +100,6 @@ class EmptyFileTest(BaseRobotTest, unittest.TestCase): robots_txt = "" -class NoRequestRateAndCrawlDelayTest(BaseRobotTest, unittest.TestCase): - robots_txt = """\ -User-agent: * -Disallow: /bar - """ - - class BaseRequestRateTest(BaseRobotTest): def test_request_rate(self): diff --git a/Misc/python-embed.pc b/Misc/python-embed.pc new file mode 100644 index 000000000000000..46dc3cbdb122046 --- /dev/null +++ b/Misc/python-embed.pc @@ -0,0 +1,13 @@ +# See: man pkg-config +prefix=/usr/local +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include + +Name: Python +Description: Embed Python into an application +Requires: +Version: 3.9 +Libs.private: -ldl -framework CoreFoundation +Libs: -L${libdir} -lpython3.9d +Cflags: -I${includedir}/python3.9d From 1e2a03eb93f43824a0229ee0126eb4ae9219d743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Tue, 11 Jun 2019 13:34:31 +0200 Subject: [PATCH 07/16] Remove useless attributes in DifferentAgentTest --- Lib/test/test_robotparser.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 6865963c6d15ac3..d350a9f2a29cc93 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -148,8 +148,6 @@ class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): class DifferentAgentTest(CrawlDelayAndRequestRateTest): agent = 'FigTree Robot libwww-perl/5.04' - request_rate = urllib.robotparser.RequestRate(9, 30) - crawl_delay = 3 class InvalidRequestRateTest(BaseRobotTest, unittest.TestCase): From 4693573b09c7d018205435ac35212cd26fbc92c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Tue, 11 Jun 2019 13:34:48 +0200 Subject: [PATCH 08/16] Remove Misc/python-embed.pc --- Misc/python-embed.pc | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 Misc/python-embed.pc diff --git a/Misc/python-embed.pc b/Misc/python-embed.pc deleted file mode 100644 index 46dc3cbdb122046..000000000000000 --- a/Misc/python-embed.pc +++ /dev/null @@ -1,13 +0,0 @@ -# See: man pkg-config -prefix=/usr/local -exec_prefix=${prefix} -libdir=${exec_prefix}/lib -includedir=${prefix}/include - -Name: Python -Description: Embed Python into an application -Requires: -Version: 3.9 -Libs.private: -ldl -framework CoreFoundation -Libs: -L${libdir} -lpython3.9d -Cflags: -I${includedir}/python3.9d From 3125ee9ba2c4ae422c0cb5688bdaaffb56869319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Tue, 11 Jun 2019 13:38:08 +0200 Subject: [PATCH 09/16] Fix EmptyFileTest --- Lib/test/test_robotparser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index d350a9f2a29cc93..5280d74e9193c86 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -98,6 +98,7 @@ class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): class EmptyFileTest(BaseRobotTest, unittest.TestCase): robots_txt = "" + good = ['/foo'] class BaseRequestRateTest(BaseRobotTest): From 0988fb2a26652d231e1cc1f15f837fe9c95fb34c Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Tue, 11 Jun 2019 19:35:27 +0300 Subject: [PATCH 10/16] add a NEWS entry --- .../next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst diff --git a/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst b/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst new file mode 100644 index 000000000000000..e2ca0446878be28 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst @@ -0,0 +1,3 @@ +Fix RobotParser's :meth:``crawl_delay()`` and :meth:``request_rate()`` +to return ``None`` rather than raise ``AttributeError`` when no +relevant rule is defined in the robots.txt file. PR by Rémi Lapeyre. From 20dce064116859c00223974060f5c6d7db91e4c2 Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Wed, 12 Jun 2019 09:18:31 +0300 Subject: [PATCH 11/16] tidy up test_request_rate code --- Lib/test/test_robotparser.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 5280d74e9193c86..734feeb75fc454d 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -104,27 +104,25 @@ class EmptyFileTest(BaseRobotTest, unittest.TestCase): class BaseRequestRateTest(BaseRobotTest): def test_request_rate(self): + parser = self.parser for url in self.good + self.bad: agent, url = self.get_agent_and_url(url) with self.subTest(url=url, agent=agent): - self.assertEqual( - self.parser.crawl_delay(agent), self.crawl_delay - ) - self.assertEqual( - self.parser.request_rate(agent), - self.request_rate - ) + self.assertEqual(parser.crawl_delay(agent), self.crawl_delay) + + parsed_request_rate = parser.request_rate(agent) + self.assertEqual(parsed_request_rate, self.request_rate) if self.request_rate is not None: self.assertIsInstance( - self.parser.request_rate(agent), + parsed_request_rate, urllib.robotparser.RequestRate ) self.assertEqual( - self.parser.request_rate(agent).requests, + parsed_request_rate.requests, self.request_rate.requests ) self.assertEqual( - self.parser.request_rate(agent).seconds, + parsed_request_rate.seconds, self.request_rate.seconds ) From 5d64344fb6dda21c3faa41aff32a02e60e1765f4 Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Wed, 12 Jun 2019 09:19:04 +0300 Subject: [PATCH 12/16] fix BaseRequestRateTest and make EmptyFileTest inherit it --- Lib/test/test_robotparser.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 734feeb75fc454d..d0d0d4bfe69b25b 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -96,12 +96,9 @@ class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): bad = ['/cyberworld/map/index.html', '/', '/tmp/'] -class EmptyFileTest(BaseRobotTest, unittest.TestCase): - robots_txt = "" - good = ['/foo'] - - class BaseRequestRateTest(BaseRobotTest): + request_rate = None + crawl_delay = None def test_request_rate(self): parser = self.parser @@ -127,6 +124,11 @@ def test_request_rate(self): ) +class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = "" + good = ['/foo'] + + class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): robots_txt = """\ User-agent: figtree From aaade01269b75546fd896fe5d478fcb3788f89da Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Wed, 12 Jun 2019 12:55:52 +0300 Subject: [PATCH 13/16] improve NEWS entry formatting --- .../next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst b/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst index e2ca0446878be28..c7f5b61b290a2ee 100644 --- a/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst +++ b/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst @@ -1,3 +1,4 @@ -Fix RobotParser's :meth:``crawl_delay()`` and :meth:``request_rate()`` -to return ``None`` rather than raise ``AttributeError`` when no -relevant rule is defined in the robots.txt file. PR by Rémi Lapeyre. +Fix :meth:``RobotFileParser.crawl_delay()`` and +:meth:``RobotFileParser.request_rate()`` to return ``None`` rather than +raise :exc:``AttributeError`` when no relevant rule is defined in the +robots.txt file. PR by Rémi Lapeyre. From 4e6a9bdfd5064dd54275860dbdd31d4176d0849f Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Wed, 12 Jun 2019 12:59:29 +0300 Subject: [PATCH 14/16] Update 2019-06-11-19-34-29.bpo-35922.rxpzWr.rst --- .../next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst b/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst index c7f5b61b290a2ee..7a9496d61929fd0 100644 --- a/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst +++ b/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst @@ -1,4 +1,4 @@ Fix :meth:``RobotFileParser.crawl_delay()`` and :meth:``RobotFileParser.request_rate()`` to return ``None`` rather than raise :exc:``AttributeError`` when no relevant rule is defined in the -robots.txt file. PR by Rémi Lapeyre. +robots.txt file. Patch by Rémi Lapeyre. From d2e9b69cd3997a6780c03e092e3f7cebfa7bb090 Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Wed, 12 Jun 2019 13:20:18 +0300 Subject: [PATCH 15/16] fix NEWS entry formatting --- .../next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst b/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst index 7a9496d61929fd0..df3216687600575 100644 --- a/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst +++ b/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst @@ -1,4 +1,4 @@ -Fix :meth:``RobotFileParser.crawl_delay()`` and -:meth:``RobotFileParser.request_rate()`` to return ``None`` rather than -raise :exc:``AttributeError`` when no relevant rule is defined in the +Fix :meth:`RobotFileParser.crawl_delay()` and +:meth:`RobotFileParser.request_rate()` to return ``None`` rather than +raise :exc:`AttributeError` when no relevant rule is defined in the robots.txt file. Patch by Rémi Lapeyre. From 67a6ee270bf2372234b8d5ffc784e9b710b382c8 Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Wed, 12 Jun 2019 21:34:48 +0300 Subject: [PATCH 16/16] address Berker Peksag's review comments --- Lib/test/test_robotparser.py | 2 +- Lib/urllib/robotparser.py | 6 ++---- .../next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index d0d0d4bfe69b25b..77cd7c4d29dfe6f 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -125,7 +125,7 @@ def test_request_rate(self): class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): - robots_txt = "" + robots_txt = '' good = ['/foo'] diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 854db9f65d79aa6..c58565e39451461 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -188,8 +188,7 @@ def crawl_delay(self, useragent): return entry.delay if self.default_entry: return self.default_entry.delay - else: - return None + return None def request_rate(self, useragent): if not self.mtime(): @@ -199,8 +198,7 @@ def request_rate(self, useragent): return entry.req_rate if self.default_entry: return self.default_entry.req_rate - else: - return None + return None def site_maps(self): if not self.sitemaps: diff --git a/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst b/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst index df3216687600575..5271a495624d60c 100644 --- a/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst +++ b/Misc/NEWS.d/next/Library/2019-06-11-19-34-29.bpo-35922.rxpzWr.rst @@ -1,4 +1,4 @@ -Fix :meth:`RobotFileParser.crawl_delay()` and -:meth:`RobotFileParser.request_rate()` to return ``None`` rather than +Fix :meth:`RobotFileParser.crawl_delay` and +:meth:`RobotFileParser.request_rate` to return ``None`` rather than raise :exc:`AttributeError` when no relevant rule is defined in the robots.txt file. Patch by Rémi Lapeyre.