From e464c0c20cc4649718f68e16367ff9dc21d090a8 Mon Sep 17 00:00:00 2001 From: Michael Lazar Date: Mon, 14 May 2018 10:10:41 -0400 Subject: [PATCH 1/4] bpo-32861: urllib.robotparser fix incomplete __str__ methods. (GH-5711) The urllib.robotparser's __str__ representation now includes wildcard entries and the "Crawl-delay" and "Request-rate" fields. Also removes extra newlines that were being appended to the end of the string. (cherry picked from commit bd08a0af2d88c590ede762102bd42da3437e9980) Co-authored-by: Michael Lazar --- Lib/test/test_robotparser.py | 26 +++++++++++++++++++ Lib/urllib/robotparser.py | 17 ++++++++---- .../2018-04-02-20-44-54.bpo-32861.HeBjzN.rst | 4 +++ 3 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 75198b70ad4ff5f..bee8d238be6b2fa 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -246,6 +246,32 @@ class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): bad = ['/cyberworld/map/index.html'] +class StringFormattingTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/ # This is an infinite virtual URL space + +# Cybermapper knows where to go. +User-agent: cybermapper +Disallow: /some/path + """ + + expected_output = """\ +User-agent: cybermapper +Disallow: /some/path + +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/\ +""" + + def test_string_formatting(self): + self.assertEqual(str(self.parser), self.expected_output) + + class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index daac29c68dc36d2..92e4efe6865e1f8 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -190,7 +190,10 @@ def request_rate(self, useragent): return self.default_entry.req_rate def __str__(self): - return ''.join([str(entry) + "\n" for entry in self.entries]) + entries = self.entries + if self.default_entry is not None: + entries = entries + [self.default_entry] + return '\n\n'.join(map(str, entries)) class RuleLine: @@ -222,10 +225,14 @@ def __init__(self): def __str__(self): ret = [] for agent in self.useragents: - ret.extend(["User-agent: ", agent, "\n"]) - for line in self.rulelines: - ret.extend([str(line), "\n"]) - return ''.join(ret) + ret.append(f"User-agent: {agent}") + if self.delay is not None: + ret.append(f"Crawl-delay: {self.delay}") + if self.req_rate is not None: + rate = self.req_rate + ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") + ret.extend(map(str, self.rulelines)) + return '\n'.join(ret) def applies_to(self, useragent): """check if this entry applies to the specified agent""" diff --git a/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst new file mode 100644 index 000000000000000..2bb24d7edab54d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst @@ -0,0 +1,4 @@ +The urllib.robotparser's ``__str__`` representation now includes wildcard +entries and the "Crawl-delay" and "Request-rate" fields. Also removes extra +newlines that were being appended to the end of the string. Patch by +Michael Lazar. From e00ecb39aa1d09f8aa4807634bc3eea8f4879f2c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 14 May 2018 17:24:37 +0300 Subject: [PATCH 2/4] Update robotparser.py --- Lib/urllib/robotparser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 92e4efe6865e1f8..883ef249210ebce 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -193,7 +193,7 @@ def __str__(self): entries = self.entries if self.default_entry is not None: entries = entries + [self.default_entry] - return '\n\n'.join(map(str, entries)) + return '\n'.join(map(str, entries)) + '\n' class RuleLine: @@ -232,6 +232,7 @@ def __str__(self): rate = self.req_rate ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") ret.extend(map(str, self.rulelines)) + ret.append('') # for compatibility return '\n'.join(ret) def applies_to(self, useragent): From d2039d1a728eec37c2ab9242013085d4a197a65b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 14 May 2018 17:25:05 +0300 Subject: [PATCH 3/4] Update test_robotparser.py --- Lib/test/test_robotparser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index bee8d238be6b2fa..140636590aa8600 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -265,7 +265,8 @@ class StringFormattingTest(BaseRobotTest, unittest.TestCase): User-agent: * Crawl-delay: 1 Request-rate: 3/15 -Disallow: /cyberworld/map/\ +Disallow: /cyberworld/map/ + """ def test_string_formatting(self): From d43db084ee812a808e3d58e98bfc0ad5a0c9bee4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 14 May 2018 17:25:37 +0300 Subject: [PATCH 4/4] Update 2018-04-02-20-44-54.bpo-32861.HeBjzN.rst --- .../next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst index 2bb24d7edab54d8..13defbb03cff630 100644 --- a/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst +++ b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst @@ -1,4 +1,3 @@ The urllib.robotparser's ``__str__`` representation now includes wildcard -entries and the "Crawl-delay" and "Request-rate" fields. Also removes extra -newlines that were being appended to the end of the string. Patch by +entries and the "Crawl-delay" and "Request-rate" fields. Patch by Michael Lazar.