From 22ea7f52118f8bb8c4317f1f729ddd898bdc820a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 5 Jul 2026 14:13:01 +0300 Subject: [PATCH] gh-79638: Restore "Treat an unreachable robots.txt as disallow all" (GH-152525) This change was accidentally reverted by f0daba1652c (gh-106693, GH-149514), which only intended to revert the ob_sval change. The tests were already restored by GH-149569. (cherry picked from commit 70100b9ea0b43a95226eee45652a5735bb91e9c7) Co-authored-by: Serhiy Storchaka Co-authored-by: Claude Opus 4.8 (1M context) --- Lib/urllib/robotparser.py | 10 +++++++++- .../2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index e70eae80036784..0c3e5d92890935 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -65,9 +65,17 @@ def read(self): f = urllib.request.urlopen(self.url) except urllib.error.HTTPError as err: if err.code in (401, 403): + # If access to robot.txt has the status Unauthorized/Forbidden, + # then most likely this applies to the entire site. self.disallow_all = True - elif err.code >= 400 and err.code < 500: + elif 400 <= err.code < 500: + # RFC 9309, Section 2.3.1.3: the crawler MAY access any + # resources on the server. self.allow_all = True + elif 500 <= err.code < 600: + # RFC 9309, Section 2.3.1.4: the crawler MUST assume + # complete disallow. + self.disallow_all = True err.close() else: raw = f.read() diff --git a/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst new file mode 100644 index 00000000000000..bd9fff0bc2e31b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst @@ -0,0 +1,2 @@ +Disallow all access in :mod:`urllib.robotparser` if the ``robots.txt`` file +is unreachable due to server or network errors.