gh-153404: Silently ignore non-decimal digits in robots.txt Crawl-delay and Request-rate

str.isdigit() returns True for non-decimal Unicode digits such as
U+00B2 SUPERSCRIPT TWO, which int() then rejects with ValueError. In
RobotFileParser.parse() this raised a raw ValueError that aborted parsing
of the entire robots.txt file, unlike every other malformed directive
which is silently ignored. Guard the Crawl-delay and Request-rate
conversions with str.isdecimal() so a value written with non-decimal
digits is ignored instead of raising.

Author: tonghuaroot

Lib/test/test_robotparser.py

@@ -246,6 +246,28 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase):
     bad = []
 
 
+class NonDecimalCrawlDelayTest(BaseRequestRateTest, unittest.TestCase):
+    # Non-decimal Unicode digits pass str.isdigit() but int() rejects
+    # them, so the directive must be silently ignored, not raise.
+    robots_txt = """\
+User-Agent: *
+Disallow: /.
+Crawl-delay: ²
+    """
+    good = ['/foo.html']
+    bad = []
+
+
+class NonDecimalRequestRateTest(BaseRequestRateTest, unittest.TestCase):
+    robots_txt = """\
+User-agent: *
+Disallow: /tmp/
+Request-rate: ²/5
+    """
+    good = ['/foo.html']
+    bad = ['/tmp/']
+
+
 class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
     # also test that Allow and Diasallow works well with each other
     robots_txt = """\

Lib/urllib/robotparser.py

@@ -148,15 +148,15 @@ def parse(self, lines):
                         # before trying to convert to int we need to make
                         # sure that robots.txt has valid syntax otherwise
                         # it will crash
-                        if line[1].strip().isdigit():
+                        if line[1].strip().isdecimal():
                             entry.delay = int(line[1])
                         state = 2
                 elif line[0] == "request-rate":
                     if state != 0:
                         numbers = line[1].split('/')
                         # check if all values are sane
-                        if (len(numbers) == 2 and numbers[0].strip().isdigit()
-                            and numbers[1].strip().isdigit()):
+                        if (len(numbers) == 2 and numbers[0].strip().isdecimal()
+                            and numbers[1].strip().isdecimal()):
                             entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
                         state = 2
                 elif line[0] == "sitemap":

Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst

@@ -0,0 +1,4 @@
+:class:`urllib.robotparser.RobotFileParser` now silently ignores a
+``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits
+(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError`
+and aborting the parse of the whole ``robots.txt`` file.