Merge branch 'main' into gh-102591-document-urlcleanup

Author: orsenthil

Doc/library/asyncio-queue.rst

@@ -71,6 +71,8 @@ Queue
       Return an item if one is immediately available, else raise
       :exc:`QueueEmpty`.
 
+      Raises :exc:`QueueShutDown` if the queue has been shut down and is empty.
+
    .. method:: join()
       :async:
 
@@ -96,6 +98,8 @@ Queue
 
       If no free slot is immediately available, raise :exc:`QueueFull`.
 
+      Raises :exc:`QueueShutDown` if the queue has been shut down.
+
    .. method:: qsize()
 
       Return the number of items in the queue.
@@ -188,8 +192,9 @@ Exceptions
 
 .. exception:: QueueShutDown
 
-   Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is
-   called on a queue which has been shut down.
+   Exception raised when :meth:`~Queue.put`, :meth:`~Queue.put_nowait`,
+   :meth:`~Queue.get` or :meth:`~Queue.get_nowait` is called
+   on a queue which has been shut down.
 
    .. versionadded:: 3.13
 

Doc/tutorial/introduction.rst

@@ -532,10 +532,13 @@ This example introduces several new features.
   and ``!=`` (not equal to).
 
 * The *body* of the loop is *indented*: indentation is Python's way of grouping
-  statements.  At the interactive prompt, you have to type a tab or space(s) for
-  each indented line.  In practice you will prepare more complicated input
-  for Python with a text editor; all decent text editors have an auto-indent
-  facility.  When a compound statement is entered interactively, it must be
+  statements.  At the interactive prompt, the default REPL automatically
+  indents continuation lines after compound statement headers like ``if`` or
+  ``while``.  In the basic REPL (invoked with :envvar:`PYTHON_BASIC_REPL`)
+  or in older Python versions, you have to type a tab or space(s) for each
+  indented line manually.  In practice you will prepare more complicated
+  input for Python with a text editor; all decent text editors have an
+  auto-indent facility.  When a compound statement is entered interactively, it must be
   followed by a blank line to indicate completion (since the parser cannot
   guess when you have typed the last line).  Note that each line within a basic
   block must be indented by the same amount.

Lib/_pyrepl/base_eventqueue.py

@@ -54,14 +54,6 @@ def empty(self) -> bool:
         """
         return not self.events
 
-    def flush_buf(self) -> bytearray:
-        """
-        Flushes the buffer and returns its contents.
-        """
-        old = self.buf
-        self.buf = bytearray()
-        return old
-
     def insert(self, event: Event) -> None:
         """
         Inserts an event into the queue.
@@ -87,7 +79,7 @@ def push(self, char: int | bytes) -> None:
             if isinstance(k, dict):
                 self.keymap = k
             else:
-                self.insert(Event('key', k, bytes(self.flush_buf())))
+                self.insert(Event('key', k, self.buf.take_bytes()))  # type: ignore[attr-defined]
                 self.keymap = self.compiled_keymap
 
         elif self.buf and self.buf[0] == 27:  # escape
@@ -97,14 +89,14 @@ def push(self, char: int | bytes) -> None:
             trace('unrecognized escape sequence, propagating...')
             self.keymap = self.compiled_keymap
             self.insert(Event('key', '\033', b'\033'))
-            for _c in self.flush_buf()[1:]:
+            for _c in self.buf.take_bytes()[1:]:  # type: ignore[attr-defined]
                 self.push(_c)
 
         else:
             try:
-                decoded = bytes(self.buf).decode(self.encoding)
+                decoded = self.buf.decode(self.encoding)
             except UnicodeError:
                 return
             else:
-                self.insert(Event('key', decoded, bytes(self.flush_buf())))
+                self.insert(Event('key', decoded, self.buf.take_bytes()))  # type: ignore[attr-defined]
             self.keymap = self.compiled_keymap

Lib/asyncio/streams.py

@@ -216,6 +216,9 @@ def _stream_reader(self):
     def _replace_transport(self, transport):
         self._transport = transport
         self._over_ssl = transport.get_extra_info('sslcontext') is not None
+        reader = self._stream_reader
+        if reader is not None:
+            reader._replace_transport(transport)
 
     def connection_made(self, transport):
         if self._reject_connection:
@@ -473,6 +476,10 @@ def set_transport(self, transport):
         assert self._transport is None, 'Transport already set'
         self._transport = transport
 
+    def _replace_transport(self, transport):
+        assert self._transport is not None, 'Transport not set'
+        self._transport = transport
+
     def _maybe_resume_transport(self):
         if self._paused and len(self._buffer) <= self._limit:
             self._paused = False

Lib/profiling/sampling/live_collector/collector.py

@@ -702,6 +702,7 @@ def build_stats_list(self):
     def reset_stats(self):
         """Reset all collected statistics."""
         self.result.clear()
+        self.opcode_stats.clear()
         self.per_thread_data.clear()
         self.thread_ids.clear()
         self.view_mode = "ALL"

Lib/test/test_asyncio/test_streams.py

@@ -861,6 +861,34 @@ async def run_test():
         self.loop.run_until_complete(run_test())
         self.assertEqual(messages, [])
 
+    def test_streamwriter_start_tls_updates_reader_transport(self):
+        reader = asyncio.StreamReader(loop=self.loop)
+        protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop)
+        old_transport = mock.Mock()
+        old_transport.get_extra_info.return_value = None
+        old_transport.is_closing.return_value = False
+        protocol.connection_made(old_transport)
+
+        writer = asyncio.StreamWriter(old_transport, protocol, reader, self.loop)
+
+        ssl_context = mock.sentinel.ssl_context
+        new_transport = mock.Mock()
+        new_transport.get_extra_info.return_value = ssl_context
+        self.loop.start_tls = mock.AsyncMock(return_value=new_transport)
+
+        self.loop.run_until_complete(writer.start_tls(ssl_context))
+
+        self.loop.start_tls.assert_awaited_once_with(
+            old_transport, protocol, ssl_context,
+            server_side=False, server_hostname=None,
+            ssl_handshake_timeout=None,
+            ssl_shutdown_timeout=None,
+        )
+        self.assertIs(writer.transport, new_transport)
+        self.assertIs(protocol._transport, new_transport)
+        self.assertIs(reader._transport, new_transport)
+        self.assertTrue(protocol._over_ssl)
+
     def test_streamreader_constructor_without_loop(self):
         with self.assertRaisesRegex(RuntimeError, 'no current event loop'):
             asyncio.StreamReader()

Lib/test/test_fstring.py

@@ -709,6 +709,16 @@ def test_double_braces(self):
                             ["f'{ {{}} }'", # dict in a set
                              ])
 
+    def test_double_brace_ast_location_covers_both_source_braces(self):
+        value = ast.parse('f"a{{"').body[0].value.values[0]
+        self.assertIsInstance(value, ast.Constant)
+        self.assertEqual(value.value, "a{")
+        self.assertEqual(
+            (value.lineno, value.col_offset, value.end_lineno,
+             value.end_col_offset),
+            (1, 2, 1, 5),
+        )
+
     def test_compile_time_concat(self):
         x = 'def'
         self.assertEqual('abc' f'## {x}ghi', 'abc## defghi')

Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py

@@ -99,6 +99,7 @@ def test_reset_stats(self):
             "cumulative_calls": 75,
             "total_rec_calls": 0,
         }
+        self.collector.opcode_stats[("test.py", 1, "func")][100] = 5
 
         # Reset
         self.collector.reset_stats()
@@ -107,6 +108,7 @@ def test_reset_stats(self):
         self.assertEqual(self.collector.successful_samples, 0)
         self.assertEqual(self.collector.failed_samples, 0)
         self.assertEqual(len(self.collector.result), 0)
+        self.assertEqual(len(self.collector.opcode_stats), 0)
 
     def test_increase_refresh_rate(self):
         """Test increasing refresh rate (faster updates)."""

Lib/test/test_pyrepl/test_eventqueue.py

@@ -38,12 +38,6 @@ def test_empty(self):
         eq.insert(Event("key", "a", b"a"))
         self.assertFalse(eq.empty())
 
-    def test_flush_buf(self):
-        eq = self.make_eventqueue()
-        eq.buf.extend(b"test")
-        self.assertEqual(eq.flush_buf(), b"test")
-        self.assertEqual(eq.buf, bytearray())
-
     def test_insert(self):
         eq = self.make_eventqueue()
         event = Event("key", "a", b"a")
@@ -93,7 +87,7 @@ def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap):
         eq.push(b"a")
         mock_keymap.compile_keymap.assert_called()
         self.assertTrue(eq.empty())
-        eq.flush_buf()
+        eq.buf.resize(0)
         eq.push(b"\033")
         self.assertEqual(eq.events[0].evt, "key")
         self.assertEqual(eq.events[0].data, "\033")

Lib/test/test_tokenize.py

@@ -2236,6 +2236,13 @@ def test_number_starting_with_zero(self):
         self.assertEqual(tokens, expected_tokens)
 
 class CTokenizeTest(TestCase):
+    @staticmethod
+    def _get_tokens(source, *, extra_tokens=False):
+        return list(tokenize._generate_tokens_from_c_tokenizer(
+            StringIO(source).readline,
+            extra_tokens=extra_tokens,
+        ))
+
     def check_tokenize(self, s, expected):
         # Format the tokens in s in a table format.
         # The ENDMARKER and final NEWLINE are omitted.
@@ -2266,6 +2273,159 @@ def readline(encoding):
                 ))
                 self.assertEqual(tokens, expected)
 
+    def test_extra_tokens_relaxes_lexer_errors(self):
+        cases = [
+            (
+                "2sin(x)",
+                ("invalid decimal literal", (1, 1)),
+                [
+                    (token.NUMBER, "2", (1, 0), (1, 1)),
+                    (token.NAME, "sin", (1, 1), (1, 4)),
+                    (token.OP, "(", (1, 4), (1, 5)),
+                    (token.NAME, "x", (1, 5), (1, 6)),
+                    (token.OP, ")", (1, 6), (1, 7)),
+                ],
+            ),
+            (
+                "01234",
+                (
+                    "leading zeros in decimal integer literals are not permitted; "
+                    "use an 0o prefix for octal integers",
+                    (1, 1),
+                ),
+                [(token.NUMBER, "01234", (1, 0), (1, 5))],
+            ),
+            (
+                ")",
+                ("unmatched ')'", (1, 1)),
+                [(token.OP, ")", (1, 0), (1, 1))],
+            ),
+            (
+                "(]",
+                (
+                    "closing parenthesis ']' does not match opening parenthesis '('",
+                    (1, 2),
+                ),
+                [
+                    (token.OP, "(", (1, 0), (1, 1)),
+                    (token.OP, "]", (1, 1), (1, 2)),
+                ],
+            ),
+            (
+                "a☃b",
+                ("invalid character '☃' (U+2603)", (1, 2)),
+                [(token.NAME, "a☃b", (1, 0), (1, 3))],
+            ),
+        ]
+
+        for source, error, expected in cases:
+            with self.subTest(source=source):
+                with self.assertRaises(tokenize.TokenError) as caught:
+                    self._get_tokens(source)
+                self.assertEqual(caught.exception.args, error)
+
+                tokens = self._get_tokens(source, extra_tokens=True)
+                self.assertEqual(
+                    [(tok.type, tok.string, tok.start, tok.end)
+                     for tok in tokens[:-2]],
+                    expected,
+                )
+
+    def test_extra_tokens_emits_comments_and_real_indent_locations(self):
+        source = "if x:\n  # c\n  y\nz\n"
+        kinds = {token.COMMENT, token.NL, token.INDENT, token.DEDENT}
+
+        parser_tokens = self._get_tokens(source)
+        self.assertEqual(
+            [(tok.type, tok.string, tok.start, tok.end)
+             for tok in parser_tokens if tok.type in kinds],
+            [
+                (token.INDENT, "", (3, -1), (3, -1)),
+                (token.DEDENT, "", (4, -1), (4, -1)),
+            ],
+        )
+
+        tolerant_tokens = self._get_tokens(source, extra_tokens=True)
+        self.assertEqual(
+            [(tok.type, tok.string, tok.start, tok.end)
+             for tok in tolerant_tokens if tok.type in kinds],
+            [
+                (token.COMMENT, "# c", (2, 2), (2, 5)),
+                (token.NL, "\n", (2, 5), (2, 6)),
+                (token.INDENT, "  ", (3, 0), (3, 2)),
+                (token.DEDENT, "", (4, 0), (4, 0)),
+            ],
+        )
+
+    def test_printable_invalid_operator_streams_in_both_modes(self):
+        for extra_tokens in (False, True):
+            with self.subTest(extra_tokens=extra_tokens):
+                first = self._get_tokens(
+                    "1 $ 2",
+                    extra_tokens=extra_tokens,
+                )[1]
+                self.assertEqual(
+                    (first.type, first.string, first.start, first.end),
+                    (token.OP, "$", (1, 2), (1, 3)),
+                )
+
+    def test_degraded_fstring_format_spec(self):
+        tokens = self._get_tokens('f"{1:{2}{{3}}}"')
+        self.assertEqual(
+            [(tok.string, tok.start, tok.end)
+             for tok in tokens if tok.type == token.FSTRING_MIDDLE],
+            [
+                ("{", (1, 8), (1, 9)),
+                ("3", (1, 10), (1, 11)),
+                ("}", (1, 12), (1, 13)),
+            ],
+        )
+
+        tokens = self._get_tokens('f"{x:{y}}"')
+        self.assertEqual(
+            [(tok.string, tok.start, tok.end)
+             for tok in tokens if tok.type == token.FSTRING_MIDDLE],
+            [("", (1, 8), (1, 8))],
+        )
+
+        with self.assertRaises(tokenize.TokenError) as caught:
+            self._get_tokens('f"{1:{2}x}}y"')
+        self.assertEqual(
+            caught.exception.args,
+            ("f-string: single '}' is not allowed", (1, 11)),
+        )
+
+    def test_escaped_fstring_brace_has_a_position_gap(self):
+        tokens = self._get_tokens('f"a{{"', extra_tokens=True)
+        self.assertEqual(
+            [(tok.type, tok.string, tok.start, tok.end)
+             for tok in tokens
+             if tok.type in {token.FSTRING_MIDDLE, token.FSTRING_END}],
+            [
+                (token.FSTRING_MIDDLE, "a{", (1, 2), (1, 4)),
+                (token.FSTRING_END, '"', (1, 5), (1, 6)),
+            ],
+        )
+
+    def test_tolerant_incompatible_prefix_position_after_non_ascii(self):
+        with self.assertRaises(tokenize.TokenError) as caught:
+            self._get_tokens('bé )tf"2 ', extra_tokens=True)
+        self.assertEqual(
+            caught.exception.args,
+            ("'f' and 't' prefixes are incompatible", (1, 6)),
+        )
+
+    def test_tolerant_fstring_closer_at_expression_entry_depth(self):
+        source = "f'1:{]]{}}r}''"
+        for extra_tokens, position in ((False, (1, 6)), (True, (1, 7))):
+            with self.subTest(extra_tokens=extra_tokens):
+                with self.assertRaises(tokenize.TokenError) as caught:
+                    self._get_tokens(source, extra_tokens=extra_tokens)
+                self.assertEqual(
+                    caught.exception.args,
+                    ("f-string: unmatched ']'", position),
+                )
+
     def test_int(self):
 
         self.check_tokenize('0xff <= 255', """\