Skip to content

Commit bce3caa

Browse files
pablogsalpablogsal
committed
gh-153569: Base the tokenizer API on source offsets
1 parent a2d3787 commit bce3caa

50 files changed

Lines changed: 5266 additions & 3241 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Lib/test/test_codeop.py‎

Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,17 @@
22
Test cases for codeop.py
33
Nick Mathewson
44
"""
5+
import builtins
56
import unittest
67
import warnings
78
from test.support import warnings_helper
89
from textwrap import dedent
910

10-
from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
11+
from codeop import (
12+
compile_command,
13+
PyCF_ALLOW_INCOMPLETE_INPUT,
14+
PyCF_DONT_IMPLY_DEDENT,
15+
)
1116

1217
class CodeopTests(unittest.TestCase):
1318

@@ -226,6 +231,72 @@ def test_incomplete(self):
226231
ai('a = f"""')
227232
ai('a = \\')
228233

234+
def test_tokenizer_incomplete_input_classification(self):
235+
cases = [
236+
(
237+
"x = 'abc",
238+
"single",
239+
builtins._IncompleteInputError,
240+
("incomplete input", 1, 5, 1, -1),
241+
),
242+
(
243+
'f"""abc',
244+
"single",
245+
builtins._IncompleteInputError,
246+
("incomplete input", 1, 1, 1, -1),
247+
),
248+
(
249+
"x = \\\n",
250+
"single",
251+
builtins._IncompleteInputError,
252+
("incomplete input", 1, 6, 1, -1),
253+
),
254+
(
255+
"x = 'abc",
256+
"exec",
257+
SyntaxError,
258+
(
259+
"unterminated string literal (detected at line 1)",
260+
1, 5, 1, 5,
261+
),
262+
),
263+
(
264+
'f"abc',
265+
"single",
266+
SyntaxError,
267+
(
268+
"unterminated f-string literal (detected at line 1)",
269+
1, 1, 1, 1,
270+
),
271+
),
272+
(
273+
"x = \\",
274+
"single",
275+
SyntaxError,
276+
(
277+
"unexpected character after line continuation character",
278+
1, 5, 1, 0,
279+
),
280+
),
281+
]
282+
283+
for source, mode, exception_type, expected in cases:
284+
with self.subTest(source=source, mode=mode):
285+
with self.assertRaises(exception_type) as caught:
286+
compile(
287+
source,
288+
"<test>",
289+
mode,
290+
PyCF_ALLOW_INCOMPLETE_INPUT,
291+
)
292+
self.assertIs(type(caught.exception), exception_type)
293+
error = caught.exception
294+
self.assertEqual(
295+
(error.msg, error.lineno, error.offset,
296+
error.end_lineno, error.end_offset),
297+
expected,
298+
)
299+
229300
def test_invalid(self):
230301
ai = self.assertInvalid
231302
ai("a b")

‎Lib/test/test_source_encoding.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,45 @@ def test_truncated_utf8_at_eof(self):
8282
with self.subTest(seq=seq):
8383
self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
8484

85+
def test_invalid_utf8_offset_after_non_ascii(self):
86+
with self.assertRaises(SyntaxError) as caught:
87+
compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
88+
error = caught.exception
89+
self.assertEqual(
90+
(error.lineno, error.offset, error.end_lineno, error.end_offset),
91+
(1, 6, 1, 6),
92+
)
93+
94+
def test_unbounded_bom_conflict_message(self):
95+
encoding = "x" * 400
96+
source = b"\xef\xbb\xbf# coding:" + encoding.encode() + b"\n"
97+
with self.assertRaises(SyntaxError) as caught:
98+
compile(source, "<test>", "exec")
99+
self.assertEqual(
100+
caught.exception.msg,
101+
f"encoding problem: {encoding} with BOM",
102+
)
103+
104+
@support.requires_subprocess()
105+
def test_stateful_file_decoder_spans_lines(self):
106+
encoded_name = "変数".encode("iso2022_jp")
107+
payload = encoded_name[3:-3]
108+
source = (
109+
b"# coding: iso2022_jp\n"
110+
b"# \x1b$B" + payload + b"\n"
111+
+ payload + b"\x1b(B = 1\n"
112+
)
113+
with tempfile.TemporaryDirectory() as directory:
114+
filename = os.path.join(directory, "stateful.py")
115+
with open(filename, "wb") as file:
116+
file.write(source)
117+
process = subprocess.run(
118+
[sys.executable, filename],
119+
stdout=subprocess.PIPE,
120+
stderr=subprocess.PIPE,
121+
)
122+
self.assertEqual(process.returncode, 0, process.stderr)
123+
85124
@support.requires_subprocess()
86125
def test_20731(self):
87126
sub = subprocess.Popen([sys.executable,

‎Lib/test/test_syntax.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3088,6 +3088,24 @@ def test_expression_with_assignment(self):
30883088
def test_curly_brace_after_primary_raises_immediately(self):
30893089
self._check_error("f{}", "invalid syntax", mode="single")
30903090

3091+
def test_tokenizer_eof_error_offsets_after_non_ascii(self):
3092+
self._check_error(
3093+
"é + (",
3094+
re.escape("'(' was never closed"),
3095+
lineno=1,
3096+
offset=5,
3097+
end_lineno=1,
3098+
end_offset=0,
3099+
)
3100+
self._check_error(
3101+
"é + \\\n",
3102+
"unexpected EOF while parsing",
3103+
lineno=1,
3104+
offset=6,
3105+
end_lineno=1,
3106+
end_offset=-1,
3107+
)
3108+
30913109
def test_assign_call(self):
30923110
self._check_error("f() = 1", "assign")
30933111

‎Lib/test/test_tokenize.py‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
import token
88
import tokenize
99
import unittest
10+
import warnings
11+
import weakref
12+
import _tokenize
1013
from io import BytesIO, StringIO
1114
from textwrap import dedent
1215
from unittest import TestCase, mock
@@ -2243,6 +2246,38 @@ def _get_tokens(source, *, extra_tokens=False):
22432246
extra_tokens=extra_tokens,
22442247
))
22452248

2249+
def test_readline_reentry_is_rejected(self):
2250+
def make_cycle():
2251+
iterator = None
2252+
2253+
def readline():
2254+
next(iterator)
2255+
2256+
iterator = _tokenize.TokenizerIter(readline, extra_tokens=False)
2257+
with self.assertRaisesRegex(RuntimeError, "already executing"):
2258+
next(iterator)
2259+
return weakref.ref(readline)
2260+
2261+
readline_ref = make_cycle()
2262+
support.gc_collect()
2263+
self.assertIsNone(readline_ref())
2264+
2265+
def test_warning_reentry_is_rejected(self):
2266+
iterator = None
2267+
2268+
def showwarning(*args, **kwargs):
2269+
next(iterator)
2270+
2271+
with warnings.catch_warnings():
2272+
warnings.simplefilter("always")
2273+
with mock.patch.object(warnings, "showwarning", showwarning):
2274+
iterator = _tokenize.TokenizerIter(
2275+
StringIO("1if\n").readline,
2276+
extra_tokens=False,
2277+
)
2278+
with self.assertRaisesRegex(RuntimeError, "already executing"):
2279+
next(iterator)
2280+
22462281
def check_tokenize(self, s, expected):
22472282
# Format the tokens in s in a table format.
22482283
# The ENDMARKER and final NEWLINE are omitted.
@@ -2273,6 +2308,71 @@ def readline(encoding):
22732308
))
22742309
self.assertEqual(tokens, expected)
22752310

2311+
def test_stateful_encoding_across_readline_calls(self):
2312+
encoded_name = "変数".encode("iso2022_jp")
2313+
payload = encoded_name[3:-3]
2314+
lines = iter([
2315+
b"# \x1b$B" + payload + b"\n",
2316+
payload + b"\x1b(B\n",
2317+
b"",
2318+
])
2319+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2320+
lines.__next__,
2321+
extra_tokens=True,
2322+
encoding="iso2022_jp",
2323+
))
2324+
self.assertEqual(
2325+
[(tok.type, tok.string) for tok in tokens],
2326+
[
2327+
(token.COMMENT, "# 変数"),
2328+
(token.NL, "\n"),
2329+
(token.NAME, "変数"),
2330+
(token.NEWLINE, "\n"),
2331+
(token.ENDMARKER, ""),
2332+
],
2333+
)
2334+
2335+
def test_utf8_decoder_spans_readline_calls(self):
2336+
lines = iter([b"\xc3", b"\xa9\n", b""])
2337+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2338+
lines.__next__,
2339+
extra_tokens=True,
2340+
encoding="utf-8",
2341+
))
2342+
self.assertEqual(
2343+
[(tok.type, tok.string, tok.start, tok.end) for tok in tokens],
2344+
[
2345+
(token.NAME, "é", (1, 0), (1, 1)),
2346+
(token.NEWLINE, "\n", (1, 1), (1, 2)),
2347+
(token.ENDMARKER, "", (2, 0), (2, 0)),
2348+
],
2349+
)
2350+
2351+
def test_encoded_readline_replaces_invalid_bytes(self):
2352+
lines = iter([b"\xff\n", b""])
2353+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2354+
lines.__next__,
2355+
extra_tokens=True,
2356+
encoding="utf-8",
2357+
))
2358+
self.assertEqual(
2359+
[(tok.type, tok.string) for tok in tokens],
2360+
[
2361+
(token.NAME, "�"),
2362+
(token.NEWLINE, "\n"),
2363+
(token.ENDMARKER, ""),
2364+
],
2365+
)
2366+
2367+
def test_encoded_readline_codec_lookup_is_lazy(self):
2368+
iterator = _tokenize.TokenizerIter(
2369+
lambda: b"",
2370+
extra_tokens=True,
2371+
encoding="missing-tokenizer-codec",
2372+
)
2373+
with self.assertRaises(LookupError):
2374+
next(iterator)
2375+
22762376
def test_extra_tokens_relaxes_lexer_errors(self):
22772377
cases = [
22782378
(
@@ -2407,6 +2507,38 @@ def test_escaped_fstring_brace_has_a_position_gap(self):
24072507
],
24082508
)
24092509

2510+
def test_unclosed_parenthesis_error_uses_buffer_relative_position(self):
2511+
for extra_tokens in (False, True):
2512+
with self.subTest(extra_tokens=extra_tokens):
2513+
with self.assertRaises(tokenize.TokenError) as caught:
2514+
self._get_tokens("é = (\n", extra_tokens=extra_tokens)
2515+
self.assertEqual(
2516+
caught.exception.args,
2517+
("unexpected EOF in multi-line statement", (1, 0)),
2518+
)
2519+
2520+
def test_line_continuation_error_uses_logical_line_position(self):
2521+
cases = [
2522+
("é \\'f\n", (1, 6)),
2523+
("x1\\\n==\\_", (2, 9)),
2524+
]
2525+
for extra_tokens in (False, True):
2526+
for source, position in cases:
2527+
with self.subTest(
2528+
source=source,
2529+
extra_tokens=extra_tokens,
2530+
):
2531+
with self.assertRaises(tokenize.TokenError) as caught:
2532+
self._get_tokens(source, extra_tokens=extra_tokens)
2533+
self.assertEqual(
2534+
caught.exception.args,
2535+
(
2536+
"unexpected character after line continuation "
2537+
"character",
2538+
position,
2539+
),
2540+
)
2541+
24102542
def test_tolerant_incompatible_prefix_position_after_non_ascii(self):
24112543
with self.assertRaises(tokenize.TokenError) as caught:
24122544
self._get_tokens('bé )tf"2 ', extra_tokens=True)

‎Makefile.pre.in‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -393,15 +393,17 @@ PEGEN_OBJS= \
393393
Parser/peg_api.o
394394

395395
TOKENIZER_OBJS= \
396-
Parser/lexer/buffer.o \
396+
Parser/tokenizer/source.o \
397+
Parser/tokenizer/api.o \
398+
Parser/tokenizer/errors.o \
399+
Parser/tokenizer/seam.o \
400+
Parser/tokenizer/cursor.o \
397401
Parser/lexer/lexer.o \
398402
Parser/lexer/number.o \
399-
Parser/lexer/state.o \
400403
Parser/lexer/string.o \
401-
Parser/tokenizer/file_tokenizer.o \
402-
Parser/tokenizer/readline_tokenizer.o \
403-
Parser/tokenizer/string_tokenizer.o \
404-
Parser/tokenizer/utf8_tokenizer.o \
404+
Parser/lexer/state.o \
405+
Parser/tokenizer/decoder.o \
406+
Parser/tokenizer/reader.o \
405407
Parser/tokenizer/helpers.o
406408

407409
PEGEN_HEADERS= \
@@ -410,11 +412,16 @@ PEGEN_HEADERS= \
410412
$(srcdir)/Parser/string_parser.h
411413

412414
TOKENIZER_HEADERS= \
413-
Parser/lexer/buffer.h \
415+
Parser/tokenizer/source.h \
416+
Parser/tokenizer/tokenizer.h \
417+
Parser/tokenizer/errors.h \
418+
Parser/tokenizer/seam.h \
419+
Parser/tokenizer/cursor.h \
420+
Parser/tokenizer/reader.h \
414421
Parser/lexer/lexer.h \
415422
Parser/lexer/lexer_internal.h \
416423
Parser/lexer/state.h \
417-
Parser/tokenizer/tokenizer.h \
424+
Parser/tokenizer/reader_internal.h \
418425
Parser/tokenizer/helpers.h
419426

420427
POBJS= \

‎PCbuild/_freeze_module.vcxproj‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -181,15 +181,17 @@
181181
<ClCompile Include="..\Parser\action_helpers.c" />
182182
<ClCompile Include="..\Parser\string_parser.c" />
183183
<ClCompile Include="..\Parser\token.c" />
184-
<ClCompile Include="..\Parser\lexer\buffer.c" />
184+
<ClCompile Include="..\Parser\tokenizer\source.c" />
185+
<ClCompile Include="..\Parser\tokenizer\api.c" />
186+
<ClCompile Include="..\Parser\tokenizer\errors.c" />
187+
<ClCompile Include="..\Parser\tokenizer\seam.c" />
188+
<ClCompile Include="..\Parser\tokenizer\cursor.c" />
189+
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
185190
<ClCompile Include="..\Parser\lexer\state.c" />
186191
<ClCompile Include="..\Parser\lexer\lexer.c" />
187192
<ClCompile Include="..\Parser\lexer\number.c" />
188193
<ClCompile Include="..\Parser\lexer\string.c" />
189-
<ClCompile Include="..\Parser\tokenizer\string_tokenizer.c" />
190-
<ClCompile Include="..\Parser\tokenizer\file_tokenizer.c" />
191-
<ClCompile Include="..\Parser\tokenizer\utf8_tokenizer.c" />
192-
<ClCompile Include="..\Parser\tokenizer\readline_tokenizer.c" />
194+
<ClCompile Include="..\Parser\tokenizer\reader.c" />
193195
<ClCompile Include="..\Parser\tokenizer\helpers.c" />
194196
<ClCompile Include="..\PC\invalid_parameter_handler.c" />
195197
<ClCompile Include="..\PC\msvcrtmodule.c" />

0 commit comments

Comments
 (0)