Skip to content

Commit a2d3787

Browse files
pablogsalpablogsal
authored
gh-153569: Split tokenizer lexer into focused files (#153585)
1 parent 43b0e01 commit a2d3787

12 files changed

Lines changed: 1224 additions & 957 deletions

File tree

‎Lib/test/test_fstring.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -709,6 +709,16 @@ def test_double_braces(self):
709709
["f'{ {{}} }'", # dict in a set
710710
])
711711

712+
def test_double_brace_ast_location_covers_both_source_braces(self):
713+
value = ast.parse('f"a{{"').body[0].value.values[0]
714+
self.assertIsInstance(value, ast.Constant)
715+
self.assertEqual(value.value, "a{")
716+
self.assertEqual(
717+
(value.lineno, value.col_offset, value.end_lineno,
718+
value.end_col_offset),
719+
(1, 2, 1, 5),
720+
)
721+
712722
def test_compile_time_concat(self):
713723
x = 'def'
714724
self.assertEqual('abc' f'## {x}ghi', 'abc## defghi')

‎Lib/test/test_tokenize.py‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2236,6 +2236,13 @@ def test_number_starting_with_zero(self):
22362236
self.assertEqual(tokens, expected_tokens)
22372237

22382238
class CTokenizeTest(TestCase):
2239+
@staticmethod
2240+
def _get_tokens(source, *, extra_tokens=False):
2241+
return list(tokenize._generate_tokens_from_c_tokenizer(
2242+
StringIO(source).readline,
2243+
extra_tokens=extra_tokens,
2244+
))
2245+
22392246
def check_tokenize(self, s, expected):
22402247
# Format the tokens in s in a table format.
22412248
# The ENDMARKER and final NEWLINE are omitted.
@@ -2266,6 +2273,159 @@ def readline(encoding):
22662273
))
22672274
self.assertEqual(tokens, expected)
22682275

2276+
def test_extra_tokens_relaxes_lexer_errors(self):
2277+
cases = [
2278+
(
2279+
"2sin(x)",
2280+
("invalid decimal literal", (1, 1)),
2281+
[
2282+
(token.NUMBER, "2", (1, 0), (1, 1)),
2283+
(token.NAME, "sin", (1, 1), (1, 4)),
2284+
(token.OP, "(", (1, 4), (1, 5)),
2285+
(token.NAME, "x", (1, 5), (1, 6)),
2286+
(token.OP, ")", (1, 6), (1, 7)),
2287+
],
2288+
),
2289+
(
2290+
"01234",
2291+
(
2292+
"leading zeros in decimal integer literals are not permitted; "
2293+
"use an 0o prefix for octal integers",
2294+
(1, 1),
2295+
),
2296+
[(token.NUMBER, "01234", (1, 0), (1, 5))],
2297+
),
2298+
(
2299+
")",
2300+
("unmatched ')'", (1, 1)),
2301+
[(token.OP, ")", (1, 0), (1, 1))],
2302+
),
2303+
(
2304+
"(]",
2305+
(
2306+
"closing parenthesis ']' does not match opening parenthesis '('",
2307+
(1, 2),
2308+
),
2309+
[
2310+
(token.OP, "(", (1, 0), (1, 1)),
2311+
(token.OP, "]", (1, 1), (1, 2)),
2312+
],
2313+
),
2314+
(
2315+
"a☃b",
2316+
("invalid character '☃' (U+2603)", (1, 2)),
2317+
[(token.NAME, "a☃b", (1, 0), (1, 3))],
2318+
),
2319+
]
2320+
2321+
for source, error, expected in cases:
2322+
with self.subTest(source=source):
2323+
with self.assertRaises(tokenize.TokenError) as caught:
2324+
self._get_tokens(source)
2325+
self.assertEqual(caught.exception.args, error)
2326+
2327+
tokens = self._get_tokens(source, extra_tokens=True)
2328+
self.assertEqual(
2329+
[(tok.type, tok.string, tok.start, tok.end)
2330+
for tok in tokens[:-2]],
2331+
expected,
2332+
)
2333+
2334+
def test_extra_tokens_emits_comments_and_real_indent_locations(self):
2335+
source = "if x:\n # c\n y\nz\n"
2336+
kinds = {token.COMMENT, token.NL, token.INDENT, token.DEDENT}
2337+
2338+
parser_tokens = self._get_tokens(source)
2339+
self.assertEqual(
2340+
[(tok.type, tok.string, tok.start, tok.end)
2341+
for tok in parser_tokens if tok.type in kinds],
2342+
[
2343+
(token.INDENT, "", (3, -1), (3, -1)),
2344+
(token.DEDENT, "", (4, -1), (4, -1)),
2345+
],
2346+
)
2347+
2348+
tolerant_tokens = self._get_tokens(source, extra_tokens=True)
2349+
self.assertEqual(
2350+
[(tok.type, tok.string, tok.start, tok.end)
2351+
for tok in tolerant_tokens if tok.type in kinds],
2352+
[
2353+
(token.COMMENT, "# c", (2, 2), (2, 5)),
2354+
(token.NL, "\n", (2, 5), (2, 6)),
2355+
(token.INDENT, " ", (3, 0), (3, 2)),
2356+
(token.DEDENT, "", (4, 0), (4, 0)),
2357+
],
2358+
)
2359+
2360+
def test_printable_invalid_operator_streams_in_both_modes(self):
2361+
for extra_tokens in (False, True):
2362+
with self.subTest(extra_tokens=extra_tokens):
2363+
first = self._get_tokens(
2364+
"1 $ 2",
2365+
extra_tokens=extra_tokens,
2366+
)[1]
2367+
self.assertEqual(
2368+
(first.type, first.string, first.start, first.end),
2369+
(token.OP, "$", (1, 2), (1, 3)),
2370+
)
2371+
2372+
def test_degraded_fstring_format_spec(self):
2373+
tokens = self._get_tokens('f"{1:{2}{{3}}}"')
2374+
self.assertEqual(
2375+
[(tok.string, tok.start, tok.end)
2376+
for tok in tokens if tok.type == token.FSTRING_MIDDLE],
2377+
[
2378+
("{", (1, 8), (1, 9)),
2379+
("3", (1, 10), (1, 11)),
2380+
("}", (1, 12), (1, 13)),
2381+
],
2382+
)
2383+
2384+
tokens = self._get_tokens('f"{x:{y}}"')
2385+
self.assertEqual(
2386+
[(tok.string, tok.start, tok.end)
2387+
for tok in tokens if tok.type == token.FSTRING_MIDDLE],
2388+
[("", (1, 8), (1, 8))],
2389+
)
2390+
2391+
with self.assertRaises(tokenize.TokenError) as caught:
2392+
self._get_tokens('f"{1:{2}x}}y"')
2393+
self.assertEqual(
2394+
caught.exception.args,
2395+
("f-string: single '}' is not allowed", (1, 11)),
2396+
)
2397+
2398+
def test_escaped_fstring_brace_has_a_position_gap(self):
2399+
tokens = self._get_tokens('f"a{{"', extra_tokens=True)
2400+
self.assertEqual(
2401+
[(tok.type, tok.string, tok.start, tok.end)
2402+
for tok in tokens
2403+
if tok.type in {token.FSTRING_MIDDLE, token.FSTRING_END}],
2404+
[
2405+
(token.FSTRING_MIDDLE, "a{", (1, 2), (1, 4)),
2406+
(token.FSTRING_END, '"', (1, 5), (1, 6)),
2407+
],
2408+
)
2409+
2410+
def test_tolerant_incompatible_prefix_position_after_non_ascii(self):
2411+
with self.assertRaises(tokenize.TokenError) as caught:
2412+
self._get_tokens('bé )tf"2 ', extra_tokens=True)
2413+
self.assertEqual(
2414+
caught.exception.args,
2415+
("'f' and 't' prefixes are incompatible", (1, 6)),
2416+
)
2417+
2418+
def test_tolerant_fstring_closer_at_expression_entry_depth(self):
2419+
source = "f'1:{]]{}}r}''"
2420+
for extra_tokens, position in ((False, (1, 6)), (True, (1, 7))):
2421+
with self.subTest(extra_tokens=extra_tokens):
2422+
with self.assertRaises(tokenize.TokenError) as caught:
2423+
self._get_tokens(source, extra_tokens=extra_tokens)
2424+
self.assertEqual(
2425+
caught.exception.args,
2426+
("f-string: unmatched ']'", position),
2427+
)
2428+
22692429
def test_int(self):
22702430

22712431
self.check_tokenize('0xff <= 255', """\

‎Makefile.pre.in‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,9 @@ PEGEN_OBJS= \
395395
TOKENIZER_OBJS= \
396396
Parser/lexer/buffer.o \
397397
Parser/lexer/lexer.o \
398+
Parser/lexer/number.o \
398399
Parser/lexer/state.o \
400+
Parser/lexer/string.o \
399401
Parser/tokenizer/file_tokenizer.o \
400402
Parser/tokenizer/readline_tokenizer.o \
401403
Parser/tokenizer/string_tokenizer.o \
@@ -410,6 +412,7 @@ PEGEN_HEADERS= \
410412
TOKENIZER_HEADERS= \
411413
Parser/lexer/buffer.h \
412414
Parser/lexer/lexer.h \
415+
Parser/lexer/lexer_internal.h \
413416
Parser/lexer/state.h \
414417
Parser/tokenizer/tokenizer.h \
415418
Parser/tokenizer/helpers.h

‎PCbuild/_freeze_module.vcxproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,8 @@
184184
<ClCompile Include="..\Parser\lexer\buffer.c" />
185185
<ClCompile Include="..\Parser\lexer\state.c" />
186186
<ClCompile Include="..\Parser\lexer\lexer.c" />
187+
<ClCompile Include="..\Parser\lexer\number.c" />
188+
<ClCompile Include="..\Parser\lexer\string.c" />
187189
<ClCompile Include="..\Parser\tokenizer\string_tokenizer.c" />
188190
<ClCompile Include="..\Parser\tokenizer\file_tokenizer.c" />
189191
<ClCompile Include="..\Parser\tokenizer\utf8_tokenizer.c" />

‎PCbuild/_freeze_module.vcxproj.filters‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,12 @@
463463
<ClCompile Include="..\Parser\lexer\lexer.c">
464464
<Filter>Source Files</Filter>
465465
</ClCompile>
466+
<ClCompile Include="..\Parser\lexer\number.c">
467+
<Filter>Source Files</Filter>
468+
</ClCompile>
469+
<ClCompile Include="..\Parser\lexer\string.c">
470+
<Filter>Source Files</Filter>
471+
</ClCompile>
466472
<ClCompile Include="..\Parser\lexer\buffer.c">
467473
<Filter>Source Files</Filter>
468474
</ClCompile>

‎PCbuild/pythoncore.vcxproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,7 @@
420420
<ClInclude Include="..\Objects\unicodetype_db.h" />
421421
<ClInclude Include="..\Parser\lexer\state.h" />
422422
<ClInclude Include="..\Parser\lexer\lexer.h" />
423+
<ClInclude Include="..\Parser\lexer\lexer_internal.h" />
423424
<ClInclude Include="..\Parser\lexer\buffer.h" />
424425
<ClInclude Include="..\Parser\tokenizer\helpers.h" />
425426
<ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
@@ -584,6 +585,8 @@
584585
<ClCompile Include="..\Parser\myreadline.c" />
585586
<ClCompile Include="..\Parser\lexer\state.c" />
586587
<ClCompile Include="..\Parser\lexer\lexer.c" />
588+
<ClCompile Include="..\Parser\lexer\number.c" />
589+
<ClCompile Include="..\Parser\lexer\string.c" />
587590
<ClCompile Include="..\Parser\lexer\buffer.c" />
588591
<ClCompile Include="..\Parser\tokenizer\string_tokenizer.c" />
589592
<ClCompile Include="..\Parser\tokenizer\file_tokenizer.c" />

‎PCbuild/pythoncore.vcxproj.filters‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,9 @@
327327
<ClInclude Include="..\Parser\lexer\state.h">
328328
<Filter>Parser</Filter>
329329
</ClInclude>
330+
<ClInclude Include="..\Parser\lexer\lexer_internal.h">
331+
<Filter>Parser</Filter>
332+
</ClInclude>
330333
<ClInclude Include="..\Parser\lexer\buffer.h">
331334
<Filter>Parser</Filter>
332335
</ClInclude>
@@ -1331,6 +1334,12 @@
13311334
<ClCompile Include="..\Parser\lexer\lexer.c">
13321335
<Filter>Parser</Filter>
13331336
</ClCompile>
1337+
<ClCompile Include="..\Parser\lexer\number.c">
1338+
<Filter>Parser</Filter>
1339+
</ClCompile>
1340+
<ClCompile Include="..\Parser\lexer\string.c">
1341+
<Filter>Parser</Filter>
1342+
</ClCompile>
13341343
<ClCompile Include="..\Parser\lexer\state.c">
13351344
<Filter>Parser</Filter>
13361345
</ClCompile>

0 commit comments

Comments
 (0)