gh-153568: Don't materialize parser token text that is never read

Only tokens whose text is actually consumed get a bytes object;
operators and structural tokens no longer allocate one.

Author: pablogsal

Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst

@@ -0,0 +1,2 @@
+Speed up the parser by not materializing the text of tokens whose text is
+never read.

Parser/pegen.c

@@ -178,18 +178,50 @@ _get_keyword_or_name_type(Parser *p, struct token *new_token)
     return NAME;
 }
 
+// Token types whose text is consumed by grammar actions or helpers, other
+// than NAME-derived tokens (identifiers and keywords), which always keep
+// their text: error actions may print keyword text (e.g. invalid_kwarg's
+// "cannot assign to True"). For every other type the token text is never
+// read again, so materializing a PyBytes for it is wasted work.
+static inline int
+token_needs_text(int type)
+{
+    switch (type) {
+        case NAME:
+        case NUMBER:
+        case STRING:
+        case FSTRING_START:
+        case FSTRING_MIDDLE:
+        case FSTRING_END:
+        case TSTRING_START:
+        case TSTRING_MIDDLE:
+        case TSTRING_END:
+        case TYPE_COMMENT:
+        case NOTEQUAL:  // _PyPegen_check_barry_as_flufl() reads its text
+            return 1;
+        default:
+            return 0;
+    }
+}
+
 static int
 initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) {
     assert(parser_token != NULL);
 
     parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type;
-    parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start);
-    if (parser_token->bytes == NULL) {
-        return -1;
+    if (token_type == NAME || token_needs_text(parser_token->type)) {
+        parser_token->bytes = PyBytes_FromStringAndSize(
+            new_token->start, new_token->end - new_token->start);
+        if (parser_token->bytes == NULL) {
+            return -1;
+        }
+        if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) {
+            Py_DECREF(parser_token->bytes);
+            return -1;
+        }
     }
-    if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) {
-        Py_DECREF(parser_token->bytes);
-        return -1;
+    else {
+        parser_token->bytes = NULL;
     }
 
     parser_token->metadata = NULL;