Skip to content

Commit 974742a

Browse files
cmaloneyclaude
andcommitted
gh-101703: Replace sprintf with _PyLong_FormatBytesWriter in bytesobject
Format %d, %i, %u, %ld, %lu, %zd, %zu and %x with _PyLong_FormatBytesWriter(), the code bytes % int uses, which writes straight into the PyBytesWriter instead of going through a stack buffer. Output is unchanged. This leaves the current %p as it has some platform specific variation which changing requires more validation and a NEWS. Each row is 1000 calls made from a C extension module (below), so 1 us is 1 ns per call. Linux x86-64, clang PGO+LTO, pyperf, upstream/main b90572a against this change. The %s and %p rows are not changed. | Benchmark | main | PR | |------------------------------|:-------:|:---------------------:| | run(b'%x', 123456789, 1000) | 36.9 us | 26.9 us: 1.37x faster | | run(b'%d', 5, 1000) | 28.1 us | 20.6 us: 1.37x faster | | run(b'%d.%d.%d', 42, 1000) | 74.7 us | 55.0 us: 1.36x faster | | run(b'%i', 42, 1000) | 34.1 us | 26.0 us: 1.31x faster | | run(b'%d', 42, 1000) | 33.9 us | 26.3 us: 1.29x faster | | run(b'%d', -123456789, 1000) | 40.5 us | 35.1 us: 1.16x faster | | run(b'%d', 123456789, 1000) | 40.0 us | 34.6 us: 1.16x faster | | run(b'%u', 123456789, 1000) | 39.7 us | 34.6 us: 1.15x faster | | run(b'%ld', 10**18, 1000) | 50.5 us | 46.0 us: 1.10x faster | | run(b'%lu', 2**64 - 1, 1000) | 50.1 us | 47.3 us: 1.06x faster | | run(b'%zu', 2**64 - 1, 1000) | 49.2 us | 47.7 us: 1.03x faster | | run(b'%s', 0, 1000) | 19.4 us | 18.8 us: 1.03x faster | | run(b'%p', 0, 1000) | 35.8 us | 34.8 us: 1.03x faster | Not significant: run(b'%p', 2**47 - 4096, 1000), run(b'%zd', 2**62, 1000). Benchmark script (bench.py): # PyBytes_FromFormat() integer formatting, called from C by # fromformat_bench.c (build it first, see its header). Each row is 1000 # calls, so 1 us is 1 ns per call. The %s and %p rows are formats the PR # does not change. # python bench.py -o main.json (main build) # python bench.py -o pr.json (PR build) # python -m pyperf compare_to main.json pr.json --table import pyperf runner = pyperf.Runner() for stmt in [ "run(b'%d', 5, 1000)", "run(b'%d', 42, 1000)", "run(b'%d', 123456789, 1000)", "run(b'%d', -123456789, 1000)", "run(b'%i', 42, 1000)", "run(b'%u', 123456789, 1000)", "run(b'%ld', 10**18, 1000)", "run(b'%lu', 2**64 - 1, 1000)", "run(b'%zd', 2**62, 1000)", "run(b'%zu', 2**64 - 1, 1000)", "run(b'%x', 123456789, 1000)", "run(b'%d.%d.%d', 42, 1000)", "run(b'%p', 2**47 - 4096, 1000)", "run(b'%p', 0, 1000)", "run(b'%s', 0, 1000)", ]: runner.timeit(stmt, stmt, setup="from fromformat_bench import run") C extension module (fromformat_bench.c): // run(format, value, loops): call PyBytes_FromFormat(format, value, value, // value) loops times. The first conversion in format picks the C type of // value; %s gets "abc". // // cc -shared -fPIC -O2 $(python3-config --includes) fromformat_bench.c // -o fromformat_bench.abi3.so #define Py_LIMITED_API 0x030c0000 #include <Python.h> #include <string.h> static PyObject * run(PyObject *self, PyObject *args) { const char *format; unsigned long long value; Py_ssize_t loops; if (!PyArg_ParseTuple(args, "yKn", &format, &value, &loops)) { return NULL; } const char *conv = strchr(format, '%'); if (conv == NULL) { PyErr_SetString(PyExc_ValueError, "no conversion in format"); return NULL; } conv++; for (Py_ssize_t i = 0; i < loops; i++) { PyObject *bytes; if (*conv == 'l') { long v = (long)value; bytes = PyBytes_FromFormat(format, v, v, v); } else if (*conv == 'z') { size_t v = (size_t)value; bytes = PyBytes_FromFormat(format, v, v, v); } else if (*conv == 'p') { void *v = (void *)(uintptr_t)value; bytes = PyBytes_FromFormat(format, v, v, v); } else if (*conv == 's') { bytes = PyBytes_FromFormat(format, "abc", "abc", "abc"); } else { int v = (int)value; bytes = PyBytes_FromFormat(format, v, v, v); } if (bytes == NULL) { return NULL; } Py_DECREF(bytes); } Py_RETURN_NONE; } static PyMethodDef methods[] = { {"run", run, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, }; static struct PyModuleDef module = { PyModuleDef_HEAD_INIT, "fromformat_bench", NULL, 0, methods, }; PyMODINIT_FUNC PyInit_fromformat_bench(void) { return PyModule_Create(&module); } Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
1 parent b90572a commit 974742a

1 file changed

Lines changed: 27 additions & 22 deletions

File tree

‎Objects/bytesobject.c‎

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -237,12 +237,7 @@ static char*
237237
bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos,
238238
const char *format, va_list vargs)
239239
{
240-
/* Longest 64-bit formatted numbers:
241-
- "18446744073709551615\0" (21 bytes)
242-
- "-9223372036854775808\0" (21 bytes)
243-
Decimal takes the most space (it isn't enough for octal.)
244-
245-
Longest 64-bit pointer representation:
240+
/* Longest 64-bit pointer representation:
246241
"0xffffffffffffffff\0" (19 bytes).
247242
248243
Longest 64-bit "%p" with "0x" prefix: len(hex(2**64-1)+'\0') = 19.
@@ -265,6 +260,20 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos,
265260
s += _len; \
266261
} while (0)
267262

263+
/* Format an integer with the same code as bytes % int. */
264+
#define WRITE_INT(obj_expr, base, alternate) \
265+
do { \
266+
PyObject *_obj = (obj_expr); \
267+
if (_obj == NULL) { \
268+
return NULL; \
269+
} \
270+
s = _PyLong_FormatBytesWriter(writer, s, _obj, (base), (alternate)); \
271+
Py_DECREF(_obj); \
272+
if (s == NULL) { \
273+
return NULL; \
274+
} \
275+
} while (0)
276+
268277
for (const char *f = format; *f; f++) {
269278
if (*f != '%') {
270279
*s++ = *f;
@@ -324,42 +333,37 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos,
324333

325334
case 'd':
326335
if (longflag) {
327-
len = sprintf(buffer, "%ld", va_arg(vargs, long));
336+
WRITE_INT(PyLong_FromLong(va_arg(vargs, long)), 10, 0);
328337
}
329338
else if (size_tflag) {
330-
len = sprintf(buffer, "%zd", va_arg(vargs, Py_ssize_t));
339+
WRITE_INT(PyLong_FromSsize_t(va_arg(vargs, Py_ssize_t)), 10, 0);
331340
}
332341
else {
333-
len = sprintf(buffer, "%d", va_arg(vargs, int));
342+
WRITE_INT(PyLong_FromLong(va_arg(vargs, int)), 10, 0);
334343
}
335-
assert(1 <= len && len < (Py_ssize_t)sizeof(buffer));
336-
WRITE_BYTES(buffer, len);
337344
break;
338345

339346
case 'u':
340347
if (longflag) {
341-
len = sprintf(buffer, "%lu", va_arg(vargs, unsigned long));
348+
WRITE_INT(PyLong_FromUnsignedLong(va_arg(vargs, unsigned long)),
349+
10, 0);
342350
}
343351
else if (size_tflag) {
344-
len = sprintf(buffer, "%zu", va_arg(vargs, size_t));
352+
WRITE_INT(PyLong_FromSize_t(va_arg(vargs, size_t)), 10, 0);
345353
}
346354
else {
347-
len = sprintf(buffer, "%u", va_arg(vargs, unsigned int));
355+
WRITE_INT(PyLong_FromUnsignedLong(va_arg(vargs, unsigned int)),
356+
10, 0);
348357
}
349-
assert(1 <= len && len < (Py_ssize_t)sizeof(buffer));
350-
WRITE_BYTES(buffer, len);
351358
break;
352359

353360
case 'i':
354-
len = sprintf(buffer, "%i", va_arg(vargs, int));
355-
assert(1 <= len && len < (Py_ssize_t)sizeof(buffer));
356-
WRITE_BYTES(buffer, len);
361+
WRITE_INT(PyLong_FromLong(va_arg(vargs, int)), 10, 0);
357362
break;
358363

359364
case 'x':
360-
len = sprintf(buffer, "%x", va_arg(vargs, int));
361-
assert(1 <= len && len < (Py_ssize_t)sizeof(buffer));
362-
WRITE_BYTES(buffer, len);
365+
WRITE_INT(PyLong_FromUnsignedLong((unsigned int)va_arg(vargs, int)),
366+
16, 0);
363367
break;
364368

365369
case 's':
@@ -414,6 +418,7 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos,
414418

415419
return s;
416420

421+
#undef WRITE_INT
417422
#undef WRITE_BYTES
418423
}
419424

0 commit comments

Comments
 (0)