From 9b709a986d286326688b6ab2029f5eedc9ba61e3 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 6 Aug 2026 21:54:53 -0400 Subject: [PATCH 1/4] feat(cmake): opt-in precompiled pybind11 static library, starting with pytypes.h Split the out-of-line pytypes.h definitions into pytypes-inl.h (fmtlib/ CLI11 style): inline by default, compiled once into a per-project static library when PYBIND11_PRECOMPILED is defined. Infrastructure: - pybind11_precompile() creates the lazy pybind11::precompiled STATIC library from the installed or in-tree src/ sources; PRECOMPILE / NO_PRECOMPILE keywords on pybind11_add_module and a global PYBIND11_PRECOMPILE switch select it per target. - A link-time guard symbol encodes PYBIND11_INTERNALS_VERSION, Py_GIL_DISABLED, PYBIND11_SIMPLE_GIL_MANAGEMENT, and PYBIND11_DETAILED_ERROR_MESSAGES, so a configuration mismatch is one readable undefined symbol. - src/ is installed to share/pybind11/src (wheel stays pure); src/pybind11_combined.cpp is a single-TU build for non-CMake use. - Tests: PYBIND11_TEST_PRECOMPILE builds the whole suite against the library, two new test_cmake_build cases, packaging file lists, tidy preset, and a 3-platform CI job. Assisted-by: ClaudeCode:claude-fable-5 --- .github/workflows/ci.yml | 22 ++ CMakeLists.txt | 14 + CMakePresets.json | 3 +- docs/Doxyfile | 1 + include/pybind11/detail/common.h | 1 + include/pybind11/detail/internals-inl.h | 28 ++ include/pybind11/detail/internals.h | 47 +++ include/pybind11/pytypes-inl.h | 280 ++++++++++++++++++ include/pybind11/pytypes.h | 259 +--------------- src/internals.cpp | 11 + src/pybind11_combined.cpp | 17 ++ src/pytypes.cpp | 13 + tests/CMakeLists.txt | 17 +- tests/extra_python_package/test_files.py | 16 +- tests/test_cmake_build/CMakeLists.txt | 2 + .../installed_precompile/CMakeLists.txt | 32 ++ .../subdirectory_precompile/CMakeLists.txt | 36 +++ tools/pybind11Common.cmake | 44 +++ tools/pybind11Config.cmake.in | 3 + tools/pybind11NewTools.cmake | 11 +- tools/pybind11Tools.cmake | 8 +- 21 files changed, 608 insertions(+), 257 deletions(-) create mode 100644 include/pybind11/detail/internals-inl.h create mode 100644 include/pybind11/pytypes-inl.h create mode 100644 src/internals.cpp create mode 100644 src/pybind11_combined.cpp create mode 100644 src/pytypes.cpp create mode 100644 tests/test_cmake_build/installed_precompile/CMakeLists.txt create mode 100644 tests/test_cmake_build/subdirectory_precompile/CMakeLists.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb2fcf2b67..12d5a1822d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,28 @@ jobs: python-version: ${{ matrix.python-version }} cmake-args: ${{ matrix.cmake-args }} + # Build the test suite against the precompiled pybind11 static library; a + # definition missing from the -inl.h split shows up here as a link error. + precompile: + if: github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + include: + - runs-on: ubuntu-latest + python-version: '3.13' + - runs-on: macos-latest + python-version: '3.13' + - runs-on: windows-2022 + python-version: '3.13' + + name: ⚡ + uses: ./.github/workflows/reusable-standard.yml + with: + runs-on: ${{ matrix.runs-on }} + python-version: ${{ matrix.python-version }} + cmake-args: -DPYBIND11_TEST_PRECOMPILE=ON + standard-large: if: github.event.pull_request.draft == false strategy: diff --git a/CMakeLists.txt b/CMakeLists.txt index 65f794a49d..b65529c9b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,6 +200,7 @@ set(PYBIND11_HEADERS include/pybind11/detail/function_ref.h include/pybind11/detail/holder_caster_foreign_helpers.h include/pybind11/detail/init.h + include/pybind11/detail/internals-inl.h include/pybind11/detail/internals.h include/pybind11/detail/native_enum_data.h include/pybind11/detail/pybind11_namespace_macros.h @@ -234,6 +235,7 @@ set(PYBIND11_HEADERS include/pybind11/numpy.h include/pybind11/operators.h include/pybind11/pybind11.h + include/pybind11/pytypes-inl.h include/pybind11/pytypes.h include/pybind11/subinterpreter.h include/pybind11/stl.h @@ -265,6 +267,11 @@ endif() list(TRANSFORM PYBIND11_HEADERS PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/") +# Library sources for the opt-in precompiled mode (pybind11_precompile()). +set(pybind11_SRC_DIR + "${CMAKE_CURRENT_SOURCE_DIR}/src" + CACHE INTERNAL "Directory containing the pybind11 library sources") + # Cache variable so this can be used in parent projects set(pybind11_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}/include" @@ -331,6 +338,8 @@ if(PYBIND11_INSTALL) install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION "${SKBUILD_HEADERS_DIR}") endif() install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/ + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src") set(PYBIND11_CMAKECONFIG_INSTALL_DIR "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}" CACHE STRING "install path for pybind11Config.cmake") @@ -340,6 +349,11 @@ if(PYBIND11_INSTALL) else() set(pybind11_INCLUDEDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_INCLUDEDIR}") endif() + if(IS_ABSOLUTE "${CMAKE_INSTALL_DATAROOTDIR}") + set(pybind11_SRCDIR "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src") + else() + set(pybind11_SRCDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src") + endif() configure_package_config_file( tools/${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" diff --git a/CMakePresets.json b/CMakePresets.json index 42bf3ade9d..6e86aa331f 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -36,7 +36,8 @@ "binaryDir": "build-tidy", "cacheVariables": { "CMAKE_CXX_CLANG_TIDY": "clang-tidy;--use-color;--warnings-as-errors=*", - "CMAKE_CXX_STANDARD": "17" + "CMAKE_CXX_STANDARD": "17", + "PYBIND11_TEST_PRECOMPILE": "ON" } } ], diff --git a/docs/Doxyfile b/docs/Doxyfile index 09138db364..d0c184ccc0 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -1,6 +1,7 @@ PROJECT_NAME = pybind11 INPUT = ../include/pybind11/ RECURSIVE = YES +EXCLUDE_PATTERNS = *-inl.h GENERATE_HTML = NO GENERATE_LATEX = NO diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 506a451a0e..00d59bd1fa 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -468,6 +468,7 @@ PyModuleDef_Init should be treated like any other PyObject (so not shared across static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \ PYBIND11_PLUGIN_IMPL(name) { \ PYBIND11_CHECK_PYTHON_VERSION \ + PYBIND11_PRECOMPILED_CONFIG_GUARD \ try { \ pybind11::detail::ensure_internals(); \ static ::pybind11::detail::slots_array mod_def_slots \ diff --git a/include/pybind11/detail/internals-inl.h b/include/pybind11/detail/internals-inl.h new file mode 100644 index 0000000000..ce2c80321d --- /dev/null +++ b/include/pybind11/detail/internals-inl.h @@ -0,0 +1,28 @@ +/* + pybind11/detail/internals-inl.h: Out-of-line definitions for internals.h + + Copyright (c) 2017 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +// Every function defined here must start with PYBIND11_INLINE (or +// PYBIND11_NOINLINE_ATTR PYBIND11_INLINE). In the default header-only mode this file is +// included at the bottom of internals.h; when PYBIND11_PRECOMPILED is defined it is only +// compiled into the pybind11 static library (see src/). + +#pragma once + +#include "internals.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +#if defined(PYBIND11_PRECOMPILED) +// Link-time configuration guard; see the declaration in internals.h. +PYBIND11_INLINE void PYBIND11_PRECOMPILED_CONFIG_CHECK() {} +#endif + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 7107b70688..ebf382cc77 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -46,8 +46,51 @@ # error "PYBIND11_INTERNALS_VERSION 12 is the minimum for all platforms for pybind11 v3.1.0" #endif +#if defined(PYBIND11_PRECOMPILED) +// PYBIND11_PRECOMPILED_CONFIG_CHECK names a do-nothing function defined in the precompiled +// pybind11 library. The identifier encodes every configuration macro that must match between +// the library and the modules linking it. PYBIND11_MODULE calls it, so a mismatch (or a +// missing library) surfaces as one readable undefined symbol at link time instead of many +// unrelated ones at run time. +# if defined(Py_GIL_DISABLED) +# define PYBIND11_PRECOMPILED_CFG_GD 1 +# else +# define PYBIND11_PRECOMPILED_CFG_GD 0 +# endif +# if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# define PYBIND11_PRECOMPILED_CFG_SG 1 +# else +# define PYBIND11_PRECOMPILED_CFG_SG 0 +# endif +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# define PYBIND11_PRECOMPILED_CFG_DE 1 +# else +# define PYBIND11_PRECOMPILED_CFG_DE 0 +# endif +// PYBIND11_CONCAT does not macro-expand its arguments (## suppresses expansion). +# define PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de) \ + pybind11_precompiled_config_v##v##_gd##gd##_sg##sg##_de##de +# define PYBIND11_PRECOMPILED_CONFIG_NAME(v, gd, sg, de) \ + PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de) +# define PYBIND11_PRECOMPILED_CONFIG_CHECK \ + PYBIND11_PRECOMPILED_CONFIG_NAME(PYBIND11_INTERNALS_VERSION, \ + PYBIND11_PRECOMPILED_CFG_GD, \ + PYBIND11_PRECOMPILED_CFG_SG, \ + PYBIND11_PRECOMPILED_CFG_DE) +# define PYBIND11_PRECOMPILED_CONFIG_GUARD \ + ::pybind11::detail::PYBIND11_PRECOMPILED_CONFIG_CHECK(); +#else +# define PYBIND11_PRECOMPILED_CONFIG_GUARD +#endif + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +#if defined(PYBIND11_PRECOMPILED) +PYBIND11_NAMESPACE_BEGIN(detail) +void PYBIND11_PRECOMPILED_CONFIG_CHECK(); +PYBIND11_NAMESPACE_END(detail) +#endif + using ExceptionTranslator = void (*)(std::exception_ptr); // The old Python Thread Local Storage (TLS) API is deprecated in Python 3.7 in favor of the new @@ -1080,3 +1123,7 @@ T &get_or_create_shared_data(const std::string &name) { } PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) + +#ifndef PYBIND11_PRECOMPILED +# include "internals-inl.h" // IWYU pragma: export +#endif diff --git a/include/pybind11/pytypes-inl.h b/include/pybind11/pytypes-inl.h new file mode 100644 index 0000000000..f2eea1070f --- /dev/null +++ b/include/pybind11/pytypes-inl.h @@ -0,0 +1,280 @@ +/* + pybind11/pytypes-inl.h: Out-of-line definitions for pytypes.h + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +// Every function defined here must start with PYBIND11_INLINE (or +// PYBIND11_NOINLINE_ATTR PYBIND11_INLINE). In the default header-only mode this file is +// included at the bottom of pytypes.h; when PYBIND11_PRECOMPILED is defined it is only +// compiled into the pybind11 static library (see src/). + +#pragma once + +#include "pytypes.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +PYBIND11_INLINE error_fetch_and_normalize::error_fetch_and_normalize(const char *called) { + PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); + if (!m_type) { + pybind11_fail("Internal error: " + std::string(called) + + " called while " + "Python error indicator not set."); + } + const char *exc_type_name_orig = detail::obj_class_name(m_type.ptr()); + if (exc_type_name_orig == nullptr) { + pybind11_fail("Internal error: " + std::string(called) + + " failed to obtain the name " + "of the original active exception type."); + } + m_lazy_error_string = exc_type_name_orig; +#if PY_VERSION_HEX >= 0x030C0000 + // The presence of __notes__ is likely due to exception normalization + // errors, although that is not necessarily true, therefore insert a + // hint only: + const int has_notes = PyObject_HasAttrString(m_value.ptr(), "__notes__"); + if (has_notes == 1) { + m_lazy_error_string += "[WITH __notes__]"; + } else if (has_notes == -1) { + // Ignore secondary errors when probing for __notes__ to avoid leaking a + // spurious exception while still reporting the original error. + PyErr_Clear(); + } +#else + // PyErr_NormalizeException() may change the exception type if there are cascading + // failures. This can potentially be extremely confusing. + PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); + if (m_type.ptr() == nullptr) { + pybind11_fail("Internal error: " + std::string(called) + + " failed to normalize the " + "active exception."); + } + const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); + if (exc_type_name_norm == nullptr) { + pybind11_fail("Internal error: " + std::string(called) + + " failed to obtain the name " + "of the normalized active exception type."); + } + if (exc_type_name_norm != m_lazy_error_string) { + std::string msg = std::string(called) + + ": MISMATCH of original and normalized " + "active exception types: "; + msg += "ORIGINAL "; + msg += m_lazy_error_string; + msg += " REPLACED BY "; + msg += exc_type_name_norm; + msg += ": " + format_value_and_trace(); + pybind11_fail(msg); + } +#endif +} + +PYBIND11_INLINE std::string error_fetch_and_normalize::format_value_and_trace() const { + std::string result; + std::string message_error_string; + if (m_value) { + auto value_str = reinterpret_steal(PyObject_Str(m_value.ptr())); + constexpr const char *message_unavailable_exc + = ""; + if (!value_str) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + // Not using `value_str.cast()`, to not potentially throw a secondary + // error_already_set that will then result in process termination (#4288). + auto value_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace")); + if (!value_bytes) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + result = std::string(buffer, static_cast(length)); + } + } + } +#if PY_VERSION_HEX >= 0x030B0000 + auto notes = reinterpret_steal(PyObject_GetAttrString(m_value.ptr(), "__notes__")); + if (!notes) { + PyErr_Clear(); // No notes is good news. + } else { + auto len_notes = PyList_Size(notes.ptr()); + if (len_notes < 0) { + result += "\nFAILURE obtaining len(__notes__): " + detail::error_string(); + } else { + result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; + for (ssize_t i = 0; i < len_notes; i++) { + PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + auto note_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); + if (!note_bytes) { + result += "\nFAILURE obtaining __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length) == -1) { + result += "\nFAILURE formatting __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + result += '\n'; + result += std::string(buffer, static_cast(length)); + } + } + } + } + } +#endif + } else { + result = ""; + } + if (result.empty()) { + result = ""; + } + + bool have_trace = false; + if (m_trace) { +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) + auto *tb = reinterpret_cast(m_trace.ptr()); + + // Get the deepest trace possible. + while (tb->tb_next) { + tb = tb->tb_next; + } + + PyFrameObject *frame = tb->tb_frame; + Py_XINCREF(frame); + result += "\n\nAt:\n"; + while (frame) { + PyCodeObject *f_code = PyFrame_GetCode(frame); + int lineno = PyFrame_GetLineNumber(frame); + result += " "; + result += handle(f_code->co_filename).cast(); + result += '('; + result += std::to_string(lineno); + result += "): "; + result += handle(f_code->co_name).cast(); + result += '\n'; + Py_DECREF(f_code); + auto *b_frame = PyFrame_GetBack(frame); + Py_DECREF(frame); + frame = b_frame; + } + + have_trace = true; +#endif //! defined(PYPY_VERSION) + } + + if (!message_error_string.empty()) { + if (!have_trace) { + result += '\n'; + } + result += "\nMESSAGE UNAVAILABLE DUE TO EXCEPTION: " + message_error_string; + } + + return result; +} + +PYBIND11_INLINE std::string const &error_fetch_and_normalize::error_string() const { + if (!m_lazy_error_string_completed) { + m_lazy_error_string += ": " + format_value_and_trace(); + m_lazy_error_string_completed = true; + } + return m_lazy_error_string; +} + +PYBIND11_INLINE void error_fetch_and_normalize::restore() { + if (m_restore_called) { + pybind11_fail("Internal error: pybind11::detail::error_fetch_and_normalize::restore() " + "called a second time. ORIGINAL ERROR: " + + error_string()); + } + PyErr_Restore(m_type.inc_ref().ptr(), m_value.inc_ref().ptr(), m_trace.inc_ref().ptr()); + m_restore_called = true; +} + +PYBIND11_INLINE std::string error_string() { + return error_fetch_and_normalize("pybind11::detail::error_string").error_string(); +} + +PYBIND11_NAMESPACE_END(detail) + +PYBIND11_INLINE void raise_from(PyObject *type, const char *message) { + // Based on _PyErr_FormatVFromCause: + // https://github.com/python/cpython/blob/467ab194fc6189d9f7310c89937c51abeac56839/Python/errors.c#L405 + // See https://github.com/pybind/pybind11/pull/2112 for details. + PyObject *exc = nullptr, *val = nullptr, *val2 = nullptr, *tb = nullptr; + + assert(PyErr_Occurred()); + PyErr_Fetch(&exc, &val, &tb); + PyErr_NormalizeException(&exc, &val, &tb); + if (tb != nullptr) { + PyException_SetTraceback(val, tb); + Py_DECREF(tb); + } + Py_DECREF(exc); + assert(!PyErr_Occurred()); + + PyErr_SetString(type, message); + + PyErr_Fetch(&exc, &val2, &tb); + PyErr_NormalizeException(&exc, &val2, &tb); + Py_INCREF(val); + PyException_SetCause(val2, val); + PyException_SetContext(val2, val); + PyErr_Restore(exc, val2, tb); +} + +PYBIND11_INLINE void raise_from(error_already_set &err, PyObject *type, const char *message) { + err.restore(); + raise_from(type, message); +} + +/// @cond DUPLICATE +PYBIND11_INLINE memoryview memoryview::from_buffer(void *ptr, + ssize_t itemsize, + const char *format, + detail::any_container shape, + detail::any_container strides, + bool readonly) { + size_t ndim = shape->size(); + if (ndim != strides->size()) { + pybind11_fail("memoryview: shape length doesn't match strides length"); + } + ssize_t size = ndim != 0u ? 1 : 0; + for (size_t i = 0; i < ndim; ++i) { + size *= (*shape)[i]; + } + Py_buffer view; + view.buf = ptr; + view.obj = nullptr; + view.len = size * itemsize; + view.readonly = static_cast(readonly); + view.itemsize = itemsize; + view.format = const_cast(format); + view.ndim = static_cast(ndim); + view.shape = shape->data(); + view.strides = strides->data(); + view.suboffsets = nullptr; + view.internal = nullptr; + PyObject *obj = PyMemoryView_FromBuffer(&view); + if (!obj) { + throw error_already_set(); + } + return memoryview(object(obj, stolen_t{})); +} +/// @endcond + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 13a6ebaa4d..71cc31b044 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -528,194 +528,16 @@ struct error_fetch_and_normalize { // would be more complex. // Starting with Python 3.12, PyErr_Fetch() normalizes exceptions immediately. // Any errors during normalization are tracked under __notes__. - explicit error_fetch_and_normalize(const char *called) { - PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); - if (!m_type) { - pybind11_fail("Internal error: " + std::string(called) - + " called while " - "Python error indicator not set."); - } - const char *exc_type_name_orig = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_orig == nullptr) { - pybind11_fail("Internal error: " + std::string(called) - + " failed to obtain the name " - "of the original active exception type."); - } - m_lazy_error_string = exc_type_name_orig; -#if PY_VERSION_HEX >= 0x030C0000 - // The presence of __notes__ is likely due to exception normalization - // errors, although that is not necessarily true, therefore insert a - // hint only: - const int has_notes = PyObject_HasAttrString(m_value.ptr(), "__notes__"); - if (has_notes == 1) { - m_lazy_error_string += "[WITH __notes__]"; - } else if (has_notes == -1) { - // Ignore secondary errors when probing for __notes__ to avoid leaking a - // spurious exception while still reporting the original error. - PyErr_Clear(); - } -#else - // PyErr_NormalizeException() may change the exception type if there are cascading - // failures. This can potentially be extremely confusing. - PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); - if (m_type.ptr() == nullptr) { - pybind11_fail("Internal error: " + std::string(called) - + " failed to normalize the " - "active exception."); - } - const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_norm == nullptr) { - pybind11_fail("Internal error: " + std::string(called) - + " failed to obtain the name " - "of the normalized active exception type."); - } - if (exc_type_name_norm != m_lazy_error_string) { - std::string msg = std::string(called) - + ": MISMATCH of original and normalized " - "active exception types: "; - msg += "ORIGINAL "; - msg += m_lazy_error_string; - msg += " REPLACED BY "; - msg += exc_type_name_norm; - msg += ": " + format_value_and_trace(); - pybind11_fail(msg); - } -#endif - } + explicit error_fetch_and_normalize(const char *called); error_fetch_and_normalize(const error_fetch_and_normalize &) = delete; error_fetch_and_normalize(error_fetch_and_normalize &&) = delete; - std::string format_value_and_trace() const { - std::string result; - std::string message_error_string; - if (m_value) { - auto value_str = reinterpret_steal(PyObject_Str(m_value.ptr())); - constexpr const char *message_unavailable_exc - = ""; - if (!value_str) { - message_error_string = detail::error_string(); - result = message_unavailable_exc; - } else { - // Not using `value_str.cast()`, to not potentially throw a secondary - // error_already_set that will then result in process termination (#4288). - auto value_bytes = reinterpret_steal( - PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace")); - if (!value_bytes) { - message_error_string = detail::error_string(); - result = message_unavailable_exc; - } else { - char *buffer = nullptr; - Py_ssize_t length = 0; - if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) { - message_error_string = detail::error_string(); - result = message_unavailable_exc; - } else { - result = std::string(buffer, static_cast(length)); - } - } - } -#if PY_VERSION_HEX >= 0x030B0000 - auto notes - = reinterpret_steal(PyObject_GetAttrString(m_value.ptr(), "__notes__")); - if (!notes) { - PyErr_Clear(); // No notes is good news. - } else { - auto len_notes = PyList_Size(notes.ptr()); - if (len_notes < 0) { - result += "\nFAILURE obtaining len(__notes__): " + detail::error_string(); - } else { - result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; - for (ssize_t i = 0; i < len_notes; i++) { - PyObject *note = PyList_GET_ITEM(notes.ptr(), i); - auto note_bytes = reinterpret_steal( - PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); - if (!note_bytes) { - result += "\nFAILURE obtaining __notes__[" + std::to_string(i) - + "]: " + detail::error_string(); - } else { - char *buffer = nullptr; - Py_ssize_t length = 0; - if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length) - == -1) { - result += "\nFAILURE formatting __notes__[" + std::to_string(i) - + "]: " + detail::error_string(); - } else { - result += '\n'; - result += std::string(buffer, static_cast(length)); - } - } - } - } - } -#endif - } else { - result = ""; - } - if (result.empty()) { - result = ""; - } - - bool have_trace = false; - if (m_trace) { -#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) - auto *tb = reinterpret_cast(m_trace.ptr()); - - // Get the deepest trace possible. - while (tb->tb_next) { - tb = tb->tb_next; - } - - PyFrameObject *frame = tb->tb_frame; - Py_XINCREF(frame); - result += "\n\nAt:\n"; - while (frame) { - PyCodeObject *f_code = PyFrame_GetCode(frame); - int lineno = PyFrame_GetLineNumber(frame); - result += " "; - result += handle(f_code->co_filename).cast(); - result += '('; - result += std::to_string(lineno); - result += "): "; - result += handle(f_code->co_name).cast(); - result += '\n'; - Py_DECREF(f_code); - auto *b_frame = PyFrame_GetBack(frame); - Py_DECREF(frame); - frame = b_frame; - } - - have_trace = true; -#endif //! defined(PYPY_VERSION) - } - - if (!message_error_string.empty()) { - if (!have_trace) { - result += '\n'; - } - result += "\nMESSAGE UNAVAILABLE DUE TO EXCEPTION: " + message_error_string; - } - - return result; - } + std::string format_value_and_trace() const; - std::string const &error_string() const { - if (!m_lazy_error_string_completed) { - m_lazy_error_string += ": " + format_value_and_trace(); - m_lazy_error_string_completed = true; - } - return m_lazy_error_string; - } + std::string const &error_string() const; - void restore() { - if (m_restore_called) { - pybind11_fail("Internal error: pybind11::detail::error_fetch_and_normalize::restore() " - "called a second time. ORIGINAL ERROR: " - + error_string()); - } - PyErr_Restore(m_type.inc_ref().ptr(), m_value.inc_ref().ptr(), m_trace.inc_ref().ptr()); - m_restore_called = true; - } + void restore(); bool matches(handle exc) const { return (PyErr_GivenExceptionMatches(m_type.ptr(), exc.ptr()) != 0); @@ -731,10 +553,6 @@ struct error_fetch_and_normalize { mutable bool m_restore_called = false; }; -inline std::string error_string() { - return error_fetch_and_normalize("pybind11::detail::error_string").error_string(); -} - PYBIND11_NAMESPACE_END(detail) /// Fetch and hold an error which was already set in Python. An instance of this is typically @@ -799,39 +617,12 @@ class PYBIND11_EXPORT_EXCEPTION error_already_set : public std::exception { /// Replaces the current Python error indicator with the chosen error, performing a /// 'raise from' to indicate that the chosen error was caused by the original error. -inline void raise_from(PyObject *type, const char *message) { - // Based on _PyErr_FormatVFromCause: - // https://github.com/python/cpython/blob/467ab194fc6189d9f7310c89937c51abeac56839/Python/errors.c#L405 - // See https://github.com/pybind/pybind11/pull/2112 for details. - PyObject *exc = nullptr, *val = nullptr, *val2 = nullptr, *tb = nullptr; - - assert(PyErr_Occurred()); - PyErr_Fetch(&exc, &val, &tb); - PyErr_NormalizeException(&exc, &val, &tb); - if (tb != nullptr) { - PyException_SetTraceback(val, tb); - Py_DECREF(tb); - } - Py_DECREF(exc); - assert(!PyErr_Occurred()); - - PyErr_SetString(type, message); - - PyErr_Fetch(&exc, &val2, &tb); - PyErr_NormalizeException(&exc, &val2, &tb); - Py_INCREF(val); - PyException_SetCause(val2, val); - PyException_SetContext(val2, val); - PyErr_Restore(exc, val2, tb); -} +void raise_from(PyObject *type, const char *message); /// Sets the current Python error indicator with the chosen error, performing a 'raise from' /// from the error contained in error_already_set to indicate that the chosen error was /// caused by the original error. -inline void raise_from(error_already_set &err, PyObject *type, const char *message) { - err.restore(); - raise_from(type, message); -} +void raise_from(error_already_set &err, PyObject *type, const char *message); /** \defgroup python_builtins const_name Unless stated otherwise, the following C++ functions behave the same @@ -2498,40 +2289,6 @@ class memoryview : public object { #endif }; -/// @cond DUPLICATE -inline memoryview memoryview::from_buffer(void *ptr, - ssize_t itemsize, - const char *format, - detail::any_container shape, - detail::any_container strides, - bool readonly) { - size_t ndim = shape->size(); - if (ndim != strides->size()) { - pybind11_fail("memoryview: shape length doesn't match strides length"); - } - ssize_t size = ndim != 0u ? 1 : 0; - for (size_t i = 0; i < ndim; ++i) { - size *= (*shape)[i]; - } - Py_buffer view; - view.buf = ptr; - view.obj = nullptr; - view.len = size * itemsize; - view.readonly = static_cast(readonly); - view.itemsize = itemsize; - view.format = const_cast(format); - view.ndim = static_cast(ndim); - view.shape = shape->data(); - view.strides = strides->data(); - view.suboffsets = nullptr; - view.internal = nullptr; - PyObject *obj = PyMemoryView_FromBuffer(&view); - if (!obj) { - throw error_already_set(); - } - return memoryview(object(obj, stolen_t{})); -} -/// @endcond /// @} pytypes /// \addtogroup python_builtins @@ -2724,3 +2481,7 @@ inline object get_module_name_if_available(handle scope) { PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) + +#ifndef PYBIND11_PRECOMPILED +# include "pytypes-inl.h" // IWYU pragma: export +#endif diff --git a/src/internals.cpp b/src/internals.cpp new file mode 100644 index 0000000000..d6522b8764 --- /dev/null +++ b/src/internals.cpp @@ -0,0 +1,11 @@ +// Copyright (c) 2025 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#if !defined(PYBIND11_PRECOMPILED) +# error "pybind11 library sources must be compiled with PYBIND11_PRECOMPILED defined." +#endif + +#include + +#include diff --git a/src/pybind11_combined.cpp b/src/pybind11_combined.cpp new file mode 100644 index 0000000000..19ff38084b --- /dev/null +++ b/src/pybind11_combined.cpp @@ -0,0 +1,17 @@ +// Copyright (c) 2025 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// Single-TU build of the pybind11 library sources, for build systems that prefer adding +// one file over one file per header (e.g. setuptools). Compile this file (and every TU +// that includes pybind11) with PYBIND11_PRECOMPILED defined. Keep in sync with the list +// of -inl.h files; the CMake path compiles the individual src/*.cpp files instead. + +#if !defined(PYBIND11_PRECOMPILED) +# error "pybind11 library sources must be compiled with PYBIND11_PRECOMPILED defined." +#endif + +#include + +#include +#include diff --git a/src/pytypes.cpp b/src/pytypes.cpp new file mode 100644 index 0000000000..9bcfd90521 --- /dev/null +++ b/src/pytypes.cpp @@ -0,0 +1,13 @@ +// Copyright (c) 2025 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#if !defined(PYBIND11_PRECOMPILED) +# error "pybind11 library sources must be compiled with PYBIND11_PRECOMPILED defined." +#endif + +// pybind11.h first: the -inl.h definitions instantiate templates (e.g. handle::cast) +// whose definitions live in other headers. +#include + +#include diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d6415b98bc..03ce0f1256 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -71,6 +71,13 @@ option(DOWNLOAD_CATCH "Download catch2 if not found" OFF) option(DOWNLOAD_EIGEN "Download EIGEN" OFF) option(PYBIND11_CUDA_TESTS "Enable building CUDA tests" OFF) option(PYBIND11_TEST_SMART_HOLDER "Change the default to smart holder" OFF) +option(PYBIND11_TEST_PRECOMPILE "Build the test modules against the precompiled library" OFF) + +if(PYBIND11_TEST_PRECOMPILE) + # Picked up by every pybind11_add_module() call below; any definition missing + # from the precompiled library shows up as a link error across the test modules. + set(PYBIND11_PRECOMPILE ON) +endif() set(PYBIND11_TEST_OVERRIDE "" CACHE STRING "Tests from ;-separated list of *.cpp files will be built instead of all tests") @@ -483,8 +490,16 @@ foreach(target ${test_targets}) set_property(SOURCE ${target}.cpp PROPERTY LANGUAGE CUDA) endif() + # These modules redefine PYBIND11_INTERNALS_VERSION inside their own TU, so they + # cannot share the precompiled library. + set(no_precompile_arg "") + if("${target}" MATCHES "^(exo_planet_pybind11|cross_module_gil_utils)$") + set(no_precompile_arg NO_PRECOMPILE) + endif() + # Create the binding library - pybind11_add_module(${target} THIN_LTO ${target}.cpp ${test_files} ${PYBIND11_HEADERS}) + pybind11_add_module(${target} THIN_LTO ${no_precompile_arg} ${target}.cpp ${test_files} + ${PYBIND11_HEADERS}) pybind11_enable_warnings(${target}) if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 164611db34..b02892a738 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -62,6 +62,7 @@ "include/pybind11/operators.h", "include/pybind11/options.h", "include/pybind11/pybind11.h", + "include/pybind11/pytypes-inl.h", "include/pybind11/pytypes.h", "include/pybind11/subinterpreter.h", "include/pybind11/stl.h", @@ -90,6 +91,7 @@ "include/pybind11/detail/function_ref.h", "include/pybind11/detail/holder_caster_foreign_helpers.h", "include/pybind11/detail/init.h", + "include/pybind11/detail/internals-inl.h", "include/pybind11/detail/internals.h", "include/pybind11/detail/native_enum_data.h", "include/pybind11/detail/pybind11_namespace_macros.h", @@ -126,6 +128,14 @@ "share/pkgconfig/pybind11.pc", } +sdist_src_files = { + "src/internals.cpp", + "src/pybind11_combined.cpp", + "src/pytypes.cpp", +} + +src_files = {f"share/pybind11/{n}" for n in sdist_src_files} + py_files = { "__init__.py", "__main__.py", @@ -138,7 +148,7 @@ } headers = main_headers | conduit_headers | detail_headers | eigen_headers | stl_headers -generated_files = cmake_files | pkgconfig_files +generated_files = cmake_files | pkgconfig_files | src_files all_files = headers | generated_files | py_files sdist_files = { @@ -204,7 +214,7 @@ def test_build_sdist(monkeypatch, tmpdir): pyproject_toml = read_tz_file(tar, "pyproject.toml") pkg_info = read_tz_file(tar, pkg_info_path).decode("utf-8") - files = headers | sdist_files + files = headers | sdist_src_files | sdist_files assert files <= simpler assert b'name = "pybind11"' in pyproject_toml @@ -241,7 +251,7 @@ def test_build_global_dist(monkeypatch, tmpdir): pyproject_toml = read_tz_file(tar, "pyproject.toml") pkg_info = read_tz_file(tar, pkg_info_path).decode("utf-8") - files = headers | sdist_files + files = headers | sdist_src_files | sdist_files assert files <= simpler assert b'name = "pybind11-global"' in pyproject_toml diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index ce63a69062..7da8f2e7ed 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -60,6 +60,7 @@ possibly_uninitialized(PYTHON_MODULE_EXTENSION Python_INTERPRETER_ID) pybind11_add_build_test(subdirectory_function) pybind11_add_build_test(subdirectory_target) +pybind11_add_build_test(subdirectory_precompile) if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy") @@ -77,6 +78,7 @@ if(PYBIND11_INSTALL) pybind11_add_build_test(installed_function INSTALL) endif() pybind11_add_build_test(installed_target INSTALL) + pybind11_add_build_test(installed_precompile INSTALL) if(NOT ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" diff --git a/tests/test_cmake_build/installed_precompile/CMakeLists.txt b/tests/test_cmake_build/installed_precompile/CMakeLists.txt new file mode 100644 index 0000000000..ad3a6dfd0a --- /dev/null +++ b/tests/test_cmake_build/installed_precompile/CMakeLists.txt @@ -0,0 +1,32 @@ +cmake_minimum_required(VERSION 3.15...4.2) + +project(test_installed_precompile CXX) + +find_package(pybind11 CONFIG REQUIRED) +message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}") + +pybind11_add_module(test_installed_precompile PRECOMPILE ../main.cpp) +set_target_properties(test_installed_precompile PROPERTIES OUTPUT_NAME test_cmake_build) + +if(NOT TARGET pybind11::precompiled) + message(FATAL_ERROR "pybind11::precompiled target missing") +endif() + +if(DEFINED Python_EXECUTABLE) + set(_Python_EXECUTABLE "${Python_EXECUTABLE}") +elseif(DEFINED PYTHON_EXECUTABLE) + set(_Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + message(FATAL_ERROR "No Python executable defined (should not be possible at this stage)") +endif() + +add_custom_target( + check_installed_precompile + ${CMAKE_COMMAND} + -E + env + PYTHONPATH=$ + ${_Python_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/../test.py + ${PROJECT_NAME} + DEPENDS test_installed_precompile) diff --git a/tests/test_cmake_build/subdirectory_precompile/CMakeLists.txt b/tests/test_cmake_build/subdirectory_precompile/CMakeLists.txt new file mode 100644 index 0000000000..9592b9649a --- /dev/null +++ b/tests/test_cmake_build/subdirectory_precompile/CMakeLists.txt @@ -0,0 +1,36 @@ +cmake_minimum_required(VERSION 3.15...4.2) + +project(test_subdirectory_precompile CXX) + +# Allow PYTHON_EXECUTABLE if in FINDPYTHON mode and building pybind11's tests +# (makes transition easier while we support both modes). +if(DEFINED PYTHON_EXECUTABLE AND NOT DEFINED Python_EXECUTABLE) + set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +endif() + +add_subdirectory("${pybind11_SOURCE_DIR}" pybind11) +pybind11_add_module(test_subdirectory_precompile PRECOMPILE ../main.cpp) +set_target_properties(test_subdirectory_precompile PROPERTIES OUTPUT_NAME test_cmake_build) + +if(NOT TARGET pybind11::precompiled) + message(FATAL_ERROR "pybind11::precompiled target missing") +endif() + +if(DEFINED Python_EXECUTABLE) + set(_Python_EXECUTABLE "${Python_EXECUTABLE}") +elseif(DEFINED PYTHON_EXECUTABLE) + set(_Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + message(FATAL_ERROR "No Python executable defined (should not be possible at this stage)") +endif() + +add_custom_target( + check_subdirectory_precompile + ${CMAKE_COMMAND} + -E + env + PYTHONPATH=$ + ${_Python_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/../test.py + ${PROJECT_NAME} + DEPENDS test_subdirectory_precompile) diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index d75fb67520..e6dfea0e76 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -466,3 +466,47 @@ function(pybind11_strip target_name) COMMAND ${CMAKE_STRIP} ${x_opt} $) endif() endfunction() + +# --------------------- pybind11_precompile ------------------------- + +# Create the pybind11::precompiled static library (once per build tree). It is +# built from the consumer's project with the consumer's flags; each extension +# module links its own copy, preserving pybind11's per-module state. Modules +# using it must compile with PYBIND11_PRECOMPILED, which the PUBLIC compile +# definition below provides automatically. +function(pybind11_precompile) + if(TARGET pybind11_precompiled) + return() + endif() + + if(PYBIND11_NOPYTHON) + message(FATAL_ERROR "pybind11_precompile requires Python headers; it cannot be used " + "with PYBIND11_NOPYTHON") + endif() + + if(NOT pybind11_SRC_DIR OR NOT EXISTS "${pybind11_SRC_DIR}") + message(FATAL_ERROR "pybind11 library sources not found (pybind11_SRC_DIR: " + "'${pybind11_SRC_DIR}')") + endif() + + file(GLOB _pybind11_precompile_sources "${pybind11_SRC_DIR}/*.cpp") + list(FILTER _pybind11_precompile_sources EXCLUDE REGEX "pybind11_combined\\.cpp$") + + add_library(pybind11_precompiled STATIC EXCLUDE_FROM_ALL ${_pybind11_precompile_sources}) + add_library(pybind11::precompiled ALIAS pybind11_precompiled) + target_compile_definitions(pybind11_precompiled PUBLIC PYBIND11_PRECOMPILED) + target_link_libraries( + pybind11_precompiled + PUBLIC pybind11::headers + PRIVATE pybind11::pybind11) + set_target_properties(pybind11_precompiled PROPERTIES POSITION_INDEPENDENT_CODE ON) + if(NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET) + set_target_properties(pybind11_precompiled PROPERTIES CXX_VISIBILITY_PRESET hidden) + endif() + if(NOT DEFINED CMAKE_VISIBILITY_INLINES_HIDDEN) + set_target_properties(pybind11_precompiled PROPERTIES VISIBILITY_INLINES_HIDDEN ON) + endif() + if(MSVC) + target_link_libraries(pybind11_precompiled PRIVATE pybind11::windows_extras) + endif() +endfunction() diff --git a/tools/pybind11Config.cmake.in b/tools/pybind11Config.cmake.in index f52b2fb891..abcd43e199 100644 --- a/tools/pybind11Config.cmake.in +++ b/tools/pybind11Config.cmake.in @@ -211,6 +211,9 @@ Using ``find_package`` with version info is not recommended except for release v # This will be relative unless explicitly set as absolute set(pybind11_INCLUDE_DIR "@pybind11_INCLUDEDIR@") +# Location of the library sources for the opt-in precompiled mode +set(pybind11_SRC_DIR "@pybind11_SRCDIR@") + set(pybind11_LIBRARY "") set(pybind11_DEFINITIONS USING_pybind11) set(pybind11_VERSION_TYPE "@pybind11_VERSION_TYPE@") diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index b0fe20768d..9519dfa111 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -255,8 +255,10 @@ endif() # WITHOUT_SOABI and WITH_SOABI will disable the custom extension handling used by pybind11. # WITH_SOABI is passed on to python_add_library. function(pybind11_add_module target_name) - cmake_parse_arguments(PARSE_ARGV 1 ARG - "STATIC;SHARED;MODULE;THIN_LTO;OPT_SIZE;NO_EXTRAS;WITHOUT_SOABI" "" "") + cmake_parse_arguments( + PARSE_ARGV 1 ARG + "STATIC;SHARED;MODULE;THIN_LTO;OPT_SIZE;NO_EXTRAS;WITHOUT_SOABI;PRECOMPILE;NO_PRECOMPILE" "" + "") if(ARG_STATIC) set(lib_type STATIC) @@ -282,6 +284,11 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::embed) endif() + if((ARG_PRECOMPILE OR PYBIND11_PRECOMPILE) AND NOT ARG_NO_PRECOMPILE) + pybind11_precompile() + target_link_libraries(${target_name} PRIVATE pybind11::precompiled) + endif() + # -fvisibility=hidden is required to allow multiple modules compiled against # different pybind versions to work properly, and for some features (e.g. # py::module_local). We force it on everything inside the `pybind11` diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 81faee7d8b..95545cd539 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -139,7 +139,8 @@ endfunction() # [NO_EXTRAS] [THIN_LTO] [OPT_SIZE] source1 [source2 ...]) # function(pybind11_add_module target_name) - set(options "MODULE;SHARED;EXCLUDE_FROM_ALL;NO_EXTRAS;SYSTEM;THIN_LTO;OPT_SIZE") + set(options + "MODULE;SHARED;EXCLUDE_FROM_ALL;NO_EXTRAS;SYSTEM;THIN_LTO;OPT_SIZE;PRECOMPILE;NO_PRECOMPILE") cmake_parse_arguments(ARG "${options}" "" "" ${ARGN}) if(ARG_MODULE AND ARG_SHARED) @@ -160,6 +161,11 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::module) + if((ARG_PRECOMPILE OR PYBIND11_PRECOMPILE) AND NOT ARG_NO_PRECOMPILE) + pybind11_precompile() + target_link_libraries(${target_name} PRIVATE pybind11::precompiled) + endif() + if(ARG_SYSTEM) message( STATUS From b6ede88e3e6eb85fe28b5c7db1f3c0d5b95d4ac0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 02:05:00 +0000 Subject: [PATCH 2/4] style: pre-commit fixes --- src/internals.cpp | 3 +-- src/pybind11_combined.cpp | 3 +-- src/pytypes.cpp | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/internals.cpp b/src/internals.cpp index d6522b8764..86a3549507 100644 --- a/src/internals.cpp +++ b/src/internals.cpp @@ -6,6 +6,5 @@ # error "pybind11 library sources must be compiled with PYBIND11_PRECOMPILED defined." #endif -#include - #include +#include diff --git a/src/pybind11_combined.cpp b/src/pybind11_combined.cpp index 19ff38084b..751b0b1dbe 100644 --- a/src/pybind11_combined.cpp +++ b/src/pybind11_combined.cpp @@ -11,7 +11,6 @@ # error "pybind11 library sources must be compiled with PYBIND11_PRECOMPILED defined." #endif -#include - #include +#include #include diff --git a/src/pytypes.cpp b/src/pytypes.cpp index 9bcfd90521..c0d655da96 100644 --- a/src/pytypes.cpp +++ b/src/pytypes.cpp @@ -9,5 +9,4 @@ // pybind11.h first: the -inl.h definitions instantiate templates (e.g. handle::cast) // whose definitions live in other headers. #include - #include From 6864b86375f1de6eee5f1c11ae7032fd603e2b90 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 6 Aug 2026 23:25:07 -0400 Subject: [PATCH 3/4] fix(tidy): keep the tidy preset header-only In precompiled mode the -inl.h definitions are intentionally non-inline, so misc-definitions-in-headers fires on every one. The header-only tidy build already analyzes all -inl.h bodies via the bottom-of-header includes. Assisted-by: ClaudeCode:claude-fable-5 --- CMakePresets.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index 6e86aa331f..42bf3ade9d 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -36,8 +36,7 @@ "binaryDir": "build-tidy", "cacheVariables": { "CMAKE_CXX_CLANG_TIDY": "clang-tidy;--use-color;--warnings-as-errors=*", - "CMAKE_CXX_STANDARD": "17", - "PYBIND11_TEST_PRECOMPILE": "ON" + "CMAKE_CXX_STANDARD": "17" } } ], From be059bad3169e18d7170724f60231d55db18ff8a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 6 Aug 2026 23:32:21 -0400 Subject: [PATCH 4/4] fix(cmake): compile the precompiled library with the interpreter ABI macros pybind11::pybind11 only carries headers; Py_GIL_DISABLED lives on Python::Module via pybind11::module. Without it the library is ABI-mismatched on free-threaded builds, and on Windows the pyconfig.h autolink pragma in its objects requests pythonXY.lib instead of pythonXYt.lib. Assisted-by: ClaudeCode:claude-fable-5 --- tools/pybind11Common.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index e6dfea0e76..7a275ef2ae 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -495,10 +495,13 @@ function(pybind11_precompile) add_library(pybind11_precompiled STATIC EXCLUDE_FROM_ALL ${_pybind11_precompile_sources}) add_library(pybind11::precompiled ALIAS pybind11_precompiled) target_compile_definitions(pybind11_precompiled PUBLIC PYBIND11_PRECOMPILED) + # pybind11::module (not just pybind11::pybind11): the library must compile with the + # interpreter's ABI macros (e.g. Py_GIL_DISABLED, which FindPython attaches to + # Python::Module); on free-threaded Windows they select the correct autolink library. target_link_libraries( pybind11_precompiled PUBLIC pybind11::headers - PRIVATE pybind11::pybind11) + PRIVATE pybind11::module) set_target_properties(pybind11_precompiled PROPERTIES POSITION_INDEPENDENT_CODE ON) if(NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET) set_target_properties(pybind11_precompiled PROPERTIES CXX_VISIBILITY_PRESET hidden)