[feat/precompile-macro-infra] feat(cmake): opt-in precompiled static library, starting with pytypes.h

.github/workflows/ci.yml

@@ -71,6 +71,28 @@ jobs:
       python-version: ${{ matrix.python-version }}
       cmake-args: ${{ matrix.cmake-args }}
 
+  # Build the test suite against the precompiled pybind11 static library; a
+  # definition missing from the -inl.h split shows up here as a link error.
+  precompile:
+    if: github.event.pull_request.draft == false
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runs-on: ubuntu-latest
+            python-version: '3.13'
+          - runs-on: macos-latest
+            python-version: '3.13'
+          - runs-on: windows-2022
+            python-version: '3.13'
+
+    name: ⚡
+    uses: ./.github/workflows/reusable-standard.yml
+    with:
+      runs-on: ${{ matrix.runs-on }}
+      python-version: ${{ matrix.python-version }}
+      cmake-args: -DPYBIND11_TEST_PRECOMPILE=ON
+
   standard-large:
     if: github.event.pull_request.draft == false
     strategy:

CMakeLists.txt

@@ -200,6 +200,7 @@ set(PYBIND11_HEADERS
     include/pybind11/detail/function_ref.h
     include/pybind11/detail/holder_caster_foreign_helpers.h
     include/pybind11/detail/init.h
+    include/pybind11/detail/internals-inl.h
     include/pybind11/detail/internals.h
     include/pybind11/detail/native_enum_data.h
     include/pybind11/detail/pybind11_namespace_macros.h
@@ -234,6 +235,7 @@ set(PYBIND11_HEADERS
     include/pybind11/numpy.h
     include/pybind11/operators.h
     include/pybind11/pybind11.h
+    include/pybind11/pytypes-inl.h
     include/pybind11/pytypes.h
     include/pybind11/subinterpreter.h
     include/pybind11/stl.h
@@ -265,6 +267,11 @@ endif()
 
 list(TRANSFORM PYBIND11_HEADERS PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
 
+# Library sources for the opt-in precompiled mode (pybind11_precompile()).
+set(pybind11_SRC_DIR
+    "${CMAKE_CURRENT_SOURCE_DIR}/src"
+    CACHE INTERNAL "Directory containing the pybind11 library sources")
+
 # Cache variable so this can be used in parent projects
 set(pybind11_INCLUDE_DIR
     "${CMAKE_CURRENT_LIST_DIR}/include"
@@ -331,6 +338,8 @@ if(PYBIND11_INSTALL)
     install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION "${SKBUILD_HEADERS_DIR}")
   endif()
   install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/
+          DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src")
   set(PYBIND11_CMAKECONFIG_INSTALL_DIR
       "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}"
       CACHE STRING "install path for pybind11Config.cmake")
@@ -340,6 +349,11 @@ if(PYBIND11_INSTALL)
   else()
     set(pybind11_INCLUDEDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_INCLUDEDIR}")
   endif()
+  if(IS_ABSOLUTE "${CMAKE_INSTALL_DATAROOTDIR}")
+    set(pybind11_SRCDIR "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src")
+  else()
+    set(pybind11_SRCDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src")
+  endif()
 
   configure_package_config_file(
     tools/${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"

docs/Doxyfile

@@ -1,6 +1,7 @@
 PROJECT_NAME           = pybind11
 INPUT                  = ../include/pybind11/
 RECURSIVE              = YES
+EXCLUDE_PATTERNS       = *-inl.h
 
 GENERATE_HTML          = NO
 GENERATE_LATEX         = NO

include/pybind11/detail/common.h

@@ -468,6 +468,7 @@ PyModuleDef_Init should be treated like any other PyObject (so not shared across
     static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *);                                 \
     PYBIND11_PLUGIN_IMPL(name) {                                                                  \
         PYBIND11_CHECK_PYTHON_VERSION                                                             \
+        PYBIND11_PRECOMPILED_CONFIG_GUARD                                                         \
         try {                                                                                     \
             pybind11::detail::ensure_internals();                                                 \
             static ::pybind11::detail::slots_array mod_def_slots                                  \

include/pybind11/detail/internals-inl.h

@@ -0,0 +1,28 @@
+/*
+    pybind11/detail/internals-inl.h: Out-of-line definitions for internals.h
+
+    Copyright (c) 2017 Wenzel Jakob <wenzel.jakob@epfl.ch>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+// Every function defined here must start with PYBIND11_INLINE (or
+// PYBIND11_NOINLINE_ATTR PYBIND11_INLINE). In the default header-only mode this file is
+// included at the bottom of internals.h; when PYBIND11_PRECOMPILED is defined it is only
+// compiled into the pybind11 static library (see src/).
+
+#pragma once
+
+#include "internals.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+#if defined(PYBIND11_PRECOMPILED)
+// Link-time configuration guard; see the declaration in internals.h.
+PYBIND11_INLINE void PYBIND11_PRECOMPILED_CONFIG_CHECK() {}
+#endif
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/detail/internals.h

@@ -46,8 +46,51 @@
 #    error "PYBIND11_INTERNALS_VERSION 12 is the minimum for all platforms for pybind11 v3.1.0"
 #endif
 
+#if defined(PYBIND11_PRECOMPILED)
+// PYBIND11_PRECOMPILED_CONFIG_CHECK names a do-nothing function defined in the precompiled
+// pybind11 library. The identifier encodes every configuration macro that must match between
+// the library and the modules linking it. PYBIND11_MODULE calls it, so a mismatch (or a
+// missing library) surfaces as one readable undefined symbol at link time instead of many
+// unrelated ones at run time.
+#    if defined(Py_GIL_DISABLED)
+#        define PYBIND11_PRECOMPILED_CFG_GD 1
+#    else
+#        define PYBIND11_PRECOMPILED_CFG_GD 0
+#    endif
+#    if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+#        define PYBIND11_PRECOMPILED_CFG_SG 1
+#    else
+#        define PYBIND11_PRECOMPILED_CFG_SG 0
+#    endif
+#    if defined(PYBIND11_DETAILED_ERROR_MESSAGES)
+#        define PYBIND11_PRECOMPILED_CFG_DE 1
+#    else
+#        define PYBIND11_PRECOMPILED_CFG_DE 0
+#    endif
+// PYBIND11_CONCAT does not macro-expand its arguments (## suppresses expansion).
+#    define PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de)                                      \
+        pybind11_precompiled_config_v##v##_gd##gd##_sg##sg##_de##de
+#    define PYBIND11_PRECOMPILED_CONFIG_NAME(v, gd, sg, de)                                       \
+        PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de)
+#    define PYBIND11_PRECOMPILED_CONFIG_CHECK                                                     \
+        PYBIND11_PRECOMPILED_CONFIG_NAME(PYBIND11_INTERNALS_VERSION,                              \
+                                         PYBIND11_PRECOMPILED_CFG_GD,                             \
+                                         PYBIND11_PRECOMPILED_CFG_SG,                             \
+                                         PYBIND11_PRECOMPILED_CFG_DE)
+#    define PYBIND11_PRECOMPILED_CONFIG_GUARD                                                     \
+        ::pybind11::detail::PYBIND11_PRECOMPILED_CONFIG_CHECK();
+#else
+#    define PYBIND11_PRECOMPILED_CONFIG_GUARD
+#endif
+
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
+#if defined(PYBIND11_PRECOMPILED)
+PYBIND11_NAMESPACE_BEGIN(detail)
+void PYBIND11_PRECOMPILED_CONFIG_CHECK();
+PYBIND11_NAMESPACE_END(detail)
+#endif
+
 using ExceptionTranslator = void (*)(std::exception_ptr);
 
 // The old Python Thread Local Storage (TLS) API is deprecated in Python 3.7 in favor of the new
@@ -1080,3 +1123,7 @@ T &get_or_create_shared_data(const std::string &name) {
 }
 
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)
+
+#ifndef PYBIND11_PRECOMPILED
+#    include "internals-inl.h" // IWYU pragma: export
+#endif