Bump cppjit and CppInterOp to latest mirrors

bindings/pyroot/cppjit/CMakeLists.txt

@@ -11,7 +11,7 @@ include(GNUInstallDirs)
 # Perhaps this should permanently be OFF and users can build their own CppInterOp if they want to run the tests?
 option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF)
 set(CPPINTEROP_GIT_REPOSITORY "https://github.com/compiler-research/CppInterOp.git" CACHE STRING "")
-set(CPPINTEROP_GIT_TAG "c8126249dd18363000b89333b34313ca2a83fb82" CACHE STRING "")
+set(CPPINTEROP_GIT_TAG "f4adaa6b50542dd564089fe584d407bf15351c97" CACHE STRING "")
 set(CPPINTEROP_SOURCE_DIR "" CACHE PATH
     "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source")
 

bindings/pyroot/cppjit/cppjit-version.tag

@@ -1 +1 @@
-0f6cc367f2f7f169c6a09ffaa97169230e52b299
+3c66389821de2084c69cc9fdfcd26e32b47e16f5

bindings/pyroot/cppjit/python/cppjit/__init__.py

@@ -241,8 +241,10 @@ def _interpreter_lock():
 def cppdef(src, verbose=True):
     """Declare C++ source <src> to Cling."""
     with _stderr_capture() as err, _interpreter_lock():
-        errcode = gbl.Cpp.Declare(src, not verbose)
-    if not errcode == 0 or err.err:
+        ok = gbl.gInterpreter.Declare(src)
+    if not verbose:
+        err.err = ""
+    if not ok or err.err:
         if "warning" in err.err.lower() and "error" not in err.err.lower():
             warnings.warn(err.err, SyntaxWarning)
             return True
@@ -326,23 +328,22 @@ def load_library(name):
 def include(header):
     """Load (and JIT) header file <header> into Cling."""
     with _stderr_capture() as err, _interpreter_lock():
-        errcode = gbl.Cpp.Declare('#include "%s"' % header, False)
-    if not errcode == 0:
+        ok = gbl.gInterpreter.Declare('#include "%s"' % header)
+    if not ok:
         raise ImportError('Failed to load header file "%s"%s' % (header, err.err))
     return True
 
 
 def c_include(header):
     """Load (and JIT) header file <header> into Cling."""
     with _stderr_capture() as err, _interpreter_lock():
-        errcode = gbl.Cpp.Declare(
+        ok = gbl.gInterpreter.Declare(
             """extern "C" {
                                     #include "%s"
                                     }"""
-            % header,
-            False,
+            % header
         )
-    if not errcode == 0:
+    if not ok:
         raise ImportError('Failed to load header file "%s"%s' % (header, err.err))
     return True
 

bindings/pyroot/cppjit/src/cpyrt/API.h

@@ -145,10 +145,6 @@ typedef Converter* (*ConverterFactory_t)(cdims_t);
 CPYRT_EXTERN bool RegisterConverter(const std::string& name,
                                     ConverterFactory_t);
 
-// register a custom converter that is a reference to an existing converter
-CPYRT_EXTERN bool RegisterConverterAlias(const std::string& name,
-                                         const std::string& target);
-
 // remove a custom converter
 CPYRT_EXTERN bool UnregisterConverter(const std::string& name);
 
@@ -176,10 +172,6 @@ CPYRT_EXTERN void DestroyConverter(Converter* p);
 typedef Executor* (*ExecutorFactory_t)(cdims_t);
 CPYRT_EXTERN bool RegisterExecutor(const std::string& name, ExecutorFactory_t);
 
-// register a custom executor that is a reference to an existing converter
-CPYRT_EXTERN bool RegisterExecutorAlias(const std::string& name,
-                                        const std::string& target);
-
 // remove a custom executor
 CPYRT_EXTERN bool UnregisterExecutor(const std::string& name);
 

bindings/pyroot/cppjit/src/cpyrt/CPPMethod.h

@@ -80,6 +80,8 @@ class CPPMethod : public PyCallable {
   PyObject* Call(CPPInstance*& self, cpyrt_PyArgs_t args, size_t nargsf,
                  PyObject* kwds, CallContext* ctxt = nullptr) override;
 
+  interop::TCppMethod_t GetMethod() const { return fMethod; }
+
 protected:
   virtual bool ProcessArgs(PyCallArgs& args);
 
@@ -89,7 +91,6 @@ class CPPMethod : public PyCallable {
                          CallContext* ctxt = nullptr);
   PyObject* Execute(void* self, ptrdiff_t offset, CallContext* ctxt = nullptr);
 
-  interop::TCppMethod_t GetMethod() { return fMethod; }
   interop::TCppScope_t GetScope() { return fScope; }
   Executor* GetExecutor() { return fExecutor; }
   std::string GetSignatureString(bool show_formalargs = true);

bindings/pyroot/cppjit/src/cpyrt/CPPScope.cxx

@@ -18,6 +18,7 @@ using namespace cppjit;
 
 // Standard
 #include <algorithm> // for for_each
+#include <cctype>
 #include <set>
 #include <string.h>
 #include <string>
@@ -207,16 +208,11 @@ static PyObject* pt_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) {
   // creation of the python-side class; extend the size if this is a smart ptr
   //
   // A namespace can never be a smart pointer, so skip the check for namespaces.
-  // This matters beyond performance: GetSmartPtrInfo() resolves the scope name
-  // through a slow interpreter lookup that triggers autoloading of the library
-  // providing the scope, which we don't want to do unnecessarily.
   interop::TCppScope_t raw;
   interop::TCppMethod_t deref;
   if (CPPScope_CheckExact(subtype) &&
       !interop::IsNamespace(((CPPScope*)subtype)->fCppType)) {
-    if (interop::GetSmartPtrInfo(
-            interop::GetScopedFinalName(((CPPScope*)subtype)->fCppType), &raw,
-            &deref))
+    if (interop::GetSmartPtrInfo(((CPPScope*)subtype)->fCppType, &raw, &deref))
       subtype->tp_basicsize = sizeof(CPPSmartClass);
   }
 
@@ -324,6 +320,74 @@ static PyObject* pt_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) {
   return (PyObject*)result;
 }
 
+//----------------------------------------------------------------------------
+static bool IsPlainIdentifier(const std::string& name) {
+  if (name.empty() || std::isdigit((unsigned char)name[0]))
+    return false;
+  for (char c : name) {
+    if (!std::isalnum((unsigned char)c) && c != '_')
+      return false;
+  }
+  return true;
+}
+
+// Returns weak references to the proxies of the namespaces that the namespace
+// \c pyclass pulls in with "using namespace", refreshing the list as needed.
+static const std::vector<PyObject*>& GetUsingScopes(PyObject* pyclass) {
+  CPPScope* klass = (CPPScope*)pyclass;
+  const std::vector<interop::TCppScope_t>& uv =
+      interop::GetUsingNamespaces(klass->fCppType);
+  if (!klass->fImp.fUsing || uv.size() != klass->fImp.fUsing->size()) {
+    if (klass->fImp.fUsing) {
+      for (auto pyref : *klass->fImp.fUsing)
+        Py_DECREF(pyref);
+      klass->fImp.fUsing->clear();
+    } else
+      klass->fImp.fUsing = new std::vector<PyObject*>;
+
+    // reload and reset weak refs
+    for (auto uid : uv) {
+      std::string uname = interop::GetScopedFinalName(uid);
+      PyObject* pyuscope = CreateScopeProxy(uname);
+      if (pyuscope) {
+        klass->fImp.fUsing->push_back(PyWeakref_NewRef(pyuscope, nullptr));
+        // the namespace may not otherwise be held, so tie the lifetimes
+        PyObject* llname =
+            PyUnicode_FromString(("__lifeline_" + uname).c_str());
+        PyType_Type.tp_setattro(pyclass, llname, pyuscope);
+        Py_DECREF(llname);
+        Py_DECREF(pyuscope);
+      } else {
+        PyErr_Clear();
+      }
+    }
+  }
+  return *klass->fImp.fUsing;
+}
+
+// Checks whether \c pyname is set as a Python attribute on any namespace that
+// \c pyclass pulls in with "using namespace", directly or transitively.
+static bool HasPythonAttrInUsing(PyObject* pyclass, PyObject* pyname,
+                                 std::set<PyObject*>& visited) {
+  for (auto pyref : GetUsingScopes(pyclass)) {
+    PyObject* pyuscope = cpyrt_GetWeakRef(pyref);
+    if (!pyuscope)
+      continue;
+    bool found = false;
+    if (visited.insert(pyuscope).second) {
+      found = _PyType_Lookup((PyTypeObject*)pyuscope, pyname) ||
+              _PyType_Lookup(Py_TYPE(pyuscope), pyname) ||
+              (CPPScope_Check(pyuscope) &&
+               (((CPPScope*)pyuscope)->fFlags & CPPScope::kIsNamespace) &&
+               HasPythonAttrInUsing(pyuscope, pyname, visited));
+    }
+    Py_DECREF(pyuscope);
+    if (found)
+      return true;
+  }
+  return false;
+}
+
 //----------------------------------------------------------------------------
 static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) {
   // normal type-based lookup
@@ -411,8 +475,9 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) {
       }
     }
 
+    interop::TCppScope_t lookup_result = nullptr;
     if (!attr) {
-      interop::TCppScope_t lookup_result = interop::GetNamed(name, scope);
+      lookup_result = interop::GetNamed(name, scope);
       if (interop::IsVariable(lookup_result) ||
           interop::IsEnumConstant(lookup_result)) {
         attr = (PyObject*)CPPDataMember_New(scope, lookup_result);
@@ -450,7 +515,7 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) {
     // enums types requested as type (rather than the constants)
     if (!attr) {
       interop::TCppScope_t enumerator =
-          interop::GetUnderlyingScope(interop::GetNamed(name, scope));
+          interop::GetUnderlyingScope(lookup_result);
       if (interop::IsEnumScope(enumerator)) {
         // enum types (incl. named and class enums)
         attr = (PyObject*)CPPEnum_New(name, enumerator);
@@ -481,38 +546,25 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) {
   }
 
   if (!attr && (klass->fFlags & CPPScope::kIsNamespace)) {
-    // refresh using list as necessary
-    const std::vector<interop::TCppScope_t>& uv =
-        interop::GetUsingNamespaces(klass->fCppType);
-    if (!klass->fImp.fUsing || uv.size() != klass->fImp.fUsing->size()) {
-      if (klass->fImp.fUsing) {
-        for (auto pyref : *klass->fImp.fUsing)
-          Py_DECREF(pyref);
-        klass->fImp.fUsing->clear();
-      } else
-        klass->fImp.fUsing = new std::vector<PyObject*>;
-
-      // reload and reset weak refs
-      for (auto uid : uv) {
-        std::string uname = interop::GetScopedFinalName(uid);
-        PyObject* pyuscope = CreateScopeProxy(uname);
-        if (pyuscope) {
-          klass->fImp.fUsing->push_back(PyWeakref_NewRef(pyuscope, nullptr));
-          // the namespace may not otherwise be held, so tie the lifetimes
-          PyObject* llname =
-              PyUnicode_FromString(("__lifeline_" + uname).c_str());
-          PyType_Type.tp_setattro(pyclass, llname, pyuscope);
-          Py_DECREF(llname);
-          Py_DECREF(pyuscope);
-        } else {
-          PyErr_Clear();
-        }
-      }
+    const std::vector<PyObject*>& using_scopes = GetUsingScopes(pyclass);
+
+    // Searching the using namespaces repeats the full attribute lookup in each
+    // of them, which is expensive for names that don't exist. Anything C++ it
+    // can find is also found by a single unqualified lookup from within this
+    // namespace, which follows the using-directives. If that finds nothing,
+    // only attributes set on the Python proxies of the using namespaces remain.
+    bool search_using = !using_scopes.empty();
+    if (search_using && IsPlainIdentifier(name) && !interop::IsBuiltin(name) &&
+        !interop::IsVisibleName(name, klass->fCppType)) {
+      std::set<PyObject*> visited{pyclass};
+      search_using = HasPythonAttrInUsing(pyclass, pyname, visited);
     }
 
     // try all outstanding using namespaces in turn to find the attribute (will
     // cache locally later; TODO: doing so may cause pathological cases)
-    for (auto pyref : *klass->fImp.fUsing) {
+    for (auto pyref : using_scopes) {
+      if (!search_using)
+        break;
       PyObject* pyuscope = cpyrt_GetWeakRef(pyref);
       if (pyuscope) {
         attr = PyObject_GetAttr(pyuscope, pyname);

bindings/pyroot/cppjit/src/cpyrt/Compatibility.h

@@ -3,6 +3,7 @@
 
 #include "Python.h"
 
+#include "PyStrings.h"
 #include "cppjit_interop.h"
 
 #include <string>
@@ -43,6 +44,61 @@ inline std::string ObjToString(interop::TCppScope_t /*klass*/,
   return "";
 }
 
+//- cppyy legacy API ----------------------------------------------------------
+// Backwards compatibility with the API that former cppyy users have in their
+// code. The macro is deliberately not a build option: it only exists so that
+// the legacy code paths are easy to find, and to remove once there is a
+// transition plan. Keep all such shims in this section.
+#define CPPYY_LEGACY_API 1
+
+// Name under which pyclass defines its explicit pythonization hook, looked up
+// in its own __dict__ only (no fall-through to base classes). Prefers the
+// __cppjit_* spelling over the old __cppyy_* one. Returns a borrowed reference,
+// or nullptr with no error set if there is no such hook.
+inline PyObject* ExplicitPythonizeHookName(PyObject* pyclass) {
+  // go through __dict__ instead of tp_getattr, see HasAttrDirect in
+  // Pythonize.cxx for the rationale
+  PyObject* name = nullptr;
+  PyObject* dct = PyObject_GetAttr(pyclass, PyStrings::gDict);
+  if (dct) {
+    PyObject* candidates[] = {
+        PyStrings::gExPythonize,
+#if CPPYY_LEGACY_API
+        PyStrings::gCppyyExPythonize,
+#endif
+    };
+    for (PyObject* candidate : candidates) {
+      PyObject* attr = PyObject_GetItem(dct, candidate);
+      if (attr) {
+        Py_DECREF(attr);
+        name = candidate;
+        break;
+      }
+      PyErr_Clear();
+    }
+    Py_DECREF(dct);
+  }
+  PyErr_Clear();
+  return name;
+}
+
+// Normal pythonization hook of pyclass, through regular attribute lookup (so
+// it falls through to base classes). Prefers the __cppjit_* spelling over the
+// old __cppyy_* one. Returns a new reference, or nullptr with no error set if
+// there is no such hook.
+inline PyObject* GetPythonizeHook(PyObject* pyclass) {
+  PyObject* func = PyObject_GetAttr(pyclass, PyStrings::gPythonize);
+#if CPPYY_LEGACY_API
+  if (!func) {
+    PyErr_Clear();
+    func = PyObject_GetAttr(pyclass, PyStrings::gCppyyPythonize);
+  }
+#endif
+  if (!func)
+    PyErr_Clear();
+  return func;
+}
+
 } // namespace cppjit::cpyrt::compat
 
 #endif // CPYRT_COMPATIBILITY_H