update pybind11

examples/knxPython/pybind11/tests/test_embed/CMakeLists.txt

@@ -1,10 +1,20 @@
+possibly_uninitialized(PYTHON_MODULE_EXTENSION Python_INTERPRETER_ID)
+
 if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy")
+  message(STATUS "Skipping embed test on PyPy")
   add_custom_target(cpptest) # Dummy target on PyPy. Embedding is not supported.
   set(_suppress_unused_variable_warning "${DOWNLOAD_CATCH}")
   return()
 endif()
 
-find_package(Catch 2.13.0)
+if(TARGET Python::Module AND NOT TARGET Python::Python)
+  message(STATUS "Skipping embed test since no embed libs found")
+  add_custom_target(cpptest) # Dummy target since embedding is not supported.
+  set(_suppress_unused_variable_warning "${DOWNLOAD_CATCH}")
+  return()
+endif()
+
+find_package(Catch 2.13.9)
 
 if(CATCH_FOUND)
   message(STATUS "Building interpreter tests using Catch v${CATCH_VERSION}")
@@ -22,12 +32,13 @@ pybind11_enable_warnings(test_embed)
 target_link_libraries(test_embed PRIVATE pybind11::embed Catch2::Catch2 Threads::Threads)
 
 if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
-  file(COPY test_interpreter.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
+  file(COPY test_interpreter.py test_trampoline.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
 endif()
 
 add_custom_target(
   cpptest
   COMMAND "$<TARGET_FILE:test_embed>"
+  DEPENDS test_embed
   WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
 
 pybind11_add_module(external_module THIN_LTO external_module.cpp)

examples/knxPython/pybind11/tests/test_embed/catch.cpp

@@ -0,0 +1,43 @@
+// The Catch implementation is compiled here. This is a standalone
+// translation unit to avoid recompiling it for every test change.
+
+#include <pybind11/embed.h>
+
+// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to
+// catch 2.0.1; this should be fixed in the next catch release after 2.0.1).
+PYBIND11_WARNING_DISABLE_MSVC(4996)
+
+// Catch uses _ internally, which breaks gettext style defines
+#ifdef _
+#    undef _
+#endif
+
+#define CATCH_CONFIG_RUNNER
+#include <catch.hpp>
+
+namespace py = pybind11;
+
+int main(int argc, char *argv[]) {
+    // Setup for TEST_CASE in test_interpreter.cpp, tagging on a large random number:
+    std::string updated_pythonpath("pybind11_test_embed_PYTHONPATH_2099743835476552");
+    const char *preexisting_pythonpath = getenv("PYTHONPATH");
+    if (preexisting_pythonpath != nullptr) {
+#if defined(_WIN32)
+        updated_pythonpath += ';';
+#else
+        updated_pythonpath += ':';
+#endif
+        updated_pythonpath += preexisting_pythonpath;
+    }
+#if defined(_WIN32)
+    _putenv_s("PYTHONPATH", updated_pythonpath.c_str());
+#else
+    setenv("PYTHONPATH", updated_pythonpath.c_str(), /*replace=*/1);
+#endif
+
+    py::scoped_interpreter guard{};
+
+    auto result = Catch::Session().run(argc, argv);
+
+    return result < 0xff ? result : 0xff;
+}

examples/knxPython/pybind11/tests/test_embed/external_module.cpp

@@ -0,0 +1,20 @@
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+
+/* Simple test module/test class to check that the referenced internals data of external pybind11
+ * modules aren't preserved over a finalize/initialize.
+ */
+
+PYBIND11_MODULE(external_module, m, py::mod_gil_not_used()) {
+    class A {
+    public:
+        explicit A(int value) : v{value} {};
+        int v;
+    };
+
+    py::class_<A>(m, "A").def(py::init<int>()).def_readwrite("value", &A::v);
+
+    m.def("internals_at",
+          []() { return reinterpret_cast<uintptr_t>(&py::detail::get_internals()); });
+}

examples/knxPython/pybind11/tests/test_embed/test_interpreter.cpp

@@ -1,27 +1,32 @@
 #include <pybind11/embed.h>
 
-#ifdef _MSC_VER
-// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to catch
-// 2.0.1; this should be fixed in the next catch release after 2.0.1).
-#  pragma warning(disable: 4996)
-#endif
+// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to
+// catch 2.0.1; this should be fixed in the next catch release after 2.0.1).
+PYBIND11_WARNING_DISABLE_MSVC(4996)
 
 #include <catch.hpp>
-
-#include <thread>
+#include <cstdlib>
 #include <fstream>
 #include <functional>
+#include <thread>
+#include <utility>
 
 namespace py = pybind11;
 using namespace py::literals;
 
+size_t get_sys_path_size() {
+    auto sys_path = py::module::import("sys").attr("path");
+    return py::len(sys_path);
+}
+
 class Widget {
 public:
-    Widget(std::string message) : message(message) { }
+    explicit Widget(std::string message) : message(std::move(message)) {}
     virtual ~Widget() = default;
 
     std::string the_message() const { return message; }
     virtual int the_answer() const = 0;
+    virtual std::string argv0() const = 0;
 
 private:
     std::string message;
@@ -31,6 +36,23 @@ class PyWidget final : public Widget {
     using Widget::Widget;
 
     int the_answer() const override { PYBIND11_OVERRIDE_PURE(int, Widget, the_answer); }
+    std::string argv0() const override { PYBIND11_OVERRIDE_PURE(std::string, Widget, argv0); }
+};
+
+class test_override_cache_helper {
+
+public:
+    virtual int func() { return 0; }
+
+    test_override_cache_helper() = default;
+    virtual ~test_override_cache_helper() = default;
+    // Non-copyable
+    test_override_cache_helper &operator=(test_override_cache_helper const &Right) = delete;
+    test_override_cache_helper(test_override_cache_helper const &Copy) = delete;
+};
+
+class test_override_cache_helper_trampoline : public test_override_cache_helper {
+    int func() override { PYBIND11_OVERRIDE(int, test_override_cache_helper, func); }
 };
 
 PYBIND11_EMBEDDED_MODULE(widget_module, m) {
@@ -41,24 +63,39 @@ PYBIND11_EMBEDDED_MODULE(widget_module, m) {
     m.def("add", [](int i, int j) { return i + j; });
 }
 
-PYBIND11_EMBEDDED_MODULE(throw_exception, ) {
-    throw std::runtime_error("C++ Error");
+PYBIND11_EMBEDDED_MODULE(trampoline_module, m) {
+    py::class_<test_override_cache_helper,
+               test_override_cache_helper_trampoline,
+               std::shared_ptr<test_override_cache_helper>>(m, "test_override_cache_helper")
+        .def(py::init_alias<>())
+        .def("func", &test_override_cache_helper::func);
 }
 
+PYBIND11_EMBEDDED_MODULE(throw_exception, ) { throw std::runtime_error("C++ Error"); }
+
 PYBIND11_EMBEDDED_MODULE(throw_error_already_set, ) {
     auto d = py::dict();
     d["missing"].cast<py::object>();
 }
 
+TEST_CASE("PYTHONPATH is used to update sys.path") {
+    // The setup for this TEST_CASE is in catch.cpp!
+    auto sys_path = py::str(py::module_::import("sys").attr("path")).cast<std::string>();
+    REQUIRE_THAT(sys_path,
+                 Catch::Matchers::Contains("pybind11_test_embed_PYTHONPATH_2099743835476552"));
+}
+
 TEST_CASE("Pass classes and data between modules defined in C++ and Python") {
     auto module_ = py::module_::import("test_interpreter");
     REQUIRE(py::hasattr(module_, "DerivedWidget"));
 
-    auto locals = py::dict("hello"_a="Hello, World!", "x"_a=5, **module_.attr("__dict__"));
+    auto locals = py::dict("hello"_a = "Hello, World!", "x"_a = 5, **module_.attr("__dict__"));
     py::exec(R"(
         widget = DerivedWidget("{} - {}".format(hello, x))
         message = widget.the_message
-    )", py::globals(), locals);
+    )",
+             py::globals(),
+             locals);
     REQUIRE(locals["message"].cast<std::string>() == "Hello, World! - 5");
 
     auto py_widget = module_.attr("DerivedWidget")("The question");
@@ -69,12 +106,51 @@ TEST_CASE("Pass classes and data between modules defined in C++ and Python") {
     REQUIRE(cpp_widget.the_answer() == 42);
 }
 
+TEST_CASE("Override cache") {
+    auto module_ = py::module_::import("test_trampoline");
+    REQUIRE(py::hasattr(module_, "func"));
+    REQUIRE(py::hasattr(module_, "func2"));
+
+    auto locals = py::dict(**module_.attr("__dict__"));
+
+    int i = 0;
+    for (; i < 1500; ++i) {
+        std::shared_ptr<test_override_cache_helper> p_obj;
+        std::shared_ptr<test_override_cache_helper> p_obj2;
+
+        py::object loc_inst = locals["func"]();
+        p_obj = py::cast<std::shared_ptr<test_override_cache_helper>>(loc_inst);
+
+        int ret = p_obj->func();
+
+        REQUIRE(ret == 42);
+
+        loc_inst = locals["func2"]();
+
+        p_obj2 = py::cast<std::shared_ptr<test_override_cache_helper>>(loc_inst);
+
+        p_obj2->func();
+    }
+}
+
 TEST_CASE("Import error handling") {
     REQUIRE_NOTHROW(py::module_::import("widget_module"));
-    REQUIRE_THROWS_WITH(py::module_::import("throw_exception"),
-                        "ImportError: C++ Error");
+    REQUIRE_THROWS_WITH(py::module_::import("throw_exception"), "ImportError: C++ Error");
     REQUIRE_THROWS_WITH(py::module_::import("throw_error_already_set"),
-                        Catch::Contains("ImportError: KeyError"));
+                        Catch::Contains("ImportError: initialization failed"));
+
+    auto locals = py::dict("is_keyerror"_a = false, "message"_a = "not set");
+    py::exec(R"(
+        try:
+            import throw_error_already_set
+        except ImportError as e:
+            is_keyerror = type(e.__cause__) == KeyError
+            message = str(e.__cause__)
+    )",
+             py::globals(),
+             locals);
+    REQUIRE(locals["is_keyerror"].cast<bool>() == true);
+    REQUIRE(locals["message"].cast<std::string>() == "'missing'");
 }
 
 TEST_CASE("There can be only one interpreter") {
@@ -95,26 +171,111 @@ TEST_CASE("There can be only one interpreter") {
     py::initialize_interpreter();
 }
 
-bool has_pybind11_internals_builtin() {
-    auto builtins = py::handle(PyEval_GetBuiltins());
-    return builtins.contains(PYBIND11_INTERNALS_ID);
-};
+#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
+TEST_CASE("Custom PyConfig") {
+    py::finalize_interpreter();
+    PyConfig config;
+    PyConfig_InitPythonConfig(&config);
+    REQUIRE_NOTHROW(py::scoped_interpreter{&config});
+    {
+        py::scoped_interpreter p{&config};
+        REQUIRE(py::module_::import("widget_module").attr("add")(1, 41).cast<int>() == 42);
+    }
+    py::initialize_interpreter();
+}
+
+TEST_CASE("scoped_interpreter with PyConfig_InitIsolatedConfig and argv") {
+    py::finalize_interpreter();
+    {
+        PyConfig config;
+        PyConfig_InitIsolatedConfig(&config);
+        char *argv[] = {strdup("a.out")};
+        py::scoped_interpreter argv_scope{&config, 1, argv};
+        std::free(argv[0]);
+        auto module = py::module::import("test_interpreter");
+        auto py_widget = module.attr("DerivedWidget")("The question");
+        const auto &cpp_widget = py_widget.cast<const Widget &>();
+        REQUIRE(cpp_widget.argv0() == "a.out");
+    }
+    py::initialize_interpreter();
+}
+
+TEST_CASE("scoped_interpreter with PyConfig_InitPythonConfig and argv") {
+    py::finalize_interpreter();
+    {
+        PyConfig config;
+        PyConfig_InitPythonConfig(&config);
+
+        // `initialize_interpreter() overrides the default value for config.parse_argv (`1`) by
+        // changing it to `0`. This test exercises `scoped_interpreter` with the default config.
+        char *argv[] = {strdup("a.out"), strdup("arg1")};
+        py::scoped_interpreter argv_scope(&config, 2, argv);
+        std::free(argv[0]);
+        std::free(argv[1]);
+        auto module = py::module::import("test_interpreter");
+        auto py_widget = module.attr("DerivedWidget")("The question");
+        const auto &cpp_widget = py_widget.cast<const Widget &>();
+        REQUIRE(cpp_widget.argv0() == "arg1");
+    }
+    py::initialize_interpreter();
+}
+#endif
+
+TEST_CASE("Add program dir to path pre-PyConfig") {
+    py::finalize_interpreter();
+    size_t path_size_add_program_dir_to_path_false = 0;
+    {
+        py::scoped_interpreter scoped_interp{true, 0, nullptr, false};
+        path_size_add_program_dir_to_path_false = get_sys_path_size();
+    }
+    {
+        py::scoped_interpreter scoped_interp{};
+        REQUIRE(get_sys_path_size() == path_size_add_program_dir_to_path_false + 1);
+    }
+    py::initialize_interpreter();
+}
+
+#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
+TEST_CASE("Add program dir to path using PyConfig") {
+    py::finalize_interpreter();
+    size_t path_size_add_program_dir_to_path_false = 0;
+    {
+        PyConfig config;
+        PyConfig_InitPythonConfig(&config);
+        py::scoped_interpreter scoped_interp{&config, 0, nullptr, false};
+        path_size_add_program_dir_to_path_false = get_sys_path_size();
+    }
+    {
+        PyConfig config;
+        PyConfig_InitPythonConfig(&config);
+        py::scoped_interpreter scoped_interp{&config};
+        REQUIRE(get_sys_path_size() == path_size_add_program_dir_to_path_false + 1);
+    }
+    py::initialize_interpreter();
+}
+#endif
+
+bool has_state_dict_internals_obj() {
+    return bool(
+        py::detail::get_internals_obj_from_state_dict(py::detail::get_python_state_dict()));
+}
 
 bool has_pybind11_internals_static() {
     auto **&ipp = py::detail::get_internals_pp();
-    return ipp && *ipp;
+    return (ipp != nullptr) && (*ipp != nullptr);
 }
 
 TEST_CASE("Restart the interpreter") {
     // Verify pre-restart state.
     REQUIRE(py::module_::import("widget_module").attr("add")(1, 2).cast<int>() == 3);
-    REQUIRE(has_pybind11_internals_builtin());
+    REQUIRE(has_state_dict_internals_obj());
     REQUIRE(has_pybind11_internals_static());
-    REQUIRE(py::module_::import("external_module").attr("A")(123).attr("value").cast<int>() == 123);
+    REQUIRE(py::module_::import("external_module").attr("A")(123).attr("value").cast<int>()
+            == 123);
 
     // local and foreign module internals should point to the same internals:
-    REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp()) ==
-            py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>());
+    REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp())
+            == py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>());
 
     // Restart the interpreter.
     py::finalize_interpreter();
@@ -124,28 +285,31 @@ TEST_CASE("Restart the interpreter") {
     REQUIRE(Py_IsInitialized() == 1);
 
     // Internals are deleted after a restart.
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_state_dict_internals_obj());
     REQUIRE_FALSE(has_pybind11_internals_static());
     pybind11::detail::get_internals();
-    REQUIRE(has_pybind11_internals_builtin());
+    REQUIRE(has_state_dict_internals_obj());
     REQUIRE(has_pybind11_internals_static());
-    REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp()) ==
-            py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>());
+    REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp())
+            == py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>());
 
     // Make sure that an interpreter with no get_internals() created until finalize still gets the
     // internals destroyed
     py::finalize_interpreter();
     py::initialize_interpreter();
     bool ran = false;
-    py::module_::import("__main__").attr("internals_destroy_test") =
-        py::capsule(&ran, [](void *ran) { py::detail::get_internals(); *static_cast<bool *>(ran) = true; });
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    py::module_::import("__main__").attr("internals_destroy_test")
+        = py::capsule(&ran, [](void *ran) {
+              py::detail::get_internals();
+              *static_cast<bool *>(ran) = true;
+          });
+    REQUIRE_FALSE(has_state_dict_internals_obj());
     REQUIRE_FALSE(has_pybind11_internals_static());
     REQUIRE_FALSE(ran);
     py::finalize_interpreter();
     REQUIRE(ran);
     py::initialize_interpreter();
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_state_dict_internals_obj());
     REQUIRE_FALSE(has_pybind11_internals_static());
 
     // C++ modules can be reloaded.
@@ -167,17 +331,17 @@ TEST_CASE("Subinterpreter") {
 
         REQUIRE(m.attr("add")(1, 2).cast<int>() == 3);
     }
-    REQUIRE(has_pybind11_internals_builtin());
+    REQUIRE(has_state_dict_internals_obj());
     REQUIRE(has_pybind11_internals_static());
 
     /// Create and switch to a subinterpreter.
-    auto main_tstate = PyThreadState_Get();
-    auto sub_tstate = Py_NewInterpreter();
+    auto *main_tstate = PyThreadState_Get();
+    auto *sub_tstate = Py_NewInterpreter();
 
     // Subinterpreters get their own copy of builtins. detail::get_internals() still
     // works by returning from the static variable, i.e. all interpreters share a single
     // global pybind11::internals;
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_state_dict_internals_obj());
     REQUIRE(has_pybind11_internals_static());
 
     // Modules tags should be gone.
@@ -212,11 +376,10 @@ TEST_CASE("Threads") {
     REQUIRE_FALSE(has_pybind11_internals_static());
 
     constexpr auto num_threads = 10;
-    auto locals = py::dict("count"_a=0);
+    auto locals = py::dict("count"_a = 0);
 
     {
         py::gil_scoped_release gil_release{};
-        REQUIRE(has_pybind11_internals_static());
 
         auto threads = std::vector<std::thread>();
         for (auto i = 0; i < num_threads; ++i) {
@@ -238,7 +401,11 @@ TEST_CASE("Threads") {
 struct scope_exit {
     std::function<void()> f_;
     explicit scope_exit(std::function<void()> f) noexcept : f_(std::move(f)) {}
-    ~scope_exit() { if (f_) f_(); }
+    ~scope_exit() {
+        if (f_) {
+            f_();
+        }
+    }
 };
 
 TEST_CASE("Reload module from file") {
@@ -249,9 +416,8 @@ TEST_CASE("Reload module from file") {
     bool dont_write_bytecode = sys.attr("dont_write_bytecode").cast<bool>();
     sys.attr("dont_write_bytecode") = true;
     // Reset the value at scope exit
-    scope_exit reset_dont_write_bytecode([&]() {
-        sys.attr("dont_write_bytecode") = dont_write_bytecode;
-    });
+    scope_exit reset_dont_write_bytecode(
+        [&]() { sys.attr("dont_write_bytecode") = dont_write_bytecode; });
 
     std::string module_name = "test_module_reload";
     std::string module_file = module_name + ".py";
@@ -262,9 +428,7 @@ TEST_CASE("Reload module from file") {
     test_module << "    return 1\n";
     test_module.close();
     // Delete the file at scope exit
-    scope_exit delete_module_file([&]() {
-        std::remove(module_file.c_str());
-    });
+    scope_exit delete_module_file([&]() { std::remove(module_file.c_str()); });
 
     // Import the module from file
     auto module_ = py::module_::import(module_name.c_str());
@@ -282,3 +446,43 @@ TEST_CASE("Reload module from file") {
     result = module_.attr("test")().cast<int>();
     REQUIRE(result == 2);
 }
+
+TEST_CASE("sys.argv gets initialized properly") {
+    py::finalize_interpreter();
+    {
+        py::scoped_interpreter default_scope;
+        auto module = py::module::import("test_interpreter");
+        auto py_widget = module.attr("DerivedWidget")("The question");
+        const auto &cpp_widget = py_widget.cast<const Widget &>();
+        REQUIRE(cpp_widget.argv0().empty());
+    }
+
+    {
+        char *argv[] = {strdup("a.out")};
+        py::scoped_interpreter argv_scope(true, 1, argv);
+        std::free(argv[0]);
+        auto module = py::module::import("test_interpreter");
+        auto py_widget = module.attr("DerivedWidget")("The question");
+        const auto &cpp_widget = py_widget.cast<const Widget &>();
+        REQUIRE(cpp_widget.argv0() == "a.out");
+    }
+    py::initialize_interpreter();
+}
+
+TEST_CASE("make_iterator can be called before then after finalizing an interpreter") {
+    // Reproduction of issue #2101 (https://github.com/pybind/pybind11/issues/2101)
+    py::finalize_interpreter();
+
+    std::vector<int> container;
+    {
+        pybind11::scoped_interpreter g;
+        auto iter = pybind11::make_iterator(container.begin(), container.end());
+    }
+
+    REQUIRE_NOTHROW([&]() {
+        pybind11::scoped_interpreter g;
+        auto iter = pybind11::make_iterator(container.begin(), container.end());
+    }());
+
+    py::initialize_interpreter();
+}

examples/knxPython/pybind11/tests/test_embed/test_interpreter.py

@@ -1,10 +1,16 @@
-# -*- coding: utf-8 -*-
+from __future__ import annotations
+
+import sys
+
 from widget_module import Widget
 
 
 class DerivedWidget(Widget):
     def __init__(self, message):
-        super(DerivedWidget, self).__init__(message)
+        super().__init__(message)
 
     def the_answer(self):
         return 42
+
+    def argv0(self):
+        return sys.argv[0]

examples/knxPython/pybind11/tests/test_embed/test_trampoline.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import trampoline_module
+
+
+def func():
+    class Test(trampoline_module.test_override_cache_helper):
+        def func(self):
+            return 42
+
+    return Test()
+
+
+def func2():
+    class Test(trampoline_module.test_override_cache_helper):
+        pass
+
+    return Test()