From 3a8c83457aedd9dec267ab57bd5afb8c9373f29d Mon Sep 17 00:00:00 2001 From: Victor Zoubok Date: Sun, 15 Nov 2026 08:16:24 -0700 Subject: [PATCH] DX-128365: [C++][Gandiva] Add BuildScope::PreShutdown to wait for in-flight builds before exit --- cpp/src/gandiva/CMakeLists.txt | 4 + cpp/src/gandiva/filter.cc | 4 + cpp/src/gandiva/projector.cc | 4 + cpp/src/gandiva/shutdown_guard.cc | 141 +++++++++++++++++++++++++ cpp/src/gandiva/shutdown_guard.h | 89 ++++++++++++++++ cpp/src/gandiva/shutdown_guard_test.cc | 95 +++++++++++++++++ 6 files changed, 337 insertions(+) create mode 100644 cpp/src/gandiva/shutdown_guard.cc create mode 100644 cpp/src/gandiva/shutdown_guard.h create mode 100644 cpp/src/gandiva/shutdown_guard_test.cc diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index dc5fb02f5313..e56fa5340ea0 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -102,6 +102,7 @@ set(SRC_FILES regex_util.cc regex_functions_holder.cc selection_vector.cc + shutdown_guard.cc tree_expr_builder.cc to_date_holder.cc random_generator_holder.cc @@ -288,5 +289,8 @@ add_gandiva_test(internals-test re2::re2 ${GANDIVA_OPENSSL_LIBS}) +# Its own binary: PreShutdown() is one-way for the life of the process. +add_gandiva_test(shutdown-guard-test SOURCES shutdown_guard_test.cc) + add_subdirectory(precompiled) add_subdirectory(tests) diff --git a/cpp/src/gandiva/filter.cc b/cpp/src/gandiva/filter.cc index be67c12254f9..e98dcc738e25 100644 --- a/cpp/src/gandiva/filter.cc +++ b/cpp/src/gandiva/filter.cc @@ -26,6 +26,7 @@ #include "gandiva/expr_validator.h" #include "gandiva/llvm_generator.h" #include "gandiva/selection_vector_impl.h" +#include "gandiva/shutdown_guard.h" namespace gandiva { @@ -45,6 +46,9 @@ Status Filter::Make(SchemaPtr schema, ConditionPtr condition, ARROW_RETURN_IF(configuration == nullptr, Status::Invalid("Configuration cannot be null")); + BuildScope build_scope; + ARROW_RETURN_IF(!build_scope.entered(), Status::Invalid("Gandiva is shutting down")); + std::shared_ptr>> cache = LLVMGenerator::GetCache(); diff --git a/cpp/src/gandiva/projector.cc b/cpp/src/gandiva/projector.cc index eb52477ff6d6..e45da2f03a3a 100644 --- a/cpp/src/gandiva/projector.cc +++ b/cpp/src/gandiva/projector.cc @@ -26,6 +26,7 @@ #include "gandiva/cache.h" #include "gandiva/expr_validator.h" #include "gandiva/llvm_generator.h" +#include "gandiva/shutdown_guard.h" namespace gandiva { @@ -61,6 +62,9 @@ Status Projector::Make(SchemaPtr schema, const ExpressionVector& exprs, ARROW_RETURN_IF(configuration == nullptr, Status::Invalid("Configuration cannot be null")); + BuildScope build_scope; + ARROW_RETURN_IF(!build_scope.entered(), Status::Invalid("Gandiva is shutting down")); + // see if equivalent projector was already built std::shared_ptr>> cache = LLVMGenerator::GetCache(); diff --git a/cpp/src/gandiva/shutdown_guard.cc b/cpp/src/gandiva/shutdown_guard.cc new file mode 100644 index 000000000000..99ec13c77121 --- /dev/null +++ b/cpp/src/gandiva/shutdown_guard.cc @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "gandiva/shutdown_guard.h" + +#include + +namespace gandiva { + +BuildScope::State& BuildScope::state() { + static State* s = new State; // leaked on purpose, thread-safe init (C++11) + return *s; +} + +int& BuildScope::Depth() { + thread_local int depth = 0; // trivially destructible, safe during exit + return depth; +} + +BuildScope::BuildScope() { + int& depth = Depth(); + if (depth > 0) { // nested: the outermost scope already holds the count + ++depth; + entered_ = true; + return; + } + entered_ = TryAcquire(); + if (entered_) { + depth = 1; + } +} + +BuildScope::~BuildScope() { + if (!entered_) { + return; + } + if (--Depth() == 0) { // only the outermost scope releases + Release(); + } +} + +arrow::Status BuildScope::PreShutdown(std::chrono::milliseconds timeout) { + if (Depth() > 0) { + return arrow::Status::Invalid( + "PreShutdown called from inside a Gandiva build; it would wait for itself"); + } + auto& s = state(); + + // acq_rel: + // - acquire half: if builds already finished, we synchronize with their release in + // Release(), so their work is visible to us even if the count is already zero and + // we never need to sleep. + // - release half: not strictly required for correctness. It is kept so that the + // request "shutdown started" is published to threads that later observe the bit, + // and it costs nothing on x86 and little on ARM. + s.word.fetch_or(kShutdownBit, std::memory_order_acq_rel); + + // The mutex is only used to sleep and wake. It is NOT what protects the count. + std::unique_lock lock(s.mtx); + bool quiescent = s.cv.wait_for(lock, timeout, [&s] { + // acquire: pairs with the release (fetch_sub) in Release(). When we read a count of + // zero, all work done by the finished builds is visible to us. This is the load that + // makes it safe to run static destructors after PreShutdown returns OK. + return (s.word.load(std::memory_order_acquire) & kCountMask) == 0; + }); + if (!quiescent) { + // relaxed: diagnostic only. We are returning an error and no other data depends on + // this value, so no ordering is needed. + return arrow::Status::IOError("Timed out waiting for ", + s.word.load(std::memory_order_relaxed) & kCountMask, + " Gandiva build(s) in flight"); + } + return arrow::Status::OK(); +} + +bool BuildScope::TryAcquire() { + auto& s = state(); + + // relaxed: this is only a first guess for the CAS below. The CAS re-validates it + // atomically, so a stale value is harmless (the CAS fails and refreshes `v`). + uint32_t v = s.word.load(std::memory_order_relaxed); + do { + if (v & kShutdownBit) { + return false; + } + // Check-and-increment in ONE atomic step: we only increment if the word is still + // exactly `v` (bit not set). If PreShutdown sets the bit in between, the CAS fails. + // + // success = acq_rel: + // - acquire half: the build body must not be reordered before this point. This is + // the "lock acquired" side, and it also makes us see the work of builds that + // released earlier. This is the half that matters. + // - release half: not strictly needed here, kept as a harmless superset so the + // operation reads as a full read-modify-write synchronization point. + // + // failure = relaxed: a failed CAS writes nothing. It only reloads `v`, and the loop + // re-checks the shutdown bit, so no ordering is needed. + // + // compare_exchange_weak: may fail spuriously (on ARM/POWER), which is fine inside + // a retry loop and can generate cheaper code than compare_exchange_strong. + } while (!s.word.compare_exchange_weak(v, v + 1, std::memory_order_acq_rel, + std::memory_order_relaxed)); + return true; +} + +void BuildScope::Release() { + auto& s = state(); + + // acq_rel (the release half is the important one): + // - release: publishes "this build is completely finished". Everything the build did + // before this point is visible to a thread that later acquire-loads this word and + // sees the decremented value, which is exactly what PreShutdown does. Without it, + // the count could reach zero while the build's memory effects were still not + // visible to the shutdown thread. + // - acquire: not strictly needed here, kept for symmetry with a lock/unlock pair. + // + // fetch_sub returns the value BEFORE the decrement, so we know exactly whether we were + // the last build and whether the shutdown bit was set at that moment. + uint32_t prev = s.word.fetch_sub(1, std::memory_order_acq_rel); + + if ((prev & kShutdownBit) && (prev & kCountMask) == 1) { + std::lock_guard lock(s.mtx); + s.cv.notify_all(); + } +} + +} // namespace gandiva \ No newline at end of file diff --git a/cpp/src/gandiva/shutdown_guard.h b/cpp/src/gandiva/shutdown_guard.h new file mode 100644 index 000000000000..e7b1a0ffd5c1 --- /dev/null +++ b/cpp/src/gandiva/shutdown_guard.h @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/status.h" + +#include "gandiva/visibility.h" + +namespace gandiva { + +/// Held for the duration of a Projector::Make or Filter::Make. +/// +/// A compile cannot be interrupted, and exit() destroys process-wide state (static +/// destructors) that a running compile still reads. The scopes count the compiles in +/// flight, so that the embedder can call PreShutdown() before the process exits to reject +/// new builds and wait for the running ones. +/// +/// Scopes may be nested on the same thread: only the outermost scope is counted, and +/// nested scopes are always admitted, since the outermost one already keeps the build +/// "in flight". A scope must be destroyed on the thread that created it. +class GANDIVA_EXPORT BuildScope { + public: + BuildScope(); + ~BuildScope(); + + BuildScope(const BuildScope&) = delete; + BuildScope& operator=(const BuildScope&) = delete; + BuildScope(BuildScope&&) = delete; + BuildScope& operator=(BuildScope&&) = delete; + + /// False if the process is shutting down and the build must not start. + bool entered() const { return entered_; } + + /// Rejects new builds and waits until no build is in flight, or the timeout expires. + /// Returns Status::OK() when quiescent, Status::IOError on timeout, and + /// Status::Invalid if called from a thread that is itself inside a build (it would + /// wait for itself). + static arrow::Status PreShutdown(std::chrono::milliseconds timeout); + + private: + static constexpr std::uint32_t kShutdownBit = 0x80000000u; + static constexpr std::uint32_t kCountMask = 0x7FFFFFFFu; + + struct State { + // bit 31 = shutting down, low 31 bits = outermost builds in flight. + // Flag and count share one word so "check flag + increment" is a single atomic step. + std::atomic word{0}; + // Only used to put PreShutdown to sleep and wake it up. + std::mutex mtx; + std::condition_variable cv; + }; + + // Leaked on purpose: builds still running at exit must find it alive. + // Function-local static avoids static initialization order problems. + static State& state(); + + // Per-thread nesting depth of live, entered scopes. + static int& Depth(); + + /// Counts an outermost build as in flight. Fails, leaving the count unchanged, once + /// PreShutdown has started. + static bool TryAcquire(); + static void Release(); + + bool entered_ = false; +}; + +} // namespace gandiva \ No newline at end of file diff --git a/cpp/src/gandiva/shutdown_guard_test.cc b/cpp/src/gandiva/shutdown_guard_test.cc new file mode 100644 index 000000000000..d55805fd762c --- /dev/null +++ b/cpp/src/gandiva/shutdown_guard_test.cc @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "gandiva/shutdown_guard.h" + +#include +#include +#include + +#include + +namespace gandiva { + +using std::chrono::milliseconds; + +// PreShutdown() is one-way for the life of the process, so the tests in this file run in +// order: the ones that need a live state come first and the last one starts the shutdown. + +TEST(TestBuildScope, NestedScopesAreAdmitted) { + BuildScope outer; + ASSERT_TRUE(outer.entered()); + { + BuildScope nested; + EXPECT_TRUE(nested.entered()); + } + BuildScope second_nested; + EXPECT_TRUE(second_nested.entered()); +} + +TEST(TestBuildScope, PreShutdownFromInsideBuildFails) { + BuildScope scope; + ASSERT_TRUE(scope.entered()); + + auto status = BuildScope::PreShutdown(milliseconds(10)); + EXPECT_TRUE(status.IsInvalid()) << status.ToString(); +} + +TEST(TestBuildScope, PreShutdownWaitsForRunningBuild) { + std::promise holding; + std::promise shutdown_started; + std::promise nested_checked; + std::promise release; + bool nested_entered = false; + + std::thread builder([&] { + BuildScope outer; + EXPECT_TRUE(outer.entered()); + holding.set_value(); + + shutdown_started.get_future().wait(); + BuildScope nested; + nested_entered = nested.entered(); + nested_checked.set_value(); + + release.get_future().wait(); + }); + holding.get_future().wait(); + + auto timed_out = BuildScope::PreShutdown(milliseconds(50)); + EXPECT_TRUE(timed_out.IsIOError()) << timed_out.ToString(); + + // the shutdown has started: the running build may still nest, new builds are rejected + shutdown_started.set_value(); + nested_checked.get_future().wait(); + EXPECT_TRUE(nested_entered); + + BuildScope rejected; + EXPECT_FALSE(rejected.entered()); + + std::thread releaser([&] { + std::this_thread::sleep_for(milliseconds(100)); + release.set_value(); + }); + auto status = BuildScope::PreShutdown(std::chrono::seconds(30)); + EXPECT_TRUE(status.ok()) << status.ToString(); + + releaser.join(); + builder.join(); +} + +} // namespace gandiva