From c53ec87195aedac468a0029a11603847a6a89701 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:13:18 +0100 Subject: [PATCH 01/25] [C++][FlightRPC] Add an async gRPC Flight server --- cpp/examples/arrow/CMakeLists.txt | 14 + .../flight_async_generic_doget_example.cc | 379 ++++++++ .../flight_async_generic_doput_example.cc | 287 ++++++ cpp/src/arrow/flight/CMakeLists.txt | 42 + cpp/src/arrow/flight/README.md | 2 +- cpp/src/arrow/flight/api.h | 1 + cpp/src/arrow/flight/flight_data_decoder.cc | 324 +++++++ cpp/src/arrow/flight/flight_data_decoder.h | 243 +++++ cpp/src/arrow/flight/flight_internals_test.cc | 385 ++++++++ cpp/src/arrow/flight/meson.build | 59 +- cpp/src/arrow/flight/serialization_internal.h | 1 + cpp/src/arrow/flight/server.cc | 55 +- cpp/src/arrow/flight/server_async.cc | 156 ++++ cpp/src/arrow/flight/server_async.h | 381 ++++++++ cpp/src/arrow/flight/transport.cc | 27 + cpp/src/arrow/flight/transport.h | 9 + .../flight/transport/grpc/async_grpc_compat.h | 45 + .../transport/grpc/async_grpc_do_get.cc | 186 ++++ .../transport/grpc/async_grpc_do_get_test.cc | 329 +++++++ .../transport/grpc/async_grpc_do_put.cc | 255 ++++++ .../transport/grpc/async_grpc_do_put_test.cc | 762 +++++++++++++++ .../transport/grpc/async_grpc_exchange.cc | 123 +++ .../grpc/async_grpc_exchange_internal.h | 218 +++++ .../grpc/async_grpc_exchange_reader.cc | 229 +++++ .../grpc/async_grpc_exchange_test.cc | 206 +++++ .../grpc/async_grpc_exchange_writer.cc | 234 +++++ .../transport/grpc/async_grpc_handshake.cc | 82 ++ .../grpc/async_grpc_handshake_test.cc | 509 +++++++++++ .../transport/grpc/async_grpc_service.cc | 150 +++ .../transport/grpc/async_grpc_service.h | 81 ++ .../grpc/async_grpc_service_internal.h | 232 +++++ .../transport/grpc/async_grpc_service_test.cc | 268 ++++++ .../transport/grpc/async_grpc_streaming.cc | 254 +++++ .../grpc/async_grpc_streaming_test.cc | 304 ++++++ .../transport/grpc/async_grpc_test_fixtures.h | 865 ++++++++++++++++++ .../flight/transport/grpc/async_grpc_unary.cc | 124 +++ .../transport/grpc/async_grpc_unary_test.cc | 218 +++++ .../flight/transport/grpc/customize_grpc.h | 4 +- .../flight/transport/grpc/grpc_server.cc | 108 ++- .../transport/grpc/grpc_server_internal.h | 34 +- .../transport/grpc/serialization_internal.cc | 30 +- .../transport/grpc/serialization_internal.h | 30 + .../flight/transport/grpc/util_internal.cc | 23 +- .../flight/transport/grpc/util_internal.h | 7 + .../arrow/flight/transport_server_internal.cc | 56 ++ .../arrow/flight/transport_server_internal.h | 36 + cpp/src/arrow/flight/types.cc | 3 + cpp/src/arrow/flight/types.h | 2 + cpp/src/arrow/util/windows_fixup.h | 8 - 49 files changed, 8297 insertions(+), 83 deletions(-) create mode 100644 cpp/examples/arrow/flight_async_generic_doget_example.cc create mode 100644 cpp/examples/arrow/flight_async_generic_doput_example.cc create mode 100644 cpp/src/arrow/flight/flight_data_decoder.cc create mode 100644 cpp/src/arrow/flight/flight_data_decoder.h create mode 100644 cpp/src/arrow/flight/server_async.cc create mode 100644 cpp/src/arrow/flight/server_async.h create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_compat.h create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_service.h create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_unary.cc create mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc diff --git a/cpp/examples/arrow/CMakeLists.txt b/cpp/examples/arrow/CMakeLists.txt index 82c075c51df1..6d9332464a51 100644 --- a/cpp/examples/arrow/CMakeLists.txt +++ b/cpp/examples/arrow/CMakeLists.txt @@ -127,6 +127,20 @@ if(ARROW_FLIGHT) "${CMAKE_CURRENT_BINARY_DIR}/helloworld.pb.cc" "${CMAKE_CURRENT_BINARY_DIR}/helloworld.grpc.pb.cc") + add_arrow_example(flight_async_generic_doget_example + EXTRA_LINK_LIBS + ${FLIGHT_EXAMPLES_LINK_LIBS} + gRPC::grpc++ + ${ARROW_PROTOBUF_LIBPROTOBUF} + ${GFLAGS_LIBRARIES}) + + add_arrow_example(flight_async_generic_doput_example + EXTRA_LINK_LIBS + ${FLIGHT_EXAMPLES_LINK_LIBS} + gRPC::grpc++ + ${ARROW_PROTOBUF_LIBPROTOBUF} + ${GFLAGS_LIBRARIES}) + if(ARROW_FLIGHT_SQL) if(ARROW_BUILD_SHARED AND ARROW_GRPC_USE_SHARED) set(FLIGHT_SQL_EXAMPLES_LINK_LIBS arrow_flight_sql_shared) diff --git a/cpp/examples/arrow/flight_async_generic_doget_example.cc b/cpp/examples/arrow/flight_async_generic_doget_example.cc new file mode 100644 index 000000000000..07dac1c296dc --- /dev/null +++ b/cpp/examples/arrow/flight_async_generic_doget_example.cc @@ -0,0 +1,379 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifdef _WIN32 +# include +#else +# include +#endif +#include +#if defined(__linux__) +# include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +// Serve DoGet over the async generic Flight transport. +// +// Deriving from AsyncGenericFlightServerBase (arrow/flight/server_async.h) +// selects the async generic gRPC transport. DoGet is answered by DoGetAsync: +// it returns a future for a stream of payloads (AsyncFlightDataStream) instead +// of the payloads themselves, so the gRPC callback thread serving the RPC +// never waits for data. +// +// DelayedAsyncStream shows what that is for: each batch is produced on the +// stream's own worker thread, --batch_delay_ms after the transport asks for +// it. The transport adds a callback to the future and returns, so the delay +// never blocks a gRPC callback thread (which would stall every RPC multiplexed +// onto it). With --batch_delay_ms=0 the delay is gone and this is a plain +// fast DoGet. +// +// Any ticket streams the same batches. Every other RPC - GetFlightInfo +// included - answers UNIMPLEMENTED, so call DoGet directly instead of +// resolving the flight first. +// +// Usage: +// flight-async-generic-doget-example --port=31337 --batch_delay_ms=200 +// flight-async-generic-doget-example --port=0 --demo --batches=5 +// +// Run with no arguments to print this message and exit. + +DEFINE_int32(port, 0, "Port to listen on (0 picks a free port)"); +DEFINE_int32(batches, 5, "Number of batches in the DoGet stream"); +DEFINE_int32(batch_delay_ms, 50, + "Delay before each batch, spent on the stream's own worker thread"); +DEFINE_bool(demo, false, "Run the DoGet self-check against the server, then exit"); + +namespace flight = arrow::flight; + +/// \brief The current process' id, portably. +int64_t CurrentPid() { +#ifdef _WIN32 + return static_cast(::GetCurrentProcessId()); +#else + return static_cast(::getpid()); +#endif +} + +/// \brief A reader with `num_batches` batches of three int64 values. +class BatchReader : public arrow::RecordBatchReader { + public: + BatchReader(std::shared_ptr schema, int32_t num_batches) + : schema_(std::move(schema)), num_batches_(num_batches) {} + + std::shared_ptr schema() const override { return schema_; } + + arrow::Status ReadNext(std::shared_ptr* batch) override { + if (next_batch_ >= num_batches_) { + *batch = nullptr; // end of stream + return arrow::Status::OK(); + } + arrow::Int64Builder builder; + for (int64_t i = 0; i < 3; i++) { + ARROW_RETURN_NOT_OK(builder.Append(3 * next_batch_ + i)); + } + ARROW_ASSIGN_OR_RAISE(auto array, builder.Finish()); + *batch = arrow::RecordBatch::Make(schema_, array->length(), {array}); + ++next_batch_; + return arrow::Status::OK(); + } + + private: + std::shared_ptr schema_; + int32_t num_batches_; + int32_t next_batch_ = 0; +}; + +/// \brief The stream's worker state. Heap-allocated and shared with the worker +/// thread: the transport deletes the stream as soon as a payload completes +/// (the end-of-stream payload is completed *on the worker*), so the worker must +/// not touch the stream object after that. Everything it needs lives here. +struct WorkerState { + std::unique_ptr stream; + std::shared_ptr schema; + int32_t delay_ms = 0; + std::mutex mutex; + std::condition_variable ready; + bool closed = false; + /// The payload the transport is waiting for, if one was requested. + arrow::Future> pending; + bool has_pending = false; + /// Whether the wrapped stream has been closed, so Close() closes it once. + bool stream_closed = false; + std::thread thread; +}; + +/// \brief The worker: wait for a payload request, sleep delay_ms, produce it, +/// complete the future the transport holds. +void RunWorker(std::shared_ptr state) { +#if defined(__linux__) + // Name the thread, so /proc//task/*/comm distinguishes this server's own + // per-stream workers (doget_worker) from gRPC's pool threads (event_engine / + // lifeguard / grpc_global_timer). Thread names are inherited, and gRPC names + // its threads, so an anonymous thread created from a gRPC callback would + // otherwise show up as another "event_engine". comm is capped at 15 + // characters. + prctl(PR_SET_NAME, "doget_worker", 0, 0, 0); +#endif + std::unique_lock lock(state->mutex); + // Read the schema on this thread: it is the only one allowed to touch the + // wrapped stream, and the transport asks for the schema first. + state->schema = state->stream->schema(); + while (!state->closed) { + state->ready.wait(lock, [&] { return state->has_pending || state->closed; }); + if (state->closed) { + break; + } + // Take the request: after this, only this thread resolves the future. + state->has_pending = false; + arrow::Future> future = state->pending; + lock.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(state->delay_ms)); + lock.lock(); + const bool cancelled = state->closed; + lock.unlock(); + // Runs the transport's callback inline, on this thread: it serializes the + // payload, and on end of stream it calls Close() (hence the lock dance). + if (cancelled) { + future.MarkFinished(arrow::Status::Cancelled()); + } else { + auto payload = state->stream->Next(); + if (!payload.ok()) { + future.MarkFinished(payload.status()); + } else if (payload->ipc_message.metadata == nullptr) { + // End of stream: the async interface reports it as a nullopt. + future.MarkFinished(std::nullopt); + } else { + future.MarkFinished(std::move(*payload)); + } + } + lock.lock(); + } + // The worker is the only thread that touched the wrapped stream, so it is the + // one that closes it (see Close(), which only asks it to stop). + lock.unlock(); + ARROW_WARN_NOT_OK(state->stream->Close(), "closing the DoGet stream failed"); + lock.lock(); + state->ready.notify_all(); +} + +/// \brief The stream the async transport pulls from, with a sleep per batch. +/// +/// NextAsync() hands the transport a future and returns at once, the delay and +/// the serialization of the wrapped synchronous stream happen on this stream's +/// own worker thread. The transport keeps one payload in flight at a time -- +/// it asks for the next only once the write completed -- so a single worker is +/// enough. +/// +/// One thread per stream. A server with many slow streams may prefer a shared +/// timer or thread pool, swapping WorkerState's thread for a task on that pool. +class DelayedAsyncStream : public flight::AsyncFlightDataStream { + public: + DelayedAsyncStream(std::unique_ptr stream, int32_t delay_ms) + : state_(std::make_shared()) { + state_->stream = std::move(stream); + state_->delay_ms = delay_ms; + std::shared_ptr state = state_; + state_->thread = std::thread([state] { RunWorker(state); }); + } + + ~DelayedAsyncStream() override { + Stop(); + if (state_->thread.joinable()) { + if (state_->thread.get_id() == std::this_thread::get_id()) { + // Deleted from inside its own worker (the transport releases the + // stream as a payload completes): joining would deadlock. + state_->thread.detach(); + } else { + state_->thread.join(); + } + } + } + + std::shared_ptr schema() override { return state_->schema; } + + arrow::Future GetSchemaPayloadAsync() override { + // The schema is ready now: no delay, no worker. This runs before the first + // NextAsync(), so it cannot overlap the worker's use of the stream. + return arrow::Future::MakeFinished( + state_->stream->GetSchemaPayload()); + } + + arrow::Future> NextAsync() override { + arrow::Future> future = + arrow::Future>::Make(); + { + std::lock_guard lock(state_->mutex); + if (state_->closed) { + future.MarkFinished(arrow::Status::Cancelled()); + return future; + } + state_->pending = future; + state_->has_pending = true; + } + state_->ready.notify_one(); + return future; + } + + arrow::Status Close() override { + // Only ask the worker to stop: the wrapped stream belongs to the worker + // thread, which may be inside stream->Next() right now, and Next()/Close() + // both mutate the IPC reader. Closing from this (gRPC) thread would race + // with that call and can corrupt the stream. Stop() is also what resolves + // a payload the worker has not taken yet, so this stays non-blocking; the + // worker closes the stream after it stops producing. + Stop(); + return arrow::Status::OK(); + } + + private: + /// \brief Stop the worker, resolve a payload it has not taken yet. Idempotent. + /// Does not join: called by the transport, whose threads must not block on + /// the worker's sleep. + void Stop() { + arrow::Future> pending; + { + std::lock_guard lock(state_->mutex); + state_->closed = true; + if (state_->has_pending) { + state_->has_pending = false; + pending = state_->pending; + } + } + state_->ready.notify_all(); + if (pending.is_valid()) { + // Runs the transport's callback inline, it reacts by calling Close(). + pending.MarkFinished(arrow::Status::Cancelled()); + } + } + + std::shared_ptr state_; +}; + +/// \brief The server: DoGetAsync is the only handler overridden, so every other +/// RPC answers UNIMPLEMENTED. +class ExampleServer : public flight::AsyncGenericFlightServerBase { + public: + arrow::Future> DoGetAsync( + const flight::ServerCallContext& context, const flight::Ticket& request) override { + std::cout << "DoGet: ticket=" << request.ticket << std::endl; + auto schema = arrow::schema({arrow::field("value", arrow::int64())}); + auto reader = std::make_shared(schema, FLAGS_batches); + std::shared_ptr stream = + std::make_shared( + std::make_unique(reader), FLAGS_batch_delay_ms); + // The stream is ready here, but its payloads are not: the transport will + // wait on the futures NextAsync() returns. + return arrow::Future>::MakeFinished( + std::move(stream)); + } +}; + +/// \brief The --demo self-check: drain one DoGet stream and count its batches. +bool RunDemo(int port) { + auto location = flight::Location::ForGrpcTcp("127.0.0.1", port); + if (!location.ok()) { + std::cerr << "Failed to build client location: " << location.status() << std::endl; + return false; + } + auto client = flight::FlightClient::Connect(*location); + if (!client.ok()) { + std::cerr << "Failed to connect: " << client.status() << std::endl; + return false; + } + auto reader = (*client)->DoGet(flight::Ticket("demo")); + if (!reader.ok()) { + std::cerr << "DoGet failed: " << reader.status() << std::endl; + return false; + } + int64_t num_batches = 0; + while (true) { + auto chunk = (*reader)->Next(); + if (!chunk.ok()) { + std::cerr << "DoGet drain failed: " << chunk.status() << std::endl; + return false; + } + if (chunk->data == nullptr) break; // end of stream + ++num_batches; + } + if (num_batches == FLAGS_batches) { + std::cout << "PASS: DoGet streamed " << num_batches << " batches" << std::endl; + return true; + } + std::cout << "FAIL: DoGet streamed " << num_batches << " batches, expected " + << FLAGS_batches << std::endl; + return false; +} + +int main(int argc, char** argv) { + if (argc == 1) { + // As in the other examples: a bare run (e.g. from ctest) does not start a + // server, it just prints the usage. + std::cout << "Usage: " << argv[0] + << " [--port=PORT] [--batches=N] [--batch_delay_ms=MS] [--demo]" + << std::endl; + return EXIT_SUCCESS; + } + gflags::ParseCommandLineFlags(&argc, &argv, true); + + ExampleServer server; + auto location = flight::Location::ForGrpcTcp("0.0.0.0", FLAGS_port); + if (!location.ok()) { + std::cerr << location.status() << std::endl; + return EXIT_FAILURE; + } + auto status = server.Init(flight::FlightServerOptions(*location)); + if (!status.ok()) { + std::cerr << status.ToString() << std::endl; + return EXIT_FAILURE; + } + std::cout << "Serving Flight on " << server.location().ToString() << std::endl; + std::cout << "Server pid " << CurrentPid() << ", port " << server.port() << std::endl; + + if (FLAGS_demo) { + bool ok = RunDemo(server.port()); + ARROW_WARN_NOT_OK(server.Shutdown(), "Error shutting down server"); + ARROW_WARN_NOT_OK(server.Wait(), "Error waiting for server shutdown"); + return ok ? EXIT_SUCCESS : EXIT_FAILURE; + } + + status = server.SetShutdownOnSignals({SIGINT, SIGTERM}); + if (!status.ok()) { + std::cerr << status.ToString() << std::endl; + return EXIT_FAILURE; + } + status = server.Serve(); + if (!status.ok()) { + std::cerr << status.ToString() << std::endl; + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/cpp/examples/arrow/flight_async_generic_doput_example.cc b/cpp/examples/arrow/flight_async_generic_doput_example.cc new file mode 100644 index 000000000000..2adee004332b --- /dev/null +++ b/cpp/examples/arrow/flight_async_generic_doput_example.cc @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#ifdef _WIN32 +# include +#else +# include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +// Serve DoPut over the async generic Flight transport. +// +// Deriving from AsyncGenericFlightServerBase (arrow/flight/server_async.h) +// selects the async generic gRPC transport. On this path DoPut is not answered +// by a handler method: the transport hands every decoded upload to the +// AsyncFlightDataListener the server class hands out (CreateDoPutListener), one +// listener per RPC. The callbacks are: +// +// * OnDescriptor(const FlightDescriptor&) - the descriptor of the upload; +// returning non-OK rejects the upload (the client sees that status when it +// closes the writer). +// * OnSchemaDecoded(std::shared_ptr) - from ipc::Listener, the decoded +// schema. +// * OnNext(FlightStreamChunk) - once per decoded message, chunk.data is the +// RecordBatch, or nullptr for metadata-only messages. +// +// All of them run on gRPC callback threads, so they must return promptly: a +// slow callback stalls the RPC and every other call that thread would serve. +// This listener only counts and prints, a server with real work would hand the +// chunk to its own queue or thread pool and return at once. +// +// Usage: +// flight-async-generic-doput-example --port=31337 +// flight-async-generic-doput-example --port=0 --demo +// +// Run with no arguments to print this message and exit. + +DEFINE_int32(port, 0, "Port to listen on (0 picks a free port)"); +DEFINE_bool(demo, false, "Run the DoPut self-check against the server, then exit"); + +namespace flight = arrow::flight; + +/// \brief The current process' id, portably. +int64_t CurrentPid() { +#ifdef _WIN32 + return static_cast(::GetCurrentProcessId()); +#else + return static_cast(::getpid()); +#endif +} + +constexpr int kNumBatches = 3; +constexpr int64_t kRowsPerBatch = 4; + +/// \brief Make a one-column batch of `num_rows` int64 values starting at `value`. +arrow::Result> MakeInt64Batch( + const std::shared_ptr& schema, int64_t value, int64_t num_rows) { + arrow::Int64Builder builder; + for (int64_t i = 0; i < num_rows; i++) { + ARROW_RETURN_NOT_OK(builder.Append(value + i)); + } + ARROW_ASSIGN_OR_RAISE(auto array, builder.Finish()); + return arrow::RecordBatch::Make(schema, array->length(), {array}); +} + +/// \brief The AsyncFlightDataListener serving one DoPut RPC: it records what it is +/// handed as the upload streams in. The counters are atomics because the +/// callbacks run on gRPC threads. +class CountingUploadListener : public flight::AsyncFlightDataListener { + public: + arrow::Future<> OnDescriptor(const flight::FlightDescriptor& descriptor) override { + std::cout << "DoPut descriptor: " << descriptor.ToString() << std::endl; + // Returning non-OK here would reject the upload, the client sees that + // status when it closes the writer. + return arrow::Future<>::MakeFinished(arrow::Status::OK()); + } + + arrow::Status OnSchemaDecoded(std::shared_ptr schema) override { + std::cout << "DoPut schema:"; + for (const auto& field : schema->fields()) { + std::cout << " " << field->name() << ":" << field->type()->ToString(); + } + std::cout << std::endl; + return arrow::Status::OK(); + } + + arrow::Future<> OnNext(flight::FlightStreamChunk chunk) override { + if (chunk.data != nullptr) { // null for metadata-only messages + num_batches_.fetch_add(1); + num_rows_.fetch_add(chunk.data->num_rows()); + } + return arrow::Future<>::MakeFinished(arrow::Status::OK()); + } + + /// \brief The upload ended, whichever way: here is where a consumer commits. + /// status == OK means the client ended it normally, anything else is why it + /// did not complete. + arrow::Future<> OnFinish(arrow::Status status) override { + finished_ = true; + if (status.ok()) { + std::cout << "DoPut finished: the client ended the upload" << std::endl; + } else { + std::cout << "DoPut finished: " << status.ToString() << std::endl; + } + return arrow::Future<>::MakeFinished(arrow::Status::OK()); + } + + int64_t num_batches() const { return num_batches_.load(); } + int64_t num_rows() const { return num_rows_.load(); } + bool finished() const { return finished_; } + + private: + std::atomic num_batches_{0}; + std::atomic num_rows_{0}; + std::atomic finished_{false}; +}; + +/// \brief The server. Deriving from AsyncGenericFlightServerBase selects the +/// async generic transport. Uploads are served by the listener the server +/// class itself hands out: the transport calls CreateDoPutListener() once per +/// DoPut RPC, the server-class counterpart of DoGetAsync. Returning nullptr +/// refuses uploads (they answer UNIMPLEMENTED). +/// +/// One listener per RPC, as overlapping uploads require. This demo has a +/// single --demo client, so it keeps the listener it handed out in order to +/// check it in main(), once the server has stopped. +class UploadServer : public flight::AsyncGenericFlightServerBase { + public: + std::shared_ptr CreateDoPutListener( + const flight::ServerCallContext& context) override { + auto listener = std::make_shared(); + // Called concurrently, once per DoPut RPC, from gRPC callback threads: an + // unsynchronised shared_ptr assignment here is a data race that corrupts + // the heap. The lock only serializes the bookkeeping for the demo's + // assertion; a real server keeps no such member. + std::lock_guard lock(listener_mutex_); + last_listener_ = listener; + return listener; + } + + /// The listener the last DoPut RPC got, for the demo's assertions. + std::shared_ptr last_listener() const { + std::lock_guard lock(listener_mutex_); + return last_listener_; + } + + private: + mutable std::mutex listener_mutex_; + std::shared_ptr last_listener_; +}; + +/// \brief The --demo client half: upload kNumBatches batches and close the +/// writer. What the listener saw is read back in main(), once the server has +/// stopped. +bool RunDemo(int port) { + auto location = flight::Location::ForGrpcTcp("127.0.0.1", port); + if (!location.ok()) { + std::cerr << "Failed to build client location: " << location.status() << std::endl; + return false; + } + auto client = flight::FlightClient::Connect(*location); + if (!client.ok()) { + std::cerr << "Failed to connect: " << client.status() << std::endl; + return false; + } + + auto schema = arrow::schema({arrow::field("value", arrow::int64())}); + auto put = (*client)->DoPut(flight::FlightDescriptor::Command("upload"), schema); + if (!put.ok()) { + std::cerr << "DoPut failed: " << put.status() << std::endl; + return false; + } + arrow::Status status; + for (int i = 0; i < kNumBatches && status.ok(); i++) { + auto batch = MakeInt64Batch(schema, i * kRowsPerBatch, kRowsPerBatch); + if (!batch.ok()) { + std::cerr << "Failed to make a batch: " << batch.status() << std::endl; + return false; + } + status = put->writer->WriteRecordBatch(**batch); + } + if (status.ok()) { + status = put->writer->DoneWriting(); + } + if (status.ok()) { + status = put->writer->Close(); + } + if (!status.ok()) { + std::cerr << "DoPut upload failed: " << status << std::endl; + return false; + } + return true; +} + +int main(int argc, char** argv) { + if (argc == 1) { + // As in the other examples: a bare run (e.g. from ctest) does not start a + // server, it just prints the usage. + std::cout << "Usage: " << argv[0] << " [--port=PORT] [--demo]" << std::endl; + return EXIT_SUCCESS; + } + gflags::ParseCommandLineFlags(&argc, &argv, true); + + UploadServer server; + + auto location = flight::Location::ForGrpcTcp("0.0.0.0", FLAGS_port); + if (!location.ok()) { + std::cerr << location.status() << std::endl; + return EXIT_FAILURE; + } + flight::FlightServerOptions options(*location); + // UploadServer derives from AsyncGenericFlightServerBase: the server class + // alone selects the async generic transport, so there is no option to set. + // Uploads come from the server's CreateDoPutListener() override. + auto status = server.Init(options); + if (!status.ok()) { + std::cerr << status.ToString() << std::endl; + return EXIT_FAILURE; + } + int port = server.port(); + std::cout << "Serving Flight on " << server.location().ToString() << std::endl; + std::cout << "Server pid " << CurrentPid() << ", port " << port << std::endl; + + if (FLAGS_demo) { + bool ok = RunDemo(port); + ARROW_WARN_NOT_OK(server.Shutdown(), "Error shutting down server"); + ARROW_WARN_NOT_OK(server.Wait(), "Error waiting for server shutdown"); + // The listener callbacks run on gRPC threads: read its counters only once + // the server has stopped (the same pattern the flight tests use). + const auto listener = server.last_listener(); + if (listener == nullptr) { + std::cout << "FAIL: the server handed out no DoPut listener" << std::endl; + ok = false; + } else if (listener->num_batches() == kNumBatches && + listener->num_rows() == kNumBatches * kRowsPerBatch && + listener->finished()) { + std::cout << "PASS: listener saw " << listener->num_batches() << " batches, " + << listener->num_rows() << " rows, and its OnFinish said the client " + << "ended the upload" << std::endl; + } else { + std::cout << "FAIL: listener saw " << listener->num_batches() << " batches, " + << listener->num_rows() << " rows, finished=" << listener->finished() + << ", expected " << kNumBatches << " batches, " + << kNumBatches * kRowsPerBatch << " rows, finished" << std::endl; + ok = false; + } + return ok ? EXIT_SUCCESS : EXIT_FAILURE; + } + + status = server.SetShutdownOnSignals({SIGINT, SIGTERM}); + if (!status.ok()) { + std::cerr << status.ToString() << std::endl; + return EXIT_FAILURE; + } + status = server.Serve(); + if (!status.ok()) { + std::cerr << status.ToString() << std::endl; + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 09eda0e24445..0e41da9bcf8a 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -162,6 +162,7 @@ else() endif() # Was in a different namespace, or simply not supported, prior to this +# (the async *client* paths in grpc_client.cc). if(ARROW_GRPC_VERSION VERSION_GREATER_EQUAL "1.40") add_definitions(-DGRPC_ENABLE_ASYNC) endif() @@ -179,9 +180,11 @@ set(ARROW_FLIGHT_SRCS client_cookie_middleware.cc client_tracing_middleware.cc cookie_internal.cc + flight_data_decoder.cc middleware.cc serialization_internal.cc server.cc + server_async.cc server_auth.cc server_tracing_middleware.cc transport.cc @@ -196,6 +199,23 @@ set(ARROW_FLIGHT_SRCS transport/grpc/util_internal.cc types.cc) +# The async generic *server* needs gRPC's generic callback API, which exists from +# 1.65 on (see transport/grpc/async_grpc_compat.h). This floor is higher than +# GRPC_ENABLE_ASYNC above, which only gates the async client. +if(ARROW_GRPC_VERSION VERSION_GREATER_EQUAL "1.65") + list(APPEND + ARROW_FLIGHT_SRCS + transport/grpc/async_grpc_service.cc + transport/grpc/async_grpc_do_get.cc + transport/grpc/async_grpc_do_put.cc + transport/grpc/async_grpc_exchange.cc + transport/grpc/async_grpc_exchange_reader.cc + transport/grpc/async_grpc_exchange_writer.cc + transport/grpc/async_grpc_handshake.cc + transport/grpc/async_grpc_streaming.cc + transport/grpc/async_grpc_unary.cc) +endif() + # Handle Unity build header conflicts on Windows. if(CMAKE_UNITY_BUILD AND WIN32) set_source_files_properties(client.cc @@ -206,6 +226,7 @@ if(CMAKE_UNITY_BUILD AND WIN32) transport/grpc/protocol_grpc_internal.cc transport/grpc/util_internal.cc types.cc + transport_server_internal.cc PROPERTIES SKIP_UNITY_BUILD_INCLUSION TRUE) endif() @@ -336,6 +357,27 @@ add_arrow_test(flight_test LABELS "arrow_flight") +# Async Flight server over the gRPC generic callback API (>= 1.65, like the +# sources it tests; higher than the async client floor above). +if(ARROW_GRPC_VERSION VERSION_GREATER_EQUAL "1.65") + add_arrow_test(flight_async_grpc_test + SOURCES + transport/grpc/async_grpc_service_test.cc + transport/grpc/async_grpc_do_get_test.cc + transport/grpc/async_grpc_do_put_test.cc + transport/grpc/async_grpc_unary_test.cc + transport/grpc/async_grpc_streaming_test.cc + transport/grpc/async_grpc_exchange_test.cc + transport/grpc/async_grpc_handshake_test.cc + STATIC_LINK_LIBS + ${ARROW_FLIGHT_TEST_LINK_LIBS} + LABELS + "arrow_flight") + if(TEST arrow-flight-async-grpc-test) + set_tests_properties(arrow-flight-async-grpc-test PROPERTIES TIMEOUT 300) + endif() +endif() + # Build test server for unit tests or benchmarks if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) add_executable(flight-test-server test_server.cc) diff --git a/cpp/src/arrow/flight/README.md b/cpp/src/arrow/flight/README.md index 5156973acf33..f23f88c78dfe 100644 --- a/cpp/src/arrow/flight/README.md +++ b/cpp/src/arrow/flight/README.md @@ -33,4 +33,4 @@ your current working directory or you need to add it to your path, e.g. ``` PATH=debug:$PATH debug/flight-test -``` \ No newline at end of file +``` diff --git a/cpp/src/arrow/flight/api.h b/cpp/src/arrow/flight/api.h index ed31b5c8fa41..c3b0e307d7a7 100644 --- a/cpp/src/arrow/flight/api.h +++ b/cpp/src/arrow/flight/api.h @@ -23,6 +23,7 @@ #include "arrow/flight/client_tracing_middleware.h" #include "arrow/flight/middleware.h" #include "arrow/flight/server.h" +#include "arrow/flight/server_async.h" #include "arrow/flight/server_auth.h" #include "arrow/flight/server_middleware.h" #include "arrow/flight/server_tracing_middleware.h" diff --git a/cpp/src/arrow/flight/flight_data_decoder.cc b/cpp/src/arrow/flight/flight_data_decoder.cc new file mode 100644 index 000000000000..6d60ceb9692a --- /dev/null +++ b/cpp/src/arrow/flight/flight_data_decoder.cc @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/flight_data_decoder.h" + +#include +#include + +#include "arrow/flight/serialization_internal.h" +#include "arrow/flight/transport.h" +#include "arrow/ipc/message.h" +#include "arrow/ipc/reader.h" +#include "arrow/record_batch.h" +#include "arrow/result.h" +#include "arrow/status.h" + +namespace arrow::flight { + +namespace { + +// FlightDataMessageReader is an ipc::MessageReader that accepts one at a time +// FlightData messages. Analogous to MessageReader::Open(InputStream*) but for +// individual FlightData messages directly read from the received buffers. +class FlightDataMessageReader : public ipc::MessageReader { + public: + /// \brief Queue one message, with the app metadata it arrived with. + /// \param message The message to queue. + /// \param app_metadata The application metadata associated with the message. + void Push(std::unique_ptr message, std::shared_ptr app_metadata) { + queue_.emplace_back(std::move(message), std::move(app_metadata)); + } + + /// \brief Check if there is a record batch message queued. + /// \return True if a record batch message is queued, false otherwise. + bool HasRecordBatch() const { + return std::ranges::any_of(queue_, [](const QueuedMessage& queued) { + return queued.message->type() == ipc::MessageType::RECORD_BATCH; + }); + } + + /// \brief Take the app metadata of the last queued message, leaving that message + /// with none: its app metadata has been delivered already. + /// \return The app metadata of the last queued message, or nullptr if no messages are + /// queued. + std::shared_ptr TakeLastAppMetadata() { + if (queue_.empty()) { + return nullptr; + } + return std::move(queue_.back().app_metadata); + } + + /// Take the next queued message, if any, and return it. The app metadata of + /// the message taken is stored internally and can be retrieved with + /// ReadAppMetadata(). + /// \return The next queued message, or nullptr if no messages are queued. + ::arrow::Result> ReadNextMessage() override { + if (queue_.empty()) { + return nullptr; + } + QueuedMessage queued = std::move(queue_.front()); + queue_.pop_front(); + app_metadata_ = std::move(queued.app_metadata); + return std::move(queued.message); + } + + /// \brief Read the application metadata of the last message taken by ReadNextMessage(). + /// \return The application metadata of the last message taken, or nullptr if no + /// messages have been taken. + std::shared_ptr ReadAppMetadata() { return app_metadata_; } + + private: + /// \brief Queue of messages that have been pushed but not yet read. + struct QueuedMessage { + QueuedMessage() = default; + QueuedMessage(std::unique_ptr message, + std::shared_ptr app_metadata) + : message(std::move(message)), app_metadata(std::move(app_metadata)) {} + std::unique_ptr message; + std::shared_ptr app_metadata; + }; + + std::deque queue_; + std::shared_ptr app_metadata_; +}; + +// FlightDataMessageReader is owned by the decoder, not by the +// RecordBatchStreamReader that reads through it: Open() takes a unique_ptr and +// adopts it, so handing over the decoder's reader would free it on any failure +// path (a malformed first message, a first message that is not a schema), +// leaving the decoder with a dangling pointer. This forwards to the decoder's +// reader and stays owned by it. +class FlightDataMessageReaderRef final : public ipc::MessageReader { + public: + explicit FlightDataMessageReaderRef(FlightDataMessageReader* reader) + : reader_(reader) {} + + arrow::Result> ReadNextMessage() override { + return reader_->ReadNextMessage(); + } + + private: + FlightDataMessageReader* reader_; +}; + +} // namespace + +class AsyncFlightMessageDecoder::AsyncFlightMessageDecoderImpl { + public: + AsyncFlightMessageDecoderImpl(std::shared_ptr listener, + ipc::IpcReadOptions options) + : listener_(std::move(listener)), + options_(std::move(options)), + message_reader_(std::make_unique()) {} + + /// \brief Consume a chunk of Flight data. + /// \param data The Flight data to consume. + /// \return A future carrying the listener's status for this message: a + /// non-OK result rejects the upload. + /// The descriptor of an upload is eported before the rest of its first message is + /// decoded. + Future<> ConsumeData(internal::FlightData data) { + if (!data.descriptor) { + return ConsumeDataAfterDescriptor(std::move(data)); + } else { + return listener_->OnDescriptor(*data.descriptor) + .Then([this, data = std::move(data)]() mutable { + return ConsumeDataAfterDescriptor(std::move(data)); + }); + } + } + + std::shared_ptr schema() const { + return batch_reader_ ? batch_reader_->schema() : nullptr; + } + + private: + /// \brief Decode a FlightData message whose descriptor (if any) was already + /// reported to the listener. + /// \param data The Flight data to consume after the descriptor has been reported. + /// \return A future carrying the listener's status for this message: a non-OK result + /// rejects the upload. + /// \note The descriptor of an upload is assumed to have been reported + /// before calling this method. + /// + Future<> ConsumeDataAfterDescriptor(internal::FlightData data) { + if (!data.metadata) { + // Metadata-only message: no IPC content, just Flight app_metadata. + if (data.app_metadata && data.app_metadata->size() > 0) { + FlightStreamChunk chunk; + chunk.app_metadata = std::move(data.app_metadata); + return listener_->OnNext(std::move(chunk)); + } + return Status::OK(); + } + + ARROW_ASSIGN_OR_RAISE(auto message, data.OpenMessage()); + + if (!batch_reader_) { + // Initialize RecordBatchStreamReader and read the first IPC message. + // It must be a schema. + message_reader_->Push(std::move(message), std::move(data.app_metadata)); + ARROW_ASSIGN_OR_RAISE( + batch_reader_, + ipc::RecordBatchStreamReader::Open( + std::make_unique(message_reader_.get()), + options_)); + return listener_->OnSchemaDecoded(batch_reader_->schema()); + } + + message_reader_->Push(std::move(message), std::move(data.app_metadata)); + + if (!message_reader_->HasRecordBatch()) { + auto app_metadata = message_reader_->TakeLastAppMetadata(); + if (app_metadata && app_metadata->size() > 0) { + FlightStreamChunk chunk; + chunk.app_metadata = std::move(app_metadata); + return listener_->OnNext(std::move(chunk)); + } + return arrow::Future<>::MakeFinished(Status::OK()); + } + + std::shared_ptr batch; + RETURN_NOT_OK(batch_reader_->ReadNext(&batch)); + auto app_metadata = message_reader_->ReadAppMetadata(); + + if (batch) { + FlightStreamChunk chunk{std::move(batch), std::move(app_metadata)}; + return listener_->OnNext(std::move(chunk)); + } + + // This has to be a Dictionary batch. + if (app_metadata && app_metadata->size() > 0) { + FlightStreamChunk chunk{nullptr, std::move(app_metadata)}; + return listener_->OnNext(std::move(chunk)); + } + return arrow::Future<>::MakeFinished(Status::OK()); + } + + std::shared_ptr listener_; + ipc::IpcReadOptions options_; + std::unique_ptr message_reader_; + std::shared_ptr batch_reader_; +}; + +AsyncFlightMessageDecoder::AsyncFlightMessageDecoder( + std::shared_ptr listener, ipc::IpcReadOptions options) + : impl_(std::make_unique(std::move(listener), + std::move(options))) {} + +AsyncFlightMessageDecoder::~AsyncFlightMessageDecoder() = default; + +Future<> AsyncFlightMessageDecoder::Consume(std::shared_ptr buffer) { + ARROW_ASSIGN_OR_RAISE(auto data, internal::DeserializeFlightData(buffer)); + return impl_->ConsumeData(std::move(data)); +} + +Future<> AsyncFlightMessageDecoder::Consume(internal::FlightData data) { + return impl_->ConsumeData(std::move(data)); +} + +std::shared_ptr AsyncFlightMessageDecoder::schema() const { + return impl_->schema(); +} + +// --- AsyncFlightDataListener: transport state and the terminal callbacks --- + +AsyncFlightDataListener::AsyncFlightDataListener() = default; +AsyncFlightDataListener::~AsyncFlightDataListener() = default; +AsyncFlightDataWriter::~AsyncFlightDataWriter() = default; + +Status AsyncFlightDataWriter::WriteMetadata(const Buffer& app_metadata) { + // Install() and Clear() swap transport_ under writer_mutex_, so both the read + // and the call must happen under it: otherwise this can read the old pointer + // just before DoPutReactor::OnDone() runs Clear(), and then call through a + // destructed reactor. The transport's own WriteMetadata only takes its + // private write_mutex_, so there is no lock-order cycle. + std::lock_guard lock(writer_mutex_); + if (transport_ == nullptr) { + return Status::Invalid( + "no upload in flight to write metadata to: the writer is only usable " + "while the upload's RPC is running"); + } + return transport_->WriteMetadata(app_metadata); +} + +namespace internal { + +FlightDataListenerTransport::~FlightDataListenerTransport() = default; + +void FlightDataListenerTransport::Install( + const std::shared_ptr& listener, + FlightDataListenerTransport* transport) { + if (listener == nullptr) { + return; + } + std::lock_guard lock(listener->transport_mutex_); + listener->transport_ = transport; + if (listener->writer_ != nullptr) { + std::lock_guard writer_lock(listener->writer_->writer_mutex_); + listener->writer_->transport_ = transport; + } +} + +void FlightDataListenerTransport::Clear( + const std::shared_ptr& listener) { + if (listener == nullptr) { + return; + } + std::lock_guard lock(listener->transport_mutex_); + listener->transport_ = nullptr; + if (listener->writer_ != nullptr) { + std::lock_guard writer_lock(listener->writer_->writer_mutex_); + listener->writer_->transport_ = nullptr; + } +} + +Future<> FlightDataListenerTransport::ReportFinish( + const std::shared_ptr& listener, Status status) { + if (listener == nullptr) { + return Future<>::MakeFinished(); + } + // Report exactly once: the first ending wins. + bool expected = false; + if (!listener->finished_.compare_exchange_strong(expected, true)) { + return Future<>::MakeFinished(); + } + return listener->OnFinish(std::move(status)); +} + +} // namespace internal + +Future<> AsyncFlightDataListener::Cancel(Status status) { + internal::FlightDataListenerTransport* transport; + { + std::lock_guard lock(transport_mutex_); + transport = transport_; + if (transport == nullptr) { + return arrow::Future<>::MakeFinished(Status::Invalid( + "no upload in flight to cancel: Cancel() must be called while the " + "upload's RPC is running")); + } + transport->Hold(); + } + // Unlocked, so OnFinish() may re-enter Cancel() without deadlocking. + transport->CancelUpload(std::move(status)); + transport->ReleaseHold(); + return arrow::Future<>::MakeFinished(Status::OK()); +} + +} // namespace arrow::flight diff --git a/cpp/src/arrow/flight/flight_data_decoder.h b/cpp/src/arrow/flight/flight_data_decoder.h new file mode 100644 index 000000000000..315547371274 --- /dev/null +++ b/cpp/src/arrow/flight/flight_data_decoder.h @@ -0,0 +1,243 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/flight/types.h" +#include "arrow/flight/visibility.h" +#include "arrow/ipc/options.h" +#include "arrow/ipc/reader.h" +#include "arrow/status.h" + +namespace arrow::flight { + +namespace internal { +struct FlightData; +} // namespace internal + +class AsyncFlightDataListener; + +namespace internal { + +/// \brief The per-RPC handle a transport gives an AsyncFlightDataListener so the +/// application can act on the RPC while it is in flight. +/// +/// The transport owns this, the listener only holds a reference for as long as +/// the RPC is live, and asks for it under a lock the transport takes to clear +/// it (see FlightDataListenerTransport::Clear) +class ARROW_FLIGHT_EXPORT FlightDataListenerTransport { + public: + virtual ~FlightDataListenerTransport(); + + /// \brief Finish the upload's RPC with `status`, from any thread. + /// \param status The status to finish the upload with. + virtual void CancelUpload(Status status) = 0; + + /// \brief Send one application metadata message to the client, from any thread. + /// \param app_metadata The metadata to attach to the next PutResult message. + /// \return OK when the message was handed to the transport, or a non-OK status + /// when the upload is already over. + virtual Status WriteMetadata(const Buffer& app_metadata) = 0; + + /// \brief Pin this transport's lifetime, from any thread. + /// + /// Install()/Clear() publish and retire the transport through the listener's + /// mutex, but a caller that reads the pointer and then releases the mutex + /// before calling in (as Cancel() must, so OnFinish() can re-enter) still + /// races with Clear() and the reactor's deletion. Hold under the mutex and + /// pair with ReleaseHold() around the call so the object stays alive. + virtual void Hold() = 0; + + /// \brief Drop a reference taken with Hold(). Deletes the transport at zero. + virtual void ReleaseHold() = 0; + + /// \brief Install `transport` on `listener` for the life of the RPC. + /// \param listener The listener to install the transport on. + /// \param transport The transport to install. + static void Install(const std::shared_ptr& listener, + FlightDataListenerTransport* transport); + + /// \brief Clear the installed transport. + /// \param listener The listener to clear the transport from. + static void Clear(const std::shared_ptr& listener); + + /// \brief Report the upload's terminal status to the listener, once. + /// \param listener The listener to report the status to. + /// \param status The terminal status of the upload. + /// \return the listener's OnFinish() future. The caller must not block on it + /// (this runs on a gRPC callback thread) but should chain the RPC's + /// completion on it, so an asynchronous failure to commit is reported to the + /// client instead of a success. Returns a finished future when the status + /// was already reported. + static Future<> ReportFinish(const std::shared_ptr& listener, + Status status); +}; + +} // namespace internal + +/// \brief Sends application-specific metadata back to the client during an upload. +/// +/// The async counterpart of FlightMetadataWriter: the transport hands one to the +/// AsyncFlightDataListener at the start of an upload (AsyncFlightDataListener::writer) +/// and it stays valid for as long as the upload's RPC runs. Every call sends one +/// PutResult message immediately, the client reads them with ReadMetadata() on +/// its DoPutResult. +class ARROW_FLIGHT_EXPORT AsyncFlightDataWriter { + public: + virtual ~AsyncFlightDataWriter(); + + /// \brief Send a metadata message to the client. + /// + /// Safe to call from any thread while the upload is in flight. Once the + /// upload has ended the call reports the ending (Cancelled or FailedPrecondition) + /// instead of writing to a finished RPC. + /// \param app_metadata The application metadata to send. + /// \return Status indicating success or failure. + Status WriteMetadata(const Buffer& app_metadata); + + private: + friend class internal::FlightDataListenerTransport; + + /// The transport's RPC state, set and cleared through + /// FlightDataListenerTransport::Install/Clear, like + /// AsyncFlightDataListener::transport_. + internal::FlightDataListenerTransport* transport_ = nullptr; + /// Guards transport_, since WriteMetadata may be called from any thread while + /// the transport installs or clears the pointer. + mutable std::mutex writer_mutex_; +}; + +/// \brief A general listener class to receive events from FlightMessageDecoder +/// +/// User must implement callback methods for interested events. +class ARROW_FLIGHT_EXPORT AsyncFlightDataListener : public ipc::Listener { + public: + AsyncFlightDataListener(); + ~AsyncFlightDataListener() override; + + /// \brief Called for each decoded FlightStreamChunk. + /// \param chunk The decoded FlightStreamChunk. + /// \return A future that completes when the chunk has been processed. + virtual Future<> OnNext(FlightStreamChunk chunk) = 0; + + /// \brief Called when the descriptor of an upload is decoded. + /// + /// Fired before any schema or data of that upload, so the listener knows + /// which upload it is being handed. A non-OK status rejects the upload. + /// \param descriptor The decoded FlightDescriptor. + /// \return A future that completes when the descriptor has been processed. + virtual Future<> OnDescriptor(const FlightDescriptor& descriptor) { + return Future<>::MakeFinished(); + } + + /// \brief Called once, when the upload ends, whichever way it ends. + /// + /// Runs on a transport thread, so it must not block. + /// `status` is OK for an upload if the client ended normally, and the failure otherwise + /// (the client went away, the transport failed, or the upload was rejected). + /// + /// \param status The terminal status of the upload. + /// \return A future that completes when the finish has been processed. + virtual Future<> OnFinish(Status status) { return Future<>::MakeFinished(); } + + /// \brief The writer for application metadata sent back to the client. + /// + /// Valid for as long as the upload's RPC is live (until OnFinish is reported). + /// Never null, calls after the upload ended report the ending instead of writing. + const std::shared_ptr& writer() const { return writer_; } + + /// \brief Cancel the upload with `status`, from any thread. + /// + /// Finishes the RPC with that status without waiting for the client to end + /// the upload: the client's pending write or Close() reports it. + /// Use for a server-side rejection discovered mid-upload (quota, bad batch, upstream + /// error). + /// Safe to call from any thread at any time: once the upload is no + /// longer in flight this reports Invalid instead of reaching a finished RPC. + /// \param status The status to cancel the upload with. + /// + /// \return OK when the cancel was handed to the transport, or Invalid when + /// there is no upload in flight. + Future<> Cancel(Status status); + + private: + /// The transport installs and clears the state through + /// FlightDataListenerTransport::Install/Clear, both under this lock. + friend class internal::FlightDataListenerTransport; + + /// Whether OnFinish was already reported. + std::atomic finished_{false}; + + /// Never null: the listener may use it without waiting for OnDescriptor. + std::shared_ptr writer_ = + std::make_shared(); + + mutable std::mutex transport_mutex_; + /// Not owned: the transport's own RPC state, valid until Clear(). + internal::FlightDataListenerTransport* transport_ = nullptr; +}; + +/// \brief Push style stream decoder that turns raw arrow Buffers into +/// FlightStreamChunks. +/// +/// This class decodes Apache Arrow Flight data format from arrow::Buffer +/// and fires events on the provided AsyncFlightDataListener. +class ARROW_FLIGHT_EXPORT AsyncFlightMessageDecoder { + public: + /// \brief Construct an AsyncFlightMessageDecoder with the given listener and IPC read + /// options. + /// + /// \param listener The listener that will receive decoded Flight messages. + /// \param options The IPC read options to use for decoding. + explicit AsyncFlightMessageDecoder( + std::shared_ptr listener, + ipc::IpcReadOptions options = ipc::IpcReadOptions::Defaults()); + ~AsyncFlightMessageDecoder(); + + /// \brief Decode one FlightData message directly from a buffer. + /// + /// Fires listener->OnSchemaDecoded() on the first message containing + /// a schema, listener->OnNext() for each subsequent record batch, + /// metadata-only message or dictionary batch. + /// + /// \param[in] buffer a raw buffer directly from the transport. Example + /// the arrow::Buffer extracted from the grpc::ByteBuffer from the gRPC transport. + /// \return Status + Future<> Consume(std::shared_ptr buffer); + + /// \brief Decode one FlightData message already deserialized by the transport. + /// + /// Same as Consume(Buffer) but avoids a serialization round-trip for transports + /// that hand out an internal::FlightData directly (the gRPC client reads FlightData + /// straight into internal::FlightData). + /// \internal This method is intended for internal use within the Arrow library. + Future<> Consume(internal::FlightData data); + + /// \brief The decoded schema. + /// \return The decoded schema, or nullptr if no schema has been decoded yet. + std::shared_ptr schema() const; + + private: + class AsyncFlightMessageDecoderImpl; + std::unique_ptr impl_; +}; + +} // namespace arrow::flight diff --git a/cpp/src/arrow/flight/flight_internals_test.cc b/cpp/src/arrow/flight/flight_internals_test.cc index 5ec9b75dcfda..6061e120724f 100644 --- a/cpp/src/arrow/flight/flight_internals_test.cc +++ b/cpp/src/arrow/flight/flight_internals_test.cc @@ -31,10 +31,14 @@ #include "arrow/flight/client_cookie_middleware.h" #include "arrow/flight/client_middleware.h" #include "arrow/flight/cookie_internal.h" +#include "arrow/flight/flight_data_decoder.h" #include "arrow/flight/serialization_internal.h" #include "arrow/flight/server.h" +#include "arrow/flight/server_async.h" #include "arrow/flight/test_util.h" #include "arrow/flight/transport.h" +#include "arrow/flight/transport/grpc/async_grpc_compat.h" +#include "arrow/flight/transport/grpc/serialization_internal.h" #include "arrow/flight/transport/grpc/util_internal.h" #include "arrow/flight/types.h" #include "arrow/ipc/reader.h" @@ -740,6 +744,43 @@ TEST(GrpcTransport, FlightDataDeserialize) { #endif } +TEST(GrpcTransport, BytesFromBufferConcatenatesSlices) { + // gRPC hands the caller whatever slice layout the message arrived in: the + // loop has to stitch the slices back together, embedded NULs included. + const std::string first("ab\0c", 4); + const std::string empty; + const std::string third("\xff\x01", 2); + std::vector<::grpc::Slice> slices = {::grpc::Slice(first), ::grpc::Slice(empty), + ::grpc::Slice(third)}; + ::grpc::ByteBuffer buffer(slices.data(), slices.size()); + + ASSERT_OK_AND_ASSIGN(auto bytes, + flight::transport::grpc::BytesFromBuffer(buffer, "unused")); + ASSERT_EQ(6, static_cast(bytes.size())); + ASSERT_EQ(first + third, bytes); +} + +TEST(GrpcTransport, BytesFromBufferEmptyBuffer) { + // A valid buffer holding no slices is an empty message, not an error. + ::grpc::ByteBuffer buffer(nullptr, 0); + ASSERT_OK_AND_ASSIGN(auto bytes, + flight::transport::grpc::BytesFromBuffer(buffer, "unused")); + ASSERT_EQ("", bytes); +} + +TEST(GrpcTransport, BytesFromBufferInvalidBufferIsAnInternalError) { + // A default-constructed buffer has no underlying grpc_byte_buffer: the failure + // must keep the caller's message and the transport's Internal detail. + ::grpc::ByteBuffer buffer; + auto result = flight::transport::grpc::BytesFromBuffer(buffer, "no bytes"); + ASSERT_FALSE(result.ok()) << "a buffer with no underlying message cannot be dumped"; + ASSERT_TRUE(result.status().IsIOError()) << result.status(); + ASSERT_NE(std::string::npos, result.status().ToString().find("no bytes")); + auto detail = FlightStatusDetail::UnwrapStatus(result.status()); + ASSERT_NE(nullptr, detail); + ASSERT_EQ(FlightStatusCode::Internal, detail->code()); +} + // ---------------------------------------------------------------------- // Transport-agnostic serialization roundtrip tests @@ -808,6 +849,329 @@ TEST(FlightSerialization, RoundtripMetadataOnly) { ASSERT_EQ(data.app_metadata->ToString(), "metadata-only-message"); } +// ---------------------------------------------------------------------- +// Push-style decoder (AsyncFlightMessageDecoder) tests + +namespace { + +// Serialize one FlightPayload into the single contiguous buffer a transport +// hands to AsyncFlightMessageDecoder::Consume (exactly one FlightData message). +::arrow::Result> DecoderTestWireBuffer( + const FlightPayload& payload) { + ARROW_ASSIGN_OR_RAISE(auto buffers, internal::SerializePayloadToBuffers(payload)); + return ConcatenateBuffers(buffers); +} + +// Captures the events AsyncFlightMessageDecoder fires, in order. +class DecoderTestListener : public AsyncFlightDataListener { + public: + explicit DecoderTestListener(Status next_status = Status::OK(), + Status descriptor_status = Status::OK()) + : next_status_(std::move(next_status)), + descriptor_status_(std::move(descriptor_status)) {} + + Future<> OnDescriptor(const FlightDescriptor& descriptor) override { + events.emplace_back("OnDescriptor"); + descriptors.push_back(descriptor); + return Future<>::MakeFinished(descriptor_status_); + } + + Future<> OnNext(FlightStreamChunk chunk) override { + events.emplace_back("OnNext"); + chunks.push_back(std::move(chunk)); + return Future<>::MakeFinished(next_status_); + } + + // The decoder calls the single-argument overload. + Status OnSchemaDecoded(std::shared_ptr schema) override { + events.emplace_back("OnSchemaDecoded"); + schemas.push_back(std::move(schema)); + return Status::OK(); + } + + std::vector events; + std::vector> schemas; + std::vector chunks; + std::vector descriptors; + + private: + Status next_status_; + Status descriptor_status_; +}; + +} // namespace + +TEST(AsyncFlightMessageDecoder, SchemaAndBatches) { + auto schema = arrow::schema( + {arrow::field("a", arrow::int32()), arrow::field("b", arrow::int32())}); + auto batch1 = RecordBatch::Make( + schema, 2, + {ArrayFromJSON(arrow::int32(), "[1, 2]"), ArrayFromJSON(arrow::int32(), "[3, 4]")}); + auto batch2 = RecordBatch::Make( + schema, 2, + {ArrayFromJSON(arrow::int32(), "[5, 6]"), ArrayFromJSON(arrow::int32(), "[7, 8]")}); + auto reader = RecordBatchReader::Make({batch1, batch2}).ValueOrDie(); + RecordBatchStream stream(std::move(reader)); + + auto listener = std::make_shared(); + AsyncFlightMessageDecoder decoder(listener); + EXPECT_EQ(decoder.schema(), nullptr); + + // One wire message per payload: schema first, then one per record batch. + ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); + ASSERT_OK_AND_ASSIGN(auto schema_wire, DecoderTestWireBuffer(schema_payload)); + ASSERT_OK(decoder.Consume(schema_wire).status()); + + EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded")); + ASSERT_EQ(listener->schemas.size(), 1); + EXPECT_TRUE(listener->schemas[0]->Equals(*schema)); + ASSERT_NE(decoder.schema(), nullptr); + EXPECT_TRUE(decoder.schema()->Equals(*schema)); + + ASSERT_OK_AND_ASSIGN(auto payload1, stream.Next()); + ASSERT_OK_AND_ASSIGN(auto payload2, stream.Next()); + ASSERT_OK_AND_ASSIGN(auto wire1, DecoderTestWireBuffer(payload1)); + ASSERT_OK_AND_ASSIGN(auto wire2, DecoderTestWireBuffer(payload2)); + ASSERT_OK(decoder.Consume(wire1).status()); + ASSERT_OK(decoder.Consume(wire2).status()); + + EXPECT_THAT(listener->events, + ::testing::ElementsAre("OnSchemaDecoded", "OnNext", "OnNext")); + ASSERT_EQ(listener->chunks.size(), 2); + ASSERT_NE(listener->chunks[0].data, nullptr); + EXPECT_TRUE(listener->chunks[0].data->Equals(*batch1)); + EXPECT_EQ(listener->chunks[0].app_metadata, nullptr); + ASSERT_NE(listener->chunks[1].data, nullptr); + EXPECT_TRUE(listener->chunks[1].data->Equals(*batch2)); + EXPECT_EQ(listener->chunks[1].app_metadata, nullptr); +} + +TEST(AsyncFlightMessageDecoder, MetadataOnlyChunkIsAValue) { + // A message with app_metadata but no IPC body is a value, not an issue . + FlightPayload payload; + payload.app_metadata = Buffer::FromString("only-app-metadata"); + ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); + + auto listener = std::make_shared(); + AsyncFlightMessageDecoder decoder(listener); + ASSERT_OK(decoder.Consume(wire).status()); + + EXPECT_THAT(listener->events, ::testing::ElementsAre("OnNext")); + ASSERT_EQ(listener->chunks.size(), 1); + EXPECT_EQ(listener->chunks[0].data, nullptr); + ASSERT_NE(listener->chunks[0].app_metadata, nullptr); + EXPECT_EQ(listener->chunks[0].app_metadata->ToString(), "only-app-metadata"); + EXPECT_EQ(decoder.schema(), nullptr); +} + +TEST(AsyncFlightMessageDecoder, AppMetadataOnBatch) { + auto schema = arrow::schema({arrow::field("a", arrow::int32())}); + auto batch = RecordBatch::Make(schema, 3, {ArrayFromJSON(arrow::int32(), "[1, 2, 3]")}); + auto reader = RecordBatchReader::Make({batch}).ValueOrDie(); + RecordBatchStream stream(std::move(reader)); + + auto listener = std::make_shared(); + AsyncFlightMessageDecoder decoder(listener); + + ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); + ASSERT_OK_AND_ASSIGN(auto schema_wire, DecoderTestWireBuffer(schema_payload)); + ASSERT_OK(decoder.Consume(schema_wire).status()); + + ASSERT_OK_AND_ASSIGN(auto payload, stream.Next()); + payload.app_metadata = Buffer::FromString("batch-app-metadata"); + ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); + ASSERT_OK(decoder.Consume(wire).status()); + + EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded", "OnNext")); + ASSERT_EQ(listener->chunks.size(), 1); + ASSERT_NE(listener->chunks[0].data, nullptr); + EXPECT_TRUE(listener->chunks[0].data->Equals(*batch)); + ASSERT_NE(listener->chunks[0].app_metadata, nullptr); + EXPECT_EQ(listener->chunks[0].app_metadata->ToString(), "batch-app-metadata"); +} + +TEST(AsyncFlightMessageDecoder, SchemaOnlyStream) { + auto schema = arrow::schema({arrow::field("a", arrow::int32())}); + auto batch = RecordBatch::Make(schema, 1, {ArrayFromJSON(arrow::int32(), "[42]")}); + auto reader = RecordBatchReader::Make({batch}).ValueOrDie(); + RecordBatchStream stream(std::move(reader)); + + auto listener = std::make_shared(); + AsyncFlightMessageDecoder decoder(listener); + + ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); + ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(schema_payload)); + ASSERT_OK(decoder.Consume(wire).status()); + + EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded")); + EXPECT_TRUE(listener->chunks.empty()); + ASSERT_NE(decoder.schema(), nullptr); + EXPECT_TRUE(decoder.schema()->Equals(*schema)); +} + +TEST(AsyncFlightMessageDecoder, ListenerErrorIsPropagated) { + auto schema = arrow::schema({arrow::field("a", arrow::int32())}); + auto batch = RecordBatch::Make(schema, 1, {ArrayFromJSON(arrow::int32(), "[1]")}); + auto reader = RecordBatchReader::Make({batch}).ValueOrDie(); + RecordBatchStream stream(std::move(reader)); + + auto listener = + std::make_shared(Status::Invalid("listener failed")); + AsyncFlightMessageDecoder decoder(listener); + + ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); + ASSERT_OK_AND_ASSIGN(auto schema_wire, DecoderTestWireBuffer(schema_payload)); + ASSERT_OK(decoder.Consume(schema_wire).status()); + + // A listener error on a record batch must reach the caller of Consume. + ASSERT_OK_AND_ASSIGN(auto payload, stream.Next()); + ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); + auto status = decoder.Consume(wire).status(); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::Invalid); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("listener failed")); + + // and so must one on a metadata-only message. + FlightPayload metadata_only; + metadata_only.app_metadata = Buffer::FromString("meta"); + ASSERT_OK_AND_ASSIGN(auto meta_wire, DecoderTestWireBuffer(metadata_only)); + auto meta_status = decoder.Consume(meta_wire).status(); + EXPECT_FALSE(meta_status.ok()); + EXPECT_EQ(meta_status.code(), StatusCode::Invalid); +} + +TEST(AsyncFlightMessageDecoder, FailedOpenLeavesTheDecoderUsable) { + // A first IPC message that is not a schema makes RecordBatchStreamReader::Open + // fail. The decoder must survive that: it owns its message reader, rather + // than handing it over to Open, which destroys it on failure and would leave + // the decoder's cached pointer dangling for the next Consume(). + auto schema = arrow::schema({arrow::field("a", arrow::int32())}); + auto batch = RecordBatch::Make(schema, 1, {ArrayFromJSON(arrow::int32(), "[1]")}); + auto reader = RecordBatchReader::Make({batch}).ValueOrDie(); + RecordBatchStream stream(std::move(reader)); + + auto listener = std::make_shared(); + AsyncFlightMessageDecoder decoder(listener); + + // A record batch as the first message: not a schema, so Open() fails. + ASSERT_OK_AND_ASSIGN(auto batch_payload, stream.Next()); + ASSERT_OK_AND_ASSIGN(auto batch_wire, DecoderTestWireBuffer(batch_payload)); + auto failed = decoder.Consume(batch_wire).status(); + EXPECT_FALSE(failed.ok()) << "a first message that is not a schema must be rejected"; + EXPECT_TRUE(listener->events.empty()) + << "nothing may be surfaced for a rejected stream"; + + // The decoder must still work: consume a well-formed stream and watch the + // schema and the batch come through. + auto good_reader = RecordBatchReader::Make({batch}).ValueOrDie(); + RecordBatchStream good_stream(std::move(good_reader)); + ASSERT_OK_AND_ASSIGN(auto schema_payload, good_stream.GetSchemaPayload()); + ASSERT_OK_AND_ASSIGN(auto schema_wire, DecoderTestWireBuffer(schema_payload)); + ASSERT_OK(decoder.Consume(schema_wire).status()); + ASSERT_OK_AND_ASSIGN(auto good_batch, good_stream.Next()); + ASSERT_OK_AND_ASSIGN(auto good_wire, DecoderTestWireBuffer(good_batch)); + ASSERT_OK(decoder.Consume(good_wire).status()); + + EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded", "OnNext")); + ASSERT_EQ(listener->chunks.size(), 1); + ASSERT_NE(listener->chunks[0].data, nullptr); + EXPECT_TRUE(listener->chunks[0].data->Equals(*batch)); +} + +TEST(AsyncFlightMessageDecoder, DictionaryStream) { + // A dictionary-encoded stream arrives as: schema -> dictionary message -> batch. + // Tthe IPC reader reads through the dictionary message while + // looking for a record batch, so a pushed message must only be handed out once + // (returning it twice re-reads it forever). + const auto dict_type = dictionary(arrow::int32(), utf8()); + const auto schema = arrow::schema({arrow::field("d", dict_type)}); + const auto batch = + RecordBatch::Make(schema, 3, {ArrayFromJSON(dict_type, R"(["a", "b", "a"])")}); + auto reader = RecordBatchReader::Make({batch}, schema).ValueOrDie(); + RecordBatchStream stream(std::move(reader)); + + auto listener = std::make_shared(); + AsyncFlightMessageDecoder decoder(listener); + + ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); + ASSERT_OK_AND_ASSIGN(auto schema_wire, DecoderTestWireBuffer(schema_payload)); + ASSERT_OK(decoder.Consume(schema_wire).status()); + + EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded")); + ASSERT_NE(decoder.schema(), nullptr); + EXPECT_TRUE(decoder.schema()->Equals(*schema)); + + // The dictionary message is consumed without surfacing as a chunk of its own. + int payloads = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto payload, stream.Next()); + if (payload.ipc_message.metadata == nullptr) { + break; // end of stream + } + ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); + ASSERT_OK(decoder.Consume(wire).status()); + ++payloads; + } + EXPECT_EQ(payloads, 2); // dictionary message + record batch + + // and the batch arrives exactly once, decoded against the dictionary. + EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded", "OnNext")); + ASSERT_EQ(listener->chunks.size(), 1); + ASSERT_NE(listener->chunks[0].data, nullptr); + EXPECT_TRUE(listener->chunks[0].data->Equals(*batch)); + EXPECT_EQ(listener->chunks[0].app_metadata, nullptr); +} + +TEST(AsyncFlightMessageDecoder, DescriptorReachesListener) { + // An upload message carries the descriptor of the DoPut command, the + // listener sees it before anything else from that message is decoded. + const auto schema = arrow::schema({arrow::field("a", arrow::int32())}); + const auto batch = + RecordBatch::Make(schema, 1, {ArrayFromJSON(arrow::int32(), "[42]")}); + auto reader = RecordBatchReader::Make({batch}).ValueOrDie(); + RecordBatchStream stream(std::move(reader)); + + auto listener = std::make_shared(); + AsyncFlightMessageDecoder decoder(listener); + + auto descriptor = FlightDescriptor::Path({"wave", "h1"}); + ASSERT_OK_AND_ASSIGN(auto payload, stream.GetSchemaPayload()); + ASSERT_OK(internal::ToPayload(descriptor, &payload.descriptor)); + ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); + ASSERT_OK(decoder.Consume(wire).status()); + + EXPECT_THAT(listener->events, + ::testing::ElementsAre("OnDescriptor", "OnSchemaDecoded")); + ASSERT_EQ(listener->descriptors.size(), 1); + EXPECT_EQ(listener->descriptors[0], descriptor); +} + +TEST(AsyncFlightMessageDecoder, DescriptorErrorRejectsUpload) { + const auto schema = arrow::schema({arrow::field("a", arrow::int32())}); + const auto batch = + RecordBatch::Make(schema, 1, {ArrayFromJSON(arrow::int32(), "[42]")}); + auto reader = RecordBatchReader::Make({batch}).ValueOrDie(); + RecordBatchStream stream(std::move(reader)); + + auto listener = std::make_shared( + Status::OK(), Status::Invalid("descriptor rejected")); + AsyncFlightMessageDecoder decoder(listener); + + auto descriptor = FlightDescriptor::Path({"wave", "h1"}); + ASSERT_OK_AND_ASSIGN(auto payload, stream.GetSchemaPayload()); + ASSERT_OK(internal::ToPayload(descriptor, &payload.descriptor)); + ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); + auto status = decoder.Consume(wire).status(); + + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::Invalid); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("descriptor rejected")); + + // The upload is rejected before the schema is decoded. + EXPECT_THAT(listener->events, ::testing::ElementsAre("OnDescriptor")); + EXPECT_EQ(decoder.schema(), nullptr); +} + // ---------------------------------------------------------------------- // Transport abstraction tests @@ -922,5 +1286,26 @@ TEST(TestServerSignalState, ShutdownUninitializedKeepsOtherServerSignalable) { #endif +// An async server must report why it cannot start, in every build: either it +// starts, or Init() answers NotImplemented naming the gRPC requirement. Before +// this was pinned, a build against gRPC < 1.65 never registered an async server +// factory and Init() failed with the generic "no async server transport +// implementation" KeyError, hiding the real cause. +TEST(TestAsyncServerAvailability, InitReportsWhetherAsyncServerIsSupported) { + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + AsyncGenericFlightServerBase server; + const auto status = server.Init(options); +#if ARROW_FLIGHT_HAS_ASYNC_SERVER + ASSERT_OK(status); + ASSERT_OK(server.Shutdown()); +#else + ASSERT_FALSE(status.ok()); + ASSERT_EQ(StatusCode::NotImplemented, status.code()); + ASSERT_NE(std::string::npos, status.message().find("1.65")) + << "the error must name the gRPC version requirement, got: " << status.message(); +#endif +} + } // namespace flight } // namespace arrow diff --git a/cpp/src/arrow/flight/meson.build b/cpp/src/arrow/flight/meson.build index dcdacc42ffdc..465aabad2fa2 100644 --- a/cpp/src/arrow/flight/meson.build +++ b/cpp/src/arrow/flight/meson.build @@ -23,11 +23,13 @@ install_headers( 'client.h', 'client_middleware.h', 'client_tracing_middleware.h', + 'flight_data_decoder.h', 'middleware.h', 'otel_logging.h', 'platform.h', 'server_auth.h', 'server.h', + 'server_async.h', 'server_middleware.h', 'server_tracing_middleware.h', 'test_auth_handlers.h', @@ -45,6 +47,16 @@ install_headers( ) grpc_dep = dependency('grpc++') + +# GRPC_ENABLE_ASYNC gates the async *client* paths in grpc_client.cc; 1.40 is +# where those gRPC APIs landed. +has_async_client = grpc_dep.version().version_compare('>=1.40.0') +grpc_enable_async_args = has_async_client ? ['-DGRPC_ENABLE_ASYNC'] : [] + +# The async generic *server* needs gRPC's generic callback API, which exists +# from 1.65 on; see transport/grpc/async_grpc_compat.h, which does the C++ +# version check. Higher floor than the async client above. +has_async_server = grpc_dep.version().version_compare('>=1.65.0') abseil_sync_dep = dependency('absl_synchronization') fs = import('fs') @@ -81,9 +93,11 @@ arrow_flight_srcs = [ 'client_cookie_middleware.cc', 'client_tracing_middleware.cc', 'cookie_internal.cc', + 'flight_data_decoder.cc', 'middleware.cc', 'serialization_internal.cc', 'server.cc', + 'server_async.cc', 'server_auth.cc', 'server_tracing_middleware.cc', 'transport.cc', @@ -98,6 +112,24 @@ arrow_flight_srcs = [ 'types.cc', ] +# The async generic server needs gRPC's generic callback API; see +# async_grpc_compat.h and the CMake equivalent. +if has_async_server + arrow_flight_srcs += [ + 'transport/grpc/async_grpc_service.cc', + 'transport/grpc/async_grpc_do_get.cc', + 'transport/grpc/async_grpc_do_put.cc', + 'transport/grpc/async_grpc_exchange.cc', + 'transport/grpc/async_grpc_exchange_reader.cc', + 'transport/grpc/async_grpc_exchange_writer.cc', + 'transport/grpc/async_grpc_handshake.cc', + 'transport/grpc/async_grpc_streaming.cc', + 'transport/grpc/async_grpc_unary.cc', + ] +endif + + + thread_dep = dependency('threads') arrow_flight = library( @@ -117,8 +149,8 @@ arrow_flight = library( abseil_sync_dep, thread_dep, ], - cpp_shared_args: ['-DARROW_FLIGHT_EXPORTING'], - cpp_static_args: ['-DARROW_FLIGHT_STATIC'], + cpp_shared_args: ['-DARROW_FLIGHT_EXPORTING'] + grpc_enable_async_args, + cpp_static_args: ['-DARROW_FLIGHT_STATIC'] + grpc_enable_async_args, gnu_symbol_visibility: 'inlineshidden', ) @@ -166,6 +198,29 @@ foreach flight_test : flight_tests test(test_name, exc) endforeach +if has_async_server + async_grpc_test = executable( + 'flight-async-grpc-test', + sources: [ + 'transport/grpc/async_grpc_service_test.cc', + 'transport/grpc/async_grpc_do_get_test.cc', + 'transport/grpc/async_grpc_do_put_test.cc', + 'transport/grpc/async_grpc_unary_test.cc', + 'transport/grpc/async_grpc_streaming_test.cc', + 'transport/grpc/async_grpc_exchange_test.cc', + 'transport/grpc/async_grpc_handshake_test.cc', + # these transitively include Flight.grpc.pb.h via the fixtures + # header, so we must declare it here to avoid a race condition + flight_proto_grpc_files[1], + ], + cpp_args: grpc_enable_async_args, + dependencies: [arrow_test_dep, arrow_flight_test_dep], + ) + # The timeout bounds the tests that would otherwise hang the suite instead + # of failing when a cancel/finish path regresses. + test('flight-async-grpc-test', async_grpc_test, timeout: 300) +endif + flight_test_dep_no_main = [ arrow_dep, arrow_flight_test_dep, diff --git a/cpp/src/arrow/flight/serialization_internal.h b/cpp/src/arrow/flight/serialization_internal.h index 896ca5b67567..b4f6a3f0e69b 100644 --- a/cpp/src/arrow/flight/serialization_internal.h +++ b/cpp/src/arrow/flight/serialization_internal.h @@ -180,6 +180,7 @@ ARROW_FLIGHT_EXPORT Status ToProto(const CloseSessionRequest& request, ARROW_FLIGHT_EXPORT Status ToProto(const CloseSessionResult& result, pb::CloseSessionResult* pb_result); +ARROW_FLIGHT_EXPORT Status ToPayload(const FlightDescriptor& descr, std::shared_ptr* out); /// \brief Serialize a FlightPayload to a vector of buffers. diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index da66d704c56c..1bb50ec191aa 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -43,8 +43,7 @@ namespace flight { /// Server implementation. Manages the lifecycle of the "real" server /// (ServerTransport) and contains struct FlightServerBase::Impl { - std::unique_ptr transport_; - internal::ServerSignalState signal_state_; + internal::ServerLifecycle lifecycle{"FlightServerBase"}; }; FlightServerOptions::FlightServerOptions(const Location& location_) @@ -67,63 +66,29 @@ Status FlightServerBase::Init(const FlightServerOptions& options) { flight::transport::grpc::InitializeFlightGrpcServer(); const auto scheme = options.location.scheme(); - ARROW_ASSIGN_OR_RAISE(impl_->transport_, + ARROW_ASSIGN_OR_RAISE(auto transport, internal::GetDefaultTransportRegistry()->MakeServer( scheme, this, options.memory_manager)); - ARROW_ASSIGN_OR_RAISE(auto uri, options.location.uri()); - return impl_->transport_->Init(options, uri); + return impl_->lifecycle.Init(options, std::move(transport)); } -int FlightServerBase::port() const { return location().port(); } +int FlightServerBase::port() const { return impl_->lifecycle.port(); } -Location FlightServerBase::location() const { return impl_->transport_->location(); } +Location FlightServerBase::location() const { return impl_->lifecycle.location(); } Status FlightServerBase::SetShutdownOnSignals(const std::vector sigs) { - return impl_->signal_state_.SetShutdownOnSignals(sigs); + return impl_->lifecycle.SetShutdownOnSignals(sigs); } -Status FlightServerBase::Serve() { - // Check before entering the signal helper: a failing wait() callback would - // otherwise leave the process signal handlers overridden, and clearing the - // signal target on the error path would disable signals for another server. - if (!impl_->transport_) { - return Status::UnknownError("Server did not start properly"); - } - return impl_->signal_state_.Serve( - [this]() -> Status { return impl_->transport_->Wait(); }, - [this](const std::chrono::system_clock::time_point* deadline) -> Status { - if (deadline) { - return impl_->transport_->Shutdown(*deadline); - } - return impl_->transport_->Shutdown(); - }, - "Error shutting down server"); -} +Status FlightServerBase::Serve() { return impl_->lifecycle.Serve(); } -int FlightServerBase::GotSignal() const { return impl_->signal_state_.GotSignal(); } +int FlightServerBase::GotSignal() const { return impl_->lifecycle.GotSignal(); } Status FlightServerBase::Shutdown(const std::chrono::system_clock::time_point* deadline) { - if (!impl_->transport_) { - return Status::Invalid("Shutdown() on uninitialized FlightServerBase"); - } - return impl_->signal_state_.Shutdown( - [this](const std::chrono::system_clock::time_point* maybe_deadline) -> Status { - if (maybe_deadline) { - return impl_->transport_->Shutdown(*maybe_deadline); - } - return impl_->transport_->Shutdown(); - }, - deadline); + return impl_->lifecycle.Shutdown(deadline); } -Status FlightServerBase::Wait() { - return impl_->signal_state_.Wait([this] { - if (!impl_->transport_) { - return Status::Invalid("Wait() on uninitialized FlightServerBase"); - } - return impl_->transport_->Wait(); - }); -} +Status FlightServerBase::Wait() { return impl_->lifecycle.Wait(); } Status FlightServerBase::ListFlights(const ServerCallContext& context, const Criteria* criteria, diff --git a/cpp/src/arrow/flight/server_async.cc b/cpp/src/arrow/flight/server_async.cc new file mode 100644 index 000000000000..19529c79479a --- /dev/null +++ b/cpp/src/arrow/flight/server_async.cc @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Standalone async Flight server implementation on top of the transport +// interface. + +#include "arrow/flight/server_async.h" + +#include +#include +#include +#include +#include + +#include "arrow/flight/transport.h" +#include "arrow/flight/transport/grpc/grpc_server.h" +#include "arrow/flight/transport_server.h" +#include "arrow/flight/transport_server_internal.h" +#include "arrow/flight/types.h" + +namespace arrow::flight { + +AsyncFlightDataStream::~AsyncFlightDataStream() = default; +AsyncFlightListing::~AsyncFlightListing() = default; +AsyncResultStream::~AsyncResultStream() = default; +AsyncFlightMessageReader::~AsyncFlightMessageReader() = default; +AsyncFlightMessageWriter::~AsyncFlightMessageWriter() = default; + +struct AsyncGenericFlightServerBase::Impl { + internal::ServerLifecycle lifecycle{"AsyncGenericFlightServerBase"}; +}; + +AsyncGenericFlightServerBase::AsyncGenericFlightServerBase() : impl_(new Impl) {} + +AsyncGenericFlightServerBase::~AsyncGenericFlightServerBase() = default; + +Status AsyncGenericFlightServerBase::Init(const FlightServerOptions& options) { + flight::transport::grpc::InitializeFlightGrpcServer(); + + const auto scheme = options.location.scheme(); + ARROW_ASSIGN_OR_RAISE(auto transport, + internal::GetDefaultTransportRegistry()->MakeServerAsync( + scheme, this, options.memory_manager)); + return impl_->lifecycle.Init(options, std::move(transport)); +} + +int AsyncGenericFlightServerBase::port() const { return impl_->lifecycle.port(); } + +Location AsyncGenericFlightServerBase::location() const { + return impl_->lifecycle.location(); +} + +Status AsyncGenericFlightServerBase::SetShutdownOnSignals( + const std::vector signals) { + return impl_->lifecycle.SetShutdownOnSignals(signals); +} + +Status AsyncGenericFlightServerBase::Serve() { return impl_->lifecycle.Serve(); } + +int AsyncGenericFlightServerBase::GotSignal() const { + return impl_->lifecycle.GotSignal(); +} + +Status AsyncGenericFlightServerBase::Shutdown( + const std::chrono::system_clock::time_point* deadline) { + return impl_->lifecycle.Shutdown(deadline); +} + +Status AsyncGenericFlightServerBase::Wait() { return impl_->lifecycle.Wait(); } + +// --- handler defaults: the same answers FlightServerBase's defaults give --- + +arrow::Future> +AsyncGenericFlightServerBase::ListFlightsAsync(const ServerCallContext&, + const Criteria*) { + return arrow::Future>::MakeFinished( + Status::NotImplemented("NYI")); +} + +arrow::Future> +AsyncGenericFlightServerBase::GetFlightInfoAsync(const ServerCallContext&, + const FlightDescriptor&) { + return arrow::Future>::MakeFinished( + Status::NotImplemented("NYI")); +} + +arrow::Future> +AsyncGenericFlightServerBase::PollFlightInfoAsync(const ServerCallContext&, + const FlightDescriptor&) { + return arrow::Future>::MakeFinished( + Status::NotImplemented("NYI")); +} + +arrow::Future> AsyncGenericFlightServerBase::GetSchemaAsync( + const ServerCallContext&, const FlightDescriptor&) { + return arrow::Future>::MakeFinished( + Status::NotImplemented("NYI")); +} + +arrow::Future> +AsyncGenericFlightServerBase::DoActionAsync(const ServerCallContext&, const Action&) { + return arrow::Future>::MakeFinished( + Status::NotImplemented("NYI")); +} + +arrow::Future> AsyncGenericFlightServerBase::ListActionsAsync( + const ServerCallContext&) { + return arrow::Future>::MakeFinished( + Status::NotImplemented("NYI")); +} + +arrow::Future<> AsyncGenericFlightServerBase::DoExchangeAsync( + const ServerCallContext&, std::shared_ptr, + std::shared_ptr) { + return arrow::Future<>::MakeFinished(Status::NotImplemented("NYI")); +} + +arrow::Future> +AsyncGenericFlightServerBase::DoGetAsync(const ServerCallContext&, const Ticket&) { + return arrow::Future>::MakeFinished( + Status::NotImplemented("DoGetAsync is not implemented")); +} + +std::shared_ptr +AsyncGenericFlightServerBase::CreateDoPutListener(const ServerCallContext&) { + // The default refuses uploads, a server that accepts them overrides this. + return nullptr; +} + +Status AsyncGenericFlightServerBase::Handshake(const ServerCallContext&, + const std::string&, std::string*) { + return Status::NotImplemented( + "This service does not have an authentication mechanism enabled."); +} + +Status AsyncGenericFlightServerBase::ValidateToken(const ServerCallContext&, + const std::string&, std::string*) { + // Keep the transport-level (TLS) identity, ignore the token. + return Status::OK(); +} + +} // namespace arrow::flight diff --git a/cpp/src/arrow/flight/server_async.h b/cpp/src/arrow/flight/server_async.h new file mode 100644 index 000000000000..84532d6103d6 --- /dev/null +++ b/cpp/src/arrow/flight/server_async.h @@ -0,0 +1,381 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/flight/flight_data_decoder.h" +#include "arrow/flight/server.h" +#include "arrow/flight/types.h" +#include "arrow/util/future.h" +#include "arrow/util/macros.h" + +namespace arrow::flight { + +/// \brief Interface that produces a sequence of IPC payloads to be sent in +/// FlightData protobuf messages, where a payload may arrive later. +class ARROW_FLIGHT_EXPORT AsyncFlightDataStream { + public: + virtual ~AsyncFlightDataStream(); + + /// \brief The schema of the payloads this stream produces. + /// + /// The transport never calls this: the schema the client sees is the one + /// GetSchemaPayloadAsync() returns. + /// \return The schema of the payloads this stream produces. + virtual std::shared_ptr schema() = 0; + + /// \brief Like GetSchemaPayload(), but the payload arrives later. + /// + /// Called once, before any NextAsync() payload is requested. + /// \return A future completed with the schema payload. A non-OK status fails the RPC. + virtual arrow::Future GetSchemaPayloadAsync() = 0; + + /// \brief Like Next(), but the payload arrives later. + /// + /// The transport keeps one call in flight at a time and only asks again once + /// the previous payload has been written. + /// + /// \return A future completed with the next payload, or with a nullopt once the stream + /// is exhausted. A non-OK status fails the RPC. + virtual arrow::Future> NextAsync() = 0; + + /// \brief Stop the stream: the RPC it was producing for is over. + /// + /// Called at most once, possibly from any thread. A pending + /// NextAsync()/GetSchemaPayloadAsync() future must be resolved here (or + /// abandoned knowingly): the transport does not wait for it. + /// Its return status becomes the RPC's final status only on the normal end-of-stream + /// path (after NextAsync() reports end of stream). + /// On every other path (write failure, cancellation) the RPC already has a status and + /// the returned status is discarded. + /// \return The status of closing the stream. + virtual Status Close() { return Status::OK(); } +}; + +/// \brief An asynchronous iterator of FlightInfo instances, returned by +/// ListFlightsAsync. +/// +/// The transport pulls it one NextAsync() at a time and writes each +/// FlightInfo to the client as it becomes available. +class ARROW_FLIGHT_EXPORT AsyncFlightListing { + public: + virtual ~AsyncFlightListing(); + + /// \brief The next FlightInfo in the listing. + /// \return A future completed with the next FlightInfo, or with a nullptr + /// once the listing is exhausted. A non-OK status fails the RPC. + virtual arrow::Future> NextAsync() = 0; +}; + +/// \brief An asynchronous iterator of Result instances, returned by +/// DoActionAsync. +class ARROW_FLIGHT_EXPORT AsyncResultStream { + public: + virtual ~AsyncResultStream(); + + /// \brief The next Result in the stream. + /// \return A future completed with the next Result, or with a nullptr once + /// the stream is exhausted. A non-OK status fails the RPC. + virtual arrow::Future> NextAsync() = 0; +}; + +/// \brief The reader of one DoExchange: the messages the client sends, pulled +/// one NextAsync() at a time. +/// +/// Runs on the thread that calls it, at most one NextAsync() may be +/// outstanding. +/// Schema and dictionary messages are read through (they are not chunks), a metadata-only +/// message is a chunk with a null data member, and the chunk whose members are both null +/// ends the exchange. +/// The reader must not be used after the future returned by DoExchangeAsync completes. +class ARROW_FLIGHT_EXPORT AsyncFlightMessageReader { + public: + virtual ~AsyncFlightMessageReader(); + + /// \brief The descriptor of this exchange. + virtual const FlightDescriptor& descriptor() const = 0; + + /// \brief The next message of the exchange, decoded. + /// \return A future completed with the next chunk, both of its members are + /// null at the end of the exchange. A non-OK status fails the RPC. + virtual arrow::Future NextAsync() = 0; +}; + +/// \brief The writer of one DoExchange: the messages the server sends back. +/// +/// Runs on the thread that calls it, await each returned future before +/// writing again. The writer must not be used after the future returned by +/// DoExchangeAsync completes. +class ARROW_FLIGHT_EXPORT AsyncFlightMessageWriter { + public: + virtual ~AsyncFlightMessageWriter(); + + /// \brief Start the writer with the schema of the messages it will write. + /// Must be called before any record batch is written. + /// \param schema The schema of the messages to be written. + /// \return A future that completes once the writer is ready to accept messages. + virtual arrow::Future<> BeginAsync(std::shared_ptr schema) = 0; + + /// \brief Write one record batch. + /// \param batch The record batch to write. + /// \return A future that completes once the batch has been written. + virtual arrow::Future<> WriteRecordBatchAsync(const RecordBatch& batch) = 0; + + /// \brief Write one application-metadata-only message. + /// \param app_metadata The application metadata to write. + /// \return A future that completes once the metadata has been written. + virtual arrow::Future<> WriteMetadataAsync(std::shared_ptr app_metadata) = 0; + + /// \brief Write one record batch with application metadata attached. + /// \param batch The record batch to write. + /// \param app_metadata The application metadata to attach. + /// \return A future that completes once the batch and metadata have been written. + virtual arrow::Future<> WriteWithMetadataAsync( + const RecordBatch& batch, std::shared_ptr app_metadata) = 0; + + /// \brief Finish the write side: end the IPC stream. + /// + /// Call once, after the last message, then await it before completing + /// DoExchangeAsync's future. + /// \return A future that completes once the write side has been closed. + virtual arrow::Future<> CloseAsync() = 0; +}; + +/// \brief a Flight server served by the async generic gRPC +/// callback service. +/// +/// Derive from this class alone and implement the handlers you serve: +/// +/// \code +/// class MyServer : public arrow::flight::AsyncGenericFlightServerBase { +/// arrow::Future> GetFlightInfoAsync( +/// const arrow::flight::ServerCallContext&, +/// const arrow::flight::FlightDescriptor&) override; +/// +/// arrow::Future> DoGetAsync( +/// const arrow::flight::ServerCallContext&, +/// const arrow::flight::Ticket&) override; +/// }; +/// \endcode +/// +/// Every handler returns a future: complete it from your own thread or pool +/// when the answer is ready, and no gRPC callback thread is ever held waiting +/// for one. Handlers that do run inline should still return promptly: blocking +/// a callback thread starves new connections. +/// +/// All handlers must be non-blocking and return promptly. +/// Non overriding handlers will return `Status::NotImplemented` by default. +/// +/// Every request argument a handler receives (`Ticket`, `FlightDescriptor`, +/// `Criteria`, `Action`) is only valid for the duration of that handler call: +/// copy what you need before completing the returned future from another +/// thread. The `ServerCallContext` handed to a handler is likewise the call's +/// own context and must not outlive it. +/// +/// Authentication goes through Handshake()/ValidateToken(): a blocking +/// ServerAuthHandler in FlightServerOptions is not supported, and Init() +/// rejects it. +class ARROW_FLIGHT_EXPORT AsyncGenericFlightServerBase { + public: + AsyncGenericFlightServerBase(); + virtual ~AsyncGenericFlightServerBase(); + + /// \brief Initialize the server listening at the given location. + /// Must be called before any other method. + /// \param options The server options to use for initialization. + /// \return Status indicating success or failure of the initialization. + Status Init(const FlightServerOptions& options); + + /// \brief Get the port that the Flight server is listening on. + /// + /// This method must only be called after Init(). Will return a non-positive + /// value if no port exists (e.g. when listening on a domain socket). + /// \return The port number the server is listening on, or a non-positive value. + int port() const; + + /// \return The address the server is listening on. + /// \warning Only valid after Init() has been called. + Location location() const; + + /// \brief Set the server to stop on any of the given signal numbers. + /// \warning Must be called before Serve(). + /// \param signals The signal numbers on which the server should shut down. + /// \return Status indicating success or failure of setting the shutdown signals. + Status SetShutdownOnSignals(const std::vector signals); + + /// \brief Start serving. Blocks until the server shuts down, which happens + /// when Shutdown() is called or a registered signal arrives. + /// \return Status indicating success or failure of starting the server. + Status Serve(); + + /// \return The signal number that interrupted Serve(), or 0. + /// \note Only valid after Serve() has returned. + int GotSignal() const; + + /// \brief Shut down the server, blocking until in-flight RPCs finish. + /// May be called from another thread or a signal handler while Serve() + /// blocks, with an optional deadline. + /// + /// An RPC whose handler never completes its future is not finished, so this + /// blocks until that future resolves: a handler that abandons its future + /// (see the per-handler notes on the transport not waiting for it) needs a + /// deadline here, or the future must be given up on before shutdown. + /// \param[in] deadline is an optional time point by which the server should complete + /// shutdown. + /// \return Status indicating success or failure of the shutdown operation. + /// \warning Call at most once. + Status Shutdown(const std::chrono::system_clock::time_point* deadline = NULLPTR); + + /// \brief Block until the server shuts down. Does not respond to signals. + /// + /// Like Shutdown(), this does not return while a handler future on a live RPC + /// is still unresolved. + Status Wait(); + + /// \brief Asynchronously retrieve a list of available fields given an + /// optional opaque criteria. + /// \param[in] context is the server call context. + /// \param[in] criteria is the optional opaque criteria for listing flights. + /// \see FlightServerBase::ListFlights + /// + /// \return A future completed with the listing, or with a non-OK status to + /// fail the RPC. + /// A future completed with a nullptr listing serves an empty listing. + /// The transport pulls the listing one NextAsync() at a time, the criteria pointer is + /// never null and is only valid for the duration of the call. + virtual arrow::Future> ListFlightsAsync( + const ServerCallContext& context, const Criteria* criteria); + + /// \brief Asynchronously retrieve the schema and an access plan for the + /// indicated descriptor. + /// \see FlightServerBase::GetFlightInfo + /// \param[in] context is the server call context. + /// \param[in] request is the flight descriptor for which to retrieve the info. + /// + /// \return A future completed with the info, or with a non-OK status to + /// fail the RPC. A `nullptr` info answers `KeyError("Flight not found")`, as + /// a synchronous server that returns OK without setting the response does. + virtual arrow::Future> GetFlightInfoAsync( + const ServerCallContext& context, const FlightDescriptor& request); + + /// \brief Asynchronously retrieve the current status of the target query. + /// \param[in] context is the server call context. + /// \param[in] request is the flight descriptor for which to retrieve the status. + /// \see FlightServerBase::PollFlightInfo. + /// \return A future completed with the info, or with a non-OK status to + /// fail the RPC. A nullptr info answers `KeyError("Flight not found")`. + virtual arrow::Future> PollFlightInfoAsync( + const ServerCallContext& context, const FlightDescriptor& request); + + /// \brief Asynchronously retrieve the schema for the indicated descriptor. + /// \param[in] context is the server call context. + /// \param[in] request is the flight descriptor for which to retrieve the schema. + /// \see FlightServerBase::GetSchema. + /// \return A future completed with the schema, or with a non-OK status to + /// fail the RPC. + /// A `nullptr` schema answers `KeyError("Flight not found")`. + virtual arrow::Future> GetSchemaAsync( + const ServerCallContext& context, const FlightDescriptor& request); + + /// \brief Asynchronously execute an action, returning a stream of zero or + /// more results. + /// \param[in] context is the server call context. + /// \param[in] action is the action to execute. + /// \see FlightServerBase::DoAction. + /// \return A future completed with the result stream, or with a non-OK + /// status to fail the RPC. A future completed with a `nullptr` stream + /// answers CANCELLED, as a synchronous server that returns OK without a + /// stream does. + /// The transport pulls the stream one NextAsync() at a time. + virtual arrow::Future> DoActionAsync( + const ServerCallContext& context, const Action& action); + + /// \brief Asynchronously retrieve the list of available actions. + /// \param[in] context is the server call context. + /// \see FlightServerBase::ListActions. + virtual arrow::Future> ListActionsAsync( + const ServerCallContext& context); + + /// \brief Asynchronously process a bidirectional stream of IPC payloads. + /// \see FlightServerBase::DoExchange. + /// \param[in] context is the server call context. + /// \param[in] reader is the asynchronous reader for the client's messages. + /// \param[in] writer is the asynchronous writer for the server's messages. + /// \return A future completed when the exchange is over, its status is the + /// RPC's status. + /// While it runs, consume the client's messages through the reader and send the + /// server's through the writer. Both must not be used after the future completes. + virtual arrow::Future<> DoExchangeAsync( + const ServerCallContext& context, std::shared_ptr reader, + std::shared_ptr writer); + + /// \brief The stream serving one DoGet RPC. + /// + /// Called after the ticket is parsed, the request is only valid for the + /// duration of the call. + /// \param[in] context is the server call context. + /// \param[in] request is an opaque ticket. + /// \return A future completed with the stream to serve, or with a non-OK + /// status to fail the RPC. + /// A future completed with a `nullptr` stream answers `KeyError("No data in this + /// flight")`. Complete the future from your ownthread or pool: the transport writes + /// whatever stream it resolves to, and must not be blocked waiting for it. + /// + /// If the RPC is cancelled while the returned future is pending, the + /// transport finishes the RPC without waiting for it, and the stream it + /// eventually resolves to is dropped without Close() ever being delivered + /// (the transport no longer holds the stream). + virtual arrow::Future> DoGetAsync( + const ServerCallContext& context, const Ticket& request); + + /// \brief Create the AsyncFlightDataListener serving one DoPut RPC. + /// \see AsyncFlightDataListener + /// \param[in] context is the server call context. + /// \return A shared pointer to the AsyncFlightDataListener handling the upload, or + /// `nullptr` to refuse the upload. + virtual std::shared_ptr CreateDoPutListener( + const ServerCallContext& context); + + /// \brief Handle the handshake protocol with the client. + /// \param[in] context is the server call context. + /// \param[in] request is the client's handshake message. + /// \param[out] response is where the server writes its handshake response. + /// \return Status indicating success or failure of the handshake. + virtual Status Handshake(const ServerCallContext& context, const std::string& request, + std::string* response); + + /// \brief Validate the token sent in the `auth-token-bin` header of RPCs + /// issued after a successful Handshake(). + /// \param[in] context is the server call context. + /// \param[in] token is the authentication token sent by the client. + /// \param[out] peer_identity is set to the authenticated identity on success. + /// \return Status indicating success or failure of the token validation. + virtual Status ValidateToken(const ServerCallContext& context, const std::string& token, + std::string* peer_identity); + + private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace arrow::flight diff --git a/cpp/src/arrow/flight/transport.cc b/cpp/src/arrow/flight/transport.cc index fd74b6d95a59..b14b2735bd4c 100644 --- a/cpp/src/arrow/flight/transport.cc +++ b/cpp/src/arrow/flight/transport.cc @@ -136,6 +136,15 @@ class TransportRegistry::Impl final { } return it->second(base, std::move(memory_manager)); } + arrow::Result> MakeServerAsync( + const std::string& scheme, AsyncGenericFlightServerBase* async_base, + std::shared_ptr memory_manager) const { + auto it = async_server_factories_.find(scheme); + if (it == async_server_factories_.end()) { + return Status::KeyError("No async server transport implementation for ", scheme); + } + return it->second(async_base, std::move(memory_manager)); + } Status RegisterClient(const std::string& scheme, ClientFactory factory) { auto it = client_factories_.insert({scheme, std::move(factory)}); if (!it.second) { @@ -150,10 +159,19 @@ class TransportRegistry::Impl final { } return Status::OK(); } + Status RegisterAsyncServer(const std::string& scheme, AsyncServerFactory factory) { + auto it = async_server_factories_.insert({scheme, std::move(factory)}); + if (!it.second) { + return Status::Invalid("Async server transport already registered for ", scheme); + } + return Status::OK(); + } private: std::unordered_map client_factories_; std::unordered_map server_factories_; + std::unordered_map + async_server_factories_; }; TransportRegistry::TransportRegistry() { impl_ = std::make_unique(); } @@ -167,6 +185,11 @@ arrow::Result> TransportRegistry::MakeServer( std::shared_ptr memory_manager) const { return impl_->MakeServer(scheme, base, std::move(memory_manager)); } +arrow::Result> TransportRegistry::MakeServerAsync( + const std::string& scheme, AsyncGenericFlightServerBase* async_base, + std::shared_ptr memory_manager) const { + return impl_->MakeServerAsync(scheme, async_base, std::move(memory_manager)); +} Status TransportRegistry::RegisterClient(const std::string& scheme, ClientFactory factory) { return impl_->RegisterClient(scheme, std::move(factory)); @@ -175,6 +198,10 @@ Status TransportRegistry::RegisterServer(const std::string& scheme, ServerFactory factory) { return impl_->RegisterServer(scheme, std::move(factory)); } +Status TransportRegistry::RegisterAsyncServer(const std::string& scheme, + AsyncServerFactory factory) { + return impl_->RegisterAsyncServer(scheme, std::move(factory)); +} TransportRegistry* GetDefaultTransportRegistry() { static TransportRegistry kRegistry; diff --git a/cpp/src/arrow/flight/transport.h b/cpp/src/arrow/flight/transport.h index 6b9694318915..4f30afc7927d 100644 --- a/cpp/src/arrow/flight/transport.h +++ b/cpp/src/arrow/flight/transport.h @@ -72,6 +72,7 @@ namespace ipc { class Message; } namespace flight { +class AsyncGenericFlightServerBase; class FlightStatusDetail; namespace internal { @@ -216,6 +217,9 @@ class ARROW_FLIGHT_EXPORT TransportRegistry { using ClientFactory = std::function>()>; using ServerFactory = std::function>( FlightServerBase*, std::shared_ptr memory_manager)>; + using AsyncServerFactory = + std::function>( + AsyncGenericFlightServerBase*, std::shared_ptr memory_manager)>; TransportRegistry(); ~TransportRegistry(); @@ -226,8 +230,13 @@ class ARROW_FLIGHT_EXPORT TransportRegistry { const std::string& scheme, FlightServerBase* base, std::shared_ptr memory_manager) const; + arrow::Result> MakeServerAsync( + const std::string& scheme, AsyncGenericFlightServerBase* async_base, + std::shared_ptr memory_manager) const; + Status RegisterClient(const std::string& scheme, ClientFactory factory); Status RegisterServer(const std::string& scheme, ServerFactory factory); + Status RegisterAsyncServer(const std::string& scheme, AsyncServerFactory factory); private: class Impl; diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_compat.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_compat.h new file mode 100644 index 000000000000..1647c3895dc5 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_compat.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +// Version gating for the async Flight server, which is built on gRPC's generic +// callback API (grpc::CallbackGenericService, grpc::GenericCallbackServerContext +// and grpc::ServerGenericBidiReactor). +// +// That API appeared in gRPC 1.65. One header covers every supported version: +// grpcpp/generic/async_generic_service.h declares the callback API itself in +// 1.65, and from 1.66 on it just includes +// grpcpp/generic/callback_generic_service.h (which does not exist before 1.66, +// so including it directly would break a 1.65 build). +// +// ARROW_FLIGHT_HAS_ASYNC_SERVER hides that detail from the rest of the +// transport: when it is 0 the async server is not compiled and the gRPC +// transport answers async server requests with NotImplemented. +// +// GRPC_CPP_VERSION_CHECK comes from util_internal.h and evaluates to 0 for +// gRPC releases predating the version macros, which covers the older system +// gRPC packages some builds link against. + +#include "arrow/flight/transport/grpc/util_internal.h" + +#if GRPC_CPP_VERSION_CHECK(1, 65, 0) +# define ARROW_FLIGHT_HAS_ASYNC_SERVER 1 +# include +#else +# define ARROW_FLIGHT_HAS_ASYNC_SERVER 0 +#endif diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc new file mode 100644 index 000000000000..1f5fd6a18bd0 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/transport/grpc/async_grpc_service_internal.h" + +#include +#include +#include +#include +#include + +namespace arrow::flight::transport::grpc::detail { + +namespace { + +/// Serve one DoGet RPC over the generic callback API. +class DoGetReactor : public AsyncReactorBase { + public: + DoGetReactor(AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) + : AsyncReactorBase(std::move(flight_context)), base_(base) { + StartRead(&request_buf_); + } + + void OnReadDone(bool ok) override { + if (!ok) { + FinishOnce(MakeFlightError(FlightStatusCode::Internal, "Failed to read request")); + return; + } + + const auto ticket = ParseTicket(); + if (!ticket.ok()) { + FinishOnce(ticket.status()); + return; + } + + arrow::Future> future = + base_->DoGetAsync(flight_context(), *ticket); + future.AddCallback( + [this, token = hold()]( + const arrow::Result>& result) { + if (!finished()) { + if (!result.ok()) { + FinishOnce(result.status()); + } else { + std::shared_ptr stream = result.ValueOrDie(); + if (stream == nullptr) { + FinishOnce(arrow::Status::KeyError("No data in this flight")); + } else { + bool already_closed; + { + std::lock_guard lock(stream_mutex_); + async_data_stream_ = stream; + already_closed = closed_; + } + if (already_closed) { + ARROW_WARN_NOT_OK(stream->Close(), + "DoGet: closing a cancelled call's stream failed"); + } else { + WriteNextPayload(); + } + } + } + } + }); + } + + void OnWriteDone(bool ok) override { + if (finished()) { + return; + } + if (!ok) { + FinishOnce(MakeFlightError(FlightStatusCode::Internal, "Write failed")); + ARROW_WARN_NOT_OK(CloseStreamOnce(), + "DoGet: closing the data stream after a failed write failed"); + return; + } + WriteNextPayload(); + } + + void OnCancel() override { + FinishOnce(arrow::Status::Cancelled()); + ARROW_WARN_NOT_OK(CloseStreamOnce(), + "DoGet: closing the stream of a cancelled call failed"); + } + + private: + arrow::Result ParseTicket() { + return ParseProtoRequest(request_buf_, "Ticket"); + } + + Status CloseStreamOnce() { + std::shared_ptr stream; + { + std::lock_guard lock(stream_mutex_); + if (closed_) { + return Status::OK(); + } + closed_ = true; + stream = async_data_stream_; + } + if (stream == nullptr) { + return Status::OK(); + } + return stream->Close(); + } + + void WriteNextPayload() { + if (finished()) { + return; + } + HoldToken token = hold(); + + arrow::Future> next; + if (!wrote_schema_) { + next = async_data_stream_->GetSchemaPayloadAsync().Then( + [](FlightPayload payload) -> std::optional { + return {std::move(payload)}; + }); + } else { + next = async_data_stream_->NextAsync(); + } + next.AddCallback([this, token = token.release()]( + arrow::Result> result) { + if (!finished()) { + if (!result.ok()) { + FinishOnce(result.status()); + ARROW_WARN_NOT_OK( + CloseStreamOnce(), + "DoGet: closing the data stream after a failed payload failed"); + } else { + wrote_schema_ = true; + std::optional payload = std::move(*result); + if (!payload.has_value()) { + FinishOnce(CloseStreamOnce()); + } else { + bool own_buffer = false; + const ::grpc::Status grpc_status = + FlightDataSerialize(*payload, &write_buf_, &own_buffer); + if (!grpc_status.ok()) { + FinishOnce(MakeFlightError(FlightStatusCode::Internal, + grpc_status.error_message())); + ARROW_WARN_NOT_OK( + CloseStreamOnce(), + "DoGet: closing the data stream after a failed serialization failed"); + } else { + WriteIfLive([this] { StartWrite(&write_buf_); }); + } + } + } + } + }); + } + + AsyncGenericFlightServerBase* base_; + ::grpc::ByteBuffer request_buf_; + ::grpc::ByteBuffer write_buf_; + std::shared_ptr async_data_stream_; + bool wrote_schema_ = false; + /// Guards async_data_stream_ and closed_, so adopting the stream and the cancellation + /// teardown cannot interleave. + std::mutex stream_mutex_; + bool closed_ = false; +}; + +} // namespace + +::grpc::ServerGenericBidiReactor* MakeDoGetReactor(AsyncCallContext flight_context, + AsyncGenericFlightServerBase* base) { + return new DoGetReactor(std::move(flight_context), base); +} + +} // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc new file mode 100644 index 000000000000..1bed2683febe --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc @@ -0,0 +1,329 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/transport/grpc/async_grpc_test_fixtures.h" + +namespace arrow::flight::transport::grpc { + +TEST(AsyncGrpcTest, BasicDoGet) { + TestFlightServer flight_server; + AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); + + int port = 0; + ::grpc::ServerBuilder builder; + builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.RegisterCallbackGenericService(&service); + auto server = builder.BuildAndStart(); + ASSERT_NE(server, nullptr); + ASSERT_NE(port, 0); + + // Connect the existing arrow::flight::FlightClient implementation. + std::string uri = "grpc://localhost:" + std::to_string(port); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + + // Call DoGet: the ticket must reach TestFlightServer::DoGetAsync. + Ticket ticket{"basic-do-get-ticket"}; + ASSERT_OK_AND_ASSIGN(auto reader, client->DoGet(ticket)); + + // Read schema. + ASSERT_OK_AND_ASSIGN(auto schema, reader->GetSchema()); + ASSERT_EQ(schema->num_fields(), 2); + ASSERT_EQ(schema->field(0)->name(), "a"); + ASSERT_EQ(schema->field(1)->name(), "b"); + + // Read batches. + int batch_count = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto chunk, reader->Next()); + if (!chunk.data) break; + ASSERT_EQ(chunk.data->num_rows(), 5); + ASSERT_OK_AND_ASSIGN(auto value, chunk.data->column(0)->GetScalar(0)); + ASSERT_TRUE(value->Equals(*arrow::MakeScalar(int64_t(1)))); + batch_count++; + } + ASSERT_EQ(batch_count, 3); + + // Cleanup. The ticket is asserted only after the server stopped, so the + // DoGet callback cannot race with the read. + ASSERT_OK(client->Close()); + server->Shutdown(); + server->Wait(); + + ASSERT_EQ(flight_server.ticket(), "basic-do-get-ticket"); +} + +TEST(AsyncGrpcTest, SchemaOnlyDoGet) { + TestFlightServer flight_server; + AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); + + int port = 0; + ::grpc::ServerBuilder builder; + builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.RegisterCallbackGenericService(&service); + auto server = builder.BuildAndStart(); + ASSERT_NE(server, nullptr); + ASSERT_NE(port, 0); + + std::string uri = "grpc://localhost:" + std::to_string(port); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + + // This ticket is served as a stream with no batches at all. + Ticket ticket{kSchemaOnlyTicket}; + ASSERT_OK_AND_ASSIGN(auto reader, client->DoGet(ticket)); + + // The schema is served... + ASSERT_OK_AND_ASSIGN(auto schema, reader->GetSchema()); + ASSERT_EQ(schema->num_fields(), 2); + ASSERT_EQ(schema->field(0)->name(), "a"); + ASSERT_EQ(schema->field(1)->name(), "b"); + + // ...and the very first read reports the end of stream instead of failing. + ASSERT_OK_AND_ASSIGN(auto chunk, reader->Next()); + ASSERT_EQ(chunk.data, nullptr); + + // Cleanup. The ticket is asserted only after the server stopped, so the + // DoGet callback cannot race with the read. + ASSERT_OK(client->Close()); + server->Shutdown(); + server->Wait(); + + ASSERT_EQ(flight_server.ticket(), kSchemaOnlyTicket); +} + +// A DoGet stream whose first payload never arrives: the producer is parked +// forever, so only a client cancel can ever end the RPC. `close_count` and +// `next_awaits` let the test observe, from the test thread, that the payload +// demand is live and that Close() was delivered exactly once. +class HangingDoGetStream final : public AsyncFlightDataStream { + public: + HangingDoGetStream(std::shared_ptr resolved, + std::atomic* next_awaits, std::atomic* close_count) + : resolved_(std::move(resolved)), + next_awaits_(next_awaits), + close_count_(close_count) {} + + std::shared_ptr schema() override { return resolved_->schema(); } + + arrow::Future GetSchemaPayloadAsync() override { + return resolved_->GetSchemaPayloadAsync(); + } + + arrow::Future> NextAsync() override { + next_awaits_->fetch_add(1); + std::lock_guard lock(mutex_); + pending_ = arrow::Future>::Make(); + return pending_; + } + + // The transport's stop signal: resolve the pending payload so the producer + // does not park forever (mirrors the example's TimedAsyncStream::Close). + Status Close() override { + close_count_->fetch_add(1); + std::lock_guard lock(mutex_); + if (pending_.is_valid() && !pending_.is_finished()) { + pending_.MarkFinished(arrow::Status::Cancelled()); + } + return resolved_->Close(); + } + + private: + std::shared_ptr resolved_; + std::atomic* next_awaits_; + std::atomic* close_count_; + std::mutex mutex_; + arrow::Future> pending_; +}; + +// DoGet server with a configurable hang: kPrepare leaves DoGetAsync() pending +// (no stream ever exists), kStream resolves the stream but leaves the first +// payload pending. +class HangingDoGetServer final : public AsyncGenericFlightServerBase { + public: + enum class HangAt { kPrepare, kStream }; + explicit HangingDoGetServer(HangAt hang) : hang_(hang) {} + + arrow::Future> DoGetAsync( + const ServerCallContext&, const Ticket&) override { + if (hang_ == HangAt::kPrepare) { + return arrow::Future>::Make(); + } + auto reader = arrow::RecordBatchReader::Make( + {}, arrow::schema({arrow::field("value", arrow::int64())})) + .ValueOrDie(); + return arrow::Future>::MakeFinished( + std::make_shared( + std::make_shared(std::move(reader)), &next_awaits, + &stream_closes)); + } + + std::atomic next_awaits{0}; + std::atomic stream_closes{0}; + + private: + HangAt hang_; +}; + +TEST(AsyncGrpcTest, DoGetClientCancelWithPendingPayloadFinishes) { + // The client gives up while the producer's NextAsync() is pending: the + // transport must finish the RPC itself and deliver Close() to the producer, + // instead of waiting for a payload that will not come. + const int64_t before = detail::AsyncReactorBase::live_reactors(); + auto trace = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + HangingDoGetServer flight_server(HangingDoGetServer::HangAt::kStream); + FlightServerOptions options(location); + options.middleware = { + {"recording", std::make_shared(trace)}}; + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + // DoGet returns once the schema payload is written,the payload after it is + // the one that never comes. Wait until the server has actually asked for + // it, so the cancel lands on a live demand (not a race with the schema + // write). + ASSERT_OK_AND_ASSIGN(auto reader, client->DoGet(Ticket{"hang"})); + ASSERT_TRUE(WaitFor([&flight_server] { return flight_server.next_awaits.load() >= 1; })) + << "the server never asked for its first payload"; + + reader->Cancel(); + reader.reset(); + ASSERT_OK(client->Close()); + + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + // The producer was told to stop, exactly once. + EXPECT_EQ(1, flight_server.stream_closes.load()); + // And the reactor is gone even though its payload future never completed. + EXPECT_TRUE(WaitFor([&] { + return detail::AsyncReactorBase::live_reactors() == before; + })) << "a reactor leaked: still alive after the server stopped (before=" + << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; + // And the call reported completion exactly once (the CAS in FinishOnce). + std::lock_guard guard(trace->mutex); + EXPECT_EQ(1, trace->call_completed); +} + +TEST(AsyncGrpcTest, DoGetClientDeadlineWhilePreparingFinishes) { + // DoGetAsync() never resolves and the client's deadline expires: the RPC + // must end anyway: the Shutdown()/Wait() below would otherwise block. No + // stream exists here, so Close() has nothing to deliver and the producer + // notices through ServerCallContext::is_cancelled(). + auto trace = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + HangingDoGetServer flight_server(HangingDoGetServer::HangAt::kPrepare); + FlightServerOptions options(location); + options.middleware = { + {"recording", std::make_shared(trace)}}; + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + FlightCallOptions impatient; + impatient.timeout = std::chrono::milliseconds(500); + auto reader = client->DoGet(impatient, Ticket{"hang"}); + EXPECT_FALSE(reader.ok()) << "the deadline must expire while the server prepares"; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + EXPECT_EQ(0, flight_server.stream_closes.load()) + << "no stream was ever prepared, so there is nothing to Close"; + std::lock_guard guard(trace->mutex); + EXPECT_EQ(1, trace->call_completed); +} + +/// A DoGet stream whose first payload is an error: the transport must fail the +/// RPC and still tell the producer the RPC is over through Close(). +class ErrorPayloadDoGetStream final : public AsyncFlightDataStream { + public: + explicit ErrorPayloadDoGetStream(std::atomic* close_count) + : close_count_(close_count) {} + + std::shared_ptr schema() override { + return arrow::schema({arrow::field("value", arrow::int64())}); + } + + arrow::Future GetSchemaPayloadAsync() override { + return resolved_->GetSchemaPayloadAsync(); + } + + arrow::Future> NextAsync() override { + return arrow::Future>::MakeFinished( + arrow::Status::IOError("the producer failed")); + } + + Status Close() override { + close_count_->fetch_add(1); + return Status::OK(); + } + + std::shared_ptr resolved_ = std::make_shared( + arrow::RecordBatchReader::Make( + {}, arrow::schema({arrow::field("value", arrow::int64())})) + .ValueOrDie()); + + private: + std::atomic* close_count_; +}; + +/// Serves ErrorPayloadDoGetStream and counts the Close() it was handed. +class ErrorPayloadDoGetServer final : public AsyncGenericFlightServerBase { + public: + arrow::Future> DoGetAsync( + const ServerCallContext&, const Ticket&) override { + return arrow::Future>::MakeFinished( + std::make_shared(&stream_closes)); + } + + std::atomic stream_closes{0}; +}; + +TEST(AsyncGrpcTest, DoGetPumpErrorStillClosesTheStream) { + // The producer's first NextAsync() fails: the RPC ends with that status, and + // the producer must still be told the RPC is over through Close(), like the + // write-failure and cancel paths already do. + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ErrorPayloadDoGetServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + // DoGet returns once the schema payload is written, the error payload follows. + ASSERT_OK_AND_ASSIGN(auto reader, client->DoGet(Ticket{"error-payload"})); + while (true) { + auto chunk = reader->Next(); + if (!chunk.ok()) { + EXPECT_EQ(chunk.status().code(), arrow::StatusCode::IOError) << chunk.status(); + break; + } + if (!chunk->data) { + break; + } + } + reader.reset(); + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + EXPECT_EQ(1, flight_server.stream_closes.load()) + << "the producer was never told the RPC is over"; +} + +} // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc new file mode 100644 index 000000000000..6449912f8e1d --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/transport/grpc/async_grpc_service_internal.h" + +#include "arrow/flight/transport/grpc/customize_grpc.h" + +#include +#include +#include + +namespace arrow::flight::transport::grpc::detail { + +namespace { + +/// Serve one DoPut RPC over the generic callback API. +class DoPutReactor : public AsyncReactorBase, + public arrow::flight::internal::FlightDataListenerTransport { + public: + DoPutReactor(AsyncCallContext flight_context, + std::shared_ptr listener) + : AsyncReactorBase(std::move(flight_context)), + listener_(std::move(listener)), + decoder_(listener_) { + arrow::flight::internal::FlightDataListenerTransport::Install(listener_, this); + StartRead(&request_buf_); + } + + void Hold() override { AsyncReactorBase::Hold(); } + + void ReleaseHold() override { AsyncReactorBase::ReleaseHold(); } + + void CancelUpload(Status status) override { + if (status.ok()) { + status = Status::Invalid("Cancel() needs a non-OK status"); + } + AsyncReactorBase::Hold(); + FinishUpload(std::move(status)); + AsyncReactorBase::ReleaseHold(); + } + + Status WriteMetadata(const Buffer& app_metadata) override { + std::lock_guard lock(write_mutex()); + if (finished()) { + return Status::Invalid( + "the upload is over: WriteMetadata() is only usable while the " + "upload's RPC is running"); + } + if (metadata_in_flight_ || ending_) { + // One message in flight at a time: OnWriteDone serializes the PutResults. + return Status::Invalid( + "a PutResult message is already in flight: await it before writing " + "another metadata message"); + } + metadata_in_flight_ = true; + pb::PutResult pb_result; + if (app_metadata.size() > 0) { + pb_result.set_app_metadata(app_metadata.data(), app_metadata.size()); + } + write_buf_ = MakeWriteBuffer(pb_result); + StartWrite(&write_buf_); + return Status::OK(); + } + + void OnReadDone(bool ok) override { + if (!ok) { + // Decide under the lock, act outside it: FinishUpload() runs the + // listener's OnFinish(), which is application code and may itself call + // WriteMetadata(). That takes writer_mutex_ and then write_mutex_, the + // reverse of a caller holding write_mutex_ and waiting for writer_mutex_, + // so invoking it here would deadlock the two threads. + { + std::lock_guard lock(write_mutex()); + if (finished()) { + return; + } + if (read_descriptor_) { + // The client half-closed: this ack ends the upload. It takes the + // same write slot as a mid-upload WriteMetadata, which may be on the + // wire right now: starting a second write would clobber write_buf_ + // and the RPC would never see its OnWriteDone. + ending_ = true; + if (metadata_in_flight_) { + return; // OnWriteDone sends the ending ack once the slot frees up + } + StartEndingAck(); + return; + } + } + FinishUpload( + MakeFlightError(FlightStatusCode::Failed, + "Client sent an empty upload: the first message must carry the " + "descriptor")); + return; + } + std::shared_ptr arrow_buf; + const Status wrap_status = WrapGrpcBuffer(&request_buf_, &arrow_buf); + if (!wrap_status.ok()) { + FinishUpload( + MakeFlightError(FlightStatusCode::Internal, + "Failed to wrap gRPC buffer: " + wrap_status.message())); + return; + } + + if (!read_descriptor_) { + // The first message must carry the descriptor of the upload, exactly as + // the sync transport requires. + // Without this the listener never hears OnDescriptor and the upload is + // acknowledged as if it were well formed. + read_descriptor_ = true; + internal::FlightData data; + const ::grpc::Status deserialized = FlightDataDeserialize(&request_buf_, &data); + if (!deserialized.ok()) { + FinishUpload( + MakeFlightError(FlightStatusCode::Internal, deserialized.error_message())); + return; + } + if (!data.descriptor) { + FinishUpload(Status::IOError("Descriptor missing on first message")); + return; + } + } + + Future<> decode_status = decoder_.Consume(std::move(arrow_buf)); + decode_status.AddCallback([this, token = hold()](arrow::Status status) { + if (finished()) { + // The RPC is over (cancelled while the listener was working): the + // message is abandoned. + return; + } + if (!status.ok()) { + FinishUpload(std::move(status)); + return; + } + // Arm the next read through WriteIfLive(): this continuation runs on the + // decoder's thread, so a cancellation can finish the RPC right after the + // finished() check above. WriteIfLive re-checks under write_mutex_, which + // FinishOnce also holds, so the read is never started on a dead call. + WriteIfLive([this] { StartRead(&request_buf_); }); + }); + } + + void OnWriteDone(bool ok) override { + // Decide under the lock, act outside it: FinishUpload() runs the listener's + // OnFinish(), which is application code and may call back into WriteMetadata(). + Status finish_status; + bool ended = false; + { + std::lock_guard lock(write_mutex()); + if (finished()) { + return; + } + if (!ok) { + finish_status = MakeFlightError(FlightStatusCode::Internal, "Write failed"); + ended = true; + } else if (ending_ack_in_flight_) { + // The ending ack reached the client: that is the clean end of the upload. + ended = true; + } else { + metadata_in_flight_ = false; + if (ending_) { + // The client half-closed while a metadata message was on the wire: + // with the slot free, the ending ack is the next write. + StartEndingAck(); + } + } + } + if (ended) { + FinishUpload(std::move(finish_status)); + } + } + + void OnCancel() override { + // The RPC is cancelled: report it to the listener but do not let a failing + // OnFinish replace the cancellation the client is seeing. The report is + // fire-and-forget here; keeping a reference alive for it is unnecessary as + // FinishOnce() below releases the reactor's hold and the callback only + // touches immutable state. + const auto status = Status::Cancelled("the client cancelled the upload"); + static_cast(arrow::flight::internal::FlightDataListenerTransport::ReportFinish( + listener_, status)); + FinishOnce(status); + } + + void OnDone() override { + arrow::flight::internal::FlightDataListenerTransport::Clear(listener_); + ReleaseHold(); + } + + private: + /// Send the empty ack that ends the upload. Caller holds write_mutex_. + void StartEndingAck() { + ending_ack_in_flight_ = true; + pb::PutResult pb_result; + write_buf_ = MakeWriteBuffer(pb_result); + StartWrite(&write_buf_); + } + + void FinishUpload(Status status) { + // Chain the RPC's completion on the listener's OnFinish() future instead of + // waiting for it: FinishOnce() must not run on a gRPC callback thread while + // the application is still committing, and a listener that fails + // asynchronously must not make the client see success. Keep the reactor + // alive across the wait. + Status reported = status; + arrow::flight::internal::FlightDataListenerTransport::ReportFinish(listener_, + std::move(status)) + .AddCallback([this, token = hold(), + reported = std::move(reported)](const Status& finish_status) { + // A failing OnFinish overrides a successful upload, but must not mask + // a status the upload already failed with. + FinishOnce(reported.ok() ? finish_status : reported); + }); + } + + std::shared_ptr listener_; + AsyncFlightMessageDecoder decoder_; + /// Whether the descriptor of the first message has been checked. + bool read_descriptor_ = false; + // The write side is serialized by AsyncReactorBase::write_mutex_, which + // FinishOnce() also takes: WriteMetadata() runs on the listener's thread + // while OnReadDone()/OnWriteDone() run on gRPC callback threads, and this + // must be the same mutex the base uses or finishing can race a write. + /// Whether a mid-upload metadata message is on the wire. + bool metadata_in_flight_ = false; + /// Whether the client half-closed, so the ending ack is owed. + bool ending_ = false; + /// Whether that ending ack is the write in flight. + bool ending_ack_in_flight_ = false; + ::grpc::ByteBuffer request_buf_; + ::grpc::ByteBuffer write_buf_; +}; + +} // namespace + +::grpc::ServerGenericBidiReactor* MakeDoPutReactor( + AsyncCallContext flight_context, std::shared_ptr listener) { + return new DoPutReactor(std::move(flight_context), std::move(listener)); +} + +} // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc new file mode 100644 index 000000000000..ceccdb91ed6f --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc @@ -0,0 +1,762 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// DoPut over the async generic service: the push decoder feeding the per-RPC +// listener, the acknowledgement that makes the client's Close() return, the +// PutResult metadata channel, and every way an upload ends -- clean, rejected by +// the listener, rejected on its descriptor, cancelled from the server side, +// client deadline, and the client vanishing while the listener is parked. + +#include "arrow/flight/transport/grpc/async_grpc_test_fixtures.h" + +namespace arrow::flight::transport::grpc { + +TEST(AsyncGrpcTest, BasicDoPut) { + // One listener for the whole test: shared with the service's factory and + // kept here so it can be asserted on after the RPC. + auto listener = std::make_shared(); + TestFlightServer flight_server; + flight_server.set_listener(listener); + AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); + + int port = 0; + ::grpc::ServerBuilder builder; + builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.RegisterCallbackGenericService(&service); + auto server = builder.BuildAndStart(); + ASSERT_NE(server, nullptr); + ASSERT_NE(port, 0); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); + + auto schema = TwoColumnSchema(); + auto batch = TwoColumnBatch(schema); + + // Call DoPut and upload the same batch twice. Close() returns the server's + // final status, i.e. it only succeeds once the acknowledgement written on + // end of stream has been read. + FlightDescriptor descriptor = FlightDescriptor::Path({"test"}); + ASSERT_OK_AND_ASSIGN(auto result, client->DoPut(descriptor, schema)); + ASSERT_OK(result.writer->WriteRecordBatch(*batch)); + ASSERT_OK(result.writer->WriteRecordBatch(*batch)); + ASSERT_OK(result.writer->Close()); + + // Cleanup. The listener is asserted only after the server stopped, so the + // DoPut callback cannot race with the reads. + ASSERT_OK(client->Close()); + server->Shutdown(); + server->Wait(); + + // The listener saw the schema once and both batches, in order. + ASSERT_EQ(listener->schema_count(), 1); + AssertTwoColumnSchema(listener->schema()); + ASSERT_EQ(listener->batches().size(), 2); + for (const auto& chunk : listener->batches()) { + ASSERT_EQ(chunk->num_rows(), 3); + ASSERT_EQ(chunk->num_columns(), 2); + } + // No metadata-only (error/app-metadata) chunks, and nothing the consumer + // rejected: a non-OK OnNext status would have failed DoPut with no ack. + ASSERT_EQ(listener->metadata_only_count(), 0); + ASSERT_OK(listener->last_status()); +} + +TEST(AsyncGrpcTest, EmptyDoPut) { + TestFlightServer flight_server; + auto listener = std::make_shared(); + flight_server.set_listener(listener); + AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); + + int port = 0; + ::grpc::ServerBuilder builder; + builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.RegisterCallbackGenericService(&service); + auto server = builder.BuildAndStart(); + ASSERT_NE(server, nullptr); + ASSERT_NE(port, 0); + + std::string uri = "grpc://localhost:" + std::to_string(port); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + + auto schema = arrow::schema( + {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); + + // Upload nothing at all: the acknowledgement the server writes back on the + // client's half-close is what makes Close() return. + FlightDescriptor descriptor = FlightDescriptor::Path({"empty-put"}); + ASSERT_OK_AND_ASSIGN(auto result, client->DoPut(descriptor, schema)); + ASSERT_OK(result.writer->Close()); + + // Cleanup before looking at the listener, so the DoPut callback cannot race + // with the reads. + ASSERT_OK(client->Close()); + server->Shutdown(); + server->Wait(); + + // No data of any kind reached the listener. + ASSERT_EQ(listener->batches().size(), 0); + ASSERT_EQ(listener->metadata_only_count(), 0); + ASSERT_OK(listener->last_status()); + // The sync client sends the schema payload eagerly, even when no batch + // follows, so the listener does see the schema here. Deliberately not + // asserted: that is the client's business, not the transport's. +} + +TEST(AsyncGrpcTest, MetadataOnlyPutChunk) { + TestFlightServer flight_server; + auto listener = std::make_shared(); + flight_server.set_listener(listener); + AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); + + int port = 0; + ::grpc::ServerBuilder builder; + builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.RegisterCallbackGenericService(&service); + auto server = builder.BuildAndStart(); + ASSERT_NE(server, nullptr); + ASSERT_NE(port, 0); + + std::string uri = "grpc://localhost:" + std::to_string(port); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + + auto schema = arrow::schema( + {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); + + FlightDescriptor descriptor = FlightDescriptor::Path({"metadata-only-put"}); + ASSERT_OK_AND_ASSIGN(auto result, client->DoPut(descriptor, schema)); + auto app_metadata = arrow::Buffer::FromString("metadata-only-upload"); + ASSERT_OK(result.writer->WriteMetadata(app_metadata)); + ASSERT_OK(result.writer->Close()); + + ASSERT_OK(client->Close()); + server->Shutdown(); + server->Wait(); + + // Exactly one chunk reached the listener: no data, that exact metadata. + ASSERT_EQ(listener->batches().size(), 0); + ASSERT_EQ(listener->metadata_only_count(), 1); + ASSERT_EQ(listener->metadata_chunks().size(), 1); + ASSERT_NE(listener->metadata_chunks()[0], nullptr); + ASSERT_EQ(listener->metadata_chunks()[0]->ToString(), "metadata-only-upload"); +} + +TEST(AsyncGrpcTest, DoPutRejectedByListener) { + TestFlightServer flight_server; + auto listener = std::make_shared(); + // The consumer rejects the upload. + listener->set_next_status(arrow::Status::Invalid("listener rejected this upload")); + flight_server.set_listener(listener); + AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); + + int port = 0; + ::grpc::ServerBuilder builder; + builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.RegisterCallbackGenericService(&service); + auto server = builder.BuildAndStart(); + ASSERT_NE(server, nullptr); + ASSERT_NE(port, 0); + + std::string uri = "grpc://localhost:" + std::to_string(port); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + + auto schema = TwoColumnSchema(); + auto batch = TwoColumnBatch(schema); + + // The rejection must reach the client as the upload's status, whichever of + // the three surfaces reports it first (in practice the writer's Close()). + FlightDescriptor descriptor = FlightDescriptor::Path({"rejected-put"}); + auto [surface, status] = UploadOneBatch(client.get(), descriptor, schema, batch); + ASSERT_FALSE(status.ok()) + << "the listener's rejection never reached the client, surfaced on " << surface; + ASSERT_EQ(status.code(), arrow::StatusCode::Invalid); + ASSERT_NE(status.message().find("listener rejected this upload"), std::string::npos) + << "unexpected status: " << status; + // The rejection must also be what OnFinish reports. + ASSERT_EQ(listener->finish_status().code(), arrow::StatusCode::Invalid) + << "OnFinish got: " << listener->finish_status(); + + ASSERT_OK(client->Close()); + server->Shutdown(); + server->Wait(); +} + +TEST(AsyncGrpcTest, CreateDoPutListenerServesUpload) { + // The upload half of the server class API: CreateDoPutListener() hands out + // the per-RPC listener, with no options-level factory. + TestUploadServer flight_server; + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_GT(flight_server.port(), 0); + + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = TwoColumnSchema(); + auto batch = TwoColumnBatch(schema); + + const FlightDescriptor descriptor = FlightDescriptor::Path({"virtual", "h2"}); + auto [surface, status] = UploadOneBatch(client.get(), descriptor, schema, batch); + ASSERT_TRUE(status.ok()) << "upload failed on " << surface << ": " << status; + + // Cleanup before reading the recordings: the listener's callbacks run on a + // gRPC thread. + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + // The virtual was consulted exactly once, and its listener saw the upload. + ASSERT_EQ(flight_server.calls(), 1); + const auto& listener = flight_server.listener(0); + ASSERT_EQ(listener->descriptor_count(), 1); + ASSERT_EQ(listener->descriptors()[0], descriptor); + ASSERT_EQ(listener->schema_count(), 1); + ASSERT_EQ(listener->batches().size(), 1); + ASSERT_EQ(listener->batches()[0]->num_rows(), 3); +} + +TEST(AsyncGrpcTest, DoPutWithoutListenerIsUnimplemented) { + // A server class that does not hand out a listener (CreateDoPutListener() + // returns nullptr, the base default) does not accept uploads: the RPC is + // answered UNIMPLEMENTED. There is no options-level fallback. + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = arrow::schema({arrow::field("a", arrow::int64())}); + auto batch = arrow::RecordBatch::Make(schema, 2, + {arrow::ArrayFromJSON(arrow::int64(), "[7, 8]")}); + auto [surface, status] = UploadOneBatch( + client.get(), FlightDescriptor::Path({"no-listener"}), schema, batch); + ASSERT_FALSE(status.ok()) << "the upload succeeded on " << surface + << " with no listener to serve it"; + ASSERT_EQ(status.code(), arrow::StatusCode::NotImplemented) << status; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, DoPutListenerSeesCleanFinish) { + // A client that ends the upload normally (DoneWriting + Close) makes the + // listener's OnFinish fire once, with OK: that is how a consumer knows the + // upload ended and it can commit what it accumulated. + TestUploadServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = OneColumnSchema(); + auto batch = OneColumnBatch(schema); + auto [surface, status] = + UploadOneBatch(client.get(), FlightDescriptor::Path({"finish"}), schema, batch); + ASSERT_TRUE(status.ok()) << "upload failed on " << surface << ": " << status; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + // OnFinish ran exactly once, and said the upload ended cleanly. + ASSERT_EQ(flight_server.listener(0)->finish_count(), 1); + ASSERT_TRUE(flight_server.listener(0)->finish_status().ok()) + << "unexpected finish status: " << flight_server.listener(0)->finish_status(); +} + +TEST(AsyncGrpcTest, OnFinishMayWriteMetadataWithoutDeadlocking) { + // OnFinish is application code and may call back into the listener's writer. + // The transport must not hold its write lock across that call: doing so + // deadlocks this upload (verified: the RPC never finishes). + RecordingUploadServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = OneColumnSchema(); + ASSERT_OK_AND_ASSIGN(auto put, + client->DoPut(FlightDescriptor::Path({"onfinish"}), schema)); + ASSERT_OK(put.writer->WriteRecordBatch(*OneColumnBatch(schema))); + ARROW_UNUSED(put.writer->Close()); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + auto listener = flight_server.WaitForListener(0); + ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; + // The write is refused (the upload is ending) rather than deadlocking. + EXPECT_TRUE(listener->finish_write.IsInvalid()) << listener->finish_write; + ASSERT_EQ(1, listener->finish_count()); +} + +TEST(AsyncGrpcTest, DoPutWritesMetadataBackToTheClient) { + // The listener's writer() sends PutResult messages during the upload, the way + // the sync FlightMetadataWriter does: the client reads them back with + // ReadMetadata() between writes. + MetadataReplyingUploadServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = OneColumnSchema(); + auto batch = OneColumnBatch(schema); + + ASSERT_OK_AND_ASSIGN(auto put, + client->DoPut(FlightDescriptor::Path({"metadata"}), schema)); + auto& writer = *put.writer; + auto& reader = *put.reader; + + // Write one batch, then read the acknowledgement the server sent for it. + ASSERT_OK(writer.WriteRecordBatch(*batch)); + std::shared_ptr metadata; + ASSERT_OK(reader.ReadMetadata(&metadata)); + ASSERT_NE(nullptr, metadata) << "the server's metadata never reached the client"; + ASSERT_EQ("ack 2 rows", metadata->ToString()); + + // A second batch and its acknowledgement prove the channel survives a round. + ASSERT_OK(writer.WriteRecordBatch(*batch)); + ASSERT_OK(reader.ReadMetadata(&metadata)); + ASSERT_NE(nullptr, metadata); + ASSERT_EQ("ack 2 rows", metadata->ToString()); + + ASSERT_OK(writer.Close()); + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + auto listener = flight_server.WaitForListener(); + ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; + ASSERT_EQ(2, listener->wrote()) << "every batch must have been acknowledged"; + ASSERT_OK(listener->last_write()) << listener->last_write(); +} + +TEST(AsyncGrpcTest, DoPutCancelFromServerSideFinishesTheUpload) { + // The server aborts the upload without waiting for the client: Cancel() + // finishes the RPC with the given status, the client's write or Close + // reports it, and the listener hears it as its OnFinish status. + TestUploadServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = arrow::schema({arrow::field("a", arrow::int64())}); + auto batch = arrow::RecordBatch::Make(schema, 2, + {arrow::ArrayFromJSON(arrow::int64(), "[3, 4]")}); + + // Open the upload and write one batch, but do not end it. + auto put = client->DoPut(FlightDescriptor::Path({"cancel"}), schema); + ASSERT_TRUE(put.ok()) << put.status(); + auto& writer = *put->writer; + ASSERT_OK(writer.WriteRecordBatch(*batch)); + + // Wait for the listener the virtual handed out, then cancel it from this + // (non-gRPC) thread: that is the mid-upload server-side abort. + auto listener = flight_server.WaitForListener(0); + ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; + listener->CancelWith(arrow::Status::Cancelled("server said stop")); + + // The client sees the cancellation on the surface that observes the RPC. + const Status close_status = writer.Close(); + ASSERT_FALSE(close_status.ok()) << "the server-side cancel never reached the client"; + ASSERT_EQ(close_status.code(), arrow::StatusCode::Cancelled); + ASSERT_NE(close_status.message().find("server said stop"), std::string::npos) + << "unexpected status: " << close_status; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + // Cancel() reached the transport, and the listener was told the upload ended + // with that status exactly once. + ASSERT_EQ(listener->cancel_count(), 1); + ASSERT_OK(listener->cancel_status()) << listener->cancel_status(); + ASSERT_EQ(listener->finish_count(), 1); + // OnFinish must carry the ending that caused it, not a generic status. + ASSERT_EQ(listener->finish_status().code(), arrow::StatusCode::Cancelled) + << "OnFinish got: " << listener->finish_status(); + + // After the upload, Cancel() has nothing to cancel. + listener->CancelWith(arrow::Status::Cancelled("too late")); + ASSERT_EQ(listener->cancel_status().code(), arrow::StatusCode::Invalid); +} + +/// A listener stuck inside OnNext(): the upload is parked in the application +/// while the RPC underneath it dies. +class HangingDoPutListener final : public FinishRecordingListener { + public: + arrow::Future<> OnNext(FlightStreamChunk) override { + next_awaits_.fetch_add(1); + return arrow::Future<>::Make(); + } + + std::atomic next_awaits_{0}; +}; + +/// A DoPut server that hands out a hanging listener, one per upload. +using HangingDoPutServer = RecordingUploadServer; + +TEST(AsyncGrpcTest, DoPutClientDeadlineWithPendingHandlerFinishes) { + // The client's deadline expires while the listener is still processing a + // chunk (its OnNext() future never resolves): the transport must finish the + // RPC itself instead of waiting for the listener: Shutdown()/Wait() below + // would otherwise block, since gRPC keeps the call alive until Finish() + // runs. + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + HangingDoPutServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = OneColumnSchema(); + auto batch = OneColumnBatch(schema); + + FlightCallOptions impatient; + impatient.timeout = std::chrono::milliseconds(1000); + auto put = client->DoPut(impatient, FlightDescriptor::Path({"hang"}), schema); + ASSERT_TRUE(put.ok()) << put.status(); + (void)put->writer->WriteRecordBatch(*batch); + + // From here the listener is inside OnNext() and the deadline does the rest. + ASSERT_TRUE(WaitFor([&flight_server] { + auto listener = flight_server.WaitForListener(0); + return listener != nullptr && listener->next_awaits_.load() >= 1; + })) << "the listener never saw a chunk"; + + ARROW_UNUSED(put->writer->Close()); + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + // The upload ended anyway, and the listener heard about it exactly once. + ASSERT_EQ(1, flight_server.listener(0)->finish_count()); +} + +TEST(AsyncGrpcTest, DoPutClientDeadlineWhileIdleIsNotACleanFinish) { + // The deadline expires while the server is idle (the batch was consumed and + // a read is armed): the upload was cut short by the client, so OnFinish() + // must not report it as a clean end, only the acknowledgement write + // completing cleanly may say OK. + TestUploadServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = OneColumnSchema(); + auto batch = OneColumnBatch(schema); + + FlightCallOptions impatient; + impatient.timeout = std::chrono::milliseconds(600); + auto put = client->DoPut(impatient, FlightDescriptor::Path({"idle"}), schema); + ASSERT_TRUE(put.ok()) << put.status(); + ASSERT_OK(put->writer->WriteRecordBatch(*batch)); + + auto listener = flight_server.WaitForListener(0); + ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; + ASSERT_TRUE(WaitFor([&listener] { return listener->batches().size() >= 1; })) + << "the server never consumed the batch"; + + // Wait for the deadline to end the upload instead of sleeping out its + // duration: the listener's OnFinish runs on the transport thread that ends + // the RPC, so the wait is on the observable event, not on wall-clock time. + // DoPut writes the schema eagerly, so the client's DoPut() above already + // showed the channel was alive when its deadline was armed. + ASSERT_TRUE(WaitFor([&listener] { return listener->finish_count() >= 1; })) + << "the deadline never fired; inconclusive"; + + const auto close_status = put->writer->Close(); + ASSERT_FALSE(close_status.ok()) << "the deadline never fired; inconclusive"; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + ASSERT_EQ(1, listener->finish_count()); + EXPECT_FALSE(listener->finish_status().ok()) + << "a deadline-cancelled upload was reported as a clean finish: " + << listener->finish_status(); +} + +/// A listener whose OnNext() future the test resolves on demand: the upload +/// parks in the application until the late-continuation test releases it. +class DeferredDoPutListener final : public FinishRecordingListener { + public: + arrow::Future<> OnNext(FlightStreamChunk) override { + next_awaits_.fetch_add(1); + std::lock_guard lock(mutex_); + pending_ = arrow::Future<>::Make(); + return pending_; + } + + /// Resolve the parked OnNext, OK releases the decode continuation. + void ResolvePending() { + std::lock_guard lock(mutex_); + if (pending_.is_valid() && !pending_.is_finished()) { + pending_.MarkFinished(arrow::Status::OK()); + } + } + + std::atomic next_awaits_{0}; + + private: + std::mutex mutex_; + arrow::Future<> pending_; +}; + +using DeferredDoPutServer = RecordingUploadServer; + +TEST(AsyncGrpcTest, DoPutClientCancelWhileListenerParkedStaysSafe) { + // The listener's OnNext() resolves after the deadline already finished the + // RPC: the parked decode continuation must stand down (release its hold, + // never touch the finished call) rather than reach into a destroyed reactor. + // The test resolves the parked future from its own thread, and the server + // keeps its shared_ptr alive, so the resolution lands on the real listener. + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + DeferredDoPutServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = OneColumnSchema(); + auto batch = OneColumnBatch(schema); + + // The deadline is what ends this RPC (the listener stays parked, so only the + // deadline can finish it), and DoPut writes the schema eagerly, so it must be + // long enough to survive the setup on a slow machine. The wait below must + // outlast it, or the poll loop gives up at the moment the deadline fires: + // that race is what made this test fail on the macOS runner. + constexpr auto kDeadline = std::chrono::milliseconds(2000); + FlightCallOptions impatient; + impatient.timeout = kDeadline; + auto put = client->DoPut(impatient, FlightDescriptor::Path({"late"}), schema); + ASSERT_TRUE(put.ok()) << put.status(); + ASSERT_OK(put->writer->WriteRecordBatch(*batch)); + + auto listener = flight_server.WaitForListener(0); + ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; + ASSERT_TRUE(WaitFor([&listener] { return listener->next_awaits_.load() >= 1; })) + << "the listener never saw a chunk"; + + // Walk away: the client stops reading, so the server's write has nowhere to + // go and the RPC ends while the listener is still parked. Wait for that to + // have happened rather than sleeping blindly, and fail clearly instead of + // reporting a vacuous pass if it did not. + ASSERT_OK(client->Close()); + ASSERT_TRUE(WaitFor([&listener] { return listener->finish_count() >= 1; }, + kDeadline + std::chrono::seconds(10))) + << "the RPC never ended; the listener is not the parked one from this upload"; + + // Now resolve the parked listener: the continuation runs on a finished RPC, + // which is the case the guards and holds are there to make safe. + listener->ResolvePending(); + + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + // The upload had ended once, and the late resolution added no second report. + ASSERT_EQ(1, listener->finish_count()); + + // Closing the writer after the RPC is gone must report a failure, not a + // silent success. + ASSERT_FALSE(put->writer->Close().ok()); +} + +TEST(AsyncGrpcTest, DoPutWithoutDescriptorOnFirstMessageIsRejected) { + // A client that starts an upload without a descriptor must be refused with + // the same status as the sync transport ("Descriptor missing on first + // message"): the listener must not hear a descriptor, and the upload must not + // be acknowledged as well formed. + TestFlightServer flight_server; + auto listener = std::make_shared(); + flight_server.set_listener(listener); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_GT(flight_server.port(), 0); + + // Hand-build the first FlightData of a DoPut: the schema IPC message with no + // descriptor field at all. + auto schema = OneColumnSchema(); + auto batch = OneColumnBatch(schema); + auto reader = arrow::RecordBatchReader::Make({batch}, schema).ValueOrDie(); + RecordBatchStream stream(std::move(reader)); + ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); + schema_payload.descriptor = nullptr; + + ::grpc::ByteBuffer request; + bool own_buffer = false; + ASSERT_TRUE(FlightDataSerialize(schema_payload, &request, &own_buffer).ok()); + + // Drive the bidi call through the raw generic stub: the regular FlightClient + // always sets the descriptor, which is exactly what this case lacks. + std::shared_ptr<::grpc::Channel> channel = + ::grpc::CreateChannel("127.0.0.1:" + std::to_string(flight_server.port()), + ::grpc::InsecureChannelCredentials()); + ::grpc::GenericStub stub(channel); + ::grpc::ClientContext context; + // Bound the call: were the descriptor ever accepted, the client would wait + // forever for an acknowledgement, turning a logic error into a hung suite. + context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(10)); + ::grpc::CompletionQueue cq; + void* tag = nullptr; + bool ok = false; + + auto call = stub.Call(&context, "/arrow.flight.protocol.FlightService/DoPut", &cq, + reinterpret_cast(1)); + ASSERT_NE(call, nullptr); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never started"; + + call->Write(request, reinterpret_cast(2)); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "the message was never written"; + + call->WritesDone(reinterpret_cast(3)); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "WritesDone never completed"; + + ::grpc::Status status; + call->Finish(&status, reinterpret_cast(4)); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never finished"; + + EXPECT_FALSE(status.ok()) << "an upload without a descriptor must be refused"; + EXPECT_THAT(status.error_message(), ::testing::HasSubstr("Descriptor missing")); + + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + // The listener never saw a descriptor, and never saw the message as valid. + EXPECT_EQ(0, listener->descriptor_count()); + EXPECT_EQ(0, listener->batches().size()); +} + +TEST(AsyncGrpcTest, DoPutReportsAFailingOnFinishToTheClient) { + // The listener's OnFinish() is where an application commits the upload. When + // it fails asynchronously, the client must not see a successful Close(): the + // RPC's status is chained on that future instead of being reported OK as soon + // as the upload's data arrived. + TestFlightServer flight_server; + auto listener = std::make_shared(); + flight_server.set_listener(listener); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = OneColumnSchema(); + auto batch = OneColumnBatch(schema); + auto put = client->DoPut(FlightDescriptor::Path({"fail-finish"}), schema); + ASSERT_TRUE(put.ok()) << put.status(); + ASSERT_OK(put->writer->WriteRecordBatch(*batch)); + + // Close() acknowledges the ending and then sees the RPC's final status, which + // is the listener's failure. + const auto close_status = put->writer->Close(); + EXPECT_FALSE(close_status.ok()) + << "a failing OnFinish must not be reported to the client as success"; + EXPECT_THAT(close_status.ToString(), ::testing::HasSubstr("could not be committed")); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + EXPECT_EQ(1, listener->finish_count()); +} + +TEST(AsyncGrpcTest, DoPutHalfCloseWithoutAnyMessageIsRejected) { + // A client that half-closes without sending a single message must not be + // acknowledged: there is no descriptor, so there is no upload. The sync + // transport rejects this ("Stream finished before first message sent"); + // without the check the reactor sent the clean-ending ack and the listener + // heard OnFinish(OK) for a request that was never well formed. + TestFlightServer flight_server; + auto listener = std::make_shared(); + flight_server.set_listener(listener); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_GT(flight_server.port(), 0); + + std::shared_ptr<::grpc::Channel> channel = + ::grpc::CreateChannel("127.0.0.1:" + std::to_string(flight_server.port()), + ::grpc::InsecureChannelCredentials()); + ::grpc::GenericStub stub(channel); + ::grpc::ClientContext context; + context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(10)); + ::grpc::CompletionQueue cq; + void* tag = nullptr; + bool ok = false; + + auto call = stub.Call(&context, "/arrow.flight.protocol.FlightService/DoPut", &cq, + reinterpret_cast(1)); + ASSERT_NE(call, nullptr); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never started"; + + // Half-close with no message written at all. + call->WritesDone(reinterpret_cast(2)); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "WritesDone never completed"; + + ::grpc::Status status; + call->Finish(&status, reinterpret_cast(3)); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never finished"; + + EXPECT_FALSE(status.ok()) << "an upload with no message at all must be refused"; + EXPECT_THAT(status.error_message(), ::testing::HasSubstr("empty upload")); + + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + // The listener must not have been told the upload ended cleanly. + EXPECT_EQ(0, listener->descriptor_count()); + EXPECT_EQ(0, listener->batches().size()); +} + +TEST(AsyncGrpcTest, UseAsyncGrpcFlagUploadRejectedOnDescriptor) { + // A handler that rejects the descriptor rejects the whole upload through + // the same surface as an OnNext rejection: the writer's status. + TestFlightServer flight_server; + auto listener = std::make_shared(); + listener->set_descriptor_status( + arrow::Status::Invalid("listener rejected this descriptor")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + flight_server.set_listener(listener); + ASSERT_OK(flight_server.Init(options)); + ASSERT_GT(flight_server.port(), 0); + + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = TwoColumnSchema(); + auto batch = TwoColumnBatch(schema); + + const FlightDescriptor descriptor = FlightDescriptor::Path({"rejected-h2"}); + auto [surface, status] = UploadOneBatch(client.get(), descriptor, schema, batch); + ASSERT_FALSE(status.ok()) + << "the descriptor rejection never reached the client, surfaced on " << surface; + ASSERT_EQ(status.code(), arrow::StatusCode::Invalid); + ASSERT_NE(status.message().find("listener rejected this descriptor"), std::string::npos) + << "unexpected status: " << status; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + // The upload was rejected on its descriptor: the handler saw it, and no + // schema or data followed it. + ASSERT_EQ(listener->descriptor_count(), 1); + ASSERT_EQ(listener->descriptors()[0], descriptor); + ASSERT_EQ(listener->schema_count(), 0); + ASSERT_EQ(listener->batches().size(), 0); +} + +} // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc new file mode 100644 index 000000000000..16ae8f3ac723 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/transport/grpc/async_grpc_exchange_internal.h" + +namespace arrow::flight::transport::grpc::detail { + +ExchangeReactor::ExchangeReactor(AsyncCallContext flight_context, + AsyncGenericFlightServerBase* base) + : AsyncReactorBase(std::move(flight_context)), + base_(base), + reader_(std::make_shared()), + writer_(std::make_shared(this)) { + reader_->SetReactor(this); + // The first message carries the request descriptor. + StartRead(&read_buf_); +} + +void ExchangeReactor::OnReadDone(bool ok) { + if (finished()) { + return; + } + if (!ok) { + if (!started_) { + FinishOnce(MakeFlightError(FlightStatusCode::Internal, "Failed to read request")); + return; + } + reader_->OnReadClosed(); + return; + } + const ::grpc::Status deserialized = FlightDataDeserialize(&read_buf_, &read_data_); + if (!deserialized.ok()) { + if (!started_) { + FinishOnce( + MakeFlightError(FlightStatusCode::Internal, deserialized.error_message())); + return; + } + reader_->OnReadFailed( + MakeFlightError(FlightStatusCode::Internal, deserialized.error_message())); + return; + } + if (!started_) { + started_ = true; + if (read_data_.descriptor == nullptr) { + FinishOnce(MakeFlightError(FlightStatusCode::Internal, + "Descriptor missing on first message")); + return; + } + reader_->SetDescriptor(*read_data_.descriptor); + // The first message may carry data,the reader consumes it on its first + // NextAsync, so hand it over before the handler can ask. + reader_->OfferFirstRead(std::move(read_data_)); + arrow::Future<> exchange = base_->DoExchangeAsync(flight_context(), reader_, writer_); + exchange.AddCallback([this, token = hold()](const arrow::Status& status) { + if (!finished()) { + FinishOnce(status); + } + }); + return; + } + // A message read because the reader asked for one. + reader_->OnMessage(std::move(read_data_)); +} + +void ExchangeReactor::OnWriteDone(bool ok) { + if (finished()) { + return; + } + writer_->OnWriteDone(ok); +} + +void ExchangeReactor::OnCancel() { + // Finish first (the handler's future may never resolve), then unwind the + // reader and writer so a handler parked on their futures wakes up. + FinishOnce(arrow::Status::Cancelled()); + reader_->OnCancelled(); + writer_->OnCancelled(); +} + +void ExchangeReactor::OnDone() { + reader_->DetachReactor(); + writer_->DetachReactor(); + AsyncReactorBase::OnDone(); +} + +void ExchangeReactor::StartReadNext() { + WriteIfLive([this] { StartRead(&read_buf_); }); +} + +arrow::Status ExchangeReactor::WritePayload(FlightPayload payload) { + bool own_buffer = false; + const ::grpc::Status serialize = FlightDataSerialize(payload, &write_buf_, &own_buffer); + (void)own_buffer; + if (!serialize.ok()) { + return MakeFlightError(FlightStatusCode::Internal, serialize.error_message()); + } + const bool written = WriteIfLive([this] { StartWrite(&write_buf_); }); + if (!written) { + return arrow::Status::Cancelled("the exchange is over"); + } + return arrow::Status::OK(); +} + +::grpc::ServerGenericBidiReactor* MakeExchangeReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { + return new ExchangeReactor(std::move(flight_context), base); +} + +} // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h new file mode 100644 index 000000000000..e64edf4d4e1f --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h @@ -0,0 +1,218 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/flight/transport/grpc/async_grpc_service_internal.h" +#include "arrow/ipc/writer.h" + +namespace arrow::flight::transport::grpc::detail { + +// DoExchange: the bidirectional RPC. +// +// Its handler is asynchronous user code: it receives an +// AsyncFlightMessageReader/AsyncFlightMessageWriter pair and returns a future +// the reactor finishes the RPC with. The three classes below are mutually +// coupled on purpose -- the reader asks the reactor for the next read, the +// writer asks it to write a payload, and the reactor drives both from the +// gRPC callbacks -- so they are declared together here and implemented one +// per translation unit (async_grpc_exchange_reader.cc, +// async_grpc_exchange_writer.cc, async_grpc_exchange_reactor.cc). + +class ExchangeReactor; + +/// \brief The AsyncFlightMessageReader of one DoExchange RPC. +/// +/// Decoding reuses the push decoder DoPut uses, so the wire shapes behave +/// identically: the schema and dictionary messages are read through (they are +/// not chunks), a metadata-only message is a chunk with a null data member, +/// and the end of the exchange is the chunk whose members are both null. +/// Messages are pulled one demand at a time: NextAsync() arms the next read, +/// and the chunk (or the end, or the failure) resolves the demand. +/// +/// The reactor drives the I/O (it owns the read buffer and calls OnMessage / +/// OnReadClosed / OnReadFailed), this class owns the decode state machine. +class ExchangeReader final : public AsyncFlightMessageReader, + public std::enable_shared_from_this { + public: + ExchangeReader(); + + const FlightDescriptor& descriptor() const override; + + arrow::Future NextAsync() override; + + /// \brief The reactor wires itself before the handler starts. + void SetReactor(ExchangeReactor* reactor); + /// \brief Called by the reactor once the exchange is over: from then on the + /// handler's reader must refuse to start reads instead of calling into a + /// freed reactor. + void DetachReactor(); + /// \brief The reactor sets the descriptor from the first message, before the + /// handler starts. + void SetDescriptor(const FlightDescriptor& descriptor); + /// \brief The reactor hands over the already-read first message, the reader + /// consumes it on its first NextAsync(). + void OfferFirstRead(internal::FlightData data); + + /// \brief The reactor's OnReadDone: a message was read. + void OnMessage(internal::FlightData data); + /// \brief The reactor's OnReadDone with ok == false: the read side is over. + void OnReadClosed(); + /// \brief The reactor read a message that could not be deserialized. + void OnReadFailed(Status status); + /// \brief The RPC was cancelled: fail the demand so the handler unwinds. + void OnCancelled(); + + private: + /// The decoder's listener: chunks are handed to the reader. + class ChunkListener; + + void ResolvePending(FlightStreamChunk chunk); + void ResolvePending(const Status& status); + + /// \brief Decode one message and resolve the demand it satisfies, or keep + /// reading when the message was read through (a schema or dictionary + /// message). + void Consume(internal::FlightData data); + + /// \brief Arm the next read through the reactor, defined in + /// async_grpc_exchange_reader.cc, which has the reactor's definition. + void RequestRead(); + + std::atomic reactor_{nullptr}; + std::shared_ptr listener_; + AsyncFlightMessageDecoder decoder_; + FlightDescriptor descriptor_; + std::optional first_read_; + std::optional buffered_; + arrow::Future pending_; + Status read_error_; + bool read_in_flight_ = false; + bool read_closed_ = false; + std::mutex mutex_; +}; + +/// \brief The AsyncFlightMessageWriter of one DoExchange RPC. +/// +/// Record batches are serialized through the same IPC writer stack the +/// synchronous transport uses (dictionary messages included) into an +/// IpcPayloadWriter sink that queues FlightPayloads, the reactor then writes +/// them one at a time. Each method's future resolves when its messages have +/// been handed to the reactor, or with the first failure. +class ExchangeWriter final : public AsyncFlightMessageWriter { + public: + explicit ExchangeWriter(ExchangeReactor* reactor); + + /// \brief Called by the reactor once the exchange is over: from then on the + /// handler's writer must not call into a freed reactor. + void DetachReactor(); + + arrow::Future<> BeginAsync(std::shared_ptr schema) override; + arrow::Future<> WriteRecordBatchAsync(const RecordBatch& batch) override; + arrow::Future<> WriteWithMetadataAsync(const RecordBatch& batch, + std::shared_ptr app_metadata) override; + arrow::Future<> WriteMetadataAsync(std::shared_ptr app_metadata) override; + arrow::Future<> CloseAsync() override; + + /// \brief The reactor's OnWriteDone: continue flushing the queue. + void OnWriteDone(bool ok); + /// \brief The RPC was cancelled: fail the pending write so the handler + /// unwinds. + void OnCancelled(); + + private: + /// The IpcPayloadWriter that queues the IPC payloads the writer stack + /// produces, the schema, dictionary and record-batch messages all travel + /// this way, as on the synchronous path. + class PayloadSink; + + arrow::Future<> WriteBatch(const RecordBatch& batch); + + /// \brief Write the queued payloads one at a time, the returned future + /// resolves when the queue is empty, or with the first failure. + arrow::Future<> Flush(); + + /// \brief Write queued payloads through the reactor, defined in + /// async_grpc_exchange_writer.cc, which has the reactor's definition. + void Pump(); + + void FailFlush(Status status); + + std::atomic reactor_; + std::unique_ptr batch_writer_; + ipc::IpcWriteOptions options_ = ipc::IpcWriteOptions::Defaults(); + std::shared_ptr pending_app_metadata_; + std::deque queue_; + arrow::Future<> flush_; + bool write_in_flight_ = false; + std::mutex mutex_; +}; + +/// \brief Serve one DoExchange RPC. +/// +/// The handler receives an AsyncFlightMessageReader/Writer pair and returns a +/// future, the RPC then runs entirely on continuations. The reader pulls one +/// message per NextAsync() demand (the reactor arms the read), the writer +/// serializes through the IPC writer stack and the reactor writes one message +/// per StartWrite, and the handler's future completing finishes the RPC with +/// its status. No callback thread is ever held by the handler. +class ExchangeReactor final : public AsyncReactorBase { + public: + ExchangeReactor(AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); + + void OnReadDone(bool ok) override; + void OnWriteDone(bool ok) override; + void OnCancel() override; + void OnDone() override; + + /// \brief The reader's demand: arm the next read. The demand guarantees at + /// most one read is outstanding (ExchangeReader::RequestRead is the only + /// caller). + void StartReadNext(); + + /// \brief Serialize one payload into the write buffer and start the write. + /// Called by ExchangeWriter, OnWriteDone() resumes it. + arrow::Status WritePayload(FlightPayload payload); + + /// \brief Take a reference across a call made through a back-pointer. + /// + /// OnDone() detaches the reader/writer back-pointers and then releases the + /// reactor's last reference, so a reader/writer that loads the raw pointer and + /// calls in can reach freed memory. These re-expose the base's protected + /// Hold()/ReleaseHold() publicly (the same way DoPutReactor does for its + /// transport interface); callers pair them around the call. + void Hold() { AsyncReactorBase::Hold(); } + void ReleaseHold() { AsyncReactorBase::ReleaseHold(); } + + private: + AsyncGenericFlightServerBase* base_; + std::shared_ptr reader_; + std::shared_ptr writer_; + ::grpc::ByteBuffer read_buf_; + ::grpc::ByteBuffer write_buf_; + internal::FlightData read_data_; + bool started_ = false; +}; + +} // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc new file mode 100644 index 000000000000..fda5dd3a6f2c --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/transport/grpc/async_grpc_exchange_internal.h" + +namespace arrow::flight::transport::grpc::detail { + +/// The decoder's listener: chunks are handed to the reader, the descriptor and +/// the schema are consumed (the reactor sets the descriptor before the handler +/// starts). +class ExchangeReader::ChunkListener final : public AsyncFlightDataListener { + public: + explicit ChunkListener(ExchangeReader* reader) : reader_(reader) {} + + arrow::Future<> OnNext(FlightStreamChunk chunk) override { + bool has_demand; + { + std::lock_guard lock(reader_->mutex_); + has_demand = reader_->pending_.is_valid() && !reader_->pending_.is_finished(); + if (!has_demand) { + reader_->buffered_ = std::move(chunk); + } + } + if (has_demand) { + reader_->ResolvePending(std::move(chunk)); + } + return arrow::Future<>::MakeFinished(); + } + + arrow::Future<> OnDescriptor(const FlightDescriptor&) override { + return arrow::Future<>::MakeFinished(); + } + + arrow::Status OnSchemaDecoded(std::shared_ptr) override { + return arrow::Status::OK(); + } + + private: + ExchangeReader* reader_; +}; + +ExchangeReader::ExchangeReader() + : listener_(std::make_shared(this)), decoder_(listener_) {} + +const FlightDescriptor& ExchangeReader::descriptor() const { return descriptor_; } + +arrow::Future ExchangeReader::NextAsync() { + using FSCFuture = arrow::Future; + + arrow::Future returned; + { + std::lock_guard lock(mutex_); + if (pending_.is_valid() && !pending_.is_finished()) { + return FSCFuture::MakeFinished(arrow::Status::Invalid("one NextAsync at a time")); + } + if (buffered_.has_value()) { + auto out = FSCFuture::MakeFinished(std::move(*buffered_)); + buffered_.reset(); + return out; + } + + if (!read_error_.ok()) { + return FSCFuture::MakeFinished(read_error_); + } + + if (read_closed_ || reactor_.load() == nullptr) { + // End of exchange: the reactor either saw the read side close or the RPC + // is over (and the reactor is gone), so no further read can be started. + return FSCFuture::MakeFinished(FlightStreamChunk{}); + } + + pending_ = FSCFuture::Make(); + returned = pending_; + } + // Consume()/RequestRead() take the lock themselves and may resolve pending_ + // from a callback, so they must run unlocked. + if (first_read_.has_value()) { + internal::FlightData data; + { + std::lock_guard lock(mutex_); + data = std::move(*first_read_); + first_read_.reset(); + } + Consume(std::move(data)); + } else { + RequestRead(); + } + return returned; +} + +void ExchangeReader::SetReactor(ExchangeReactor* reactor) { + std::lock_guard lock(mutex_); + reactor_.store(reactor); +} + +void ExchangeReader::DetachReactor() { + std::lock_guard lock(mutex_); + reactor_.store(nullptr); +} + +void ExchangeReader::SetDescriptor(const FlightDescriptor& descriptor) { + descriptor_ = descriptor; +} + +void ExchangeReader::OfferFirstRead(internal::FlightData data) { + // Set during reactor setup, but NextAsync() reads it under mutex_. + std::lock_guard lock(mutex_); + first_read_ = std::move(data); +} + +void ExchangeReader::OnMessage(internal::FlightData data) { + { + std::lock_guard lock(mutex_); + read_in_flight_ = false; + } + Consume(std::move(data)); +} + +void ExchangeReader::OnReadClosed() { + { + std::lock_guard lock(mutex_); + read_in_flight_ = false; + read_closed_ = true; + } + ResolvePending(FlightStreamChunk{}); +} + +void ExchangeReader::OnReadFailed(Status status) { + { + std::lock_guard lock(mutex_); + read_in_flight_ = false; + read_error_ = status; + } + ResolvePending(status); +} + +void ExchangeReader::OnCancelled() { + { + std::lock_guard lock(mutex_); + read_closed_ = true; + } + ResolvePending(arrow::Status::Cancelled("the client cancelled the exchange")); +} + +void ExchangeReader::ResolvePending(FlightStreamChunk chunk) { + arrow::Future pending; + { + std::lock_guard lock(mutex_); + if (!pending_.is_valid() || pending_.is_finished()) { + return; + } + pending = std::move(pending_); + pending_ = arrow::Future(); + } + pending.MarkFinished(std::move(chunk)); +} + +void ExchangeReader::ResolvePending(const Status& status) { + arrow::Future pending; + { + std::lock_guard lock(mutex_); + if (!pending_.is_valid() || pending_.is_finished()) { + return; + } + pending = std::move(pending_); + pending_ = arrow::Future(); + } + pending.MarkFinished(status); +} + +void ExchangeReader::Consume(internal::FlightData data) { + { + std::lock_guard lock(mutex_); + if (read_closed_) { + return; + } + } + auto self = shared_from_this(); + decoder_.Consume(std::move(data)) + .AddCallback([this, self](const arrow::Status& status) { + if (!status.ok()) { + { + std::lock_guard lock(mutex_); + read_error_ = status; + } + ResolvePending(status); + } else { + bool has_demand; + { + std::lock_guard lock(mutex_); + has_demand = pending_.is_valid() && !pending_.is_finished(); + } + if (has_demand) { + RequestRead(); + } + } + }); +} + +void ExchangeReader::RequestRead() { + ExchangeReactor* reactor = nullptr; + { + std::lock_guard lock(mutex_); + reactor = reactor_.load(); + if (read_in_flight_ || read_closed_ || reactor == nullptr) { + return; + } + reactor->Hold(); + read_in_flight_ = true; + } + reactor->StartReadNext(); + reactor->ReleaseHold(); +} + +} // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc new file mode 100644 index 000000000000..f1480cea72e5 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// DoExchange over the async generic service: the bidirectional RPC, whose handler +// awaits the reader's demand and each write in turn, so reads and writes are +// interleaved without blocking a callback thread. Covers the echo path, a +// metadata-only exchange, the client vanishing mid-exchange, and a handler whose +// future never resolves. + +#include "arrow/flight/transport/grpc/async_grpc_test_fixtures.h" + +namespace arrow::flight::transport::grpc { + +// --------------------------------------------------------------------------- +// DoExchange: the bidirectional RPC. Its handler is DoExchangeAsync, which +// awaits the reader's demand and each write before the next one, so reads and +// writes are interleaved without blocking the callback thread. +// --------------------------------------------------------------------------- + +TEST(AsyncGrpcTest, DoExchangeEchoesBatches) { + // Two batches up, two batches back, in order: this is the case that needs a + // read and a write outstanding at the same time. + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + EchoExchangeTestServer flight_server; + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN( + auto client_location, + Location::Parse("grpc://localhost:" + std::to_string(flight_server.port()))); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(client_location)); + + ASSERT_OK_AND_ASSIGN(auto exchange, + client->DoExchange(FlightDescriptor::Command("echo"))); + + auto schema = arrow::schema({arrow::field("a", arrow::int64())}); + std::vector> sent = { + arrow::RecordBatch::Make(schema, 2, + {arrow::ArrayFromJSON(arrow::int64(), "[1, 2]")}), + arrow::RecordBatch::Make(schema, 3, + {arrow::ArrayFromJSON(arrow::int64(), "[3, 4, 5]")})}; + ASSERT_OK(exchange.writer->Begin(schema)); + for (const auto& batch : sent) { + ASSERT_OK(exchange.writer->WriteRecordBatch(*batch)); + } + ASSERT_OK(exchange.writer->DoneWriting()); + + ASSERT_OK_AND_ASSIGN(auto server_schema, exchange.reader->GetSchema()); + AssertSchemaEqual(*schema, *server_schema); + for (const auto& batch : sent) { + ASSERT_OK_AND_ASSIGN(auto chunk, exchange.reader->Next()); + ASSERT_NE(nullptr, chunk.data); + ASSERT_BATCHES_EQUAL(*batch, *chunk.data); + } + ASSERT_OK_AND_ASSIGN(auto end, exchange.reader->Next()); + ASSERT_EQ(nullptr, end.data); + ASSERT_OK(exchange.writer->Close()); + + // Cleanup before looking at the recording: it was written on a gRPC thread. + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + ASSERT_EQ(1, flight_server.exchange_count()); + ASSERT_EQ(0, flight_server.exchanges_in_flight()); +} + +TEST(AsyncGrpcTest, DoExchangeMetadataOnlyDoesNotHang) { + // A client that writes metadata and nothing else, then reads: the server's + // handler must see the end of the exchange and reply, not wait for a schema + // message that will never come (the case DataTest::TestDoExchangeNoData + // covers). A hang here is the failure mode this test exists for. + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + EchoExchangeTestServer flight_server; + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN( + auto client_location, + Location::Parse("grpc://localhost:" + std::to_string(flight_server.port()))); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(client_location)); + + ASSERT_OK_AND_ASSIGN(auto exchange, + client->DoExchange(FlightDescriptor::Command("echo"))); + + ASSERT_OK(exchange.writer->WriteMetadata(arrow::Buffer::FromString("only-metadata"))); + ASSERT_OK(exchange.writer->DoneWriting()); + + // The echo server reflects the metadata chunk back with no data. + ASSERT_OK_AND_ASSIGN(auto chunk, exchange.reader->Next()); + ASSERT_EQ(nullptr, chunk.data); + ASSERT_NE(nullptr, chunk.app_metadata); + ASSERT_EQ("only-metadata", chunk.app_metadata->ToString()); + ASSERT_OK_AND_ASSIGN(auto end, exchange.reader->Next()); + ASSERT_EQ(nullptr, end.data); + ASSERT_OK(exchange.writer->Close()); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + ASSERT_EQ(1, flight_server.exchange_count()); + ASSERT_EQ(0, flight_server.exchanges_in_flight()); +} + +TEST(AsyncGrpcTest, DoExchangeClientDisconnectFinishes) { + // A client that vanishes mid-exchange: the server's read fails, the reactor + // must let the handler unwind and finish the RPC, and the exchange counter + // added to the test server must come back to zero -- a hang or a leaked + // handler leaves it above zero. + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + EchoExchangeTestServer flight_server; + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN( + auto client_location, + Location::Parse("grpc://localhost:" + std::to_string(flight_server.port()))); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(client_location)); + + ASSERT_OK_AND_ASSIGN(auto exchange, + client->DoExchange(FlightDescriptor::Command("echo"))); + + auto schema = arrow::schema({arrow::field("a", arrow::int64())}); + ASSERT_OK(exchange.writer->Begin(schema)); + ASSERT_OK(exchange.writer->WriteRecordBatch(*arrow::RecordBatch::Make( + schema, 2, {arrow::ArrayFromJSON(arrow::int64(), "[1, 2]")}))); + // The server has entered its handler by now (it answered the schema), the + // exchange is live. + ASSERT_TRUE(WaitFor([&flight_server] { return flight_server.exchange_count() == 1; })); + + // Walk away without finishing: no DoneWriting, no Close, no reader drain. + // The client's destructor closes the channel, which cancels the RPC. + exchange.reader->Cancel(); + exchange.writer.reset(); + exchange.reader.reset(); + ASSERT_OK(client->Close()); + + // The handler must leave, whichever status path it takes. + const bool drained = + WaitFor([&flight_server] { return flight_server.exchanges_in_flight() == 0; }); + ASSERT_TRUE(drained) << "the exchange handler never left (in flight: " + << flight_server.exchanges_in_flight() << ")"; + + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +// A DoExchange handler whose future never resolves. +class HangingExchangeServer final : public AsyncGenericFlightServerBase { + public: + arrow::Future<> DoExchangeAsync(const ServerCallContext&, + std::shared_ptr, + std::shared_ptr) override { + handler_started.fetch_add(1); + return arrow::Future<>::Make(); // never resolved + } + + std::atomic handler_started{0}; +}; + +TEST(AsyncGrpcTest, DoExchangeClientCancelWithPendingHandlerFinishes) { + // The client walks away with the handler still parked: the transport must + // finish the RPC (and unwind reader/writer), not wait for the handler. + const int64_t before = detail::AsyncReactorBase::live_reactors(); + + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + HangingExchangeServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + ASSERT_OK_AND_ASSIGN(auto exchange, + client->DoExchange(FlightDescriptor::Command("hang"))); + // Wait until the handler is in (the descriptor reached it), so the cancel + // lands on a live handler rather than a race with dispatch. + ASSERT_TRUE(WaitFor([&flight_server] { + return flight_server.handler_started.load() >= 1; + })) << "the exchange handler never started"; + + exchange.reader->Cancel(); + exchange.writer.reset(); + exchange.reader.reset(); + ASSERT_OK(client->Close()); + + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + EXPECT_TRUE(WaitFor([&] { + return detail::AsyncReactorBase::live_reactors() == before; + })) << "a reactor leaked: still alive after the server stopped (before=" + << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; +} + +} // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc new file mode 100644 index 000000000000..ce9ccf04caff --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/transport/grpc/async_grpc_exchange_internal.h" + +namespace arrow::flight::transport::grpc::detail { + +/// The IpcPayloadWriter that queues the IPC payloads the writer stack +/// produces, the schema, dictionary and record-batch messages all travel this +/// way, as on the synchronous path. +class ExchangeWriter::PayloadSink final : public ipc::internal::IpcPayloadWriter { + public: + explicit PayloadSink(ExchangeWriter* writer) : writer_(writer) {} + + arrow::Status Start() override { return arrow::Status::OK(); } + + arrow::Status WritePayload(const ipc::IpcPayload& ipc_payload) override { + FlightPayload payload; + payload.ipc_message = ipc_payload; + if (ipc_payload.type == ipc::MessageType::RECORD_BATCH && + writer_->pending_app_metadata_ != nullptr) { + payload.app_metadata = std::move(writer_->pending_app_metadata_); + } + writer_->queue_.push_back(std::move(payload)); + return arrow::Status::OK(); + } + + arrow::Status Close() override { return arrow::Status::OK(); } + + private: + ExchangeWriter* writer_; +}; + +ExchangeWriter::ExchangeWriter(ExchangeReactor* reactor) : reactor_(reactor) {} + +void ExchangeWriter::DetachReactor() { + std::lock_guard lock(mutex_); + reactor_.store(nullptr); +} + +arrow::Future<> ExchangeWriter::BeginAsync(std::shared_ptr schema) { + if (reactor_.load() == nullptr) { + return arrow::Future<>::MakeFinished(arrow::Status::Cancelled( + "the exchange is over: the client disconnected or cancelled")); + } + if (batch_writer_ != nullptr) { + return arrow::Future<>::MakeFinished( + arrow::Status::Invalid("This writer has already been started.")); + } + auto sink = std::make_unique(this); + ARROW_ASSIGN_OR_RAISE(batch_writer_, ipc::internal::OpenRecordBatchWriter( + std::move(sink), schema, options_)); + return Flush(); +} + +arrow::Future<> ExchangeWriter::WriteRecordBatchAsync(const RecordBatch& batch) { + pending_app_metadata_ = nullptr; + return WriteBatch(batch); +} + +arrow::Future<> ExchangeWriter::WriteWithMetadataAsync( + const RecordBatch& batch, std::shared_ptr app_metadata) { + pending_app_metadata_ = std::move(app_metadata); + return WriteBatch(batch); +} + +arrow::Future<> ExchangeWriter::WriteMetadataAsync(std::shared_ptr app_metadata) { + FlightPayload payload; + payload.app_metadata = std::move(app_metadata); + queue_.push_back(std::move(payload)); + return Flush(); +} + +arrow::Future<> ExchangeWriter::CloseAsync() { + if (batch_writer_ != nullptr) { + auto status = batch_writer_->Close(); + batch_writer_ = nullptr; + if (!status.ok()) { + return arrow::Future<>::MakeFinished(std::move(status)); + } + } + return arrow::Future<>::MakeFinished(); +} + +void ExchangeWriter::OnWriteDone(bool ok) { + if (!ok) { + { + std::lock_guard lock(mutex_); + write_in_flight_ = false; + } + FailFlush(MakeFlightError(FlightStatusCode::Internal, "Failed to write response")); + return; + } + { + std::lock_guard lock(mutex_); + write_in_flight_ = false; + } + Pump(); +} + +void ExchangeWriter::OnCancelled() { + bool had_write; + { + std::lock_guard lock(mutex_); + had_write = write_in_flight_; + write_in_flight_ = false; + } + if (had_write) { + FailFlush(arrow::Status::Cancelled("the client cancelled the exchange")); + } +} + +arrow::Future<> ExchangeWriter::WriteBatch(const RecordBatch& batch) { + if (batch_writer_ == nullptr) { + return arrow::Future<>::MakeFinished(arrow::Status::Invalid( + "This writer is not started. Call BeginAsync() with a schema")); + } + auto status = batch_writer_->WriteRecordBatch(batch); + if (!status.ok()) { + return arrow::Future<>::MakeFinished(std::move(status)); + } + return Flush(); +} + +arrow::Future<> ExchangeWriter::Flush() { + arrow::Future<> flush; + { + std::lock_guard lock(mutex_); + if (reactor_.load() == nullptr) { + // The exchange is over: report it rather than pumping into a dead reactor. + return arrow::Future<>::MakeFinished(arrow::Status::Cancelled( + "the exchange is over: the client disconnected or cancelled")); + } + if (!flush_.is_valid() || flush_.is_finished()) { + flush_ = arrow::Future<>::Make(); + } + flush = flush_; + } + // Pump() takes the lock itself and may complete flush_ from a callback. + Pump(); + return flush; +} + +void ExchangeWriter::Pump() { + FlightPayload payload; + ExchangeReactor* reactor = nullptr; + enum class Action { kIdle, kDrained, kNoReactor, kWrite } action = Action::kIdle; + arrow::Future<> drained; + { + std::lock_guard lock(mutex_); + if (write_in_flight_) { + return; + } + reactor = reactor_.load(); + if (reactor != nullptr) { + reactor->Hold(); + } + if (queue_.empty()) { + drained = std::move(flush_); + flush_ = arrow::Future<>(); + action = Action::kDrained; + } else if (reactor == nullptr) { + write_in_flight_ = false; + action = Action::kNoReactor; + } else { + payload = std::move(queue_.front()); + queue_.pop_front(); + write_in_flight_ = true; + action = Action::kWrite; + } + } + // Everything below runs without mutex_. Every path that pinned above must + // unpin exactly once, including the ones that never reach WritePayload(). + switch (action) { + case Action::kDrained: + if (reactor != nullptr) { + reactor->ReleaseHold(); + } + if (drained.is_valid() && !drained.is_finished()) { + drained.MarkFinished(); + } + return; + case Action::kNoReactor: + // reactor is null here, so nothing was pinned. + FailFlush(arrow::Status::Cancelled( + "the exchange is over: the client disconnected or cancelled")); + return; + case Action::kWrite: { + const auto status = reactor->WritePayload(std::move(payload)); + reactor->ReleaseHold(); + if (!status.ok()) { + { + std::lock_guard lock(mutex_); + write_in_flight_ = false; + } + FailFlush(std::move(status)); + } + return; + } + case Action::kIdle: + return; + } +} + +void ExchangeWriter::FailFlush(Status status) { + // Claim the future under the lock, complete it outside: MarkFinished() runs + // continuations inline, which may re-enter the writer. + arrow::Future<> flush; + { + std::lock_guard lock(mutex_); + if (!flush_.is_valid() || flush_.is_finished()) { + return; + } + flush = std::move(flush_); + flush_ = arrow::Future<>(); + } + flush.MarkFinished(std::move(status)); +} + +} // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc new file mode 100644 index 000000000000..c64fc3d3efea --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/transport/grpc/async_grpc_service_internal.h" + +#include +#include + +namespace arrow::flight::transport::grpc::detail { + +namespace { + +/// Serve the Handshake RPC over the generic callback API. The handler is +/// synchronous, so every path finishes the RPC from the callback that started +/// it: no callback outlives its read or write, and the single reference the base +/// class holds is enough. +class HandshakeReactor final : public AsyncReactorBase { + public: + HandshakeReactor(AsyncCallContext flight_context, HandshakeFn handshake_handler) + : AsyncReactorBase(std::move(flight_context)), + handshake_handler_(std::move(handshake_handler)) { + StartRead(&request_buf_); + } + + void OnReadDone(bool ok) override { + if (!ok) { + FinishOnce(MakeFlightError(FlightStatusCode::Internal, "Failed to read request")); + return; + } + auto request = + ParseProtoRequest(request_buf_, "HandshakeRequest"); + if (!request.ok()) { + FinishOnce(std::move(request).status()); + return; + } + std::string response; + const auto status = + handshake_handler_(flight_context(), request->payload(), &response); + if (!status.ok()) { + FinishOnce(status); + return; + } + pb::HandshakeResponse pb_response; + pb_response.set_payload(std::move(response)); + write_buf_ = MakeWriteBuffer(pb_response); + WriteIfLive([this] { StartWrite(&write_buf_); }); + } + + void OnWriteDone(bool ok) override { + FinishOnce( + ok ? arrow::Status::OK() + : MakeFlightError(FlightStatusCode::Internal, "Failed to write response")); + } + + private: + HandshakeFn handshake_handler_; + ::grpc::ByteBuffer request_buf_; + ::grpc::ByteBuffer write_buf_; +}; + +} // namespace + +::grpc::ServerGenericBidiReactor* MakeHandshakeReactor(AsyncCallContext flight_context, + HandshakeFn handshake_handler) { + return new HandshakeReactor(std::move(flight_context), std::move(handshake_handler)); +} + +} // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc new file mode 100644 index 000000000000..a68e0abc5de9 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc @@ -0,0 +1,509 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Handshake and per-RPC token auth on the async generic service: the server +// class's Handshake()/ValidateToken() hooks, the token being required on data +// calls, payload fidelity (binary, empty), an abandoned or late handshake, and +// concurrency on fresh and reused channels. + +#include "arrow/flight/transport/grpc/async_grpc_test_fixtures.h" + +namespace arrow::flight::transport::grpc { + +// --------------------------------------------------------------------------- +// Handshake + per-RPC token auth on the async generic service. +// --------------------------------------------------------------------------- + +// Counters shared between the server callbacks and the test thread. +struct AsyncAuthState { + std::mutex mutex; + int handshakes = 0; + int validations = 0; + std::string peer_identity; +}; + +// Async-transport twin of TestServerAuthHandler: the handshake answers the +// client's password with the username, later RPCs must carry that password as +// the token, which is what TestClientAuthHandler::GetToken() puts in the +// `auth-token-bin` header (the sync twin validates it the same way in +// TestServerAuthHandler::IsValid). It is a server class: the two hooks below +// are the only configuration the transport needs. +class AsyncAuthTestServer : public AsyncGenericFlightServerBase { + public: + AsyncAuthTestServer(std::string username, std::string password, std::string ticket, + std::shared_ptr state = nullptr) + : username_(std::move(username)), + password_(std::move(password)), + ticket_(std::move(ticket)), + state_(state ? std::move(state) : std::make_shared()) {} + + const std::shared_ptr& state() const { return state_; } + + arrow::Future> DoGetAsync( + const ServerCallContext& context, const Ticket& request) override { + { + std::lock_guard guard(state_->mutex); + state_->peer_identity = context.peer_identity(); + } + if (request.ticket != ticket_) { + return arrow::Future>::MakeFinished( + Status::KeyError("No such ticket: ", request.ticket)); + } + auto schema = arrow::schema({arrow::field("a", arrow::int64())}); + auto batch = arrow::RecordBatch::Make( + schema, 2, {arrow::ArrayFromJSON(arrow::int64(), "[1, 2]")}); + ARROW_ASSIGN_OR_RAISE(auto reader, arrow::RecordBatchReader::Make({batch})); + return arrow::Future>::MakeFinished( + std::make_shared(std::move(reader))); + } + + // The hook: the transport read the client's password, answer with the + // username, or reject the RPC the way the sync twin does. + Status Handshake(const ServerCallContext& /*context*/, const std::string& password, + std::string* response) override { + { + std::lock_guard guard(state_->mutex); + state_->handshakes++; + } + if (password != password_) { + return MakeFlightError(FlightStatusCode::Unauthenticated, "Invalid token"); + } + *response = username_; + return Status::OK(); + } + + Status ValidateToken(const ServerCallContext& /*context*/, const std::string& token, + std::string* peer_identity) override { + std::lock_guard guard(state_->mutex); + state_->validations++; + if (token != password_) { + return MakeFlightError(FlightStatusCode::Unauthenticated, "Invalid token"); + } + *peer_identity = username_; + return Status::OK(); + } + + private: + std::string username_; + std::string password_; + std::string ticket_; + std::shared_ptr state_; +}; + +// A server with the async auth hooks installed, and a client connected to it. +// `status` carries the setup result, construction never fails a test. +struct AuthenticatedHarness { + AsyncAuthTestServer server{"user", "p4ssw0rd", "auth-ticket"}; + std::unique_ptr client; + Status status; + + AuthenticatedHarness() : status(Start()) {} + + ~AuthenticatedHarness() { + if (client) { + ARROW_WARN_NOT_OK(client->Close(), "Close()"); + } + ARROW_WARN_NOT_OK(server.Shutdown(), "Shutdown()"); + } + + private: + Status Start() { + ARROW_ASSIGN_OR_RAISE(auto location, Location::Parse("grpc://localhost:0")); + // The server class alone selects the async transport: deriving from it and + // overriding its virtuals are the whole configuration. + FlightServerOptions options(location); + RETURN_NOT_OK(server.Init(options)); + ARROW_ASSIGN_OR_RAISE(auto client_location, + Location::ForScheme("grpc", "127.0.0.1", server.port())); + ARROW_ASSIGN_OR_RAISE(client, FlightClient::Connect(client_location)); + return Status::OK(); + } +}; + +// A minimal server that mixes in the async hooks: the server class alone +// selects the async generic transport. +class AsyncGenericTestServer : public AsyncGenericFlightServerBase {}; + +TEST(AsyncGrpcTest, HandshakeWithoutHookIsUnimplemented) { + // A bare async server: Handshake is answered by the class's default virtual + // with UNIMPLEMENTED, and the client's Authenticate() surfaces it. + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + AsyncGenericTestServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto status = client->Authenticate( + {}, std::make_unique("user", "p4ssw0rd")); + ASSERT_FALSE(status.ok()) << "handshake must not succeed without a hook"; + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("authentication mechanism")); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, AsyncGenericFlightServerBaseImpliesAsyncTransport) { + // Deriving from the async class is enough: no hooks and no option to set, + // yet the RPC is served by the generic callback service. DoPut is the + // discriminator: an async server with no listener factory answers with its own + // message rather than "NYI". + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + AsyncGenericTestServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + // DoPut is the discriminator: an async server with no listener factory + // answers with its own message rather than "NYI". The refusal can surface on + // either call: FlightClient::DoPut + // writes the schema payload eagerly, so when the server's UNIMPLEMENTED + // arrives first the DoPut call itself fails, otherwise the writer's Close() + // reports it. + auto put = client->DoPut(FlightDescriptor::Path({"x"}), arrow::schema({})); + auto status = put.status(); + if (status.ok()) { + status = put->writer->Close(); + } + EXPECT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("no listener available")) + << "the async generic service must have served this call, not the sync path"; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, HandshakeAuthenticatesAndTokenIsRequired) { + AuthenticatedHarness harness; + ASSERT_OK(harness.status); + + // 1. Without a handshake the data call is rejected before it is served. + ASSERT_OK_AND_ASSIGN(auto unauthenticated_location, + Location::ForScheme("grpc", "127.0.0.1", harness.server.port())); + ASSERT_OK_AND_ASSIGN(auto unauthenticated_client, + FlightClient::Connect(unauthenticated_location)); + auto status = unauthenticated_client->DoGet(Ticket{"auth-ticket"}).status(); + EXPECT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid token")); + ASSERT_OK(unauthenticated_client->Close()); + + // 2. After the handshake the same call succeeds and the server sees the + // validated identity. + ASSERT_OK(harness.client->Authenticate( + {}, std::make_unique("user", "p4ssw0rd"))); + ASSERT_OK_AND_ASSIGN(auto reader, harness.client->DoGet(Ticket{"auth-ticket"})); + int batches = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto chunk, reader->Next()); + if (!chunk.data) break; + batches++; + } + EXPECT_EQ(1, batches); + { + std::lock_guard guard(harness.server.state()->mutex); + EXPECT_EQ(1, harness.server.state()->handshakes); + EXPECT_EQ("user", harness.server.state()->peer_identity); + EXPECT_GE(harness.server.state()->validations, 1); + } +} + +TEST(AsyncGrpcTest, BadPasswordIsRejectedAtHandshake) { + AuthenticatedHarness harness; + ASSERT_OK(harness.status); + auto status = harness.client->Authenticate( + {}, std::make_unique("user", "wrong")); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid token")); +} + +TEST(AsyncGrpcTest, UnauthenticatedUnknownMethodIsRejectedBeforeUnimplemented) { + AuthenticatedHarness harness; + ASSERT_OK(harness.status); + // GetFlightInfo is not served by the async generic service, but auth runs + // first: the client must see the auth failure, not UNIMPLEMENTED. + auto status = harness.client->GetFlightInfo(FlightDescriptor::Path({"x"})).status(); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid token")); + EXPECT_EQ(arrow::StatusCode::IOError, status.code()); +} + +// --------------------------------------------------------------------------- +// The synchronous request/response Handshake hook: payload fidelity, edge +// cases, and reactor cleanup (the reactor finishes on the strength of "one +// terminal path per direction", so the concurrency test is its real check). +// --------------------------------------------------------------------------- + +// Send `password`, require the server to echo `expect` back. +class EchoClientAuthHandler : public ClientAuthHandler { + public: + EchoClientAuthHandler(std::string password, std::string expect) + : password_(std::move(password)), expect_(std::move(expect)) {} + + Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) override { + ARROW_RETURN_NOT_OK(outgoing->Write(password_)); + ARROW_RETURN_NOT_OK(incoming->Read(&response_)); + if (response_ != expect_) { + return MakeFlightError(FlightStatusCode::Unauthenticated, + "unexpected handshake response"); + } + return Status::OK(); + } + + Status GetToken(std::string* token) override { + *token = password_; + return Status::OK(); + } + + const std::string& response() const { return response_; } + + private: + std::string password_; + std::string expect_; + std::string response_; +}; + +TEST(AsyncGrpcTest, HandshakePayloadIsBinarySafe) { + // The hook's request is a std::string taken from the protobuf and its + // response is echoed back through the client's reader: an embedded NUL and + // non-ASCII bytes must survive both directions unchanged. + const std::string binary("p4ss\0w0rd\xC3\xA9", 11); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + AsyncAuthTestServer server(binary, binary, "bin-ticket"); + FlightServerOptions options(location); + ASSERT_OK(server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(server.port())); + + auto handler = std::make_unique(binary, binary); + const auto* observed = handler.get(); + ASSERT_OK(client->Authenticate({}, std::move(handler))); + EXPECT_EQ(binary, observed->response()); + + // The token is the password, so an authenticated call must also work. + ASSERT_OK_AND_ASSIGN(auto reader, client->DoGet(Ticket{"bin-ticket"})); + ASSERT_OK_AND_ASSIGN(auto chunk, reader->Next()); + EXPECT_NE(nullptr, chunk.data); + + ASSERT_OK(client->Close()); + ASSERT_OK(server.Shutdown()); + ASSERT_OK(server.Wait()); +} + +TEST(AsyncGrpcTest, EmptyHandshakeRequestIsRejectedNotHung) { + // An empty HandshakeRequest is a valid protobuf message: the read succeeds, + // the hook sees an empty string, and its rejection must reach the client. + class SilentClientAuthHandler : public ClientAuthHandler { + public: + Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader*) override { + return outgoing->Write(""); + } + Status GetToken(std::string*) override { return Status::OK(); } + }; + + AuthenticatedHarness harness; + ASSERT_OK(harness.status); + auto status = + harness.client->Authenticate({}, std::make_unique()); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid token")); + std::lock_guard guard(harness.server.state()->mutex); + EXPECT_EQ(1, harness.server.state()->handshakes) + << "the hook must run on an empty request, not fail the read"; +} + +TEST(AsyncGrpcTest, AbandonedHandshakeLeavesServerUsable) { + // A client that never sends a message: the server's read completes with + // ok=false, the reactor must still finish, and the server must keep serving. + class AbandoningClientAuthHandler : public ClientAuthHandler { + public: + Status Authenticate(ClientAuthSender*, ClientAuthReader*) override { + return Status::Invalid("client never speaks"); + } + Status GetToken(std::string*) override { return Status::OK(); } + }; + + AuthenticatedHarness harness; + ASSERT_OK(harness.status); + EXPECT_FALSE( + harness.client->Authenticate({}, std::make_unique()) + .ok()); + + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(harness.server.port())); + ASSERT_OK(client->Authenticate( + {}, std::make_unique("user", "p4ssw0rd"))); + ASSERT_OK_AND_ASSIGN(auto reader, client->DoGet(Ticket{"auth-ticket"})); + ASSERT_OK_AND_ASSIGN(auto chunk, reader->Next()); + EXPECT_NE(nullptr, chunk.data); + ASSERT_OK(client->Close()); + + std::lock_guard guard(harness.server.state()->mutex); + EXPECT_EQ(1, harness.server.state()->handshakes) + << "the abandoned call must never have reached the hook"; +} + +TEST(AsyncGrpcTest, ConcurrentHandshakesAllComplete) { + constexpr int kClients = 32; + AuthenticatedHarness harness; + ASSERT_OK(harness.status); + + std::vector threads; + std::vector statuses(kClients); + for (int i = 0; i < kClients; i++) { + threads.emplace_back([&harness, &statuses, i] { + auto location = Location::ForScheme("grpc", "127.0.0.1", harness.server.port()); + if (!location.ok()) { + statuses[i] = location.status(); + return; + } + auto client = FlightClient::Connect(*location); + if (!client.ok()) { + statuses[i] = client.status(); + return; + } + auto authenticated = (*client)->Authenticate( + {}, std::make_unique("user", "p4ssw0rd")); + if (!authenticated.ok()) { + statuses[i] = authenticated; + return; + } + auto reader = (*client)->DoGet(Ticket{"auth-ticket"}); + if (!reader.ok()) { + statuses[i] = reader.status(); + return; + } + auto chunk = (*reader)->Next(); + if (!chunk.ok()) { + statuses[i] = chunk.status(); + return; + } + statuses[i] = (*client)->Close(); + }); + } + for (auto& thread : threads) thread.join(); + for (int i = 0; i < kClients; i++) { + EXPECT_TRUE(statuses[i].ok()) << "client " << i << ": " << statuses[i]; + } + + std::lock_guard guard(harness.server.state()->mutex); + EXPECT_EQ(kClients, harness.server.state()->handshakes); +} + +TEST(AsyncGrpcTest, RepeatedHandshakesOnFreshChannels) { + constexpr int kRounds = 50; + AuthenticatedHarness harness; + ASSERT_OK(harness.status); + for (int i = 0; i < kRounds; i++) { + ASSERT_OK_AND_ASSIGN(auto location, + Location::ForScheme("grpc", "127.0.0.1", harness.server.port())); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + ASSERT_OK(client->Authenticate( + {}, std::make_unique("user", "p4ssw0rd"))); + ASSERT_OK(client->Close()); + } + std::lock_guard guard(harness.server.state()->mutex); + EXPECT_EQ(kRounds, harness.server.state()->handshakes); +} + +TEST(AsyncGrpcTest, LateHandshakeResponseLeavesServerUsable) { + // The client gives up while the hook is still running, so the server's write + // fails: the reactor's write-side terminal path must still Finish, and the + // server must keep serving. + class SlowAnsweringServer : public AsyncGenericFlightServerBase { + public: + Status Handshake(const ServerCallContext&, const std::string&, + std::string* response) override { + handshakes.fetch_add(1); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + *response = "user"; + return Status::OK(); + } + std::atomic handshakes{0}; + }; + + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + SlowAnsweringServer server; + FlightServerOptions options(location); + ASSERT_OK(server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(server.port())); + + FlightCallOptions impatient; + impatient.timeout = std::chrono::milliseconds(50); + auto status = client->Authenticate( + impatient, std::make_unique("user", "p4ssw0rd")); + EXPECT_FALSE(status.ok()) << "the client's deadline must expire during the hook"; + EXPECT_NE(std::string::npos, status.ToString().find("Deadline")) + << "expected a deadline error, got: " << status.ToString(); + EXPECT_EQ(1, server.handshakes.load()) + << "the first call must have reached the hook, so its write had to finish"; + + ASSERT_OK_AND_ASSIGN(auto second_location, + Location::ForScheme("grpc", "127.0.0.1", server.port())); + ASSERT_OK_AND_ASSIGN(auto second_client, FlightClient::Connect(second_location)); + ASSERT_OK(second_client->Authenticate( + {}, std::make_unique("user", "p4ssw0rd"))); + + ASSERT_OK(second_client->Close()); + ASSERT_OK(client->Close()); + ASSERT_OK(server.Shutdown()); + ASSERT_OK(server.Wait()); +} + +TEST(AsyncGrpcTest, EmptyHandshakeResponseIsDelivered) { + // A hook that answers nothing still produces exactly one (empty) response + // message: the client's Read must return an empty string rather than hang. + class NoResponseServer : public AsyncGenericFlightServerBase { + public: + Status Handshake(const ServerCallContext&, const std::string&, + std::string*) override { + return Status::OK(); + } + }; + + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + NoResponseServer server; + FlightServerOptions options(location); + ASSERT_OK(server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(server.port())); + + auto handler = std::make_unique("anything", std::string()); + const auto* observed = handler.get(); + ASSERT_OK(client->Authenticate({}, std::move(handler))); + EXPECT_EQ(std::string(), observed->response()); + + ASSERT_OK(client->Close()); + ASSERT_OK(server.Shutdown()); + ASSERT_OK(server.Wait()); +} + +TEST(AsyncGrpcTest, RehandshakeOnTheSameChannelWorks) { + // The Handshake RPC is per-call, so authenticating twice on one channel must + // work: the second handshake reaches the hook and the channel keeps serving. + AuthenticatedHarness harness; + ASSERT_OK(harness.status); + ASSERT_OK(harness.client->Authenticate( + {}, std::make_unique("user", "p4ssw0rd"))); + ASSERT_OK(harness.client->Authenticate( + {}, std::make_unique("user", "p4ssw0rd"))); + ASSERT_OK_AND_ASSIGN(auto reader, harness.client->DoGet(Ticket{"auth-ticket"})); + ASSERT_OK_AND_ASSIGN(auto chunk, reader->Next()); + EXPECT_NE(nullptr, chunk.data); + + std::lock_guard guard(harness.server.state()->mutex); + EXPECT_EQ(2, harness.server.state()->handshakes); +} + +} // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc new file mode 100644 index 000000000000..e4daa60a85d8 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/transport/grpc/async_grpc_service.h" + +#include +#include +#include + +#include "arrow/flight/transport/grpc/async_grpc_service_internal.h" + +namespace arrow::flight::transport::grpc { + +namespace { + +namespace detail = arrow::flight::transport::grpc::detail; + +constexpr std::string_view kPrefix = "/arrow.flight.protocol.FlightService/"; +constexpr std::string_view kHandshakeMethod = "Handshake"; +constexpr std::string_view kDoGetMethod = "DoGet"; +constexpr std::string_view kDoPutMethod = "DoPut"; +constexpr std::string_view kDoActionMethod = "DoAction"; +constexpr std::string_view kListActionsMethod = "ListActions"; +constexpr std::string_view kDoExchangeMethod = "DoExchange"; +constexpr std::string_view kPollFlightInfoMethod = "PollFlightInfo"; +constexpr std::string_view kListFlightsMethod = "ListFlights"; +constexpr std::string_view kGetFlightInfoMethod = "GetFlightInfo"; +constexpr std::string_view kGetSchemaMethod = "GetSchema"; + +static const std::unordered_map kMethodMap = { + {kHandshakeMethod, FlightMethod::Handshake}, + {kListFlightsMethod, FlightMethod::ListFlights}, + {kGetFlightInfoMethod, FlightMethod::GetFlightInfo}, + {kGetSchemaMethod, FlightMethod::GetSchema}, + {kDoGetMethod, FlightMethod::DoGet}, + {kDoPutMethod, FlightMethod::DoPut}, + {kDoActionMethod, FlightMethod::DoAction}, + {kListActionsMethod, FlightMethod::ListActions}, + {kDoExchangeMethod, FlightMethod::DoExchange}, + {kPollFlightInfoMethod, FlightMethod::PollFlightInfo}, +}; + +FlightMethod MethodFromName(std::string_view method) { + if (!method.starts_with(kPrefix)) { + return FlightMethod::Invalid; + } + method.remove_prefix(kPrefix.size()); + const auto it = kMethodMap.find(method); + return it != kMethodMap.end() ? it->second : FlightMethod::Invalid; +} + +class Unimplemented : public ::grpc::ServerGenericBidiReactor { + public: + Unimplemented(detail::AsyncCallContext& flight_context, ::grpc::Status status) { + Finish(flight_context.FinishRequest(status)); + } + + explicit Unimplemented(::grpc::Status status) { Finish(status); } + + void OnDone() override { delete this; } +}; + +} // namespace + +AsyncGenericFlightService::AsyncGenericFlightService( + AsyncGenericFlightServerBase* async_base, + std::shared_ptr> helper, + HandshakeFn handshake_handler) + : base_(async_base), + helper_(std::move(helper)), + handshake_handler_(std::move(handshake_handler)) {} + +::grpc::ServerGenericBidiReactor* AsyncGenericFlightService::CreateReactor( + ::grpc::GenericCallbackServerContext* context) { + const std::string_view full_method = context->method(); + const std::string_view method = + full_method.starts_with(kPrefix) ? full_method.substr(kPrefix.size()) : full_method; + const FlightMethod flight_method = MethodFromName(full_method); + detail::AsyncCallContext flight_context(context); + + const auto prepare_status = + method == kHandshakeMethod + ? helper_->MakeCallContext(flight_method, context, &flight_context) + : helper_->CheckAuth(flight_method, context, &flight_context); + if (!prepare_status.ok()) { + // CheckAuth()/MakeCallContext() already ran FinishRequest() for this call, + // so only the gRPC status is left to send. + return new Unimplemented(prepare_status); + } + + switch (flight_method) { + case FlightMethod::Handshake: + if (!handshake_handler_) { + return new Unimplemented( + flight_context, + ::grpc::Status( + ::grpc::StatusCode::UNIMPLEMENTED, + "This service does not have an authentication mechanism enabled.")); + } + return detail::MakeHandshakeReactor(std::move(flight_context), handshake_handler_); + case FlightMethod::GetFlightInfo: + return detail::MakeGetFlightInfoReactor(std::move(flight_context), base_); + case FlightMethod::GetSchema: + return detail::MakeGetSchemaReactor(std::move(flight_context), base_); + case FlightMethod::PollFlightInfo: + return detail::MakePollFlightInfoReactor(std::move(flight_context), base_); + case FlightMethod::ListActions: + return detail::MakeListActionsReactor(std::move(flight_context), base_); + case FlightMethod::DoAction: + return detail::MakeDoActionReactor(std::move(flight_context), base_); + case FlightMethod::ListFlights: + return detail::MakeListFlightsReactor(std::move(flight_context), base_); + case FlightMethod::DoExchange: + return detail::MakeExchangeReactor(std::move(flight_context), base_); + case FlightMethod::DoGet: + return detail::MakeDoGetReactor(std::move(flight_context), base_); + case FlightMethod::DoPut: { + std::shared_ptr listener = + base_->CreateDoPutListener(flight_context); + if (!listener) { + return new Unimplemented( + flight_context, + ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, + "DoPut is not implemented: no listener available")); + } + return detail::MakeDoPutReactor(std::move(flight_context), std::move(listener)); + } + case FlightMethod::Invalid: + break; + } + return new Unimplemented( + flight_context, + ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "Unknown method")); +} + +} // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.h new file mode 100644 index 000000000000..ae4f13a5fe02 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.h @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/flight/transport/grpc/async_grpc_compat.h" +#include "arrow/flight/transport/grpc/grpc_server_internal.h" + +#include "arrow/flight/server_async.h" + +namespace arrow::flight::transport::grpc { + +#if ARROW_FLIGHT_HAS_ASYNC_SERVER + +using HandshakeFn = + GrpcServerCallContextHelper<::grpc::CallbackServerContext>::HandshakeFn; + +/// \brief Generic callback service dispatching Flight methods to reactors. +/// +/// Constructed with the AsyncGenericFlightServerBase whose handlers are served. +/// The server's middleware runs for every call, through the shared context +/// helper, which also authenticates every method except Handshake. Handshake, +/// DoGet, DoPut and the synchronous Flight methods (GetFlightInfo, GetSchema, +/// PollFlightInfo, ListActions, DoAction, ListFlights, DoExchange) have a +/// reactor. Every other method is answered UNIMPLEMENTED. + +// Silence warning +// "non dll-interface class CallbackGenericService used as base for dll-interface class" +# ifdef _MSC_VER +# pragma warning(push) +# pragma warning(disable : 4275) +# endif +class ARROW_FLIGHT_EXPORT AsyncGenericFlightService + : public ::grpc::CallbackGenericService { + public: + /// \param async_base is the async server whose handlers are served, it must + /// outlive the service. + /// \param helper runs middleware and auth and builds the call context. + /// \param handshake_handler is the server's Handshake hook, an empty hook + /// means the server has no authentication mechanism. + AsyncGenericFlightService( + AsyncGenericFlightServerBase* async_base, + std::shared_ptr> helper, + HandshakeFn handshake_handler = {}); + + /// \brief Creates a reactor for the incoming RPC. + /// \param context is the gRPC callback server context for the RPC. + /// \return a new reactor handling the RPC, or an Unimplemented reactor if the method is + /// unknown. + ::grpc::ServerGenericBidiReactor* CreateReactor( + ::grpc::GenericCallbackServerContext* context) override; + + private: + AsyncGenericFlightServerBase* base_; + std::shared_ptr> helper_; + /// The server class's Handshake hook. Empty when the server has none. + HandshakeFn handshake_handler_; +}; +# ifdef _MSC_VER +# pragma warning(pop) +# endif + +#endif // ARROW_FLIGHT_HAS_ASYNC_SERVER + +} // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h new file mode 100644 index 000000000000..394a61b4512a --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "arrow/flight/flight_data_decoder.h" +#include "arrow/flight/serialization_internal.h" +#include "arrow/flight/server_async.h" +#include "arrow/flight/transport/grpc/async_grpc_compat.h" +#include "arrow/flight/transport/grpc/async_grpc_service.h" +#include "arrow/flight/transport/grpc/serialization_internal.h" + +namespace arrow::flight::transport::grpc::detail { + +namespace pb = arrow::flight::protocol; + +#if ARROW_FLIGHT_HAS_ASYNC_SERVER + +using AsyncCallContext = GrpcServerCallContext<::grpc::CallbackServerContext>; + +template +::grpc::ByteBuffer MakeWriteBuffer(const ProtoT& message) { + const std::string bytes = message.SerializeAsString(); + ::grpc::Slice slice(bytes); + return {&slice, 1}; +} + +template +arrow::Result ParseProtoRequest(const ::grpc::ByteBuffer& buf, + std::string_view what) { + ARROW_ASSIGN_OR_RAISE(std::string bytes, + BytesFromBuffer(buf, "Failed to read request")); + PbT pb; + if (!pb.ParseFromArray(bytes.data(), static_cast(bytes.size()))) { + return arrow::Status::Invalid("Failed to parse ", what); + } + return pb; +} + +template +arrow::Result ParseProtoRequest(const ::grpc::ByteBuffer& buf, std::string_view what) { + ARROW_ASSIGN_OR_RAISE(auto pb, ParseProtoRequest(buf, what)); + T out; + ARROW_RETURN_NOT_OK(arrow::flight::internal::FromProto(pb, &out)); + return out; +} + +template +arrow::Status SerializeOrNotFound(const std::shared_ptr& value, PbT* out) { + if (value == nullptr) { + return arrow::Status::KeyError("Flight not found"); + } + return arrow::flight::internal::ToProto(*value, out); +} + +/// \brief Lifetime and termination handling shared by the callback reactors. +/// +/// A reactor's callbacks run on arbitrary gRPC threads, so it cannot be +/// destroyed while one is outstanding: the RPC's own lifetime holds the initial +/// reference (released in OnDone(), after which gRPC never touches the reactor), +/// and every callback that may outlive its StartRead()/StartWrite() takes +/// another one. The last ReleaseHold() deletes the reactor. +class AsyncReactorBase : public ::grpc::ServerGenericBidiReactor { + public: + explicit AsyncReactorBase(AsyncCallContext flight_context) + : flight_context_(std::move(flight_context)) { + live_reactors_.fetch_add(1, std::memory_order_relaxed); + } + + ~AsyncReactorBase() override { live_reactors_.fetch_sub(1, std::memory_order_relaxed); } + + /// \brief How many reactors are alive right now. + /// + /// A reactor is deleted by the last ReleaseHold(), so this returns to its + /// starting value once every RPC (including the ones whose handler abandoned + /// its future) has been torn down. + static int64_t live_reactors() { + return live_reactors_.load(std::memory_order_relaxed); + } + + /// The default cancellation: report the RPC as cancelled and finish it. + /// Reactors with their own teardown override this and call it first. + void OnCancel() override { FinishOnce(arrow::Status::Cancelled()); } + + void OnDone() override { ReleaseHold(); } + + protected: + /// \brief Finish the RPC with this status, later calls are no-ops. + /// + /// The status is the one the client sees, so the first one wins: the other + /// paths (cancel after error, error after cancel) must not overwrite it. + void FinishOnce(arrow::Status status) { + bool expected = false; + if (finished_.compare_exchange_strong(expected, true)) { + std::lock_guard lock(write_mutex_); + Finish(flight_context_.FinishRequest(status)); + } + } + + /// Whether FinishOnce() has already run (the RPC's status is settled). + bool finished() const { return finished_.load(); } + + /// The call context: handlers and the teardown paths need it. + AsyncCallContext& flight_context() { return flight_context_; } + + /// \brief Run `start_write` unless the RPC has already been finished. + /// + /// A handler continuation runs on the application's thread and may pass a + /// bare finished() check just before another path (a cancel, a deadline) + /// calls Finish() and gRPC tears the call down. Running the write under + /// write_mutex_ closes that window: FinishOnce() takes the same mutex, so + /// either the write wins and Finish() waits for it to be issued, or + /// finished() is already set and no write is started. A HoldToken keeps the + /// reactor alive but not the underlying gRPC call, which is why the check + /// alone is not enough. + /// \return true if `start_write` ran. + template + bool WriteIfLive(StartWriteFn&& start_write) { + std::lock_guard lock(write_mutex_); + if (finished()) { + return false; + } + start_write(); + return true; + } + + /// \brief The mutex guarding write initiation, see WriteIfLive(). + std::mutex& write_mutex() { return write_mutex_; } + + /// Account for work that may outlive the callback that started it. + void Hold() { refs_.fetch_add(1, std::memory_order_relaxed); } + + /// Drop a reference, deleting the reactor once the last one is gone. + void ReleaseHold() { + if (refs_.fetch_sub(1, std::memory_order_acq_rel) == 1) { + delete this; + } + } + + /// A reference the reactor owns, to be handed to a callback. + class HoldToken { + public: + HoldToken() = default; + explicit HoldToken(AsyncReactorBase* reactor) : reactor_(reactor) { + reactor_->Hold(); + } + HoldToken(const HoldToken&) = delete; + HoldToken& operator=(const HoldToken&) = delete; + HoldToken(HoldToken&& other) noexcept + : reactor_(std::exchange(other.reactor_, nullptr)) {} + HoldToken& operator=(HoldToken&& other) noexcept { + if (this != &other) { + Release(); + reactor_ = std::exchange(other.reactor_, nullptr); + } + return *this; + } + ~HoldToken() { Release(); } + + /// \brief The reference, to be captured by the callback and moved out of. + HoldToken&& release() { return std::move(*this); } + + private: + void Release() { + if (reactor_ != nullptr) { + reactor_->ReleaseHold(); + reactor_ = nullptr; + } + } + AsyncReactorBase* reactor_ = nullptr; + }; + + /// Take a reference and return the token that holds it. + HoldToken hold() { return HoldToken(this); } + + private: + AsyncCallContext flight_context_; + std::mutex write_mutex_; + std::atomic finished_{false}; + std::atomic refs_{1}; + inline static std::atomic live_reactors_{0}; +}; + +::grpc::ServerGenericBidiReactor* MakeHandshakeReactor(AsyncCallContext flight_context, + HandshakeFn handshake_handler); +::grpc::ServerGenericBidiReactor* MakeGetFlightInfoReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); +::grpc::ServerGenericBidiReactor* MakeGetSchemaReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); +::grpc::ServerGenericBidiReactor* MakePollFlightInfoReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); +::grpc::ServerGenericBidiReactor* MakeDoGetReactor(AsyncCallContext flight_context, + AsyncGenericFlightServerBase* base); +::grpc::ServerGenericBidiReactor* MakeDoPutReactor( + AsyncCallContext flight_context, std::shared_ptr listener); +::grpc::ServerGenericBidiReactor* MakeListActionsReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); +::grpc::ServerGenericBidiReactor* MakeDoActionReactor(AsyncCallContext flight_context, + AsyncGenericFlightServerBase* base); +::grpc::ServerGenericBidiReactor* MakeListFlightsReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); +::grpc::ServerGenericBidiReactor* MakeExchangeReactor(AsyncCallContext flight_context, + AsyncGenericFlightServerBase* base); + +#endif // ARROW_FLIGHT_HAS_ASYNC_SERVER + +} // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc new file mode 100644 index 000000000000..086ab00938ed --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Dispatch-level tests of the async generic gRPC Flight service: the +// UNIMPLEMENTED fallback reactor, middleware on every call (including the calls +// that never reach a handler), the options an async server accepts, and the two +// ways it can be started -- its own Init() or a hand-built grpc::ServerBuilder. +// +// Per-RPC behaviour lives in async_grpc_do_get_test.cc, async_grpc_do_put_test.cc, +// async_grpc_unary_test.cc, async_grpc_streaming_test.cc and +// async_grpc_exchange_test.cc. + +#include "arrow/flight/transport/grpc/async_grpc_test_fixtures.h" + +namespace arrow::flight::transport::grpc { + +TEST(AsyncGrpcTest, MiddlewareCallCompletedRunsOnUnknownMethod) { + // Middleware that started for a call must also see CallCompleted when the + // call is answered by the UNIMPLEMENTED fallback reactor: otherwise tracing + // middleware leaks a span per unknown/unimplemented RPC. + auto trace = std::make_shared(); + TestFlightServer flight_server; + auto helper = + std::make_shared>( + /*auth_handler=*/nullptr, + MiddlewareFactoryList{ + {"recording", std::make_shared(trace)}}); + AsyncGenericFlightService service(&flight_server, helper); + + int port = 0; + ::grpc::ServerBuilder builder; + builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.RegisterCallbackGenericService(&service); + auto server = builder.BuildAndStart(); + ASSERT_NE(server, nullptr); + ASSERT_NE(port, 0); + + const ::grpc::Status status = CallUnknownMethod(port); + ASSERT_EQ(::grpc::StatusCode::UNIMPLEMENTED, status.error_code()) + << status.error_message(); + + server->Shutdown(); + server->Wait(); + + std::lock_guard guard(trace->mutex); + EXPECT_EQ(1, trace->start_call) << "the middleware must have started for the call"; + EXPECT_EQ(1, trace->call_completed) + << "CallCompleted must run for an UNIMPLEMENTED reply too"; +} + +TEST(AsyncGrpcTest, OtherMethodsAreUnimplemented) { + TestFlightServer flight_server; + auto listener = std::make_shared(); + flight_server.set_listener(listener); + AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); + + int port = 0; + ::grpc::ServerBuilder builder; + builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.RegisterCallbackGenericService(&service); + auto server = builder.BuildAndStart(); + ASSERT_NE(server, nullptr); + ASSERT_NE(port, 0); + + std::string uri = "grpc://localhost:" + std::to_string(port); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + + // Every typed FlightService method is served now, the fallback reactor is + // still reachable through a method name no stub knows (which maps to + // FlightMethod::Invalid in MethodFromName), and that is the only pin left + // that an unknown method cannot be answered by a typed path. + const ::grpc::Status status = CallUnknownMethod(port); + ASSERT_FALSE(status.ok()) << "an unknown method must not be served"; + ASSERT_EQ(::grpc::StatusCode::UNIMPLEMENTED, status.error_code()) + << status.error_message(); + + ASSERT_OK(client->Close()); + server->Shutdown(); + server->Wait(); +} + +TEST(AsyncGrpcTest, UseAsyncGrpcFlag) { + ASSERT_OK_AND_ASSIGN(auto default_location, Location::Parse("grpc://localhost:0")); + FlightServerOptions default_options(default_location); + + // Same adapter as the tests above, but now started through the async + // server's own Init instead of a hand-built grpc::ServerBuilder. + TestFlightServer flight_server; + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_GT(flight_server.port(), 0); + + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + // (i) DoGet went through the generic reactor: the ticket reached + // TestFlightServer::DoGetAsync and the whole stream came back. + Ticket ticket{"flag-path-do-get-ticket"}; + ASSERT_OK_AND_ASSIGN(auto reader, client->DoGet(ticket)); + ASSERT_OK_AND_ASSIGN(auto schema, reader->GetSchema()); + ASSERT_EQ(schema->num_fields(), 2); + ASSERT_EQ(schema->field(0)->name(), "a"); + ASSERT_EQ(schema->field(1)->name(), "b"); + int batch_count = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto chunk, reader->Next()); + if (!chunk.data) break; + ASSERT_EQ(chunk.data->num_rows(), 5); + ASSERT_OK_AND_ASSIGN(auto value, chunk.data->column(0)->GetScalar(0)); + ASSERT_TRUE(value->Equals(*arrow::MakeScalar(int64_t(1)))); + batch_count++; + } + ASSERT_EQ(batch_count, 3); + + // (ii) Every typed method is served on this path too, the fallback reactor + // still answers a method name no stub knows, which is what pins "the typed + // sync service is NOT registered alongside the generic one". + const ::grpc::Status unknown_status = CallUnknownMethod(flight_server.port()); + ASSERT_FALSE(unknown_status.ok()) << "an unknown method must not be served"; + ASSERT_EQ(::grpc::StatusCode::UNIMPLEMENTED, unknown_status.error_code()) + << unknown_status.error_message(); + + // (iii) The async server takes no listener factory here, so uploads are + // refused the same way. + FlightDescriptor descriptor = FlightDescriptor::Path({"not-served"}); + auto put_schema = arrow::schema({arrow::field("a", arrow::int64())}); + auto batch = arrow::RecordBatch::Make(put_schema, 1, + {arrow::ArrayFromJSON(arrow::int64(), "[1]")}); + auto [surface, status] = UploadOneBatch(client.get(), descriptor, put_schema, batch); + ASSERT_FALSE(status.ok()) << "DoPut must be refused, surfaced on " << surface; + ASSERT_EQ(status.code(), arrow::StatusCode::NotImplemented); + + // Cleanup. The ticket is asserted only after the server stopped, so the + // DoGet callback cannot race with the read. + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + ASSERT_EQ(flight_server.ticket(), "flag-path-do-get-ticket"); +} + +TEST(AsyncGrpcTest, UseAsyncGrpcFlagServesDoPut) { + // The async server serves uploads because the server class hands out the + // per-RPC listener through CreateDoPutListener(), instead of the RPC falling + // through to UNIMPLEMENTED. + TestFlightServer flight_server; + auto listener = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + FlightServerOptions options(location); + flight_server.set_listener(listener); + ASSERT_OK(flight_server.Init(options)); + ASSERT_GT(flight_server.port(), 0); + + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = arrow::schema( + {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); + auto batch = + arrow::RecordBatch::Make(schema, 3, + {arrow::ArrayFromJSON(arrow::int64(), "[1, 2, 3]"), + arrow::ArrayFromJSON(arrow::int64(), "[10, 20, 30]")}); + + const FlightDescriptor descriptor = FlightDescriptor::Path({"wave", "h2"}); + auto [surface, status] = UploadOneBatch(client.get(), descriptor, schema, batch); + ASSERT_TRUE(status.ok()) << "upload failed on " << surface << ": " << status; + + // Cleanup before looking at the listener: its callbacks run on a gRPC + // thread, and nothing may read the recordings while that thread writes. + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + // The listener saw the descriptor the client sent, the schema, and the batch. + ASSERT_EQ(listener->descriptor_count(), 1); + ASSERT_EQ(listener->descriptors()[0], descriptor); + ASSERT_EQ(listener->schema_count(), 1); + ASSERT_EQ(listener->schema()->num_fields(), 2); + ASSERT_EQ(listener->schema()->field(0)->name(), "a"); + ASSERT_EQ(listener->schema()->field(1)->name(), "b"); + ASSERT_EQ(listener->batches().size(), 1); + const std::shared_ptr chunk = listener->batches()[0]; + ASSERT_EQ(chunk->num_rows(), 3); + ASSERT_EQ(chunk->num_columns(), 2); + ASSERT_OK_AND_ASSIGN(auto value, chunk->column(0)->GetScalar(0)); + ASSERT_TRUE(value->Equals(*arrow::MakeScalar(int64_t(1)))); +} + +TEST(AsyncGrpcTest, UseAsyncGrpcFlagRefusesBlockingAuthHandler) { + // The callback path cannot run the blocking ServerAuthHandler (it would run + // on callback threads), so a server configured with one must refuse to start + // instead of silently serving unauthenticated requests. Middleware *is* + // supported now - see UseAsyncGrpcFlagAllowsMiddleware. + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + + TestFlightServer auth_server; + FlightServerOptions with_auth(location); + with_auth.auth_handler = std::make_shared("user", "pass"); + auto auth_status = auth_server.Init(with_auth); + ASSERT_FALSE(auth_status.ok()) << "an auth handler must not be silently skipped"; + ASSERT_EQ(auth_status.code(), arrow::StatusCode::NotImplemented); + ASSERT_NE(auth_status.message().find("AsyncGenericFlightServerBase"), std::string::npos) + << auth_status; +} + +TEST(AsyncGrpcTest, UseAsyncGrpcFlagAllowsMiddleware) { + // Middleware configured through FlightServerOptions must run on the async + // path: StartCall at call creation, SendingHeaders and CallCompleted at the + // end - success or failure alike. + auto trace = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + + TestFlightServer flight_server; + FlightServerOptions options(location); + options.middleware = { + {"recording", std::make_shared(trace)}}; + ASSERT_OK(flight_server.Init(options)); + + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + ASSERT_OK_AND_ASSIGN(auto reader, client->DoGet(Ticket{"middleware-ticket"})); + while (true) { + ASSERT_OK_AND_ASSIGN(auto chunk, reader->Next()); + if (!chunk.data) break; + } + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + std::lock_guard guard(trace->mutex); + EXPECT_EQ(1, trace->start_call); + EXPECT_EQ(1, trace->sending_headers); + EXPECT_EQ(1, trace->call_completed); +} + +TEST(AsyncGrpcTest, UseAsyncGrpcFlagMiddlewareCanRejectCall) { + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + TestFlightServer flight_server; + FlightServerOptions options(location); + options.middleware = { + {"rejecting", std::make_shared()}}; + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto status = client->DoGet(Ticket{"rejected"}).status(); + EXPECT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("rejected by middleware")); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +} // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc new file mode 100644 index 000000000000..61757c0474ab --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/transport/grpc/async_grpc_service_internal.h" + +#include +#include +#include +#include +#include + +namespace arrow::flight::transport::grpc::detail { + +namespace { + +/** + * A generic streaming reactor for handling async gRPC calls in the Flight service. + * One request message is read, followed by N response messages, then the stream finishes. + */ +class StreamingReactor : public AsyncReactorBase { + public: + StreamingReactor(AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) + : AsyncReactorBase(std::move(flight_context)), base_(base) { + StartRead(&request_buf_); + } + + void OnReadDone(bool ok) override { + if (!ok) { + FinishOnce(MakeFlightError(FlightStatusCode::Internal, "Failed to read request")); + return; + } + Start().AddCallback([this, token = hold()](const arrow::Status& status) { + if (!finished()) { + if (!status.ok()) { + FinishOnce(status); + } else { + WriteNextMessage(); + } + } + }); + } + + void OnWriteDone(bool ok) override { + if (finished()) { + return; + } + if (!ok) { + FinishOnce(MakeFlightError(FlightStatusCode::Internal, "Failed to write response")); + return; + } + WriteNextMessage(); + } + + protected: + virtual arrow::Future<> Start() = 0; + virtual arrow::Future NextMessage() = 0; + + void WriteNextMessage() { + if (finished()) { + return; + } + NextMessage().AddCallback( + [this, token = hold()](const arrow::Result& has_next) { + if (!finished()) { + if (!has_next.ok()) { + FinishOnce(has_next.status()); + } else if (!*has_next) { + FinishOnce(arrow::Status::OK()); + } else { + WriteIfLive([this] { StartWrite(&write_buf_); }); + } + } + }); + } + + /// \brief Guards the subclass state that Start()'s continuation publishes and + /// NextMessage() consumes. NextMessage() runs on a gRPC callback thread + /// (OnWriteDone) while Start()'s continuation runs on the handler's, so both + /// share this one lock rather than each reactor declaring its own. + std::mutex& state_mutex() { return state_mutex_; } + + protected: + AsyncGenericFlightServerBase* base_; + ::grpc::ByteBuffer request_buf_; + ::grpc::ByteBuffer write_buf_; + + private: + std::mutex state_mutex_; +}; + +class ListActionsReactor final : public StreamingReactor { + public: + using StreamingReactor::StreamingReactor; + + protected: + arrow::Future<> Start() override { + return base_->ListActionsAsync(flight_context()) + .Then([this](const std::vector& actions) -> arrow::Status { + // Published on the handler's thread, read from NextMessage() which + // OnWriteDone() also calls from a gRPC thread: guard both. + std::lock_guard lock(state_mutex()); + actions_ = actions; + next_ = 0; + return arrow::Status::OK(); + }); + } + + arrow::Future NextMessage() override { + pb::ActionType pb_type; + { + std::lock_guard lock(state_mutex()); + if (next_ >= actions_.size()) { + return arrow::Future::MakeFinished(false); + } + const auto status = internal::ToProto(actions_[next_++], &pb_type); + if (!status.ok()) { + return arrow::Future::MakeFinished(status); + } + } + write_buf_ = MakeWriteBuffer(pb_type); + return arrow::Future::MakeFinished(true); + } + + private: + std::vector actions_; + size_t next_ = 0; +}; + +class DoActionReactor final : public StreamingReactor { + public: + using StreamingReactor::StreamingReactor; + + protected: + arrow::Future<> Start() override { + auto action = ParseProtoRequest(request_buf_, "Action"); + if (!action.ok()) { + return arrow::Future<>::MakeFinished(action.status()); + } + return base_->DoActionAsync(flight_context(), *action) + .Then([this](const std::shared_ptr& stream) -> arrow::Status { + if (stream == nullptr) { + return arrow::Status::Cancelled(); + } + { + std::lock_guard lock(state_mutex()); + stream_ = stream; + } + return arrow::Status::OK(); + }); + } + + arrow::Future NextMessage() override { + std::shared_ptr stream; + { + std::lock_guard lock(state_mutex()); + stream = stream_; + } + if (stream == nullptr) { + return arrow::Future::MakeFinished(false); + } + return stream->NextAsync().Then( + [this](const std::shared_ptr& result) -> arrow::Result { + if (result == nullptr) { + return false; + } + pb::Result pb_result; + ARROW_RETURN_NOT_OK(internal::ToProto(*result, &pb_result)); + write_buf_ = MakeWriteBuffer(pb_result); + return true; + }); + } + + private: + std::shared_ptr stream_; +}; + +class ListFlightsReactor final : public StreamingReactor { + public: + using StreamingReactor::StreamingReactor; + + protected: + arrow::Future<> Start() override { + auto criteria = ParseProtoRequest(request_buf_, "Criteria"); + if (!criteria.ok()) { + return arrow::Future<>::MakeFinished(criteria.status()); + } + criteria_ = *criteria; + return base_->ListFlightsAsync(flight_context(), &criteria_) + .Then( + [this](const std::shared_ptr& listing) -> arrow::Status { + std::lock_guard lock(state_mutex()); + listing_ = listing; + return arrow::Status::OK(); + }); + } + + arrow::Future NextMessage() override { + std::shared_ptr listing; + { + std::lock_guard lock(state_mutex()); + listing = listing_; + } + if (listing == nullptr) { + return arrow::Future::MakeFinished(false); + } + return listing->NextAsync().Then( + [this](const std::shared_ptr& info) -> arrow::Result { + if (info == nullptr) { + return false; + } + pb::FlightInfo pb_info; + ARROW_RETURN_NOT_OK(internal::ToProto(*info, &pb_info)); + write_buf_ = MakeWriteBuffer(pb_info); + return true; + }); + } + + private: + Criteria criteria_; + std::shared_ptr listing_; +}; + +} // namespace + +::grpc::ServerGenericBidiReactor* MakeListActionsReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { + return new ListActionsReactor(std::move(flight_context), base); +} + +::grpc::ServerGenericBidiReactor* MakeDoActionReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { + return new DoActionReactor(std::move(flight_context), base); +} + +::grpc::ServerGenericBidiReactor* MakeListFlightsReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { + return new ListFlightsReactor(std::move(flight_context), base); +} + +} // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc new file mode 100644 index 000000000000..9b0a8cadd2be --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The streaming RPCs served by StreamingReactor (one request in, N responses +// out, driven by OnWriteDone): ListActions (a vector), DoAction (an +// AsyncResultStream) and ListFlights (an AsyncFlightListing). Covers the +// empty-stream case (no write is ever armed), the mid-stream error rule, a +// listing completed on another thread, and a handler deadline. + +#include "arrow/flight/transport/grpc/async_grpc_test_fixtures.h" + +namespace arrow::flight::transport::grpc { + +// --------------------------------------------------------------------------- +// The streaming trio: ListActions (a vector), DoAction (an AsyncResultStream) +// and ListFlights (an AsyncFlightListing) share StreamingReactor (one request +// in, N responses out, driven by OnWriteDone -> WriteNext). Each RPC has a +// happy-path test, the reactor never writing a message must still finish the +// RPC (the empty tests), and the two iterator-backed ones pin the mid-stream +// error rule. +// --------------------------------------------------------------------------- + +TEST(AsyncGrpcTest, ListActionsIsServedAsync) { + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + ASSERT_OK_AND_ASSIGN(auto actions, client->ListActions()); + ASSERT_EQ(actions.size(), 2); + ASSERT_EQ(actions[0].type, "echo"); + ASSERT_EQ(actions[0].description, "echo back"); + ASSERT_EQ(actions[1].type, "fail"); + ASSERT_EQ(actions[1].description, "always fails"); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, ListActionsEmptyFinishesCleanly) { + // Zero response messages: WriteNext() finds the vector exhausted before it + // ever calls StartWrite(), so the RPC must be finished with OK rather than + // left waiting for a write completion that will never arrive. + TestFlightServer flight_server; + flight_server.set_empty_actions(); + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + ASSERT_OK_AND_ASSIGN(auto actions, client->ListActions()); + ASSERT_TRUE(actions.empty()); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, DoActionIsServedAsync) { + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + const Action action{"ping", Buffer::FromString("request-body")}; + ASSERT_OK_AND_ASSIGN(auto results, client->DoAction(action)); + ASSERT_OK_AND_ASSIGN(auto result, results->Next()); + ASSERT_NE(result, nullptr); + ASSERT_EQ(result->body->ToString(), "one"); + // The stream ends with a null Result, not with an error. + ASSERT_OK_AND_ASSIGN(auto end, results->Next()); + ASSERT_EQ(end, nullptr); + + // Cleanup before looking at the recording: it was written on a gRPC thread. + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + ASSERT_EQ("ping", flight_server.last_action_type()); +} + +TEST(AsyncGrpcTest, DoActionStreamsMultipleResults) { + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + const Action action{"multi", Buffer::FromString("")}; + ASSERT_OK_AND_ASSIGN(auto results, client->DoAction(action)); + + ASSERT_OK_AND_ASSIGN(auto bodies, DrainResultBodies(results.get())); + ASSERT_EQ(bodies, (std::vector{"one", "two", "three"})); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, DoActionMidStreamErrorFinishesTheRpc) { + // The stream yields one Result and then fails. The error must become the + // RPC status -- the client must not see a truncated success. + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + const Action action{"fail-mid-stream", Buffer::FromString("")}; + ASSERT_OK_AND_ASSIGN(auto results, client->DoAction(action)); + ASSERT_OK_AND_ASSIGN(auto first, results->Next()); + ASSERT_NE(first, nullptr); + ASSERT_EQ(first->body->ToString(), "first"); + + auto second = results->Next(); + ASSERT_FALSE(second.ok()) << "the mid-stream error must reach the client"; + ASSERT_EQ(second.status().code(), arrow::StatusCode::IOError); + ASSERT_THAT(second.status().message(), + ::testing::HasSubstr("mid-stream action failure")); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, DoActionWithNullResultStreamIsCancelled) { + // A handler that answers OK without a ResultStream answers CANCELLED on the + // sync transport (grpc_server.cc:404-406), before any message is written, the + // async reactor must answer the same. The client may see it on DoAction() or + // on the first Next(), so both surfaces are accepted. + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + const Action action{"null-result-stream", Buffer::FromString("")}; + auto results = client->DoAction(action); + if (results.ok()) { + // The stream was handed out before the status arrived: CANCELLED is then + // reported by the first read. + ASSERT_RAISES(Cancelled, (*results)->Next()); + } else { + ASSERT_RAISES(Cancelled, results.status()); + } + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, ListFlightsIsServedAsync) { + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + const Criteria criteria("filter-me"); + ASSERT_OK_AND_ASSIGN(auto listing, client->ListFlights({}, criteria)); + + ASSERT_OK_AND_ASSIGN(auto paths, DrainListingPaths(listing.get())); + ASSERT_EQ(paths, (std::vector>{{"first-listing"}, + {"second-listing"}})); + + // Cleanup before looking at the recording: it was written on a gRPC thread. + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + ASSERT_EQ("filter-me", flight_server.last_criteria().expression); +} + +TEST(AsyncGrpcTest, ListFlightsEmptyFinishesCleanly) { + // A listing with no flights: the server answers OK with zero messages, and + // the client sees an exhausted listing rather than a hang. + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + ASSERT_OK_AND_ASSIGN(auto listing, client->ListFlights({}, Criteria(""))); + ASSERT_OK_AND_ASSIGN(auto info, listing->Next()); + ASSERT_EQ(info, nullptr); + // The client sent no criteria, the server must still have received the + // (empty) Criteria, exactly as the sync handler does. + ASSERT_EQ("", flight_server.last_criteria().expression); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, ListFlightsMidStreamErrorFinishesTheRpc) { + // A listing that fails on its second Next(): the error must become the RPC + // status, not a silently truncated success. + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto listing = client->ListFlights({}, Criteria("fail-mid-stream")); + ASSERT_FALSE(listing.ok()) << "the mid-stream error must reach the client"; + ASSERT_EQ(listing.status().code(), arrow::StatusCode::IOError); + ASSERT_THAT(listing.status().message(), + ::testing::HasSubstr("mid-stream listing failure")); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, ListFlightsAsyncCompletesOnAnotherThread) { + // The listing object itself arrives on a helper thread: the transport must + // serve the response from wherever the future resolves. + class DeferredListingServer : public AsyncGenericFlightServerBase { + public: + arrow::Future> ListFlightsAsync( + const ServerCallContext&, const Criteria*) override { + auto out = arrow::Future>::Make(); + std::thread([out]() mutable { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::vector flights; + auto info = MakeTestFlightInfo(FlightDescriptor::Path({"deferred"})); + if (!info.ok()) { + out.MarkFinished(info.status()); + return; + } + flights.push_back(std::move(**info)); + out.MarkFinished(std::make_shared(std::move(flights))); + }).detach(); + return out; + } + }; + + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + DeferredListingServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN( + auto client_location, + Location::Parse("grpc://localhost:" + std::to_string(flight_server.port()))); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(client_location)); + + ASSERT_OK_AND_ASSIGN(auto listing, client->ListFlights()); + int seen = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto info, listing->Next()); + if (info == nullptr) break; + ++seen; + } + ASSERT_EQ(1, seen); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +// A ListFlights handler whose future never resolves. +class HangingListingServer final : public AsyncGenericFlightServerBase { + public: + arrow::Future> ListFlightsAsync( + const ServerCallContext&, const Criteria*) override { + return arrow::Future>::Make(); + } +}; + +TEST(AsyncGrpcTest, ListFlightsDeadlineWithPendingHandlerFinishes) { + // The handler's future never resolves and the client's deadline ends the RPC: + // the reactor must be released even though the handler never completes. + const int64_t before = detail::AsyncReactorBase::live_reactors(); + + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + HangingListingServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + FlightCallOptions impatient; + impatient.timeout = std::chrono::milliseconds(500); + auto listing = client->ListFlights(impatient, Criteria("hang")); + EXPECT_FALSE(listing.ok()) << "the deadline must expire while the handler hangs"; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + EXPECT_TRUE(WaitFor([&] { + return detail::AsyncReactorBase::live_reactors() == before; + })) << "a reactor leaked: still alive after the server stopped (before=" + << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; +} + +} // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h new file mode 100644 index 000000000000..e08960ad7119 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h @@ -0,0 +1,865 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Shared fixtures for the async generic gRPC Flight transport tests. +// +// One translation unit per part of the transport would otherwise duplicate the +// adapters, the canned test servers and the gRPC call helpers, so they live +// here. This header is internal (the `_fixtures` suffix keeps it out of the +// installed headers) and is only included by the async_grpc_*_test.cc files. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "arrow/flight/client.h" +#include "arrow/flight/server.h" +#include "arrow/flight/server_async.h" +#include "arrow/flight/server_middleware.h" +#include "arrow/flight/test_auth_handlers.h" +#include "arrow/flight/transport/grpc/async_grpc_service.h" +#include "arrow/flight/transport/grpc/async_grpc_service_internal.h" +#include "arrow/ipc/writer.h" +#include "arrow/scalar.h" +#include "arrow/testing/gtest_util.h" + +namespace arrow::flight::transport::grpc { + +// The ticket the test server answers with a stream that has a schema and no +// batches at all, every other ticket gets the 3-batch stream. +constexpr const char* kSchemaOnlyTicket = "schema-only-do-get-ticket"; + +// One canned FlightInfo, `descriptor` names it for the assertions. +inline arrow::Result> MakeTestFlightInfo( + FlightDescriptor descriptor) { + ARROW_ASSIGN_OR_RAISE( + auto made, FlightInfo::Make(*arrow::schema({arrow::field("value", arrow::int64())}), + std::move(descriptor), std::vector{}, + /*total_records=*/-1, /*total_bytes=*/-1)); + return std::make_unique(std::move(made)); +} + +// An AsyncFlightDataStream over an in-memory RecordBatchReader: every payload is +// already available, so each future is finished when it is created. +class ResolvedStream : public AsyncFlightDataStream { + public: + explicit ResolvedStream(std::shared_ptr reader) + : reader_(std::move(reader)), mapper_(*reader_->schema()) {} + + std::shared_ptr schema() override { return reader_->schema(); } + + arrow::Future GetSchemaPayloadAsync() override { + FlightPayload payload; + auto status = + ipc::GetSchemaPayload(*schema(), ipc_options_, mapper_, &payload.ipc_message); + if (!status.ok()) { + return arrow::Future::MakeFinished(status); + } + return arrow::Future::MakeFinished(std::move(payload)); + } + + arrow::Future> NextAsync() override { + auto batch = reader_->Next(); + if (!batch.ok()) { + return arrow::Future>::MakeFinished(batch.status()); + } + if (*batch == nullptr) { + // End of stream: the async interface reports it as a nullopt. + return arrow::Future>::MakeFinished(std::nullopt); + } + FlightPayload payload; + auto status = ipc::GetRecordBatchPayload(**batch, ipc_options_, &payload.ipc_message); + if (!status.ok()) { + return arrow::Future>::MakeFinished(status); + } + return arrow::Future>::MakeFinished(std::move(payload)); + } + + Status Close() override { return reader_->Close(); } + + private: + std::shared_ptr reader_; + ipc::DictionaryFieldMapper mapper_; + ipc::IpcWriteOptions ipc_options_ = ipc::IpcWriteOptions::Defaults(); +}; + +// A vector-backed async iterator: every NextAsync() resolves immediately, and +// exhaustion is reported as a null item. Instantiated below for both iterator +// interfaces the streaming RPCs use. +template +class VectorAsyncIterator final : public Base { + public: + explicit VectorAsyncIterator(std::vector items) : items_(std::move(items)) {} + + arrow::Future> NextAsync() override { + if (next_ >= items_.size()) { + return arrow::Future>::MakeFinished(std::shared_ptr{}); + } + return arrow::Future>::MakeFinished( + std::make_shared(std::move(items_[next_++]))); + } + + private: + std::vector items_; + size_t next_ = 0; +}; + +using SimpleListingAdapter = VectorAsyncIterator; +using VectorAsyncResultStream = VectorAsyncIterator; + +// An async iterator that yields one item and then fails: the error must become +// the RPC status, not truncate the stream into a success. `make_item` produces +// that first item, or the status that fails the RPC before it. +template +class FailAfterFirstAsyncIterator final : public Base { + public: + FailAfterFirstAsyncIterator(std::function()> make_item, + std::string error_message) + : make_item_(std::move(make_item)), error_message_(std::move(error_message)) {} + + arrow::Future> NextAsync() override { + if (served_) { + return arrow::Future>::MakeFinished( + arrow::Status::IOError(error_message_)); + } + served_ = true; + auto item = make_item_(); + if (!item.ok()) { + return arrow::Future>::MakeFinished(item.status()); + } + return arrow::Future>::MakeFinished( + std::make_shared(std::move(*item))); + } + + private: + std::function()> make_item_; + std::string error_message_; + bool served_ = false; +}; + +using MidStreamErrorAsyncResultStream = + FailAfterFirstAsyncIterator; +using MidStreamErrorAsyncListing = + FailAfterFirstAsyncIterator; + +// Serves the canned answers the async-transport tests drive: 3 batches of 5 +// rows for any ticket except kSchemaOnlyTicket, uploads through the listener the +// test installs, and one canned answer per unary/streaming RPC. The handlers +// run on gRPC threads, the tests read the recordings only after the server has +// shut down. +class TestFlightServer final : public AsyncGenericFlightServerBase { + public: + // The listener the next DoPut RPC is handed, nullptr (the default) refuses + // uploads, which is the base class behavior. + std::shared_ptr CreateDoPutListener( + const ServerCallContext&) override { + return listener_; + } + + void set_listener(std::shared_ptr listener) { + listener_ = std::move(listener); + } + + arrow::Future> DoGetAsync( + const ServerCallContext&, const Ticket& request) override { + ticket_ = request.ticket; + auto schema = arrow::schema( + {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); + if (request.ticket == kSchemaOnlyTicket) { + // An empty reader still carries the schema, NextAsync() reports the end + // of stream right away. + auto reader = arrow::RecordBatchReader::Make({}, schema); + if (!reader.ok()) { + return arrow::Future>::MakeFinished( + reader.status()); + } + return arrow::Future>::MakeFinished( + std::make_shared(std::move(*reader))); + } + auto batch = arrow::RecordBatch::Make( + schema, 5, + {arrow::ArrayFromJSON(arrow::int64(), "[1, 2, 3, 4, 5]"), + arrow::ArrayFromJSON(arrow::int64(), "[10, 20, 30, 40, 50]")}); + auto reader = arrow::RecordBatchReader::Make({batch, batch, batch}); + if (!reader.ok()) { + return arrow::Future>::MakeFinished( + reader.status()); + } + return arrow::Future>::MakeFinished( + std::make_shared(std::move(*reader))); + } + + // The unary trio: canned responses. `last_descriptor_` records the request + // of the last one served (it is written on a gRPC thread, so tests read it + // only after the server has shut down), Command("fail") is the sentinel that + // drives the error-path test. + arrow::Future> GetFlightInfoAsync( + const ServerCallContext&, const FlightDescriptor& request) override { + last_descriptor_ = request; + if (request == FlightDescriptor::Command("fail")) { + return arrow::Future>::MakeFinished( + arrow::Status::KeyError("no such flight")); + } + if (request == FlightDescriptor::Command("null-info")) { + // Answered with a null FlightInfo: the transport must answer what the + // sync transport does for that case (grpc_server.cc: "Flight not found"). + return arrow::Future>::MakeFinished( + std::shared_ptr{}); + } + auto made = + FlightInfo::Make(*arrow::schema({arrow::field("value", arrow::int64())}), + FlightDescriptor{}, std::vector{}, -1, -1); + if (!made.ok()) { + return arrow::Future>::MakeFinished(made.status()); + } + return arrow::Future>::MakeFinished( + std::make_shared(std::move(*made))); + } + + arrow::Future> GetSchemaAsync( + const ServerCallContext&, const FlightDescriptor& request) override { + last_descriptor_ = request; + auto made = + SchemaResult::Make(*arrow::schema({arrow::field("value", arrow::int64())})); + if (!made.ok()) { + return arrow::Future>::MakeFinished(made.status()); + } + return arrow::Future>::MakeFinished( + std::shared_ptr(std::move(*made))); + } + + arrow::Future> PollFlightInfoAsync( + const ServerCallContext&, const FlightDescriptor& request) override { + last_descriptor_ = request; + auto poll_info = std::make_shared(); + poll_info->descriptor = FlightDescriptor::Command("poll-next"); + poll_info->progress = 0.5; + return arrow::Future>::MakeFinished(std::move(poll_info)); + } + + // The streaming trio: ListActions (a vector), DoAction (an AsyncResultStream) + // and ListFlights (an AsyncFlightListing). `last_criteria_` records the + // filter the client sent (read only after the server stopped), the + // "fail-mid-stream" sentinels drive the error-path tests. + arrow::Future> ListActionsAsync( + const ServerCallContext&) override { + std::vector actions = {ActionType("echo", "echo back"), + ActionType("fail", "always fails")}; + if (empty_actions_) { + actions.clear(); + } + return arrow::Future>::MakeFinished(std::move(actions)); + } + + arrow::Future> DoActionAsync( + const ServerCallContext&, const Action& action) override { + last_action_type_ = action.type; + if (action.type == "fail-mid-stream") { + return arrow::Future>::MakeFinished( + std::make_shared( + [] { return Result{arrow::Buffer::FromString("first")}; }, + "mid-stream action failure")); + } + if (action.type == "null-result-stream") { + // Answered with a null stream: the transport answers CANCELLED for this + // case (grpc_server.cc:404-406), before writing anything. + return arrow::Future>::MakeFinished( + std::shared_ptr{}); + } + std::vector made; + made.emplace_back(arrow::Buffer::FromString("one")); + if (action.type == "multi") { + made.emplace_back(arrow::Buffer::FromString("two")); + made.emplace_back(arrow::Buffer::FromString("three")); + } + return arrow::Future>::MakeFinished( + std::make_shared(std::move(made))); + } + + arrow::Future> ListFlightsAsync( + const ServerCallContext&, const Criteria* criteria) override { + if (criteria != nullptr) { + last_criteria_ = *criteria; + } + const std::string expression = criteria == nullptr ? "" : criteria->expression; + if (expression == "fail-mid-stream") { + return arrow::Future>::MakeFinished( + std::make_shared( + []() -> arrow::Result { + ARROW_ASSIGN_OR_RAISE( + auto info, + MakeTestFlightInfo(FlightDescriptor::Path({"first-listing"}))); + return std::move(*info); + }, + "mid-stream listing failure")); + } + if (expression.empty()) { + // No criteria: the sync handler passes an empty (never null) Criteria + // to the handler and the async reactor must do the same, so an empty + // expression is reachable here. + return arrow::Future>::MakeFinished( + std::make_shared(std::vector{})); + } + std::vector flights; + for (const auto& name : {"first-listing", "second-listing"}) { + auto info = MakeTestFlightInfo(FlightDescriptor::Path({name})); + if (!info.ok()) { + return arrow::Future>::MakeFinished( + info.status()); + } + flights.push_back(std::move(**info)); + } + return arrow::Future>::MakeFinished( + std::make_shared(std::move(flights))); + } + + void set_empty_actions() { empty_actions_ = true; } + + const std::string& ticket() const { return ticket_; } + const FlightDescriptor& last_descriptor() const { return last_descriptor_; } + const std::string& last_action_type() const { return last_action_type_; } + const Criteria& last_criteria() const { return last_criteria_; } + + private: + std::shared_ptr listener_; + std::string ticket_; + FlightDescriptor last_descriptor_; + std::string last_action_type_; + Criteria last_criteria_; + bool empty_actions_ = false; +}; + +// Echoes every chunk of a DoExchange back, in order, on the async API: the +// reader's demand drives the loop, each write is awaited, and the all-null +// chunk ends the exchange. `exchanges_in_flight` proves an abandoned +// exchange still leaves the handler: the state object is freed when the last +// continuation dies and decrements the counter on destruction. +class EchoExchangeTestServer : public AsyncGenericFlightServerBase { + public: + arrow::Future<> DoExchangeAsync( + const ServerCallContext&, std::shared_ptr reader, + std::shared_ptr writer) override { + exchange_count_.fetch_add(1); + exchanges_in_flight_.fetch_add(1); + auto state = std::make_shared(); + state->reader = std::move(reader); + state->writer = std::move(writer); + state->in_flight = &exchanges_in_flight_; + return EchoNext(state); + } + + int exchanges_in_flight() const { return exchanges_in_flight_.load(); } + int exchange_count() const { return exchange_count_.load(); } + + private: + struct EchoState { + std::shared_ptr reader; + std::shared_ptr writer; + std::atomic* in_flight = nullptr; + bool begun = false; + ~EchoState() { in_flight->fetch_sub(1); } + }; + + arrow::Future<> EchoNext(const std::shared_ptr& state) { + // The recursion depth follows the batch count of one exchange. Flatten + // into a loop if a single exchange ever carries thousands. + return state->reader->NextAsync().Then( + [this, state](const FlightStreamChunk& chunk) -> arrow::Future<> { + if (chunk.data == nullptr && chunk.app_metadata == nullptr) { + // End of the exchange. + return arrow::Future<>::MakeFinished(); + } + arrow::Future<> written = arrow::Future<>::MakeFinished(); + if (chunk.data != nullptr && !state->begun) { + state->begun = true; + written = state->writer->BeginAsync(chunk.data->schema()); + } + if (chunk.data != nullptr && chunk.app_metadata != nullptr) { + written = written.Then([state, chunk]() { + return state->writer->WriteWithMetadataAsync(*chunk.data, + chunk.app_metadata); + }); + } else if (chunk.data != nullptr) { + written = written.Then([state, chunk]() { + return state->writer->WriteRecordBatchAsync(*chunk.data); + }); + } else { + written = written.Then([state, chunk]() { + return state->writer->WriteMetadataAsync(chunk.app_metadata); + }); + } + return written.Then([this, state]() { return EchoNext(state); }); + }); + } + + std::atomic exchanges_in_flight_{0}; + std::atomic exchange_count_{0}; +}; + +// Records what the decoder hands to the application: the schema, every chunk, +// and whether the consumer accepted it. OnNext returns whatever status the +// test configured, so the upload's error path is reachable from here. +class RecordingListener : public AsyncFlightDataListener { + public: + arrow::Status OnSchemaDecoded(std::shared_ptr schema) override { + std::lock_guard lock(mutex_); + ++schema_count_; + schema_ = std::move(schema); + return arrow::Status::OK(); + } + + arrow::Future<> OnNext(FlightStreamChunk chunk) override { + std::lock_guard lock(mutex_); + if (chunk.data) { + batches_.push_back(std::move(chunk.data)); + } else { + ++metadata_only_count_; + metadata_chunks_.push_back(std::move(chunk.app_metadata)); + } + last_status_ = next_status_; + return arrow::Future<>::MakeFinished(next_status_); + } + + // Records the upload's descriptor and reports whatever status the test + // configured, so the descriptor rejection path is reachable from here too. + arrow::Future<> OnDescriptor(const FlightDescriptor& descriptor) override { + std::lock_guard lock(mutex_); + descriptors_.push_back(descriptor); + return arrow::Future<>::MakeFinished(descriptor_status_); + } + + // Consumer-side rejection of the upload, as the decoder tests do. + void set_next_status(arrow::Status status) { + std::lock_guard lock(mutex_); + next_status_ = std::move(status); + } + // Rejection before any schema or data arrives. + void set_descriptor_status(arrow::Status status) { + std::lock_guard lock(mutex_); + descriptor_status_ = std::move(status); + } + + int schema_count() const { + std::lock_guard lock(mutex_); + return schema_count_; + } + int descriptor_count() const { + std::lock_guard lock(mutex_); + return static_cast(descriptors_.size()); + } + int metadata_only_count() const { + std::lock_guard lock(mutex_); + return metadata_only_count_; + } + std::shared_ptr schema() const { + std::lock_guard lock(mutex_); + return schema_; + } + std::vector descriptors() const { + std::lock_guard lock(mutex_); + return descriptors_; + } + std::vector> batches() const { + std::lock_guard lock(mutex_); + return batches_; + } + std::vector> metadata_chunks() const { + std::lock_guard lock(mutex_); + return metadata_chunks_; + } + arrow::Status last_status() const { + std::lock_guard lock(mutex_); + return last_status_; + } + + protected: + /// The callbacks run on gRPC threads while the test thread reads the + /// accessors, so everything they share goes through this. + mutable std::mutex mutex_; + int schema_count_ = 0; + int metadata_only_count_ = 0; + std::shared_ptr schema_; + std::vector descriptors_; + std::vector> batches_; + std::vector> metadata_chunks_; + arrow::Status descriptor_status_; + arrow::Status next_status_; + arrow::Status last_status_; + + protected: + int finish_count_ = 0; + int cancel_count_ = 0; + Status finish_status_; + Status cancel_status_; +}; + +// A listener that records the terminal events on top of RecordingListener: +// OnFinish (the upload ended, whichever way) and what Cancel() reported, so +// the tests can assert on an upload cancelled from the server side. +class FinishRecordingListener : public RecordingListener { + public: + arrow::Future<> OnFinish(Status status) override { + std::lock_guard lock(mutex_); + finish_count_++; + finish_status_ = std::move(status); + return arrow::Future<>::MakeFinished(arrow::Status::OK()); + } + + /// Cancel from whatever thread the test drives, records what Cancel() said. + void CancelWith(Status status) { + const arrow::Status cancel_status = Cancel(std::move(status)).status(); + std::lock_guard lock(mutex_); + cancel_status_ = cancel_status; + ++cancel_count_; + } + + int finish_count() const { + std::lock_guard lock(mutex_); + return finish_count_; + } + int cancel_count() const { + std::lock_guard lock(mutex_); + return cancel_count_; + } + Status finish_status() const { + std::lock_guard lock(mutex_); + return finish_status_; + } + Status cancel_status() const { + std::lock_guard lock(mutex_); + return cancel_status_; + } +}; + +// A server that accepts uploads: the transport asks the server for the per-RPC +// listener through CreateDoPutListener(). One fresh listener per RPC, as +// overlapping uploads require. The test either reads them after the server has +// stopped, or waits for them with WaitForListener when it must act mid-upload. +// +// The listener type is the fixture's parameter, so a test can plug in one that +// parks, defers or cancels an upload without re-implementing the seam. +template +class RecordingUploadServer : public AsyncGenericFlightServerBase { + public: + std::shared_ptr CreateDoPutListener( + const ServerCallContext&) override { + auto listener = std::make_shared(); + std::lock_guard lock(mutex_); + listeners_.push_back(listener); + return listener; + } + + int calls() const { + std::lock_guard lock(mutex_); + return static_cast(listeners_.size()); + } + + std::shared_ptr listener(size_t i) const { + std::lock_guard lock(mutex_); + return listeners_[i]; + } + + /// \brief Wait (bounded) for the listener of the i-th upload to exist. + /// The factory runs on a gRPC thread, so a test that must act mid-upload + /// waits for it here instead of racing the read. + std::shared_ptr WaitForListener(size_t i = 0) const { + for (int attempt = 0; attempt < 500; attempt++) { + { + std::lock_guard lock(mutex_); + if (listeners_.size() > i) return listeners_[i]; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + return nullptr; + } + + private: + mutable std::mutex mutex_; + std::vector> listeners_; +}; + +using TestUploadServer = RecordingUploadServer; + +/// A listener that acknowledges every batch it is handed with one application +/// metadata message, the way an application reports per-chunk progress. The +/// batch recording itself is RecordingListener's. +class MetadataReplyingListener final : public RecordingListener { + public: + arrow::Future<> OnNext(FlightStreamChunk chunk) override { + if (chunk.data) { + const Status status = writer()->WriteMetadata(*arrow::Buffer::FromString( + "ack " + std::to_string(chunk.data->num_rows()) + " rows")); + std::lock_guard lock(mutex_); + last_write_ = status; + ++wrote_; + } + return RecordingListener::OnNext(std::move(chunk)); + } + + int wrote() const { + std::lock_guard lock(mutex_); + return wrote_; + } + Status last_write() const { + std::lock_guard lock(mutex_); + return last_write_; + } + + private: + int wrote_ = 0; + Status last_write_; +}; + +/// A listener whose OnFinish fails asynchronously: the commit it performs is +/// what decides whether the upload succeeded, so the client must not be told +/// the upload is OK. +class FailingFinishListener final : public FinishRecordingListener { + public: + arrow::Future<> OnFinish(Status status) override { + static_cast(FinishRecordingListener::OnFinish(std::move(status))); + return arrow::Future<>::MakeFinished( + arrow::Status::IOError("the upload could not be committed")); + } +}; + +/// A listener that calls back into the writer from OnFinish: OnFinish is +/// application code running on a transport thread, so the transport must not +/// hold its own write lock across it. +class FinishingAckListener final : public FinishRecordingListener { + public: + arrow::Future<> OnFinish(Status status) override { + finish_write = writer()->WriteMetadata(*arrow::Buffer::FromString("from-onfinish")); + return FinishRecordingListener::OnFinish(std::move(status)); + } + + Status finish_write; +}; + +/// Hands out a MetadataReplyingListener for every upload. +using MetadataReplyingUploadServer = RecordingUploadServer; + +// Middleware execution counts, shared between the factory, the middleware and +// the test thread (all three touch it). +struct MiddlewareTrace { + std::mutex mutex; + int start_call = 0; + int sending_headers = 0; + int call_completed = 0; +}; + +// Records middleware execution so a test can assert StartCall/CallCompleted +// ran on the async path (the sync path is covered by TestMiddleware). +class RecordingServerMiddleware : public ServerMiddleware { + public: + explicit RecordingServerMiddleware(std::shared_ptr trace) + : trace_(std::move(trace)) {} + + std::string name() const override { return "recording"; } + void SendingHeaders(AddCallHeaders* /*outgoing_headers*/) override { + std::lock_guard guard(trace_->mutex); + trace_->sending_headers++; + } + void CallCompleted(const Status& /*status*/) override { + std::lock_guard guard(trace_->mutex); + trace_->call_completed++; + } + + private: + std::shared_ptr trace_; +}; + +class RecordingServerMiddlewareFactory : public ServerMiddlewareFactory { + public: + explicit RecordingServerMiddlewareFactory(std::shared_ptr trace) + : trace_(std::move(trace)) {} + + Status StartCall(const CallInfo& /*info*/, const ServerCallContext& /*context*/, + std::shared_ptr* middleware) override { + { + std::lock_guard guard(trace_->mutex); + trace_->start_call++; + } + *middleware = std::make_shared(trace_); + return Status::OK(); + } + + private: + std::shared_ptr trace_; +}; + +class RejectingServerMiddlewareFactory : public ServerMiddlewareFactory { + public: + Status StartCall(const CallInfo&, const ServerCallContext&, + std::shared_ptr*) override { + return Status::Invalid("rejected by middleware"); + } +}; + +// The gRPC transport builds the generic service with the shared context +// helper (which runs middleware and auth). Tests that register the service +// on a hand-built grpc::ServerBuilder must pass it explicitly. +inline std::shared_ptr> +MakeAsyncHelper() { + return std::make_shared>( + /*auth_handler=*/nullptr, MiddlewareFactoryList{}); +} + +// The method name no FlightService stub knows: the transport must answer it +// with the UNIMPLEMENTED fallback reactor rather than any typed path. It is +// called over a raw generic stub, because only a generic client can name a +// method the service does not have. +constexpr const char* kUnknownMethod = + "/arrow.flight.protocol.FlightService/NoSuchMethod"; + +// Call `kUnknownMethod` on the server listening on `port` with the sync-style +// generic stub API, and return the server's status. +inline ::grpc::Status CallUnknownMethod(int port) { + std::shared_ptr<::grpc::Channel> channel = ::grpc::CreateChannel( + "127.0.0.1:" + std::to_string(port), ::grpc::InsecureChannelCredentials()); + ::grpc::GenericStub stub(channel); + ::grpc::ClientContext context; + ::grpc::ByteBuffer request; + ::grpc::ByteBuffer response; + ::grpc::CompletionQueue cq; + std::unique_ptr<::grpc::ClientAsyncResponseReader<::grpc::ByteBuffer>> call = + stub.PrepareUnaryCall(&context, kUnknownMethod, request, &cq); + call->StartCall(); + ::grpc::Status status; + call->Finish(&response, &status, reinterpret_cast(1)); + void* tag = nullptr; + bool ok = false; + cq.Next(&tag, &ok); + return status; +} + +// The single int64 column ("a") the upload tests use. +inline std::shared_ptr OneColumnSchema() { + return arrow::schema({arrow::field("a", arrow::int64())}); +} + +// One 2-row batch of OneColumnSchema(), rows [1, 2]. +inline std::shared_ptr OneColumnBatch( + const std::shared_ptr& schema = OneColumnSchema()) { + return arrow::RecordBatch::Make(schema, 2, + {arrow::ArrayFromJSON(arrow::int64(), "[1, 2]")}); +} + +// The listen address the tests start their servers on: an ephemeral port. +inline const Location& LocalhostLocation() { + static const Location location = *Location::Parse("grpc://localhost:0"); + return location; +} + +// Connect a FlightClient to the server listening on `port`. +inline arrow::Result> ConnectTo(int port) { + ARROW_ASSIGN_OR_RAISE(auto location, Location::ForScheme("grpc", "127.0.0.1", port)); + return FlightClient::Connect(location); +} + +// The "a", "b" int64 schema the canned streams serve. +inline std::shared_ptr TwoColumnSchema() { + return arrow::schema( + {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); +} + +// One 3-row batch of TwoColumnSchema(), the payload the DoPut tests upload. +inline std::shared_ptr TwoColumnBatch( + const std::shared_ptr& schema = TwoColumnSchema()) { + return arrow::RecordBatch::Make(schema, 3, + {arrow::ArrayFromJSON(arrow::int64(), "[1, 2, 3]"), + arrow::ArrayFromJSON(arrow::int64(), "[10, 20, 30]")}); +} + +inline void AssertTwoColumnSchema(const std::shared_ptr& schema) { + ASSERT_NE(schema, nullptr); + ASSERT_EQ(schema->num_fields(), 2); + ASSERT_EQ(schema->field(0)->name(), "a"); + ASSERT_EQ(schema->field(1)->name(), "b"); +} + +// Drain a DoAction result stream, projecting each Result to a string. +inline arrow::Result> DrainResultBodies(ResultStream* results) { + std::vector out; + while (true) { + ARROW_ASSIGN_OR_RAISE(auto result, results->Next()); + if (result == nullptr) { + return out; + } + out.push_back(result->body->ToString()); + } +} + +// Drain a ListFlights listing, projecting each FlightInfo to its path. +inline arrow::Result>> DrainListingPaths( + FlightListing* listing) { + std::vector> out; + while (true) { + ARROW_ASSIGN_OR_RAISE(auto info, listing->Next()); + if (info == nullptr) { + return out; + } + out.push_back(info->descriptor().path); + } +} + +// Poll `predicate` until it holds or the timeout expires, the DoExchange tests +// use it to bound server-side cleanup instead of sleeping a fixed time. +template +inline bool WaitFor(Predicate predicate, + std::chrono::milliseconds timeout = std::chrono::seconds(10)) { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) { + if (predicate()) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return predicate(); +} + +// Upload one batch and report whichever surface first reports an error: the +// DoPut call, the write, or Close. +inline std::pair UploadOneBatch( + FlightClient* client, const FlightDescriptor& descriptor, + const std::shared_ptr& schema, + const std::shared_ptr& batch) { + auto result = client->DoPut(descriptor, schema); + if (!result.ok()) { + return {"DoPut", result.status()}; + } + auto write_status = result->writer->WriteRecordBatch(*batch); + if (!write_status.ok()) { + return {"WriteRecordBatch", write_status}; + } + return {"Close", result->writer->Close()}; +} + +} // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_unary.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary.cc new file mode 100644 index 000000000000..fcba29fe5a98 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary.cc @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/flight/transport/grpc/async_grpc_service_internal.h" + +#include +#include +#include + +namespace arrow::flight::transport::grpc::detail { + +namespace { + +/** + * A generic unary reactor for handling async gRPC calls in the Flight service. + * T is the type of the response object, and PbT is the corresponding protobuf type. + */ +template +class UnaryReactor final : public AsyncReactorBase { + public: + using HandlerFn = std::function>( + const ServerCallContext&, const FlightDescriptor&)>; + + UnaryReactor(AsyncCallContext flight_context, HandlerFn handler) + : AsyncReactorBase(std::move(flight_context)), handler_(std::move(handler)) { + StartRead(&request_buf_); + } + + void OnReadDone(bool ok) override { + if (!ok) { + FinishOnce(MakeFlightError(FlightStatusCode::Internal, "Failed to read request")); + return; + } + const auto descriptor = ParseProtoRequest( + request_buf_, "FlightDescriptor"); + if (!descriptor.ok()) { + FinishOnce(descriptor.status()); + return; + } + arrow::Future> future = handler_(flight_context(), *descriptor); + future.AddCallback( + [this, token = hold()](const arrow::Result>& result) mutable { + if (!finished()) { + if (!result.ok()) { + FinishOnce(result.status()); + } else { + PbT response; + const auto status = SerializeOrNotFound(*result, &response); + if (!status.ok()) { + FinishOnce(status); + } else { + response_buf_ = MakeWriteBuffer(response); + WriteIfLive([this] { StartWrite(&response_buf_); }); + } + } + } + }); + } + + void OnWriteDone(bool ok) override { + if (finished()) { + return; + } + FinishOnce( + ok ? arrow::Status::OK() + : MakeFlightError(FlightStatusCode::Internal, "Failed to write response")); + } + + private: + HandlerFn handler_; + ::grpc::ByteBuffer request_buf_; + ::grpc::ByteBuffer response_buf_; +}; + +template +::grpc::ServerGenericBidiReactor* MakeUnaryReactor(AsyncCallContext flight_context, + HandlerFn handler) { + return new UnaryReactor(std::move(flight_context), std::move(handler)); +} + +} // namespace + +::grpc::ServerGenericBidiReactor* MakeGetFlightInfoReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { + return MakeUnaryReactor( + std::move(flight_context), + [base](const ServerCallContext& context, const FlightDescriptor& descriptor) { + return base->GetFlightInfoAsync(context, descriptor); + }); +} + +::grpc::ServerGenericBidiReactor* MakeGetSchemaReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { + return MakeUnaryReactor( + std::move(flight_context), + [base](const ServerCallContext& context, const FlightDescriptor& descriptor) { + return base->GetSchemaAsync(context, descriptor); + }); +} + +::grpc::ServerGenericBidiReactor* MakePollFlightInfoReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { + return MakeUnaryReactor( + std::move(flight_context), + [base](const ServerCallContext& context, const FlightDescriptor& descriptor) { + return base->PollFlightInfoAsync(context, descriptor); + }); +} + +} // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc new file mode 100644 index 000000000000..204c4b527b79 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc @@ -0,0 +1,218 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The unary RPCs served by UnaryReactor (one request in, one response out): +// GetFlightInfo, GetSchema and PollFlightInfo, plus the rules they share -- a +// null answer maps to the sync transport's "Flight not found", a server error +// propagates, an answer may complete on another thread, and a handler future +// that never resolves must not leak its reactor. + +#include "arrow/flight/transport/grpc/async_grpc_test_fixtures.h" + +namespace arrow::flight::transport::grpc { + +// --------------------------------------------------------------------------- +// The unary trio: GetFlightInfo, GetSchema and PollFlightInfo share the +// UnaryReactor (one request in, one response out), so the trio has one +// happy-path test each plus one error-path test that pins the base class. +// --------------------------------------------------------------------------- + +TEST(AsyncGrpcTest, GetFlightInfoIsServedAsync) { + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + ASSERT_OK_AND_ASSIGN(auto info, + client->GetFlightInfo(FlightDescriptor::Command("ping"))); + ipc::DictionaryMemo dict_memo; + ASSERT_OK_AND_ASSIGN(auto schema, info->GetSchema(&dict_memo)); + ASSERT_EQ(1, schema->num_fields()); + ASSERT_EQ("value", schema->field(0)->name()); + + // Cleanup before looking at the recording: it was written on a gRPC thread. + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + ASSERT_EQ("ping", flight_server.last_descriptor().cmd); +} + +TEST(AsyncGrpcTest, GetFlightInfoWithNullResponseIsNotFound) { + // A handler that answers OK without setting the response: the sync transport + // answers NOT_FOUND with "Flight not found" (grpc_server.cc:282-286), and the + // async reactor must answer the same, not INTERNAL. + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + const auto status = + client->GetFlightInfo(FlightDescriptor::Command("null-info")).status(); + ASSERT_RAISES(KeyError, status); + ASSERT_THAT(status.message(), ::testing::HasSubstr("Flight not found")); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, GetSchemaIsServedAsync) { + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + ASSERT_OK_AND_ASSIGN(auto schema_result, + client->GetSchema(FlightDescriptor::Command("ping"))); + ipc::DictionaryMemo dict_memo; + ASSERT_OK_AND_ASSIGN(auto schema, schema_result->GetSchema(&dict_memo)); + ASSERT_EQ(1, schema->num_fields()); + ASSERT_EQ("value", schema->field(0)->name()); + + // Cleanup before looking at the recording: it was written on a gRPC thread. + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + ASSERT_EQ("ping", flight_server.last_descriptor().cmd); +} + +TEST(AsyncGrpcTest, PollFlightInfoIsServedAsync) { + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + ASSERT_OK_AND_ASSIGN(auto poll_info, + client->PollFlightInfo(FlightDescriptor::Command("ping"))); + ASSERT_TRUE(poll_info->progress.has_value()); + ASSERT_DOUBLE_EQ(0.5, *poll_info->progress); + ASSERT_TRUE(poll_info->descriptor.has_value()); + ASSERT_EQ("poll-next", poll_info->descriptor->cmd); + + // Cleanup before looking at the recording: it was written on a gRPC thread. + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + ASSERT_EQ("ping", flight_server.last_descriptor().cmd); +} + +// A GetFlightInfo handler whose future never resolves: the deadline is the +// only thing that can end the RPC. +class HangingUnaryServer final : public AsyncGenericFlightServerBase { + public: + arrow::Future> GetFlightInfoAsync( + const ServerCallContext&, const FlightDescriptor&) override { + return arrow::Future>::Make(); + } +}; + +TEST(AsyncGrpcTest, GetFlightInfoDeadlineWithPendingHandlerFinishes) { + // The handler's future never resolves and the client's deadline ends the + // RPC: the reactor must be released even though the handler never completed, + // or Shutdown() below would block. + const int64_t before = detail::AsyncReactorBase::live_reactors(); + + HangingUnaryServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + FlightCallOptions impatient; + impatient.timeout = std::chrono::milliseconds(500); + auto status = + client->GetFlightInfo(impatient, FlightDescriptor::Command("hang")).status(); + EXPECT_FALSE(status.ok()) << "the deadline must expire while the handler hangs"; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + EXPECT_TRUE(WaitFor([&] { + return detail::AsyncReactorBase::live_reactors() == before; + })) << "a reactor leaked: still alive after the server stopped (before=" + << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; +} + +TEST(AsyncGrpcTest, UnaryReactorPropagatesServerError) { + TestFlightServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + // The server's KeyError must reach the client as a status failure carrying + // the server's message, not as a successful (empty) response. The message is + // matched by substring because this build defines ARROW_EXTRA_ERROR_CONTEXT, + // which appends the call-site trail to the server's message. + auto status = client->GetFlightInfo(FlightDescriptor::Command("fail")).status(); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(arrow::StatusCode::KeyError, status.code()); + ASSERT_THAT(status.message(), ::testing::HasSubstr("no such flight")); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +TEST(AsyncGrpcTest, GetFlightInfoAsyncCompletesOnAnotherThread) { + // The handler completes its future from a helper thread after the dispatch + // has returned: the answer must arrive on the client anyway, which is the + // whole point of the async handler. + class DeferredInfoServer : public AsyncGenericFlightServerBase { + public: + arrow::Future> GetFlightInfoAsync( + const ServerCallContext&, const FlightDescriptor& request) override { + auto out = arrow::Future>::Make(); + std::thread([out, request]() mutable { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + auto info = + FlightInfo::Make(*arrow::schema({arrow::field("value", arrow::int64())}), + request, std::vector{}, -1, -1); + if (!info.ok()) { + out.MarkFinished(info.status()); + return; + } + out.MarkFinished(std::make_shared(std::move(*info))); + }).detach(); + return out; + } + }; + + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + DeferredInfoServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN( + auto client_location, + Location::Parse("grpc://localhost:" + std::to_string(flight_server.port()))); + ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(client_location)); + + ASSERT_OK_AND_ASSIGN(auto info, + client->GetFlightInfo(FlightDescriptor::Command("deferred"))); + ipc::DictionaryMemo dict_memo; + ASSERT_OK_AND_ASSIGN(auto schema, info->GetSchema(&dict_memo)); + ASSERT_EQ(1, schema->num_fields()); + ASSERT_EQ("value", schema->field(0)->name()); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + +} // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/customize_grpc.h b/cpp/src/arrow/flight/transport/grpc/customize_grpc.h index 4a64e4e07214..1974111709b4 100644 --- a/cpp/src/arrow/flight/transport/grpc/customize_grpc.h +++ b/cpp/src/arrow/flight/transport/grpc/customize_grpc.h @@ -105,8 +105,8 @@ namespace grpc { // Those two functions are defined in serialization_internal.cc // Write FlightData to a grpc::ByteBuffer without extra copying -::grpc::Status FlightDataSerialize(const arrow::flight::FlightPayload& msg, - ::grpc::ByteBuffer* out, bool* own_buffer); +ARROW_FLIGHT_EXPORT ::grpc::Status FlightDataSerialize( + const arrow::flight::FlightPayload& msg, ::grpc::ByteBuffer* out, bool* own_buffer); // Read internal::FlightData from grpc::ByteBuffer containing FlightData // protobuf without copying diff --git a/cpp/src/arrow/flight/transport/grpc/grpc_server.cc b/cpp/src/arrow/flight/transport/grpc/grpc_server.cc index 2a100caeff76..507bef4693f7 100644 --- a/cpp/src/arrow/flight/transport/grpc/grpc_server.cc +++ b/cpp/src/arrow/flight/transport/grpc/grpc_server.cc @@ -25,6 +25,10 @@ #include #include +#include "arrow/flight/transport/grpc/async_grpc_compat.h" +#if ARROW_FLIGHT_HAS_ASYNC_SERVER +# include "arrow/flight/transport/grpc/async_grpc_service.h" +#endif #include "arrow/flight/transport/grpc/customize_grpc.h" #include @@ -113,7 +117,7 @@ class GrpcServerAuthSender : public ServerAuthSender { ::grpc::ServerReaderWriter* stream_; }; -using GrpcServerCallContext = transport::grpc::GrpcServerCallContext; +using SyncServerCallContext = transport::grpc::GrpcServerCallContext; using GrpcContextHelper = transport::grpc::GrpcServerCallContextHelper; // A ServerDataStream for streaming data to the client. @@ -224,7 +228,7 @@ class GrpcServiceHandler final : public FlightService::Service { ::grpc::Status Handshake( ServerContext* context, ::grpc::ServerReaderWriter* stream) { - GrpcServerCallContext flight_context(context); + SyncServerCallContext flight_context(context); GRPC_RETURN_NOT_GRPC_OK( helper_.MakeCallContext(FlightMethod::Handshake, context, &flight_context)); if (!helper_.auth_handler()) { @@ -242,7 +246,7 @@ class GrpcServiceHandler final : public FlightService::Service { ::grpc::Status ListFlights(ServerContext* context, const pb::Criteria* request, ServerWriter* writer) { - GrpcServerCallContext flight_context(context); + SyncServerCallContext flight_context(context); GRPC_RETURN_NOT_GRPC_OK( helper_.CheckAuth(FlightMethod::ListFlights, context, &flight_context)); // Retrieve the listing from the implementation @@ -265,7 +269,7 @@ class GrpcServiceHandler final : public FlightService::Service { ::grpc::Status GetFlightInfo(ServerContext* context, const pb::FlightDescriptor* request, pb::FlightInfo* response) { - GrpcServerCallContext flight_context(context); + SyncServerCallContext flight_context(context); GRPC_RETURN_NOT_GRPC_OK( helper_.CheckAuth(FlightMethod::GetFlightInfo, context, &flight_context)); @@ -292,7 +296,7 @@ class GrpcServiceHandler final : public FlightService::Service { ::grpc::Status PollFlightInfo(ServerContext* context, const pb::FlightDescriptor* request, pb::PollInfo* response) { - GrpcServerCallContext flight_context(context); + SyncServerCallContext flight_context(context); GRPC_RETURN_NOT_GRPC_OK( helper_.CheckAuth(FlightMethod::PollFlightInfo, context, &flight_context)); @@ -317,7 +321,7 @@ class GrpcServiceHandler final : public FlightService::Service { ::grpc::Status GetSchema(ServerContext* context, const pb::FlightDescriptor* request, pb::SchemaResult* response) { - GrpcServerCallContext flight_context(context); + SyncServerCallContext flight_context(context); GRPC_RETURN_NOT_GRPC_OK( helper_.CheckAuth(FlightMethod::GetSchema, context, &flight_context)); @@ -342,7 +346,7 @@ class GrpcServiceHandler final : public FlightService::Service { ::grpc::Status DoGet(ServerContext* context, const pb::Ticket* request, ServerWriter* writer) { - GrpcServerCallContext flight_context(context); + SyncServerCallContext flight_context(context); GRPC_RETURN_NOT_GRPC_OK( helper_.CheckAuth(FlightMethod::DoGet, context, &flight_context)); CHECK_ARG_NOT_NULL(flight_context, request, "ticket cannot be null"); @@ -358,7 +362,7 @@ class GrpcServiceHandler final : public FlightService::Service { ::grpc::Status DoPut( ServerContext* context, ::grpc::ServerReaderWriter* reader) { - GrpcServerCallContext flight_context(context); + SyncServerCallContext flight_context(context); GRPC_RETURN_NOT_GRPC_OK( helper_.CheckAuth(FlightMethod::DoPut, context, &flight_context)); PutDataStream stream(reader); @@ -368,7 +372,7 @@ class GrpcServiceHandler final : public FlightService::Service { ::grpc::Status DoExchange( ServerContext* context, ::grpc::ServerReaderWriter* stream) { - GrpcServerCallContext flight_context(context); + SyncServerCallContext flight_context(context); GRPC_RETURN_NOT_GRPC_OK( helper_.CheckAuth(FlightMethod::DoExchange, context, &flight_context)); ExchangeDataStream data_stream(stream); @@ -378,7 +382,7 @@ class GrpcServiceHandler final : public FlightService::Service { ::grpc::Status ListActions(ServerContext* context, const pb::Empty* request, ServerWriter* writer) { - GrpcServerCallContext flight_context(context); + SyncServerCallContext flight_context(context); GRPC_RETURN_NOT_GRPC_OK( helper_.CheckAuth(FlightMethod::ListActions, context, &flight_context)); // Retrieve the listing from the implementation @@ -390,7 +394,7 @@ class GrpcServiceHandler final : public FlightService::Service { ::grpc::Status DoAction(ServerContext* context, const pb::Action* request, ServerWriter* writer) { - GrpcServerCallContext flight_context(context); + SyncServerCallContext flight_context(context); GRPC_RETURN_NOT_GRPC_OK( helper_.CheckAuth(FlightMethod::DoAction, context, &flight_context)); CHECK_ARG_NOT_NULL(flight_context, request, "Action cannot be null"); @@ -430,7 +434,16 @@ class GrpcServiceHandler final : public FlightService::Service { // The ServerTransport implementation for gRPC. Manages the gRPC server itself. class GrpcServerTransport : public internal::ServerTransport { public: - using internal::ServerTransport::ServerTransport; + GrpcServerTransport(FlightServerBase* base, + std::shared_ptr memory_manager, + AsyncGenericFlightServerBase* async_base = nullptr) + : internal::ServerTransport(base, std::move(memory_manager)) { +#if ARROW_FLIGHT_HAS_ASYNC_SERVER + async_base_ = async_base; +#else + ARROW_UNUSED(async_base); +#endif + } static arrow::Result> Make( FlightServerBase* base, std::shared_ptr memory_manager) { @@ -438,9 +451,56 @@ class GrpcServerTransport : public internal::ServerTransport { new GrpcServerTransport(base, std::move(memory_manager))); } + /// \brief Like Make(), for a server that is an AsyncGenericFlightServerBase. + /// + /// This is the async mode: the transport serves the async server's handlers + /// through the generic callback service, with no FlightServerBase involved. + static arrow::Result> MakeAsync( + AsyncGenericFlightServerBase* async_base, + std::shared_ptr memory_manager) { +#if ARROW_FLIGHT_HAS_ASYNC_SERVER + return std::unique_ptr( + new GrpcServerTransport(/*base=*/nullptr, std::move(memory_manager), async_base)); +#else + ARROW_UNUSED(async_base); + ARROW_UNUSED(memory_manager); + return Status::NotImplemented( + "The async Flight server requires gRPC's generic callback API " + "(arrow was built against gRPC < 1.65)"); +#endif + } + Status Init(const FlightServerOptions& options, const arrow::util::Uri& uri) override { - grpc_service_.reset( - new GrpcServiceHandler(options.auth_handler, options.middleware, this)); +#if ARROW_FLIGHT_HAS_ASYNC_SERVER + if (async_base_ != nullptr) { + if (options.auth_handler) { + return Status::NotImplemented( + "AsyncGenericFlightServerBase does not support a blocking auth handler; " + "override Handshake/ValidateToken instead"); + } + using AsyncHelper = GrpcServerCallContextHelper<::grpc::CallbackServerContext>; + AsyncHelper::HandshakeFn handshake = + [base = async_base_](const ServerCallContext& context, + const std::string& request, std::string* response) { + return base->Handshake(context, request, response); + }; + AsyncHelper::ValidateTokenFn validate_token = + [base = async_base_](const ServerCallContext& context, const std::string& token, + std::string* peer_identity) { + return base->ValidateToken(context, token, peer_identity); + }; + async_helper_ = std::make_shared( + /*auth_handler=*/nullptr, options.middleware, std::move(validate_token)); + async_service_ = std::make_unique( + async_base_, async_helper_, std::move(handshake)); + // The else and its body are split by #endif, so cpplint cannot pair the + // braces; the #if branch above opens the block it closes. + } else // NOLINT(readability/braces) +#endif + { + grpc_service_.reset( + new GrpcServiceHandler(options.auth_handler, options.middleware, this)); + } ::grpc::ServerBuilder builder; int port = 0; @@ -452,7 +512,14 @@ class GrpcServerTransport : public internal::ServerTransport { builder.AddListeningPort(endpoint.address, endpoint.credentials, &port); } - builder.RegisterService(grpc_service_.get()); +#if ARROW_FLIGHT_HAS_ASYNC_SERVER + if (async_service_) { + builder.RegisterCallbackGenericService(async_service_.get()); + } else // NOLINT(readability/braces) +#endif + { + builder.RegisterService(grpc_service_.get()); + } ConfigureServerBuilderOptions(options, &builder); grpc_server_ = builder.BuildAndStart(); @@ -484,6 +551,12 @@ class GrpcServerTransport : public internal::ServerTransport { private: std::unique_ptr grpc_service_; +#if ARROW_FLIGHT_HAS_ASYNC_SERVER + AsyncGenericFlightServerBase* async_base_ = nullptr; + std::shared_ptr> + async_helper_; + std::unique_ptr async_service_; +#endif std::unique_ptr<::grpc::Server> grpc_server_; Location location_; }; @@ -496,6 +569,11 @@ void InitializeFlightGrpcServer() { auto* registry = flight::internal::GetDefaultTransportRegistry(); for (const auto& transport : {"grpc", "grpc+tls", "grpc+tcp", "grpc+unix"}) { ARROW_CHECK_OK(registry->RegisterServer(transport, GrpcServerTransport::Make)); + // Registered even when gRPC is too old for the async server: MakeAsync then + // answers NotImplemented with the reason, instead of Init() failing with the + // generic "no async server transport implementation" error. + ARROW_CHECK_OK( + registry->RegisterAsyncServer(transport, GrpcServerTransport::MakeAsync)); } }); } diff --git a/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h b/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h index cdafb53a3e12..7964ce87df22 100644 --- a/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -26,7 +27,9 @@ #include #include "arrow/flight/server.h" +#include "arrow/flight/server_auth.h" #include "arrow/flight/server_middleware.h" +#include "arrow/flight/transport/grpc/async_grpc_compat.h" #include "arrow/flight/transport/grpc/util_internal.h" #include "arrow/util/uri.h" @@ -112,9 +115,24 @@ class GrpcServerCallContext : public ServerCallContext { template class GrpcServerCallContextHelper { public: + /// A token validator for servers that have no ServerAuthHandler: the async + /// server plugs AsyncGenericFlightServerBase::ValidateToken in here. + using ValidateTokenFn = + std::function; + + /// The async handshake hook: the transport reads one Handshake request, + /// calls this, then writes the response it filled in. + using HandshakeFn = + std::function; + GrpcServerCallContextHelper(std::shared_ptr auth_handler, - MiddlewareFactoryList middleware) - : auth_handler_(std::move(auth_handler)), middleware_(std::move(middleware)) {} + MiddlewareFactoryList middleware, + ValidateTokenFn validate_token = {}) + : auth_handler_(std::move(auth_handler)), + middleware_(std::move(middleware)), + validate_token_(std::move(validate_token)) {} // Authenticate the client (if applicable) and construct the call context ::grpc::Status MakeCallContext(FlightMethod method, GrpcContext* context, @@ -165,7 +183,11 @@ class GrpcServerCallContextHelper { } else { flight_context->peer_identity_ = ""; } - } else { + if (!validate_token_) { + return MakeCallContext(method, context, flight_context); + } + } + { const auto client_metadata = context->client_metadata(); const auto [auth_header, auth_header_end] = client_metadata.equal_range(kGrpcAuthHeader); @@ -174,7 +196,10 @@ class GrpcServerCallContextHelper { token = std::string(auth_header->second.data(), auth_header->second.length()); } auto auth_status = - auth_handler_->IsValid(*flight_context, token, &flight_context->peer_identity_); + validate_token_ + ? validate_token_(*flight_context, token, &flight_context->peer_identity_) + : auth_handler_->IsValid(*flight_context, token, + &flight_context->peer_identity_); if (!auth_status.ok()) { return flight_context->FinishRequest(auth_status); } @@ -187,6 +212,7 @@ class GrpcServerCallContextHelper { private: std::shared_ptr auth_handler_; MiddlewareFactoryList middleware_; + ValidateTokenFn validate_token_; }; struct GrpcServerEndpoint { diff --git a/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc b/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc index ee13a8e2022f..a2ac73c70dc0 100644 --- a/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc +++ b/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc @@ -17,6 +17,10 @@ #include "arrow/flight/transport/grpc/serialization_internal.h" +#include + +#include "arrow/flight/types.h" + #include #include #include @@ -125,7 +129,12 @@ class GrpcBuffer : public MutableBuffer { grpc_slice slice_; }; -// Destructor callback for grpc::Slice +} // namespace + +// Destructor callback for grpc::Slice. Declared here (rather than in the +// header) so -Wmissing-declarations is happy. +void ReleaseBuffer(void* buf_ptr); + void ReleaseBuffer(void* buf_ptr) { delete reinterpret_cast*>(buf_ptr); } @@ -149,7 +158,24 @@ arrow::Result<::grpc::Slice> SliceFromBuffer(const std::shared_ptr& buf) return slice; } -} // namespace +// Initialize arrow Buffer from gRPC ByteBuffer +Status WrapGrpcBuffer(::grpc::ByteBuffer* cpp_buf, std::shared_ptr* out) { + return GrpcBuffer::Wrap(cpp_buf, out); +} + +arrow::Result BytesFromBuffer(const ::grpc::ByteBuffer& buffer, + std::string_view error_message) { + std::vector<::grpc::Slice> slices; + if (!buffer.Dump(&slices).ok()) { + return MakeFlightError(FlightStatusCode::Internal, std::string(error_message)); + } + std::string bytes; + bytes.reserve(buffer.Length()); + for (const auto& slice : slices) { + bytes.append(reinterpret_cast(slice.begin()), slice.size()); + } + return bytes; +} ::grpc::Status FlightDataSerialize(const FlightPayload& msg, ByteBuffer* out, bool* own_buffer) { diff --git a/cpp/src/arrow/flight/transport/grpc/serialization_internal.h b/cpp/src/arrow/flight/transport/grpc/serialization_internal.h index 5c347fd4f819..36285e0bdec8 100644 --- a/cpp/src/arrow/flight/transport/grpc/serialization_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/serialization_internal.h @@ -21,6 +21,8 @@ #pragma once #include +#include +#include #include "arrow/flight/protocol_internal.h" #include "arrow/flight/transport/grpc/protocol_grpc_internal.h" @@ -28,12 +30,40 @@ #include "arrow/result.h" namespace arrow { + +class Buffer; + namespace flight { namespace transport { namespace grpc { namespace pb = arrow::flight::protocol; +/// Zero-copy conversion of an Arrow Buffer to a gRPC Slice. +/// +/// The Buffer lifetime is tied to the Slice (a shared_ptr is attached to the +/// slice's destructor callback) +/// \param[in] buf The Arrow Buffer to convert. +/// \return A gRPC Slice that shares ownership of the Buffer. +arrow::Result<::grpc::Slice> SliceFromBuffer(const std::shared_ptr& buf); + +/// Zero-copy conversion of a gRPC ByteBuffer into an Arrow Buffer. +/// +/// The Arrow Buffer owns a reference to the gRPC slice, so it stays valid +/// after the ByteBuffer goes away. +/// \param[in] cpp_buf The gRPC ByteBuffer to wrap. +/// \param[out] out The resulting Arrow Buffer that shares ownership of the gRPC slices. +/// \return Status indicating success or failure. +Status WrapGrpcBuffer(::grpc::ByteBuffer* cpp_buf, std::shared_ptr* out); + +/// Copy a gRPC ByteBuffer's slices into one contiguous string. +/// \param[in] buffer The gRPC ByteBuffer to copy from. +/// \param[in] error_message The error message to use if the buffer cannot be dumped. +/// \return The buffer's bytes, or an IOError carrying FlightStatusCode::Internal +/// on failure. +ARROW_FLIGHT_EXPORT arrow::Result BytesFromBuffer( + const ::grpc::ByteBuffer& buffer, std::string_view error_message); + /// Write Flight message on gRPC stream with zero-copy optimizations. // Returns Invalid if the payload is ill-formed // Returns true if the payload was written, false if it was not diff --git a/cpp/src/arrow/flight/transport/grpc/util_internal.cc b/cpp/src/arrow/flight/transport/grpc/util_internal.cc index 95b205d67808..8dbdf3eb1952 100644 --- a/cpp/src/arrow/flight/transport/grpc/util_internal.cc +++ b/cpp/src/arrow/flight/transport/grpc/util_internal.cc @@ -310,9 +310,10 @@ static ::grpc::Status ToRawGrpcStatus(const Status& arrow_status) { return {grpc_code, std::move(transport_status.message)}; } -/// Convert an Arrow status to a gRPC status, and add extra headers to -/// the response to encode the original Arrow status. -::grpc::Status ToGrpcStatus(const Status& arrow_status, ::grpc::ServerContext* ctx) { +namespace { +// Shared body: convert and attach the Arrow status as trailing metadata. +template +::grpc::Status ToGrpcStatusWithContext(const Status& arrow_status, Context* ctx) { ::grpc::Status status = ToRawGrpcStatus(arrow_status); if (!status.ok() && ctx) { const std::string code = ToChars(static_cast(arrow_status.code())); @@ -327,9 +328,23 @@ ::grpc::Status ToGrpcStatus(const Status& arrow_status, ::grpc::ServerContext* c ctx->AddTrailingMetadata(kBinaryErrorDetailsKey, fsd->extra_info()); } } - return status; } +} // namespace + +/// Convert an Arrow status to a gRPC status, and add extra headers to +/// the response to encode the original Arrow status. +::grpc::Status ToGrpcStatus(const Status& arrow_status, ::grpc::ServerContext* ctx) { + return ToGrpcStatusWithContext(arrow_status, ctx); +} + +#if GRPC_CPP_VERSION_CHECK(1, 65, 0) +/// Same, for the callback API's server context. +::grpc::Status ToGrpcStatus(const Status& arrow_status, + ::grpc::CallbackServerContext* ctx) { + return ToGrpcStatusWithContext(arrow_status, ctx); +} +#endif #if GRPC_CPP_VERSION_CHECK(1, 80, 0) Status FromAbslStatus(const ::absl::Status& absl_status) { diff --git a/cpp/src/arrow/flight/transport/grpc/util_internal.h b/cpp/src/arrow/flight/transport/grpc/util_internal.h index 7e306d574a58..5d215620221c 100644 --- a/cpp/src/arrow/flight/transport/grpc/util_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/util_internal.h @@ -42,6 +42,7 @@ namespace grpc { class Status; +class CallbackServerContext; } // namespace grpc @@ -107,6 +108,12 @@ ARROW_FLIGHT_EXPORT ::grpc::Status ToGrpcStatus(const Status& arrow_status, ::grpc::ServerContext* ctx = nullptr); +#if GRPC_CPP_VERSION_CHECK(1, 65, 0) +ARROW_FLIGHT_EXPORT +::grpc::Status ToGrpcStatus(const Status& arrow_status, + ::grpc::CallbackServerContext* ctx); +#endif + // gRPC 1.80.0 or later use absl::Status. #if GRPC_CPP_VERSION_CHECK(1, 80, 0) /// Convert an Abseil status to an Arrow status. diff --git a/cpp/src/arrow/flight/transport_server_internal.cc b/cpp/src/arrow/flight/transport_server_internal.cc index be159afa5d9e..9a5afa19f46b 100644 --- a/cpp/src/arrow/flight/transport_server_internal.cc +++ b/cpp/src/arrow/flight/transport_server_internal.cc @@ -298,6 +298,62 @@ void ServerSignalState::WaitForSignals( } } +ServerLifecycle::ServerLifecycle(const char* server_name) : server_name_(server_name) {} + +ServerLifecycle::~ServerLifecycle() = default; + +Status ServerLifecycle::Init(const FlightServerOptions& options, + std::unique_ptr transport) { + ARROW_ASSIGN_OR_RAISE(auto uri, options.location.uri()); + transport_ = std::move(transport); + return transport_->Init(options, uri); +} + +int ServerLifecycle::port() const { return location().port(); } + +Location ServerLifecycle::location() const { return transport_->location(); } + +Status ServerLifecycle::SetShutdownOnSignals(const std::vector& signals) { + return signal_state_.SetShutdownOnSignals(signals); +} + +Status ServerLifecycle::Serve() { + if (!transport_) { + return Status::UnknownError("Server did not start properly"); + } + return signal_state_.Serve( + [this]() -> Status { return transport_->Wait(); }, + [this](const std::chrono::system_clock::time_point* deadline) -> Status { + return Shutdown(deadline); + }, + "Error shutting down server"); +} + +int ServerLifecycle::GotSignal() const { return signal_state_.GotSignal(); } + +Status ServerLifecycle::Shutdown(const std::chrono::system_clock::time_point* deadline) { + if (!transport_) { + return Status::Invalid("Shutdown() on uninitialized ", server_name_); + } + return signal_state_.Shutdown( + [this](const std::chrono::system_clock::time_point* maybe_deadline) -> Status { + if (maybe_deadline) { + return transport_->Shutdown(*maybe_deadline); + } + return transport_->Shutdown(); + }, + deadline); +} + +Status ServerLifecycle::Wait() { + return signal_state_.Wait([this] { + if (!transport_) { + return Status::Invalid("Wait() on uninitialized ", server_name_); + } + return transport_->Wait(); + }); +} + Status WriteDataStream(std::unique_ptr data_stream, ServerDataStream* stream) { if (!data_stream) { diff --git a/cpp/src/arrow/flight/transport_server_internal.h b/cpp/src/arrow/flight/transport_server_internal.h index 2112d613c6b1..d1f28319787e 100644 --- a/cpp/src/arrow/flight/transport_server_internal.h +++ b/cpp/src/arrow/flight/transport_server_internal.h @@ -37,6 +37,7 @@ namespace arrow::flight::internal { class ServerDataStream; class PeekableFlightDataReader; +class ServerTransport; /// \brief Adapt TransportDataStream to the FlightMessageReader /// interface for DoPut. @@ -302,4 +303,39 @@ class ARROW_FLIGHT_EXPORT ServerSignalState { const char* shutdown_warning_ = nullptr; }; +/// \brief The server lifecycle shared by FlightServerBase and +/// AsyncGenericFlightServerBase. +/// +/// Owns the server's transport and signal state and implements Init(), the +/// signal-aware Serve()/Shutdown()/Wait() trio and the location accessors, so +/// both server classes have one implementation of them. +class ServerLifecycle { + public: + /// \param[in] server_name names the server class. Used in error logs. + explicit ServerLifecycle(const char* server_name); + ~ServerLifecycle(); + + /// \brief Take ownership of \p transport and initialize it. + /// + /// \param[in] options the server options the transport is initialized with. + /// \param[in] transport the transport serving the server, created by the + /// server class (the sync and async servers come from different registry + /// factories). + Status Init(const FlightServerOptions& options, + std::unique_ptr transport); + + int port() const; + Location location() const; + Status SetShutdownOnSignals(const std::vector& signals); + Status Serve(); + int GotSignal() const; + Status Shutdown(const std::chrono::system_clock::time_point* deadline); + Status Wait(); + + private: + const char* server_name_; + std::unique_ptr transport_; + ServerSignalState signal_state_; +}; + } // namespace arrow::flight::internal diff --git a/cpp/src/arrow/flight/types.cc b/cpp/src/arrow/flight/types.cc index c4b81184cc67..08e891557b42 100644 --- a/cpp/src/arrow/flight/types.cc +++ b/cpp/src/arrow/flight/types.cc @@ -1052,6 +1052,9 @@ Status ResultStream::Drain() { } FlightStreamChunk::FlightStreamChunk() noexcept = default; +FlightStreamChunk::FlightStreamChunk(std::shared_ptr data, + std::shared_ptr app_metadata) noexcept + : data(std::move(data)), app_metadata(std::move(app_metadata)) {} FlightStreamChunk::~FlightStreamChunk() = default; arrow::Result>> diff --git a/cpp/src/arrow/flight/types.h b/cpp/src/arrow/flight/types.h index dd18c0e0786a..d3049a5e8eed 100644 --- a/cpp/src/arrow/flight/types.h +++ b/cpp/src/arrow/flight/types.h @@ -1166,6 +1166,8 @@ class ARROW_FLIGHT_EXPORT ResultStream { struct ARROW_FLIGHT_EXPORT FlightStreamChunk { public: FlightStreamChunk() noexcept; + FlightStreamChunk(std::shared_ptr data, + std::shared_ptr app_metadata) noexcept; ~FlightStreamChunk(); std::shared_ptr data; diff --git a/cpp/src/arrow/util/windows_fixup.h b/cpp/src/arrow/util/windows_fixup.h index 42e74f4a7857..03dc238820cb 100644 --- a/cpp/src/arrow/util/windows_fixup.h +++ b/cpp/src/arrow/util/windows_fixup.h @@ -38,14 +38,6 @@ # undef DeleteFile # endif -// Other annoying Windows macro definitions... -# ifdef IN -# undef IN -# endif -# ifdef OUT -# undef OUT -# endif - // Note that we can't undefine OPTIONAL, because it can be used in other // Windows headers... From 94a7e251ba1dd242419a44cf4c2c4370323aed17 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:56 +0100 Subject: [PATCH 02/25] [C++][FlightRPC] Bind the async server tests to 127.0.0.1 localhost resolves to both [::1] and 127.0.0.1: gRPC binds the ephemeral port on both, and when another process already holds it on 127.0.0.1 the test server keeps only [::1] while the client reaches that other process. Name the address once in the fixtures (kLoopbackHost, kListenAddress, kServerUri) and connect every client through ConnectTo(). --- .../transport/grpc/async_grpc_do_get_test.cc | 18 +++----- .../transport/grpc/async_grpc_do_put_test.cc | 46 ++++++++----------- .../grpc/async_grpc_exchange_test.cc | 23 +++------- .../grpc/async_grpc_handshake_test.cc | 36 +++++---------- .../transport/grpc/async_grpc_service_test.cc | 20 ++++---- .../grpc/async_grpc_streaming_test.cc | 9 ++-- .../transport/grpc/async_grpc_test_fixtures.h | 21 +++++++-- .../transport/grpc/async_grpc_unary_test.cc | 7 +-- 8 files changed, 75 insertions(+), 105 deletions(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc index 1bed2683febe..f575327c9603 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc @@ -25,16 +25,14 @@ TEST(AsyncGrpcTest, BasicDoGet) { int port = 0; ::grpc::ServerBuilder builder; - builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); builder.RegisterCallbackGenericService(&service); auto server = builder.BuildAndStart(); ASSERT_NE(server, nullptr); ASSERT_NE(port, 0); // Connect the existing arrow::flight::FlightClient implementation. - std::string uri = "grpc://localhost:" + std::to_string(port); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); // Call DoGet: the ticket must reach TestFlightServer::DoGetAsync. Ticket ticket{"basic-do-get-ticket"}; @@ -73,15 +71,13 @@ TEST(AsyncGrpcTest, SchemaOnlyDoGet) { int port = 0; ::grpc::ServerBuilder builder; - builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); builder.RegisterCallbackGenericService(&service); auto server = builder.BuildAndStart(); ASSERT_NE(server, nullptr); ASSERT_NE(port, 0); - std::string uri = "grpc://localhost:" + std::to_string(port); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); // This ticket is served as a stream with no batches at all. Ticket ticket{kSchemaOnlyTicket}; @@ -185,7 +181,7 @@ TEST(AsyncGrpcTest, DoGetClientCancelWithPendingPayloadFinishes) { // instead of waiting for a payload that will not come. const int64_t before = detail::AsyncReactorBase::live_reactors(); auto trace = std::make_shared(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); HangingDoGetServer flight_server(HangingDoGetServer::HangAt::kStream); FlightServerOptions options(location); options.middleware = { @@ -226,7 +222,7 @@ TEST(AsyncGrpcTest, DoGetClientDeadlineWhilePreparingFinishes) { // stream exists here, so Close() has nothing to deliver and the producer // notices through ServerCallContext::is_cancelled(). auto trace = std::make_shared(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); HangingDoGetServer flight_server(HangingDoGetServer::HangAt::kPrepare); FlightServerOptions options(location); options.middleware = { @@ -299,7 +295,7 @@ TEST(AsyncGrpcTest, DoGetPumpErrorStillClosesTheStream) { // The producer's first NextAsync() fails: the RPC ends with that status, and // the producer must still be told the RPC is over through Close(), like the // write-failure and cancel paths already do. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); ErrorPayloadDoGetServer flight_server; FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc index ceccdb91ed6f..579eb3fd89e5 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc @@ -35,7 +35,7 @@ TEST(AsyncGrpcTest, BasicDoPut) { int port = 0; ::grpc::ServerBuilder builder; - builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); builder.RegisterCallbackGenericService(&service); auto server = builder.BuildAndStart(); ASSERT_NE(server, nullptr); @@ -82,15 +82,13 @@ TEST(AsyncGrpcTest, EmptyDoPut) { int port = 0; ::grpc::ServerBuilder builder; - builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); builder.RegisterCallbackGenericService(&service); auto server = builder.BuildAndStart(); ASSERT_NE(server, nullptr); ASSERT_NE(port, 0); - std::string uri = "grpc://localhost:" + std::to_string(port); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); auto schema = arrow::schema( {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); @@ -124,15 +122,13 @@ TEST(AsyncGrpcTest, MetadataOnlyPutChunk) { int port = 0; ::grpc::ServerBuilder builder; - builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); builder.RegisterCallbackGenericService(&service); auto server = builder.BuildAndStart(); ASSERT_NE(server, nullptr); ASSERT_NE(port, 0); - std::string uri = "grpc://localhost:" + std::to_string(port); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); auto schema = arrow::schema( {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); @@ -165,15 +161,13 @@ TEST(AsyncGrpcTest, DoPutRejectedByListener) { int port = 0; ::grpc::ServerBuilder builder; - builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); builder.RegisterCallbackGenericService(&service); auto server = builder.BuildAndStart(); ASSERT_NE(server, nullptr); ASSERT_NE(port, 0); - std::string uri = "grpc://localhost:" + std::to_string(port); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); auto schema = TwoColumnSchema(); auto batch = TwoColumnBatch(schema); @@ -200,7 +194,7 @@ TEST(AsyncGrpcTest, CreateDoPutListenerServesUpload) { // The upload half of the server class API: CreateDoPutListener() hands out // the per-RPC listener, with no options-level factory. TestUploadServer flight_server; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); ASSERT_GT(flight_server.port(), 0); @@ -416,7 +410,7 @@ TEST(AsyncGrpcTest, DoPutClientDeadlineWithPendingHandlerFinishes) { // RPC itself instead of waiting for the listener: Shutdown()/Wait() below // would otherwise block, since gRPC keeps the call alive until Finish() // runs. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); HangingDoPutServer flight_server; FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); @@ -525,7 +519,7 @@ TEST(AsyncGrpcTest, DoPutClientCancelWhileListenerParkedStaysSafe) { // never touch the finished call) rather than reach into a destroyed reactor. // The test resolves the parked future from its own thread, and the server // keeps its shared_ptr alive, so the resolution lands on the real listener. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); DeferredDoPutServer flight_server; FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); @@ -583,7 +577,7 @@ TEST(AsyncGrpcTest, DoPutWithoutDescriptorOnFirstMessageIsRejected) { TestFlightServer flight_server; auto listener = std::make_shared(); flight_server.set_listener(listener); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); ASSERT_GT(flight_server.port(), 0); @@ -603,9 +597,9 @@ TEST(AsyncGrpcTest, DoPutWithoutDescriptorOnFirstMessageIsRejected) { // Drive the bidi call through the raw generic stub: the regular FlightClient // always sets the descriptor, which is exactly what this case lacks. - std::shared_ptr<::grpc::Channel> channel = - ::grpc::CreateChannel("127.0.0.1:" + std::to_string(flight_server.port()), - ::grpc::InsecureChannelCredentials()); + std::shared_ptr<::grpc::Channel> channel = ::grpc::CreateChannel( + std::string(kLoopbackHost) + ":" + std::to_string(flight_server.port()), + ::grpc::InsecureChannelCredentials()); ::grpc::GenericStub stub(channel); ::grpc::ClientContext context; // Bound the call: were the descriptor ever accepted, the client would wait @@ -649,7 +643,7 @@ TEST(AsyncGrpcTest, DoPutReportsAFailingOnFinishToTheClient) { TestFlightServer flight_server; auto listener = std::make_shared(); flight_server.set_listener(listener); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -682,14 +676,14 @@ TEST(AsyncGrpcTest, DoPutHalfCloseWithoutAnyMessageIsRejected) { TestFlightServer flight_server; auto listener = std::make_shared(); flight_server.set_listener(listener); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); ASSERT_GT(flight_server.port(), 0); - std::shared_ptr<::grpc::Channel> channel = - ::grpc::CreateChannel("127.0.0.1:" + std::to_string(flight_server.port()), - ::grpc::InsecureChannelCredentials()); + std::shared_ptr<::grpc::Channel> channel = ::grpc::CreateChannel( + std::string(kLoopbackHost) + ":" + std::to_string(flight_server.port()), + ::grpc::InsecureChannelCredentials()); ::grpc::GenericStub stub(channel); ::grpc::ClientContext context; context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(10)); @@ -728,7 +722,7 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagUploadRejectedOnDescriptor) { auto listener = std::make_shared(); listener->set_descriptor_status( arrow::Status::Invalid("listener rejected this descriptor")); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); FlightServerOptions options(location); flight_server.set_listener(listener); ASSERT_OK(flight_server.Init(options)); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc index f1480cea72e5..5ad40bd23bf0 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc @@ -34,14 +34,11 @@ namespace arrow::flight::transport::grpc { TEST(AsyncGrpcTest, DoExchangeEchoesBatches) { // Two batches up, two batches back, in order: this is the case that needs a // read and a write outstanding at the same time. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); FlightServerOptions options(location); EchoExchangeTestServer flight_server; ASSERT_OK(flight_server.Init(options)); - ASSERT_OK_AND_ASSIGN( - auto client_location, - Location::Parse("grpc://localhost:" + std::to_string(flight_server.port()))); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(client_location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); ASSERT_OK_AND_ASSIGN(auto exchange, client->DoExchange(FlightDescriptor::Command("echo"))); @@ -83,14 +80,11 @@ TEST(AsyncGrpcTest, DoExchangeMetadataOnlyDoesNotHang) { // handler must see the end of the exchange and reply, not wait for a schema // message that will never come (the case DataTest::TestDoExchangeNoData // covers). A hang here is the failure mode this test exists for. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); FlightServerOptions options(location); EchoExchangeTestServer flight_server; ASSERT_OK(flight_server.Init(options)); - ASSERT_OK_AND_ASSIGN( - auto client_location, - Location::Parse("grpc://localhost:" + std::to_string(flight_server.port()))); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(client_location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); ASSERT_OK_AND_ASSIGN(auto exchange, client->DoExchange(FlightDescriptor::Command("echo"))); @@ -120,14 +114,11 @@ TEST(AsyncGrpcTest, DoExchangeClientDisconnectFinishes) { // must let the handler unwind and finish the RPC, and the exchange counter // added to the test server must come back to zero -- a hang or a leaked // handler leaves it above zero. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); FlightServerOptions options(location); EchoExchangeTestServer flight_server; ASSERT_OK(flight_server.Init(options)); - ASSERT_OK_AND_ASSIGN( - auto client_location, - Location::Parse("grpc://localhost:" + std::to_string(flight_server.port()))); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(client_location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); ASSERT_OK_AND_ASSIGN(auto exchange, client->DoExchange(FlightDescriptor::Command("echo"))); @@ -175,7 +166,7 @@ TEST(AsyncGrpcTest, DoExchangeClientCancelWithPendingHandlerFinishes) { // finish the RPC (and unwind reader/writer), not wait for the handler. const int64_t before = detail::AsyncReactorBase::live_reactors(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); HangingExchangeServer flight_server; FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc index a68e0abc5de9..54ff0f3ea77c 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc @@ -122,14 +122,12 @@ struct AuthenticatedHarness { private: Status Start() { - ARROW_ASSIGN_OR_RAISE(auto location, Location::Parse("grpc://localhost:0")); + ARROW_ASSIGN_OR_RAISE(auto location, Location::Parse(kServerUri)); // The server class alone selects the async transport: deriving from it and // overriding its virtuals are the whole configuration. FlightServerOptions options(location); RETURN_NOT_OK(server.Init(options)); - ARROW_ASSIGN_OR_RAISE(auto client_location, - Location::ForScheme("grpc", "127.0.0.1", server.port())); - ARROW_ASSIGN_OR_RAISE(client, FlightClient::Connect(client_location)); + ARROW_ASSIGN_OR_RAISE(client, ConnectTo(server.port())); return Status::OK(); } }; @@ -141,7 +139,7 @@ class AsyncGenericTestServer : public AsyncGenericFlightServerBase {}; TEST(AsyncGrpcTest, HandshakeWithoutHookIsUnimplemented) { // A bare async server: Handshake is answered by the class's default virtual // with UNIMPLEMENTED, and the client's Authenticate() surfaces it. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); AsyncGenericTestServer flight_server; FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); @@ -162,7 +160,7 @@ TEST(AsyncGrpcTest, AsyncGenericFlightServerBaseImpliesAsyncTransport) { // yet the RPC is served by the generic callback service. DoPut is the // discriminator: an async server with no listener factory answers with its own // message rather than "NYI". - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); AsyncGenericTestServer flight_server; FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); @@ -193,10 +191,7 @@ TEST(AsyncGrpcTest, HandshakeAuthenticatesAndTokenIsRequired) { ASSERT_OK(harness.status); // 1. Without a handshake the data call is rejected before it is served. - ASSERT_OK_AND_ASSIGN(auto unauthenticated_location, - Location::ForScheme("grpc", "127.0.0.1", harness.server.port())); - ASSERT_OK_AND_ASSIGN(auto unauthenticated_client, - FlightClient::Connect(unauthenticated_location)); + ASSERT_OK_AND_ASSIGN(auto unauthenticated_client, ConnectTo(harness.server.port())); auto status = unauthenticated_client->DoGet(Ticket{"auth-ticket"}).status(); EXPECT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid token")); @@ -282,7 +277,7 @@ TEST(AsyncGrpcTest, HandshakePayloadIsBinarySafe) { // response is echoed back through the client's reader: an embedded NUL and // non-ASCII bytes must survive both directions unchanged. const std::string binary("p4ss\0w0rd\xC3\xA9", 11); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); AsyncAuthTestServer server(binary, binary, "bin-ticket"); FlightServerOptions options(location); ASSERT_OK(server.Init(options)); @@ -364,12 +359,7 @@ TEST(AsyncGrpcTest, ConcurrentHandshakesAllComplete) { std::vector statuses(kClients); for (int i = 0; i < kClients; i++) { threads.emplace_back([&harness, &statuses, i] { - auto location = Location::ForScheme("grpc", "127.0.0.1", harness.server.port()); - if (!location.ok()) { - statuses[i] = location.status(); - return; - } - auto client = FlightClient::Connect(*location); + auto client = ConnectTo(harness.server.port()); if (!client.ok()) { statuses[i] = client.status(); return; @@ -407,9 +397,7 @@ TEST(AsyncGrpcTest, RepeatedHandshakesOnFreshChannels) { AuthenticatedHarness harness; ASSERT_OK(harness.status); for (int i = 0; i < kRounds; i++) { - ASSERT_OK_AND_ASSIGN(auto location, - Location::ForScheme("grpc", "127.0.0.1", harness.server.port())); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(harness.server.port())); ASSERT_OK(client->Authenticate( {}, std::make_unique("user", "p4ssw0rd"))); ASSERT_OK(client->Close()); @@ -434,7 +422,7 @@ TEST(AsyncGrpcTest, LateHandshakeResponseLeavesServerUsable) { std::atomic handshakes{0}; }; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); SlowAnsweringServer server; FlightServerOptions options(location); ASSERT_OK(server.Init(options)); @@ -450,9 +438,7 @@ TEST(AsyncGrpcTest, LateHandshakeResponseLeavesServerUsable) { EXPECT_EQ(1, server.handshakes.load()) << "the first call must have reached the hook, so its write had to finish"; - ASSERT_OK_AND_ASSIGN(auto second_location, - Location::ForScheme("grpc", "127.0.0.1", server.port())); - ASSERT_OK_AND_ASSIGN(auto second_client, FlightClient::Connect(second_location)); + ASSERT_OK_AND_ASSIGN(auto second_client, ConnectTo(server.port())); ASSERT_OK(second_client->Authenticate( {}, std::make_unique("user", "p4ssw0rd"))); @@ -473,7 +459,7 @@ TEST(AsyncGrpcTest, EmptyHandshakeResponseIsDelivered) { } }; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); NoResponseServer server; FlightServerOptions options(location); ASSERT_OK(server.Init(options)); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc index 086ab00938ed..3195169348b7 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc @@ -43,7 +43,7 @@ TEST(AsyncGrpcTest, MiddlewareCallCompletedRunsOnUnknownMethod) { int port = 0; ::grpc::ServerBuilder builder; - builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); builder.RegisterCallbackGenericService(&service); auto server = builder.BuildAndStart(); ASSERT_NE(server, nullptr); @@ -70,15 +70,13 @@ TEST(AsyncGrpcTest, OtherMethodsAreUnimplemented) { int port = 0; ::grpc::ServerBuilder builder; - builder.AddListeningPort("localhost:0", ::grpc::InsecureServerCredentials(), &port); + builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); builder.RegisterCallbackGenericService(&service); auto server = builder.BuildAndStart(); ASSERT_NE(server, nullptr); ASSERT_NE(port, 0); - std::string uri = "grpc://localhost:" + std::to_string(port); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(uri)); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); // Every typed FlightService method is served now, the fallback reactor is // still reachable through a method name no stub knows (which maps to @@ -95,13 +93,13 @@ TEST(AsyncGrpcTest, OtherMethodsAreUnimplemented) { } TEST(AsyncGrpcTest, UseAsyncGrpcFlag) { - ASSERT_OK_AND_ASSIGN(auto default_location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto default_location, Location::Parse(kServerUri)); FlightServerOptions default_options(default_location); // Same adapter as the tests above, but now started through the async // server's own Init instead of a hand-built grpc::ServerBuilder. TestFlightServer flight_server; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); ASSERT_GT(flight_server.port(), 0); @@ -160,7 +158,7 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagServesDoPut) { // through to UNIMPLEMENTED. TestFlightServer flight_server; auto listener = std::make_shared(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); FlightServerOptions options(location); flight_server.set_listener(listener); ASSERT_OK(flight_server.Init(options)); @@ -205,7 +203,7 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagRefusesBlockingAuthHandler) { // on callback threads), so a server configured with one must refuse to start // instead of silently serving unauthenticated requests. Middleware *is* // supported now - see UseAsyncGrpcFlagAllowsMiddleware. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); TestFlightServer auth_server; FlightServerOptions with_auth(location); @@ -222,7 +220,7 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagAllowsMiddleware) { // path: StartCall at call creation, SendingHeaders and CallCompleted at the // end - success or failure alike. auto trace = std::make_shared(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); TestFlightServer flight_server; FlightServerOptions options(location); @@ -248,7 +246,7 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagAllowsMiddleware) { } TEST(AsyncGrpcTest, UseAsyncGrpcFlagMiddlewareCanRejectCall) { - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); TestFlightServer flight_server; FlightServerOptions options(location); options.middleware = { diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc index 9b0a8cadd2be..162b7769f967 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc @@ -243,14 +243,11 @@ TEST(AsyncGrpcTest, ListFlightsAsyncCompletesOnAnotherThread) { } }; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); DeferredListingServer flight_server; FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); - ASSERT_OK_AND_ASSIGN( - auto client_location, - Location::Parse("grpc://localhost:" + std::to_string(flight_server.port()))); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(client_location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); ASSERT_OK_AND_ASSIGN(auto listing, client->ListFlights()); int seen = 0; @@ -280,7 +277,7 @@ TEST(AsyncGrpcTest, ListFlightsDeadlineWithPendingHandlerFinishes) { // the reactor must be released even though the handler never completes. const int64_t before = detail::AsyncReactorBase::live_reactors(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); HangingListingServer flight_server; FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h index e08960ad7119..b6abde071560 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h @@ -732,6 +732,16 @@ MakeAsyncHelper() { /*auth_handler=*/nullptr, MiddlewareFactoryList{}); } +// The host every test server listens on and every client connects to: +// 127.0.0.1, not localhost. localhost also resolves to [::1], gRPC then binds +// the ephemeral port on both, and when another process already holds it on +// 127.0.0.1 the server keeps only [::1] while the client reaches that process. +static const char kLoopbackHost[] = "127.0.0.1"; +// kLoopbackHost with an ephemeral port, for a hand-built grpc::ServerBuilder. +static const char kListenAddress[] = "127.0.0.1:0"; +// kLoopbackHost with an ephemeral port, for a FlightServerBase's options. +static const char kServerUri[] = "grpc://127.0.0.1:0"; + // The method name no FlightService stub knows: the transport must answer it // with the UNIMPLEMENTED fallback reactor rather than any typed path. It is // called over a raw generic stub, because only a generic client can name a @@ -742,8 +752,9 @@ constexpr const char* kUnknownMethod = // Call `kUnknownMethod` on the server listening on `port` with the sync-style // generic stub API, and return the server's status. inline ::grpc::Status CallUnknownMethod(int port) { - std::shared_ptr<::grpc::Channel> channel = ::grpc::CreateChannel( - "127.0.0.1:" + std::to_string(port), ::grpc::InsecureChannelCredentials()); + std::shared_ptr<::grpc::Channel> channel = + ::grpc::CreateChannel(std::string(kLoopbackHost) + ":" + std::to_string(port), + ::grpc::InsecureChannelCredentials()); ::grpc::GenericStub stub(channel); ::grpc::ClientContext context; ::grpc::ByteBuffer request; @@ -772,15 +783,15 @@ inline std::shared_ptr OneColumnBatch( {arrow::ArrayFromJSON(arrow::int64(), "[1, 2]")}); } -// The listen address the tests start their servers on: an ephemeral port. +// The location the tests start their servers on, see kServerUri. inline const Location& LocalhostLocation() { - static const Location location = *Location::Parse("grpc://localhost:0"); + static const Location location = *Location::Parse(kServerUri); return location; } // Connect a FlightClient to the server listening on `port`. inline arrow::Result> ConnectTo(int port) { - ARROW_ASSIGN_OR_RAISE(auto location, Location::ForScheme("grpc", "127.0.0.1", port)); + ARROW_ASSIGN_OR_RAISE(auto location, Location::ForScheme("grpc", kLoopbackHost, port)); return FlightClient::Connect(location); } diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc index 204c4b527b79..4a1e7bfa172f 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc @@ -194,14 +194,11 @@ TEST(AsyncGrpcTest, GetFlightInfoAsyncCompletesOnAnotherThread) { } }; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse("grpc://localhost:0")); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); DeferredInfoServer flight_server; FlightServerOptions options(location); ASSERT_OK(flight_server.Init(options)); - ASSERT_OK_AND_ASSIGN( - auto client_location, - Location::Parse("grpc://localhost:" + std::to_string(flight_server.port()))); - ASSERT_OK_AND_ASSIGN(auto client, FlightClient::Connect(client_location)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); ASSERT_OK_AND_ASSIGN(auto info, client->GetFlightInfo(FlightDescriptor::Command("deferred"))); From a10692b4e683d00c8e36fea95f231237764c6d32 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:56 +0100 Subject: [PATCH 03/25] [C++][FlightRPC] Wait for the upload before cancelling it in the DoPut cancel test WaitForListener() returns as soon as CreateDoPutListener() does, before the transport is installed on the listener: a Cancel() in that window is refused with Invalid and the test failed. Wait for the first batch, which proves the transport is installed. --- .../flight/transport/grpc/async_grpc_do_put_test.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc index 579eb3fd89e5..6aab1de43294 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc @@ -358,10 +358,15 @@ TEST(AsyncGrpcTest, DoPutCancelFromServerSideFinishesTheUpload) { auto& writer = *put->writer; ASSERT_OK(writer.WriteRecordBatch(*batch)); - // Wait for the listener the virtual handed out, then cancel it from this - // (non-gRPC) thread: that is the mid-upload server-side abort. + // Wait for the upload to reach the listener the virtual handed out, then + // cancel it from this (non-gRPC) thread: that is the mid-upload server-side + // abort. The listener exists as soon as CreateDoPutListener() returns, but + // Cancel() only reaches the RPC once the transport is installed, which the + // first batch proves. auto listener = flight_server.WaitForListener(0); ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; + ASSERT_TRUE(WaitFor([&listener] { return listener->batches().size() >= 1; })) + << "the server never consumed the batch"; listener->CancelWith(arrow::Status::Cancelled("server said stop")); // The client sees the cancellation on the surface that observes the RPC. From bdf123f4184593c6979eb456767eac6b1b407f0f Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:57 +0100 Subject: [PATCH 04/25] [C++][FlightRPC] Push cancellation to async handlers Add AsyncGenericFlightServerBase::AddCancelCallback(): it runs a callback once if the RPC is cancelled, or at once if it already is, so a handler can stop the work behind a pending future instead of polling is_cancelled(). A context that no async server handed out is refused with Invalid. AsyncReactorBase::OnCancel() is now final: the reactor's TearDownOnCancel() settles the RPC's status, then the registered callbacks run. DoGet now delivers Close() to a stream that DoGetAsync() resolves after the RPC was cancelled; such a stream used to be dropped without it. --- cpp/src/arrow/flight/server_async.cc | 19 ++++ cpp/src/arrow/flight/server_async.h | 36 +++++-- .../transport/grpc/async_grpc_do_get.cc | 51 +++++----- .../transport/grpc/async_grpc_do_get_test.cc | 97 +++++++++++++++++++ .../transport/grpc/async_grpc_do_put.cc | 2 +- .../transport/grpc/async_grpc_exchange.cc | 2 +- .../grpc/async_grpc_exchange_internal.h | 2 +- .../grpc/async_grpc_service_internal.h | 64 +++++++++++- .../transport/grpc/async_grpc_service_test.cc | 28 ++++++ 9 files changed, 265 insertions(+), 36 deletions(-) diff --git a/cpp/src/arrow/flight/server_async.cc b/cpp/src/arrow/flight/server_async.cc index 19529c79479a..590f9c61dfde 100644 --- a/cpp/src/arrow/flight/server_async.cc +++ b/cpp/src/arrow/flight/server_async.cc @@ -21,13 +21,18 @@ #include "arrow/flight/server_async.h" #include +#include #include #include #include #include #include "arrow/flight/transport.h" +#include "arrow/flight/transport/grpc/async_grpc_compat.h" #include "arrow/flight/transport/grpc/grpc_server.h" +#if ARROW_FLIGHT_HAS_ASYNC_SERVER +# include "arrow/flight/transport/grpc/async_grpc_service_internal.h" +#endif #include "arrow/flight/transport_server.h" #include "arrow/flight/transport_server_internal.h" #include "arrow/flight/types.h" @@ -82,6 +87,20 @@ Status AsyncGenericFlightServerBase::Shutdown( Status AsyncGenericFlightServerBase::Wait() { return impl_->lifecycle.Wait(); } +Status AsyncGenericFlightServerBase::AddCancelCallback(const ServerCallContext& context, + std::function callback) { +#if ARROW_FLIGHT_HAS_ASYNC_SERVER + // Only the contexts the async transport hands its handlers carry a registry. + if (const auto* async_context = + dynamic_cast(&context)) { + return async_context->AddCancelCallback(std::move(callback)); + } +#endif + return Status::Invalid( + "AddCancelCallback() needs the context an AsyncGenericFlightServerBase handed " + "to a handler"); +} + // --- handler defaults: the same answers FlightServerBase's defaults give --- arrow::Future> diff --git a/cpp/src/arrow/flight/server_async.h b/cpp/src/arrow/flight/server_async.h index 84532d6103d6..39b444cc3fe1 100644 --- a/cpp/src/arrow/flight/server_async.h +++ b/cpp/src/arrow/flight/server_async.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include #include @@ -61,9 +62,10 @@ class ARROW_FLIGHT_EXPORT AsyncFlightDataStream { /// \brief Stop the stream: the RPC it was producing for is over. /// - /// Called at most once, possibly from any thread. A pending - /// NextAsync()/GetSchemaPayloadAsync() future must be resolved here (or - /// abandoned knowingly): the transport does not wait for it. + /// Called once, possibly from any thread, whichever way the RPC ends, including for a + /// stream DoGetAsync() resolves after the RPC was cancelled. A pending + /// NextAsync()/GetSchemaPayloadAsync() future must be resolved here (or abandoned + /// knowingly): the transport does not wait for it. /// Its return status becomes the RPC's final status only on the normal end-of-stream /// path (after NextAsync() reports end of stream). /// On every other path (write failure, cancellation) the RPC already has a status and @@ -252,6 +254,26 @@ class ARROW_FLIGHT_EXPORT AsyncGenericFlightServerBase { /// is still unresolved. Status Wait(); + /// \brief Run `callback` once if the RPC behind `context` is cancelled: the + /// push counterpart of ServerCallContext::is_cancelled(), to stop work a + /// handler started instead of polling. + /// + /// The callback runs when the RPC is cancelled (by the client, a deadline, a + /// network error, etc.), on the transport thread that observes it and after + /// the transport has settled the RPC's status; or before this call returns, + /// on the calling thread, if the RPC is already cancelled. It never runs if + /// the RPC ends without being cancelled. It must not block. + /// + /// Call this while the RPC is live: during a handler call, or while a future + /// the transport awaits from that handler is still pending. + /// \param[in] context The context an AsyncGenericFlightServerBase handed to the + /// handler. + /// \param[in] callback The function to run on cancellation. + /// \return OK once the callback is registered (or has run); Invalid when + /// `context` comes from anywhere else. + static Status AddCancelCallback(const ServerCallContext& context, + std::function callback); + /// \brief Asynchronously retrieve a list of available fields given an /// optional opaque criteria. /// \param[in] context is the server call context. @@ -342,9 +364,11 @@ class ARROW_FLIGHT_EXPORT AsyncGenericFlightServerBase { /// whatever stream it resolves to, and must not be blocked waiting for it. /// /// If the RPC is cancelled while the returned future is pending, the - /// transport finishes the RPC without waiting for it, and the stream it - /// eventually resolves to is dropped without Close() ever being delivered - /// (the transport no longer holds the stream). + /// transport finishes the RPC without waiting for it. The stream the future + /// eventually resolves to is never read from, but still receives Close(): + /// on the thread that completes the future, or on the transport's thread when + /// that completion races the cancellation. To stop the work behind a pending + /// future, register a callback with AddCancelCallback(). virtual arrow::Future> DoGetAsync( const ServerCallContext& context, const Ticket& request); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc index 1f5fd6a18bd0..1a7a6fe816bf 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc @@ -52,28 +52,33 @@ class DoGetReactor : public AsyncReactorBase { future.AddCallback( [this, token = hold()]( const arrow::Result>& result) { - if (!finished()) { - if (!result.ok()) { - FinishOnce(result.status()); - } else { - std::shared_ptr stream = result.ValueOrDie(); - if (stream == nullptr) { - FinishOnce(arrow::Status::KeyError("No data in this flight")); - } else { - bool already_closed; - { - std::lock_guard lock(stream_mutex_); - async_data_stream_ = stream; - already_closed = closed_; - } - if (already_closed) { - ARROW_WARN_NOT_OK(stream->Close(), - "DoGet: closing a cancelled call's stream failed"); - } else { - WriteNextPayload(); - } - } - } + if (!result.ok()) { + FinishOnce(result.status()); + return; + } + std::shared_ptr stream = result.ValueOrDie(); + if (stream == nullptr) { + FinishOnce(arrow::Status::KeyError("No data in this flight")); + return; + } + bool already_closed; + { + std::lock_guard lock(stream_mutex_); + async_data_stream_ = stream; + already_closed = closed_; + } + if (already_closed) { + // The RPC ended before the stream arrived: the teardown found no + // stream to close, so this one is closed here. + ARROW_WARN_NOT_OK(stream->Close(), + "DoGet: closing a cancelled call's stream failed"); + } else if (finished()) { + // Cancelled, but the teardown's CloseStreamOnce() has not run yet: + // whichever CloseStreamOnce() runs first closes the stream. + ARROW_WARN_NOT_OK(CloseStreamOnce(), + "DoGet: closing a finished call's stream failed"); + } else { + WriteNextPayload(); } }); } @@ -91,7 +96,7 @@ class DoGetReactor : public AsyncReactorBase { WriteNextPayload(); } - void OnCancel() override { + void TearDownOnCancel() override { FinishOnce(arrow::Status::Cancelled()); ARROW_WARN_NOT_OK(CloseStreamOnce(), "DoGet: closing the stream of a cancelled call failed"); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc index f575327c9603..450ee1d5cd27 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc @@ -245,6 +245,103 @@ TEST(AsyncGrpcTest, DoGetClientDeadlineWhilePreparingFinishes) { EXPECT_EQ(1, trace->call_completed); } +/// A DoGet server whose DoGetAsync() registers a cancel callback and parks its +/// future until the test resolves it: the test drives a cancellation, a +/// callback registered once the RPC is already cancelled, and a stream that +/// arrives after the RPC ended. +class CancelCallbackDoGetServer final : public AsyncGenericFlightServerBase { + public: + arrow::Future> DoGetAsync( + const ServerCallContext& context, const Ticket&) override { + auto pending = arrow::Future>::Make(); + { + std::lock_guard lock(mutex_); + context_ = &context; + pending_ = pending; + } + ARROW_RETURN_NOT_OK( + AddCancelCallback(context, [this] { cancel_callbacks.fetch_add(1); })); + return pending; + } + + /// Register on the parked call's context from the test thread: the call is + /// still live, since the transport awaits the parked future. + Status AddCancelCallbackFromTest(std::function callback) { + std::lock_guard lock(mutex_); + return AddCancelCallback(*context_, std::move(callback)); + } + + /// Resolve the parked DoGetAsync() future with a stream, on this thread. + void ResolveWithStream() { + arrow::Future> pending; + { + std::lock_guard lock(mutex_); + pending = pending_; + } + auto reader = arrow::RecordBatchReader::Make( + {}, arrow::schema({arrow::field("value", arrow::int64())})) + .ValueOrDie(); + std::shared_ptr stream = std::make_shared( + std::make_shared(std::move(reader)), &next_awaits, + &stream_closes); + pending.MarkFinished(std::move(stream)); + } + + std::atomic cancel_callbacks{0}; + std::atomic next_awaits{0}; + std::atomic stream_closes{0}; + + private: + std::mutex mutex_; + const ServerCallContext* context_ = nullptr; + arrow::Future> pending_; +}; + +TEST(AsyncGrpcTest, DoGetCancelCallbackReachesPendingHandler) { + // The client's deadline expires while DoGetAsync() is pending: the callback + // the handler registered runs once without any polling, a callback + // registered once the RPC is cancelled runs at once, and the stream the + // handler then resolves to is never read from but still receives Close(). + const int64_t before = detail::AsyncReactorBase::live_reactors(); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); + CancelCallbackDoGetServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + FlightCallOptions impatient; + impatient.timeout = std::chrono::milliseconds(500); + EXPECT_FALSE(client->DoGet(impatient, Ticket{"hang"}).ok()) + << "the deadline must expire while the handler is pending"; + ASSERT_TRUE(WaitFor([&] { return flight_server.cancel_callbacks.load() >= 1; })) + << "the cancellation never reached the handler's callback"; + + auto late_callbacks = std::make_shared>(0); + ASSERT_OK(flight_server.AddCancelCallbackFromTest( + [late_callbacks] { late_callbacks->fetch_add(1); })); + EXPECT_EQ(1, late_callbacks->load()) + << "a callback registered after the cancellation must run before " + "AddCancelCallback() returns"; + + // The transport's continuation runs inline, on this thread. + flight_server.ResolveWithStream(); + EXPECT_EQ(1, flight_server.stream_closes.load()) + << "a stream that arrives after the RPC ended must still be closed"; + EXPECT_EQ(0, flight_server.next_awaits.load()) + << "a stream that arrives after the RPC ended must not be read from"; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + EXPECT_EQ(1, flight_server.cancel_callbacks.load()) + << "the cancel callback must run exactly once"; + EXPECT_TRUE(WaitFor([&] { + return detail::AsyncReactorBase::live_reactors() == before; + })) << "a reactor leaked: still alive after the server stopped (before=" + << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; +} + /// A DoGet stream whose first payload is an error: the transport must fail the /// RPC and still tell the producer the RPC is over through Close(). class ErrorPayloadDoGetStream final : public AsyncFlightDataStream { diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc index 6449912f8e1d..b1c46960e1ed 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc @@ -184,7 +184,7 @@ class DoPutReactor : public AsyncReactorBase, } } - void OnCancel() override { + void TearDownOnCancel() override { // The RPC is cancelled: report it to the listener but do not let a failing // OnFinish replace the cancellation the client is seeing. The report is // fire-and-forget here; keeping a reference alive for it is unnecessary as diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc index 16ae8f3ac723..87adb56aefc1 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc @@ -83,7 +83,7 @@ void ExchangeReactor::OnWriteDone(bool ok) { writer_->OnWriteDone(ok); } -void ExchangeReactor::OnCancel() { +void ExchangeReactor::TearDownOnCancel() { // Finish first (the handler's future may never resolve), then unwind the // reader and writer so a handler parked on their futures wakes up. FinishOnce(arrow::Status::Cancelled()); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h index e64edf4d4e1f..5457d8c6bbdc 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h @@ -183,7 +183,7 @@ class ExchangeReactor final : public AsyncReactorBase { void OnReadDone(bool ok) override; void OnWriteDone(bool ok) override; - void OnCancel() override; + void TearDownOnCancel() override; void OnDone() override; /// \brief The reader's demand: arm the next read. The demand guarantees at diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h index 394a61b4512a..6a3a9ad2e950 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h @@ -19,11 +19,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include @@ -41,7 +43,51 @@ namespace pb = arrow::flight::protocol; #if ARROW_FLIGHT_HAS_ASYNC_SERVER -using AsyncCallContext = GrpcServerCallContext<::grpc::CallbackServerContext>; +/// \brief The context of one async call: GrpcServerCallContext plus the +/// callbacks AsyncGenericFlightServerBase::AddCancelCallback() registers, which +/// AsyncReactorBase::OnCancel() runs. +class AsyncCallContext : public GrpcServerCallContext<::grpc::CallbackServerContext> { + public: + explicit AsyncCallContext(::grpc::CallbackServerContext* context) + : GrpcServerCallContext<::grpc::CallbackServerContext>(context) {} + + arrow::Status AddCancelCallback(std::function callback) const { + { + std::lock_guard lock(cancel_->mutex); + if (!cancel_->cancelled) { + cancel_->callbacks.push_back(std::move(callback)); + return arrow::Status::OK(); + } + } + // Already cancelled: run it now, on the caller's thread, unlocked. + callback(); + return arrow::Status::OK(); + } + + /// \brief Run the registered callbacks once, unlocked, so a callback may + /// register another (which then runs inline). Called by + /// AsyncReactorBase::OnCancel() only. + void RunCancelCallbacks() { + std::vector> callbacks; + { + std::lock_guard lock(cancel_->mutex); + cancel_->cancelled = true; + callbacks.swap(cancel_->callbacks); + } + for (auto& callback : callbacks) { + callback(); + } + } + + private: + struct CancelCallbacks { + std::mutex mutex; + bool cancelled = false; + std::vector> callbacks; + }; + /// Behind a pointer so the context stays movable. + std::unique_ptr cancel_ = std::make_unique(); +}; template ::grpc::ByteBuffer MakeWriteBuffer(const ProtoT& message) { @@ -103,9 +149,15 @@ class AsyncReactorBase : public ::grpc::ServerGenericBidiReactor { return live_reactors_.load(std::memory_order_relaxed); } - /// The default cancellation: report the RPC as cancelled and finish it. - /// Reactors with their own teardown override this and call it first. - void OnCancel() override { FinishOnce(arrow::Status::Cancelled()); } + /// \brief gRPC's cancellation (client cancel, deadline, network failure). + /// + /// Final, so every reactor does both halves in this order: the transport's + /// teardown settles the RPC's status, then the callbacks handlers registered + /// with AsyncGenericFlightServerBase::AddCancelCallback() run. + void OnCancel() final { + TearDownOnCancel(); + flight_context_.RunCancelCallbacks(); + } void OnDone() override { ReleaseHold(); } @@ -125,6 +177,10 @@ class AsyncReactorBase : public ::grpc::ServerGenericBidiReactor { /// Whether FinishOnce() has already run (the RPC's status is settled). bool finished() const { return finished_.load(); } + /// The transport's teardown on cancellation: finish the RPC as cancelled. + /// Reactors with more to unwind override it and call FinishOnce() first. + virtual void TearDownOnCancel() { FinishOnce(arrow::Status::Cancelled()); } + /// The call context: handlers and the teardown paths need it. AsyncCallContext& flight_context() { return flight_context_; } diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc index 3195169348b7..4151952407bd 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc @@ -263,4 +263,32 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagMiddlewareCanRejectCall) { ASSERT_OK(flight_server.Wait()); } +/// A ServerCallContext no async server handed out, as a synchronous +/// FlightServerBase handler would hold. +class ForeignCallContext final : public ServerCallContext { + public: + const std::string& peer_identity() const override { return empty_; } + const std::string& peer() const override { return empty_; } + void AddHeader(const std::string&, const std::string&) const override {} + void AddTrailer(const std::string&, const std::string&) const override {} + ServerMiddleware* GetMiddleware(const std::string&) const override { return nullptr; } + bool is_cancelled() const override { return false; } + const CallHeaders& incoming_headers() const override { return headers_; } + + private: + std::string empty_; + CallHeaders headers_; +}; + +TEST(AsyncGrpcTest, AddCancelCallbackRefusesAContextItDidNotHandOut) { + // Cancellation is only pushed for calls the async server serves: any other + // context is refused, and its callback never runs. + ForeignCallContext context; + bool ran = false; + const Status status = + AsyncGenericFlightServerBase::AddCancelCallback(context, [&ran] { ran = true; }); + EXPECT_TRUE(status.IsInvalid()) << status; + EXPECT_FALSE(ran) << "a refused callback must not run"; +} + } // namespace arrow::flight::transport::grpc From c29454b5674f5328018cde36326b7ab5f2e2f42b Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:57 +0100 Subject: [PATCH 05/25] [C++][FlightRPC] Make DoPut metadata acknowledgements awaitable AsyncFlightDataWriter::WriteMetadata() returned a Status and refused a second write while one was in flight, with nothing to await. Replace it with WriteMetadataAsync(), whose future completes once the PutResult is written (IOError if the RPC failed first): a listener can chain OnNext() on it to apply backpressure. --- cpp/src/arrow/flight/flight_data_decoder.cc | 10 +-- cpp/src/arrow/flight/flight_data_decoder.h | 34 ++++++---- .../transport/grpc/async_grpc_do_put.cc | 64 ++++++++++++------- .../transport/grpc/async_grpc_do_put_test.cc | 8 ++- .../transport/grpc/async_grpc_test_fixtures.h | 34 +++++----- 5 files changed, 89 insertions(+), 61 deletions(-) diff --git a/cpp/src/arrow/flight/flight_data_decoder.cc b/cpp/src/arrow/flight/flight_data_decoder.cc index 6d60ceb9692a..e7e591a76eb1 100644 --- a/cpp/src/arrow/flight/flight_data_decoder.cc +++ b/cpp/src/arrow/flight/flight_data_decoder.cc @@ -242,19 +242,19 @@ AsyncFlightDataListener::AsyncFlightDataListener() = default; AsyncFlightDataListener::~AsyncFlightDataListener() = default; AsyncFlightDataWriter::~AsyncFlightDataWriter() = default; -Status AsyncFlightDataWriter::WriteMetadata(const Buffer& app_metadata) { +Future<> AsyncFlightDataWriter::WriteMetadataAsync(const Buffer& app_metadata) { // Install() and Clear() swap transport_ under writer_mutex_, so both the read // and the call must happen under it: otherwise this can read the old pointer // just before DoPutReactor::OnDone() runs Clear(), and then call through a - // destructed reactor. The transport's own WriteMetadata only takes its + // destructed reactor. The transport's own WriteMetadataAsync only takes its // private write_mutex_, so there is no lock-order cycle. std::lock_guard lock(writer_mutex_); if (transport_ == nullptr) { - return Status::Invalid( + return Future<>::MakeFinished(Status::Invalid( "no upload in flight to write metadata to: the writer is only usable " - "while the upload's RPC is running"); + "while the upload's RPC is running")); } - return transport_->WriteMetadata(app_metadata); + return transport_->WriteMetadataAsync(app_metadata); } namespace internal { diff --git a/cpp/src/arrow/flight/flight_data_decoder.h b/cpp/src/arrow/flight/flight_data_decoder.h index 315547371274..fcd0ced9759e 100644 --- a/cpp/src/arrow/flight/flight_data_decoder.h +++ b/cpp/src/arrow/flight/flight_data_decoder.h @@ -51,11 +51,13 @@ class ARROW_FLIGHT_EXPORT FlightDataListenerTransport { /// \param status The status to finish the upload with. virtual void CancelUpload(Status status) = 0; - /// \brief Send one application metadata message to the client, from any thread. - /// \param app_metadata The metadata to attach to the next PutResult message. - /// \return OK when the message was handed to the transport, or a non-OK status - /// when the upload is already over. - virtual Status WriteMetadata(const Buffer& app_metadata) = 0; + /// \brief Start sending one application metadata message to the client, from + /// any thread. + /// \param app_metadata The metadata to attach to the next PutResult message, + /// copied before the call returns. + /// \return A future completed once the message has been written, or already + /// failed when a message is still in flight or the upload is ending or over. + virtual Future<> WriteMetadataAsync(const Buffer& app_metadata) = 0; /// \brief Pin this transport's lifetime, from any thread. /// @@ -98,20 +100,26 @@ class ARROW_FLIGHT_EXPORT FlightDataListenerTransport { /// The async counterpart of FlightMetadataWriter: the transport hands one to the /// AsyncFlightDataListener at the start of an upload (AsyncFlightDataListener::writer) /// and it stays valid for as long as the upload's RPC runs. Every call sends one -/// PutResult message immediately, the client reads them with ReadMetadata() on -/// its DoPutResult. +/// PutResult message, the client reads them with ReadMetadata() on its DoPutResult. +/// One message is in flight at a time: await each returned future before writing +/// the next. Returning a future chained on it from OnNext() holds the next read +/// until the acknowledgement is written, which is how a listener applies +/// backpressure. class ARROW_FLIGHT_EXPORT AsyncFlightDataWriter { public: virtual ~AsyncFlightDataWriter(); /// \brief Send a metadata message to the client. /// - /// Safe to call from any thread while the upload is in flight. Once the - /// upload has ended the call reports the ending (Cancelled or FailedPrecondition) - /// instead of writing to a finished RPC. + /// Safe to call from any thread while the upload is in flight: the write is + /// started on the calling thread, and `app_metadata` is copied before the call + /// returns. /// \param app_metadata The application metadata to send. - /// \return Status indicating success or failure. - Status WriteMetadata(const Buffer& app_metadata); + /// \return A future completed, on the gRPC thread that sees the write finish, + /// once the message has been written (IOError if the write failed or the + /// upload's RPC ended first). It is already failed with Invalid when the + /// previous message is still in flight, or the upload is ending or over. + Future<> WriteMetadataAsync(const Buffer& app_metadata); private: friend class internal::FlightDataListenerTransport; @@ -120,7 +128,7 @@ class ARROW_FLIGHT_EXPORT AsyncFlightDataWriter { /// FlightDataListenerTransport::Install/Clear, like /// AsyncFlightDataListener::transport_. internal::FlightDataListenerTransport* transport_ = nullptr; - /// Guards transport_, since WriteMetadata may be called from any thread while + /// Guards transport_, since WriteMetadataAsync may be called from any thread while /// the transport installs or clears the pointer. mutable std::mutex writer_mutex_; }; diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc index b1c46960e1ed..cbeda880829d 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc @@ -53,34 +53,35 @@ class DoPutReactor : public AsyncReactorBase, AsyncReactorBase::ReleaseHold(); } - Status WriteMetadata(const Buffer& app_metadata) override { + Future<> WriteMetadataAsync(const Buffer& app_metadata) override { std::lock_guard lock(write_mutex()); if (finished()) { - return Status::Invalid( - "the upload is over: WriteMetadata() is only usable while the " - "upload's RPC is running"); + return Future<>::MakeFinished(Status::Invalid( + "the upload is over: WriteMetadataAsync() is only usable while the " + "upload's RPC is running")); } if (metadata_in_flight_ || ending_) { // One message in flight at a time: OnWriteDone serializes the PutResults. - return Status::Invalid( - "a PutResult message is already in flight: await it before writing " - "another metadata message"); + return Future<>::MakeFinished(Status::Invalid( + "a PutResult message is already in flight: await the previous " + "WriteMetadataAsync() future before writing another metadata message")); } metadata_in_flight_ = true; + metadata_written_ = Future<>::Make(); pb::PutResult pb_result; if (app_metadata.size() > 0) { pb_result.set_app_metadata(app_metadata.data(), app_metadata.size()); } write_buf_ = MakeWriteBuffer(pb_result); StartWrite(&write_buf_); - return Status::OK(); + return metadata_written_; } void OnReadDone(bool ok) override { if (!ok) { // Decide under the lock, act outside it: FinishUpload() runs the // listener's OnFinish(), which is application code and may itself call - // WriteMetadata(). That takes writer_mutex_ and then write_mutex_, the + // WriteMetadataAsync(). That takes writer_mutex_ and then write_mutex_, the // reverse of a caller holding write_mutex_ and waiting for writer_mutex_, // so invoking it here would deadlock the two threads. { @@ -90,7 +91,7 @@ class DoPutReactor : public AsyncReactorBase, } if (read_descriptor_) { // The client half-closed: this ack ends the upload. It takes the - // same write slot as a mid-upload WriteMetadata, which may be on the + // same write slot as a mid-upload WriteMetadataAsync, which may be on the // wire right now: starting a second write would clobber write_buf_ // and the RPC would never see its OnWriteDone. ending_ = true; @@ -155,30 +156,43 @@ class DoPutReactor : public AsyncReactorBase, } void OnWriteDone(bool ok) override { - // Decide under the lock, act outside it: FinishUpload() runs the listener's - // OnFinish(), which is application code and may call back into WriteMetadata(). + // Decide under the lock, act outside it: completing a WriteMetadataAsync() + // future runs its continuation and FinishUpload() runs the listener's + // OnFinish(); both are application code that may call WriteMetadataAsync(). Status finish_status; bool ended = false; + // The mid-upload message this write carried, unless it was the ending ack. + Future<> written; { std::lock_guard lock(write_mutex()); - if (finished()) { - return; - } - if (!ok) { - finish_status = MakeFlightError(FlightStatusCode::Internal, "Write failed"); - ended = true; - } else if (ending_ack_in_flight_) { - // The ending ack reached the client: that is the clean end of the upload. - ended = true; - } else { + if (!ending_ack_in_flight_) { + written = std::move(metadata_written_); + metadata_written_ = Future<>(); metadata_in_flight_ = false; - if (ending_) { + } + if (!finished()) { + if (!ok) { + finish_status = MakeFlightError(FlightStatusCode::Internal, "Write failed"); + ended = true; + // The call is broken: a continuation of the failed future must not + // start another write before FinishOnce() runs. + ending_ = true; + } else if (ending_ack_in_flight_) { + // The ending ack reached the client: that is the clean end of the upload. + ended = true; + } else if (ending_) { // The client half-closed while a metadata message was on the wire: // with the slot free, the ending ack is the next write. StartEndingAck(); } } } + if (written.is_valid()) { + written.MarkFinished( + ok ? Status::OK() + : Status::IOError("the PutResult message was not written: " + "the upload's RPC failed or was cancelled")); + } if (ended) { FinishUpload(std::move(finish_status)); } @@ -232,11 +246,13 @@ class DoPutReactor : public AsyncReactorBase, /// Whether the descriptor of the first message has been checked. bool read_descriptor_ = false; // The write side is serialized by AsyncReactorBase::write_mutex_, which - // FinishOnce() also takes: WriteMetadata() runs on the listener's thread + // FinishOnce() also takes: WriteMetadataAsync() runs on the listener's thread // while OnReadDone()/OnWriteDone() run on gRPC callback threads, and this // must be the same mutex the base uses or finishing can race a write. /// Whether a mid-upload metadata message is on the wire. bool metadata_in_flight_ = false; + /// Completed by OnWriteDone() once that mid-upload message is written. + Future<> metadata_written_; /// Whether the client half-closed, so the ending ack is owed. bool ending_ = false; /// Whether that ending ack is the write in flight. diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc index 6aab1de43294..0a1d33c18bdc 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc @@ -293,9 +293,12 @@ TEST(AsyncGrpcTest, OnFinishMayWriteMetadataWithoutDeadlocking) { auto listener = flight_server.WaitForListener(0); ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; - // The write is refused (the upload is ending) rather than deadlocking. - EXPECT_TRUE(listener->finish_write.IsInvalid()) << listener->finish_write; ASSERT_EQ(1, listener->finish_count()); + // The write is refused (the upload is ending) rather than deadlocking. + ASSERT_TRUE(listener->finish_write.is_finished()) + << "a write refused while the upload ends must fail at once"; + EXPECT_TRUE(listener->finish_write.status().IsInvalid()) + << listener->finish_write.status(); } TEST(AsyncGrpcTest, DoPutWritesMetadataBackToTheClient) { @@ -336,7 +339,6 @@ TEST(AsyncGrpcTest, DoPutWritesMetadataBackToTheClient) { auto listener = flight_server.WaitForListener(); ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; ASSERT_EQ(2, listener->wrote()) << "every batch must have been acknowledged"; - ASSERT_OK(listener->last_write()) << listener->last_write(); } TEST(AsyncGrpcTest, DoPutCancelFromServerSideFinishesTheUpload) { diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h index b6abde071560..60739944675b 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h @@ -609,33 +609,34 @@ class RecordingUploadServer : public AsyncGenericFlightServerBase { using TestUploadServer = RecordingUploadServer; /// A listener that acknowledges every batch it is handed with one application -/// metadata message, the way an application reports per-chunk progress. The -/// batch recording itself is RecordingListener's. +/// metadata message, the way an application reports per-chunk progress, and +/// takes the next batch only once that ack is written: chaining OnNext()'s +/// future on WriteMetadataAsync()'s is the upload's backpressure. The batch +/// recording itself is RecordingListener's. class MetadataReplyingListener final : public RecordingListener { public: arrow::Future<> OnNext(FlightStreamChunk chunk) override { - if (chunk.data) { - const Status status = writer()->WriteMetadata(*arrow::Buffer::FromString( - "ack " + std::to_string(chunk.data->num_rows()) + " rows")); - std::lock_guard lock(mutex_); - last_write_ = status; - ++wrote_; + if (!chunk.data) { + return RecordingListener::OnNext(std::move(chunk)); } - return RecordingListener::OnNext(std::move(chunk)); + const auto ack = arrow::Buffer::FromString( + "ack " + std::to_string(chunk.data->num_rows()) + " rows"); + return writer()->WriteMetadataAsync(*ack).Then([this, chunk]() { + { + std::lock_guard lock(mutex_); + ++wrote_; + } + return RecordingListener::OnNext(chunk); + }); } int wrote() const { std::lock_guard lock(mutex_); return wrote_; } - Status last_write() const { - std::lock_guard lock(mutex_); - return last_write_; - } private: int wrote_ = 0; - Status last_write_; }; /// A listener whose OnFinish fails asynchronously: the commit it performs is @@ -656,11 +657,12 @@ class FailingFinishListener final : public FinishRecordingListener { class FinishingAckListener final : public FinishRecordingListener { public: arrow::Future<> OnFinish(Status status) override { - finish_write = writer()->WriteMetadata(*arrow::Buffer::FromString("from-onfinish")); + finish_write = + writer()->WriteMetadataAsync(*arrow::Buffer::FromString("from-onfinish")); return FinishRecordingListener::OnFinish(std::move(status)); } - Status finish_write; + arrow::Future<> finish_write; }; /// Hands out a MetadataReplyingListener for every upload. From 7341f744c020a4ee6f1c3d81bac3791e182f2cba Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:57 +0100 Subject: [PATCH 06/25] [C++][FlightRPC] Document the async server's threading contract The transport has no executor: its continuations, including the next StartWrite(), run on the thread that completes the futures handlers and listeners return. --- cpp/src/arrow/flight/flight_data_decoder.h | 4 ++++ cpp/src/arrow/flight/server_async.h | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/cpp/src/arrow/flight/flight_data_decoder.h b/cpp/src/arrow/flight/flight_data_decoder.h index fcd0ced9759e..84d46aea04f3 100644 --- a/cpp/src/arrow/flight/flight_data_decoder.h +++ b/cpp/src/arrow/flight/flight_data_decoder.h @@ -136,6 +136,10 @@ class ARROW_FLIGHT_EXPORT AsyncFlightDataWriter { /// \brief A general listener class to receive events from FlightMessageDecoder /// /// User must implement callback methods for interested events. +/// +/// When it serves a DoPut, the transport continues on the thread that +/// completes the futures this listener returns: see the threading notes on +/// AsyncGenericFlightServerBase. class ARROW_FLIGHT_EXPORT AsyncFlightDataListener : public ipc::Listener { public: AsyncFlightDataListener(); diff --git a/cpp/src/arrow/flight/server_async.h b/cpp/src/arrow/flight/server_async.h index 39b444cc3fe1..e70c8fafbc25 100644 --- a/cpp/src/arrow/flight/server_async.h +++ b/cpp/src/arrow/flight/server_async.h @@ -188,6 +188,19 @@ class ARROW_FLIGHT_EXPORT AsyncFlightMessageWriter { /// All handlers must be non-blocking and return promptly. /// Non overriding handlers will return `Status::NotImplemented` by default. /// +/// Threading: the transport has no thread or executor of its own. It chains +/// its continuation on every future you return (a handler's result, +/// AsyncFlightDataStream::GetSchemaPayloadAsync()/NextAsync(), +/// AsyncFlightListing::NextAsync(), AsyncResultStream::NextAsync(), and an +/// AsyncFlightDataListener's OnDescriptor()/OnNext()/OnFinish()) with the +/// default CallbackOptions, so the continuation runs on whichever thread +/// completes your future, or inline on the gRPC callback thread that called +/// you if the future is already complete when you return it. That +/// continuation starts the RPC's next operation: the next StartWrite() (a +/// payload, a listing entry, an action result, a unary response), the next +/// StartRead(), or Finish(). Complete your futures on the executor you want +/// the RPC's I/O issued from. +/// /// Every request argument a handler receives (`Ticket`, `FlightDescriptor`, /// `Criteria`, `Action`) is only valid for the duration of that handler call: /// copy what you need before completing the returned future from another From 138e0fb57993bd67437876b70a0a80437bf49f12 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:57 +0100 Subject: [PATCH 07/25] [C++][FlightRPC] Let the first ending of a DoPut decide its status A server-side Cancel() whose OnFinish() was still pending let the upload keep reading and delivering OnNext() after OnFinish(), and the client ending the upload normally could then finish the RPC with OK ahead of the cancellation. The first ending now claims the upload: nothing is read, handed to the listener or written after it, and later endings no longer finish the RPC. On cancellation the RPC's status is settled before OnFinish() runs. --- .../transport/grpc/async_grpc_do_put.cc | 44 +++++++---- .../transport/grpc/async_grpc_do_put_test.cc | 76 +++++++++++++++++++ 2 files changed, 107 insertions(+), 13 deletions(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc index cbeda880829d..c50e51afce87 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc @@ -19,6 +19,7 @@ #include "arrow/flight/transport/grpc/customize_grpc.h" +#include #include #include #include @@ -55,7 +56,7 @@ class DoPutReactor : public AsyncReactorBase, Future<> WriteMetadataAsync(const Buffer& app_metadata) override { std::lock_guard lock(write_mutex()); - if (finished()) { + if (finished() || upload_ending_) { return Future<>::MakeFinished(Status::Invalid( "the upload is over: WriteMetadataAsync() is only usable while the " "upload's RPC is running")); @@ -78,6 +79,12 @@ class DoPutReactor : public AsyncReactorBase, } void OnReadDone(bool ok) override { + if (upload_ending_) { + // The upload is ending (its OnFinish() was reported or is being + // reported): nothing read from now on reaches the listener, and no + // further read is armed. + return; + } if (!ok) { // Decide under the lock, act outside it: FinishUpload() runs the // listener's OnFinish(), which is application code and may itself call @@ -138,9 +145,9 @@ class DoPutReactor : public AsyncReactorBase, Future<> decode_status = decoder_.Consume(std::move(arrow_buf)); decode_status.AddCallback([this, token = hold()](arrow::Status status) { - if (finished()) { - // The RPC is over (cancelled while the listener was working): the - // message is abandoned. + if (upload_ending_) { + // The upload is ending (cancelled, or ended while the listener was + // working): the message is abandoned and no further read is armed. return; } if (!status.ok()) { @@ -148,9 +155,10 @@ class DoPutReactor : public AsyncReactorBase, return; } // Arm the next read through WriteIfLive(): this continuation runs on the - // decoder's thread, so a cancellation can finish the RPC right after the - // finished() check above. WriteIfLive re-checks under write_mutex_, which - // FinishOnce also holds, so the read is never started on a dead call. + // decoder's thread, so the RPC can be finished right after the check + // above. WriteIfLive re-checks under write_mutex_, which FinishOnce also + // holds, so the read is never started on a dead call (and OnReadDone + // drops what a read armed as the upload starts ending brings in). WriteIfLive([this] { StartRead(&request_buf_); }); }); } @@ -199,15 +207,14 @@ class DoPutReactor : public AsyncReactorBase, } void TearDownOnCancel() override { - // The RPC is cancelled: report it to the listener but do not let a failing - // OnFinish replace the cancellation the client is seeing. The report is - // fire-and-forget here; keeping a reference alive for it is unnecessary as - // FinishOnce() below releases the reactor's hold and the callback only - // touches immutable state. + // The RPC is cancelled: settle the status the client sees first, then tell + // the listener unless another ending already did. OnFinish()'s future is + // ignored: a failing OnFinish must not replace the cancellation. + upload_ending_ = true; const auto status = Status::Cancelled("the client cancelled the upload"); + FinishOnce(status); static_cast(arrow::flight::internal::FlightDataListenerTransport::ReportFinish( listener_, status)); - FinishOnce(status); } void OnDone() override { @@ -225,6 +232,13 @@ class DoPutReactor : public AsyncReactorBase, } void FinishUpload(Status status) { + // Only the first ending reports to the listener and decides the RPC's + // status: a later one (the client's half-close racing a server-side + // Cancel()) must not finish the RPC with its own status while the first + // one's OnFinish() is still running. + if (upload_ending_.exchange(true)) { + return; + } // Chain the RPC's completion on the listener's OnFinish() future instead of // waiting for it: FinishOnce() must not run on a gRPC callback thread while // the application is still committing, and a listener that fails @@ -245,6 +259,10 @@ class DoPutReactor : public AsyncReactorBase, AsyncFlightMessageDecoder decoder_; /// Whether the descriptor of the first message has been checked. bool read_descriptor_ = false; + /// Set by the first ending (FinishUpload() or the cancellation), before the + /// listener hears OnFinish(): from then on nothing more is read, handed to + /// the listener or written. + std::atomic upload_ending_{false}; // The write side is serialized by AsyncReactorBase::write_mutex_, which // FinishOnce() also takes: WriteMetadataAsync() runs on the listener's thread // while OnReadDone()/OnWriteDone() run on gRPC callback threads, and this diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc index 0a1d33c18bdc..2d57bc7f26b1 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc @@ -396,6 +396,82 @@ TEST(AsyncGrpcTest, DoPutCancelFromServerSideFinishesTheUpload) { ASSERT_EQ(listener->cancel_status().code(), arrow::StatusCode::Invalid); } +/// A listener whose OnFinish() stays pending until the test releases it, the +/// way an application committing the upload asynchronously does: the window in +/// which an ending is reported but the RPC is not finished yet. +class ParkedFinishListener final : public FinishRecordingListener { + public: + arrow::Future<> OnFinish(Status status) override { + static_cast(FinishRecordingListener::OnFinish(std::move(status))); + std::lock_guard lock(mutex_); + parked_ = arrow::Future<>::Make(); + return parked_; + } + + /// Complete the parked OnFinish() future, on the calling thread. + void ReleaseFinish() { + arrow::Future<> parked; + { + std::lock_guard lock(mutex_); + parked = parked_; + } + if (parked.is_valid()) { + parked.MarkFinished(); + } + } + + private: + arrow::Future<> parked_; +}; + +TEST(AsyncGrpcTest, DoPutServerCancelEndsTheUploadWithItsOwnStatus) { + // A server-side Cancel() whose OnFinish() is still committing: what the + // client sends afterwards must not reach the listener, and the client ending + // the upload normally must not finish the RPC with OK ahead of the + // cancellation: the first ending decides the RPC's status. + RecordingUploadServer flight_server; + FlightServerOptions options(LocalhostLocation()); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + auto schema = OneColumnSchema(); + auto batch = OneColumnBatch(schema); + ASSERT_OK_AND_ASSIGN(auto put, + client->DoPut(FlightDescriptor::Path({"parked-finish"}), schema)); + ASSERT_OK(put.writer->WriteRecordBatch(*batch)); + auto listener = flight_server.WaitForListener(0); + ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; + ASSERT_TRUE(WaitFor([&listener] { return listener->batches().size() >= 1; })) + << "the server never consumed the first batch"; + + listener->CancelWith(arrow::Status::Cancelled("server said stop")); + ASSERT_OK(listener->cancel_status()); + ASSERT_EQ(1, listener->finish_count()) << "Cancel() must report OnFinish() at once"; + + // Sent after OnFinish() was reported: must never reach the listener. + ARROW_UNUSED(put.writer->WriteRecordBatch(*batch)); + // Keep OnFinish() pending long enough for the client's normal end of the + // upload to reach the server first. + std::thread release([&listener] { + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + listener->ReleaseFinish(); + }); + const Status close_status = put.writer->Close(); + release.join(); + + EXPECT_EQ(close_status.code(), arrow::StatusCode::Cancelled) + << "the client's end of the upload won over the server's cancel: " << close_status; + EXPECT_NE(close_status.message().find("server said stop"), std::string::npos) + << close_status; + EXPECT_EQ(1, listener->batches().size()) + << "a batch reached the listener after its OnFinish()"; + EXPECT_EQ(1, listener->finish_count()); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); +} + /// A listener stuck inside OnNext(): the upload is parked in the application /// while the RPC underneath it dies. class HangingDoPutListener final : public FinishRecordingListener { From f5d86c73b1d50a91155d6df7757688b217235657 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:58 +0100 Subject: [PATCH 08/25] [C++][FlightRPC] Never ask a DoGet stream for a payload during or after Close() A cancellation could deliver Close() while the transport was inside GetSchemaPayloadAsync() or NextAsync(), or just before it asked again. Issue each demand under the stream lock that CloseStreamOnce() takes. --- cpp/src/arrow/flight/server_async.h | 4 +- .../transport/grpc/async_grpc_do_get.cc | 34 +++--- .../transport/grpc/async_grpc_do_get_test.cc | 103 ++++++++++++++++++ 3 files changed, 126 insertions(+), 15 deletions(-) diff --git a/cpp/src/arrow/flight/server_async.h b/cpp/src/arrow/flight/server_async.h index e70c8fafbc25..266020c3eaa2 100644 --- a/cpp/src/arrow/flight/server_async.h +++ b/cpp/src/arrow/flight/server_async.h @@ -65,7 +65,9 @@ class ARROW_FLIGHT_EXPORT AsyncFlightDataStream { /// Called once, possibly from any thread, whichever way the RPC ends, including for a /// stream DoGetAsync() resolves after the RPC was cancelled. A pending /// NextAsync()/GetSchemaPayloadAsync() future must be resolved here (or abandoned - /// knowingly): the transport does not wait for it. + /// knowingly): the transport does not wait for it. Close() never overlaps a + /// GetSchemaPayloadAsync()/NextAsync() call, and neither is called once Close() + /// has started. /// Its return status becomes the RPC's final status only on the normal end-of-stream /// path (after NextAsync() reports end of stream). /// On every other path (write failure, cancellation) the RPC already has a status and diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc index 1a7a6fe816bf..da9b01759730 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc @@ -124,19 +124,26 @@ class DoGetReactor : public AsyncReactorBase { } void WriteNextPayload() { - if (finished()) { - return; - } HoldToken token = hold(); arrow::Future> next; - if (!wrote_schema_) { - next = async_data_stream_->GetSchemaPayloadAsync().Then( - [](FlightPayload payload) -> std::optional { - return {std::move(payload)}; - }); - } else { - next = async_data_stream_->NextAsync(); + { + // Ask under stream_mutex_, which CloseStreamOnce() takes to claim the + // stream before calling Close(): a demand never overlaps Close() and is + // never made once Close() has started. + std::lock_guard lock(stream_mutex_); + if (closed_ || finished()) { + return; + } + if (!wrote_schema_) { + wrote_schema_ = true; + next = async_data_stream_->GetSchemaPayloadAsync().Then( + [](FlightPayload payload) -> std::optional { + return {std::move(payload)}; + }); + } else { + next = async_data_stream_->NextAsync(); + } } next.AddCallback([this, token = token.release()]( arrow::Result> result) { @@ -147,7 +154,6 @@ class DoGetReactor : public AsyncReactorBase { CloseStreamOnce(), "DoGet: closing the data stream after a failed payload failed"); } else { - wrote_schema_ = true; std::optional payload = std::move(*result); if (!payload.has_value()) { FinishOnce(CloseStreamOnce()); @@ -174,11 +180,11 @@ class DoGetReactor : public AsyncReactorBase { ::grpc::ByteBuffer request_buf_; ::grpc::ByteBuffer write_buf_; std::shared_ptr async_data_stream_; - bool wrote_schema_ = false; - /// Guards async_data_stream_ and closed_, so adopting the stream and the cancellation - /// teardown cannot interleave. + /// Guards async_data_stream_, closed_ and wrote_schema_: adopting the stream, + /// each demand on it and the teardown's Close() never interleave. std::mutex stream_mutex_; bool closed_ = false; + bool wrote_schema_ = false; }; } // namespace diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc index 450ee1d5cd27..b58cf9549053 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc @@ -342,6 +342,109 @@ TEST(AsyncGrpcTest, DoGetCancelCallbackReachesPendingHandler) { << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; } +/// A DoGet stream asked for its schema while the RPC dies: GetSchemaPayloadAsync() +/// stays inside the call until the client's deadline cancelled the RPC, then +/// gives the cancellation's teardown time to deliver Close(). It records whether +/// Close() arrived while it was still being asked, or a demand after Close(). +class SlowSchemaDoGetStream final : public AsyncFlightDataStream { + public: + SlowSchemaDoGetStream(const ServerCallContext* context, + std::shared_ptr resolved, + std::atomic* closes, std::atomic* closed_while_asked, + std::atomic* asked_after_close) + : context_(context), + resolved_(std::move(resolved)), + closes_(closes), + closed_while_asked_(closed_while_asked), + asked_after_close_(asked_after_close) {} + + std::shared_ptr schema() override { return resolved_->schema(); } + + arrow::Future GetSchemaPayloadAsync() override { + CountDemandAfterClose(); + WaitFor([this] { return context_->is_cancelled(); }); + if (WaitFor([this] { return closes_->load() > 0; }, std::chrono::milliseconds(300))) { + closed_while_asked_->store(true); + } + return resolved_->GetSchemaPayloadAsync(); + } + + arrow::Future> NextAsync() override { + CountDemandAfterClose(); + return resolved_->NextAsync(); + } + + Status Close() override { + closes_->fetch_add(1); + return resolved_->Close(); + } + + private: + void CountDemandAfterClose() { + if (closes_->load() > 0) { + asked_after_close_->fetch_add(1); + } + } + + const ServerCallContext* context_; + std::shared_ptr resolved_; + std::atomic* closes_; + std::atomic* closed_while_asked_; + std::atomic* asked_after_close_; +}; + +/// Serves every DoGet with a SlowSchemaDoGetStream, resolved at once. +class SlowSchemaDoGetServer final : public AsyncGenericFlightServerBase { + public: + arrow::Future> DoGetAsync( + const ServerCallContext& context, const Ticket&) override { + auto reader = arrow::RecordBatchReader::Make( + {}, arrow::schema({arrow::field("value", arrow::int64())})) + .ValueOrDie(); + std::shared_ptr stream = + std::make_shared( + &context, std::make_shared(std::move(reader)), &closes, + &closed_while_asked, &asked_after_close); + return arrow::Future>::MakeFinished( + std::move(stream)); + } + + std::atomic closes{0}; + std::atomic closed_while_asked{false}; + std::atomic asked_after_close{0}; +}; + +TEST(AsyncGrpcTest, DoGetNeverClosesAStreamItIsAskingForAPayload) { + // The client's deadline expires while the transport is inside the stream's + // GetSchemaPayloadAsync(): the cancellation's Close() must wait for that + // call to return, and the stream must not be asked for anything after it. + const int64_t before = detail::AsyncReactorBase::live_reactors(); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); + SlowSchemaDoGetServer flight_server; + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + FlightCallOptions impatient; + impatient.timeout = std::chrono::milliseconds(300); + EXPECT_FALSE(client->DoGet(impatient, Ticket{"slow-schema"}).ok()) + << "the deadline must expire while the stream is asked for its schema"; + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + EXPECT_FALSE(flight_server.closed_while_asked.load()) + << "Close() ran while the transport was inside GetSchemaPayloadAsync()"; + EXPECT_EQ(0, flight_server.asked_after_close.load()) + << "the stream was asked for a payload after Close()"; + EXPECT_EQ(1, flight_server.closes.load()) << "Close() must be delivered exactly once"; + EXPECT_TRUE(WaitFor([&] { + return detail::AsyncReactorBase::live_reactors() == before; + })) << "a reactor leaked: still alive after the server stopped (before=" + << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; +} + /// A DoGet stream whose first payload is an error: the transport must fail the /// RPC and still tell the producer the RPC is over through Close(). class ErrorPayloadDoGetStream final : public AsyncFlightDataStream { From 0a8db9097e31bc400ca7ded872c8ef8a0e9b9dc8 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:58 +0100 Subject: [PATCH 09/25] [C++][FlightRPC] Run middleware CallCompleted() outside the reactor's write lock FinishOnce() ran the middleware's CallCompleted(), application code, while holding write_mutex_: a middleware reaching back into the RPC deadlocked the finishing thread, and a slow one stalled writes. Compute the status first; finished_ already keeps new writes out. --- .../transport/grpc/async_grpc_do_put_test.cc | 111 ++++++++++++++++++ .../grpc/async_grpc_service_internal.h | 6 +- 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc index 2d57bc7f26b1..d150e8f4c02f 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc @@ -301,6 +301,117 @@ TEST(AsyncGrpcTest, OnFinishMayWriteMetadataWithoutDeadlocking) { << listener->finish_write.status(); } +/// Middleware that calls back into the upload from CallCompleted(): middleware +/// is application code too, so the transport must not hold its write lock +/// across it. +class UploadWritingMiddleware final : public ServerMiddleware { + public: + std::string name() const override { return "upload-writing"; } + void SendingHeaders(AddCallHeaders*) override {} + + void CallCompleted(const Status&) override { + std::shared_ptr upload; + { + std::lock_guard lock(mutex_); + upload = upload_; + } + if (upload == nullptr) { + return; + } + auto write = upload->writer()->WriteMetadataAsync( + *arrow::Buffer::FromString("from-middleware")); + std::lock_guard lock(mutex_); + completed_write_ = std::move(write); + } + + void set_upload(std::shared_ptr upload) { + std::lock_guard lock(mutex_); + upload_ = std::move(upload); + } + + arrow::Future<> completed_write() const { + std::lock_guard lock(mutex_); + return completed_write_; + } + + private: + mutable std::mutex mutex_; + std::shared_ptr upload_; + arrow::Future<> completed_write_; +}; + +class UploadWritingMiddlewareFactory final : public ServerMiddlewareFactory { + public: + Status StartCall(const CallInfo&, const ServerCallContext&, + std::shared_ptr* middleware) override { + auto instance = std::make_shared(); + { + std::lock_guard lock(mutex_); + last_ = instance; + } + *middleware = std::move(instance); + return Status::OK(); + } + + std::shared_ptr last() const { + std::lock_guard lock(mutex_); + return last_; + } + + private: + mutable std::mutex mutex_; + std::shared_ptr last_; +}; + +/// Hands every upload's listener to the call's UploadWritingMiddleware. +class MiddlewareWiredUploadServer final : public AsyncGenericFlightServerBase { + public: + std::shared_ptr CreateDoPutListener( + const ServerCallContext& context) override { + auto listener = std::make_shared(); + auto* middleware = + static_cast(context.GetMiddleware("upload-writing")); + if (middleware != nullptr) { + middleware->set_upload(listener); + } + return listener; + } +}; + +TEST(AsyncGrpcTest, CallCompletedMayWriteMetadataWithoutDeadlocking) { + // Middleware's CallCompleted() runs as the RPC finishes and may reach the + // upload from there. The transport must not hold its write lock across that + // call: doing so deadlocks the finishing thread (verified: the RPC never + // finishes). The write itself is refused, the upload being over. + auto factory = std::make_shared(); + MiddlewareWiredUploadServer flight_server; + FlightServerOptions options(LocalhostLocation()); + options.middleware = {{"upload-writing", factory}}; + ASSERT_OK(flight_server.Init(options)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); + + // Bounded, so a deadlocked server fails the Close() below instead of hanging it. + FlightCallOptions bounded; + bounded.timeout = std::chrono::seconds(10); + auto schema = OneColumnSchema(); + ASSERT_OK_AND_ASSIGN( + auto put, client->DoPut(bounded, FlightDescriptor::Path({"middleware"}), schema)); + ASSERT_OK(put.writer->WriteRecordBatch(*OneColumnBatch(schema))); + ASSERT_OK(put.writer->Close()); + + ASSERT_OK(client->Close()); + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + const auto middleware = factory->last(); + ASSERT_NE(middleware, nullptr) << "the upload never created its middleware"; + const auto write = middleware->completed_write(); + ASSERT_TRUE(write.is_valid()) << "CallCompleted() never reached the upload's writer"; + ASSERT_TRUE(write.is_finished()) + << "a write refused once the upload ended must fail at once"; + EXPECT_TRUE(write.status().IsInvalid()) << write.status(); +} + TEST(AsyncGrpcTest, DoPutWritesMetadataBackToTheClient) { // The listener's writer() sends PutResult messages during the upload, the way // the sync FlightMetadataWriter does: the client reads them back with diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h index 6a3a9ad2e950..b0436911be9a 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h @@ -169,8 +169,12 @@ class AsyncReactorBase : public ::grpc::ServerGenericBidiReactor { void FinishOnce(arrow::Status status) { bool expected = false; if (finished_.compare_exchange_strong(expected, true)) { + // FinishRequest() runs the middleware's CallCompleted(), application code + // that may call back into this RPC, so it runs before taking write_mutex_: + // finished_ is already set, so WriteIfLive() starts nothing meanwhile. + const ::grpc::Status grpc_status = flight_context_.FinishRequest(status); std::lock_guard lock(write_mutex_); - Finish(flight_context_.FinishRequest(status)); + Finish(grpc_status); } } From 3f57e819a825d2fbc6d1d354c92f8fdbd8cd15d7 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:58 +0100 Subject: [PATCH 10/25] [C++][FlightRPC] Drop the StreamingReactor state mutex Start()'s continuation publishes the subclass state before the first NextMessage(), and every later NextMessage() follows the previous write's completion: the state is never accessed concurrently. --- .../transport/grpc/async_grpc_streaming.cc | 57 +++++-------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc index 61757c0474ab..c94adfc0de88 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc @@ -18,7 +18,6 @@ #include "arrow/flight/transport/grpc/async_grpc_service_internal.h" #include -#include #include #include #include @@ -66,6 +65,10 @@ class StreamingReactor : public AsyncReactorBase { } protected: + /// Run the handler. Its continuation publishes the state NextMessage() reads + /// before the first NextMessage(), and every later NextMessage() follows the + /// previous write's completion: that state is never shared concurrently, so + /// it needs no lock. virtual arrow::Future<> Start() = 0; virtual arrow::Future NextMessage() = 0; @@ -87,19 +90,9 @@ class StreamingReactor : public AsyncReactorBase { }); } - /// \brief Guards the subclass state that Start()'s continuation publishes and - /// NextMessage() consumes. NextMessage() runs on a gRPC callback thread - /// (OnWriteDone) while Start()'s continuation runs on the handler's, so both - /// share this one lock rather than each reactor declaring its own. - std::mutex& state_mutex() { return state_mutex_; } - - protected: AsyncGenericFlightServerBase* base_; ::grpc::ByteBuffer request_buf_; ::grpc::ByteBuffer write_buf_; - - private: - std::mutex state_mutex_; }; class ListActionsReactor final : public StreamingReactor { @@ -110,9 +103,6 @@ class ListActionsReactor final : public StreamingReactor { arrow::Future<> Start() override { return base_->ListActionsAsync(flight_context()) .Then([this](const std::vector& actions) -> arrow::Status { - // Published on the handler's thread, read from NextMessage() which - // OnWriteDone() also calls from a gRPC thread: guard both. - std::lock_guard lock(state_mutex()); actions_ = actions; next_ = 0; return arrow::Status::OK(); @@ -120,16 +110,13 @@ class ListActionsReactor final : public StreamingReactor { } arrow::Future NextMessage() override { + if (next_ >= actions_.size()) { + return arrow::Future::MakeFinished(false); + } pb::ActionType pb_type; - { - std::lock_guard lock(state_mutex()); - if (next_ >= actions_.size()) { - return arrow::Future::MakeFinished(false); - } - const auto status = internal::ToProto(actions_[next_++], &pb_type); - if (!status.ok()) { - return arrow::Future::MakeFinished(status); - } + const auto status = internal::ToProto(actions_[next_++], &pb_type); + if (!status.ok()) { + return arrow::Future::MakeFinished(status); } write_buf_ = MakeWriteBuffer(pb_type); return arrow::Future::MakeFinished(true); @@ -155,24 +142,16 @@ class DoActionReactor final : public StreamingReactor { if (stream == nullptr) { return arrow::Status::Cancelled(); } - { - std::lock_guard lock(state_mutex()); - stream_ = stream; - } + stream_ = stream; return arrow::Status::OK(); }); } arrow::Future NextMessage() override { - std::shared_ptr stream; - { - std::lock_guard lock(state_mutex()); - stream = stream_; - } - if (stream == nullptr) { + if (stream_ == nullptr) { return arrow::Future::MakeFinished(false); } - return stream->NextAsync().Then( + return stream_->NextAsync().Then( [this](const std::shared_ptr& result) -> arrow::Result { if (result == nullptr) { return false; @@ -202,22 +181,16 @@ class ListFlightsReactor final : public StreamingReactor { return base_->ListFlightsAsync(flight_context(), &criteria_) .Then( [this](const std::shared_ptr& listing) -> arrow::Status { - std::lock_guard lock(state_mutex()); listing_ = listing; return arrow::Status::OK(); }); } arrow::Future NextMessage() override { - std::shared_ptr listing; - { - std::lock_guard lock(state_mutex()); - listing = listing_; - } - if (listing == nullptr) { + if (listing_ == nullptr) { return arrow::Future::MakeFinished(false); } - return listing->NextAsync().Then( + return listing_->NextAsync().Then( [this](const std::shared_ptr& info) -> arrow::Result { if (info == nullptr) { return false; From 8ed7a727c26be4330c236914d5aea1c7c2ded394 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:58 +0100 Subject: [PATCH 11/25] [C++][FlightRPC] Detect the async handshake from the parsed method CreateReactor() parsed the method name a second time only to compare it with "Handshake". --- cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc index e4daa60a85d8..013f8b0e7953 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc @@ -86,14 +86,11 @@ AsyncGenericFlightService::AsyncGenericFlightService( ::grpc::ServerGenericBidiReactor* AsyncGenericFlightService::CreateReactor( ::grpc::GenericCallbackServerContext* context) { - const std::string_view full_method = context->method(); - const std::string_view method = - full_method.starts_with(kPrefix) ? full_method.substr(kPrefix.size()) : full_method; - const FlightMethod flight_method = MethodFromName(full_method); + const FlightMethod flight_method = MethodFromName(context->method()); detail::AsyncCallContext flight_context(context); const auto prepare_status = - method == kHandshakeMethod + flight_method == FlightMethod::Handshake ? helper_->MakeCallContext(flight_method, context, &flight_context) : helper_->CheckAuth(flight_method, context, &flight_context); if (!prepare_status.ok()) { From d21e1bdf69a37b48430c87b4711a6c58b521f0cd Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:00:59 +0100 Subject: [PATCH 12/25] [C++][FlightRPC] Deserialize each DoPut message once The first message was wrapped and deserialized twice: once for the descriptor check and again by the decoder. Deserialize once and hand the result to the decoder; a malformed first message now fails with the parser's status, as later messages already did. --- .../transport/grpc/async_grpc_do_put.cc | 18 ++++--- .../transport/grpc/async_grpc_do_put_test.cc | 47 +++++++++++++++++++ 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc index c50e51afce87..8f29084e1ad4 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc @@ -123,27 +123,25 @@ class DoPutReactor : public AsyncReactorBase, "Failed to wrap gRPC buffer: " + wrap_status.message())); return; } - + // Deserialize once: the descriptor check and the decoder share the result. + auto data = arrow::flight::internal::DeserializeFlightData(arrow_buf); + if (!data.ok()) { + FinishUpload(data.status()); + return; + } if (!read_descriptor_) { // The first message must carry the descriptor of the upload, exactly as // the sync transport requires. // Without this the listener never hears OnDescriptor and the upload is // acknowledged as if it were well formed. read_descriptor_ = true; - internal::FlightData data; - const ::grpc::Status deserialized = FlightDataDeserialize(&request_buf_, &data); - if (!deserialized.ok()) { - FinishUpload( - MakeFlightError(FlightStatusCode::Internal, deserialized.error_message())); - return; - } - if (!data.descriptor) { + if (!data->descriptor) { FinishUpload(Status::IOError("Descriptor missing on first message")); return; } } - Future<> decode_status = decoder_.Consume(std::move(arrow_buf)); + Future<> decode_status = decoder_.Consume(std::move(*data)); decode_status.AddCallback([this, token = hold()](arrow::Status status) { if (upload_ending_) { // The upload is ending (cancelled, or ended while the listener was diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc index d150e8f4c02f..b5c0e1f93531 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc @@ -829,6 +829,53 @@ TEST(AsyncGrpcTest, DoPutWithoutDescriptorOnFirstMessageIsRejected) { EXPECT_EQ(0, listener->batches().size()); } +TEST(AsyncGrpcTest, DoPutMalformedFirstMessageIsRejected) { + // A first message that does not parse as FlightData (here a descriptor field + // cut short) fails the upload before the listener hears anything. + TestFlightServer flight_server; + auto listener = std::make_shared(); + flight_server.set_listener(listener); + ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); + FlightServerOptions options(location); + ASSERT_OK(flight_server.Init(options)); + + // Field 1 (the descriptor), length-delimited, announcing 127 bytes of 2. + ::grpc::Slice truncated(std::string("\x0a\x7f\x01\x02", 4)); + ::grpc::ByteBuffer request(&truncated, 1); + + std::shared_ptr<::grpc::Channel> channel = ::grpc::CreateChannel( + std::string(kLoopbackHost) + ":" + std::to_string(flight_server.port()), + ::grpc::InsecureChannelCredentials()); + ::grpc::GenericStub stub(channel); + ::grpc::ClientContext context; + context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(10)); + ::grpc::CompletionQueue cq; + void* tag = nullptr; + bool ok = false; + + auto call = stub.Call(&context, "/arrow.flight.protocol.FlightService/DoPut", &cq, + reinterpret_cast(1)); + ASSERT_NE(call, nullptr); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never started"; + call->Write(request, reinterpret_cast(2)); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "the message was never written"; + call->WritesDone(reinterpret_cast(3)); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "WritesDone never completed"; + ::grpc::Status status; + call->Finish(&status, reinterpret_cast(4)); + ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never finished"; + + EXPECT_FALSE(status.ok()) << "a malformed first message must be refused"; + EXPECT_THAT(status.error_message(), + ::testing::HasSubstr("Unable to read FlightDescriptor")); + + ASSERT_OK(flight_server.Shutdown()); + ASSERT_OK(flight_server.Wait()); + + EXPECT_EQ(0, listener->descriptor_count()); + EXPECT_EQ(0, listener->batches().size()); +} + TEST(AsyncGrpcTest, DoPutReportsAFailingOnFinishToTheClient) { // The listener's OnFinish() is where an application commits the upload. When // it fails asynchronously, the client must not see a successful Close(): the From 6c9d411fc2a8faa1e3b2b496896e24adb3e7c55b Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:35:09 +0100 Subject: [PATCH 13/25] [C++][FlightRPC] Pin the async reactors through a unique_ptr HoldToken --- cpp/src/arrow/flight/flight_data_decoder.h | 1 + .../transport/grpc/async_grpc_do_get.cc | 4 +- .../transport/grpc/async_grpc_do_put.cc | 2 - .../grpc/async_grpc_exchange_internal.h | 17 ++---- .../grpc/async_grpc_exchange_reader.cc | 17 +++--- .../grpc/async_grpc_exchange_writer.cc | 34 ++++++----- .../grpc/async_grpc_service_internal.h | 57 +++++++------------ 7 files changed, 53 insertions(+), 79 deletions(-) diff --git a/cpp/src/arrow/flight/flight_data_decoder.h b/cpp/src/arrow/flight/flight_data_decoder.h index 84d46aea04f3..ba1262379eb6 100644 --- a/cpp/src/arrow/flight/flight_data_decoder.h +++ b/cpp/src/arrow/flight/flight_data_decoder.h @@ -48,6 +48,7 @@ class ARROW_FLIGHT_EXPORT FlightDataListenerTransport { virtual ~FlightDataListenerTransport(); /// \brief Finish the upload's RPC with `status`, from any thread. + /// The caller keeps the transport alive across the call (see Hold()). /// \param status The status to finish the upload with. virtual void CancelUpload(Status status) = 0; diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc index da9b01759730..61316933be6d 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc @@ -124,8 +124,6 @@ class DoGetReactor : public AsyncReactorBase { } void WriteNextPayload() { - HoldToken token = hold(); - arrow::Future> next; { // Ask under stream_mutex_, which CloseStreamOnce() takes to claim the @@ -145,7 +143,7 @@ class DoGetReactor : public AsyncReactorBase { next = async_data_stream_->NextAsync(); } } - next.AddCallback([this, token = token.release()]( + next.AddCallback([this, token = hold()]( arrow::Result> result) { if (!finished()) { if (!result.ok()) { diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc index 8f29084e1ad4..8db8d7c76230 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc @@ -49,9 +49,7 @@ class DoPutReactor : public AsyncReactorBase, if (status.ok()) { status = Status::Invalid("Cancel() needs a non-OK status"); } - AsyncReactorBase::Hold(); FinishUpload(std::move(status)); - AsyncReactorBase::ReleaseHold(); } Future<> WriteMetadataAsync(const Buffer& app_metadata) override { diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h index 5457d8c6bbdc..cc6a2e1c6fee 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include #include @@ -100,7 +99,8 @@ class ExchangeReader final : public AsyncFlightMessageReader, /// async_grpc_exchange_reader.cc, which has the reactor's definition. void RequestRead(); - std::atomic reactor_{nullptr}; + /// Guarded by mutex_. + ExchangeReactor* reactor_ = nullptr; std::shared_ptr listener_; AsyncFlightMessageDecoder decoder_; FlightDescriptor descriptor_; @@ -159,7 +159,8 @@ class ExchangeWriter final : public AsyncFlightMessageWriter { void FailFlush(Status status); - std::atomic reactor_; + /// Set by the constructor, cleared by DetachReactor(); read under mutex_. + ExchangeReactor* reactor_; std::unique_ptr batch_writer_; ipc::IpcWriteOptions options_ = ipc::IpcWriteOptions::Defaults(); std::shared_ptr pending_app_metadata_; @@ -195,16 +196,6 @@ class ExchangeReactor final : public AsyncReactorBase { /// Called by ExchangeWriter, OnWriteDone() resumes it. arrow::Status WritePayload(FlightPayload payload); - /// \brief Take a reference across a call made through a back-pointer. - /// - /// OnDone() detaches the reader/writer back-pointers and then releases the - /// reactor's last reference, so a reader/writer that loads the raw pointer and - /// calls in can reach freed memory. These re-expose the base's protected - /// Hold()/ReleaseHold() publicly (the same way DoPutReactor does for its - /// transport interface); callers pair them around the call. - void Hold() { AsyncReactorBase::Hold(); } - void ReleaseHold() { AsyncReactorBase::ReleaseHold(); } - private: AsyncGenericFlightServerBase* base_; std::shared_ptr reader_; diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc index fda5dd3a6f2c..9bbc9ed7e378 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc @@ -77,7 +77,7 @@ arrow::Future ExchangeReader::NextAsync() { return FSCFuture::MakeFinished(read_error_); } - if (read_closed_ || reactor_.load() == nullptr) { + if (read_closed_ || reactor_ == nullptr) { // End of exchange: the reactor either saw the read side close or the RPC // is over (and the reactor is gone), so no further read can be started. return FSCFuture::MakeFinished(FlightStreamChunk{}); @@ -104,12 +104,12 @@ arrow::Future ExchangeReader::NextAsync() { void ExchangeReader::SetReactor(ExchangeReactor* reactor) { std::lock_guard lock(mutex_); - reactor_.store(reactor); + reactor_ = reactor; } void ExchangeReader::DetachReactor() { std::lock_guard lock(mutex_); - reactor_.store(nullptr); + reactor_ = nullptr; } void ExchangeReader::SetDescriptor(const FlightDescriptor& descriptor) { @@ -213,17 +213,20 @@ void ExchangeReader::Consume(internal::FlightData data) { void ExchangeReader::RequestRead() { ExchangeReactor* reactor = nullptr; + // Taken under mutex_: DetachReactor() takes the same lock, and OnDone() + // drops the RPC's reference only after detaching, so the reactor outlives + // the unlocked call below. Released when this function returns. + AsyncReactorBase::HoldToken token; { std::lock_guard lock(mutex_); - reactor = reactor_.load(); - if (read_in_flight_ || read_closed_ || reactor == nullptr) { + if (read_in_flight_ || read_closed_ || reactor_ == nullptr) { return; } - reactor->Hold(); + reactor = reactor_; + token = reactor->hold(); read_in_flight_ = true; } reactor->StartReadNext(); - reactor->ReleaseHold(); } } // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc index ce9ccf04caff..01bb15886958 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc @@ -49,13 +49,16 @@ ExchangeWriter::ExchangeWriter(ExchangeReactor* reactor) : reactor_(reactor) {} void ExchangeWriter::DetachReactor() { std::lock_guard lock(mutex_); - reactor_.store(nullptr); + reactor_ = nullptr; } arrow::Future<> ExchangeWriter::BeginAsync(std::shared_ptr schema) { - if (reactor_.load() == nullptr) { - return arrow::Future<>::MakeFinished(arrow::Status::Cancelled( - "the exchange is over: the client disconnected or cancelled")); + { + std::lock_guard lock(mutex_); + if (reactor_ == nullptr) { + return arrow::Future<>::MakeFinished(arrow::Status::Cancelled( + "the exchange is over: the client disconnected or cancelled")); + } } if (batch_writer_ != nullptr) { return arrow::Future<>::MakeFinished( @@ -140,7 +143,7 @@ arrow::Future<> ExchangeWriter::Flush() { arrow::Future<> flush; { std::lock_guard lock(mutex_); - if (reactor_.load() == nullptr) { + if (reactor_ == nullptr) { // The exchange is over: report it rather than pumping into a dead reactor. return arrow::Future<>::MakeFinished(arrow::Status::Cancelled( "the exchange is over: the client disconnected or cancelled")); @@ -158,6 +161,10 @@ arrow::Future<> ExchangeWriter::Flush() { void ExchangeWriter::Pump() { FlightPayload payload; ExchangeReactor* reactor = nullptr; + // Taken under mutex_ on the write path only: DetachReactor() takes the same + // lock, and OnDone() drops the RPC's reference only after detaching, so the + // reactor outlives the unlocked WritePayload() call. Released on return. + AsyncReactorBase::HoldToken token; enum class Action { kIdle, kDrained, kNoReactor, kWrite } action = Action::kIdle; arrow::Future<> drained; { @@ -165,43 +172,34 @@ void ExchangeWriter::Pump() { if (write_in_flight_) { return; } - reactor = reactor_.load(); - if (reactor != nullptr) { - reactor->Hold(); - } if (queue_.empty()) { drained = std::move(flush_); flush_ = arrow::Future<>(); action = Action::kDrained; - } else if (reactor == nullptr) { - write_in_flight_ = false; + } else if (reactor_ == nullptr) { action = Action::kNoReactor; } else { + reactor = reactor_; + token = reactor->hold(); payload = std::move(queue_.front()); queue_.pop_front(); write_in_flight_ = true; action = Action::kWrite; } } - // Everything below runs without mutex_. Every path that pinned above must - // unpin exactly once, including the ones that never reach WritePayload(). + // Everything below runs without mutex_. switch (action) { case Action::kDrained: - if (reactor != nullptr) { - reactor->ReleaseHold(); - } if (drained.is_valid() && !drained.is_finished()) { drained.MarkFinished(); } return; case Action::kNoReactor: - // reactor is null here, so nothing was pinned. FailFlush(arrow::Status::Cancelled( "the exchange is over: the client disconnected or cancelled")); return; case Action::kWrite: { const auto status = reactor->WritePayload(std::move(payload)); - reactor->ReleaseHold(); if (!status.ok()) { { std::lock_guard lock(mutex_); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h index b0436911be9a..0108125c58a1 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h @@ -161,6 +161,27 @@ class AsyncReactorBase : public ::grpc::ServerGenericBidiReactor { void OnDone() override { ReleaseHold(); } + /// \brief HoldToken's deleter: drops the reference the token owns. + struct ReleaseHoldDeleter { + void operator()(AsyncReactorBase* reactor) const { reactor->ReleaseHold(); } + }; + + /// \brief A counted reference to a reactor, dropped when the token is destroyed. + /// + /// Capture one in every callback that may run after OnDone(), and hand it on + /// with std::move. Never call release() on it: unique_ptr::release() gives the + /// pointer up without dropping the reference, so the reactor is never deleted. + using HoldToken = std::unique_ptr; + + /// \brief Take a reference: the reactor stays alive until the token is destroyed. + /// + /// Public so the DoExchange reader and writer can pin the reactor they reach + /// through a back-pointer; the raw Hold()/ReleaseHold() stay protected. + [[nodiscard]] HoldToken hold() { + Hold(); + return HoldToken(this); + } + protected: /// \brief Finish the RPC with this status, later calls are no-ops. /// @@ -222,42 +243,6 @@ class AsyncReactorBase : public ::grpc::ServerGenericBidiReactor { } } - /// A reference the reactor owns, to be handed to a callback. - class HoldToken { - public: - HoldToken() = default; - explicit HoldToken(AsyncReactorBase* reactor) : reactor_(reactor) { - reactor_->Hold(); - } - HoldToken(const HoldToken&) = delete; - HoldToken& operator=(const HoldToken&) = delete; - HoldToken(HoldToken&& other) noexcept - : reactor_(std::exchange(other.reactor_, nullptr)) {} - HoldToken& operator=(HoldToken&& other) noexcept { - if (this != &other) { - Release(); - reactor_ = std::exchange(other.reactor_, nullptr); - } - return *this; - } - ~HoldToken() { Release(); } - - /// \brief The reference, to be captured by the callback and moved out of. - HoldToken&& release() { return std::move(*this); } - - private: - void Release() { - if (reactor_ != nullptr) { - reactor_->ReleaseHold(); - reactor_ = nullptr; - } - } - AsyncReactorBase* reactor_ = nullptr; - }; - - /// Take a reference and return the token that holds it. - HoldToken hold() { return HoldToken(this); } - private: AsyncCallContext flight_context_; std::mutex write_mutex_; From 2eaa97f13fb808aa1752df584ab901bca6ec4b23 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:34:20 +0100 Subject: [PATCH 14/25] [C++][FlightRPC] Drop the unused AsyncFlightDataStream::schema() --- cpp/examples/arrow/flight_async_generic_doget_example.cc | 6 ------ cpp/src/arrow/flight/server_async.h | 7 ------- .../arrow/flight/transport/grpc/async_grpc_do_get_test.cc | 8 -------- .../flight/transport/grpc/async_grpc_test_fixtures.h | 6 ++---- 4 files changed, 2 insertions(+), 25 deletions(-) diff --git a/cpp/examples/arrow/flight_async_generic_doget_example.cc b/cpp/examples/arrow/flight_async_generic_doget_example.cc index 07dac1c296dc..3eea1d8ae539 100644 --- a/cpp/examples/arrow/flight_async_generic_doget_example.cc +++ b/cpp/examples/arrow/flight_async_generic_doget_example.cc @@ -118,7 +118,6 @@ class BatchReader : public arrow::RecordBatchReader { /// not touch the stream object after that. Everything it needs lives here. struct WorkerState { std::unique_ptr stream; - std::shared_ptr schema; int32_t delay_ms = 0; std::mutex mutex; std::condition_variable ready; @@ -144,9 +143,6 @@ void RunWorker(std::shared_ptr state) { prctl(PR_SET_NAME, "doget_worker", 0, 0, 0); #endif std::unique_lock lock(state->mutex); - // Read the schema on this thread: it is the only one allowed to touch the - // wrapped stream, and the transport asks for the schema first. - state->schema = state->stream->schema(); while (!state->closed) { state->ready.wait(lock, [&] { return state->has_pending || state->closed; }); if (state->closed) { @@ -218,8 +214,6 @@ class DelayedAsyncStream : public flight::AsyncFlightDataStream { } } - std::shared_ptr schema() override { return state_->schema; } - arrow::Future GetSchemaPayloadAsync() override { // The schema is ready now: no delay, no worker. This runs before the first // NextAsync(), so it cannot overlap the worker's use of the stream. diff --git a/cpp/src/arrow/flight/server_async.h b/cpp/src/arrow/flight/server_async.h index 266020c3eaa2..db69a16ee7eb 100644 --- a/cpp/src/arrow/flight/server_async.h +++ b/cpp/src/arrow/flight/server_async.h @@ -38,13 +38,6 @@ class ARROW_FLIGHT_EXPORT AsyncFlightDataStream { public: virtual ~AsyncFlightDataStream(); - /// \brief The schema of the payloads this stream produces. - /// - /// The transport never calls this: the schema the client sees is the one - /// GetSchemaPayloadAsync() returns. - /// \return The schema of the payloads this stream produces. - virtual std::shared_ptr schema() = 0; - /// \brief Like GetSchemaPayload(), but the payload arrives later. /// /// Called once, before any NextAsync() payload is requested. diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc index b58cf9549053..f380114ff4fb 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc @@ -114,8 +114,6 @@ class HangingDoGetStream final : public AsyncFlightDataStream { next_awaits_(next_awaits), close_count_(close_count) {} - std::shared_ptr schema() override { return resolved_->schema(); } - arrow::Future GetSchemaPayloadAsync() override { return resolved_->GetSchemaPayloadAsync(); } @@ -358,8 +356,6 @@ class SlowSchemaDoGetStream final : public AsyncFlightDataStream { closed_while_asked_(closed_while_asked), asked_after_close_(asked_after_close) {} - std::shared_ptr schema() override { return resolved_->schema(); } - arrow::Future GetSchemaPayloadAsync() override { CountDemandAfterClose(); WaitFor([this] { return context_->is_cancelled(); }); @@ -452,10 +448,6 @@ class ErrorPayloadDoGetStream final : public AsyncFlightDataStream { explicit ErrorPayloadDoGetStream(std::atomic* close_count) : close_count_(close_count) {} - std::shared_ptr schema() override { - return arrow::schema({arrow::field("value", arrow::int64())}); - } - arrow::Future GetSchemaPayloadAsync() override { return resolved_->GetSchemaPayloadAsync(); } diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h index 60739944675b..bffb8b09a8e9 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h @@ -74,12 +74,10 @@ class ResolvedStream : public AsyncFlightDataStream { explicit ResolvedStream(std::shared_ptr reader) : reader_(std::move(reader)), mapper_(*reader_->schema()) {} - std::shared_ptr schema() override { return reader_->schema(); } - arrow::Future GetSchemaPayloadAsync() override { FlightPayload payload; - auto status = - ipc::GetSchemaPayload(*schema(), ipc_options_, mapper_, &payload.ipc_message); + auto status = ipc::GetSchemaPayload(*reader_->schema(), ipc_options_, mapper_, + &payload.ipc_message); if (!status.ok()) { return arrow::Future::MakeFinished(status); } From de58592ec92bc22c3645d407bf1384b0c17e436b Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:34:20 +0100 Subject: [PATCH 15/25] [C++][FlightRPC] Simplify the async Flight examples --- .../flight_async_generic_doget_example.cc | 74 ++++-------------- .../flight_async_generic_doput_example.cc | 78 +++++-------------- 2 files changed, 35 insertions(+), 117 deletions(-) diff --git a/cpp/examples/arrow/flight_async_generic_doget_example.cc b/cpp/examples/arrow/flight_async_generic_doget_example.cc index 3eea1d8ae539..ce1df2a841d3 100644 --- a/cpp/examples/arrow/flight_async_generic_doget_example.cc +++ b/cpp/examples/arrow/flight_async_generic_doget_example.cc @@ -15,15 +15,7 @@ // specific language governing permissions and limitations // under the License. -#ifdef _WIN32 -# include -#else -# include -#endif #include -#if defined(__linux__) -# include -#endif #include #include @@ -74,15 +66,6 @@ DEFINE_bool(demo, false, "Run the DoGet self-check against the server, then exit namespace flight = arrow::flight; -/// \brief The current process' id, portably. -int64_t CurrentPid() { -#ifdef _WIN32 - return static_cast(::GetCurrentProcessId()); -#else - return static_cast(::getpid()); -#endif -} - /// \brief A reader with `num_batches` batches of three int64 values. class BatchReader : public arrow::RecordBatchReader { public: @@ -133,15 +116,6 @@ struct WorkerState { /// \brief The worker: wait for a payload request, sleep delay_ms, produce it, /// complete the future the transport holds. void RunWorker(std::shared_ptr state) { -#if defined(__linux__) - // Name the thread, so /proc//task/*/comm distinguishes this server's own - // per-stream workers (doget_worker) from gRPC's pool threads (event_engine / - // lifeguard / grpc_global_timer). Thread names are inherited, and gRPC names - // its threads, so an anonymous thread created from a gRPC callback would - // otherwise show up as another "event_engine". comm is capped at 15 - // characters. - prctl(PR_SET_NAME, "doget_worker", 0, 0, 0); -#endif std::unique_lock lock(state->mutex); while (!state->closed) { state->ready.wait(lock, [&] { return state->has_pending || state->closed; }); @@ -292,39 +266,22 @@ class ExampleServer : public flight::AsyncGenericFlightServerBase { }; /// \brief The --demo self-check: drain one DoGet stream and count its batches. -bool RunDemo(int port) { - auto location = flight::Location::ForGrpcTcp("127.0.0.1", port); - if (!location.ok()) { - std::cerr << "Failed to build client location: " << location.status() << std::endl; - return false; - } - auto client = flight::FlightClient::Connect(*location); - if (!client.ok()) { - std::cerr << "Failed to connect: " << client.status() << std::endl; - return false; - } - auto reader = (*client)->DoGet(flight::Ticket("demo")); - if (!reader.ok()) { - std::cerr << "DoGet failed: " << reader.status() << std::endl; - return false; - } +arrow::Status RunDemo(int port) { + ARROW_ASSIGN_OR_RAISE(auto location, flight::Location::ForGrpcTcp("127.0.0.1", port)); + ARROW_ASSIGN_OR_RAISE(auto client, flight::FlightClient::Connect(location)); + ARROW_ASSIGN_OR_RAISE(auto reader, client->DoGet(flight::Ticket("demo"))); int64_t num_batches = 0; while (true) { - auto chunk = (*reader)->Next(); - if (!chunk.ok()) { - std::cerr << "DoGet drain failed: " << chunk.status() << std::endl; - return false; - } - if (chunk->data == nullptr) break; // end of stream + ARROW_ASSIGN_OR_RAISE(auto chunk, reader->Next()); + if (chunk.data == nullptr) break; // end of stream ++num_batches; } - if (num_batches == FLAGS_batches) { - std::cout << "PASS: DoGet streamed " << num_batches << " batches" << std::endl; - return true; + if (num_batches != FLAGS_batches) { + return arrow::Status::Invalid("DoGet streamed ", num_batches, " batches, expected ", + FLAGS_batches); } - std::cout << "FAIL: DoGet streamed " << num_batches << " batches, expected " - << FLAGS_batches << std::endl; - return false; + std::cout << "PASS: DoGet streamed " << num_batches << " batches" << std::endl; + return arrow::Status::OK(); } int main(int argc, char** argv) { @@ -350,13 +307,16 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } std::cout << "Serving Flight on " << server.location().ToString() << std::endl; - std::cout << "Server pid " << CurrentPid() << ", port " << server.port() << std::endl; if (FLAGS_demo) { - bool ok = RunDemo(server.port()); + status = RunDemo(server.port()); ARROW_WARN_NOT_OK(server.Shutdown(), "Error shutting down server"); ARROW_WARN_NOT_OK(server.Wait(), "Error waiting for server shutdown"); - return ok ? EXIT_SUCCESS : EXIT_FAILURE; + if (!status.ok()) { + std::cout << "FAIL: " << status.ToString() << std::endl; + return EXIT_FAILURE; + } + return EXIT_SUCCESS; } status = server.SetShutdownOnSignals({SIGINT, SIGTERM}); diff --git a/cpp/examples/arrow/flight_async_generic_doput_example.cc b/cpp/examples/arrow/flight_async_generic_doput_example.cc index 2adee004332b..850f284d4728 100644 --- a/cpp/examples/arrow/flight_async_generic_doput_example.cc +++ b/cpp/examples/arrow/flight_async_generic_doput_example.cc @@ -16,11 +16,6 @@ // under the License. #include -#ifdef _WIN32 -# include -#else -# include -#endif #include #include @@ -66,15 +61,6 @@ DEFINE_bool(demo, false, "Run the DoPut self-check against the server, then exit namespace flight = arrow::flight; -/// \brief The current process' id, portably. -int64_t CurrentPid() { -#ifdef _WIN32 - return static_cast(::GetCurrentProcessId()); -#else - return static_cast(::getpid()); -#endif -} - constexpr int kNumBatches = 3; constexpr int64_t kRowsPerBatch = 4; @@ -96,8 +82,6 @@ class CountingUploadListener : public flight::AsyncFlightDataListener { public: arrow::Future<> OnDescriptor(const flight::FlightDescriptor& descriptor) override { std::cout << "DoPut descriptor: " << descriptor.ToString() << std::endl; - // Returning non-OK here would reject the upload, the client sees that - // status when it closes the writer. return arrow::Future<>::MakeFinished(arrow::Status::OK()); } @@ -178,44 +162,19 @@ class UploadServer : public flight::AsyncGenericFlightServerBase { /// \brief The --demo client half: upload kNumBatches batches and close the /// writer. What the listener saw is read back in main(), once the server has /// stopped. -bool RunDemo(int port) { - auto location = flight::Location::ForGrpcTcp("127.0.0.1", port); - if (!location.ok()) { - std::cerr << "Failed to build client location: " << location.status() << std::endl; - return false; - } - auto client = flight::FlightClient::Connect(*location); - if (!client.ok()) { - std::cerr << "Failed to connect: " << client.status() << std::endl; - return false; - } - +arrow::Status RunDemo(int port) { + ARROW_ASSIGN_OR_RAISE(auto location, flight::Location::ForGrpcTcp("127.0.0.1", port)); + ARROW_ASSIGN_OR_RAISE(auto client, flight::FlightClient::Connect(location)); auto schema = arrow::schema({arrow::field("value", arrow::int64())}); - auto put = (*client)->DoPut(flight::FlightDescriptor::Command("upload"), schema); - if (!put.ok()) { - std::cerr << "DoPut failed: " << put.status() << std::endl; - return false; - } - arrow::Status status; - for (int i = 0; i < kNumBatches && status.ok(); i++) { - auto batch = MakeInt64Batch(schema, i * kRowsPerBatch, kRowsPerBatch); - if (!batch.ok()) { - std::cerr << "Failed to make a batch: " << batch.status() << std::endl; - return false; - } - status = put->writer->WriteRecordBatch(**batch); - } - if (status.ok()) { - status = put->writer->DoneWriting(); - } - if (status.ok()) { - status = put->writer->Close(); - } - if (!status.ok()) { - std::cerr << "DoPut upload failed: " << status << std::endl; - return false; - } - return true; + ARROW_ASSIGN_OR_RAISE( + auto put, client->DoPut(flight::FlightDescriptor::Command("upload"), schema)); + for (int i = 0; i < kNumBatches; i++) { + ARROW_ASSIGN_OR_RAISE(auto batch, + MakeInt64Batch(schema, i * kRowsPerBatch, kRowsPerBatch)); + ARROW_RETURN_NOT_OK(put.writer->WriteRecordBatch(*batch)); + } + ARROW_RETURN_NOT_OK(put.writer->DoneWriting()); + return put.writer->Close(); } int main(int argc, char** argv) { @@ -234,21 +193,20 @@ int main(int argc, char** argv) { std::cerr << location.status() << std::endl; return EXIT_FAILURE; } - flight::FlightServerOptions options(*location); - // UploadServer derives from AsyncGenericFlightServerBase: the server class - // alone selects the async generic transport, so there is no option to set. - // Uploads come from the server's CreateDoPutListener() override. - auto status = server.Init(options); + auto status = server.Init(flight::FlightServerOptions(*location)); if (!status.ok()) { std::cerr << status.ToString() << std::endl; return EXIT_FAILURE; } int port = server.port(); std::cout << "Serving Flight on " << server.location().ToString() << std::endl; - std::cout << "Server pid " << CurrentPid() << ", port " << port << std::endl; if (FLAGS_demo) { - bool ok = RunDemo(port); + status = RunDemo(port); + bool ok = status.ok(); + if (!ok) { + std::cout << "FAIL: " << status.ToString() << std::endl; + } ARROW_WARN_NOT_OK(server.Shutdown(), "Error shutting down server"); ARROW_WARN_NOT_OK(server.Wait(), "Error waiting for server shutdown"); // The listener callbacks run on gRPC threads: read its counters only once From 0e6fbe48d32b9b1f2cb2ae0f39c03e7fd4045ecc Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:34:20 +0100 Subject: [PATCH 16/25] [C++][FlightRPC] Share the request and write buffers through AsyncReactorBase --- cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc | 2 -- cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc | 2 -- cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc | 5 ++--- .../flight/transport/grpc/async_grpc_exchange_internal.h | 2 -- cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc | 2 -- .../flight/transport/grpc/async_grpc_service_internal.h | 4 ++++ cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc | 2 -- 7 files changed, 6 insertions(+), 13 deletions(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc index 61316933be6d..457e25a4a150 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc @@ -175,8 +175,6 @@ class DoGetReactor : public AsyncReactorBase { } AsyncGenericFlightServerBase* base_; - ::grpc::ByteBuffer request_buf_; - ::grpc::ByteBuffer write_buf_; std::shared_ptr async_data_stream_; /// Guards async_data_stream_, closed_ and wrote_schema_: adopting the stream, /// each demand on it and the teardown's Close() never interleave. diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc index 8db8d7c76230..62ca1164b978 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc @@ -271,8 +271,6 @@ class DoPutReactor : public AsyncReactorBase, bool ending_ = false; /// Whether that ending ack is the write in flight. bool ending_ack_in_flight_ = false; - ::grpc::ByteBuffer request_buf_; - ::grpc::ByteBuffer write_buf_; }; } // namespace diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc index 87adb56aefc1..b4e79f96b6eb 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc @@ -27,7 +27,7 @@ ExchangeReactor::ExchangeReactor(AsyncCallContext flight_context, writer_(std::make_shared(this)) { reader_->SetReactor(this); // The first message carries the request descriptor. - StartRead(&read_buf_); + StartRead(&request_buf_); } void ExchangeReactor::OnReadDone(bool ok) { @@ -42,7 +42,7 @@ void ExchangeReactor::OnReadDone(bool ok) { reader_->OnReadClosed(); return; } - const ::grpc::Status deserialized = FlightDataDeserialize(&read_buf_, &read_data_); + const ::grpc::Status deserialized = FlightDataDeserialize(&request_buf_, &read_data_); if (!deserialized.ok()) { if (!started_) { FinishOnce( @@ -98,7 +98,6 @@ void ExchangeReactor::OnDone() { } void ExchangeReactor::StartReadNext() { - WriteIfLive([this] { StartRead(&read_buf_); }); } arrow::Status ExchangeReactor::WritePayload(FlightPayload payload) { diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h index cc6a2e1c6fee..9ee0f7314445 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h @@ -200,8 +200,6 @@ class ExchangeReactor final : public AsyncReactorBase { AsyncGenericFlightServerBase* base_; std::shared_ptr reader_; std::shared_ptr writer_; - ::grpc::ByteBuffer read_buf_; - ::grpc::ByteBuffer write_buf_; internal::FlightData read_data_; bool started_ = false; }; diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc index c64fc3d3efea..7ac377158f3b 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc @@ -68,8 +68,6 @@ class HandshakeReactor final : public AsyncReactorBase { private: HandshakeFn handshake_handler_; - ::grpc::ByteBuffer request_buf_; - ::grpc::ByteBuffer write_buf_; }; } // namespace diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h index 0108125c58a1..ea925faed8a5 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h @@ -243,6 +243,10 @@ class AsyncReactorBase : public ::grpc::ServerGenericBidiReactor { } } + /// The request message StartRead() fills and the message StartWrite() sends. + ::grpc::ByteBuffer request_buf_; + ::grpc::ByteBuffer write_buf_; + private: AsyncCallContext flight_context_; std::mutex write_mutex_; diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc index c94adfc0de88..af53d27fbc8c 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc @@ -91,8 +91,6 @@ class StreamingReactor : public AsyncReactorBase { } AsyncGenericFlightServerBase* base_; - ::grpc::ByteBuffer request_buf_; - ::grpc::ByteBuffer write_buf_; }; class ListActionsReactor final : public StreamingReactor { From a413a2429e0910cd92f088e89936683e33f5fc63 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:34:20 +0100 Subject: [PATCH 17/25] [C++][FlightRPC] Move the unary reactor into async_grpc_service_internal.h --- cpp/src/arrow/flight/CMakeLists.txt | 3 +- cpp/src/arrow/flight/meson.build | 3 - .../transport/grpc/async_grpc_service.cc | 19 ++- .../grpc/async_grpc_service_internal.h | 59 ++++++++- .../flight/transport/grpc/async_grpc_unary.cc | 124 ------------------ 5 files changed, 70 insertions(+), 138 deletions(-) delete mode 100644 cpp/src/arrow/flight/transport/grpc/async_grpc_unary.cc diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 0e41da9bcf8a..45844a593a67 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -212,8 +212,7 @@ if(ARROW_GRPC_VERSION VERSION_GREATER_EQUAL "1.65") transport/grpc/async_grpc_exchange_reader.cc transport/grpc/async_grpc_exchange_writer.cc transport/grpc/async_grpc_handshake.cc - transport/grpc/async_grpc_streaming.cc - transport/grpc/async_grpc_unary.cc) + transport/grpc/async_grpc_streaming.cc) endif() # Handle Unity build header conflicts on Windows. diff --git a/cpp/src/arrow/flight/meson.build b/cpp/src/arrow/flight/meson.build index 465aabad2fa2..97a2557b411c 100644 --- a/cpp/src/arrow/flight/meson.build +++ b/cpp/src/arrow/flight/meson.build @@ -124,12 +124,9 @@ if has_async_server 'transport/grpc/async_grpc_exchange_writer.cc', 'transport/grpc/async_grpc_handshake.cc', 'transport/grpc/async_grpc_streaming.cc', - 'transport/grpc/async_grpc_unary.cc', ] endif - - thread_dep = dependency('threads') arrow_flight = library( diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc index 013f8b0e7953..f156898aa87e 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc @@ -28,6 +28,7 @@ namespace arrow::flight::transport::grpc { namespace { namespace detail = arrow::flight::transport::grpc::detail; +namespace pb = arrow::flight::protocol; constexpr std::string_view kPrefix = "/arrow.flight.protocol.FlightService/"; constexpr std::string_view kHandshakeMethod = "Handshake"; @@ -110,11 +111,23 @@ ::grpc::ServerGenericBidiReactor* AsyncGenericFlightService::CreateReactor( } return detail::MakeHandshakeReactor(std::move(flight_context), handshake_handler_); case FlightMethod::GetFlightInfo: - return detail::MakeGetFlightInfoReactor(std::move(flight_context), base_); + return new detail::UnaryReactor( + std::move(flight_context), [base = base_](const ServerCallContext& context, + const FlightDescriptor& descriptor) { + return base->GetFlightInfoAsync(context, descriptor); + }); case FlightMethod::GetSchema: - return detail::MakeGetSchemaReactor(std::move(flight_context), base_); + return new detail::UnaryReactor( + std::move(flight_context), [base = base_](const ServerCallContext& context, + const FlightDescriptor& descriptor) { + return base->GetSchemaAsync(context, descriptor); + }); case FlightMethod::PollFlightInfo: - return detail::MakePollFlightInfoReactor(std::move(flight_context), base_); + return new detail::UnaryReactor( + std::move(flight_context), [base = base_](const ServerCallContext& context, + const FlightDescriptor& descriptor) { + return base->PollFlightInfoAsync(context, descriptor); + }); case FlightMethod::ListActions: return detail::MakeListActionsReactor(std::move(flight_context), base_); case FlightMethod::DoAction: diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h index ea925faed8a5..819f5b44895b 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h @@ -255,14 +255,61 @@ class AsyncReactorBase : public ::grpc::ServerGenericBidiReactor { inline static std::atomic live_reactors_{0}; }; +/// \brief A unary RPC: read one FlightDescriptor, write the handler's response. +/// +/// T is the type of the response object, and PbT is the corresponding protobuf type. +template +class UnaryReactor final : public AsyncReactorBase { + public: + using HandlerFn = std::function>( + const ServerCallContext&, const FlightDescriptor&)>; + + UnaryReactor(AsyncCallContext flight_context, HandlerFn handler) + : AsyncReactorBase(std::move(flight_context)), handler_(std::move(handler)) { + StartRead(&request_buf_); + } + + void OnReadDone(bool ok) override { + if (!ok) { + FinishOnce(MakeFlightError(FlightStatusCode::Internal, "Failed to read request")); + return; + } + const auto descriptor = ParseProtoRequest( + request_buf_, "FlightDescriptor"); + if (!descriptor.ok()) { + FinishOnce(descriptor.status()); + return; + } + arrow::Future> future = handler_(flight_context(), *descriptor); + future.AddCallback( + [this, token = hold()](const arrow::Result>& result) mutable { + if (!result.ok()) { + FinishOnce(result.status()); + return; + } + PbT response; + const auto status = SerializeOrNotFound(*result, &response); + if (!status.ok()) { + FinishOnce(status); + return; + } + write_buf_ = MakeWriteBuffer(response); + WriteIfLive([this] { StartWrite(&write_buf_); }); + }); + } + + void OnWriteDone(bool ok) override { + FinishOnce( + ok ? arrow::Status::OK() + : MakeFlightError(FlightStatusCode::Internal, "Failed to write response")); + } + + private: + HandlerFn handler_; +}; + ::grpc::ServerGenericBidiReactor* MakeHandshakeReactor(AsyncCallContext flight_context, HandshakeFn handshake_handler); -::grpc::ServerGenericBidiReactor* MakeGetFlightInfoReactor( - AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); -::grpc::ServerGenericBidiReactor* MakeGetSchemaReactor( - AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); -::grpc::ServerGenericBidiReactor* MakePollFlightInfoReactor( - AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); ::grpc::ServerGenericBidiReactor* MakeDoGetReactor(AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); ::grpc::ServerGenericBidiReactor* MakeDoPutReactor( diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_unary.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary.cc deleted file mode 100644 index fcba29fe5a98..000000000000 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_unary.cc +++ /dev/null @@ -1,124 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "arrow/flight/transport/grpc/async_grpc_service_internal.h" - -#include -#include -#include - -namespace arrow::flight::transport::grpc::detail { - -namespace { - -/** - * A generic unary reactor for handling async gRPC calls in the Flight service. - * T is the type of the response object, and PbT is the corresponding protobuf type. - */ -template -class UnaryReactor final : public AsyncReactorBase { - public: - using HandlerFn = std::function>( - const ServerCallContext&, const FlightDescriptor&)>; - - UnaryReactor(AsyncCallContext flight_context, HandlerFn handler) - : AsyncReactorBase(std::move(flight_context)), handler_(std::move(handler)) { - StartRead(&request_buf_); - } - - void OnReadDone(bool ok) override { - if (!ok) { - FinishOnce(MakeFlightError(FlightStatusCode::Internal, "Failed to read request")); - return; - } - const auto descriptor = ParseProtoRequest( - request_buf_, "FlightDescriptor"); - if (!descriptor.ok()) { - FinishOnce(descriptor.status()); - return; - } - arrow::Future> future = handler_(flight_context(), *descriptor); - future.AddCallback( - [this, token = hold()](const arrow::Result>& result) mutable { - if (!finished()) { - if (!result.ok()) { - FinishOnce(result.status()); - } else { - PbT response; - const auto status = SerializeOrNotFound(*result, &response); - if (!status.ok()) { - FinishOnce(status); - } else { - response_buf_ = MakeWriteBuffer(response); - WriteIfLive([this] { StartWrite(&response_buf_); }); - } - } - } - }); - } - - void OnWriteDone(bool ok) override { - if (finished()) { - return; - } - FinishOnce( - ok ? arrow::Status::OK() - : MakeFlightError(FlightStatusCode::Internal, "Failed to write response")); - } - - private: - HandlerFn handler_; - ::grpc::ByteBuffer request_buf_; - ::grpc::ByteBuffer response_buf_; -}; - -template -::grpc::ServerGenericBidiReactor* MakeUnaryReactor(AsyncCallContext flight_context, - HandlerFn handler) { - return new UnaryReactor(std::move(flight_context), std::move(handler)); -} - -} // namespace - -::grpc::ServerGenericBidiReactor* MakeGetFlightInfoReactor( - AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { - return MakeUnaryReactor( - std::move(flight_context), - [base](const ServerCallContext& context, const FlightDescriptor& descriptor) { - return base->GetFlightInfoAsync(context, descriptor); - }); -} - -::grpc::ServerGenericBidiReactor* MakeGetSchemaReactor( - AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { - return MakeUnaryReactor( - std::move(flight_context), - [base](const ServerCallContext& context, const FlightDescriptor& descriptor) { - return base->GetSchemaAsync(context, descriptor); - }); -} - -::grpc::ServerGenericBidiReactor* MakePollFlightInfoReactor( - AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { - return MakeUnaryReactor( - std::move(flight_context), - [base](const ServerCallContext& context, const FlightDescriptor& descriptor) { - return base->PollFlightInfoAsync(context, descriptor); - }); -} - -} // namespace arrow::flight::transport::grpc::detail From b582edbeefe45820682486a6aa6357e03c03f818 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:34:20 +0100 Subject: [PATCH 18/25] [C++][FlightRPC] Drop redundant checks from the streaming reactor --- .../transport/grpc/async_grpc_streaming.cc | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc index af53d27fbc8c..06b2faf23b4c 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc @@ -43,20 +43,15 @@ class StreamingReactor : public AsyncReactorBase { return; } Start().AddCallback([this, token = hold()](const arrow::Status& status) { - if (!finished()) { - if (!status.ok()) { - FinishOnce(status); - } else { - WriteNextMessage(); - } + if (!status.ok()) { + FinishOnce(status); + return; } + WriteNextMessage(); }); } void OnWriteDone(bool ok) override { - if (finished()) { - return; - } if (!ok) { FinishOnce(MakeFlightError(FlightStatusCode::Internal, "Failed to write response")); return; @@ -78,14 +73,12 @@ class StreamingReactor : public AsyncReactorBase { } NextMessage().AddCallback( [this, token = hold()](const arrow::Result& has_next) { - if (!finished()) { - if (!has_next.ok()) { - FinishOnce(has_next.status()); - } else if (!*has_next) { - FinishOnce(arrow::Status::OK()); - } else { - WriteIfLive([this] { StartWrite(&write_buf_); }); - } + if (!has_next.ok()) { + FinishOnce(has_next.status()); + } else if (!*has_next) { + FinishOnce(arrow::Status::OK()); + } else { + WriteIfLive([this] { StartWrite(&write_buf_); }); } }); } @@ -146,9 +139,6 @@ class DoActionReactor final : public StreamingReactor { } arrow::Future NextMessage() override { - if (stream_ == nullptr) { - return arrow::Future::MakeFinished(false); - } return stream_->NextAsync().Then( [this](const std::shared_ptr& result) -> arrow::Result { if (result == nullptr) { From b8b8a1142e3ee85fdc2be9d5e5c895fe72f79cef Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:34:20 +0100 Subject: [PATCH 19/25] [C++][FlightRPC] Replace FlightDataListenerTransport::Install/Clear with SetTransport --- cpp/src/arrow/flight/flight_data_decoder.cc | 27 ++++---------- cpp/src/arrow/flight/flight_data_decoder.h | 36 +++++++++---------- .../transport/grpc/async_grpc_do_put.cc | 5 +-- 3 files changed, 26 insertions(+), 42 deletions(-) diff --git a/cpp/src/arrow/flight/flight_data_decoder.cc b/cpp/src/arrow/flight/flight_data_decoder.cc index e7e591a76eb1..5a9942d59104 100644 --- a/cpp/src/arrow/flight/flight_data_decoder.cc +++ b/cpp/src/arrow/flight/flight_data_decoder.cc @@ -243,9 +243,9 @@ AsyncFlightDataListener::~AsyncFlightDataListener() = default; AsyncFlightDataWriter::~AsyncFlightDataWriter() = default; Future<> AsyncFlightDataWriter::WriteMetadataAsync(const Buffer& app_metadata) { - // Install() and Clear() swap transport_ under writer_mutex_, so both the read - // and the call must happen under it: otherwise this can read the old pointer - // just before DoPutReactor::OnDone() runs Clear(), and then call through a + // SetTransport() swaps transport_ under writer_mutex_, so both the read and + // the call must happen under it: otherwise this can read the old pointer just + // before DoPutReactor::OnDone() clears it, and then call through a // destructed reactor. The transport's own WriteMetadataAsync only takes its // private write_mutex_, so there is no lock-order cycle. std::lock_guard lock(writer_mutex_); @@ -261,7 +261,7 @@ namespace internal { FlightDataListenerTransport::~FlightDataListenerTransport() = default; -void FlightDataListenerTransport::Install( +void FlightDataListenerTransport::SetTransport( const std::shared_ptr& listener, FlightDataListenerTransport* transport) { if (listener == nullptr) { @@ -269,23 +269,8 @@ void FlightDataListenerTransport::Install( } std::lock_guard lock(listener->transport_mutex_); listener->transport_ = transport; - if (listener->writer_ != nullptr) { - std::lock_guard writer_lock(listener->writer_->writer_mutex_); - listener->writer_->transport_ = transport; - } -} - -void FlightDataListenerTransport::Clear( - const std::shared_ptr& listener) { - if (listener == nullptr) { - return; - } - std::lock_guard lock(listener->transport_mutex_); - listener->transport_ = nullptr; - if (listener->writer_ != nullptr) { - std::lock_guard writer_lock(listener->writer_->writer_mutex_); - listener->writer_->transport_ = nullptr; - } + std::lock_guard writer_lock(listener->writer_->writer_mutex_); + listener->writer_->transport_ = transport; } Future<> FlightDataListenerTransport::ReportFinish( diff --git a/cpp/src/arrow/flight/flight_data_decoder.h b/cpp/src/arrow/flight/flight_data_decoder.h index ba1262379eb6..35b499f6063c 100644 --- a/cpp/src/arrow/flight/flight_data_decoder.h +++ b/cpp/src/arrow/flight/flight_data_decoder.h @@ -42,7 +42,7 @@ namespace internal { /// /// The transport owns this, the listener only holds a reference for as long as /// the RPC is live, and asks for it under a lock the transport takes to clear -/// it (see FlightDataListenerTransport::Clear) +/// it (see FlightDataListenerTransport::SetTransport) class ARROW_FLIGHT_EXPORT FlightDataListenerTransport { public: virtual ~FlightDataListenerTransport(); @@ -62,25 +62,23 @@ class ARROW_FLIGHT_EXPORT FlightDataListenerTransport { /// \brief Pin this transport's lifetime, from any thread. /// - /// Install()/Clear() publish and retire the transport through the listener's + /// SetTransport() publishes and retires the transport through the listener's /// mutex, but a caller that reads the pointer and then releases the mutex /// before calling in (as Cancel() must, so OnFinish() can re-enter) still - /// races with Clear() and the reactor's deletion. Hold under the mutex and - /// pair with ReleaseHold() around the call so the object stays alive. + /// races with SetTransport(listener, nullptr) and the reactor's deletion. Hold + /// under the mutex and pair with ReleaseHold() around the call so the object + /// stays alive. virtual void Hold() = 0; /// \brief Drop a reference taken with Hold(). Deletes the transport at zero. virtual void ReleaseHold() = 0; - /// \brief Install `transport` on `listener` for the life of the RPC. - /// \param listener The listener to install the transport on. - /// \param transport The transport to install. - static void Install(const std::shared_ptr& listener, - FlightDataListenerTransport* transport); - - /// \brief Clear the installed transport. - /// \param listener The listener to clear the transport from. - static void Clear(const std::shared_ptr& listener); + /// \brief Set the transport of `listener`: the RPC's at its start, nullptr at + /// its end. + /// \param listener The listener to set the transport on. + /// \param transport The transport, or nullptr to clear it. + static void SetTransport(const std::shared_ptr& listener, + FlightDataListenerTransport* transport); /// \brief Report the upload's terminal status to the listener, once. /// \param listener The listener to report the status to. @@ -125,12 +123,12 @@ class ARROW_FLIGHT_EXPORT AsyncFlightDataWriter { private: friend class internal::FlightDataListenerTransport; - /// The transport's RPC state, set and cleared through - /// FlightDataListenerTransport::Install/Clear, like + /// The transport's RPC state, set through + /// FlightDataListenerTransport::SetTransport, like /// AsyncFlightDataListener::transport_. internal::FlightDataListenerTransport* transport_ = nullptr; /// Guards transport_, since WriteMetadataAsync may be called from any thread while - /// the transport installs or clears the pointer. + /// the transport sets or clears the pointer. mutable std::mutex writer_mutex_; }; @@ -192,8 +190,8 @@ class ARROW_FLIGHT_EXPORT AsyncFlightDataListener : public ipc::Listener { Future<> Cancel(Status status); private: - /// The transport installs and clears the state through - /// FlightDataListenerTransport::Install/Clear, both under this lock. + /// The transport sets and clears the state through + /// FlightDataListenerTransport::SetTransport, under this lock. friend class internal::FlightDataListenerTransport; /// Whether OnFinish was already reported. @@ -204,7 +202,7 @@ class ARROW_FLIGHT_EXPORT AsyncFlightDataListener : public ipc::Listener { std::make_shared(); mutable std::mutex transport_mutex_; - /// Not owned: the transport's own RPC state, valid until Clear(). + /// Not owned: the transport's own RPC state, valid until it is cleared. internal::FlightDataListenerTransport* transport_ = nullptr; }; diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc index 62ca1164b978..bc13a6fa399d 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc @@ -37,7 +37,7 @@ class DoPutReactor : public AsyncReactorBase, : AsyncReactorBase(std::move(flight_context)), listener_(std::move(listener)), decoder_(listener_) { - arrow::flight::internal::FlightDataListenerTransport::Install(listener_, this); + arrow::flight::internal::FlightDataListenerTransport::SetTransport(listener_, this); StartRead(&request_buf_); } @@ -214,7 +214,8 @@ class DoPutReactor : public AsyncReactorBase, } void OnDone() override { - arrow::flight::internal::FlightDataListenerTransport::Clear(listener_); + arrow::flight::internal::FlightDataListenerTransport::SetTransport(listener_, + nullptr); ReleaseHold(); } From c805eb3ebf42b9d51bd1ffafdf141bf29d0530f8 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:34:20 +0100 Subject: [PATCH 20/25] [C++][FlightRPC] Simplify the DoExchange reader and writer --- .../transport/grpc/async_grpc_exchange.cc | 2 +- .../grpc/async_grpc_exchange_internal.h | 15 ++++------- .../grpc/async_grpc_exchange_reader.cc | 25 ++---------------- .../grpc/async_grpc_exchange_writer.cc | 26 +++++++------------ 4 files changed, 17 insertions(+), 51 deletions(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc index b4e79f96b6eb..fe9d69ae0b7b 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc @@ -98,12 +98,12 @@ void ExchangeReactor::OnDone() { } void ExchangeReactor::StartReadNext() { + WriteIfLive([this] { StartRead(&request_buf_); }); } arrow::Status ExchangeReactor::WritePayload(FlightPayload payload) { bool own_buffer = false; const ::grpc::Status serialize = FlightDataSerialize(payload, &write_buf_, &own_buffer); - (void)own_buffer; if (!serialize.ok()) { return MakeFlightError(FlightStatusCode::Internal, serialize.error_message()); } diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h index 9ee0f7314445..cdbcc08d6319 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h @@ -21,7 +21,6 @@ #include #include #include -#include #include "arrow/flight/transport/grpc/async_grpc_service_internal.h" #include "arrow/ipc/writer.h" @@ -36,8 +35,8 @@ namespace arrow::flight::transport::grpc::detail { // coupled on purpose -- the reader asks the reactor for the next read, the // writer asks it to write a payload, and the reactor drives both from the // gRPC callbacks -- so they are declared together here and implemented one -// per translation unit (async_grpc_exchange_reader.cc, -// async_grpc_exchange_writer.cc, async_grpc_exchange_reactor.cc). +// per translation unit (async_grpc_exchange.cc, async_grpc_exchange_reader.cc, +// async_grpc_exchange_writer.cc). class ExchangeReactor; @@ -87,16 +86,14 @@ class ExchangeReader final : public AsyncFlightMessageReader, /// The decoder's listener: chunks are handed to the reader. class ChunkListener; - void ResolvePending(FlightStreamChunk chunk); - void ResolvePending(const Status& status); + void ResolvePending(arrow::Result result); /// \brief Decode one message and resolve the demand it satisfies, or keep /// reading when the message was read through (a schema or dictionary /// message). void Consume(internal::FlightData data); - /// \brief Arm the next read through the reactor, defined in - /// async_grpc_exchange_reader.cc, which has the reactor's definition. + /// \brief Arm the next read through the reactor. void RequestRead(); /// Guarded by mutex_. @@ -153,8 +150,7 @@ class ExchangeWriter final : public AsyncFlightMessageWriter { /// resolves when the queue is empty, or with the first failure. arrow::Future<> Flush(); - /// \brief Write queued payloads through the reactor, defined in - /// async_grpc_exchange_writer.cc, which has the reactor's definition. + /// \brief Write queued payloads through the reactor. void Pump(); void FailFlush(Status status); @@ -162,7 +158,6 @@ class ExchangeWriter final : public AsyncFlightMessageWriter { /// Set by the constructor, cleared by DetachReactor(); read under mutex_. ExchangeReactor* reactor_; std::unique_ptr batch_writer_; - ipc::IpcWriteOptions options_ = ipc::IpcWriteOptions::Defaults(); std::shared_ptr pending_app_metadata_; std::deque queue_; arrow::Future<> flush_; diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc index 9bbc9ed7e378..78f25e45baa2 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc @@ -41,14 +41,6 @@ class ExchangeReader::ChunkListener final : public AsyncFlightDataListener { return arrow::Future<>::MakeFinished(); } - arrow::Future<> OnDescriptor(const FlightDescriptor&) override { - return arrow::Future<>::MakeFinished(); - } - - arrow::Status OnSchemaDecoded(std::shared_ptr) override { - return arrow::Status::OK(); - } - private: ExchangeReader* reader_; }; @@ -156,20 +148,7 @@ void ExchangeReader::OnCancelled() { ResolvePending(arrow::Status::Cancelled("the client cancelled the exchange")); } -void ExchangeReader::ResolvePending(FlightStreamChunk chunk) { - arrow::Future pending; - { - std::lock_guard lock(mutex_); - if (!pending_.is_valid() || pending_.is_finished()) { - return; - } - pending = std::move(pending_); - pending_ = arrow::Future(); - } - pending.MarkFinished(std::move(chunk)); -} - -void ExchangeReader::ResolvePending(const Status& status) { +void ExchangeReader::ResolvePending(arrow::Result result) { arrow::Future pending; { std::lock_guard lock(mutex_); @@ -179,7 +158,7 @@ void ExchangeReader::ResolvePending(const Status& status) { pending = std::move(pending_); pending_ = arrow::Future(); } - pending.MarkFinished(status); + pending.MarkFinished(std::move(result)); } void ExchangeReader::Consume(internal::FlightData data) { diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc index 01bb15886958..a0ed08b413db 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc @@ -65,14 +65,13 @@ arrow::Future<> ExchangeWriter::BeginAsync(std::shared_ptr schema) { arrow::Status::Invalid("This writer has already been started.")); } auto sink = std::make_unique(this); - ARROW_ASSIGN_OR_RAISE(batch_writer_, ipc::internal::OpenRecordBatchWriter( - std::move(sink), schema, options_)); + ARROW_ASSIGN_OR_RAISE(batch_writer_, + ipc::internal::OpenRecordBatchWriter(std::move(sink), schema)); return Flush(); } arrow::Future<> ExchangeWriter::WriteRecordBatchAsync(const RecordBatch& batch) { - pending_app_metadata_ = nullptr; - return WriteBatch(batch); + return WriteWithMetadataAsync(batch, nullptr); } arrow::Future<> ExchangeWriter::WriteWithMetadataAsync( @@ -100,31 +99,24 @@ arrow::Future<> ExchangeWriter::CloseAsync() { } void ExchangeWriter::OnWriteDone(bool ok) { - if (!ok) { - { - std::lock_guard lock(mutex_); - write_in_flight_ = false; - } - FailFlush(MakeFlightError(FlightStatusCode::Internal, "Failed to write response")); - return; - } { std::lock_guard lock(mutex_); write_in_flight_ = false; } + if (!ok) { + FailFlush(MakeFlightError(FlightStatusCode::Internal, "Failed to write response")); + return; + } Pump(); } void ExchangeWriter::OnCancelled() { - bool had_write; { std::lock_guard lock(mutex_); - had_write = write_in_flight_; write_in_flight_ = false; } - if (had_write) { - FailFlush(arrow::Status::Cancelled("the client cancelled the exchange")); - } + // A no-op unless a flush is pending. + FailFlush(arrow::Status::Cancelled("the client cancelled the exchange")); } arrow::Future<> ExchangeWriter::WriteBatch(const RecordBatch& batch) { From d86a4a19eab980c2651cc70913c816e4dcf38a05 Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:34:20 +0100 Subject: [PATCH 21/25] [C++][FlightRPC] Remove dead code from the gRPC transport --- cpp/src/arrow/flight/flight_data_decoder.h | 1 - cpp/src/arrow/flight/meson.build | 1 - cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc | 1 - .../arrow/flight/transport/grpc/async_grpc_streaming.cc | 1 - cpp/src/arrow/flight/transport/grpc/grpc_server.cc | 6 ++---- .../arrow/flight/transport/grpc/grpc_server_internal.h | 1 - .../flight/transport/grpc/serialization_internal.cc | 9 +++------ .../arrow/flight/transport/grpc/serialization_internal.h | 8 -------- 8 files changed, 5 insertions(+), 23 deletions(-) diff --git a/cpp/src/arrow/flight/flight_data_decoder.h b/cpp/src/arrow/flight/flight_data_decoder.h index 35b499f6063c..e26f614bfa27 100644 --- a/cpp/src/arrow/flight/flight_data_decoder.h +++ b/cpp/src/arrow/flight/flight_data_decoder.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include diff --git a/cpp/src/arrow/flight/meson.build b/cpp/src/arrow/flight/meson.build index 97a2557b411c..97dd1a05d37d 100644 --- a/cpp/src/arrow/flight/meson.build +++ b/cpp/src/arrow/flight/meson.build @@ -210,7 +210,6 @@ if has_async_server # header, so we must declare it here to avoid a race condition flight_proto_grpc_files[1], ], - cpp_args: grpc_enable_async_args, dependencies: [arrow_test_dep, arrow_flight_test_dep], ) # The timeout bounds the tests that would otherwise hang the suite instead diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc index 457e25a4a150..c752735a1da1 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get.cc @@ -17,7 +17,6 @@ #include "arrow/flight/transport/grpc/async_grpc_service_internal.h" -#include #include #include #include diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc index 06b2faf23b4c..1ba570bc59f0 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc @@ -18,7 +18,6 @@ #include "arrow/flight/transport/grpc/async_grpc_service_internal.h" #include -#include #include #include diff --git a/cpp/src/arrow/flight/transport/grpc/grpc_server.cc b/cpp/src/arrow/flight/transport/grpc/grpc_server.cc index 507bef4693f7..23fa0a678b5c 100644 --- a/cpp/src/arrow/flight/transport/grpc/grpc_server.cc +++ b/cpp/src/arrow/flight/transport/grpc/grpc_server.cc @@ -489,10 +489,10 @@ class GrpcServerTransport : public internal::ServerTransport { std::string* peer_identity) { return base->ValidateToken(context, token, peer_identity); }; - async_helper_ = std::make_shared( + auto async_helper = std::make_shared( /*auth_handler=*/nullptr, options.middleware, std::move(validate_token)); async_service_ = std::make_unique( - async_base_, async_helper_, std::move(handshake)); + async_base_, std::move(async_helper), std::move(handshake)); // The else and its body are split by #endif, so cpplint cannot pair the // braces; the #if branch above opens the block it closes. } else // NOLINT(readability/braces) @@ -553,8 +553,6 @@ class GrpcServerTransport : public internal::ServerTransport { std::unique_ptr grpc_service_; #if ARROW_FLIGHT_HAS_ASYNC_SERVER AsyncGenericFlightServerBase* async_base_ = nullptr; - std::shared_ptr> - async_helper_; std::unique_ptr async_service_; #endif std::unique_ptr<::grpc::Server> grpc_server_; diff --git a/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h b/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h index 7964ce87df22..f1b679942cf6 100644 --- a/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h @@ -29,7 +29,6 @@ #include "arrow/flight/server.h" #include "arrow/flight/server_auth.h" #include "arrow/flight/server_middleware.h" -#include "arrow/flight/transport/grpc/async_grpc_compat.h" #include "arrow/flight/transport/grpc/util_internal.h" #include "arrow/util/uri.h" diff --git a/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc b/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc index a2ac73c70dc0..0393b42db852 100644 --- a/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc +++ b/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc @@ -129,12 +129,7 @@ class GrpcBuffer : public MutableBuffer { grpc_slice slice_; }; -} // namespace - -// Destructor callback for grpc::Slice. Declared here (rather than in the -// header) so -Wmissing-declarations is happy. -void ReleaseBuffer(void* buf_ptr); - +// Destructor callback for grpc::Slice void ReleaseBuffer(void* buf_ptr) { delete reinterpret_cast*>(buf_ptr); } @@ -158,6 +153,8 @@ arrow::Result<::grpc::Slice> SliceFromBuffer(const std::shared_ptr& buf) return slice; } +} // namespace + // Initialize arrow Buffer from gRPC ByteBuffer Status WrapGrpcBuffer(::grpc::ByteBuffer* cpp_buf, std::shared_ptr* out) { return GrpcBuffer::Wrap(cpp_buf, out); diff --git a/cpp/src/arrow/flight/transport/grpc/serialization_internal.h b/cpp/src/arrow/flight/transport/grpc/serialization_internal.h index 36285e0bdec8..80806f09966c 100644 --- a/cpp/src/arrow/flight/transport/grpc/serialization_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/serialization_internal.h @@ -39,14 +39,6 @@ namespace grpc { namespace pb = arrow::flight::protocol; -/// Zero-copy conversion of an Arrow Buffer to a gRPC Slice. -/// -/// The Buffer lifetime is tied to the Slice (a shared_ptr is attached to the -/// slice's destructor callback) -/// \param[in] buf The Arrow Buffer to convert. -/// \return A gRPC Slice that shares ownership of the Buffer. -arrow::Result<::grpc::Slice> SliceFromBuffer(const std::shared_ptr& buf); - /// Zero-copy conversion of a gRPC ByteBuffer into an Arrow Buffer. /// /// The Arrow Buffer owns a reference to the gRPC slice, so it stays valid From 3fdae835ce4eb148abf11949b04c06aa0709759d Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:34:21 +0100 Subject: [PATCH 22/25] [C++][FlightRPC] Share the async server test helpers and drop duplicate tests --- .../transport/grpc/async_grpc_do_get_test.cc | 72 ++---- .../transport/grpc/async_grpc_do_put_test.cc | 220 ++++-------------- .../grpc/async_grpc_exchange_test.cc | 24 +- .../grpc/async_grpc_handshake_test.cc | 24 +- .../transport/grpc/async_grpc_service_test.cc | 109 +-------- .../grpc/async_grpc_streaming_test.cc | 66 ++---- .../transport/grpc/async_grpc_test_fixtures.h | 148 ++++++++---- .../transport/grpc/async_grpc_unary_test.cc | 39 +--- 8 files changed, 205 insertions(+), 497 deletions(-) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc index f380114ff4fb..f895db57473e 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc @@ -21,18 +21,12 @@ namespace arrow::flight::transport::grpc { TEST(AsyncGrpcTest, BasicDoGet) { TestFlightServer flight_server; - AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); - - int port = 0; - ::grpc::ServerBuilder builder; - builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); - builder.RegisterCallbackGenericService(&service); - auto server = builder.BuildAndStart(); - ASSERT_NE(server, nullptr); - ASSERT_NE(port, 0); + HandBuiltServer hand_built = StartHandBuilt(&flight_server); + ASSERT_NE(hand_built.server, nullptr); + ASSERT_NE(hand_built.port, 0); // Connect the existing arrow::flight::FlightClient implementation. - ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(hand_built.port)); // Call DoGet: the ticket must reach TestFlightServer::DoGetAsync. Ticket ticket{"basic-do-get-ticket"}; @@ -59,25 +53,19 @@ TEST(AsyncGrpcTest, BasicDoGet) { // Cleanup. The ticket is asserted only after the server stopped, so the // DoGet callback cannot race with the read. ASSERT_OK(client->Close()); - server->Shutdown(); - server->Wait(); + hand_built.server->Shutdown(); + hand_built.server->Wait(); ASSERT_EQ(flight_server.ticket(), "basic-do-get-ticket"); } TEST(AsyncGrpcTest, SchemaOnlyDoGet) { TestFlightServer flight_server; - AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); - - int port = 0; - ::grpc::ServerBuilder builder; - builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); - builder.RegisterCallbackGenericService(&service); - auto server = builder.BuildAndStart(); - ASSERT_NE(server, nullptr); - ASSERT_NE(port, 0); + HandBuiltServer hand_built = StartHandBuilt(&flight_server); + ASSERT_NE(hand_built.server, nullptr); + ASSERT_NE(hand_built.port, 0); - ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(hand_built.port)); // This ticket is served as a stream with no batches at all. Ticket ticket{kSchemaOnlyTicket}; @@ -96,8 +84,8 @@ TEST(AsyncGrpcTest, SchemaOnlyDoGet) { // Cleanup. The ticket is asserted only after the server stopped, so the // DoGet callback cannot race with the read. ASSERT_OK(client->Close()); - server->Shutdown(); - server->Wait(); + hand_built.server->Shutdown(); + hand_built.server->Wait(); ASSERT_EQ(flight_server.ticket(), kSchemaOnlyTicket); } @@ -197,18 +185,12 @@ TEST(AsyncGrpcTest, DoGetClientCancelWithPendingPayloadFinishes) { reader->Cancel(); reader.reset(); - ASSERT_OK(client->Close()); - - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); // The producer was told to stop, exactly once. EXPECT_EQ(1, flight_server.stream_closes.load()); // And the reactor is gone even though its payload future never completed. - EXPECT_TRUE(WaitFor([&] { - return detail::AsyncReactorBase::live_reactors() == before; - })) << "a reactor leaked: still alive after the server stopped (before=" - << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; + ExpectNoReactorLeak(before); // And the call reported completion exactly once (the CAS in FinishOnce). std::lock_guard guard(trace->mutex); EXPECT_EQ(1, trace->call_completed); @@ -233,9 +215,7 @@ TEST(AsyncGrpcTest, DoGetClientDeadlineWhilePreparingFinishes) { auto reader = client->DoGet(impatient, Ticket{"hang"}); EXPECT_FALSE(reader.ok()) << "the deadline must expire while the server prepares"; - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); EXPECT_EQ(0, flight_server.stream_closes.load()) << "no stream was ever prepared, so there is nothing to Close"; @@ -328,16 +308,11 @@ TEST(AsyncGrpcTest, DoGetCancelCallbackReachesPendingHandler) { EXPECT_EQ(0, flight_server.next_awaits.load()) << "a stream that arrives after the RPC ended must not be read from"; - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); EXPECT_EQ(1, flight_server.cancel_callbacks.load()) << "the cancel callback must run exactly once"; - EXPECT_TRUE(WaitFor([&] { - return detail::AsyncReactorBase::live_reactors() == before; - })) << "a reactor leaked: still alive after the server stopped (before=" - << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; + ExpectNoReactorLeak(before); } /// A DoGet stream asked for its schema while the RPC dies: GetSchemaPayloadAsync() @@ -426,19 +401,14 @@ TEST(AsyncGrpcTest, DoGetNeverClosesAStreamItIsAskingForAPayload) { EXPECT_FALSE(client->DoGet(impatient, Ticket{"slow-schema"}).ok()) << "the deadline must expire while the stream is asked for its schema"; - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); EXPECT_FALSE(flight_server.closed_while_asked.load()) << "Close() ran while the transport was inside GetSchemaPayloadAsync()"; EXPECT_EQ(0, flight_server.asked_after_close.load()) << "the stream was asked for a payload after Close()"; EXPECT_EQ(1, flight_server.closes.load()) << "Close() must be delivered exactly once"; - EXPECT_TRUE(WaitFor([&] { - return detail::AsyncReactorBase::live_reactors() == before; - })) << "a reactor leaked: still alive after the server stopped (before=" - << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; + ExpectNoReactorLeak(before); } /// A DoGet stream whose first payload is an error: the transport must fail the @@ -506,9 +476,7 @@ TEST(AsyncGrpcTest, DoGetPumpErrorStillClosesTheStream) { } } reader.reset(); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); EXPECT_EQ(1, flight_server.stream_closes.load()) << "the producer was never told the RPC is over"; diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc index b5c0e1f93531..da77eef8cd41 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc @@ -31,16 +31,10 @@ TEST(AsyncGrpcTest, BasicDoPut) { auto listener = std::make_shared(); TestFlightServer flight_server; flight_server.set_listener(listener); - AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); - - int port = 0; - ::grpc::ServerBuilder builder; - builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); - builder.RegisterCallbackGenericService(&service); - auto server = builder.BuildAndStart(); - ASSERT_NE(server, nullptr); - ASSERT_NE(port, 0); - ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); + HandBuiltServer hand_built = StartHandBuilt(&flight_server); + ASSERT_NE(hand_built.server, nullptr); + ASSERT_NE(hand_built.port, 0); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(hand_built.port)); auto schema = TwoColumnSchema(); auto batch = TwoColumnBatch(schema); @@ -57,12 +51,15 @@ TEST(AsyncGrpcTest, BasicDoPut) { // Cleanup. The listener is asserted only after the server stopped, so the // DoPut callback cannot race with the reads. ASSERT_OK(client->Close()); - server->Shutdown(); - server->Wait(); + hand_built.server->Shutdown(); + hand_built.server->Wait(); // The listener saw the schema once and both batches, in order. ASSERT_EQ(listener->schema_count(), 1); - AssertTwoColumnSchema(listener->schema()); + ASSERT_NE(listener->schema(), nullptr); + ASSERT_EQ(listener->schema()->num_fields(), 2); + ASSERT_EQ(listener->schema()->field(0)->name(), "a"); + ASSERT_EQ(listener->schema()->field(1)->name(), "b"); ASSERT_EQ(listener->batches().size(), 2); for (const auto& chunk : listener->batches()) { ASSERT_EQ(chunk->num_rows(), 3); @@ -78,17 +75,11 @@ TEST(AsyncGrpcTest, EmptyDoPut) { TestFlightServer flight_server; auto listener = std::make_shared(); flight_server.set_listener(listener); - AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); - - int port = 0; - ::grpc::ServerBuilder builder; - builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); - builder.RegisterCallbackGenericService(&service); - auto server = builder.BuildAndStart(); - ASSERT_NE(server, nullptr); - ASSERT_NE(port, 0); + HandBuiltServer hand_built = StartHandBuilt(&flight_server); + ASSERT_NE(hand_built.server, nullptr); + ASSERT_NE(hand_built.port, 0); - ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(hand_built.port)); auto schema = arrow::schema( {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); @@ -102,8 +93,8 @@ TEST(AsyncGrpcTest, EmptyDoPut) { // Cleanup before looking at the listener, so the DoPut callback cannot race // with the reads. ASSERT_OK(client->Close()); - server->Shutdown(); - server->Wait(); + hand_built.server->Shutdown(); + hand_built.server->Wait(); // No data of any kind reached the listener. ASSERT_EQ(listener->batches().size(), 0); @@ -118,17 +109,11 @@ TEST(AsyncGrpcTest, MetadataOnlyPutChunk) { TestFlightServer flight_server; auto listener = std::make_shared(); flight_server.set_listener(listener); - AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); + HandBuiltServer hand_built = StartHandBuilt(&flight_server); + ASSERT_NE(hand_built.server, nullptr); + ASSERT_NE(hand_built.port, 0); - int port = 0; - ::grpc::ServerBuilder builder; - builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); - builder.RegisterCallbackGenericService(&service); - auto server = builder.BuildAndStart(); - ASSERT_NE(server, nullptr); - ASSERT_NE(port, 0); - - ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(hand_built.port)); auto schema = arrow::schema( {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); @@ -140,8 +125,8 @@ TEST(AsyncGrpcTest, MetadataOnlyPutChunk) { ASSERT_OK(result.writer->Close()); ASSERT_OK(client->Close()); - server->Shutdown(); - server->Wait(); + hand_built.server->Shutdown(); + hand_built.server->Wait(); // Exactly one chunk reached the listener: no data, that exact metadata. ASSERT_EQ(listener->batches().size(), 0); @@ -157,17 +142,11 @@ TEST(AsyncGrpcTest, DoPutRejectedByListener) { // The consumer rejects the upload. listener->set_next_status(arrow::Status::Invalid("listener rejected this upload")); flight_server.set_listener(listener); - AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); - - int port = 0; - ::grpc::ServerBuilder builder; - builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); - builder.RegisterCallbackGenericService(&service); - auto server = builder.BuildAndStart(); - ASSERT_NE(server, nullptr); - ASSERT_NE(port, 0); + HandBuiltServer hand_built = StartHandBuilt(&flight_server); + ASSERT_NE(hand_built.server, nullptr); + ASSERT_NE(hand_built.port, 0); - ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); + ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(hand_built.port)); auto schema = TwoColumnSchema(); auto batch = TwoColumnBatch(schema); @@ -186,8 +165,8 @@ TEST(AsyncGrpcTest, DoPutRejectedByListener) { << "OnFinish got: " << listener->finish_status(); ASSERT_OK(client->Close()); - server->Shutdown(); - server->Wait(); + hand_built.server->Shutdown(); + hand_built.server->Wait(); } TEST(AsyncGrpcTest, CreateDoPutListenerServesUpload) { @@ -210,9 +189,7 @@ TEST(AsyncGrpcTest, CreateDoPutListenerServesUpload) { // Cleanup before reading the recordings: the listener's callbacks run on a // gRPC thread. - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); // The virtual was consulted exactly once, and its listener saw the upload. ASSERT_EQ(flight_server.calls(), 1); @@ -220,31 +197,13 @@ TEST(AsyncGrpcTest, CreateDoPutListenerServesUpload) { ASSERT_EQ(listener->descriptor_count(), 1); ASSERT_EQ(listener->descriptors()[0], descriptor); ASSERT_EQ(listener->schema_count(), 1); + ASSERT_EQ(listener->schema()->num_fields(), 2); + ASSERT_EQ(listener->schema()->field(0)->name(), "a"); + ASSERT_EQ(listener->schema()->field(1)->name(), "b"); ASSERT_EQ(listener->batches().size(), 1); ASSERT_EQ(listener->batches()[0]->num_rows(), 3); -} - -TEST(AsyncGrpcTest, DoPutWithoutListenerIsUnimplemented) { - // A server class that does not hand out a listener (CreateDoPutListener() - // returns nullptr, the base default) does not accept uploads: the RPC is - // answered UNIMPLEMENTED. There is no options-level fallback. - TestFlightServer flight_server; - FlightServerOptions options(LocalhostLocation()); - ASSERT_OK(flight_server.Init(options)); - ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); - - auto schema = arrow::schema({arrow::field("a", arrow::int64())}); - auto batch = arrow::RecordBatch::Make(schema, 2, - {arrow::ArrayFromJSON(arrow::int64(), "[7, 8]")}); - auto [surface, status] = UploadOneBatch( - client.get(), FlightDescriptor::Path({"no-listener"}), schema, batch); - ASSERT_FALSE(status.ok()) << "the upload succeeded on " << surface - << " with no listener to serve it"; - ASSERT_EQ(status.code(), arrow::StatusCode::NotImplemented) << status; - - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK_AND_ASSIGN(auto value, listener->batches()[0]->column(0)->GetScalar(0)); + ASSERT_TRUE(value->Equals(*arrow::MakeScalar(int64_t(1)))); } TEST(AsyncGrpcTest, DoPutListenerSeesCleanFinish) { @@ -262,9 +221,7 @@ TEST(AsyncGrpcTest, DoPutListenerSeesCleanFinish) { UploadOneBatch(client.get(), FlightDescriptor::Path({"finish"}), schema, batch); ASSERT_TRUE(status.ok()) << "upload failed on " << surface << ": " << status; - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); // OnFinish ran exactly once, and said the upload ended cleanly. ASSERT_EQ(flight_server.listener(0)->finish_count(), 1); @@ -287,9 +244,7 @@ TEST(AsyncGrpcTest, OnFinishMayWriteMetadataWithoutDeadlocking) { ASSERT_OK(put.writer->WriteRecordBatch(*OneColumnBatch(schema))); ARROW_UNUSED(put.writer->Close()); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); auto listener = flight_server.WaitForListener(0); ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; @@ -399,9 +354,7 @@ TEST(AsyncGrpcTest, CallCompletedMayWriteMetadataWithoutDeadlocking) { ASSERT_OK(put.writer->WriteRecordBatch(*OneColumnBatch(schema))); ASSERT_OK(put.writer->Close()); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); const auto middleware = factory->last(); ASSERT_NE(middleware, nullptr) << "the upload never created its middleware"; @@ -443,9 +396,7 @@ TEST(AsyncGrpcTest, DoPutWritesMetadataBackToTheClient) { ASSERT_EQ("ack 2 rows", metadata->ToString()); ASSERT_OK(writer.Close()); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); auto listener = flight_server.WaitForListener(); ASSERT_NE(listener, nullptr) << "the server never handed out a DoPut listener"; @@ -489,9 +440,7 @@ TEST(AsyncGrpcTest, DoPutCancelFromServerSideFinishesTheUpload) { ASSERT_NE(close_status.message().find("server said stop"), std::string::npos) << "unexpected status: " << close_status; - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); // Cancel() reached the transport, and the listener was told the upload ended // with that status exactly once. @@ -578,9 +527,7 @@ TEST(AsyncGrpcTest, DoPutServerCancelEndsTheUploadWithItsOwnStatus) { << "a batch reached the listener after its OnFinish()"; EXPECT_EQ(1, listener->finish_count()); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } /// A listener stuck inside OnNext(): the upload is parked in the application @@ -626,9 +573,7 @@ TEST(AsyncGrpcTest, DoPutClientDeadlineWithPendingHandlerFinishes) { })) << "the listener never saw a chunk"; ARROW_UNUSED(put->writer->Close()); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); // The upload ended anyway, and the listener heard about it exactly once. ASSERT_EQ(1, flight_server.listener(0)->finish_count()); @@ -669,9 +614,7 @@ TEST(AsyncGrpcTest, DoPutClientDeadlineWhileIdleIsNotACleanFinish) { const auto close_status = put->writer->Close(); ASSERT_FALSE(close_status.ok()) << "the deadline never fired; inconclusive"; - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); ASSERT_EQ(1, listener->finish_count()); EXPECT_FALSE(listener->finish_status().ok()) @@ -791,32 +734,7 @@ TEST(AsyncGrpcTest, DoPutWithoutDescriptorOnFirstMessageIsRejected) { // Drive the bidi call through the raw generic stub: the regular FlightClient // always sets the descriptor, which is exactly what this case lacks. - std::shared_ptr<::grpc::Channel> channel = ::grpc::CreateChannel( - std::string(kLoopbackHost) + ":" + std::to_string(flight_server.port()), - ::grpc::InsecureChannelCredentials()); - ::grpc::GenericStub stub(channel); - ::grpc::ClientContext context; - // Bound the call: were the descriptor ever accepted, the client would wait - // forever for an acknowledgement, turning a logic error into a hung suite. - context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(10)); - ::grpc::CompletionQueue cq; - void* tag = nullptr; - bool ok = false; - - auto call = stub.Call(&context, "/arrow.flight.protocol.FlightService/DoPut", &cq, - reinterpret_cast(1)); - ASSERT_NE(call, nullptr); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never started"; - - call->Write(request, reinterpret_cast(2)); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "the message was never written"; - - call->WritesDone(reinterpret_cast(3)); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "WritesDone never completed"; - - ::grpc::Status status; - call->Finish(&status, reinterpret_cast(4)); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never finished"; + const ::grpc::Status status = CallDoPutRaw(flight_server.port(), {request}); EXPECT_FALSE(status.ok()) << "an upload without a descriptor must be refused"; EXPECT_THAT(status.error_message(), ::testing::HasSubstr("Descriptor missing")); @@ -843,27 +761,7 @@ TEST(AsyncGrpcTest, DoPutMalformedFirstMessageIsRejected) { ::grpc::Slice truncated(std::string("\x0a\x7f\x01\x02", 4)); ::grpc::ByteBuffer request(&truncated, 1); - std::shared_ptr<::grpc::Channel> channel = ::grpc::CreateChannel( - std::string(kLoopbackHost) + ":" + std::to_string(flight_server.port()), - ::grpc::InsecureChannelCredentials()); - ::grpc::GenericStub stub(channel); - ::grpc::ClientContext context; - context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(10)); - ::grpc::CompletionQueue cq; - void* tag = nullptr; - bool ok = false; - - auto call = stub.Call(&context, "/arrow.flight.protocol.FlightService/DoPut", &cq, - reinterpret_cast(1)); - ASSERT_NE(call, nullptr); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never started"; - call->Write(request, reinterpret_cast(2)); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "the message was never written"; - call->WritesDone(reinterpret_cast(3)); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "WritesDone never completed"; - ::grpc::Status status; - call->Finish(&status, reinterpret_cast(4)); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never finished"; + const ::grpc::Status status = CallDoPutRaw(flight_server.port(), {request}); EXPECT_FALSE(status.ok()) << "a malformed first message must be refused"; EXPECT_THAT(status.error_message(), @@ -902,9 +800,7 @@ TEST(AsyncGrpcTest, DoPutReportsAFailingOnFinishToTheClient) { << "a failing OnFinish must not be reported to the client as success"; EXPECT_THAT(close_status.ToString(), ::testing::HasSubstr("could not be committed")); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); EXPECT_EQ(1, listener->finish_count()); } @@ -922,28 +818,8 @@ TEST(AsyncGrpcTest, DoPutHalfCloseWithoutAnyMessageIsRejected) { ASSERT_OK(flight_server.Init(options)); ASSERT_GT(flight_server.port(), 0); - std::shared_ptr<::grpc::Channel> channel = ::grpc::CreateChannel( - std::string(kLoopbackHost) + ":" + std::to_string(flight_server.port()), - ::grpc::InsecureChannelCredentials()); - ::grpc::GenericStub stub(channel); - ::grpc::ClientContext context; - context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(10)); - ::grpc::CompletionQueue cq; - void* tag = nullptr; - bool ok = false; - - auto call = stub.Call(&context, "/arrow.flight.protocol.FlightService/DoPut", &cq, - reinterpret_cast(1)); - ASSERT_NE(call, nullptr); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never started"; - // Half-close with no message written at all. - call->WritesDone(reinterpret_cast(2)); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "WritesDone never completed"; - - ::grpc::Status status; - call->Finish(&status, reinterpret_cast(3)); - ASSERT_TRUE(cq.Next(&tag, &ok)) << "the call never finished"; + const ::grpc::Status status = CallDoPutRaw(flight_server.port(), {}); EXPECT_FALSE(status.ok()) << "an upload with no message at all must be refused"; EXPECT_THAT(status.error_message(), ::testing::HasSubstr("empty upload")); @@ -982,9 +858,7 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagUploadRejectedOnDescriptor) { ASSERT_NE(status.message().find("listener rejected this descriptor"), std::string::npos) << "unexpected status: " << status; - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); // The upload was rejected on its descriptor: the handler saw it, and no // schema or data followed it. diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc index 5ad40bd23bf0..2c3533951da6 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc @@ -25,12 +25,6 @@ namespace arrow::flight::transport::grpc { -// --------------------------------------------------------------------------- -// DoExchange: the bidirectional RPC. Its handler is DoExchangeAsync, which -// awaits the reader's demand and each write before the next one, so reads and -// writes are interleaved without blocking the callback thread. -// --------------------------------------------------------------------------- - TEST(AsyncGrpcTest, DoExchangeEchoesBatches) { // Two batches up, two batches back, in order: this is the case that needs a // read and a write outstanding at the same time. @@ -67,9 +61,7 @@ TEST(AsyncGrpcTest, DoExchangeEchoesBatches) { ASSERT_OK(exchange.writer->Close()); // Cleanup before looking at the recording: it was written on a gRPC thread. - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); ASSERT_EQ(1, flight_server.exchange_count()); ASSERT_EQ(0, flight_server.exchanges_in_flight()); @@ -101,9 +93,7 @@ TEST(AsyncGrpcTest, DoExchangeMetadataOnlyDoesNotHang) { ASSERT_EQ(nullptr, end.data); ASSERT_OK(exchange.writer->Close()); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); ASSERT_EQ(1, flight_server.exchange_count()); ASSERT_EQ(0, flight_server.exchanges_in_flight()); @@ -183,15 +173,9 @@ TEST(AsyncGrpcTest, DoExchangeClientCancelWithPendingHandlerFinishes) { exchange.reader->Cancel(); exchange.writer.reset(); exchange.reader.reset(); - ASSERT_OK(client->Close()); - - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); - EXPECT_TRUE(WaitFor([&] { - return detail::AsyncReactorBase::live_reactors() == before; - })) << "a reactor leaked: still alive after the server stopped (before=" - << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; + ExpectNoReactorLeak(before); } } // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc index 54ff0f3ea77c..e52a2d88f9f9 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc @@ -24,10 +24,6 @@ namespace arrow::flight::transport::grpc { -// --------------------------------------------------------------------------- -// Handshake + per-RPC token auth on the async generic service. -// --------------------------------------------------------------------------- - // Counters shared between the server callbacks and the test thread. struct AsyncAuthState { std::mutex mutex; @@ -150,9 +146,7 @@ TEST(AsyncGrpcTest, HandshakeWithoutHookIsUnimplemented) { ASSERT_FALSE(status.ok()) << "handshake must not succeed without a hook"; EXPECT_THAT(status.ToString(), ::testing::HasSubstr("authentication mechanism")); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, AsyncGenericFlightServerBaseImpliesAsyncTransport) { @@ -181,9 +175,7 @@ TEST(AsyncGrpcTest, AsyncGenericFlightServerBaseImpliesAsyncTransport) { EXPECT_THAT(status.ToString(), ::testing::HasSubstr("no listener available")) << "the async generic service must have served this call, not the sync path"; - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, HandshakeAuthenticatesAndTokenIsRequired) { @@ -293,9 +285,7 @@ TEST(AsyncGrpcTest, HandshakePayloadIsBinarySafe) { ASSERT_OK_AND_ASSIGN(auto chunk, reader->Next()); EXPECT_NE(nullptr, chunk.data); - ASSERT_OK(client->Close()); - ASSERT_OK(server.Shutdown()); - ASSERT_OK(server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &server)); } TEST(AsyncGrpcTest, EmptyHandshakeRequestIsRejectedNotHung) { @@ -443,9 +433,7 @@ TEST(AsyncGrpcTest, LateHandshakeResponseLeavesServerUsable) { {}, std::make_unique("user", "p4ssw0rd"))); ASSERT_OK(second_client->Close()); - ASSERT_OK(client->Close()); - ASSERT_OK(server.Shutdown()); - ASSERT_OK(server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &server)); } TEST(AsyncGrpcTest, EmptyHandshakeResponseIsDelivered) { @@ -470,9 +458,7 @@ TEST(AsyncGrpcTest, EmptyHandshakeResponseIsDelivered) { ASSERT_OK(client->Authenticate({}, std::move(handler))); EXPECT_EQ(std::string(), observed->response()); - ASSERT_OK(client->Close()); - ASSERT_OK(server.Shutdown()); - ASSERT_OK(server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &server)); } TEST(AsyncGrpcTest, RehandshakeOnTheSameChannelWorks) { diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc index 4151952407bd..743973cf1eaf 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc @@ -39,22 +39,16 @@ TEST(AsyncGrpcTest, MiddlewareCallCompletedRunsOnUnknownMethod) { /*auth_handler=*/nullptr, MiddlewareFactoryList{ {"recording", std::make_shared(trace)}}); - AsyncGenericFlightService service(&flight_server, helper); + HandBuiltServer hand_built = StartHandBuilt(&flight_server, helper); + ASSERT_NE(hand_built.server, nullptr); + ASSERT_NE(hand_built.port, 0); - int port = 0; - ::grpc::ServerBuilder builder; - builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); - builder.RegisterCallbackGenericService(&service); - auto server = builder.BuildAndStart(); - ASSERT_NE(server, nullptr); - ASSERT_NE(port, 0); - - const ::grpc::Status status = CallUnknownMethod(port); + const ::grpc::Status status = CallUnknownMethod(hand_built.port); ASSERT_EQ(::grpc::StatusCode::UNIMPLEMENTED, status.error_code()) << status.error_message(); - server->Shutdown(); - server->Wait(); + hand_built.server->Shutdown(); + hand_built.server->Wait(); std::lock_guard guard(trace->mutex); EXPECT_EQ(1, trace->start_call) << "the middleware must have started for the call"; @@ -62,40 +56,7 @@ TEST(AsyncGrpcTest, MiddlewareCallCompletedRunsOnUnknownMethod) { << "CallCompleted must run for an UNIMPLEMENTED reply too"; } -TEST(AsyncGrpcTest, OtherMethodsAreUnimplemented) { - TestFlightServer flight_server; - auto listener = std::make_shared(); - flight_server.set_listener(listener); - AsyncGenericFlightService service(&flight_server, MakeAsyncHelper()); - - int port = 0; - ::grpc::ServerBuilder builder; - builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), &port); - builder.RegisterCallbackGenericService(&service); - auto server = builder.BuildAndStart(); - ASSERT_NE(server, nullptr); - ASSERT_NE(port, 0); - - ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(port)); - - // Every typed FlightService method is served now, the fallback reactor is - // still reachable through a method name no stub knows (which maps to - // FlightMethod::Invalid in MethodFromName), and that is the only pin left - // that an unknown method cannot be answered by a typed path. - const ::grpc::Status status = CallUnknownMethod(port); - ASSERT_FALSE(status.ok()) << "an unknown method must not be served"; - ASSERT_EQ(::grpc::StatusCode::UNIMPLEMENTED, status.error_code()) - << status.error_message(); - - ASSERT_OK(client->Close()); - server->Shutdown(); - server->Wait(); -} - TEST(AsyncGrpcTest, UseAsyncGrpcFlag) { - ASSERT_OK_AND_ASSIGN(auto default_location, Location::Parse(kServerUri)); - FlightServerOptions default_options(default_location); - // Same adapter as the tests above, but now started through the async // server's own Init instead of a hand-built grpc::ServerBuilder. TestFlightServer flight_server; @@ -145,59 +106,11 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlag) { // Cleanup. The ticket is asserted only after the server stopped, so the // DoGet callback cannot race with the read. - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); ASSERT_EQ(flight_server.ticket(), "flag-path-do-get-ticket"); } -TEST(AsyncGrpcTest, UseAsyncGrpcFlagServesDoPut) { - // The async server serves uploads because the server class hands out the - // per-RPC listener through CreateDoPutListener(), instead of the RPC falling - // through to UNIMPLEMENTED. - TestFlightServer flight_server; - auto listener = std::make_shared(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); - flight_server.set_listener(listener); - ASSERT_OK(flight_server.Init(options)); - ASSERT_GT(flight_server.port(), 0); - - ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); - - auto schema = arrow::schema( - {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); - auto batch = - arrow::RecordBatch::Make(schema, 3, - {arrow::ArrayFromJSON(arrow::int64(), "[1, 2, 3]"), - arrow::ArrayFromJSON(arrow::int64(), "[10, 20, 30]")}); - - const FlightDescriptor descriptor = FlightDescriptor::Path({"wave", "h2"}); - auto [surface, status] = UploadOneBatch(client.get(), descriptor, schema, batch); - ASSERT_TRUE(status.ok()) << "upload failed on " << surface << ": " << status; - - // Cleanup before looking at the listener: its callbacks run on a gRPC - // thread, and nothing may read the recordings while that thread writes. - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); - - // The listener saw the descriptor the client sent, the schema, and the batch. - ASSERT_EQ(listener->descriptor_count(), 1); - ASSERT_EQ(listener->descriptors()[0], descriptor); - ASSERT_EQ(listener->schema_count(), 1); - ASSERT_EQ(listener->schema()->num_fields(), 2); - ASSERT_EQ(listener->schema()->field(0)->name(), "a"); - ASSERT_EQ(listener->schema()->field(1)->name(), "b"); - ASSERT_EQ(listener->batches().size(), 1); - const std::shared_ptr chunk = listener->batches()[0]; - ASSERT_EQ(chunk->num_rows(), 3); - ASSERT_EQ(chunk->num_columns(), 2); - ASSERT_OK_AND_ASSIGN(auto value, chunk->column(0)->GetScalar(0)); - ASSERT_TRUE(value->Equals(*arrow::MakeScalar(int64_t(1)))); -} - TEST(AsyncGrpcTest, UseAsyncGrpcFlagRefusesBlockingAuthHandler) { // The callback path cannot run the blocking ServerAuthHandler (it would run // on callback threads), so a server configured with one must refuse to start @@ -235,9 +148,7 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagAllowsMiddleware) { if (!chunk.data) break; } - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); std::lock_guard guard(trace->mutex); EXPECT_EQ(1, trace->start_call); @@ -258,9 +169,7 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagMiddlewareCanRejectCall) { EXPECT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("rejected by middleware")); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } /// A ServerCallContext no async server handed out, as a synchronous diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc index 162b7769f967..379023515385 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc @@ -25,15 +25,6 @@ namespace arrow::flight::transport::grpc { -// --------------------------------------------------------------------------- -// The streaming trio: ListActions (a vector), DoAction (an AsyncResultStream) -// and ListFlights (an AsyncFlightListing) share StreamingReactor (one request -// in, N responses out, driven by OnWriteDone -> WriteNext). Each RPC has a -// happy-path test, the reactor never writing a message must still finish the -// RPC (the empty tests), and the two iterator-backed ones pin the mid-stream -// error rule. -// --------------------------------------------------------------------------- - TEST(AsyncGrpcTest, ListActionsIsServedAsync) { TestFlightServer flight_server; FlightServerOptions options(LocalhostLocation()); @@ -47,9 +38,7 @@ TEST(AsyncGrpcTest, ListActionsIsServedAsync) { ASSERT_EQ(actions[1].type, "fail"); ASSERT_EQ(actions[1].description, "always fails"); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, ListActionsEmptyFinishesCleanly) { @@ -65,9 +54,7 @@ TEST(AsyncGrpcTest, ListActionsEmptyFinishesCleanly) { ASSERT_OK_AND_ASSIGN(auto actions, client->ListActions()); ASSERT_TRUE(actions.empty()); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, DoActionIsServedAsync) { @@ -86,9 +73,7 @@ TEST(AsyncGrpcTest, DoActionIsServedAsync) { ASSERT_EQ(end, nullptr); // Cleanup before looking at the recording: it was written on a gRPC thread. - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); ASSERT_EQ("ping", flight_server.last_action_type()); } @@ -105,9 +90,7 @@ TEST(AsyncGrpcTest, DoActionStreamsMultipleResults) { ASSERT_OK_AND_ASSIGN(auto bodies, DrainResultBodies(results.get())); ASSERT_EQ(bodies, (std::vector{"one", "two", "three"})); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, DoActionMidStreamErrorFinishesTheRpc) { @@ -130,9 +113,7 @@ TEST(AsyncGrpcTest, DoActionMidStreamErrorFinishesTheRpc) { ASSERT_THAT(second.status().message(), ::testing::HasSubstr("mid-stream action failure")); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, DoActionWithNullResultStreamIsCancelled) { @@ -155,9 +136,7 @@ TEST(AsyncGrpcTest, DoActionWithNullResultStreamIsCancelled) { ASSERT_RAISES(Cancelled, results.status()); } - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, ListFlightsIsServedAsync) { @@ -174,9 +153,7 @@ TEST(AsyncGrpcTest, ListFlightsIsServedAsync) { {"second-listing"}})); // Cleanup before looking at the recording: it was written on a gRPC thread. - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); ASSERT_EQ("filter-me", flight_server.last_criteria().expression); } @@ -196,9 +173,7 @@ TEST(AsyncGrpcTest, ListFlightsEmptyFinishesCleanly) { // (empty) Criteria, exactly as the sync handler does. ASSERT_EQ("", flight_server.last_criteria().expression); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, ListFlightsMidStreamErrorFinishesTheRpc) { @@ -215,9 +190,7 @@ TEST(AsyncGrpcTest, ListFlightsMidStreamErrorFinishesTheRpc) { ASSERT_THAT(listing.status().message(), ::testing::HasSubstr("mid-stream listing failure")); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, ListFlightsAsyncCompletesOnAnotherThread) { @@ -231,12 +204,8 @@ TEST(AsyncGrpcTest, ListFlightsAsyncCompletesOnAnotherThread) { std::thread([out]() mutable { std::this_thread::sleep_for(std::chrono::milliseconds(100)); std::vector flights; - auto info = MakeTestFlightInfo(FlightDescriptor::Path({"deferred"})); - if (!info.ok()) { - out.MarkFinished(info.status()); - return; - } - flights.push_back(std::move(**info)); + flights.push_back( + std::move(*MakeTestFlightInfo(FlightDescriptor::Path({"deferred"})))); out.MarkFinished(std::make_shared(std::move(flights))); }).detach(); return out; @@ -258,9 +227,7 @@ TEST(AsyncGrpcTest, ListFlightsAsyncCompletesOnAnotherThread) { } ASSERT_EQ(1, seen); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } // A ListFlights handler whose future never resolves. @@ -288,14 +255,9 @@ TEST(AsyncGrpcTest, ListFlightsDeadlineWithPendingHandlerFinishes) { auto listing = client->ListFlights(impatient, Criteria("hang")); EXPECT_FALSE(listing.ok()) << "the deadline must expire while the handler hangs"; - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); - EXPECT_TRUE(WaitFor([&] { - return detail::AsyncReactorBase::live_reactors() == before; - })) << "a reactor leaked: still alive after the server stopped (before=" - << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; + ExpectNoReactorLeak(before); } } // namespace arrow::flight::transport::grpc diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h index bffb8b09a8e9..d18220951ead 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h @@ -58,13 +58,12 @@ namespace arrow::flight::transport::grpc { constexpr const char* kSchemaOnlyTicket = "schema-only-do-get-ticket"; // One canned FlightInfo, `descriptor` names it for the assertions. -inline arrow::Result> MakeTestFlightInfo( - FlightDescriptor descriptor) { - ARROW_ASSIGN_OR_RAISE( - auto made, FlightInfo::Make(*arrow::schema({arrow::field("value", arrow::int64())}), - std::move(descriptor), std::vector{}, - /*total_records=*/-1, /*total_bytes=*/-1)); - return std::make_unique(std::move(made)); +inline std::unique_ptr MakeTestFlightInfo(FlightDescriptor descriptor) { + return std::make_unique( + FlightInfo::Make(*arrow::schema({arrow::field("value", arrow::int64())}), + std::move(descriptor), std::vector{}, + /*total_records=*/-1, /*total_bytes=*/-1) + .ValueOrDie()); } // An AsyncFlightDataStream over an in-memory RecordBatchReader: every payload is @@ -194,25 +193,17 @@ class TestFlightServer final : public AsyncGenericFlightServerBase { if (request.ticket == kSchemaOnlyTicket) { // An empty reader still carries the schema, NextAsync() reports the end // of stream right away. - auto reader = arrow::RecordBatchReader::Make({}, schema); - if (!reader.ok()) { - return arrow::Future>::MakeFinished( - reader.status()); - } return arrow::Future>::MakeFinished( - std::make_shared(std::move(*reader))); + std::make_shared( + arrow::RecordBatchReader::Make({}, schema).ValueOrDie())); } auto batch = arrow::RecordBatch::Make( schema, 5, {arrow::ArrayFromJSON(arrow::int64(), "[1, 2, 3, 4, 5]"), arrow::ArrayFromJSON(arrow::int64(), "[10, 20, 30, 40, 50]")}); - auto reader = arrow::RecordBatchReader::Make({batch, batch, batch}); - if (!reader.ok()) { - return arrow::Future>::MakeFinished( - reader.status()); - } return arrow::Future>::MakeFinished( - std::make_shared(std::move(*reader))); + std::make_shared( + arrow::RecordBatchReader::Make({batch, batch, batch}).ValueOrDie())); } // The unary trio: canned responses. `last_descriptor_` records the request @@ -232,26 +223,20 @@ class TestFlightServer final : public AsyncGenericFlightServerBase { return arrow::Future>::MakeFinished( std::shared_ptr{}); } - auto made = - FlightInfo::Make(*arrow::schema({arrow::field("value", arrow::int64())}), - FlightDescriptor{}, std::vector{}, -1, -1); - if (!made.ok()) { - return arrow::Future>::MakeFinished(made.status()); - } return arrow::Future>::MakeFinished( - std::make_shared(std::move(*made))); + std::make_shared( + FlightInfo::Make(*arrow::schema({arrow::field("value", arrow::int64())}), + FlightDescriptor{}, std::vector{}, -1, -1) + .ValueOrDie())); } arrow::Future> GetSchemaAsync( const ServerCallContext&, const FlightDescriptor& request) override { last_descriptor_ = request; - auto made = - SchemaResult::Make(*arrow::schema({arrow::field("value", arrow::int64())})); - if (!made.ok()) { - return arrow::Future>::MakeFinished(made.status()); - } return arrow::Future>::MakeFinished( - std::shared_ptr(std::move(*made))); + std::shared_ptr( + SchemaResult::Make(*arrow::schema({arrow::field("value", arrow::int64())})) + .ValueOrDie())); } arrow::Future> PollFlightInfoAsync( @@ -311,11 +296,9 @@ class TestFlightServer final : public AsyncGenericFlightServerBase { if (expression == "fail-mid-stream") { return arrow::Future>::MakeFinished( std::make_shared( - []() -> arrow::Result { - ARROW_ASSIGN_OR_RAISE( - auto info, - MakeTestFlightInfo(FlightDescriptor::Path({"first-listing"}))); - return std::move(*info); + [] { + return std::move( + *MakeTestFlightInfo(FlightDescriptor::Path({"first-listing"}))); }, "mid-stream listing failure")); } @@ -328,12 +311,7 @@ class TestFlightServer final : public AsyncGenericFlightServerBase { } std::vector flights; for (const auto& name : {"first-listing", "second-listing"}) { - auto info = MakeTestFlightInfo(FlightDescriptor::Path({name})); - if (!info.ok()) { - return arrow::Future>::MakeFinished( - info.status()); - } - flights.push_back(std::move(**info)); + flights.push_back(std::move(*MakeTestFlightInfo(FlightDescriptor::Path({name})))); } return arrow::Future>::MakeFinished( std::make_shared(std::move(flights))); @@ -771,6 +749,62 @@ inline ::grpc::Status CallUnknownMethod(int port) { return status; } +// Upload `messages` over a raw generic DoPut call to the server listening on +// `port`, half-close, and return the server's status. Only a raw call can send +// FlightData the FlightClient would refuse to build. The 10 s deadline bounds +// the call: were a bad upload ever accepted, the client would wait forever for +// an acknowledgement, turning a logic error into a hung suite. +inline ::grpc::Status CallDoPutRaw(int port, + const std::vector<::grpc::ByteBuffer>& messages) { + std::shared_ptr<::grpc::Channel> channel = + ::grpc::CreateChannel(std::string(kLoopbackHost) + ":" + std::to_string(port), + ::grpc::InsecureChannelCredentials()); + ::grpc::GenericStub stub(channel); + ::grpc::ClientContext context; + context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(10)); + ::grpc::CompletionQueue cq; + std::unique_ptr<::grpc::GenericClientAsyncReaderWriter> call = + stub.PrepareCall(&context, "/arrow.flight.protocol.FlightService/DoPut", &cq); + void* tag = nullptr; + bool ok = false; + call->StartCall(reinterpret_cast(1)); + EXPECT_TRUE(cq.Next(&tag, &ok)) << "the call never started"; + for (const auto& message : messages) { + call->Write(message, reinterpret_cast(2)); + EXPECT_TRUE(cq.Next(&tag, &ok)) << "the message was never written"; + } + call->WritesDone(reinterpret_cast(3)); + EXPECT_TRUE(cq.Next(&tag, &ok)) << "WritesDone never completed"; + ::grpc::Status status; + call->Finish(&status, reinterpret_cast(4)); + EXPECT_TRUE(cq.Next(&tag, &ok)) << "the call never finished"; + return status; +} + +// The generic service and the server a test registered it on by hand, with the +// port the server bound. `service` is declared first so it outlives `server`. +struct HandBuiltServer { + std::unique_ptr service; + std::unique_ptr<::grpc::Server> server; + int port = 0; +}; + +// Serve `base` on kListenAddress from a hand-built grpc::ServerBuilder. The +// caller checks `server` is non-null and `port` non-zero. +inline HandBuiltServer StartHandBuilt( + AsyncGenericFlightServerBase* base, + std::shared_ptr> helper = + MakeAsyncHelper()) { + HandBuiltServer out; + out.service = std::make_unique(base, std::move(helper)); + ::grpc::ServerBuilder builder; + builder.AddListeningPort(kListenAddress, ::grpc::InsecureServerCredentials(), + &out.port); + builder.RegisterCallbackGenericService(out.service.get()); + out.server = builder.BuildAndStart(); + return out; +} + // The single int64 column ("a") the upload tests use. inline std::shared_ptr OneColumnSchema() { return arrow::schema({arrow::field("a", arrow::int64())}); @@ -809,13 +843,6 @@ inline std::shared_ptr TwoColumnBatch( arrow::ArrayFromJSON(arrow::int64(), "[10, 20, 30]")}); } -inline void AssertTwoColumnSchema(const std::shared_ptr& schema) { - ASSERT_NE(schema, nullptr); - ASSERT_EQ(schema->num_fields(), 2); - ASSERT_EQ(schema->field(0)->name(), "a"); - ASSERT_EQ(schema->field(1)->name(), "b"); -} - // Drain a DoAction result stream, projecting each Result to a string. inline arrow::Result> DrainResultBodies(ResultStream* results) { std::vector out; @@ -856,6 +883,27 @@ inline bool WaitFor(Predicate predicate, return predicate(); } +// Expect every reactor alive after `before` was sampled to be gone, within +// WaitFor's bound: a leaked reactor would hold its call's state forever. +inline void ExpectNoReactorLeak(int64_t before) { + EXPECT_TRUE(WaitFor([&] { + return detail::AsyncReactorBase::live_reactors() == before; + })) << "a reactor leaked: still alive after the server stopped (before=" + << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; +} + +// Close the client and stop the server, all three unconditionally, and return +// the first error. +inline Status StopClientAndServer(FlightClient* client, + AsyncGenericFlightServerBase* server) { + Status status = client->Close(); + Status shutdown = server->Shutdown(); + Status wait = server->Wait(); + if (status.ok()) status = shutdown; + if (status.ok()) status = wait; + return status; +} + // Upload one batch and report whichever surface first reports an error: the // DoPut call, the write, or Close. inline std::pair UploadOneBatch( diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc index 4a1e7bfa172f..63ad549a7e68 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc @@ -25,12 +25,6 @@ namespace arrow::flight::transport::grpc { -// --------------------------------------------------------------------------- -// The unary trio: GetFlightInfo, GetSchema and PollFlightInfo share the -// UnaryReactor (one request in, one response out), so the trio has one -// happy-path test each plus one error-path test that pins the base class. -// --------------------------------------------------------------------------- - TEST(AsyncGrpcTest, GetFlightInfoIsServedAsync) { TestFlightServer flight_server; FlightServerOptions options(LocalhostLocation()); @@ -45,9 +39,7 @@ TEST(AsyncGrpcTest, GetFlightInfoIsServedAsync) { ASSERT_EQ("value", schema->field(0)->name()); // Cleanup before looking at the recording: it was written on a gRPC thread. - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); ASSERT_EQ("ping", flight_server.last_descriptor().cmd); } @@ -66,9 +58,7 @@ TEST(AsyncGrpcTest, GetFlightInfoWithNullResponseIsNotFound) { ASSERT_RAISES(KeyError, status); ASSERT_THAT(status.message(), ::testing::HasSubstr("Flight not found")); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, GetSchemaIsServedAsync) { @@ -85,9 +75,7 @@ TEST(AsyncGrpcTest, GetSchemaIsServedAsync) { ASSERT_EQ("value", schema->field(0)->name()); // Cleanup before looking at the recording: it was written on a gRPC thread. - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); ASSERT_EQ("ping", flight_server.last_descriptor().cmd); } @@ -106,9 +94,7 @@ TEST(AsyncGrpcTest, PollFlightInfoIsServedAsync) { ASSERT_EQ("poll-next", poll_info->descriptor->cmd); // Cleanup before looking at the recording: it was written on a gRPC thread. - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); ASSERT_EQ("ping", flight_server.last_descriptor().cmd); } @@ -140,14 +126,9 @@ TEST(AsyncGrpcTest, GetFlightInfoDeadlineWithPendingHandlerFinishes) { client->GetFlightInfo(impatient, FlightDescriptor::Command("hang")).status(); EXPECT_FALSE(status.ok()) << "the deadline must expire while the handler hangs"; - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); - EXPECT_TRUE(WaitFor([&] { - return detail::AsyncReactorBase::live_reactors() == before; - })) << "a reactor leaked: still alive after the server stopped (before=" - << before << ", now=" << detail::AsyncReactorBase::live_reactors() << ")"; + ExpectNoReactorLeak(before); } TEST(AsyncGrpcTest, UnaryReactorPropagatesServerError) { @@ -165,9 +146,7 @@ TEST(AsyncGrpcTest, UnaryReactorPropagatesServerError) { ASSERT_EQ(arrow::StatusCode::KeyError, status.code()); ASSERT_THAT(status.message(), ::testing::HasSubstr("no such flight")); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } TEST(AsyncGrpcTest, GetFlightInfoAsyncCompletesOnAnotherThread) { @@ -207,9 +186,7 @@ TEST(AsyncGrpcTest, GetFlightInfoAsyncCompletesOnAnotherThread) { ASSERT_EQ(1, schema->num_fields()); ASSERT_EQ("value", schema->field(0)->name()); - ASSERT_OK(client->Close()); - ASSERT_OK(flight_server.Shutdown()); - ASSERT_OK(flight_server.Wait()); + ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); } } // namespace arrow::flight::transport::grpc From 0f0f5eca55f4ad53ab4c21797310d3e9753ce2df Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:03:44 +0100 Subject: [PATCH 23/25] [C++][FlightRPC][Docs] Document the async Flight server --- docs/source/cpp/api/flight.rst | 41 +++++ docs/source/cpp/flight.rst | 221 ++++++++++++++++++++++++++ docs/source/developers/cpp/flight.rst | 174 ++++++++++++++++++++ docs/source/developers/cpp/index.rst | 1 + 4 files changed, 437 insertions(+) create mode 100644 docs/source/developers/cpp/flight.rst diff --git a/docs/source/cpp/api/flight.rst b/docs/source/cpp/api/flight.rst index 9b9e2b1e44f8..500911aa8668 100644 --- a/docs/source/cpp/api/flight.rst +++ b/docs/source/cpp/api/flight.rst @@ -184,6 +184,47 @@ Servers :project: arrow_cpp :members: +Asynchronous Servers +==================== + +See :ref:`cpp-flight-async-server` for an overview. + +.. doxygenclass:: arrow::flight::AsyncGenericFlightServerBase + :project: arrow_cpp + :members: + +.. doxygenclass:: arrow::flight::AsyncFlightDataStream + :project: arrow_cpp + :members: + +.. doxygenclass:: arrow::flight::AsyncFlightListing + :project: arrow_cpp + :members: + +.. doxygenclass:: arrow::flight::AsyncResultStream + :project: arrow_cpp + :members: + +.. doxygenclass:: arrow::flight::AsyncFlightMessageReader + :project: arrow_cpp + :members: + +.. doxygenclass:: arrow::flight::AsyncFlightMessageWriter + :project: arrow_cpp + :members: + +.. doxygenclass:: arrow::flight::AsyncFlightDataListener + :project: arrow_cpp + :members: + +.. doxygenclass:: arrow::flight::AsyncFlightDataWriter + :project: arrow_cpp + :members: + +.. doxygenclass:: arrow::flight::AsyncFlightMessageDecoder + :project: arrow_cpp + :members: + Error Handling ============== diff --git a/docs/source/cpp/flight.rst b/docs/source/cpp/flight.rst index 1bc6f62b0512..975bc10a165b 100644 --- a/docs/source/cpp/flight.rst +++ b/docs/source/cpp/flight.rst @@ -94,6 +94,227 @@ server stops. std::cout << "Server listening on localhost:" << server->port() << std::endl; ARROW_CHECK_OK(server->Serve()); +.. _cpp-flight-async-server: + +Writing an Asynchronous Flight Service +====================================== + +A :class:`arrow::flight::FlightServerBase` handler holds a gRPC thread until +it returns. When the work behind an RPC waits on something else (another +service, a query engine, storage), derive from +:class:`arrow::flight::AsyncGenericFlightServerBase` instead: every handler +returns an :class:`arrow::Future`, and no gRPC thread waits for it. + +.. note:: The asynchronous server needs Arrow to be built against gRPC 1.65 + or later. With an older gRPC, + :func:`arrow::flight::AsyncGenericFlightServerBase::Init` returns + ``NotImplemented``. + +.. code-block:: cpp + + class MyAsyncServer : public arrow::flight::AsyncGenericFlightServerBase { + public: + explicit MyAsyncServer(std::shared_ptr catalog) + : catalog_(std::move(catalog)) {} + + arrow::Future> GetFlightInfoAsync( + const arrow::flight::ServerCallContext& context, + const arrow::flight::FlightDescriptor& request) override { + // `request` is only valid during this call: hand on a copy. + return catalog_->LookUp(request); + } + + private: + std::shared_ptr catalog_; + }; + +Start, serve and stop the server exactly like a synchronous one: + +.. code-block:: cpp + + MyAsyncServer server(catalog); + ARROW_ASSIGN_OR_RAISE(auto location, + arrow::flight::Location::ForGrpcTcp("0.0.0.0", 0)); + ARROW_RETURN_NOT_OK(server.Init(arrow::flight::FlightServerOptions(location))); + ARROW_RETURN_NOT_OK(server.SetShutdownOnSignals({SIGTERM})); + ARROW_RETURN_NOT_OK(server.Serve()); + +Handlers you do not override answer ``NotImplemented``. Each request argument +(:class:`arrow::flight::Ticket`, :class:`arrow::flight::FlightDescriptor`, +:class:`arrow::flight::Criteria`, :class:`arrow::flight::Action`) and the +:class:`arrow::flight::ServerCallContext` are only valid during the handler +call. Authentication goes through ``Handshake()`` and ``ValidateToken()``: a +blocking :class:`arrow::flight::ServerAuthHandler` in the server options is +rejected by ``Init()``. Server middleware is supported. + +The streaming RPCs return a source the transport pulls one future at a time: + +================= ===================================================== +RPC Handler returns +================= ===================================================== +``ListFlights`` :class:`arrow::flight::AsyncFlightListing` +``DoAction`` :class:`arrow::flight::AsyncResultStream` +``DoGet`` :class:`arrow::flight::AsyncFlightDataStream` +``DoPut`` an :class:`arrow::flight::AsyncFlightDataListener`, + from ``CreateDoPutListener()`` +``DoExchange`` a future that completes when the exchange is over; + the handler is given an + :class:`arrow::flight::AsyncFlightMessageReader` and an + :class:`arrow::flight::AsyncFlightMessageWriter` +================= ===================================================== + +Serving DoGet +------------- + +An :class:`arrow::flight::AsyncFlightDataStream` produces IPC payloads (see +:func:`arrow::ipc::GetSchemaPayload` and +:func:`arrow::ipc::GetRecordBatchPayload`). The transport asks for the schema +first, then for one payload at a time, and asks again only once the previous +payload has been written: + +.. code-block:: cpp + + class QueryStream : public arrow::flight::AsyncFlightDataStream { + public: + explicit QueryStream(std::shared_ptr query) : query_(std::move(query)) {} + + arrow::Future GetSchemaPayloadAsync() override { + return query_->SchemaPayload(); + } + + arrow::Future> NextAsync() override { + // Complete with std::nullopt at the end of the stream. + return query_->NextPayload(); + } + + arrow::Status Close() override { + // The RPC is over: stop producing and resolve any pending NextAsync(). + return query_->Stop(); + } + + private: + std::shared_ptr query_; + }; + + arrow::Future> DoGetAsync( + const arrow::flight::ServerCallContext& context, + const arrow::flight::Ticket& request) override { + return catalog_->Open(request).Then( + [](const std::shared_ptr& query) + -> std::shared_ptr { + return std::make_shared(query); + }); + } + +``Close()`` is called exactly once, however the RPC ends. That includes a +stream ``DoGetAsync()`` only completes after the client cancelled: such a +stream is never read from, but it is still closed. ``Close()`` never overlaps +``GetSchemaPayloadAsync()`` or ``NextAsync()``, and neither is called once +``Close()`` has started. + +Serving DoPut +------------- + +``CreateDoPutListener()`` returns one fresh +:class:`arrow::flight::AsyncFlightDataListener` per upload, or ``nullptr`` to +refuse uploads (the client then sees ``NotImplemented``). The transport reads +the next message only once the future returned by ``OnNext()`` completes. +:func:`arrow::flight::AsyncFlightDataWriter::WriteMetadataAsync` sends one +acknowledgement and completes when it has been written, so returning a future +chained on it holds back the upload until the client has been answered: + +.. code-block:: cpp + + class IngestListener : public arrow::flight::AsyncFlightDataListener { + public: + explicit IngestListener(std::shared_ptr sink) : sink_(std::move(sink)) {} + + arrow::Future<> OnNext(arrow::flight::FlightStreamChunk chunk) override { + if (chunk.data == nullptr) { + return arrow::Future<>::MakeFinished(); // a metadata-only message + } + return sink_->Append(chunk.data).Then([this](int64_t rows) { + const auto ack = arrow::Buffer::FromString(std::to_string(rows)); + // The next message is read only once this ack has been written. + return writer()->WriteMetadataAsync(*ack); + }); + } + + arrow::Future<> OnFinish(arrow::Status status) override { + // Called once, however the upload ends. The client is answered only + // once this future completes, and a failure here fails the upload. + return status.ok() ? sink_->Commit() : sink_->Abort(status); + } + + private: + std::shared_ptr sink_; + }; + + std::shared_ptr CreateDoPutListener( + const arrow::flight::ServerCallContext& context) override { + return std::make_shared(catalog_->NewSink()); + } + +Only one acknowledgement is in flight at a time: ``WriteMetadataAsync()`` +returns an already-failed ``Invalid`` future while the previous one is still +being written. To reject an upload from the server side (a quota, a bad +batch), call :func:`arrow::flight::AsyncFlightDataListener::Cancel` from any +thread. + +Reacting to Cancellation +------------------------ + +When the client cancels, its deadline expires, or the connection drops, the +transport finishes the RPC without waiting for the handler's pending future, +and calls ``Close()`` on a DoGet stream. To stop the work behind a pending +future promptly instead of polling +:func:`arrow::flight::ServerCallContext::is_cancelled`, register a callback +with :func:`arrow::flight::AsyncGenericFlightServerBase::AddCancelCallback`: + +.. code-block:: cpp + + arrow::Future> DoGetAsync( + const arrow::flight::ServerCallContext& context, + const arrow::flight::Ticket& request) override { + std::shared_ptr query = catalog_->Start(request); + ARROW_RETURN_NOT_OK(AddCancelCallback(context, [query] { query->Cancel(); })); + return query->Ready().Then( + [query]() -> std::shared_ptr { + return std::make_shared(query); + }); + } + +The callback runs once, on the transport thread that sees the cancellation, +after the RPC's status is settled. If the RPC is already cancelled, it runs +before ``AddCancelCallback()`` returns. It never runs when the RPC ends +normally. It must not block. + +Threading +--------- + +The transport has no thread or executor of its own. Its continuation on each +future a handler, stream or listener returns runs on whichever thread +completes that future, or inline on the gRPC thread that called you if the +future is already complete. That continuation issues the RPC's next +operation: the next write, the next read, or the end of the RPC. You choose +the thread that issues the RPC's I/O by choosing where you complete your +futures: + +.. code-block:: cpp + + arrow::Future> NextAsync() override { + // Produced on io_executor_, so the next write is issued from that thread. + return arrow::DeferNotOk(io_executor_->Submit([this] { return ProduceNext(); })); + } + +Complete futures promptly and never block inside a handler: a blocked gRPC +callback thread stalls every other RPC served by it. + +Two complete programs ship with the C++ examples: +``cpp/examples/arrow/flight_async_generic_doget_example.cc`` and +``cpp/examples/arrow/flight_async_generic_doput_example.cc``. How the +transport is built is described in :ref:`cpp-flight-async-internals`. + Using the Flight Client ======================= diff --git a/docs/source/developers/cpp/flight.rst b/docs/source/developers/cpp/flight.rst new file mode 100644 index 000000000000..65c50af890e8 --- /dev/null +++ b/docs/source/developers/cpp/flight.rst @@ -0,0 +1,174 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +.. default-domain:: cpp +.. highlight:: cpp + +.. _cpp-flight-async-internals: + +========================================= +Developing the Asynchronous Flight Server +========================================= + +This page explains how the asynchronous gRPC Flight server +(:class:`arrow::flight::AsyncGenericFlightServerBase`) is built, for +developers changing it. The user-facing guide is :ref:`cpp-flight-async-server`. + +Building and layout +=================== + +The server uses gRPC's generic callback API, which exists from gRPC 1.65. +``transport/grpc/async_grpc_compat.h`` defines +``ARROW_FLIGHT_HAS_ASYNC_SERVER``. CMake and Meson compile the +``async_grpc_*.cc`` sources and the ``arrow-flight-async-grpc-test`` binary +only for gRPC 1.65 or later. With an older gRPC, the transport still registers +an async factory, and that factory answers ``NotImplemented`` so ``Init()`` +reports the reason. + +All sources live in ``cpp/src/arrow/flight/transport/grpc/``: + +================================== =========================================== +File Contents +================================== =========================================== +``async_grpc_service.cc`` ``AsyncGenericFlightService``: maps a method + name to a reactor +``async_grpc_service_internal.h`` ``AsyncCallContext``, ``AsyncReactorBase``, + the unary reactor +``async_grpc_streaming.cc`` ListActions, DoAction, ListFlights +``async_grpc_do_get.cc`` DoGet +``async_grpc_do_put.cc`` DoPut +``async_grpc_exchange*.cc`` DoExchange: reactor, reader, writer +``async_grpc_handshake.cc`` Handshake +================================== =========================================== + +``arrow/flight/server_async.h`` declares the public handler interfaces, and +``arrow/flight/flight_data_decoder.h`` declares the DoPut listener, its +metadata writer and the push decoder that DoPut and DoExchange share. + +Dispatch +======== + +``AsyncGenericFlightService::CreateReactor()`` runs once per RPC on a gRPC +thread. It builds the call's ``AsyncCallContext`` (running authentication and +server middleware through ``GrpcServerCallContextHelper``), then creates the +reactor for the method. An unknown method, a Handshake with no handshake hook, +and a DoPut whose ``CreateDoPutListener()`` returns ``nullptr`` get a reactor +that only finishes the call as ``UNIMPLEMENTED``. Every other reactor derives +from ``AsyncReactorBase``; a handler the server class does not override answers +``NotImplemented`` through that reactor. + +Reactor lifetime +================ + +gRPC forbids deleting a reactor before its ``OnDone()``, but continuations on +application futures may still run after ``OnDone()``. So ``AsyncReactorBase`` +counts references (``refs_``): + +* The RPC holds the first reference, and ``OnDone()`` releases it. +* Every continuation that may run after ``OnDone()`` captures a + ``HoldToken`` (``token = hold()``). This is a ``std::unique_ptr`` whose + deleter releases one reference. +* The last release deletes the reactor. + +Never call ``release()`` on a ``HoldToken``: that gives up the pointer without +dropping the reference, and the reactor leaks. ``hold()`` is ``[[nodiscard]]``. + +A ``HoldToken`` keeps the reactor alive, but not the gRPC call. That is why the +reactor never starts I/O on a bare ``finished()`` check. + +Finishing and writing +===================== + +* ``FinishOnce(status)`` finishes the call once: the first status wins, and + later calls are no-ops. It runs the middleware's ``CallCompleted()`` first, + outside ``write_mutex_``, because that is application code that may call + back into the RPC. Then it calls ``Finish()`` under ``write_mutex_``. +* ``WriteIfLive(fn)`` runs ``fn`` (a ``StartWrite()`` or ``StartRead()``) + under the same mutex, only while the call is unfinished. Either the + operation is issued before ``Finish()``, or it is not issued at all. + +Every continuation that starts I/O goes through ``WriteIfLive()``. + +Cancellation +============ + +``AsyncReactorBase::OnCancel()`` is ``final`` and does two things in order: + +1. ``TearDownOnCancel()``: the reactor's own teardown, which settles the + status with ``FinishOnce()`` and then unwinds (DoGet closes its stream, + DoExchange wakes its reader and writer, DoPut reports ``OnFinish()``). +2. ``AsyncCallContext::RunCancelCallbacks()``: runs the callbacks handlers + registered through + :func:`arrow::flight::AsyncGenericFlightServerBase::AddCancelCallback`. + +The callback registry lives behind a ``std::unique_ptr`` inside +``AsyncCallContext``, so the context stays movable from ``CreateReactor()`` +into the reactor, and registering never touches gRPC state. +``AddCancelCallback()`` finds the registry with a ``dynamic_cast`` from +``ServerCallContext``, and refuses (``Invalid``) contexts that the async +server did not create. + +Per-method notes +================ + +DoGet + ``stream_mutex_`` guards the stream, ``closed_`` and ``wrote_schema_``. + Each ``GetSchemaPayloadAsync()`` / ``NextAsync()`` demand is made under it, + and ``CloseStreamOnce()`` claims the stream under it. So ``Close()`` never + overlaps a demand, and runs exactly once. A stream that arrives after + cancellation is closed by whichever path sees it last. + +DoPut + The reactor is the listener's + ``internal::FlightDataListenerTransport``. It is published with + ``SetTransport(listener, this)`` and retired with + ``SetTransport(listener, nullptr)`` in ``OnDone()``, under the listener's + mutexes. ``AsyncFlightDataListener::Cancel()`` pins it with ``Hold()`` + under that mutex before calling in. One PutResult write slot is shared by + ``WriteMetadataAsync()`` and the final acknowledgement, and + ``OnWriteDone()`` completes the pending metadata future. ``upload_ending_`` + makes the first ending (client, server cancel, failure) decide the + upload's status. + +DoExchange + ``ExchangeReader`` and ``ExchangeWriter`` reach the reactor through plain + back-pointers guarded by their own mutex. ``OnDone()`` detaches both + before releasing the RPC's reference. The reader and writer take a + ``HoldToken`` under that mutex around each call into the reactor. + +Testing +======= + +``arrow-flight-async-grpc-test`` holds the transport tests, with shared +fixtures in ``async_grpc_test_fixtures.h``: + +* ``StopClientAndServer()`` closes the client and stops the server. +* ``HandBuiltServer`` / ``StartHandBuilt()`` serve a base on a hand-built + ``grpc::ServerBuilder``. +* ``CallDoPutRaw()`` sends FlightData the client would refuse to build. +* ``WaitFor()`` polls a condition with a bound. + +Tests that end an RPC early (cancellation, deadlines, parked handlers) record +``AsyncReactorBase::live_reactors()`` at the start and finish with +``ExpectNoReactorLeak(before)``: a reactor still alive once the server has +stopped is a leaked reference. Run lifetime-sensitive changes repeatedly, for +example: + +.. code-block:: shell + + arrow-flight-async-grpc-test --gtest_repeat=20 \ + --gtest_filter='*DoExchange*:*Cancel*:*PendingHandlerFinishes*' diff --git a/docs/source/developers/cpp/index.rst b/docs/source/developers/cpp/index.rst index b95cc69ea654..cbcbf1866a67 100644 --- a/docs/source/developers/cpp/index.rst +++ b/docs/source/developers/cpp/index.rst @@ -32,3 +32,4 @@ C++ Development fuzzing compute acero + flight From cc5de48f4cba2d38e64a838669b447809818b8db Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:43:16 +0100 Subject: [PATCH 24/25] [C++][FlightRPC] Simplify the async Flight server internals Remove the machinery the async transports carry but do not need: - AsyncFlightMessageDecoder: drop Consume(std::shared_ptr) and schema(). Nothing calls them; the transports drive the decoder through Consume(internal::FlightData). Keep the class and its export for the internals test, and document that the transports drive it rather than the applications. - AsyncFlightDataListener::Cancel() returns Status instead of an always-finished Future<>, and futures that carry a plain status are returned as one. - Handshake goes through AsyncGenericFlightServerBase::Handshake() instead of a separate HandshakeFn, so the reactor needs no per-call handler plumbing. - FlightStreamChunk loses its two-argument constructor, QueuedMessage and the test helpers lose their forwarding constructors and accessors. - Share GrpcServerCallContextHelper construction between the transport and the hand-built test servers instead of duplicating it. Run the DoGet example's per-payload task on Arrow's IO thread pool: the simulated delay and the serialization of the wrapped synchronous stream are blocking work, so they no longer occupy a CPU-pool thread. The API reference stops listing AsyncFlightMessageDecoder. --- .../flight_async_generic_doget_example.cc | 247 +++++------------- .../flight_async_generic_doput_example.cc | 23 +- cpp/src/arrow/flight/flight_data_decoder.cc | 78 ++---- cpp/src/arrow/flight/flight_data_decoder.h | 32 +-- cpp/src/arrow/flight/flight_internals_test.cc | 94 ++----- cpp/src/arrow/flight/server_async.cc | 26 +- .../transport/grpc/async_grpc_do_get_test.cc | 15 +- .../transport/grpc/async_grpc_do_put.cc | 28 +- .../transport/grpc/async_grpc_do_put_test.cc | 35 +-- .../transport/grpc/async_grpc_exchange.cc | 3 +- .../grpc/async_grpc_exchange_internal.h | 6 +- .../grpc/async_grpc_exchange_reader.cc | 25 +- .../grpc/async_grpc_exchange_test.cc | 12 +- .../grpc/async_grpc_exchange_writer.cc | 42 ++- .../transport/grpc/async_grpc_handshake.cc | 16 +- .../grpc/async_grpc_handshake_test.cc | 26 +- .../transport/grpc/async_grpc_service.cc | 34 +-- .../transport/grpc/async_grpc_service.h | 10 +- .../grpc/async_grpc_service_internal.h | 39 +-- .../transport/grpc/async_grpc_service_test.cc | 66 +---- .../transport/grpc/async_grpc_streaming.cc | 27 +- .../grpc/async_grpc_streaming_test.cc | 9 +- .../transport/grpc/async_grpc_test_fixtures.h | 96 +++---- .../transport/grpc/async_grpc_unary_test.cc | 3 +- .../flight/transport/grpc/grpc_server.cc | 7 +- .../transport/grpc/grpc_server_internal.h | 6 - .../transport/grpc/serialization_internal.cc | 23 -- .../transport/grpc/serialization_internal.h | 22 -- cpp/src/arrow/flight/types.cc | 3 - cpp/src/arrow/flight/types.h | 2 - docs/source/cpp/api/flight.rst | 4 - docs/source/developers/cpp/flight.rst | 8 +- 32 files changed, 309 insertions(+), 758 deletions(-) diff --git a/cpp/examples/arrow/flight_async_generic_doget_example.cc b/cpp/examples/arrow/flight_async_generic_doget_example.cc index ce1df2a841d3..58ca04d9e5fd 100644 --- a/cpp/examples/arrow/flight_async_generic_doget_example.cc +++ b/cpp/examples/arrow/flight_async_generic_doget_example.cc @@ -18,19 +18,20 @@ #include #include -#include #include #include #include #include #include #include -#include #include #include +#include #include #include +#include +#include #include // Serve DoGet over the async generic Flight transport. @@ -41,12 +42,13 @@ // of the payloads themselves, so the gRPC callback thread serving the RPC // never waits for data. // -// DelayedAsyncStream shows what that is for: each batch is produced on the -// stream's own worker thread, --batch_delay_ms after the transport asks for -// it. The transport adds a callback to the future and returns, so the delay -// never blocks a gRPC callback thread (which would stall every RPC multiplexed -// onto it). With --batch_delay_ms=0 the delay is gone and this is a plain -// fast DoGet. +// DelayedAsyncStream shows what that is for: each batch is produced on Arrow's +// IO thread pool, --batch_delay_ms after the transport asks for it. The +// transport adds a callback to the future and returns, so the delay never +// blocks a gRPC callback thread (which would stall every RPC multiplexed onto +// it). The transport continues on the thread that completes the future, so +// each batch's write is issued from that IO thread too. With +// --batch_delay_ms=0 the delay is gone and this is a plain fast DoGet. // // Any ticket streams the same batches. Every other RPC - GetFlightInfo // included - answers UNIMPLEMENTED, so call DoGet directly instead of @@ -61,191 +63,83 @@ DEFINE_int32(port, 0, "Port to listen on (0 picks a free port)"); DEFINE_int32(batches, 5, "Number of batches in the DoGet stream"); DEFINE_int32(batch_delay_ms, 50, - "Delay before each batch, spent on the stream's own worker thread"); + "Delay before each batch, spent on Arrow's IO thread pool"); DEFINE_bool(demo, false, "Run the DoGet self-check against the server, then exit"); namespace flight = arrow::flight; -/// \brief A reader with `num_batches` batches of three int64 values. -class BatchReader : public arrow::RecordBatchReader { - public: - BatchReader(std::shared_ptr schema, int32_t num_batches) - : schema_(std::move(schema)), num_batches_(num_batches) {} - - std::shared_ptr schema() const override { return schema_; } - - arrow::Status ReadNext(std::shared_ptr* batch) override { - if (next_batch_ >= num_batches_) { - *batch = nullptr; // end of stream - return arrow::Status::OK(); - } - arrow::Int64Builder builder; - for (int64_t i = 0; i < 3; i++) { - ARROW_RETURN_NOT_OK(builder.Append(3 * next_batch_ + i)); - } - ARROW_ASSIGN_OR_RAISE(auto array, builder.Finish()); - *batch = arrow::RecordBatch::Make(schema_, array->length(), {array}); - ++next_batch_; - return arrow::Status::OK(); - } - - private: - std::shared_ptr schema_; - int32_t num_batches_; - int32_t next_batch_ = 0; -}; - -/// \brief The stream's worker state. Heap-allocated and shared with the worker -/// thread: the transport deletes the stream as soon as a payload completes -/// (the end-of-stream payload is completed *on the worker*), so the worker must -/// not touch the stream object after that. Everything it needs lives here. -struct WorkerState { - std::unique_ptr stream; - int32_t delay_ms = 0; - std::mutex mutex; - std::condition_variable ready; - bool closed = false; - /// The payload the transport is waiting for, if one was requested. - arrow::Future> pending; - bool has_pending = false; - /// Whether the wrapped stream has been closed, so Close() closes it once. - bool stream_closed = false; - std::thread thread; -}; - -/// \brief The worker: wait for a payload request, sleep delay_ms, produce it, -/// complete the future the transport holds. -void RunWorker(std::shared_ptr state) { - std::unique_lock lock(state->mutex); - while (!state->closed) { - state->ready.wait(lock, [&] { return state->has_pending || state->closed; }); - if (state->closed) { - break; - } - // Take the request: after this, only this thread resolves the future. - state->has_pending = false; - arrow::Future> future = state->pending; - lock.unlock(); - std::this_thread::sleep_for(std::chrono::milliseconds(state->delay_ms)); - lock.lock(); - const bool cancelled = state->closed; - lock.unlock(); - // Runs the transport's callback inline, on this thread: it serializes the - // payload, and on end of stream it calls Close() (hence the lock dance). - if (cancelled) { - future.MarkFinished(arrow::Status::Cancelled()); - } else { - auto payload = state->stream->Next(); - if (!payload.ok()) { - future.MarkFinished(payload.status()); - } else if (payload->ipc_message.metadata == nullptr) { - // End of stream: the async interface reports it as a nullopt. - future.MarkFinished(std::nullopt); - } else { - future.MarkFinished(std::move(*payload)); - } - } - lock.lock(); - } - // The worker is the only thread that touched the wrapped stream, so it is the - // one that closes it (see Close(), which only asks it to stop). - lock.unlock(); - ARROW_WARN_NOT_OK(state->stream->Close(), "closing the DoGet stream failed"); - lock.lock(); - state->ready.notify_all(); -} - /// \brief The stream the async transport pulls from, with a sleep per batch. /// -/// NextAsync() hands the transport a future and returns at once, the delay and -/// the serialization of the wrapped synchronous stream happen on this stream's -/// own worker thread. The transport keeps one payload in flight at a time -- -/// it asks for the next only once the write completed -- so a single worker is -/// enough. -/// -/// One thread per stream. A server with many slow streams may prefer a shared -/// timer or thread pool, swapping WorkerState's thread for a task on that pool. +/// NextAsync() hands the transport a future and returns at once: the delay and +/// the serialization of the wrapped synchronous stream run as a task on Arrow's +/// IO thread pool, which is meant for blocking work, so the sleep parks an IO +/// thread instead of tying up a gRPC callback thread or a thread that other RPCs +/// share. The transport keeps one payload in flight at a time -- it asks for +/// the next only once the write completed -- so at most one task reads the +/// stream at a time. class DelayedAsyncStream : public flight::AsyncFlightDataStream { public: - DelayedAsyncStream(std::unique_ptr stream, int32_t delay_ms) - : state_(std::make_shared()) { - state_->stream = std::move(stream); - state_->delay_ms = delay_ms; - std::shared_ptr state = state_; - state_->thread = std::thread([state] { RunWorker(state); }); - } - - ~DelayedAsyncStream() override { - Stop(); - if (state_->thread.joinable()) { - if (state_->thread.get_id() == std::this_thread::get_id()) { - // Deleted from inside its own worker (the transport releases the - // stream as a payload completes): joining would deadlock. - state_->thread.detach(); - } else { - state_->thread.join(); - } - } - } + DelayedAsyncStream(std::shared_ptr reader, int32_t delay_ms) + : state_(std::make_shared(std::move(reader), delay_ms)) {} arrow::Future GetSchemaPayloadAsync() override { - // The schema is ready now: no delay, no worker. This runs before the first - // NextAsync(), so it cannot overlap the worker's use of the stream. - return arrow::Future::MakeFinished( - state_->stream->GetSchemaPayload()); + // The schema is ready now: no delay, no task. + return state_->stream.GetSchemaPayload(); } arrow::Future> NextAsync() override { - arrow::Future> future = - arrow::Future>::Make(); - { - std::lock_guard lock(state_->mutex); - if (state_->closed) { - future.MarkFinished(arrow::Status::Cancelled()); - return future; - } - state_->pending = future; - state_->has_pending = true; - } - state_->ready.notify_one(); - return future; + // The task owns a reference to the state: the transport may delete this + // stream as soon as the payload completes. + return arrow::DeferNotOk(arrow::io::default_io_context().executor()->Submit( + [state = state_]() -> arrow::Result> { + std::this_thread::sleep_for(std::chrono::milliseconds(state->delay_ms)); + std::lock_guard lock(state->mutex); + if (state->closed) { + return arrow::Status::Cancelled("the DoGet stream was closed"); + } + ARROW_ASSIGN_OR_RAISE(auto payload, state->stream.Next()); + if (payload.ipc_message.metadata == nullptr) { + return std::nullopt; // end of stream + } + return payload; + })); } arrow::Status Close() override { - // Only ask the worker to stop: the wrapped stream belongs to the worker - // thread, which may be inside stream->Next() right now, and Next()/Close() - // both mutate the IPC reader. Closing from this (gRPC) thread would race - // with that call and can corrupt the stream. Stop() is also what resolves - // a payload the worker has not taken yet, so this stays non-blocking; the - // worker closes the stream after it stops producing. - Stop(); - return arrow::Status::OK(); + // Next() and Close() both touch the IPC reader, so Close() takes the lock a + // running task holds while it reads (never while it sleeps). A task still + // sleeping completes its future as Cancelled. + std::lock_guard lock(state_->mutex); + state_->closed = true; + return state_->stream.Close(); } private: - /// \brief Stop the worker, resolve a payload it has not taken yet. Idempotent. - /// Does not join: called by the transport, whose threads must not block on - /// the worker's sleep. - void Stop() { - arrow::Future> pending; - { - std::lock_guard lock(state_->mutex); - state_->closed = true; - if (state_->has_pending) { - state_->has_pending = false; - pending = state_->pending; - } - } - state_->ready.notify_all(); - if (pending.is_valid()) { - // Runs the transport's callback inline, it reacts by calling Close(). - pending.MarkFinished(arrow::Status::Cancelled()); - } - } - - std::shared_ptr state_; + struct State { + State(std::shared_ptr reader, int32_t delay_ms) + : stream(std::move(reader)), delay_ms(delay_ms) {} + flight::RecordBatchStream stream; + int32_t delay_ms; + std::mutex mutex; + bool closed = false; + }; + std::shared_ptr state_; }; +/// \brief `num_batches` batches of three int64 values each. +arrow::Result> MakeReader( + const std::shared_ptr& schema, int32_t num_batches) { + arrow::RecordBatchVector batches; + for (int64_t b = 0; b < num_batches; b++) { + arrow::Int64Builder builder; + ARROW_RETURN_NOT_OK( + builder.AppendValues(std::vector{3 * b, 3 * b + 1, 3 * b + 2})); + ARROW_ASSIGN_OR_RAISE(auto array, builder.Finish()); + batches.push_back(arrow::RecordBatch::Make(schema, array->length(), {array})); + } + return arrow::RecordBatchReader::Make(std::move(batches), schema); +} + /// \brief The server: DoGetAsync is the only handler overridden, so every other /// RPC answers UNIMPLEMENTED. class ExampleServer : public flight::AsyncGenericFlightServerBase { @@ -254,14 +148,11 @@ class ExampleServer : public flight::AsyncGenericFlightServerBase { const flight::ServerCallContext& context, const flight::Ticket& request) override { std::cout << "DoGet: ticket=" << request.ticket << std::endl; auto schema = arrow::schema({arrow::field("value", arrow::int64())}); - auto reader = std::make_shared(schema, FLAGS_batches); - std::shared_ptr stream = - std::make_shared( - std::make_unique(reader), FLAGS_batch_delay_ms); + ARROW_ASSIGN_OR_RAISE(auto reader, MakeReader(schema, FLAGS_batches)); // The stream is ready here, but its payloads are not: the transport will // wait on the futures NextAsync() returns. - return arrow::Future>::MakeFinished( - std::move(stream)); + return std::shared_ptr( + std::make_shared(std::move(reader), FLAGS_batch_delay_ms)); } }; diff --git a/cpp/examples/arrow/flight_async_generic_doput_example.cc b/cpp/examples/arrow/flight_async_generic_doput_example.cc index 850f284d4728..4094f63018bb 100644 --- a/cpp/examples/arrow/flight_async_generic_doput_example.cc +++ b/cpp/examples/arrow/flight_async_generic_doput_example.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -63,18 +64,6 @@ namespace flight = arrow::flight; constexpr int kNumBatches = 3; constexpr int64_t kRowsPerBatch = 4; - -/// \brief Make a one-column batch of `num_rows` int64 values starting at `value`. -arrow::Result> MakeInt64Batch( - const std::shared_ptr& schema, int64_t value, int64_t num_rows) { - arrow::Int64Builder builder; - for (int64_t i = 0; i < num_rows; i++) { - ARROW_RETURN_NOT_OK(builder.Append(value + i)); - } - ARROW_ASSIGN_OR_RAISE(auto array, builder.Finish()); - return arrow::RecordBatch::Make(schema, array->length(), {array}); -} - /// \brief The AsyncFlightDataListener serving one DoPut RPC: it records what it is /// handed as the upload streams in. The counters are atomics because the /// callbacks run on gRPC threads. @@ -168,10 +157,12 @@ arrow::Status RunDemo(int port) { auto schema = arrow::schema({arrow::field("value", arrow::int64())}); ARROW_ASSIGN_OR_RAISE( auto put, client->DoPut(flight::FlightDescriptor::Command("upload"), schema)); - for (int i = 0; i < kNumBatches; i++) { - ARROW_ASSIGN_OR_RAISE(auto batch, - MakeInt64Batch(schema, i * kRowsPerBatch, kRowsPerBatch)); - ARROW_RETURN_NOT_OK(put.writer->WriteRecordBatch(*batch)); + for (int64_t i = 0; i < kNumBatches; i++) { + arrow::Int64Builder builder; + ARROW_RETURN_NOT_OK(builder.AppendValues(std::vector(kRowsPerBatch, i))); + ARROW_ASSIGN_OR_RAISE(auto array, builder.Finish()); + ARROW_RETURN_NOT_OK(put.writer->WriteRecordBatch( + *arrow::RecordBatch::Make(schema, kRowsPerBatch, {array}))); } ARROW_RETURN_NOT_OK(put.writer->DoneWriting()); return put.writer->Close(); diff --git a/cpp/src/arrow/flight/flight_data_decoder.cc b/cpp/src/arrow/flight/flight_data_decoder.cc index 5a9942d59104..19d65870a444 100644 --- a/cpp/src/arrow/flight/flight_data_decoder.cc +++ b/cpp/src/arrow/flight/flight_data_decoder.cc @@ -41,7 +41,7 @@ class FlightDataMessageReader : public ipc::MessageReader { /// \param message The message to queue. /// \param app_metadata The application metadata associated with the message. void Push(std::unique_ptr message, std::shared_ptr app_metadata) { - queue_.emplace_back(std::move(message), std::move(app_metadata)); + queue_.push_back({std::move(message), std::move(app_metadata)}); } /// \brief Check if there is a record batch message queued. @@ -85,10 +85,6 @@ class FlightDataMessageReader : public ipc::MessageReader { private: /// \brief Queue of messages that have been pushed but not yet read. struct QueuedMessage { - QueuedMessage() = default; - QueuedMessage(std::unique_ptr message, - std::shared_ptr app_metadata) - : message(std::move(message)), app_metadata(std::move(app_metadata)) {} std::unique_ptr message; std::shared_ptr app_metadata; }; @@ -143,11 +139,18 @@ class AsyncFlightMessageDecoder::AsyncFlightMessageDecoderImpl { } } - std::shared_ptr schema() const { - return batch_reader_ ? batch_reader_->schema() : nullptr; + private: + /// \brief Hand a message's app metadata to the listener as a metadata-only + /// chunk, unless it is empty. + Future<> DeliverMetadata(std::shared_ptr app_metadata) { + if (app_metadata == nullptr || app_metadata->size() == 0) { + return Status::OK(); + } + FlightStreamChunk chunk; + chunk.app_metadata = std::move(app_metadata); + return listener_->OnNext(std::move(chunk)); } - private: /// \brief Decode a FlightData message whose descriptor (if any) was already /// reported to the listener. /// \param data The Flight data to consume after the descriptor has been reported. @@ -159,12 +162,7 @@ class AsyncFlightMessageDecoder::AsyncFlightMessageDecoderImpl { Future<> ConsumeDataAfterDescriptor(internal::FlightData data) { if (!data.metadata) { // Metadata-only message: no IPC content, just Flight app_metadata. - if (data.app_metadata && data.app_metadata->size() > 0) { - FlightStreamChunk chunk; - chunk.app_metadata = std::move(data.app_metadata); - return listener_->OnNext(std::move(chunk)); - } - return Status::OK(); + return DeliverMetadata(std::move(data.app_metadata)); } ARROW_ASSIGN_OR_RAISE(auto message, data.OpenMessage()); @@ -184,30 +182,19 @@ class AsyncFlightMessageDecoder::AsyncFlightMessageDecoderImpl { message_reader_->Push(std::move(message), std::move(data.app_metadata)); if (!message_reader_->HasRecordBatch()) { - auto app_metadata = message_reader_->TakeLastAppMetadata(); - if (app_metadata && app_metadata->size() > 0) { - FlightStreamChunk chunk; - chunk.app_metadata = std::move(app_metadata); - return listener_->OnNext(std::move(chunk)); - } - return arrow::Future<>::MakeFinished(Status::OK()); + return DeliverMetadata(message_reader_->TakeLastAppMetadata()); } std::shared_ptr batch; RETURN_NOT_OK(batch_reader_->ReadNext(&batch)); - auto app_metadata = message_reader_->ReadAppMetadata(); - - if (batch) { - FlightStreamChunk chunk{std::move(batch), std::move(app_metadata)}; - return listener_->OnNext(std::move(chunk)); + if (!batch) { + // This has to be a Dictionary batch. + return DeliverMetadata(message_reader_->ReadAppMetadata()); } - - // This has to be a Dictionary batch. - if (app_metadata && app_metadata->size() > 0) { - FlightStreamChunk chunk{nullptr, std::move(app_metadata)}; - return listener_->OnNext(std::move(chunk)); - } - return arrow::Future<>::MakeFinished(Status::OK()); + FlightStreamChunk chunk; + chunk.data = std::move(batch); + chunk.app_metadata = message_reader_->ReadAppMetadata(); + return listener_->OnNext(std::move(chunk)); } std::shared_ptr listener_; @@ -223,19 +210,10 @@ AsyncFlightMessageDecoder::AsyncFlightMessageDecoder( AsyncFlightMessageDecoder::~AsyncFlightMessageDecoder() = default; -Future<> AsyncFlightMessageDecoder::Consume(std::shared_ptr buffer) { - ARROW_ASSIGN_OR_RAISE(auto data, internal::DeserializeFlightData(buffer)); - return impl_->ConsumeData(std::move(data)); -} - Future<> AsyncFlightMessageDecoder::Consume(internal::FlightData data) { return impl_->ConsumeData(std::move(data)); } -std::shared_ptr AsyncFlightMessageDecoder::schema() const { - return impl_->schema(); -} - // --- AsyncFlightDataListener: transport state and the terminal callbacks --- AsyncFlightDataListener::AsyncFlightDataListener() = default; @@ -250,9 +228,9 @@ Future<> AsyncFlightDataWriter::WriteMetadataAsync(const Buffer& app_metadata) { // private write_mutex_, so there is no lock-order cycle. std::lock_guard lock(writer_mutex_); if (transport_ == nullptr) { - return Future<>::MakeFinished(Status::Invalid( + return Status::Invalid( "no upload in flight to write metadata to: the writer is only usable " - "while the upload's RPC is running")); + "while the upload's RPC is running"); } return transport_->WriteMetadataAsync(app_metadata); } @@ -276,34 +254,34 @@ void FlightDataListenerTransport::SetTransport( Future<> FlightDataListenerTransport::ReportFinish( const std::shared_ptr& listener, Status status) { if (listener == nullptr) { - return Future<>::MakeFinished(); + return Status::OK(); } // Report exactly once: the first ending wins. bool expected = false; if (!listener->finished_.compare_exchange_strong(expected, true)) { - return Future<>::MakeFinished(); + return Status::OK(); } return listener->OnFinish(std::move(status)); } } // namespace internal -Future<> AsyncFlightDataListener::Cancel(Status status) { +Status AsyncFlightDataListener::Cancel(Status status) { internal::FlightDataListenerTransport* transport; { std::lock_guard lock(transport_mutex_); transport = transport_; if (transport == nullptr) { - return arrow::Future<>::MakeFinished(Status::Invalid( + return Status::Invalid( "no upload in flight to cancel: Cancel() must be called while the " - "upload's RPC is running")); + "upload's RPC is running"); } transport->Hold(); } // Unlocked, so OnFinish() may re-enter Cancel() without deadlocking. transport->CancelUpload(std::move(status)); transport->ReleaseHold(); - return arrow::Future<>::MakeFinished(Status::OK()); + return Status::OK(); } } // namespace arrow::flight diff --git a/cpp/src/arrow/flight/flight_data_decoder.h b/cpp/src/arrow/flight/flight_data_decoder.h index e26f614bfa27..7e58cf8bc9ea 100644 --- a/cpp/src/arrow/flight/flight_data_decoder.h +++ b/cpp/src/arrow/flight/flight_data_decoder.h @@ -186,7 +186,7 @@ class ARROW_FLIGHT_EXPORT AsyncFlightDataListener : public ipc::Listener { /// /// \return OK when the cancel was handed to the transport, or Invalid when /// there is no upload in flight. - Future<> Cancel(Status status); + Status Cancel(Status status); private: /// The transport sets and clears the state through @@ -205,11 +205,12 @@ class ARROW_FLIGHT_EXPORT AsyncFlightDataListener : public ipc::Listener { internal::FlightDataListenerTransport* transport_ = nullptr; }; -/// \brief Push style stream decoder that turns raw arrow Buffers into +/// \brief Push style stream decoder that turns FlightData messages into /// FlightStreamChunks. /// -/// This class decodes Apache Arrow Flight data format from arrow::Buffer -/// and fires events on the provided AsyncFlightDataListener. +/// The Flight transports drive it, to feed each incoming message to the RPC's +/// AsyncFlightDataListener: applications implement the listener, they do not +/// construct a decoder. class ARROW_FLIGHT_EXPORT AsyncFlightMessageDecoder { public: /// \brief Construct an AsyncFlightMessageDecoder with the given listener and IPC read @@ -222,29 +223,16 @@ class ARROW_FLIGHT_EXPORT AsyncFlightMessageDecoder { ipc::IpcReadOptions options = ipc::IpcReadOptions::Defaults()); ~AsyncFlightMessageDecoder(); - /// \brief Decode one FlightData message directly from a buffer. - /// - /// Fires listener->OnSchemaDecoded() on the first message containing - /// a schema, listener->OnNext() for each subsequent record batch, - /// metadata-only message or dictionary batch. - /// - /// \param[in] buffer a raw buffer directly from the transport. Example - /// the arrow::Buffer extracted from the grpc::ByteBuffer from the gRPC transport. - /// \return Status - Future<> Consume(std::shared_ptr buffer); - /// \brief Decode one FlightData message already deserialized by the transport. /// - /// Same as Consume(Buffer) but avoids a serialization round-trip for transports - /// that hand out an internal::FlightData directly (the gRPC client reads FlightData - /// straight into internal::FlightData). + /// Fires listener->OnDescriptor() for a message carrying a descriptor, + /// listener->OnSchemaDecoded() on the first message containing a schema, and + /// listener->OnNext() for each subsequent record batch, metadata-only message or + /// dictionary batch carrying app metadata. /// \internal This method is intended for internal use within the Arrow library. + /// \return A future completed with the listener's status for this message. Future<> Consume(internal::FlightData data); - /// \brief The decoded schema. - /// \return The decoded schema, or nullptr if no schema has been decoded yet. - std::shared_ptr schema() const; - private: class AsyncFlightMessageDecoderImpl; std::unique_ptr impl_; diff --git a/cpp/src/arrow/flight/flight_internals_test.cc b/cpp/src/arrow/flight/flight_internals_test.cc index 6061e120724f..152f6e7b2a65 100644 --- a/cpp/src/arrow/flight/flight_internals_test.cc +++ b/cpp/src/arrow/flight/flight_internals_test.cc @@ -38,7 +38,6 @@ #include "arrow/flight/test_util.h" #include "arrow/flight/transport.h" #include "arrow/flight/transport/grpc/async_grpc_compat.h" -#include "arrow/flight/transport/grpc/serialization_internal.h" #include "arrow/flight/transport/grpc/util_internal.h" #include "arrow/flight/types.h" #include "arrow/ipc/reader.h" @@ -744,43 +743,6 @@ TEST(GrpcTransport, FlightDataDeserialize) { #endif } -TEST(GrpcTransport, BytesFromBufferConcatenatesSlices) { - // gRPC hands the caller whatever slice layout the message arrived in: the - // loop has to stitch the slices back together, embedded NULs included. - const std::string first("ab\0c", 4); - const std::string empty; - const std::string third("\xff\x01", 2); - std::vector<::grpc::Slice> slices = {::grpc::Slice(first), ::grpc::Slice(empty), - ::grpc::Slice(third)}; - ::grpc::ByteBuffer buffer(slices.data(), slices.size()); - - ASSERT_OK_AND_ASSIGN(auto bytes, - flight::transport::grpc::BytesFromBuffer(buffer, "unused")); - ASSERT_EQ(6, static_cast(bytes.size())); - ASSERT_EQ(first + third, bytes); -} - -TEST(GrpcTransport, BytesFromBufferEmptyBuffer) { - // A valid buffer holding no slices is an empty message, not an error. - ::grpc::ByteBuffer buffer(nullptr, 0); - ASSERT_OK_AND_ASSIGN(auto bytes, - flight::transport::grpc::BytesFromBuffer(buffer, "unused")); - ASSERT_EQ("", bytes); -} - -TEST(GrpcTransport, BytesFromBufferInvalidBufferIsAnInternalError) { - // A default-constructed buffer has no underlying grpc_byte_buffer: the failure - // must keep the caller's message and the transport's Internal detail. - ::grpc::ByteBuffer buffer; - auto result = flight::transport::grpc::BytesFromBuffer(buffer, "no bytes"); - ASSERT_FALSE(result.ok()) << "a buffer with no underlying message cannot be dumped"; - ASSERT_TRUE(result.status().IsIOError()) << result.status(); - ASSERT_NE(std::string::npos, result.status().ToString().find("no bytes")); - auto detail = FlightStatusDetail::UnwrapStatus(result.status()); - ASSERT_NE(nullptr, detail); - ASSERT_EQ(FlightStatusCode::Internal, detail->code()); -} - // ---------------------------------------------------------------------- // Transport-agnostic serialization roundtrip tests @@ -854,12 +816,13 @@ TEST(FlightSerialization, RoundtripMetadataOnly) { namespace { -// Serialize one FlightPayload into the single contiguous buffer a transport -// hands to AsyncFlightMessageDecoder::Consume (exactly one FlightData message). -::arrow::Result> DecoderTestWireBuffer( +// Serialize one FlightPayload into the single FlightData message a transport +// hands to AsyncFlightMessageDecoder::Consume, deserialized as it would be. +::arrow::Result DecoderTestWireBuffer( const FlightPayload& payload) { ARROW_ASSIGN_OR_RAISE(auto buffers, internal::SerializePayloadToBuffers(payload)); - return ConcatenateBuffers(buffers); + ARROW_ASSIGN_OR_RAISE(auto buffer, ConcatenateBuffers(buffers)); + return internal::DeserializeFlightData(buffer); } // Captures the events AsyncFlightMessageDecoder fires, in order. @@ -915,25 +878,22 @@ TEST(AsyncFlightMessageDecoder, SchemaAndBatches) { auto listener = std::make_shared(); AsyncFlightMessageDecoder decoder(listener); - EXPECT_EQ(decoder.schema(), nullptr); // One wire message per payload: schema first, then one per record batch. ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); ASSERT_OK_AND_ASSIGN(auto schema_wire, DecoderTestWireBuffer(schema_payload)); - ASSERT_OK(decoder.Consume(schema_wire).status()); + ASSERT_OK(decoder.Consume(std::move(schema_wire)).status()); EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded")); ASSERT_EQ(listener->schemas.size(), 1); EXPECT_TRUE(listener->schemas[0]->Equals(*schema)); - ASSERT_NE(decoder.schema(), nullptr); - EXPECT_TRUE(decoder.schema()->Equals(*schema)); ASSERT_OK_AND_ASSIGN(auto payload1, stream.Next()); ASSERT_OK_AND_ASSIGN(auto payload2, stream.Next()); ASSERT_OK_AND_ASSIGN(auto wire1, DecoderTestWireBuffer(payload1)); ASSERT_OK_AND_ASSIGN(auto wire2, DecoderTestWireBuffer(payload2)); - ASSERT_OK(decoder.Consume(wire1).status()); - ASSERT_OK(decoder.Consume(wire2).status()); + ASSERT_OK(decoder.Consume(std::move(wire1)).status()); + ASSERT_OK(decoder.Consume(std::move(wire2)).status()); EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded", "OnNext", "OnNext")); @@ -954,14 +914,13 @@ TEST(AsyncFlightMessageDecoder, MetadataOnlyChunkIsAValue) { auto listener = std::make_shared(); AsyncFlightMessageDecoder decoder(listener); - ASSERT_OK(decoder.Consume(wire).status()); + ASSERT_OK(decoder.Consume(std::move(wire)).status()); EXPECT_THAT(listener->events, ::testing::ElementsAre("OnNext")); ASSERT_EQ(listener->chunks.size(), 1); EXPECT_EQ(listener->chunks[0].data, nullptr); ASSERT_NE(listener->chunks[0].app_metadata, nullptr); EXPECT_EQ(listener->chunks[0].app_metadata->ToString(), "only-app-metadata"); - EXPECT_EQ(decoder.schema(), nullptr); } TEST(AsyncFlightMessageDecoder, AppMetadataOnBatch) { @@ -975,12 +934,12 @@ TEST(AsyncFlightMessageDecoder, AppMetadataOnBatch) { ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); ASSERT_OK_AND_ASSIGN(auto schema_wire, DecoderTestWireBuffer(schema_payload)); - ASSERT_OK(decoder.Consume(schema_wire).status()); + ASSERT_OK(decoder.Consume(std::move(schema_wire)).status()); ASSERT_OK_AND_ASSIGN(auto payload, stream.Next()); payload.app_metadata = Buffer::FromString("batch-app-metadata"); ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); - ASSERT_OK(decoder.Consume(wire).status()); + ASSERT_OK(decoder.Consume(std::move(wire)).status()); EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded", "OnNext")); ASSERT_EQ(listener->chunks.size(), 1); @@ -1001,12 +960,12 @@ TEST(AsyncFlightMessageDecoder, SchemaOnlyStream) { ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(schema_payload)); - ASSERT_OK(decoder.Consume(wire).status()); + ASSERT_OK(decoder.Consume(std::move(wire)).status()); EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded")); EXPECT_TRUE(listener->chunks.empty()); - ASSERT_NE(decoder.schema(), nullptr); - EXPECT_TRUE(decoder.schema()->Equals(*schema)); + ASSERT_EQ(listener->schemas.size(), 1); + EXPECT_TRUE(listener->schemas[0]->Equals(*schema)); } TEST(AsyncFlightMessageDecoder, ListenerErrorIsPropagated) { @@ -1021,12 +980,12 @@ TEST(AsyncFlightMessageDecoder, ListenerErrorIsPropagated) { ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); ASSERT_OK_AND_ASSIGN(auto schema_wire, DecoderTestWireBuffer(schema_payload)); - ASSERT_OK(decoder.Consume(schema_wire).status()); + ASSERT_OK(decoder.Consume(std::move(schema_wire)).status()); // A listener error on a record batch must reach the caller of Consume. ASSERT_OK_AND_ASSIGN(auto payload, stream.Next()); ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); - auto status = decoder.Consume(wire).status(); + auto status = decoder.Consume(std::move(wire)).status(); EXPECT_FALSE(status.ok()); EXPECT_EQ(status.code(), StatusCode::Invalid); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("listener failed")); @@ -1035,7 +994,7 @@ TEST(AsyncFlightMessageDecoder, ListenerErrorIsPropagated) { FlightPayload metadata_only; metadata_only.app_metadata = Buffer::FromString("meta"); ASSERT_OK_AND_ASSIGN(auto meta_wire, DecoderTestWireBuffer(metadata_only)); - auto meta_status = decoder.Consume(meta_wire).status(); + auto meta_status = decoder.Consume(std::move(meta_wire)).status(); EXPECT_FALSE(meta_status.ok()); EXPECT_EQ(meta_status.code(), StatusCode::Invalid); } @@ -1056,7 +1015,7 @@ TEST(AsyncFlightMessageDecoder, FailedOpenLeavesTheDecoderUsable) { // A record batch as the first message: not a schema, so Open() fails. ASSERT_OK_AND_ASSIGN(auto batch_payload, stream.Next()); ASSERT_OK_AND_ASSIGN(auto batch_wire, DecoderTestWireBuffer(batch_payload)); - auto failed = decoder.Consume(batch_wire).status(); + auto failed = decoder.Consume(std::move(batch_wire)).status(); EXPECT_FALSE(failed.ok()) << "a first message that is not a schema must be rejected"; EXPECT_TRUE(listener->events.empty()) << "nothing may be surfaced for a rejected stream"; @@ -1067,10 +1026,10 @@ TEST(AsyncFlightMessageDecoder, FailedOpenLeavesTheDecoderUsable) { RecordBatchStream good_stream(std::move(good_reader)); ASSERT_OK_AND_ASSIGN(auto schema_payload, good_stream.GetSchemaPayload()); ASSERT_OK_AND_ASSIGN(auto schema_wire, DecoderTestWireBuffer(schema_payload)); - ASSERT_OK(decoder.Consume(schema_wire).status()); + ASSERT_OK(decoder.Consume(std::move(schema_wire)).status()); ASSERT_OK_AND_ASSIGN(auto good_batch, good_stream.Next()); ASSERT_OK_AND_ASSIGN(auto good_wire, DecoderTestWireBuffer(good_batch)); - ASSERT_OK(decoder.Consume(good_wire).status()); + ASSERT_OK(decoder.Consume(std::move(good_wire)).status()); EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded", "OnNext")); ASSERT_EQ(listener->chunks.size(), 1); @@ -1095,11 +1054,11 @@ TEST(AsyncFlightMessageDecoder, DictionaryStream) { ASSERT_OK_AND_ASSIGN(auto schema_payload, stream.GetSchemaPayload()); ASSERT_OK_AND_ASSIGN(auto schema_wire, DecoderTestWireBuffer(schema_payload)); - ASSERT_OK(decoder.Consume(schema_wire).status()); + ASSERT_OK(decoder.Consume(std::move(schema_wire)).status()); EXPECT_THAT(listener->events, ::testing::ElementsAre("OnSchemaDecoded")); - ASSERT_NE(decoder.schema(), nullptr); - EXPECT_TRUE(decoder.schema()->Equals(*schema)); + ASSERT_EQ(listener->schemas.size(), 1); + EXPECT_TRUE(listener->schemas[0]->Equals(*schema)); // The dictionary message is consumed without surfacing as a chunk of its own. int payloads = 0; @@ -1109,7 +1068,7 @@ TEST(AsyncFlightMessageDecoder, DictionaryStream) { break; // end of stream } ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); - ASSERT_OK(decoder.Consume(wire).status()); + ASSERT_OK(decoder.Consume(std::move(wire)).status()); ++payloads; } EXPECT_EQ(payloads, 2); // dictionary message + record batch @@ -1138,7 +1097,7 @@ TEST(AsyncFlightMessageDecoder, DescriptorReachesListener) { ASSERT_OK_AND_ASSIGN(auto payload, stream.GetSchemaPayload()); ASSERT_OK(internal::ToPayload(descriptor, &payload.descriptor)); ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); - ASSERT_OK(decoder.Consume(wire).status()); + ASSERT_OK(decoder.Consume(std::move(wire)).status()); EXPECT_THAT(listener->events, ::testing::ElementsAre("OnDescriptor", "OnSchemaDecoded")); @@ -1161,7 +1120,7 @@ TEST(AsyncFlightMessageDecoder, DescriptorErrorRejectsUpload) { ASSERT_OK_AND_ASSIGN(auto payload, stream.GetSchemaPayload()); ASSERT_OK(internal::ToPayload(descriptor, &payload.descriptor)); ASSERT_OK_AND_ASSIGN(auto wire, DecoderTestWireBuffer(payload)); - auto status = decoder.Consume(wire).status(); + auto status = decoder.Consume(std::move(wire)).status(); EXPECT_FALSE(status.ok()); EXPECT_EQ(status.code(), StatusCode::Invalid); @@ -1169,7 +1128,6 @@ TEST(AsyncFlightMessageDecoder, DescriptorErrorRejectsUpload) { // The upload is rejected before the schema is decoded. EXPECT_THAT(listener->events, ::testing::ElementsAre("OnDescriptor")); - EXPECT_EQ(decoder.schema(), nullptr); } // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/flight/server_async.cc b/cpp/src/arrow/flight/server_async.cc index 590f9c61dfde..53ffc7548a65 100644 --- a/cpp/src/arrow/flight/server_async.cc +++ b/cpp/src/arrow/flight/server_async.cc @@ -93,7 +93,8 @@ Status AsyncGenericFlightServerBase::AddCancelCallback(const ServerCallContext& // Only the contexts the async transport hands its handlers carry a registry. if (const auto* async_context = dynamic_cast(&context)) { - return async_context->AddCancelCallback(std::move(callback)); + async_context->AddCancelCallback(std::move(callback)); + return Status::OK(); } #endif return Status::Invalid( @@ -106,52 +107,45 @@ Status AsyncGenericFlightServerBase::AddCancelCallback(const ServerCallContext& arrow::Future> AsyncGenericFlightServerBase::ListFlightsAsync(const ServerCallContext&, const Criteria*) { - return arrow::Future>::MakeFinished( - Status::NotImplemented("NYI")); + return Status::NotImplemented("NYI"); } arrow::Future> AsyncGenericFlightServerBase::GetFlightInfoAsync(const ServerCallContext&, const FlightDescriptor&) { - return arrow::Future>::MakeFinished( - Status::NotImplemented("NYI")); + return Status::NotImplemented("NYI"); } arrow::Future> AsyncGenericFlightServerBase::PollFlightInfoAsync(const ServerCallContext&, const FlightDescriptor&) { - return arrow::Future>::MakeFinished( - Status::NotImplemented("NYI")); + return Status::NotImplemented("NYI"); } arrow::Future> AsyncGenericFlightServerBase::GetSchemaAsync( const ServerCallContext&, const FlightDescriptor&) { - return arrow::Future>::MakeFinished( - Status::NotImplemented("NYI")); + return Status::NotImplemented("NYI"); } arrow::Future> AsyncGenericFlightServerBase::DoActionAsync(const ServerCallContext&, const Action&) { - return arrow::Future>::MakeFinished( - Status::NotImplemented("NYI")); + return Status::NotImplemented("NYI"); } arrow::Future> AsyncGenericFlightServerBase::ListActionsAsync( const ServerCallContext&) { - return arrow::Future>::MakeFinished( - Status::NotImplemented("NYI")); + return Status::NotImplemented("NYI"); } arrow::Future<> AsyncGenericFlightServerBase::DoExchangeAsync( const ServerCallContext&, std::shared_ptr, std::shared_ptr) { - return arrow::Future<>::MakeFinished(Status::NotImplemented("NYI")); + return Status::NotImplemented("NYI"); } arrow::Future> AsyncGenericFlightServerBase::DoGetAsync(const ServerCallContext&, const Ticket&) { - return arrow::Future>::MakeFinished( - Status::NotImplemented("DoGetAsync is not implemented")); + return Status::NotImplemented("DoGetAsync is not implemented"); } std::shared_ptr diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc index f895db57473e..47e86f9cc348 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_get_test.cc @@ -167,9 +167,8 @@ TEST(AsyncGrpcTest, DoGetClientCancelWithPendingPayloadFinishes) { // instead of waiting for a payload that will not come. const int64_t before = detail::AsyncReactorBase::live_reactors(); auto trace = std::make_shared(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); HangingDoGetServer flight_server(HangingDoGetServer::HangAt::kStream); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); options.middleware = { {"recording", std::make_shared(trace)}}; ASSERT_OK(flight_server.Init(options)); @@ -202,9 +201,8 @@ TEST(AsyncGrpcTest, DoGetClientDeadlineWhilePreparingFinishes) { // stream exists here, so Close() has nothing to deliver and the producer // notices through ServerCallContext::is_cancelled(). auto trace = std::make_shared(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); HangingDoGetServer flight_server(HangingDoGetServer::HangAt::kPrepare); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); options.middleware = { {"recording", std::make_shared(trace)}}; ASSERT_OK(flight_server.Init(options)); @@ -281,9 +279,8 @@ TEST(AsyncGrpcTest, DoGetCancelCallbackReachesPendingHandler) { // registered once the RPC is cancelled runs at once, and the stream the // handler then resolves to is never read from but still receives Close(). const int64_t before = detail::AsyncReactorBase::live_reactors(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); CancelCallbackDoGetServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -390,9 +387,8 @@ TEST(AsyncGrpcTest, DoGetNeverClosesAStreamItIsAskingForAPayload) { // GetSchemaPayloadAsync(): the cancellation's Close() must wait for that // call to return, and the stream must not be asked for anything after it. const int64_t before = detail::AsyncReactorBase::live_reactors(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); SlowSchemaDoGetServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -457,9 +453,8 @@ TEST(AsyncGrpcTest, DoGetPumpErrorStillClosesTheStream) { // The producer's first NextAsync() fails: the RPC ends with that status, and // the producer must still be told the RPC is over through Close(), like the // write-failure and cancel paths already do. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); ErrorPayloadDoGetServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc index bc13a6fa399d..6f5583141bd3 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put.cc @@ -55,15 +55,15 @@ class DoPutReactor : public AsyncReactorBase, Future<> WriteMetadataAsync(const Buffer& app_metadata) override { std::lock_guard lock(write_mutex()); if (finished() || upload_ending_) { - return Future<>::MakeFinished(Status::Invalid( + return Status::Invalid( "the upload is over: WriteMetadataAsync() is only usable while the " - "upload's RPC is running")); + "upload's RPC is running"); } if (metadata_in_flight_ || ending_) { // One message in flight at a time: OnWriteDone serializes the PutResults. - return Future<>::MakeFinished(Status::Invalid( + return Status::Invalid( "a PutResult message is already in flight: await the previous " - "WriteMetadataAsync() future before writing another metadata message")); + "WriteMetadataAsync() future before writing another metadata message"); } metadata_in_flight_ = true; metadata_written_ = Future<>::Make(); @@ -113,18 +113,10 @@ class DoPutReactor : public AsyncReactorBase, "descriptor")); return; } - std::shared_ptr arrow_buf; - const Status wrap_status = WrapGrpcBuffer(&request_buf_, &arrow_buf); - if (!wrap_status.ok()) { - FinishUpload( - MakeFlightError(FlightStatusCode::Internal, - "Failed to wrap gRPC buffer: " + wrap_status.message())); - return; - } - // Deserialize once: the descriptor check and the decoder share the result. - auto data = arrow::flight::internal::DeserializeFlightData(arrow_buf); - if (!data.ok()) { - FinishUpload(data.status()); + internal::FlightData data; + const ::grpc::Status deserialized = FlightDataDeserialize(&request_buf_, &data); + if (!deserialized.ok()) { + FinishUpload(FromGrpcStatus(deserialized)); return; } if (!read_descriptor_) { @@ -133,13 +125,13 @@ class DoPutReactor : public AsyncReactorBase, // Without this the listener never hears OnDescriptor and the upload is // acknowledged as if it were well formed. read_descriptor_ = true; - if (!data->descriptor) { + if (!data.descriptor) { FinishUpload(Status::IOError("Descriptor missing on first message")); return; } } - Future<> decode_status = decoder_.Consume(std::move(*data)); + Future<> decode_status = decoder_.Consume(std::move(data)); decode_status.AddCallback([this, token = hold()](arrow::Status status) { if (upload_ending_) { // The upload is ending (cancelled, or ended while the listener was diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc index da77eef8cd41..e665f24b8e13 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_do_put_test.cc @@ -65,10 +65,8 @@ TEST(AsyncGrpcTest, BasicDoPut) { ASSERT_EQ(chunk->num_rows(), 3); ASSERT_EQ(chunk->num_columns(), 2); } - // No metadata-only (error/app-metadata) chunks, and nothing the consumer - // rejected: a non-OK OnNext status would have failed DoPut with no ack. + // No metadata-only (error/app-metadata) chunks. ASSERT_EQ(listener->metadata_only_count(), 0); - ASSERT_OK(listener->last_status()); } TEST(AsyncGrpcTest, EmptyDoPut) { @@ -81,8 +79,7 @@ TEST(AsyncGrpcTest, EmptyDoPut) { ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(hand_built.port)); - auto schema = arrow::schema( - {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); + auto schema = TwoColumnSchema(); // Upload nothing at all: the acknowledgement the server writes back on the // client's half-close is what makes Close() return. @@ -99,7 +96,6 @@ TEST(AsyncGrpcTest, EmptyDoPut) { // No data of any kind reached the listener. ASSERT_EQ(listener->batches().size(), 0); ASSERT_EQ(listener->metadata_only_count(), 0); - ASSERT_OK(listener->last_status()); // The sync client sends the schema payload eagerly, even when no batch // follows, so the listener does see the schema here. Deliberately not // asserted: that is the client's business, not the transport's. @@ -115,8 +111,7 @@ TEST(AsyncGrpcTest, MetadataOnlyPutChunk) { ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(hand_built.port)); - auto schema = arrow::schema( - {arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); + auto schema = TwoColumnSchema(); FlightDescriptor descriptor = FlightDescriptor::Path({"metadata-only-put"}); ASSERT_OK_AND_ASSIGN(auto result, client->DoPut(descriptor, schema)); @@ -173,8 +168,7 @@ TEST(AsyncGrpcTest, CreateDoPutListenerServesUpload) { // The upload half of the server class API: CreateDoPutListener() hands out // the per-RPC listener, with no options-level factory. TestUploadServer flight_server; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_GT(flight_server.port(), 0); @@ -551,9 +545,8 @@ TEST(AsyncGrpcTest, DoPutClientDeadlineWithPendingHandlerFinishes) { // RPC itself instead of waiting for the listener: Shutdown()/Wait() below // would otherwise block, since gRPC keeps the call alive until Finish() // runs. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); HangingDoPutServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -656,9 +649,8 @@ TEST(AsyncGrpcTest, DoPutClientCancelWhileListenerParkedStaysSafe) { // never touch the finished call) rather than reach into a destroyed reactor. // The test resolves the parked future from its own thread, and the server // keeps its shared_ptr alive, so the resolution lands on the real listener. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); DeferredDoPutServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -714,8 +706,7 @@ TEST(AsyncGrpcTest, DoPutWithoutDescriptorOnFirstMessageIsRejected) { TestFlightServer flight_server; auto listener = std::make_shared(); flight_server.set_listener(listener); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_GT(flight_server.port(), 0); @@ -753,8 +744,7 @@ TEST(AsyncGrpcTest, DoPutMalformedFirstMessageIsRejected) { TestFlightServer flight_server; auto listener = std::make_shared(); flight_server.set_listener(listener); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); // Field 1 (the descriptor), length-delimited, announcing 127 bytes of 2. @@ -782,8 +772,7 @@ TEST(AsyncGrpcTest, DoPutReportsAFailingOnFinishToTheClient) { TestFlightServer flight_server; auto listener = std::make_shared(); flight_server.set_listener(listener); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -813,8 +802,7 @@ TEST(AsyncGrpcTest, DoPutHalfCloseWithoutAnyMessageIsRejected) { TestFlightServer flight_server; auto listener = std::make_shared(); flight_server.set_listener(listener); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_GT(flight_server.port(), 0); @@ -839,8 +827,7 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagUploadRejectedOnDescriptor) { auto listener = std::make_shared(); listener->set_descriptor_status( arrow::Status::Invalid("listener rejected this descriptor")); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); flight_server.set_listener(listener); ASSERT_OK(flight_server.Init(options)); ASSERT_GT(flight_server.port(), 0); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc index fe9d69ae0b7b..5e23bcde775f 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange.cc @@ -23,9 +23,8 @@ ExchangeReactor::ExchangeReactor(AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) : AsyncReactorBase(std::move(flight_context)), base_(base), - reader_(std::make_shared()), + reader_(std::make_shared(this)), writer_(std::make_shared(this)) { - reader_->SetReactor(this); // The first message carries the request descriptor. StartRead(&request_buf_); } diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h index cdbcc08d6319..5c67749f6353 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_internal.h @@ -54,14 +54,12 @@ class ExchangeReactor; class ExchangeReader final : public AsyncFlightMessageReader, public std::enable_shared_from_this { public: - ExchangeReader(); + explicit ExchangeReader(ExchangeReactor* reactor); const FlightDescriptor& descriptor() const override; arrow::Future NextAsync() override; - /// \brief The reactor wires itself before the handler starts. - void SetReactor(ExchangeReactor* reactor); /// \brief Called by the reactor once the exchange is over: from then on the /// handler's reader must refuse to start reads instead of calling into a /// freed reactor. @@ -144,8 +142,6 @@ class ExchangeWriter final : public AsyncFlightMessageWriter { /// this way, as on the synchronous path. class PayloadSink; - arrow::Future<> WriteBatch(const RecordBatch& batch); - /// \brief Write the queued payloads one at a time, the returned future /// resolves when the queue is empty, or with the first failure. arrow::Future<> Flush(); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc index 78f25e45baa2..a2741bce7425 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_reader.cc @@ -38,44 +38,44 @@ class ExchangeReader::ChunkListener final : public AsyncFlightDataListener { if (has_demand) { reader_->ResolvePending(std::move(chunk)); } - return arrow::Future<>::MakeFinished(); + return Status::OK(); } private: ExchangeReader* reader_; }; -ExchangeReader::ExchangeReader() - : listener_(std::make_shared(this)), decoder_(listener_) {} +ExchangeReader::ExchangeReader(ExchangeReactor* reactor) + : reactor_(reactor), + listener_(std::make_shared(this)), + decoder_(listener_) {} const FlightDescriptor& ExchangeReader::descriptor() const { return descriptor_; } arrow::Future ExchangeReader::NextAsync() { - using FSCFuture = arrow::Future; - arrow::Future returned; { std::lock_guard lock(mutex_); if (pending_.is_valid() && !pending_.is_finished()) { - return FSCFuture::MakeFinished(arrow::Status::Invalid("one NextAsync at a time")); + return Status::Invalid("one NextAsync at a time"); } if (buffered_.has_value()) { - auto out = FSCFuture::MakeFinished(std::move(*buffered_)); + FlightStreamChunk out = std::move(*buffered_); buffered_.reset(); return out; } if (!read_error_.ok()) { - return FSCFuture::MakeFinished(read_error_); + return read_error_; } if (read_closed_ || reactor_ == nullptr) { // End of exchange: the reactor either saw the read side close or the RPC // is over (and the reactor is gone), so no further read can be started. - return FSCFuture::MakeFinished(FlightStreamChunk{}); + return FlightStreamChunk{}; } - pending_ = FSCFuture::Make(); + pending_ = arrow::Future::Make(); returned = pending_; } // Consume()/RequestRead() take the lock themselves and may resolve pending_ @@ -94,11 +94,6 @@ arrow::Future ExchangeReader::NextAsync() { return returned; } -void ExchangeReader::SetReactor(ExchangeReactor* reactor) { - std::lock_guard lock(mutex_); - reactor_ = reactor; -} - void ExchangeReader::DetachReactor() { std::lock_guard lock(mutex_); reactor_ = nullptr; diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc index 2c3533951da6..ffc1bd7e331b 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_test.cc @@ -28,8 +28,7 @@ namespace arrow::flight::transport::grpc { TEST(AsyncGrpcTest, DoExchangeEchoesBatches) { // Two batches up, two batches back, in order: this is the case that needs a // read and a write outstanding at the same time. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); EchoExchangeTestServer flight_server; ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -72,8 +71,7 @@ TEST(AsyncGrpcTest, DoExchangeMetadataOnlyDoesNotHang) { // handler must see the end of the exchange and reply, not wait for a schema // message that will never come (the case DataTest::TestDoExchangeNoData // covers). A hang here is the failure mode this test exists for. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); EchoExchangeTestServer flight_server; ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -104,8 +102,7 @@ TEST(AsyncGrpcTest, DoExchangeClientDisconnectFinishes) { // must let the handler unwind and finish the RPC, and the exchange counter // added to the test server must come back to zero -- a hang or a leaked // handler leaves it above zero. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); EchoExchangeTestServer flight_server; ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -156,9 +153,8 @@ TEST(AsyncGrpcTest, DoExchangeClientCancelWithPendingHandlerFinishes) { // finish the RPC (and unwind reader/writer), not wait for the handler. const int64_t before = detail::AsyncReactorBase::live_reactors(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); HangingExchangeServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc index a0ed08b413db..5eaa92971d79 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_exchange_writer.cc @@ -56,13 +56,12 @@ arrow::Future<> ExchangeWriter::BeginAsync(std::shared_ptr schema) { { std::lock_guard lock(mutex_); if (reactor_ == nullptr) { - return arrow::Future<>::MakeFinished(arrow::Status::Cancelled( - "the exchange is over: the client disconnected or cancelled")); + return arrow::Status::Cancelled( + "the exchange is over: the client disconnected or cancelled"); } } if (batch_writer_ != nullptr) { - return arrow::Future<>::MakeFinished( - arrow::Status::Invalid("This writer has already been started.")); + return arrow::Status::Invalid("This writer has already been started."); } auto sink = std::make_unique(this); ARROW_ASSIGN_OR_RAISE(batch_writer_, @@ -76,8 +75,13 @@ arrow::Future<> ExchangeWriter::WriteRecordBatchAsync(const RecordBatch& batch) arrow::Future<> ExchangeWriter::WriteWithMetadataAsync( const RecordBatch& batch, std::shared_ptr app_metadata) { + if (batch_writer_ == nullptr) { + return arrow::Status::Invalid( + "This writer is not started. Call BeginAsync() with a schema"); + } pending_app_metadata_ = std::move(app_metadata); - return WriteBatch(batch); + ARROW_RETURN_NOT_OK(batch_writer_->WriteRecordBatch(batch)); + return Flush(); } arrow::Future<> ExchangeWriter::WriteMetadataAsync(std::shared_ptr app_metadata) { @@ -88,14 +92,12 @@ arrow::Future<> ExchangeWriter::WriteMetadataAsync(std::shared_ptr app_m } arrow::Future<> ExchangeWriter::CloseAsync() { - if (batch_writer_ != nullptr) { - auto status = batch_writer_->Close(); - batch_writer_ = nullptr; - if (!status.ok()) { - return arrow::Future<>::MakeFinished(std::move(status)); - } + if (batch_writer_ == nullptr) { + return arrow::Status::OK(); } - return arrow::Future<>::MakeFinished(); + auto status = batch_writer_->Close(); + batch_writer_ = nullptr; + return status; } void ExchangeWriter::OnWriteDone(bool ok) { @@ -119,26 +121,14 @@ void ExchangeWriter::OnCancelled() { FailFlush(arrow::Status::Cancelled("the client cancelled the exchange")); } -arrow::Future<> ExchangeWriter::WriteBatch(const RecordBatch& batch) { - if (batch_writer_ == nullptr) { - return arrow::Future<>::MakeFinished(arrow::Status::Invalid( - "This writer is not started. Call BeginAsync() with a schema")); - } - auto status = batch_writer_->WriteRecordBatch(batch); - if (!status.ok()) { - return arrow::Future<>::MakeFinished(std::move(status)); - } - return Flush(); -} - arrow::Future<> ExchangeWriter::Flush() { arrow::Future<> flush; { std::lock_guard lock(mutex_); if (reactor_ == nullptr) { // The exchange is over: report it rather than pumping into a dead reactor. - return arrow::Future<>::MakeFinished(arrow::Status::Cancelled( - "the exchange is over: the client disconnected or cancelled")); + return arrow::Status::Cancelled( + "the exchange is over: the client disconnected or cancelled"); } if (!flush_.is_valid() || flush_.is_finished()) { flush_ = arrow::Future<>::Make(); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc index 7ac377158f3b..0f393044f8a6 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake.cc @@ -30,9 +30,8 @@ namespace { /// class holds is enough. class HandshakeReactor final : public AsyncReactorBase { public: - HandshakeReactor(AsyncCallContext flight_context, HandshakeFn handshake_handler) - : AsyncReactorBase(std::move(flight_context)), - handshake_handler_(std::move(handshake_handler)) { + HandshakeReactor(AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) + : AsyncReactorBase(std::move(flight_context)), base_(base) { StartRead(&request_buf_); } @@ -48,8 +47,7 @@ class HandshakeReactor final : public AsyncReactorBase { return; } std::string response; - const auto status = - handshake_handler_(flight_context(), request->payload(), &response); + const auto status = base_->Handshake(flight_context(), request->payload(), &response); if (!status.ok()) { FinishOnce(status); return; @@ -67,14 +65,14 @@ class HandshakeReactor final : public AsyncReactorBase { } private: - HandshakeFn handshake_handler_; + AsyncGenericFlightServerBase* base_; }; } // namespace -::grpc::ServerGenericBidiReactor* MakeHandshakeReactor(AsyncCallContext flight_context, - HandshakeFn handshake_handler) { - return new HandshakeReactor(std::move(flight_context), std::move(handshake_handler)); +::grpc::ServerGenericBidiReactor* MakeHandshakeReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base) { + return new HandshakeReactor(std::move(flight_context), base); } } // namespace arrow::flight::transport::grpc::detail diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc index e52a2d88f9f9..2ffd71e9d2ec 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_handshake_test.cc @@ -40,12 +40,10 @@ struct AsyncAuthState { // are the only configuration the transport needs. class AsyncAuthTestServer : public AsyncGenericFlightServerBase { public: - AsyncAuthTestServer(std::string username, std::string password, std::string ticket, - std::shared_ptr state = nullptr) + AsyncAuthTestServer(std::string username, std::string password, std::string ticket) : username_(std::move(username)), password_(std::move(password)), - ticket_(std::move(ticket)), - state_(state ? std::move(state) : std::make_shared()) {} + ticket_(std::move(ticket)) {} const std::shared_ptr& state() const { return state_; } @@ -97,7 +95,7 @@ class AsyncAuthTestServer : public AsyncGenericFlightServerBase { std::string username_; std::string password_; std::string ticket_; - std::shared_ptr state_; + std::shared_ptr state_ = std::make_shared(); }; // A server with the async auth hooks installed, and a client connected to it. @@ -118,10 +116,9 @@ struct AuthenticatedHarness { private: Status Start() { - ARROW_ASSIGN_OR_RAISE(auto location, Location::Parse(kServerUri)); // The server class alone selects the async transport: deriving from it and // overriding its virtuals are the whole configuration. - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); RETURN_NOT_OK(server.Init(options)); ARROW_ASSIGN_OR_RAISE(client, ConnectTo(server.port())); return Status::OK(); @@ -135,9 +132,8 @@ class AsyncGenericTestServer : public AsyncGenericFlightServerBase {}; TEST(AsyncGrpcTest, HandshakeWithoutHookIsUnimplemented) { // A bare async server: Handshake is answered by the class's default virtual // with UNIMPLEMENTED, and the client's Authenticate() surfaces it. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); AsyncGenericTestServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -154,9 +150,8 @@ TEST(AsyncGrpcTest, AsyncGenericFlightServerBaseImpliesAsyncTransport) { // yet the RPC is served by the generic callback service. DoPut is the // discriminator: an async server with no listener factory answers with its own // message rather than "NYI". - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); AsyncGenericTestServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -269,9 +264,8 @@ TEST(AsyncGrpcTest, HandshakePayloadIsBinarySafe) { // response is echoed back through the client's reader: an embedded NUL and // non-ASCII bytes must survive both directions unchanged. const std::string binary("p4ss\0w0rd\xC3\xA9", 11); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); AsyncAuthTestServer server(binary, binary, "bin-ticket"); - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(server.port())); @@ -412,9 +406,8 @@ TEST(AsyncGrpcTest, LateHandshakeResponseLeavesServerUsable) { std::atomic handshakes{0}; }; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); SlowAnsweringServer server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(server.port())); @@ -447,9 +440,8 @@ TEST(AsyncGrpcTest, EmptyHandshakeResponseIsDelivered) { } }; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); NoResponseServer server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(server.port())); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc index f156898aa87e..b644cc75916f 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.cc @@ -79,11 +79,8 @@ class Unimplemented : public ::grpc::ServerGenericBidiReactor { AsyncGenericFlightService::AsyncGenericFlightService( AsyncGenericFlightServerBase* async_base, - std::shared_ptr> helper, - HandshakeFn handshake_handler) - : base_(async_base), - helper_(std::move(helper)), - handshake_handler_(std::move(handshake_handler)) {} + std::shared_ptr> helper) + : base_(async_base), helper_(std::move(helper)) {} ::grpc::ServerGenericBidiReactor* AsyncGenericFlightService::CreateReactor( ::grpc::GenericCallbackServerContext* context) { @@ -102,32 +99,19 @@ ::grpc::ServerGenericBidiReactor* AsyncGenericFlightService::CreateReactor( switch (flight_method) { case FlightMethod::Handshake: - if (!handshake_handler_) { - return new Unimplemented( - flight_context, - ::grpc::Status( - ::grpc::StatusCode::UNIMPLEMENTED, - "This service does not have an authentication mechanism enabled.")); - } - return detail::MakeHandshakeReactor(std::move(flight_context), handshake_handler_); + return detail::MakeHandshakeReactor(std::move(flight_context), base_); case FlightMethod::GetFlightInfo: return new detail::UnaryReactor( - std::move(flight_context), [base = base_](const ServerCallContext& context, - const FlightDescriptor& descriptor) { - return base->GetFlightInfoAsync(context, descriptor); - }); + std::move(flight_context), base_, + &AsyncGenericFlightServerBase::GetFlightInfoAsync); case FlightMethod::GetSchema: return new detail::UnaryReactor( - std::move(flight_context), [base = base_](const ServerCallContext& context, - const FlightDescriptor& descriptor) { - return base->GetSchemaAsync(context, descriptor); - }); + std::move(flight_context), base_, + &AsyncGenericFlightServerBase::GetSchemaAsync); case FlightMethod::PollFlightInfo: return new detail::UnaryReactor( - std::move(flight_context), [base = base_](const ServerCallContext& context, - const FlightDescriptor& descriptor) { - return base->PollFlightInfoAsync(context, descriptor); - }); + std::move(flight_context), base_, + &AsyncGenericFlightServerBase::PollFlightInfoAsync); case FlightMethod::ListActions: return detail::MakeListActionsReactor(std::move(flight_context), base_); case FlightMethod::DoAction: diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.h index ae4f13a5fe02..ed345d532263 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service.h @@ -28,9 +28,6 @@ namespace arrow::flight::transport::grpc { #if ARROW_FLIGHT_HAS_ASYNC_SERVER -using HandshakeFn = - GrpcServerCallContextHelper<::grpc::CallbackServerContext>::HandshakeFn; - /// \brief Generic callback service dispatching Flight methods to reactors. /// /// Constructed with the AsyncGenericFlightServerBase whose handlers are served. @@ -52,12 +49,9 @@ class ARROW_FLIGHT_EXPORT AsyncGenericFlightService /// \param async_base is the async server whose handlers are served, it must /// outlive the service. /// \param helper runs middleware and auth and builds the call context. - /// \param handshake_handler is the server's Handshake hook, an empty hook - /// means the server has no authentication mechanism. AsyncGenericFlightService( AsyncGenericFlightServerBase* async_base, - std::shared_ptr> helper, - HandshakeFn handshake_handler = {}); + std::shared_ptr> helper); /// \brief Creates a reactor for the incoming RPC. /// \param context is the gRPC callback server context for the RPC. @@ -69,8 +63,6 @@ class ARROW_FLIGHT_EXPORT AsyncGenericFlightService private: AsyncGenericFlightServerBase* base_; std::shared_ptr> helper_; - /// The server class's Handshake hook. Empty when the server has none. - HandshakeFn handshake_handler_; }; # ifdef _MSC_VER # pragma warning(pop) diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h index 819f5b44895b..3cd6844c6353 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_internal.h @@ -28,6 +28,7 @@ #include #include +#include #include #include "arrow/flight/flight_data_decoder.h" @@ -51,17 +52,16 @@ class AsyncCallContext : public GrpcServerCallContext<::grpc::CallbackServerCont explicit AsyncCallContext(::grpc::CallbackServerContext* context) : GrpcServerCallContext<::grpc::CallbackServerContext>(context) {} - arrow::Status AddCancelCallback(std::function callback) const { + void AddCancelCallback(std::function callback) const { { std::lock_guard lock(cancel_->mutex); if (!cancel_->cancelled) { cancel_->callbacks.push_back(std::move(callback)); - return arrow::Status::OK(); + return; } } // Already cancelled: run it now, on the caller's thread, unlocked. callback(); - return arrow::Status::OK(); } /// \brief Run the registered callbacks once, unlocked, so a callback may @@ -97,19 +97,20 @@ ::grpc::ByteBuffer MakeWriteBuffer(const ProtoT& message) { } template -arrow::Result ParseProtoRequest(const ::grpc::ByteBuffer& buf, - std::string_view what) { - ARROW_ASSIGN_OR_RAISE(std::string bytes, - BytesFromBuffer(buf, "Failed to read request")); +arrow::Result ParseProtoRequest(::grpc::ByteBuffer& buf, std::string_view what) { + ::grpc::ProtoBufferReader reader(&buf); + if (!reader.status().ok()) { + return MakeFlightError(FlightStatusCode::Internal, "Failed to read request"); + } PbT pb; - if (!pb.ParseFromArray(bytes.data(), static_cast(bytes.size()))) { + if (!pb.ParseFromZeroCopyStream(&reader)) { return arrow::Status::Invalid("Failed to parse ", what); } return pb; } template -arrow::Result ParseProtoRequest(const ::grpc::ByteBuffer& buf, std::string_view what) { +arrow::Result ParseProtoRequest(::grpc::ByteBuffer& buf, std::string_view what) { ARROW_ASSIGN_OR_RAISE(auto pb, ParseProtoRequest(buf, what)); T out; ARROW_RETURN_NOT_OK(arrow::flight::internal::FromProto(pb, &out)); @@ -261,11 +262,13 @@ class AsyncReactorBase : public ::grpc::ServerGenericBidiReactor { template class UnaryReactor final : public AsyncReactorBase { public: - using HandlerFn = std::function>( - const ServerCallContext&, const FlightDescriptor&)>; + /// The AsyncGenericFlightServerBase handler this RPC calls. + using Handler = arrow::Future> (AsyncGenericFlightServerBase::*)( + const ServerCallContext&, const FlightDescriptor&); - UnaryReactor(AsyncCallContext flight_context, HandlerFn handler) - : AsyncReactorBase(std::move(flight_context)), handler_(std::move(handler)) { + UnaryReactor(AsyncCallContext flight_context, AsyncGenericFlightServerBase* base, + Handler handler) + : AsyncReactorBase(std::move(flight_context)), base_(base), handler_(handler) { StartRead(&request_buf_); } @@ -280,7 +283,8 @@ class UnaryReactor final : public AsyncReactorBase { FinishOnce(descriptor.status()); return; } - arrow::Future> future = handler_(flight_context(), *descriptor); + arrow::Future> future = + (base_->*handler_)(flight_context(), *descriptor); future.AddCallback( [this, token = hold()](const arrow::Result>& result) mutable { if (!result.ok()) { @@ -305,11 +309,12 @@ class UnaryReactor final : public AsyncReactorBase { } private: - HandlerFn handler_; + AsyncGenericFlightServerBase* base_; + Handler handler_; }; -::grpc::ServerGenericBidiReactor* MakeHandshakeReactor(AsyncCallContext flight_context, - HandshakeFn handshake_handler); +::grpc::ServerGenericBidiReactor* MakeHandshakeReactor( + AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); ::grpc::ServerGenericBidiReactor* MakeDoGetReactor(AsyncCallContext flight_context, AsyncGenericFlightServerBase* base); ::grpc::ServerGenericBidiReactor* MakeDoPutReactor( diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc index 743973cf1eaf..940a6277d28e 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_service_test.cc @@ -56,70 +56,13 @@ TEST(AsyncGrpcTest, MiddlewareCallCompletedRunsOnUnknownMethod) { << "CallCompleted must run for an UNIMPLEMENTED reply too"; } -TEST(AsyncGrpcTest, UseAsyncGrpcFlag) { - // Same adapter as the tests above, but now started through the async - // server's own Init instead of a hand-built grpc::ServerBuilder. - TestFlightServer flight_server; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - FlightServerOptions options(location); - ASSERT_OK(flight_server.Init(options)); - ASSERT_GT(flight_server.port(), 0); - - ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); - - // (i) DoGet went through the generic reactor: the ticket reached - // TestFlightServer::DoGetAsync and the whole stream came back. - Ticket ticket{"flag-path-do-get-ticket"}; - ASSERT_OK_AND_ASSIGN(auto reader, client->DoGet(ticket)); - ASSERT_OK_AND_ASSIGN(auto schema, reader->GetSchema()); - ASSERT_EQ(schema->num_fields(), 2); - ASSERT_EQ(schema->field(0)->name(), "a"); - ASSERT_EQ(schema->field(1)->name(), "b"); - int batch_count = 0; - while (true) { - ASSERT_OK_AND_ASSIGN(auto chunk, reader->Next()); - if (!chunk.data) break; - ASSERT_EQ(chunk.data->num_rows(), 5); - ASSERT_OK_AND_ASSIGN(auto value, chunk.data->column(0)->GetScalar(0)); - ASSERT_TRUE(value->Equals(*arrow::MakeScalar(int64_t(1)))); - batch_count++; - } - ASSERT_EQ(batch_count, 3); - - // (ii) Every typed method is served on this path too, the fallback reactor - // still answers a method name no stub knows, which is what pins "the typed - // sync service is NOT registered alongside the generic one". - const ::grpc::Status unknown_status = CallUnknownMethod(flight_server.port()); - ASSERT_FALSE(unknown_status.ok()) << "an unknown method must not be served"; - ASSERT_EQ(::grpc::StatusCode::UNIMPLEMENTED, unknown_status.error_code()) - << unknown_status.error_message(); - - // (iii) The async server takes no listener factory here, so uploads are - // refused the same way. - FlightDescriptor descriptor = FlightDescriptor::Path({"not-served"}); - auto put_schema = arrow::schema({arrow::field("a", arrow::int64())}); - auto batch = arrow::RecordBatch::Make(put_schema, 1, - {arrow::ArrayFromJSON(arrow::int64(), "[1]")}); - auto [surface, status] = UploadOneBatch(client.get(), descriptor, put_schema, batch); - ASSERT_FALSE(status.ok()) << "DoPut must be refused, surfaced on " << surface; - ASSERT_EQ(status.code(), arrow::StatusCode::NotImplemented); - - // Cleanup. The ticket is asserted only after the server stopped, so the - // DoGet callback cannot race with the read. - ASSERT_OK(StopClientAndServer(client.get(), &flight_server)); - - ASSERT_EQ(flight_server.ticket(), "flag-path-do-get-ticket"); -} - TEST(AsyncGrpcTest, UseAsyncGrpcFlagRefusesBlockingAuthHandler) { // The callback path cannot run the blocking ServerAuthHandler (it would run // on callback threads), so a server configured with one must refuse to start // instead of silently serving unauthenticated requests. Middleware *is* // supported now - see UseAsyncGrpcFlagAllowsMiddleware. - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - TestFlightServer auth_server; - FlightServerOptions with_auth(location); + FlightServerOptions with_auth(LocalhostLocation()); with_auth.auth_handler = std::make_shared("user", "pass"); auto auth_status = auth_server.Init(with_auth); ASSERT_FALSE(auth_status.ok()) << "an auth handler must not be silently skipped"; @@ -133,10 +76,8 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagAllowsMiddleware) { // path: StartCall at call creation, SendingHeaders and CallCompleted at the // end - success or failure alike. auto trace = std::make_shared(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); - TestFlightServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); options.middleware = { {"recording", std::make_shared(trace)}}; ASSERT_OK(flight_server.Init(options)); @@ -157,9 +98,8 @@ TEST(AsyncGrpcTest, UseAsyncGrpcFlagAllowsMiddleware) { } TEST(AsyncGrpcTest, UseAsyncGrpcFlagMiddlewareCanRejectCall) { - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); TestFlightServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); options.middleware = { {"rejecting", std::make_shared()}}; ASSERT_OK(flight_server.Init(options)); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc index 1ba570bc59f0..701a6a2b254f 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming.cc @@ -94,22 +94,18 @@ class ListActionsReactor final : public StreamingReactor { return base_->ListActionsAsync(flight_context()) .Then([this](const std::vector& actions) -> arrow::Status { actions_ = actions; - next_ = 0; return arrow::Status::OK(); }); } arrow::Future NextMessage() override { if (next_ >= actions_.size()) { - return arrow::Future::MakeFinished(false); + return false; } pb::ActionType pb_type; - const auto status = internal::ToProto(actions_[next_++], &pb_type); - if (!status.ok()) { - return arrow::Future::MakeFinished(status); - } + ARROW_RETURN_NOT_OK(internal::ToProto(actions_[next_++], &pb_type)); write_buf_ = MakeWriteBuffer(pb_type); - return arrow::Future::MakeFinished(true); + return true; } private: @@ -123,11 +119,9 @@ class DoActionReactor final : public StreamingReactor { protected: arrow::Future<> Start() override { - auto action = ParseProtoRequest(request_buf_, "Action"); - if (!action.ok()) { - return arrow::Future<>::MakeFinished(action.status()); - } - return base_->DoActionAsync(flight_context(), *action) + ARROW_ASSIGN_OR_RAISE( + auto action, (ParseProtoRequest(request_buf_, "Action"))); + return base_->DoActionAsync(flight_context(), action) .Then([this](const std::shared_ptr& stream) -> arrow::Status { if (stream == nullptr) { return arrow::Status::Cancelled(); @@ -160,11 +154,8 @@ class ListFlightsReactor final : public StreamingReactor { protected: arrow::Future<> Start() override { - auto criteria = ParseProtoRequest(request_buf_, "Criteria"); - if (!criteria.ok()) { - return arrow::Future<>::MakeFinished(criteria.status()); - } - criteria_ = *criteria; + ARROW_ASSIGN_OR_RAISE( + criteria_, (ParseProtoRequest(request_buf_, "Criteria"))); return base_->ListFlightsAsync(flight_context(), &criteria_) .Then( [this](const std::shared_ptr& listing) -> arrow::Status { @@ -175,7 +166,7 @@ class ListFlightsReactor final : public StreamingReactor { arrow::Future NextMessage() override { if (listing_ == nullptr) { - return arrow::Future::MakeFinished(false); + return false; } return listing_->NextAsync().Then( [this](const std::shared_ptr& info) -> arrow::Result { diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc index 379023515385..b2c548ffe85c 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_streaming_test.cc @@ -204,17 +204,15 @@ TEST(AsyncGrpcTest, ListFlightsAsyncCompletesOnAnotherThread) { std::thread([out]() mutable { std::this_thread::sleep_for(std::chrono::milliseconds(100)); std::vector flights; - flights.push_back( - std::move(*MakeTestFlightInfo(FlightDescriptor::Path({"deferred"})))); + flights.push_back(MakeTestFlightInfo(FlightDescriptor::Path({"deferred"}))); out.MarkFinished(std::make_shared(std::move(flights))); }).detach(); return out; } }; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); DeferredListingServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); @@ -244,9 +242,8 @@ TEST(AsyncGrpcTest, ListFlightsDeadlineWithPendingHandlerFinishes) { // the reactor must be released even though the handler never completes. const int64_t before = detail::AsyncReactorBase::live_reactors(); - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); HangingListingServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h index d18220951ead..c55cdeffc977 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_test_fixtures.h @@ -58,12 +58,11 @@ namespace arrow::flight::transport::grpc { constexpr const char* kSchemaOnlyTicket = "schema-only-do-get-ticket"; // One canned FlightInfo, `descriptor` names it for the assertions. -inline std::unique_ptr MakeTestFlightInfo(FlightDescriptor descriptor) { - return std::make_unique( - FlightInfo::Make(*arrow::schema({arrow::field("value", arrow::int64())}), - std::move(descriptor), std::vector{}, - /*total_records=*/-1, /*total_bytes=*/-1) - .ValueOrDie()); +inline FlightInfo MakeTestFlightInfo(FlightDescriptor descriptor) { + return FlightInfo::Make(*arrow::schema({arrow::field("value", arrow::int64())}), + std::move(descriptor), std::vector{}, + /*total_records=*/-1, /*total_bytes=*/-1) + .ValueOrDie(); } // An AsyncFlightDataStream over an in-memory RecordBatchReader: every payload is @@ -71,41 +70,24 @@ inline std::unique_ptr MakeTestFlightInfo(FlightDescriptor descripto class ResolvedStream : public AsyncFlightDataStream { public: explicit ResolvedStream(std::shared_ptr reader) - : reader_(std::move(reader)), mapper_(*reader_->schema()) {} + : stream_(std::move(reader)) {} arrow::Future GetSchemaPayloadAsync() override { - FlightPayload payload; - auto status = ipc::GetSchemaPayload(*reader_->schema(), ipc_options_, mapper_, - &payload.ipc_message); - if (!status.ok()) { - return arrow::Future::MakeFinished(status); - } - return arrow::Future::MakeFinished(std::move(payload)); + return stream_.GetSchemaPayload(); } arrow::Future> NextAsync() override { - auto batch = reader_->Next(); - if (!batch.ok()) { - return arrow::Future>::MakeFinished(batch.status()); - } - if (*batch == nullptr) { - // End of stream: the async interface reports it as a nullopt. - return arrow::Future>::MakeFinished(std::nullopt); - } - FlightPayload payload; - auto status = ipc::GetRecordBatchPayload(**batch, ipc_options_, &payload.ipc_message); - if (!status.ok()) { - return arrow::Future>::MakeFinished(status); + ARROW_ASSIGN_OR_RAISE(auto payload, stream_.Next()); + if (payload.ipc_message.metadata == nullptr) { + return std::optional(); // end of stream } - return arrow::Future>::MakeFinished(std::move(payload)); + return std::optional(std::move(payload)); } - Status Close() override { return reader_->Close(); } + Status Close() override { return stream_.Close(); } private: - std::shared_ptr reader_; - ipc::DictionaryFieldMapper mapper_; - ipc::IpcWriteOptions ipc_options_ = ipc::IpcWriteOptions::Defaults(); + RecordBatchStream stream_; }; // A vector-backed async iterator: every NextAsync() resolves immediately, and @@ -297,8 +279,7 @@ class TestFlightServer final : public AsyncGenericFlightServerBase { return arrow::Future>::MakeFinished( std::make_shared( [] { - return std::move( - *MakeTestFlightInfo(FlightDescriptor::Path({"first-listing"}))); + return MakeTestFlightInfo(FlightDescriptor::Path({"first-listing"})); }, "mid-stream listing failure")); } @@ -311,7 +292,7 @@ class TestFlightServer final : public AsyncGenericFlightServerBase { } std::vector flights; for (const auto& name : {"first-listing", "second-listing"}) { - flights.push_back(std::move(*MakeTestFlightInfo(FlightDescriptor::Path({name})))); + flights.push_back(MakeTestFlightInfo(FlightDescriptor::Path({name}))); } return arrow::Future>::MakeFinished( std::make_shared(std::move(flights))); @@ -420,8 +401,7 @@ class RecordingListener : public AsyncFlightDataListener { ++metadata_only_count_; metadata_chunks_.push_back(std::move(chunk.app_metadata)); } - last_status_ = next_status_; - return arrow::Future<>::MakeFinished(next_status_); + return next_status_; } // Records the upload's descriptor and reports whatever status the test @@ -429,7 +409,7 @@ class RecordingListener : public AsyncFlightDataListener { arrow::Future<> OnDescriptor(const FlightDescriptor& descriptor) override { std::lock_guard lock(mutex_); descriptors_.push_back(descriptor); - return arrow::Future<>::MakeFinished(descriptor_status_); + return descriptor_status_; } // Consumer-side rejection of the upload, as the decoder tests do. @@ -471,10 +451,6 @@ class RecordingListener : public AsyncFlightDataListener { std::lock_guard lock(mutex_); return metadata_chunks_; } - arrow::Status last_status() const { - std::lock_guard lock(mutex_); - return last_status_; - } protected: /// The callbacks run on gRPC threads while the test thread reads the @@ -488,13 +464,6 @@ class RecordingListener : public AsyncFlightDataListener { std::vector> metadata_chunks_; arrow::Status descriptor_status_; arrow::Status next_status_; - arrow::Status last_status_; - - protected: - int finish_count_ = 0; - int cancel_count_ = 0; - Status finish_status_; - Status cancel_status_; }; // A listener that records the terminal events on top of RecordingListener: @@ -506,12 +475,12 @@ class FinishRecordingListener : public RecordingListener { std::lock_guard lock(mutex_); finish_count_++; finish_status_ = std::move(status); - return arrow::Future<>::MakeFinished(arrow::Status::OK()); + return arrow::Status::OK(); } /// Cancel from whatever thread the test drives, records what Cancel() said. void CancelWith(Status status) { - const arrow::Status cancel_status = Cancel(std::move(status)).status(); + const arrow::Status cancel_status = Cancel(std::move(status)); std::lock_guard lock(mutex_); cancel_status_ = cancel_status; ++cancel_count_; @@ -533,6 +502,12 @@ class FinishRecordingListener : public RecordingListener { std::lock_guard lock(mutex_); return cancel_status_; } + + private: + int finish_count_ = 0; + int cancel_count_ = 0; + Status finish_status_; + Status cancel_status_; }; // A server that accepts uploads: the transport asks the server for the per-RPC @@ -701,15 +676,6 @@ class RejectingServerMiddlewareFactory : public ServerMiddlewareFactory { } }; -// The gRPC transport builds the generic service with the shared context -// helper (which runs middleware and auth). Tests that register the service -// on a hand-built grpc::ServerBuilder must pass it explicitly. -inline std::shared_ptr> -MakeAsyncHelper() { - return std::make_shared>( - /*auth_handler=*/nullptr, MiddlewareFactoryList{}); -} - // The host every test server listens on and every client connects to: // 127.0.0.1, not localhost. localhost also resolves to [::1], gRPC then binds // the ephemeral port on both, and when another process already holds it on @@ -789,12 +755,14 @@ struct HandBuiltServer { int port = 0; }; -// Serve `base` on kListenAddress from a hand-built grpc::ServerBuilder. The -// caller checks `server` is non-null and `port` non-zero. +// Serve `base` on kListenAddress from a hand-built grpc::ServerBuilder, with the +// context helper (middleware and auth) the gRPC transport would otherwise +// build. The caller checks `server` is non-null and `port` non-zero. inline HandBuiltServer StartHandBuilt( AsyncGenericFlightServerBase* base, std::shared_ptr> helper = - MakeAsyncHelper()) { + std::make_shared>( + /*auth_handler=*/nullptr, MiddlewareFactoryList{})) { HandBuiltServer out; out.service = std::make_unique(base, std::move(helper)); ::grpc::ServerBuilder builder; @@ -812,7 +780,7 @@ inline std::shared_ptr OneColumnSchema() { // One 2-row batch of OneColumnSchema(), rows [1, 2]. inline std::shared_ptr OneColumnBatch( - const std::shared_ptr& schema = OneColumnSchema()) { + const std::shared_ptr& schema) { return arrow::RecordBatch::Make(schema, 2, {arrow::ArrayFromJSON(arrow::int64(), "[1, 2]")}); } @@ -837,7 +805,7 @@ inline std::shared_ptr TwoColumnSchema() { // One 3-row batch of TwoColumnSchema(), the payload the DoPut tests upload. inline std::shared_ptr TwoColumnBatch( - const std::shared_ptr& schema = TwoColumnSchema()) { + const std::shared_ptr& schema) { return arrow::RecordBatch::Make(schema, 3, {arrow::ArrayFromJSON(arrow::int64(), "[1, 2, 3]"), arrow::ArrayFromJSON(arrow::int64(), "[10, 20, 30]")}); diff --git a/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc index 63ad549a7e68..71c91abcdaff 100644 --- a/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc +++ b/cpp/src/arrow/flight/transport/grpc/async_grpc_unary_test.cc @@ -173,9 +173,8 @@ TEST(AsyncGrpcTest, GetFlightInfoAsyncCompletesOnAnotherThread) { } }; - ASSERT_OK_AND_ASSIGN(auto location, Location::Parse(kServerUri)); DeferredInfoServer flight_server; - FlightServerOptions options(location); + FlightServerOptions options(LocalhostLocation()); ASSERT_OK(flight_server.Init(options)); ASSERT_OK_AND_ASSIGN(auto client, ConnectTo(flight_server.port())); diff --git a/cpp/src/arrow/flight/transport/grpc/grpc_server.cc b/cpp/src/arrow/flight/transport/grpc/grpc_server.cc index 23fa0a678b5c..3148c4d62ce6 100644 --- a/cpp/src/arrow/flight/transport/grpc/grpc_server.cc +++ b/cpp/src/arrow/flight/transport/grpc/grpc_server.cc @@ -479,11 +479,6 @@ class GrpcServerTransport : public internal::ServerTransport { "override Handshake/ValidateToken instead"); } using AsyncHelper = GrpcServerCallContextHelper<::grpc::CallbackServerContext>; - AsyncHelper::HandshakeFn handshake = - [base = async_base_](const ServerCallContext& context, - const std::string& request, std::string* response) { - return base->Handshake(context, request, response); - }; AsyncHelper::ValidateTokenFn validate_token = [base = async_base_](const ServerCallContext& context, const std::string& token, std::string* peer_identity) { @@ -492,7 +487,7 @@ class GrpcServerTransport : public internal::ServerTransport { auto async_helper = std::make_shared( /*auth_handler=*/nullptr, options.middleware, std::move(validate_token)); async_service_ = std::make_unique( - async_base_, std::move(async_helper), std::move(handshake)); + async_base_, std::move(async_helper)); // The else and its body are split by #endif, so cpplint cannot pair the // braces; the #if branch above opens the block it closes. } else // NOLINT(readability/braces) diff --git a/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h b/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h index f1b679942cf6..03ff4d1f5e5c 100644 --- a/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/grpc_server_internal.h @@ -120,12 +120,6 @@ class GrpcServerCallContextHelper { std::function; - /// The async handshake hook: the transport reads one Handshake request, - /// calls this, then writes the response it filled in. - using HandshakeFn = - std::function; - GrpcServerCallContextHelper(std::shared_ptr auth_handler, MiddlewareFactoryList middleware, ValidateTokenFn validate_token = {}) diff --git a/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc b/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc index 0393b42db852..ee13a8e2022f 100644 --- a/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc +++ b/cpp/src/arrow/flight/transport/grpc/serialization_internal.cc @@ -17,10 +17,6 @@ #include "arrow/flight/transport/grpc/serialization_internal.h" -#include - -#include "arrow/flight/types.h" - #include #include #include @@ -155,25 +151,6 @@ arrow::Result<::grpc::Slice> SliceFromBuffer(const std::shared_ptr& buf) } // namespace -// Initialize arrow Buffer from gRPC ByteBuffer -Status WrapGrpcBuffer(::grpc::ByteBuffer* cpp_buf, std::shared_ptr* out) { - return GrpcBuffer::Wrap(cpp_buf, out); -} - -arrow::Result BytesFromBuffer(const ::grpc::ByteBuffer& buffer, - std::string_view error_message) { - std::vector<::grpc::Slice> slices; - if (!buffer.Dump(&slices).ok()) { - return MakeFlightError(FlightStatusCode::Internal, std::string(error_message)); - } - std::string bytes; - bytes.reserve(buffer.Length()); - for (const auto& slice : slices) { - bytes.append(reinterpret_cast(slice.begin()), slice.size()); - } - return bytes; -} - ::grpc::Status FlightDataSerialize(const FlightPayload& msg, ByteBuffer* out, bool* own_buffer) { // Retrieve BufferVector from the transport-agnostic serialization. diff --git a/cpp/src/arrow/flight/transport/grpc/serialization_internal.h b/cpp/src/arrow/flight/transport/grpc/serialization_internal.h index 80806f09966c..5c347fd4f819 100644 --- a/cpp/src/arrow/flight/transport/grpc/serialization_internal.h +++ b/cpp/src/arrow/flight/transport/grpc/serialization_internal.h @@ -21,8 +21,6 @@ #pragma once #include -#include -#include #include "arrow/flight/protocol_internal.h" #include "arrow/flight/transport/grpc/protocol_grpc_internal.h" @@ -30,32 +28,12 @@ #include "arrow/result.h" namespace arrow { - -class Buffer; - namespace flight { namespace transport { namespace grpc { namespace pb = arrow::flight::protocol; -/// Zero-copy conversion of a gRPC ByteBuffer into an Arrow Buffer. -/// -/// The Arrow Buffer owns a reference to the gRPC slice, so it stays valid -/// after the ByteBuffer goes away. -/// \param[in] cpp_buf The gRPC ByteBuffer to wrap. -/// \param[out] out The resulting Arrow Buffer that shares ownership of the gRPC slices. -/// \return Status indicating success or failure. -Status WrapGrpcBuffer(::grpc::ByteBuffer* cpp_buf, std::shared_ptr* out); - -/// Copy a gRPC ByteBuffer's slices into one contiguous string. -/// \param[in] buffer The gRPC ByteBuffer to copy from. -/// \param[in] error_message The error message to use if the buffer cannot be dumped. -/// \return The buffer's bytes, or an IOError carrying FlightStatusCode::Internal -/// on failure. -ARROW_FLIGHT_EXPORT arrow::Result BytesFromBuffer( - const ::grpc::ByteBuffer& buffer, std::string_view error_message); - /// Write Flight message on gRPC stream with zero-copy optimizations. // Returns Invalid if the payload is ill-formed // Returns true if the payload was written, false if it was not diff --git a/cpp/src/arrow/flight/types.cc b/cpp/src/arrow/flight/types.cc index 08e891557b42..c4b81184cc67 100644 --- a/cpp/src/arrow/flight/types.cc +++ b/cpp/src/arrow/flight/types.cc @@ -1052,9 +1052,6 @@ Status ResultStream::Drain() { } FlightStreamChunk::FlightStreamChunk() noexcept = default; -FlightStreamChunk::FlightStreamChunk(std::shared_ptr data, - std::shared_ptr app_metadata) noexcept - : data(std::move(data)), app_metadata(std::move(app_metadata)) {} FlightStreamChunk::~FlightStreamChunk() = default; arrow::Result>> diff --git a/cpp/src/arrow/flight/types.h b/cpp/src/arrow/flight/types.h index d3049a5e8eed..dd18c0e0786a 100644 --- a/cpp/src/arrow/flight/types.h +++ b/cpp/src/arrow/flight/types.h @@ -1166,8 +1166,6 @@ class ARROW_FLIGHT_EXPORT ResultStream { struct ARROW_FLIGHT_EXPORT FlightStreamChunk { public: FlightStreamChunk() noexcept; - FlightStreamChunk(std::shared_ptr data, - std::shared_ptr app_metadata) noexcept; ~FlightStreamChunk(); std::shared_ptr data; diff --git a/docs/source/cpp/api/flight.rst b/docs/source/cpp/api/flight.rst index 500911aa8668..a026989a503a 100644 --- a/docs/source/cpp/api/flight.rst +++ b/docs/source/cpp/api/flight.rst @@ -221,10 +221,6 @@ See :ref:`cpp-flight-async-server` for an overview. :project: arrow_cpp :members: -.. doxygenclass:: arrow::flight::AsyncFlightMessageDecoder - :project: arrow_cpp - :members: - Error Handling ============== diff --git a/docs/source/developers/cpp/flight.rst b/docs/source/developers/cpp/flight.rst index 65c50af890e8..476ccd282489 100644 --- a/docs/source/developers/cpp/flight.rst +++ b/docs/source/developers/cpp/flight.rst @@ -65,10 +65,10 @@ Dispatch ``AsyncGenericFlightService::CreateReactor()`` runs once per RPC on a gRPC thread. It builds the call's ``AsyncCallContext`` (running authentication and server middleware through ``GrpcServerCallContextHelper``), then creates the -reactor for the method. An unknown method, a Handshake with no handshake hook, -and a DoPut whose ``CreateDoPutListener()`` returns ``nullptr`` get a reactor -that only finishes the call as ``UNIMPLEMENTED``. Every other reactor derives -from ``AsyncReactorBase``; a handler the server class does not override answers +reactor for the method. An unknown method and a DoPut whose +``CreateDoPutListener()`` returns ``nullptr`` get a reactor that only finishes +the call as ``UNIMPLEMENTED``. Every other reactor derives from +``AsyncReactorBase``; a handler the server class does not override answers ``NotImplemented`` through that reactor. Reactor lifetime From 134afd301c33210f3e168f2da2b6d1f914678e4b Mon Sep 17 00:00:00 2001 From: Alexis Placet <2400067+Alex-PLACET@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:51:04 +0100 Subject: [PATCH 25/25] fix(flight): force the Abseil Mutex workaround into Flight translation units --- cpp/examples/arrow/CMakeLists.txt | 9 ++++ cpp/src/arrow/flight/CMakeLists.txt | 9 ++++ .../flight/transport/grpc/customize_absl.h | 35 ++++++++++++++ .../flight/transport/grpc/customize_grpc.h | 46 +------------------ 4 files changed, 54 insertions(+), 45 deletions(-) create mode 100644 cpp/src/arrow/flight/transport/grpc/customize_absl.h diff --git a/cpp/examples/arrow/CMakeLists.txt b/cpp/examples/arrow/CMakeLists.txt index 6d9332464a51..44b1a45c0e04 100644 --- a/cpp/examples/arrow/CMakeLists.txt +++ b/cpp/examples/arrow/CMakeLists.txt @@ -127,6 +127,15 @@ if(ARROW_FLIGHT) "${CMAKE_CURRENT_BINARY_DIR}/helloworld.pb.cc" "${CMAKE_CURRENT_BINARY_DIR}/helloworld.grpc.pb.cc") + if(MSVC) + set(FLIGHT_GRPC_EXAMPLE_FORCE_INCLUDE + "/FI${ARROW_SOURCE_DIR}/src/arrow/flight/transport/grpc/customize_absl.h") + else() + set(FLIGHT_GRPC_EXAMPLE_FORCE_INCLUDE + "-include" "${ARROW_SOURCE_DIR}/src/arrow/flight/transport/grpc/customize_absl.h") + endif() + target_compile_options(flight-grpc-example PRIVATE ${FLIGHT_GRPC_EXAMPLE_FORCE_INCLUDE}) + add_arrow_example(flight_async_generic_doget_example EXTRA_LINK_LIBS ${FLIGHT_EXAMPLES_LINK_LIBS} diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 45844a593a67..1f49d2b67d67 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -17,6 +17,15 @@ add_custom_target(arrow_flight) +if(MSVC) + set(ARROW_FLIGHT_FORCE_INCLUDE + "/FI${CMAKE_CURRENT_SOURCE_DIR}/transport/grpc/customize_absl.h") + add_compile_options($<$:${ARROW_FLIGHT_FORCE_INCLUDE}>) +else() + add_compile_options("$<$:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/transport/grpc/customize_absl.h>" + ) +endif() + arrow_install_all_headers("arrow/flight") # If libarrow_flight.a is only built, "pkg-config --cflags --libs diff --git a/cpp/src/arrow/flight/transport/grpc/customize_absl.h b/cpp/src/arrow/flight/transport/grpc/customize_absl.h new file mode 100644 index 000000000000..5fffe8589346 --- /dev/null +++ b/cpp/src/arrow/flight/transport/grpc/customize_absl.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if __has_include() + +# if !defined(_MSC_VER) || defined(ABSL_CONSUME_DLL) +# ifndef NDEBUG +# define ARROW_NO_NDEBUG +# define NDEBUG +# endif +# endif + +# include + +# ifdef ARROW_NO_NDEBUG +# undef NDEBUG +# endif + +#endif diff --git a/cpp/src/arrow/flight/transport/grpc/customize_grpc.h b/cpp/src/arrow/flight/transport/grpc/customize_grpc.h index 1974111709b4..d49cfdd9f103 100644 --- a/cpp/src/arrow/flight/transport/grpc/customize_grpc.h +++ b/cpp/src/arrow/flight/transport/grpc/customize_grpc.h @@ -20,51 +20,7 @@ #include #include -#if __has_include() - -// HACK: Workaround absl::Mutex ABI incompatibility by making sure the -// non-debug version of Abseil is included -// (https://github.com/conda-forge/abseil-cpp-feedstock/issues/104, -// https://github.com/abseil/abseil-cpp/issues/1624) -// -// Abseil picks where Mutex::Dtor lives based on NDEBUG: -// -// mutex.h: #if defined(NDEBUG) && !ABSL_HAVE_THREAD_SANITIZER && -// !ABSL_BUILD_DLL -> inline, emitted into this TU -// mutex.cc: #if !defined(NDEBUG) || ABSL_HAVE_THREAD_SANITIZER || -// ABSL_BUILD_DLL -> strong, out-of-line -// -// Defining NDEBUG here makes this TU emit its own inline Mutex::Dtor. Whether -// that helps or hurts depends on how the Abseil we link against was built: -// -// - Abseil built *with* NDEBUG (the conda case above): it inlined Dtor away -// and never exported it, so a debug TU that expects the out-of-line symbol -// gets an undefined reference. Emitting our own copy fixes that. -// -// - Abseil built *without* NDEBUG (a Debug vcpkg/system build): it has the -// strong out-of-line Dtor. Emitting our own copy is then a duplicate -// definition, which a *static* Abseil turns into LNK2005 (GH-49585). With a -// shared Abseil it is harmless, since the definitions live in separate -// images. -// -// So restrict the hack to the cases where it cannot produce a duplicate: -// anything that is not MSVC-linking-static-Abseil. On MSVC, Abseil's CMake puts -// ABSL_CONSUME_DLL in the INTERFACE compile definitions of its targets, so its -// presence tells us we are consuming the DLL rather than the static library. -# if !defined(_MSC_VER) || defined(ABSL_CONSUME_DLL) -# ifndef NDEBUG -# define ARROW_NO_NDEBUG -# define NDEBUG -# endif -# endif - -# include - -# ifdef ARROW_NO_NDEBUG -# undef NDEBUG -# endif - -#endif +#include "arrow/flight/transport/grpc/customize_absl.h" #include "arrow/flight/platform.h" #include "arrow/flight/type_fwd.h"