Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 95 additions & 23 deletions cpp/src/arrow/dataset/scanner.cc
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,41 @@ Status ScanV2Options::AddFieldsNeededForFilter(ScanV2Options* options) {
}

namespace {
void StopScan(const std::weak_ptr<acero::ExecPlan>& weak_plan) {
auto plan = weak_plan.lock();
if (plan && !plan->finished().is_finished()) {
plan->StopProducing();
}
}

/// \brief An iterator over a scan that stops the scan when destroyed early
///
/// Without threads, destroying the underlying iterator runs the rest of the scan, so
/// the plan is stopped first. The underlying iterator owns the plan and releases it,
/// with the files it reads, at the end of the scan, so only a weak reference is kept.
template <typename T>
class StoppingIterator {
public:
StoppingIterator(Iterator<T> it, std::weak_ptr<acero::ExecPlan> plan)
: it_(std::move(it)), plan_(std::move(plan)) {}
StoppingIterator(StoppingIterator&&) = default;
~StoppingIterator() { StopScan(plan_); }

Result<T> Next() { return it_.Next(); }

private:
Iterator<T> it_;
std::weak_ptr<acero::ExecPlan> plan_;
};

class ScannerRecordBatchReader : public RecordBatchReader {
public:
explicit ScannerRecordBatchReader(std::shared_ptr<Schema> schema,
TaggedRecordBatchIterator delegate)
: schema_(std::move(schema)), delegate_(std::move(delegate)) {}
ScannerRecordBatchReader(std::shared_ptr<Schema> schema,
TaggedRecordBatchIterator delegate,
std::weak_ptr<acero::ExecPlan> plan)
: schema_(std::move(schema)),
delegate_(std::move(delegate)),
plan_(std::move(plan)) {}

std::shared_ptr<Schema> schema() const override { return schema_; }
Status ReadNext(std::shared_ptr<RecordBatch>* batch) override {
Expand All @@ -112,17 +142,23 @@ class ScannerRecordBatchReader : public RecordBatchReader {
}

Status Close() override {
// End the scan and read from the iterator until the plan has finished
StopScan(plan_);
std::shared_ptr<RecordBatch> batch;
RETURN_NOT_OK(ReadNext(&batch));
while (batch != nullptr) {
RETURN_NOT_OK(ReadNext(&batch));
}
return Status::OK();
Status st;
do {
st = ReadNext(&batch);
} while (st.ok() && batch != nullptr);
// Release the scan now rather than when the reader is destroyed
delegate_ = TaggedRecordBatchIterator();
// A stopped plan finishes with Cancelled
return st.IsCancelled() ? Status::OK() : st;
}

private:
std::shared_ptr<Schema> schema_;
TaggedRecordBatchIterator delegate_;
std::weak_ptr<acero::ExecPlan> plan_;
};

const FieldVector kAugmentedFields{
Expand Down Expand Up @@ -282,8 +318,17 @@ class AsyncScanner : public Scanner, public std::enable_shared_from_this<AsyncSc
private:
Future<> VisitBatchesAsync(std::function<Status(TaggedRecordBatch)> visitor,
Executor* executor);
// Iterate over a scan synchronously, with an iterator that stops the scan when it is
// destroyed early. If plan is not null, it is set to the scan's plan.
template <typename T, typename MakeGenerator>
Iterator<T> IterateScan(MakeGenerator&& make_generator,
std::weak_ptr<acero::ExecPlan>* plan = nullptr);
// If plan is not null, it is set to the plan backing the returned generator
Result<TaggedRecordBatchGenerator> ScanBatchesAsyncImpl(
Executor* executor, std::shared_ptr<acero::ExecPlan>* plan);
Result<EnumeratedRecordBatchGenerator> ScanBatchesUnorderedAsync(
Executor* executor, bool sequence_fragments, bool use_legacy_batching = false);
Executor* executor, bool sequence_fragments, bool use_legacy_batching = false,
std::shared_ptr<acero::ExecPlan>* plan = nullptr);
Future<std::shared_ptr<Table>> ToTableAsync(Executor* executor);
Future<int64_t> CountRowsAsync(Executor* executor);

Expand Down Expand Up @@ -383,20 +428,33 @@ Result<FragmentGenerator> AsyncScanner::GetFragments() const {
return MakeVectorGenerator(std::move(fragments_vec));
}

Result<TaggedRecordBatchIterator> AsyncScanner::ScanBatches() {
return ::arrow::internal::IterateSynchronously<TaggedRecordBatch>(
[this](::arrow::internal::Executor* executor) {
return ScanBatchesAsync(executor);
},
template <typename T, typename MakeGenerator>
Iterator<T> AsyncScanner::IterateScan(MakeGenerator&& make_generator,
std::weak_ptr<acero::ExecPlan>* out_plan) {
std::shared_ptr<acero::ExecPlan> plan;
// IterateSynchronously calls the function before it returns
auto it = ::arrow::internal::IterateSynchronously<T>(
[&](Executor* executor) { return make_generator(executor, &plan); },
scan_options_->use_threads, scan_options_->cpu_executor);
if (out_plan != nullptr) {
*out_plan = plan;
}
return Iterator<T>(StoppingIterator<T>(std::move(it), plan));
}

Result<TaggedRecordBatchIterator> AsyncScanner::ScanBatches() {
return IterateScan<TaggedRecordBatch>(
[this](Executor* executor, std::shared_ptr<acero::ExecPlan>* plan) {
return ScanBatchesAsyncImpl(executor, plan);
});
}

Result<EnumeratedRecordBatchIterator> AsyncScanner::ScanBatchesUnordered() {
return ::arrow::internal::IterateSynchronously<EnumeratedRecordBatch>(
[this](::arrow::internal::Executor* executor) {
return ScanBatchesUnorderedAsync(executor);
},
scan_options_->use_threads, scan_options_->cpu_executor);
return IterateScan<EnumeratedRecordBatch>(
[this](Executor* executor, std::shared_ptr<acero::ExecPlan>* plan) {
return ScanBatchesUnorderedAsync(executor, /*sequence_fragments=*/false,
/*use_legacy_batching=*/false, plan);
});
}

Result<std::shared_ptr<Table>> AsyncScanner::ToTable() {
Expand Down Expand Up @@ -435,7 +493,8 @@ Result<EnumeratedRecordBatch> ToEnumeratedRecordBatch(
}

Result<EnumeratedRecordBatchGenerator> AsyncScanner::ScanBatchesUnorderedAsync(
Executor* cpu_executor, bool sequence_fragments, bool use_legacy_batching) {
Executor* cpu_executor, bool sequence_fragments, bool use_legacy_batching,
std::shared_ptr<acero::ExecPlan>* out_plan) {
RETURN_NOT_OK(NormalizeScanOptions(scan_options_, dataset_->schema()));

auto exec_context =
Expand Down Expand Up @@ -491,6 +550,9 @@ Result<EnumeratedRecordBatchGenerator> AsyncScanner::ScanBatchesUnorderedAsync(
return ToEnumeratedRecordBatch(batch, *options, *shared_fragments);
});

if (out_plan != nullptr) {
*out_plan = plan;
}
return [mapped_gen = std::move(mapped_gen), plan = std::move(plan),
stop_producing = std::move(stop_producing)] {
auto next = mapped_gen();
Expand Down Expand Up @@ -616,9 +678,14 @@ Result<TaggedRecordBatchGenerator> AsyncScanner::ScanBatchesAsync() {

Result<TaggedRecordBatchGenerator> AsyncScanner::ScanBatchesAsync(
Executor* cpu_executor) {
return ScanBatchesAsyncImpl(cpu_executor, /*plan=*/nullptr);
}

Result<TaggedRecordBatchGenerator> AsyncScanner::ScanBatchesAsyncImpl(
Executor* cpu_executor, std::shared_ptr<acero::ExecPlan>* plan) {
ARROW_ASSIGN_OR_RAISE(
auto unordered, ScanBatchesUnorderedAsync(cpu_executor, /*sequence_fragments=*/true,
/*use_legacy_batching=*/true));
/*use_legacy_batching=*/true, plan));
// We need an initial value sentinel, so we use one with fragment.index < 0
auto is_before_any = [](const EnumeratedRecordBatch& batch) {
return batch.fragment.index < 0;
Expand Down Expand Up @@ -800,9 +867,14 @@ Result<int64_t> AsyncScanner::CountRows() {
}

Result<std::shared_ptr<RecordBatchReader>> AsyncScanner::ToRecordBatchReader() {
ARROW_ASSIGN_OR_RAISE(auto it, ScanBatches());
std::weak_ptr<acero::ExecPlan> scan_plan;
auto it = IterateScan<TaggedRecordBatch>(
[this](Executor* executor, std::shared_ptr<acero::ExecPlan>* plan) {
return ScanBatchesAsyncImpl(executor, plan);
},
&scan_plan);
return std::make_shared<ScannerRecordBatchReader>(options()->projected_schema,
std::move(it));
std::move(it), std::move(scan_plan));
}

const std::shared_ptr<Dataset>& AsyncScanner::dataset() const { return dataset_; }
Expand Down
10 changes: 9 additions & 1 deletion cpp/src/arrow/dataset/scanner.h
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,9 @@ class ARROW_DS_EXPORT Scanner {
/// much in memory (this is goverended by the readahead options and use_threads option).
/// If the readahead queue fills up then I/O will pause until the calling thread catches
/// up.
///
/// Destroying the iterator before the end of the stream stops the scan. Reads that
/// were already in flight may finish in the background afterwards.
virtual Result<TaggedRecordBatchIterator> ScanBatches() = 0;
virtual Result<TaggedRecordBatchGenerator> ScanBatchesAsync() = 0;
virtual Result<TaggedRecordBatchGenerator> ScanBatchesAsync(
Expand All @@ -434,7 +437,8 @@ class ARROW_DS_EXPORT Scanner {
/// may be cached in RAM or just happen to get scheduled earlier by the I/O)
///
/// To make up for the out-of-order iteration each batch is further tagged with
/// positional information.
/// positional information. As with ScanBatches, destroying the iterator early stops
/// the scan.
virtual Result<EnumeratedRecordBatchIterator> ScanBatchesUnordered() = 0;
virtual Result<EnumeratedRecordBatchGenerator> ScanBatchesUnorderedAsync() = 0;
virtual Result<EnumeratedRecordBatchGenerator> ScanBatchesUnorderedAsync(
Expand All @@ -453,6 +457,10 @@ class ARROW_DS_EXPORT Scanner {
virtual Future<int64_t> CountRowsAsync() = 0;
/// \brief Convert the Scanner to a RecordBatchReader so it can be
/// easily used with APIs that expect a reader.
///
/// Closing or destroying the reader before the end of the stream stops the scan.
/// Close() returns once the scan's plan has finished, but reads that were already in
/// flight may finish in the background afterwards.
virtual Result<std::shared_ptr<RecordBatchReader>> ToRecordBatchReader() = 0;

/// \brief Get the options for this scan.
Expand Down
124 changes: 124 additions & 0 deletions cpp/src/arrow/dataset/scanner_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2044,6 +2044,130 @@ TEST_F(TestBackpressure, ScanBatchesOrdered) {
Finish(std::move(gen));
}

// A ControlledFragment whose batches are handed over by tasks on an executor, so that
// the scan waits for each batch like it does when reading a file
class ExecutorControlledFragment : public ControlledFragment {
public:
ExecutorControlledFragment(std::shared_ptr<Schema> schema,
::arrow::internal::Executor* executor)
: ControlledFragment(std::move(schema)), executor_(executor) {}

Result<RecordBatchGenerator> ScanBatchesAsync(
const std::shared_ptr<ScanOptions>& options) override {
ARROW_ASSIGN_OR_RAISE(auto gen, ControlledFragment::ScanBatchesAsync(options));
return [gen, executor = executor_]() { return DeferNotOk(executor->Submit(gen)); };
}

private:
::arrow::internal::Executor* executor_;
};

// GH-40653: stopping a scan early should not read the rest of the dataset
class TestScanEarlyStop : public ::testing::TestWithParam<bool> {
protected:
static constexpr int kNumBatches = 1000;
static constexpr int kNumRows = 10;
// Without threads, batches delivered before the first read. The scan runs only while
// it is read, so the rest can be delivered after the first batch has been read.
static constexpr int kFirstBatches = 10;

void SetUp() override {
ASSERT_OK_AND_ASSIGN(thread_pool_, ::arrow::internal::ThreadPool::Make(1));
auto options = std::make_shared<ScanOptions>();
if (GetParam()) {
fragment_ =
std::make_shared<ExecutorControlledFragment>(schema_, thread_pool_.get());
options->cpu_executor = thread_pool_.get();
DeliverBatches(kNumBatches);
fragment_->Finish();
} else {
fragment_ = std::make_shared<ControlledFragment>(schema_);
DeliverBatches(kFirstBatches);
}
ScannerBuilder builder(
std::make_shared<FragmentDataset>(schema_, FragmentVector{fragment_}), options);
ASSERT_OK(builder.UseThreads(GetParam()));
// With threads, backpressure pauses the scan while nothing is read
ASSERT_OK(builder.Backpressure(
acero::BackpressureOptions(kResumeIfBelowBytes, kPauseIfAboveBytes)));
ASSERT_OK_AND_ASSIGN(scanner_, builder.Finish());
}

void DeliverBatches(int num_batches) {
for (int i = 0; i < num_batches; i++) {
fragment_->DeliverBatch(kNumRows);
}
}

// Without threads, deliver the rest of the batches after the first one has been read
void DeliverRemainingBatches() {
if (!GetParam()) {
DeliverBatches(kNumBatches - kFirstBatches);
fragment_->Finish();
}
}

void AssertStoppedEarly() {
// Let the tasks queued before the scan stopped run before the pool is destroyed
thread_pool_->WaitForIdle();
// The scan stopped early: fewer than a tenth of the batches were requested
ASSERT_LT(fragment_->NumBatchesRead(), kNumBatches / 10);
}

std::shared_ptr<::arrow::internal::ThreadPool> thread_pool_;
std::shared_ptr<Schema> schema_ = schema({field("values", int32())});
std::shared_ptr<ControlledFragment> fragment_;
std::shared_ptr<Scanner> scanner_;
};

TEST_P(TestScanEarlyStop, CloseReader) {
ASSERT_OK_AND_ASSIGN(auto reader, scanner_->ToRecordBatchReader());
ASSERT_OK_AND_ASSIGN(auto batch, reader->Next());
ASSERT_NE(batch, nullptr);
DeliverRemainingBatches();
ASSERT_OK(reader->Close());
AssertStoppedEarly();
// The reader is exhausted and can be closed again
ASSERT_OK_AND_ASSIGN(batch, reader->Next());
ASSERT_EQ(batch, nullptr);
ASSERT_OK(reader->Close());
}

TEST_P(TestScanEarlyStop, CloseReaderAfterEnd) {
ASSERT_OK_AND_ASSIGN(auto reader, scanner_->ToRecordBatchReader());
ASSERT_OK_AND_ASSIGN(auto batch, reader->Next());
ASSERT_NE(batch, nullptr);
DeliverRemainingBatches();
ASSERT_OK_AND_ASSIGN(auto table, reader->ToTable());
ASSERT_EQ(table->num_rows(), (kNumBatches - 1) * kNumRows);
ASSERT_OK(reader->Close());
thread_pool_->WaitForIdle();
}

TEST_P(TestScanEarlyStop, DestroyReader) {
ASSERT_OK_AND_ASSIGN(auto reader, scanner_->ToRecordBatchReader());
ASSERT_OK_AND_ASSIGN(auto batch, reader->Next());
ASSERT_NE(batch, nullptr);
DeliverRemainingBatches();
reader.reset();
AssertStoppedEarly();
}

// Head(), TakeRows() and pyarrow's to_batches() stop reading this iterator early
TEST_P(TestScanEarlyStop, DestroyIterator) {
ASSERT_OK_AND_ASSIGN(auto batch_it, scanner_->ScanBatches());
ASSERT_OK_AND_ASSIGN(auto batch, batch_it.Next());
ASSERT_NE(batch.record_batch, nullptr);
DeliverRemainingBatches();
batch_it = TaggedRecordBatchIterator();
AssertStoppedEarly();
}

INSTANTIATE_TEST_SUITE_P(TestScanEarlyStop, TestScanEarlyStop, ::testing::Bool(),
[](const ::testing::TestParamInfo<bool>& info) {
return info.param ? "Threaded" : "Serial";
});

struct BatchConsumer {
explicit BatchConsumer(EnumeratedRecordBatchGenerator generator)
: generator(std::move(generator)), next() {}
Expand Down
Loading