Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions cpp/src/arrow/array/array_list_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1048,6 +1048,43 @@ class TestListConversions : public ::testing::Test {
*checked_pointer_cast<SrcListViewArrayClass>(list_view_wo_nulls), pool_));
ASSERT_OK(result->ValidateFull());
AssertArraysEqual(*expected_list_wo_nulls, *result, /*verbose=*/true);

// The views are contiguous, so the values are kept as they are (GH-51612).
ASSERT_EQ(result->values()->data()->buffers[1].get(),
checked_pointer_cast<SrcListViewArrayClass>(list_view_wo_nulls)
->values()
->data()
->buffers[1]
.get());
}

template <typename DestListType, typename SrcListViewType>
void DoTestListFromSlicedListView() {
using SrcListViewArrayClass = typename TypeTraits<SrcListViewType>::ArrayType;
using DestListArrayClass = typename TypeTraits<DestListType>::ArrayType;
auto list_view_type = std::make_shared<SrcListViewType>(int32());
auto list_type = std::make_shared<DestListType>(int32());

auto list_view =
ArrayFromJSON(list_view_type, "[[1, 2], [3], null, [], [4], [5, 6], null, [7]]");

auto check = [&](const Array& sliced, const std::shared_ptr<Array>& expected) {
ASSERT_OK_AND_ASSIGN(
auto result, DestListArrayClass::FromListView(
checked_cast<const SrcListViewArrayClass&>(sliced), pool_));
ASSERT_OK(result->ValidateFull());
AssertArraysEqual(*expected, *result, /*verbose=*/true);
};

// A slice whose element offset is not a multiple of 8, so the bitmap's bit
// offset and byte offset differ (GH-51613).
check(*list_view->Slice(1, 6),
ArrayFromJSON(list_type, "[[3], null, [], [4], [5, 6], null]"));
// A slice starting on a byte boundary, so that the bit offset is zero.
check(*list_view->Slice(2, 4), ArrayFromJSON(list_type, "[null, [], [4], [5, 6]]"));
// A slice whose only null is the last slot.
check(*list_view->Slice(0, 8),
ArrayFromJSON(list_type, "[[1, 2], [3], null, [], [4], [5, 6], null, [7]]"));
}
};

Expand All @@ -1061,6 +1098,11 @@ TEST_F(TestListConversions, ListFromListView) {
this->DoTestListFromListView<LargeListType, LargeListViewType>();
}

TEST_F(TestListConversions, ListFromSlicedListView) {
this->DoTestListFromSlicedListView<ListType, ListViewType>();
this->DoTestListFromSlicedListView<LargeListType, LargeListViewType>();
}

// ----------------------------------------------------------------------
// Map tests

Expand Down
33 changes: 2 additions & 31 deletions cpp/src/arrow/array/array_nested.cc
Original file line number Diff line number Diff line change
Expand Up @@ -415,37 +415,8 @@ Result<std::shared_ptr<ArrayData>> ListFromListViewImpl(
std::is_same<typename SrcListViewType::offset_type,
typename DestListType::offset_type>::value,
"Offset types between list type and list-view type are expected to match");
using offset_type = typename DestListType::offset_type;
using ListBuilderType = typename TypeTraits<DestListType>::BuilderType;

const auto& list_view_type =
checked_cast<const SrcListViewType&>(*list_view_data->type);
const auto& value_type = list_view_type.value_type();
const auto list_type = std::make_shared<DestListType>(value_type);

ARROW_ASSIGN_OR_RAISE(auto sum_of_list_view_sizes,
list_util::internal::SumOfLogicalListSizes(*list_view_data));
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ArrayBuilder> value_builder,
MakeBuilder(value_type, pool));
RETURN_NOT_OK(value_builder->Reserve(sum_of_list_view_sizes));
auto list_builder = std::make_shared<ListBuilderType>(pool, value_builder, list_type);
RETURN_NOT_OK(list_builder->Reserve(list_view_data->length));

ArraySpan values{*list_view_data->child_data[0]};
const auto* in_validity_bitmap = list_view_data->GetValues<uint8_t>(0);
const auto* in_offsets = list_view_data->GetValues<offset_type>(1);
const auto* in_sizes = list_view_data->GetValues<offset_type>(2);
for (int64_t i = 0; i < list_view_data->length; ++i) {
const bool is_valid =
!in_validity_bitmap ||
bit_util::GetBit(in_validity_bitmap, list_view_data->offset + i);
const int64_t size = is_valid ? in_sizes[i] : 0;
RETURN_NOT_OK(list_builder->Append(is_valid, size));
RETURN_NOT_OK(value_builder->AppendArraySlice(values, in_offsets[i], size));
}
std::shared_ptr<ArrayData> list_array_data;
RETURN_NOT_OK(list_builder->FinishInternal(&list_array_data));
return list_array_data;
return list_util::internal::ListFromListView(ArraySpan{*list_view_data},
DestListType::type_id, pool);
Comment on lines +418 to +419

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The more common order is a function that takes dynamically typed values to delegate to the statically typed (templated/specialized). Here the opposite is happening.

This template has the types already worked out, but then it delegates to a function that is going to switch on the type ids again.

}

} // namespace
Expand Down
31 changes: 31 additions & 0 deletions cpp/src/arrow/compute/kernels/scalar_cast_nested.cc
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include "arrow/compute/kernels/scalar_cast_internal.h"
#include "arrow/util/bitmap_ops.h"
#include "arrow/util/int_util.h"
#include "arrow/util/list_util.h"
#include "arrow/util/logging_internal.h"

namespace arrow {
Expand Down Expand Up @@ -105,7 +106,37 @@ struct CastList {
return Status::OK();
}

/// \brief Cast a list-view array to a (large)list array
///
/// The values referenced by a list-view have to be gathered according to the
/// views' offsets and sizes, so the views' offsets buffer cannot be reused as
/// a list-offsets buffer (GH-51612).
static Status ExecListView(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
const CastOptions& options = CastState::Get(ctx);
auto child_type = checked_cast<const DestType&>(*out->type()).value_type();

ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ArrayData> list_data,
list_util::internal::ListFromListView(
batch[0].array, DestType::type_id, ctx->memory_pool()));

ARROW_ASSIGN_OR_RAISE(Datum cast_values, Cast(list_data->child_data[0], child_type,
options, ctx->exec_context()));
DCHECK(cast_values.is_array());

ArrayData* out_array = out->array_data().get();
out_array->buffers[0] = std::move(list_data->buffers[0]);
out_array->buffers[1] = std::move(list_data->buffers[1]);
out_array->null_count.store(list_data->null_count.load());
out_array->offset = list_data->offset;
out_array->child_data.push_back(cast_values.array());
return Status::OK();
}

static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
if constexpr (is_list_view(SrcType::type_id)) {
return ExecListView(ctx, batch, out);
}

Comment on lines +136 to +139

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a weird short-circuit for this logic that is meant to be generic for nested arrays. It would make more sense to have a branch around HandleOffsets and the handling of values. And what you want is not always a cast of the entire array to list-view. You want to make the cast values packed in sequence according to the slices created by the offset-size pairs.

From those values you can decide to output a list-view or a *list. Always returning list is going to be weird (i.e. list-view in, list out).

const CastOptions& options = CastState::Get(ctx);

auto child_type = checked_cast<const DestType&>(*out->type()).value_type();
Expand Down
Loading
Loading