diff --git a/cpp/src/arrow/array/array_test.cc b/cpp/src/arrow/array/array_test.cc index 6e1bfed0d81e..f128f7236f04 100644 --- a/cpp/src/arrow/array/array_test.cc +++ b/cpp/src/arrow/array/array_test.cc @@ -4506,6 +4506,91 @@ TEST_F(TestHalfFloatBuilder, TestBulkAppend) { } } +TEST_F(TestArray, ArangeInvalidInputs) { + // step == 0 is always invalid + ASSERT_RAISES(Invalid, Arange(0, 10, 0, pool_)); + + // Byte count overflows uint64 (size = 2^63 - 1) + ASSERT_RAISES(Invalid, Arange(0, INT64_MAX, 1, pool_)); + + // Byte count fits uint64 but exceeds INT64_MAX (size = 2^60 + 8) + ASSERT_RAISES(Invalid, Arange(0, (1LL << 60) + 8, 1, pool_)); + + // Negative step over the full domain (range = 2^64 - 1) + ASSERT_RAISES(Invalid, Arange(INT64_MAX, INT64_MIN, -1, pool_)); +} + +TEST_F(TestArray, ArangeBasicPositiveStep) { + auto expected = ArrayFromJSON(int64(), "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"); + ASSERT_OK_AND_ASSIGN(auto result, Arange(0, 10, 1, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); + + expected = ArrayFromJSON(int64(), "[0, 3, 6, 9]"); + ASSERT_OK_AND_ASSIGN(result, Arange(0, 10, 3, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); + + expected = ArrayFromJSON(int64(), "[-5, -3, -1, 1, 3]"); + ASSERT_OK_AND_ASSIGN(result, Arange(-5, 5, 2, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); +} + +TEST_F(TestArray, ArangeBasicNegativeStep) { + auto expected = ArrayFromJSON(int64(), "[10, 9, 8, 7, 6, 5, 4, 3, 2, 1]"); + ASSERT_OK_AND_ASSIGN(auto result, Arange(10, 0, -1, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); + + expected = ArrayFromJSON(int64(), "[10, 7, 4, 1]"); + ASSERT_OK_AND_ASSIGN(result, Arange(10, 0, -3, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); + + expected = ArrayFromJSON(int64(), "[5, 2, -1, -4]"); + ASSERT_OK_AND_ASSIGN(result, Arange(5, -7, -3, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); +} + +TEST_F(TestArray, ArangeEmptyRanges) { + ASSERT_OK_AND_ASSIGN(auto result, Arange(10, 5, 1, pool_)); + ASSERT_EQ(result->length(), 0); + + ASSERT_OK_AND_ASSIGN(result, Arange(5, 10, -1, pool_)); + ASSERT_EQ(result->length(), 0); + + ASSERT_OK_AND_ASSIGN(result, Arange(5, 5, 1, pool_)); + ASSERT_EQ(result->length(), 0); +} + +TEST_F(TestArray, ArangeBoundaryValues) { + // Positive-step, with range = 2^64 - 1 + auto expected = ArrayFromJSON( + int64(), "[-9223372036854775808, -4611686018427387904, 0, 4611686018427387904]"); + ASSERT_OK_AND_ASSIGN(auto result, Arange(INT64_MIN, INT64_MAX, 1LL << 62, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); + + // Negative-step mirror, also with range = 2^64 - 1 + expected = ArrayFromJSON( + int64(), "[9223372036854775807, 4611686018427387903, -1, -4611686018427387905]"); + ASSERT_OK_AND_ASSIGN(result, Arange(INT64_MAX, INT64_MIN, -(1LL << 62), pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); + + // step == INT64_MIN is valid: |step| is computed in unsigned arithmetic + expected = ArrayFromJSON(int64(), "[10]"); + ASSERT_OK_AND_ASSIGN(result, Arange(10, 0, INT64_MIN, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); + + expected = ArrayFromJSON(int64(), "[9223372036854775807, -1]"); + ASSERT_OK_AND_ASSIGN(result, Arange(INT64_MAX, INT64_MIN, INT64_MIN, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); + + // Endpoints on the int64 rails + expected = ArrayFromJSON(int64(), "[-9223372036854775808, -9223372036854775807]"); + ASSERT_OK_AND_ASSIGN(result, Arange(INT64_MIN, INT64_MIN + 2, 1, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); + + expected = ArrayFromJSON(int64(), "[9223372036854775805, 9223372036854775806]"); + ASSERT_OK_AND_ASSIGN(result, Arange(INT64_MAX - 2, INT64_MAX, 1, pool_)); + ASSERT_ARRAYS_EQUAL(*expected, *result); +} + class TestDayTimeIntervalBuilder : public ::testing::Test { public: void VerifyValue(const DayTimeIntervalBuilder& builder, int64_t index, diff --git a/cpp/src/arrow/array/util.cc b/cpp/src/arrow/array/util.cc index d97e2f7f85d9..48d1e99622dd 100644 --- a/cpp/src/arrow/array/util.cc +++ b/cpp/src/arrow/array/util.cc @@ -43,6 +43,7 @@ #include "arrow/util/checked_cast.h" #include "arrow/util/decimal.h" #include "arrow/util/endian.h" +#include "arrow/util/int_util_overflow.h" #include "arrow/util/logging_internal.h" #include "arrow/util/sort_internal.h" #include "arrow/visit_data_inline.h" @@ -926,6 +927,53 @@ Result> MakeEmptyArray(std::shared_ptr type, return builder->Finish(); } +Result> Arange(int64_t start, int64_t stop, int64_t step, + MemoryPool* pool) { + if (step == 0) { + return Status::Invalid("Arange step cannot be zero"); + } + + auto error = [&]() { + return Status::Invalid("Arange: range [", start, ", ", stop, ") with step ", step, + " would produce an array that is too large"); + }; + + uint64_t size; + if (step > 0 && stop > start) { + const uint64_t range = static_cast(stop) - static_cast(start); + const auto step_u = static_cast(step); + size = (range - 1) / step_u + 1; + } else if (step < 0 && stop < start) { + const uint64_t range = static_cast(start) - static_cast(stop); + const uint64_t step_u = 0 - static_cast(step); + size = (range - 1) / step_u + 1; + } else { + return MakeEmptyArray(int64(), pool); + } + + uint64_t buffer_size_u; + if (internal::MultiplyWithOverflow(size, sizeof(int64_t), &buffer_size_u) || + buffer_size_u > static_cast(std::numeric_limits::max())) { + return error(); + } + const auto size_i64 = static_cast(size); + const auto buffer_size = static_cast(buffer_size_u); + + ARROW_ASSIGN_OR_RAISE(auto data_buffer, AllocateBuffer(buffer_size, pool)); + + auto values = reinterpret_cast(data_buffer->mutable_data()); + int64_t value = start; + int64_t i = 0; + for (; i < size_i64 - 1; ++i) { + values[i] = value; + value += step; + } + values[i] = value; + + auto data = ArrayData::Make(int64(), size_i64, {nullptr, std::move(data_buffer)}, 0); + return MakeArray(data); +} + namespace internal { std::vector RechunkArraysConsistently( diff --git a/cpp/src/arrow/array/util.h b/cpp/src/arrow/array/util.h index fd8e75ddb864..2e9a992c2523 100644 --- a/cpp/src/arrow/array/util.h +++ b/cpp/src/arrow/array/util.h @@ -69,6 +69,21 @@ ARROW_EXPORT Result> MakeEmptyArray(std::shared_ptr type, MemoryPool* pool = default_memory_pool()); +/// \brief Create an array of evenly spaced integer values +/// +/// Generates a sequence of integers from start (inclusive) to stop (exclusive) +/// with the given step, similar to Python's range() or NumPy's arange(). +/// The resulting array will have type Int64. +/// +/// \param[in] start First value in the sequence +/// \param[in] stop End value (exclusive) +/// \param[in] step Spacing between values (must not be zero) +/// \param[in] pool the memory pool to allocate memory from +/// \return the resulting Int64Array +ARROW_EXPORT +Result> Arange(int64_t start, int64_t stop, int64_t step, + MemoryPool* pool = default_memory_pool()); + /// @} namespace internal { diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 750f0f06f576..4db1ec7045cf 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -405,8 +405,7 @@ set(PYARROW_CPP_SRCS ${PYARROW_CPP_SOURCE_DIR}/python_test.cc ${PYARROW_CPP_SOURCE_DIR}/python_to_arrow.cc ${PYARROW_CPP_SOURCE_DIR}/pyarrow.cc - ${PYARROW_CPP_SOURCE_DIR}/udf.cc - ${PYARROW_CPP_SOURCE_DIR}/util.cc) + ${PYARROW_CPP_SOURCE_DIR}/udf.cc) set_source_files_properties(${PYARROW_CPP_SOURCE_DIR}/numpy_init.cc PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 296148f52816..25a9aa8da6f5 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -290,6 +290,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: CResult[shared_ptr[CArray]] MakeArrayFromScalar( const CScalar& scalar, int64_t length, CMemoryPool* pool) + CResult[shared_ptr[CArray]] Arange(int64_t start, int64_t stop, + int64_t step, CMemoryPool* pool) + CStatus DebugPrint(const CArray& arr, int indent) cdef cppclass CFixedWidthType" arrow::FixedWidthType"(CDataType): diff --git a/python/pyarrow/includes/libarrow_python.pxd b/python/pyarrow/includes/libarrow_python.pxd index 385a2924d1da..e6fbd23e5913 100644 --- a/python/pyarrow/includes/libarrow_python.pxd +++ b/python/pyarrow/includes/libarrow_python.pxd @@ -73,9 +73,6 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: object obj, object mask, const PyConversionOptions& options, CMemoryPool* pool) - CResult[shared_ptr[CArray]] Arange(int64_t start, int64_t stop, - int64_t step, CMemoryPool* pool) - CResult[shared_ptr[CDataType]] NumPyDtypeToArrow(object dtype) CStatus NdarrayToArrow(CMemoryPool* pool, object ao, object mo, diff --git a/python/pyarrow/src/arrow/python/api.h b/python/pyarrow/src/arrow/python/api.h index 2af0963a9c04..e66bf49dfecf 100644 --- a/python/pyarrow/src/arrow/python/api.h +++ b/python/pyarrow/src/arrow/python/api.h @@ -26,4 +26,3 @@ #include "arrow/python/numpy_convert.h" #include "arrow/python/numpy_to_arrow.h" #include "arrow/python/python_to_arrow.h" -#include "arrow/python/util.h" diff --git a/python/pyarrow/src/arrow/python/util.cc b/python/pyarrow/src/arrow/python/util.cc deleted file mode 100644 index cffe1eb956b3..000000000000 --- a/python/pyarrow/src/arrow/python/util.cc +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "arrow/python/util.h" - -#include "arrow/array.h" -#include "arrow/python/common.h" - -namespace arrow ::py { - -Result> Arange(int64_t start, int64_t stop, int64_t step, - MemoryPool* pool) { - int64_t size; - if (step == 0) { - return Status::Invalid("Step must not be zero"); - } - if (step > 0 && stop > start) { - // Ceiling division for positive step - size = (stop - start + step - 1) / step; - } else if (step < 0 && stop < start) { - // Ceiling division for negative step - size = (start - stop - step - 1) / (-step); - } else { - return MakeEmptyArray(int64()); - } - std::shared_ptr data_buffer; - ARROW_ASSIGN_OR_RAISE(data_buffer, AllocateBuffer(size * sizeof(int64_t), pool)); - auto values = reinterpret_cast(data_buffer->mutable_data()); - for (int64_t i = 0; i < size; ++i) { - values[i] = start + i * step; - } - auto data = ArrayData::Make(int64(), size, {nullptr, data_buffer}, 0); - return MakeArray(data); -} - -} // namespace arrow::py diff --git a/python/pyarrow/src/arrow/python/util.h b/python/pyarrow/src/arrow/python/util.h deleted file mode 100644 index ff2ffcaea9cf..000000000000 --- a/python/pyarrow/src/arrow/python/util.h +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "arrow/python/common.h" -#include "arrow/python/visibility.h" - -namespace arrow::py { - -/// \brief Create an array of evenly spaced values within a given interval. -/// This function is similar to Python's `range` function. -/// The resulting array will contain values starting from `start` up to but not -/// including `stop`, with a step size of `step`. If `step` is zero, the function -/// will return an error. -/// The resulting array will have a data type of `int64`. -/// \param[in] start initial value of the sequence. -/// \param[in] stop final value of the sequence (exclusive). -/// \param[in] step step size between consecutive values. -/// \param[in] pool Memory pool for any memory allocations. -/// \return Result Array -ARROW_PYTHON_EXPORT -Result> Arange(int64_t start, int64_t stop, int64_t step, - MemoryPool* pool); - -} // namespace arrow::py