diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index dac7b4609c56..e4bfe34545d6 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -1708,6 +1708,10 @@ parquet___arrow___FileReader__num_columns <- function(reader) { .Call(`_arrow_parquet___arrow___FileReader__num_columns`, reader) } +parquet___arrow___FileReader__FieldIndicesToColumnIndices <- function(reader, field_indices) { + .Call(`_arrow_parquet___arrow___FileReader__FieldIndicesToColumnIndices`, reader, field_indices) +} + parquet___arrow___FileReader__num_row_groups <- function(reader) { .Call(`_arrow_parquet___arrow___FileReader__num_row_groups`, reader) } diff --git a/r/R/parquet.R b/r/R/parquet.R index 55638c66f098..cf8169771f64 100644 --- a/r/R/parquet.R +++ b/r/R/parquet.R @@ -60,7 +60,13 @@ read_parquet <- function( if (!quo_is_null(col_select)) { # infer which columns to keep from schema sim_df <- as.data.frame(reader$GetSchema()) - indices <- eval_select(col_select, sim_df) - 1L + field_indices <- eval_select(col_select, sim_df) - 1L + # ReadTable() takes Parquet leaf column indices, which differ from the + # positions of the fields in the schema when there are nested columns + indices <- parquet___arrow___FileReader__FieldIndicesToColumnIndices( + reader, + field_indices + ) tab <- tryCatch( reader$ReadTable(indices), error = read_compressed_error @@ -489,6 +495,12 @@ ParquetFileWriter$create <- function( #' - `$GetSchema()`: get the `arrow::Schema` of the data in the file #' - `$ReadColumn(i)`: read the `i`th column (0-based) as a [ChunkedArray]. #' +#' `column_indices` refer to Parquet leaf columns, not to the fields of the +#' schema returned by `$GetSchema()`. The two differ when the file has nested +#' columns, as each nested field is stored as one leaf column per child. Selecting +#' only some of a nested field's leaves returns that field containing only the +#' selected children. +#' #' @section Active bindings: #' #' - `$num_rows`: number of rows. diff --git a/r/man/ParquetFileReader.Rd b/r/man/ParquetFileReader.Rd index 8638e24c747a..a04b15201a0e 100644 --- a/r/man/ParquetFileReader.Rd +++ b/r/man/ParquetFileReader.Rd @@ -35,6 +35,12 @@ The optional \verb{column_indices=} argument is a 0-based integer vector indicat \item \verb{$GetSchema()}: get the \code{arrow::Schema} of the data in the file \item \verb{$ReadColumn(i)}: read the \code{i}th column (0-based) as a \link{ChunkedArray}. } + +\code{column_indices} refer to Parquet leaf columns, not to the fields of the +schema returned by \verb{$GetSchema()}. The two differ when the file has nested +columns, as each nested field is stored as one leaf column per child. Selecting +only some of a nested field's leaves returns that field containing only the +selected children. } \section{Active bindings}{ diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 4b47e635f7ec..7acc0ef38c9f 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -4474,6 +4474,22 @@ extern "C" SEXP _arrow_parquet___arrow___FileReader__num_columns(SEXP reader_sex } #endif +// parquet.cpp +#if defined(ARROW_R_WITH_PARQUET) +std::vector parquet___arrow___FileReader__FieldIndicesToColumnIndices(const std::shared_ptr& reader, const std::vector& field_indices); +extern "C" SEXP _arrow_parquet___arrow___FileReader__FieldIndicesToColumnIndices(SEXP reader_sexp, SEXP field_indices_sexp){ +BEGIN_CPP11 + arrow::r::Input&>::type reader(reader_sexp); + arrow::r::Input&>::type field_indices(field_indices_sexp); + return cpp11::as_sexp(parquet___arrow___FileReader__FieldIndicesToColumnIndices(reader, field_indices)); +END_CPP11 +} +#else +extern "C" SEXP _arrow_parquet___arrow___FileReader__FieldIndicesToColumnIndices(SEXP reader_sexp, SEXP field_indices_sexp){ + Rf_error("Cannot call parquet___arrow___FileReader__FieldIndicesToColumnIndices(). See https://arrow.apache.org/docs/r/articles/install.html for help installing Arrow C++ libraries. "); +} +#endif + // parquet.cpp #if defined(ARROW_R_WITH_PARQUET) int parquet___arrow___FileReader__num_row_groups(const std::shared_ptr& reader); @@ -6194,6 +6210,7 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_parquet___arrow___FileReader__ReadRowGroups2", (DL_FUNC) &_arrow_parquet___arrow___FileReader__ReadRowGroups2, 3}, { "_arrow_parquet___arrow___FileReader__num_rows", (DL_FUNC) &_arrow_parquet___arrow___FileReader__num_rows, 1}, { "_arrow_parquet___arrow___FileReader__num_columns", (DL_FUNC) &_arrow_parquet___arrow___FileReader__num_columns, 1}, + { "_arrow_parquet___arrow___FileReader__FieldIndicesToColumnIndices", (DL_FUNC) &_arrow_parquet___arrow___FileReader__FieldIndicesToColumnIndices, 2}, { "_arrow_parquet___arrow___FileReader__num_row_groups", (DL_FUNC) &_arrow_parquet___arrow___FileReader__num_row_groups, 1}, { "_arrow_parquet___arrow___FileReader__ReadColumn", (DL_FUNC) &_arrow_parquet___arrow___FileReader__ReadColumn, 2}, { "_arrow_parquet___ArrowWriterProperties___create", (DL_FUNC) &_arrow_parquet___ArrowWriterProperties___create, 3}, diff --git a/r/src/parquet.cpp b/r/src/parquet.cpp index efdc584d87bf..2f40287ff6c8 100644 --- a/r/src/parquet.cpp +++ b/r/src/parquet.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -191,6 +192,42 @@ int parquet___arrow___FileReader__num_columns( return reader->parquet_reader()->metadata()->num_columns(); } +namespace { + +// Append the Parquet leaf column indices below `field` to `out` +void AppendLeafColumnIndices(const parquet::arrow::SchemaField& field, + std::vector* out) { + if (field.is_leaf()) { + out->push_back(field.column_index); + return; + } + for (const auto& child : field.children) { + AppendLeafColumnIndices(child, out); + } +} + +} // namespace + +// Translate 0-based indices of top-level fields in the Arrow schema to the 0-based +// Parquet leaf column indices that FileReader::ReadTable() etc. expect. The two +// differ whenever the file has nested columns. +// [[parquet::export]] +std::vector parquet___arrow___FileReader__FieldIndicesToColumnIndices( + const std::shared_ptr& reader, + const std::vector& field_indices) { + const auto& schema_fields = reader->manifest().schema_fields; + const int num_fields = static_cast(schema_fields.size()); + + std::vector column_indices; + for (int field_index : field_indices) { + if (field_index < 0 || field_index >= num_fields) { + cpp11::stop("Field index %d is not valid", field_index); + } + AppendLeafColumnIndices(schema_fields[field_index], &column_indices); + } + return column_indices; +} + // [[parquet::export]] int parquet___arrow___FileReader__num_row_groups( const std::shared_ptr& reader) { diff --git a/r/tests/testthat/test-parquet.R b/r/tests/testthat/test-parquet.R index faa8d41e23db..edc1af41819c 100644 --- a/r/tests/testthat/test-parquet.R +++ b/r/tests/testthat/test-parquet.R @@ -559,3 +559,31 @@ test_that("WriteBatch on ParquetFileWriter errors when called on closed sink", { batch <- RecordBatch$create(data.frame(a = 1:10)) expect_error(writer$WriteBatch(batch, chunk_size = 10), "Operation on closed file") }) + +test_that("read_parquet() col_select works with nested columns (GH-55578)", { + df <- tibble::tibble(a = 1:2) + df$nested <- list( + tibble::tibble(x = 1, y = "p"), + tibble::tibble(x = 2, y = "q") + ) + df$b <- c("b1", "b2") + df$c <- c(TRUE, FALSE) + + tf <- tempfile(fileext = ".parquet") + on.exit(unlink(tf)) + write_parquet(df, tf) + + # columns after the nested one + expect_identical(names(read_parquet(tf, col_select = "b")), "b") + expect_identical(names(read_parquet(tf, col_select = "c")), "c") + expect_identical(names(read_parquet(tf, col_select = c("a", "c"))), c("a", "c")) + expect_identical(read_parquet(tf, col_select = c("a", "c")), df[, c("a", "c")]) + + # selection order is preserved + expect_identical(names(read_parquet(tf, col_select = c("c", "a"))), c("c", "a")) + + # the nested column itself is read in full + nested <- read_parquet(tf, col_select = "nested", as_data_frame = FALSE) + expect_identical(names(nested), "nested") + expect_equal(nested, read_parquet(tf, as_data_frame = FALSE)[, "nested"]) +})