Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions r/R/arrowExports.R

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 13 additions & 1 deletion r/R/parquet.R
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,13 @@ read_parquet <- function(
if (!quo_is_null(col_select)) {
# infer which columns to keep from schema
sim_df <- as.data.frame(reader$GetSchema())
indices <- eval_select(col_select, sim_df) - 1L
field_indices <- eval_select(col_select, sim_df) - 1L
# ReadTable() takes Parquet leaf column indices, which differ from the
# positions of the fields in the schema when there are nested columns
indices <- parquet___arrow___FileReader__FieldIndicesToColumnIndices(
reader,
field_indices
)
tab <- tryCatch(
reader$ReadTable(indices),
error = read_compressed_error
Expand Down Expand Up @@ -489,6 +495,12 @@ ParquetFileWriter$create <- function(
#' - `$GetSchema()`: get the `arrow::Schema` of the data in the file
#' - `$ReadColumn(i)`: read the `i`th column (0-based) as a [ChunkedArray].
#'
#' `column_indices` refer to Parquet leaf columns, not to the fields of the
#' schema returned by `$GetSchema()`. The two differ when the file has nested
#' columns, as each nested field is stored as one leaf column per child. Selecting
#' only some of a nested field's leaves returns that field containing only the
#' selected children.
#'
#' @section Active bindings:
#'
#' - `$num_rows`: number of rows.
Expand Down
6 changes: 6 additions & 0 deletions r/man/ParquetFileReader.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 17 additions & 0 deletions r/src/arrowExports.cpp

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

37 changes: 37 additions & 0 deletions r/src/parquet.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

#include <arrow/table.h>
#include <parquet/arrow/reader.h>
#include <parquet/arrow/schema.h>
#include <parquet/arrow/writer.h>
#include <parquet/exception.h>

Expand Down Expand Up @@ -191,6 +192,42 @@ int parquet___arrow___FileReader__num_columns(
return reader->parquet_reader()->metadata()->num_columns();
}

namespace {

// Append the Parquet leaf column indices below `field` to `out`
void AppendLeafColumnIndices(const parquet::arrow::SchemaField& field,
std::vector<int>* out) {
if (field.is_leaf()) {
out->push_back(field.column_index);
return;
}
for (const auto& child : field.children) {
AppendLeafColumnIndices(child, out);
}
}

} // namespace

// Translate 0-based indices of top-level fields in the Arrow schema to the 0-based
// Parquet leaf column indices that FileReader::ReadTable() etc. expect. The two
// differ whenever the file has nested columns.
// [[parquet::export]]
std::vector<int> parquet___arrow___FileReader__FieldIndicesToColumnIndices(
const std::shared_ptr<parquet::arrow::FileReader>& reader,
const std::vector<int>& field_indices) {
const auto& schema_fields = reader->manifest().schema_fields;
const int num_fields = static_cast<int>(schema_fields.size());

std::vector<int> column_indices;
for (int field_index : field_indices) {
if (field_index < 0 || field_index >= num_fields) {
cpp11::stop("Field index %d is not valid", field_index);
}
AppendLeafColumnIndices(schema_fields[field_index], &column_indices);
}
return column_indices;
}

// [[parquet::export]]
int parquet___arrow___FileReader__num_row_groups(
const std::shared_ptr<parquet::arrow::FileReader>& reader) {
Expand Down
28 changes: 28 additions & 0 deletions r/tests/testthat/test-parquet.R
Original file line number Diff line number Diff line change
Expand Up @@ -559,3 +559,31 @@ test_that("WriteBatch on ParquetFileWriter errors when called on closed sink", {
batch <- RecordBatch$create(data.frame(a = 1:10))
expect_error(writer$WriteBatch(batch, chunk_size = 10), "Operation on closed file")
})

test_that("read_parquet() col_select works with nested columns (GH-55578)", {
df <- tibble::tibble(a = 1:2)
df$nested <- list(
tibble::tibble(x = 1, y = "p"),
tibble::tibble(x = 2, y = "q")
)
df$b <- c("b1", "b2")
df$c <- c(TRUE, FALSE)

tf <- tempfile(fileext = ".parquet")
on.exit(unlink(tf))
write_parquet(df, tf)

# columns after the nested one
expect_identical(names(read_parquet(tf, col_select = "b")), "b")
expect_identical(names(read_parquet(tf, col_select = "c")), "c")
expect_identical(names(read_parquet(tf, col_select = c("a", "c"))), c("a", "c"))
expect_identical(read_parquet(tf, col_select = c("a", "c")), df[, c("a", "c")])

# selection order is preserved
expect_identical(names(read_parquet(tf, col_select = c("c", "a"))), c("c", "a"))

# the nested column itself is read in full
nested <- read_parquet(tf, col_select = "nested", as_data_frame = FALSE)
expect_identical(names(nested), "nested")
expect_equal(nested, read_parquet(tf, as_data_frame = FALSE)[, "nested"])
})
Loading