From 7a0287a90cf9d796a24c7a7618c1399fd255de28 Mon Sep 17 00:00:00 2001 From: Akshay Chitneni Date: Tue, 17 Nov 2026 16:04:07 -0800 Subject: [PATCH] docs: add a DataFrame::cache() example in staged and pipelined modes The example caches a prepared DataFrame in one session, builds the cache with the first query and reads it in staged mode, then switches the session to pipelined mode and reads the same cache with three parallel queries that stream their rows as they arrive. The tuning guide's "Caching DataFrames" section includes it, and the pipelined guide notes how caching works there. --- docs/source/user-guide/pipelined-execution.md | 9 ++ docs/source/user-guide/tuning-guide.md | 13 ++ examples/Cargo.toml | 5 + examples/README.md | 8 + .../examples/standalone-dataframe-cache.rs | 153 ++++++++++++++++++ 5 files changed, 188 insertions(+) create mode 100644 examples/examples/standalone-dataframe-cache.rs diff --git a/docs/source/user-guide/pipelined-execution.md b/docs/source/user-guide/pipelined-execution.md index 9b0d7b2adb..e331458148 100644 --- a/docs/source/user-guide/pipelined-execution.md +++ b/docs/source/user-guide/pipelined-execution.md @@ -137,6 +137,15 @@ does not work. executors more memory, use fewer partitions, or run such queries in staged mode. +## Queries over a cached DataFrame + +`DataFrame::cache()` works in a pipelined session. A cache is always built by +an ordinary staged job, run by the first query that reads it; later queries +read the cached result pipelined, streaming their rows as they run, so +expensive preparation is done once and the interactive queries over it stay +short. See [Caching DataFrames](tuning-guide.md#caching-dataframes) for how to +turn caching on, and a full example. + ## Settings | Setting | Default | Meaning | diff --git a/docs/source/user-guide/tuning-guide.md b/docs/source/user-guide/tuning-guide.md index 6b8fd70880..0782200f07 100644 --- a/docs/source/user-guide/tuning-guide.md +++ b/docs/source/user-guide/tuning-guide.md @@ -469,6 +469,19 @@ cached.collect().await?; // reads it `--disable-dataframe-cache` flag; cached DataFrames are then computed as part of each query. +A cache works in both execution modes. Its build always runs staged, and +queries over it run in the session's mode, so a +[pipelined](pipelined-execution.md) session reads a cache with streaming +queries. + +A full example, in staged and pipelined modes, from +[`examples/examples/standalone-dataframe-cache.rs`](https://github.com/apache/datafusion-ballista/blob/main/examples/examples/standalone-dataframe-cache.rs): + +```{literalinclude} ../../../examples/examples/standalone-dataframe-cache.rs +:language: rust +:start-after: under the License. +``` + See the [DataFrame cache design](../contributors-guide/dataframe-cache.md) for how it works. diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 52e72ce1a6..e6fca97a7d 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -38,6 +38,11 @@ name = "standalone_broadcast_join" path = "examples/standalone-broadcast-join.rs" required-features = ["ballista/standalone"] +[[example]] +name = "standalone_dataframe_cache" +path = "examples/standalone-dataframe-cache.rs" +required-features = ["ballista/standalone"] + [[example]] name = "mtls-cluster" path = "examples/mtls-cluster.rs" diff --git a/examples/README.md b/examples/README.md index a96db79402..b880d85dfa 100644 --- a/examples/README.md +++ b/examples/README.md @@ -32,6 +32,14 @@ cargo run --example standalone_sql Source: [`examples/examples/standalone-sql.rs`](https://github.com/apache/datafusion-ballista/blob/main/examples/examples/standalone-sql.rs) +`DataFrame::cache()` in staged and pipelined modes: + +```bash +cargo run --example standalone_dataframe_cache +``` + +Source: [`examples/examples/standalone-dataframe-cache.rs`](https://github.com/apache/datafusion-ballista/blob/main/examples/examples/standalone-dataframe-cache.rs) + ```bash cargo run --example standalone-substrait --features="ballista-scheduler/substrait" ``` diff --git a/examples/examples/standalone-dataframe-cache.rs b/examples/examples/standalone-dataframe-cache.rs new file mode 100644 index 0000000000..4ea16b3a40 --- /dev/null +++ b/examples/examples/standalone-dataframe-cache.rs @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `DataFrame::cache()` keeps a DataFrame's result on the executors, so queries +//! over it read the stored result instead of computing it again. +//! +//! This example: +//! 1. turns caching on (it is off by default), +//! 2. caches a prepared DataFrame (nothing runs yet), +//! 3. builds the cache with the first query and reads it with the next ones, +//! in staged mode, +//! 4. switches the session to pipelined mode, which reads the same cache, +//! 5. runs three pipelined queries in parallel, printing rows as they arrive. +//! +//! Good to know: a cache belongs to its session and is not evicted yet, so +//! cache only what you reuse. Cached data lost with an executor is rebuilt, and +//! `EXPLAIN` never builds a cache. See the tuning guide for details. + +use ballista::datafusion::{ + arrow::{record_batch::RecordBatch, util::pretty::pretty_format_batches}, + common::Result, + execution::{SessionStateBuilder, options::CsvReadOptions}, + functions_aggregate::expr_fn::{count, sum}, + prelude::{DataFrame, SessionConfig, SessionContext, col, lit}, +}; +use ballista::prelude::{SessionConfigExt, SessionContextExt}; +use ballista_examples::test_util; +use futures::StreamExt; + +#[tokio::main] +async fn main() -> Result<()> { + // 1. A standalone cluster, with caching turned on. Queries start staged. + let ctx = standalone().await?; + + // 2. The DataFrame the queries start from. Nothing runs here: `cache()` + // only marks it to be kept. In an application this is the expensive + // part, such as scans, joins and aggregations. + let path = format!("{}/aggregate_test_100.csv", test_util::examples_test_data()); + let sales = ctx + .read_csv(path, CsvReadOptions::new()) + .await? + .select(vec![ + col("c1").alias("category"), + col("c2").alias("store"), + col("c4").alias("amount"), + ])? + .filter(col("amount").gt(lit(0)))? + .cache() + .await?; + + // 3. Staged: the first query builds the cache, as a staged job, and then + // reads it. Later queries, and DataFrames derived from `sales`, read it. + println!("== staged: the first query builds the cache =="); + println!("{} sales", sales.clone().count().await?); + + println!("== staged: later queries read the cache =="); + let staged = by_category(sales.clone())?.collect().await?; + println!("{}", pretty_format_batches(&staged)?); + + // 4. Pipelined: one SET. A DataFrame keeps the settings of the session it + // came from, so rebuild it from the session's state to pick up the new + // mode. The plan, and the cache it reads, are the same. + ctx.sql("SET ballista.execution.mode = 'pipelined'").await?; + let sales = DataFrame::new(ctx.state(), sales.into_unoptimized_plan()); + + // 5. Three queries in parallel, each streaming its rows from the cache as + // they are produced. Read every stream you start, and keep the queries + // running at once within the cluster: each holds the vcores of its + // widest stage until it ends, here 8 vcores / 2 partitions = 4 at once. + println!("== pipelined: three queries in parallel =="); + let (pipelined, _, _) = tokio::try_join!( + stream("by category", by_category(sales.clone())?), + stream( + "sales per store", + sales + .clone() + .aggregate(vec![col("store")], vec![count(lit(1)).alias("sales")])? + ), + stream( + "top 5 amounts", + sales + .clone() + .sort(vec![col("amount").sort(false, false)])? + .limit(0, Some(5))? + ), + )?; + + // The cache gives the same result in either mode. + assert_eq!( + pretty_format_batches(&staged)?.to_string(), + pretty_format_batches(&pipelined)?.to_string(), + ); + Ok(()) +} + +/// Total amount per category, a query over the cached `sales`. +fn by_category(sales: DataFrame) -> Result { + sales + .aggregate( + vec![col("category")], + vec![sum(col("amount")).alias("total")], + )? + .sort_by(vec![col("category")]) +} + +/// Starts a standalone cluster that can run staged and pipelined queries, and +/// returns a session on it with caching turned on. +async fn standalone() -> Result { + let mut config = SessionConfig::new_with_ballista() + .with_target_partitions(2) + .with_ballista_standalone_parallelism(8); + // Pipelined queries need the scheduler to push tasks to executors. + config + .options_mut() + .set("ballista.standalone.task_scheduling_policy", "push-staged")?; + // `DataFrame::cache()` does nothing unless caching is turned on. + config.options_mut().set("ballista.cache.noop", "false")?; + + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .build(); + SessionContext::standalone_with_state(state).await +} + +/// Runs `df`, printing each batch of rows as it arrives, and returns them. +async fn stream(name: &str, df: DataFrame) -> Result> { + let mut stream = df.execute_stream().await?; + let mut batches = vec![]; + while let Some(batch) = stream.next().await { + let batch = batch?; + println!( + "{name}:\n{}", + pretty_format_batches(std::slice::from_ref(&batch))? + ); + batches.push(batch); + } + Ok(batches) +}