diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b172db01..c0443695 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -51,3 +51,20 @@ jobs: run: cargo check --no-default-features -p datafusion-spark - name: Check datafusion-spark (core) run: cargo check --no-default-features --features core -p datafusion-spark + + function-docs: + name: function docs up to date + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: rustup toolchain install stable --profile minimal --no-self-update && rustup default stable + - uses: Swatinem/rust-cache@v2 + - run: dev/update_function_docs.sh + - name: Check docs/functions is up to date + run: | + if [ -n "$(git status --porcelain -- docs/functions)" ]; then + git status --porcelain -- docs/functions + git diff -- docs/functions + echo "docs/functions is out of date. Run dev/update_function_docs.sh and commit the result." + exit 1 + fi diff --git a/AGENTS.md b/AGENTS.md index 48ba53c5..32ef188e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,9 +10,11 @@ DataFusion expressions that emulate Apache Spark's behavior. - `crates/spark` – the `datafusion-spark` crate - `crates/sqllogictest` – `.slt` tests for the Spark functions +- `crates/docgen` – generates the `docs/functions` reference from `#[user_doc]` (run `dev/update_function_docs.sh`) Before opening a PR, run `cargo fmt --all`, `cargo clippy --all-targets --all-features -- -D warnings`, `cargo test -p datafusion-spark` and `cargo test -p datafusion-spark-sqllogictest`. +If you changed any `#[user_doc]` attribute, also run `dev/update_function_docs.sh` and commit the changes under `docs/functions`. ## Agent Skills diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 606232e9..ec981b4a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,6 +104,16 @@ impl ScalarUDFImpl for SparkMyFunc { See `sha2` (`crates/spark/src/function/hash/sha2.rs`) and `substring` (`crates/spark/src/function/string/substring.rs`) for complete examples. +The function reference in [`docs/functions`](docs/functions/README.md) is generated from these +attributes, with one page per function category. After adding or changing a function's documentation, regenerate it and commit the +result (CI checks that it is up to date): + +```shell +dev/update_function_docs.sh +``` + +The script also lists, on stderr, the functions that don't implement `documentation()` yet. + ## Testing Guidelines Prefer adding tests via SQLLogicTests where possible, see the [Spark SQLLogicTest README]. diff --git a/Cargo.lock b/Cargo.lock index 65f1326f..1c219383 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1653,6 +1653,14 @@ dependencies = [ "url", ] +[[package]] +name = "datafusion-spark-docgen" +version = "56.0.0" +dependencies = [ + "datafusion-expr", + "datafusion-spark 56.0.0", +] + [[package]] name = "datafusion-spark-sqllogictest" version = "56.0.0" diff --git a/Cargo.toml b/Cargo.toml index e02a721b..b1c66485 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ [workspace] resolver = "2" -members = ["crates/spark", "crates/sqllogictest"] +members = ["crates/docgen", "crates/spark", "crates/sqllogictest"] [workspace.package] authors = ["Apache DataFusion "] diff --git a/README.md b/README.md index a9caae6d..54a54b12 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,8 @@ let ctx = SessionContext::new_with_state(state); let df = ctx.sql("SELECT sha2('Spark', 256)").await?; ``` -See the [crate README](crates/spark/README.md) and the [API documentation](https://docs.rs/datafusion-spark) +See the [function reference](docs/functions/README.md) for the documented functions, the +[crate README](crates/spark/README.md) and the [API documentation](https://docs.rs/datafusion-spark) for more. ## Repository layout diff --git a/crates/docgen/Cargo.toml b/crates/docgen/Cargo.toml new file mode 100644 index 00000000..25ac4aee --- /dev/null +++ b/crates/docgen/Cargo.toml @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +[package] +name = "datafusion-spark-docgen" +description = "Generates the function reference for datafusion-spark from #[user_doc]" +version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } +edition = { workspace = true } +publish = false + +[lints] +workspace = true + +[dependencies] +datafusion-expr = { workspace = true } +datafusion-spark = { path = "../spark" } + +[[bin]] +name = "print_functions_docs" +path = "src/main.rs" diff --git a/crates/docgen/src/main.rs b/crates/docgen/src/main.rs new file mode 100644 index 00000000..4597eb24 --- /dev/null +++ b/crates/docgen/src/main.rs @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Writes the function reference for `datafusion-spark` as Markdown. +//! +//! The content comes from each function's `documentation()`, which is generated by the +//! `#[user_doc]` attribute. Usage: +//! +//! ```text +//! cargo run -p datafusion-spark-docgen --bin print_functions_docs -- +//! ``` +//! +//! The output directory gets a `README.md` index and one page per function type and doc +//! section, for example `scalar/math.md`. Called from `dev/update_function_docs.sh`, which +//! writes to `docs/functions`. Functions that don't implement `documentation()` yet are listed +//! on stderr. + +use datafusion_expr::{ + DocSection, Documentation, aggregate_doc_sections, scalar_doc_sections, + window_doc_sections, +}; +use std::collections::HashMap; +use std::fmt::Write as _; +use std::path::Path; + +const LICENSE_HEADER: &str = " + + +"; + +/// Name, aliases and documentation of a scalar, aggregate or window function. +struct FunctionDoc { + name: String, + aliases: Vec, + documentation: Option, +} + +/// Collects [`FunctionDoc`]s from a list of `ScalarUDF`s, `AggregateUDF`s or `WindowUDF`s, +/// which share these methods but no common trait. +macro_rules! function_docs { + ($udfs:expr) => { + $udfs + .iter() + .map(|f| FunctionDoc { + name: f.name().to_string(), + aliases: f.aliases().to_vec(), + documentation: f.documentation().cloned(), + }) + .collect::>() + }; +} + +/// One generated page: the functions of one type (scalar, aggregate or window) in one doc +/// section. +struct Page<'a> { + /// Path relative to the output directory, for example `scalar/math.md` + path: String, + type_title: &'static str, + section: &'a DocSection, + functions: Vec<&'a FunctionDoc>, +} + +impl Page<'_> { + /// Function names and aliases, sorted, so that aliases can be looked up too + fn names(&self) -> Vec<(&str, &FunctionDoc)> { + let mut names: Vec<(&str, &FunctionDoc)> = self + .functions + .iter() + .flat_map(|f| { + std::iter::once(f.name.as_str()) + .chain(f.aliases.iter().map(String::as_str)) + .map(move |n| (n, *f)) + }) + .collect(); + names.sort_by_key(|(n, _)| *n); + names + } +} + +fn main() { + let Some(out_dir) = std::env::args().nth(1) else { + eprintln!("usage: print_functions_docs "); + std::process::exit(2); + }; + let out_dir = Path::new(&out_dir); + + let scalar: Vec = + function_docs!(datafusion_spark::all_default_scalar_functions()); + let aggregate: Vec = + function_docs!(datafusion_spark::all_default_aggregate_functions()); + let window: Vec = + function_docs!(datafusion_spark::all_default_window_functions()); + + let mut pages = Vec::new(); + let mut undocumented = Vec::new(); + for (title, dir, functions, known_sections) in [ + ( + "Scalar Functions", + "scalar", + &scalar, + scalar_doc_sections::doc_sections(), + ), + ( + "Aggregate Functions", + "aggregate", + &aggregate, + aggregate_doc_sections::doc_sections(), + ), + ( + "Window Functions", + "window", + &window, + window_doc_sections::doc_sections(), + ), + ] { + pages.extend(pages_for_type(title, dir, functions, &known_sections)); + undocumented.extend( + functions + .iter() + .filter(|f| f.documentation.is_none()) + .map(|f| format!("{} ({})", f.name, title.to_lowercase())), + ); + } + + // Page of every function name and alias, so links can cross pages + let page_of: HashMap<&str, &str> = pages + .iter() + .flat_map(|p| p.names().into_iter().map(|(n, _)| (n, p.path.as_str()))) + .collect(); + + write_file(&out_dir.join("README.md"), &index_page(&pages)); + for page in &pages { + write_file(&out_dir.join(&page.path), §ion_page(page, &page_of)); + } + + if !undocumented.is_empty() { + undocumented.sort(); + eprintln!( + "INFO: {} functions do not implement `documentation()` yet:", + undocumented.len() + ); + for name in undocumented { + eprintln!(" - {name}"); + } + } +} + +fn write_file(path: &Path, contents: &str) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .unwrap_or_else(|e| panic!("creating {}: {e}", parent.display())); + } + std::fs::write(path, contents) + .unwrap_or_else(|e| panic!("writing {}: {e}", path.display())); +} + +/// File name for a doc section, for example "Time and Date Functions" -> `time_and_date.md`. +fn section_file_name(label: &str) -> String { + let label = label.strip_suffix(" Functions").unwrap_or(label); + let slug: String = label + .to_lowercase() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect(); + format!("{slug}.md") +} + +/// Builds one page per doc section of a function type (scalar, aggregate or window). +/// +/// Sections follow DataFusion's order. Sections that DataFusion doesn't define (for example +/// "URL Functions") come after it, sorted by label. +fn pages_for_type<'a>( + type_title: &'static str, + dir: &str, + functions: &'a [FunctionDoc], + known_sections: &[DocSection], +) -> Vec> { + let mut sections: Vec<&DocSection> = Vec::new(); + for section in functions + .iter() + .filter_map(|f| f.documentation.as_ref()) + .map(|d| &d.doc_section) + .filter(|s| s.include) + { + if !sections.iter().any(|s| s.label == section.label) { + sections.push(section); + } + } + let rank = |s: &DocSection| { + known_sections + .iter() + .position(|k| k.label == s.label) + .unwrap_or(known_sections.len()) + }; + sections.sort_by(|a, b| rank(a).cmp(&rank(b)).then(a.label.cmp(b.label))); + + sections + .into_iter() + .map(|section| Page { + path: format!("{dir}/{}", section_file_name(section.label)), + type_title, + section, + functions: functions + .iter() + .filter(|f| { + f.documentation + .as_ref() + .is_some_and(|d| d.doc_section.label == section.label) + }) + .collect(), + }) + .collect() +} + +/// The `README.md` index: each section page with its function count, then every function +/// name in alphabetical order. +fn index_page(pages: &[Page]) -> String { + let mut docs = String::from(LICENSE_HEADER); + let _ = writeln!( + docs, + "\n# datafusion-spark Function Reference\n\n\ + Functions in this crate emulate Apache Spark's behavior. Each entry describes how the\n\ + function differs from the DataFusion built-in of the same name, if there is one." + ); + + let mut current_type = ""; + for page in pages { + if page.type_title != current_type { + current_type = page.type_title; + let _ = writeln!(docs, "\n## {current_type}\n"); + } + let _ = writeln!( + docs, + "- [{}]({}) ({})", + page.section.label, + page.path, + page.functions.len() + ); + } + + let mut all: Vec<(&str, &Page)> = pages + .iter() + .flat_map(|p| p.names().into_iter().map(move |(n, _)| (n, p))) + .collect(); + all.sort_by_key(|(n, p)| (*n, p.path.as_str())); + let _ = writeln!(docs, "\n## All Functions\n"); + for (name, page) in all { + let _ = writeln!(docs, "- [{name}]({}#{name})", page.path); + } + docs +} + +/// A section page, for example the scalar math functions. +fn section_page(page: &Page, page_of: &HashMap<&str, &str>) -> String { + let mut docs = String::from(LICENSE_HEADER); + let section = page.section; + let _ = writeln!( + docs, + "\n# {} ({})\n\nPart of the [datafusion-spark function reference](../README.md).\n", + section.label, + page.type_title.to_lowercase() + ); + if let Some(description) = section.description { + let _ = writeln!(docs, "{description}\n"); + } + + let names = page.names(); + for (name, _) in &names { + let _ = writeln!(docs, "- [{name}](#{name})"); + } + + for (name, f) in names { + if name != f.name { + let _ = writeln!(docs, "\n## `{name}`\n"); + let _ = writeln!(docs, "_Alias of [{0}](#{0})._", f.name); + continue; + } + let Some(d) = &f.documentation else { + unreachable!("only documented functions are in a section") + }; + + let _ = writeln!(docs, "\n## `{name}`\n"); + let _ = writeln!(docs, "{}\n", d.description); + let _ = writeln!(docs, "```sql\n{}\n```", d.syntax_example); + + if let Some(args) = &d.arguments { + let _ = writeln!(docs, "\n### Arguments\n"); + for (arg_name, arg_desc) in args { + let _ = writeln!(docs, "- **{arg_name}**: {arg_desc}"); + } + } + + if let Some(example) = &d.sql_example { + let _ = writeln!(docs, "\n### Example\n\n{example}"); + } + + if let Some(alt_syntax) = &d.alternative_syntax { + let _ = writeln!(docs, "\n### Alternative Syntax\n"); + for syntax in alt_syntax { + let _ = writeln!(docs, "```sql\n{syntax}\n```"); + } + } + + if !f.aliases.is_empty() { + let _ = writeln!(docs, "\n### Aliases\n"); + for alias in &f.aliases { + let _ = writeln!(docs, "- {}", alias.replace('_', r"\_")); + } + } + + if let Some(related) = &d.related_udfs { + let _ = writeln!(docs, "\n**Related functions**:\n"); + for r in related { + match page_of.get(r.as_str()) { + Some(path) if *path == page.path => { + let _ = writeln!(docs, "- [{r}](#{r})"); + } + Some(path) => { + let _ = writeln!(docs, "- [{r}](../{path}#{r})"); + } + // Not a datafusion-spark function, so there is no page to link to + None => { + let _ = writeln!(docs, "- {r}"); + } + } + } + } + } + docs +} diff --git a/crates/spark/README.md b/crates/spark/README.md index 2095ab54..a14760ee 100644 --- a/crates/spark/README.md +++ b/crates/spark/README.md @@ -24,7 +24,8 @@ errors, ...). This crate is useful when you are: - running SQL migrated from Spark on DataFusion. Not every Spark function is implemented yet. Progress is tracked in the -[`datafusion-spark` epic]; see the [API documentation] for the functions available today. +[`datafusion-spark` epic]. See the [function reference] for the documented functions, with +descriptions, arguments and examples, and the [API documentation] for the Rust API. ## Quick start @@ -102,4 +103,5 @@ and the [DataFusion contributor guide](https://datafusion.apache.org/contributor [apache spark]: https://spark.apache.org/ [apache datafusion comet]: https://datafusion.apache.org/comet/ [api documentation]: https://docs.rs/datafusion-spark +[function reference]: https://github.com/apache/datafusion-spark/blob/main/docs/functions/README.md [`datafusion-spark` epic]: https://github.com/apache/datafusion/issues/15914 diff --git a/crates/spark/src/lib.rs b/crates/spark/src/lib.rs index 462274ae..97cc28fb 100644 --- a/crates/spark/src/lib.rs +++ b/crates/spark/src/lib.rs @@ -46,7 +46,8 @@ //! * one accessor per function (for example `function::hash::sha2()`) returning the UDF //! //! Not every Spark function is implemented yet; coverage is growing. The -//! [`datafusion-spark` epic] tracks progress. +//! [`datafusion-spark` epic] tracks progress. The [function reference] lists +//! the documented functions with their descriptions, arguments and examples. //! //! # Semantics and caveats //! @@ -71,6 +72,7 @@ //! [Apache DataFusion]: https://datafusion.apache.org/ //! [Apache DataFusion Comet]: https://datafusion.apache.org/comet/ //! [`datafusion-spark` epic]: https://github.com/apache/datafusion/issues/15914 +//! [function reference]: https://github.com/apache/datafusion-spark/blob/main/docs/functions/README.md //! //! # Example: using all function packages //! diff --git a/dev/update_function_docs.sh b/dev/update_function_docs.sh new file mode 100755 index 00000000..4996fc97 --- /dev/null +++ b/dev/update_function_docs.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Regenerates docs/functions from the #[user_doc] attributes on each function. +# +# Usage: dev/update_function_docs.sh + +set -euo pipefail + +ROOT_DIR="$(git rev-parse --show-toplevel)" +cd "${ROOT_DIR}" + +TARGET_DIR="docs/functions" +mkdir -p "$(dirname "${TARGET_DIR}")" + +# Generate into a temporary directory so a failed build leaves the existing reference untouched, +# and so pages for removed sections don't linger. +TMP_DIR="$(mktemp -d "${TARGET_DIR}.XXXXXX")" +trap 'rm -rf "${TMP_DIR}"' EXIT + +cargo run --quiet -p datafusion-spark-docgen --bin print_functions_docs -- "${TMP_DIR}" + +chmod 755 "${TMP_DIR}" +rm -rf "${TARGET_DIR}" +mv "${TMP_DIR}" "${TARGET_DIR}" + +echo "Wrote ${TARGET_DIR}" diff --git a/docs/functions/README.md b/docs/functions/README.md new file mode 100644 index 00000000..2b87a101 --- /dev/null +++ b/docs/functions/README.md @@ -0,0 +1,153 @@ + + + + +# datafusion-spark Function Reference + +Functions in this crate emulate Apache Spark's behavior. Each entry describes how the +function differs from the DataFusion built-in of the same name, if there is one. + +## Scalar Functions + +- [Math Functions](scalar/math.md) (25) +- [Conditional Functions](scalar/conditional.md) (2) +- [String Functions](scalar/string.md) (17) +- [Binary String Functions](scalar/binary_string.md) (5) +- [Time and Date Functions](scalar/time_and_date.md) (26) +- [Array Functions](scalar/array.md) (10) +- [Hashing Functions](scalar/hashing.md) (4) +- [Other Functions](scalar/other.md) (6) +- [JSON Functions](scalar/json.md) (3) +- [URL Functions](scalar/url.md) (3) + +## Aggregate Functions + +- [General Functions](aggregate/general.md) (4) + +## All Functions + +- [abs](scalar/math.md#abs) +- [add_months](scalar/time_and_date.md#add_months) +- [array](scalar/array.md#array) +- [array_insert](scalar/array.md#array_insert) +- [array_max](scalar/array.md#array_max) +- [array_min](scalar/array.md#array_min) +- [array_position](scalar/array.md#array_position) +- [array_remove](scalar/array.md#array_remove) +- [array_repeat](scalar/array.md#array_repeat) +- [ascii](scalar/string.md#ascii) +- [atan2](scalar/math.md#atan2) +- [avg](aggregate/general.md#avg) +- [base64](scalar/binary_string.md#base64) +- [bin](scalar/math.md#bin) +- [bit_count](scalar/math.md#bit_count) +- [bit_get](scalar/math.md#bit_get) +- [bitmap_bit_position](scalar/other.md#bitmap_bit_position) +- [bitmap_bucket_number](scalar/other.md#bitmap_bucket_number) +- [bitmap_count](scalar/other.md#bitmap_count) +- [bitwise_not](scalar/math.md#bitwise_not) +- [bround](scalar/math.md#bround) +- [ceil](scalar/math.md#ceil) +- [ceiling](scalar/math.md#ceiling) +- [char](scalar/string.md#char) +- [char_length](scalar/string.md#char_length) +- [character_length](scalar/string.md#character_length) +- [collect_list](aggregate/general.md#collect_list) +- [collect_set](aggregate/general.md#collect_set) +- [concat](scalar/string.md#concat) +- [concat_ws](scalar/string.md#concat_ws) +- [contains](scalar/string.md#contains) +- [crc32](scalar/hashing.md#crc32) +- [csc](scalar/math.md#csc) +- [date_add](scalar/time_and_date.md#date_add) +- [date_diff](scalar/time_and_date.md#date_diff) +- [date_from_unix_date](scalar/time_and_date.md#date_from_unix_date) +- [date_part](scalar/time_and_date.md#date_part) +- [date_sub](scalar/time_and_date.md#date_sub) +- [date_trunc](scalar/time_and_date.md#date_trunc) +- [dateadd](scalar/time_and_date.md#dateadd) +- [datediff](scalar/time_and_date.md#datediff) +- [datepart](scalar/time_and_date.md#datepart) +- [dayname](scalar/time_and_date.md#dayname) +- [elt](scalar/string.md#elt) +- [encode](scalar/binary_string.md#encode) +- [equal_null](scalar/other.md#equal_null) +- [expm1](scalar/math.md#expm1) +- [factorial](scalar/math.md#factorial) +- [floor](scalar/math.md#floor) +- [format_string](scalar/string.md#format_string) +- [from_utc_timestamp](scalar/time_and_date.md#from_utc_timestamp) +- [getbit](scalar/math.md#getbit) +- [hash](scalar/hashing.md#hash) +- [hex](scalar/math.md#hex) +- [hour](scalar/time_and_date.md#hour) +- [hypot](scalar/math.md#hypot) +- [if](scalar/conditional.md#if) +- [ilike](scalar/string.md#ilike) +- [is_valid_utf8](scalar/string.md#is_valid_utf8) +- [json_array_length](scalar/json.md#json_array_length) +- [json_tuple](scalar/json.md#json_tuple) +- [last_day](scalar/time_and_date.md#last_day) +- [len](scalar/string.md#len) +- [length](scalar/string.md#length) +- [like](scalar/string.md#like) +- [luhn_check](scalar/string.md#luhn_check) +- [make_dt_interval](scalar/time_and_date.md#make_dt_interval) +- [make_interval](scalar/time_and_date.md#make_interval) +- [make_time](scalar/time_and_date.md#make_time) +- [make_valid_utf8](scalar/string.md#make_valid_utf8) +- [minute](scalar/time_and_date.md#minute) +- [monthname](scalar/time_and_date.md#monthname) +- [nanvl](scalar/conditional.md#nanvl) +- [negative](scalar/math.md#negative) +- [next_day](scalar/time_and_date.md#next_day) +- [positive](scalar/math.md#positive) +- [pow](scalar/math.md#pow) +- [power](scalar/math.md#power) +- [printf](scalar/string.md#printf) +- [quote](scalar/string.md#quote) +- [rint](scalar/math.md#rint) +- [round](scalar/math.md#round) +- [sec](scalar/math.md#sec) +- [second](scalar/time_and_date.md#second) +- [sha](scalar/hashing.md#sha) +- [sha1](scalar/hashing.md#sha1) +- [sha2](scalar/hashing.md#sha2) +- [shiftleft](scalar/math.md#shiftleft) +- [shiftright](scalar/math.md#shiftright) +- [shiftrightunsigned](scalar/math.md#shiftrightunsigned) +- [shuffle](scalar/array.md#shuffle) +- [size](scalar/array.md#size) +- [slice](scalar/array.md#slice) +- [soundex](scalar/string.md#soundex) +- [space](scalar/string.md#space) +- [spark_cast](scalar/other.md#spark_cast) +- [stack](scalar/other.md#stack) +- [substr](scalar/string.md#substr) +- [substring](scalar/string.md#substring) +- [time_trunc](scalar/time_and_date.md#time_trunc) +- [to_binary](scalar/binary_string.md#to_binary) +- [to_json](scalar/json.md#to_json) +- [to_utc_timestamp](scalar/time_and_date.md#to_utc_timestamp) +- [trunc](scalar/time_and_date.md#trunc) +- [try_sum](aggregate/general.md#try_sum) +- [try_to_binary](scalar/binary_string.md#try_to_binary) +- [try_url_decode](scalar/url.md#try_url_decode) +- [unbase64](scalar/binary_string.md#unbase64) +- [unhex](scalar/math.md#unhex) +- [unix_date](scalar/time_and_date.md#unix_date) +- [unix_micros](scalar/time_and_date.md#unix_micros) +- [unix_millis](scalar/time_and_date.md#unix_millis) +- [unix_seconds](scalar/time_and_date.md#unix_seconds) +- [url_decode](scalar/url.md#url_decode) +- [url_encode](scalar/url.md#url_encode) +- [weekday](scalar/time_and_date.md#weekday) +- [width_bucket](scalar/math.md#width_bucket) diff --git a/docs/functions/aggregate/general.md b/docs/functions/aggregate/general.md new file mode 100644 index 00000000..63104d6f --- /dev/null +++ b/docs/functions/aggregate/general.md @@ -0,0 +1,102 @@ + + + + +# General Functions (aggregate functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [avg](#avg) +- [collect_list](#collect_list) +- [collect_set](#collect_set) +- [try_sum](#try_sum) + +## `avg` + +Returns the average of the non-NULL values as a DOUBLE, or NULL if there are none. All numeric input is cast to DOUBLE first. So unlike DataFusion's `avg` and Spark's, DECIMAL input returns a DOUBLE rather than a DECIMAL ([#111](https://github.com/apache/datafusion-spark/issues/111)). + +```sql +avg(expr) +``` + +### Arguments + +- **expr**: Numeric expression to average. NULL values are skipped. + +### Example + +```sql +> SELECT avg(a) FROM (VALUES (40), (23), (17), (40), (NULL)) AS t(a); +30 +``` + +## `collect_list` + +Returns an array of the non-NULL values, including duplicates. Unlike DataFusion's `array_agg`, NULL values are skipped, so a group or window frame that is empty or holds only NULLs returns an empty array, where `array_agg` returns NULL or an array of NULLs. The order of the elements depends on the order of the rows, which may not be deterministic. `ORDER BY` inside the call is not supported and fails with an internal error ([#112](https://github.com/apache/datafusion-spark/issues/112)). + +```sql +collect_list(expr) +``` + +### Arguments + +- **expr**: Expression of any type to collect. NULL values are skipped. + +### Example + +```sql +> SELECT collect_list(a) FROM (VALUES (1), (2), (NULL), (1)) AS t(a); +[1, 2, 1] +> SELECT collect_list(a) FROM (VALUES (CAST(NULL AS INT))) AS t(a); +[] +``` + +## `collect_set` + +Returns an array of the distinct non-NULL values, in no particular order. Unlike DataFusion's `array_agg(DISTINCT ...)`, NULL values are skipped, so a group or window frame that is empty or holds only NULLs returns an empty array, where `array_agg` returns NULL or `[NULL]`. + +```sql +collect_set(expr) +``` + +### Arguments + +- **expr**: Expression of any type to collect. NULL values are skipped. + +### Example + +```sql +> SELECT array_sort(collect_set(a)) FROM (VALUES (1), (2), (NULL), (2)) AS t(a); +[1, 2] +> SELECT collect_set(a) FROM (VALUES (CAST(NULL AS INT))) AS t(a); +[] +``` + +## `try_sum` + +Returns the sum of the non-NULL values, or NULL if the sum overflows. DataFusion has no built-in `try_sum`. Integer input returns BIGINT, floating-point input returns DOUBLE, and DECIMAL(p, s) returns DECIMAL(min(p + 10, 38), s). BIGINT and DECIMAL overflow returns NULL, while DOUBLE sums follow IEEE 754 and can return Infinity or NaN. Returns NULL if all values are NULL. Other input types, such as intervals, are not supported. + +```sql +try_sum(expr) +``` + +### Arguments + +- **expr**: Integer, floating-point or decimal expression to sum. NULL values are skipped. + +### Example + +```sql +> SELECT try_sum(x) FROM VALUES (1), (2), (3) AS tab(x); +6 +> SELECT try_sum(x) FROM VALUES (CAST(9223372036854775807 AS BIGINT)), (1) AS tab(x); +NULL +``` diff --git a/docs/functions/scalar/array.md b/docs/functions/scalar/array.md new file mode 100644 index 00000000..bcec6e29 --- /dev/null +++ b/docs/functions/scalar/array.md @@ -0,0 +1,252 @@ + + + + +# Array Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [array](#array) +- [array_insert](#array_insert) +- [array_max](#array_max) +- [array_min](#array_min) +- [array_position](#array_position) +- [array_remove](#array_remove) +- [array_repeat](#array_repeat) +- [shuffle](#shuffle) +- [size](#size) +- [slice](#slice) + +## `array` + +Returns an array of the given elements, coerced to a common type. NULL elements are kept, and with no arguments the result is an empty array. Unlike DataFusion's `make_array`, the list's element field is named `element`, as in Spark, rather than `item`. + +```sql +array(expr, ...) +``` + +### Arguments + +- **expr**: Zero or more elements of any type that can be coerced to a common type. + +### Example + +```sql +> SELECT array(1, 2, 3); +[1, 2, 3] +> SELECT array(1, NULL, 3); +[1, NULL, 3] +> SELECT array(); +[] +``` + +## `array_insert` + +Returns `array` with `item` inserted at index `pos`. DataFusion has no built-in `array_insert`. Indexes are 1-based, and a negative index counts from the end, so -1 inserts after the last element. An index past the end pads with NULLs before the item. A negative index before the start places the item first, then NULL padding, then the original elements. Returns NULL if `array` or `pos` is NULL. An index of 0 is an error (`INVALID_INDEX_OF_ZERO`), as is a result longer than Spark's maximum array length (`COLLECTION_SIZE_LIMIT_EXCEEDED`). With Spark's `spark.sql.legacy.negativeIndexInArrayInsert`, -1 inserts before the last element instead, and a negative index before the start pads one more NULL, so -(n+1) on an array of n elements gives the item, one NULL, then the elements. + +```sql +array_insert(array, pos, item) +``` + +### Arguments + +- **array**: Array expression to insert into. NULL returns NULL. +- **pos**: Integer index, 1-based, of any integer type (cast to INT). A negative value counts from the end. 0 is an error. NULL returns NULL. +- **item**: Value to insert. Must share a type family with the array's elements. A NULL item is inserted as a NULL element. + +### Example + +```sql +> SELECT array_insert(make_array(1, 2, 3), 2, 10); +[1, 10, 2, 3] +> SELECT array_insert(make_array(5, 4, 3, 2), -1, 1); +[5, 4, 3, 2, 1] +> SELECT array_insert(make_array(1, 2, 3), 5, 100); +[1, 2, 3, NULL, 100] +> SELECT array_insert(make_array(1), -3, 100); +[100, NULL, 1] +> SELECT array_insert(make_array(1, 2), NULL::INT, 5); +NULL +``` + +## `array_max` + +Returns the maximum element of the array, using Spark's ordering. NULL elements are skipped, and an empty array or an array of only NULL elements returns NULL. For floating point values, also inside nested arrays and structs, NaN is greater than every other value, including infinity, and -0.0 is equal to 0.0. On a tie the first of the tied elements is returned as it is, so the sign of a zero and the bits of a NaN are kept. Strings and binary values compare bytewise. Arrays compare element by element and then by length, and structs compare field by field; NULLs nested inside them sort before other values. Map and INTERVAL elements, at any depth, are an error, as in Spark. DataFusion's built-in of the same name differs: it orders -0.0 before 0.0 rather than returning the first of them, sorts NULLs nested inside arrays and structs after other values, rejects a NULL argument at planning instead of returning NULL, and fails at execution on map elements. + +```sql +array_max(array) +``` + +### Arguments + +- **array**: Array expression. NULL returns NULL. + +### Example + +```sql +> SELECT array_max(array(1, 20, NULL, 3)); +20 +``` + +## `array_min` + +Returns the minimum element of the array, using Spark's ordering. NULL elements are skipped, and an empty array or an array of only NULL elements returns NULL. For floating point values, also inside nested arrays and structs, NaN is greater than every other value, including infinity, and -0.0 is equal to 0.0. On a tie the first of the tied elements is returned as it is, so the sign of a zero and the bits of a NaN are kept. Strings and binary values compare bytewise. Arrays compare element by element and then by length, and structs compare field by field; NULLs nested inside them sort before other values. Map and INTERVAL elements, at any depth, are an error, as in Spark. DataFusion's built-in of the same name differs: it orders -0.0 before 0.0 rather than returning the first of them, sorts NULLs nested inside arrays and structs after other values, rejects a NULL argument at planning instead of returning NULL, and fails at execution on map elements. + +```sql +array_min(array) +``` + +### Arguments + +- **array**: Array expression. NULL returns NULL. + +### Example + +```sql +> SELECT array_min(array(1, 20, NULL, 3)); +1 +``` + +## `array_position` + +Returns the 1-based position of the first occurrence of the element in the array, as a `BIGINT`. The DataFusion built-in returns a `UInt64`, and returns NULL when the element is not found, where this returns 0. The result is NULL only when the array or the element is NULL: a NULL element returns NULL even if the array contains NULL, while in the DataFusion built-in a NULL element matches the first NULL in the array. Top-level NULL elements in the array never match, while corresponding NULLs nested inside compared arrays or structs are equal. For floating point values, also inside nested arrays and structs, NaN is equal to NaN and -0.0 is equal to 0.0. As in Spark, the array element and the element are cast to their tightest common type: numbers widen, an integer widens to a decimal only if the decimal can hold all its values, and a date widens to a timestamp. Other combinations are an error, for example a string and a number, two different decimals, or structs whose field names or field types differ, as in Spark with ANSI mode off (with ANSI mode on, Spark widens struct field types). A NULL-typed array, and map or INTERVAL elements at any depth, are also an error. + +```sql +array_position(array, element) +``` + +### Arguments + +- **array**: Array expression. NULL returns NULL. +- **element**: Element to search for. NULL returns NULL. + +### Example + +```sql +> SELECT array_position(array(3, 2, 1), 1); +3 +> SELECT array_position(array(3, 2, 1), 4); +0 +``` + +## `array_remove` + +Removes all elements that are equal to the element from the array, while the DataFusion built-in removes only the first one. The result is NULL when the array or the element is NULL. NULL elements in the array are always kept. Corresponding NULLs nested inside compared arrays or structs are equal. For floating point values, also inside nested arrays and structs, NaN is equal to NaN and -0.0 is equal to 0.0; the elements that are kept keep their original values. As in Spark, the array element and the element are cast to their tightest common type, and the result is an array of that type: numbers widen, an integer widens to a decimal only if the decimal can hold all its values, and a date widens to a timestamp. Other combinations are an error, for example a string and a number, two different decimals, or structs whose field names or field types differ, as in Spark with ANSI mode off (with ANSI mode on, Spark widens struct field types). Map and INTERVAL elements, at any depth, are also an error. + +```sql +array_remove(array, element) +``` + +### Arguments + +- **array**: Array expression. NULL returns NULL. +- **element**: Element to remove. NULL returns NULL. + +### Example + +```sql +> SELECT array_remove(array(1, 2, 3, NULL, 3), 3); +[1, 2, NULL] +``` + +## `array_repeat` + +Returns an array that contains `element` `count` times. Unlike DataFusion's `array_repeat`, a NULL `count` returns NULL. A NULL `element` is repeated like any other value, and a `count` of 0 or less returns an empty array. + +```sql +array_repeat(element, count) +``` + +### Arguments + +- **element**: Value to repeat, of any type. NULL is repeated. +- **count**: Integer number of times to repeat `element`. Non-integer types are an error. NULL returns NULL. + +### Example + +```sql +> SELECT array_repeat('123', 2); +[123, 123] +> SELECT array_repeat(NULL, 2); +[NULL, NULL] +> SELECT array_repeat('123', NULL); +NULL +``` + +## `shuffle` + +Returns a random permutation of the array. DataFusion has no built-in `shuffle`. NULL elements are kept and shuffled with the others, and a NULL array returns NULL. With a `seed`, the order is the same on every run. It comes from a different random number generator than Spark's, so it does not match Spark's order for the same seed. + +```sql +shuffle(array[, seed]) +``` + +### Arguments + +- **array**: Array to shuffle. NULL returns NULL. +- **seed**: Optional constant integer seed. A NULL seed is the same as no seed, which gives a different order on each call. + +### Example + +```sql +> SELECT shuffle(array(1, 2, 3, 4, 5, NULL), 1); +[1, 4, NULL, 2, 5, 3] +> SELECT shuffle(NULL); +NULL +``` + +## `size` + +Returns the number of elements in an array, or the number of entries in a map, as an INT. NULL elements are counted. DataFusion has no built-in `size`. A NULL input returns -1 whatever the ANSI mode setting, so the result is never NULL. This is Spark's legacy behavior (`spark.sql.legacy.sizeOfNull` true and ANSI mode off). Spark with ANSI mode on, the default since Spark 4.0, returns NULL instead. + +```sql +size(expr) +``` + +### Arguments + +- **expr**: Array or map expression. NULL returns -1. + +### Example + +```sql +> SELECT size(array(1, NULL, 3)); +3 +> SELECT size(map(array('a', 'b'), array(1, 2))); +2 +> SELECT size(CAST(NULL AS ARRAY)); +-1 +``` + +## `slice` + +Returns the part of array `x` that starts at `start` and is at most `length` elements long. Unlike DataFusion's `array_slice`, the third argument is a length, not an end index. Positions are 1-based, and a negative `start` counts from the end of the array (-1 is the last element). A `start` past the end, or one that counts back past the first element, returns an empty array, and a `length` that runs past the end is clamped. A `start` of 0 or a negative `length` is an error. Returns NULL if any argument is NULL. + +```sql +slice(x, start, length) +``` + +### Arguments + +- **x**: Array to slice. NULL returns NULL. +- **start**: Integer start position, 1-based. A negative value counts from the end of the array. 0 is an error. +- **length**: Integer maximum number of elements to return. A negative value is an error. + +### Example + +```sql +> SELECT slice(array(1, 2, 3, 4), 2, 2); +[2, 3] +> SELECT slice(array(1, 2, 3, 4), -2, 2); +[3, 4] +> SELECT slice(array(1, 2, 3, 4), 100, 2); +[] +``` diff --git a/docs/functions/scalar/binary_string.md b/docs/functions/scalar/binary_string.md new file mode 100644 index 00000000..2c0ad65f --- /dev/null +++ b/docs/functions/scalar/binary_string.md @@ -0,0 +1,137 @@ + + + + +# Binary String Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [base64](#base64) +- [encode](#encode) +- [to_binary](#to_binary) +- [try_to_binary](#try_to_binary) +- [unbase64](#unbase64) + +## `base64` + +Encodes binary as a padded base64 string, using the standard alphabet. It is the same as DataFusion's `encode(expr, 'base64pad')`; DataFusion's `encode(expr, 'base64')` leaves out the padding. String input is encoded as its UTF-8 bytes. Returns NULL if the input is NULL. + +```sql +base64(bin) +``` + +### Arguments + +- **bin**: Binary or string expression to encode. NULL returns NULL. + +### Example + +```sql +> SELECT base64('Spark SQL'); +U3BhcmsgU1FM +> SELECT base64('Spark S'); +U3BhcmsgUw== +``` + +## `encode` + +Encodes a string into binary using a character set. Unlike DataFusion's `encode`, which takes `base64` or `hex` and returns a string, this takes a character set and returns binary. Supported character sets are `US-ASCII`, `ISO-8859-1`, `UTF-8`, `UTF-16`, `UTF-16BE`, `UTF-16LE`, and `UTF-32`, matched case-insensitively. Any other character set raises `[INVALID_PARAMETER_VALUE.CHARSET]`. `UTF-16` writes a big-endian byte order mark first. `UTF-32` is big-endian with no byte order mark. Characters the character set cannot represent (non-ASCII in `US-ASCII`, code points above U+00FF in `ISO-8859-1`) raise `[MALFORMED_CHARACTER_CODING]`. Binary input is decoded as UTF-8 first, with each invalid byte replaced by U+FFFD. Returns NULL if either input is NULL. This follows Spark 4.0. Pre-Spark 4.0 accepted Java aliases (legacy mode supports a fixed set of aliases: `UTF8`, `ASCII`, `LATIN1`, `ISO88591`, `UTF16`, `UTF16BE`, `UTF16LE`, `UTF32`, `UTF32BE`, `UTF32LE`) and replaced unmappable characters with `?`; those legacy behaviors can be enabled programmatically via `with_legacy_java_charsets(true)` and `with_legacy_coding_error_action(true)`. + +```sql +encode(str, charset) +``` + +### Arguments + +- **str**: String or binary expression to encode. Binary input is decoded as UTF-8, with invalid bytes replaced by U+FFFD. NULL returns NULL. +- **charset**: String name of the character set, matched case-insensitively. May vary per row. NULL returns NULL. + +### Example + +```sql +> SELECT encode('Spark SQL', 'utf-8'); +537061726b2053514c +> SELECT encode('AB', 'utf-16le'); +41004200 +> SELECT encode('Spark SQL', 'us-ascii'); +537061726b2053514c +``` + +## `to_binary` + +Converts a string to binary using the given format. DataFusion has no built-in `to_binary`. `hex` decodes hex digits, and an odd number of digits is padded with a leading zero. `utf-8` (or `utf8`) returns the string's UTF-8 bytes. `base64` decodes like Java's MIME decoder, as Spark does: padding is optional and characters outside the base64 alphabet are skipped. An empty string converts to empty binary in every format. A value that cannot be converted, or a format that is not recognised, is an error; use `try_to_binary` to get NULL instead. Returns NULL if either input is NULL. + +```sql +to_binary(str[, fmt]) +``` + +### Arguments + +- **str**: String expression to convert. NULL returns NULL. +- **fmt**: Optional constant string format, matched case-insensitively: `hex` (the default), `utf-8`, `utf8` or `base64`. A NULL format returns NULL. + +### Example + +```sql +> SELECT to_binary('537061726b'); +537061726b +> SELECT to_binary('abc', 'utf-8'); +616263 +> SELECT to_binary('U3Bhcms', 'base64'); +537061726b +``` + +## `try_to_binary` + +Same as `to_binary`, except that a value which cannot be converted returns NULL for that row, and a format that is not recognised returns NULL, instead of an error. DataFusion has no built-in `try_to_binary`. + +```sql +try_to_binary(str[, fmt]) +``` + +### Arguments + +- **str**: String expression to convert. NULL, or a value that cannot be converted, returns NULL. +- **fmt**: Optional constant string format, matched case-insensitively: `hex` (the default), `utf-8`, `utf8` or `base64`. A NULL or unrecognised format returns NULL. + +### Example + +```sql +> SELECT try_to_binary('U3Bhcms=', 'base64'); +537061726b +> SELECT try_to_binary('zz', 'hex'); +NULL +> SELECT try_to_binary('abc', 'invalidFormat'); +NULL +``` + +## `unbase64` + +Decodes a base64 string to binary, the same way as Java's MIME decoder, which Spark uses. DataFusion has no built-in `unbase64`. Padding is optional, and characters outside the base64 alphabet (such as spaces and line breaks) are skipped. A single leftover character at the end, malformed padding, or base64 characters after the padding are errors. Returns NULL if the input is NULL. + +```sql +unbase64(str) +``` + +### Arguments + +- **str**: String or binary expression holding base64 text. NULL returns NULL. + +### Example + +```sql +> SELECT arrow_cast(unbase64('U3BhcmsgU1FM'), 'Utf8'); +Spark SQL +> SELECT arrow_cast(unbase64('YW Jj'), 'Utf8'); +abc +> SELECT hex(unbase64('123')); +D76D +``` diff --git a/docs/functions/scalar/conditional.md b/docs/functions/scalar/conditional.md new file mode 100644 index 00000000..c05378f9 --- /dev/null +++ b/docs/functions/scalar/conditional.md @@ -0,0 +1,69 @@ + + + + +# Conditional Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [if](#if) +- [nanvl](#nanvl) + +## `if` + +Returns `expr1` if `cond` is true, and `expr2` otherwise. A NULL condition counts as false. It is rewritten to `CASE WHEN cond THEN expr1 ELSE expr2 END`, so an error such as division by zero in the branch that isn't taken is not raised. `expr1` and `expr2` are coerced to a common type (`if(true, 10, 20.5)` returns a DOUBLE). It is an error if `cond` is not a boolean or NULL. DataFusion has no built-in `if`. + +```sql +if(cond, expr1, expr2) +``` + +### Arguments + +- **cond**: Boolean expression. NULL is treated as false. +- **expr1**: Expression returned when `cond` is true. +- **expr2**: Expression returned when `cond` is false or NULL. + +### Example + +```sql +> SELECT if(1 < 2, 'a', 'b'); +a +> SELECT if(NULL, 'yes', 'no'); +no +> SELECT if(true, 1 / 1, 1 / 0); +1 +``` + +## `nanvl` + +Returns `expr1` if it is not NaN, or `expr2` if `expr1` is NaN. Unlike DataFusion's built-in `nanvl`, which evaluates both arguments eagerly, this function only evaluates `expr2` when `expr1` is NaN, matching Spark's short-circuiting behavior. If `expr1` is NULL the result is NULL, even when `expr2` is not NULL; a NULL `expr2` only matters when `expr1` is NaN. Both arguments are implicitly cast to DOUBLE (integer, DECIMAL and STRING arguments included); if both arguments are FLOAT the result is FLOAT, otherwise the result is DOUBLE. +/-Infinity and -0.0 are not NaN and are returned unchanged, with the sign of zero preserved. + +```sql +nanvl(expr1, expr2) +``` + +### Arguments + +- **expr1**: Numeric expression, cast to DOUBLE (or FLOAT if both arguments are FLOAT). NULL returns NULL regardless of expr2. +- **expr2**: Numeric expression, cast to match expr1. Only evaluated, and only matters, when expr1 is NaN. + +### Example + +```sql +> SELECT nanvl(cast('NaN' as double), 123); +123 +> SELECT nanvl(5.0, CAST(NULL AS DOUBLE)); +5 +> SELECT nanvl(CAST(NULL AS DOUBLE), 5.0); +NULL +> SELECT nanvl(CAST('NaN' AS DOUBLE), CAST('NaN' AS DOUBLE)); +NaN +``` diff --git a/docs/functions/scalar/hashing.md b/docs/functions/scalar/hashing.md new file mode 100644 index 00000000..615e2306 --- /dev/null +++ b/docs/functions/scalar/hashing.md @@ -0,0 +1,127 @@ + + + + +# Hashing Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [crc32](#crc32) +- [hash](#hash) +- [sha](#sha) +- [sha1](#sha1) +- [sha2](#sha2) + +## `crc32` + +Returns the CRC-32 checksum of the input as a BIGINT. The checksum is an unsigned 32-bit value, so the result is never negative. DataFusion has no built-in `crc32`. + +```sql +crc32(expr) +``` + +### Arguments + +- **expr**: String or binary expression. Strings are checksummed as UTF-8 bytes. NULL returns NULL. + +### Example + +```sql +> SELECT crc32('Spark'); +1557323817 +> SELECT crc32(''); +0 +> SELECT crc32(NULL); +NULL +``` + +## `hash` + +Returns the 32-bit Murmur3 hash (seed 42) of the arguments as an INT, matching Spark's `hash`. DataFusion has no built-in equivalent. Arguments are hashed in order, and each argument's hash seeds the next, so `hash(a, b)` equals `hash(named_struct('a', a, 'b', b))` and argument order matters. NULL arguments, NULL list elements and NULL struct fields are skipped, so the result is never NULL: `hash(NULL)` returns 42. The result depends on the input type, as in Spark: INT hashes 4 bytes and BIGINT 8, and integer literals are BIGINT in DataFusion but INT in Spark, so cast them to INT to match Spark's results. `-0.0` hashes like `0.0`, and every NaN hashes like the canonical NaN. Dates hash as INT days and timestamps as BIGINT microseconds, whatever the Arrow unit (DataFusion's TIMESTAMP literals are nanoseconds, floored to microseconds). TIME hashes as BIGINT nanoseconds since midnight, a year-month interval (`Interval(YearMonth)`) as INT months and a day-time interval (`Duration(Microsecond)`) as BIGINT microseconds. A calendar interval (`Interval(MonthDayNano)`) hashes its microseconds and then its months, and leaves out its days, as Spark 4.2's generated code does (Spark 4.3 includes the days, see SPARK-58236). DataFusion types `INTERVAL '1' DAY` as a calendar interval, so it hashes like a zero interval, where Spark hashes it as a day-time interval. Use `arrow_cast(86400000000, 'Duration(Microsecond)')` to match Spark. Decimals with precision up to 18 hash as their unscaled long, and wider decimals hash the minimal big-endian two's complement bytes of the unscaled value. Lists hash each element in order (nested lists flatten into the same chain, and an empty list leaves the seed unchanged), structs hash each field, and maps hash each key then its value. This crate hashes maps as if Spark's `spark.sql.legacy.allowHashOnMapType` were set, where Spark 4 rejects them by default (see https://github.com/apache/datafusion-spark/issues/69). Dictionary-encoded values hash like their decoded values. At least one argument is required. + +```sql +hash(expr1[, ..., exprN]) +``` + +### Arguments + +- **expr1**: Expression of any type to hash. NULL is skipped. +- **exprN**: Further expressions, hashed in order with the previous hash as the seed. + +### Example + +```sql +> SELECT hash('Spark', [CAST(123 AS INT)], CAST(2 AS INT)); +-1321691492 +> SELECT hash(CAST(1 AS INT)); +-559580957 +> SELECT hash(1); +-1712319331 +> SELECT hash(NULL); +42 +``` + +## `sha` + +_Alias of [sha1](#sha1)._ + +## `sha1` + +Returns the SHA-1 digest of the input as a 40-character lowercase hex string. DataFusion has no built-in `sha1`. + +```sql +sha1(expr) +``` + +### Arguments + +- **expr**: String or binary expression to hash. Strings are hashed as UTF-8 bytes. NULL returns NULL. + +### Example + +```sql +> SELECT sha1('Spark'); +85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c +> SELECT sha1(NULL); +NULL +``` + +### Alternative Syntax + +```sql +sha(expr) +``` + +### Aliases + +- sha + +## `sha2` + +Returns the SHA-2 digest of the input as a lowercase hex string. Unlike DataFusion's `sha224`/`sha256`/..., the result is a hex string rather than binary, and the bit length may vary per row. An unsupported bit length returns NULL rather than an error. + +```sql +sha2(expr, bit_length) +``` + +### Arguments + +- **expr**: String or binary expression to hash. Strings are hashed as UTF-8 bytes. NULL returns NULL. +- **bit_length**: SHA-2 variant: 224, 256, 384 or 512, cast to INT as in Spark. 0 is treated as 256. Any other value, or NULL, returns NULL. + +### Example + +```sql +> SELECT sha2('Spark', 256); +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b +> SELECT sha2('Spark', 128); +NULL +``` diff --git a/docs/functions/scalar/json.md b/docs/functions/scalar/json.md new file mode 100644 index 00000000..a66d3651 --- /dev/null +++ b/docs/functions/scalar/json.md @@ -0,0 +1,89 @@ + + + + +# JSON Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [json_array_length](#json_array_length) +- [json_tuple](#json_tuple) +- [to_json](#to_json) + +## `json_array_length` + +Returns the number of elements in the outermost JSON array. Returns NULL for a NULL input, an empty or blank string, invalid or truncated JSON, and any valid JSON value that is not an array. Like Spark, the array is read up to its closing bracket and anything after it is ignored. Known differences from Spark, which parses with Jackson configured to accept single-quoted strings and unescaped control characters (Hive compatibility): those inputs return NULL here. Jackson also limits nesting to 1000 levels and number literals to 1000 digits, and Spark returns NULL beyond those limits, while such arrays are still counted here. + +```sql +json_array_length(json_array) +``` + +### Arguments + +- **json_array**: String expression containing a JSON array. Other types are rejected at planning time. NULL returns NULL. + +### Example + +```sql +> SELECT json_array_length('[1,2,3,[33,44],{"key":[2,3,4]}]'); +5 +> SELECT json_array_length('[1,2] trailing'); +2 +> SELECT json_array_length('{"key":"not a json array"}'); +NULL +> SELECT json_array_length('[1,2'); +NULL +``` + +## `json_tuple` + +Returns the values of the given top-level fields of a JSON object, as a struct with one string field per key, named `c0`, `c1`, ... DataFusion has no built-in `json_tuple`. In Spark, `json_tuple` is a generator that returns one column per key. A scalar function returns one value per row, so here the columns are wrapped in a struct, which the caller (e.g. Comet) is expected to unpack. Keys are case-sensitive. A missing field, a JSON `null` value or a NULL key gives a NULL field, and non-string values such as numbers are returned as their JSON text. Returns NULL if the input is NULL, is not valid JSON, or is not a JSON object. + +```sql +json_tuple(json_str, key1[, key2, ...]) +``` + +### Arguments + +- **json_str**: String containing a JSON object. NULL, invalid JSON or JSON that is not an object returns NULL. +- **key**: One or more string keys of top-level fields to return. A NULL key gives a NULL field. + +### Example + +```sql +> SELECT json_tuple('{"a":1, "b":"x"}', 'a', 'b', 'c'); +{c0: 1, c1: x, c2: NULL} +> SELECT json_tuple('[1, 2]', 'a'); +NULL +``` + +## `to_json` + +Serializes a struct to a JSON string, one object per row, as Spark's `to_json` does with its default options. DataFusion has no built-in `to_json`. A NULL struct returns NULL. NULL struct fields are omitted (`ignoreNullFields`), while NULL map entries are kept as `"key":null`. Nested structs, lists and maps are serialized recursively. Floats are written like Java's `Double.toString` (`1.0`, `1.0E-5`; `NaN` and `Infinity` as strings), decimals like `BigDecimal.toString` (`1.50`, `0E-10`), binary as Base64, dates as `yyyy-MM-dd`, timestamps as `yyyy-MM-dd'T'HH:mm:ss.SSS` with the session time zone offset (`datafusion.execution.time_zone`, `Z` for UTC) and without offset for a timestamp without time zone. Map keys are written as strings holding Spark's internal value: dates as days since the epoch, timestamps as microseconds, numbers as above. Types with no Spark JSON representation (intervals, durations, times, unions, map keys other than strings, booleans, numbers, dates and timestamps) are rejected when the query is planned. The options argument and map or array input are not supported. + +```sql +to_json(struct_expr) +``` + +### Arguments + +- **struct_expr**: Struct expression to serialize, e.g. a struct column or a `named_struct` call. The field names become the JSON keys. NULL returns NULL. + +### Example + +```sql +> SELECT to_json(named_struct('a', 1, 'b', 'x', 'c', CAST(NULL AS INT))); +{"a":1,"b":"x"} +> SELECT to_json(named_struct('t', TIMESTAMP '2024-01-01 01:02:03.123456', 'd', 1e-5)); +{"t":"2024-01-01T01:02:03.123","d":1.0E-5} +> SELECT to_json(named_struct('m', map([DATE '2024-01-01'], [1]))); +{"m":{"19723":1}} +``` diff --git a/docs/functions/scalar/math.md b/docs/functions/scalar/math.md new file mode 100644 index 00000000..0a0a673d --- /dev/null +++ b/docs/functions/scalar/math.md @@ -0,0 +1,657 @@ + + + + +# Math Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [abs](#abs) +- [atan2](#atan2) +- [bin](#bin) +- [bit_count](#bit_count) +- [bit_get](#bit_get) +- [bitwise_not](#bitwise_not) +- [bround](#bround) +- [ceil](#ceil) +- [ceiling](#ceiling) +- [csc](#csc) +- [expm1](#expm1) +- [factorial](#factorial) +- [floor](#floor) +- [getbit](#getbit) +- [hex](#hex) +- [hypot](#hypot) +- [negative](#negative) +- [positive](#positive) +- [pow](#pow) +- [power](#power) +- [rint](#rint) +- [round](#round) +- [sec](#sec) +- [shiftleft](#shiftleft) +- [shiftright](#shiftright) +- [shiftrightunsigned](#shiftrightunsigned) +- [unhex](#unhex) +- [width_bucket](#width_bucket) + +## `abs` + +Returns the absolute value of `expr`, with the same type as the input. Unlike DataFusion's `abs`, which always errors on overflow, the result for the minimum value of a signed integer type (for example `-2147483648` for INT) depends on ANSI mode: with `datafusion.execution.enable_ansi_mode` off it is returned unchanged, and with it on the function fails with `ARITHMETIC_OVERFLOW`. Returns NULL if the input is NULL and NaN if it is NaN, and the absolute value of -0.0 is 0. Spark's interval types are not supported yet. + +```sql +abs(expr) +``` + +### Arguments + +- **expr**: Numeric expression: integer, floating-point or decimal. NULL returns NULL. + +### Example + +```sql +> SELECT abs(-1); +1 +> SELECT abs(-99999999.99::DECIMAL(10, 2)); +99999999.99 +> SELECT abs((-2147483648)::INT); +-2147483648 +``` + +## `atan2` + +Returns the angle in radians, between -pi and pi, between the positive x-axis and the point (`exprX`, `exprY`). Note that the y coordinate comes first. Both arguments are cast to DOUBLE and the result is a DOUBLE. Returns NULL if either input is NULL, and NaN if either input is NaN (and neither is NULL). Infinite inputs follow `java.lang.Math.atan2`, but both arguments have `0.0` added first (as in Spark), treating `-0.0` as `0.0`: for example `atan2(-0.0, -1.0)` is pi. + +```sql +atan2(exprY, exprX) +``` + +### Arguments + +- **exprY**: Numeric y coordinate, cast to DOUBLE. NULL returns NULL. +- **exprX**: Numeric x coordinate, cast to DOUBLE. NULL returns NULL. + +### Example + +```sql +> SELECT atan2(1, 1); +0.785398163397448 +> SELECT atan2(0, -1); +3.141592653589793 +> SELECT atan2(-0.0::double, -1.0::double); +3.141592653589793 +``` + +## `bin` + +Returns the binary representation of `expr` as a string, with no leading zeros. DataFusion has no built-in `bin`. The input is cast to BIGINT first, so fractional values are truncated and every negative value is rendered as its 64-bit two's complement, whatever its original integer width. Returns NULL if the input is NULL. + +```sql +bin(expr) +``` + +### Arguments + +- **expr**: Numeric expression, cast to BIGINT. NULL returns NULL. + +### Example + +```sql +> SELECT bin(13); +1101 +> SELECT bin(13.3); +1101 +> SELECT bin(-13); +1111111111111111111111111111111111111111111111111111111111110011 +``` + +## `bit_count` + +Returns the number of bits that are set in `expr`, as an INT. As in Spark, signed integers are sign-extended to 64 bits before counting, so a negative value of any signed type counts the extra high bits (`bit_count(-1)` is 64 for TINYINT, SMALLINT, INT and BIGINT). Unsigned integers are counted at their own width. A boolean counts as 1 for true and 0 for false. NULL returns NULL. DataFusion has no built-in `bit_count`. + +```sql +bit_count(expr) +``` + +### Arguments + +- **expr**: Boolean or integer expression (signed or unsigned, 8 to 64 bits). NULL returns NULL. + +### Example + +```sql +> SELECT bit_count(7); +3 +> SELECT bit_count(-1::int); +64 +> SELECT bit_count(NULL::int); +NULL +``` + +## `bit_get` + +Returns the bit (0 or 1) at position `pos` of an integer, as a TINYINT. Positions count from 0 at the least significant bit. Returns NULL if either input is NULL. Errors if `pos` is negative or not less than the bit width of the type of `expr`. Integer literals are BIGINT in DataFusion but INT in Spark, so for an uncast literal the limit is 64 here and 32 in Spark. DataFusion has no built-in `bit_get`. + +```sql +bit_get(expr, pos) +``` + +### Arguments + +- **expr**: Integer expression. NULL returns NULL. +- **pos**: Integer bit position, from 0 to the bit width of `expr` minus 1. Any other value is an error. NULL returns NULL. + +### Example + +```sql +> SELECT bit_get(11, 0); +1 +> SELECT bit_get(11, 2); +0 +> SELECT bit_get(11, NULL); +NULL +``` + +### Alternative Syntax + +```sql +getbit(expr, pos) +``` + +### Aliases + +- getbit + +## `bitwise_not` + +Returns the bitwise NOT of a signed integer, with every bit flipped, in the same type as the input. In two's complement this is `-expr - 1`, and it never overflows (`bitwise_not` of the maximum value is the minimum value). NULL returns NULL. Spark SQL spells this `~expr`. DataFusion has no built-in `bitwise_not` function. + +```sql +bitwise_not(expr) +``` + +### Arguments + +- **expr**: TINYINT, SMALLINT, INT or BIGINT expression. NULL returns NULL. + +### Example + +```sql +> SELECT bitwise_not(0::int); +-1 +> SELECT bitwise_not(-3::int); +2 +> SELECT bitwise_not(2147483647::int); +-2147483648 +``` + +## `bround` + +Rounds `expr` to `d` decimal places using HALF_EVEN rounding: a tie goes to the even neighbour (2.5 becomes 2, 3.5 becomes 4). A negative `d` rounds to the left of the decimal point. Integers and floating-point values keep their type, and a string is cast to DOUBLE (a malformed string is NULL with `datafusion.execution.enable_ansi_mode` off and an error with it on). A decimal `DECIMAL(p, s)` returns `DECIMAL(min(p - s + 1 + min(s, d), 38), min(s, d))` for `d >= 0` and `DECIMAL(min(max(p - s + 1, 1 - d), 38), 0)` for `d < 0`, like Spark. Floating-point values are rounded from their shortest decimal representation, so `bround(2.675, 2)` is 2.68. Integer overflow wraps with ANSI mode off and fails with it on. Infinity and NaN are returned unchanged. Returns NULL if `expr` or `d` is NULL. + +```sql +bround(expr[, d]) +``` + +### Arguments + +- **expr**: Numeric or string expression. NULL returns NULL. +- **d**: Optional constant integer number of decimal places. Defaults to 0. NULL returns NULL. + +### Example + +```sql +> SELECT bround(2.5, 0); +2 +> SELECT bround(25, -1); +20 +``` + +## `ceil` + +Returns the smallest whole number not less than `expr`. Unlike DataFusion's `ceil`, which keeps the input type, integer and floating-point inputs return BIGINT, and `DECIMAL(p, s)` with `s > 0` returns `DECIMAL(p - s + 1, 0)`. Decimals with a scale of 0 are returned unchanged. Returns NULL if the input is NULL. Decimal32, Decimal64 and Decimal256 inputs are not supported, and the two-argument form `ceil(expr, scale)` is not implemented yet (). + +```sql +ceil(expr) +``` + +### Arguments + +- **expr**: Numeric expression: integer, floating-point or decimal. NULL returns NULL. + +### Example + +```sql +> SELECT ceil(-1.1); +-1 +> SELECT ceil(1.50::DECIMAL(10, 2)); +2 +> SELECT ceil(5::INT); +5 +``` + +### Alternative Syntax + +```sql +ceiling(expr) +``` + +### Aliases + +- ceiling + +## `ceiling` + +_Alias of [ceil](#ceil)._ + +## `csc` + +Returns the cosecant of `expr` (`1 / sin(expr)`) as a DOUBLE. DataFusion has no built-in `csc`. The argument is cast to DOUBLE. `csc(0)` returns Infinity rather than an error. Returns NULL if the input is NULL, and NaN if it is NaN. + +```sql +csc(expr) +``` + +### Arguments + +- **expr**: Numeric angle in radians, cast to DOUBLE. NULL returns NULL. + +### Example + +```sql +> SELECT csc(1); +1.188395105778121 +> SELECT csc(0); +Infinity +``` + +## `expm1` + +Returns `exp(expr) - 1` as a DOUBLE, computed without the precision loss of subtracting 1 from `exp(expr)` when `expr` is close to 0. DataFusion has no built-in `expm1`. Numeric arguments are cast to DOUBLE. Returns NULL if the input is NULL. + +```sql +expm1(expr) +``` + +### Arguments + +- **expr**: Numeric expression, cast to DOUBLE. NULL returns NULL. + +### Example + +```sql +> SELECT expm1(0); +0 +> SELECT expm1(1); +1.718281828459045 +``` + +## `factorial` + +Returns the factorial of `expr` as a BIGINT. Only 0 to 20 have a result: a value outside that range returns NULL, where DataFusion's `factorial` returns an error. The argument is cast to INT. Returns NULL if the input is NULL. + +```sql +factorial(expr) +``` + +### Arguments + +- **expr**: Integer expression, cast to INT. Values below 0 or above 20, and NULL, return NULL. + +### Example + +```sql +> SELECT factorial(5); +120 +> SELECT factorial(20); +2432902008176640000 +> SELECT factorial(21); +NULL +``` + +## `floor` + +Returns the largest whole number not greater than `expr`. Unlike DataFusion's `floor`, which keeps the input type, integer and floating-point inputs return BIGINT, and `DECIMAL(p, s)` with `s > 0` returns `DECIMAL(p - s + 1, 0)`. Decimals with a scale of 0 are returned unchanged. Floating-point values outside the BIGINT range saturate to its minimum or maximum (so Infinity returns 9223372036854775807), and NaN returns 0. Returns NULL if the input is NULL. Decimal32, Decimal64 and Decimal256 inputs are not supported. + +```sql +floor(expr) +``` + +### Arguments + +- **expr**: Numeric expression: integer, floating-point or decimal. NULL returns NULL. + +### Example + +```sql +> SELECT floor(-1.1); +-2 +> SELECT floor(-123.45::DECIMAL(5, 2)); +-124 +> SELECT floor('Infinity'::DOUBLE); +9223372036854775807 +``` + +## `getbit` + +_Alias of [bit_get](#bit_get)._ + +## `hex` + +Returns the hexadecimal representation of `expr` as an upper-case string. Numbers are cast to BIGINT and rendered with no leading zeros; negative values are rendered as their 64-bit two's complement (so -1 is `FFFFFFFFFFFFFFFF`). Strings are encoded as the hex of their UTF-8 bytes, and binary values as the hex of their bytes. DataFusion's `to_hex` only accepts integers and returns lower case. Dictionary-encoded input returns a dictionary-encoded string. Returns NULL if the input is NULL. + +```sql +hex(expr) +``` + +### Arguments + +- **expr**: Numeric, string or binary expression. Numbers are cast to BIGINT. NULL returns NULL. + +### Example + +```sql +> SELECT hex(1234); +4D2 +> SELECT hex('Spark SQL'); +537061726B2053514C +``` + +## `hypot` + +Returns `sqrt(expr1^2 + expr2^2)` as a DOUBLE, computed without intermediate overflow or underflow, like `java.lang.Math.hypot`. DataFusion has no built-in `hypot`. Both arguments are cast to DOUBLE. Returns NULL if either input is NULL. If either input is infinite the result is Infinity, even if the other is NaN. Otherwise a NaN input returns NaN. + +```sql +hypot(expr1, expr2) +``` + +### Arguments + +- **expr1**: Numeric expression, cast to DOUBLE. NULL returns NULL. +- **expr2**: Numeric expression, cast to DOUBLE. NULL returns NULL. + +### Example + +```sql +> SELECT hypot(3, 4); +5 +> SELECT hypot('Infinity'::double, 'NaN'::double); +Infinity +> SELECT hypot(NULL::double, 4.0::double); +NULL +``` + +## `negative` + +Returns the negation of `expr` (the same as unary minus), with the same type as the input. DataFusion has no built-in `negative`. Accepts integer, floating-point, decimal and interval values. Negating the minimum value of a signed integer type (for example `-2147483648` for INT) depends on ANSI mode: with `datafusion.execution.enable_ansi_mode` off it wraps around and returns the same value, and with it on the function fails with an overflow error. Returns NULL if the input is NULL and NaN if it is NaN. + +```sql +negative(expr) +``` + +### Arguments + +- **expr**: Numeric or interval expression. NULL returns NULL. + +### Example + +```sql +> SELECT negative(1); +-1 +> SELECT negative(-99.99::DECIMAL(10, 2)); +99.99 +> SELECT negative((-2147483648)::INT); +-2147483648 +``` + +## `positive` + +Returns `expr` unchanged. Accepts integer, floating-point, decimal and interval values. A string is cast to DOUBLE: a malformed string returns NULL with `datafusion.execution.enable_ansi_mode` off and fails with it on. Unlike Spark, a string with a type suffix or a hex float (`'1.5d'`, `'0x1p3'`) is malformed. Returns NULL if the input is NULL. + +```sql +positive(expr) +``` + +### Arguments + +- **expr**: Numeric, interval or string expression. NULL returns NULL. + +### Example + +```sql +> SELECT positive(1); +1 +> SELECT positive('-1.11'); +-1.11 +``` + +## `pow` + +Returns a base expression raised to the power of an exponent. + +```sql +power(base, exponent) +``` + +### Arguments + +- **base**: Numeric expression to operate on. Can be a constant, column, or function, and any combination of operators. +- **exponent**: Exponent numeric expression to operate on. Can be a constant, column, or function, and any combination of operators. + +### Example + +```sql +> SELECT power(2, 3); ++-------------+ +| power(2,3) | ++-------------+ +| 8 | ++-------------+ +``` + +### Aliases + +- power + +## `power` + +_Alias of [pow](#pow)._ + +## `rint` + +Returns the whole number closest to `expr`, as a DOUBLE. Halves round to the even neighbor (2.5 becomes 2, 3.5 becomes 4, -2.5 becomes -2), unlike `round`, which rounds halves away from zero. DataFusion has no built-in `rint`. Integer inputs are returned as DOUBLE. Returns NULL if the input is NULL. + +```sql +rint(expr) +``` + +### Arguments + +- **expr**: Numeric expression. NULL returns NULL. + +### Example + +```sql +> SELECT rint(12.3456); +12 +> SELECT rint(2.5); +2 +> SELECT rint(3.5); +4 +``` + +## `round` + +Rounds `expr` to `scale` decimal places, rounding halves away from zero (HALF_UP: 2.5 becomes 3, -2.5 becomes -3). A negative `scale` rounds to the left of the decimal point, so `round(25, -1)` is 30. The result has the same type as `expr`: integers stay integers, and decimals keep their precision and scale (`round(2.345::DECIMAL(10, 3), 2)` is 2.350), where Spark and DataFusion's `round` reduce the decimal scale (Spark returns 2.35 as DECIMAL(10, 2); see [#37](https://github.com/apache/datafusion-spark/issues/37)). Integer overflow depends on ANSI mode: with `datafusion.execution.enable_ansi_mode` off the result wraps around (`round(2147483647, -1)` is -2147483646), and with it on the function fails with an overflow error. A `scale` beyond the precision of the value returns the value unchanged for positive `scale` and 0 for negative `scale`. Infinity and NaN are returned unchanged. Returns NULL if `expr` or `scale` is NULL. Strings, booleans and timestamps are rejected. + +```sql +round(expr[, scale]) +``` + +### Arguments + +- **expr**: Numeric expression: integer, floating-point or decimal. NULL returns NULL. +- **scale**: Optional constant integer number of decimal places. Defaults to 0. A negative value rounds to the left of the decimal point. NULL returns NULL. + +### Example + +```sql +> SELECT round(2.5::DOUBLE); +3 +> SELECT round(2.345::DOUBLE, 2); +2.35 +> SELECT round(25, -1); +30 +> SELECT round(2.345::DECIMAL(10, 3), 2); +2.350 +``` + +## `sec` + +Returns the secant of `expr` (`1 / cos(expr)`) as a DOUBLE. DataFusion has no built-in `sec`. The argument is cast to DOUBLE. Returns NULL if the input is NULL, and NaN if it is NaN. + +```sql +sec(expr) +``` + +### Arguments + +- **expr**: Numeric angle in radians, cast to DOUBLE. NULL returns NULL. + +### Example + +```sql +> SELECT sec(0); +1 +> SELECT sec(1); +1.850815717680926 +``` + +## `shiftleft` + +Shifts the bits of `expr` left by `n` positions. Bits shifted past the top are lost, so the result can change sign or wrap to 0. As in Spark (and Java), the shift amount is taken modulo the bit width of `expr` (32 for INT, 64 for BIGINT), so `shiftleft(1::int, 32)` is 1 and a negative `n` shifts by the bit width plus `n`. TINYINT and SMALLINT are widened to INT, and the result has the type of the widened `expr`. Returns NULL if either input is NULL. DataFusion has no built-in `shiftleft` function. + +```sql +shiftleft(expr, n) +``` + +### Arguments + +- **expr**: Integer expression to shift. NULL returns NULL. +- **n**: Integer number of bit positions, taken modulo the bit width of `expr`. NULL returns NULL. + +### Example + +```sql +> SELECT shiftleft(2::int, 1); +4 +> SELECT shiftleft(2147483647::int, 1); +-2 +> SELECT shiftleft(1::int, 32); +1 +``` + +## `shiftright` + +Shifts the bits of `expr` right by `n` positions, copying the sign bit into the vacated high bits (an arithmetic shift, so negative values stay negative). As in Spark (and Java), the shift amount is taken modulo the bit width of `expr` (32 for INT, 64 for BIGINT), so `shiftright(1::int, 32)` is 1 and a negative `n` shifts by the bit width plus `n`. TINYINT and SMALLINT are widened to INT, and the result has the type of the widened `expr`. Returns NULL if either input is NULL. See `shiftrightunsigned` for a shift that fills with zeros. DataFusion has no built-in `shiftright` function. + +```sql +shiftright(expr, n) +``` + +### Arguments + +- **expr**: Integer expression to shift. NULL returns NULL. +- **n**: Integer number of bit positions, taken modulo the bit width of `expr`. NULL returns NULL. + +### Example + +```sql +> SELECT shiftright(16::int, 3); +2 +> SELECT shiftright(-16::int, 3); +-2 +> SELECT shiftright(4::int, -1); +0 +``` + +## `shiftrightunsigned` + +Shifts the bits of `expr` right by `n` positions, filling the vacated high bits with zeros (a logical shift, Java's `>>>`), so a negative value becomes a large positive one. As in Spark (and Java), the shift amount is taken modulo the bit width of `expr` (32 for INT, 64 for BIGINT), so `shiftrightunsigned(1::int, 32)` is 1 and a negative `n` shifts by the bit width plus `n`. TINYINT and SMALLINT are widened to INT, and the result has the type of the widened `expr`. Returns NULL if either input is NULL. DataFusion has no built-in `shiftrightunsigned` function. + +```sql +shiftrightunsigned(expr, n) +``` + +### Arguments + +- **expr**: Integer expression to shift. NULL returns NULL. +- **n**: Integer number of bit positions, taken modulo the bit width of `expr`. NULL returns NULL. + +### Example + +```sql +> SELECT shiftrightunsigned(16::int, 3); +2 +> SELECT shiftrightunsigned(-16::int, 3); +536870910 +``` + +## `unhex` + +Converts a string of hexadecimal digits to BINARY: each pair of digits becomes one byte. This is the inverse of `hex`. DataFusion has no built-in `unhex`. Digits may be upper or lower case. A string with an odd number of digits is treated as if it had a leading `0`. Returns NULL if the input is NULL or contains anything other than hex digits (including spaces), and an empty BINARY value for an empty string. + +```sql +unhex(str) +``` + +### Arguments + +- **str**: String expression of hexadecimal digits. NULL returns NULL. + +### Example + +```sql +> SELECT arrow_cast(unhex('537061726B2053514C'), 'Utf8'); +Spark SQL +> SELECT unhex('ABC'); +0abc +> SELECT unhex('GGHH'); +NULL +``` + +## `width_bucket` + +Returns the number of the bucket that `expr` falls into, when the range from `min` to `max` is split into `num_buckets` buckets of equal width. DataFusion has no built-in `width_bucket`. Buckets are numbered from 1. A value before the range returns 0, and a value at or beyond the end of the range returns `num_buckets + 1`. `min` may be greater than `max`, in which case buckets count down from `min`. Works on numbers (cast to DOUBLE), durations and intervals. Returns NULL if any input is NULL, if `num_buckets` is 0 or less, if `min` equals `max`, if a number is NaN or infinite, or if an interval mixes months with days or smaller units. + +```sql +width_bucket(expr, min, max, num_buckets) +``` + +### Arguments + +- **expr**: Numeric, duration or interval value to place in a bucket. NULL returns NULL. +- **min**: Start of the range, of the same kind as `expr`. NULL returns NULL. +- **max**: End of the range, of the same kind as `expr`. NULL returns NULL. +- **num_buckets**: Integer number of buckets. 0 or less, or NULL, returns NULL. + +### Example + +```sql +> SELECT width_bucket(5.3, 0.2, 10.6, 5); +3 +> SELECT width_bucket(-0.9, 5.2, 0.5, 2); +3 +> SELECT width_bucket(8.1, 0.0, 5.7, 4); +5 +> SELECT width_bucket(INTERVAL '1' DAY, INTERVAL '0' DAY, INTERVAL '10' DAY, 10); +2 +``` diff --git a/docs/functions/scalar/other.md b/docs/functions/scalar/other.md new file mode 100644 index 00000000..bf8f1a40 --- /dev/null +++ b/docs/functions/scalar/other.md @@ -0,0 +1,158 @@ + + + + +# Other Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [bitmap_bit_position](#bitmap_bit_position) +- [bitmap_bucket_number](#bitmap_bucket_number) +- [bitmap_count](#bitmap_count) +- [equal_null](#equal_null) +- [spark_cast](#spark_cast) +- [stack](#stack) + +## `bitmap_bit_position` + +Returns the bit position of a value within its bitmap bucket, as a BIGINT from 0 to 32767. Spark's bitmap functions split integers into buckets of 32768 values (one 4 KiB bitmap each); `bitmap_bucket_number` gives the bucket. A positive value maps to `(value - 1) % 32768`, and zero or a negative value maps to `-value % 32768`. NULL returns NULL. DataFusion has no built-in `bitmap_bit_position`. + +```sql +bitmap_bit_position(expr) +``` + +### Arguments + +- **expr**: TINYINT, SMALLINT, INT or BIGINT expression. NULL returns NULL. + +### Example + +```sql +> SELECT bitmap_bit_position(1); +0 +> SELECT bitmap_bit_position(32769); +0 +> SELECT bitmap_bit_position(-1); +1 +``` + +## `bitmap_bucket_number` + +Returns the number of the bitmap bucket that a value falls in, as a BIGINT. Spark's bitmap functions split integers into buckets of 32768 values (one 4 KiB bitmap each); `bitmap_bit_position` gives the position within the bucket. A positive value maps to `1 + (value - 1) / 32768`, so 1 to 32768 are bucket 1. Zero or a negative value maps to `value / 32768`, rounded toward zero, so -32767 to 0 are bucket 0. NULL returns NULL. DataFusion has no built-in `bitmap_bucket_number`. + +```sql +bitmap_bucket_number(expr) +``` + +### Arguments + +- **expr**: TINYINT, SMALLINT, INT or BIGINT expression. NULL returns NULL. + +### Example + +```sql +> SELECT bitmap_bucket_number(1); +1 +> SELECT bitmap_bucket_number(65537); +3 +> SELECT bitmap_bucket_number(-65536); +-2 +``` + +## `bitmap_count` + +Returns the number of bits that are set in a binary bitmap, as a BIGINT. NULL returns NULL. DataFusion has no built-in `bitmap_count`. + +```sql +bitmap_count(bitmap) +``` + +### Arguments + +- **bitmap**: Binary expression, such as a bitmap built by Spark's `bitmap_construct_agg`. NULL returns NULL. + +### Example + +```sql +> SELECT bitmap_count(X'1010'); +2 +> SELECT bitmap_count(X'FFFF'); +16 +``` + +## `equal_null` + +Returns true if the two arguments are equal or both NULL, and false otherwise. Never returns NULL. Same as Spark's `<=>` operator and DataFusion's `IS NOT DISTINCT FROM`, which it is rewritten to. The arguments are coerced to a common type, so `equal_null(1::int, 1.0::double)` is true; it is an error if they can't be compared. For floating-point values NaN equals NaN and 0.0 equals -0.0, including inside arrays and structs. DataFusion has no built-in `equal_null`. + +```sql +equal_null(expr1, expr2) +``` + +### Arguments + +- **expr1**: Expression of any comparable type. +- **expr2**: Expression of a type comparable with `expr1`. + +### Example + +```sql +> SELECT equal_null(3, 3); +true +> SELECT equal_null(1, NULL); +false +> SELECT equal_null(NULL, NULL); +true +``` + +## `spark_cast` + +Casts `expr` to `type` as Spark's `CAST(expr AS type)` does. The target type is a string literal in Spark's SQL type syntax: BOOLEAN, TINYINT (BYTE), SMALLINT (SHORT), INT (INTEGER), BIGINT (LONG), FLOAT (REAL), DOUBLE, DECIMAL(p, s) (DEC, NUMERIC), STRING, CHAR(n), VARCHAR(n), BINARY, DATE, TIMESTAMP (TIMESTAMP_LTZ), TIMESTAMP_NTZ, and ARRAY<...>, MAP<..., ...> and STRUCT of these. Type names are case-insensitive. CHAR and VARCHAR require a length and cast to STRING. INTERVAL, TIME, VARIANT, VOID and the geospatial types are not supported. The cast follows `datafusion.execution.enable_ansi_mode`, as Spark's follows `spark.sql.ansi.enabled`, and uses `datafusion.execution.time_zone` as the session time zone. Casts are being ported from Comet (https://github.com/apache/datafusion-spark/issues/88): a cast that Spark supports but this function does not yet returns a "not implemented yet" error, and a cast that Spark does not support returns a "not supported" error. Returns NULL if `expr` is NULL. + +```sql +spark_cast(expr, type) +``` + +### Arguments + +- **expr**: Expression to cast. +- **type**: The target type, as a string literal in Spark's SQL type syntax, such as 'int', 'decimal(10, 2)' or 'array'. + +### Example + +```sql +> SELECT spark_cast(true, 'int'); +1 +> SELECT spark_cast(0, 'timestamp'); +1970-01-01T00:00:00Z +``` + +## `stack` + +Separates `expr1`, ..., `exprk` into `n` rows of `ceil(k / n)` columns named `col0`, `col1`, ... The values fill the rows in order, and the last rows are padded with NULLs. DataFusion has no built-in `stack`. In Spark, `stack` is a generator that returns rows. A scalar function returns one value per row, so here the rows are returned as a list of structs, which the caller (e.g. Comet, or `unnest` in SQL) is expected to expand. As in Spark, the list is never NULL, but its elements and fields can be. As in Spark, `n` must be an INT from 1 to 2147483647, and the values of a column must have the same type, which is the type of its first non-NULL value. NULLs, Arrow types of the same Spark type (such as `Utf8` and `Utf8View`), and integer literals that fit an INT column are cast to it. Unlike Spark, which accepts any constant `n` (such as `3 - 1`), `n` must be an integer literal, because the return type is planned before expressions are folded. The rows are materialized, so a very large `n`, such as `stack(2000000000, 1)`, is an error. + +```sql +stack(n, expr1[, ..., exprk]) +``` + +### Arguments + +- **n**: Integer literal number of rows, from 1 to 2147483647. +- **expr**: One or more values to separate into rows. The values of each column must have the same type. + +### Example + +```sql +> SELECT stack(2, 1, 2, 3); +[{col0: 1, col1: 2}, {col0: 3, col1: NULL}] +> SELECT unnest(stack(2, 'a', 1, 'b', 2)); +{col0: a, col1: 1} +{col0: b, col1: 2} +``` diff --git a/docs/functions/scalar/string.md b/docs/functions/scalar/string.md new file mode 100644 index 00000000..09fffe61 --- /dev/null +++ b/docs/functions/scalar/string.md @@ -0,0 +1,499 @@ + + + + +# String Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [ascii](#ascii) +- [char](#char) +- [char_length](#char_length) +- [character_length](#character_length) +- [concat](#concat) +- [concat_ws](#concat_ws) +- [contains](#contains) +- [elt](#elt) +- [format_string](#format_string) +- [ilike](#ilike) +- [is_valid_utf8](#is_valid_utf8) +- [len](#len) +- [length](#length) +- [like](#like) +- [luhn_check](#luhn_check) +- [make_valid_utf8](#make_valid_utf8) +- [printf](#printf) +- [quote](#quote) +- [soundex](#soundex) +- [space](#space) +- [substr](#substr) +- [substring](#substring) + +## `ascii` + +Returns the numeric code point of the first character of `str`. Despite the name, this is the full Unicode code point, not just ASCII (`ascii('😀')` is 128512). An empty string returns 0, and NULL returns NULL. Unlike DataFusion's `ascii`, numeric input is accepted and cast to a string first, so `ascii(2)` is the code point of `'2'`. + +```sql +ascii(str) +``` + +### Arguments + +- **str**: String expression. Numeric values are cast to a string. NULL returns NULL. + +### Example + +```sql +> SELECT ascii('Spark'); +83 +> SELECT ascii(''); +0 +> SELECT ascii(2); +50 +``` + +## `char` + +Returns the character whose code point is `n` modulo 256, so `char(321)` is `A`. A negative `n` returns an empty string, and NULL returns NULL. DataFusion's `chr` instead uses `n` as a full Unicode code point and errors on negative or invalid values. + +```sql +char(n) +``` + +### Arguments + +- **n**: Integer expression, cast to BIGINT. NULL returns NULL. + +### Example + +```sql +> SELECT char(65); +A +> SELECT char(321); +A +> SELECT char(-1); +-- empty string +``` + +## `char_length` + +_Alias of [length](#length)._ + +## `character_length` + +_Alias of [length](#length)._ + +## `concat` + +Concatenates the arguments into a single string, or binary if all arguments are binary. Returns NULL if any argument is NULL, whereas DataFusion's `concat` skips NULL arguments. Unlike DataFusion's `concat`, it also accepts zero arguments and then returns an empty string. + +```sql +concat([expr1[, ..., exprN]]) +``` + +### Arguments + +- **expr**: String or binary expressions to concatenate. Any NULL makes the result NULL. + +### Example + +```sql +> SELECT concat('Spark', 'SQL'); +SparkSQL +> SELECT concat('Spark', 'SQL', NULL); +NULL +> SELECT concat(); +-- empty string +``` + +## `concat_ws` + +Concatenates the arguments after the first, separated by `sep`. NULL arguments and NULL array elements are skipped, so if every argument is NULL the result is an empty string. A NULL separator returns NULL. With only a separator, the result is an empty string. Unlike DataFusion's `concat_ws`, array arguments are expanded into their elements, non-string arguments (numbers, booleans, dates, binary, array elements) are cast to strings, and the result is always `Utf8`. + +```sql +concat_ws(sep[, expr1[, ..., exprN]]) +``` + +### Arguments + +- **sep**: Separator, cast to a string. NULL returns NULL. +- **expr**: Values or arrays to join. Arrays contribute each element. NULL values and elements are skipped. Structs are not supported. + +### Example + +```sql +> SELECT concat_ws(' ', 'Spark', 'SQL'); +Spark SQL +> SELECT concat_ws('/', 'foo', NULL, 'bar'); +foo/bar +> SELECT concat_ws(',', 'x', array('a', 'b'), 'y'); +x,a,b,y +> SELECT concat_ws(NULL, 'a', 'b'); +NULL +``` + +## `contains` + +Returns true if `right` is found inside `left`. DataFusion's own `contains` takes only strings; this one also accepts binary. When both arguments are binary the match is on bytes. When one argument is a string and the other is binary, both are compared as bytes, which is the same as Spark casting the binary to a string. Integer, boolean and date arguments are cast to a string first. Other types are rejected at planning, including floating-point, decimal and timestamp arguments, which Spark casts to a string (see https://github.com/apache/datafusion-spark/issues/347). The match is case-sensitive, and an empty `right` is found in every value, including an empty `left`. Returns NULL if either input is NULL. + +```sql +contains(left, right) +``` + +### Arguments + +- **left**: String or binary expression to search in. NULL returns NULL. +- **right**: String or binary expression to search for. NULL returns NULL. + +### Example + +```sql +> SELECT contains('Spark SQL', 'Spark'); +true +> SELECT contains('Spark SQL', 'SPARK'); +false +> SELECT contains(x'537061726b2053514c', x'537061726b'); +true +``` + +## `elt` + +Returns the `n`-th of the following arguments as a string (1-based). DataFusion has no built-in `elt`. A NULL `n` returns NULL, as does picking a NULL argument. If `n` is less than 1 or greater than the number of arguments, the result is NULL, or an error when `datafusion.execution.enable_ansi_mode` is true. + +```sql +elt(n, expr1[, ..., exprN]) +``` + +### Arguments + +- **n**: Integer index, 1-based, cast to BIGINT. NULL returns NULL. +- **expr**: At least one value to choose from. Each is cast to a string. + +### Example + +```sql +> SELECT elt(1, 'scala', 'java'); +scala +> SELECT elt(2, 'a', 1); +1 +> SELECT elt(3, 'scala', 'java'); +NULL +``` + +## `format_string` + +Returns a string formatted from a printf-style format string, following Java's `java.util.Formatter` as Spark does. DataFusion has no built-in equivalent. Supports the `%s`, `%b`, `%c`, `%d`, `%o`, `%x`, `%e`, `%f`, `%g`, `%a` and `%t` (date/time) conversions and their uppercase forms, the `-`, `+`, ` `, `0`, `#`, `,` and `(` flags, width and precision, and positional (`%2$s`) and relative (`% SELECT format_string('Hello World %d %s', 100, 'days'); +Hello World 100 days +> SELECT format_string('%2$s %1$d', 42, 'test'); +test 42 +> SELECT format_string('Precision: %.2f', 3.14159); +Precision: 3.14 +> SELECT format_string('Value: %s', NULL); +Value: null +``` + +### Alternative Syntax + +```sql +printf(format[, arg1[, ..., argN]]) +``` + +### Aliases + +- printf + +## `ilike` + +Returns true if `str` matches `pattern`, ignoring case. In the pattern, `_` matches any one character and `%` matches any sequence of characters. Returns NULL if either argument is NULL. This is the function form of the `ILIKE` operator, which DataFusion supports only as an operator. Spark's optional third `escape` argument is not supported. + +```sql +ilike(str, pattern) +``` + +### Arguments + +- **str**: String expression to match. NULL returns NULL. +- **pattern**: String pattern, using `_` and `%` as wildcards. NULL returns NULL. + +### Example + +```sql +> SELECT ilike('Spark', '_PARK'); +true +> SELECT ilike('Spark', 'xyz'); +false +> SELECT ilike('Spark', NULL); +NULL +``` + +## `is_valid_utf8` + +Returns true if the input is valid UTF-8. DataFusion has no built-in `is_valid_utf8`. String input is always valid. Binary input is invalid if it has truncated sequences, stray continuation bytes, overlong encodings, encoded surrogates (U+D800 to U+DFFF) or code points above U+10FFFF. Empty input is valid. Returns NULL if the input is NULL. + +```sql +is_valid_utf8(str) +``` + +### Arguments + +- **str**: String or binary expression. NULL returns NULL. + +### Example + +```sql +> SELECT is_valid_utf8('Spark'); +true +> SELECT is_valid_utf8(X'C2A9'); +true +> SELECT is_valid_utf8(X'61C262'); +false +``` + +## `len` + +_Alias of [length](#length)._ + +## `length` + +Returns the number of characters in a string, or the number of bytes in a binary value. Trailing spaces and binary zeros are counted. NULL returns NULL. Unlike DataFusion's `length`/`character_length`, binary input is accepted, and the result is always `Int32`, including for `LargeUtf8` input. + +```sql +length(expr) +``` + +### Arguments + +- **expr**: String or binary expression. NULL returns NULL. + +### Example + +```sql +> SELECT length('Spark SQL '); +10 +> SELECT length(x'537061726b2053514c'); +9 +``` + +### Alternative Syntax + +```sql +char_length(expr) +``` +```sql +character_length(expr) +``` +```sql +len(expr) +``` + +### Aliases + +- character\_length +- char\_length +- len + +## `like` + +Returns true if `str` matches `pattern`, case-sensitively. In the pattern, `_` matches any one character and `%` matches any sequence of characters. Returns NULL if either argument is NULL. This is the function form of the `LIKE` operator, which DataFusion supports only as an operator. Spark's optional third `escape` argument is not supported. + +```sql +like(str, pattern) +``` + +### Arguments + +- **str**: String expression to match. NULL returns NULL. +- **pattern**: String pattern, using `_` and `%` as wildcards. NULL returns NULL. + +### Example + +```sql +> SELECT like('Spark', '_park'); +true +> SELECT like('Spark', 'SP%'); +false +> SELECT like(NULL, '_park'); +NULL +``` + +## `luhn_check` + +Returns true if the string is a number that passes the Luhn checksum, as used for credit card numbers. DataFusion has no built-in `luhn_check`. The string must contain only the digits 0-9: spaces, letters or any other character make the result false, and so does an empty string. Numeric input is cast to a string first. Returns NULL if the input is NULL. + +```sql +luhn_check(str) +``` + +### Arguments + +- **str**: String of digits to check. Numeric input is cast to a string. NULL returns NULL. + +### Example + +```sql +> SELECT luhn_check('79927398713'); +true +> SELECT luhn_check('79927398714'); +false +> SELECT luhn_check('4111111 111111111'); +false +``` + +## `make_valid_utf8` + +Returns the input as a valid UTF-8 string, replacing each invalid byte sequence with the replacement character U+FFFD. DataFusion has no built-in `make_valid_utf8`. String input is already valid and is returned unchanged. Binary input is decoded as UTF-8; overlong encodings and encoded surrogates count as invalid. Returns NULL if the input is NULL. + +```sql +make_valid_utf8(str) +``` + +### Arguments + +- **str**: String or binary expression. NULL returns NULL. + +### Example + +```sql +> SELECT make_valid_utf8('Spark'); +Spark +> SELECT make_valid_utf8(X'C3A9'); +é +> SELECT make_valid_utf8(X'61C262'); +a�b +``` + +## `printf` + +_Alias of [format_string](#format_string)._ + +## `quote` + +Returns `str` wrapped in single quotes, with each single quote inside it escaped with a backslash. No other characters are escaped. DataFusion has no built-in equivalent. Values of other types are cast to a string first. NULL returns NULL. + +```sql +quote(str) +``` + +### Arguments + +- **str**: Expression to quote. Non-string values are cast to a string. NULL returns NULL. + +### Example + +```sql +> SELECT quote('Spark'); +'Spark' +> SELECT quote('it''s'); +'it\'s' +> SELECT quote(NULL); +NULL +``` + +## `soundex` + +Returns the four-character Soundex code of `str`: its first letter in upper case followed by three digits, padded with `0`. Letters are matched case-insensitively. A string that does not start with an ASCII letter (for example a digit, space, symbol or `Ñ`) is returned unchanged, as is the empty string. NULL returns NULL. DataFusion has no built-in equivalent. + +```sql +soundex(str) +``` + +### Arguments + +- **str**: String expression. NULL returns NULL. + +### Example + +```sql +> SELECT soundex('Miller'); +M460 +> SELECT soundex('Robert'); +R163 +> SELECT soundex('123'); +123 +``` + +## `space` + +Returns a string of `n` spaces. A value of 0 or less returns an empty string, and NULL returns NULL. DataFusion has no built-in equivalent. `n` must be an INT, and integer literals are BIGINT in DataFusion, so cast them (`space(2::INT)`; see [#97](https://github.com/apache/datafusion-spark/issues/97)). + +```sql +space(n) +``` + +### Arguments + +- **n**: INT number of spaces. NULL returns NULL. + +### Example + +```sql +> SELECT concat(space(2::INT), 'Spark'); + Spark +> SELECT space(-1::INT); +-- empty string +``` + +## `substr` + +_Alias of [substring](#substring)._ + +## `substring` + +Returns the substring of `str` that starts at `pos` and is `length` characters long. Positions are 1-based and count characters (bytes for binary input). A position of 0 is treated as 1. A negative position counts from the end of the string (-1 is the last character). If it reaches past the start, the part before the start comes off `length` (so `substring('abc', -10, 3)` is empty). A position past the end, or a length of 0 or less, returns an empty string, and a length that runs past the end is clamped. Returns NULL if any input is NULL. DataFusion's `substr` also takes the part before the start off `length`, but counts positions below 1 from the start instead of the end (0 is one before the first character, -1 two before), so `substring('abc', -3, 6)` returns `abc`, where DataFusion's `substr('abc', -3, 6)` returns `ab`. + +```sql +substring(str, pos[, length]) +``` + +### Arguments + +- **str**: String or binary expression. For binary input, positions count bytes. +- **pos**: Integer start position, 1-based. 0 is treated as 1. A negative value counts from the end of the string. +- **length**: Optional integer maximum length. If omitted, the rest of the string is returned. A value of 0 or less returns an empty string. + +### Example + +```sql +> SELECT substring('Spark SQL', 5); +k SQL +> SELECT substring('Spark SQL', -3); +SQL +> SELECT substring('Spark SQL', 0); +Spark SQL +> SELECT substring('Spark SQL', 3, 0); +-- empty string +> SELECT substring('Spark SQL', NULL); +NULL +``` + +### Alternative Syntax + +```sql +substr(str, pos[, length]) +``` + +### Aliases + +- substr diff --git a/docs/functions/scalar/time_and_date.md b/docs/functions/scalar/time_and_date.md new file mode 100644 index 00000000..2880b8c3 --- /dev/null +++ b/docs/functions/scalar/time_and_date.md @@ -0,0 +1,713 @@ + + + + +# Time and Date Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [add_months](#add_months) +- [date_add](#date_add) +- [date_diff](#date_diff) +- [date_from_unix_date](#date_from_unix_date) +- [date_part](#date_part) +- [date_sub](#date_sub) +- [date_trunc](#date_trunc) +- [dateadd](#dateadd) +- [datediff](#datediff) +- [datepart](#datepart) +- [dayname](#dayname) +- [from_utc_timestamp](#from_utc_timestamp) +- [hour](#hour) +- [last_day](#last_day) +- [make_dt_interval](#make_dt_interval) +- [make_interval](#make_interval) +- [make_time](#make_time) +- [minute](#minute) +- [monthname](#monthname) +- [next_day](#next_day) +- [second](#second) +- [time_trunc](#time_trunc) +- [to_utc_timestamp](#to_utc_timestamp) +- [trunc](#trunc) +- [unix_date](#unix_date) +- [unix_micros](#unix_micros) +- [unix_millis](#unix_millis) +- [unix_seconds](#unix_seconds) +- [weekday](#weekday) + +## `add_months` + +Returns the date that is `num_months` months after `start_date`. A negative `num_months` moves the date backwards. DataFusion has no built-in `add_months`. If the day of the month does not exist in the result month, the last day of that month is returned (so subtracting 5 months from 2016-07-30 gives 2016-02-29). Returns NULL if either input is NULL. A result outside the supported date range returns an error. `num_months` must be an INT. Integer literals are BIGINT in DataFusion, so cast them (for example `1::INT`; see [#97](https://github.com/apache/datafusion-spark/issues/97)). `start_date` must be a DATE; strings are not implicitly cast as they are in Spark ([#41](https://github.com/apache/datafusion-spark/issues/41)). + +```sql +add_months(start_date, num_months) +``` + +### Arguments + +- **start_date**: DATE expression. NULL returns NULL. +- **num_months**: INT number of months to add. Negative values subtract months. NULL returns NULL. + +### Example + +```sql +> SELECT add_months(DATE '2016-07-30', 1::INT); +2016-08-30 +> SELECT add_months(DATE '2016-07-30', -5::INT); +2016-02-29 +> SELECT add_months(DATE '2016-07-30', NULL::INT); +NULL +``` + +## `date_add` + +Returns the date that is `num_days` days after `start_date`. A negative `num_days` moves the date backwards. DataFusion has no built-in `date_add`. Returns NULL if either input is NULL. The arithmetic wraps around on overflow instead of returning an error. `num_days` must be INT, SMALLINT or TINYINT. Integer literals are BIGINT in DataFusion, so cast them (for example `1::INT`; see [#97](https://github.com/apache/datafusion-spark/issues/97)). `start_date` must be a DATE; strings are not implicitly cast as they are in Spark ([#41](https://github.com/apache/datafusion-spark/issues/41)). + +```sql +date_add(start_date, num_days) +``` + +### Arguments + +- **start_date**: DATE expression. NULL returns NULL. +- **num_days**: INT, SMALLINT or TINYINT number of days to add. Negative values subtract days. NULL returns NULL. + +### Example + +```sql +> SELECT date_add(DATE '2016-07-30', 1::INT); +2016-07-31 +> SELECT date_add(DATE '2016-07-30', -5::INT); +2016-07-25 +> SELECT date_add(DATE '2016-07-30', NULL::INT); +NULL +``` + +### Alternative Syntax + +```sql +dateadd(start_date, num_days) +``` + +### Aliases + +- dateadd + +## `date_diff` + +Returns the number of days from `start_date` to `end_date`, as an INT. The result is negative when `end_date` is before `start_date`. DataFusion has no built-in `date_diff`. Strings and timestamps are cast to DATE first, so the time of day is ignored. Returns NULL if either input is NULL. + +```sql +date_diff(end_date, start_date) +``` + +### Arguments + +- **end_date**: DATE, TIMESTAMP or string expression. NULL returns NULL. +- **start_date**: DATE, TIMESTAMP or string expression. NULL returns NULL. + +### Example + +```sql +> SELECT date_diff(DATE '2009-07-31', DATE '2009-07-30'); +1 +> SELECT date_diff('2009-07-30', '2009-07-31'); +-1 +> SELECT date_diff(TIMESTAMP '2009-07-31 23:45:01', TIMESTAMP '2009-07-30 12:34:56'); +1 +``` + +### Alternative Syntax + +```sql +datediff(end_date, start_date) +``` + +### Aliases + +- datediff + +## `date_from_unix_date` + +Returns the date that is `days` days after 1970-01-01. This is the inverse of `unix_date`. DataFusion has no built-in equivalent. Negative values give dates before the epoch. Any INT is a valid date. Other numeric and string inputs are cast to INT first, which truncates fractions and raises an error if the value is out of the INT range or is not a valid number. These errors match Spark with ANSI mode enabled. With ANSI mode disabled, Spark wraps an out-of-range BIGINT and returns NULL for a malformed string, while this function still errors (see https://github.com/apache/datafusion-spark/issues/88). Returns NULL if the input is NULL. + +```sql +date_from_unix_date(days) +``` + +### Arguments + +- **days**: Number of days since 1970-01-01, as an INT. Other integer, floating-point, decimal and string inputs are cast to INT. BOOLEAN is rejected. NULL returns NULL. + +### Example + +```sql +> SELECT date_from_unix_date(1000); +1972-09-27 +> SELECT date_from_unix_date(-1); +1969-12-31 +> SELECT date_from_unix_date(NULL); +NULL +``` + +## `date_part` + +Extracts a field from a date or timestamp, as an INT. Unlike DataFusion's `date_part`, it accepts Spark's field names and abbreviations (case-insensitive), and `DAYOFWEEK`/`DOW` counts from 1 (Sunday) to 7 (Saturday) where DataFusion's `dow` counts from 0 (Sunday). `DAYOFWEEK_ISO`/`DOW_ISO` counts from 1 (Monday) to 7 (Sunday), and `YEAROFWEEK` is the ISO week-numbering year. A field name Spark does not accept (such as `QUARTERS` or `DAYOFYEAR`) is an error. A NULL field or source returns NULL. `SECOND` returns whole seconds; Spark returns DECIMAL(8, 6) including the fraction of a second. + +```sql +date_part(field, source) +``` + +### Arguments + +- **field**: String literal naming the field to extract: `YEAR` (`Y`, `YEARS`, `YR`, `YRS`), `YEAROFWEEK`, `QUARTER` (`QTR`), `MONTH` (`MON`, `MONS`, `MONTHS`), `WEEK` (`W`, `WEEKS`), `DAY` (`D`, `DAYS`), `DAYOFWEEK` (`DOW`), `DAYOFWEEK_ISO` (`DOW_ISO`), `DOY`, `HOUR` (`H`, `HOURS`, `HR`, `HRS`), `MINUTE` (`M`, `MIN`, `MINS`, `MINUTES`) or `SECOND` (`S`, `SEC`, `SECONDS`, `SECS`). NULL returns NULL. +- **source**: DATE or TIMESTAMP expression. NULL returns NULL. + +### Example + +```sql +> SELECT date_part('YEAR', DATE '2000-01-01'); +2000 +> SELECT date_part('DOW', DATE '2000-01-01'); +7 +> SELECT date_part('MINS', TIMESTAMP '2000-01-01 12:30:45'); +30 +> SELECT date_part('year', NULL::TIMESTAMP); +NULL +``` + +### Alternative Syntax + +```sql +datepart(field, source) +``` + +### Aliases + +- datepart + +## `date_sub` + +Returns the date that is `num_days` days before `start_date`. A negative `num_days` moves the date forwards. DataFusion has no built-in `date_sub`. Returns NULL if either input is NULL. The arithmetic wraps around on overflow instead of returning an error. `num_days` must be INT, SMALLINT or TINYINT. Integer literals are BIGINT in DataFusion, so cast them (for example `1::INT`; see [#97](https://github.com/apache/datafusion-spark/issues/97)). `start_date` must be a DATE; strings are not implicitly cast as they are in Spark ([#41](https://github.com/apache/datafusion-spark/issues/41)). + +```sql +date_sub(start_date, num_days) +``` + +### Arguments + +- **start_date**: DATE expression. NULL returns NULL. +- **num_days**: INT, SMALLINT or TINYINT number of days to subtract. Negative values add days. NULL returns NULL. + +### Example + +```sql +> SELECT date_sub(DATE '2016-07-30', 1::INT); +2016-07-29 +> SELECT date_sub(DATE '2016-07-30', -5::INT); +2016-08-04 +> SELECT date_sub(NULL::DATE, 1::INT); +NULL +``` + +## `date_trunc` + +Returns `ts` truncated to the unit given by `fmt`, with the same type as `ts`. Unlike DataFusion's `date_trunc`, it also accepts Spark's unit aliases `yyyy` and `yy` for year, `mon` and `mm` for month and `dd` for day, and it truncates a timestamp with a time zone in the session time zone (`datafusion.execution.time_zone`) rather than the timestamp's own time zone. A string `ts` is cast to a timestamp. Returns NULL if `ts` is NULL. `fmt` must be a constant string: a NULL `fmt` is a planning error and an unsupported `fmt` is an execution error, where Spark returns NULL for both. + +```sql +date_trunc(fmt, ts) +``` + +### Arguments + +- **fmt**: Constant string unit, case-insensitive: `year` (`yyyy`, `yy`), `quarter`, `month` (`mon`, `mm`), `week` (truncates to Monday), `day` (`dd`), `hour`, `minute`, `second`, `millisecond` or `microsecond`. +- **ts**: Timestamp expression to truncate. A string is cast to a timestamp. NULL returns NULL. + +### Example + +```sql +> SELECT date_trunc('YEAR', '2015-03-05T09:32:05.123456'::timestamp); +2015-01-01T00:00:00 +> SELECT date_trunc('MM', '2015-03-05T09:32:05.123456'::timestamp); +2015-03-01T00:00:00 +> SELECT date_trunc('HOUR', '2015-03-05T09:32:05.123456'::timestamp); +2015-03-05T09:00:00 +> SELECT date_trunc('MILLISECOND', '2015-03-05T09:32:05.123456'::timestamp); +2015-03-05T09:32:05.123 +``` + +## `dateadd` + +_Alias of [date_add](#date_add)._ + +## `datediff` + +_Alias of [date_diff](#date_diff)._ + +## `datepart` + +_Alias of [date_part](#date_part)._ + +## `dayname` + +Returns the three-letter abbreviated English day name (`Mon` to `Sun`) of a date. DataFusion has no built-in `dayname`. Strings are implicitly cast to DATE. A malformed string is an error, even with ANSI mode off where Spark returns NULL. Timestamps are cast to DATE first, so the time of day is ignored. A timestamp with a time zone is converted in its own time zone, not the session time zone that Spark uses. Returns NULL if the input is NULL. + +```sql +dayname(date) +``` + +### Arguments + +- **date**: DATE, TIMESTAMP or string expression. NULL returns NULL. + +### Example + +```sql +> SELECT dayname('2008-02-20'::DATE); +Wed +> SELECT dayname('2008-02-20'); +Wed +> SELECT dayname(arrow_cast('2024-05-20 23:59:59', 'Timestamp(Microsecond, None)')); +Mon +> SELECT dayname(NULL::DATE); +NULL +``` + +## `from_utc_timestamp` + +Treats `ts` as a UTC wall-clock time and returns the wall-clock time in time zone `tz`. DataFusion has no built-in equivalent. A string `ts` is cast to a timestamp, so a string with an offset is first converted to UTC. The result has the same type as `ts`, including any time zone it carries. Returns NULL if either argument is NULL. `tz` accepts Spark's spellings: region IDs, offsets such as `+01:00`, `GMT+1`, `UTC+1:2` and `UT-010203`, Java short IDs such as `PST`, and legacy `SystemV/` IDs. An offset beyond 18 hours or an unknown zone is an error. Differences from Spark: `tz` is only parsed for rows where both arguments are non-NULL, so an invalid constant `tz` can return NULL where Spark raises an error; conversion is limited to Chrono's calendar range, which is narrower than the range Spark implements but covers Spark's documented range; and region rules from `chrono-tz` may differ after 2099, because it does not extrapolate later daylight saving time transitions. + +```sql +from_utc_timestamp(ts, tz) +``` + +### Arguments + +- **ts**: Timestamp expression, interpreted as UTC. A string is cast to a timestamp. NULL returns NULL. +- **tz**: String time zone to convert to. NULL returns NULL. + +### Example + +```sql +> SELECT from_utc_timestamp('2016-08-31'::timestamp, 'Asia/Seoul'); +2016-08-31T09:00:00 +> SELECT from_utc_timestamp('2018-03-13T06:18:23+02:00', 'America/New_York'); +2018-03-13T00:18:23 +> SELECT from_utc_timestamp('2024-07-01T00:00:00'::timestamp, 'GMT+1'); +2024-07-01T01:00:00 +> SELECT from_utc_timestamp('2016-08-31'::timestamp, NULL); +NULL +``` + +## `hour` + +Returns the hour of the day (0 to 23) of a timestamp, as an INT. DataFusion has no built-in `hour`. Returns NULL if the input is NULL. The input must be a TIMESTAMP; strings are not implicitly cast as they are in Spark ([#41](https://github.com/apache/datafusion-spark/issues/41)). + +```sql +hour(timestamp) +``` + +### Arguments + +- **timestamp**: TIMESTAMP expression. NULL returns NULL. + +### Example + +```sql +> SELECT hour(TIMESTAMP '2009-07-30 12:58:59'); +12 +> SELECT hour(NULL::TIMESTAMP); +NULL +``` + +## `last_day` + +Returns the last day of the month that `date` falls in, taking leap years into account. DataFusion has no built-in `last_day`. Returns NULL if `date` is NULL. The input must be a DATE; strings and timestamps are not implicitly cast as they are in Spark ([#41](https://github.com/apache/datafusion-spark/issues/41)). + +```sql +last_day(date) +``` + +### Arguments + +- **date**: DATE expression. NULL returns NULL. + +### Example + +```sql +> SELECT last_day(DATE '2009-01-12'); +2009-01-31 +> SELECT last_day(DATE '2016-02-07'); +2016-02-29 +> SELECT last_day(NULL::DATE); +NULL +``` + +## `make_dt_interval` + +Returns a day-time interval built from days, hours, minutes and seconds. DataFusion has no built-in equivalent. All arguments are optional from the right and default to 0, so `make_dt_interval()` is a zero interval. Components can be negative and are added together, so `make_dt_interval(1, -24)` is zero. The result is a `Duration(Microsecond)` rather than an Arrow interval, because Arrow's day-time interval only has millisecond precision while Spark's has microsecond precision. Returns NULL if any argument is NULL or `secs` is NaN or infinite. Overflow returns NULL, where Spark raises an error ([#20](https://github.com/apache/datafusion-spark/issues/20)). Unlike Spark, where `secs` is a DECIMAL(18, 6), `secs` is cast to DOUBLE and the fraction is rounded to microseconds. + +```sql +make_dt_interval([days[, hours[, mins[, secs]]]]) +``` + +### Arguments + +- **days**: Integer number of days, cast to INT. NULL returns NULL. +- **hours**: Integer number of hours, cast to INT. NULL returns NULL. +- **mins**: Integer number of minutes, cast to INT. NULL returns NULL. +- **secs**: Numeric number of seconds, which may have a fraction, cast to DOUBLE. NULL, NaN or an infinite value returns NULL. + +### Example + +```sql +> SELECT make_dt_interval(1, 12, 30, 1.001001); +1 days 12 hours 30 mins 1.001001 secs +> SELECT make_dt_interval(2); +2 days 0 hours 0 mins 0.000000 secs +> SELECT make_dt_interval(100, NULL, 3); +NULL +``` + +## `make_interval` + +Returns an interval built from years, months, weeks, days, hours, minutes and seconds. DataFusion has no built-in equivalent. All arguments are optional from the right and default to 0, so `make_interval()` is a zero interval. The result is a month-day-nanosecond interval: years and months become months, weeks and days become days, and the rest becomes nanoseconds. Returns NULL if any argument is NULL or `secs` is NaN or infinite. Overflow returns NULL; ANSI mode is not applied. Unlike Spark, where `secs` is a DECIMAL(18, 6), `secs` is cast to DOUBLE and the fraction is rounded to nanoseconds. + +```sql +make_interval([years[, months[, weeks[, days[, hours[, mins[, secs]]]]]]]) +``` + +### Arguments + +- **years**: Integer number of years, cast to INT. NULL returns NULL. +- **months**: Integer number of months, cast to INT. NULL returns NULL. +- **weeks**: Integer number of weeks, cast to INT. NULL returns NULL. +- **days**: Integer number of days, cast to INT. NULL returns NULL. +- **hours**: Integer number of hours, cast to INT. NULL returns NULL. +- **mins**: Integer number of minutes, cast to INT. NULL returns NULL. +- **secs**: Numeric number of seconds, which may have a fraction, cast to DOUBLE. NULL, NaN or an infinite value returns NULL. + +### Example + +```sql +> SELECT make_interval(1, 1, 1, 1, 1, 1, 1.0); +13 mons 8 days 1 hours 1 mins 1.000000000 secs +> SELECT make_interval(2147483647, 1, 0, 0, 0, 0, 0.0); +NULL +``` + +## `make_time` + +Creates a time of day from hour, minute and second fields. The result is a `Time64(Nanosecond)` with microsecond precision, like Spark's `TIME(6)`. Returns NULL if any argument is NULL. Out-of-range fields raise a `DATETIME_FIELD_OUT_OF_BOUNDS` error whether or not ANSI mode is enabled. Seconds that are negative or above 2147483647 are checked first, then hour, minute and second, so the error names the same field as Spark's. Arguments are cast to INT, INT and DECIMAL(16, 6) with DataFusion's cast rather than Spark's, so a value the cast cannot represent (an integer outside the INT range, a malformed string, seconds that do not fit DECIMAL(16, 6), NaN or Infinity) raises a cast error in both ANSI modes. With ANSI mode off Spark wraps the integer or returns NULL instead, and it returns NULL for NaN and Infinity in both modes (https://github.com/apache/datafusion-spark/issues/88). + +```sql +make_time(hour, minute, second) +``` + +### Arguments + +- **hour**: Hour of the day, 0 to 23. Numeric and string values are cast to INT, and a fractional number is truncated. +- **minute**: Minute of the hour, 0 to 59. Numeric and string values are cast to INT, and a fractional number is truncated. +- **second**: Second of the minute with an optional fraction, 0 to 59.999999. Numeric and string values are cast to DECIMAL(16, 6), so extra fractional digits are rounded half up (59.9999995 becomes 60 and is out of range). + +### Example + +```sql +> SELECT make_time(6, 30, 45.887); +06:30:45.887 +> SELECT make_time(23, 59, 59.999999); +23:59:59.999999 +> SELECT make_time(1, 18, NULL); +NULL +> SELECT make_time(24, 18, 19.87); +-- error: Invalid value for HourOfDay (valid values 0 - 23): 24 +``` + +## `minute` + +Returns the minute of the hour (0 to 59) of a timestamp, as an INT. DataFusion has no built-in `minute`. Returns NULL if the input is NULL. The input must be a TIMESTAMP; strings are not implicitly cast as they are in Spark ([#41](https://github.com/apache/datafusion-spark/issues/41)). + +```sql +minute(timestamp) +``` + +### Arguments + +- **timestamp**: TIMESTAMP expression. NULL returns NULL. + +### Example + +```sql +> SELECT minute(TIMESTAMP '2009-07-30 12:58:59'); +58 +> SELECT minute(NULL::TIMESTAMP); +NULL +``` + +## `monthname` + +Returns the three-letter abbreviated English month name (`Jan` to `Dec`) of a date. DataFusion has no built-in `monthname`. Strings are implicitly cast to DATE. A malformed string is an error, even with ANSI mode off where Spark returns NULL. Timestamps are cast to DATE first, so the time of day is ignored. A timestamp with a time zone is converted in its own time zone, not the session time zone that Spark uses. Returns NULL if the input is NULL. + +```sql +monthname(date) +``` + +### Arguments + +- **date**: DATE, TIMESTAMP or string expression. NULL returns NULL. + +### Example + +```sql +> SELECT monthname('2008-02-20'::DATE); +Feb +> SELECT monthname('2008-02-20'); +Feb +> SELECT monthname('2024-03-15 12:34:56'::TIMESTAMP); +Mar +> SELECT monthname(NULL::DATE); +NULL +``` + +## `next_day` + +Returns the first date after `start_date` that falls on `day_of_week`. The result is always later than `start_date`: if `start_date` is already that day, the date one week later is returned. DataFusion has no built-in `next_day`. Strings and timestamps are cast to DATE first. Returns NULL if either input is NULL, or if `day_of_week` is not a recognized day name (including names padded with whitespace) when ANSI mode is disabled. Spark raises an `[ILLEGAL_DAY_OF_WEEK]` error for an unrecognized day name when ANSI mode is enabled. + +```sql +next_day(start_date, day_of_week) +``` + +### Arguments + +- **start_date**: DATE, TIMESTAMP or string expression. NULL returns NULL. +- **day_of_week**: Day of the week, case-insensitive: a two-letter abbreviation (`MO`), a three-letter abbreviation (`Mon`) or the full name (`Monday`). Other types are cast to string. NULL or an unrecognized value returns NULL (or an error when `datafusion.execution.enable_ansi_mode` is true). + +### Example + +```sql +> SELECT next_day(DATE '2015-01-14', 'TU'); +2015-01-20 +> SELECT next_day(DATE '2015-07-27', 'Sun'); +2015-08-02 +> SELECT next_day(TIMESTAMP '2015-07-23 12:12:12', 'Mon'); +2015-07-27 +> SELECT next_day(DATE '2020-01-01', 'invalid_day'); +NULL +``` + +## `second` + +Returns the second of the minute (0 to 59) of a timestamp, as an INT. Fractions of a second are dropped. DataFusion has no built-in `second`. Returns NULL if the input is NULL. The input must be a TIMESTAMP; strings are not implicitly cast as they are in Spark ([#41](https://github.com/apache/datafusion-spark/issues/41)). + +```sql +second(timestamp) +``` + +### Arguments + +- **timestamp**: TIMESTAMP expression. NULL returns NULL. + +### Example + +```sql +> SELECT second(TIMESTAMP '2009-07-30 12:58:59'); +59 +> SELECT second(NULL::TIMESTAMP); +NULL +``` + +## `time_trunc` + +Returns the time `time` truncated to the unit given by `unit`, with the same type as `time`. DataFusion has no `time_trunc`; this rewrites to DataFusion's `date_trunc`, but only accepts time input and the units `HOUR`, `MINUTE`, `SECOND`, `MILLISECOND` and `MICROSECOND` (case-insensitive). Any other unit is an error, with the same message and SQLSTATE 22023 as Spark. Returns NULL if `unit` or `time` is NULL. + +```sql +time_trunc(unit, time) +``` + +### Arguments + +- **unit**: Constant string unit: `HOUR`, `MINUTE`, `SECOND`, `MILLISECOND` or `MICROSECOND`, case-insensitive. NULL returns NULL. +- **time**: Time expression to truncate. NULL returns NULL. + +### Example + +```sql +> SELECT time_trunc('HOUR', '09:32:05.123456'::time); +09:00:00 +> SELECT time_trunc('MILLISECOND', '09:32:05.123456'::time); +09:32:05.123 +> SELECT time_trunc(NULL, '09:32:05.123456'::time); +NULL +``` + +## `to_utc_timestamp` + +Treats `ts` as a wall-clock time in time zone `tz` and returns the UTC wall-clock time. This is the inverse of `from_utc_timestamp`. DataFusion has no built-in equivalent. A string `ts` is cast to a timestamp, so a string with an offset is first converted to UTC. The result has the same type as `ts`, including any time zone it carries. Returns NULL if either argument is NULL. Unlike `from_utc_timestamp`, `tz` is parsed by Arrow, so it accepts region IDs and `+HH:MM` offsets but not all of Spark's spellings (such as `GMT+1` or `PST`). An unknown zone is an error. Within a few hours of a daylight saving time change, the offset can be taken from the wrong side of the change, so the result can differ from Spark by the size of the change ([#110](https://github.com/apache/datafusion-spark/issues/110)). + +```sql +to_utc_timestamp(ts, tz) +``` + +### Arguments + +- **ts**: Timestamp expression, interpreted as a wall-clock time in `tz`. A string is cast to a timestamp. NULL returns NULL. +- **tz**: String time zone that `ts` is in: a region ID such as `Asia/Seoul`, or an offset such as `+09:00`. NULL returns NULL. + +### Example + +```sql +> SELECT to_utc_timestamp('2016-08-31'::timestamp, 'Asia/Seoul'); +2016-08-30T15:00:00 +> SELECT to_utc_timestamp('2018-03-13T06:18:23+02:00', 'America/New_York'); +2018-03-13T08:18:23 +> SELECT to_utc_timestamp('2016-08-31'::timestamp, NULL); +NULL +``` + +## `trunc` + +Returns `date` truncated to the unit given by `fmt`, as a date. Unlike DataFusion's `date_trunc`, it takes the date first and the unit second, only accepts dates (a string is cast to a date), and also accepts Spark's unit aliases `yyyy` and `yy` for year and `mon` and `mm` for month. Returns NULL if `date` is NULL. `fmt` must be a constant string: a NULL or unsupported `fmt` is a planning error, where Spark returns NULL. + +```sql +trunc(date, fmt) +``` + +### Arguments + +- **date**: Date expression to truncate. A string is cast to a date. NULL returns NULL. +- **fmt**: Constant string unit, case-insensitive: `year` (`yyyy`, `yy`), `quarter`, `month` (`mon`, `mm`), `week` (truncates to Monday) or `day`, or another spelling of these that DataFusion accepts (such as `months`). + +### Example + +```sql +> SELECT trunc('2009-02-12'::date, 'YEAR'); +2009-01-01 +> SELECT trunc('2009-02-12'::date, 'MM'); +2009-02-01 +> SELECT trunc('2009-02-12'::date, 'WEEK'); +2009-02-09 +``` + +## `unix_date` + +Returns the number of days since 1970-01-01, as an INT. Dates before 1970-01-01 return a negative number. DataFusion has no built-in `unix_date`. Returns NULL if the input is NULL. The input must be a DATE; strings are not implicitly cast. + +```sql +unix_date(date) +``` + +### Arguments + +- **date**: DATE expression. NULL returns NULL. + +### Example + +```sql +> SELECT unix_date(DATE '1970-01-02'); +1 +> SELECT unix_date(DATE '1900-01-02'); +-25566 +> SELECT unix_date(NULL::DATE); +NULL +``` + +## `unix_micros` + +Returns the number of microseconds since 1970-01-01 00:00:00 UTC, as a BIGINT. DataFusion has no built-in `unix_micros`. A timestamp without a time zone is read as UTC, and a timestamp's time zone does not change the result. Any finer part is rounded down (toward negative infinity, like Spark), so timestamps before 1970 with a fractional part move one unit further from zero. Returns NULL if the input is NULL. The input must be a TIMESTAMP; strings are not implicitly cast. + +```sql +unix_micros(timestamp) +``` + +### Arguments + +- **timestamp**: TIMESTAMP expression. NULL returns NULL. + +### Example + +```sql +> SELECT unix_micros(TIMESTAMP '1970-01-01 00:00:01'); +1000000 +> SELECT unix_micros(TIMESTAMP '1900-01-01 00:00:01'); +-2208988799000000 +> SELECT unix_micros(NULL::TIMESTAMP); +NULL +``` + +## `unix_millis` + +Returns the number of milliseconds since 1970-01-01 00:00:00 UTC, as a BIGINT. DataFusion has no built-in `unix_millis`. A timestamp without a time zone is read as UTC, and a timestamp's time zone does not change the result. Any finer part is rounded down (toward negative infinity, like Spark), so timestamps before 1970 with a fractional part move one unit further from zero. Returns NULL if the input is NULL. The input must be a TIMESTAMP; strings are not implicitly cast. + +```sql +unix_millis(timestamp) +``` + +### Arguments + +- **timestamp**: TIMESTAMP expression. NULL returns NULL. + +### Example + +```sql +> SELECT unix_millis(TIMESTAMP '1970-01-01 00:00:01'); +1000 +> SELECT unix_millis(TIMESTAMP '1969-12-31 23:59:59.9985'); +-2 +> SELECT unix_millis(NULL::TIMESTAMP); +NULL +``` + +## `unix_seconds` + +Returns the number of seconds since 1970-01-01 00:00:00 UTC, as a BIGINT. DataFusion has no built-in `unix_seconds`. A timestamp without a time zone is read as UTC, and a timestamp's time zone does not change the result. Any finer part is rounded down (toward negative infinity, like Spark), so timestamps before 1970 with a fractional part move one unit further from zero. Returns NULL if the input is NULL. The input must be a TIMESTAMP; strings are not implicitly cast. + +```sql +unix_seconds(timestamp) +``` + +### Arguments + +- **timestamp**: TIMESTAMP expression. NULL returns NULL. + +### Example + +```sql +> SELECT unix_seconds(TIMESTAMP '1970-01-01 00:00:01'); +1 +> SELECT unix_seconds(TIMESTAMP '1969-12-31 23:59:58.5'); +-2 +> SELECT unix_seconds(NULL::TIMESTAMP); +NULL +``` + +## `weekday` + +Returns the day of the week as an INT from 0 (Monday) to 6 (Sunday). This differs from `dayofweek`, which counts from 1 (Sunday) to 7 (Saturday). DataFusion has no built-in `weekday`. Timestamps are cast to DATE first. Strings are not implicitly cast as they are in Spark ([#41](https://github.com/apache/datafusion-spark/issues/41)). Returns NULL if the input is NULL. + +```sql +weekday(date) +``` + +### Arguments + +- **date**: DATE or TIMESTAMP expression. NULL returns NULL. + +### Example + +```sql +> SELECT weekday(DATE '2009-07-30'); +3 +> SELECT weekday(TIMESTAMP '2024-01-07 23:59:59'); +6 +> SELECT weekday(NULL::DATE); +NULL +``` diff --git a/docs/functions/scalar/url.md b/docs/functions/scalar/url.md new file mode 100644 index 00000000..66235fe3 --- /dev/null +++ b/docs/functions/scalar/url.md @@ -0,0 +1,86 @@ + + + + +# URL Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [try_url_decode](#try_url_decode) +- [url_decode](#url_decode) +- [url_encode](#url_encode) + +## `try_url_decode` + +Same as `url_decode`, except that a value which cannot be decoded returns NULL for that row instead of an error. DataFusion has no built-in `try_url_decode`. + +```sql +try_url_decode(str) +``` + +### Arguments + +- **str**: String expression to decode. NULL, or a value that cannot be decoded, returns NULL. + +### Example + +```sql +> SELECT try_url_decode('Spark+SQL%21'); +Spark SQL! +> SELECT try_url_decode('https%3%2F%2Fspark.apache.org'); +NULL +``` + +## `url_decode` + +Decodes a string in `application/x-www-form-urlencoded` format, the inverse of `url_encode`. DataFusion has no built-in `url_decode`. `+` becomes a space and each `%XX` escape becomes the byte it encodes, and the bytes are read as UTF-8. A `%` that is not followed by two hex digits, or bytes that are not valid UTF-8, are an error; use `try_url_decode` to get NULL instead. Returns NULL if the input is NULL. + +```sql +url_decode(str) +``` + +### Arguments + +- **str**: String expression to decode. NULL returns NULL. + +### Example + +```sql +> SELECT url_decode('https%3A%2F%2Fspark.apache.org'); +https://spark.apache.org +> SELECT url_decode('Spark+SQL%21'); +Spark SQL! +> SELECT url_decode('%E4%BD%A0%E5%A5%BD'); +你好 +``` + +## `url_encode` + +Encodes a string in `application/x-www-form-urlencoded` format, like Java's `URLEncoder` with UTF-8, which Spark uses. DataFusion has no built-in `url_encode`. Letters, digits and `.`, `-`, `*` and `_` are kept, a space becomes `+`, and every other byte of the UTF-8 encoding becomes `%XX` with uppercase hex digits. Returns NULL if the input is NULL. + +```sql +url_encode(str) +``` + +### Arguments + +- **str**: String expression to encode. NULL returns NULL. + +### Example + +```sql +> SELECT url_encode('https://spark.apache.org'); +https%3A%2F%2Fspark.apache.org +> SELECT url_encode('Spark SQL'); +Spark+SQL +> SELECT url_encode('你好'); +%E4%BD%A0%E5%A5%BD +```