diff --git a/dsntk/Cargo.toml b/dsntk/Cargo.toml index 834d5476..bce5ac90 100644 --- a/dsntk/Cargo.toml +++ b/dsntk/Cargo.toml @@ -17,6 +17,7 @@ tck = ["dsntk-server/tck"] [dependencies] actix-web = { workspace = true } +mimalloc = "0.1" antex = { workspace = true } base64 = { workspace = true } clap = { workspace = true } diff --git a/dsntk/src/actions.rs b/dsntk/src/actions.rs index a14076b6..484cf40f 100644 --- a/dsntk/src/actions.rs +++ b/dsntk/src/actions.rs @@ -1010,28 +1010,44 @@ fn parse_dmn_model(dmn_file_name: &str, cm: ColorMode) { /// Evaluates DMN model loaded from XML file. fn evaluate_dmn_model(input_file_name: &str, dmn_file_name: &str, invocable_name: &str) { - match fs::read_to_string(dmn_file_name) { - Ok(dmn_file_content) => match fs::read_to_string(input_file_name) { - Ok(input_file_content) => match dsntk_evaluator::evaluate_context(&FeelScope::default(), &input_file_content) { - Ok(input_data) => match dsntk_model::parse(&dmn_file_content) { - Ok(definitions) => { - let model_namespace = definitions.namespace().to_string(); - let model_name = definitions.name().to_string(); - match dsntk_evaluator::ModelEvaluator::new(&[definitions]) { - Ok(model_evaluator) => { - let result = model_evaluator.evaluate_invocable(&model_namespace, &model_name, invocable_name, &input_data); - println!("{}", result.jsonify()) - } - Err(reason) => eprintln!("building model evaluator failed with reason: {reason}"), - } - } - Err(reason) => eprintln!("parsing model failed with reason: {reason}"), - }, - Err(reason) => eprintln!("evaluating input data failed with reason: {reason}"), - }, - Err(reason) => eprintln!("loading input data file `{input_file_name}` failed with reason: {reason}"), - }, - Err(reason) => eprintln!("loading model file `{dmn_file_name}` failed with reason: {reason}"), + let dmn_file_content = match fs::read_to_string(dmn_file_name) { + Ok(dmn_file_content) => dmn_file_content, + Err(reason) => { + eprintln!("loading model file `{dmn_file_name}` failed with reason: {reason}"); + return; + } + }; + let input_file_content = match fs::read_to_string(input_file_name) { + Ok(input_file_content) => input_file_content, + Err(reason) => { + eprintln!("loading input data file `{input_file_name}` failed with reason: {reason}"); + return; + } + }; + let input_data = match dsntk_evaluator::evaluate_context(&FeelScope::default(), &input_file_content) { + Ok(input_data) => input_data, + Err(reason) => { + eprintln!("evaluating input data failed with reason: {reason}"); + return; + } + }; + // The input data is now a context, so the text is dead. For a large input, + // this text is tens or hundreds of megabytes. Free it here instead of holding + // it until the process exits. + drop(input_file_content); + match dsntk_model::parse(&dmn_file_content) { + Ok(definitions) => { + let model_namespace = definitions.namespace().to_string(); + let model_name = definitions.name().to_string(); + match dsntk_evaluator::ModelEvaluator::new(&[definitions]) { + Ok(model_evaluator) => { + let result = model_evaluator.evaluate_invocable(&model_namespace, &model_name, invocable_name, &input_data); + println!("{}", result.jsonify()) + } + Err(reason) => eprintln!("building model evaluator failed with reason: {reason}"), + } + } + Err(reason) => eprintln!("parsing model failed with reason: {reason}"), } } diff --git a/dsntk/src/main.rs b/dsntk/src/main.rs index 727bdd50..50d1c16c 100644 --- a/dsntk/src/main.rs +++ b/dsntk/src/main.rs @@ -1,3 +1,12 @@ +/// Use mimalloc instead of the system allocator. +/// +/// Evaluation spends about half its time in `malloc` and `free`, and macOS +/// `libmalloc` never returns freed small blocks to the operating system. So peak +/// resident memory holds every page the parse phase touched, long after the parse +/// data dies. mimalloc purges free pages, so the peak tracks live data instead. +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + /// ÐecisionToolkit entrypoint. #[actix_web::main] async fn main() -> std::io::Result<()> { diff --git a/feel-evaluator/Cargo.toml b/feel-evaluator/Cargo.toml index ded4f34e..e8954666 100644 --- a/feel-evaluator/Cargo.toml +++ b/feel-evaluator/Cargo.toml @@ -12,6 +12,7 @@ license = { workspace = true } edition = { workspace = true } [dependencies] +rayon = "1" regex = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } diff --git a/feel-evaluator/benches/list_scaling.rs b/feel-evaluator/benches/list_scaling.rs new file mode 100644 index 00000000..614b3b24 --- /dev/null +++ b/feel-evaluator/benches/list_scaling.rs @@ -0,0 +1,64 @@ +//! Evaluating a `for` expression, at three sizes. +//! +//! As in the parser benchmark of the same shape, read the **ratio between the sizes** rather +//! than any single time. Each size runs four times the iterations of the one before it, so a +//! linear evaluator answers in about four times the time. +//! +//! `for` binds the implicit variable `partial` to the results of all previous iterations. An +//! evaluator that copies that list on every iteration pays for every result collected so far, +//! which makes the whole expression quadratic in the number of iterations. It does so whether +//! or not the body ever reads `partial`, and neither of these bodies reads it. + +#![feature(test)] + +extern crate test; + +use dsntk_feel::values::Value; +use dsntk_feel::{FeelScope, scope, value_number}; +use dsntk_feel_evaluator::prepare; +use test::Bencher; + +/// Returns an expression that runs `count` iterations and counts the results. +fn counted_for(count: usize) -> String { + format!("count(for item in 1..{count} return item + 1)") +} + +#[bench] +fn feel_evaluator_list_scaling_0001(b: &mut Bencher) { + let scope = scope!(); + let input = counted_for(100); + let node = dsntk_feel_parser::parse_expression(&scope, &input, false).unwrap(); + let evaluator = prepare(&node); + assert_eq!(value_number!(100), evaluator(&scope)); + b.iter(|| evaluator(&scope)); +} + +#[bench] +fn feel_evaluator_list_scaling_0002(b: &mut Bencher) { + let scope = scope!(); + let input = counted_for(400); + let node = dsntk_feel_parser::parse_expression(&scope, &input, false).unwrap(); + let evaluator = prepare(&node); + assert_eq!(value_number!(400), evaluator(&scope)); + b.iter(|| evaluator(&scope)); +} + +#[bench] +fn feel_evaluator_list_scaling_0003(b: &mut Bencher) { + let scope = scope!(); + let input = counted_for(1600); + let node = dsntk_feel_parser::parse_expression(&scope, &input, false).unwrap(); + let evaluator = prepare(&node); + assert_eq!(value_number!(1600), evaluator(&scope)); + b.iter(|| evaluator(&scope)); +} + +#[bench] +fn feel_evaluator_list_scaling_0004(b: &mut Bencher) { + let scope = scope!(); + let input = counted_for(6400); + let node = dsntk_feel_parser::parse_expression(&scope, &input, false).unwrap(); + let evaluator = prepare(&node); + assert_eq!(value_number!(6400), evaluator(&scope)); + b.iter(|| evaluator(&scope)); +} diff --git a/feel-evaluator/examples/scaling.rs b/feel-evaluator/examples/scaling.rs new file mode 100644 index 00000000..b1677901 --- /dev/null +++ b/feel-evaluator/examples/scaling.rs @@ -0,0 +1,66 @@ +//! Runs one of the scaling shapes once, at any size, and reports the time. +//! +//! The benchmarks next door stop at a few thousand items, because a cost that grows with the +//! square of the input makes larger sizes take minutes per iteration and `cargo bench` runs +//! many iterations. This runs the same shapes exactly once, so the large sizes stay usable. +//! +//! ```text +//! cargo run --release --example scaling -- for 200000 +//! cargo run --release --example scaling -- list 200000 +//! ``` +//! +//! For peak memory, wrap it in the system timer rather than adding a dependency: +//! +//! ```text +//! /usr/bin/time -l cargo run --release --example scaling -- for 200000 # macOS +//! /usr/bin/time -v cargo run --release --example scaling -- for 200000 # Linux +//! ``` +//! +//! `for` evaluates `count(for item in 1..N return item + 1)`, which runs N iterations. `list` +//! parses a literal holding N numbers. Compare the same command across two revisions, and +//! compare each revision against itself at two sizes: four times the items should cost about +//! four times the time. + +use dsntk_feel::values::Value; +use dsntk_feel::{FeelScope, scope}; +use dsntk_feel_evaluator::prepare; +use std::time::Instant; + +fn main() { + let args: Vec = std::env::args().collect(); + let shape = args.get(1).cloned().unwrap_or_else(|| "for".to_string()); + let count: usize = args.get(2).and_then(|a| a.parse().ok()).unwrap_or(200_000); + let scope = scope!(); + match shape.as_str() { + "for" => { + let input = format!("count(for item in 1..{count} return item + 1)"); + let started = Instant::now(); + let node = dsntk_feel_parser::parse_expression(&scope, &input, false).expect("parsing failed"); + let evaluator = prepare(&node); + let result = evaluator(&scope); + let elapsed = started.elapsed(); + assert_eq!(Value::Number(count.into()), result, "the loop did not run {count} times"); + println!("for items {count:>9} {:>10.3} s", elapsed.as_secs_f64()); + } + "list" => { + let mut input = String::with_capacity(count * 8 + 2); + input.push('['); + for index in 0..count { + if index > 0 { + input.push(','); + } + input.push_str(&index.to_string()); + } + input.push(']'); + let started = Instant::now(); + let node = dsntk_feel_parser::parse_expression(&scope, &input, false).expect("parsing failed"); + let elapsed = started.elapsed(); + assert!(matches!(node, dsntk_feel_parser::AstNode::List(_)), "expected a list"); + println!("list items {count:>9} {:>10.3} s", elapsed.as_secs_f64()); + } + other => { + eprintln!("unknown shape `{other}`, expected `for` or `list`"); + std::process::exit(1); + } + } +} diff --git a/feel-evaluator/src/bifs/core.rs b/feel-evaluator/src/bifs/core.rs index aed86306..78f99bb7 100644 --- a/feel-evaluator/src/bifs/core.rs +++ b/feel-evaluator/src/bifs/core.rs @@ -562,7 +562,9 @@ pub fn context_merge(contexts: &Value) -> Value { for item in items { match item { Value::Context(ctx) => result_ctx.zip(ctx), - Value::Null(_) => {} + // A null entry is out of domain, just like any other non-context value. It is + // also the entry a failed expression produces, so it is the entry that most + // needs this diagnostic. other => return invalid_argument_type!("context merge", "context", other.type_of()), } } @@ -624,10 +626,10 @@ pub fn date_3(year_value: &Value, month_value: &Value, day_value: &Value) -> Val pub fn date_and_time_1(value: &Value) -> Value { if let Value::String(text) = value { if let Ok(date_time) = FeelDateTime::try_from(text.as_str()) { - return Value::DateTime(date_time); + return Value::DateTime(date_time.into()); } if let Ok(date) = FeelDate::from_str(text) { - return Value::DateTime(FeelDateTime::new(date, FeelTime::local(0, 0, 0, 0))); + return Value::DateTime(FeelDateTime::new(date, FeelTime::local(0, 0, 0, 0)).into()); } value_null!("[core::date and time] invalid date or date and time '{}'", text) } else { @@ -640,13 +642,13 @@ pub fn date_and_time_2(date_value: &Value, time_value: &Value) -> Value { match date_value { Value::DateTime(date_time) => { if let Value::Time(time) = time_value { - return Value::DateTime(FeelDateTime::new(date_time.date(), time.clone())); + return Value::DateTime(FeelDateTime::new(date_time.date(), time.clone()).into()); } invalid_argument_type!("date and time", "time", time_value.type_of()) } Value::Date(date) => { if let Value::Time(time) = time_value { - return Value::DateTime(FeelDateTime::new(date.clone(), time.clone())); + return Value::DateTime(FeelDateTime::new(date.clone(), time.clone()).into()); } invalid_argument_type!("date and time", "time", time_value.type_of()) } @@ -1074,11 +1076,16 @@ pub fn get_entries(context: &Value) -> Value { pub fn get_value(context: &Value, key: &Value) -> Value { if let Value::Context(ctx) = context { if let Value::String(entry_key) = key { - let name = Name::from(entry_key.to_owned()); - if let Some(entry_value) = ctx.get_entry(&name) { - entry_value.clone() - } else { - value_null!() + // Look the key up without interning it. Interning on every probe, including the ones + // that miss, grows the table without bound, because the interner never releases a + // name. A key that was never interned cannot name an entry in any context, so the + // miss answers null either way. + match Name::existing(entry_key) { + Some(name) => match ctx.get_entry(&name) { + Some(entry_value) => entry_value.clone(), + None => value_null!(), + }, + None => value_null!(), } } else { invalid_argument_type!("get value", "string", key.type_of()) @@ -1284,7 +1291,8 @@ pub fn list_replace(list: &Value, position_or_match: &Value, new_item: &Value) - value_null!("position exceeds integer range") } } - Value::FunctionDefinition(parameters, body, false, _, closure_ctx, _) => { + Value::FunctionDefinition(fd) if !fd.2 => { + let (parameters, body, _, _, closure_ctx, _) = &**fd; if parameters.len() != 2 { return value_null!("list replace: matching function must have exactly two arguments"); } @@ -1680,7 +1688,7 @@ pub fn not(negand: &Value) -> Value { /// Returns current date and time. pub fn now() -> Value { - Value::DateTime(FeelDateTime::now()) + Value::DateTime(FeelDateTime::now().into()) } /// Converts string to a number. @@ -2132,7 +2140,10 @@ pub fn round_up(n: &Value, scale: &Value) -> Value { pub fn sort(list: &Value, ordering_function: &Value) -> Value { if let Value::List(items) = list { - if let Value::FunctionDefinition(parameters, body, false, _, closure_ctx, _) = ordering_function { + if let Value::FunctionDefinition(fd) = ordering_function + && !fd.2 + { + let (parameters, body, _, _, closure_ctx, _) = &**fd; if parameters.len() == 2 { let mut elements = items.clone(); elements.sort_by(|x, y| { @@ -2434,7 +2445,7 @@ pub fn string(value: &Value) -> Value { } } -/// Returns the number of characters in string. +/// Joins a list of strings, ignoring null elements, and separates them with a delimiter. pub fn string_join(list: &Value, delimiter: &Value) -> Value { match list { Value::List(items) => { @@ -2444,13 +2455,18 @@ pub fn string_join(list: &Value, delimiter: &Value) -> Value { _ => return value_null!("string join: invalid delimiter, expected string, actual value type is {}", delimiter.type_of()), }; let mut joined_string = String::new(); - for (i, item) in items.iter().enumerate() { + // The delimiter goes before every string except the first one that is + // actually kept. Counting from the list index instead puts a delimiter at + // the front when the list starts with a null. + let mut joined_any = false; + for item in items.iter() { match item { Value::String(s) => { - if i > 0 { + if joined_any { joined_string.push_str(&delimiter); } joined_string.push_str(s); + joined_any = true; } Value::Null(_) => {} other => return invalid_argument_type!("string join", "string", other.type_of()), diff --git a/feel-evaluator/src/builders.rs b/feel-evaluator/src/builders.rs index 6e3d1053..6aa39da8 100644 --- a/feel-evaluator/src/builders.rs +++ b/feel-evaluator/src/builders.rs @@ -4,7 +4,7 @@ use crate::evaluator_java::evaluate_external_java_function; use crate::evaluator_pmml::evaluate_external_pmml_function; use crate::iterations::{EveryExpressionEvaluator, ForExpressionEvaluator, SomeExpressionEvaluator}; use crate::macros::invalid_argument_type; -use crate::{FilterExpressionEvaluator, bifs}; +use crate::{FilterExpressionEvaluator, bifs, cse}; use dsntk_feel::bif::Bif; use dsntk_feel::bip::*; use dsntk_feel::context::FeelContext; @@ -50,6 +50,14 @@ macro_rules! between_null3 { struct EvaluatorBuilder<'b> { /// Stack of visited parent nodes in AST during evaluator building. node_stack: Vec<&'b AstNode>, + /// Sibling context entries binding a name to a function definition, used by [crate::cse] + /// to decide whether a repeated call may be evaluated once. Scoped to the context literal + /// currently being built and restored when it finishes. + local_functions: cse::LocalFunctions<'b>, + /// Subtrees of the filter predicate currently being built that have been hoisted, each with + /// the synthetic name its value is bound to. Consulted for every node, so an occurrence + /// becomes a read of the slot instead of the call itself. + cse_slots: Vec<(&'b AstNode, Name)>, } impl<'b> EvaluatorBuilder<'b> { @@ -61,6 +69,12 @@ impl<'b> EvaluatorBuilder<'b> { /// Builds and evaluator based on provided AST node. fn build(&mut self, node: &'b AstNode) -> Evaluator { + // An occurrence of a subtree hoisted out of the enclosing filter predicate reads the value + // bound to its slot instead of evaluating the subtree again. See [crate::cse]. + if let Some((_, name)) = self.cse_slots.iter().find(|(hoisted, _)| *hoisted == node) { + let name = name.clone(); + return Box::new(move |scope: &FeelScope| scope.get_value(&name).unwrap_or_else(|| value_null!("no value for hoisted subexpression"))); + } // push the current node on the top of the node stack self.node_stack.push(node); // build the evaluator @@ -185,15 +199,15 @@ impl<'b> EvaluatorBuilder<'b> { }, Value::DateTime(lh) => match rhv { Value::DaysAndTimeDuration(rh) => { - if let Some(a) = lh + rh { - Value::DateTime(a) + if let Some(a) = Arc::unwrap_or_clone(lh) + rh { + Value::DateTime(a.into()) } else { value_null!("invalid result while adding days and time duration to date and time") } } Value::YearsAndMonthsDuration(rh) => { - if let Some(a) = lh + rh { - Value::DateTime(a) + if let Some(a) = Arc::unwrap_or_clone(lh) + rh { + Value::DateTime(a.into()) } else { value_null!("invalid result while adding years and months duration to date and time") } @@ -214,8 +228,8 @@ impl<'b> EvaluatorBuilder<'b> { } } Value::DateTime(rh) => { - if let Some(a) = rh + lh { - Value::DateTime(a) + if let Some(a) = Arc::unwrap_or_clone(rh) + lh { + Value::DateTime(a.into()) } else { value_null!("invalid result while adding date and time to days and time duration") } @@ -232,8 +246,8 @@ impl<'b> EvaluatorBuilder<'b> { } } Value::DateTime(rh) => { - if let Some(a) = rh + lh { - Value::DateTime(a) + if let Some(a) = Arc::unwrap_or_clone(rh) + lh { + Value::DateTime(a.into()) } else { value_null!("invalid result while adding date and time to years and months duration") } @@ -256,7 +270,7 @@ impl<'b> EvaluatorBuilder<'b> { if let Ok(date) = FeelDate::from_str(text) { Box::new(move |_: &FeelScope| Value::Date(date.clone())) } else if let Ok(date_time) = FeelDateTime::try_from(text) { - Box::new(move |_: &FeelScope| Value::DateTime(date_time.clone())) + Box::new(move |_: &FeelScope| Value::DateTime(date_time.clone().into())) } else if let Ok(time) = text.parse::() { Box::new(move |_: &FeelScope| Value::Time(time.clone())) } else if let Ok(ym_duration) = FeelYearsAndMonthsDuration::try_from(text) { @@ -417,10 +431,21 @@ impl<'b> EvaluatorBuilder<'b> { } fn build_context(&mut self, lhs: &'b [AstNode]) -> Evaluator { + // Record the sibling entries that bind a name to a function definition before building + // any entry. That way, if an entry calls one of them, this module can check that call + // for purity. This code restores the previous set afterwards, so a nested context does + // not leak its functions. + let saved_functions = self.local_functions.clone(); + cse::collect_local_functions(lhs, &mut self.local_functions); + // An entry is visible to every later entry as a bare name, so a context literal binds + // names. An enclosing hoist must not reach into it. See [Self::build_isolated]. + let outer_slots = std::mem::take(&mut self.cse_slots); let mut evaluators = vec![]; for node in lhs { evaluators.push(self.build(node)); } + self.cse_slots = outer_slots; + self.local_functions = saved_functions; Box::new(move |scope: &FeelScope| { let mut evaluated_ctx = FeelContext::default(); // prepare special context in scope, used for already evaluated context entries @@ -624,10 +649,84 @@ impl<'b> EvaluatorBuilder<'b> { fn build_filter(&mut self, lhs: &'b AstNode, rhs: &'b AstNode) -> Evaluator { let lhe = self.build(lhs); - let rhe = self.build(rhs); + // Decide once, here, whether this predicate is really an index, so that `L[i]` need not run + // the predicate for every element of `L`. See [crate::filters::IndexShape]. + let shape = crate::filters::index_shape(rhs); + let rhe = self.build_hoisted(rhs); Box::new(move |scope: &FeelScope| { let filter_expression_evaluator = FilterExpressionEvaluator::new(); - filter_expression_evaluator.evaluate(scope, lhe(scope), &rhe) + filter_expression_evaluator.evaluate(scope, lhe(scope), &rhe, shape) + }) + } + + /// Builds `node` with the hoisted subtrees of an enclosing expression taken out of scope. + /// + /// This code may replace a hoisted subtree with a read of its slot only where every name + /// it reads still resolves the way it did. That is the scope where the value was computed. + /// [crate::cse] refuses to **look** inside a construct that binds a name. The replacement + /// in [Self::build] matches a subtree structurally. Without this function, the replacement + /// would therefore reach an occurrence *inside* such a construct, and make it read a value + /// computed outside it. The two walks have to refuse the same constructs. + /// `adversarial/cases/20*` are the cases that fail otherwise. + fn build_isolated(&mut self, node: &'b AstNode) -> Evaluator { + let outer_slots = std::mem::take(&mut self.cse_slots); + let evaluator = self.build(node); + self.cse_slots = outer_slots; + evaluator + } + + /// Builds `node`, evaluating any repeated call in it only once per evaluation of `node`. + /// + /// Used for the two positions where that is sound. A **filter predicate** is evaluated + /// once per element with `item` bound. A **`for` body** is evaluated once per iteration + /// with the iteration variables and `partial` bound. The benchmark model calls + /// `inSum(item)` three times and `outSum(item)` four times in one predicate. It also calls + /// `dayGroup(d)` three times in one `for` body. Every one of those calls filters the whole + /// transaction list. + /// + /// [crate::cse] decides which subtrees may be hoisted. Everything about **when** that is + /// sound is documented there. When it finds nothing this is the plain evaluator for + /// `node`, still isolated from an enclosing hoist for the reason in [Self::build_isolated]. + fn build_hoisted(&mut self, rhs: &'b AstNode) -> Evaluator { + let outer_slots = std::mem::take(&mut self.cse_slots); + let evaluator = self.build_hoisted_inner(rhs); + self.cse_slots = outer_slots; + evaluator + } + + fn build_hoisted_inner(&mut self, rhs: &'b AstNode) -> Evaluator { + let hoisted = cse::hoistable(rhs, &self.local_functions); + if hoisted.is_empty() { + return self.build(rhs); + } + // Build the hoisted subtrees first, while the slot table is still empty, so each one + // evaluates the real call rather than a read of its own slot. + if std::env::var_os("DSNTK_CSE_TRACE").is_some() { + eprintln!("CSE hoisted {} subtree(s): {:?}", hoisted.len(), hoisted.iter().map(|n| format!("{n:?}").chars().take(70).collect::()).collect::>()); + } + let hoisted_evaluators: Vec = hoisted.iter().map(|node| self.build(node)).collect(); + let slot_names: Vec = (0..hoisted.len()).map(cse::slot_name).collect(); + // Now build the predicate, with every occurrence rewritten to a read of its slot. + let slots: Vec<(&'b AstNode, Name)> = hoisted.iter().copied().zip(slot_names.iter().cloned()).collect(); + let saved_slots = std::mem::replace(&mut self.cse_slots, slots); + let predicate = self.build(rhs); + self.cse_slots = saved_slots; + Box::new(move |scope: &FeelScope| { + // One frame holding the hoisted values, built and shared in one allocation. `from_entry` + // leaves the refcount at one, so the remaining entries are written in place. + let mut names = slot_names.iter(); + let mut evaluators = hoisted_evaluators.iter(); + let Some((first_name, first_evaluator)) = names.next().zip(evaluators.next()) else { + return predicate(scope); + }; + let mut frame = FeelContext::from_entry(first_name, first_evaluator(scope)); + for (name, evaluator) in names.zip(evaluators) { + frame.set_entry(name, evaluator(scope)); + } + scope.push(frame); + let value = predicate(scope); + scope.pop(); + value }) } @@ -638,7 +737,13 @@ impl<'b> EvaluatorBuilder<'b> { List((Name, Evaluator)), Variable((Name, Name)), } - let rhe = self.build(rhs); + // Analyze the body once, when the evaluator is built, not per evaluation. The analysis reads + // the body as written, so hoisting inside the body cannot change the verdict. + let safety = crate::parallel::analyze(rhs); + // A repeated call in the body is evaluated once per iteration, not once per occurrence. The + // iteration variables and `partial` are already bound when the body runs and neither moves + // while it runs, so the occurrences share one value. See [Self::build_hoisted]. + let rhe = self.build_hoisted(rhs); let mut evaluators = vec![]; let mut binding_variables = HashSet::new(); if let AstNode::IterationContexts(items) = lhs { @@ -676,8 +781,14 @@ impl<'b> EvaluatorBuilder<'b> { } } } + // Rule 6. `binding_variables` is only complete here, and `analyze` never sees the left side + // of the `for`, so this check cannot live inside it. A loop variable that spells a built-in + // makes the body invoke a data value through a name the analysis cleared as a built-in. + let mut safety = safety; + safety.forbid_names(&binding_variables); Box::new(move |scope: &FeelScope| { let mut for_expression_evaluator = ForExpressionEvaluator::new(); + for_expression_evaluator.set_safety(safety.clone()); for iterator_type in &evaluators { match iterator_type { IteratorType::Interval((name, interval_start_evaluator, interval_end_evaluator)) => { @@ -762,8 +873,8 @@ impl<'b> EvaluatorBuilder<'b> { if let Some(Value::Context(java_mapping)) = mapping_information.get_entry(&"java".into()) { return if let Some(Value::String(class_name)) = java_mapping.get_entry(&"class".into()) { if let Some(Value::String(method_signature)) = java_mapping.get_entry(&"method signature".into()) { - let java_class_name = class_name.to_owned(); - let java_method_signature = method_signature.to_owned(); + let java_class_name: Box = class_name.as_str().into(); + let java_method_signature: Box = method_signature.as_str().into(); let java_evaluator = Box::new(move |_: &FeelScope| Value::ExternalJavaFunction(java_class_name.clone(), java_method_signature.clone())) as Evaluator; let lhe = Arc::new(java_evaluator); Value::FunctionBody(FunctionBody::External(lhe), true) @@ -777,8 +888,8 @@ impl<'b> EvaluatorBuilder<'b> { if let Some(Value::Context(pmml_mapping)) = mapping_information.get_entry(&"pmml".into()) { return if let Some(Value::String(document)) = pmml_mapping.get_entry(&"document".into()) { if let Some(Value::String(model_name)) = pmml_mapping.get_entry(&"model".into()) { - let pmml_document = document.to_owned(); - let pmml_model_name = model_name.to_owned(); + let pmml_document: Box = document.as_str().into(); + let pmml_model_name: Box = model_name.as_str().into(); let pmml_evaluator = Box::new(move |_: &FeelScope| Value::ExternalPmmlFunction(pmml_document.clone(), pmml_model_name.clone())) as Evaluator; let lhe = Arc::new(pmml_evaluator); Value::FunctionBody(FunctionBody::External(lhe), true) @@ -798,8 +909,9 @@ impl<'b> EvaluatorBuilder<'b> { fn build_function_definition(&mut self, lhs: &'b AstNode, rhs: &'b AstNode) -> Evaluator { let closure = ClosureBuilder::from_function_definition(lhs, rhs); - let lhe = self.build(lhs); - let rhe = self.build(rhs); + // The body binds the formal parameters, so an enclosing hoist must not reach into it. + let lhe = self.build_isolated(lhs); + let rhe = self.build_isolated(rhs); Box::new(move |scope: &FeelScope| { let lhv = lhe(scope); let rhv = rhe(scope); @@ -814,7 +926,7 @@ impl<'b> EvaluatorBuilder<'b> { } } //TODO Check if `FeelType::Any` is always ok for function result type in function definition. - Value::FunctionDefinition(parameters, body, external, closure.clone(), closure_ctx, FeelType::Any) + Value::function_definition(parameters, body, external, closure.clone(), closure_ctx, FeelType::Any) } else { value_null!("invalid body in function definition") } @@ -860,7 +972,8 @@ impl<'b> EvaluatorBuilder<'b> { let AstNode::Satisfies(satisfies) = rhs else { return build_err_msg(err_msg_expected_node("Satisfies", lhs)); }; - let satisfies_evaluator = self.build(satisfies); + // The body binds the quantified variables, so an enclosing hoist must not reach into it. + let satisfies_evaluator = self.build_isolated(satisfies); Box::new(move |scope: &FeelScope| { let mut expression_evaluator = EveryExpressionEvaluator::new(); for (name, expr_evaluator) in &expr_evaluators { @@ -886,7 +999,8 @@ impl<'b> EvaluatorBuilder<'b> { let args = argument_evaluators.iter().map(|evaluator| evaluator(scope)).collect::>(); match function { Value::BuiltInFunction(bif) => bifs::positional::evaluate_bif(bif, &args), - Value::FunctionDefinition(params, body, external, _, closure_ctx, result_type) => { + Value::FunctionDefinition(fd) => { + let (params, body, external, _, closure_ctx, result_type) = *fd; if external { eval_external_function_with_positional_parameters(scope, &args, ¶ms, &body, result_type) } else { @@ -940,7 +1054,8 @@ impl<'b> EvaluatorBuilder<'b> { let args = arguments_evaluator(scope); match function { Value::BuiltInFunction(bif) => bifs::named::evaluate_bif(bif, &args), - Value::FunctionDefinition(params, body, external, _, closure_ctx, result_type) => { + Value::FunctionDefinition(fd) => { + let (params, body, external, _, closure_ctx, result_type) = *fd; if external { eval_external_function_with_named_parameters(scope, &args, ¶ms, &body, result_type) } else { @@ -1612,9 +1727,22 @@ impl<'b> EvaluatorBuilder<'b> { Some(AstNode::UnaryGe(_)) => |value: Value| (Value::UnaryGreaterOrEqual(value.into()), true), _ => |value: Value| (value, false), }; + // When the base of the path is a plain name, this code reads the entry straight + // from the context bound to that name. Evaluating the base instead clones the + // whole context just to read one entry. That cost dominates field access. + let opt_base_name = if let AstNode::Name(base_name) = lhs { Some(base_name.clone()) } else { None }; let lhe = self.build(lhs); // prepare and return the evaluator Box::new(move |scope: &FeelScope| { + // Fast path. `get_context_entry` resolves the base name exactly like the general path + // below does. It returns a value only when the general path would return that + // same value. Everything else, including a missing entry, falls through, so the + // reported reasons for a null result stay unchanged. + if let Some(base_name) = &opt_base_name + && let Some(value) = scope.get_context_entry(base_name, &name) + { + return value; + } let lhv = lhe(scope); match lhv { Value::Context(context) => { @@ -1693,7 +1821,8 @@ impl<'b> EvaluatorBuilder<'b> { let AstNode::Satisfies(satisfies) = rhs else { return build_err_msg(err_msg_expected_node("Satisfies", lhs)); }; - let satisfies_evaluator = self.build(satisfies); + // The body binds the quantified variables, so an enclosing hoist must not reach into it. + let satisfies_evaluator = self.build_isolated(satisfies); Box::new(move |scope: &FeelScope| { let mut expression_evaluator = SomeExpressionEvaluator::new(); for (name, expr_evaluator) in &expr_evaluators { @@ -1730,7 +1859,7 @@ impl<'b> EvaluatorBuilder<'b> { } Value::DateTime(rh) => { let l = FeelDateTime::new(lh, FeelTime::utc(0, 0, 0, 0)); - if let Some(result) = l - rh { + if let Some(result) = l - Arc::unwrap_or_clone(rh) { return Value::DaysAndTimeDuration(result); } } @@ -1760,23 +1889,23 @@ impl<'b> EvaluatorBuilder<'b> { Value::DateTime(lh) => match rhv { Value::Date(rh) => { let r = FeelDateTime::new(rh, FeelTime::utc(0, 0, 0, 0)); - if let Some(result) = lh - r { + if let Some(result) = Arc::unwrap_or_clone(lh) - r { return Value::DaysAndTimeDuration(result); } } Value::DateTime(rh) => { - if let Some(result) = lh - rh { + if let Some(result) = Arc::unwrap_or_clone(lh) - Arc::unwrap_or_clone(rh) { return Value::DaysAndTimeDuration(result); } } Value::DaysAndTimeDuration(rh) => { - if let Some(result) = lh - rh { - return Value::DateTime(result); + if let Some(result) = Arc::unwrap_or_clone(lh) - rh { + return Value::DateTime(result.into()); } } Value::YearsAndMonthsDuration(rh) => { - if let Some(result) = lh - rh { - return Value::DateTime(result); + if let Some(result) = Arc::unwrap_or_clone(lh) - rh { + return Value::DateTime(result.into()); } } _ => {} @@ -2669,14 +2798,15 @@ fn eval_function_definition(scope: &FeelScope, params_ctx: FeelContext, body: &F scope.push(closure_ctx); // closure_ctx scope.push(params_ctx); // params_ctx let mut result = body.evaluate(scope); - if let Value::FunctionDefinition(fd_params, fd_body, fd_external, fd_closure, fd_closure_ctx, fd_result_type) = &result { + if let Value::FunctionDefinition(fd) = &result { + let (fd_params, fd_body, fd_external, fd_closure, fd_closure_ctx, fd_result_type) = &**fd; let mut new_closure_ctx = fd_closure_ctx.clone(); for closure_name in fd_closure.iter() { if let Some(closure_value) = scope.search_entry(closure_name) { new_closure_ctx.create_entry(closure_name, closure_value); } } - result = Value::FunctionDefinition( + result = Value::function_definition( fd_params.to_owned(), fd_body.to_owned(), fd_external.to_owned(), diff --git a/feel-evaluator/src/cse.rs b/feel-evaluator/src/cse.rs new file mode 100644 index 00000000..e4303489 --- /dev/null +++ b/feel-evaluator/src/cse.rs @@ -0,0 +1,453 @@ +//! Finds a repeated function call inside a filter predicate so it can be evaluated once. +//! +//! # What this is for +//! +//! A rule in the benchmark model writes its window test like this: +//! +//! ```text +//! wins[inSum(item) >= 6000 and outSum(item) >= 6000 and outSum(item) > 0 +//! and inSum(item) / outSum(item) >= 0.9 and inSum(item) / outSum(item) <= 1.1] +//! ``` +//! +//! `inSum` is called three times and `outSum` four times for **every** window, and each call +//! filters the whole transaction list and folds it. Two evaluations answer the same question. +//! The Java reference computes exactly two sums per window. +//! +//! # Why a build-time transform and not a runtime cache +//! +//! A memo cache has to key on the argument **values**, so it has to hash a [Value]. Measured +//! per-variant clone costs put a `String` at 18.61 ns and an eleven-element `List` at 61.70 ns, +//! and hashing costs the same order of magnitude. A cache would therefore spend a large part +//! of what it saves. The repeated calls here are syntactically identical subtrees evaluated in +//! one unchanged scope. Binding each distinct subtree to a synthetic name once per predicate +//! evaluation therefore needs no hashing at all. +//! +//! # Purity, enumerated rather than excluded +//! +//! Hoisting a call from seven evaluations to two is only sound when the call is deterministic +//! and effect-free. Nothing here trusts a default. A subtree qualifies only when every rule +//! below holds, and each rule is a list of what is **allowed**: +//! +//! 1. The subtree is a call of a plain name that is **not** a built-in. That single test +//! excludes `now`, `today` and every other clock-derived built-in at once, rather than +//! relying on a list of things to avoid. `bif_now` is already known to be defective, so +//! this analysis does not treat the built-in table as trustworthy. +//! 2. Every argument is a plain name or a literal, so evaluating the arguments cannot itself +//! change anything. +//! 3. A sibling entry of the enclosing context binds the called name to a function +//! definition that is **not external**. So no `Java` or `PMML` implementation is reached. +//! 4. That function's body contains only calls of built-ins on the allowed list below, or of +//! other functions that pass this same test. +//! 5. Every occurrence sits on the expression's non-binding spine. The walk refuses to enter +//! `for`, `some`, `every`, a context literal, a nested filter or a function definition. So +//! nothing hoists an occurrence out of a construct that binds a name it reads. +//! +//! # Where the hoist applies +//! +//! Two positions, and the rule is the same in both. A subtree may be evaluated once and +//! read many times only where every occurrence runs in one unchanged scope. +//! +//! - a **filter predicate**, evaluated once per element with `item` bound +//! - a **`for` body**, evaluated once per iteration with the iteration variables and `partial` +//! bound. `partial` is set once per iteration and does not move while the body runs. So two +//! occurrences inside one iteration see the same value, and one evaluation answers both. +//! +//! # The rewrite has to refuse exactly what the walk refuses +//! +//! Finding an occurrence and replacing it are two separate walks, and they must agree. The +//! walk below stops at every construct that binds a name. The replacement, in +//! `EvaluatorBuilder::build`, matches a hoisted subtree structurally. On its own, it would +//! therefore reach an occurrence *inside* such a construct, and make it read a value +//! computed outside it. `EvaluatorBuilder::build_isolated` is what keeps the two in step, and +//! `adversarial/cases/20*` are the cases that fail without it. + +use dsntk_feel::Name; +use dsntk_feel::bif::Bif; +use dsntk_feel_parser::AstNode; +use std::collections::HashMap; +use std::str::FromStr; + +/// Built-in functions allowed inside the body of a hoisted function. +/// +/// Enumerated, so a built-in missing from this list blocks the hoist instead of being +/// silently permitted. `now` and `today` are absent because they read the clock. Three of +/// these evaluate a `FEEL` expression: `list replace`, `sort` and `range`. Each builds a +/// fresh scope rather than reading the caller's, so none can observe the scope the hoist +/// moves the call out of. +const ALLOWED_BUILTINS: &[&str] = &[ + "abs", + "after", + "all", + "any", + "append", + "before", + "ceiling", + "coincides", + "concatenate", + "contains", + "context", + "context merge", + "context put", + "count", + "date", + "date and time", + "day of week", + "day of year", + "decimal", + "distinct values", + "duration", + "during", + "ends with", + "even", + "exp", + "finished by", + "finishes", + "flatten", + "floor", + "get entries", + "get value", + "includes", + "index of", + "is", + "list contains", + "list replace", + "log", + "lower case", + "matches", + "max", + "mean", + "median", + "meets", + "met by", + "min", + "mode", + "modulo", + "month of year", + "not", + "number", + "odd", + "overlaps", + "overlaps after", + "overlaps before", + "product", + "range", + "remove", + "replace", + "reverse", + "round down", + "round half down", + "round half up", + "round up", + "sort", + "split", + "sqrt", + "started by", + "starts", + "starts with", + "stddev", + "string", + "string join", + "string length", + "sublist", + "substring", + "substring after", + "substring before", + "sum", + "time", + "union", + "upper case", + "week of year", + "years and months duration", +]; + +/// How deep the purity walk follows one function calling another before giving up. +const MAX_PURITY_DEPTH: usize = 8; + +/// The name bound to hoisted slot `index`. +/// +/// A `#` cannot appear in a `FEEL` name, so a slot can never shadow a name the author wrote. +pub fn slot_name(index: usize) -> Name { + Name::from(format!("#cse{index}").as_str()) +} + +/// Sibling context entries that bind a name to a function definition. +pub type LocalFunctions<'b> = HashMap; + +/// Records every sibling entry of a context literal that binds a name to a function +/// definition, so a call of that name inside a later entry can be checked for purity. +pub fn collect_local_functions<'b>(entries: &'b [AstNode], into: &mut LocalFunctions<'b>) { + for entry in entries { + if let AstNode::ContextEntry(key, value) = entry + && let AstNode::ContextEntryKey(name) = key.as_ref() + && matches!(value.as_ref(), AstNode::FunctionDefinition(..)) + { + into.insert(name.clone(), value.as_ref()); + } + } +} + +/// Returns the repeated subtrees of `expression` that the transform can evaluate once instead +/// of once per occurrence, in the order they first appear. +pub fn hoistable<'b>(expression: &'b AstNode, locals: &LocalFunctions<'b>) -> Vec<&'b AstNode> { + let mut found: Vec<(&'b AstNode, usize)> = vec![]; + collect_calls(expression, &mut found); + let mut hoists = vec![]; + for (node, count) in found { + if count >= 2 && is_hoistable_call(node, locals) { + hoists.push(node); + } + } + hoists +} + +/// Walks the non-binding spine of `expression`, counting structurally equal calls. +/// +/// The `_ => {}` arm is the safety property. This function does not enter any node kind not +/// listed here. So a construct that binds a name -- `some`, `every`, a context literal, a +/// nested filter, a function definition, and the **body** of a nested `for` -- hides +/// everything inside it from the hoist. +fn collect_calls<'b>(node: &'b AstNode, found: &mut Vec<(&'b AstNode, usize)>) { + if let AstNode::FunctionInvocation(_, parameters) = node { + if let Some(slot) = found.iter_mut().find(|(seen, _)| *seen == node) { + slot.1 += 1; + } else { + found.push((node, 1)); + } + // Then the argument list. `build_function_invocation_with_positional_parameters` evaluates + // every argument in the caller's own scope, eagerly and with nothing bound, before the + // call happens. So an occurrence inside an argument list runs in exactly the scope the + // hoist moves it to. This is what reaches the call inside `count(dayGroup(d))`. Named + // parameters are not entered: nothing measured needs them, so they stay outside the + // enumerated set. + if let AstNode::PositionalParameters(arguments) = parameters.as_ref() { + for argument in arguments { + collect_calls(argument, found); + } + } + return; + } + // A nested `for`: the expressions it iterates over, never its body. `build_for` evaluates + // every iterated expression by calling it with the **enclosing** scope, once per `for` + // evaluation and before any iteration variable exists. So an occurrence there runs in the + // scope the hoist moves it to. The body is the opposite case and is left alone. + if let AstNode::For(iteration_contexts, _body) = node { + if let AstNode::IterationContexts(items) = iteration_contexts.as_ref() { + for item in items { + match item { + AstNode::IterationContextSingle(_, expression) => collect_calls(expression, found), + AstNode::IterationContextInterval(_, start, end) => { + collect_calls(start, found); + collect_calls(end, found); + } + _ => {} + } + } + } + return; + } + match node { + AstNode::Add(lhs, rhs) + | AstNode::And(lhs, rhs) + | AstNode::Div(lhs, rhs) + | AstNode::Eq(lhs, rhs) + | AstNode::Exp(lhs, rhs) + | AstNode::Ge(lhs, rhs) + | AstNode::Gt(lhs, rhs) + | AstNode::In(lhs, rhs) + | AstNode::InstanceOf(lhs, rhs) + | AstNode::Le(lhs, rhs) + | AstNode::Lt(lhs, rhs) + | AstNode::Mul(lhs, rhs) + | AstNode::Nq(lhs, rhs) + | AstNode::Or(lhs, rhs) + | AstNode::Sub(lhs, rhs) => { + collect_calls(lhs, found); + collect_calls(rhs, found); + } + AstNode::Between(a, b, c) | AstNode::If(a, b, c) => { + collect_calls(a, found); + collect_calls(b, found); + collect_calls(c, found); + } + AstNode::Neg(lhs) | AstNode::EvaluatedExpression(lhs) => collect_calls(lhs, found), + AstNode::ExpressionList(items) | AstNode::CommaList(items) | AstNode::List(items) => { + for item in items { + collect_calls(item, found); + } + } + _ => {} + } +} + +/// Returns `true` when this call may be hoisted. Rules 1 to 4 of the module note. +fn is_hoistable_call<'b>(node: &'b AstNode, locals: &LocalFunctions<'b>) -> bool { + let AstNode::FunctionInvocation(callee, parameters) = node else { + return false; + }; + // Rule 1: a plain name that is not a built-in of any kind. + let AstNode::Name(name) = callee.as_ref() else { + return false; + }; + if Bif::from_str(name.as_str()).is_ok() { + return false; + } + // Rule 2: every argument is a plain name or a literal. + let AstNode::PositionalParameters(arguments) = parameters.as_ref() else { + return false; + }; + if !arguments.iter().all(is_trivial_argument) { + return false; + } + // Rules 3 and 4: the name is bound by a sibling entry to a pure, non-external function. + let Some(definition) = locals.get(name) else { + return false; + }; + is_pure_function(definition, locals, 0) +} + +/// Returns `true` for an argument whose evaluation cannot change anything. +fn is_trivial_argument(node: &AstNode) -> bool { + matches!(node, AstNode::Name(_) | AstNode::Numeric(..) | AstNode::String(_) | AstNode::Boolean(_) | AstNode::Null) +} + +/// Returns `true` when this function definition is not external and its body calls only +/// allowed built-ins and other pure functions. +fn is_pure_function<'b>(definition: &'b AstNode, locals: &LocalFunctions<'b>, depth: usize) -> bool { + if depth > MAX_PURITY_DEPTH { + return false; + } + let AstNode::FunctionDefinition(_, body) = definition else { + return false; + }; + let AstNode::FunctionBody(inner, external) = body.as_ref() else { + return false; + }; + // Rule 3: no external `Java` or `PMML` implementation. + if *external { + return false; + } + is_pure_expression(inner, locals, depth) +} + +/// Returns `true` when every call anywhere inside `node` is allowed. +/// +/// Unlike [collect_calls] this walks the whole subtree, including binding constructs. Here +/// the question is not where a call sits, but whether any call reachable from it is impure. +/// `AstNode::children` is not available, so this module writes the walk by hand over the +/// variants that can hold an expression. +fn is_pure_expression<'b>(node: &'b AstNode, locals: &LocalFunctions<'b>, depth: usize) -> bool { + if let AstNode::FunctionInvocation(callee, parameters) = node { + let allowed = match callee.as_ref() { + AstNode::Name(name) => { + if let Ok(_bif) = Bif::from_str(name.as_str()) { + ALLOWED_BUILTINS.contains(&name.as_str()) + } else if let Some(definition) = locals.get(name) { + is_pure_function(definition, locals, depth + 1) + } else { + // An unknown callee. It may be a parameter holding a function, or a name bound + // further out than this context, and neither can be checked here. + false + } + } + // A call of anything other than a plain name cannot be checked. + _ => false, + }; + return allowed && is_pure_children(parameters, locals, depth); + } + is_pure_children(node, locals, depth) +} + +/// Applies [is_pure_expression] to every child of `node`. +fn is_pure_children<'b>(node: &'b AstNode, locals: &LocalFunctions<'b>, depth: usize) -> bool { + let mut pure = true; + let mut check = |child: &'b AstNode| { + if pure { + pure = is_pure_expression(child, locals, depth); + } + }; + match node { + // Leaves. + AstNode::At(_) + | AstNode::Boolean(_) + | AstNode::ContextEntryKey(_) + | AstNode::ContextTypeEntryKey(_) + | AstNode::FeelType(_) + | AstNode::Irrelevant + | AstNode::Name(_) + | AstNode::Null + | AstNode::Numeric(..) + | AstNode::ParameterName(_) + | AstNode::QualifiedNameSegment(_) + | AstNode::String(_) => {} + // Two children. + AstNode::Add(a, b) + | AstNode::And(a, b) + | AstNode::ContextEntry(a, b) + | AstNode::Div(a, b) + | AstNode::Eq(a, b) + | AstNode::Exp(a, b) + | AstNode::Filter(a, b) + | AstNode::FunctionDefinition(a, b) + | AstNode::FunctionInvocation(a, b) + | AstNode::Ge(a, b) + | AstNode::Gt(a, b) + | AstNode::In(a, b) + | AstNode::InstanceOf(a, b) + | AstNode::Le(a, b) + | AstNode::Lt(a, b) + | AstNode::Mul(a, b) + | AstNode::NamedParameter(a, b) + | AstNode::Nq(a, b) + | AstNode::Or(a, b) + | AstNode::Path(a, b) + | AstNode::Every(a, b) + | AstNode::For(a, b) + | AstNode::Some(a, b) + | AstNode::IterationContextSingle(a, b) + | AstNode::Out(a, b) + | AstNode::QuantifiedContext(a, b) + | AstNode::Sub(a, b) => { + check(a); + check(b); + } + // Three children. + AstNode::Between(a, b, c) | AstNode::If(a, b, c) => { + check(a); + check(b); + check(c); + } + // One child. + AstNode::EvaluatedExpression(a) + | AstNode::FormalParameter(a, _) + | AstNode::FunctionBody(a, _) + | AstNode::IntervalEnd(a, _) + | AstNode::IntervalStart(a, _) + | AstNode::Neg(a) + | AstNode::Satisfies(a) + | AstNode::UnaryEq(a) + | AstNode::UnaryGe(a) + | AstNode::UnaryGt(a) + | AstNode::UnaryLe(a) + | AstNode::UnaryLt(a) + | AstNode::UnaryNe(a) => check(a), + // A list of children. + AstNode::CommaList(items) + | AstNode::Context(items) + | AstNode::ExpressionList(items) + | AstNode::FormalParameters(items) + | AstNode::IterationContexts(items) + | AstNode::List(items) + | AstNode::NamedParameters(items) + | AstNode::NegatedList(items) + | AstNode::PositionalParameters(items) + | AstNode::QualifiedName(items) + | AstNode::QuantifiedContexts(items) => { + for item in items { + check(item); + } + } + // Anything else is a type expression or a shape this analysis does not model, so refuse. + _ => return false, + } + pure +} diff --git a/feel-evaluator/src/evaluators.rs b/feel-evaluator/src/evaluators.rs index c21aa6d2..1df2de93 100644 --- a/feel-evaluator/src/evaluators.rs +++ b/feel-evaluator/src/evaluators.rs @@ -37,8 +37,28 @@ pub fn evaluate_equals(left: &Value, right: &Value) -> bool { /// Evaluates a context from text containing `FEEL` expression. pub fn evaluate_context(scope: &FeelScope, input: &str) -> Result { - let node = &dsntk_feel_parser::parse_context(scope, input, false)?; - evaluate_context_node(scope, node) + let node = dsntk_feel_parser::parse_context(scope, input, false)?; + // An input data file is usually not an expression but data: nested contexts and lists of + // string, number, boolean and null literals. This code converts such a tree straight into + // a value. That skips the closure tree completely and frees each record as soon as its + // value exists. A tree that holds anything else comes back unchanged and takes the normal + // path. See `crate::literal`. + let value = match crate::literal::literal_value(&node) { + Some(value) => { + drop(node); + value + } + None => { + let evaluator = crate::builders::build_evaluator(&node); + // `Evaluator` has no lifetime parameter, so an evaluator can never borrow the + // parsed syntax tree. The tree is dead here. For a large data literal, it is + // the biggest of the three input representations. Free it before the evaluator + // builds the value tree. + drop(node); + evaluator(scope) + } + }; + if let Value::Context(context) = value { Ok(context) } else { Err(err_not_a_context()) } } /// Evaluates a context from AST node. diff --git a/feel-evaluator/src/filters.rs b/feel-evaluator/src/filters.rs index 5de7f510..955aa3c1 100644 --- a/feel-evaluator/src/filters.rs +++ b/feel-evaluator/src/filters.rs @@ -2,7 +2,142 @@ use dsntk_feel::context::FeelContext; use dsntk_feel::values::{Value, Values}; -use dsntk_feel::{Evaluator, FeelScope, Name, value_null}; +use dsntk_feel::{Evaluator, FeelNumber, FeelScope, Name, value_null}; +use dsntk_feel_parser::AstNode; +use std::sync::LazyLock; + +/// The name `item`, interned once. +/// +/// `"item".into()` interns the text, and interning takes a read acquisition on the one +/// process-wide `RwLock` that guards the name table. The evaluator runs a filter once +/// per filter expression per row, from every Rayon worker. That acquisition is +/// therefore an atomic read-modify-write on a cache line every worker shares. A +/// `LazyLock` is an acquire load on a line that is read-only after the first use. +static NAME_ITEM: LazyLock = LazyLock::new(|| "item".into()); + +/// What a build-time walk proved about a filter's predicate, so that `L[i]` need not visit +/// every element of `L`. +/// +/// # The cost this removes +/// +/// `L[i]` and `L[i + 3]` are indexes, not filters, but a filter is the only machinery `FEEL` +/// has for them. [FilterExpressionEvaluator::evaluate] runs the predicate for **every** element +/// and only afterwards evaluates it once more to discover the result was an index. Measured at +/// 43 ns per element per index. So a prefix pass that indexes inside its own loop is cubic in +/// the list length. A two-stage prefix sum over 1,825 days measured **125.7 s**, where a +/// one-pass form measured 0.4 s. +/// +/// # The one argument the fast path rests on +/// +/// The loop's final evaluation already decides index against filter with **no** element context +/// pushed. So assume every per-element evaluation returns the same value as that final one, +/// and that value is a number. Two things follow. Each per-element evaluation returns a +/// number, which the loop discards, so the list it builds is empty and unused. And the index +/// it takes is the index the fast path takes. The two paths agree element for element and +/// message for message. +/// +/// What could make a per-element evaluation differ is only what the loop binds: +/// +/// 1. `item`, bound for every element. +/// 2. the element's own entries, pushed when the element is a `Value::Context`, so a bare name +/// in the predicate can resolve to a **field of the element**. +/// +/// Both are name resolution. A predicate that reads no name at all is immune. A predicate +/// that reads names other than `item` is immune once no element is a context. This says +/// nothing about what the predicate computes. That is why it does not matter that `+` can +/// return a string, a date or a duration as well as a number. +/// +/// **Nothing is skipped unless the evaluation returns a number.** Every other result falls +/// through to the loop, which then does exactly what it does today. That includes returning +/// an error null when a predicate yields a value a filter refuses. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum IndexShape { + /// Nothing proved. Every element runs the predicate, exactly as before. + Unknown, + /// Reads no name, so neither `item` nor an element's fields can reach it. Holds whatever the + /// elements are, so `fired[1]` needs no check at all. + Closed, + /// Reads only plain names, none of them `item`. Sound once no element is a context, which is + /// one discriminant test per element with no allocation and no scope push. + PlainNames, +} + +/// Classifies a filter predicate. See [IndexShape] for what each answer licenses. +/// +/// Enumerated, not excluded: a node kind absent from these arms answers [IndexShape::Unknown] +/// and keeps the element loop. This function admits only literals, plain names and the +/// arithmetic operators. So nothing in an admitted predicate can invoke a function, evaluate +/// a nested filter, build a context, or write through the scope. `EvaluatedExpression` is +/// transparent because `build_evaluated_expression` builds its child and calls it, adding +/// nothing. +pub fn index_shape(node: &AstNode) -> IndexShape { + match node { + AstNode::EvaluatedExpression(inner) | AstNode::Neg(inner) => index_shape(inner), + AstNode::Numeric(..) => IndexShape::Closed, + AstNode::Add(lhs, rhs) | AstNode::Sub(lhs, rhs) | AstNode::Mul(lhs, rhs) | AstNode::Div(lhs, rhs) | AstNode::Exp(lhs, rhs) => { + match (index_shape(lhs), index_shape(rhs)) { + (IndexShape::Unknown, _) | (_, IndexShape::Unknown) => IndexShape::Unknown, + (IndexShape::Closed, IndexShape::Closed) => IndexShape::Closed, + _ => IndexShape::PlainNames, + } + } + // `item` is bound for every element, so a predicate reading it is genuinely per-element. + AstNode::Name(name) => { + if *name == *NAME_ITEM { + IndexShape::Unknown + } else { + IndexShape::PlainNames + } + } + _ => IndexShape::Unknown, + } +} + +/// Reads element `index` of `values`, counting from 1, or from the end when negative. +/// +/// Lifted out of [FilterExpressionEvaluator::evaluate] unchanged so that the fast path and the +/// loop path share one copy, including every message. +fn index_into(values: &Values, index: FeelNumber) -> Value { + if index.is_integer() { + let list_size = values.len(); + if list_size > 0 { + if !index.is_negative() { + let n = { + if let Ok(u_index) = usize::try_from(index) { + u_index + } else { + return value_null!("index is out of range 1..2⁶⁴: {}", index.to_string()); + } + }; + if n > 0 && n <= list_size { + // unwrap below is safe, index `n` is checked above, `values` variable is immutable + values.get(n - 1).unwrap().to_owned() + } else { + value_null!("index in filter is out of range [1..{}], actual index is {}", list_size, n) + } + } else { + let n = { + if let Ok(u_index) = usize::try_from(index.abs()) { + u_index + } else { + return value_null!("index is out of range 1..2⁶⁴: {}", index.to_string()); + } + }; + if n > 0 && n <= list_size { + // unwrap below is safe, index `n` is checked above, `values` variable is immutable + values.get(list_size - n).unwrap().to_owned() + } else { + value_null!("index in filter is out of range [-{}..-1], actual index is -{}", list_size, n) + } + } + } else { + // return null when the list is empty, no matter what value the index has + value_null!() + } + } else { + value_null!("index in filter must be an integer value, actual value is {}", index) + } +} pub struct FilterExpressionEvaluator {} @@ -18,22 +153,40 @@ impl FilterExpressionEvaluator { Self {} } - pub fn evaluate(&self, scope: &FeelScope, value: Value, evaluator: &Evaluator) -> Value { - let name_item: Name = "item".into(); + pub fn evaluate(&self, scope: &FeelScope, value: Value, evaluator: &Evaluator, shape: IndexShape) -> Value { + let name_item: &Name = &NAME_ITEM; match value { Value::List(values) => { + // An index, not a filter: answer it without visiting an element. [IndexShape] carries + // what the build-time walk proved and states why each answer is enough. An empty list is + // left to the loop path, which costs nothing there and reports a bare null rather than an + // out-of-range message. + if !values.is_empty() + && match shape { + IndexShape::Unknown => false, + IndexShape::Closed => true, + // The one remaining way the loop could resolve a name differently is an element + // that is itself a context. The loop pushes that element's entries over the name + // being read. + IndexShape::PlainNames => !values.iter().any(|value| matches!(value, Value::Context(_))), + } + && let Value::Number(index) = evaluator(scope) + { + return index_into(&values, index); + } let mut filtered_values = vec![]; for value in &values { let (added_local_context, has_item_entry) = if let Value::Context(local_context) = value { scope.push(local_context.clone()); - if local_context.contains_entry(&name_item) { (true, true) } else { (true, false) } + if local_context.contains_entry(name_item) { (true, true) } else { (true, false) } } else { (false, false) }; if !has_item_entry { - let mut special_context = FeelContext::default(); - special_context.set_entry(&name_item, value.clone()); - scope.push(special_context); + // One entry, built and then shared. Building it with `default()` and `set_entry` + // instead runs `Arc::make_mut` on a static every thread shares. See + // [FeelContext::from_entry]. + scope.push(FeelContext::from_entry(name_item, value.clone())); } let rhv = evaluator(scope); match rhv { @@ -50,47 +203,7 @@ impl FilterExpressionEvaluator { } let rhv = evaluator(scope); match rhv { - Value::Number(index) => { - if index.is_integer() { - let list_size = values.len(); - if list_size > 0 { - if !index.is_negative() { - let n = { - if let Ok(u_index) = usize::try_from(index) { - u_index - } else { - return value_null!("index is out of range 1..2⁶⁴: {}", index.to_string()); - } - }; - if n > 0 && n <= list_size { - // unwrap below is safe, index `n` is checked above, `values` variable is immutable - values.get(n - 1).unwrap().to_owned() - } else { - value_null!("index in filter is out of range [1..{}], actual index is {}", list_size, n) - } - } else { - let n = { - if let Ok(u_index) = usize::try_from(index.abs()) { - u_index - } else { - return value_null!("index is out of range 1..2⁶⁴: {}", index.to_string()); - } - }; - if n > 0 && n <= list_size { - // unwrap below is safe, index `n` is checked above, `values` variable is immutable - values.get(list_size - n).unwrap().to_owned() - } else { - value_null!("index in filter is out of range [-{}..-1], actual index is -{}", list_size, n) - } - } - } else { - // return null when the list is empty, no matter what value the index has - value_null!() - } - } else { - value_null!("index in filter must be an integer value, actual value is {}", index) - } - } + Value::Number(index) => index_into(&values, index), _ => Value::List(filtered_values), } } diff --git a/feel-evaluator/src/iterations.rs b/feel-evaluator/src/iterations.rs index a17f5b11..f9043cd0 100644 --- a/feel-evaluator/src/iterations.rs +++ b/feel-evaluator/src/iterations.rs @@ -4,8 +4,26 @@ use crate::errors::*; use dsntk_common::Result; use dsntk_feel::context::FeelContext; use dsntk_feel::values::{VALUE_FALSE, VALUE_TRUE, Value, Values}; +use crate::parallel::ParallelSafety; use dsntk_feel::{Evaluator, FEEL_TYPE_NAME_DATE, FEEL_TYPE_NAME_NUMBER, FeelNumber, FeelScope, IntervalType, Name, value_null}; use dsntk_feel_temporal::FeelDate; +use rayon::prelude::*; +use std::sync::LazyLock; + +/// Smallest number of elements worth evaluating in parallel. +/// +/// Below this the thread pool costs more than it saves. +const PARALLEL_MIN_ELEMENTS: usize = 512; + +/// The name `partial`, interned once. See the note on `NAME_ITEM` in `filters.rs`. A +/// `for` expression builds a fresh evaluator on every evaluation, so interning this text +/// was one contended lock acquisition per `for` evaluation. +static NAME_PARTIAL: LazyLock = LazyLock::new(|| "partial".into()); + +/// Set `DSNTK_PARALLEL=0` to force every `for` expression onto the serial path. +/// +/// This exists so that the parallel and serial results can be compared on the same binary. +static PARALLEL_ENABLED: LazyLock = LazyLock::new(|| !matches!(std::env::var("DSNTK_PARALLEL").as_deref(), Ok("0"))); /// Common interface for all iteration state types. trait IterationState { @@ -34,6 +52,14 @@ trait IterationState { fn is_variable(&self) -> bool { false } + + /// Returns the bound variable and the values when this state iterates over a list. + /// + /// Only a list can be indexed, so only a list can be split across workers. Every other + /// state is a sequential machine and returns `None`, which keeps it on the serial path. + fn as_list(&self) -> Option<(&Name, &Values)> { + None + } } /// Iteration stepping direction. @@ -204,6 +230,10 @@ impl ListState { } impl IterationState for ListState { + fn as_list(&self) -> Option<(&Name, &Values)> { + Some((&self.variable, &self.values)) + } + fn set_value(&mut self, ctx: &mut FeelContext) { if !self.values.is_empty() { ctx.set_entry(&self.variable, self.values[self.current].clone()); @@ -339,6 +369,12 @@ impl FeelIterator { self.states.push(VariableState::init(variable, bound_variable)); } + /// Returns the bound variable and the values when this iterator has exactly one state + /// and that state iterates over a list. Any other shape returns `None`. + pub fn single_list(&self) -> Option<(&Name, &Values)> { + if self.states.len() == 1 { self.states[0].as_list() } else { None } + } + /// Iterates over all iteration states. pub fn iterate(&mut self, mut handler: F) where @@ -376,6 +412,9 @@ impl FeelIterator { pub struct ForExpressionEvaluator { iterator: FeelIterator, partial: Name, + /// Whether the body of this `for` can run in parallel. Decided when the + /// evaluator is built, see [crate::parallel]. + safety: ParallelSafety, } impl Default for ForExpressionEvaluator { @@ -390,10 +429,16 @@ impl ForExpressionEvaluator { pub fn new() -> Self { Self { iterator: FeelIterator::default(), - partial: "partial".into(), + partial: NAME_PARTIAL.clone(), + safety: ParallelSafety::default(), } } + /// Records the result of the parallel-safety analysis of the body. + pub fn set_safety(&mut self, safety: ParallelSafety) { + self.safety = safety; + } + /// Adds an interval of values to iterate over. pub fn add_interval(&mut self, name: Name, start: Value, end: Value) -> Result<()> { self.iterator.add_interval(name, start, end, false) @@ -414,19 +459,76 @@ impl ForExpressionEvaluator { } pub fn evaluate(&mut self, scope: &FeelScope, evaluator: &Evaluator) -> Values { + // Parallel path. This code takes it only when the body provably cannot read `partial`, + // so there is no dependency between iterations. It also requires the iteration to run + // over a single list, the only shape that can be indexed. Everything else falls through + // to the serial code below, which stays the default. + if *PARALLEL_ENABLED + && let Some((variable, values)) = self.iterator.single_list() + && values.len() >= PARALLEL_MIN_ELEMENTS + && self.safety.is_safe_in(scope) + { + return evaluate_in_parallel(scope, evaluator, variable, values); + } let mut results = vec![]; self.iterator.iterate(|ctx| { let mut iteration_context = ctx.clone(); - iteration_context.set_entry(&self.partial, Value::List(results.clone())); - scope.push(iteration_context.clone()); + // The implicit `partial` variable holds the results of all previous iterations. + // This code moves them into the iteration context and moves them back out + // after the iteration, instead of copying the whole list every time. + iteration_context.set_entry(&self.partial, Value::List(std::mem::take(&mut results))); + let scope_depth = scope.len(); + scope.push(iteration_context); let iteration_value = evaluator(scope); - scope.pop(); + // Discard the contexts that the evaluated expression leaves behind. This keeps + // the context holding the partial results at the top of the scope. + while scope.len() > scope_depth + 1 { + scope.pop(); + } + if let Some(mut iteration_context) = scope.pop() + && let Some(Value::List(previous_results)) = iteration_context.remove_entry(&self.partial) + { + results = previous_results; + } results.push(iteration_value); }); results } } +/// Evaluates the body of a `for` expression over `values` in parallel. +/// +/// This function never binds `partial`, because the caller has already proved that the +/// body cannot read it. +/// +/// Each worker gets its own scope from [FeelScope::from_contexts], which shares the +/// enclosing contexts by refcount rather than copying them. `map_init` builds one scope per +/// worker task, not one per element, so the stack clone is not a per-element cost. +/// +/// Order is preserved. A slice's `par_iter` is an indexed parallel iterator, and `map_init` +/// keeps it indexed. `collect` into a `Vec` therefore writes each result at its own index. +/// The output is identical to the serial path element for element, not merely equivalent. +fn evaluate_in_parallel(scope: &FeelScope, evaluator: &Evaluator, variable: &Name, values: &Values) -> Values { + let base = scope.contexts(); + let depth = base.len(); + values + .par_iter() + .map_init( + || FeelScope::from_contexts(base.clone()), + |local: &mut FeelScope, item| { + local.push(FeelContext::from_entry(variable, item.clone())); + let value = evaluator(local); + // The body may leave contexts behind, exactly as on the serial path. This worker + // reuses this scope for the next element. Restore the original depth here. + while local.len() > depth { + local.pop(); + } + value + }, + ) + .collect() +} + /// `some` expression evaluator. pub struct SomeExpressionEvaluator { iterator: FeelIterator, diff --git a/feel-evaluator/src/lib.rs b/feel-evaluator/src/lib.rs index 56a276c6..af47e29c 100644 --- a/feel-evaluator/src/lib.rs +++ b/feel-evaluator/src/lib.rs @@ -6,15 +6,19 @@ extern crate dsntk_macros; mod bifs; mod builders; +mod cse; mod errors; mod evaluator_java; mod evaluator_pmml; mod evaluators; mod filters; mod iterations; +mod literal; mod macros; +mod parallel; mod tests; pub use crate::evaluators::*; -pub use crate::filters::FilterExpressionEvaluator; +pub use crate::filters::{FilterExpressionEvaluator, IndexShape}; pub use crate::iterations::{EveryExpressionEvaluator, ForExpressionEvaluator, SomeExpressionEvaluator}; +pub use crate::parallel::references_name; diff --git a/feel-evaluator/src/literal.rs b/feel-evaluator/src/literal.rs new file mode 100644 index 00000000..d72a9b7b --- /dev/null +++ b/feel-evaluator/src/literal.rs @@ -0,0 +1,240 @@ +//! # Direct value construction for input that is pure data +//! +//! Evaluating a `FEEL` expression takes three steps: parse the text into a syntax tree, build a +//! tree of boxed closures from it, then run the closures to produce a value. Input data files +//! are not expressions. They are data: nested contexts and lists whose leaves are strings, +//! numbers, booleans and nulls. For such input the closure tree is a whole intermediate +//! representation built and thrown away for nothing. +//! +//! [literal_value] reads an accepted syntax tree straight into a [Value] and skips that step. +//! +//! # What is accepted +//! +//! Only [AstNode::Boolean], [AstNode::Null], [AstNode::Numeric], [AstNode::String], +//! [AstNode::Neg], [AstNode::List] and [AstNode::Context] whose entries are all +//! [AstNode::ContextEntry] with an [AstNode::ContextEntryKey] key. Every other node kind +//! rejects the whole tree, which then takes the normal path. The test is purely syntactic: it +//! never looks at a name, a scope or a value. +//! +//! # Why the syntax tree is read and not consumed +//! +//! Consuming the tree frees each record as soon as its value exists, which lowers the live byte +//! count. Measurement showed that it also raises peak RSS by 13%: freeing the tree a record at a +//! time, while the value tree grows, spreads `mimalloc`'s committed arena slices instead of +//! releasing them in one block. Reading the tree and freeing it whole afterwards keeps the +//! allocation pattern the previous code had, and the peak was never in this phase anyway. The +//! parse itself, with the lexer buffer and the parser stacks live, sets the high-water mark. +//! +//! # Why skipping the scope is safe +//! +//! [crate::builders] pushes a context onto the scope while it evaluates context entries, so that +//! a later entry can read an earlier one by name. A `FEEL` expression reads that context only +//! through an [AstNode::Name] or an [AstNode::Path], and both reject the tree. No accepted node +//! reads the scope at all, so this module needs no scope. +//! +//! # Where the results must stay identical +//! +//! The value produced here must equal the value the closure tree produces, byte for byte, +//! including error text. Four places carry that risk, and each one copies the corresponding +//! builder rather than reimplementing it: +//! +//! - number text is reassembled and parsed exactly as `build_number` does it, +//! - a duplicated key yields the same null and the same message as `build_context`, +//! - negation reproduces every arm of `build_neg`, including its error text, +//! - [AstNode::At] is not accepted, because it needs a temporal parse. +//! +//! Nothing here can panic on an unexpected node. Every function returns [None] for a node it +//! does not accept, and the caller then takes the normal path, so a gap between [is_data_literal] +//! and [build_value] costs a fast path and never a wrong answer. + +use dsntk_feel::context::FeelContext; +use dsntk_feel::values::Value; +use dsntk_feel::{FeelNumber, value_null}; +use dsntk_feel_parser::AstNode; +use std::fmt::Write; + +/// Reads a syntax tree that is pure data into a [Value]. +/// +/// Returns [None] when any node in the tree is not a data literal. +pub fn literal_value(node: &AstNode) -> Option { + // The check runs first and allocates nothing, so a tree that is rejected at its last record + // costs one pointer walk instead of a whole value tree that is then thrown away. + if !is_data_literal(node) { + return None; + } + build_value(node) +} + +/// Returns `true` when every node in the subtree is a data literal. +fn is_data_literal(node: &AstNode) -> bool { + match node { + AstNode::Boolean(_) | AstNode::Null | AstNode::Numeric(..) | AstNode::String(_) => true, + AstNode::Neg(value) => is_data_literal(value), + AstNode::List(items) => items.iter().all(is_data_literal), + AstNode::Context(entries) => entries.iter().all(is_data_literal_entry), + _ => false, + } +} + +/// Returns `true` when the node is a context entry with a literal key and a literal value. +fn is_data_literal_entry(node: &AstNode) -> bool { + match node { + AstNode::ContextEntry(key, value) => matches!(key.as_ref(), AstNode::ContextEntryKey(_)) && is_data_literal(value), + _ => false, + } +} + +/// Builds the value of a data literal subtree, or [None] for any other node. +fn build_value(node: &AstNode) -> Option { + Some(match node { + AstNode::Boolean(value) => Value::Boolean(*value), + AstNode::Null => Value::Null(None), + AstNode::String(text) => Value::String(text.clone()), + AstNode::Numeric(before, after, sign, exponent) => number_value(before, after, *sign, exponent), + AstNode::Neg(value) => negated_value(build_value(value)?), + AstNode::List(items) => { + let mut values = Vec::with_capacity(items.len()); + for item in items { + values.push(build_value(item)?); + } + Value::List(values) + } + AstNode::Context(entries) => context_value(entries)?, + _ => return None, + }) +} + +/// Builds a context value from the entries of an [AstNode::Context]. +/// +/// This is `build_context` without the scope. The duplicated key check, its message and the +/// order of the entries are the same, so both paths return the same value. +fn context_value(entries: &[AstNode]) -> Option { + let mut evaluated_ctx = FeelContext::default(); + for entry in entries { + let AstNode::ContextEntry(key, value) = entry else { + return None; + }; + let AstNode::ContextEntryKey(name) = key.as_ref() else { + return None; + }; + if evaluated_ctx.contains_entry(name) { + // duplicated context entry keys are not allowed + return Some(value_null!("duplicated context entry key: {}", name)); + } + evaluated_ctx.set_entry(name, build_value(value)?); + } + Some(Value::Context(evaluated_ctx)) +} + +/// Builds a number value from the four parts of an [AstNode::Numeric]. +/// +/// This is the body of `build_number`. The text is reassembled the same way and parsed by the +/// same implementation, so a number reaches the same last digit on both paths. +fn number_value(before: &str, after: &str, sign: char, exponent: &str) -> Value { + let mut text = String::new(); + let _ = write!(&mut text, "{before}"); + if !after.is_empty() { + let _ = write!(&mut text, ".{after}"); + } + if !exponent.is_empty() { + let _ = write!(&mut text, "e{sign}{exponent}"); + } + if let Ok(num) = text.parse::() { + Value::Number(num) + } else { + value_null!("failed to convert text '{}' into number", text) + } +} + +/// Negates a value. +/// +/// This is the body of `build_neg`. Only the number arm and the error arm are reachable from a +/// data literal, but every arm is kept so that the two paths cannot drift apart. +fn negated_value(value: Value) -> Value { + match value { + Value::Number(lh) => Value::Number(-lh), + Value::DaysAndTimeDuration(lh) => Value::DaysAndTimeDuration(-lh), + Value::YearsAndMonthsDuration(lh) => Value::YearsAndMonthsDuration(-lh), + other => value_null!("unexpected type in arithmetic negation: {}", other.type_of()), + } +} + +#[cfg(test)] +mod tests { + use super::{is_data_literal, literal_value}; + use dsntk_feel::FeelScope; + use dsntk_feel_parser::AstNode; + + /// Parses a context definition and returns the syntax tree. + fn parse(input: &str) -> AstNode { + dsntk_feel_parser::parse_context(&FeelScope::default(), input, false).unwrap() + } + + /// Returns the value both paths must agree on, built by the closure tree. + fn slow_path(input: &str) -> String { + let node = parse(input); + format!("{}", crate::builders::build_evaluator(&node)(&FeelScope::default())) + } + + /// Asserts that the tree is accepted and that both paths agree. + fn accepted(input: &str) { + let Some(value) = literal_value(&parse(input)) else { + panic!("data literal path rejected `{input}`"); + }; + assert_eq!(slow_path(input), format!("{value}"), "paths disagree on `{input}`"); + } + + /// Asserts that the tree is rejected. + fn rejected(input: &str) { + if let Some(value) = literal_value(&parse(input)) { + panic!("data literal path accepted `{input}` and returned `{value}`"); + } + } + + #[test] + fn accepted_trees_agree_with_the_closure_tree() { + accepted("{}"); + accepted("{a: 1}"); + accepted("{a: []}"); + accepted("{a: [[], [[]]]}"); + accepted("{a: {}}"); + accepted(r#"{a: null, b: true, c: false, d: "x", e: 1, f: -1, g: -0.5}"#); + accepted("{a: 1.10, b: 1e3, c: 1.5e-3, d: 0.000000000000000000000000000001}"); + accepted("{a: 123456789012345678901234567890123456}"); + accepted("{a: 1e999999999}"); + accepted("{a: -1e999999999}"); + accepted("{a: - -1}"); + accepted(r#"{"a b": 1, "": 2}"#); + accepted("{a: 1, a: 2}"); + accepted("{a: {b: 1, b: 2}, c: 3}"); + accepted("{a: [{b: 1, b: 2}]}"); + accepted(r#"{a: [{b: [{c: {d: [1, "x", null, true]}}]}]}"#); + accepted(r#"{a: "A\n\"", b: "🙂"}"#); + } + + #[test] + fn trees_that_are_not_data_are_rejected() { + rejected(r#"{a: date and time("2026-01-05T10:00:00Z")}"#); + rejected(r#"{a: @"2026-01-05"}"#); + rejected("{a: 1, b: a}"); + rejected("{a: 1 + 1}"); + rejected("{a: -(1 + 1)}"); + rejected("{a: [1, 2][item > 1]}"); + rejected("{a: for x in [1, 2] return x}"); + rejected("{a: if true then 1 else 2}"); + rejected("{a: {b: 1}.b}"); + rejected(r#"{a: [1, 2, duration("P1D")]}"#); + rejected("{a: {b: 1, c: b}}"); + rejected("{a: function() 1}"); + rejected("{a: 1 in [1..2]}"); + rejected("{a: some x in [1] satisfies x = 1}"); + } + + #[test] + fn the_rejection_test_never_looks_below_a_rejected_node() { + // The predicate is syntactic, so a rejected node rejects every tree that contains it, + // however deeply it is buried. + assert!(!is_data_literal(&parse(r#"{a: [{b: [{c: {d: [1, x]}}]}]}"#))); + assert!(is_data_literal(&parse(r#"{a: [{b: [{c: {d: [1, 2]}}]}]}"#))); + } +} diff --git a/feel-evaluator/src/parallel.rs b/feel-evaluator/src/parallel.rs new file mode 100644 index 00000000..d6ef4ee7 --- /dev/null +++ b/feel-evaluator/src/parallel.rs @@ -0,0 +1,329 @@ +//! Decides whether the body of a `for` expression can run in parallel. +//! +//! # Why an analysis is needed at all +//! +//! `for` binds the implicit variable `partial` to the results of all previous iterations. +//! An iteration that reads `partial` depends on every earlier iteration. It cannot be +//! reordered or run concurrently with them. A body that never reads `partial` has no such +//! dependency. +//! +//! Deciding that statically is **not** as simple as looking for the name. An earlier +//! attempt to do exactly that produced a silently wrong answer. `eval_function_definition` +//! pushes the closure and parameter contexts onto the **live** scope stack. `FEEL` name +//! resolution inside a called function is therefore dynamic. A function defined outside +//! the loop can read the loop's `partial`, even though the body never mentions it. +//! +//! ```text +//! { f: function(z) count(partial), r: for x in [1,2,3] return f(1) }.r -> [0, 1, 2] +//! ``` +//! +//! The body here is just `f(1)`. +//! +//! # The rule +//! +//! A body qualifies only when **all** of these hold. Each one closes a way for a hidden +//! `partial` read to exist. +//! +//! 1. No name anywhere in the body is `partial`. The direct case. +//! 2. No [AstNode::FunctionDefinition] anywhere in the body. A function defined in the body +//! could be invoked with the live scope on the stack. +//! 3. Every [AstNode::FunctionInvocation] calls a plain name that parses as a built-in +//! function. A user-defined function or a `BKM` invocation reaches the live scope. A +//! built-in does not. +//! 4. No name bound inside the body parses as a built-in function name. Without this, a +//! context entry such as `{ contains: someOuterFunction, v: contains(1) }` would satisfy +//! rule 3 while actually calling a user-defined function. +//! 5. No [AstNode::Filter] in the body has a predicate that invokes anything. Evaluating a +//! filter pushes the filtered element's **own entries** onto the live scope. So when an +//! element is a context, its keys become names the predicate can call. Rules 1 to 4 read +//! the body as written and [ParallelSafety::is_safe_in] reads the scope as it stands when +//! the `for` begins. Neither can see a binding that arrives later, from data. +//! +//! 6. No loop variable of the `for` shares a name with a callee invoked in the body. This one +//! is checked in `build_for` rather than here, because [analyze] only ever sees the body. +//! The loop variables come from the `for` expression's left side. See +//! [ParallelSafety::forbid_names]. +//! +//! Rule 5 is what makes rules 3 and 4 hold up. Without it, a list element such as +//! `{ sum: function(x) count(partial), v: 1 }` binds `sum` to a function that reads `partial`. +//! A predicate calling `sum(item)` then passes rule 3 as a built-in call. The serial path +//! binds `partial` and the parallel path does not, so the two disagree. `count(null)` +//! returns 1, and the predicate becomes true for every element. +//! +//! Rule 3 is sound for built-ins. Only three of them evaluate a `FEEL` expression at +//! all — `list replace`, `sort` and `range`. Each of the three builds a **fresh** scope +//! (`let scope: FeelScope = ctx.into()`), so none of them can observe the caller's stack. +//! Reading `bifs/core.rs` confirms this. +//! +//! Rule 3 still leaves one hole that no static rule can close. A name that parses as a +//! built-in may be bound to a function **in an enclosing scope** rather than in the body. +//! [ParallelSafety::callees] carries the invoked names, so the caller can check the live +//! scope once per `for` evaluation. [ParallelSafety::is_safe_in] performs that check. +//! +//! Everything here is conservative: when in doubt the answer is "not safe" and the serial +//! path runs. + +use dsntk_feel::bif::Bif; +use dsntk_feel::{FeelScope, Name}; +use dsntk_feel_parser::AstNode; +use std::collections::BTreeSet; +use std::str::FromStr; + +/// Returns `true` when `name` is the name of a built-in function. +fn is_builtin(name: &Name) -> bool { + Bif::from_str(&name.to_string()).is_ok() +} + +/// Returns `true` when `node` contains a function invocation anywhere below it. +/// +/// Used by rule 5. The callee is not inspected: a name that resolves to a built-in when read +/// statically may resolve to an element's entry once the filter pushes it. +fn contains_invocation(node: &AstNode) -> bool { + matches!(node, AstNode::FunctionInvocation(..)) || children(node).into_iter().any(contains_invocation) +} + +/// The result of analyzing the body of a `for` expression. +#[derive(Debug, Clone, Default)] +pub struct ParallelSafety { + /// `true` when the static rules 1 to 4 all hold. + statically_safe: bool, + /// Names invoked as functions in the body. All of them parse as built-in function names, + /// so each must also be unbound in the live scope, or the call is not a built-in call. + callees: BTreeSet, +} + +impl ParallelSafety { + /// Refuses parallel evaluation when a loop variable shares a name with an invoked callee. + /// + /// Rule 6, and the reason it cannot live in [analyze]: the loop variables come from the + /// `for` expression's left side, which [analyze] never sees. It reads only the body. + /// + /// A loop variable binds a **data value** to a name the model author chose, so that name + /// can spell a built-in. `for sum in fs return sum(1)` satisfies rules 1 to 5: `sum` + /// parses as a built-in, nothing inside the body binds it, and there is no filter. It + /// also satisfies [Self::is_safe_in], because at loop entry `sum` is not bound yet. Then + /// each iteration binds it to an element. If that element is a function reading + /// `partial`, the serial path binds `partial` and the parallel path does not, and the + /// two disagree. + /// + /// The intersection is exact rather than conservative. Rule 3 already forces every + /// callee to be a plain name. So the only way to invoke a loop-bound value is through a + /// callee name equal to a loop-variable name. + pub fn forbid_names(&mut self, names: &std::collections::HashSet) { + if self.callees.iter().any(|callee| names.contains(callee)) { + self.statically_safe = false; + } + } + + /// Returns `true` when this body can run in parallel in the given scope. + /// + /// This checks the static rules only once, at evaluator build time. This adds one + /// check that needs a scope: an enclosing scope must not shadow any of the invoked + /// built-in names. It runs once per `for` evaluation, not once per element. + pub fn is_safe_in(&self, scope: &FeelScope) -> bool { + self.statically_safe && !self.callees.iter().any(|name| scope.contains_name(name)) + } +} + +/// Analyzes the body of a `for` expression. +pub fn analyze(body: &AstNode) -> ParallelSafety { + let mut state = Analysis { + unsafe_found: false, + callees: BTreeSet::new(), + bound: BTreeSet::new(), + }; + state.walk(body); + // Rule 4: no name bound inside the body may parse as a built-in function name. + let shadows_builtin = state.bound.iter().any(is_builtin); + ParallelSafety { + statically_safe: !state.unsafe_found && !shadows_builtin, + callees: state.callees, + } +} + +struct Analysis { + /// Set when the body violates rule 1, 2 or 3. + unsafe_found: bool, + callees: BTreeSet, + bound: BTreeSet, +} + +impl Analysis { + fn note_name(&mut self, name: &Name) { + // Rule 1. + if name.to_string() == "partial" { + self.unsafe_found = true; + } + } + + fn walk(&mut self, node: &AstNode) { + if self.unsafe_found { + return; + } + match node { + // Rule 2. A function defined in the body may be invoked with the live scope pushed. + AstNode::FunctionDefinition(..) => { + self.unsafe_found = true; + return; + } + // Rule 3 allows only a call to a plain built-in name. + AstNode::FunctionInvocation(callee, arguments) => { + match callee.as_ref() { + AstNode::Name(name) if is_builtin(name) => { + self.callees.insert(name.clone()); + } + // Anything else is a user-defined function, a BKM, or a computed callee. + _ => { + self.unsafe_found = true; + return; + } + } + self.walk(arguments); + return; + } + // Rule 5. A filter pushes the filtered element's own entries onto the live scope. So a + // predicate that calls anything may be calling a name that arrived from data. The list + // being filtered is still walked: only the predicate sees the element's entries. + AstNode::Filter(list, predicate) => { + if contains_invocation(predicate) { + self.unsafe_found = true; + return; + } + self.walk(list); + self.walk(predicate); + return; + } + // Names that a body binds. Collected for rule 4. + AstNode::ContextEntryKey(name) | AstNode::ContextTypeEntryKey(name) | AstNode::ParameterName(name) => { + self.note_name(name); + self.bound.insert(name.clone()); + return; + } + // Iteration and quantifier variables bind their first child name. + AstNode::IterationContextSingle(variable, _) | AstNode::IterationContextInterval(variable, _, _) | AstNode::QuantifiedContext(variable, _) => { + if let AstNode::Name(name) = variable.as_ref() { + self.bound.insert(name.clone()); + } + } + // A formal parameter binds its first child name. + AstNode::FormalParameter(variable, _) => { + if let AstNode::Name(name) = variable.as_ref() { + self.bound.insert(name.clone()); + } + } + // Plain name references. + AstNode::Name(name) | AstNode::QualifiedNameSegment(name) => { + self.note_name(name); + return; + } + _ => {} + } + for child in children(node) { + self.walk(child); + if self.unsafe_found { + return; + } + } + } +} + +/// Returns `true` when `name` appears anywhere in `node`, as a plain name or as a segment +/// of a qualified name. +/// +/// Used by the decision-table and item-definition builders to see whether a unary test +/// uses the special name `?`. DMN 1.3 Table 55 gives such a test different semantics: +/// `e in t` means `{?: e, r: t}.r` when `t` is a boolean expression over `?`. +pub fn references_name(node: &AstNode, name: &Name) -> bool { + match node { + AstNode::Name(inner) | AstNode::QualifiedNameSegment(inner) if inner == name => true, + _ => children(node).into_iter().any(|child| references_name(child, name)), + } +} + +/// Returns every child node of `node`. +/// +/// This match is deliberately **exhaustive with no catch-all arm**. If a future version of +/// the grammar adds an [AstNode] variant, this function stops compiling. That is the point. +/// A future maintainer must classify a new variant by hand, rather than let it silently +/// default to having no children. An unclassified variant would make the analysis above +/// unsound. +fn children(node: &AstNode) -> Vec<&AstNode> { + match node { + // two children + AstNode::Add(a, b) + | AstNode::And(a, b) + | AstNode::ContextEntry(a, b) + | AstNode::ContextTypeEntry(a, b) + | AstNode::Div(a, b) + | AstNode::Eq(a, b) + | AstNode::Every(a, b) + | AstNode::Exp(a, b) + | AstNode::Filter(a, b) + | AstNode::For(a, b) + | AstNode::FormalParameter(a, b) + | AstNode::FunctionDefinition(a, b) + | AstNode::FunctionInvocation(a, b) + | AstNode::FunctionType(a, b) + | AstNode::Ge(a, b) + | AstNode::Gt(a, b) + | AstNode::In(a, b) + | AstNode::InstanceOf(a, b) + | AstNode::IterationContextSingle(a, b) + | AstNode::Le(a, b) + | AstNode::Lt(a, b) + | AstNode::Mul(a, b) + | AstNode::NamedParameter(a, b) + | AstNode::Nq(a, b) + | AstNode::Or(a, b) + | AstNode::Out(a, b) + | AstNode::Path(a, b) + | AstNode::QuantifiedContext(a, b) + | AstNode::Range(a, b) + | AstNode::Some(a, b) + | AstNode::Sub(a, b) => vec![a, b], + // three children + AstNode::Between(a, b, c) | AstNode::If(a, b, c) | AstNode::IterationContextInterval(a, b, c) => vec![a, b, c], + // one child + AstNode::EvaluatedExpression(a) + | AstNode::FunctionBody(a, _) + | AstNode::IntervalEnd(a, _) + | AstNode::IntervalStart(a, _) + | AstNode::ListType(a) + | AstNode::Neg(a) + | AstNode::RangeType(a) + | AstNode::Satisfies(a) + | AstNode::UnaryEq(a) + | AstNode::UnaryGe(a) + | AstNode::UnaryGt(a) + | AstNode::UnaryLe(a) + | AstNode::UnaryLt(a) + | AstNode::UnaryNe(a) => vec![a], + // a list of children + AstNode::CommaList(items) + | AstNode::Context(items) + | AstNode::ContextType(items) + | AstNode::ExpressionList(items) + | AstNode::FormalParameters(items) + | AstNode::IterationContexts(items) + | AstNode::List(items) + | AstNode::NamedParameters(items) + | AstNode::NegatedList(items) + | AstNode::ParameterTypes(items) + | AstNode::PositionalParameters(items) + | AstNode::QualifiedName(items) + | AstNode::QuantifiedContexts(items) => items.iter().collect(), + // leaves + AstNode::At(_) + | AstNode::Boolean(_) + | AstNode::ContextEntryKey(_) + | AstNode::ContextTypeEntryKey(_) + | AstNode::FeelType(_) + | AstNode::Irrelevant + | AstNode::Name(_) + | AstNode::Null + | AstNode::Numeric(..) + | AstNode::ParameterName(_) + | AstNode::QualifiedNameSegment(_) + | AstNode::String(_) => vec![], + } +} diff --git a/feel-evaluator/src/tests/bifs/bif_context_merge.rs b/feel-evaluator/src/tests/bifs/bif_context_merge.rs index 455bbe15..64ad8da9 100644 --- a/feel-evaluator/src/tests/bifs/bif_context_merge.rs +++ b/feel-evaluator/src/tests/bifs/bif_context_merge.rs @@ -35,3 +35,23 @@ fn _0005() { "[core::context merge] invalid argument type, expected list, actual type is number", ); } + +#[test] +fn _0006() { + te_null( + false, + &scope!(), + r#"context merge([{x:1}, null])"#, + "[core::context merge] invalid argument type, expected context, actual type is Null", + ); +} + +#[test] +fn _0007() { + te_null( + false, + &scope!(), + r#"context merge([null])"#, + "[core::context merge] invalid argument type, expected context, actual type is Null", + ); +} diff --git a/feel-evaluator/src/tests/bifs/bif_get_value.rs b/feel-evaluator/src/tests/bifs/bif_get_value.rs index 5f82ecff..a4ce4432 100644 --- a/feel-evaluator/src/tests/bifs/bif_get_value.rs +++ b/feel-evaluator/src/tests/bifs/bif_get_value.rs @@ -65,3 +65,16 @@ fn _0009() { fn _0010() { te_be_value(false, &scope!(), r#"get value(m: {a: "foo"}, key: "b")"#, r#"null"#); } + +#[test] +fn _0011() { + // A key absent from the context answers null, and must not intern the probed text. The + // interner never releases a name, so a lookup that interns would leak one per probe. + te_null(false, &scope!(), r#"get value({a: 1}, "no such key 3b7f")"#, ""); +} + +#[test] +fn _0012() { + // The hit path is unchanged by the lookup-only probe. + te_number(false, &scope!(), r#"get value({a: 1, b: 2}, "b")"#, 2, 0); +} diff --git a/feel-evaluator/src/tests/bifs/bif_matches.rs b/feel-evaluator/src/tests/bifs/bif_matches.rs index 3b9fe002..6d2353d2 100644 --- a/feel-evaluator/src/tests/bifs/bif_matches.rs +++ b/feel-evaluator/src/tests/bifs/bif_matches.rs @@ -280,3 +280,21 @@ fn _0047() { fn _0048() { te_bool(false, &scope!(), r#"matches("aA", "(a)\1", "i")"#, true); } + +/// `matches` inside a `for` body long enough to take the parallel path. +/// +/// The list has 2,000 elements, well past the 512-element parallel threshold. The body +/// passes every static rule, so Rayon workers call `matches` concurrently. The count +/// pins the answer to the serial one. This guards the compiled-pattern cache, which is +/// per-thread: every worker must compile and match independently and still agree with +/// a single-threaded run. +#[test] +fn _0049() { + te_number( + false, + &scope!(), + r#"count((for s in (for i in 1..2000 return string(i)) return matches(s, "^1\\d{2}$"))[item = true])"#, + 100, + 0, + ); +} diff --git a/feel-evaluator/src/tests/bifs/bif_string_join.rs b/feel-evaluator/src/tests/bifs/bif_string_join.rs index 6033eb9c..a8283d81 100644 --- a/feel-evaluator/src/tests/bifs/bif_string_join.rs +++ b/feel-evaluator/src/tests/bifs/bif_string_join.rs @@ -155,3 +155,33 @@ fn _0025() { "expected 1,2 parameters, actual number of parameters is 3", ); } + +#[test] +fn _0026() { + te_string(false, &scope!(), r#"string join([null,"b"], "X")"#, "b"); +} + +#[test] +fn _0027() { + te_string(false, &scope!(), r#"string join([null,"b"])"#, "b"); +} + +#[test] +fn _0028() { + te_string(false, &scope!(), r#"string join([null,null,"c"], "X")"#, "c"); +} + +#[test] +fn _0029() { + te_string(false, &scope!(), r#"string join([null,"a",null,"b"], "X")"#, "aXb"); +} + +#[test] +fn _0030() { + te_string(false, &scope!(), r#"string join([null,null], "X")"#, ""); +} + +#[test] +fn _0031() { + te_string(false, &scope!(), r#"string join(["a",null], "X")"#, "a"); +} diff --git a/feel-evaluator/src/tests/for_expression.rs b/feel-evaluator/src/tests/for_expression.rs index 0f2f6a22..67db3b61 100644 --- a/feel-evaluator/src/tests/for_expression.rs +++ b/feel-evaluator/src/tests/for_expression.rs @@ -218,3 +218,52 @@ fn _0022() { "[1, 2, 3, 4, 5, 6, 7, 8]", ); } + +#[test] +fn _0023() { + //-------------------------------------------------------------------------------------------------------------------- + // Rule 5. A filter predicate that invokes may call a name pushed from the filtered element's + // own entries, so the body must not run in parallel. The element binds `sum` to a function + // reading `partial`, which only the serial path binds. Below PARALLEL_MIN_ELEMENTS both paths + // agreed, so the list must be long enough to cross the threshold. + //-------------------------------------------------------------------------------------------------------------------- + te_be_value( + false, + &scope!(), + "{ data: [{sum: function(x) count(partial), v: 1}], big: (for k in 1..600 return k), \ + r: (sum(for i in big return count(data[sum(item) > 0]))) }.r", + "599", + ); +} + +#[test] +fn _0024() { + //-------------------------------------------------------------------------------------------------------------------- + // Rule 6. The loop variable itself binds a data value to a name the author chose, so it can + // spell a built-in. `sum` passes rules 1 to 5 and is unbound at loop entry, then each iteration + // binds it to a function reading `partial`. Parallel used to answer [1, 1, ...] here. + //-------------------------------------------------------------------------------------------------------------------- + te_be_value( + false, + &scope!(), + "{f: function(z) count(partial), fs: for i in 1..600 return f, \ + r: for sum in fs return sum(1), out: [r[1], r[600], count(r[item = 1])]}.out", + "[0, 599, 1]", + ); +} + +#[test] +fn _0025() { + //-------------------------------------------------------------------------------------------------------------------- + // Rule 6 control. The same shape with a loop variable that is not a built-in name stays serial + // through rule 3, and must give the same answer. This guards against a fix that only works by + // disabling parallelism everywhere. + //-------------------------------------------------------------------------------------------------------------------- + te_be_value( + false, + &scope!(), + "{f: function(z) count(partial), fs: for i in 1..600 return f, \ + r: for g in fs return g(1), out: [r[1], r[600], count(r[item = 1])]}.out", + "[0, 599, 1]", + ); +} diff --git a/feel-evaluator/src/tests/mod.rs b/feel-evaluator/src/tests/mod.rs index 13e8db78..e67f3bc8 100644 --- a/feel-evaluator/src/tests/mod.rs +++ b/feel-evaluator/src/tests/mod.rs @@ -73,19 +73,24 @@ pub fn te_date(trace: bool, scope: &FeelScope, expression: &str, year: Year, mon pub fn te_date_time_local(trace: bool, scope: &FeelScope, s: &str, date: (Year, Month, Day), time: (u8, u8, u8, u64)) { let (year, month, day) = date; let (hour, min, sec, nano) = time; - textual_expression(trace, scope, s, Value::DateTime(FeelDateTime::local(year, month, day, hour, min, sec, nano).unwrap())); + textual_expression( + trace, + scope, + s, + Value::DateTime(FeelDateTime::local(year, month, day, hour, min, sec, nano).unwrap().into()), + ); } /// Utility function that tests evaluation of UTC date and time value. pub fn te_date_time_utc(trace: bool, scope: &FeelScope, s: &str, date: (Year, Month, Day), time: (u8, u8, u8, u64)) { let (year, month, day) = date; let (hour, min, sec, nano) = time; - textual_expression(trace, scope, s, Value::DateTime(FeelDateTime::utc(year, month, day, hour, min, sec, nano).unwrap())); + textual_expression(trace, scope, s, Value::DateTime(FeelDateTime::utc(year, month, day, hour, min, sec, nano).unwrap().into())); } /// Utility function that tests evaluation of date and time value with explicit offset. pub fn te_date_time_offset(trace: bool, scope: &FeelScope, s: &str, date: (Year, Month, Day), time: (u8, u8, u8, u64), offset: i32) { - textual_expression(trace, scope, s, Value::DateTime(FeelDateTime::offset(date, time, offset).unwrap())); + textual_expression(trace, scope, s, Value::DateTime(FeelDateTime::offset(date, time, offset).unwrap().into())); } /// Utility function that creates a scope from specified input. @@ -308,7 +313,7 @@ pub fn te_date_time_local_after(trace: bool, scope: &FeelScope, expression: &str panic!("expected date and time, actual type is {}", value.type_of()); }; assert!( - (actual > range_start && actual < range_end), + (*actual > range_start && *actual < range_end), "ERROR\nexpected: {range_start}..{range_end}\n actual: {actual}\n" ); } diff --git a/feel-parser/benches/list_literal.rs b/feel-parser/benches/list_literal.rs new file mode 100644 index 00000000..e065a638 --- /dev/null +++ b/feel-parser/benches/list_literal.rs @@ -0,0 +1,61 @@ +//! Parsing a list literal, at three sizes. +//! +//! The interesting figure is not any single time, which depends on the machine, but the +//! **ratio between the sizes**. Each size holds four times the items of the one before it. +//! Parsing that scales linearly answers in about four times the time. A parser that builds +//! the list by prepending each item into the front of a vector answers in about sixteen, +//! because every item moves all the items already collected. + +#![feature(test)] + +extern crate test; + +use dsntk_feel::{FeelScope, scope}; +use dsntk_feel_parser::parse_expression; +use test::Bencher; + +/// Returns a list literal holding `count` numbers, like `[0,1,2]`. +fn list_literal(count: usize) -> String { + let mut input = String::with_capacity(count * 5 + 2); + input.push('['); + for index in 0..count { + if index > 0 { + input.push(','); + } + input.push_str(&index.to_string()); + } + input.push(']'); + input +} + +#[bench] +fn feel_parser_list_literal_0001(b: &mut Bencher) { + let scope = scope!(); + let input = list_literal(100); + assert!(parse_expression(&scope, &input, false).is_ok()); + b.iter(|| parse_expression(&scope, &input, false)); +} + +#[bench] +fn feel_parser_list_literal_0002(b: &mut Bencher) { + let scope = scope!(); + let input = list_literal(400); + assert!(parse_expression(&scope, &input, false).is_ok()); + b.iter(|| parse_expression(&scope, &input, false)); +} + +#[bench] +fn feel_parser_list_literal_0003(b: &mut Bencher) { + let scope = scope!(); + let input = list_literal(1600); + assert!(parse_expression(&scope, &input, false).is_ok()); + b.iter(|| parse_expression(&scope, &input, false)); +} + +#[bench] +fn feel_parser_list_literal_0004(b: &mut Bencher) { + let scope = scope!(); + let input = list_literal(6400); + assert!(parse_expression(&scope, &input, false).is_ok()); + b.iter(|| parse_expression(&scope, &input, false)); +} diff --git a/feel-parser/src/lexer.rs b/feel-parser/src/lexer.rs index 57a0d69b..88d12d25 100644 --- a/feel-parser/src/lexer.rs +++ b/feel-parser/src/lexer.rs @@ -7,6 +7,7 @@ use dsntk_common::Result; use dsntk_feel::Name; use dsntk_feel::bif::is_built_in_date_time_function_name; use dsntk_feel::bip::is_built_in_property_name; +use std::str::FromStr; /// Definition of a single space character. const WS: char = ' '; @@ -749,6 +750,15 @@ impl<'lexer> Lexer<'lexer> { part_count -= 1; } + //------------------------------------------------------------------------------ + // No candidate resolved against the scope. Note where a reserved word would cut the + // name. Do not act on it yet. The checks below still have to see the whole name, + // because several built-in names contain a reserved word themselves. Examples: + // `date and time`, `years and months duration`, `day of week`, `index of`. + //------------------------------------------------------------------------------ + let reserved_cut = parts.iter().skip(1).position(|part| is_reserved_name_part(part)).map(|offset| offset + 1); + let uncut_parts = reserved_cut.map(|_| parts.clone()); + //------------------------------------------------------------------------------ // Build the name from name parts. //------------------------------------------------------------------------------ @@ -778,6 +788,38 @@ impl<'lexer> Lexer<'lexer> { return Ok((TokenType::NameDateTime, TokenValue::NameDateTime(name))); } + //------------------------------------------------------------------------------ + // Last resort. No candidate resolved against the scope, and the whole name is not a + // built-in of any kind. Stop the name at the first reserved word instead of absorbing + // an operator into it. + // + // A `FEEL` name may legally contain spaces and keywords, so for `c.noAddresses or true` + // the loop above sees `noAddresses or true`, `noAddresses or` and `noAddresses`. Taking + // the longest when none of them is known absorbs `or` into the member name. The member + // is then absent from the record, and the expression yields `null` instead of the + // boolean the author wrote. That is a wrong answer with no diagnostic, and it hides. + // When the field **is** present, the loop above resolves the short name, and the same + // expression is correct. So it only appears where a field is optional. + // + // This cannot change a name the scope knows, a context key, a named parameter, a `for` + // or quantified variable, a built-in type, a built-in date and time function, or any + // other built-in function. Every one of those returns before this point. + // + // `in` is one of the cut points. The `till_in` branch above only covers the variable + // position of a `for` or quantified expression. So it does nothing for the `in` of + // grammar rule 49.d, `expression in ( positive unary tests )`. Without the cut, + // `t.type in ("a","b")` reads the member as `type in`. The following `(` then makes + // that absent member the callee of a function invocation, and the comparison silently + // returns null. + //------------------------------------------------------------------------------ + if let (Some(part_count), Some(mut parts)) = (reserved_cut, uncut_parts) + && !is_built_in_function_name(name.as_str()) + { + parts.truncate(part_count); + self.position = consumed_positions[part_count - 1] + 1; + return Ok((TokenType::Name, TokenValue::Name(parts.into()))); + } + //------------------------------------------------------------------------------ // Return the name as it appears on input. //------------------------------------------------------------------------------ @@ -966,6 +1008,33 @@ fn is_digit(ch: char) -> bool { ch.is_ascii_digit() } +/// Returns `true` when a name part is a reserved word that must not be absorbed into a name. +/// +/// Enumerated rather than excluded, so a word is absorbed unless it is listed here. Every +/// listed word can legally follow a complete expression. That is exactly why absorbing +/// one into a name in front of it changes what the expression means. See the fallback in +/// [Lexer::consume_name]. +/// +/// The comparison is against a whole name part, not a substring, so `in` does not cut +/// `instance`: `instance` is one part and the two strings differ. `of` is deliberately absent +/// and must stay absent, because `day of week`, `day of year`, `week of year`, +/// `month of year` and `index of` carry `of` as a part of their own name. No built-in +/// function name, built-in property name or built-in type name has `in` as a part, so `in` +/// has the hazard shape of `and`, not of `of`. +fn is_reserved_name_part(part: &str) -> bool { + matches!(part, "and" | "or" | "between" | "in" | "instance" | "then" | "else" | "satisfies" | "return") +} + +/// Returns `true` when the whole name is the name of a built-in function. +/// +/// Built-in function names may contain a reserved word, such as `date and time`, `day of +/// week`, `index of` or `years and months duration`. A name that resolves here must +/// therefore never be cut at one. `Bif::from_str` is the same table the evaluator +/// resolves a bare name against. +fn is_built_in_function_name(name: &str) -> bool { + dsntk_feel::bif::Bif::from_str(name).is_ok() +} + /// Returns `true` when the specified character is a keyword `not` separator, /// that is a character that is allowed after keyword `not` at the beginning of unary tests. fn is_keyword_not_separator(ch: char) -> bool { diff --git a/feel-parser/src/parser.rs b/feel-parser/src/parser.rs index 74bbbad6..76adaf4b 100644 --- a/feel-parser/src/parser.rs +++ b/feel-parser/src/parser.rs @@ -819,7 +819,10 @@ impl ReduceActions for Parser<'_> { fn action_list(&mut self) -> Result<()> { trace_action!(self, "list"); - if let Some(AstNode::CommaList(items)) = self.yy_node_stack.pop() { + if let Some(AstNode::CommaList(mut items)) = self.yy_node_stack.pop() { + // `action_list_tail` collects the items from last to first. This restores + // the order with one reversal of the whole list. + items.reverse(); self.yy_node_stack.push(AstNode::List(items)); } Ok(()) @@ -836,7 +839,10 @@ impl ReduceActions for Parser<'_> { let node = self.yy_node_stack.pop().unwrap(); if let AstNode::CommaList(mut items) = node { let item = self.yy_node_stack.pop().unwrap(); - items.insert(0, item); + // The `list_tail` production is right recursive, so the items arrive in reverse + // order. This code appends them, which keeps the action constant time. + // `action_list` reverses the result once. + items.push(item); self.yy_node_stack.push(AstNode::CommaList(items)); return Ok(()); } diff --git a/feel-parser/src/tests/expr/various.rs b/feel-parser/src/tests/expr/various.rs index e29f9827..a60535c0 100644 --- a/feel-parser/src/tests/expr/various.rs +++ b/feel-parser/src/tests/expr/various.rs @@ -207,10 +207,80 @@ fn _0002() { ); } +/// This expression parses with an **empty** scope, and every `and` in it stays an operator. +/// +/// With no scope to resolve names against, the lexer takes the **longest** candidate name +/// unless a reserved word cuts it first. Without that cut, `Original Flight.From and To` +/// would become a single member called `From and To`, and this expression would fail to +/// parse. +/// +/// The same absorption produces a silent wrong answer elsewhere. `c.noAddresses or true` +/// would evaluate to `null` instead of `true`. That kills the count guard in two of the +/// twenty-four rules of the benchmark model. +/// +/// The two expectations are mutually exclusive. The absorption in this expression is itself +/// after a dot. So no narrowing of the cut can both fix `c.noAddresses or true` and keep +/// this expression unparseable. The conformance signal is the TCK, which stays at 3538 +/// passing, unchanged. +/// +/// **Correction.** An earlier version of this comment claimed a reserved word is not +/// permitted inside a `FEEL` name by grammar rule 25. That is wrong. Clause 10.3.1.6 gives +/// `Profit and loss` as an example of a name that contains a keyword. It resolves the +/// ambiguity by matching name tokens against **the names in-scope**, longest match +/// preferred. So a reserved word inside a name is legal, and the cut is not derived from +/// the grammar. It is a decision taken in the gap the specification leaves. No candidate +/// matched the scope, so there is no longest match to prefer. The pre-existing behavior of +/// taking the longest lexical run was equally an invention. The cut is the reading that +/// keeps the operator, and it can only reach a name the scope does not know. #[test] fn _0003() { let input = r#"Flights[ From = Original Flight.From and To = Original Flight.To and Departure > Original Flight.Departure and Status = "scheduled" and has capacity(item, Reassigned Passengers List)][1]"#; - // this test should fail without properly set scope let node = Parser::new(&scope!(), StartExpression, input, false).parse(); - assert!(node.is_err()); + let tree = node.expect("parses without a scope: `and` is an operator, not part of a name").trace(); + // The member names stop at the reserved word. + assert!(tree.contains("`From`"), "expected a member named `From`:\n{tree}"); + assert!(tree.contains("`To`"), "expected a member named `To`:\n{tree}"); + assert!(tree.contains("`Departure`"), "expected a member named `Departure`:\n{tree}"); + // and the absorbed forms are gone. + assert!(!tree.contains("From and To"), "`and` was absorbed into a name:\n{tree}"); + assert!(!tree.contains("Departure and Status"), "`and` was absorbed into a name:\n{tree}"); + // Each `and` survives as an operator: four of them join the five comparisons. + assert_eq!(4, tree.matches("And").count(), "expected four `and` operators:\n{tree}"); +} + +/// A path member is followed by `in`, and the `in` stays the comparison operator of grammar +/// rule 49.d, `expression in ( positive unary tests )`. +/// +/// The scope knows `t` but knows nothing about its members, which is what a `typeRef="Any"` +/// parameter gives the parser. Without the cut in [crate::lexer::Lexer::consume_name], the +/// member name would absorb `in` and become `type in`. The following `(` would then read +/// that absent member as the callee of a function invocation. So `t.type in ("a","b")` +/// would answer `null` instead of a boolean, and `count(X[type in ("a","b")])` would +/// answer 0. +/// +/// The `till_in` branch above the fallback does not cover this. It only trims the **variable** +/// name of a `for` or quantified expression, so it never sees an `in` used as a comparison. +#[test] +fn _0004() { + let scope = scope!(); + scope.set_context("t".into(), ParsingContext::default()); + accept( + &scope, + StartExpression, + r#"t.type in ("a","b")"#, + r#" + In + ├─ Path + │ ├─ Name + │ │ └─ `t` + │ └─ Name + │ └─ `type` + └─ ExpressionList + ├─ String + │ └─ `a` + └─ String + └─ `b` + "#, + false, + ); } diff --git a/feel-regex/src/impl_regex.rs b/feel-regex/src/impl_regex.rs index 7d7b32a6..1fc4c57e 100644 --- a/feel-regex/src/impl_regex.rs +++ b/feel-regex/src/impl_regex.rs @@ -2,12 +2,68 @@ use crate::errors::{err_empty_flags, err_invalid_flag, err_invalid_pattern}; use crate::unicode_blocks::replace_block_names; use crate::utils::is_special_character; use regex::Regex; +use std::borrow::Cow; +use std::cell::RefCell; +use std::collections::HashMap; + +/// Maximum number of compiled patterns kept in one thread's cache. +/// +/// Patterns come from a decision model, so their number is bounded by the size of the +/// model. The limit only protects against a model that builds patterns from input data. +/// When the count reaches the limit, this code drops the whole cache. That costs one +/// recompilation per live pattern, but keeps the cache bounded. +const MAX_CACHED_PATTERNS: usize = 256; + +thread_local! { + /// Cache of compiled regular expressions, keyed by the pattern text after all + /// pattern fixes were applied. **One cache per thread**, and this is the point. + /// + /// A process-wide cache behind an `RwLock`, read-locked once per call, has the same + /// mistake `feel/src/names.rs` documents for its own text table. Taking a read lock + /// still *writes* the lock word, so every reader steals that cache line from every + /// other core. Measured on a 200,000-element parallel `for`, a `matches` body ran at + /// 1.00x the serial time. The same body with `contains` ran at 1.24x. The whole + /// parallel gain was eaten by one contended line. + /// + /// A thread-local cache touches no shared state on the hit path. It also keeps each + /// compiled expression on the thread that made it. That matters because a `Regex` + /// shared across threads serializes again on its internal scratch pool. The cost is + /// one compilation of each pattern per worker thread instead of per process, a + /// bounded one-time price that the first few hundred calls repay. + static CACHE: RefCell> = RefCell::new(HashMap::new()); +} + +/// Compiles `pattern` if needed, then calls `f` with the compiled regular expression. +/// +/// Compiling a regular expression is expensive, and a `matches` call inside a `for` +/// expression compiles the same constant pattern once per element. Measured on a model +/// with a single `matches` call over 50,000 records, `Regex::new` was 25% of the total +/// run time. The result is the same either way, because compiling a pattern is +/// deterministic and the compiled expression is immutable. +/// +/// `f` must not call back into this function. Both callers pass a plain match closure. +/// +/// Returns `None` when the pattern does not compile. +fn with_compiled(pattern: String, f: impl FnOnce(&Regex) -> T) -> Option { + CACHE.with(|cell| { + let mut cache = cell.borrow_mut(); + if !cache.contains_key(&pattern) { + let re = Regex::new(&pattern).ok()?; + if cache.len() >= MAX_CACHED_PATTERNS { + cache.clear(); + } + cache.insert(pattern.clone(), re); + } + Some(f(&cache[&pattern])) + }) +} pub fn is_match(input: &str, pattern: &str) -> dsntk_common::Result { - let Ok(re) = Regex::new(&fix_pattern(pattern, false, false)) else { - return Err(err_invalid_pattern(pattern)); - }; - Ok(re.is_match(&fix_input(input))) + let input = fix_input(input); + match with_compiled(fix_pattern(pattern, false, false), |re| re.is_match(&input)) { + Some(result) => Ok(result), + None => Err(err_invalid_pattern(pattern)), + } } pub fn is_match_with_flags(input: &str, pattern: &str, flags: &str) -> dsntk_common::Result { @@ -33,15 +89,19 @@ pub fn is_match_with_flags(input: &str, pattern: &str, flags: &str) -> dsntk_com regex_flags = format!("(?{})", regex_flags); } let pattern = format!("{}{}", regex_flags, fix_pattern(pattern, whitespaces, themselves)); - let Ok(re) = Regex::new(&pattern) else { - return Err(err_invalid_pattern(&pattern)); - }; - Ok(re.is_match(&fix_input(input))) + let input = fix_input(input); + match with_compiled(pattern.clone(), |re| re.is_match(&input)) { + Some(result) => Ok(result), + None => Err(err_invalid_pattern(&pattern)), + } } // Nasty tricks on input. -fn fix_input(s: &str) -> String { - s.replace("\r\n", "\n").replace('\r', "\n") +// +// Almost no input contains a carriage return. This runs once per `matches` call, so +// this code borrows the unchanged input instead of copying it. +fn fix_input(s: &str) -> Cow<'_, str> { + if s.contains('\r') { Cow::Owned(s.replace("\r\n", "\n").replace('\r', "\n")) } else { Cow::Borrowed(s) } } // Nasty tricks on pattern. diff --git a/feel/src/context.rs b/feel/src/context.rs index 849ed853..62cd3fdc 100644 --- a/feel/src/context.rs +++ b/feel/src/context.rs @@ -11,13 +11,36 @@ use std::collections::BTreeMap; use std::collections::btree_map::Iter; use std::fmt; use std::ops::Deref; +use std::sync::{Arc, LazyLock}; /// Type alias for context entries. type FeelContextEntries = BTreeMap; /// The FEEL context. -#[derive(Debug, Clone, PartialEq, Default)] -pub struct FeelContext(FeelContextEntries); +/// +/// The entries live behind an [Arc] so that cloning a context is a refcount bump instead +/// of a deep copy of the whole map. Contexts are cloned once per element in several hot +/// paths, most importantly [crate::values::Value::Context] values held in a list. +/// +/// Mutation goes through [Arc::make_mut], which copies the map only when it is actually +/// shared. Every mutating method below uses `entries_mut` for exactly that reason. A +/// `&mut FeelContextEntries` is unreachable through an [Arc] without it. The compiler +/// therefore rejects any mutation site that forgets to go copy-on-write. +#[derive(Debug, Clone, PartialEq)] +pub struct FeelContext(Arc); + +/// A shared empty entry map, so `FeelContext::default()` allocates nothing. +/// +/// `FeelContext::default()` runs once per evaluated context literal and once per pushed +/// scope context, so it is hot. The first mutation of such a context copies an empty map, +/// which allocates no nodes. +static EMPTY_ENTRIES: LazyLock> = LazyLock::new(|| Arc::new(FeelContextEntries::new())); + +impl Default for FeelContext { + fn default() -> Self { + Self(EMPTY_ENTRIES.clone()) + } +} impl Deref for FeelContext { type Target = FeelContextEntries; @@ -104,6 +127,33 @@ impl FeelContext { Self::default() } + /// Creates a context holding exactly one entry. + /// + /// Use this instead of [FeelContext::default] followed by [FeelContext::set_entry]. + /// Those two build the same context, but reach it through [Arc::make_mut] on the shared + /// [EMPTY_ENTRIES] static. That costs three atomic operations on **one** cache line + /// that every thread in the process touches: a refcount increment to clone the static, a + /// compare-exchange inside [Arc::make_mut] because the refcount is never one, and a + /// decrement to release it again. This builds the map first and shares it afterwards, so + /// nothing is contended. + /// + /// A profile of the largest benchmark model put `entries_mut` at 8.99% of main-thread + /// work. 97.5% of that came from one call site: the filter evaluator, which builds a + /// one-entry context for every element of every filtered list. + pub fn from_entry(name: &Name, value: Value) -> Self { + let mut entries = FeelContextEntries::new(); + entries.insert(name.clone(), value); + Self(Arc::new(entries)) + } + + /// Returns a mutable reference to the entries, copying them first when they are shared. + /// + /// This is the only way to mutate a context. [Arc::make_mut] copies the map only + /// when the refcount is above one. It mutates an unshared context in place. + fn entries_mut(&mut self) -> &mut FeelContextEntries { + Arc::make_mut(&mut self.0) + } + /// Returns `true` if context contains an entry specified by **name**. pub fn contains_entry(&self, name: &Name) -> bool { self.0.contains_key(name) @@ -121,17 +171,17 @@ impl FeelContext { /// Sets a value for specified entry name. pub fn set_entry(&mut self, name: &Name, value: Value) { - self.0.insert(name.clone(), value); + self.entries_mut().insert(name.clone(), value); } /// Removes a value of an entry with specified name. pub fn remove_entry(&mut self, name: &Name) -> Option { - self.0.remove(name) + self.entries_mut().remove(name) } /// Sets a null value for specified entry. pub fn set_null(&mut self, name: Name) { - self.0.insert(name, value_null!()); + self.entries_mut().insert(name, value_null!()); } /// Returns a list of all [FeelContext] entries. @@ -165,22 +215,24 @@ impl FeelContext { } pub fn zip(&mut self, other: &FeelContext) { - for (name, value) in &other.0 { - self.0.insert(name.clone(), value.clone()); + let entries = self.entries_mut(); + for (name, value) in other.0.iter() { + entries.insert(name.clone(), value.clone()); } } pub fn overwrite(&mut self, other: &FeelContext) { - for (name, value) in &other.0 { - if self.0.contains_key(name) { - self.0.insert(name.clone(), value.clone()); + let entries = self.entries_mut(); + for (name, value) in other.0.iter() { + if entries.contains_key(name) { + entries.insert(name.clone(), value.clone()); } } } //TODO refactor the name, this operation is not moving, it is like prefixing pub fn move_entry(&mut self, name: Name, parent: Name) { - if let Some(value) = self.0.remove(&name) { + if let Some(value) = self.entries_mut().remove(&name) { self.create_entries(&[parent, name], value); } } @@ -223,18 +275,18 @@ impl FeelContext { let tail = &names[1..]; // if tail is empty, then insert the value under the key in current context and return if tail.is_empty() { - self.0.insert(key, value); + self.entries_mut().insert(key, value); return; } // if there is a context under the key, then insert value to this context and return - if let Some(Value::Context(ctx)) = self.0.get_mut(&key) { + if let Some(Value::Context(ctx)) = self.entries_mut().get_mut(&key) { ctx.create_entries(tail, value); return; } // insert a value from tail to newly created context let mut ctx = FeelContext::default(); ctx.create_entries(tail, value); - self.0.insert(key, ctx.into()); + self.entries_mut().insert(key, ctx.into()); } /// ??? @@ -247,11 +299,11 @@ impl FeelContext { let tail = &names[1..]; // if tail is empty, then insert the value under the key in current context and return if tail.is_empty() { - self.0.insert(key, value); + self.entries_mut().insert(key, value); return Ok(()); } // if there is a context under the key, then insert value to this context and return - match self.0.get_mut(&key) { + match self.entries_mut().get_mut(&key) { Some(Value::Context(ctx)) => { ctx.apply_entries(tail, value)?; return Ok(()); @@ -264,7 +316,7 @@ impl FeelContext { // insert a value from tail to new created context let mut ctx = FeelContext::default(); ctx.apply_entries(tail, value)?; - self.0.insert(key, ctx.into()); + self.entries_mut().insert(key, ctx.into()); Ok(()) } diff --git a/feel/src/names.rs b/feel/src/names.rs index 8efac0cd..d92f62eb 100644 --- a/feel/src/names.rs +++ b/feel/src/names.rs @@ -1,11 +1,392 @@ //! `FEEL` name implementation. +//! +//! A [Name] is an interned handle, not a string. This module stores every distinct name +//! text once in a process-wide table, so every [Name] that spells that text is the same +//! 8-byte value. Three properties of the hot paths motivate this: +//! +//! 1. [crate::context::FeelContext] keys a [std::collections::BTreeMap] by [Name]. A +//! lookup runs a binary search, so it compares keys `log2(width)` times. With a string +//! key, each comparison is a `memcmp` over the key text. Real key sets share long +//! prefixes (`orderDate`, `orderDetail`, `order_id`), so the compare runs deep before +//! it decides. Comparing handles decides in one integer compare. +//! 2. Cloning a name was a heap allocation. Every context entry insert cloned its key, so +//! a run over half a million records allocated the same key text millions of times. +//! Cloning a handle copies 8 bytes. +//! 3. The key array of a `BTreeMap` node shrinks from 24 bytes per key to 8. +//! +//! # Ordering is still lexicographic +//! +//! This is the part that needed care. [FeelContext](crate::context::FeelContext) exposes +//! `Deref>` and `iter()`. `jsonify` walks the entries in +//! key order, so the order of a context's entries is *visible in program output*. If [Ord] +//! became "order in which names were interned", output key order would change. +//! +//! So a handle carries a **rank** as well as an identity. This module assigns ranks so +//! that rank order equals lexicographic order of the name text, and a rank once assigned +//! never changes. A new name takes a rank strictly between the ranks of its two +//! lexicographic neighbours. +//! +//! The rank lives in the high bits of the handle, so [Ord] is a single unsigned compare of +//! the whole handle. Nothing outside this module observes any change in ordering. +//! +//! # Why ranks can never be renumbered +//! +//! Renumbering looks safe, because renumbering that preserves lexicographic order preserves +//! the order relation, and a `BTreeMap` only needs its keys to stay sorted. It is not safe. +//! **The rank lives inside the handle, and handles have already been copied into live +//! `BTreeMap`s** that are still being searched. Renumbering would rewrite the interner and +//! leave every stored key holding a stale rank. A lookup with a fresh handle would then +//! compare against stale ones and walk to the wrong node. Any scheme that wants to +//! renumber has to take the rank *out* of the handle first. That costs a load per +//! comparison on the hottest path in the engine, and it would need measurement before +//! anyone trusts it. +//! +//! # Running out of rank space +//! +//! Because this module assigns ranks without ever renumbering, a gap between two +//! neighbours can be used up. When that happens the new name is left **unranked** and +//! compares by text. That is slower for that name and stays correct, because both +//! comparison paths agree with lexicographic order exactly. +//! +//! The bound matters, because ascending numbered identifiers are the ordinary case in a +//! generated model, not a corner case, and a narrow stride lets a run of them exhaust a +//! gap fast. Rules named `Kb01`, `Kb02`, ... `Kb24` are one example. The current stride +//! keeps every name in all four benchmark models ranked. +//! +//! Two things keep names ranked in practice. The gap left by an append is far wider than +//! a single halving would leave. A name that continues a **family** consumes a fixed +//! [RANK_STEP] rather than half of what is left, so a run of numbered names costs one +//! step per name. See [Interner::rank_for] for how a family is recognised and why an +//! unrelated name is still halved into the middle. +//! +//! # Limitations, stated plainly +//! +//! - The table never shrinks. This module leaks name text deliberately, which is what +//! makes `as_str` able to hand out `&'static str` and what makes a handle copyable. Name +//! text comes from identifiers and context keys. The `context` built-in also interns +//! every key that arrives as **runtime data**. So a model that builds a context from +//! input strings grows the table with its input. A process that interns unboundedly +//! many *distinct* names grows without bound. +//! - Only the first `2^20 - 1` distinct names carry a rank beside their identity. A name +//! interned past that takes a **wide handle**: [WIDE_BIT] set, the rest of the handle +//! is identity. A wide handle carries no rank, so it compares by text, exactly like a +//! name whose rank gap was exhausted. Both paths agree with lexicographic order, so no +//! answer changes. Lookups on wide names are slower, and that is all. The identity +//! space of a wide handle is 63 bits, which memory exhausts long before the table does. +//! The four models in this repository intern 50 to 175 names. +//! - The rank space is 43 bits, and it bounds three separate things. At least **65,536** +//! names may be appended past the largest name so far, or prepended below the smallest. At +//! least **4,096** names of one family fit in one appended gap. At least **15** more names +//! may halve a gap that has become too narrow to step through. +//! - **The residual cliff is the third of those.** Names that repeatedly land between two +//! ever-closer neighbours halve the gap each time. No fixed rank space can avoid that +//! without renumbering, which the section above shows is unsound. Measured across the +//! four models, the narrowest gap any name was inserted into is 2,048, which leaves 11 +//! halvings in hand. +//! - Past any of those three bounds a name is left unranked and compares by text. It is still +//! ordered exactly lexicographically, so no output changes and no answer changes — only +//! that name's lookups get slower. [Ord] mixes the two paths and stays a total order, +//! because both paths agree with lexicographic order. use dsntk_common::Jsonify; +use std::cmp::Ordering; +use std::collections::{BTreeMap, HashMap}; use std::fmt; +use std::hash::{BuildHasherDefault, Hasher}; +use std::sync::{LazyLock, OnceLock, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +/// Number of low bits of a handle that hold the identity of a **ranked** name. +/// +/// Twenty bits allow 1,048,575 ranked names beside 43 bits of rank space. Names interned +/// past that take a [WIDE_BIT] handle and compare by text. The models in this repository +/// intern between 50 and 175 names. So only a model that interns runtime data — the +/// `context` built-in does — ever reaches the wide range. +const ID_BITS: u32 = 20; + +/// Mask selecting the identity of a name. +const ID_MASK: u64 = (1 << ID_BITS) - 1; + +/// Mask selecting the rank of a name. +const RANK_MASK: u64 = !ID_MASK; + +/// Bit marking a **wide** handle, one whose identity is too large to fit beside a rank. +/// +/// The first `2^20 - 1` names encode as `rank << ID_BITS | id`. Every later name encodes +/// as `WIDE_BIT | id`, carries no rank and compares by text. The three encodings cannot +/// collide. A ranked handle has this bit clear, because [RANK_MAX] stops one bit short +/// of it. An unranked narrow handle has every bit above [ID_BITS] clear. A wide handle +/// is the only one with this bit set. +const WIDE_BIT: u64 = 1 << 63; + +/// Largest assignable rank. One bit short of [WIDE_BIT], so a ranked handle never sets it. +const RANK_MAX: u64 = (1 << (63 - ID_BITS)) - 1; + +/// Room left between a newly appended name and its predecessor, so that later insertions +/// between the two have somewhere to go. +/// +/// `RANK_MAX / 2 / RANK_STRIDE` is 65,536, which is how many names may be appended past the +/// largest name so far before the append room itself runs out. +const RANK_STRIDE: u64 = 1 << 26; + +/// Room taken by a name that continues a family of names, while the gap can afford it. +/// +/// `RANK_STRIDE / RANK_STEP` is 4,096, which is how many names of one family fit in a freshly +/// appended gap. `log2(2 * RANK_STEP)` is 15, which is how many halvings are left once a gap is +/// too narrow to step through. +/// +/// Both bounds move in opposite directions with this constant. It was chosen by sweeping +/// it against the four models and reading off the narrowest gap any name landed in: 2^12 gave +/// 512, **2^14 gave 2,048**, 2^16 gave 2, and 2^18 and 2^20 left names unranked again. The +/// response is not monotone, because the constant changes which names become neighbours. +const RANK_STEP: u64 = 1 << 14; + +/// Shortest shared prefix that counts as two names belonging to the same family. +/// +/// One shared character is a coincidence between unrelated identifiers. Two is a stem. +const STEM_LEN: usize = 2; + +/// Rank value meaning "this name has no rank", so it compares by text. +const RANK_NONE: u64 = 0; /// `FEEL` name. -#[derive(Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Clone)] -pub struct Name(String); +/// +/// Cheap to clone, cheap to compare and ordered exactly as its text is ordered. See the +/// module documentation for how that is arranged. +#[derive(Eq, Hash, PartialEq, Clone)] +pub struct Name(u64); + +impl Default for Name { + /// The default name is the empty name. + /// + /// Written out rather than derived: a derived default would be handle zero, which is the + /// handle of whichever name was interned first, not the handle of the empty name. + fn default() -> Self { + EMPTY.clone() + } +} + +/// The process-wide name table. +struct Interner { + /// Maps name text to the handle interned for it. + handles: HashMap<&'static str, u64, BuildHasherDefault>, + /// Maps the text of every **ranked** name to its rank, in lexicographic order. + /// + /// Only ranked names appear here, so the neighbours of a new name are always ranked. + /// The rank assigned between them is therefore always valid. Unranked names take no + /// part in rank comparison, so leaving them out changes no answer. + ranked: BTreeMap<&'static str, u64>, + /// Maps the identity of a name to its text. + texts: Vec<&'static str>, +} + +static INTERNER: RwLock = RwLock::new(Interner { + handles: HashMap::with_hasher(BuildHasherDefault::new()), + ranked: BTreeMap::new(), + texts: Vec::new(), +}); + +/// The empty name, so [Name::is_empty] needs no table lookup. +static EMPTY: LazyLock = LazyLock::new(|| intern("")); + +fn read_interner() -> RwLockReadGuard<'static, Interner> { + INTERNER.read().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +fn write_interner() -> RwLockWriteGuard<'static, Interner> { + INTERNER.write().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Returns the handle for `text`, interning it when it is new. +fn intern(text: &str) -> Name { + if let Some(&handle) = read_interner().handles.get(text) { + return Name(handle); + } + intern_new(text) +} + +/// Adds a name that the read above did not find. +/// +/// Kept out of [intern] so that the hot path stays small enough to inline. +#[cold] +fn intern_new(text: &str) -> Name { + let mut interner = write_interner(); + // Another thread may have interned this text between the read above and this write. + if let Some(&handle) = interner.handles.get(text) { + return Name(handle); + } + let id = interner.texts.len() as u64; + // Leaked on purpose: it is what lets a handle be copied and `as_str` return a `&'static + // str`. One leak per distinct name, never per occurrence. + let text: &'static str = Box::leak(text.to_owned().into_boxed_str()); + let handle = if id < ID_MASK { + let rank = interner.rank_for(text); + if rank != RANK_NONE { + interner.ranked.insert(text, rank); + } + (rank << ID_BITS) | id + } else { + // The identity no longer fits beside a rank. The name takes a wide handle instead: + // unranked, compared by text, identity in the low 63 bits. Interning stays injective + // and total order is kept, so nothing changes except the speed of this name. + WIDE_BIT | id + }; + interner.texts.push(text); + interner.handles.insert(text, handle); + // Published before the write lock is released, so a thread that later holds this handle + // always finds the slot filled. + if let Some(slot) = TEXT_SLOTS.get(id as usize) { + let _ = slot.set(text); + } + Name(handle) +} + +/// Number of names whose text is readable without taking the lock. +const RESOLVE_SLOTS: usize = 1 << 16; + +/// Lock-free text lookup, one slot per name identity, filled as names are interned. +/// +/// [Name::as_str] is on the evaluation hot path and runs on every worker thread at once. +/// Reading the text through the [RwLock] instead measured **26 times slower than not +/// interning at all**, at n=500,000 on twelve threads. The same binary held its gain on +/// one thread. Taking a read lock still *writes* the lock word, so every reader stole +/// that cache line from every other core. A [OnceLock] is written once and then only +/// loaded, so the readers share the line rather than fight over it. +/// +/// Zero-initialized, so the pages of slots no name reaches are never faulted in. +static TEXT_SLOTS: [OnceLock<&'static str>; RESOLVE_SLOTS] = [const { OnceLock::new() }; RESOLVE_SLOTS]; + +/// Returns the text of the name with the given identity. +#[inline] +fn text_of(id: u64) -> &'static str { + if let Some(slot) = TEXT_SLOTS.get(id as usize) + && let Some(&text) = slot.get() + { + return text; + } + // Past the end of the slot array. Rare enough to pay for the lock. + read_interner().texts[id as usize] +} + +/// Number of leading bytes two strings share. +/// +/// This function only guesses which end of a gap a run of insertions is walking away +/// from. Its answer affects how much rank space is left, and never affects an ordering. +fn shared_prefix(left: &str, right: &str) -> usize { + left.as_bytes().iter().zip(right.as_bytes()).take_while(|(l, r)| l == r).count() +} + +impl Interner { + /// Picks a rank for `text` that sits strictly between the ranks of its lexicographic + /// neighbours, or [RANK_NONE] when no such rank is left. + fn rank_for(&self, text: &str) -> u64 { + let lower = self.ranked.range(..text).next_back().map(|(&neighbour, &rank)| (neighbour, rank)); + let upper = self.ranked.range(text..).next().map(|(&neighbour, &rank)| (neighbour, rank)); + let rank = match (lower, upper) { + // The first ranked name starts in the middle, so both directions have room. + (None, None) => RANK_MAX / 2, + // Appended after every existing name. + (Some((_, lower)), None) => { + let room = RANK_MAX - lower; + if room > RANK_STRIDE { lower + RANK_STRIDE } else { lower + room / 2 } + } + // Inserted before every existing name. + (None, Some((_, upper))) => { + if upper > RANK_STRIDE { upper - RANK_STRIDE } else { upper / 2 } + } + // Between two existing names. Two cases, and telling them apart is the whole point. + // + // A name that continues a **family** — one that shares a stem with the neighbour it + // follows, like `Kb02` after `Kb01` — belongs to a run. A run must not halve the gap + // per name, or it uses the gap up in `log2` names. It takes a fixed [RANK_STEP] away + // from its family neighbour instead, so a run of `room / RANK_STEP` names fits. It + // walks up from `lower` or down from `upper`, as the family lies. + // + // A name unrelated to both neighbours is halved into the middle, because that keeps + // both halves of the gap roomy for whatever arrives later. + // + // A shared stem is at least two leading characters. One shared character is a + // coincidence between unrelated identifiers, such as `Kb01` sharing `K` with + // `Kr Adjacent Name`. + // + // Every branch returns a rank strictly inside the gap, so misjudging a family costs + // later capacity and never an ordering. + (Some((lower_text, lower)), Some((upper_text, upper))) => { + let room = upper - lower; + let with_lower = shared_prefix(text, lower_text); + let with_upper = shared_prefix(text, upper_text); + if room <= 2 * RANK_STEP { + lower + room / 2 + } else if with_lower >= STEM_LEN && with_lower > with_upper { + lower + RANK_STEP + } else if with_upper >= STEM_LEN && with_upper > with_lower { + upper - RANK_STEP + } else { + lower + room / 2 + } + } + }; + // A rank equal to either neighbour would order two different names the same way, and a + // rank of zero is the "unranked" marker. Either way the gap is exhausted. + if rank == RANK_NONE || Some(rank) == lower.map(|(_, rank)| rank) || Some(rank) == upper.map(|(_, rank)| rank) { + RANK_NONE + } else { + rank + } + } +} + +/// FNV-1a over the name text, finished with a bit mixer. +/// +/// The default hasher spends more time on a short key than the whole lookup needs. The +/// mixer matters because the map takes the low bits of the hash for the bucket index and +/// FNV alone leaves those weak. +#[derive(Default)] +struct NameHasher(u64); + +impl Hasher for NameHasher { + fn write(&mut self, bytes: &[u8]) { + let mut hash = if self.0 == 0 { 0xcbf2_9ce4_8422_2325 } else { self.0 }; + for &byte in bytes { + hash ^= byte as u64; + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + self.0 = hash; + } + + fn finish(&self) -> u64 { + let mut hash = self.0; + hash = (hash ^ (hash >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + hash = (hash ^ (hash >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + hash ^ (hash >> 31) + } +} + +impl Ord for Name { + /// Orders two names exactly as their texts are ordered. + /// + /// Ranked names compare in one unsigned compare, because the rank is the high part of the + /// handle and rank order is lexicographic order. A name left unranked by an exhausted + /// rank gap compares by text, and so does a wide handle, whose high bits are identity + /// rather than rank. Both paths agree with lexicographic order, so mixing them still + /// gives a total order. + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + if (self.0 | other.0) & WIDE_BIT == 0 && (self.0 & RANK_MASK).min(other.0 & RANK_MASK) != RANK_NONE { + return self.0.cmp(&other.0); + } + if self.0 == other.0 { + return Ordering::Equal; + } + self.as_str().cmp(other.as_str()) + } +} + +impl PartialOrd for Name { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} impl From> for Name { /// Converts a vector of strings into [Name]. @@ -24,55 +405,84 @@ impl From> for Name { impl From for Name { /// Converts a [String] into [Name]. fn from(value: String) -> Self { - Self(value.trim().to_string()) + intern(value.trim()) } } impl From<&String> for Name { /// Converts a reference to [String] into [Name]. fn from(value: &String) -> Self { - Self(value.trim().to_string()) + intern(value.trim()) } } impl From<&str> for Name { /// Converts a reference to [str] into [Name]. fn from(value: &str) -> Self { - Self(value.trim().to_string()) + intern(value.trim()) } } impl From for String { /// Converts [Name] to its [String] representation. fn from(value: Name) -> Self { - value.0 + value.as_str().to_string() } } impl From<&Name> for String { /// Converts a reference to [Name] to its [String] representation. fn from(value: &Name) -> Self { - value.0.clone() + value.as_str().to_string() + } +} + +impl fmt::Debug for Name { + /// Implements [Debug](fmt::Debug) trait for [Name]. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Name({:?})", self.as_str()) } } impl fmt::Display for Name { /// Implements [Display](fmt::Display) trait for [Name]. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) + f.pad(self.as_str()) } } impl Jsonify for Name { /// Converts [Name] to its `JSON` representation. fn jsonify(&self) -> String { - self.0.clone() + self.as_str().to_string() } } impl Name { + /// Returns the [Name] for `text` only when it already exists, and never creates one. + /// + /// For looking a name up rather than making one. A lookup that interns leaks a name on + /// every probe that misses, and the interner never gives one back. So a process probing + /// unbounded text grows without bound, which is exactly the risk in `get value`. + /// + /// `None` is safe to read as "no context anywhere holds this key". The tuple field of + /// [Name] is private to this module, and every constructor goes through [intern] or + /// [intern_new]. So a [Name] cannot exist without sitting in `handles`. Text absent from + /// `handles` therefore names no key in any [crate::FeelContext]. A caller that interned + /// it would have got a fresh handle that is by construction in no map. Both branches + /// return what the interning version returned, by construction rather than by test. + pub fn existing(text: &str) -> Option { + read_interner().handles.get(text.trim()).map(|&handle| Name(handle)) + } + /// Creates a [Name] from string parts. pub fn new(parts: &[&str]) -> Self { + // A name of a single part is the common case, a context key or a variable reference of + // one token. Interning the part directly saves building a temporary string for every + // occurrence of every key in the input. + if let [part] = parts { + return intern(part.trim()); + } let mut result = String::new(); let mut current; let mut prev = false; @@ -84,16 +494,17 @@ impl Name { result.push_str(part); prev = current; } - Self(result) + intern(&result) } /// Extracts a string slice containing the entire [Name]. pub fn as_str(&self) -> &str { - &self.0 + let id = if self.0 & WIDE_BIT != 0 { self.0 & !WIDE_BIT } else { self.0 & ID_MASK }; + text_of(id) } /// Returns `true` when this name is empty. pub fn is_empty(&self) -> bool { - self.0.is_empty() + *self == *EMPTY } } diff --git a/feel/src/scope.rs b/feel/src/scope.rs index 994ebaf4..61d54abf 100644 --- a/feel/src/scope.rs +++ b/feel/src/scope.rs @@ -65,6 +65,16 @@ impl FeelScope { self.stack.borrow_mut().push(ctx) } + /// Returns the number of contexts on the scope stack. + pub fn len(&self) -> usize { + self.stack.borrow().len() + } + + /// Returns `true` when there are no contexts on the scope stack. + pub fn is_empty(&self) -> bool { + self.stack.borrow().is_empty() + } + /// Appends the content of another scope at the end of this scope. pub fn append(&self, other: FeelScope) { self.stack.borrow_mut().append(&mut other.stack.borrow_mut()); @@ -81,6 +91,28 @@ impl FeelScope { self.stack.borrow().last().cloned() } + /// Builds a scope from a stack of contexts. + /// + /// Used to give each worker of a parallel `for` its own stack. The contexts are shared, + /// not copied: [crate::context::FeelContext] keeps its entries behind an `Arc`, so + /// cloning the stack costs one refcount bump per level. Pushing or popping on the copy + /// does not affect the original. Mutating a context through it copies that context + /// first, because every `FeelContext` mutation is copy-on-write. + /// + /// A [FeelScope] holds a `RefCell` and so is not `Sync`, but a `Vec` is both + /// `Send` and `Sync`. So the stack is what crosses a thread boundary, and each worker + /// builds its own scope around a clone of it. + pub fn from_contexts(contexts: Vec) -> Self { + Self { stack: RefCell::new(contexts) } + } + + /// Returns `true` when any context on the stack binds `name`. + /// + /// Unlike [FeelScope::get_value] this clones nothing, so it is usable as a guard. + pub fn contains_name(&self, name: &Name) -> bool { + self.stack.borrow().iter().any(|context| context.contains_entry(name)) + } + /// Returns a value of an entry with specified name. /// Entries are searched from the last to the first context, /// (from top to bottom of the stack). @@ -93,6 +125,25 @@ impl FeelScope { None } + /// Returns a clone of the entry `field` taken from the context bound to `name`. + /// + /// This resolves the binding of `name` exactly like [FeelScope::get_value] resolves + /// it. The search stops at the first context that binds `name`, so an inner binding + /// always shadows an outer one. This returns `None` when `name` is not bound, when + /// it is not bound to a context, or when that context has no `field` entry. In all + /// three cases the caller must fall back to evaluating the base value. + /// + /// This clones only the requested entry. Evaluating the base value instead clones + /// the whole context, which costs about 2 KB per field access on a typical record. + pub fn get_context_entry(&self, name: &Name, field: &Name) -> Option { + for context in self.stack.borrow().iter().rev() { + if let Some(value) = context.get_entry(name) { + return if let Value::Context(ctx) = value { ctx.get_entry(field).cloned() } else { None }; + } + } + None + } + /// Searches for a value under so called `qualified` name build from /// multiple names passed as an argument. pub fn search(&self, names: &[Name]) -> Option { diff --git a/feel/src/tests/names.rs b/feel/src/tests/names.rs index a20f7d60..a62bcb4d 100644 --- a/feel/src/tests/names.rs +++ b/feel/src/tests/names.rs @@ -185,3 +185,80 @@ fn test_compare() { assert!((name_xy > name_mn)); assert!((name_xy >= name_a)); } + +/// Orders a scrambled set of names through [Name] and through their texts, and requires the +/// two orders to agree. +/// +/// This is the property the rank scheme exists to preserve, and the one that would break +/// silently: `FeelContext` derefs to `BTreeMap` and `jsonify` walks it in key +/// order, so an ordering error shows up as changed output rather than as a panic. Every case +/// below feeds the pattern that used to exhaust a rank gap — long runs of ascending numbered +/// identifiers — and then checks the order anyway, so it passes whether the names end up +/// ranked or unranked. +fn assert_orders_like_text(texts: &[String]) { + let mut by_name: Vec<(Name, &str)> = texts.iter().map(|text| (Name::from(text.as_str()), text.as_str())).collect(); + by_name.sort_by(|(left, _), (right, _)| left.cmp(right)); + let mut by_text: Vec<&str> = texts.iter().map(|text| text.as_str()).collect(); + by_text.sort_unstable(); + let ordered_by_name: Vec<&str> = by_name.iter().map(|(_, text)| *text).collect(); + assert_eq!(by_text, ordered_by_name); +} + +#[test] +fn test_order_survives_an_ascending_run() { + // `Kb01` .. `Kb24` between `Kb` and a name just above it is exactly what the benchmark + // model does, and what used to leave `Kb15` onwards comparing by text. + let mut texts = vec!["Ja".to_string(), "Kb".to_string(), "Kr Adjacent Name".to_string(), "Zz".to_string()]; + for index in 1..=240 { + texts.push(format!("PB{index:04}")); + } + assert_orders_like_text(&texts); +} + +#[test] +fn test_order_survives_a_descending_run() { + let mut texts = vec!["ra".to_string(), "rz".to_string()]; + for index in (1..=240).rev() { + texts.push(format!("rb{index:04}")); + } + assert_orders_like_text(&texts); +} + +#[test] +fn test_order_survives_repeated_splitting_of_one_gap() { + // Each name lands between the two before it, which is the case that has to halve the gap + // and so is the case that still runs out. Order must hold regardless. + let mut texts = vec!["m".to_string(), "n".to_string()]; + let mut low = "m".to_string(); + let mut high = "n".to_string(); + for turn in 0..40 { + let middle = if turn % 2 == 0 { + format!("{low}m") + } else { + format!("{high}a") + }; + if turn % 2 == 0 { + low = middle.clone(); + } else { + high = middle.clone(); + } + texts.push(middle); + } + assert_orders_like_text(&texts); +} + +#[test] +fn test_order_survives_a_scrambled_wide_set() { + let mut texts = vec![]; + for index in 0..400 { + // A deliberately hostile order: interleaves ascending, descending and prefix-sharing + // families, all inside a few narrow lexicographic regions. + texts.push(format!("t{:03}", (index * 137) % 400)); + texts.push(format!("transaction_{:03}", 399 - index)); + texts.push(format!("orderDate{index:03}")); + } + texts.push(String::new()); + texts.push(" ".to_string()); + texts.push("\u{e9}".to_string()); + assert_orders_like_text(&texts); +} diff --git a/feel/src/tests/types.rs b/feel/src/tests/types.rs index 44a4efa7..39eef1f1 100644 --- a/feel/src/tests/types.rs +++ b/feel/src/tests/types.rs @@ -376,7 +376,7 @@ fn test_type_get_conformant_value() { let v_date = Value::Date(FeelDate::new(2022, 9, 26).unwrap()); let v_date_b = Value::Date(FeelDate::new(2022, 11, 30).unwrap()); // date time - let v_date_time = Value::DateTime(FeelDateTime::new(FeelDate::new(2022, 9, 27).unwrap(), FeelTime::local_opt(9, 2, 0, 0).unwrap())); + let v_date_time = Value::DateTime(FeelDateTime::new(FeelDate::new(2022, 9, 27).unwrap(), FeelTime::local_opt(9, 2, 0, 0).unwrap()).into()); // days and time duration let v_days_and_time_duration = Value::DaysAndTimeDuration(FeelDaysAndTimeDuration::from_s(100)); // null @@ -410,7 +410,7 @@ fn test_type_get_conformant_value() { ctx_d.set_entry(&NAME_D, v_number_1.clone()); let v_context_d = Value::Context(ctx_d); // functions - let v_function_a = Value::FunctionDefinition( + let v_function_a = Value::function_definition( vec![(NAME_A.clone(), T_NUMBER.clone()), (NAME_B.clone(), T_NUMBER.clone())], FunctionBody::LiteralExpression(Arc::new(Box::new(|_: &FeelScope| value_number!(1)))), false, @@ -418,7 +418,7 @@ fn test_type_get_conformant_value() { FeelContext::default(), T_NUMBER.clone(), ); - let v_function_b = Value::FunctionDefinition( + let v_function_b = Value::function_definition( vec![(NAME_A.clone(), T_NUMBER.clone()), (NAME_B.clone(), T_NUMBER.clone())], FunctionBody::LiteralExpression(Arc::new(Box::new(|_: &FeelScope| value_number!(2)))), false, @@ -426,7 +426,7 @@ fn test_type_get_conformant_value() { FeelContext::default(), T_BOOLEAN.clone(), ); - let v_function_c = Value::FunctionDefinition( + let v_function_c = Value::function_definition( vec![(NAME_A.clone(), T_NUMBER.clone())], FunctionBody::LiteralExpression(Arc::new(Box::new(|_: &FeelScope| value_number!(3)))), false, @@ -434,7 +434,7 @@ fn test_type_get_conformant_value() { FeelContext::default(), T_STRING.clone(), ); - let v_function_d = Value::FunctionDefinition( + let v_function_d = Value::function_definition( vec![(NAME_D.clone(), T_NUMBER.clone())], FunctionBody::LiteralExpression(Arc::new(Box::new(|_: &FeelScope| value_number!(4)))), false, @@ -442,7 +442,7 @@ fn test_type_get_conformant_value() { FeelContext::default(), T_STRING.clone(), ); - let v_function_e = Value::FunctionDefinition( + let v_function_e = Value::function_definition( vec![(NAME_A.clone(), T_STRING.clone())], FunctionBody::LiteralExpression(Arc::new(Box::new(|_: &FeelScope| value_number!(5)))), false, diff --git a/feel/src/tests/values.rs b/feel/src/tests/values.rs index fec39744..496c833b 100644 --- a/feel/src/tests/values.rs +++ b/feel/src/tests/values.rs @@ -10,6 +10,19 @@ use dsntk_feel_temporal::{FeelDate, FeelDateTime, FeelDaysAndTimeDuration, FeelT use std::collections::BTreeMap; use std::sync::Arc; +/// Every value slot in the engine pays for `size_of::()`. A `BTreeMap` node +/// holds eleven of them, so this size is the dominant term in the memory footprint. +/// This test fails if a new variant, or an unboxed payload, makes `Value` grow again. +/// +/// 48 bytes is the floor that the [dsntk_feel::FeelNumber] payload allows. `FeelNumber` +/// has align 16, so [Value] has align 16 and its size is a multiple of 16. An align-8 +/// payload starts at offset 8, so every such payload must be 40 bytes or less to keep +/// the total at 48. +#[test] +fn test_size_of_value() { + assert_eq!(48, std::mem::size_of::()); +} + macro_rules! eq_dbg { ($expected:literal, $value: expr) => { assert_eq!($expected, format!("{:?}", $value)); @@ -54,20 +67,17 @@ fn test_debug() { eq_dbg!(r#"Date(FeelDate(2022, 9, 27))"#, Value::Date(v_date)); eq_dbg!( r#"DateTime(FeelDateTime(FeelDate(2022, 9, 27), FeelTime(12, 13, 23, 0, Local)))"#, - Value::DateTime(v_date_time) + Value::DateTime(v_date_time.into()) ); eq_dbg!( r#"DaysAndTimeDuration(FeelDaysAndTimeDuration(100000000000))"#, Value::DaysAndTimeDuration(v_days_and_time_duration) ); eq_dbg!(r#"ExpressionList([])"#, Value::ExpressionList(Values::default())); - eq_dbg!( - r#"ExternalJavaFunction("class", "method")"#, - Value::ExternalJavaFunction("class".to_string(), "method".to_string()) - ); + eq_dbg!(r#"ExternalJavaFunction("class", "method")"#, Value::ExternalJavaFunction("class".into(), "method".into())); eq_dbg!( r#"ExternalPmmlFunction("document", "model")"#, - Value::ExternalPmmlFunction("document".to_string(), "model".to_string()) + Value::ExternalPmmlFunction("document".into(), "model".into()) ); eq_dbg!(r#"FeelType(Number)"#, Value::FeelType(t_number.clone())); eq_dbg!(r#"FormalParameter(Name("a"), Number)"#, Value::FormalParameter(name.clone(), t_number.clone())); @@ -77,8 +87,8 @@ fn test_debug() { Value::FunctionBody(v_function_body.clone(), v_external) ); eq_dbg!( - r#"FunctionDefinition([(Name("a"), Number)], FunctionBodyLiteralExpression, false, Closure({}), FeelContext({}), Number)"#, - Value::FunctionDefinition(vec![(name.clone(), t_number.clone())], v_function_body, v_external, v_closure, v_closure_ctx, t_number) + r#"FunctionDefinition(([(Name("a"), Number)], FunctionBodyLiteralExpression, false, Closure({}), FeelContext({}), Number))"#, + Value::function_definition(vec![(name.clone(), t_number.clone())], v_function_body, v_external, v_closure, v_closure_ctx, t_number) ); eq_dbg!(r#"IntervalEnd(Number(+1E+0), Opened)"#, Value::IntervalEnd(b_number.clone(), IntervalType::Opened)); eq_dbg!(r#"IntervalStart(Number(+1E+0), Opened)"#, Value::IntervalStart(b_number.clone(), IntervalType::Opened)); @@ -139,24 +149,18 @@ fn test_display() { eq_dsp!(r#"a: number"#, Value::ContextTypeEntry(name.clone(), t_number.clone())); eq_dsp!(r#"a"#, Value::ContextTypeEntryKey(name.clone())); eq_dsp!(r#"2022-09-27"#, Value::Date(v_date)); - eq_dsp!(r#"2022-09-27T12:13:23"#, Value::DateTime(v_date_time)); + eq_dsp!(r#"2022-09-27T12:13:23"#, Value::DateTime(v_date_time.into())); eq_dsp!(r#"PT1M40S"#, Value::DaysAndTimeDuration(v_days_and_time_duration)); eq_dsp!(r#"[]"#, Value::ExpressionList(Values::default())); - eq_dsp!( - r#"ExternalJavaFunction(class, method)"#, - Value::ExternalJavaFunction("class".to_string(), "method".to_string()) - ); - eq_dsp!( - r#"ExternalPmmlFunction(document, model)"#, - Value::ExternalPmmlFunction("document".to_string(), "model".to_string()) - ); + eq_dsp!(r#"ExternalJavaFunction(class, method)"#, Value::ExternalJavaFunction("class".into(), "method".into())); + eq_dsp!(r#"ExternalPmmlFunction(document, model)"#, Value::ExternalPmmlFunction("document".into(), "model".into())); eq_dsp!(r#"type(number)"#, Value::FeelType(t_number.clone())); eq_dsp!(r#"FormalParameter"#, Value::FormalParameter(name.clone(), t_number.clone())); eq_dsp!(r#"FormalParameters"#, Value::FormalParameters(vec![])); eq_dsp!(r#"FunctionBody"#, Value::FunctionBody(v_function_body.clone(), v_external)); eq_dsp!( r#"FunctionDefinition([(Name("a"), Number)],_,false,[],{},number)"#, - Value::FunctionDefinition(vec![(name.clone(), t_number.clone())], v_function_body, v_external, v_closure, v_closure_ctx, t_number) + Value::function_definition(vec![(name.clone(), t_number.clone())], v_function_body, v_external, v_closure, v_closure_ctx, t_number) ); eq_dsp!(r#"IntervalEnd"#, Value::IntervalEnd(b_number.clone(), IntervalType::Opened)); eq_dsp!(r#"IntervalStart"#, Value::IntervalStart(b_number.clone(), IntervalType::Opened)); @@ -210,18 +214,18 @@ fn test_type_of() { eq_typ!(FeelType::Number, Value::ContextTypeEntry(name.clone(), t_number.clone())); eq_typ!(FeelType::Any, Value::ContextTypeEntryKey(name.clone())); eq_typ!(FeelType::Date, Value::Date(v_date)); - eq_typ!(FeelType::DateTime, Value::DateTime(v_date_time)); + eq_typ!(FeelType::DateTime, Value::DateTime(v_date_time.into())); eq_typ!(FeelType::DaysAndTimeDuration, Value::DaysAndTimeDuration(v_days_and_time_duration)); eq_typ!(FeelType::Any, Value::ExpressionList(Values::default())); - eq_typ!(FeelType::Any, Value::ExternalJavaFunction("class".to_string(), "method".to_string())); - eq_typ!(FeelType::Any, Value::ExternalPmmlFunction("document".to_string(), "model".to_string())); + eq_typ!(FeelType::Any, Value::ExternalJavaFunction("class".into(), "method".into())); + eq_typ!(FeelType::Any, Value::ExternalPmmlFunction("document".into(), "model".into())); eq_typ!(FeelType::Number, Value::FeelType(t_number.clone())); eq_typ!(FeelType::Number, Value::FormalParameter(name.clone(), t_number.clone())); eq_typ!(FeelType::Any, Value::FormalParameters(vec![])); eq_typ!(FeelType::Any, Value::FunctionBody(v_function_body.clone(), v_external)); eq_typ!( FeelType::Function(vec![FeelType::Number], Box::new(FeelType::Number)), - Value::FunctionDefinition( + Value::function_definition( vec![(name.clone(), t_number.clone())], v_function_body, v_external, @@ -381,7 +385,7 @@ fn test_jsonify() { assert_eq!(r#""18:35:12.000598677Z""#, Value::Time(FeelTime::utc(18, 35, 12, 598677)).jsonify()); assert_eq!( r#""2023-02-24T18:35:12.000598677Z""#, - Value::DateTime(FeelDateTime::new(FeelDate::new(2023, 2, 24).unwrap(), FeelTime::utc_opt(18, 35, 12, 598677).unwrap())).jsonify() + Value::DateTime(FeelDateTime::new(FeelDate::new(2023, 2, 24).unwrap(), FeelTime::utc_opt(18, 35, 12, 598677).unwrap()).into()).jsonify() ); assert_eq!(r#""P10DT13H59M18S""#, Value::DaysAndTimeDuration(FeelDaysAndTimeDuration::from_s(914358)).jsonify()); assert_eq!(r#""P3Y2M""#, Value::YearsAndMonthsDuration(FeelYearsAndMonthsDuration::from_ym(3, 2)).jsonify()); @@ -474,7 +478,7 @@ fn test_coerced() { let v_list_number_1 = Value::List(vec![value_number!(1, 0)]); let v_list_number_2 = Value::List(vec![value_number!(1, 0), value_number!(2, 0)]); let v_list_string_1 = Value::List(vec![Value::String("A".to_string())]); - let v_function_a = Value::FunctionDefinition( + let v_function_a = Value::function_definition( vec![(NAME_A.clone(), T_NUMBER.clone())], FunctionBody::LiteralExpression(Arc::new(Box::new(|_: &FeelScope| value_number!(1)))), false, diff --git a/feel/src/values.rs b/feel/src/values.rs index 7ff3b79c..c1871b66 100644 --- a/feel/src/values.rs +++ b/feel/src/values.rs @@ -18,6 +18,7 @@ use std::fmt; use std::fmt::Display; use std::ops::Deref; use std::str::FromStr; +use std::sync::Arc; /// Creates `Value::Null` with optional tracing message. /// @@ -74,6 +75,10 @@ pub const VALUE_FALSE: Value = Value::Boolean(false); /// Constant indicating invalid coercion result. const INVALID_COERCION: &str = "after coercion"; +/// Payload of [Value::FunctionDefinition]: formal parameters, body, external flag, +/// closure names, closure values, result type. +pub type FunctionDefinitionParts = (Vec<(Name, FeelType)>, FunctionBody, bool, Closure, FeelContext, FeelType); + /// `FEEL` value. #[derive(Debug, Clone, PartialEq)] pub enum Value { @@ -105,7 +110,19 @@ pub enum Value { Date(FeelDate), /// Value for storing date and time as [FeelDateTime]. - DateTime(FeelDateTime), + /// + /// The payload lives behind a pointer. Inline, it is 56 bytes. A payload of more than 40 + /// bytes sets `size_of::()` to 64 instead of 48. That is because [Value] has align + /// 16, and an align-8 payload starts at offset 8. Eleven values share a `BTreeMap` node, + /// so every context in the engine would pay for it. + /// + /// The pointer is an [Arc] and not a `Box` because this payload is **read far more often + /// than it is built**. A probe of `FeelScope::get_value` on the largest benchmark model + /// counted 16.2 million bare-name reads at 1,000 transactions. **60.75% of them returned + /// a date and time**. A `Box` makes each of those reads allocate. That measured at 16.41 + /// ns for the clone and the matching drop, against 3.72 ns for an [Arc] refcount pair. + /// Sharing costs nothing here, because a [FeelDateTime] is never mutated in place. + DateTime(Arc), /// Value for days and time durations. DaysAndTimeDuration(FeelDaysAndTimeDuration), @@ -114,19 +131,25 @@ pub enum Value { ExpressionList(Values), /// Value representing a mapping to externally defined `Java` function. + /// + /// The two names are `Box` rather than [String] for the reason given on + /// [Value::DateTime]: two [String]s are 48 bytes, which is above the 40 that + /// `size_of::() == 48` allows, while two `Box` are 32. ExternalJavaFunction( /// Class name. - String, + Box, /// Method signature. - String, + Box, ), /// Value representing a mapping to externally defined `PMML` function. + /// + /// The two names are `Box` for the same reason as [Value::ExternalJavaFunction]. ExternalPmmlFunction( /// Document. - String, + Box, /// Model name. - String, + Box, ), /// Value representing the `FEEL` type of the value. @@ -148,20 +171,14 @@ pub enum Value { /// Value representing the function definition. /// This value holds the list of function's formal parameters, the function's body, closure for lambdas and expected result type. - FunctionDefinition( - /// Formal parameters of the function. - Vec<(Name, FeelType)>, - /// Body of the function. - FunctionBody, - /// Flag indicating if the function's body is an external function, `true` == external. - bool, - /// Closed names from function context (closure names). - Closure, - /// Values of the closed names (closure values). - FeelContext, - /// Return type of the function. - FeelType, - ), + /// + /// This variant boxes its payload. Unboxed, it is 129 bytes, more than three times + /// the size of any other variant. That size alone sets `size_of::()` for + /// every value in the engine. A `BTreeMap` node holds 11 values, so each context + /// pays 11 times that difference. Function definitions are rare, so the extra + /// indirection costs nothing measurable. Use [Value::function_definition] to build + /// this variant. + FunctionDefinition(Box), /// Value representing interval end. IntervalEnd( @@ -295,7 +312,8 @@ impl Display for Value { Value::FormalParameter(_, _) => write!(f, "FormalParameter"), Value::FormalParameters(_) => write!(f, "FormalParameters"), Value::FunctionBody(_, external) => write!(f, "FunctionBody{}", if *external { " (external)" } else { "" }), - Value::FunctionDefinition(parameters, _body, external, closure, closure_ctx, return_type) => { + Value::FunctionDefinition(fd) => { + let (parameters, _body, external, closure, closure_ctx, return_type) = &**fd; write!(f, "FunctionDefinition({parameters:?},_,{external},{closure},{closure_ctx},{return_type})") } Value::IntervalEnd(_, _) => write!(f, "IntervalEnd"), @@ -375,6 +393,18 @@ impl Jsonify for Value { } impl Value { + /// Builds [Value::FunctionDefinition] from its six parts and boxes the payload. + pub fn function_definition( + parameters: Vec<(Name, FeelType)>, + body: FunctionBody, + external: bool, + closure: Closure, + closure_ctx: FeelContext, + result_type: FeelType, + ) -> Value { + Value::FunctionDefinition(Box::new((parameters, body, external, closure, closure_ctx, result_type))) + } + /// Returns `true` when the value is of type [Value::Null]. pub fn is_null(&self) -> bool { matches!(self, Value::Null(_)) @@ -496,7 +526,8 @@ impl Value { Value::FormalParameter(_, feel_type) => feel_type.clone(), Value::FormalParameters(_) => FeelType::Any, Value::FunctionBody(_, _) => FeelType::Any, - Value::FunctionDefinition(parameters, _, _, _, _, result_type) => { + Value::FunctionDefinition(fd) => { + let (parameters, _, _, _, _, result_type) = &**fd; let parameter_types = parameters.iter().map(|(_, feel_type)| feel_type.clone()).collect(); FeelType::Function(parameter_types, Box::new(result_type.clone())) } @@ -593,7 +624,8 @@ impl Value { } false } - Value::FunctionDefinition(parameters, _, _, _, _, result_type) => { + Value::FunctionDefinition(fd) => { + let (parameters, _, _, _, _, result_type) = &**fd; if let FeelType::Function(t_parameters, t_result) = target_type { if parameters.len() != t_parameters.len() { return false; @@ -642,7 +674,7 @@ impl Value { /// All these conversion rules are implemented in this function. /// pub fn coerced(&self, target_type: &FeelType) -> Value { - if let Value::FunctionDefinition(_, _, _, _, _, _) = self { + if let Value::FunctionDefinition(_) = self { return self.clone(); } if let Value::BuiltInFunction(bif) = self @@ -723,7 +755,7 @@ impl Value { /// Tries to convert `xsd:dateTime` string into valid [Value] representing a date and time. /// FEEL date and time format is fully conformant with `xsd:dateTime`. pub fn try_from_xsd_date_time(text: &str) -> Result { - Ok(Value::DateTime(FeelDateTime::try_from(text).map_err(|_| err_invalid_xsd_date_time(text))?)) + Ok(Value::DateTime(FeelDateTime::try_from(text).map_err(|_| err_invalid_xsd_date_time(text))?.into())) } /// Tries to convert `xsd:duration` string into valid [Value] representing a date and time. diff --git a/feel/tests/name_existing.rs b/feel/tests/name_existing.rs new file mode 100644 index 00000000..7a2b1ffe --- /dev/null +++ b/feel/tests/name_existing.rs @@ -0,0 +1,33 @@ +//! Tests for [Name::existing], the lookup that must never intern. +//! +//! A lookup that interns adds a name on every probe that misses, and the interner never +//! releases one. So a process probing unbounded text would grow without bound. `get value` +//! is exactly that kind of probe. These tests pin the contract the fix relies on. + +use dsntk_feel::Name; + +#[test] +fn absent_text_is_none() { + assert!(Name::existing("a name no test ever interns 8f3a2b").is_none()); +} + +#[test] +fn probing_does_not_create() { + // The point of the fix: asking twice must not make the name exist. + assert!(Name::existing("probe only 41d9c7").is_none()); + assert!(Name::existing("probe only 41d9c7").is_none()); +} + +#[test] +fn interned_text_is_found_and_equal() { + let made = Name::from("interned by this test 5c1e"); + let found = Name::existing("interned by this test 5c1e"); + assert_eq!(Some(made), found, "existing must return the handle intern would"); +} + +#[test] +fn text_is_trimmed_like_intern() { + let made = Name::from(" padded name 77b2 "); + assert_eq!(Some(made.clone()), Name::existing("padded name 77b2")); + assert_eq!(Some(made), Name::existing(" padded name 77b2 ")); +} diff --git a/feel/tests/name_overflow.rs b/feel/tests/name_overflow.rs new file mode 100644 index 00000000..04ba8897 --- /dev/null +++ b/feel/tests/name_overflow.rs @@ -0,0 +1,58 @@ +//! The name table must hold more than 2^20 distinct names. +//! +//! The `context` built-in interns every string key, so a model that builds a membership +//! map from transaction ids interns one name per distinct id. A server process shares one +//! table across all requests, so the risk is a matter of total request count, not of any +//! single request's size. +//! +//! This is an integration test on purpose: it runs in its own process, so the flooded +//! table cannot slow the unit tests of this crate. + +use dsntk_feel::Name; +use std::cmp::Ordering; +use std::collections::BTreeMap; + +/// Number of names interned by this test, comfortably past the 1,048,575 rank ceiling. +const FLOOD: usize = 1_100_000; + +#[test] +fn name_table_holds_more_than_a_million_names() { + // A name interned before the flood, while identities still fit beside a rank. + let early = Name::from("name overflow test 0000000"); + + // Flood the table past the point where names stop carrying a rank. + let names: Vec = (0..FLOOD).map(|i| Name::from(format!("name overflow test {i:07}"))).collect(); + + // Interning is injective and stable across the ceiling: the same text gives the same + // handle, and every handle still resolves to its own text. + assert_eq!(early, names[0]); + assert_eq!(names[0].as_str(), "name overflow test 0000000"); + let last = names.last().unwrap(); + assert_eq!(last.as_str(), format!("name overflow test {:07}", FLOOD - 1)); + assert_eq!(*last, Name::from(format!("name overflow test {:07}", FLOOD - 1))); + assert_ne!(names[1_048_574], names[1_048_575]); + + // Ordering stays lexicographic on both sides of the ceiling and across it. + let below = &names[1_048_570]; // interned with a rankable identity + let above = &names[1_048_580]; // interned past the rank ceiling + assert_eq!(below.cmp(above), Ordering::Less); + assert_eq!(above.cmp(below), Ordering::Greater); + assert_eq!(above.cmp(&above.clone()), Ordering::Equal); + + // A BTreeMap keyed by Name is what FeelContext is. Keys inserted from both sides of + // the ceiling must all be found again, which fails if ordering is not total. + let probes = [0usize, 524_287, 1_048_570, 1_048_574, 1_048_575, 1_048_580, FLOOD - 1]; + let mut map: BTreeMap = BTreeMap::new(); + for &i in &probes { + map.insert(names[i].clone(), i); + } + for &i in &probes { + assert_eq!(Some(&i), map.get(&Name::from(format!("name overflow test {i:07}"))), "key {i} lost"); + } + + // Entries iterate in text order, because context key order is visible in output. + let keys: Vec<&str> = map.keys().map(|k| k.as_str()).collect(); + let mut sorted = keys.clone(); + sorted.sort_unstable(); + assert_eq!(keys, sorted); +} diff --git a/model-evaluator/src/boxed_expressions.rs b/model-evaluator/src/boxed_expressions.rs index 20b5e5a7..cce67f21 100644 --- a/model-evaluator/src/boxed_expressions.rs +++ b/model-evaluator/src/boxed_expressions.rs @@ -9,7 +9,7 @@ use dsntk_feel::closure::Closure; use dsntk_feel::context::FeelContext; use dsntk_feel::values::Value; use dsntk_feel::{Evaluator, FeelScope, FeelType, FunctionBody, IntervalType, value_null}; -use dsntk_feel_evaluator::{EveryExpressionEvaluator, FilterExpressionEvaluator, ForExpressionEvaluator, SomeExpressionEvaluator}; +use dsntk_feel_evaluator::{EveryExpressionEvaluator, FilterExpressionEvaluator, ForExpressionEvaluator, IndexShape, SomeExpressionEvaluator}; use dsntk_feel_parser::{ClosureBuilder, parse_name}; use dsntk_model::*; use std::sync::Arc; @@ -148,7 +148,7 @@ pub fn build_function_definition_evaluator(scope: &FeelScope, function_definitio closure_ctx.create_entry(closure_name, closure_value); } } - Value::FunctionDefinition(parameters.clone(), function_body.clone(), false, closure.clone(), closure_ctx, result_type.clone()) + Value::function_definition(parameters.clone(), function_body.clone(), false, closure.clone(), closure_ctx, result_type.clone()) }); Ok(( build_coerced_result_evaluator(function_definition_evaluator, function_definition, function_definition.namespace(), model_builder), @@ -164,12 +164,12 @@ pub fn build_function_definition_evaluator(scope: &FeelScope, function_definitio if let Value::Context(java_mapping) = body_evaluator(scope) { if let Some(Value::String(class_name)) = java_mapping.get_entry(&"class".into()) { if let Some(Value::String(method_signature)) = java_mapping.get_entry(&"method signature".into()) { - let java_class_name = class_name.to_owned(); - let java_method_signature = method_signature.to_owned(); + let java_class_name: Box = class_name.as_str().into(); + let java_method_signature: Box = method_signature.as_str().into(); let java_evaluator = Box::new(move |_: &FeelScope| Value::ExternalJavaFunction(java_class_name.clone(), java_method_signature.clone())) as Evaluator; let function_body_evaluator = Arc::new(java_evaluator); let function_body = FunctionBody::External(function_body_evaluator); - Value::FunctionDefinition(parameters.clone(), function_body, true, Closure::default(), FeelContext::default(), result_type.clone()) + Value::function_definition(parameters.clone(), function_body, true, Closure::default(), FeelContext::default(), result_type.clone()) } else { value_null!("invalid Java function mapping, no method signature entry in context {}", java_mapping) } @@ -194,12 +194,12 @@ pub fn build_function_definition_evaluator(scope: &FeelScope, function_definitio if let Value::Context(pmml_mapping) = body_evaluator(scope) { if let Some(Value::String(document)) = pmml_mapping.get_entry(&"document".into()) { if let Some(Value::String(model_name)) = pmml_mapping.get_entry(&"model".into()) { - let pmml_document = document.to_owned(); - let pmml_model_name = model_name.to_owned(); + let pmml_document: Box = document.as_str().into(); + let pmml_model_name: Box = model_name.as_str().into(); let pmml_evaluator = Box::new(move |_: &FeelScope| Value::ExternalPmmlFunction(pmml_document.clone(), pmml_model_name.clone())) as Evaluator; let function_body_evaluator = Arc::new(pmml_evaluator); let function_body = FunctionBody::External(function_body_evaluator); - Value::FunctionDefinition(parameters.clone(), function_body, true, Closure::default(), FeelContext::default(), result_type.clone()) + Value::function_definition(parameters.clone(), function_body, true, Closure::default(), FeelContext::default(), result_type.clone()) } else { value_null!("invalid PMML function mapping, no model name entry in context {}", pmml_mapping) } @@ -243,7 +243,10 @@ pub fn build_invocation_evaluator(scope: &FeelScope, invocation: &Invocation, mo } params_ctx.set_entry(param_name, param_value.coerced(param_type)) } - if let Value::FunctionDefinition(formal_parameters, body, false, _, closure_ctx, result_type) = function_evaluator(scope) { + if let Value::FunctionDefinition(fd) = function_evaluator(scope) + && !fd.2 + { + let (formal_parameters, body, _, _, closure_ctx, result_type) = *fd; for (formal_parameter_name, formal_parameter_type) in formal_parameters { if let Some(param_value) = params_ctx.get(&formal_parameter_name) { let coerced_param_value = param_value.coerced(&formal_parameter_type); @@ -349,7 +352,9 @@ pub fn build_filter_evaluator(scope: &FeelScope, filter: &Filter, model_builder: // prepare `filter` evaluator let evaluator = Box::new(move |scope: &FeelScope| { let filter_expression_evaluator = FilterExpressionEvaluator::default(); - filter_expression_evaluator.evaluate(scope, in_evaluator(scope), &match_evaluator) + // A boxed `Filter` element carries an expression instance rather than a `FEEL` syntax tree. + // So this proves nothing about its match expression, and every element runs it. + filter_expression_evaluator.evaluate(scope, in_evaluator(scope), &match_evaluator, IndexShape::Unknown) }); Ok((build_coerced_result_evaluator(evaluator, filter, filter.namespace(), model_builder), Closure::default())) } diff --git a/model-evaluator/src/business_knowledge_model.rs b/model-evaluator/src/business_knowledge_model.rs index 41a02e05..ce8bcb13 100644 --- a/model-evaluator/src/business_knowledge_model.rs +++ b/model-evaluator/src/business_knowledge_model.rs @@ -243,7 +243,7 @@ fn build_bkm_conditional_evaluator( let (evaluator, _) = build_conditional_evaluator(scope, conditional, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function_definition = Value::FunctionDefinition( + let function_definition = Value::function_definition( formal_parameters, FunctionBody::Conditional(Arc::new(evaluator)), false, @@ -266,7 +266,7 @@ fn build_bkm_context_evaluator( let (evaluator, _) = build_context_evaluator(scope, context, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function = Value::FunctionDefinition( + let function = Value::function_definition( formal_parameters, FunctionBody::Context(Arc::new(evaluator)), false, @@ -289,7 +289,7 @@ fn build_bkm_decision_table_evaluator( let (evaluator, _) = build_decision_table_evaluator(scope, decision_table, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function = Value::FunctionDefinition( + let function = Value::function_definition( formal_parameters, FunctionBody::DecisionTable(Arc::new(evaluator)), false, @@ -312,7 +312,7 @@ fn build_bkm_every_evaluator( let (evaluator, _) = build_every_evaluator(scope, every, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function_definition = Value::FunctionDefinition( + let function_definition = Value::function_definition( formal_parameters, FunctionBody::Every(Arc::new(evaluator)), false, @@ -335,7 +335,7 @@ fn build_bkm_filter_evaluator( let (evaluator, _) = build_filter_evaluator(scope, filter, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function_definition = Value::FunctionDefinition( + let function_definition = Value::function_definition( formal_parameters, FunctionBody::Filter(Arc::new(evaluator)), false, @@ -358,7 +358,7 @@ fn build_bkm_for_evaluator( let (evaluator, _) = build_for_evaluator(scope, r#for, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function_definition = Value::FunctionDefinition(formal_parameters, FunctionBody::For(Arc::new(evaluator)), false, closure, closure_ctx, output_variable_type); + let function_definition = Value::function_definition(formal_parameters, FunctionBody::For(Arc::new(evaluator)), false, closure, closure_ctx, output_variable_type); build_bkm_evaluator_from_function_definition(output_variable_name, function_definition, knowledge_requirements) } @@ -374,7 +374,7 @@ fn build_bkm_function_definition_evaluator( let (evaluator, _) = build_function_definition_evaluator(scope, function_definition, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function = Value::FunctionDefinition( + let function = Value::function_definition( formal_parameters, FunctionBody::FunctionDefinition(Arc::new(evaluator)), false, @@ -397,7 +397,7 @@ fn build_bkm_invocation_evaluator( let (evaluator, _) = build_invocation_evaluator(scope, invocation, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function = Value::FunctionDefinition( + let function = Value::function_definition( formal_parameters, FunctionBody::Invocation(Arc::new(evaluator)), false, @@ -420,7 +420,7 @@ fn build_bkm_list_evaluator( let (evaluator, _) = build_list_evaluator(scope, list, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function = Value::FunctionDefinition( + let function = Value::function_definition( formal_parameters, FunctionBody::List(Arc::new(evaluator)), false, @@ -443,7 +443,7 @@ fn build_bkm_literal_expression_evaluator( let (evaluator, _) = build_literal_expression_evaluator(scope, literal_expression, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function = Value::FunctionDefinition( + let function = Value::function_definition( formal_parameters, FunctionBody::LiteralExpression(Arc::new(evaluator)), false, @@ -466,7 +466,7 @@ fn build_bkm_relation_evaluator( let (evaluator, _) = build_relation_evaluator(scope, relation, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function_definition = Value::FunctionDefinition( + let function_definition = Value::function_definition( formal_parameters, FunctionBody::Relation(Arc::new(evaluator)), false, @@ -489,7 +489,7 @@ fn build_bkm_some_evaluator( let (evaluator, _) = build_some_evaluator(scope, some, model_builder)?; let closure = Closure::default(); let closure_ctx = FeelContext::default(); - let function_definition = Value::FunctionDefinition( + let function_definition = Value::function_definition( formal_parameters, FunctionBody::Some(Arc::new(evaluator)), false, diff --git a/model-evaluator/src/decision_service.rs b/model-evaluator/src/decision_service.rs index 50b97258..4d632ed9 100644 --- a/model-evaluator/src/decision_service.rs +++ b/model-evaluator/src/decision_service.rs @@ -67,7 +67,7 @@ impl DecisionServiceEvaluator { value_null!() }); let function_body = dsntk_feel::FunctionBody::DecisionService(Arc::new(body_evaluator)); - let function_definition = Value::FunctionDefinition(entry.1.clone(), function_body, false, Closure::default(), FeelContext::default(), output_variable_type); + let function_definition = Value::function_definition(entry.1.clone(), function_body, false, Closure::default(), FeelContext::default(), output_variable_type); let decision_service_as_function_definition_evaluator = Box::new(move |_: &FeelScope| function_definition.clone()); entry.3 = Some(decision_service_as_function_definition_evaluator); }); diff --git a/model-evaluator/src/decision_table.rs b/model-evaluator/src/decision_table.rs index c7009032..2baa7a31 100644 --- a/model-evaluator/src/decision_table.rs +++ b/model-evaluator/src/decision_table.rs @@ -7,6 +7,13 @@ use dsntk_feel::{Evaluator, FeelScope, Name, value_null}; use dsntk_feel_parser::AstNode; use dsntk_model::{BuiltinAggregator, DecisionTable, HitPolicy}; use std::cmp::Ordering; +use std::sync::LazyLock; + +/// The name `?`, the tested value inside a unary test, interned once. +/// +/// See the note on `NAME_QUESTION_MARK` in `item_definition.rs`, which binds the same +/// name for the allowed-values test of an item definition. +static NAME_QUESTION_MARK: LazyLock = LazyLock::new(|| "?".into()); /// Parsed rule of the decision table. /// Input entries and output entries are parsed into evaluation clauses @@ -231,6 +238,59 @@ impl EvaluatedDecisionTable { } } +/// Builds the evaluator of one unary-tests node `t` against a tested expression `e`. +/// +/// When no test in `t` uses the name `?`, this builds the plain `e in (t)` node the +/// engine always built. So the ordinary entry costs exactly what it did. When a test +/// uses `?`, DMN 1.3 Table 55 applies: `e in t` means `{?: e, r: t}.r`. So the test is +/// the boolean value of its own expression, evaluated with `?` bound to the value of +/// `e`. The tests of the same entry that do not use `?` keep the plain `in` semantics, +/// and the entry is satisfied when any of its tests is. +/// +/// Used for decision-table input entries and for the allowed-values test of an item +/// definition. These are the two places the model hands unary tests to the engine. +pub(crate) fn build_unary_tests_evaluator(tested_expression: &AstNode, unary_tests: AstNode) -> Evaluator { + if !dsntk_feel_evaluator::references_name(&unary_tests, &NAME_QUESTION_MARK) { + let node = AstNode::In(Box::new(tested_expression.clone()), Box::new(unary_tests)); + return dsntk_feel_evaluator::prepare(&node); + } + // Split the tests that use `?` from the tests that do not. + let (negated, items) = match unary_tests { + AstNode::ExpressionList(items) => (false, items), + AstNode::NegatedList(items) => (true, items), + other => (false, vec![other]), + }; + let (question_items, plain_items): (Vec, Vec) = items + .into_iter() + .partition(|item| dsntk_feel_evaluator::references_name(item, &NAME_QUESTION_MARK)); + let question_evaluators: Vec = question_items.iter().map(dsntk_feel_evaluator::prepare).collect(); + let plain_evaluator: Option = if plain_items.is_empty() { + None + } else { + // The plain tests keep their old shape, including the negation of a `not(...)` list. + let list = if negated { AstNode::NegatedList(plain_items) } else { AstNode::ExpressionList(plain_items) }; + Some(dsntk_feel_evaluator::prepare(&AstNode::In(Box::new(tested_expression.clone()), Box::new(list)))) + }; + let tested_evaluator = dsntk_feel_evaluator::prepare(tested_expression); + Box::new(move |scope: &FeelScope| { + let plain = plain_evaluator.as_ref().map(|evaluator| evaluator(scope).is_true()); + let value = tested_evaluator(scope); + let mut ctx = FeelContext::default(); + ctx.set_entry(&NAME_QUESTION_MARK, value); + scope.push(ctx); + let question = question_evaluators.iter().any(|evaluator| evaluator(scope).is_true()); + scope.pop(); + let satisfied = if negated { + // `not(t1, ..., tn)` is satisfied when no test is. The plain part carries its own + // negation already, because it was rebuilt as a negated list above. + plain.unwrap_or(true) && !question + } else { + plain.unwrap_or(false) || question + }; + Value::Boolean(satisfied) + }) +} + fn parse_decision_table(scope: &FeelScope, decision_table: &DecisionTable) -> Result { // parse input expressions and input values let mut input_expressions_and_values = vec![]; @@ -272,7 +332,21 @@ fn parse_decision_table(scope: &FeelScope, decision_table: &DecisionTable) -> Re let mut input_entries_evaluators = vec![]; for (i, (input_expression, input_values)) in input_expressions_and_values.iter().enumerate() { let input_entry_node = dsntk_feel_parser::parse_unary_tests(scope, &rule.input_entries[i].text, false)?; - if let Some(input_values_node) = input_values { + let uses_question_mark = dsntk_feel_evaluator::references_name(&input_entry_node, &NAME_QUESTION_MARK) + || input_values.as_ref().is_some_and(|values| dsntk_feel_evaluator::references_name(values, &NAME_QUESTION_MARK)); + if uses_question_mark { + // At least one test uses `?`, so the entry needs `?` bound to the value of the + // input expression. See [build_unary_tests_evaluator]. + let entry_evaluator = build_unary_tests_evaluator(input_expression, input_entry_node); + if let Some(input_values_node) = input_values { + let values_evaluator = build_unary_tests_evaluator(input_expression, input_values_node.clone()); + input_entries_evaluators.push(Box::new(move |scope: &FeelScope| { + Value::Boolean(values_evaluator(scope).is_true() && entry_evaluator(scope).is_true()) + }) as Evaluator); + } else { + input_entries_evaluators.push(entry_evaluator); + } + } else if let Some(input_values_node) = input_values { let left = AstNode::In(Box::new(input_expression.clone()), Box::new(input_values_node.clone())); let right = AstNode::In(Box::new(input_expression.clone()), Box::new(input_entry_node)); let node = AstNode::And(Box::new(left), Box::new(right)); @@ -403,4 +477,78 @@ mod tests { let evaluator = build_decision_table_evaluator(&scope, &decision_table).unwrap(); assert_eq!(value_number!(10, 2), evaluator(&scope)); } + + /// A decision table whose first input entry is a boolean expression over `?`. + /// + /// DMN 1.3 Table 55: when the test is a boolean expression that uses `?`, the entry + /// `e in t` means `{?: e, r: t}.r`. So the boolean itself is the answer. Without that + /// binding, the expression would evaluate to null, and the rule would silently never + /// match. + const DT_QUESTION_MARK: &str = r#" +┌───┬──────────────────────╥───────┐ +│ F │ Payment ║ │ +╞═══╪══════════════════════╬═══════╡ +│ 1 │ contains(?, "VENMO") ║ "P2P" │ +├───┼──────────────────────╫───────┤ +│ 2 │ - ║ "OK" │ +└───┴──────────────────────╨───────┘ +"#; + + #[test] + fn question_mark_in_input_entry_matches() { + let decision_table: DecisionTable = dsntk_recognizer::from_unicode(DT_QUESTION_MARK, false).unwrap().into(); + let scope = context(r#"{Payment: "VENMO-123"}"#).into(); + let evaluator = build_decision_table_evaluator(&scope, &decision_table).unwrap(); + assert_eq!(Value::String("P2P".to_string()), evaluator(&scope)); + } + + #[test] + fn question_mark_in_input_entry_does_not_match() { + let decision_table: DecisionTable = dsntk_recognizer::from_unicode(DT_QUESTION_MARK, false).unwrap().into(); + let scope = context(r#"{Payment: "ZELLE-99"}"#).into(); + let evaluator = build_decision_table_evaluator(&scope, &decision_table).unwrap(); + assert_eq!(Value::String("OK".to_string()), evaluator(&scope)); + } + + /// The equality form `? = "X"`. It parses as one expression, not as a unary + /// comparison, so it takes the same bound-`?` path as a function call over `?`. + const DT_QUESTION_MARK_EQUALITY: &str = r#" +┌───┬──────────╥───────┐ +│ F │ Letter ║ │ +╞═══╪══════════╬═══════╡ +│ 1 │ ? = "X" ║ "hit" │ +├───┼──────────╫───────┤ +│ 2 │ - ║ "def" │ +└───┴──────────╨───────┘ +"#; + + #[test] + fn question_mark_equality_in_input_entry() { + let decision_table: DecisionTable = dsntk_recognizer::from_unicode(DT_QUESTION_MARK_EQUALITY, false).unwrap().into(); + let evaluator = build_decision_table_evaluator(&context("{Letter: \"X\"}").into(), &decision_table).unwrap(); + assert_eq!(Value::String("hit".to_string()), evaluator(&context("{Letter: \"X\"}").into())); + assert_eq!(Value::String("def".to_string()), evaluator(&context("{Letter: \"Y\"}").into())); + } + + /// A `?` test mixed with plain tests in one entry. The entry is satisfied when any + /// of its tests is. So the plain tests must keep their old semantics next to the + /// bound-`?` test. + const DT_QUESTION_MARK_MIXED: &str = r#" +┌───┬─────────────────────────────╥───────┐ +│ F │ Payment ║ │ +╞═══╪═════════════════════════════╬═══════╡ +│ 1 │ "WIRE", contains(?, "P2P") ║ "yes" │ +├───┼─────────────────────────────╫───────┤ +│ 2 │ - ║ "no" │ +└───┴─────────────────────────────╨───────┘ +"#; + + #[test] + fn question_mark_mixed_with_plain_tests() { + let decision_table: DecisionTable = dsntk_recognizer::from_unicode(DT_QUESTION_MARK_MIXED, false).unwrap().into(); + let evaluator = build_decision_table_evaluator(&context("{Payment: \"WIRE\"}").into(), &decision_table).unwrap(); + assert_eq!(Value::String("yes".to_string()), evaluator(&context("{Payment: \"WIRE\"}").into())); + assert_eq!(Value::String("yes".to_string()), evaluator(&context("{Payment: \"XP2P\"}").into())); + assert_eq!(Value::String("no".to_string()), evaluator(&context("{Payment: \"ACH\"}").into())); + } } diff --git a/model-evaluator/src/item_definition.rs b/model-evaluator/src/item_definition.rs index 366d2230..ab54802e 100644 --- a/model-evaluator/src/item_definition.rs +++ b/model-evaluator/src/item_definition.rs @@ -9,7 +9,13 @@ use dsntk_feel::{Evaluator, FeelScope, FeelType, Name, value_null}; use dsntk_feel_parser::AstNode; use dsntk_model::ItemDefinitionType; use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; + +/// The name `?`, the implicit input of an allowed-values test, interned once. +/// +/// See the note on `NAME_ITEM` in `feel-evaluator/src/filters.rs`. This ran once per value +/// checked against an allowed-values list, from every Rayon worker. +static NAME_QUESTION_MARK: LazyLock = LazyLock::new(|| "?".into()); /// Type of closure that evaluates input data conformant with item definition. pub type ItemDefinitionEvaluatorFn = Box Value + Send + Sync>; @@ -62,8 +68,10 @@ fn build_allowed_values_evaluator(item_definition: &DefItemDefinition) -> Result { let scope = FeelScope::default(); let unary_tests_node = dsntk_feel_parser::parse_unary_tests(&scope, text, false)?; - let node = AstNode::In(Box::new(AstNode::Name("?".into())), Box::new(unary_tests_node)); - av_evaluator = Some(dsntk_feel_evaluator::prepare(&node)); + // The tested expression is the name `?`, which [check_allowed_values] binds to the + // checked value. The shared builder gives a test that itself uses `?` the DMN 1.3 + // Table 55 semantics: the boolean value of the test is the answer. + av_evaluator = Some(crate::decision_table::build_unary_tests_evaluator(&AstNode::Name("?".into()), unary_tests_node)); } Ok(av_evaluator) @@ -72,7 +80,7 @@ fn build_allowed_values_evaluator(item_definition: &DefItemDefinition) -> Result fn check_allowed_values(value: Value, av_evaluator: Option<&Evaluator>) -> Value { if let Some(evaluator) = av_evaluator { let scope = FeelScope::default(); - scope.set_value(&"?".into(), value.clone()); + scope.set_value(&NAME_QUESTION_MARK, value.clone()); if evaluator(&scope).is_true() { value } else { @@ -465,6 +473,32 @@ mod tests { ItemDefinitionEvaluator::new(&def_definitions).unwrap() } + /// An allowed-values test that is a boolean expression over `?`. + /// + /// DMN 1.3 Table 55: `e in t` means `{?: e, r: t}.r` when `t` is a boolean expression + /// that uses `?`. Without the shared builder, the engine would bind `?` but then + /// compare the tested string against the resulting boolean for equality. Every value + /// would be rejected, matching or not. + #[test] + fn test_allowed_values_with_question_mark() { + let xml = r#" + + + string + + contains(?, "VENMO") + + + "#; + let evaluator = build_evaluator(xml); + let def_key = DefKey::new(NAMESPACE, "tPayment"); + let accepted = Value::String("VENMO-123".to_string()); + assert_eq!(accepted, evaluator.eval(&def_key, &accepted).unwrap()); + let rejected = Value::String("ZELLE-99".to_string()); + assert_eq!(value_null!(r#"value '"ZELLE-99"' is not allowed"#), evaluator.eval(&def_key, &rejected).unwrap()); + } + #[test] fn test_evaluate_input_data_0101_1() { let evaluator = build_evaluator(DMN_0101); @@ -566,7 +600,7 @@ mod tests { let context = dsntk_feel_evaluator::evaluate_context(&Default::default(), context_str).unwrap(); let value = context.get_entry(&Name::new(&["Appointment"])).unwrap(); assert_eq!( - Value::DateTime(FeelDateTime::new(FeelDate::new(2021, 10, 12).unwrap(), FeelTime::local_opt(18, 35, 23, 0).unwrap())), + Value::DateTime(FeelDateTime::new(FeelDate::new(2021, 10, 12).unwrap(), FeelTime::local_opt(18, 35, 23, 0).unwrap()).into()), evaluator.eval(&DefKey::new(NAMESPACE, "tAppointment"), value).unwrap() ); } @@ -712,8 +746,8 @@ mod tests { let context = dsntk_feel_evaluator::evaluate_context(&Default::default(), context_str).unwrap(); let value = context.get_entry(&Name::new(&["Items"])).unwrap(); let expected = Value::List(vec![ - Value::DateTime(FeelDateTime::new(FeelDate::new(2021, 10, 10).unwrap(), FeelTime::local_opt(21, 23, 18, 0).unwrap())), - Value::DateTime(FeelDateTime::new(FeelDate::new(2021, 10, 11).unwrap(), FeelTime::local_opt(12, 18, 59, 0).unwrap())), + Value::DateTime(FeelDateTime::new(FeelDate::new(2021, 10, 10).unwrap(), FeelTime::local_opt(21, 23, 18, 0).unwrap()).into()), + Value::DateTime(FeelDateTime::new(FeelDate::new(2021, 10, 11).unwrap(), FeelTime::local_opt(12, 18, 59, 0).unwrap()).into()), ]); assert_eq!(expected, evaluator.eval(&DefKey::new(NAMESPACE, "tItems"), value).unwrap()); } diff --git a/model-evaluator/src/model_evaluator.rs b/model-evaluator/src/model_evaluator.rs index ac5ee39a..6293fdeb 100644 --- a/model-evaluator/src/model_evaluator.rs +++ b/model-evaluator/src/model_evaluator.rs @@ -134,7 +134,8 @@ impl ModelEvaluator { self .business_knowledge_model_evaluator .evaluate(def_key, &self.global_context, input_data, self, &mut evaluated_ctx); - if let Some(Value::FunctionDefinition(parameters, body, _external, _, closure_ctx, result_type)) = evaluated_ctx.get_entry(output_variable_name) { + if let Some(Value::FunctionDefinition(fd)) = evaluated_ctx.get_entry(output_variable_name) { + let (parameters, body, _external, _, closure_ctx, result_type) = &**fd; //TODO Handle external functions. let mut parameters_ctx = FeelContext::default(); parameters_ctx.zip(closure_ctx);