From cc8459bcff12e6be9adf1daad5f372f0c0f3f931 Mon Sep 17 00:00:00 2001 From: ZHU Yuhao Date: Thu, 9 Jul 2026 22:49:36 +0200 Subject: [PATCH] Move expression evaluator into core --- docs/changelog.md | 10 +++ src/cli/calculator/__init__.mojo | 13 +++- src/cli/calculator/engine.mojo | 4 +- src/cli/calculator/repl.mojo | 2 +- src/decimo/__init__.mojo | 3 + src/decimo/expression/__init__.mojo | 64 +++++++++++++++++ .../expression}/evaluator.mojo | 69 ++++++++++++++++--- .../expression}/parser.mojo | 2 +- .../expression}/tokenizer.mojo | 8 +-- src/decimo/prelude.mojo | 1 + .../test_error_handling.mojo | 3 +- tests/{cli => expression}/test_evaluator.mojo | 11 +-- tests/{cli => expression}/test_parser.mojo | 5 +- tests/{cli => expression}/test_tokenizer.mojo | 2 +- tests/test.sh | 6 +- 15 files changed, 171 insertions(+), 32 deletions(-) create mode 100644 src/decimo/expression/__init__.mojo rename src/{cli/calculator => decimo/expression}/evaluator.mojo (85%) rename src/{cli/calculator => decimo/expression}/parser.mojo (98%) rename src/{cli/calculator => decimo/expression}/tokenizer.mojo (98%) rename tests/{cli => expression}/test_error_handling.mojo (99%) rename tests/{cli => expression}/test_evaluator.mojo (98%) rename tests/{cli => expression}/test_parser.mojo (99%) rename tests/{cli => expression}/test_tokenizer.mojo (99%) diff --git a/docs/changelog.md b/docs/changelog.md index 00764b4b..0d1513a9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,16 @@ This is a list of changes for the Decimo package (formerly DeciMojo). +## Unreleased + +### ⭐️ New + +**Expression engine (`decimo.expression`):** + +1. The arithmetic-expression engine (tokenizer, shunting-yard parser, and RPN evaluator) that previously lived inside the CLI is now a first-class part of the core library under `decimo/expression/`. Users can evaluate a string in one call with the new high-level API `decimo.eval(expr, precision=50, variables={}, rounding_mode=...)`, e.g. `decimo.eval("100 + e * pi")`. Advanced users can import the individual stages via `from decimo.expression import tokenize, parse_to_rpn, evaluate_rpn`. `eval` optionally accepts a `variables` map so expressions can reference externally supplied named values (e.g. `eval("x^2 + y", variables=vars)`). `decimo.evaluate` is kept as an alias. +1. The CLI now re-uses this shared engine instead of its own copy, eliminating duplicated logic. Its presentation layer (`display`, `io`, `repl`, `settings`, `engine`) stays in the CLI. +1. The expression tokenizer now treats newline (`\n`) and carriage return (`\r`) as whitespace, so `eval` accepts strings with leading/trailing/embedded line breaks (e.g. triple-quoted expressions). + ## 20260701 (v0.11.0) Decimo v0.11.0 retargets the codebase to **Mojo v1.0.0b2**, adds the `factorial()` and `permutation()` functions to `BigInt` and `BigDecimal`, and includes a series of performance improvements for `BigDecimal` and `BigUInt` arithmetic. It also renames the `BigDecimal` `round_to_precision` APIs to `*_inplace`, which is a breaking change for code that calls them directly. diff --git a/src/cli/calculator/__init__.mojo b/src/cli/calculator/__init__.mojo index a6306a38..20c6e500 100644 --- a/src/cli/calculator/__init__.mojo +++ b/src/cli/calculator/__init__.mojo @@ -26,9 +26,18 @@ var result = evaluate("100 * 12 - 23/17", precision=50) ``` """ -from .tokenizer import ( +from decimo.expression import ( Token, tokenize, + parse_to_rpn, + evaluate_rpn, + final_round, + eval, + evaluate, + is_known_function, + is_known_constant, + is_alpha_or_underscore, + is_alnum_or_underscore, TOKEN_NUMBER, TOKEN_PLUS, TOKEN_MINUS, @@ -43,8 +52,6 @@ from .tokenizer import ( TOKEN_COMMA, TOKEN_VARIABLE, ) -from .parser import parse_to_rpn -from .evaluator import evaluate_rpn, evaluate from .engine import ( evaluate_and_print, evaluate_and_return, diff --git a/src/cli/calculator/engine.mojo b/src/cli/calculator/engine.mojo index 50bb07bf..e2074caf 100644 --- a/src/cli/calculator/engine.mojo +++ b/src/cli/calculator/engine.mojo @@ -25,9 +25,7 @@ used by both one-shot/pipe/file modes (main.mojo) and the interactive REPL from decimo import Decimal from decimo.rounding_mode import RoundingMode from std.collections import Dict -from .tokenizer import tokenize -from .parser import parse_to_rpn -from .evaluator import evaluate_rpn, final_round +from decimo.expression import tokenize, parse_to_rpn, evaluate_rpn, final_round from .display import print_error diff --git a/src/cli/calculator/repl.mojo b/src/cli/calculator/repl.mojo index 6e0c32a8..168eb4a5 100644 --- a/src/cli/calculator/repl.mojo +++ b/src/cli/calculator/repl.mojo @@ -43,7 +43,7 @@ from .display import print_error, format_about from .engine import evaluate_and_return from .io import strip, is_comment_or_blank from .settings import Settings, parse_settings, split_inline_settings, to_lower -from .tokenizer import ( +from decimo.expression import ( is_alpha_or_underscore, is_alnum_or_underscore, is_known_function, diff --git a/src/decimo/__init__.mojo b/src/decimo/__init__.mojo index dc7a0c19..96d20668 100644 --- a/src/decimo/__init__.mojo +++ b/src/decimo/__init__.mojo @@ -57,3 +57,6 @@ from .rounding_mode import ( # Core functions from .bigint.number_theory import gcd, lcm, extended_gcd, mod_inverse, mod_pow + +# Expression engine (high-level: `eval`; mid-level: `decimo.expression`) +from .expression import eval, evaluate diff --git a/src/decimo/expression/__init__.mojo b/src/decimo/expression/__init__.mojo new file mode 100644 index 00000000..c572bb42 --- /dev/null +++ b/src/decimo/expression/__init__.mojo @@ -0,0 +1,64 @@ +# ===----------------------------------------------------------------------=== # +# Copyright 2025-2026 Yuhao Zhu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ===----------------------------------------------------------------------=== # + +""" +Decimo expression engine. + +Parses and evaluates arbitrary-precision arithmetic expressions such as +`"100 + e * pi"` or `"sqrt(2) + 1/3"` using BigDecimal arithmetic. + +High-level API — evaluate a string in one call: + +```mojo +from decimo import eval + +var r = eval("100 + e * pi", precision=50) +``` + +Mid-level API — import the individual stages (tokenizer, parser, +evaluator) for advanced use: + +```mojo +from decimo.expression import tokenize, parse_to_rpn, evaluate_rpn + +var rpn = parse_to_rpn(tokenize("1 + 2 * 3")^) +var value = evaluate_rpn(rpn^, precision=50) +``` +""" + +from .tokenizer import ( + Token, + tokenize, + is_known_function, + is_known_constant, + is_alpha_or_underscore, + is_alnum_or_underscore, + TOKEN_NUMBER, + TOKEN_PLUS, + TOKEN_MINUS, + TOKEN_STAR, + TOKEN_SLASH, + TOKEN_LPAREN, + TOKEN_RPAREN, + TOKEN_UNARY_MINUS, + TOKEN_CARET, + TOKEN_FUNC, + TOKEN_CONST, + TOKEN_COMMA, + TOKEN_VARIABLE, +) +from .parser import parse_to_rpn +from .evaluator import evaluate_rpn, final_round, eval, evaluate diff --git a/src/cli/calculator/evaluator.mojo b/src/decimo/expression/evaluator.mojo similarity index 85% rename from src/cli/calculator/evaluator.mojo rename to src/decimo/expression/evaluator.mojo index 4679c5b1..d0ea9322 100644 --- a/src/cli/calculator/evaluator.mojo +++ b/src/decimo/expression/evaluator.mojo @@ -15,13 +15,13 @@ # ===----------------------------------------------------------------------=== # """ -RPN evaluator for the Decimo CLI calculator. +RPN evaluator for the Decimo expression engine. Evaluates a Reverse Polish Notation token list using BigDecimal arithmetic. """ -from decimo import Decimal -from decimo.rounding_mode import RoundingMode +from ..bigdecimal.bigdecimal import Decimal +from ..rounding_mode import RoundingMode from std.collections import Dict from .tokenizer import ( @@ -359,20 +359,45 @@ def final_round( return result^ -def evaluate( +def eval( expr: String, precision: Int = 50, + variables: Dict[String, Decimal] = Dict[String, Decimal](), rounding_mode: RoundingMode = RoundingMode.half_even(), ) raises -> Decimal: """Evaluate a math expression string and return a BigDecimal result. - This is the main entry point for the calculator engine. - It tokenizes, parses (shunting-yard), and evaluates (RPN) the expression. - The result is rounded to `precision` significant digits. + This is the high-level entry point for the Decimo expression engine. + It tokenizes, parses (shunting-yard), and evaluates (RPN) the + expression, then rounds the result to `precision` significant digits. + + ```mojo + from decimo import eval + + var r = eval("100 + e * pi") # default precision = 50 + var q = eval("1/3", precision=100) # 100 significant digits + ``` + + User-defined values can be injected via `variables`, letting an + expression reference named quantities supplied from outside: + + ```mojo + from std.collections import Dict + from decimo import eval, Decimal + + var vars = Dict[String, Decimal]() + vars["x"] = Decimal.from_string("10") + vars["y"] = Decimal.from_string("3") + var r = eval("x^2 + y", variables=vars) # -> 103 + ``` Args: expr: The math expression to evaluate (e.g. "100 * 12 - 23/17"). precision: The number of significant digits (default: 50). + variables: Optional name->value mapping of user-defined variables. + Identifiers matching a key resolve to the given value; unknown + identifiers (other than built-in constants `pi`/`e` and the + supported functions) raise an error. rounding_mode: The rounding mode for the final result (default: half_even). @@ -384,7 +409,33 @@ def evaluate( evaluated (e.g., syntax error, unknown identifier, division by zero, domain error in a math function). """ - var tokens = tokenize(expr) + var tokens = tokenize(expr, variables) var rpn = parse_to_rpn(tokens^) - var result = evaluate_rpn(rpn^, precision) + var result = evaluate_rpn(rpn^, precision, variables) return final_round(result, precision, rounding_mode) + + +def evaluate( + expr: String, + precision: Int = 50, + variables: Dict[String, Decimal] = Dict[String, Decimal](), + rounding_mode: RoundingMode = RoundingMode.half_even(), +) raises -> Decimal: + """Alias of `eval` kept for backwards compatibility. + + See `eval` for the full description and examples. + + Args: + expr: The math expression to evaluate. + precision: The number of significant digits (default: 50). + variables: Optional name->value mapping of user-defined variables. + rounding_mode: The rounding mode for the final result + (default: half_even). + + Returns: + The result as a BigDecimal, rounded to `precision` significant digits. + + Raises: + Error: If the expression cannot be tokenized, parsed, or evaluated. + """ + return eval(expr, precision, variables, rounding_mode) diff --git a/src/cli/calculator/parser.mojo b/src/decimo/expression/parser.mojo similarity index 98% rename from src/cli/calculator/parser.mojo rename to src/decimo/expression/parser.mojo index da3f243e..c427ef8f 100644 --- a/src/cli/calculator/parser.mojo +++ b/src/decimo/expression/parser.mojo @@ -15,7 +15,7 @@ # ===----------------------------------------------------------------------=== # """ -Shunting-Yard parser for the Decimo CLI calculator. +Shunting-Yard parser for the Decimo expression engine. Converts infix token lists to Reverse Polish Notation (RPN). """ diff --git a/src/cli/calculator/tokenizer.mojo b/src/decimo/expression/tokenizer.mojo similarity index 98% rename from src/cli/calculator/tokenizer.mojo rename to src/decimo/expression/tokenizer.mojo index eddefb11..a8e043f9 100644 --- a/src/cli/calculator/tokenizer.mojo +++ b/src/decimo/expression/tokenizer.mojo @@ -15,13 +15,13 @@ # ===----------------------------------------------------------------------=== # """ -Tokenizer for the Decimo CLI calculator. +Tokenizer for the Decimo expression engine. Converts an expression string into a list of tokens for the parser. """ from std.collections import Dict -from decimo import Decimal +from ..bigdecimal.bigdecimal import Decimal # ===----------------------------------------------------------------------=== # # Token kinds @@ -269,8 +269,8 @@ def tokenize( while i < n: var c = ptr[i] - # Skip whitespace (space, tab) - if c == 32 or c == 9: + # Skip whitespace (space, tab, newline, carriage return) + if c == 32 or c == 9 or c == 10 or c == 13: i += 1 continue diff --git a/src/decimo/prelude.mojo b/src/decimo/prelude.mojo index 476069be..9c962347 100644 --- a/src/decimo/prelude.mojo +++ b/src/decimo/prelude.mojo @@ -25,6 +25,7 @@ from decimo.prelude import * ``` """ +import decimo import decimo as dm from decimo.decimal128.decimal128 import Decimal128, Dec128 from decimo.bigdecimal.bigdecimal import BigDecimal, BDec, Decimal diff --git a/tests/cli/test_error_handling.mojo b/tests/expression/test_error_handling.mojo similarity index 99% rename from tests/cli/test_error_handling.mojo rename to tests/expression/test_error_handling.mojo index a78f6bad..37b14f52 100644 --- a/tests/cli/test_error_handling.mojo +++ b/tests/expression/test_error_handling.mojo @@ -7,8 +7,7 @@ negative sqrt, etc.). from std import testing -from calculator import evaluate -from calculator.tokenizer import tokenize +from decimo.expression import evaluate, tokenize # ===----------------------------------------------------------------------=== # diff --git a/tests/cli/test_evaluator.mojo b/tests/expression/test_evaluator.mojo similarity index 98% rename from tests/cli/test_evaluator.mojo rename to tests/expression/test_evaluator.mojo index c6b52fbd..96b9bb22 100644 --- a/tests/cli/test_evaluator.mojo +++ b/tests/expression/test_evaluator.mojo @@ -3,10 +3,13 @@ from std import testing from std.collections import Dict -from calculator import evaluate -from calculator.tokenizer import tokenize -from calculator.parser import parse_to_rpn -from calculator.evaluator import evaluate_rpn, final_round +from decimo.expression import ( + evaluate, + tokenize, + parse_to_rpn, + evaluate_rpn, + final_round, +) from decimo import Decimal from decimo.rounding_mode import RoundingMode diff --git a/tests/cli/test_parser.mojo b/tests/expression/test_parser.mojo similarity index 99% rename from tests/cli/test_parser.mojo rename to tests/expression/test_parser.mojo index 78c527c0..7fd5a164 100644 --- a/tests/cli/test_parser.mojo +++ b/tests/expression/test_parser.mojo @@ -2,7 +2,7 @@ from std import testing -from calculator.tokenizer import ( +from decimo.expression import ( Token, tokenize, TOKEN_NUMBER, @@ -15,9 +15,8 @@ from calculator.tokenizer import ( TOKEN_FUNC, TOKEN_CONST, TOKEN_COMMA, + parse_to_rpn, ) -from calculator.parser import parse_to_rpn - # ===----------------------------------------------------------------------=== # # Helper: convert token list to a compact string for easy assertion diff --git a/tests/cli/test_tokenizer.mojo b/tests/expression/test_tokenizer.mojo similarity index 99% rename from tests/cli/test_tokenizer.mojo rename to tests/expression/test_tokenizer.mojo index a567a405..545d7689 100644 --- a/tests/cli/test_tokenizer.mojo +++ b/tests/expression/test_tokenizer.mojo @@ -4,7 +4,7 @@ from std import testing from std.collections import Dict from decimo import Decimal -from calculator.tokenizer import ( +from decimo.expression import ( Token, tokenize, TOKEN_NUMBER, diff --git a/tests/test.sh b/tests/test.sh index 7522757a..9d895727 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -76,6 +76,7 @@ run_biguint() { run_mojo_suite biguint; } run_bigint10() { run_mojo_suite bigint10; } run_decimal128() { run_mojo_suite decimal128; } run_rational() { run_mojo_suite rational; } +run_expression() { run_mojo_suite expression; } run_toml() { run_mojo_suite toml; } run_bigfloat() { @@ -324,6 +325,7 @@ run_decimo() { run_bigint10 run_decimal128 run_rational + run_expression } run_all() { @@ -344,6 +346,7 @@ resolve() { bigint10|bint10|int10) echo "run_bigint10" ;; decimal128|dec128|d128) echo "run_decimal128" ;; rational|rat|frac) echo "run_rational" ;; + expression|expr|eval) echo "run_expression" ;; bigfloat|bfloat|float) echo "run_bigfloat" ;; toml) echo "run_toml" ;; cli) echo "run_cli" ;; @@ -365,11 +368,12 @@ list_suites() { printf " %-28s %s\n" "bigint10, bint10, int10" "BigInt10 tests" printf " %-28s %s\n" "decimal128, dec128, d128" "Decimal128 tests" printf " %-28s %s\n" "rational, rat, frac" "Rational number tests" + printf " %-28s %s\n" "expression, expr, eval" "Expression engine tests" printf " %-28s %s\n" "bigfloat, bfloat, float" "BigFloat tests (requires MPFR)" printf " %-28s %s\n" "toml" "TOML parser tests" printf " %-28s %s\n" "cli" "CLI calculator tests" printf " %-28s %s\n" "python, py" "Python binding tests" - printf " %-28s %s\n" "decimo, core" "All core suites (bdec+bint+buint+bint10+dec128+rational)" + printf " %-28s %s\n" "decimo, core" "All core suites (bdec+bint+buint+bint10+dec128+rational+expression)" printf " %-28s %s\n" "all" "Everything (decimo + toml + cli)" }