From 1eb18aa5a481e041dad0624ef3a500f7baeda85c Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Mon, 9 Nov 2026 14:02:01 -0600 Subject: [PATCH 01/10] Validate temporal training bounds before generation --- .../kumo-core/src/kumo_core/model_plan.py | 15 +++ .../src/kumo_core/training/bounds.py | 64 +++++++++++ .../kumo-core/tests/test_training_bounds.py | 104 ++++++++++++++++++ 3 files changed, 183 insertions(+) create mode 100644 packages/kumo-core/src/kumo_core/training/bounds.py create mode 100644 packages/kumo-core/tests/test_training_bounds.py diff --git a/packages/kumo-core/src/kumo_core/model_plan.py b/packages/kumo-core/src/kumo_core/model_plan.py index b2eddde77..2e6eba069 100644 --- a/packages/kumo-core/src/kumo_core/model_plan.py +++ b/packages/kumo-core/src/kumo_core/model_plan.py @@ -23,6 +23,7 @@ import kumoapi.encoder as encoder_api import kumoapi.model_plan as api from kumo_core.errors import TrainingValidationError as ValidationException +from kumo_core.training.bounds import validate_time_bounds from kumo_core.types import TaskType from kumoapi.common import ValidationWarning from kumoapi.encoder import ( @@ -838,6 +839,7 @@ def validate( "model plan."))) errors.extend(unsupported_entity_candidate_errors(self)) + split = None if self.split != InferredType.VALUE: try: split = self.get_split(self.split) @@ -954,6 +956,19 @@ def validate( f"Forecasting requires a {acceptable_splits} " f"split. Please choose one of these splits or " f"set `split` to `inferred`.")) + if query_type == QueryType.TEMPORAL and task_type.is_node_pred: + try: + validate_time_bounds( + inferred_or_missing_type_to_none(self.start_time), + inferred_or_missing_type_to_none(self.end_time), + train_start_offset=inferred_or_missing_type_to_none( + self.train_start_offset), + train_end_offset=inferred_or_missing_type_to_none( + self.train_end_offset), + split_family=None if split is None else split.name, + ) + except ValueError as error: + errors.append(error) return errors, warnings @classmethod diff --git a/packages/kumo-core/src/kumo_core/training/bounds.py b/packages/kumo-core/src/kumo_core/training/bounds.py new file mode 100644 index 000000000..f42d0956c --- /dev/null +++ b/packages/kumo-core/src/kumo_core/training/bounds.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Validation and UTC normalization for temporal label-window bounds.""" + +from pandas import Timestamp, isna + + +def parse_time_bound(value: int | str | None, + name: str) -> int | Timestamp | None: + """Keep relative offsets numerical and normalize absolute times to UTC.""" + if value is None or (name == 'start_time' and value == '-INF'): + return None + if isinstance(value, int): + if value > 0: + raise ValueError(f"'{name}' must be a non-positive relative offset") + return value + try: + timestamp = Timestamp(value) + if isna(timestamp): + raise ValueError('timestamp is NaT') + if timestamp.tzinfo is not None: + timestamp = timestamp.tz_convert('UTC').tz_localize(None) + # Generation stores timestamps as signed nanoseconds. + timestamp = Timestamp(timestamp.value) + except (ValueError, TypeError, OverflowError) as error: + raise ValueError(f"'{name}' must be a valid timestamp: {value!r}") from error + return timestamp + + +def validate_time_bounds( + start: int | str | None, + end: int | str | None, + *, + train_start_offset: int | None = None, + train_end_offset: int | None = None, + split_family: str | None = None, +) -> tuple[int | Timestamp | None, int | Timestamp | None]: + """Validate explicit parameters without changing their serialized form.""" + if ((start is not None or end is not None) + and (train_start_offset is not None + or train_end_offset is not None)): + raise ValueError('Cannot specify both start_time/end_time and ' + 'train_start_offset/train_end_offset.') + parsed_start = parse_time_bound(start, 'start_time') + parsed_end = parse_time_bound(end, 'end_time') + if parsed_start is not None and parsed_end is not None: + if isinstance(parsed_start, int) != isinstance(parsed_end, int): + raise ValueError('start_time and end_time must both be offsets ' + 'or both be timestamps.') + if ((isinstance(parsed_start, int) and isinstance(parsed_end, int) + and parsed_start >= parsed_end) + or (isinstance(parsed_start, Timestamp) + and isinstance(parsed_end, Timestamp) + and parsed_start >= parsed_end)): + raise ValueError('start_time must be smaller than end_time.') + if (isinstance(parsed_start, Timestamp) + or isinstance(parsed_end, Timestamp)) and split_family in { + 'BacktestingSplit', 'DateOffsetSplit', 'TimeRangeSplit', + 'TimeSplit', + }: + raise ValueError('Absolute start_time/end_time cannot be combined ' + f'with {split_family}; use a ratio split or inferred.') + return parsed_start, parsed_end diff --git a/packages/kumo-core/tests/test_training_bounds.py b/packages/kumo-core/tests/test_training_bounds.py new file mode 100644 index 000000000..c98e5d700 --- /dev/null +++ b/packages/kumo-core/tests/test_training_bounds.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +from kumo_core.model_plan import TrainingTableGenerationPlan +from kumo_core.training.bounds import validate_time_bounds +from kumo_core.types.enums import TaskType +from kumoapi.model_plan import RunMode +from kumoapi.pquery import QueryType +from pandas import Timestamp + + +@pytest.mark.parametrize('start,end,message', [ + ('invalid-date', None, 'valid timestamp'), + (None, 'NaT', 'valid timestamp'), + ('2024-01-02', '2024-01-02', 'smaller than'), + ('2024-01-03', '2024-01-02', 'smaller than'), + (-2, '2024-01-02', 'both be offsets'), + (1, None, 'non-positive'), + (None, 1, 'non-positive'), + (-1, -2, 'smaller than'), + (None, '3000-01-01', 'valid timestamp'), +]) +def test_invalid_bounds(start, end, message): + with pytest.raises(ValueError, match=message): + validate_time_bounds(start, end) + + +def test_bounds_normalize_utc(): + assert validate_time_bounds( + '2024-01-01T00:00:00-06:00', + '2024-01-01T08:00:00+01:00', + ) == (Timestamp('2024-01-01T06:00:00'), + Timestamp('2024-01-01T07:00:00')) + assert validate_time_bounds('-INF', '2024-01-02') == ( + None, Timestamp('2024-01-02')) + assert validate_time_bounds(None, -1) == (None, -1) + + +@pytest.mark.parametrize('split', + ['BacktestingSplit', 'DateOffsetSplit', + 'TimeRangeSplit']) +def test_absolute_split_conflict(split): + with pytest.raises(ValueError, match='cannot be combined'): + validate_time_bounds(None, '2024-01-02', split_family=split) + + +@pytest.mark.parametrize('split', + ['ApproxDateOffsetSplit', 'RandomSplit', + 'TemporalSplit']) +def test_absolute_ratio_split(split): + assert validate_time_bounds('2024-01-01', None, split_family=split) == ( + Timestamp('2024-01-01'), None) + + +@pytest.mark.parametrize('flag,expected_errors', [ + ('false', True), ('true', False), +]) +def test_integer_bounds_keep_gate(flag, expected_errors, monkeypatch): + monkeypatch.setenv('USE_DECOUPLED_SPLITS', flag) + errors, _ = TrainingTableGenerationPlan(start_time=-10, end_time=0).validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + assert bool(errors) == expected_errors + + +@pytest.mark.parametrize('task_type', TaskType.get_node_pred_tasks()) +def test_node_bound_validation(task_type, monkeypatch): + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'true') + plan = TrainingTableGenerationPlan( + start_time='2024-01-01T00:00:00-06:00', + end_time='2024-01-02T00:00:00-06:00', + ) + errors, _ = plan.validate( + task_type=task_type, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + assert not errors + assert plan.start_time == '2024-01-01T00:00:00-06:00' + + +@pytest.mark.parametrize('options,message', [ + ({'start_time': 'NaT'}, 'valid timestamp'), + ({'start_time': '2024-01-01', 'train_start_offset': 2}, + 'Cannot specify both'), + ({'start_time': '2024-01-01', + 'split': "TimeRangeSplit([('2024-01-01', '2024-01-02')])"}, + 'cannot be combined'), +]) +def test_plan_rejects_invalid_bounds(options, message, monkeypatch): + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'true') + errors, _ = TrainingTableGenerationPlan(**options).validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + assert any(message in str(error) for error in errors) From 1ca2c547baa9e771df706626332a8351c065caec Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Mon, 9 Nov 2026 14:08:15 -0600 Subject: [PATCH 02/10] Support absolute label bounds for temporal node training --- .../kumo-core/src/kumo_core/model_plan.py | 11 +- packages/kumo-core/src/kumo_core/pquery.py | 68 +++++++++-- packages/kumo-core/tests/test_pquery.py | 108 +++++++++++++++++- .../kumo-core/tests/test_training_bounds.py | 20 +++- .../kumo-engine/test/test_predictive_query.py | 17 +-- test/config/test_model_plan.py | 17 ++- 6 files changed, 203 insertions(+), 38 deletions(-) diff --git a/packages/kumo-core/src/kumo_core/model_plan.py b/packages/kumo-core/src/kumo_core/model_plan.py index 2e6eba069..6dff9a534 100644 --- a/packages/kumo-core/src/kumo_core/model_plan.py +++ b/packages/kumo-core/src/kumo_core/model_plan.py @@ -796,11 +796,14 @@ def is_valid_option( has_train_table_weight_col): return False - # Check split-related options based on decoupled splits setting - # When using decoupled splits, only allow start/end time - # When using legacy splits, only allow train start/end offset + # Absolute node bounds also work with the default split path. + # Relative start/end offsets retain the decoupled-splits gate. if name in ['start_time', 'end_time']: - return _use_decoupled_splits() + value = self.start_time if name == 'start_time' else self.end_time + return (_use_decoupled_splits() + or (task_type.is_node_pred and isinstance(value, str) + and not _is_missing(value) + and not _is_inferred(value))) if name in ['train_start_offset', 'train_end_offset']: return not _use_decoupled_splits() diff --git a/packages/kumo-core/src/kumo_core/pquery.py b/packages/kumo-core/src/kumo_core/pquery.py index 60517d2f8..7cc47bd39 100644 --- a/packages/kumo-core/src/kumo_core/pquery.py +++ b/packages/kumo-core/src/kumo_core/pquery.py @@ -59,6 +59,7 @@ PADDING_SIZE, YOY_COL, ) +from .training.bounds import validate_time_bounds from .training.splits import ( SplitArguments, TrainTableTime, @@ -1032,19 +1033,53 @@ def date_offset(value: int) -> DateOffset: ) if error is not None: raise PredictiveQueryPlanError(f"{error.title}. {error.message}") - if explicit: - return (tuple(_timestamp_ns(value) for value in explicit), - "flexible_boundaries" in split) - minimum, maximum, filter_buffer = _temporal_target_time_range( - query, index_path) - def relative(value: Any) -> DateOffset | Timestamp | None: + def relative(value: int | str | None) -> DateOffset | Timestamp | None: if value is None or value == "-INF": return None if isinstance(value, Integral): return date_offset(int(value)) return Timestamp(Timestamp(value).value) + start_time: DateOffset | Timestamp | int | None + end_time: DateOffset | Timestamp | int | None + node_bounds = query.task_type in TaskType.get_node_pred_tasks() + if not node_bounds: + start_time = relative( + inferred_or_missing_type_to_none(options.start_time)) + end_time = relative(inferred_or_missing_type_to_none(options.end_time)) + else: + try: + start_time, end_time = validate_time_bounds( + inferred_or_missing_type_to_none(options.start_time), + inferred_or_missing_type_to_none(options.end_time), + train_start_offset=train_start, + train_end_offset=train_end, + split_family=(split_family + if inferred_or_missing_type_to_none( + options.split) is not None else None), + ) + except ValueError as error: + raise PredictiveQueryPlanError(str(error)) from error + if explicit: + return (tuple(_timestamp_ns(value) for value in explicit), + "flexible_boundaries" in split) + minimum, maximum, filter_buffer = _temporal_target_time_range( + query, index_path) + absolute_bounds = (isinstance(start_time, Timestamp) + or isinstance(end_time, Timestamp)) and node_bounds + if absolute_bounds and isinstance(start_time, Timestamp): + start_time = max(start_time, minimum) + if absolute_bounds and isinstance(end_time, Timestamp): + end_time = min(end_time, maximum) + if absolute_bounds and query.task_type == "forecasting": + horizon = query.forecast_length - 1 + if horizon > 0: + timeframe = DateOffsetRange.date_offset_subtract( + offset.end_date_offset, date_offset(int(offset.start or 0))) + cutoff = end_time if isinstance(end_time, Timestamp) else maximum + end_time = cutoff - horizon * timeframe + step_count = int( inferred_or_missing_type_to_none(options.timeframe_step) or (int(offset.end) - int(offset.start or 0))) @@ -1072,11 +1107,26 @@ def relative(value: Any) -> DateOffset | Timestamp | None: int(train_start))), train_end_offset=(None if train_end is None else date_offset( int(train_end))), - start_time=relative( - inferred_or_missing_type_to_none(options.start_time)), - end_time=relative(inferred_or_missing_type_to_none(options.end_time)), + start_time=(date_offset(start_time) + if isinstance(start_time, int) else start_time), + end_time=(date_offset(end_time) + if isinstance(end_time, int) else end_time), drop_earliest=True, ) + if absolute_bounds: + # Inferred offset splits may extend their first range to fill a stage. + # Absolute label bounds remain authoritative over that adjustment. + lower = start_time if isinstance(start_time, Timestamp) else minimum + cutoff = end_time if isinstance(end_time, Timestamp) else maximum + upper = cutoff - offset.end_date_offset + timestamps = [ + timestamp for timestamp in timestamps if lower <= timestamp <= upper + ] + if absolute_bounds and not timestamps: + raise PredictiveQueryPlanError( + "No usable training examples within start_time/end_time and " + "the available label data. Widen the bounds or provide more data " + "for the target horizon.") return tuple(timestamp.value for timestamp in timestamps), fixed_boundaries diff --git a/packages/kumo-core/tests/test_pquery.py b/packages/kumo-core/tests/test_pquery.py index f8d26fc14..d34a4c40b 100644 --- a/packages/kumo-core/tests/test_pquery.py +++ b/packages/kumo-core/tests/test_pquery.py @@ -703,11 +703,6 @@ def test_plan_rejects_explicit_timestamps_on_a_static_query(tmp_path): "start_time": "-INF", "end_time": -10 }, 1, 29), - ({ - "train_end_offset": 10, - "start_time": "2024-01-11", - "end_time": "2024-01-21" - }, 0, 29), ]) def test_generation_plan_time_bounds(tmp_path, monkeypatch, options, first_day, last_day, unit): @@ -742,6 +737,106 @@ def test_generation_plan_time_bounds(tmp_path, monkeypatch, options, first_day, start + day * index for index in range(last_day, first_day - 1, -1)) +@pytest.mark.parametrize('start,end,step,window,expected', [ + ('2024-01-05', '2024-01-15', 3, (0, 3), [12, 9, 6]), + ('2023-12-01', '2024-02-01', 3, (0, 3), [17, 14, 11, 8, 5, 2]), + ('2024-01-05', None, 3, (0, 3), [17, 14, 11, 8, 5]), + (None, '2024-01-15', 3, (0, 3), [12, 9, 6]), + ('2024-01-05', '2024-01-16', 4, (0, 3), [13, 9, 5]), + ('2024-01-05', '2024-01-15', 2, (1, 3), [12, 10, 8, 6]), + ('2024-01-05T01:00:00+01:00', '2024-01-15T01:00:00+01:00', + 3, (0, 3), [12, 9, 6]), +]) +def test_absolute_bounds_intersect_available_labels( + monkeypatch, start, end, step, window, expected): + from pandas import Timestamp + + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + f'PREDICT COUNT(events.*, {window[0]}, {window[1]}, days) ' + 'FOR EACH articles.aid', graph) + monkeypatch.setattr( + 'kumo_core.pquery._temporal_target_time_range', + lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) + options = TrainingTableGenerationPlan(timeframe_step=step) + if start is not None: + options.start_time = start + if end is not None: + options.end_time = end + timestamps, _ = _temporal_timestamps( + query, '/synthetic-index', options, {}, 'ApproxDateOffsetSplit', ()) + assert timestamps == tuple( + Timestamp(f'2024-01-{day:02}').value for day in expected) + + +@pytest.mark.parametrize('start,end', [ + ('2024-02-01', '2024-02-10'), + ('2023-12-01', '2023-12-10'), + ('2024-01-19', '2024-01-20'), +]) +def test_absolute_bounds_require_usable_windows(monkeypatch, start, end): + from pandas import Timestamp + + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + 'PREDICT COUNT(events.*, 0, 3, days) FOR EACH articles.aid', graph) + monkeypatch.setattr( + 'kumo_core.pquery._temporal_target_time_range', + lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) + with pytest.raises(PredictiveQueryPlanError, match='No usable training'): + _temporal_timestamps( + query, '/synthetic-index', TrainingTableGenerationPlan( + start_time=start, end_time=end), {}, 'ApproxDateOffsetSplit', ()) + + +@pytest.mark.parametrize('options', [ + {'start_time': 'NaT'}, + {'start_time': '2024-01-11', 'end_time': '2024-01-21', + 'train_end_offset': 10}, + {'start_time': '2024-01-11', + 'split': "TimeRangeSplit([('2024-01-01', '2024-02-01')])"}, +]) +@pytest.mark.parametrize('explicit', [(), (1704067200000000000,)]) +def test_invalid_absolute_bounds_rejected_before_stats( + monkeypatch, options, explicit): + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + 'PREDICT COUNT(events.*, 0, 1, days) FOR EACH articles.aid', graph) + monkeypatch.setattr( + 'kumo_core.pquery._temporal_target_time_range', + lambda *_args: pytest.fail('Read stats before rejecting invalid bounds')) + with pytest.raises(PredictiveQueryPlanError): + _temporal_timestamps( + query, '/synthetic-index', TrainingTableGenerationPlan(**options), + {}, 'TimeRangeSplit' if 'split' in options else 'ApproxDateOffsetSplit', + explicit) + + +@pytest.mark.parametrize('end', [None, '2024-01-15']) +def test_absolute_forecast_bounds_reserve_full_horizon(monkeypatch, end): + from pandas import DateOffset, Timestamp + + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + 'PREDICT COUNT(events.*, 1, 3, days) FORECAST 3 TIMEFRAMES ' + 'FOR EACH articles.aid', graph) + monkeypatch.setattr( + 'kumo_core.pquery._temporal_target_time_range', + lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) + options = TrainingTableGenerationPlan( + start_time='2024-01-05', timeframe_step=2) + if end is not None: + options.end_time = end + split, family = _split_options(query, graph, options) + timestamps, _ = _temporal_timestamps( + query, '/synthetic-index', options, split, family, ()) + cutoff = Timestamp(end or '2024-01-20') + newest = Timestamp(timestamps[0]) + assert newest == cutoff - DateOffset(days=7) + assert all(Timestamp(value) >= Timestamp('2024-01-05') + for value in timestamps) + + def test_latest_temporal_timestamp_uses_source_maximum(tmp_path, monkeypatch): minimum = 1_704_067_200_000_000_000 maximum = 1_711_843_200_000_000_000 @@ -972,8 +1067,9 @@ def test_date_offset_split_uses_bounded_range_for_each_phase( graph_index_path=str(tmp_path / "index"), scratch_dir=str(tmp_path), artifact_dir=str(tmp_path)) + # Reserve all three future labels before restarting steps at each split. assert plan.train_table.timestamps == tuple( - (origin + index * day).value for index in (9, 7, 6, 4, 3, 1)) + (origin + index * day).value for index in (7, 5, 4, 2, 1)) def test_nonzero_target_offset_restarts_step_at_each_split_boundary( diff --git a/packages/kumo-core/tests/test_training_bounds.py b/packages/kumo-core/tests/test_training_bounds.py index c98e5d700..1d76ce3fc 100644 --- a/packages/kumo-core/tests/test_training_bounds.py +++ b/packages/kumo-core/tests/test_training_bounds.py @@ -69,8 +69,12 @@ def test_integer_bounds_keep_gate(flag, expected_errors, monkeypatch): @pytest.mark.parametrize('task_type', TaskType.get_node_pred_tasks()) -def test_node_bound_validation(task_type, monkeypatch): - monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'true') +@pytest.mark.parametrize('flag', [None, 'false', 'true']) +def test_node_bound_validation(task_type, flag, monkeypatch): + if flag is None: + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + else: + monkeypatch.setenv('USE_DECOUPLED_SPLITS', flag) plan = TrainingTableGenerationPlan( start_time='2024-01-01T00:00:00-06:00', end_time='2024-01-02T00:00:00-06:00', @@ -85,6 +89,18 @@ def test_node_bound_validation(task_type, monkeypatch): assert plan.start_time == '2024-01-01T00:00:00-06:00' +def test_link_bounds_keep_gate(monkeypatch): + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + errors, _ = TrainingTableGenerationPlan(start_time='2024-01-01').validate( + task_type=TaskType.TEMPORAL_LINK_PREDICTION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + assert any("'start_time' field is not supported" in str(error) + for error in errors) + + @pytest.mark.parametrize('options,message', [ ({'start_time': 'NaT'}, 'valid timestamp'), ({'start_time': '2024-01-01', 'train_start_offset': 2}, diff --git a/packages/kumo-engine/test/test_predictive_query.py b/packages/kumo-engine/test/test_predictive_query.py index c4af2df11..7ad89704f 100644 --- a/packages/kumo-engine/test/test_predictive_query.py +++ b/packages/kumo-engine/test/test_predictive_query.py @@ -334,8 +334,13 @@ def test_generated_temporal_split_requires_time_column(graph_snapshot, split='TemporalSplit([0.6, 0.3, 0.1])')) -def test_generated_absolute_bounds_name_supported_alternative( - graph_snapshot, monkeypatch): +@pytest.mark.parametrize('start,end,message', [ + ('NaT', '2025-01-21', 'valid timestamp'), + ('2025-01-21', '2025-01-02', 'smaller than'), + ('2025-01-02', -1, 'both be offsets'), +]) +def test_generated_invalid_bounds_fail_before_artifacts( + graph_snapshot, monkeypatch, start, end, message): graph, snapshot = graph_snapshot monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) monkeypatch.setattr( @@ -343,17 +348,15 @@ def test_generated_absolute_bounds_name_supported_alternative( lambda: pytest.fail('allocated a bounded-generation run id'), ) - with pytest.raises( - ValueError, - match="Use 'train_start_offset' and 'train_end_offset' instead"): + with pytest.raises(ValueError, match=message): PredictiveQuery( graph, 'PREDICT COUNT(orders.*, 0, 1, days) > 0 FOR EACH users.id', ).generate_training_table( snapshot, TrainingTableGenerationPlan( - start_time='2025-01-02T12:00:00', - end_time='2025-01-21T12:00:00', + start_time=start, + end_time=end, timeframe_step=1, split='ApproxDateOffsetSplit([0.6, 0.2, 0.2])', ), diff --git a/test/config/test_model_plan.py b/test/config/test_model_plan.py index 2b8afefa1..74fbdfadb 100644 --- a/test/config/test_model_plan.py +++ b/test/config/test_model_plan.py @@ -2368,18 +2368,15 @@ def test_model_plan_decoupled_splits(monkeypatch): plan.training_table_generation.train_start_offset == InferredType.VALUE) assert plan.training_table_generation.start_time == MissingType.VALUE - # Verify validation names the supported relative-bound alternative. + # Absolute bounds are supported alongside inferred legacy defaults. plan.training_table_generation.start_time = "2024-01-01" plan.training_table_generation.end_time = "2024-02-01" - with pytest.raises(ValueError) as error: - plan.validate( - task_type=TaskType.BINARY_CLASSIFICATION, - query_type=QueryType.TEMPORAL, - has_train_table_time_col=True, - run_mode=RunMode.FAST, - ) - assert str(error.value).count( - "Use 'train_start_offset' and 'train_end_offset' instead.") == 2 + plan.validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) # Static queries cannot use either absolute or relative bounds. static_plan = ModelPlan.from_default( From 0f0a0552a56ad7a5e48571e867ea59531ed9b468 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Mon, 9 Nov 2026 14:15:22 -0600 Subject: [PATCH 03/10] Verify absolute training bounds through model reload Document temporal node bounds and cover native generation, full forecast horizons, early validation, and CPU fit/save/load/prediction. Share compiled forecast shifts and reject unusable edge windows safely. --- packages/kumo-api/kumoapi/model_plan.py | 39 ++- packages/kumo-core/src/kumo_core/pquery.py | 46 ++-- .../src/kumo_core/training/bounds.py | 6 +- packages/kumo-core/tests/test_pquery.py | 47 ++++ packages/kumo-engine/README.md | 37 +++ .../test/test_absolute_training_bounds.py | 238 ++++++++++++++++++ .../kumo-engine/test/test_predictive_query.py | 37 ++- 7 files changed, 407 insertions(+), 43 deletions(-) create mode 100644 packages/kumo-engine/test/test_absolute_training_bounds.py diff --git a/packages/kumo-api/kumoapi/model_plan.py b/packages/kumo-api/kumoapi/model_plan.py index 95c8c070d..e6dea7173 100644 --- a/packages/kumo-api/kumoapi/model_plan.py +++ b/packages/kumo-api/kumoapi/model_plan.py @@ -619,19 +619,32 @@ class TrainingTableGenerationPlan(PlanMixin): like to migrate this parameter to end_time. **Supported Task Types:** Temporal - :ivar start_time: (``int`` | ``"inferred"`` | ``str``) Defines the absolute - start time to use for generating training data. Can be specified - either as a pandas Timestamp compatible string (e.g. '2024-04-01') or - as a non-positive integer offset in days from the current time - (e.g. -30 for 30 days ago) (*default:* ``"inferred"``). - **Supported Task Types:** Temporal - - :ivar end_time: (``int`` | ``"inferred"`` | ``str``) Defines the absolute - end time to use for generating training data. Can be specified either - as a pandas Timestamp compatible string (e.g. '2024-04-01') or as a - non-positive integer offset in days from the current time - (e.g. -30 for 30 days ago) (*default:* ``"inferred"``). - **Supported Task Types:** Temporal + :ivar start_time: (``int`` | ``"inferred"`` | ``str``) Inclusive lower + bound on generated prediction anchors. Temporal node tasks accept + pandas Timestamp compatible strings (e.g. '2024-04-01') with default + splits. Timezone-aware strings are normalized to UTC; naive strings + are read as UTC. ``"inferred"`` or ``"-INF"`` uses the available-data + lower bound. Bounds are intersected with available label data. + Direct absolute bounds cannot be combined with train_start_offset, + train_end_offset, or explicit date/offset window splits. Ratio splits + remain subject to existing task restrictions. + Non-positive integer offsets retain the decoupled-splits gate and + are relative to the latest label data, in the aggregation's time unit. + (*default:* ``"inferred"``). + **Supported Task Types:** Temporal node tasks by default; temporal + link prediction retains its existing decoupled-splits requirement. + + :ivar end_time: (``int`` | ``"inferred"`` | ``str``) Label observation + cutoff, rather than the latest prediction anchor. Each generated + example must fit its complete target horizon before this cutoff, + including every future label for forecasting. Accepts the same + timestamp strings and gated integer offsets as start_time; + ``"inferred"`` uses available label data. Either absolute bound can + be omitted. Empty intersections or windows too short for a complete + example fail generation. Existing time-unit and split constraints + continue to apply. (*default:* ``"inferred"``). + **Supported Task Types:** Temporal node tasks by default; temporal + link prediction retains its existing decoupled-splits requirement. :ivar timeframe_step: (``int`` | ``"inferred"``) Defines the step size of generating time intervals for training table generation diff --git a/packages/kumo-core/src/kumo_core/pquery.py b/packages/kumo-core/src/kumo_core/pquery.py index 7cc47bd39..e5632ef17 100644 --- a/packages/kumo-core/src/kumo_core/pquery.py +++ b/packages/kumo-core/src/kumo_core/pquery.py @@ -756,6 +756,13 @@ def resolve_query_forecast( ) +def _forecast_offsets(query: ValidatedNativeQuery) -> tuple[int, ...]: + """The exact future shifts passed to the native label compiler.""" + timeframe = query.query.get_combined_date_offset_range().timeframe + step_seconds = int(timeframe.total_seconds()) + return tuple(step_seconds * index for index in range(query.forecast_length)) + + def _forecast_options( query: ValidatedNativeQuery, options: TrainingTableGenerationPlan, @@ -774,8 +781,7 @@ def _forecast_options( step_seconds = int(timeframe.total_seconds()) lag_offsets = tuple(-step_seconds * index for index in range(int(lag_timesteps), 0, -1)) - future_offsets = tuple(step_seconds * index - for index in range(query.forecast_length)) + future_offsets = _forecast_offsets(query) compiler_options = { "future_offsets_secs": future_offsets, "lag_offsets_secs": lag_offsets, @@ -1072,13 +1078,26 @@ def relative(value: int | str | None) -> DateOffset | Timestamp | None: start_time = max(start_time, minimum) if absolute_bounds and isinstance(end_time, Timestamp): end_time = min(end_time, maximum) + label_cutoff = end_time if isinstance(end_time, Timestamp) else maximum + lower = start_time if isinstance(start_time, Timestamp) else minimum + empty_bounds = ( + "No usable training examples within start_time/end_time and " + "the available label data. Widen the bounds or provide more data " + "for the target horizon.") + if absolute_bounds and lower >= label_cutoff: + raise PredictiveQueryPlanError(empty_bounds) + future_shift = Timedelta(0) if absolute_bounds and query.task_type == "forecasting": - horizon = query.forecast_length - 1 - if horizon > 0: - timeframe = DateOffsetRange.date_offset_subtract( - offset.end_date_offset, date_offset(int(offset.start or 0))) - cutoff = end_time if isinstance(end_time, Timestamp) else maximum - end_time = cutoff - horizon * timeframe + future_shift = Timedelta(seconds=max(_forecast_offsets(query))) + if label_cutoff.value - lower.value <= future_shift.value: + raise PredictiveQueryPlanError(empty_bounds) + shifted_cutoff = label_cutoff - future_shift + if not isinstance(shifted_cutoff, Timestamp): + raise PredictiveQueryPlanError(empty_bounds) + end_time = shifted_cutoff + cutoff = end_time if isinstance(end_time, Timestamp) else maximum + if absolute_bounds and lower >= cutoff: + raise PredictiveQueryPlanError(empty_bounds) step_count = int( inferred_or_missing_type_to_none(options.timeframe_step) @@ -1116,17 +1135,12 @@ def relative(value: int | str | None) -> DateOffset | Timestamp | None: if absolute_bounds: # Inferred offset splits may extend their first range to fill a stage. # Absolute label bounds remain authoritative over that adjustment. - lower = start_time if isinstance(start_time, Timestamp) else minimum - cutoff = end_time if isinstance(end_time, Timestamp) else maximum - upper = cutoff - offset.end_date_offset timestamps = [ - timestamp for timestamp in timestamps if lower <= timestamp <= upper + timestamp for timestamp in timestamps if timestamp >= lower + and timestamp + future_shift + offset.end_date_offset <= label_cutoff ] if absolute_bounds and not timestamps: - raise PredictiveQueryPlanError( - "No usable training examples within start_time/end_time and " - "the available label data. Widen the bounds or provide more data " - "for the target horizon.") + raise PredictiveQueryPlanError(empty_bounds) return tuple(timestamp.value for timestamp in timestamps), fixed_boundaries diff --git a/packages/kumo-core/src/kumo_core/training/bounds.py b/packages/kumo-core/src/kumo_core/training/bounds.py index f42d0956c..33e1a8ae5 100644 --- a/packages/kumo-core/src/kumo_core/training/bounds.py +++ b/packages/kumo-core/src/kumo_core/training/bounds.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """Validation and UTC normalization for temporal label-window bounds.""" -from pandas import Timestamp, isna +from pandas import Timestamp def parse_time_bound(value: int | str | None, @@ -17,12 +17,14 @@ def parse_time_bound(value: int | str | None, return value try: timestamp = Timestamp(value) - if isna(timestamp): + if not isinstance(timestamp, Timestamp): raise ValueError('timestamp is NaT') if timestamp.tzinfo is not None: timestamp = timestamp.tz_convert('UTC').tz_localize(None) # Generation stores timestamps as signed nanoseconds. timestamp = Timestamp(timestamp.value) + if not isinstance(timestamp, Timestamp): + raise ValueError('timestamp is NaT') except (ValueError, TypeError, OverflowError) as error: raise ValueError(f"'{name}' must be a valid timestamp: {value!r}") from error return timestamp diff --git a/packages/kumo-core/tests/test_pquery.py b/packages/kumo-core/tests/test_pquery.py index d34a4c40b..8af614132 100644 --- a/packages/kumo-core/tests/test_pquery.py +++ b/packages/kumo-core/tests/test_pquery.py @@ -773,6 +773,8 @@ def test_absolute_bounds_intersect_available_labels( ('2024-02-01', '2024-02-10'), ('2023-12-01', '2023-12-10'), ('2024-01-19', '2024-01-20'), + ('1678-01-01', '1679-01-01'), + ('2260-01-01', '2261-01-01'), ]) def test_absolute_bounds_require_usable_windows(monkeypatch, start, end): from pandas import Timestamp @@ -837,6 +839,51 @@ def test_absolute_forecast_bounds_reserve_full_horizon(monkeypatch, end): for value in timestamps) +@pytest.mark.parametrize('minimum,maximum', [ + ('2024-01-01', '2024-01-20'), + ('1677-09-22', '1677-09-23'), +]) +def test_forecast_bounds_rejected_before_horizon_underflow( + monkeypatch, minimum, maximum): + from pandas import Timestamp + + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + 'PREDICT COUNT(events.*, 0, 1, days) FORECAST 3 TIMEFRAMES ' + 'FOR EACH articles.aid', graph) + monkeypatch.setattr( + 'kumo_core.pquery._temporal_target_time_range', + lambda *_args: (Timestamp(minimum), Timestamp(maximum), None)) + with pytest.raises(PredictiveQueryPlanError, match='No usable training'): + _temporal_timestamps( + query, '/synthetic-index', TrainingTableGenerationPlan( + start_time='1677-09-22', end_time='1677-09-23'), + {}, 'DateOffsetSplit', ()) + + +def test_absolute_monthly_forecast_uses_compiled_future_shifts(monkeypatch): + from kumo_core.pquery import _forecast_options + from pandas import DateOffset, Timedelta, Timestamp + + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + 'PREDICT COUNT(events.*, 0, 1, months) FORECAST 3 TIMEFRAMES ' + 'FOR EACH articles.aid', graph) + monkeypatch.setattr( + 'kumo_core.pquery._temporal_target_time_range', + lambda *_args: (Timestamp('2023-01-01'), Timestamp('2024-12-31'), None)) + options = TrainingTableGenerationPlan( + start_time='2023-01-01', end_time='2024-05-31', timeframe_step=1) + split, family = _split_options(query, graph, options) + timestamps, _ = _temporal_timestamps( + query, '/synthetic-index', options, split, family, ()) + _, compiler = _forecast_options(query, options) + assert compiler['future_offsets_secs'] == (0, 31 * 86400, 62 * 86400) + assert timestamps + assert all(Timestamp(value) + Timedelta(days=62) + DateOffset(months=1) + <= Timestamp('2024-05-31') for value in timestamps) + + def test_latest_temporal_timestamp_uses_source_maximum(tmp_path, monkeypatch): minimum = 1_704_067_200_000_000_000 maximum = 1_711_843_200_000_000_000 diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index 6e7fc1418..20607544e 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -114,6 +114,43 @@ To use your own data, point `FileConnector` at a directory of CSV or Parquet fil `graph.snapshot()` ingests the tables into an on-disk graph index under the scratch directory, which also holds training runs, model bundles, and predictions. It defaults to `~/.cache/kumo-engine`; to put it on a large local disk, call `kumo.engine.configure(scratch_dir=...)` first. +### Absolute training bounds + +For temporal node classification, regression, ranking, and forecasting, pass +absolute bounds when generating the training table: + +```python +from kumoapi.model_plan import TrainingTableGenerationPlan + +table = pquery.generate_training_table( + snapshot, + TrainingTableGenerationPlan( + start_time="2024-01-01T00:00:00Z", + end_time="2024-07-01T00:00:00Z", + ), +) +training_result = trainer.fit(table, snapshot=snapshot) +``` + +`start_time` includes prediction anchors at that timestamp. `end_time` limits +the label observation window: a 30-day target needs an anchor at least 30 days +before the cutoff. Forecasting reserves the complete future-label horizon. +Bounds are intersected with available label data; generation fails when no +complete examples fit. Omit either bound, or use `"inferred"`, to infer it from +the data. `start_time="-INF"` also uses the available lower bound. Aware +timestamps are normalized to UTC; naive timestamps are read as UTC. + +The bounds cover all generated train, validation, and test stages. Default +splits and supported ratio splits operate within that window. Choose direct +bounds or an explicit temporal window split (`TimeRangeSplit`, +`DateOffsetSplit`, or `BacktestingSplit`) separately. Direct bounds also cannot +be combined with `train_start_offset` or `train_end_offset`. Existing task, +time-unit, and split restrictions continue to apply. Integer `start_time` and +`end_time` offsets retain their existing decoupled-splits gate and measure +backward from the latest label data in the aggregation's time unit. Temporal +link prediction retains its existing gate. Fit and saved model bundles retain +the effective generation plan. + ## Agent skills Kumo Engine ships an agent skill, `kumo-data-science`, that teaches a coding agent to build, train, and run Kumo models with this version of the SDK. Install it for your agent: diff --git a/packages/kumo-engine/test/test_absolute_training_bounds.py b/packages/kumo-engine/test/test_absolute_training_bounds.py new file mode 100644 index 000000000..d10992d2f --- /dev/null +++ b/packages/kumo-engine/test/test_absolute_training_bounds.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +import torch +from kumo_core.io.json import to_json_dict +from kumoapi.graph import Edge +from kumoapi.model_plan import ModelPlan, TrainingTableGenerationPlan +from kumoapi.typing import Stype + +from io_utils import fsutil +from io_utils.parquet_util import read_parquet +from kumo.engine import ( + FileConnector, + Graph, + PredictiveQuery, + Table, + Trainer, + TrainingTableOutputConfig, + configure, +) +from kumo.engine._scratch import get_scratch_dir + +START = pd.Timestamp('2024-01-10') +END = pd.Timestamp('2024-02-20') +SOURCE_END = pd.Timestamp('2024-02-29') +TASKS = [ + ('binary_classification', 'COUNT(events.*, 0, 2, days) > 0'), + ('regression', 'COUNT(events.*, 0, 2, days)'), + ('multiclass_classification', 'FIRST(events.category, 0, 2, days)'), + ('multilabel_classification', + 'LIST_DISTINCT(events.category, 0, 2, days) CLASSIFY TOP 3'), + ('multilabel_ranking', + 'LIST_DISTINCT(events.category, 0, 2, days) RANK TOP 3'), + ('forecasting', 'COUNT(events.*, 0, 2, days) FORECAST 3 TIMEFRAMES'), +] + + +@pytest.fixture(scope='module') +def temporal_graph(tmp_path_factory, request): + root = tmp_path_factory.mktemp('absolute-bounds') + previous = get_scratch_dir() + configure(scratch_dir=root / 'scratch') + request.addfinalizer(lambda: configure(scratch_dir=previous)) + source = root / 'data' + fsutil.makedirs(str(source)) + events = [(user, day) for user in range(24) for day in range(60) + if (user + day) % 5 < 2] + frames = { + 'users': pa.table({'id': range(24), 'value': range(24)}), + 'events': pa.table({ + 'id': range(len(events)), + 'user_id': [user for user, _ in events], + 'ts': [pd.Timestamp('2024-01-01') + pd.Timedelta(days=day) + for _, day in events], + 'category': [f'category-{day % 3}' for _, day in events], + }), + } + for name, frame in frames.items(): + with fsutil.get_binary_handle(str(source / f'{name}.parquet'), + 'wb') as handle: + pq.write_table(frame, handle) + connector = FileConnector(root_dir=str(source)) + graph = Graph({ + 'users': Table.from_source_table(connector['users'], primary_key='id'), + 'events': Table.from_source_table( + connector['events'], primary_key='id', time_column='ts' + ).with_column('user_id', stype=Stype.ID).with_column( + 'category', stype=Stype.categorical), + }, [Edge('events', 'user_id', 'users')]) + return graph, graph.snapshot(), frames['events'].to_pandas() + + +def _generate(temporal_graph, tmp_path, target, **options): + graph, snapshot, _ = temporal_graph + query = PredictiveQuery(graph, f'PREDICT {target} FOR EACH users.id') + table = query.generate_training_table( + snapshot, TrainingTableGenerationPlan(timeframe_step=2, **options)) + destination = tmp_path / 'export' + fsutil.makedirs(str(destination)) + table.export(TrainingTableOutputConfig( + FileConnector(root_dir=str(destination)), 'rows')) + return query, table, read_parquet(str(destination / 'rows')) + + +@pytest.mark.parametrize('task,target', TASKS) +def test_temporal_node_bounds(temporal_graph, tmp_path, monkeypatch, task, + target): + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + options = dict(start_time=START.isoformat(), end_time=END.isoformat()) + if task == 'forecasting': + options.update(lag_timesteps=3, year_over_year=False) + query, table, rows = _generate(temporal_graph, tmp_path, target, **options) + assert query.get_task_type().value == task + assert set(rows['SPLIT']) == {0, 1, 2} + assert rows['TIMESTAMP'].min() >= START + horizon = 6 if task == 'forecasting' else 2 + assert rows['TIMESTAMP'].max() + pd.Timedelta(days=horizon) <= END + assert table.generation_plan.start_time == START.isoformat() + assert table.generation_plan.end_time == END.isoformat() + if task in {'binary_classification', 'regression'}: + events = temporal_graph[2] + for row in rows.itertuples(): + count = len(events[(events['user_id'] == row.ENTITY1) + & (events['ts'] > row.TIMESTAMP) + & (events['ts'] <= row.TIMESTAMP + + pd.Timedelta(days=2))]) + assert row.TARGET1 == (count > 0 if task.startswith('binary') + else count) + + +@pytest.mark.parametrize('bounds', [ + {'start_time': '2023-12-01', 'end_time': '2024-04-01'}, + {'start_time': '2024-01-10'}, + {'end_time': '2024-02-20'}, + {'start_time': '-INF', 'end_time': '2024-02-20'}, +]) +def test_partial_and_clipped_bounds(temporal_graph, tmp_path, monkeypatch, + bounds): + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'false') + _, _, rows = _generate(temporal_graph, tmp_path, + 'COUNT(events.*, 0, 2, days)', **bounds) + lower = max(pd.Timestamp('2024-01-01'), + pd.Timestamp(bounds['start_time']) + if bounds.get('start_time') not in {None, '-INF'} else + pd.Timestamp('2024-01-01')) + upper = min(SOURCE_END, pd.Timestamp(bounds.get('end_time', SOURCE_END))) + assert not rows.empty + assert rows['TIMESTAMP'].min() >= lower + assert rows['TIMESTAMP'].max() + pd.Timedelta(days=2) <= upper + + +@pytest.mark.parametrize('override', [False, True]) +def test_forecast_complete_horizon(temporal_graph, tmp_path, monkeypatch, + override): + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'false') + target = 'COUNT(events.*, 0, 2, days)' + options = dict(start_time=START.isoformat(), lag_timesteps=3, + year_over_year=False) + if override: + options.update(forecast_length=3, end_time=END.isoformat()) + else: + target += ' FORECAST 3 TIMEFRAMES' + _, table, rows = _generate(temporal_graph, tmp_path, target, **options) + cutoff = END if override else SOURCE_END + assert not rows.empty + assert rows['TIMESTAMP'].max() + pd.Timedelta(days=6) <= cutoff + assert table.generation_plan.forecast_length == 3 + + +@pytest.mark.parametrize('split', [None, 'RandomSplit([0.6, 0.2, 0.2])', + 'TemporalSplit([0.6, 0.2, 0.2])', + 'ApproxDateOffsetSplit([0.6, 0.2, 0.2])']) +def test_bound_timezone_equivalence(temporal_graph, tmp_path, monkeypatch, + split): + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + options = {} if split is None else {'split': split} + _, _, utc = _generate( + temporal_graph, tmp_path / 'utc', 'COUNT(events.*, 0, 2, days)', + start_time='2024-01-10T00:00:00Z', + end_time='2024-02-20T00:00:00Z', **options) + _, _, local = _generate( + temporal_graph, tmp_path / 'local', 'COUNT(events.*, 0, 2, days)', + start_time='2024-01-09T18:00:00-06:00', + end_time='2024-02-20T01:00:00+01:00', **options) + columns = ['TIMESTAMP', 'ENTITY1', 'SPLIT'] + pd.testing.assert_frame_equal( + utc.sort_values(columns).reset_index(drop=True), + local.sort_values(columns).reset_index(drop=True)) + if split == 'ApproxDateOffsetSplit([0.6, 0.2, 0.2])': + expected = pd.date_range('2024-01-11', '2024-02-18', freq='2D') + assert set(utc['TIMESTAMP']) == set(expected) + + +@pytest.mark.parametrize('options', [ + {}, + {'train_start_offset': 40, 'train_end_offset': 10}, + {'split': "TimeRangeSplit([('2024-01-10', '2024-01-24'), " + "('2024-01-24', '2024-02-07'), " + "('2024-02-07', '2024-02-20')])"}, +]) +def test_existing_window_controls(temporal_graph, tmp_path, monkeypatch, + options): + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + _, _, rows = _generate(temporal_graph, tmp_path, + 'COUNT(events.*, 0, 2, days)', **options) + assert set(rows['SPLIT']) == {0, 1, 2} + assert rows['TIMESTAMP'].min() >= pd.Timestamp('2024-01-01') + assert rows['TIMESTAMP'].max() + pd.Timedelta(days=2) <= SOURCE_END + + +@pytest.mark.parametrize('start,end', [ + ('2024-03-10', '2024-03-20'), + ('2024-01-10', '2024-01-11'), +]) +def test_no_complete_window(temporal_graph, tmp_path, monkeypatch, start, end): + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + with pytest.raises(ValueError, match='No usable training examples'): + _generate(temporal_graph, tmp_path, 'COUNT(events.*, 0, 2, days)', + start_time=start, end_time=end) + + +def test_bounds_fit_reload_predict(temporal_graph, tmp_path, monkeypatch, + request): + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + threads = torch.get_num_threads() + torch.set_num_threads(2) + request.addfinalizer(lambda: torch.set_num_threads(threads)) + query, table, _ = _generate( + temporal_graph, tmp_path, 'COUNT(events.*, 0, 2, days) > 0', + start_time='2024-01-10T00:00:00Z', + end_time='2024-02-19T18:00:00-06:00') + plan = ModelPlan() + plan.training_job.num_experiments = 1 + plan.training_job.disable_explain = True + plan.training_job.refit_trainval = False + plan.optimization.max_epochs = 1 + plan.optimization.max_steps_per_epoch = 2 + plan.optimization.batch_size = [32] + plan.model_architecture.channels = [16] + snapshot = temporal_graph[1] + trainer = Trainer(plan, accelerator='cpu') + result = trainer.fit(table, snapshot=snapshot) + restored = Trainer.load(result.bundle_dir, accelerator='cpu') + expected = to_json_dict(table.generation_plan) + assert to_json_dict(trainer.training_table_generation_plan) == expected + assert to_json_dict(restored.training_table_generation_plan) == expected + prediction_table = query.generate_prediction_table(snapshot) + before = trainer.predict(prediction_table, snapshot=snapshot, + binary_classification_threshold=0.5).frame + after = restored.predict(prediction_table, snapshot=snapshot, + binary_classification_threshold=0.5).frame + assert len(after) == 24 + pd.testing.assert_frame_equal(before, after, atol=0.005, rtol=0) diff --git a/packages/kumo-engine/test/test_predictive_query.py b/packages/kumo-engine/test/test_predictive_query.py index 7ad89704f..36636ca72 100644 --- a/packages/kumo-engine/test/test_predictive_query.py +++ b/packages/kumo-engine/test/test_predictive_query.py @@ -334,15 +334,33 @@ def test_generated_temporal_split_requires_time_column(graph_snapshot, split='TemporalSplit([0.6, 0.3, 0.1])')) -@pytest.mark.parametrize('start,end,message', [ - ('NaT', '2025-01-21', 'valid timestamp'), - ('2025-01-21', '2025-01-02', 'smaller than'), - ('2025-01-02', -1, 'both be offsets'), +@pytest.mark.parametrize('flag', [None, 'false']) +@pytest.mark.parametrize('options,message', [ + ({'start_time': 'NaT', 'end_time': '2025-01-21'}, 'valid timestamp'), + ({'start_time': '2025-01-21', 'end_time': '2025-01-02'}, 'smaller than'), + ({'start_time': '2025-01-02', 'end_time': -1}, 'both be offsets'), + ({'start_time': '2025-01-02', 'train_start_offset': 10}, + 'Cannot specify both'), + ({'end_time': '2025-01-21', 'train_end_offset': 1}, + 'Cannot specify both'), + ({'start_time': '2025-01-02', + 'split': "TimeRangeSplit([('2025-01-02', '2025-01-10'), " + "('2025-01-10', '2025-01-21')])"}, + 'cannot be combined'), + ({'end_time': '2025-01-21', + 'split': 'DateOffsetSplit([-4, -2], timeframe=1)'}, + 'cannot be combined'), + ({'end_time': '2025-01-21', + 'split': 'TimeSplit([0.6, 0.2, 0.2])'}, + 'TimeSplit'), ]) def test_generated_invalid_bounds_fail_before_artifacts( - graph_snapshot, monkeypatch, start, end, message): + graph_snapshot, monkeypatch, options, message, flag): graph, snapshot = graph_snapshot - monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + if flag is None: + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + else: + monkeypatch.setenv('USE_DECOUPLED_SPLITS', flag) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.new_run_id', lambda: pytest.fail('allocated a bounded-generation run id'), @@ -354,12 +372,7 @@ def test_generated_invalid_bounds_fail_before_artifacts( 'PREDICT COUNT(orders.*, 0, 1, days) > 0 FOR EACH users.id', ).generate_training_table( snapshot, - TrainingTableGenerationPlan( - start_time=start, - end_time=end, - timeframe_step=1, - split='ApproxDateOffsetSplit([0.6, 0.2, 0.2])', - ), + TrainingTableGenerationPlan(timeframe_step=1, **options), ) From 906547da1e8a83986ab6dfdaec9fdbeaaa0d107a Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Mon, 9 Nov 2026 16:27:52 -0600 Subject: [PATCH 04/10] Separate absolute bounds example from executable quickstart --- packages/kumo-engine/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index 20607544e..d625459d7 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -114,7 +114,7 @@ To use your own data, point `FileConnector` at a directory of CSV or Parquet fil `graph.snapshot()` ingests the tables into an on-disk graph index under the scratch directory, which also holds training runs, model bundles, and predictions. It defaults to `~/.cache/kumo-engine`; to put it on a large local disk, call `kumo.engine.configure(scratch_dir=...)` first. -### Absolute training bounds +## Absolute training bounds For temporal node classification, regression, ranking, and forecasting, pass absolute bounds when generating the training table: From bdf16a2df81c65e9237820a0df19e471ce83a91c Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Tue, 10 Nov 2026 12:47:02 -0600 Subject: [PATCH 05/10] Validate explicit training anchors and inferred bounds Apply absolute training windows to explicit anchors and retain prediction and label-evaluation anchors by clearing fit bounds on copied plans. Accept explicit inferred endpoints without changing default YAML or relative-offset gates. --- .../kumo-core/src/kumo_core/model_plan.py | 3 +- packages/kumo-core/src/kumo_core/pquery.py | 34 ++++++++--- packages/kumo-core/tests/test_pquery.py | 59 +++++++++++++++++++ .../kumo-core/tests/test_training_bounds.py | 29 ++++++++- .../kumo/engine/pquery/prediction_table.py | 9 ++- .../kumo/engine/pquery/predictive_query.py | 6 +- .../test/test_absolute_training_bounds.py | 8 ++- .../kumo-engine/test/test_predictive_query.py | 50 ++++++++++++++++ 8 files changed, 179 insertions(+), 19 deletions(-) diff --git a/packages/kumo-core/src/kumo_core/model_plan.py b/packages/kumo-core/src/kumo_core/model_plan.py index 6dff9a534..1d0ee188c 100644 --- a/packages/kumo-core/src/kumo_core/model_plan.py +++ b/packages/kumo-core/src/kumo_core/model_plan.py @@ -802,8 +802,7 @@ def is_valid_option( value = self.start_time if name == 'start_time' else self.end_time return (_use_decoupled_splits() or (task_type.is_node_pred and isinstance(value, str) - and not _is_missing(value) - and not _is_inferred(value))) + and not _is_missing(value))) if name in ['train_start_offset', 'train_end_offset']: return not _use_decoupled_splits() diff --git a/packages/kumo-core/src/kumo_core/pquery.py b/packages/kumo-core/src/kumo_core/pquery.py index e5632ef17..849d56f1e 100644 --- a/packages/kumo-core/src/kumo_core/pquery.py +++ b/packages/kumo-core/src/kumo_core/pquery.py @@ -39,6 +39,7 @@ from kumopql.parser import PQLParser from kumopql.validator import PredictiveQueryValidator from pandas import DateOffset, Timedelta, Timestamp +from pandas.errors import OutOfBoundsDatetime from pydantic import TypeAdapter from .encoder_selection import ( @@ -1067,13 +1068,13 @@ def relative(value: int | str | None) -> DateOffset | Timestamp | None: ) except ValueError as error: raise PredictiveQueryPlanError(str(error)) from error - if explicit: + absolute_bounds = (isinstance(start_time, Timestamp) + or isinstance(end_time, Timestamp)) and node_bounds + if explicit and not absolute_bounds: return (tuple(_timestamp_ns(value) for value in explicit), "flexible_boundaries" in split) minimum, maximum, filter_buffer = _temporal_target_time_range( query, index_path) - absolute_bounds = (isinstance(start_time, Timestamp) - or isinstance(end_time, Timestamp)) and node_bounds if absolute_bounds and isinstance(start_time, Timestamp): start_time = max(start_time, minimum) if absolute_bounds and isinstance(end_time, Timestamp): @@ -1099,6 +1100,25 @@ def relative(value: int | str | None) -> DateOffset | Timestamp | None: if absolute_bounds and lower >= cutoff: raise PredictiveQueryPlanError(empty_bounds) + def within_bounds(value: int) -> bool: + shifted = value + future_shift.value + if value < lower.value or shifted > label_cutoff.value: + return False + try: + return (Timestamp(shifted) + offset.end_date_offset <= label_cutoff) + except (OutOfBoundsDatetime, OverflowError): + return False + + def bounded(values: Sequence[int]) -> tuple[int, ...]: + result = tuple(value for value in values if within_bounds(value)) + if not result: + raise PredictiveQueryPlanError(empty_bounds) + return result + + if explicit: + return (bounded(tuple(_timestamp_ns(value) for value in explicit)), + "flexible_boundaries" in split) + step_count = int( inferred_or_missing_type_to_none(options.timeframe_step) or (int(offset.end) - int(offset.start or 0))) @@ -1135,12 +1155,8 @@ def relative(value: int | str | None) -> DateOffset | Timestamp | None: if absolute_bounds: # Inferred offset splits may extend their first range to fill a stage. # Absolute label bounds remain authoritative over that adjustment. - timestamps = [ - timestamp for timestamp in timestamps if timestamp >= lower - and timestamp + future_shift + offset.end_date_offset <= label_cutoff - ] - if absolute_bounds and not timestamps: - raise PredictiveQueryPlanError(empty_bounds) + return (bounded(tuple(timestamp.value for timestamp in timestamps)), + fixed_boundaries) return tuple(timestamp.value for timestamp in timestamps), fixed_boundaries diff --git a/packages/kumo-core/tests/test_pquery.py b/packages/kumo-core/tests/test_pquery.py index 8af614132..50f212271 100644 --- a/packages/kumo-core/tests/test_pquery.py +++ b/packages/kumo-core/tests/test_pquery.py @@ -791,6 +791,65 @@ def test_absolute_bounds_require_usable_windows(monkeypatch, start, end): start_time=start, end_time=end), {}, 'ApproxDateOffsetSplit', ()) +@pytest.mark.parametrize('clause,expected', [ + (' > 0', [13, 5, 12, 5]), + (' FORECAST 3 TIMEFRAMES', [5, 12, 5]), +]) +@pytest.mark.parametrize('split', [{}, {'flexible_boundaries': False}]) +def test_absolute_bounds_filter_explicit_training_anchors( + monkeypatch, clause, expected, split): + from pandas import Timestamp + + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + f'PREDICT COUNT(events.*, 0, 1, days){clause} ' + 'FOR EACH articles.aid', graph) + monkeypatch.setattr( + 'kumo_core.pquery._temporal_target_time_range', + lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) + off_grid = Timestamp('2024-01-06T12:34:56.123456789').value + explicit = (off_grid,) + tuple( + Timestamp(f'2024-01-{day:02}').value + for day in [13, 4, 5, 12, 5, 19]) + (Timestamp.max.value,) + timestamps, fixed = _temporal_timestamps( + query, '/synthetic-index', TrainingTableGenerationPlan( + start_time='2024-01-05', end_time='2024-01-15', timeframe_step=99), + split, 'ApproxDateOffsetSplit', explicit) + assert timestamps == (off_grid,) + tuple( + Timestamp(f'2024-01-{day:02}').value for day in expected) + assert fixed == ('flexible_boundaries' in split) + + +@pytest.mark.parametrize('explicit,expected', [ + (['2024-01-01', '2024-01-19', '2024-01-20'], + ['2024-01-01', '2024-01-19']), + (['2024-01-20', '2024-01-21'], []), +]) +def test_explicit_absolute_anchors_respect_available_labels( + monkeypatch, explicit, expected): + from pandas import Timestamp + + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + 'PREDICT COUNT(events.*, 0, 1, days) > 0 FOR EACH articles.aid', graph) + monkeypatch.setattr( + 'kumo_core.pquery._temporal_target_time_range', + lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) + options = TrainingTableGenerationPlan( + start_time='2023-12-01', end_time='2024-02-01') + anchors = tuple(Timestamp(value).value for value in explicit) + if not expected: + with pytest.raises(PredictiveQueryPlanError, match='No usable training'): + _temporal_timestamps( + query, '/synthetic-index', options, {}, + 'ApproxDateOffsetSplit', anchors) + else: + timestamps, _ = _temporal_timestamps( + query, '/synthetic-index', options, {}, + 'ApproxDateOffsetSplit', anchors) + assert timestamps == tuple(Timestamp(value).value for value in expected) + + @pytest.mark.parametrize('options', [ {'start_time': 'NaT'}, {'start_time': '2024-01-11', 'end_time': '2024-01-21', diff --git a/packages/kumo-core/tests/test_training_bounds.py b/packages/kumo-core/tests/test_training_bounds.py index 1d76ce3fc..c1368b2cb 100644 --- a/packages/kumo-core/tests/test_training_bounds.py +++ b/packages/kumo-core/tests/test_training_bounds.py @@ -3,10 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 import pytest +from kumo_core.io.json import from_json_dict, to_json_dict from kumo_core.model_plan import TrainingTableGenerationPlan from kumo_core.training.bounds import validate_time_bounds from kumo_core.types.enums import TaskType -from kumoapi.model_plan import RunMode +from kumoapi.model_plan import InferredType, RunMode +from kumoapi.model_plan import \ + TrainingTableGenerationPlan as PublicGenerationPlan from kumoapi.pquery import QueryType from pandas import Timestamp @@ -89,6 +92,30 @@ def test_node_bound_validation(task_type, flag, monkeypatch): assert plan.start_time == '2024-01-01T00:00:00-06:00' +@pytest.mark.parametrize('flag', [None, 'false', 'true']) +@pytest.mark.parametrize('inferred', ['inferred', InferredType.VALUE]) +@pytest.mark.parametrize('bound', ['start_time', 'end_time']) +def test_public_inferred_bound(flag, inferred, bound, monkeypatch): + if flag is None: + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + else: + monkeypatch.setenv('USE_DECOUPLED_SPLITS', flag) + public = PublicGenerationPlan(**{ + 'start_time': '2024-01-01', + 'end_time': '2024-02-01', + bound: inferred, + }) + # Match generate_training_table's public-plan preflight conversion. + plan = from_json_dict(to_json_dict(public), TrainingTableGenerationPlan) + errors, _ = plan.validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + assert not errors + + def test_link_bounds_keep_gate(monkeypatch): monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) errors, _ = TrainingTableGenerationPlan(start_time='2024-01-01').validate( diff --git a/packages/kumo-engine/kumo/engine/pquery/prediction_table.py b/packages/kumo-engine/kumo/engine/pquery/prediction_table.py index db6147642..35de92943 100644 --- a/packages/kumo-engine/kumo/engine/pquery/prediction_table.py +++ b/packages/kumo-engine/kumo/engine/pquery/prediction_table.py @@ -188,9 +188,12 @@ def _build_prediction_table_request( if generation_plan is not None: from kumo_core.io.json import from_json_dict from kumoapi.json_serde import to_json_dict - options = from_json_dict( - to_json_dict(generation_plan), - CoreGenerationPlan, + + # Prediction anchors do not inherit the fitted label window. + options = replace( + from_json_dict(to_json_dict(generation_plan), CoreGenerationPlan), + start_time=options.start_time, + end_time=options.end_time, ) parsed = resolve_query_forecast(parsed, native_graph, options) diff --git a/packages/kumo-engine/kumo/engine/pquery/predictive_query.py b/packages/kumo-engine/kumo/engine/pquery/predictive_query.py index 34df0f44a..985d46332 100644 --- a/packages/kumo-engine/kumo/engine/pquery/predictive_query.py +++ b/packages/kumo-engine/kumo/engine/pquery/predictive_query.py @@ -350,12 +350,14 @@ def _label_parts( from kumo_core.io.json import from_json_dict - # The fit's split ranges would not cover later anchors. + # Fit windows and split ranges do not constrain evaluation anchors. options = CoreGenerationPlan() if plan is not None: options = replace(from_json_dict(to_json_dict(plan), CoreGenerationPlan), - split=options.split) + split=options.split, + start_time=options.start_time, + end_time=options.end_time) run_dir = train_table_scratch_dir(new_run_id()) run_dir.mkdir(parents=True, exist_ok=True) staging = None diff --git a/packages/kumo-engine/test/test_absolute_training_bounds.py b/packages/kumo-engine/test/test_absolute_training_bounds.py index d10992d2f..8ccce3447 100644 --- a/packages/kumo-engine/test/test_absolute_training_bounds.py +++ b/packages/kumo-engine/test/test_absolute_training_bounds.py @@ -117,6 +117,8 @@ def test_temporal_node_bounds(temporal_graph, tmp_path, monkeypatch, task, {'start_time': '2023-12-01', 'end_time': '2024-04-01'}, {'start_time': '2024-01-10'}, {'end_time': '2024-02-20'}, + {'start_time': '2024-01-10', 'end_time': 'inferred'}, + {'start_time': 'inferred', 'end_time': '2024-02-20'}, {'start_time': '-INF', 'end_time': '2024-02-20'}, ]) def test_partial_and_clipped_bounds(temporal_graph, tmp_path, monkeypatch, @@ -126,9 +128,11 @@ def test_partial_and_clipped_bounds(temporal_graph, tmp_path, monkeypatch, 'COUNT(events.*, 0, 2, days)', **bounds) lower = max(pd.Timestamp('2024-01-01'), pd.Timestamp(bounds['start_time']) - if bounds.get('start_time') not in {None, '-INF'} else + if bounds.get('start_time') not in {None, '-INF', 'inferred'} else pd.Timestamp('2024-01-01')) - upper = min(SOURCE_END, pd.Timestamp(bounds.get('end_time', SOURCE_END))) + end = bounds.get('end_time', 'inferred') + upper = min(SOURCE_END, pd.Timestamp(end) if end != 'inferred' else + SOURCE_END) assert not rows.empty assert rows['TIMESTAMP'].min() >= lower assert rows['TIMESTAMP'].max() + pd.Timedelta(days=2) <= upper diff --git a/packages/kumo-engine/test/test_predictive_query.py b/packages/kumo-engine/test/test_predictive_query.py index 36636ca72..2bcbd9d12 100644 --- a/packages/kumo-engine/test/test_predictive_query.py +++ b/packages/kumo-engine/test/test_predictive_query.py @@ -465,6 +465,56 @@ def test_prediction_compiler_uses_plan_forecast_horizon(graph_snapshot, 259200) +def test_evaluate_prediction_after_absolute_training_bounds( + graph, tmp_path, monkeypatch, request): + import pandas as pd + import torch + + from kumo.engine._scratch import get_scratch_dir + + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + previous = get_scratch_dir() + configure(scratch_dir=tmp_path / 'scratch') + request.addfinalizer(lambda: configure(scratch_dir=previous)) + threads = torch.get_num_threads() + torch.set_num_threads(2) + request.addfinalizer(lambda: torch.set_num_threads(threads)) + snapshot = graph.snapshot() + query = PredictiveQuery( + graph, 'PREDICT COUNT(events.*, 0, 7, days) > 0 FOR EACH users.ID') + table = query.generate_training_table( + snapshot, TrainingTableGenerationPlan( + start_time='2024-01-10', end_time='2024-02-20', timeframe_step=2)) + plan = ModelPlan() + plan.training_job.num_experiments = 1 + plan.training_job.disable_explain = True + plan.training_job.refit_trainval = False + plan.optimization.max_epochs = 1 + plan.optimization.max_steps_per_epoch = 2 + plan.optimization.batch_size = [32] + plan.model_architecture.channels = [16] + trainer = Trainer(plan, accelerator='cpu') + trainer.fit(table, snapshot=snapshot) + fitted = to_json_dict(trainer.training_table_generation_plan) + prediction = query.generate_prediction_table( + snapshot, prediction_time=pd.Timestamp('2024-03-01').to_pydatetime(), + plan=trainer.training_table_generation_plan) + run = trainer.predict(prediction, snapshot=snapshot, + binary_classification_threshold=0.5) + result = trainer.evaluate(run, labels_snapshot=snapshot) + assert result.matured + assert result.labelled_fraction == 1.0 + assert pd.Timestamp(result.anchors[0]).tz_convert(None) == pd.Timestamp( + '2024-03-01') + future_anchor = pd.Timestamp('2025-01-01') + future = query.generate_prediction_table( + snapshot, prediction_time=future_anchor.to_pydatetime(), + plan=trainer.training_table_generation_plan) + assert future.timestamps == (future_anchor.value,) + assert future.row_count == 50 + assert to_json_dict(trainer.training_table_generation_plan) == fitted + + def test_custom_training_table_materializes_weight_and_manual_split( graph_snapshot, tmp_path, monkeypatch): graph, snapshot = graph_snapshot From e08672310ed6930310358a08a00a6ba274b75744 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Tue, 10 Nov 2026 14:41:12 -0600 Subject: [PATCH 06/10] Share temporal bounds between Studio and Engine --- kumo/data/generic/diskgraph_train_table.py | 6 +- .../pre_train_table_validator.py | 98 +++---------------- kumo/pquery/train_table_generator.py | 60 ++++++------ packages/kumo-core/src/kumo_core/pquery.py | 66 +++++-------- .../src/kumo_core/training/bounds.py | 30 ++++-- .../src/kumo_core/training/timestamps.py | 63 +++++++++++- packages/kumo-core/tests/test_pquery.py | 40 ++------ .../test/test_absolute_training_bounds.py | 1 + test/pquery/test_predictive_query_v2.py | 4 +- .../test_train_generator_basic.py | 42 +++++++- 10 files changed, 197 insertions(+), 213 deletions(-) diff --git a/kumo/data/generic/diskgraph_train_table.py b/kumo/data/generic/diskgraph_train_table.py index e052d6427..e3fced8ec 100644 --- a/kumo/data/generic/diskgraph_train_table.py +++ b/kumo/data/generic/diskgraph_train_table.py @@ -21,6 +21,7 @@ build_split_spec as build_native_split_spec from kumo_core.training.table import \ generate_train_table as generate_native_train_table +from kumo_core.training.timestamps import forecast_offsets from kumoapi.pquery.validated_predictive_query import ValidatedPredictiveQuery from io_utils import fsutil, logging @@ -676,10 +677,7 @@ def _ttng_forecasting_offset_kwargs( step = time_range.timeframe assert step is not None - future_offsets_secs = [ - int(((pt + i * step) - pt).total_seconds()) - for i in range(forecast_length) - ] + future_offsets_secs = list(forecast_offsets(step, forecast_length)) lag_offsets_secs = None if lag_timesteps is not None: lag_ts = get_lag_timestamps(pt, lag_timesteps, step) diff --git a/kumo/pquery/pquery_validators/pre_train_table_validator.py b/kumo/pquery/pquery_validators/pre_train_table_validator.py index 772c3a4c7..792716246 100644 --- a/kumo/pquery/pquery_validators/pre_train_table_validator.py +++ b/kumo/pquery/pquery_validators/pre_train_table_validator.py @@ -5,6 +5,7 @@ from typing import Dict, List, Optional, Union import pandas as pd +from kumo_core.training.bounds import validate_time_bounds from kumo_core.training.validation import validate_train_offsets from kumo_core.types import TaskType from kumoapi.pquery import QueryType, ValidatedPredictiveQuery @@ -774,94 +775,17 @@ def validate_limits( return validation_response - def validate_train_time_compatibility(self) -> ValidationResponse: - r"""Validates that start_time/end_time and train offsets are not used - together. - - Note: This validation can be removed once we fully migrate from using - train_start_offset/train_end_offset to start_time/end_time parameters. - - Returns: - ValidationResponse: Response containing any validation errors. - """ - validation_response = ValidationResponse() - - if ((self.start_time is not None or self.end_time is not None) - and (self.train_start_offset is not None - or self.train_end_offset is not None)): - validation_response.errors.append( - ValidationError( - title="Incompatible time parameters", - message=("Cannot specify both start_time/end_time and " - "train_start_offset/train_end_offset. Use either " - "absolute/relative timestamps or train offset " - "parameters, but not both."))) - - return validation_response - - def validate_train_time(self) -> ValidationResponse: - r"""Validates start_time and end_time parameters. - - Checks: - 1. Both must be offsets or both timestamps - 2. For timestamps: start_time must be smaller than end_time - 3. For offsets: both must be <= 0 and start_time must be < end_time - - Returns: - ValidationResponse: Response containing any validation errors. - """ + def validate_time_bounds(self) -> ValidationResponse: + """Adapt the shared time-bound checks to service validation errors.""" validation_response = ValidationResponse() - - if self.start_time is None or self.end_time is None: - return validation_response - - # Check if both are same type (offset or timestamp) - is_start_offset = isinstance(self.start_time, pd.DateOffset) - is_end_offset = isinstance(self.end_time, pd.DateOffset) - - if is_start_offset != is_end_offset: - validation_response.errors.append( - ValidationError( - title="Inconsistent time parameter types", - message=("start_time and end_time must both be offsets " - "or both be timestamps, but got mixed types."))) - return validation_response - - # Validate timestamps - if not is_start_offset: - assert isinstance(self.start_time, pd.Timestamp) - assert isinstance(self.end_time, pd.Timestamp) - if self.start_time >= self.end_time: - validation_response.errors.append( - ValidationError( - title="Invalid timestamp range", message=( - f"start_time ({self.start_time}) must be smaller " - f"than end_time ({self.end_time})"))) - # Validate offsets - else: - assert isinstance(self.start_time, pd.DateOffset) - assert isinstance(self.end_time, pd.DateOffset) - now = pd.Timestamp.now() - start_ts = now + self.start_time - end_ts = now + self.end_time - - if start_ts <= now and end_ts <= now: - if start_ts >= end_ts: - validation_response.errors.append( - ValidationError( - title="Invalid offset range", message=( - f"For relative offsets, start_time must be " - f"more negative than end_time, but got " - f"start_time={self.start_time}, " - f"end_time={self.end_time}"))) - else: - validation_response.errors.append( - ValidationError( - title="Invalid offset values", message=( - f"Relative offsets must be <= 0 (i.e. in the " - f"past), but got start_time={self.start_time}, " - f"end_time={self.end_time}"))) - + try: + validate_time_bounds( + self.start_time, self.end_time, + train_start_offset=self.train_start_offset, + train_end_offset=self.train_end_offset) + except ValueError as error: + validation_response.errors.append(ValidationError( + title='Invalid training time bounds', message=str(error))) return validation_response diff --git a/kumo/pquery/train_table_generator.py b/kumo/pquery/train_table_generator.py index 8d7c8d309..9995ab2ec 100644 --- a/kumo/pquery/train_table_generator.py +++ b/kumo/pquery/train_table_generator.py @@ -8,11 +8,13 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union import kumo_core.types as core_types +from kumo_core.training.bounds import parse_time_bound from kumo_core.training.options import allow_array_targets from kumo_core.training.splits import TrainTableTime, train_table_time from kumo_core.training.timestamps import ( generate_decoupled_timestamps, generate_legacy_timestamps, + resolve_training_window, ) from kumo_core.training.validation import GenerationValidationError from kumo_core.types.conversions import DataFrame @@ -286,26 +288,12 @@ def __init__( or isinstance(train_end_offset, DateOffset)) self.train_end_offset = train_end_offset - # Handle new time parameters - self.start_time = None - if start_time is not None: - if start_time == '-INF': - pass # Keep start_time as None for -INF case - elif isinstance(start_time, int): - self.start_time = self.get_date_offset(int(start_time)) - else: - assert isinstance(start_time, str) - self.start_time = Timestamp(start_time) - assert isinstance(self.start_time, Timestamp) - - self.end_time = None - if end_time is not None: - if isinstance(end_time, int): - self.end_time = self.get_date_offset(int(end_time)) - else: - assert isinstance(end_time, str) - self.end_time = Timestamp(end_time) - assert isinstance(self.end_time, Timestamp) + parsed_start = parse_time_bound(start_time, 'start_time') + parsed_end = parse_time_bound(end_time, 'end_time') + self.start_time = (self.get_date_offset(parsed_start) + if isinstance(parsed_start, int) else parsed_start) + self.end_time = (self.get_date_offset(parsed_end) + if isinstance(parsed_end, int) else parsed_end) self.timeframe_step = None if timeframe_step is not None: @@ -425,6 +413,12 @@ def generate_timestamps(self) -> List[Timestamp]: values = self._timestamp_inputs() (original_start, original_end, filter_buffer, offset_range, timeframe, step) = values + window = None + if self.task_type.is_node_pred: + window = resolve_training_window( + self.start_time, self.end_time, original_start, original_end) + start_time = self.start_time if window is None else window.start + end_time = self.end_time if window is None else window.end logger.info( f"For raw data, the min_date is {original_start} and the max_date " f"is {original_end}.") @@ -463,13 +457,16 @@ def fix_boundaries() -> None: split=descriptor, train_start_offset=self.train_start_offset, train_end_offset=self.train_end_offset, - start_time=self.start_time, - end_time=self.end_time, + start_time=start_time, + end_time=end_time, drop_earliest=self.drop_earliest_timeframe, on_fixed_boundaries=fix_boundaries, ) if short_window: - return [] + return [] if window is None else window.filter( + timestamps, offset_range.end_date_offset) + if window is not None: + timestamps = window.filter(timestamps, offset_range.end_date_offset) logger.info(f'Generated {len(timestamps)} timestamps:') for i, timestamp in enumerate(timestamps): logger.info(f'Timestamp {i:03d}: {timestamp}') @@ -489,6 +486,10 @@ def generate_timestamps_v2(self) -> List[Timestamp]: values = self._timestamp_inputs() (original_start, original_end, filter_buffer, offset_range, timeframe, step) = values + window = None + if self.task_type.is_node_pred: + window = resolve_training_window( + self.start_time, self.end_time, original_start, original_end) timestamps = generate_decoupled_timestamps( original_start=original_start, original_end=original_end, @@ -496,10 +497,12 @@ def generate_timestamps_v2(self) -> List[Timestamp]: target_timeframe=timeframe, target_start_offset=offset_range.start_date_offset, step=step, - start_time=self.start_time, - end_time=self.end_time, + start_time=self.start_time if window is None else window.start, + end_time=self.end_time if window is None else window.end, drop_earliest=self.drop_earliest_timeframe, ) + if window is not None: + timestamps = window.filter(timestamps, offset_range.end_date_offset) logger.info(f'Generated {len(timestamps)} timestamps:') for i, timestamp in enumerate(timestamps): logger.info(f'Timestamp {i:03d}: {timestamp}') @@ -585,13 +588,8 @@ def validate_pre_train_table_generate( validation_response.merge( self.pre_train_table_validator.validate_temporal_splits()) - # Add new validation for time parameter compatibility - validation_response.merge(self.pre_train_table_validator. - validate_train_time_compatibility()) - - # Add new validation for time parameter values validation_response.merge( - self.pre_train_table_validator.validate_train_time()) + self.pre_train_table_validator.validate_time_bounds()) validation_response.merge( self.pre_train_table_validator.validate_train_offsets( diff --git a/packages/kumo-core/src/kumo_core/pquery.py b/packages/kumo-core/src/kumo_core/pquery.py index 849d56f1e..5c7533f22 100644 --- a/packages/kumo-core/src/kumo_core/pquery.py +++ b/packages/kumo-core/src/kumo_core/pquery.py @@ -39,7 +39,6 @@ from kumopql.parser import PQLParser from kumopql.validator import PredictiveQueryValidator from pandas import DateOffset, Timedelta, Timestamp -from pandas.errors import OutOfBoundsDatetime from pydantic import TypeAdapter from .encoder_selection import ( @@ -75,7 +74,9 @@ ) from .training.timestamps import ( default_target_time_range, + forecast_offsets, generate_legacy_timestamps, + resolve_training_window, ) from .training.validation import validate_train_offsets from .types import TaskType @@ -760,8 +761,7 @@ def resolve_query_forecast( def _forecast_offsets(query: ValidatedNativeQuery) -> tuple[int, ...]: """The exact future shifts passed to the native label compiler.""" timeframe = query.query.get_combined_date_offset_range().timeframe - step_seconds = int(timeframe.total_seconds()) - return tuple(step_seconds * index for index in range(query.forecast_length)) + return forecast_offsets(timeframe, query.forecast_length) def _forecast_options( @@ -1075,45 +1075,25 @@ def relative(value: int | str | None) -> DateOffset | Timestamp | None: "flexible_boundaries" in split) minimum, maximum, filter_buffer = _temporal_target_time_range( query, index_path) - if absolute_bounds and isinstance(start_time, Timestamp): - start_time = max(start_time, minimum) - if absolute_bounds and isinstance(end_time, Timestamp): - end_time = min(end_time, maximum) - label_cutoff = end_time if isinstance(end_time, Timestamp) else maximum - lower = start_time if isinstance(start_time, Timestamp) else minimum - empty_bounds = ( - "No usable training examples within start_time/end_time and " - "the available label data. Widen the bounds or provide more data " - "for the target horizon.") - if absolute_bounds and lower >= label_cutoff: - raise PredictiveQueryPlanError(empty_bounds) - future_shift = Timedelta(0) - if absolute_bounds and query.task_type == "forecasting": - future_shift = Timedelta(seconds=max(_forecast_offsets(query))) - if label_cutoff.value - lower.value <= future_shift.value: - raise PredictiveQueryPlanError(empty_bounds) - shifted_cutoff = label_cutoff - future_shift - if not isinstance(shifted_cutoff, Timestamp): - raise PredictiveQueryPlanError(empty_bounds) - end_time = shifted_cutoff - cutoff = end_time if isinstance(end_time, Timestamp) else maximum - if absolute_bounds and lower >= cutoff: - raise PredictiveQueryPlanError(empty_bounds) - - def within_bounds(value: int) -> bool: - shifted = value + future_shift.value - if value < lower.value or shifted > label_cutoff.value: - return False - try: - return (Timestamp(shifted) + offset.end_date_offset <= label_cutoff) - except (OutOfBoundsDatetime, OverflowError): - return False + start_time = (date_offset(start_time) + if isinstance(start_time, int) else start_time) + end_time = (date_offset(end_time) + if isinstance(end_time, int) else end_time) + try: + window = (resolve_training_window(start_time, end_time, minimum, maximum) + if absolute_bounds else None) + except ValueError as error: + raise PredictiveQueryPlanError(str(error)) from error + if window is not None: + start_time, end_time = window.start, window.end def bounded(values: Sequence[int]) -> tuple[int, ...]: - result = tuple(value for value in values if within_bounds(value)) - if not result: - raise PredictiveQueryPlanError(empty_bounds) - return result + if window is None: + return tuple(values) + try: + return tuple(window.filter(values, offset.end_date_offset)) + except ValueError as error: + raise PredictiveQueryPlanError(str(error)) from error if explicit: return (bounded(tuple(_timestamp_ns(value) for value in explicit)), @@ -1146,10 +1126,8 @@ def bounded(values: Sequence[int]) -> tuple[int, ...]: int(train_start))), train_end_offset=(None if train_end is None else date_offset( int(train_end))), - start_time=(date_offset(start_time) - if isinstance(start_time, int) else start_time), - end_time=(date_offset(end_time) - if isinstance(end_time, int) else end_time), + start_time=start_time, + end_time=end_time, drop_earliest=True, ) if absolute_bounds: diff --git a/packages/kumo-core/src/kumo_core/training/bounds.py b/packages/kumo-core/src/kumo_core/training/bounds.py index 33e1a8ae5..cd182f862 100644 --- a/packages/kumo-core/src/kumo_core/training/bounds.py +++ b/packages/kumo-core/src/kumo_core/training/bounds.py @@ -3,11 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 """Validation and UTC normalization for temporal label-window bounds.""" -from pandas import Timestamp +from pandas import DateOffset, Timestamp -def parse_time_bound(value: int | str | None, - name: str) -> int | Timestamp | None: +def parse_time_bound(value: int | str | DateOffset | Timestamp | None, + name: str) -> int | DateOffset | Timestamp | None: """Keep relative offsets numerical and normalize absolute times to UTC.""" if value is None or (name == 'start_time' and value == '-INF'): return None @@ -15,6 +15,10 @@ def parse_time_bound(value: int | str | None, if value > 0: raise ValueError(f"'{name}' must be a non-positive relative offset") return value + if isinstance(value, DateOffset): + if Timestamp('1970-01-01') + value > Timestamp('1970-01-01'): + raise ValueError(f"'{name}' must be a non-positive relative offset") + return value try: timestamp = Timestamp(value) if not isinstance(timestamp, Timestamp): @@ -31,13 +35,14 @@ def parse_time_bound(value: int | str | None, def validate_time_bounds( - start: int | str | None, - end: int | str | None, + start: int | str | DateOffset | Timestamp | None, + end: int | str | DateOffset | Timestamp | None, *, - train_start_offset: int | None = None, - train_end_offset: int | None = None, + train_start_offset: int | DateOffset | None = None, + train_end_offset: int | DateOffset | None = None, split_family: str | None = None, -) -> tuple[int | Timestamp | None, int | Timestamp | None]: +) -> tuple[int | DateOffset | Timestamp | None, + int | DateOffset | Timestamp | None]: """Validate explicit parameters without changing their serialized form.""" if ((start is not None or end is not None) and (train_start_offset is not None @@ -47,14 +52,19 @@ def validate_time_bounds( parsed_start = parse_time_bound(start, 'start_time') parsed_end = parse_time_bound(end, 'end_time') if parsed_start is not None and parsed_end is not None: - if isinstance(parsed_start, int) != isinstance(parsed_end, int): + if (isinstance(parsed_start, Timestamp) + != isinstance(parsed_end, Timestamp)): raise ValueError('start_time and end_time must both be offsets ' 'or both be timestamps.') if ((isinstance(parsed_start, int) and isinstance(parsed_end, int) and parsed_start >= parsed_end) or (isinstance(parsed_start, Timestamp) and isinstance(parsed_end, Timestamp) - and parsed_start >= parsed_end)): + and parsed_start >= parsed_end) + or (isinstance(parsed_start, DateOffset) + and isinstance(parsed_end, DateOffset) + and Timestamp('1970-01-01') + parsed_start + >= Timestamp('1970-01-01') + parsed_end)): raise ValueError('start_time must be smaller than end_time.') if (isinstance(parsed_start, Timestamp) or isinstance(parsed_end, Timestamp)) and split_family in { diff --git a/packages/kumo-core/src/kumo_core/training/timestamps.py b/packages/kumo-core/src/kumo_core/training/timestamps.py index 981ca78bf..266e5de7d 100644 --- a/packages/kumo-core/src/kumo_core/training/timestamps.py +++ b/packages/kumo-core/src/kumo_core/training/timestamps.py @@ -3,10 +3,69 @@ # SPDX-License-Identifier: Apache-2.0 """Shared temporal anchor enumeration for hosted and portable generation.""" -from typing import Any, Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Callable, Mapping, Sequence, TypeVar -from kumoapi.pquery.AST import DateOffsetRange +from kumoapi.pquery.AST.date_offset_range import DateOffsetRange from pandas import DateOffset, Timedelta, Timestamp +from pandas.errors import OutOfBoundsDatetime + +Anchor = TypeVar('Anchor', int, Timestamp) +_EMPTY_WINDOW = ('No usable training examples within start_time/end_time and ' + 'the available label data. Widen the bounds or provide more ' + 'data for the target horizon.') + + +@dataclass(frozen=True) +class TrainingWindow: + """Available base label windows; native routing owns forecast tails.""" + + start: DateOffset | Timestamp | None + end: DateOffset | Timestamp | None + lower: Timestamp + cutoff: Timestamp + + def filter(self, timestamps: Sequence[Anchor], + target_end: DateOffset) -> list[Anchor]: + result = [ + value for value in timestamps if self.contains( + value.value if isinstance(value, Timestamp) else value, + target_end) + ] + if not result: + raise ValueError(_EMPTY_WINDOW) + return result + + def contains(self, value: int, target_end: DateOffset) -> bool: + if value < self.lower.value or value > self.cutoff.value: + return False + try: + return Timestamp(value) + target_end <= self.cutoff + except (OutOfBoundsDatetime, OverflowError): + return False + + +def resolve_training_window( + start: DateOffset | Timestamp | None, + end: DateOffset | Timestamp | None, + minimum: Timestamp, + maximum: Timestamp, +) -> TrainingWindow | None: + if not isinstance(start, Timestamp) and not isinstance(end, Timestamp): + return None + start = max(start, minimum) if isinstance(start, Timestamp) else start + end = min(end, maximum) if isinstance(end, Timestamp) else end + lower = start if isinstance(start, Timestamp) else minimum + cutoff = end if isinstance(end, Timestamp) else maximum + if lower >= cutoff: + raise ValueError(_EMPTY_WINDOW) + return TrainingWindow(start, end, lower, cutoff) + + +def forecast_offsets(timeframe: Timedelta, length: int) -> tuple[int, ...]: + """Future shifts shared by hosted and portable native compilers.""" + step_seconds = int(timeframe.total_seconds()) + return tuple(step_seconds * index for index in range(length)) def end_anchored_timestamps( diff --git a/packages/kumo-core/tests/test_pquery.py b/packages/kumo-core/tests/test_pquery.py index 50f212271..aa8078f9c 100644 --- a/packages/kumo-core/tests/test_pquery.py +++ b/packages/kumo-core/tests/test_pquery.py @@ -793,7 +793,7 @@ def test_absolute_bounds_require_usable_windows(monkeypatch, start, end): @pytest.mark.parametrize('clause,expected', [ (' > 0', [13, 5, 12, 5]), - (' FORECAST 3 TIMEFRAMES', [5, 12, 5]), + (' FORECAST 3 TIMEFRAMES', [13, 5, 12, 5]), ]) @pytest.mark.parametrize('split', [{}, {'flexible_boundaries': False}]) def test_absolute_bounds_filter_explicit_training_anchors( @@ -874,7 +874,7 @@ def test_invalid_absolute_bounds_rejected_before_stats( @pytest.mark.parametrize('end', [None, '2024-01-15']) -def test_absolute_forecast_bounds_reserve_full_horizon(monkeypatch, end): +def test_absolute_forecast_candidates_fit_base_window(monkeypatch, end): from pandas import DateOffset, Timestamp graph = normalize_graph(forecasting_graph_config()) @@ -893,36 +893,15 @@ def test_absolute_forecast_bounds_reserve_full_horizon(monkeypatch, end): query, '/synthetic-index', options, split, family, ()) cutoff = Timestamp(end or '2024-01-20') newest = Timestamp(timestamps[0]) - assert newest == cutoff - DateOffset(days=7) + # Native generation reserves the remaining forecast tail once. + assert newest == cutoff - DateOffset(days=3) assert all(Timestamp(value) >= Timestamp('2024-01-05') for value in timestamps) -@pytest.mark.parametrize('minimum,maximum', [ - ('2024-01-01', '2024-01-20'), - ('1677-09-22', '1677-09-23'), -]) -def test_forecast_bounds_rejected_before_horizon_underflow( - monkeypatch, minimum, maximum): - from pandas import Timestamp - - graph = normalize_graph(forecasting_graph_config()) - query = parse_and_validate_query( - 'PREDICT COUNT(events.*, 0, 1, days) FORECAST 3 TIMEFRAMES ' - 'FOR EACH articles.aid', graph) - monkeypatch.setattr( - 'kumo_core.pquery._temporal_target_time_range', - lambda *_args: (Timestamp(minimum), Timestamp(maximum), None)) - with pytest.raises(PredictiveQueryPlanError, match='No usable training'): - _temporal_timestamps( - query, '/synthetic-index', TrainingTableGenerationPlan( - start_time='1677-09-22', end_time='1677-09-23'), - {}, 'DateOffsetSplit', ()) - - -def test_absolute_monthly_forecast_uses_compiled_future_shifts(monkeypatch): +def test_absolute_monthly_forecast_keeps_base_candidates(monkeypatch): from kumo_core.pquery import _forecast_options - from pandas import DateOffset, Timedelta, Timestamp + from pandas import DateOffset, Timestamp graph = normalize_graph(forecasting_graph_config()) query = parse_and_validate_query( @@ -939,7 +918,7 @@ def test_absolute_monthly_forecast_uses_compiled_future_shifts(monkeypatch): _, compiler = _forecast_options(query, options) assert compiler['future_offsets_secs'] == (0, 31 * 86400, 62 * 86400) assert timestamps - assert all(Timestamp(value) + Timedelta(days=62) + DateOffset(months=1) + assert all(Timestamp(value) + DateOffset(months=1) <= Timestamp('2024-05-31') for value in timestamps) @@ -1173,9 +1152,10 @@ def test_date_offset_split_uses_bounded_range_for_each_phase( graph_index_path=str(tmp_path / "index"), scratch_dir=str(tmp_path), artifact_dir=str(tmp_path)) - # Reserve all three future labels before restarting steps at each split. + # Candidate steps restart at each split; native generation reserves the + # complete future-label horizon. assert plan.train_table.timestamps == tuple( - (origin + index * day).value for index in (7, 5, 4, 2, 1)) + (origin + index * day).value for index in (9, 7, 6, 4, 3, 1)) def test_nonzero_target_offset_restarts_step_at_each_split_boundary( diff --git a/packages/kumo-engine/test/test_absolute_training_bounds.py b/packages/kumo-engine/test/test_absolute_training_bounds.py index 8ccce3447..87a7a4dfa 100644 --- a/packages/kumo-engine/test/test_absolute_training_bounds.py +++ b/packages/kumo-engine/test/test_absolute_training_bounds.py @@ -153,6 +153,7 @@ def test_forecast_complete_horizon(temporal_graph, tmp_path, monkeypatch, cutoff = END if override else SOURCE_END assert not rows.empty assert rows['TIMESTAMP'].max() + pd.Timedelta(days=6) <= cutoff + assert rows['TIMESTAMP'].max() == cutoff - pd.Timedelta(days=6) assert table.generation_plan.forecast_length == 3 diff --git a/test/pquery/test_predictive_query_v2.py b/test/pquery/test_predictive_query_v2.py index f95d14691..6e3d236da 100644 --- a/test/pquery/test_predictive_query_v2.py +++ b/test/pquery/test_predictive_query_v2.py @@ -414,7 +414,7 @@ def test_predictive_query_time_range_validation(basic_test_graph, tempdir): ) validate_response = pquery.get_validate_response() assert not validate_response.ok - assert validate_response.errors[0].title == 'Incompatible time parameters' + assert validate_response.errors[0].title == 'Invalid training time bounds' # Test validation when end_time is before start_time pquery = PredictiveQueryV2( @@ -426,7 +426,7 @@ def test_predictive_query_time_range_validation(basic_test_graph, tempdir): ) validate_response = pquery.get_validate_response() assert not validate_response.ok - assert validate_response.errors[0].title == 'Invalid timestamp range' + assert validate_response.errors[0].title == 'Invalid training time bounds' def test_get_default_split_with_decoupled_splits(basic_test_graph): diff --git a/test/pquery/train_table_generator/test_train_generator_basic.py b/test/pquery/train_table_generator/test_train_generator_basic.py index 0236b0a5a..a9cea5331 100644 --- a/test/pquery/train_table_generator/test_train_generator_basic.py +++ b/test/pquery/train_table_generator/test_train_generator_basic.py @@ -14,6 +14,8 @@ import pyarrow as pa import pyarrow.parquet as pq import pytest +from kumo_core.model_plan import TrainingTableGenerationPlan +from kumo_core.pquery import ValidatedNativeQuery, _temporal_timestamps from kumo_core.types import ColStatType, TaskType from kumoapi.pquery import QueryType from kumoapi.train import TrainingStage @@ -26,6 +28,7 @@ from kumo.stats.stats import Stats from kumo.stats.train_table_stats import STATS_PER_TIMEFRAME from kumo.task.split import ( + ApproxDateOffsetSplit, BacktestingSplit, DateOffsetSplit, RandomSplit, @@ -44,6 +47,39 @@ NANOSECONDS_PER_SECOND = 1_000_000_000 +def test_absolute_bounds_generator_parity(create_node_pred_graph, tempdir, + monkeypatch): + graph = create_node_pred_graph('test_graph', tempdir.name) + query = ('PREDICT COUNT(TRANSACTIONS.*, 0, 3, days) > 0 ' + 'FOR EACH USERS.USER_ID') + options = TrainingTableGenerationPlan( + start_time='2023-12-01T00:00:00Z', end_time='2024-02-01T00:00:00Z', + timeframe_step=3) + available = (pd.Timestamp('2024-01-01'), pd.Timestamp('2024-01-20'), None) + studio = TrainingTableGeneratorV2( + graph, get_validated_query(graph, query), + ApproxDateOffsetSplit([0.8, 0.1, 0.1]), + start_time=options.start_time, end_time=options.end_time, + timeframe_step=3, use_decoupled_splits=False) + monkeypatch.setattr(studio, 'get_default_target_time_range', + lambda: available) + monkeypatch.setattr('kumo_core.pquery._temporal_target_time_range', + lambda *_args: available) + native_query = ValidatedNativeQuery( + query=studio.query, task_type=studio.task_type.value, + task_family='node_prediction', entity_table='USERS', + entity_column='USER_ID', target_stype=studio.query.target_stype.value, + query_type='temporal', rhs_node_type=None, rhs_is_temporal=False, + list_distinct_target_is_array=False, forecast_length=1) + native, _ = _temporal_timestamps( + native_query, '/synthetic-index', options, {}, + 'ApproxDateOffsetSplit', ()) + expected = [pd.Timestamp(f'2024-01-{day:02}') + for day in [17, 14, 11, 8, 5, 2]] + assert studio.generate_timestamps() == expected + assert [pd.Timestamp(value) for value in native] == expected + + def test_generate_time_intervals_single_aggr(create_node_pred_graph, tempdir): graph = create_node_pred_graph('test_graph', tempdir.name) query = ('PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 2) ' @@ -950,8 +986,7 @@ def test_generate_time_intervals_invalid_start_end_time(create_node_pred_graph, end_time='1990-01-02') validation_response = generator.validate_pre_train_table_generate() assert not validation_response.ok - assert ("start_time (1990-01-04 00:00:00) must be smaller than " - "end_time (1990-01-02 00:00:00)" + assert ('start_time must be smaller than end_time.' in validation_response.error_message()) # Test with relative offsets @@ -964,7 +999,7 @@ def test_generate_time_intervals_invalid_start_end_time(create_node_pred_graph, ) validation_response = generator.validate_pre_train_table_generate() assert not validation_response.ok - assert ("start_time must be more negative than end_time" + assert ('start_time must be smaller than end_time.' in validation_response.error_message()) @@ -1214,6 +1249,7 @@ def test_non_aligned_time_bounds_keep_newest_window(method): from types import MethodType, SimpleNamespace generator = SimpleNamespace( + task_type=TaskType.BINARY_CLASSIFICATION, get_default_target_time_range=lambda: (pd.Timestamp("2024-01-01"), pd.Timestamp("2024-02-10"), None), split=None, From e60f3ca92e3883cdc998d6d2259df0b0d0b39bec Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Tue, 10 Nov 2026 14:48:47 -0600 Subject: [PATCH 07/10] Apply temporal bounds only to training plans --- kumo/pquery/train_table_generator.py | 8 +-- packages/kumo-api/kumoapi/model_plan.py | 20 +++--- .../kumo-core/src/kumo_core/model_plan.py | 20 ++++-- packages/kumo-core/src/kumo_core/pquery.py | 48 +++++++------ .../src/kumo_core/training/bounds.py | 20 +++--- .../src/kumo_core/training/workflow.py | 7 +- packages/kumo-core/tests/test_pquery.py | 39 ++++++++--- .../kumo-core/tests/test_training_bounds.py | 25 ++++--- packages/kumo-engine/README.md | 9 +-- .../kumo/engine/pquery/prediction_table.py | 9 +-- .../kumo/engine/pquery/predictive_query.py | 15 +++-- .../kumo-engine/test/test_predictive_query.py | 8 +-- test/config/test_model_plan.py | 2 + .../test_train_generator_basic.py | 67 +++++-------------- 14 files changed, 146 insertions(+), 151 deletions(-) diff --git a/kumo/pquery/train_table_generator.py b/kumo/pquery/train_table_generator.py index 9995ab2ec..0a3082478 100644 --- a/kumo/pquery/train_table_generator.py +++ b/kumo/pquery/train_table_generator.py @@ -288,12 +288,8 @@ def __init__( or isinstance(train_end_offset, DateOffset)) self.train_end_offset = train_end_offset - parsed_start = parse_time_bound(start_time, 'start_time') - parsed_end = parse_time_bound(end_time, 'end_time') - self.start_time = (self.get_date_offset(parsed_start) - if isinstance(parsed_start, int) else parsed_start) - self.end_time = (self.get_date_offset(parsed_end) - if isinstance(parsed_end, int) else parsed_end) + self.start_time = parse_time_bound(start_time, 'start_time') + self.end_time = parse_time_bound(end_time, 'end_time') self.timeframe_step = None if timeframe_step is not None: diff --git a/packages/kumo-api/kumoapi/model_plan.py b/packages/kumo-api/kumoapi/model_plan.py index e6dea7173..267f6f5fb 100644 --- a/packages/kumo-api/kumoapi/model_plan.py +++ b/packages/kumo-api/kumoapi/model_plan.py @@ -619,7 +619,7 @@ class TrainingTableGenerationPlan(PlanMixin): like to migrate this parameter to end_time. **Supported Task Types:** Temporal - :ivar start_time: (``int`` | ``"inferred"`` | ``str``) Inclusive lower + :ivar start_time: (``"inferred"`` | ``str``) Inclusive lower bound on generated prediction anchors. Temporal node tasks accept pandas Timestamp compatible strings (e.g. '2024-04-01') with default splits. Timezone-aware strings are normalized to UTC; naive strings @@ -628,23 +628,23 @@ class TrainingTableGenerationPlan(PlanMixin): Direct absolute bounds cannot be combined with train_start_offset, train_end_offset, or explicit date/offset window splits. Ratio splits remain subject to existing task restrictions. - Non-positive integer offsets retain the decoupled-splits gate and - are relative to the latest label data, in the aggregation's time unit. + Integer bounds are not supported. For relative training windows, + use train_start_offset/train_end_offset in the aggregation's time unit. (*default:* ``"inferred"``). - **Supported Task Types:** Temporal node tasks by default; temporal - link prediction retains its existing decoupled-splits requirement. + **Supported Task Types:** Temporal node classification, regression, + ranking, and forecasting. - :ivar end_time: (``int`` | ``"inferred"`` | ``str``) Label observation + :ivar end_time: (``"inferred"`` | ``str``) Label observation cutoff, rather than the latest prediction anchor. Each generated - example must fit its complete target horizon before this cutoff, + example must fit its complete target horizon by this cutoff, including every future label for forecasting. Accepts the same - timestamp strings and gated integer offsets as start_time; + timestamp strings as start_time; ``"inferred"`` uses available label data. Either absolute bound can be omitted. Empty intersections or windows too short for a complete example fail generation. Existing time-unit and split constraints continue to apply. (*default:* ``"inferred"``). - **Supported Task Types:** Temporal node tasks by default; temporal - link prediction retains its existing decoupled-splits requirement. + **Supported Task Types:** Temporal node classification, regression, + ranking, and forecasting. :ivar timeframe_step: (``int`` | ``"inferred"``) Defines the step size of generating time intervals for training table generation diff --git a/packages/kumo-core/src/kumo_core/model_plan.py b/packages/kumo-core/src/kumo_core/model_plan.py index 1d0ee188c..61ccf7c65 100644 --- a/packages/kumo-core/src/kumo_core/model_plan.py +++ b/packages/kumo-core/src/kumo_core/model_plan.py @@ -23,7 +23,7 @@ import kumoapi.encoder as encoder_api import kumoapi.model_plan as api from kumo_core.errors import TrainingValidationError as ValidationException -from kumo_core.training.bounds import validate_time_bounds +from kumo_core.training.bounds import parse_time_bound, validate_time_bounds from kumo_core.types import TaskType from kumoapi.common import ValidationWarning from kumoapi.encoder import ( @@ -796,13 +796,9 @@ def is_valid_option( has_train_table_weight_col): return False - # Absolute node bounds also work with the default split path. - # Relative start/end offsets retain the decoupled-splits gate. + # Absolute node bounds are available independently of split routing. if name in ['start_time', 'end_time']: - value = self.start_time if name == 'start_time' else self.end_time - return (_use_decoupled_splits() - or (task_type.is_node_pred and isinstance(value, str) - and not _is_missing(value))) + return _use_decoupled_splits() or task_type.is_node_pred if name in ['train_start_offset', 'train_end_offset']: return not _use_decoupled_splits() @@ -971,6 +967,16 @@ def validate( ) except ValueError as error: errors.append(error) + elif query_type == QueryType.TEMPORAL: + # Preserve existing link routing while rejecting public integer + # bounds consistently with temporal node plans. + for name, value in (('start_time', self.start_time), + ('end_time', self.end_time)): + if isinstance(value, int): + try: + parse_time_bound(value, name) + except ValueError as error: + errors.append(error) return errors, warnings @classmethod diff --git a/packages/kumo-core/src/kumo_core/pquery.py b/packages/kumo-core/src/kumo_core/pquery.py index 5c7533f22..d81880675 100644 --- a/packages/kumo-core/src/kumo_core/pquery.py +++ b/packages/kumo-core/src/kumo_core/pquery.py @@ -59,7 +59,7 @@ PADDING_SIZE, YOY_COL, ) -from .training.bounds import validate_time_bounds +from .training.bounds import parse_time_bound, validate_time_bounds from .training.splits import ( SplitArguments, TrainTableTime, @@ -86,6 +86,12 @@ class PredictiveQueryPlanError(ValueError): """A notebook graph/query cannot be converted to a native task plan.""" +class TaskPlanPurpose(str, Enum): + TRAINING = 'training' + PREDICTION = 'prediction' + LABELS = 'labels' + + _NATIVE_TRAINING_TASK_TYPES = frozenset({ "node_prediction", "binary_classification", @@ -1025,7 +1031,12 @@ def _temporal_timestamps( split: Mapping[str, Any], split_family: str, explicit: tuple[int, ...], + *, + purpose: TaskPlanPurpose = TaskPlanPurpose.TRAINING, ) -> tuple[tuple[int, ...], bool]: + if purpose is not TaskPlanPurpose.TRAINING: + return (tuple(_timestamp_ns(value) for value in explicit), + 'flexible_boundaries' in split) offset = query.query.get_combined_date_offset_range() def date_offset(value: int) -> DateOffset: @@ -1041,22 +1052,17 @@ def date_offset(value: int) -> DateOffset: if error is not None: raise PredictiveQueryPlanError(f"{error.title}. {error.message}") - def relative(value: int | str | None) -> DateOffset | Timestamp | None: - if value is None or value == "-INF": - return None - if isinstance(value, Integral): - return date_offset(int(value)) - return Timestamp(Timestamp(value).value) - - start_time: DateOffset | Timestamp | int | None - end_time: DateOffset | Timestamp | int | None + start_time: DateOffset | Timestamp | None + end_time: DateOffset | Timestamp | None node_bounds = query.task_type in TaskType.get_node_pred_tasks() - if not node_bounds: - start_time = relative( - inferred_or_missing_type_to_none(options.start_time)) - end_time = relative(inferred_or_missing_type_to_none(options.end_time)) - else: - try: + try: + if not node_bounds: + start_time = parse_time_bound( + inferred_or_missing_type_to_none(options.start_time), + 'start_time') + end_time = parse_time_bound( + inferred_or_missing_type_to_none(options.end_time), 'end_time') + else: start_time, end_time = validate_time_bounds( inferred_or_missing_type_to_none(options.start_time), inferred_or_missing_type_to_none(options.end_time), @@ -1066,8 +1072,8 @@ def relative(value: int | str | None) -> DateOffset | Timestamp | None: if inferred_or_missing_type_to_none( options.split) is not None else None), ) - except ValueError as error: - raise PredictiveQueryPlanError(str(error)) from error + except ValueError as error: + raise PredictiveQueryPlanError(str(error)) from error absolute_bounds = (isinstance(start_time, Timestamp) or isinstance(end_time, Timestamp)) and node_bounds if explicit and not absolute_bounds: @@ -1075,10 +1081,6 @@ def relative(value: int | str | None) -> DateOffset | Timestamp | None: "flexible_boundaries" in split) minimum, maximum, filter_buffer = _temporal_target_time_range( query, index_path) - start_time = (date_offset(start_time) - if isinstance(start_time, int) else start_time) - end_time = (date_offset(end_time) - if isinstance(end_time, int) else end_time) try: window = (resolve_training_window(start_time, end_time, minimum, maximum) if absolute_bounds else None) @@ -1448,6 +1450,7 @@ def build_native_pquery_task_plan( graph_index_id: str | None = None, timestamps: tuple[int, ...] = (), supplied_rows: bool = False, + purpose: TaskPlanPurpose = TaskPlanPurpose.TRAINING, ) -> NativePqueryTaskPlan: """Resolve task inputs for generated or already-supplied rows. @@ -1481,6 +1484,7 @@ def build_native_pquery_task_plan( split, split_family, timestamps, + purpose=purpose, ) if fixed_boundaries: split = {**split, "flexible_boundaries": False} diff --git a/packages/kumo-core/src/kumo_core/training/bounds.py b/packages/kumo-core/src/kumo_core/training/bounds.py index cd182f862..faa5a85c2 100644 --- a/packages/kumo-core/src/kumo_core/training/bounds.py +++ b/packages/kumo-core/src/kumo_core/training/bounds.py @@ -7,14 +7,15 @@ def parse_time_bound(value: int | str | DateOffset | Timestamp | None, - name: str) -> int | DateOffset | Timestamp | None: - """Keep relative offsets numerical and normalize absolute times to UTC.""" + name: str) -> DateOffset | Timestamp | None: + """Normalize absolute times to UTC and retain parsed internal offsets.""" if value is None or (name == 'start_time' and value == '-INF'): return None if isinstance(value, int): - if value > 0: - raise ValueError(f"'{name}' must be a non-positive relative offset") - return value + raise ValueError( + f"Integer '{name}' bounds are not supported. Use timestamp strings " + "or 'inferred'; use train_start_offset/train_end_offset for " + "relative training bounds.") if isinstance(value, DateOffset): if Timestamp('1970-01-01') + value > Timestamp('1970-01-01'): raise ValueError(f"'{name}' must be a non-positive relative offset") @@ -41,8 +42,7 @@ def validate_time_bounds( train_start_offset: int | DateOffset | None = None, train_end_offset: int | DateOffset | None = None, split_family: str | None = None, -) -> tuple[int | DateOffset | Timestamp | None, - int | DateOffset | Timestamp | None]: +) -> tuple[DateOffset | Timestamp | None, DateOffset | Timestamp | None]: """Validate explicit parameters without changing their serialized form.""" if ((start is not None or end is not None) and (train_start_offset is not None @@ -56,11 +56,9 @@ def validate_time_bounds( != isinstance(parsed_end, Timestamp)): raise ValueError('start_time and end_time must both be offsets ' 'or both be timestamps.') - if ((isinstance(parsed_start, int) and isinstance(parsed_end, int) + if ((isinstance(parsed_start, Timestamp) + and isinstance(parsed_end, Timestamp) and parsed_start >= parsed_end) - or (isinstance(parsed_start, Timestamp) - and isinstance(parsed_end, Timestamp) - and parsed_start >= parsed_end) or (isinstance(parsed_start, DateOffset) and isinstance(parsed_end, DateOffset) and Timestamp('1970-01-01') + parsed_start diff --git a/packages/kumo-core/src/kumo_core/training/workflow.py b/packages/kumo-core/src/kumo_core/training/workflow.py index 7f611ff69..f48124838 100644 --- a/packages/kumo-core/src/kumo_core/training/workflow.py +++ b/packages/kumo-core/src/kumo_core/training/workflow.py @@ -43,6 +43,7 @@ NativeGraph, NativePqueryTaskPlan, PlanGraph, + TaskPlanPurpose, ValidatedNativeQuery, build_native_pquery_task_plan, normalize_graph, @@ -350,7 +351,10 @@ def from_stage_output( def build_training_task_plan( - request: TrainingWorkflowRequest, ) -> NativePqueryTaskPlan: + request: TrainingWorkflowRequest, + *, + purpose: TaskPlanPurpose = TaskPlanPurpose.TRAINING, +) -> NativePqueryTaskPlan: """Build the exact train-table and task-dataset plan for a request.""" return build_native_pquery_task_plan( @@ -363,6 +367,7 @@ def build_training_task_plan( artifact_dir=request.artifact_dir, graph_index_id=request.prepared_graph.index_id, timestamps=request.timestamps, + purpose=purpose, ) diff --git a/packages/kumo-core/tests/test_pquery.py b/packages/kumo-core/tests/test_pquery.py index aa8078f9c..b75844faf 100644 --- a/packages/kumo-core/tests/test_pquery.py +++ b/packages/kumo-core/tests/test_pquery.py @@ -12,6 +12,7 @@ from kumo_core.model_plan import TrainingTableGenerationPlan from kumo_core.pquery import ( PredictiveQueryPlanError, + TaskPlanPurpose, _column_stats, _default_split, _split_options, @@ -687,10 +688,6 @@ def test_plan_rejects_explicit_timestamps_on_a_static_query(tmp_path): "train_start_offset": 20, "train_end_offset": 10 }, 20, 29), - ({ - "start_time": -20, - "end_time": -10 - }, 20, 29), ({ "start_time": "2024-01-11", "end_time": "2024-01-21" @@ -699,10 +696,6 @@ def test_plan_rejects_explicit_timestamps_on_a_static_query(tmp_path): "start_time": "2024-01-11T00:00:00Z", "end_time": "2024-01-21T00:00:00Z" }, 10, 19), - ({ - "start_time": "-INF", - "end_time": -10 - }, 1, 29), ]) def test_generation_plan_time_bounds(tmp_path, monkeypatch, options, first_day, last_day, unit): @@ -850,15 +843,41 @@ def test_explicit_absolute_anchors_respect_available_labels( assert timestamps == tuple(Timestamp(value).value for value in expected) +@pytest.mark.parametrize('purpose', [ + TaskPlanPurpose.PREDICTION, TaskPlanPurpose.LABELS, +]) +def test_prediction_anchors_ignore_training_window(monkeypatch, purpose): + from pandas import Timestamp + + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + 'PREDICT COUNT(events.*, 0, 1, days) > 0 FOR EACH articles.aid', graph) + monkeypatch.setattr( + 'kumo_core.pquery._temporal_target_time_range', + lambda *_args: pytest.fail('Prediction read training label bounds')) + options = TrainingTableGenerationPlan( + start_time='2024-01-05', end_time='2024-01-15') + explicit = tuple(Timestamp(value).value for value in [ + '2025-01-01', '2023-01-01', '2025-01-01']) + timestamps, _ = _temporal_timestamps( + query, '/synthetic-index', options, {}, 'ApproxDateOffsetSplit', + explicit, purpose=purpose) + assert timestamps == explicit + assert options.start_time == '2024-01-05' + assert options.end_time == '2024-01-15' + + @pytest.mark.parametrize('options', [ {'start_time': 'NaT'}, + {'start_time': -20, 'end_time': -10}, + {'start_time': '-INF', 'end_time': -10}, {'start_time': '2024-01-11', 'end_time': '2024-01-21', 'train_end_offset': 10}, {'start_time': '2024-01-11', 'split': "TimeRangeSplit([('2024-01-01', '2024-02-01')])"}, ]) @pytest.mark.parametrize('explicit', [(), (1704067200000000000,)]) -def test_invalid_absolute_bounds_rejected_before_stats( +def test_invalid_bounds_rejected_before_stats( monkeypatch, options, explicit): graph = normalize_graph(forecasting_graph_config()) query = parse_and_validate_query( @@ -1987,8 +2006,6 @@ def unexpected_read(*args): ("2023-04-29", {}, (0, 1), ["2023-03-29", "2023-02-28"]), ("2024-06-30", {"timeframe_step": 2}, (0, 1), ["2024-05-30", "2024-03-30"]), - ("2024-06-30", {"start_time": -3, "end_time": -1}, (0, 1), - ["2024-04-30", "2024-03-30"]), ("2024-06-30", {"train_start_offset": 3, "train_end_offset": 1}, (0, 1), ["2024-04-30", "2024-03-30"]), ("2024-06-30", {"start_time": "2024-01-31"}, (1, 2), diff --git a/packages/kumo-core/tests/test_training_bounds.py b/packages/kumo-core/tests/test_training_bounds.py index c1368b2cb..2a97a106d 100644 --- a/packages/kumo-core/tests/test_training_bounds.py +++ b/packages/kumo-core/tests/test_training_bounds.py @@ -11,7 +11,7 @@ from kumoapi.model_plan import \ TrainingTableGenerationPlan as PublicGenerationPlan from kumoapi.pquery import QueryType -from pandas import Timestamp +from pandas import DateOffset, Timestamp @pytest.mark.parametrize('start,end,message', [ @@ -19,10 +19,10 @@ (None, 'NaT', 'valid timestamp'), ('2024-01-02', '2024-01-02', 'smaller than'), ('2024-01-03', '2024-01-02', 'smaller than'), - (-2, '2024-01-02', 'both be offsets'), - (1, None, 'non-positive'), - (None, 1, 'non-positive'), - (-1, -2, 'smaller than'), + (-2, '2024-01-02', 'Integer'), + (1, None, 'Integer'), + (None, 1, 'Integer'), + (-1, -2, 'Integer'), (None, '3000-01-01', 'valid timestamp'), ]) def test_invalid_bounds(start, end, message): @@ -38,7 +38,8 @@ def test_bounds_normalize_utc(): Timestamp('2024-01-01T07:00:00')) assert validate_time_bounds('-INF', '2024-01-02') == ( None, Timestamp('2024-01-02')) - assert validate_time_bounds(None, -1) == (None, -1) + offset = DateOffset(days=-1) + assert validate_time_bounds(None, offset) == (None, offset) @pytest.mark.parametrize('split', @@ -57,18 +58,20 @@ def test_absolute_ratio_split(split): Timestamp('2024-01-01'), None) -@pytest.mark.parametrize('flag,expected_errors', [ - ('false', True), ('true', False), +@pytest.mark.parametrize('flag', ['false', 'true']) +@pytest.mark.parametrize('task', [ + TaskType.BINARY_CLASSIFICATION, TaskType.TEMPORAL_LINK_PREDICTION, ]) -def test_integer_bounds_keep_gate(flag, expected_errors, monkeypatch): +def test_integer_bounds_rejected(flag, task, monkeypatch): monkeypatch.setenv('USE_DECOUPLED_SPLITS', flag) errors, _ = TrainingTableGenerationPlan(start_time=-10, end_time=0).validate( - task_type=TaskType.BINARY_CLASSIFICATION, + task_type=task, query_type=QueryType.TEMPORAL, has_train_table_time_col=True, run_mode=RunMode.FAST, ) - assert bool(errors) == expected_errors + assert any("Integer 'start_time' bounds are not supported" in str(error) + for error in errors) @pytest.mark.parametrize('task_type', TaskType.get_node_pred_tasks()) diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index d625459d7..30708cb43 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -146,10 +146,11 @@ bounds or an explicit temporal window split (`TimeRangeSplit`, `DateOffsetSplit`, or `BacktestingSplit`) separately. Direct bounds also cannot be combined with `train_start_offset` or `train_end_offset`. Existing task, time-unit, and split restrictions continue to apply. Integer `start_time` and -`end_time` offsets retain their existing decoupled-splits gate and measure -backward from the latest label data in the aggregation's time unit. Temporal -link prediction retains its existing gate. Fit and saved model bundles retain -the effective generation plan. +`end_time` bounds are not supported. Use `train_start_offset` and +`train_end_offset` for relative windows measured backward from the latest label +data in the aggregation's time unit. Absolute bounds for temporal link +prediction are outside this feature's scope. Fit and saved model bundles +retain the effective generation plan. ## Agent skills diff --git a/packages/kumo-engine/kumo/engine/pquery/prediction_table.py b/packages/kumo-engine/kumo/engine/pquery/prediction_table.py index 35de92943..80b348f57 100644 --- a/packages/kumo-engine/kumo/engine/pquery/prediction_table.py +++ b/packages/kumo-engine/kumo/engine/pquery/prediction_table.py @@ -13,6 +13,7 @@ from kumo_core.model_plan import \ TrainingTableGenerationPlan as CoreGenerationPlan from kumo_core.pquery import ( + TaskPlanPurpose, ValidatedNativeQuery, build_native_pquery_task_plan, latest_temporal_timestamp, @@ -189,12 +190,7 @@ def _build_prediction_table_request( from kumo_core.io.json import from_json_dict from kumoapi.json_serde import to_json_dict - # Prediction anchors do not inherit the fitted label window. - options = replace( - from_json_dict(to_json_dict(generation_plan), CoreGenerationPlan), - start_time=options.start_time, - end_time=options.end_time, - ) + options = from_json_dict(to_json_dict(generation_plan), CoreGenerationPlan) parsed = resolve_query_forecast(parsed, native_graph, options) with no_encoder_inference_warnings(): @@ -209,6 +205,7 @@ def _build_prediction_table_request( graph_index_id=snapshot.id, timestamps=timestamps, supplied_rows=supplied_rows, + purpose=TaskPlanPurpose.PREDICTION, ) forecast_timeframe_step = None if parsed.task_type == 'forecasting' and generation_plan is not None: diff --git a/packages/kumo-engine/kumo/engine/pquery/predictive_query.py b/packages/kumo-engine/kumo/engine/pquery/predictive_query.py index 985d46332..bc2d8fb3e 100644 --- a/packages/kumo-engine/kumo/engine/pquery/predictive_query.py +++ b/packages/kumo-engine/kumo/engine/pquery/predictive_query.py @@ -59,6 +59,7 @@ from kumo_core.model_plan_conversion import public_model_plan_from_internal from kumo_core.pquery import ( NativeGraph, + TaskPlanPurpose, ValidatedNativeQuery, parse_and_validate_query, query_is_temporal, @@ -323,7 +324,8 @@ def generate_training_table( def _generate(self, snapshot: GraphSnapshot, options: CoreGenerationPlan, run_dir: Path, *, timestamps: tuple[int, ...] = (), - validate_targets: bool = True) -> tuple: + validate_targets: bool = True, + purpose: TaskPlanPurpose = TaskPlanPurpose.TRAINING) -> tuple: workflow = TrainingWorkflowRequest.from_spec( graph=snapshot.graph_config, predictive_query=self.query, @@ -338,7 +340,7 @@ def _generate(self, snapshot: GraphSnapshot, options: CoreGenerationPlan, timestamps=timestamps, ) with no_encoder_inference_warnings(): - task_plan = build_training_task_plan(workflow) + task_plan = build_training_task_plan(workflow, purpose=purpose) return workflow, task_plan, generate_train_table( task_plan.train_table, validate_targets=validate_targets) @@ -350,21 +352,20 @@ def _label_parts( from kumo_core.io.json import from_json_dict - # Fit windows and split ranges do not constrain evaluation anchors. + # The fit's split ranges do not cover later evaluation anchors. options = CoreGenerationPlan() if plan is not None: options = replace(from_json_dict(to_json_dict(plan), CoreGenerationPlan), - split=options.split, - start_time=options.start_time, - end_time=options.end_time) + split=options.split) run_dir = train_table_scratch_dir(new_run_id()) run_dir.mkdir(parents=True, exist_ok=True) staging = None try: _, _, generated = self._generate(snapshot, options, run_dir, timestamps=timestamps, - validate_targets=False) + validate_targets=False, + purpose=TaskPlanPurpose.LABELS) staging = Path(generated.staging_dir) yield sorted(str(part) for part in Path( generated.train_table_path).glob('*.parquet')) diff --git a/packages/kumo-engine/test/test_predictive_query.py b/packages/kumo-engine/test/test_predictive_query.py index 2bcbd9d12..70414be04 100644 --- a/packages/kumo-engine/test/test_predictive_query.py +++ b/packages/kumo-engine/test/test_predictive_query.py @@ -275,7 +275,7 @@ def test_generated_weight_rejected_before_artifacts(connector_kind, tmp_path, ) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.build_training_task_plan', - lambda workflow: pytest.fail('built a weighted native task plan'), + lambda workflow, *, purpose: pytest.fail('built a weighted native task plan'), ) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.generate_train_table', @@ -306,7 +306,7 @@ def test_generated_weight_rejected_before_artifacts(connector_kind, tmp_path, lambda: 'unweighted') monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.build_training_task_plan', - lambda workflow: native_plan, + lambda workflow, *, purpose: native_plan, ) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.generate_train_table', @@ -338,7 +338,7 @@ def test_generated_temporal_split_requires_time_column(graph_snapshot, @pytest.mark.parametrize('options,message', [ ({'start_time': 'NaT', 'end_time': '2025-01-21'}, 'valid timestamp'), ({'start_time': '2025-01-21', 'end_time': '2025-01-02'}, 'smaller than'), - ({'start_time': '2025-01-02', 'end_time': -1}, 'both be offsets'), + ({'start_time': '2025-01-02', 'end_time': -1}, 'Integer'), ({'start_time': '2025-01-02', 'train_start_offset': 10}, 'Cannot specify both'), ({'end_time': '2025-01-21', 'train_end_offset': 1}, @@ -409,7 +409,7 @@ def test_forecast_generation_retains_effective_plan(graph_snapshot, tmp_path, ) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.build_training_task_plan', - lambda workflow: native_plan, + lambda workflow, *, purpose: native_plan, ) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.generate_train_table', diff --git a/test/config/test_model_plan.py b/test/config/test_model_plan.py index 74fbdfadb..6bdc0d968 100644 --- a/test/config/test_model_plan.py +++ b/test/config/test_model_plan.py @@ -82,6 +82,8 @@ def test_model_plan_basic(): split: inferred train_start_offset: ??? train_end_offset: ??? + start_time: ??? + end_time: ??? timeframe_step: ??? training_job: num_experiments: ??? diff --git a/test/pquery/train_table_generator/test_train_generator_basic.py b/test/pquery/train_table_generator/test_train_generator_basic.py index a9cea5331..a0e4adf89 100644 --- a/test/pquery/train_table_generator/test_train_generator_basic.py +++ b/test/pquery/train_table_generator/test_train_generator_basic.py @@ -916,9 +916,9 @@ def test_generate_time_intervals_start_end_time_absolute( assert task_type == TaskType.BINARY_CLASSIFICATION -def test_generate_time_intervals_start_end_time_relative( +def test_generate_time_intervals_legacy_offsets( create_node_pred_graph, tempdir): - """Test using relative offsets for start_time and end_time""" + """Legacy offsets remain relative to the latest label data.""" graph = create_node_pred_graph('test_graph', tempdir.name) query = ('PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 1) > 30 ' 'FOR EACH USERS.USER_ID') @@ -929,8 +929,8 @@ def test_generate_time_intervals_start_end_time_relative( graph, validated_query, split, - start_time=-4, # 4 days before latest date - end_time=-1 # 1 day before latest date + train_start_offset=4, + train_end_offset=1, ) task_type = generator.task_type @@ -989,40 +989,19 @@ def test_generate_time_intervals_invalid_start_end_time(create_node_pred_graph, assert ('start_time must be smaller than end_time.' in validation_response.error_message()) - # Test with relative offsets - generator = TrainingTableGeneratorV2( - graph, - validated_query, - split, - start_time=-1, # 1 day before latest - end_time=-3 # 3 days before latest - ) - validation_response = generator.validate_pre_train_table_generate() - assert not validation_response.ok - assert ('start_time must be smaller than end_time.' - in validation_response.error_message()) - -def test_generate_time_intervals_mixed_types_error(create_node_pred_graph, - tempdir): - """Test error when mixing absolute and relative timestamps""" +def test_generate_time_intervals_integer_bound_rejected(create_node_pred_graph, + tempdir): + """Integer start bounds have an actionable error before generation.""" graph = create_node_pred_graph('test_graph', tempdir.name) query = ('PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 1) > 30 ' 'FOR EACH USERS.USER_ID') validated_query = get_validated_query(graph, query) split = DateOffsetSplit([-1, 0], unit=TimeUnit.DAYS, timeframe=1) - generator = TrainingTableGeneratorV2( - graph, - validated_query, - split, - start_time=-2, # Relative offset - end_time='1990-01-04' # Absolute timestamp - ) - validation_response = generator.validate_pre_train_table_generate() - assert not validation_response.ok - assert ("start_time and end_time must both be offsets or both be timestamps" - in validation_response.error_message()) + with pytest.raises(ValueError, match="Integer 'start_time' bounds"): + TrainingTableGeneratorV2( + graph, validated_query, split, start_time=-2, end_time='1990-01-04') def test_generate_time_intervals_with_legacy_params_error( @@ -1159,9 +1138,9 @@ def test_generate_timestamps_v2_with_start_end_time(create_node_pred_graph, assert task_type == TaskType.BINARY_CLASSIFICATION -def test_generate_timestamps_v2_with_relative_start_end_time( +def test_generate_timestamps_v2_integer_bound_rejected( create_node_pred_graph, tempdir): - """Test generate_timestamps_v2 with relative start_time and end_time""" + """Decoupled routing also rejects integer end bounds.""" graph = create_node_pred_graph('test_graph', tempdir.name) query = ('PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 8, hours) > 30 ' 'FOR EACH USERS.USER_ID') @@ -1173,24 +1152,10 @@ def test_generate_timestamps_v2_with_relative_start_end_time( anchor_start_offset=0, timeframe_size=8, ) - generator = TrainingTableGeneratorV2( - graph, - validated_query, - split, - start_time=-48, # 2 days before latest date - end_time=-24, # 1 day before latest date - timeframe_step=8, # 24 hour steps - ) - task_type = generator.task_type - - timestamps = generator.generate_timestamps_v2() - # Should only include timestamps between max_date-3 and max_date-1 - assert timestamps == [ - pd.Timestamp('1990-01-03 16:00:00'), # Test - pd.Timestamp('1990-01-03 08:00:00'), # Val - pd.Timestamp('1990-01-03 00:00:00'), # Train - ] - assert task_type == TaskType.BINARY_CLASSIFICATION + with pytest.raises(ValueError, match="Integer 'end_time' bounds"): + TrainingTableGeneratorV2( + graph, validated_query, split, start_time='1990-01-02', end_time=-24, + timeframe_step=8, use_decoupled_splits=True) def test_generate_timestamps_v2_legacy_params_error(create_node_pred_graph, From a5dad59b486c2897d5036dd05bb9f17547c820ed Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Tue, 10 Nov 2026 14:53:41 -0600 Subject: [PATCH 08/10] Consolidate temporal bounds workflow coverage --- packages/kumo-core/tests/test_pquery.py | 214 +++--------------- .../kumo-core/tests/test_training_bounds.py | 150 ------------ .../test/test_absolute_training_bounds.py | 153 +------------ .../kumo-engine/test/test_predictive_query.py | 28 +-- test/config/test_model_plan.py | 10 + 5 files changed, 64 insertions(+), 491 deletions(-) delete mode 100644 packages/kumo-core/tests/test_training_bounds.py diff --git a/packages/kumo-core/tests/test_pquery.py b/packages/kumo-core/tests/test_pquery.py index b75844faf..f2ae8a1bc 100644 --- a/packages/kumo-core/tests/test_pquery.py +++ b/packages/kumo-core/tests/test_pquery.py @@ -730,24 +730,22 @@ def test_generation_plan_time_bounds(tmp_path, monkeypatch, options, first_day, start + day * index for index in range(last_day, first_day - 1, -1)) -@pytest.mark.parametrize('start,end,step,window,expected', [ - ('2024-01-05', '2024-01-15', 3, (0, 3), [12, 9, 6]), - ('2023-12-01', '2024-02-01', 3, (0, 3), [17, 14, 11, 8, 5, 2]), - ('2024-01-05', None, 3, (0, 3), [17, 14, 11, 8, 5]), - (None, '2024-01-15', 3, (0, 3), [12, 9, 6]), - ('2024-01-05', '2024-01-16', 4, (0, 3), [13, 9, 5]), - ('2024-01-05', '2024-01-15', 2, (1, 3), [12, 10, 8, 6]), +@pytest.mark.parametrize('start,end,step,expected', [ + ('2023-12-01', '2024-02-01', 3, [17, 14, 11, 8, 5, 2]), + ('2024-01-05', None, 3, [17, 14, 11, 8, 5]), + (None, '2024-01-15', 3, [12, 9, 6]), + ('2024-01-05', '2024-01-16', 4, [13, 9, 5]), ('2024-01-05T01:00:00+01:00', '2024-01-15T01:00:00+01:00', - 3, (0, 3), [12, 9, 6]), + 3, [12, 9, 6]), + ('2024-02-01', '2024-02-10', 3, None), ]) def test_absolute_bounds_intersect_available_labels( - monkeypatch, start, end, step, window, expected): + monkeypatch, start, end, step, expected): from pandas import Timestamp graph = normalize_graph(forecasting_graph_config()) query = parse_and_validate_query( - f'PREDICT COUNT(events.*, {window[0]}, {window[1]}, days) ' - 'FOR EACH articles.aid', graph) + 'PREDICT COUNT(events.*, 1, 3, days) FOR EACH articles.aid', graph) monkeypatch.setattr( 'kumo_core.pquery._temporal_target_time_range', lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) @@ -756,189 +754,45 @@ def test_absolute_bounds_intersect_available_labels( options.start_time = start if end is not None: options.end_time = end - timestamps, _ = _temporal_timestamps( - query, '/synthetic-index', options, {}, 'ApproxDateOffsetSplit', ()) - assert timestamps == tuple( - Timestamp(f'2024-01-{day:02}').value for day in expected) - - -@pytest.mark.parametrize('start,end', [ - ('2024-02-01', '2024-02-10'), - ('2023-12-01', '2023-12-10'), - ('2024-01-19', '2024-01-20'), - ('1678-01-01', '1679-01-01'), - ('2260-01-01', '2261-01-01'), -]) -def test_absolute_bounds_require_usable_windows(monkeypatch, start, end): - from pandas import Timestamp - - graph = normalize_graph(forecasting_graph_config()) - query = parse_and_validate_query( - 'PREDICT COUNT(events.*, 0, 3, days) FOR EACH articles.aid', graph) - monkeypatch.setattr( - 'kumo_core.pquery._temporal_target_time_range', - lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) - with pytest.raises(PredictiveQueryPlanError, match='No usable training'): - _temporal_timestamps( - query, '/synthetic-index', TrainingTableGenerationPlan( - start_time=start, end_time=end), {}, 'ApproxDateOffsetSplit', ()) - - -@pytest.mark.parametrize('clause,expected', [ - (' > 0', [13, 5, 12, 5]), - (' FORECAST 3 TIMEFRAMES', [13, 5, 12, 5]), -]) -@pytest.mark.parametrize('split', [{}, {'flexible_boundaries': False}]) -def test_absolute_bounds_filter_explicit_training_anchors( - monkeypatch, clause, expected, split): - from pandas import Timestamp - - graph = normalize_graph(forecasting_graph_config()) - query = parse_and_validate_query( - f'PREDICT COUNT(events.*, 0, 1, days){clause} ' - 'FOR EACH articles.aid', graph) - monkeypatch.setattr( - 'kumo_core.pquery._temporal_target_time_range', - lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) - off_grid = Timestamp('2024-01-06T12:34:56.123456789').value - explicit = (off_grid,) + tuple( - Timestamp(f'2024-01-{day:02}').value - for day in [13, 4, 5, 12, 5, 19]) + (Timestamp.max.value,) - timestamps, fixed = _temporal_timestamps( - query, '/synthetic-index', TrainingTableGenerationPlan( - start_time='2024-01-05', end_time='2024-01-15', timeframe_step=99), - split, 'ApproxDateOffsetSplit', explicit) - assert timestamps == (off_grid,) + tuple( - Timestamp(f'2024-01-{day:02}').value for day in expected) - assert fixed == ('flexible_boundaries' in split) - - -@pytest.mark.parametrize('explicit,expected', [ - (['2024-01-01', '2024-01-19', '2024-01-20'], - ['2024-01-01', '2024-01-19']), - (['2024-01-20', '2024-01-21'], []), -]) -def test_explicit_absolute_anchors_respect_available_labels( - monkeypatch, explicit, expected): - from pandas import Timestamp - - graph = normalize_graph(forecasting_graph_config()) - query = parse_and_validate_query( - 'PREDICT COUNT(events.*, 0, 1, days) > 0 FOR EACH articles.aid', graph) - monkeypatch.setattr( - 'kumo_core.pquery._temporal_target_time_range', - lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) - options = TrainingTableGenerationPlan( - start_time='2023-12-01', end_time='2024-02-01') - anchors = tuple(Timestamp(value).value for value in explicit) - if not expected: + if expected is None: with pytest.raises(PredictiveQueryPlanError, match='No usable training'): _temporal_timestamps( query, '/synthetic-index', options, {}, - 'ApproxDateOffsetSplit', anchors) + 'ApproxDateOffsetSplit', ()) else: timestamps, _ = _temporal_timestamps( - query, '/synthetic-index', options, {}, - 'ApproxDateOffsetSplit', anchors) - assert timestamps == tuple(Timestamp(value).value for value in expected) + query, '/synthetic-index', options, {}, 'ApproxDateOffsetSplit', ()) + assert timestamps == tuple( + Timestamp(f'2024-01-{day:02}').value for day in expected) -@pytest.mark.parametrize('purpose', [ - TaskPlanPurpose.PREDICTION, TaskPlanPurpose.LABELS, -]) -def test_prediction_anchors_ignore_training_window(monkeypatch, purpose): +@pytest.mark.parametrize('purpose', list(TaskPlanPurpose)) +def test_explicit_anchors_follow_planning_purpose(monkeypatch, purpose): from pandas import Timestamp graph = normalize_graph(forecasting_graph_config()) query = parse_and_validate_query( 'PREDICT COUNT(events.*, 0, 1, days) > 0 FOR EACH articles.aid', graph) - monkeypatch.setattr( - 'kumo_core.pquery._temporal_target_time_range', - lambda *_args: pytest.fail('Prediction read training label bounds')) - options = TrainingTableGenerationPlan( - start_time='2024-01-05', end_time='2024-01-15') - explicit = tuple(Timestamp(value).value for value in [ - '2025-01-01', '2023-01-01', '2025-01-01']) - timestamps, _ = _temporal_timestamps( - query, '/synthetic-index', options, {}, 'ApproxDateOffsetSplit', - explicit, purpose=purpose) - assert timestamps == explicit - assert options.start_time == '2024-01-05' - assert options.end_time == '2024-01-15' - - -@pytest.mark.parametrize('options', [ - {'start_time': 'NaT'}, - {'start_time': -20, 'end_time': -10}, - {'start_time': '-INF', 'end_time': -10}, - {'start_time': '2024-01-11', 'end_time': '2024-01-21', - 'train_end_offset': 10}, - {'start_time': '2024-01-11', - 'split': "TimeRangeSplit([('2024-01-01', '2024-02-01')])"}, -]) -@pytest.mark.parametrize('explicit', [(), (1704067200000000000,)]) -def test_invalid_bounds_rejected_before_stats( - monkeypatch, options, explicit): - graph = normalize_graph(forecasting_graph_config()) - query = parse_and_validate_query( - 'PREDICT COUNT(events.*, 0, 1, days) FOR EACH articles.aid', graph) - monkeypatch.setattr( - 'kumo_core.pquery._temporal_target_time_range', - lambda *_args: pytest.fail('Read stats before rejecting invalid bounds')) - with pytest.raises(PredictiveQueryPlanError): - _temporal_timestamps( - query, '/synthetic-index', TrainingTableGenerationPlan(**options), - {}, 'TimeRangeSplit' if 'split' in options else 'ApproxDateOffsetSplit', - explicit) - -@pytest.mark.parametrize('end', [None, '2024-01-15']) -def test_absolute_forecast_candidates_fit_base_window(monkeypatch, end): - from pandas import DateOffset, Timestamp + def available_bounds(query, index_path): + assert purpose is TaskPlanPurpose.TRAINING + return Timestamp('2024-01-01'), Timestamp('2024-01-20'), None - graph = normalize_graph(forecasting_graph_config()) - query = parse_and_validate_query( - 'PREDICT COUNT(events.*, 1, 3, days) FORECAST 3 TIMEFRAMES ' - 'FOR EACH articles.aid', graph) - monkeypatch.setattr( - 'kumo_core.pquery._temporal_target_time_range', - lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) - options = TrainingTableGenerationPlan( - start_time='2024-01-05', timeframe_step=2) - if end is not None: - options.end_time = end - split, family = _split_options(query, graph, options) - timestamps, _ = _temporal_timestamps( - query, '/synthetic-index', options, split, family, ()) - cutoff = Timestamp(end or '2024-01-20') - newest = Timestamp(timestamps[0]) - # Native generation reserves the remaining forecast tail once. - assert newest == cutoff - DateOffset(days=3) - assert all(Timestamp(value) >= Timestamp('2024-01-05') - for value in timestamps) - - -def test_absolute_monthly_forecast_keeps_base_candidates(monkeypatch): - from kumo_core.pquery import _forecast_options - from pandas import DateOffset, Timestamp - - graph = normalize_graph(forecasting_graph_config()) - query = parse_and_validate_query( - 'PREDICT COUNT(events.*, 0, 1, months) FORECAST 3 TIMEFRAMES ' - 'FOR EACH articles.aid', graph) - monkeypatch.setattr( - 'kumo_core.pquery._temporal_target_time_range', - lambda *_args: (Timestamp('2023-01-01'), Timestamp('2024-12-31'), None)) - options = TrainingTableGenerationPlan( - start_time='2023-01-01', end_time='2024-05-31', timeframe_step=1) - split, family = _split_options(query, graph, options) - timestamps, _ = _temporal_timestamps( - query, '/synthetic-index', options, split, family, ()) - _, compiler = _forecast_options(query, options) - assert compiler['future_offsets_secs'] == (0, 31 * 86400, 62 * 86400) - assert timestamps - assert all(Timestamp(value) + DateOffset(months=1) - <= Timestamp('2024-05-31') for value in timestamps) + monkeypatch.setattr('kumo_core.pquery._temporal_target_time_range', + available_bounds) + off_grid = Timestamp('2024-01-06T12:34:56.123456789').value + explicit = (off_grid,) + tuple( + Timestamp(f'2024-01-{day:02}').value for day in [19, 4, 5, 20, 25, 5]) + timestamps, fixed = _temporal_timestamps( + query, '/synthetic-index', TrainingTableGenerationPlan( + start_time='2024-01-05', end_time='2024-02-01', timeframe_step=99), + {'flexible_boundaries': False}, 'ApproxDateOffsetSplit', explicit, + purpose=purpose) + expected = ((off_grid,) + tuple( + Timestamp(f'2024-01-{day:02}').value for day in [19, 5, 5]) + if purpose is TaskPlanPurpose.TRAINING else explicit) + assert timestamps == expected + assert fixed def test_latest_temporal_timestamp_uses_source_maximum(tmp_path, monkeypatch): diff --git a/packages/kumo-core/tests/test_training_bounds.py b/packages/kumo-core/tests/test_training_bounds.py deleted file mode 100644 index 2a97a106d..000000000 --- a/packages/kumo-core/tests/test_training_bounds.py +++ /dev/null @@ -1,150 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest -from kumo_core.io.json import from_json_dict, to_json_dict -from kumo_core.model_plan import TrainingTableGenerationPlan -from kumo_core.training.bounds import validate_time_bounds -from kumo_core.types.enums import TaskType -from kumoapi.model_plan import InferredType, RunMode -from kumoapi.model_plan import \ - TrainingTableGenerationPlan as PublicGenerationPlan -from kumoapi.pquery import QueryType -from pandas import DateOffset, Timestamp - - -@pytest.mark.parametrize('start,end,message', [ - ('invalid-date', None, 'valid timestamp'), - (None, 'NaT', 'valid timestamp'), - ('2024-01-02', '2024-01-02', 'smaller than'), - ('2024-01-03', '2024-01-02', 'smaller than'), - (-2, '2024-01-02', 'Integer'), - (1, None, 'Integer'), - (None, 1, 'Integer'), - (-1, -2, 'Integer'), - (None, '3000-01-01', 'valid timestamp'), -]) -def test_invalid_bounds(start, end, message): - with pytest.raises(ValueError, match=message): - validate_time_bounds(start, end) - - -def test_bounds_normalize_utc(): - assert validate_time_bounds( - '2024-01-01T00:00:00-06:00', - '2024-01-01T08:00:00+01:00', - ) == (Timestamp('2024-01-01T06:00:00'), - Timestamp('2024-01-01T07:00:00')) - assert validate_time_bounds('-INF', '2024-01-02') == ( - None, Timestamp('2024-01-02')) - offset = DateOffset(days=-1) - assert validate_time_bounds(None, offset) == (None, offset) - - -@pytest.mark.parametrize('split', - ['BacktestingSplit', 'DateOffsetSplit', - 'TimeRangeSplit']) -def test_absolute_split_conflict(split): - with pytest.raises(ValueError, match='cannot be combined'): - validate_time_bounds(None, '2024-01-02', split_family=split) - - -@pytest.mark.parametrize('split', - ['ApproxDateOffsetSplit', 'RandomSplit', - 'TemporalSplit']) -def test_absolute_ratio_split(split): - assert validate_time_bounds('2024-01-01', None, split_family=split) == ( - Timestamp('2024-01-01'), None) - - -@pytest.mark.parametrize('flag', ['false', 'true']) -@pytest.mark.parametrize('task', [ - TaskType.BINARY_CLASSIFICATION, TaskType.TEMPORAL_LINK_PREDICTION, -]) -def test_integer_bounds_rejected(flag, task, monkeypatch): - monkeypatch.setenv('USE_DECOUPLED_SPLITS', flag) - errors, _ = TrainingTableGenerationPlan(start_time=-10, end_time=0).validate( - task_type=task, - query_type=QueryType.TEMPORAL, - has_train_table_time_col=True, - run_mode=RunMode.FAST, - ) - assert any("Integer 'start_time' bounds are not supported" in str(error) - for error in errors) - - -@pytest.mark.parametrize('task_type', TaskType.get_node_pred_tasks()) -@pytest.mark.parametrize('flag', [None, 'false', 'true']) -def test_node_bound_validation(task_type, flag, monkeypatch): - if flag is None: - monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) - else: - monkeypatch.setenv('USE_DECOUPLED_SPLITS', flag) - plan = TrainingTableGenerationPlan( - start_time='2024-01-01T00:00:00-06:00', - end_time='2024-01-02T00:00:00-06:00', - ) - errors, _ = plan.validate( - task_type=task_type, - query_type=QueryType.TEMPORAL, - has_train_table_time_col=True, - run_mode=RunMode.FAST, - ) - assert not errors - assert plan.start_time == '2024-01-01T00:00:00-06:00' - - -@pytest.mark.parametrize('flag', [None, 'false', 'true']) -@pytest.mark.parametrize('inferred', ['inferred', InferredType.VALUE]) -@pytest.mark.parametrize('bound', ['start_time', 'end_time']) -def test_public_inferred_bound(flag, inferred, bound, monkeypatch): - if flag is None: - monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) - else: - monkeypatch.setenv('USE_DECOUPLED_SPLITS', flag) - public = PublicGenerationPlan(**{ - 'start_time': '2024-01-01', - 'end_time': '2024-02-01', - bound: inferred, - }) - # Match generate_training_table's public-plan preflight conversion. - plan = from_json_dict(to_json_dict(public), TrainingTableGenerationPlan) - errors, _ = plan.validate( - task_type=TaskType.BINARY_CLASSIFICATION, - query_type=QueryType.TEMPORAL, - has_train_table_time_col=True, - run_mode=RunMode.FAST, - ) - assert not errors - - -def test_link_bounds_keep_gate(monkeypatch): - monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) - errors, _ = TrainingTableGenerationPlan(start_time='2024-01-01').validate( - task_type=TaskType.TEMPORAL_LINK_PREDICTION, - query_type=QueryType.TEMPORAL, - has_train_table_time_col=True, - run_mode=RunMode.FAST, - ) - assert any("'start_time' field is not supported" in str(error) - for error in errors) - - -@pytest.mark.parametrize('options,message', [ - ({'start_time': 'NaT'}, 'valid timestamp'), - ({'start_time': '2024-01-01', 'train_start_offset': 2}, - 'Cannot specify both'), - ({'start_time': '2024-01-01', - 'split': "TimeRangeSplit([('2024-01-01', '2024-01-02')])"}, - 'cannot be combined'), -]) -def test_plan_rejects_invalid_bounds(options, message, monkeypatch): - monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'true') - errors, _ = TrainingTableGenerationPlan(**options).validate( - task_type=TaskType.BINARY_CLASSIFICATION, - query_type=QueryType.TEMPORAL, - has_train_table_time_col=True, - run_mode=RunMode.FAST, - ) - assert any(message in str(error) for error in errors) diff --git a/packages/kumo-engine/test/test_absolute_training_bounds.py b/packages/kumo-engine/test/test_absolute_training_bounds.py index 87a7a4dfa..f4757395c 100644 --- a/packages/kumo-engine/test/test_absolute_training_bounds.py +++ b/packages/kumo-engine/test/test_absolute_training_bounds.py @@ -6,10 +6,8 @@ import pyarrow as pa import pyarrow.parquet as pq import pytest -import torch -from kumo_core.io.json import to_json_dict from kumoapi.graph import Edge -from kumoapi.model_plan import ModelPlan, TrainingTableGenerationPlan +from kumoapi.model_plan import TrainingTableGenerationPlan from kumoapi.typing import Stype from io_utils import fsutil @@ -19,7 +17,6 @@ Graph, PredictiveQuery, Table, - Trainer, TrainingTableOutputConfig, configure, ) @@ -27,16 +24,11 @@ START = pd.Timestamp('2024-01-10') END = pd.Timestamp('2024-02-20') -SOURCE_END = pd.Timestamp('2024-02-29') TASKS = [ ('binary_classification', 'COUNT(events.*, 0, 2, days) > 0'), - ('regression', 'COUNT(events.*, 0, 2, days)'), - ('multiclass_classification', 'FIRST(events.category, 0, 2, days)'), + ('forecasting', 'COUNT(events.*, 0, 2, days) FORECAST 2 TIMEFRAMES'), ('multilabel_classification', 'LIST_DISTINCT(events.category, 0, 2, days) CLASSIFY TOP 3'), - ('multilabel_ranking', - 'LIST_DISTINCT(events.category, 0, 2, days) RANK TOP 3'), - ('forecasting', 'COUNT(events.*, 0, 2, days) FORECAST 3 TIMEFRAMES'), ] @@ -93,7 +85,8 @@ def test_temporal_node_bounds(temporal_graph, tmp_path, monkeypatch, task, monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) options = dict(start_time=START.isoformat(), end_time=END.isoformat()) if task == 'forecasting': - options.update(lag_timesteps=3, year_over_year=False) + options.update(forecast_length=3, lag_timesteps=3, + year_over_year=False) query, table, rows = _generate(temporal_graph, tmp_path, target, **options) assert query.get_task_type().value == task assert set(rows['SPLIT']) == {0, 1, 2} @@ -102,142 +95,14 @@ def test_temporal_node_bounds(temporal_graph, tmp_path, monkeypatch, task, assert rows['TIMESTAMP'].max() + pd.Timedelta(days=horizon) <= END assert table.generation_plan.start_time == START.isoformat() assert table.generation_plan.end_time == END.isoformat() - if task in {'binary_classification', 'regression'}: + if task == 'forecasting': + assert table.generation_plan.forecast_length == 3 + assert rows['TIMESTAMP'].max() == END - pd.Timedelta(days=6) + if task == 'binary_classification': events = temporal_graph[2] for row in rows.itertuples(): count = len(events[(events['user_id'] == row.ENTITY1) & (events['ts'] > row.TIMESTAMP) & (events['ts'] <= row.TIMESTAMP + pd.Timedelta(days=2))]) - assert row.TARGET1 == (count > 0 if task.startswith('binary') - else count) - - -@pytest.mark.parametrize('bounds', [ - {'start_time': '2023-12-01', 'end_time': '2024-04-01'}, - {'start_time': '2024-01-10'}, - {'end_time': '2024-02-20'}, - {'start_time': '2024-01-10', 'end_time': 'inferred'}, - {'start_time': 'inferred', 'end_time': '2024-02-20'}, - {'start_time': '-INF', 'end_time': '2024-02-20'}, -]) -def test_partial_and_clipped_bounds(temporal_graph, tmp_path, monkeypatch, - bounds): - monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'false') - _, _, rows = _generate(temporal_graph, tmp_path, - 'COUNT(events.*, 0, 2, days)', **bounds) - lower = max(pd.Timestamp('2024-01-01'), - pd.Timestamp(bounds['start_time']) - if bounds.get('start_time') not in {None, '-INF', 'inferred'} else - pd.Timestamp('2024-01-01')) - end = bounds.get('end_time', 'inferred') - upper = min(SOURCE_END, pd.Timestamp(end) if end != 'inferred' else - SOURCE_END) - assert not rows.empty - assert rows['TIMESTAMP'].min() >= lower - assert rows['TIMESTAMP'].max() + pd.Timedelta(days=2) <= upper - - -@pytest.mark.parametrize('override', [False, True]) -def test_forecast_complete_horizon(temporal_graph, tmp_path, monkeypatch, - override): - monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'false') - target = 'COUNT(events.*, 0, 2, days)' - options = dict(start_time=START.isoformat(), lag_timesteps=3, - year_over_year=False) - if override: - options.update(forecast_length=3, end_time=END.isoformat()) - else: - target += ' FORECAST 3 TIMEFRAMES' - _, table, rows = _generate(temporal_graph, tmp_path, target, **options) - cutoff = END if override else SOURCE_END - assert not rows.empty - assert rows['TIMESTAMP'].max() + pd.Timedelta(days=6) <= cutoff - assert rows['TIMESTAMP'].max() == cutoff - pd.Timedelta(days=6) - assert table.generation_plan.forecast_length == 3 - - -@pytest.mark.parametrize('split', [None, 'RandomSplit([0.6, 0.2, 0.2])', - 'TemporalSplit([0.6, 0.2, 0.2])', - 'ApproxDateOffsetSplit([0.6, 0.2, 0.2])']) -def test_bound_timezone_equivalence(temporal_graph, tmp_path, monkeypatch, - split): - monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) - options = {} if split is None else {'split': split} - _, _, utc = _generate( - temporal_graph, tmp_path / 'utc', 'COUNT(events.*, 0, 2, days)', - start_time='2024-01-10T00:00:00Z', - end_time='2024-02-20T00:00:00Z', **options) - _, _, local = _generate( - temporal_graph, tmp_path / 'local', 'COUNT(events.*, 0, 2, days)', - start_time='2024-01-09T18:00:00-06:00', - end_time='2024-02-20T01:00:00+01:00', **options) - columns = ['TIMESTAMP', 'ENTITY1', 'SPLIT'] - pd.testing.assert_frame_equal( - utc.sort_values(columns).reset_index(drop=True), - local.sort_values(columns).reset_index(drop=True)) - if split == 'ApproxDateOffsetSplit([0.6, 0.2, 0.2])': - expected = pd.date_range('2024-01-11', '2024-02-18', freq='2D') - assert set(utc['TIMESTAMP']) == set(expected) - - -@pytest.mark.parametrize('options', [ - {}, - {'train_start_offset': 40, 'train_end_offset': 10}, - {'split': "TimeRangeSplit([('2024-01-10', '2024-01-24'), " - "('2024-01-24', '2024-02-07'), " - "('2024-02-07', '2024-02-20')])"}, -]) -def test_existing_window_controls(temporal_graph, tmp_path, monkeypatch, - options): - monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) - _, _, rows = _generate(temporal_graph, tmp_path, - 'COUNT(events.*, 0, 2, days)', **options) - assert set(rows['SPLIT']) == {0, 1, 2} - assert rows['TIMESTAMP'].min() >= pd.Timestamp('2024-01-01') - assert rows['TIMESTAMP'].max() + pd.Timedelta(days=2) <= SOURCE_END - - -@pytest.mark.parametrize('start,end', [ - ('2024-03-10', '2024-03-20'), - ('2024-01-10', '2024-01-11'), -]) -def test_no_complete_window(temporal_graph, tmp_path, monkeypatch, start, end): - monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) - with pytest.raises(ValueError, match='No usable training examples'): - _generate(temporal_graph, tmp_path, 'COUNT(events.*, 0, 2, days)', - start_time=start, end_time=end) - - -def test_bounds_fit_reload_predict(temporal_graph, tmp_path, monkeypatch, - request): - monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) - threads = torch.get_num_threads() - torch.set_num_threads(2) - request.addfinalizer(lambda: torch.set_num_threads(threads)) - query, table, _ = _generate( - temporal_graph, tmp_path, 'COUNT(events.*, 0, 2, days) > 0', - start_time='2024-01-10T00:00:00Z', - end_time='2024-02-19T18:00:00-06:00') - plan = ModelPlan() - plan.training_job.num_experiments = 1 - plan.training_job.disable_explain = True - plan.training_job.refit_trainval = False - plan.optimization.max_epochs = 1 - plan.optimization.max_steps_per_epoch = 2 - plan.optimization.batch_size = [32] - plan.model_architecture.channels = [16] - snapshot = temporal_graph[1] - trainer = Trainer(plan, accelerator='cpu') - result = trainer.fit(table, snapshot=snapshot) - restored = Trainer.load(result.bundle_dir, accelerator='cpu') - expected = to_json_dict(table.generation_plan) - assert to_json_dict(trainer.training_table_generation_plan) == expected - assert to_json_dict(restored.training_table_generation_plan) == expected - prediction_table = query.generate_prediction_table(snapshot) - before = trainer.predict(prediction_table, snapshot=snapshot, - binary_classification_threshold=0.5).frame - after = restored.predict(prediction_table, snapshot=snapshot, - binary_classification_threshold=0.5).frame - assert len(after) == 24 - pd.testing.assert_frame_equal(before, after, atol=0.005, rtol=0) + assert row.TARGET1 == (count > 0) diff --git a/packages/kumo-engine/test/test_predictive_query.py b/packages/kumo-engine/test/test_predictive_query.py index 70414be04..84485b42c 100644 --- a/packages/kumo-engine/test/test_predictive_query.py +++ b/packages/kumo-engine/test/test_predictive_query.py @@ -334,33 +334,19 @@ def test_generated_temporal_split_requires_time_column(graph_snapshot, split='TemporalSplit([0.6, 0.3, 0.1])')) -@pytest.mark.parametrize('flag', [None, 'false']) @pytest.mark.parametrize('options,message', [ ({'start_time': 'NaT', 'end_time': '2025-01-21'}, 'valid timestamp'), - ({'start_time': '2025-01-21', 'end_time': '2025-01-02'}, 'smaller than'), - ({'start_time': '2025-01-02', 'end_time': -1}, 'Integer'), ({'start_time': '2025-01-02', 'train_start_offset': 10}, 'Cannot specify both'), - ({'end_time': '2025-01-21', 'train_end_offset': 1}, - 'Cannot specify both'), ({'start_time': '2025-01-02', 'split': "TimeRangeSplit([('2025-01-02', '2025-01-10'), " "('2025-01-10', '2025-01-21')])"}, 'cannot be combined'), - ({'end_time': '2025-01-21', - 'split': 'DateOffsetSplit([-4, -2], timeframe=1)'}, - 'cannot be combined'), - ({'end_time': '2025-01-21', - 'split': 'TimeSplit([0.6, 0.2, 0.2])'}, - 'TimeSplit'), ]) def test_generated_invalid_bounds_fail_before_artifacts( - graph_snapshot, monkeypatch, options, message, flag): + graph_snapshot, monkeypatch, options, message): graph, snapshot = graph_snapshot - if flag is None: - monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) - else: - monkeypatch.setenv('USE_DECOUPLED_SPLITS', flag) + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.new_run_id', lambda: pytest.fail('allocated a bounded-generation run id'), @@ -494,8 +480,9 @@ def test_evaluate_prediction_after_absolute_training_bounds( plan.optimization.batch_size = [32] plan.model_architecture.channels = [16] trainer = Trainer(plan, accelerator='cpu') - trainer.fit(table, snapshot=snapshot) + training_result = trainer.fit(table, snapshot=snapshot) fitted = to_json_dict(trainer.training_table_generation_plan) + assert fitted == to_json_dict(table.generation_plan) prediction = query.generate_prediction_table( snapshot, prediction_time=pd.Timestamp('2024-03-01').to_pydatetime(), plan=trainer.training_table_generation_plan) @@ -513,6 +500,13 @@ def test_evaluate_prediction_after_absolute_training_bounds( assert future.timestamps == (future_anchor.value,) assert future.row_count == 50 assert to_json_dict(trainer.training_table_generation_plan) == fitted + restored = Trainer.load(training_result.bundle_dir, accelerator='cpu') + assert to_json_dict(restored.training_table_generation_plan) == fitted + reloaded_run = restored.predict( + prediction, snapshot=snapshot, binary_classification_threshold=0.5) + assert len(reloaded_run.frame) == 50 + pd.testing.assert_frame_equal(run.frame, reloaded_run.frame, + atol=0.005, rtol=0) def test_custom_training_table_materializes_weight_and_manual_split( diff --git a/test/config/test_model_plan.py b/test/config/test_model_plan.py index 6bdc0d968..5c25031c3 100644 --- a/test/config/test_model_plan.py +++ b/test/config/test_model_plan.py @@ -2345,6 +2345,16 @@ def test_model_plan_decoupled_splits(monkeypatch): assert ( plan.training_table_generation.train_start_offset == MissingType.VALUE) + plan.training_table_generation.start_time = -10 + with pytest.raises(ValueError, match="Integer 'start_time' bounds"): + plan.validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + plan.training_table_generation.start_time = InferredType.VALUE + # Verify validation fails if train_start_offset is set plan.training_table_generation.train_start_offset = 10 with pytest.raises(ValueError, match="field is not supported"): From d3e3fd1703dd7c802cb37f3d4a142bf251b5299b Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Tue, 10 Nov 2026 15:10:58 -0600 Subject: [PATCH 09/10] Address remaining absolute bounds review details --- packages/kumo-api/kumoapi/model_plan.py | 12 ++---- packages/kumo-core/src/kumo_core/pquery.py | 13 ++++++- .../src/kumo_core/training/bounds.py | 6 +-- packages/kumo-core/tests/test_pquery.py | 20 ++++++++-- packages/kumo-engine/README.md | 38 ------------------- .../test/test_absolute_training_bounds.py | 6 ++- 6 files changed, 39 insertions(+), 56 deletions(-) diff --git a/packages/kumo-api/kumoapi/model_plan.py b/packages/kumo-api/kumoapi/model_plan.py index 267f6f5fb..8b7dbb653 100644 --- a/packages/kumo-api/kumoapi/model_plan.py +++ b/packages/kumo-api/kumoapi/model_plan.py @@ -608,15 +608,13 @@ class TrainingTableGenerationPlan(PlanMixin): :ivar train_start_offset: (``int`` | ``"inferred``") Defines the numerical offset from the most recent entry to use to generate training data labels. Unless a custom time unit is specified in the aggregation, this - value is in days (*default:* ``"inferred"``). Eventually we would - like to migrate this parameter to start_time. + value is in days (*default:* ``"inferred"``). **Supported Task Types:** Temporal :ivar train_end_offset: (``int`` | ``"inferred"``) Defines the numerical offset from the most recent entry to not use to generate training data labels. Unless a custom time unit is specified in the aggregation, this - value is in days (*default:* ``"inferred"``). Eventually we would - like to migrate this parameter to end_time. + value is in days (*default:* ``"inferred"``). **Supported Task Types:** Temporal :ivar start_time: (``"inferred"`` | ``str``) Inclusive lower @@ -626,8 +624,7 @@ class TrainingTableGenerationPlan(PlanMixin): are read as UTC. ``"inferred"`` or ``"-INF"`` uses the available-data lower bound. Bounds are intersected with available label data. Direct absolute bounds cannot be combined with train_start_offset, - train_end_offset, or explicit date/offset window splits. Ratio splits - remain subject to existing task restrictions. + train_end_offset, or explicit date/offset window splits. Integer bounds are not supported. For relative training windows, use train_start_offset/train_end_offset in the aggregation's time unit. (*default:* ``"inferred"``). @@ -641,8 +638,7 @@ class TrainingTableGenerationPlan(PlanMixin): timestamp strings as start_time; ``"inferred"`` uses available label data. Either absolute bound can be omitted. Empty intersections or windows too short for a complete - example fail generation. Existing time-unit and split constraints - continue to apply. (*default:* ``"inferred"``). + example fail generation. (*default:* ``"inferred"``). **Supported Task Types:** Temporal node classification, regression, ranking, and forecasting. diff --git a/packages/kumo-core/src/kumo_core/pquery.py b/packages/kumo-core/src/kumo_core/pquery.py index d81880675..92e7c6af7 100644 --- a/packages/kumo-core/src/kumo_core/pquery.py +++ b/packages/kumo-core/src/kumo_core/pquery.py @@ -13,6 +13,7 @@ import hashlib import json +import logging import os import re from dataclasses import dataclass @@ -81,6 +82,8 @@ from .training.validation import validate_train_offsets from .types import TaskType +logger = logging.getLogger(__name__) + class PredictiveQueryPlanError(ValueError): """A notebook graph/query cannot be converted to a native task plan.""" @@ -1098,8 +1101,14 @@ def bounded(values: Sequence[int]) -> tuple[int, ...]: raise PredictiveQueryPlanError(str(error)) from error if explicit: - return (bounded(tuple(_timestamp_ns(value) for value in explicit)), - "flexible_boundaries" in split) + timestamps = bounded(tuple(_timestamp_ns(value) for value in explicit)) + dropped = len(explicit) - len(timestamps) + if dropped: + logger.warning( + 'Dropped %d of %d explicit training anchors outside ' + 'start_time/end_time or available label data.', + dropped, len(explicit)) + return timestamps, "flexible_boundaries" in split step_count = int( inferred_or_missing_type_to_none(options.timeframe_step) diff --git a/packages/kumo-core/src/kumo_core/training/bounds.py b/packages/kumo-core/src/kumo_core/training/bounds.py index faa5a85c2..47e8d78a6 100644 --- a/packages/kumo-core/src/kumo_core/training/bounds.py +++ b/packages/kumo-core/src/kumo_core/training/bounds.py @@ -27,9 +27,7 @@ def parse_time_bound(value: int | str | DateOffset | Timestamp | None, if timestamp.tzinfo is not None: timestamp = timestamp.tz_convert('UTC').tz_localize(None) # Generation stores timestamps as signed nanoseconds. - timestamp = Timestamp(timestamp.value) - if not isinstance(timestamp, Timestamp): - raise ValueError('timestamp is NaT') + _ = timestamp.value except (ValueError, TypeError, OverflowError) as error: raise ValueError(f"'{name}' must be a valid timestamp: {value!r}") from error return timestamp @@ -43,7 +41,7 @@ def validate_time_bounds( train_end_offset: int | DateOffset | None = None, split_family: str | None = None, ) -> tuple[DateOffset | Timestamp | None, DateOffset | Timestamp | None]: - """Validate explicit parameters without changing their serialized form.""" + """Parse start/end bounds and reject invalid combinations.""" if ((start is not None or end is not None) and (train_start_offset is not None or train_end_offset is not None)): diff --git a/packages/kumo-core/tests/test_pquery.py b/packages/kumo-core/tests/test_pquery.py index f2ae8a1bc..f4140050c 100644 --- a/packages/kumo-core/tests/test_pquery.py +++ b/packages/kumo-core/tests/test_pquery.py @@ -767,7 +767,7 @@ def test_absolute_bounds_intersect_available_labels( @pytest.mark.parametrize('purpose', list(TaskPlanPurpose)) -def test_explicit_anchors_follow_planning_purpose(monkeypatch, purpose): +def test_explicit_anchors_follow_planning_purpose(monkeypatch, caplog, purpose): from pandas import Timestamp graph = normalize_graph(forecasting_graph_config()) @@ -783,9 +783,10 @@ def available_bounds(query, index_path): off_grid = Timestamp('2024-01-06T12:34:56.123456789').value explicit = (off_grid,) + tuple( Timestamp(f'2024-01-{day:02}').value for day in [19, 4, 5, 20, 25, 5]) + options = TrainingTableGenerationPlan( + start_time='2024-01-05', end_time='2024-02-01', timeframe_step=99) timestamps, fixed = _temporal_timestamps( - query, '/synthetic-index', TrainingTableGenerationPlan( - start_time='2024-01-05', end_time='2024-02-01', timeframe_step=99), + query, '/synthetic-index', options, {'flexible_boundaries': False}, 'ApproxDateOffsetSplit', explicit, purpose=purpose) expected = ((off_grid,) + tuple( @@ -793,6 +794,19 @@ def available_bounds(query, index_path): if purpose is TaskPlanPurpose.TRAINING else explicit) assert timestamps == expected assert fixed + messages = [record.getMessage() for record in caplog.records + if record.name == 'kumo_core.pquery'] + assert messages == ([ + 'Dropped 3 of 7 explicit training anchors outside ' + 'start_time/end_time or available label data.' + ] if purpose is TaskPlanPurpose.TRAINING else []) + if purpose is TaskPlanPurpose.TRAINING: + caplog.clear() + with pytest.raises(PredictiveQueryPlanError, match='No usable training'): + _temporal_timestamps( + query, '/synthetic-index', options, {}, 'ApproxDateOffsetSplit', + (Timestamp('2024-01-25').value,)) + assert not caplog.records def test_latest_temporal_timestamp_uses_source_maximum(tmp_path, monkeypatch): diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index 30708cb43..6e7fc1418 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -114,44 +114,6 @@ To use your own data, point `FileConnector` at a directory of CSV or Parquet fil `graph.snapshot()` ingests the tables into an on-disk graph index under the scratch directory, which also holds training runs, model bundles, and predictions. It defaults to `~/.cache/kumo-engine`; to put it on a large local disk, call `kumo.engine.configure(scratch_dir=...)` first. -## Absolute training bounds - -For temporal node classification, regression, ranking, and forecasting, pass -absolute bounds when generating the training table: - -```python -from kumoapi.model_plan import TrainingTableGenerationPlan - -table = pquery.generate_training_table( - snapshot, - TrainingTableGenerationPlan( - start_time="2024-01-01T00:00:00Z", - end_time="2024-07-01T00:00:00Z", - ), -) -training_result = trainer.fit(table, snapshot=snapshot) -``` - -`start_time` includes prediction anchors at that timestamp. `end_time` limits -the label observation window: a 30-day target needs an anchor at least 30 days -before the cutoff. Forecasting reserves the complete future-label horizon. -Bounds are intersected with available label data; generation fails when no -complete examples fit. Omit either bound, or use `"inferred"`, to infer it from -the data. `start_time="-INF"` also uses the available lower bound. Aware -timestamps are normalized to UTC; naive timestamps are read as UTC. - -The bounds cover all generated train, validation, and test stages. Default -splits and supported ratio splits operate within that window. Choose direct -bounds or an explicit temporal window split (`TimeRangeSplit`, -`DateOffsetSplit`, or `BacktestingSplit`) separately. Direct bounds also cannot -be combined with `train_start_offset` or `train_end_offset`. Existing task, -time-unit, and split restrictions continue to apply. Integer `start_time` and -`end_time` bounds are not supported. Use `train_start_offset` and -`train_end_offset` for relative windows measured backward from the latest label -data in the aggregation's time unit. Absolute bounds for temporal link -prediction are outside this feature's scope. Fit and saved model bundles -retain the effective generation plan. - ## Agent skills Kumo Engine ships an agent skill, `kumo-data-science`, that teaches a coding agent to build, train, and run Kumo models with this version of the SDK. Install it for your agent: diff --git a/packages/kumo-engine/test/test_absolute_training_bounds.py b/packages/kumo-engine/test/test_absolute_training_bounds.py index f4757395c..6effb4b0a 100644 --- a/packages/kumo-engine/test/test_absolute_training_bounds.py +++ b/packages/kumo-engine/test/test_absolute_training_bounds.py @@ -97,7 +97,11 @@ def test_temporal_node_bounds(temporal_graph, tmp_path, monkeypatch, task, assert table.generation_plan.end_time == END.isoformat() if task == 'forecasting': assert table.generation_plan.forecast_length == 3 - assert rows['TIMESTAMP'].max() == END - pd.Timedelta(days=6) + stage_horizon = pd.Timedelta(days=2 * 3) + # Each default forecast stage owns one complete forecast horizon. + expected = {stage: END - (3 - stage) * stage_horizon + for stage in range(3)} + assert rows.groupby('SPLIT')['TIMESTAMP'].max().to_dict() == expected if task == 'binary_classification': events = temporal_graph[2] for row in rows.itertuples(): From 1ac0eaca6cb6c5aba3f091e1ba1a05ee440b665f Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Tue, 10 Nov 2026 16:05:47 -0600 Subject: [PATCH 10/10] Preserve decoupled relative integer training bounds --- kumo/pquery/train_table_generator.py | 16 ++++-- packages/kumo-api/kumoapi/model_plan.py | 4 +- .../kumo-core/src/kumo_core/model_plan.py | 20 ++++++-- packages/kumo-core/src/kumo_core/pquery.py | 20 +++++--- packages/kumo-core/tests/test_pquery.py | 3 ++ test/config/test_model_plan.py | 50 ++++++++++++++++++- .../test_train_generator_basic.py | 28 ++++++++--- 7 files changed, 115 insertions(+), 26 deletions(-) diff --git a/kumo/pquery/train_table_generator.py b/kumo/pquery/train_table_generator.py index 0a3082478..d43edce01 100644 --- a/kumo/pquery/train_table_generator.py +++ b/kumo/pquery/train_table_generator.py @@ -8,6 +8,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union import kumo_core.types as core_types +from kumo_core.model_plan import _use_decoupled_splits from kumo_core.training.bounds import parse_time_bound from kumo_core.training.options import allow_array_targets from kumo_core.training.splits import TrainTableTime, train_table_time @@ -288,8 +289,18 @@ def __init__( or isinstance(train_end_offset, DateOffset)) self.train_end_offset = train_end_offset - self.start_time = parse_time_bound(start_time, 'start_time') - self.end_time = parse_time_bound(end_time, 'end_time') + self.use_decoupled_splits = ( + _use_decoupled_splits() + if use_decoupled_splits is None else use_decoupled_splits) + start_bound: int | str | DateOffset | None = start_time + end_bound: int | str | DateOffset | None = end_time + if self.use_decoupled_splits: + if isinstance(start_bound, int): + start_bound = self.get_date_offset(start_bound) + if isinstance(end_bound, int): + end_bound = self.get_date_offset(end_bound) + self.start_time = parse_time_bound(start_bound, 'start_time') + self.end_time = parse_time_bound(end_bound, 'end_time') self.timeframe_step = None if timeframe_step is not None: @@ -298,7 +309,6 @@ def __init__( self.lag_timesteps = lag_timesteps self.forecast_length = forecast_length self.year_over_year = year_over_year - self.use_decoupled_splits = use_decoupled_splits self.weight_col = normalize_weight_col(weight_col) self.task_type: core_types.TaskType = get_task_type( diff --git a/packages/kumo-api/kumoapi/model_plan.py b/packages/kumo-api/kumoapi/model_plan.py index 8b7dbb653..6f4e9b21b 100644 --- a/packages/kumo-api/kumoapi/model_plan.py +++ b/packages/kumo-api/kumoapi/model_plan.py @@ -625,8 +625,8 @@ class TrainingTableGenerationPlan(PlanMixin): lower bound. Bounds are intersected with available label data. Direct absolute bounds cannot be combined with train_start_offset, train_end_offset, or explicit date/offset window splits. - Integer bounds are not supported. For relative training windows, - use train_start_offset/train_end_offset in the aggregation's time unit. + For relative training windows, use train_start_offset/train_end_offset + in the aggregation's time unit. (*default:* ``"inferred"``). **Supported Task Types:** Temporal node classification, regression, ranking, and forecasting. diff --git a/packages/kumo-core/src/kumo_core/model_plan.py b/packages/kumo-core/src/kumo_core/model_plan.py index 61ccf7c65..1b3c5b98b 100644 --- a/packages/kumo-core/src/kumo_core/model_plan.py +++ b/packages/kumo-core/src/kumo_core/model_plan.py @@ -68,6 +68,7 @@ ) from kumoapi.pquery import QueryType from kumoapi.typing import Stype, TimeUnit +from pandas import DateOffset from pydantic import BaseModel, ValidationError, conlist, field_validator from pydantic.dataclasses import dataclass from pydantic_core import InitErrorDetails, PydanticCustomError @@ -955,10 +956,19 @@ def validate( f"split. Please choose one of these splits or " f"set `split` to `inferred`.")) if query_type == QueryType.TEMPORAL and task_type.is_node_pred: + start = inferred_or_missing_type_to_none(self.start_time) + end = inferred_or_missing_type_to_none(self.end_time) + # The plan has no aggregation unit. Check relative ordering here; + # query planning resolves integers using the actual aggregation unit. + if _use_decoupled_splits(): + if isinstance(start, int): + start = DateOffset(days=start) + if isinstance(end, int): + end = DateOffset(days=end) try: validate_time_bounds( - inferred_or_missing_type_to_none(self.start_time), - inferred_or_missing_type_to_none(self.end_time), + start, + end, train_start_offset=inferred_or_missing_type_to_none( self.train_start_offset), train_end_offset=inferred_or_missing_type_to_none( @@ -967,9 +977,9 @@ def validate( ) except ValueError as error: errors.append(error) - elif query_type == QueryType.TEMPORAL: - # Preserve existing link routing while rejecting public integer - # bounds consistently with temporal node plans. + elif (query_type == QueryType.TEMPORAL + and not _use_decoupled_splits()): + # Reject integer bounds in default mode, matching temporal node plans. for name, value in (('start_time', self.start_time), ('end_time', self.end_time)): if isinstance(value, int): diff --git a/packages/kumo-core/src/kumo_core/pquery.py b/packages/kumo-core/src/kumo_core/pquery.py index 92e7c6af7..9bfa75f02 100644 --- a/packages/kumo-core/src/kumo_core/pquery.py +++ b/packages/kumo-core/src/kumo_core/pquery.py @@ -49,6 +49,7 @@ ) from .model_plan import ( TrainingTableGenerationPlan, + _use_decoupled_splits, inferred_or_missing_type_to_false, inferred_or_missing_type_to_none, ) @@ -1058,17 +1059,22 @@ def date_offset(value: int) -> DateOffset: start_time: DateOffset | Timestamp | None end_time: DateOffset | Timestamp | None node_bounds = query.task_type in TaskType.get_node_pred_tasks() + start_bound = inferred_or_missing_type_to_none(options.start_time) + end_bound = inferred_or_missing_type_to_none(options.end_time) + if isinstance(start_bound, Integral): + start_bound = (date_offset(int(start_bound)) + if _use_decoupled_splits() else int(start_bound)) + if isinstance(end_bound, Integral): + end_bound = (date_offset(int(end_bound)) + if _use_decoupled_splits() else int(end_bound)) try: if not node_bounds: - start_time = parse_time_bound( - inferred_or_missing_type_to_none(options.start_time), - 'start_time') - end_time = parse_time_bound( - inferred_or_missing_type_to_none(options.end_time), 'end_time') + start_time = parse_time_bound(start_bound, 'start_time') + end_time = parse_time_bound(end_bound, 'end_time') else: start_time, end_time = validate_time_bounds( - inferred_or_missing_type_to_none(options.start_time), - inferred_or_missing_type_to_none(options.end_time), + start_bound, + end_bound, train_start_offset=train_start, train_end_offset=train_end, split_family=(split_family diff --git a/packages/kumo-core/tests/test_pquery.py b/packages/kumo-core/tests/test_pquery.py index f4140050c..34c384495 100644 --- a/packages/kumo-core/tests/test_pquery.py +++ b/packages/kumo-core/tests/test_pquery.py @@ -1876,6 +1876,8 @@ def unexpected_read(*args): ["2024-05-30", "2024-03-30"]), ("2024-06-30", {"train_start_offset": 3, "train_end_offset": 1}, (0, 1), ["2024-04-30", "2024-03-30"]), + ("2024-06-30", {"start_time": -3, "end_time": -1}, + (0, 1), ["2024-04-30", "2024-03-30"]), ("2024-06-30", {"start_time": "2024-01-31"}, (1, 2), ["2024-04-30", "2024-03-30", "2024-02-29"]), ], @@ -1884,6 +1886,7 @@ def test_month_planning_preserves_calendar_windows( monkeypatch, maximum, options, window, expected): from pandas import Timestamp + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'true') graph = normalize_graph(forecasting_graph_config()) query = parse_and_validate_query( f"PREDICT COUNT(events.*, {window[0]}, {window[1]}, months) " diff --git a/test/config/test_model_plan.py b/test/config/test_model_plan.py index 5c25031c3..2f85648f1 100644 --- a/test/config/test_model_plan.py +++ b/test/config/test_model_plan.py @@ -2346,7 +2346,23 @@ def test_model_plan_decoupled_splits(monkeypatch): plan.training_table_generation.train_start_offset == MissingType.VALUE) plan.training_table_generation.start_time = -10 - with pytest.raises(ValueError, match="Integer 'start_time' bounds"): + plan.training_table_generation.end_time = -1 + plan.validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + plan.training_table_generation.end_time = -20 + with pytest.raises(ValueError, match="start_time must be smaller"): + plan.validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + plan.training_table_generation.end_time = '2024-02-01' + with pytest.raises(ValueError, match="both be offsets"): plan.validate( task_type=TaskType.BINARY_CLASSIFICATION, query_type=QueryType.TEMPORAL, @@ -2354,6 +2370,22 @@ def test_model_plan_decoupled_splits(monkeypatch): run_mode=RunMode.FAST, ) plan.training_table_generation.start_time = InferredType.VALUE + plan.training_table_generation.end_time = InferredType.VALUE + + link_plan = ModelPlan.from_default( + task_type=TaskType.TEMPORAL_LINK_PREDICTION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + link_plan.training_table_generation.start_time = -10 + link_plan.training_table_generation.end_time = -1 + link_plan.validate( + task_type=TaskType.TEMPORAL_LINK_PREDICTION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) # Verify validation fails if train_start_offset is set plan.training_table_generation.train_start_offset = 10 @@ -2380,6 +2412,22 @@ def test_model_plan_decoupled_splits(monkeypatch): plan.training_table_generation.train_start_offset == InferredType.VALUE) assert plan.training_table_generation.start_time == MissingType.VALUE + plan.training_table_generation.start_time = -10 + with pytest.raises(ValueError, match="Integer 'start_time' bounds"): + plan.validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + with pytest.raises(ValueError, match="Integer 'start_time' bounds"): + link_plan.validate( + task_type=TaskType.TEMPORAL_LINK_PREDICTION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + # Absolute bounds are supported alongside inferred legacy defaults. plan.training_table_generation.start_time = "2024-01-01" plan.training_table_generation.end_time = "2024-02-01" diff --git a/test/pquery/train_table_generator/test_train_generator_basic.py b/test/pquery/train_table_generator/test_train_generator_basic.py index a0e4adf89..c158868cb 100644 --- a/test/pquery/train_table_generator/test_train_generator_basic.py +++ b/test/pquery/train_table_generator/test_train_generator_basic.py @@ -991,8 +991,9 @@ def test_generate_time_intervals_invalid_start_end_time(create_node_pred_graph, def test_generate_time_intervals_integer_bound_rejected(create_node_pred_graph, - tempdir): + tempdir, monkeypatch): """Integer start bounds have an actionable error before generation.""" + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'false') graph = create_node_pred_graph('test_graph', tempdir.name) query = ('PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 1) > 30 ' 'FOR EACH USERS.USER_ID') @@ -1002,6 +1003,11 @@ def test_generate_time_intervals_integer_bound_rejected(create_node_pred_graph, with pytest.raises(ValueError, match="Integer 'start_time' bounds"): TrainingTableGeneratorV2( graph, validated_query, split, start_time=-2, end_time='1990-01-04') + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'true') + with pytest.raises(ValueError, match="Integer 'start_time' bounds"): + TrainingTableGeneratorV2( + graph, validated_query, split, start_time=-2, end_time='1990-01-04', + use_decoupled_splits=False) def test_generate_time_intervals_with_legacy_params_error( @@ -1138,9 +1144,10 @@ def test_generate_timestamps_v2_with_start_end_time(create_node_pred_graph, assert task_type == TaskType.BINARY_CLASSIFICATION -def test_generate_timestamps_v2_integer_bound_rejected( - create_node_pred_graph, tempdir): - """Decoupled routing also rejects integer end bounds.""" +def test_generate_timestamps_v2_with_relative_start_end_time( + create_node_pred_graph, tempdir, monkeypatch): + """Decoupled integer bounds use the aggregation's time unit.""" + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'true') graph = create_node_pred_graph('test_graph', tempdir.name) query = ('PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 8, hours) > 30 ' 'FOR EACH USERS.USER_ID') @@ -1152,10 +1159,15 @@ def test_generate_timestamps_v2_integer_bound_rejected( anchor_start_offset=0, timeframe_size=8, ) - with pytest.raises(ValueError, match="Integer 'end_time' bounds"): - TrainingTableGeneratorV2( - graph, validated_query, split, start_time='1990-01-02', end_time=-24, - timeframe_step=8, use_decoupled_splits=True) + generator = TrainingTableGeneratorV2( + graph, validated_query, split, start_time=-48, end_time=-24, + timeframe_step=8, use_decoupled_splits=True) + assert generator.generate_timestamps_v2() == [ + pd.Timestamp('1990-01-03 16:00:00'), + pd.Timestamp('1990-01-03 08:00:00'), + pd.Timestamp('1990-01-03 00:00:00'), + ] + assert generator.task_type == TaskType.BINARY_CLASSIFICATION def test_generate_timestamps_v2_legacy_params_error(create_node_pred_graph,