diff --git a/kumo/data/generic/diskgraph_train_table.py b/kumo/data/generic/diskgraph_train_table.py index e052d6427..e3fced8ec 100644 --- a/kumo/data/generic/diskgraph_train_table.py +++ b/kumo/data/generic/diskgraph_train_table.py @@ -21,6 +21,7 @@ build_split_spec as build_native_split_spec from kumo_core.training.table import \ generate_train_table as generate_native_train_table +from kumo_core.training.timestamps import forecast_offsets from kumoapi.pquery.validated_predictive_query import ValidatedPredictiveQuery from io_utils import fsutil, logging @@ -676,10 +677,7 @@ def _ttng_forecasting_offset_kwargs( step = time_range.timeframe assert step is not None - future_offsets_secs = [ - int(((pt + i * step) - pt).total_seconds()) - for i in range(forecast_length) - ] + future_offsets_secs = list(forecast_offsets(step, forecast_length)) lag_offsets_secs = None if lag_timesteps is not None: lag_ts = get_lag_timestamps(pt, lag_timesteps, step) diff --git a/kumo/pquery/pquery_validators/pre_train_table_validator.py b/kumo/pquery/pquery_validators/pre_train_table_validator.py index 772c3a4c7..792716246 100644 --- a/kumo/pquery/pquery_validators/pre_train_table_validator.py +++ b/kumo/pquery/pquery_validators/pre_train_table_validator.py @@ -5,6 +5,7 @@ from typing import Dict, List, Optional, Union import pandas as pd +from kumo_core.training.bounds import validate_time_bounds from kumo_core.training.validation import validate_train_offsets from kumo_core.types import TaskType from kumoapi.pquery import QueryType, ValidatedPredictiveQuery @@ -774,94 +775,17 @@ def validate_limits( return validation_response - def validate_train_time_compatibility(self) -> ValidationResponse: - r"""Validates that start_time/end_time and train offsets are not used - together. - - Note: This validation can be removed once we fully migrate from using - train_start_offset/train_end_offset to start_time/end_time parameters. - - Returns: - ValidationResponse: Response containing any validation errors. - """ - validation_response = ValidationResponse() - - if ((self.start_time is not None or self.end_time is not None) - and (self.train_start_offset is not None - or self.train_end_offset is not None)): - validation_response.errors.append( - ValidationError( - title="Incompatible time parameters", - message=("Cannot specify both start_time/end_time and " - "train_start_offset/train_end_offset. Use either " - "absolute/relative timestamps or train offset " - "parameters, but not both."))) - - return validation_response - - def validate_train_time(self) -> ValidationResponse: - r"""Validates start_time and end_time parameters. - - Checks: - 1. Both must be offsets or both timestamps - 2. For timestamps: start_time must be smaller than end_time - 3. For offsets: both must be <= 0 and start_time must be < end_time - - Returns: - ValidationResponse: Response containing any validation errors. - """ + def validate_time_bounds(self) -> ValidationResponse: + """Adapt the shared time-bound checks to service validation errors.""" validation_response = ValidationResponse() - - if self.start_time is None or self.end_time is None: - return validation_response - - # Check if both are same type (offset or timestamp) - is_start_offset = isinstance(self.start_time, pd.DateOffset) - is_end_offset = isinstance(self.end_time, pd.DateOffset) - - if is_start_offset != is_end_offset: - validation_response.errors.append( - ValidationError( - title="Inconsistent time parameter types", - message=("start_time and end_time must both be offsets " - "or both be timestamps, but got mixed types."))) - return validation_response - - # Validate timestamps - if not is_start_offset: - assert isinstance(self.start_time, pd.Timestamp) - assert isinstance(self.end_time, pd.Timestamp) - if self.start_time >= self.end_time: - validation_response.errors.append( - ValidationError( - title="Invalid timestamp range", message=( - f"start_time ({self.start_time}) must be smaller " - f"than end_time ({self.end_time})"))) - # Validate offsets - else: - assert isinstance(self.start_time, pd.DateOffset) - assert isinstance(self.end_time, pd.DateOffset) - now = pd.Timestamp.now() - start_ts = now + self.start_time - end_ts = now + self.end_time - - if start_ts <= now and end_ts <= now: - if start_ts >= end_ts: - validation_response.errors.append( - ValidationError( - title="Invalid offset range", message=( - f"For relative offsets, start_time must be " - f"more negative than end_time, but got " - f"start_time={self.start_time}, " - f"end_time={self.end_time}"))) - else: - validation_response.errors.append( - ValidationError( - title="Invalid offset values", message=( - f"Relative offsets must be <= 0 (i.e. in the " - f"past), but got start_time={self.start_time}, " - f"end_time={self.end_time}"))) - + try: + validate_time_bounds( + self.start_time, self.end_time, + train_start_offset=self.train_start_offset, + train_end_offset=self.train_end_offset) + except ValueError as error: + validation_response.errors.append(ValidationError( + title='Invalid training time bounds', message=str(error))) return validation_response diff --git a/kumo/pquery/train_table_generator.py b/kumo/pquery/train_table_generator.py index 8d7c8d309..d43edce01 100644 --- a/kumo/pquery/train_table_generator.py +++ b/kumo/pquery/train_table_generator.py @@ -8,11 +8,14 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union import kumo_core.types as core_types +from kumo_core.model_plan import _use_decoupled_splits +from kumo_core.training.bounds import parse_time_bound from kumo_core.training.options import allow_array_targets from kumo_core.training.splits import TrainTableTime, train_table_time from kumo_core.training.timestamps import ( generate_decoupled_timestamps, generate_legacy_timestamps, + resolve_training_window, ) from kumo_core.training.validation import GenerationValidationError from kumo_core.types.conversions import DataFrame @@ -286,26 +289,18 @@ def __init__( or isinstance(train_end_offset, DateOffset)) self.train_end_offset = train_end_offset - # Handle new time parameters - self.start_time = None - if start_time is not None: - if start_time == '-INF': - pass # Keep start_time as None for -INF case - elif isinstance(start_time, int): - self.start_time = self.get_date_offset(int(start_time)) - else: - assert isinstance(start_time, str) - self.start_time = Timestamp(start_time) - assert isinstance(self.start_time, Timestamp) - - self.end_time = None - if end_time is not None: - if isinstance(end_time, int): - self.end_time = self.get_date_offset(int(end_time)) - else: - assert isinstance(end_time, str) - self.end_time = Timestamp(end_time) - assert isinstance(self.end_time, Timestamp) + self.use_decoupled_splits = ( + _use_decoupled_splits() + if use_decoupled_splits is None else use_decoupled_splits) + start_bound: int | str | DateOffset | None = start_time + end_bound: int | str | DateOffset | None = end_time + if self.use_decoupled_splits: + if isinstance(start_bound, int): + start_bound = self.get_date_offset(start_bound) + if isinstance(end_bound, int): + end_bound = self.get_date_offset(end_bound) + self.start_time = parse_time_bound(start_bound, 'start_time') + self.end_time = parse_time_bound(end_bound, 'end_time') self.timeframe_step = None if timeframe_step is not None: @@ -314,7 +309,6 @@ def __init__( self.lag_timesteps = lag_timesteps self.forecast_length = forecast_length self.year_over_year = year_over_year - self.use_decoupled_splits = use_decoupled_splits self.weight_col = normalize_weight_col(weight_col) self.task_type: core_types.TaskType = get_task_type( @@ -425,6 +419,12 @@ def generate_timestamps(self) -> List[Timestamp]: values = self._timestamp_inputs() (original_start, original_end, filter_buffer, offset_range, timeframe, step) = values + window = None + if self.task_type.is_node_pred: + window = resolve_training_window( + self.start_time, self.end_time, original_start, original_end) + start_time = self.start_time if window is None else window.start + end_time = self.end_time if window is None else window.end logger.info( f"For raw data, the min_date is {original_start} and the max_date " f"is {original_end}.") @@ -463,13 +463,16 @@ def fix_boundaries() -> None: split=descriptor, train_start_offset=self.train_start_offset, train_end_offset=self.train_end_offset, - start_time=self.start_time, - end_time=self.end_time, + start_time=start_time, + end_time=end_time, drop_earliest=self.drop_earliest_timeframe, on_fixed_boundaries=fix_boundaries, ) if short_window: - return [] + return [] if window is None else window.filter( + timestamps, offset_range.end_date_offset) + if window is not None: + timestamps = window.filter(timestamps, offset_range.end_date_offset) logger.info(f'Generated {len(timestamps)} timestamps:') for i, timestamp in enumerate(timestamps): logger.info(f'Timestamp {i:03d}: {timestamp}') @@ -489,6 +492,10 @@ def generate_timestamps_v2(self) -> List[Timestamp]: values = self._timestamp_inputs() (original_start, original_end, filter_buffer, offset_range, timeframe, step) = values + window = None + if self.task_type.is_node_pred: + window = resolve_training_window( + self.start_time, self.end_time, original_start, original_end) timestamps = generate_decoupled_timestamps( original_start=original_start, original_end=original_end, @@ -496,10 +503,12 @@ def generate_timestamps_v2(self) -> List[Timestamp]: target_timeframe=timeframe, target_start_offset=offset_range.start_date_offset, step=step, - start_time=self.start_time, - end_time=self.end_time, + start_time=self.start_time if window is None else window.start, + end_time=self.end_time if window is None else window.end, drop_earliest=self.drop_earliest_timeframe, ) + if window is not None: + timestamps = window.filter(timestamps, offset_range.end_date_offset) logger.info(f'Generated {len(timestamps)} timestamps:') for i, timestamp in enumerate(timestamps): logger.info(f'Timestamp {i:03d}: {timestamp}') @@ -585,13 +594,8 @@ def validate_pre_train_table_generate( validation_response.merge( self.pre_train_table_validator.validate_temporal_splits()) - # Add new validation for time parameter compatibility - validation_response.merge(self.pre_train_table_validator. - validate_train_time_compatibility()) - - # Add new validation for time parameter values validation_response.merge( - self.pre_train_table_validator.validate_train_time()) + self.pre_train_table_validator.validate_time_bounds()) validation_response.merge( self.pre_train_table_validator.validate_train_offsets( diff --git a/packages/kumo-api/kumoapi/model_plan.py b/packages/kumo-api/kumoapi/model_plan.py index 95c8c070d..6f4e9b21b 100644 --- a/packages/kumo-api/kumoapi/model_plan.py +++ b/packages/kumo-api/kumoapi/model_plan.py @@ -608,30 +608,39 @@ class TrainingTableGenerationPlan(PlanMixin): :ivar train_start_offset: (``int`` | ``"inferred``") Defines the numerical offset from the most recent entry to use to generate training data labels. Unless a custom time unit is specified in the aggregation, this - value is in days (*default:* ``"inferred"``). Eventually we would - like to migrate this parameter to start_time. + value is in days (*default:* ``"inferred"``). **Supported Task Types:** Temporal :ivar train_end_offset: (``int`` | ``"inferred"``) Defines the numerical offset from the most recent entry to not use to generate training data labels. Unless a custom time unit is specified in the aggregation, this - value is in days (*default:* ``"inferred"``). Eventually we would - like to migrate this parameter to end_time. + value is in days (*default:* ``"inferred"``). **Supported Task Types:** Temporal - :ivar start_time: (``int`` | ``"inferred"`` | ``str``) Defines the absolute - start time to use for generating training data. Can be specified - either as a pandas Timestamp compatible string (e.g. '2024-04-01') or - as a non-positive integer offset in days from the current time - (e.g. -30 for 30 days ago) (*default:* ``"inferred"``). - **Supported Task Types:** Temporal - - :ivar end_time: (``int`` | ``"inferred"`` | ``str``) Defines the absolute - end time to use for generating training data. Can be specified either - as a pandas Timestamp compatible string (e.g. '2024-04-01') or as a - non-positive integer offset in days from the current time - (e.g. -30 for 30 days ago) (*default:* ``"inferred"``). - **Supported Task Types:** Temporal + :ivar start_time: (``"inferred"`` | ``str``) Inclusive lower + bound on generated prediction anchors. Temporal node tasks accept + pandas Timestamp compatible strings (e.g. '2024-04-01') with default + splits. Timezone-aware strings are normalized to UTC; naive strings + are read as UTC. ``"inferred"`` or ``"-INF"`` uses the available-data + lower bound. Bounds are intersected with available label data. + Direct absolute bounds cannot be combined with train_start_offset, + train_end_offset, or explicit date/offset window splits. + For relative training windows, use train_start_offset/train_end_offset + in the aggregation's time unit. + (*default:* ``"inferred"``). + **Supported Task Types:** Temporal node classification, regression, + ranking, and forecasting. + + :ivar end_time: (``"inferred"`` | ``str``) Label observation + cutoff, rather than the latest prediction anchor. Each generated + example must fit its complete target horizon by this cutoff, + including every future label for forecasting. Accepts the same + timestamp strings as start_time; + ``"inferred"`` uses available label data. Either absolute bound can + be omitted. Empty intersections or windows too short for a complete + example fail generation. (*default:* ``"inferred"``). + **Supported Task Types:** Temporal node classification, regression, + ranking, and forecasting. :ivar timeframe_step: (``int`` | ``"inferred"``) Defines the step size of generating time intervals for training table generation diff --git a/packages/kumo-core/src/kumo_core/model_plan.py b/packages/kumo-core/src/kumo_core/model_plan.py index b2eddde77..1b3c5b98b 100644 --- a/packages/kumo-core/src/kumo_core/model_plan.py +++ b/packages/kumo-core/src/kumo_core/model_plan.py @@ -23,6 +23,7 @@ import kumoapi.encoder as encoder_api import kumoapi.model_plan as api from kumo_core.errors import TrainingValidationError as ValidationException +from kumo_core.training.bounds import parse_time_bound, validate_time_bounds from kumo_core.types import TaskType from kumoapi.common import ValidationWarning from kumoapi.encoder import ( @@ -67,6 +68,7 @@ ) from kumoapi.pquery import QueryType from kumoapi.typing import Stype, TimeUnit +from pandas import DateOffset from pydantic import BaseModel, ValidationError, conlist, field_validator from pydantic.dataclasses import dataclass from pydantic_core import InitErrorDetails, PydanticCustomError @@ -795,11 +797,9 @@ def is_valid_option( has_train_table_weight_col): return False - # Check split-related options based on decoupled splits setting - # When using decoupled splits, only allow start/end time - # When using legacy splits, only allow train start/end offset + # Absolute node bounds are available independently of split routing. if name in ['start_time', 'end_time']: - return _use_decoupled_splits() + return _use_decoupled_splits() or task_type.is_node_pred if name in ['train_start_offset', 'train_end_offset']: return not _use_decoupled_splits() @@ -838,6 +838,7 @@ def validate( "model plan."))) errors.extend(unsupported_entity_candidate_errors(self)) + split = None if self.split != InferredType.VALUE: try: split = self.get_split(self.split) @@ -954,6 +955,38 @@ def validate( f"Forecasting requires a {acceptable_splits} " f"split. Please choose one of these splits or " f"set `split` to `inferred`.")) + if query_type == QueryType.TEMPORAL and task_type.is_node_pred: + start = inferred_or_missing_type_to_none(self.start_time) + end = inferred_or_missing_type_to_none(self.end_time) + # The plan has no aggregation unit. Check relative ordering here; + # query planning resolves integers using the actual aggregation unit. + if _use_decoupled_splits(): + if isinstance(start, int): + start = DateOffset(days=start) + if isinstance(end, int): + end = DateOffset(days=end) + try: + validate_time_bounds( + start, + end, + train_start_offset=inferred_or_missing_type_to_none( + self.train_start_offset), + train_end_offset=inferred_or_missing_type_to_none( + self.train_end_offset), + split_family=None if split is None else split.name, + ) + except ValueError as error: + errors.append(error) + elif (query_type == QueryType.TEMPORAL + and not _use_decoupled_splits()): + # Reject integer bounds in default mode, matching temporal node plans. + for name, value in (('start_time', self.start_time), + ('end_time', self.end_time)): + if isinstance(value, int): + try: + parse_time_bound(value, name) + except ValueError as error: + errors.append(error) return errors, warnings @classmethod diff --git a/packages/kumo-core/src/kumo_core/pquery.py b/packages/kumo-core/src/kumo_core/pquery.py index 60517d2f8..9bfa75f02 100644 --- a/packages/kumo-core/src/kumo_core/pquery.py +++ b/packages/kumo-core/src/kumo_core/pquery.py @@ -13,6 +13,7 @@ import hashlib import json +import logging import os import re from dataclasses import dataclass @@ -48,6 +49,7 @@ ) from .model_plan import ( TrainingTableGenerationPlan, + _use_decoupled_splits, inferred_or_missing_type_to_false, inferred_or_missing_type_to_none, ) @@ -59,6 +61,7 @@ PADDING_SIZE, YOY_COL, ) +from .training.bounds import parse_time_bound, validate_time_bounds from .training.splits import ( SplitArguments, TrainTableTime, @@ -73,16 +76,26 @@ ) from .training.timestamps import ( default_target_time_range, + forecast_offsets, generate_legacy_timestamps, + resolve_training_window, ) from .training.validation import validate_train_offsets from .types import TaskType +logger = logging.getLogger(__name__) + class PredictiveQueryPlanError(ValueError): """A notebook graph/query cannot be converted to a native task plan.""" +class TaskPlanPurpose(str, Enum): + TRAINING = 'training' + PREDICTION = 'prediction' + LABELS = 'labels' + + _NATIVE_TRAINING_TASK_TYPES = frozenset({ "node_prediction", "binary_classification", @@ -755,6 +768,12 @@ def resolve_query_forecast( ) +def _forecast_offsets(query: ValidatedNativeQuery) -> tuple[int, ...]: + """The exact future shifts passed to the native label compiler.""" + timeframe = query.query.get_combined_date_offset_range().timeframe + return forecast_offsets(timeframe, query.forecast_length) + + def _forecast_options( query: ValidatedNativeQuery, options: TrainingTableGenerationPlan, @@ -773,8 +792,7 @@ def _forecast_options( step_seconds = int(timeframe.total_seconds()) lag_offsets = tuple(-step_seconds * index for index in range(int(lag_timesteps), 0, -1)) - future_offsets = tuple(step_seconds * index - for index in range(query.forecast_length)) + future_offsets = _forecast_offsets(query) compiler_options = { "future_offsets_secs": future_offsets, "lag_offsets_secs": lag_offsets, @@ -1017,7 +1035,12 @@ def _temporal_timestamps( split: Mapping[str, Any], split_family: str, explicit: tuple[int, ...], + *, + purpose: TaskPlanPurpose = TaskPlanPurpose.TRAINING, ) -> tuple[tuple[int, ...], bool]: + if purpose is not TaskPlanPurpose.TRAINING: + return (tuple(_timestamp_ns(value) for value in explicit), + 'flexible_boundaries' in split) offset = query.query.get_combined_date_offset_range() def date_offset(value: int) -> DateOffset: @@ -1032,18 +1055,66 @@ def date_offset(value: int) -> DateOffset: ) if error is not None: raise PredictiveQueryPlanError(f"{error.title}. {error.message}") - if explicit: + + start_time: DateOffset | Timestamp | None + end_time: DateOffset | Timestamp | None + node_bounds = query.task_type in TaskType.get_node_pred_tasks() + start_bound = inferred_or_missing_type_to_none(options.start_time) + end_bound = inferred_or_missing_type_to_none(options.end_time) + if isinstance(start_bound, Integral): + start_bound = (date_offset(int(start_bound)) + if _use_decoupled_splits() else int(start_bound)) + if isinstance(end_bound, Integral): + end_bound = (date_offset(int(end_bound)) + if _use_decoupled_splits() else int(end_bound)) + try: + if not node_bounds: + start_time = parse_time_bound(start_bound, 'start_time') + end_time = parse_time_bound(end_bound, 'end_time') + else: + start_time, end_time = validate_time_bounds( + start_bound, + end_bound, + train_start_offset=train_start, + train_end_offset=train_end, + split_family=(split_family + if inferred_or_missing_type_to_none( + options.split) is not None else None), + ) + except ValueError as error: + raise PredictiveQueryPlanError(str(error)) from error + absolute_bounds = (isinstance(start_time, Timestamp) + or isinstance(end_time, Timestamp)) and node_bounds + if explicit and not absolute_bounds: return (tuple(_timestamp_ns(value) for value in explicit), "flexible_boundaries" in split) minimum, maximum, filter_buffer = _temporal_target_time_range( query, index_path) + try: + window = (resolve_training_window(start_time, end_time, minimum, maximum) + if absolute_bounds else None) + except ValueError as error: + raise PredictiveQueryPlanError(str(error)) from error + if window is not None: + start_time, end_time = window.start, window.end + + def bounded(values: Sequence[int]) -> tuple[int, ...]: + if window is None: + return tuple(values) + try: + return tuple(window.filter(values, offset.end_date_offset)) + except ValueError as error: + raise PredictiveQueryPlanError(str(error)) from error - def relative(value: Any) -> DateOffset | Timestamp | None: - if value is None or value == "-INF": - return None - if isinstance(value, Integral): - return date_offset(int(value)) - return Timestamp(Timestamp(value).value) + if explicit: + timestamps = bounded(tuple(_timestamp_ns(value) for value in explicit)) + dropped = len(explicit) - len(timestamps) + if dropped: + logger.warning( + 'Dropped %d of %d explicit training anchors outside ' + 'start_time/end_time or available label data.', + dropped, len(explicit)) + return timestamps, "flexible_boundaries" in split step_count = int( inferred_or_missing_type_to_none(options.timeframe_step) @@ -1072,11 +1143,15 @@ def relative(value: Any) -> DateOffset | Timestamp | None: int(train_start))), train_end_offset=(None if train_end is None else date_offset( int(train_end))), - start_time=relative( - inferred_or_missing_type_to_none(options.start_time)), - end_time=relative(inferred_or_missing_type_to_none(options.end_time)), + start_time=start_time, + end_time=end_time, drop_earliest=True, ) + if absolute_bounds: + # Inferred offset splits may extend their first range to fill a stage. + # Absolute label bounds remain authoritative over that adjustment. + return (bounded(tuple(timestamp.value for timestamp in timestamps)), + fixed_boundaries) return tuple(timestamp.value for timestamp in timestamps), fixed_boundaries @@ -1390,6 +1465,7 @@ def build_native_pquery_task_plan( graph_index_id: str | None = None, timestamps: tuple[int, ...] = (), supplied_rows: bool = False, + purpose: TaskPlanPurpose = TaskPlanPurpose.TRAINING, ) -> NativePqueryTaskPlan: """Resolve task inputs for generated or already-supplied rows. @@ -1423,6 +1499,7 @@ def build_native_pquery_task_plan( split, split_family, timestamps, + purpose=purpose, ) if fixed_boundaries: split = {**split, "flexible_boundaries": False} diff --git a/packages/kumo-core/src/kumo_core/training/bounds.py b/packages/kumo-core/src/kumo_core/training/bounds.py new file mode 100644 index 000000000..47e8d78a6 --- /dev/null +++ b/packages/kumo-core/src/kumo_core/training/bounds.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Validation and UTC normalization for temporal label-window bounds.""" + +from pandas import DateOffset, Timestamp + + +def parse_time_bound(value: int | str | DateOffset | Timestamp | None, + name: str) -> DateOffset | Timestamp | None: + """Normalize absolute times to UTC and retain parsed internal offsets.""" + if value is None or (name == 'start_time' and value == '-INF'): + return None + if isinstance(value, int): + raise ValueError( + f"Integer '{name}' bounds are not supported. Use timestamp strings " + "or 'inferred'; use train_start_offset/train_end_offset for " + "relative training bounds.") + if isinstance(value, DateOffset): + if Timestamp('1970-01-01') + value > Timestamp('1970-01-01'): + raise ValueError(f"'{name}' must be a non-positive relative offset") + return value + try: + timestamp = Timestamp(value) + if not isinstance(timestamp, Timestamp): + raise ValueError('timestamp is NaT') + if timestamp.tzinfo is not None: + timestamp = timestamp.tz_convert('UTC').tz_localize(None) + # Generation stores timestamps as signed nanoseconds. + _ = timestamp.value + except (ValueError, TypeError, OverflowError) as error: + raise ValueError(f"'{name}' must be a valid timestamp: {value!r}") from error + return timestamp + + +def validate_time_bounds( + start: int | str | DateOffset | Timestamp | None, + end: int | str | DateOffset | Timestamp | None, + *, + train_start_offset: int | DateOffset | None = None, + train_end_offset: int | DateOffset | None = None, + split_family: str | None = None, +) -> tuple[DateOffset | Timestamp | None, DateOffset | Timestamp | None]: + """Parse start/end bounds and reject invalid combinations.""" + if ((start is not None or end is not None) + and (train_start_offset is not None + or train_end_offset is not None)): + raise ValueError('Cannot specify both start_time/end_time and ' + 'train_start_offset/train_end_offset.') + parsed_start = parse_time_bound(start, 'start_time') + parsed_end = parse_time_bound(end, 'end_time') + if parsed_start is not None and parsed_end is not None: + if (isinstance(parsed_start, Timestamp) + != isinstance(parsed_end, Timestamp)): + raise ValueError('start_time and end_time must both be offsets ' + 'or both be timestamps.') + if ((isinstance(parsed_start, Timestamp) + and isinstance(parsed_end, Timestamp) + and parsed_start >= parsed_end) + or (isinstance(parsed_start, DateOffset) + and isinstance(parsed_end, DateOffset) + and Timestamp('1970-01-01') + parsed_start + >= Timestamp('1970-01-01') + parsed_end)): + raise ValueError('start_time must be smaller than end_time.') + if (isinstance(parsed_start, Timestamp) + or isinstance(parsed_end, Timestamp)) and split_family in { + 'BacktestingSplit', 'DateOffsetSplit', 'TimeRangeSplit', + 'TimeSplit', + }: + raise ValueError('Absolute start_time/end_time cannot be combined ' + f'with {split_family}; use a ratio split or inferred.') + return parsed_start, parsed_end diff --git a/packages/kumo-core/src/kumo_core/training/timestamps.py b/packages/kumo-core/src/kumo_core/training/timestamps.py index 981ca78bf..266e5de7d 100644 --- a/packages/kumo-core/src/kumo_core/training/timestamps.py +++ b/packages/kumo-core/src/kumo_core/training/timestamps.py @@ -3,10 +3,69 @@ # SPDX-License-Identifier: Apache-2.0 """Shared temporal anchor enumeration for hosted and portable generation.""" -from typing import Any, Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Callable, Mapping, Sequence, TypeVar -from kumoapi.pquery.AST import DateOffsetRange +from kumoapi.pquery.AST.date_offset_range import DateOffsetRange from pandas import DateOffset, Timedelta, Timestamp +from pandas.errors import OutOfBoundsDatetime + +Anchor = TypeVar('Anchor', int, Timestamp) +_EMPTY_WINDOW = ('No usable training examples within start_time/end_time and ' + 'the available label data. Widen the bounds or provide more ' + 'data for the target horizon.') + + +@dataclass(frozen=True) +class TrainingWindow: + """Available base label windows; native routing owns forecast tails.""" + + start: DateOffset | Timestamp | None + end: DateOffset | Timestamp | None + lower: Timestamp + cutoff: Timestamp + + def filter(self, timestamps: Sequence[Anchor], + target_end: DateOffset) -> list[Anchor]: + result = [ + value for value in timestamps if self.contains( + value.value if isinstance(value, Timestamp) else value, + target_end) + ] + if not result: + raise ValueError(_EMPTY_WINDOW) + return result + + def contains(self, value: int, target_end: DateOffset) -> bool: + if value < self.lower.value or value > self.cutoff.value: + return False + try: + return Timestamp(value) + target_end <= self.cutoff + except (OutOfBoundsDatetime, OverflowError): + return False + + +def resolve_training_window( + start: DateOffset | Timestamp | None, + end: DateOffset | Timestamp | None, + minimum: Timestamp, + maximum: Timestamp, +) -> TrainingWindow | None: + if not isinstance(start, Timestamp) and not isinstance(end, Timestamp): + return None + start = max(start, minimum) if isinstance(start, Timestamp) else start + end = min(end, maximum) if isinstance(end, Timestamp) else end + lower = start if isinstance(start, Timestamp) else minimum + cutoff = end if isinstance(end, Timestamp) else maximum + if lower >= cutoff: + raise ValueError(_EMPTY_WINDOW) + return TrainingWindow(start, end, lower, cutoff) + + +def forecast_offsets(timeframe: Timedelta, length: int) -> tuple[int, ...]: + """Future shifts shared by hosted and portable native compilers.""" + step_seconds = int(timeframe.total_seconds()) + return tuple(step_seconds * index for index in range(length)) def end_anchored_timestamps( diff --git a/packages/kumo-core/src/kumo_core/training/workflow.py b/packages/kumo-core/src/kumo_core/training/workflow.py index 7f611ff69..f48124838 100644 --- a/packages/kumo-core/src/kumo_core/training/workflow.py +++ b/packages/kumo-core/src/kumo_core/training/workflow.py @@ -43,6 +43,7 @@ NativeGraph, NativePqueryTaskPlan, PlanGraph, + TaskPlanPurpose, ValidatedNativeQuery, build_native_pquery_task_plan, normalize_graph, @@ -350,7 +351,10 @@ def from_stage_output( def build_training_task_plan( - request: TrainingWorkflowRequest, ) -> NativePqueryTaskPlan: + request: TrainingWorkflowRequest, + *, + purpose: TaskPlanPurpose = TaskPlanPurpose.TRAINING, +) -> NativePqueryTaskPlan: """Build the exact train-table and task-dataset plan for a request.""" return build_native_pquery_task_plan( @@ -363,6 +367,7 @@ def build_training_task_plan( artifact_dir=request.artifact_dir, graph_index_id=request.prepared_graph.index_id, timestamps=request.timestamps, + purpose=purpose, ) diff --git a/packages/kumo-core/tests/test_pquery.py b/packages/kumo-core/tests/test_pquery.py index f8d26fc14..34c384495 100644 --- a/packages/kumo-core/tests/test_pquery.py +++ b/packages/kumo-core/tests/test_pquery.py @@ -12,6 +12,7 @@ from kumo_core.model_plan import TrainingTableGenerationPlan from kumo_core.pquery import ( PredictiveQueryPlanError, + TaskPlanPurpose, _column_stats, _default_split, _split_options, @@ -687,10 +688,6 @@ def test_plan_rejects_explicit_timestamps_on_a_static_query(tmp_path): "train_start_offset": 20, "train_end_offset": 10 }, 20, 29), - ({ - "start_time": -20, - "end_time": -10 - }, 20, 29), ({ "start_time": "2024-01-11", "end_time": "2024-01-21" @@ -699,15 +696,6 @@ def test_plan_rejects_explicit_timestamps_on_a_static_query(tmp_path): "start_time": "2024-01-11T00:00:00Z", "end_time": "2024-01-21T00:00:00Z" }, 10, 19), - ({ - "start_time": "-INF", - "end_time": -10 - }, 1, 29), - ({ - "train_end_offset": 10, - "start_time": "2024-01-11", - "end_time": "2024-01-21" - }, 0, 29), ]) def test_generation_plan_time_bounds(tmp_path, monkeypatch, options, first_day, last_day, unit): @@ -742,6 +730,85 @@ def test_generation_plan_time_bounds(tmp_path, monkeypatch, options, first_day, start + day * index for index in range(last_day, first_day - 1, -1)) +@pytest.mark.parametrize('start,end,step,expected', [ + ('2023-12-01', '2024-02-01', 3, [17, 14, 11, 8, 5, 2]), + ('2024-01-05', None, 3, [17, 14, 11, 8, 5]), + (None, '2024-01-15', 3, [12, 9, 6]), + ('2024-01-05', '2024-01-16', 4, [13, 9, 5]), + ('2024-01-05T01:00:00+01:00', '2024-01-15T01:00:00+01:00', + 3, [12, 9, 6]), + ('2024-02-01', '2024-02-10', 3, None), +]) +def test_absolute_bounds_intersect_available_labels( + monkeypatch, start, end, step, expected): + from pandas import Timestamp + + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + 'PREDICT COUNT(events.*, 1, 3, days) FOR EACH articles.aid', graph) + monkeypatch.setattr( + 'kumo_core.pquery._temporal_target_time_range', + lambda *_args: (Timestamp('2024-01-01'), Timestamp('2024-01-20'), None)) + options = TrainingTableGenerationPlan(timeframe_step=step) + if start is not None: + options.start_time = start + if end is not None: + options.end_time = end + if expected is None: + with pytest.raises(PredictiveQueryPlanError, match='No usable training'): + _temporal_timestamps( + query, '/synthetic-index', options, {}, + 'ApproxDateOffsetSplit', ()) + else: + timestamps, _ = _temporal_timestamps( + query, '/synthetic-index', options, {}, 'ApproxDateOffsetSplit', ()) + assert timestamps == tuple( + Timestamp(f'2024-01-{day:02}').value for day in expected) + + +@pytest.mark.parametrize('purpose', list(TaskPlanPurpose)) +def test_explicit_anchors_follow_planning_purpose(monkeypatch, caplog, purpose): + from pandas import Timestamp + + graph = normalize_graph(forecasting_graph_config()) + query = parse_and_validate_query( + 'PREDICT COUNT(events.*, 0, 1, days) > 0 FOR EACH articles.aid', graph) + + def available_bounds(query, index_path): + assert purpose is TaskPlanPurpose.TRAINING + return Timestamp('2024-01-01'), Timestamp('2024-01-20'), None + + monkeypatch.setattr('kumo_core.pquery._temporal_target_time_range', + available_bounds) + off_grid = Timestamp('2024-01-06T12:34:56.123456789').value + explicit = (off_grid,) + tuple( + Timestamp(f'2024-01-{day:02}').value for day in [19, 4, 5, 20, 25, 5]) + options = TrainingTableGenerationPlan( + start_time='2024-01-05', end_time='2024-02-01', timeframe_step=99) + timestamps, fixed = _temporal_timestamps( + query, '/synthetic-index', options, + {'flexible_boundaries': False}, 'ApproxDateOffsetSplit', explicit, + purpose=purpose) + expected = ((off_grid,) + tuple( + Timestamp(f'2024-01-{day:02}').value for day in [19, 5, 5]) + if purpose is TaskPlanPurpose.TRAINING else explicit) + assert timestamps == expected + assert fixed + messages = [record.getMessage() for record in caplog.records + if record.name == 'kumo_core.pquery'] + assert messages == ([ + 'Dropped 3 of 7 explicit training anchors outside ' + 'start_time/end_time or available label data.' + ] if purpose is TaskPlanPurpose.TRAINING else []) + if purpose is TaskPlanPurpose.TRAINING: + caplog.clear() + with pytest.raises(PredictiveQueryPlanError, match='No usable training'): + _temporal_timestamps( + query, '/synthetic-index', options, {}, 'ApproxDateOffsetSplit', + (Timestamp('2024-01-25').value,)) + assert not caplog.records + + def test_latest_temporal_timestamp_uses_source_maximum(tmp_path, monkeypatch): minimum = 1_704_067_200_000_000_000 maximum = 1_711_843_200_000_000_000 @@ -972,6 +1039,8 @@ def test_date_offset_split_uses_bounded_range_for_each_phase( graph_index_path=str(tmp_path / "index"), scratch_dir=str(tmp_path), artifact_dir=str(tmp_path)) + # Candidate steps restart at each split; native generation reserves the + # complete future-label horizon. assert plan.train_table.timestamps == tuple( (origin + index * day).value for index in (9, 7, 6, 4, 3, 1)) @@ -1805,10 +1874,10 @@ def unexpected_read(*args): ("2023-04-29", {}, (0, 1), ["2023-03-29", "2023-02-28"]), ("2024-06-30", {"timeframe_step": 2}, (0, 1), ["2024-05-30", "2024-03-30"]), - ("2024-06-30", {"start_time": -3, "end_time": -1}, (0, 1), - ["2024-04-30", "2024-03-30"]), ("2024-06-30", {"train_start_offset": 3, "train_end_offset": 1}, (0, 1), ["2024-04-30", "2024-03-30"]), + ("2024-06-30", {"start_time": -3, "end_time": -1}, + (0, 1), ["2024-04-30", "2024-03-30"]), ("2024-06-30", {"start_time": "2024-01-31"}, (1, 2), ["2024-04-30", "2024-03-30", "2024-02-29"]), ], @@ -1817,6 +1886,7 @@ def test_month_planning_preserves_calendar_windows( monkeypatch, maximum, options, window, expected): from pandas import Timestamp + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'true') graph = normalize_graph(forecasting_graph_config()) query = parse_and_validate_query( f"PREDICT COUNT(events.*, {window[0]}, {window[1]}, months) " diff --git a/packages/kumo-engine/kumo/engine/pquery/prediction_table.py b/packages/kumo-engine/kumo/engine/pquery/prediction_table.py index db6147642..80b348f57 100644 --- a/packages/kumo-engine/kumo/engine/pquery/prediction_table.py +++ b/packages/kumo-engine/kumo/engine/pquery/prediction_table.py @@ -13,6 +13,7 @@ from kumo_core.model_plan import \ TrainingTableGenerationPlan as CoreGenerationPlan from kumo_core.pquery import ( + TaskPlanPurpose, ValidatedNativeQuery, build_native_pquery_task_plan, latest_temporal_timestamp, @@ -188,10 +189,8 @@ def _build_prediction_table_request( if generation_plan is not None: from kumo_core.io.json import from_json_dict from kumoapi.json_serde import to_json_dict - options = from_json_dict( - to_json_dict(generation_plan), - CoreGenerationPlan, - ) + + options = from_json_dict(to_json_dict(generation_plan), CoreGenerationPlan) parsed = resolve_query_forecast(parsed, native_graph, options) with no_encoder_inference_warnings(): @@ -206,6 +205,7 @@ def _build_prediction_table_request( graph_index_id=snapshot.id, timestamps=timestamps, supplied_rows=supplied_rows, + purpose=TaskPlanPurpose.PREDICTION, ) forecast_timeframe_step = None if parsed.task_type == 'forecasting' and generation_plan is not None: diff --git a/packages/kumo-engine/kumo/engine/pquery/predictive_query.py b/packages/kumo-engine/kumo/engine/pquery/predictive_query.py index 34df0f44a..bc2d8fb3e 100644 --- a/packages/kumo-engine/kumo/engine/pquery/predictive_query.py +++ b/packages/kumo-engine/kumo/engine/pquery/predictive_query.py @@ -59,6 +59,7 @@ from kumo_core.model_plan_conversion import public_model_plan_from_internal from kumo_core.pquery import ( NativeGraph, + TaskPlanPurpose, ValidatedNativeQuery, parse_and_validate_query, query_is_temporal, @@ -323,7 +324,8 @@ def generate_training_table( def _generate(self, snapshot: GraphSnapshot, options: CoreGenerationPlan, run_dir: Path, *, timestamps: tuple[int, ...] = (), - validate_targets: bool = True) -> tuple: + validate_targets: bool = True, + purpose: TaskPlanPurpose = TaskPlanPurpose.TRAINING) -> tuple: workflow = TrainingWorkflowRequest.from_spec( graph=snapshot.graph_config, predictive_query=self.query, @@ -338,7 +340,7 @@ def _generate(self, snapshot: GraphSnapshot, options: CoreGenerationPlan, timestamps=timestamps, ) with no_encoder_inference_warnings(): - task_plan = build_training_task_plan(workflow) + task_plan = build_training_task_plan(workflow, purpose=purpose) return workflow, task_plan, generate_train_table( task_plan.train_table, validate_targets=validate_targets) @@ -350,7 +352,7 @@ def _label_parts( from kumo_core.io.json import from_json_dict - # The fit's split ranges would not cover later anchors. + # The fit's split ranges do not cover later evaluation anchors. options = CoreGenerationPlan() if plan is not None: options = replace(from_json_dict(to_json_dict(plan), @@ -362,7 +364,8 @@ def _label_parts( try: _, _, generated = self._generate(snapshot, options, run_dir, timestamps=timestamps, - validate_targets=False) + validate_targets=False, + purpose=TaskPlanPurpose.LABELS) staging = Path(generated.staging_dir) yield sorted(str(part) for part in Path( generated.train_table_path).glob('*.parquet')) diff --git a/packages/kumo-engine/test/test_absolute_training_bounds.py b/packages/kumo-engine/test/test_absolute_training_bounds.py new file mode 100644 index 000000000..6effb4b0a --- /dev/null +++ b/packages/kumo-engine/test/test_absolute_training_bounds.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +from kumoapi.graph import Edge +from kumoapi.model_plan import TrainingTableGenerationPlan +from kumoapi.typing import Stype + +from io_utils import fsutil +from io_utils.parquet_util import read_parquet +from kumo.engine import ( + FileConnector, + Graph, + PredictiveQuery, + Table, + TrainingTableOutputConfig, + configure, +) +from kumo.engine._scratch import get_scratch_dir + +START = pd.Timestamp('2024-01-10') +END = pd.Timestamp('2024-02-20') +TASKS = [ + ('binary_classification', 'COUNT(events.*, 0, 2, days) > 0'), + ('forecasting', 'COUNT(events.*, 0, 2, days) FORECAST 2 TIMEFRAMES'), + ('multilabel_classification', + 'LIST_DISTINCT(events.category, 0, 2, days) CLASSIFY TOP 3'), +] + + +@pytest.fixture(scope='module') +def temporal_graph(tmp_path_factory, request): + root = tmp_path_factory.mktemp('absolute-bounds') + previous = get_scratch_dir() + configure(scratch_dir=root / 'scratch') + request.addfinalizer(lambda: configure(scratch_dir=previous)) + source = root / 'data' + fsutil.makedirs(str(source)) + events = [(user, day) for user in range(24) for day in range(60) + if (user + day) % 5 < 2] + frames = { + 'users': pa.table({'id': range(24), 'value': range(24)}), + 'events': pa.table({ + 'id': range(len(events)), + 'user_id': [user for user, _ in events], + 'ts': [pd.Timestamp('2024-01-01') + pd.Timedelta(days=day) + for _, day in events], + 'category': [f'category-{day % 3}' for _, day in events], + }), + } + for name, frame in frames.items(): + with fsutil.get_binary_handle(str(source / f'{name}.parquet'), + 'wb') as handle: + pq.write_table(frame, handle) + connector = FileConnector(root_dir=str(source)) + graph = Graph({ + 'users': Table.from_source_table(connector['users'], primary_key='id'), + 'events': Table.from_source_table( + connector['events'], primary_key='id', time_column='ts' + ).with_column('user_id', stype=Stype.ID).with_column( + 'category', stype=Stype.categorical), + }, [Edge('events', 'user_id', 'users')]) + return graph, graph.snapshot(), frames['events'].to_pandas() + + +def _generate(temporal_graph, tmp_path, target, **options): + graph, snapshot, _ = temporal_graph + query = PredictiveQuery(graph, f'PREDICT {target} FOR EACH users.id') + table = query.generate_training_table( + snapshot, TrainingTableGenerationPlan(timeframe_step=2, **options)) + destination = tmp_path / 'export' + fsutil.makedirs(str(destination)) + table.export(TrainingTableOutputConfig( + FileConnector(root_dir=str(destination)), 'rows')) + return query, table, read_parquet(str(destination / 'rows')) + + +@pytest.mark.parametrize('task,target', TASKS) +def test_temporal_node_bounds(temporal_graph, tmp_path, monkeypatch, task, + target): + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + options = dict(start_time=START.isoformat(), end_time=END.isoformat()) + if task == 'forecasting': + options.update(forecast_length=3, lag_timesteps=3, + year_over_year=False) + query, table, rows = _generate(temporal_graph, tmp_path, target, **options) + assert query.get_task_type().value == task + assert set(rows['SPLIT']) == {0, 1, 2} + assert rows['TIMESTAMP'].min() >= START + horizon = 6 if task == 'forecasting' else 2 + assert rows['TIMESTAMP'].max() + pd.Timedelta(days=horizon) <= END + assert table.generation_plan.start_time == START.isoformat() + assert table.generation_plan.end_time == END.isoformat() + if task == 'forecasting': + assert table.generation_plan.forecast_length == 3 + stage_horizon = pd.Timedelta(days=2 * 3) + # Each default forecast stage owns one complete forecast horizon. + expected = {stage: END - (3 - stage) * stage_horizon + for stage in range(3)} + assert rows.groupby('SPLIT')['TIMESTAMP'].max().to_dict() == expected + if task == 'binary_classification': + events = temporal_graph[2] + for row in rows.itertuples(): + count = len(events[(events['user_id'] == row.ENTITY1) + & (events['ts'] > row.TIMESTAMP) + & (events['ts'] <= row.TIMESTAMP + + pd.Timedelta(days=2))]) + assert row.TARGET1 == (count > 0) diff --git a/packages/kumo-engine/test/test_predictive_query.py b/packages/kumo-engine/test/test_predictive_query.py index c4af2df11..84485b42c 100644 --- a/packages/kumo-engine/test/test_predictive_query.py +++ b/packages/kumo-engine/test/test_predictive_query.py @@ -275,7 +275,7 @@ def test_generated_weight_rejected_before_artifacts(connector_kind, tmp_path, ) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.build_training_task_plan', - lambda workflow: pytest.fail('built a weighted native task plan'), + lambda workflow, *, purpose: pytest.fail('built a weighted native task plan'), ) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.generate_train_table', @@ -306,7 +306,7 @@ def test_generated_weight_rejected_before_artifacts(connector_kind, tmp_path, lambda: 'unweighted') monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.build_training_task_plan', - lambda workflow: native_plan, + lambda workflow, *, purpose: native_plan, ) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.generate_train_table', @@ -334,8 +334,17 @@ def test_generated_temporal_split_requires_time_column(graph_snapshot, split='TemporalSplit([0.6, 0.3, 0.1])')) -def test_generated_absolute_bounds_name_supported_alternative( - graph_snapshot, monkeypatch): +@pytest.mark.parametrize('options,message', [ + ({'start_time': 'NaT', 'end_time': '2025-01-21'}, 'valid timestamp'), + ({'start_time': '2025-01-02', 'train_start_offset': 10}, + 'Cannot specify both'), + ({'start_time': '2025-01-02', + 'split': "TimeRangeSplit([('2025-01-02', '2025-01-10'), " + "('2025-01-10', '2025-01-21')])"}, + 'cannot be combined'), +]) +def test_generated_invalid_bounds_fail_before_artifacts( + graph_snapshot, monkeypatch, options, message): graph, snapshot = graph_snapshot monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) monkeypatch.setattr( @@ -343,20 +352,13 @@ def test_generated_absolute_bounds_name_supported_alternative( lambda: pytest.fail('allocated a bounded-generation run id'), ) - with pytest.raises( - ValueError, - match="Use 'train_start_offset' and 'train_end_offset' instead"): + with pytest.raises(ValueError, match=message): PredictiveQuery( graph, 'PREDICT COUNT(orders.*, 0, 1, days) > 0 FOR EACH users.id', ).generate_training_table( snapshot, - TrainingTableGenerationPlan( - start_time='2025-01-02T12:00:00', - end_time='2025-01-21T12:00:00', - timeframe_step=1, - split='ApproxDateOffsetSplit([0.6, 0.2, 0.2])', - ), + TrainingTableGenerationPlan(timeframe_step=1, **options), ) @@ -393,7 +395,7 @@ def test_forecast_generation_retains_effective_plan(graph_snapshot, tmp_path, ) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.build_training_task_plan', - lambda workflow: native_plan, + lambda workflow, *, purpose: native_plan, ) monkeypatch.setattr( 'kumo.engine.pquery.predictive_query.generate_train_table', @@ -449,6 +451,64 @@ def test_prediction_compiler_uses_plan_forecast_horizon(graph_snapshot, 259200) +def test_evaluate_prediction_after_absolute_training_bounds( + graph, tmp_path, monkeypatch, request): + import pandas as pd + import torch + + from kumo.engine._scratch import get_scratch_dir + + monkeypatch.delenv('USE_DECOUPLED_SPLITS', raising=False) + previous = get_scratch_dir() + configure(scratch_dir=tmp_path / 'scratch') + request.addfinalizer(lambda: configure(scratch_dir=previous)) + threads = torch.get_num_threads() + torch.set_num_threads(2) + request.addfinalizer(lambda: torch.set_num_threads(threads)) + snapshot = graph.snapshot() + query = PredictiveQuery( + graph, 'PREDICT COUNT(events.*, 0, 7, days) > 0 FOR EACH users.ID') + table = query.generate_training_table( + snapshot, TrainingTableGenerationPlan( + start_time='2024-01-10', end_time='2024-02-20', timeframe_step=2)) + plan = ModelPlan() + plan.training_job.num_experiments = 1 + plan.training_job.disable_explain = True + plan.training_job.refit_trainval = False + plan.optimization.max_epochs = 1 + plan.optimization.max_steps_per_epoch = 2 + plan.optimization.batch_size = [32] + plan.model_architecture.channels = [16] + trainer = Trainer(plan, accelerator='cpu') + training_result = trainer.fit(table, snapshot=snapshot) + fitted = to_json_dict(trainer.training_table_generation_plan) + assert fitted == to_json_dict(table.generation_plan) + prediction = query.generate_prediction_table( + snapshot, prediction_time=pd.Timestamp('2024-03-01').to_pydatetime(), + plan=trainer.training_table_generation_plan) + run = trainer.predict(prediction, snapshot=snapshot, + binary_classification_threshold=0.5) + result = trainer.evaluate(run, labels_snapshot=snapshot) + assert result.matured + assert result.labelled_fraction == 1.0 + assert pd.Timestamp(result.anchors[0]).tz_convert(None) == pd.Timestamp( + '2024-03-01') + future_anchor = pd.Timestamp('2025-01-01') + future = query.generate_prediction_table( + snapshot, prediction_time=future_anchor.to_pydatetime(), + plan=trainer.training_table_generation_plan) + assert future.timestamps == (future_anchor.value,) + assert future.row_count == 50 + assert to_json_dict(trainer.training_table_generation_plan) == fitted + restored = Trainer.load(training_result.bundle_dir, accelerator='cpu') + assert to_json_dict(restored.training_table_generation_plan) == fitted + reloaded_run = restored.predict( + prediction, snapshot=snapshot, binary_classification_threshold=0.5) + assert len(reloaded_run.frame) == 50 + pd.testing.assert_frame_equal(run.frame, reloaded_run.frame, + atol=0.005, rtol=0) + + def test_custom_training_table_materializes_weight_and_manual_split( graph_snapshot, tmp_path, monkeypatch): graph, snapshot = graph_snapshot diff --git a/test/config/test_model_plan.py b/test/config/test_model_plan.py index 2b8afefa1..2f85648f1 100644 --- a/test/config/test_model_plan.py +++ b/test/config/test_model_plan.py @@ -82,6 +82,8 @@ def test_model_plan_basic(): split: inferred train_start_offset: ??? train_end_offset: ??? + start_time: ??? + end_time: ??? timeframe_step: ??? training_job: num_experiments: ??? @@ -2343,6 +2345,48 @@ def test_model_plan_decoupled_splits(monkeypatch): assert ( plan.training_table_generation.train_start_offset == MissingType.VALUE) + plan.training_table_generation.start_time = -10 + plan.training_table_generation.end_time = -1 + plan.validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + plan.training_table_generation.end_time = -20 + with pytest.raises(ValueError, match="start_time must be smaller"): + plan.validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + plan.training_table_generation.end_time = '2024-02-01' + with pytest.raises(ValueError, match="both be offsets"): + plan.validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + plan.training_table_generation.start_time = InferredType.VALUE + plan.training_table_generation.end_time = InferredType.VALUE + + link_plan = ModelPlan.from_default( + task_type=TaskType.TEMPORAL_LINK_PREDICTION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + link_plan.training_table_generation.start_time = -10 + link_plan.training_table_generation.end_time = -1 + link_plan.validate( + task_type=TaskType.TEMPORAL_LINK_PREDICTION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + # Verify validation fails if train_start_offset is set plan.training_table_generation.train_start_offset = 10 with pytest.raises(ValueError, match="field is not supported"): @@ -2368,18 +2412,31 @@ def test_model_plan_decoupled_splits(monkeypatch): plan.training_table_generation.train_start_offset == InferredType.VALUE) assert plan.training_table_generation.start_time == MissingType.VALUE - # Verify validation names the supported relative-bound alternative. - plan.training_table_generation.start_time = "2024-01-01" - plan.training_table_generation.end_time = "2024-02-01" - with pytest.raises(ValueError) as error: + plan.training_table_generation.start_time = -10 + with pytest.raises(ValueError, match="Integer 'start_time' bounds"): plan.validate( task_type=TaskType.BINARY_CLASSIFICATION, query_type=QueryType.TEMPORAL, has_train_table_time_col=True, run_mode=RunMode.FAST, ) - assert str(error.value).count( - "Use 'train_start_offset' and 'train_end_offset' instead.") == 2 + with pytest.raises(ValueError, match="Integer 'start_time' bounds"): + link_plan.validate( + task_type=TaskType.TEMPORAL_LINK_PREDICTION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) + + # Absolute bounds are supported alongside inferred legacy defaults. + plan.training_table_generation.start_time = "2024-01-01" + plan.training_table_generation.end_time = "2024-02-01" + plan.validate( + task_type=TaskType.BINARY_CLASSIFICATION, + query_type=QueryType.TEMPORAL, + has_train_table_time_col=True, + run_mode=RunMode.FAST, + ) # Static queries cannot use either absolute or relative bounds. static_plan = ModelPlan.from_default( diff --git a/test/pquery/test_predictive_query_v2.py b/test/pquery/test_predictive_query_v2.py index f95d14691..6e3d236da 100644 --- a/test/pquery/test_predictive_query_v2.py +++ b/test/pquery/test_predictive_query_v2.py @@ -414,7 +414,7 @@ def test_predictive_query_time_range_validation(basic_test_graph, tempdir): ) validate_response = pquery.get_validate_response() assert not validate_response.ok - assert validate_response.errors[0].title == 'Incompatible time parameters' + assert validate_response.errors[0].title == 'Invalid training time bounds' # Test validation when end_time is before start_time pquery = PredictiveQueryV2( @@ -426,7 +426,7 @@ def test_predictive_query_time_range_validation(basic_test_graph, tempdir): ) validate_response = pquery.get_validate_response() assert not validate_response.ok - assert validate_response.errors[0].title == 'Invalid timestamp range' + assert validate_response.errors[0].title == 'Invalid training time bounds' def test_get_default_split_with_decoupled_splits(basic_test_graph): diff --git a/test/pquery/train_table_generator/test_train_generator_basic.py b/test/pquery/train_table_generator/test_train_generator_basic.py index 0236b0a5a..c158868cb 100644 --- a/test/pquery/train_table_generator/test_train_generator_basic.py +++ b/test/pquery/train_table_generator/test_train_generator_basic.py @@ -14,6 +14,8 @@ import pyarrow as pa import pyarrow.parquet as pq import pytest +from kumo_core.model_plan import TrainingTableGenerationPlan +from kumo_core.pquery import ValidatedNativeQuery, _temporal_timestamps from kumo_core.types import ColStatType, TaskType from kumoapi.pquery import QueryType from kumoapi.train import TrainingStage @@ -26,6 +28,7 @@ from kumo.stats.stats import Stats from kumo.stats.train_table_stats import STATS_PER_TIMEFRAME from kumo.task.split import ( + ApproxDateOffsetSplit, BacktestingSplit, DateOffsetSplit, RandomSplit, @@ -44,6 +47,39 @@ NANOSECONDS_PER_SECOND = 1_000_000_000 +def test_absolute_bounds_generator_parity(create_node_pred_graph, tempdir, + monkeypatch): + graph = create_node_pred_graph('test_graph', tempdir.name) + query = ('PREDICT COUNT(TRANSACTIONS.*, 0, 3, days) > 0 ' + 'FOR EACH USERS.USER_ID') + options = TrainingTableGenerationPlan( + start_time='2023-12-01T00:00:00Z', end_time='2024-02-01T00:00:00Z', + timeframe_step=3) + available = (pd.Timestamp('2024-01-01'), pd.Timestamp('2024-01-20'), None) + studio = TrainingTableGeneratorV2( + graph, get_validated_query(graph, query), + ApproxDateOffsetSplit([0.8, 0.1, 0.1]), + start_time=options.start_time, end_time=options.end_time, + timeframe_step=3, use_decoupled_splits=False) + monkeypatch.setattr(studio, 'get_default_target_time_range', + lambda: available) + monkeypatch.setattr('kumo_core.pquery._temporal_target_time_range', + lambda *_args: available) + native_query = ValidatedNativeQuery( + query=studio.query, task_type=studio.task_type.value, + task_family='node_prediction', entity_table='USERS', + entity_column='USER_ID', target_stype=studio.query.target_stype.value, + query_type='temporal', rhs_node_type=None, rhs_is_temporal=False, + list_distinct_target_is_array=False, forecast_length=1) + native, _ = _temporal_timestamps( + native_query, '/synthetic-index', options, {}, + 'ApproxDateOffsetSplit', ()) + expected = [pd.Timestamp(f'2024-01-{day:02}') + for day in [17, 14, 11, 8, 5, 2]] + assert studio.generate_timestamps() == expected + assert [pd.Timestamp(value) for value in native] == expected + + def test_generate_time_intervals_single_aggr(create_node_pred_graph, tempdir): graph = create_node_pred_graph('test_graph', tempdir.name) query = ('PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 2) ' @@ -880,9 +916,9 @@ def test_generate_time_intervals_start_end_time_absolute( assert task_type == TaskType.BINARY_CLASSIFICATION -def test_generate_time_intervals_start_end_time_relative( +def test_generate_time_intervals_legacy_offsets( create_node_pred_graph, tempdir): - """Test using relative offsets for start_time and end_time""" + """Legacy offsets remain relative to the latest label data.""" graph = create_node_pred_graph('test_graph', tempdir.name) query = ('PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 1) > 30 ' 'FOR EACH USERS.USER_ID') @@ -893,8 +929,8 @@ def test_generate_time_intervals_start_end_time_relative( graph, validated_query, split, - start_time=-4, # 4 days before latest date - end_time=-1 # 1 day before latest date + train_start_offset=4, + train_end_offset=1, ) task_type = generator.task_type @@ -950,44 +986,28 @@ def test_generate_time_intervals_invalid_start_end_time(create_node_pred_graph, end_time='1990-01-02') validation_response = generator.validate_pre_train_table_generate() assert not validation_response.ok - assert ("start_time (1990-01-04 00:00:00) must be smaller than " - "end_time (1990-01-02 00:00:00)" - in validation_response.error_message()) - - # Test with relative offsets - generator = TrainingTableGeneratorV2( - graph, - validated_query, - split, - start_time=-1, # 1 day before latest - end_time=-3 # 3 days before latest - ) - validation_response = generator.validate_pre_train_table_generate() - assert not validation_response.ok - assert ("start_time must be more negative than end_time" + assert ('start_time must be smaller than end_time.' in validation_response.error_message()) -def test_generate_time_intervals_mixed_types_error(create_node_pred_graph, - tempdir): - """Test error when mixing absolute and relative timestamps""" +def test_generate_time_intervals_integer_bound_rejected(create_node_pred_graph, + tempdir, monkeypatch): + """Integer start bounds have an actionable error before generation.""" + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'false') graph = create_node_pred_graph('test_graph', tempdir.name) query = ('PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 1) > 30 ' 'FOR EACH USERS.USER_ID') validated_query = get_validated_query(graph, query) split = DateOffsetSplit([-1, 0], unit=TimeUnit.DAYS, timeframe=1) - generator = TrainingTableGeneratorV2( - graph, - validated_query, - split, - start_time=-2, # Relative offset - end_time='1990-01-04' # Absolute timestamp - ) - validation_response = generator.validate_pre_train_table_generate() - assert not validation_response.ok - assert ("start_time and end_time must both be offsets or both be timestamps" - in validation_response.error_message()) + with pytest.raises(ValueError, match="Integer 'start_time' bounds"): + TrainingTableGeneratorV2( + graph, validated_query, split, start_time=-2, end_time='1990-01-04') + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'true') + with pytest.raises(ValueError, match="Integer 'start_time' bounds"): + TrainingTableGeneratorV2( + graph, validated_query, split, start_time=-2, end_time='1990-01-04', + use_decoupled_splits=False) def test_generate_time_intervals_with_legacy_params_error( @@ -1125,8 +1145,9 @@ def test_generate_timestamps_v2_with_start_end_time(create_node_pred_graph, def test_generate_timestamps_v2_with_relative_start_end_time( - create_node_pred_graph, tempdir): - """Test generate_timestamps_v2 with relative start_time and end_time""" + create_node_pred_graph, tempdir, monkeypatch): + """Decoupled integer bounds use the aggregation's time unit.""" + monkeypatch.setenv('USE_DECOUPLED_SPLITS', 'true') graph = create_node_pred_graph('test_graph', tempdir.name) query = ('PREDICT SUM(TRANSACTIONS.AMOUNT, 0, 8, hours) > 30 ' 'FOR EACH USERS.USER_ID') @@ -1139,23 +1160,14 @@ def test_generate_timestamps_v2_with_relative_start_end_time( timeframe_size=8, ) generator = TrainingTableGeneratorV2( - graph, - validated_query, - split, - start_time=-48, # 2 days before latest date - end_time=-24, # 1 day before latest date - timeframe_step=8, # 24 hour steps - ) - task_type = generator.task_type - - timestamps = generator.generate_timestamps_v2() - # Should only include timestamps between max_date-3 and max_date-1 - assert timestamps == [ - pd.Timestamp('1990-01-03 16:00:00'), # Test - pd.Timestamp('1990-01-03 08:00:00'), # Val - pd.Timestamp('1990-01-03 00:00:00'), # Train + graph, validated_query, split, start_time=-48, end_time=-24, + timeframe_step=8, use_decoupled_splits=True) + assert generator.generate_timestamps_v2() == [ + pd.Timestamp('1990-01-03 16:00:00'), + pd.Timestamp('1990-01-03 08:00:00'), + pd.Timestamp('1990-01-03 00:00:00'), ] - assert task_type == TaskType.BINARY_CLASSIFICATION + assert generator.task_type == TaskType.BINARY_CLASSIFICATION def test_generate_timestamps_v2_legacy_params_error(create_node_pred_graph, @@ -1214,6 +1226,7 @@ def test_non_aligned_time_bounds_keep_newest_window(method): from types import MethodType, SimpleNamespace generator = SimpleNamespace( + task_type=TaskType.BINARY_CLASSIFICATION, get_default_target_time_range=lambda: (pd.Timestamp("2024-01-01"), pd.Timestamp("2024-02-10"), None), split=None,