Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions kumo/data/generic/diskgraph_train_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
build_split_spec as build_native_split_spec
from kumo_core.training.table import \
generate_train_table as generate_native_train_table
from kumo_core.training.timestamps import forecast_offsets
from kumoapi.pquery.validated_predictive_query import ValidatedPredictiveQuery

from io_utils import fsutil, logging
Expand Down Expand Up @@ -676,10 +677,7 @@ def _ttng_forecasting_offset_kwargs(
step = time_range.timeframe
assert step is not None

future_offsets_secs = [
int(((pt + i * step) - pt).total_seconds())
for i in range(forecast_length)
]
future_offsets_secs = list(forecast_offsets(step, forecast_length))
lag_offsets_secs = None
if lag_timesteps is not None:
lag_ts = get_lag_timestamps(pt, lag_timesteps, step)
Expand Down
98 changes: 11 additions & 87 deletions kumo/pquery/pquery_validators/pre_train_table_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import Dict, List, Optional, Union

import pandas as pd
from kumo_core.training.bounds import validate_time_bounds
from kumo_core.training.validation import validate_train_offsets
from kumo_core.types import TaskType
from kumoapi.pquery import QueryType, ValidatedPredictiveQuery
Expand Down Expand Up @@ -774,94 +775,17 @@ def validate_limits(

return validation_response

def validate_train_time_compatibility(self) -> ValidationResponse:
r"""Validates that start_time/end_time and train offsets are not used
together.

Note: This validation can be removed once we fully migrate from using
train_start_offset/train_end_offset to start_time/end_time parameters.

Returns:
ValidationResponse: Response containing any validation errors.
"""
validation_response = ValidationResponse()

if ((self.start_time is not None or self.end_time is not None)
and (self.train_start_offset is not None
or self.train_end_offset is not None)):
validation_response.errors.append(
ValidationError(
title="Incompatible time parameters",
message=("Cannot specify both start_time/end_time and "
"train_start_offset/train_end_offset. Use either "
"absolute/relative timestamps or train offset "
"parameters, but not both.")))

return validation_response

def validate_train_time(self) -> ValidationResponse:
r"""Validates start_time and end_time parameters.

Checks:
1. Both must be offsets or both timestamps
2. For timestamps: start_time must be smaller than end_time
3. For offsets: both must be <= 0 and start_time must be < end_time

Returns:
ValidationResponse: Response containing any validation errors.
"""
def validate_time_bounds(self) -> ValidationResponse:
"""Adapt the shared time-bound checks to service validation errors."""
validation_response = ValidationResponse()

if self.start_time is None or self.end_time is None:
return validation_response

# Check if both are same type (offset or timestamp)
is_start_offset = isinstance(self.start_time, pd.DateOffset)
is_end_offset = isinstance(self.end_time, pd.DateOffset)

if is_start_offset != is_end_offset:
validation_response.errors.append(
ValidationError(
title="Inconsistent time parameter types",
message=("start_time and end_time must both be offsets "
"or both be timestamps, but got mixed types.")))
return validation_response

# Validate timestamps
if not is_start_offset:
assert isinstance(self.start_time, pd.Timestamp)
assert isinstance(self.end_time, pd.Timestamp)
if self.start_time >= self.end_time:
validation_response.errors.append(
ValidationError(
title="Invalid timestamp range", message=(
f"start_time ({self.start_time}) must be smaller "
f"than end_time ({self.end_time})")))
# Validate offsets
else:
assert isinstance(self.start_time, pd.DateOffset)
assert isinstance(self.end_time, pd.DateOffset)
now = pd.Timestamp.now()
start_ts = now + self.start_time
end_ts = now + self.end_time

if start_ts <= now and end_ts <= now:
if start_ts >= end_ts:
validation_response.errors.append(
ValidationError(
title="Invalid offset range", message=(
f"For relative offsets, start_time must be "
f"more negative than end_time, but got "
f"start_time={self.start_time}, "
f"end_time={self.end_time}")))
else:
validation_response.errors.append(
ValidationError(
title="Invalid offset values", message=(
f"Relative offsets must be <= 0 (i.e. in the "
f"past), but got start_time={self.start_time}, "
f"end_time={self.end_time}")))

try:
validate_time_bounds(
self.start_time, self.end_time,
train_start_offset=self.train_start_offset,
train_end_offset=self.train_end_offset)
except ValueError as error:
validation_response.errors.append(ValidationError(
title='Invalid training time bounds', message=str(error)))
return validation_response


Expand Down
68 changes: 36 additions & 32 deletions kumo/pquery/train_table_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,14 @@
from typing import Any, Callable, Dict, List, Optional, Tuple, Union

import kumo_core.types as core_types
from kumo_core.model_plan import _use_decoupled_splits
from kumo_core.training.bounds import parse_time_bound
from kumo_core.training.options import allow_array_targets
from kumo_core.training.splits import TrainTableTime, train_table_time
from kumo_core.training.timestamps import (
generate_decoupled_timestamps,
generate_legacy_timestamps,
resolve_training_window,
)
from kumo_core.training.validation import GenerationValidationError
from kumo_core.types.conversions import DataFrame
Expand Down Expand Up @@ -286,26 +289,18 @@ def __init__(
or isinstance(train_end_offset, DateOffset))
self.train_end_offset = train_end_offset

# Handle new time parameters
self.start_time = None
if start_time is not None:
if start_time == '-INF':
pass # Keep start_time as None for -INF case
elif isinstance(start_time, int):
self.start_time = self.get_date_offset(int(start_time))
else:
assert isinstance(start_time, str)
self.start_time = Timestamp(start_time)
assert isinstance(self.start_time, Timestamp)

self.end_time = None
if end_time is not None:
if isinstance(end_time, int):
self.end_time = self.get_date_offset(int(end_time))
else:
assert isinstance(end_time, str)
self.end_time = Timestamp(end_time)
assert isinstance(self.end_time, Timestamp)
self.use_decoupled_splits = (
_use_decoupled_splits()
if use_decoupled_splits is None else use_decoupled_splits)
start_bound: int | str | DateOffset | None = start_time
end_bound: int | str | DateOffset | None = end_time
if self.use_decoupled_splits:
if isinstance(start_bound, int):
start_bound = self.get_date_offset(start_bound)
if isinstance(end_bound, int):
end_bound = self.get_date_offset(end_bound)
self.start_time = parse_time_bound(start_bound, 'start_time')
self.end_time = parse_time_bound(end_bound, 'end_time')

self.timeframe_step = None
if timeframe_step is not None:
Expand All @@ -314,7 +309,6 @@ def __init__(
self.lag_timesteps = lag_timesteps
self.forecast_length = forecast_length
self.year_over_year = year_over_year
self.use_decoupled_splits = use_decoupled_splits
self.weight_col = normalize_weight_col(weight_col)

self.task_type: core_types.TaskType = get_task_type(
Expand Down Expand Up @@ -425,6 +419,12 @@ def generate_timestamps(self) -> List[Timestamp]:
values = self._timestamp_inputs()
(original_start, original_end, filter_buffer, offset_range, timeframe,
step) = values
window = None
if self.task_type.is_node_pred:
window = resolve_training_window(
self.start_time, self.end_time, original_start, original_end)
start_time = self.start_time if window is None else window.start
end_time = self.end_time if window is None else window.end
logger.info(
f"For raw data, the min_date is {original_start} and the max_date "
f"is {original_end}.")
Expand Down Expand Up @@ -463,13 +463,16 @@ def fix_boundaries() -> None:
split=descriptor,
train_start_offset=self.train_start_offset,
train_end_offset=self.train_end_offset,
start_time=self.start_time,
end_time=self.end_time,
start_time=start_time,
end_time=end_time,
drop_earliest=self.drop_earliest_timeframe,
on_fixed_boundaries=fix_boundaries,
)
if short_window:
return []
return [] if window is None else window.filter(
timestamps, offset_range.end_date_offset)
if window is not None:
timestamps = window.filter(timestamps, offset_range.end_date_offset)
logger.info(f'Generated {len(timestamps)} timestamps:')
for i, timestamp in enumerate(timestamps):
logger.info(f'Timestamp {i:03d}: {timestamp}')
Expand All @@ -489,17 +492,23 @@ def generate_timestamps_v2(self) -> List[Timestamp]:
values = self._timestamp_inputs()
(original_start, original_end, filter_buffer, offset_range, timeframe,
step) = values
window = None
if self.task_type.is_node_pred:
window = resolve_training_window(
self.start_time, self.end_time, original_start, original_end)
timestamps = generate_decoupled_timestamps(
original_start=original_start,
original_end=original_end,
filter_buffer=filter_buffer,
target_timeframe=timeframe,
target_start_offset=offset_range.start_date_offset,
step=step,
start_time=self.start_time,
end_time=self.end_time,
start_time=self.start_time if window is None else window.start,
end_time=self.end_time if window is None else window.end,
drop_earliest=self.drop_earliest_timeframe,
)
if window is not None:
timestamps = window.filter(timestamps, offset_range.end_date_offset)
logger.info(f'Generated {len(timestamps)} timestamps:')
for i, timestamp in enumerate(timestamps):
logger.info(f'Timestamp {i:03d}: {timestamp}')
Expand Down Expand Up @@ -585,13 +594,8 @@ def validate_pre_train_table_generate(
validation_response.merge(
self.pre_train_table_validator.validate_temporal_splits())

# Add new validation for time parameter compatibility
validation_response.merge(self.pre_train_table_validator.
validate_train_time_compatibility())

# Add new validation for time parameter values
validation_response.merge(
self.pre_train_table_validator.validate_train_time())
self.pre_train_table_validator.validate_time_bounds())

validation_response.merge(
self.pre_train_table_validator.validate_train_offsets(
Expand Down
43 changes: 26 additions & 17 deletions packages/kumo-api/kumoapi/model_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -608,30 +608,39 @@ class TrainingTableGenerationPlan(PlanMixin):
:ivar train_start_offset: (``int`` | ``"inferred``") Defines the numerical
offset from the most recent entry to use to generate training data
labels. Unless a custom time unit is specified in the aggregation, this
value is in days (*default:* ``"inferred"``). Eventually we would
like to migrate this parameter to start_time.
value is in days (*default:* ``"inferred"``).
**Supported Task Types:** Temporal

:ivar train_end_offset: (``int`` | ``"inferred"``) Defines the numerical
offset from the most recent entry to not use to generate training data
labels. Unless a custom time unit is specified in the aggregation, this
value is in days (*default:* ``"inferred"``). Eventually we would
like to migrate this parameter to end_time.
value is in days (*default:* ``"inferred"``).
**Supported Task Types:** Temporal

:ivar start_time: (``int`` | ``"inferred"`` | ``str``) Defines the absolute
start time to use for generating training data. Can be specified
either as a pandas Timestamp compatible string (e.g. '2024-04-01') or
as a non-positive integer offset in days from the current time
(e.g. -30 for 30 days ago) (*default:* ``"inferred"``).
**Supported Task Types:** Temporal

:ivar end_time: (``int`` | ``"inferred"`` | ``str``) Defines the absolute
end time to use for generating training data. Can be specified either
as a pandas Timestamp compatible string (e.g. '2024-04-01') or as a
non-positive integer offset in days from the current time
(e.g. -30 for 30 days ago) (*default:* ``"inferred"``).
**Supported Task Types:** Temporal
:ivar start_time: (``"inferred"`` | ``str``) Inclusive lower
bound on generated prediction anchors. Temporal node tasks accept
pandas Timestamp compatible strings (e.g. '2024-04-01') with default
splits. Timezone-aware strings are normalized to UTC; naive strings
are read as UTC. ``"inferred"`` or ``"-INF"`` uses the available-data
lower bound. Bounds are intersected with available label data.
Direct absolute bounds cannot be combined with train_start_offset,
train_end_offset, or explicit date/offset window splits.
For relative training windows, use train_start_offset/train_end_offset
in the aggregation's time unit.
(*default:* ``"inferred"``).
**Supported Task Types:** Temporal node classification, regression,
ranking, and forecasting.

:ivar end_time: (``"inferred"`` | ``str``) Label observation
cutoff, rather than the latest prediction anchor. Each generated
example must fit its complete target horizon by this cutoff,
including every future label for forecasting. Accepts the same
timestamp strings as start_time;
``"inferred"`` uses available label data. Either absolute bound can
be omitted. Empty intersections or windows too short for a complete
example fail generation. (*default:* ``"inferred"``).
**Supported Task Types:** Temporal node classification, regression,
ranking, and forecasting.

:ivar timeframe_step: (``int`` | ``"inferred"``) Defines the step size of
generating time intervals for training table generation
Expand Down
Loading
Loading