Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions packages/kumo-api/kumoapi/model_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -1056,6 +1056,9 @@ class OptimizationPlan(PlanMixin):
`Kumo documentation <https://docs.kumo.ai/docs/advanced-operations
#optimization>`_ for more information.
GPU memory retries increase evaluation step limits as batches shrink to
preserve the configured evaluation row budget.
:ivar max_epochs: (``int``) The maximum number of epochs to train a model
for (*default:* ``run_mode``-dependent).
**Supported Task Types:** All
Expand Down Expand Up @@ -1169,17 +1172,15 @@ class OptimizationPlan(PlanMixin):
metadata=Metadata(),
)

max_val_steps: Union[conint(ge=1, le=8000), MissingType,
None] = compat_field(
default_factory=lambda: MissingType.VALUE,
metadata=Metadata(),
)
max_val_steps: Union[conint(ge=1), MissingType, None] = compat_field(
default_factory=lambda: MissingType.VALUE,
metadata=Metadata(),
)

max_test_steps: Union[conint(ge=1, le=8000), MissingType,
None] = compat_field(
default_factory=lambda: MissingType.VALUE,
metadata=Metadata(),
)
max_test_steps: Union[conint(ge=1), MissingType, None] = compat_field(
default_factory=lambda: MissingType.VALUE,
metadata=Metadata(),
)

loss: Union[compat_conlist(
Union[
Expand Down
15 changes: 10 additions & 5 deletions packages/kumo-core/src/kumo_core/automl/auto_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,6 @@ def __init__(
self.max_epochs = space.optimization.max_epochs
self.min_steps_per_epoch = space.optimization.min_steps_per_epoch
self.max_steps_per_epoch = space.optimization.max_steps_per_epoch
self.max_val_steps = space.optimization.max_val_steps
self.max_test_steps = space.optimization.max_test_steps
self.refit_trainval = space.training_job.refit_trainval
self.refit_full = space.training_job.refit_full
self.disable_compilation = space.training_job.disable_compilation
Expand Down Expand Up @@ -689,8 +687,8 @@ def _train(
max_epochs=self.max_epochs,
min_steps_per_epoch=self.min_steps_per_epoch,
max_steps_per_epoch=self.max_steps_per_epoch,
max_val_steps=self.max_val_steps,
max_test_steps=self.max_test_steps,
max_val_steps=cfg.optimization.max_val_steps,
max_test_steps=cfg.optimization.max_test_steps,
# General arguments:
metrics_client_job_name=metrics_client_job_name,
raise_fake_trial_error_msg=self._raise_fake_trial_error_msg,
Expand Down Expand Up @@ -996,13 +994,20 @@ def _handle_oom_retry(
f"{try_idx + 1}/{self.max_tries} tries")

if try_idx < self.max_tries:
batch_size = cfg.optimization.batch_size[0] // 2
previous_batch_size = cfg.optimization.batch_size[0]
batch_size = previous_batch_size // 2
if batch_size < self.min_retry_batch_size:
logger.warning(f"{msg}. Cannot re-run by reducing the batch "
f"size to {batch_size}. The minimum retry "
f"batch size is {self.min_retry_batch_size}.")
return None

for field in ('max_val_steps', 'max_test_steps'):
steps = getattr(cfg.optimization, field)
if steps is not None and not isinstance(steps, MissingType):
setattr(cfg.optimization, field,
(steps * previous_batch_size + batch_size - 1) //
batch_size)
cfg.optimization.batch_size = [batch_size]
logger.warning(f"{msg}. Trying to re-run by "
f"reducing the batch size by half.")
Expand Down
3 changes: 3 additions & 0 deletions packages/kumo-core/src/kumo_core/train/base_module.py
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,9 @@ def on_train_epoch_end(self):

def on_evaluation_start(self, stage: TrainingStage):
self.outputs: dict[str, Tensor | None] | None = None
# Discard partial outputs when a failed evaluation restarts.
self._validation_step_outputs.clear()
self._test_step_outputs.clear()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if self._trainer is None:
# Baseline module somehow uses this lightning module without a
# PyTorch Lightning's trainer.
Expand Down
26 changes: 26 additions & 0 deletions packages/kumo-core/src/kumo_core/train/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,9 +208,14 @@ def handle_gpu_oom(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
orig_batch_size = batch_size = None
orig_test_steps = None
is_test = func.__name__ == 'test'
orig_test_config = (self.eval_loader_kwargs,
self.max_test_steps) if is_test else None

max_retries = 3
for i in range(max_retries + 1):
retrying = False
try:
out = func(self, *args, **kwargs)
if orig_batch_size is not None:
Expand All @@ -221,6 +226,8 @@ def wrapper(self, *args, **kwargs):
eval_loader_kwargs = self.datamodule.eval_loader_kwargs
if i == 0:
orig_batch_size = eval_loader_kwargs['batch_size']
if is_test:
orig_test_steps = self.datamodule.max_test_steps

if i == max_retries:
raise e
Expand All @@ -230,6 +237,18 @@ def wrapper(self, *args, **kwargs):
gc.collect()
batch_size = math.ceil(eval_loader_kwargs['batch_size'] / 2)
eval_loader_kwargs['batch_size'] = batch_size
if is_test:
if orig_test_steps is not None:
self.datamodule.max_test_steps = (
orig_test_steps * orig_batch_size + batch_size -
1) // batch_size
# Refit init() rebuilds the datamodule from Trainer state.
self.eval_loader_kwargs = {
**(self.eval_loader_kwargs or {}),
'batch_size': batch_size,
}
self.max_test_steps = self.datamodule.max_test_steps
retrying = True
logger.warning(f"{e}. Temporarily decreasing the batch size "
f"from '{orig_batch_size}' to '{batch_size}'")
except RuntimeError as e:
Expand All @@ -251,6 +270,13 @@ def wrapper(self, *args, **kwargs):
f"For tables over 5 million rows, we recommend setting "
f"'output_embedding_dim' to {rhs_dim_limit} or "
f"lower.") from e
finally:
if (orig_test_config is not None
and orig_batch_size is not None and not retrying):
self.datamodule.eval_loader_kwargs[
'batch_size'] = orig_batch_size
self.datamodule.max_test_steps = orig_test_steps
self.eval_loader_kwargs, self.max_test_steps = orig_test_config

return wrapper

Expand Down
112 changes: 112 additions & 0 deletions packages/kumo-core/tests/test_evaluation_budget.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock

import pytest
import torch
from kumo_core.automl import auto_trainer
from kumo_core.automl.auto_trainer import AutoTrainer
from kumo_core.instrumentation import NoOpTrainingInstrumentation
from kumo_core.model_plan import ModelPlan
from kumo_core.train.trainer import handle_gpu_oom
from kumo_core.types import TaskType
from kumoapi.model_plan import MissingType, RunMode
from kumoapi.pquery import QueryType


@pytest.mark.parametrize('steps,expected', [
((100, 200), (204, 407)),
((10_000, 20_000), (20_313, 40_625)),
((None, None), (None, None)),
((MissingType.VALUE, MissingType.VALUE),
(MissingType.VALUE, MissingType.VALUE)),
])
def test_retry_budget_reaches_the_trainer_constructor(tmp_path, monkeypatch,
steps, expected):
cfg = ModelPlan.from_default(task_type=TaskType.BINARY_CLASSIFICATION,
query_type=QueryType.STATIC,
has_train_table_time_col=False,
run_mode=RunMode.DEBUG)
cfg.optimization.batch_size = [65]
cfg.optimization.max_val_steps, cfg.optimization.max_test_steps = steps
trainer = SimpleNamespace(
log_dir=str(tmp_path), random_state_seed=0, num_experiments=1,
max_tries=3, min_retry_batch_size=16, max_epochs=1,
min_steps_per_epoch=1, max_steps_per_epoch=1, tune_metric='auroc',
disable_compilation=True, _raise_fake_trial_error_msg=None,
checkpoint_path=None, encoders=None,
diskgraph_task_dataset_bundle_dir='unused',
instrumentation=NoOpTrainingInstrumentation(),
task=Mock(task_type=TaskType.BINARY_CLASSIFICATION),
runtime=SimpleNamespace(trainer_runtime_kwargs=lambda: {}))
for name in ('_get_encoder', '_get_model', '_get_head', '_get_past_encoder',
'_get_year_over_year_encoder'):
setattr(trainer, name, lambda *_: None)
cfg = AutoTrainer._handle_oom_retry(trainer, cfg, trial=0, try_idx=0)
constructor = Mock(side_effect=RuntimeError('stop before initialization'))
monkeypatch.setattr(auto_trainer, 'DiskgraphNestedTrainer', constructor)
with pytest.raises(RuntimeError, match='stop before initialization'):
AutoTrainer._train(trainer, cfg, metrics_client_job_name='test')
received = constructor.call_args.kwargs
assert (received['loader_kwargs']['batch_size'], received['max_val_steps'],
received['max_test_steps']) == (32, *expected)


@pytest.mark.parametrize('stage,limit,caps,rebuild', [
('test', 100, [100, 200, 400, 800], False),
('test', None, [None, None, None, None], False),
('predict', 100, [100, 100, 100, 100], False),
('test', 100, [100, 200, 400, 800], True),
('test', None, [None, None, None, None], True),
])
@pytest.mark.parametrize('loader_options',
[None, {}, {'batch_size': 64, 'num_workers': 2}])
@pytest.mark.parametrize('failure',
[None, torch.cuda.OutOfMemoryError, RuntimeError])
def test_evaluation_retry_preserves_budget_and_restores_state(
stage, limit, caps, rebuild, loader_options, failure, monkeypatch):
monkeypatch.setattr(torch.cuda, 'empty_cache', lambda: None)
module = SimpleNamespace(eval_loader_kwargs={'batch_size': 64},
max_test_steps=limit)
trainer = SimpleNamespace(
datamodule=module,
eval_loader_kwargs=(None if loader_options is None else
loader_options.copy()),
max_test_steps=limit,
task=SimpleNamespace(task_type=TaskType.BINARY_CLASSIFICATION),
pl_model=Mock())
original_options = trainer.eval_loader_kwargs
attempted = []

def evaluate(self):
if rebuild:
self.datamodule = SimpleNamespace(
eval_loader_kwargs={
'batch_size': (self.eval_loader_kwargs or {}).get(
'batch_size', 64)}, max_test_steps=self.max_test_steps)
module = self.datamodule
batch = module.eval_loader_kwargs['batch_size']
attempted.append((batch, module.max_test_steps))
if batch > 16:
raise torch.cuda.OutOfMemoryError('simulated')
if failure is not None:
raise failure('simulated')
return 'complete'

evaluate.__name__ = stage
run = handle_gpu_oom(evaluate)
with pytest.raises(failure) if failure else nullcontext():
assert run(trainer) == 'complete'
expected = list(zip([64, 32, 16, 8], caps))
assert attempted == expected[:4 if failure is
torch.cuda.OutOfMemoryError else 3]
if stage == 'test' or failure is None:
assert trainer.datamodule.eval_loader_kwargs == {'batch_size': 64}
assert trainer.datamodule.max_test_steps == limit
assert trainer.eval_loader_kwargs is original_options
assert trainer.eval_loader_kwargs == loader_options
assert trainer.max_test_steps == limit
6 changes: 5 additions & 1 deletion packages/kumo-core/tests/test_model_plan_conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,13 @@ def test_round_trip_defaults_internal_only_sections(case: Case) -> None:


@pytest.mark.parametrize("case", _CASES, ids=_CASE_IDS)
@pytest.mark.parametrize('steps',
[api.MissingType.VALUE, 8001, 1_000_000, None])
def test_round_trip_with_base_preserves_internal_only_sections(
case: Case) -> None:
case: Case, steps) -> None:
internal = case.internal_plan()
limits = internal.optimization
limits.max_val_steps = limits.max_test_steps = steps

restored = case.to_internal(case.to_public(internal), base=internal)

Expand Down
38 changes: 38 additions & 0 deletions test/train/test_base_node_module.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,3 +165,41 @@ def test_validation_keeps_step_outputs_unless_fitting_unexplained(
module.on_validation_epoch_end()
assert bool(module.outputs) == bool(kept)
assert ('pred' in module.outputs) == bool(kept)


@pytest.mark.parametrize('name, fn', [
('validation', TrainerFn.VALIDATING),
('test', TrainerFn.TESTING),
])
def test_evaluation_retry_discards_outputs_from_failed_attempt(
monkeypatch, name, fn):
monkeypatch.setattr(BaseNodeLightningModule, '__abstractmethods__', set())
module = BaseNodeLightningModule(TaskType.BINARY_CLASSIFICATION)
module.metric_store.init_metrics([], module)
module._trainer = SimpleNamespace(
ckpt_path=None, sanity_checking=False, num_val_batches=[2],
num_test_batches=[2], state=SimpleNamespace(fn=fn))
first = dict(pkey=torch.tensor([10]), logits=torch.tensor([[-1.0]]),
target=torch.tensor([[0.0]]))
second = dict(pkey=torch.tensor([11]), logits=torch.tensor([[1.0]]),
target=torch.tensor([[1.0]]))
module.evaluation_step = Mock(side_effect=[
first, torch.cuda.OutOfMemoryError('simulated'), first, second,
])
start = getattr(module, f'on_{name}_start')
step = getattr(module, f'{name}_step')
end = getattr(module, f'on_{name}_epoch_end')

start()
step(SimpleNamespace(), 0)
with pytest.raises(torch.cuda.OutOfMemoryError, match='simulated'):
step(SimpleNamespace(), 1)

start()
step(SimpleNamespace(), 0)
step(SimpleNamespace(), 1)
end()

assert module.outputs['pkey'].tolist() == [10, 11]
assert module.outputs['target'].flatten().tolist() == [0.0, 1.0]
assert module.outputs['pred'].tolist() == [False, True]
8 changes: 6 additions & 2 deletions ui/src/gen/spec/v0_openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -13399,11 +13399,13 @@
"anyOf": [
{
"type": "integer",
"maximum": 8000.0,
"minimum": 1.0
},
{
"$ref": "#/components/schemas/MissingType"
},
{
"type": "null"
}
],
"title": "Max Val Steps",
Expand All @@ -13426,11 +13428,13 @@
"anyOf": [
{
"type": "integer",
"maximum": 8000.0,
"minimum": 1.0
},
{
"$ref": "#/components/schemas/MissingType"
},
{
"type": "null"
}
],
"title": "Max Test Steps",
Expand Down
8 changes: 6 additions & 2 deletions ui/src/gen/spec/v1_openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -7600,11 +7600,13 @@
"anyOf": [
{
"type": "integer",
"maximum": 8000.0,
"minimum": 1.0
},
{
"$ref": "#/components/schemas/MissingType"
},
{
"type": "null"
}
],
"title": "Max Val Steps",
Expand All @@ -7627,11 +7629,13 @@
"anyOf": [
{
"type": "integer",
"maximum": 8000.0,
"minimum": 1.0
},
{
"$ref": "#/components/schemas/MissingType"
},
{
"type": "null"
}
],
"title": "Max Test Steps",
Expand Down
Loading
Loading