From 9cef1010d93fb6b4846ec1e8da0058fccee3e6f2 Mon Sep 17 00:00:00 2001 From: bghira Date: Sun, 2 Aug 2026 13:10:12 -0600 Subject: [PATCH 1/2] Add segmented checkpointing base controls --- documentation/OPTIONS.es.md | 35 +- documentation/OPTIONS.hi.md | 35 +- documentation/OPTIONS.ja.md | 35 +- documentation/OPTIONS.md | 35 +- documentation/OPTIONS.pt-BR.md | 35 +- documentation/OPTIONS.zh.md | 35 +- .../SEGMENTED_CHECKPOINTING.es.md | 1007 +++++++++++++++++ .../SEGMENTED_CHECKPOINTING.hi.md | 1007 +++++++++++++++++ .../SEGMENTED_CHECKPOINTING.ja.md | 1007 +++++++++++++++++ .../experimental/SEGMENTED_CHECKPOINTING.md | 1007 +++++++++++++++++ .../SEGMENTED_CHECKPOINTING.pt-BR.md | 1007 +++++++++++++++++ .../SEGMENTED_CHECKPOINTING.zh.md | 1007 +++++++++++++++++ .../experimental/UNSLOTH_CHECKPOINTING.es.md | 8 +- .../experimental/UNSLOTH_CHECKPOINTING.hi.md | 8 +- .../experimental/UNSLOTH_CHECKPOINTING.ja.md | 8 +- .../experimental/UNSLOTH_CHECKPOINTING.md | 8 +- .../UNSLOTH_CHECKPOINTING.pt-BR.md | 8 +- .../experimental/UNSLOTH_CHECKPOINTING.zh.md | 8 +- documentation/index.es.md | 4 +- documentation/index.hi.md | 4 +- documentation/index.ja.md | 4 +- documentation/index.md | 4 +- documentation/index.pt-BR.md | 4 +- documentation/index.zh.md | 4 +- mkdocs.yml | 1 + simpletuner/helpers/configuration/env_file.py | 4 + simpletuner/helpers/models/common.py | 43 + simpletuner/helpers/musubi_block_swap.py | 78 +- .../helpers/ramtorch/modules/linear.py | 125 +- simpletuner/helpers/ramtorch/profiling.py | 18 + simpletuner/helpers/ramtorch_extensions.py | 76 +- .../training/default_settings/safety_check.py | 45 + .../gradient_checkpointing_interval.py | 36 +- .../offloaded_gradient_checkpointer.py | 854 +++++++++++++- simpletuner/helpers/training/trainer.py | 144 ++- simpletuner/helpers/utils/ramtorch.py | 27 +- .../services/field_registry/sections/model.py | 26 +- .../field_registry/sections/training.py | 62 + tests/test_gradient_checkpointing_backend.py | 636 +++++++++-- tests/test_musubi_block_swap.py | 34 + tests/test_ramtorch.py | 69 ++ 41 files changed, 8373 insertions(+), 229 deletions(-) create mode 100644 documentation/experimental/SEGMENTED_CHECKPOINTING.es.md create mode 100644 documentation/experimental/SEGMENTED_CHECKPOINTING.hi.md create mode 100644 documentation/experimental/SEGMENTED_CHECKPOINTING.ja.md create mode 100644 documentation/experimental/SEGMENTED_CHECKPOINTING.md create mode 100644 documentation/experimental/SEGMENTED_CHECKPOINTING.pt-BR.md create mode 100644 documentation/experimental/SEGMENTED_CHECKPOINTING.zh.md diff --git a/documentation/OPTIONS.es.md b/documentation/OPTIONS.es.md index 51f0ea1d2..8c4024c34 100644 --- a/documentation/OPTIONS.es.md +++ b/documentation/OPTIONS.es.md @@ -282,8 +282,14 @@ Donde `foo` es tu entorno de configuración; o simplemente usa `config/config.js ### `--gradient_checkpointing_interval` -- **Qué**: Hace checkpoint de chunks contiguos de *n* bloques, donde *n* es un valor mayor que cero. Un valor de 1 equivale a dejar `--gradient_checkpointing` habilitado, y 2 checkpointa chunks de dos bloques. -- **Nota**: Flux y MageFlow usan checkpointing en chunks contiguos en rutas whole-block. Valores más altos reducen recompute pero dejan más activaciones en VRAM. +- **Qué**: Intervalo dependiente del modelo para checkpointing de bloques transformer. Un valor de 1 equivale básicamente a dejar `--gradient_checkpointing` habilitado. +- **Nota**: Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image y Wan usan chunks contiguos de *n* bloques en rutas whole-block. Otras familias que exponen esta opción pueden seguir usando el comportamiento anterior de "checkpoint cada *n* bloques". Valores más altos pueden reducir recompute, pero normalmente dejan más activaciones en VRAM. + +### `--gradient_checkpointing_segment_stride` + +- **Qué**: Inicia un segmento con checkpoint cada *n* bloques en rutas segmented whole-block compatibles. +- **Ejemplo**: Con `--gradient_checkpointing_interval=2` y `--gradient_checkpointing_segment_stride=4`, SimpleTuner checkpointa dos bloques, ejecuta los dos siguientes normalmente y repite. +- **Nota**: Compatible con rutas segmentadas de Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image y Wan. El stride debe ser al menos igual al interval. Consulta [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md). ### `--gradient_checkpointing_backend` @@ -293,7 +299,27 @@ Donde `foo` es tu entorno de configuración; o simplemente usa `config/config.js - `torch-ffn`: checkpoint solo del lado feed-forward en modelos con un límite FFN limpio. - `unsloth`: checkpoint del bloque completo compatible y offload de tensores guardados a CPU. - `unsloth-ffn`: checkpoint solo del lado feed-forward y offload de sus tensores guardados a CPU. -- **Nota**: Solo efectivo cuando `--gradient_checkpointing` está habilitado. Las variantes `unsloth` requieren CUDA. Las variantes FFN-only soportan actualmente bloques estilo Flux.1 y MageFlow, y fallan de forma explícita si no existe ese scope. Consulta [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) para tradeoffs medidos. +- **Nota**: Solo efectivo cuando `--gradient_checkpointing` está habilitado. Las variantes `unsloth` requieren CUDA. Las variantes FFN-only soportan actualmente Chroma, Flux, Krea 2, LTXVideo2, MageFlow, Wan y Z-Image, y fallan de forma explícita si no existe ese scope. Consulta [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) para tradeoffs medidos. + +### `--gradient_checkpointing_offload_attention` + +- **Qué**: Hace offload a CPU de las activaciones guardadas del lado attention en modelos con un límite attention/FFN limpio. +- **Por qué**: Cuando transferir es más barato que recomputar attention, reduce VRAM sin pagar todo el coste de rematerializar attention. +- **Nota**: Puede activarse por si solo. Tambien puede combinarse con cualquier checkpoint backend que soporte el modelo. Attention offload actualmente soporta Chroma, Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Wan y Z-Image; las familias no soportadas fallan de forma explícita. + +### `--gradient_checkpointing_offload_pin_memory_max_buckets` + +- **Predeterminado**: `12` +- **Qué**: Número máximo de buckets distintos de tensores CPU pinned usados por activation offload. +- **Por qué**: Pinned memory mejora las transferencias CPU/GPU, pero resoluciones y longitudes de texto variables pueden crear formas raras. Al alcanzar este límite, las nuevas formas de bucket usan memoria CPU normal. +- **Nota**: Usa `0` para desactivar el pooling de pinned memory para activation offload. + +### `--gradient_checkpointing_offload_prefetch` + +- **Predeterminado**: `false` +- **Qué**: Aprende el orden de restore en backward para activations offloaded y precarga en GPU el tensor que probablemente venga después. +- **Por qué**: El restore H2D justo a tiempo casi no se puede solapar. Con un orden estable, prefetch puede ocultar parte de la transferencia detrás del backward compute. +- **Nota**: Experimental y solo activo con `--gradient_checkpointing_offload_attention`. ### `--refiner_training` @@ -1741,6 +1767,7 @@ usage: train.py [-h] --model_family [--text_encoder_3_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--text_encoder_4_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL] + [--gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE] [--offload_during_startup [OFFLOAD_DURING_STARTUP]] [--quantize_via {cpu,accelerator,pipeline}] [--quantization_config QUANTIZATION_CONFIG] @@ -2058,6 +2085,8 @@ options: memory. --gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL Checkpoint every N transformer blocks + --gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE + Start a checkpointed segment every N transformer blocks --offload_during_startup [OFFLOAD_DURING_STARTUP] Offload text encoders to CPU during VAE caching --quantize_via {cpu,accelerator,pipeline} diff --git a/documentation/OPTIONS.hi.md b/documentation/OPTIONS.hi.md index 499ab72c1..a4f55e186 100644 --- a/documentation/OPTIONS.hi.md +++ b/documentation/OPTIONS.hi.md @@ -282,8 +282,14 @@ simpletuner configure config/foo/config.json ### `--gradient_checkpointing_interval` -- **What**: *n* contiguous blocks के chunks checkpoint करें, जहाँ *n* शून्य से बड़ा मान है। 1 का मान `--gradient_checkpointing` enabled जैसा है, और 2 दो-block chunks checkpoint करता है। -- **Note**: Flux और MageFlow whole-block paths पर contiguous chunked checkpointing use करते हैं। Higher values recompute overhead घटाती हैं लेकिन VRAM में ज्यादा activations रखती हैं। +- **What**: Transformer block checkpointing के लिए model-dependent interval। 1 का मान लगभग `--gradient_checkpointing` enabled जैसा है। +- **Note**: Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image, और Wan whole-block paths पर *n* contiguous block chunks use करते हैं। इस option को expose करने वाली दूसरी families अभी भी पुराने "हर *n*-th block checkpoint" behavior का उपयोग कर सकती हैं। Higher values recompute overhead घटा सकती हैं, लेकिन आम तौर पर VRAM में ज्यादा activations रखती हैं। + +### `--gradient_checkpointing_segment_stride` + +- **What**: Supported segmented whole-block paths पर हर *n* blocks में checkpointed segment शुरू करें। +- **Example**: `--gradient_checkpointing_interval=2` और `--gradient_checkpointing_segment_stride=4` के साथ SimpleTuner दो blocks checkpoint करता है, अगले दो blocks normally चलाता है, और repeat करता है। +- **Note**: Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image, और Wan segmented paths पर supported। stride interval से कम नहीं हो सकता। [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) देखें। ### `--gradient_checkpointing_backend` @@ -293,7 +299,27 @@ simpletuner configure config/foo/config.json - `torch-ffn`: साफ FFN boundary वाले models पर केवल feed-forward side checkpoint करता है। - `unsloth`: पूरे supported block को checkpoint करता है और saved tensors CPU पर offload करता है। - `unsloth-ffn`: केवल feed-forward side checkpoint करता है और उसके saved tensors CPU पर offload करता है। -- **Note**: केवल `--gradient_checkpointing` enabled होने पर प्रभावी। `unsloth` variants के लिए CUDA आवश्यक है। FFN-only variants अभी Flux.1-style blocks और MageFlow support करते हैं, और unsupported scope पर साफ error मिलता है। Measured tradeoffs के लिए [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) देखें। +- **Note**: केवल `--gradient_checkpointing` enabled होने पर प्रभावी। `unsloth` variants के लिए CUDA आवश्यक है। FFN-only variants अभी Chroma, Flux, Krea 2, LTXVideo2, MageFlow, Wan, और Z-Image support करते हैं, और unsupported scope पर साफ error मिलता है। Measured tradeoffs के लिए [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) देखें। + +### `--gradient_checkpointing_offload_attention` + +- **What**: साफ attention/FFN boundary वाले models पर attention-side saved activations CPU पर offload करें। +- **Why**: जब transfer attention recompute से सस्ता हो, तो full attention rematerialization cost दिए बिना VRAM घट सकती है। +- **Note**: इसे अकेले enable किया जा सकता है। Model जिस भी checkpoint backend को support करे, उसके साथ भी combine किया जा सकता है। Attention offload अभी Chroma, Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Wan, और Z-Image support करता है; unsupported model families साफ error देती हैं। + +### `--gradient_checkpointing_offload_pin_memory_max_buckets` + +- **Default**: `12` +- **What**: activation offload द्वारा इस्तेमाल किए जाने वाले अलग-अलग pinned CPU tensor buckets की maximum संख्या। +- **Why**: pinned memory CPU/GPU transfer में मदद करती है, लेकिन variable resolution और text length rare tensor shapes बना सकते हैं। limit पूरी होने के बाद नए bucket shapes normal CPU memory use करेंगे। +- **Note**: activation offload के pinned-memory pooling को बंद करने के लिए `0` set करें। + +### `--gradient_checkpointing_offload_prefetch` + +- **Default**: `false` +- **What**: offloaded activations के backward restore order को learn करके likely next tensor को GPU पर prefetch करता है। +- **Why**: JIT H2D restore अक्सर overlap नहीं कर पाता। order stable होने के बाद prefetch transfer के कुछ हिस्से को backward compute के पीछे छिपा सकता है। +- **Note**: Experimental है और केवल `--gradient_checkpointing_offload_attention` के साथ active होता है। ### `--refiner_training` @@ -1739,6 +1765,7 @@ usage: train.py [-h] --model_family [--text_encoder_3_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--text_encoder_4_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL] + [--gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE] [--offload_during_startup [OFFLOAD_DURING_STARTUP]] [--quantize_via {cpu,accelerator,pipeline}] [--quantization_config QUANTIZATION_CONFIG] @@ -2056,6 +2083,8 @@ options: memory. --gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL Checkpoint every N transformer blocks + --gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE + Start a checkpointed segment every N transformer blocks --offload_during_startup [OFFLOAD_DURING_STARTUP] Offload text encoders to CPU during VAE caching --quantize_via {cpu,accelerator,pipeline} diff --git a/documentation/OPTIONS.ja.md b/documentation/OPTIONS.ja.md index 95b1eedcb..d7f24dc47 100644 --- a/documentation/OPTIONS.ja.md +++ b/documentation/OPTIONS.ja.md @@ -283,8 +283,14 @@ simpletuner configure config/foo/config.json ### `--gradient_checkpointing_interval` -- **内容**: 連続した *n* block chunk を checkpoint します。値は 0 より大きい必要があります。1 は `--gradient_checkpointing` と同等で、2 は 2-block chunk を checkpoint します。 -- **注記**: Flux と MageFlow は whole-block path で連続 chunk checkpointing を使います。値を大きくすると再計算 overhead は減りますが、VRAM に残る activation は増えます。 +- **内容**: transformer block checkpointing のモデル依存 interval です。1 は `--gradient_checkpointing` を有効にした状態とほぼ同じです。 +- **注記**: Flux、Flux.2、Krea 2、LTXVideo2、MageFlow、Z-Image、Wan は whole-block path で連続した *n* block chunk を使います。このオプションを持つ他の family は、従来の「*n* block ごとに checkpoint」挙動のままの場合があります。値を大きくすると再計算 overhead は減ることがありますが、通常は VRAM に残る activation が増えます。 + +### `--gradient_checkpointing_segment_stride` + +- **内容**: 対応する segmented whole-block path で、*n* block ごとに checkpointed segment を開始します。 +- **例**: `--gradient_checkpointing_interval=2` と `--gradient_checkpointing_segment_stride=4` では、SimpleTuner は 2 block を checkpoint し、次の 2 block を通常実行し、それを繰り返します。 +- **注記**: Flux、Flux.2、Krea 2、LTXVideo2、MageFlow、Z-Image、Wan の segmented path で対応しています。stride は interval 以上である必要があります。[Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) を参照してください。 ### `--gradient_checkpointing_backend` @@ -294,7 +300,27 @@ simpletuner configure config/foo/config.json - `torch-ffn`: 明確な FFN 境界があるモデルで feed-forward 側だけを checkpoint します。 - `unsloth`: 対応 block 全体を checkpoint し、保存 tensor を CPU に offload します。 - `unsloth-ffn`: feed-forward 側だけを checkpoint し、保存 tensor を CPU に offload します。 -- **注記**: `--gradient_checkpointing` が有効な場合のみ機能します。`unsloth` 系は CUDA が必要です。FFN-only 系は現在 Flux.1-style blocks と MageFlow に対応し、対応していない scope では明示的に失敗します。実測 tradeoff は [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) を参照してください。 +- **注記**: `--gradient_checkpointing` が有効な場合のみ機能します。`unsloth` 系は CUDA が必要です。FFN-only 系は現在 Chroma、Flux、Krea 2、LTXVideo2、MageFlow、Wan、Z-Image に対応し、対応していない scope では明示的に失敗します。実測 tradeoff は [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) を参照してください。 + +### `--gradient_checkpointing_offload_attention` + +- **内容**: 明確な attention/FFN 境界があるモデルで、attention 側の保存 activations を CPU に offload します。 +- **理由**: attention を再計算するより転送が安い場合、完全な attention rematerialization cost を払わずに VRAM を減らせます。 +- **注記**: 単体でも有効化できます。そのモデルがサポートする任意の checkpoint backend とも組み合わせられます。Attention offload は現在 Chroma、Flux、Flux.2、Krea 2、LTXVideo2、MageFlow、Wan、Z-Image に対応し、未対応モデルでは明示的に失敗します。 + +### `--gradient_checkpointing_offload_pin_memory_max_buckets` + +- **デフォルト**: `12` +- **内容**: activation offload が使う pinned CPU tensor bucket の最大数です。 +- **理由**: pinned memory は CPU/GPU 転送に有利ですが、可変解像度や可変 text length では珍しい tensor shape が出ます。上限に達した後の新しい bucket shape は通常の CPU memory を使います。 +- **注記**: `0` にすると activation offload の pinned-memory pooling を無効化します。 + +### `--gradient_checkpointing_offload_prefetch` + +- **デフォルト**: `false` +- **内容**: offload された activations の backward restore order を学習し、次に必要になりそうな tensor を GPU に prefetch します。 +- **理由**: JIT H2D restore はほとんど overlap できません。順序が安定すると、prefetch は一部の転送を backward compute の裏に隠せます。 +- **注記**: 実験的機能で、`--gradient_checkpointing_offload_attention` が有効な場合のみ動作します。 ### `--refiner_training` @@ -1742,6 +1768,7 @@ usage: train.py [-h] --model_family [--text_encoder_3_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--text_encoder_4_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL] + [--gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE] [--offload_during_startup [OFFLOAD_DURING_STARTUP]] [--quantize_via {cpu,accelerator,pipeline}] [--quantization_config QUANTIZATION_CONFIG] @@ -2058,6 +2085,8 @@ options: memory. --gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL Checkpoint every N transformer blocks + --gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE + Start a checkpointed segment every N transformer blocks --offload_during_startup [OFFLOAD_DURING_STARTUP] Offload text encoders to CPU during VAE caching --quantize_via {cpu,accelerator,pipeline} diff --git a/documentation/OPTIONS.md b/documentation/OPTIONS.md index 5a9e03f13..087476bae 100644 --- a/documentation/OPTIONS.md +++ b/documentation/OPTIONS.md @@ -289,8 +289,14 @@ Where `foo` is your config environment - or just use `config/config.json` if you ### `--gradient_checkpointing_interval` -- **What**: Checkpoint contiguous chunks of *n* blocks, where *n* is a value greater than zero. A value of 1 is effectively the same as just leaving `--gradient_checkpointing` enabled, and a value of 2 checkpoints two-block chunks. -- **Note**: Flux and MageFlow use contiguous chunked checkpointing on whole-block paths. Higher values reduce recompute overhead but keep more activations in VRAM. +- **What**: Model-dependent interval for transformer block checkpointing. A value of 1 is effectively the same as leaving `--gradient_checkpointing` enabled. +- **Note**: Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image, and Wan use contiguous chunks of *n* blocks on whole-block paths. Other families that expose this option may use the older "checkpoint every *n*-th block" behavior. Higher values can reduce recompute overhead, but usually keep more activations in VRAM. + +### `--gradient_checkpointing_segment_stride` + +- **What**: Start a checkpointed segment every *n* blocks on supported segmented whole-block paths. +- **Example**: With `--gradient_checkpointing_interval=2` and `--gradient_checkpointing_segment_stride=4`, SimpleTuner checkpoints two blocks, runs the next two blocks normally, and repeats. +- **Note**: Supported by Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image, and Wan segmented paths. The stride must be at least the interval. See [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md). ### `--gradient_checkpointing_backend` @@ -300,7 +306,27 @@ Where `foo` is your config environment - or just use `config/config.json` if you - `torch-ffn`: checkpoint only the feed-forward side on models that expose a clean FFN boundary. - `unsloth`: checkpoint the whole supported block and offload saved tensors to CPU. - `unsloth-ffn`: checkpoint only the feed-forward side and offload its saved tensors to CPU. -- **Note**: Only effective when `--gradient_checkpointing` is enabled. The `unsloth` variants require CUDA. FFN-only variants currently support Flux.1-style blocks and MageFlow, and fail loudly when the model does not expose that scope. See [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) for measured tradeoffs. +- **Note**: Only effective when `--gradient_checkpointing` is enabled. The `unsloth` variants require CUDA. FFN-only variants currently support Chroma, Flux, Krea 2, LTXVideo2, MageFlow, Wan, and Z-Image, and fail loudly when the model does not expose that scope. See [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) for measured tradeoffs. + +### `--gradient_checkpointing_offload_attention` + +- **What**: Offload attention-side saved activations to CPU on models with a clean attention/FFN boundary. +- **Why**: When transfer is cheaper than recomputing attention, this can reduce VRAM without paying the full attention rematerialization cost. +- **Note**: This can be enabled by itself. It can also be combined with any checkpoint backend that the model supports. Attention offload is currently supported by Chroma, Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Wan, and Z-Image; unsupported model families fail loudly. + +### `--gradient_checkpointing_offload_pin_memory_max_buckets` + +- **Default**: `12` +- **What**: Maximum number of distinct pinned CPU tensor buckets used by activation offload. +- **Why**: Pinned memory improves CPU/GPU transfer behavior, but variable resolutions and text lengths can create rare tensor shapes. Once this cap is reached, new bucket shapes use normal CPU memory instead. +- **Note**: Set `0` to disable pinned-memory pooling for activation offload. + +### `--gradient_checkpointing_offload_prefetch` + +- **Default**: `false` +- **What**: Learn the backward restore order for labeled offloaded activations and prefetch likely next tensors back to GPU. +- **Why**: JIT H2D restore usually cannot overlap much. Prefetch can hide some transfer behind backward compute after the order stabilizes. +- **Note**: Experimental and only active with `--gradient_checkpointing_offload_attention`. ### `--refiner_training` @@ -1745,6 +1771,7 @@ usage: train.py [-h] --model_family [--text_encoder_3_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--text_encoder_4_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL] + [--gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE] [--offload_during_startup [OFFLOAD_DURING_STARTUP]] [--quantize_via {cpu,accelerator,pipeline}] [--quantization_config QUANTIZATION_CONFIG] @@ -2062,6 +2089,8 @@ options: memory. --gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL Checkpoint every N transformer blocks + --gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE + Start a checkpointed segment every N transformer blocks --offload_during_startup [OFFLOAD_DURING_STARTUP] Offload text encoders to CPU during VAE caching --quantize_via {cpu,accelerator,pipeline} diff --git a/documentation/OPTIONS.pt-BR.md b/documentation/OPTIONS.pt-BR.md index ae3e2023d..a7355132c 100644 --- a/documentation/OPTIONS.pt-BR.md +++ b/documentation/OPTIONS.pt-BR.md @@ -282,8 +282,14 @@ Onde `foo` e seu ambiente de config — ou use `config/config.json` se nao estiv ### `--gradient_checkpointing_interval` -- **O que**: Faz checkpoint de chunks contiguos de *n* blocos, onde *n* e um valor maior que zero. Um valor 1 e efetivamente o mesmo que deixar `--gradient_checkpointing` habilitado, e 2 faz checkpoint de chunks de dois blocos. -- **Nota**: Flux e MageFlow usam checkpointing em chunks contiguos nos caminhos whole-block. Valores maiores reduzem recompute, mas mantêm mais activations na VRAM. +- **O que**: Intervalo dependente do modelo para checkpointing de blocos transformer. Um valor 1 e basicamente o mesmo que deixar `--gradient_checkpointing` habilitado. +- **Nota**: Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image e Wan usam chunks contiguos de *n* blocos nos caminhos whole-block. Outras familias que expoem esta opcao podem continuar usando o comportamento antigo de "checkpoint a cada *n* blocos". Valores maiores podem reduzir recompute, mas normalmente mantem mais activations na VRAM. + +### `--gradient_checkpointing_segment_stride` + +- **O que**: Inicia um segmento com checkpoint a cada *n* blocos nos caminhos segmented whole-block suportados. +- **Exemplo**: Com `--gradient_checkpointing_interval=2` e `--gradient_checkpointing_segment_stride=4`, o SimpleTuner faz checkpoint de dois blocos, executa os dois blocos seguintes normalmente e repete. +- **Nota**: Suportado pelos caminhos segmentados de Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image e Wan. O stride deve ser pelo menos igual ao interval. Veja [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md). ### `--gradient_checkpointing_backend` @@ -293,7 +299,27 @@ Onde `foo` e seu ambiente de config — ou use `config/config.json` se nao estiv - `torch-ffn`: checkpoint só do lado feed-forward em modelos com fronteira FFN limpa. - `unsloth`: checkpoint do bloco completo compatível e offload dos tensores salvos para CPU. - `unsloth-ffn`: checkpoint só do lado feed-forward e offload dos tensores salvos para CPU. -- **Nota**: So funciona quando `--gradient_checkpointing` esta habilitado. As variantes `unsloth` requerem CUDA. As variantes FFN-only atualmente suportam blocos estilo Flux.1 e MageFlow, e falham explicitamente quando o modelo nao expoe esse escopo. Veja [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) para tradeoffs medidos. +- **Nota**: So funciona quando `--gradient_checkpointing` esta habilitado. As variantes `unsloth` requerem CUDA. As variantes FFN-only atualmente suportam Chroma, Flux, Krea 2, LTXVideo2, MageFlow, Wan e Z-Image, e falham explicitamente quando o modelo nao expoe esse escopo. Veja [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) para tradeoffs medidos. + +### `--gradient_checkpointing_offload_attention` + +- **O que**: Faz offload para CPU das activations salvas do lado attention em modelos com fronteira attention/FFN limpa. +- **Por que**: Quando transferencia e mais barata que recomputar attention, reduz VRAM sem pagar todo o custo de rematerializar attention. +- **Nota**: Pode ser ativado sozinho. Tambem pode ser combinado com qualquer checkpoint backend que o modelo suportar. Attention offload atualmente suporta Chroma, Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Wan e Z-Image; familias nao suportadas falham explicitamente. + +### `--gradient_checkpointing_offload_pin_memory_max_buckets` + +- **Padrao**: `12` +- **O que**: Numero maximo de buckets distintos de tensores CPU pinned usados por activation offload. +- **Por que**: Pinned memory melhora transferencias CPU/GPU, mas resolucoes e comprimentos de texto variaveis podem criar shapes raros. Ao atingir esse limite, novos bucket shapes usam memoria CPU normal. +- **Nota**: Use `0` para desativar o pooling de pinned memory para activation offload. + +### `--gradient_checkpointing_offload_prefetch` + +- **Padrao**: `false` +- **O que**: Aprende a ordem de restore no backward para activations offloaded e prefetch o provavel proximo tensor para a GPU. +- **Por que**: Restore H2D just-in-time quase nao consegue overlap. Com ordem estavel, prefetch pode esconder parte da transferencia atras do backward compute. +- **Nota**: Experimental e ativo apenas com `--gradient_checkpointing_offload_attention`. ### `--refiner_training` @@ -1737,6 +1763,7 @@ usage: train.py [-h] --model_family [--text_encoder_3_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--text_encoder_4_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL] + [--gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE] [--offload_during_startup [OFFLOAD_DURING_STARTUP]] [--quantize_via {cpu,accelerator,pipeline}] [--quantization_config QUANTIZATION_CONFIG] @@ -2053,6 +2080,8 @@ options: memory. --gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL Checkpoint every N transformer blocks + --gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE + Start a checkpointed segment every N transformer blocks --offload_during_startup [OFFLOAD_DURING_STARTUP] Offload text encoders to CPU during VAE caching --quantize_via {cpu,accelerator,pipeline} diff --git a/documentation/OPTIONS.zh.md b/documentation/OPTIONS.zh.md index 5db8f63fe..66461d886 100644 --- a/documentation/OPTIONS.zh.md +++ b/documentation/OPTIONS.zh.md @@ -283,8 +283,14 @@ simpletuner configure config/foo/config.json ### `--gradient_checkpointing_interval` -- **内容**：checkpoint 连续的 *n* 个 block chunk，*n* 必须大于 0。1 等同于启用 `--gradient_checkpointing`，2 会 checkpoint 两个 block 的 chunk。 -- **说明**：Flux 和 MageFlow 会在 whole-block 路径上使用连续 chunk checkpointing。值越大，重算开销越低，但 VRAM 里保留的 activation 越多。 +- **内容**：transformer block checkpointing 的模型相关 interval。1 基本等同于启用 `--gradient_checkpointing`。 +- **说明**：Flux、Flux.2、Krea 2、LTXVideo2、MageFlow、Z-Image 和 Wan 在 whole-block 路径上使用连续的 *n* 个 block chunk。其他支持此选项的模型族可能仍使用旧的“每第 *n* 个 block checkpoint”行为。值越大可能降低重算开销，但通常会在 VRAM 中保留更多 activation。 + +### `--gradient_checkpointing_segment_stride` + +- **内容**：在支持 segmented whole-block 路径的模型上，每隔 *n* 个 block 启动一个 checkpointed segment。 +- **示例**：设置 `--gradient_checkpointing_interval=2` 且 `--gradient_checkpointing_segment_stride=4` 时，SimpleTuner 会 checkpoint 两个 block，然后正常运行接下来的两个 block，如此重复。 +- **说明**：Flux、Flux.2、Krea 2、LTXVideo2、MageFlow、Z-Image 和 Wan segmented 路径支持此选项。stride 必须大于或等于 interval。参见 [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md)。 ### `--gradient_checkpointing_backend` @@ -294,7 +300,27 @@ simpletuner configure config/foo/config.json - `torch-ffn`：在有清晰 FFN 边界的模型上，只 checkpoint feed-forward 部分。 - `unsloth`：checkpoint 整个受支持 block，并把保存的 tensor 卸载到 CPU。 - `unsloth-ffn`：只 checkpoint feed-forward 部分，并把保存的 tensor 卸载到 CPU。 -- **说明**：仅在启用 `--gradient_checkpointing` 时生效。`unsloth` 变体需要 CUDA。FFN-only 变体目前支持 Flux.1 风格 blocks 和 MageFlow；模型不支持该 scope 时会直接报错。实测 tradeoff 见 [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md)。 +- **说明**：仅在启用 `--gradient_checkpointing` 时生效。`unsloth` 变体需要 CUDA。FFN-only 变体目前支持 Chroma、Flux、Krea 2、LTXVideo2、MageFlow、Wan 和 Z-Image；模型不支持该 scope 时会直接报错。实测 tradeoff 见 [Unsloth-style checkpointing](experimental/UNSLOTH_CHECKPOINTING.md)。 + +### `--gradient_checkpointing_offload_attention` + +- **内容**：在有清晰 attention/FFN 边界的模型上，把 attention 侧保存的 activations 卸载到 CPU。 +- **原因**：当传输比重算 attention 更便宜时，可降低 VRAM，且不需要支付完整 attention 重算成本。 +- **说明**：它可以单独启用。也可与模型支持的任意 checkpoint backend 组合。Attention offload 目前支持 Chroma、Flux、Flux.2、Krea 2、LTXVideo2、MageFlow、Wan 和 Z-Image；不支持的模型会直接报错。 + +### `--gradient_checkpointing_offload_pin_memory_max_buckets` + +- **默认值**：`12` +- **内容**：activation offload 可使用的 pinned CPU tensor bucket 最大数量。 +- **原因**：pinned memory 有利于 CPU/GPU 传输，但可变分辨率和文本长度会产生少见 tensor 形状。达到上限后，新的 bucket 形状会改用普通 CPU memory。 +- **说明**：设为 `0` 可关闭 activation offload 的 pinned-memory pooling。 + +### `--gradient_checkpointing_offload_prefetch` + +- **默认值**：`false` +- **内容**：学习 offloaded activations 的 backward restore 顺序，并把可能下一个需要的 tensor 预取回 GPU。 +- **原因**：JIT H2D restore 通常很难重叠。顺序稳定后，prefetch 可把部分传输隐藏在 backward compute 后面。 +- **说明**：实验性功能，并且只在启用 `--gradient_checkpointing_offload_attention` 时生效。 ### `--refiner_training` @@ -1744,6 +1770,7 @@ usage: train.py [-h] --model_family [--text_encoder_3_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--text_encoder_4_precision {no_change,int8-quanto,int4-quanto,int2-quanto,int8-torchao,int8dq-torchao,int8dq-int4-torchao,nf4-bnb,int4-torchao,fp8-quanto,fp8uz-quanto,fp8-native,fp8-torchao,fp8wo-torchao,fp8-int4-torchao,fp8-transformerengine}] [--gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL] + [--gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE] [--offload_during_startup [OFFLOAD_DURING_STARTUP]] [--quantize_via {cpu,accelerator,pipeline}] [--quantization_config QUANTIZATION_CONFIG] @@ -2060,6 +2087,8 @@ options: memory. --gradient_checkpointing_interval GRADIENT_CHECKPOINTING_INTERVAL Checkpoint every N transformer blocks + --gradient_checkpointing_segment_stride GRADIENT_CHECKPOINTING_SEGMENT_STRIDE + Start a checkpointed segment every N transformer blocks --offload_during_startup [OFFLOAD_DURING_STARTUP] Offload text encoders to CPU during VAE caching --quantize_via {cpu,accelerator,pipeline} diff --git a/documentation/experimental/SEGMENTED_CHECKPOINTING.es.md b/documentation/experimental/SEGMENTED_CHECKPOINTING.es.md new file mode 100644 index 000000000..4a232d5ef --- /dev/null +++ b/documentation/experimental/SEGMENTED_CHECKPOINTING.es.md @@ -0,0 +1,1007 @@ +# Checkpointing Segmentado + +El checkpointing segmentado queda entre checkpointing en cada block y no usar checkpointing. + +Usa el backend de activation checkpointing de PyTorch. SimpleTuner ejecuta un grupo contiguo de transformer blocks bajo una llamada de checkpoint y pasa el hidden state devuelto al grupo siguiente. Grupos mas anchos recomputan menos en backward, pero mantienen mas activations vivas. + +Para CPU offload y FFN-only checkpointing, usa [Unsloth-style checkpointing](UNSLOTH_CHECKPOINTING.es.md#controls). La regla corta sigue en [Decision Rule](UNSLOTH_CHECKPOINTING.es.md#decision-rule). + +## Controles + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2 +} +``` + +En rutas whole-block soportadas, `gradient_checkpointing_interval` es el ancho del segment. `2` significa checkpoint de blocks `0-1`, `2-3`, `4-5`, etc. + +Para controlar VRAM con mas detalle, agrega stride: + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2, + "gradient_checkpointing_segment_stride": 4 +} +``` + +Esto checkpointa blocks `0-1`, ejecuta `2-3` normalmente, checkpointa `4-5`, ejecuta `6-7` normalmente y repite. El stride debe ser al menos el interval; schedules solapados no son validos. + +Rutas segmented whole-block soportadas: Flux.1, Flux.2, HunyuanVideo, Krea 2, LongCat Image, LongCat Video, LTXVideo 0.9, LTXVideo2, Lumina2, MageFlow, PixArt, SD3, SanaVideo, Z-Image, ZLab I1 y Wan. + +Stable Cascade stage C tambien soporta interval y stride, pero aplica el schedule a la secuencia de micro-bloques Res/Timestep/Attention del UNet en vez de a grupos transformer whole-block. + +Algunas familias usan semantica de interval especifica del modelo: + +| Family | `gradient_checkpointing_interval` | `gradient_checkpointing_segment_stride` | +| --- | --- | --- | +| Sana | Checkpoint cada N-th block | Ignorado | +| Stable Cascade stage C | Checkpoint de micro-bloques UNet por interval | Stride alterna ventanas UNet checkpointed y no checkpointed | +| SD1x, SDXL | Sin soporte segmented whole-block | Ignorado | + +No compares filas stride cuando stride se ignora. Si los numeros son identicos, normalmente el option no se aplico. + +## Cuando Usarlo + +Usalo despues de que el checkpointing normal por block ya entra en VRAM pero cuesta demasiado tiempo por step. Empieza con `2`. Si hay VRAM, prueba `2` con stride `4` en modelos muy profundos. + +No esperes que ayude cuando el peak viene sobre todo de pesos entrenables, optimizer state, validation, cache VAE, block swapping o routing. SimpleTuner vuelve a la ruta per-block mas segura cuando una funcion del modelo necesita control por block. + +`dynamo_use_regional_compilation` no es una victoria universal. Ayudo o fue neutral en varios image models, pero fue mala opcion para los perfiles Wan/RamTorch y LTXVideo2 de abajo. + +## Benchmarks + +Medido con ejemplos reales de SimpleTuner en pods single-GPU H100 y L40S. Validation y checkpoint saves estuvieron desactivados, cache preparation quedo fuera, y el compile/setup del primer step dentro del train loop se excluye cuando existe timing post-warmup. + +Cada celda medida es `post-warmup sec/step / peak VRAM GiB`. Las celdas de estado significan: `OOM` agoto memoria de GPU, `failed` no llego a training steps medidos, `unsupported` significa que esa opcion no estaba conectada para esa familia, y `not run` significa que el sweep no incluyo esa combinacion. + +Compara modos dentro de una familia primero. Las comparaciones entre familias son aproximadas porque cambian resolution, frame count, attention backend, model depth, trainable adapter type y dataset shape. + +La matriz de abajo es la fuente de verdad para este sweep. Las notas especificas por modelo marcan caveats cuando una fila es coverage data y no una recomendacion. + + + +### Resultados Por Familia + +### ACE Step 1.5 + +Example: `ace_step-v1-5.peft-lora`. Resolution: 512. + +Note: This sweep did not produce a usable ACE Step throughput row. The status-only entries below should be treated as coverage gaps, not as a recommendation. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | OOM | OOM | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | OOM | OOM | +| int8-sdnq-hadamard | interval2 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Destilacion AnyFlow en Anima + +Ejemplo: `anima-anyflow.peft-lora`. Resolucion: 1024x1024. Esta fila mide destilacion AnyFlow usando Anima, no el ejemplo LoRA de Anima puro. Usa `anima.peft-lora` para entrenamiento de imagen Anima puro a 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.022 / 17.26 | 0.719 / 17.21 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.232 / 5.60 | 0.903 / 5.56 | +| bf16 | interval2 | 1.252 / 5.60 | 0.897 / 5.56 | +| bf16 | seg2-stride4 | 1.244 / 5.60 | 0.898 / 5.56 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 5.417 / 18.61 | 4.974 / 18.57 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.562 / 4.36 | 4.019 / 4.31 | +| int8-sdnq-hadamard | interval2 | 3.723 / 4.36 | 3.196 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4 | 3.658 / 4.36 | 3.140 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.242 / 45.71 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.810 / 5.51 | 2.576 / 5.46 | +| fp8-torchao | interval2 | 2.846 / 5.51 | 2.581 / 5.46 | +| fp8-torchao | seg2-stride4 | 2.766 / 5.51 | 2.567 / 5.46 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### AuraFlow + +Example: `auraflow.peft-lora`. Resolution: 1024x1024. + +Nota: AuraFlow soporta cuantizacion con SDNQ y TorchAO. Las filas cuantizadas sin checkpointing estan abajo; las filas cuantizadas con checkpointing necesitan una nueva medicion completa antes de publicar numeros aqui. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.180 / 19.19 | 0.233 / 19.12 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.824 / 13.37 | 0.833 / 13.32 | +| bf16 | interval2 | 1.764 / 16.21 | 0.877 / 16.14 | +| bf16 | seg2-stride4 | 1.771 / 16.21 | 0.887 / 16.14 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.642 / 12.88 | 0.610 / 12.87 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | not run | not run | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.621 / 24.53 | 0.757 / 24.44 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | not run | not run | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Boogu Image + +Example: `boogu-image-v0.1.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.694 / 59.14 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.907 / 23.44 | 2.641 / 23.39 | +| bf16 | interval2 | 0.912 / 23.44 | 2.649 / 23.39 | +| bf16 | seg2-stride4 | 0.911 / 23.44 | 2.648 / 23.39 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.488 / 53.24 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.878 / 15.20 | 3.309 / 15.15 | +| int8-sdnq-hadamard | interval2 | 1.630 / 34.12 | 2.577 / 34.07 | +| int8-sdnq-hadamard | seg2-stride4 | 1.656 / 34.11 | 2.574 / 34.06 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.713 / 18.48 | 3.778 / 18.44 | +| fp8-torchao | interval2 | 1.731 / 18.48 | 3.777 / 18.44 | +| fp8-torchao | seg2-stride4 | 1.721 / 18.48 | 3.779 / 18.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Chroma + +Example: `chroma.peft-lora`. Resolution: 1024x1024. + +Note: Checkpointed Chroma rows use `attention_mechanism=native-efficient`, which was the stable attention path for this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.454 / 26.18 | 0.559 / 26.13 | +| bf16 | activation-offload | 4.873 / 18.74 | 4.793 / 18.69 | +| bf16 | layer | 1.276 / 17.67 | 1.430 / 17.63 | +| bf16 | interval2 | 1.204 / 21.80 | 1.349 / 21.75 | +| bf16 | seg2-stride4 | 1.200 / 21.78 | 1.382 / 21.74 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.083 / 21.42 | 1.061 / 21.37 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.714 / 10.41 | 1.646 / 10.36 | +| int8-sdnq-hadamard | interval2 | 1.443 / 15.72 | 1.391 / 15.68 | +| int8-sdnq-hadamard | seg2-stride4 | 1.428 / 15.71 | 1.323 / 15.67 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.122 / 45.44 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.821 / 10.64 | 2.104 / 10.60 | +| fp8-torchao | interval2 | 1.447 / 27.49 | 1.871 / 27.44 | +| fp8-torchao | seg2-stride4 | 1.431 / 27.49 | 1.877 / 27.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 2 Image + +Example: `cosmos2image.lycoris-lokr`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.336 / 8.05 | 0.316 / 8.00 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.567 / 4.09 | 0.559 / 4.04 | +| bf16 | interval2 | 0.595 / 4.09 | 0.544 / 4.04 | +| bf16 | seg2-stride4 | 0.598 / 4.09 | 0.546 / 4.04 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.831 / 6.56 | 0.783 / 6.56 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.315 / 2.44 | 1.288 / 2.39 | +| int8-sdnq-hadamard | interval2 | 1.346 / 2.44 | 1.321 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4 | 1.413 / 2.44 | 1.274 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.850 / 13.30 | 0.840 / 13.25 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.692 / 2.70 | 1.560 / 2.65 | +| fp8-torchao | interval2 | 1.626 / 2.70 | 1.544 / 2.65 | +| fp8-torchao | seg2-stride4 | 1.607 / 2.70 | 1.555 / 2.65 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 3 + +Example: `cosmos3-edge-image-24g.lycoris-lokr`. Resolution: 1024 px. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 2.747 / 8.90 | 2.904 / 8.86 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.602 / 8.90 | 2.606 / 8.86 | +| bf16 | interval2 | 2.658 / 8.90 | 2.567 / 8.86 | +| bf16 | seg2-stride4 | 2.628 / 8.90 | 2.899 / 8.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 17.112 / 6.21 | 17.965 / 6.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 3.238 / 6.21 | 2.891 / 6.16 | +| int8-sdnq-hadamard | interval2 | 3.253 / 6.21 | 2.916 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4 | 3.254 / 6.21 | 2.923 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.849 / 17.84 | 1.559 / 17.80 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.869 / 17.84 | 1.599 / 17.79 | +| fp8-torchao | interval2 | 1.855 / 17.84 | 1.600 / 17.80 | +| fp8-torchao | seg2-stride4 | 1.859 / 17.84 | 1.523 / 17.80 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### ERNIE 4.5 Image + +Example: `ernie.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.711 / 15.61 | 1.282 / 15.56 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.110 / 4.94 | 1.840 / 4.90 | +| bf16 | interval2 | 0.876 / 10.16 | 1.536 / 10.12 | +| bf16 | seg2-stride4 | 0.874 / 10.16 | 1.532 / 10.12 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 2.782 / 13.75 | 2.722 / 13.70 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.262 / 2.98 | 4.393 / 2.94 | +| int8-sdnq-hadamard | interval2 | 3.547 / 8.26 | 3.380 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4 | 3.366 / 8.26 | 3.457 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.432 / 14.03 | 2.303 / 13.98 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.920 / 3.26 | 4.008 / 3.22 | +| fp8-torchao | interval2 | 3.316 / 8.54 | 2.973 / 8.49 | +| fp8-torchao | seg2-stride4 | 2.994 / 8.54 | 3.046 / 8.49 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HeartMula + +Example: `heartmula.peft-lora`. Entrenamiento con tokens de audio. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HiDream + +Example: `hidream.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.515 / 44.58 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.928 / 33.57 | 1.043 / 33.52 | +| bf16 | interval2 | 0.971 / 33.57 | 1.002 / 33.52 | +| bf16 | seg2-stride4 | 0.952 / 33.57 | 1.010 / 33.52 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | not measured | not measured | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.497 / 17.99 | 2.058 / 17.93 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | not measured | not measured | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 4.920 / 18.10 | 4.338 / 18.05 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +SDNQ Hadamard numbers use `sdnq_compile_mode=eager`. The compiled SDNQ path quantizes HiDream, but this sweep spent the first training step in Inductor dequantizer compilation, so it is not listed as a throughput row. + +### HunyuanVideo + +Ejemplo: `hunyuanvideo-1.5-t2v.peft-lora`. Forma de training: video buckets de 480 pixel-area, 48 frames, batch 2. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | not run | not run | +| bf16 | layer | 7.682 / 26.35 | 22.816 / 26.30 | +| bf16 | interval2 | 7.398 / 26.11 | 22.772 / 26.06 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 10.679 / 58.37 | not run | +| int8-sdnq-hadamard | none | not run | not run | +| int8-sdnq-hadamard | activation-offload | not run | not run | +| int8-sdnq-hadamard | layer | 11.765 / 25.96 | 34.464 / 25.92 | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | not run | not run | +| fp8-torchao | none | not run | not run | +| fp8-torchao | activation-offload | not run | not run | +| fp8-torchao | layer | 10.516 / 33.55 | 32.003 / 33.53 | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | not run | not run | + +HunyuanVideo usa muchas activations con esta forma de training. Per-block e interval-2 encajan bien; sin checkpointing no entra en un H100 de 80 GB. `seg2-stride4` solo entro en este sweep con attention activation offload, y esa fila es una salida de memoria, no una recomendacion de velocidad. SDNQ Hadamard funciona, pero las formas variables de conditioning todavia disparan compilacion de kernels dinamicos durante la medicion. + +### Ideogram 4.0 + +Ejemplo: `ideogram-fp8.peft-lora`. Resolucion: 1024x1024. El flavour fp8 usa el checkpoint fp8 nativo weight-only de Ideogram 4 (`base_model_precision=no_change`). + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| fp8-native | none | failed | OOM | +| fp8-native | activation-offload | unsupported | unsupported | +| fp8-native | layer | 1.033 / 12.57 | 3.101 / 11.82 | +| fp8-native | interval2 | 1.030 / 12.33 | 3.098 / 11.82 | +| fp8-native | seg2-stride4 | 1.031 / 12.33 | 3.088 / 11.82 | +| fp8-native | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.702 / 61.78 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.033 / 12.33 | 3.030 / 11.82 | +| int8-sdnq-hadamard | interval2 | 1.032 / 12.33 | 3.034 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4 | 1.028 / 12.33 | 3.035 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | + +### Kandinsky 5 Image + +Example: `kandinsky5-image-6b-t2i.lycoris-lokr`. Resolution: 1024x1024. + +Note: batch 3 at 1024x1024 needs full checkpointing on both cards. SDNQ with Hadamard is the best low-VRAM row; H100 can also use partial checkpointing with SDNQ, but only near the top of the card. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.956 / 25.52 | 10.189 / 25.42 | +| bf16 | layer | 6.590 / 25.58 | 9.458 / 25.55 | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 7.186 / 20.00 | 10.141 / 19.95 | +| int8-sdnq-hadamard | layer | 6.830 / 20.12 | 9.362 / 20.08 | +| int8-sdnq-hadamard | interval2 | 5.746 / 75.67 | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 6.057 / 71.46 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 8.319 / 24.29 | 14.716 / 24.24 | +| fp8-torchao | layer | 7.976 / 24.40 | 13.949 / 24.35 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kandinsky 5 Video + +Example: `kandinsky5-video-2b-t2v.peft-lora`. Resolution: 768x512, 81f. + +Kandinsky 5 video is activation-heavy at this frame count. Full block checkpointing is the practical baseline on both cards. On H100, `interval2` and `seg2-stride4` are faster when they fit; on L40S, SDNQ `interval2` is the only partial-checkpoint row here that fits without attention activation offload. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 2.580 / 9.49 | 7.136 / 9.45 | +| bf16 | layer | 2.267 / 9.83 | 6.379 / 9.79 | +| bf16 | interval2 | 1.967 / 44.57 | OOM | +| bf16 | seg2-stride4 | 1.971 / 46.62 | OOM | +| bf16 | seg2-stride4-offload | 2.275 / 37.99 | 6.249 / 37.94 | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 2.844 / 8.07 | 7.234 / 8.02 | +| int8-sdnq-hadamard | layer | 2.460 / 8.40 | 6.509 / 8.35 | +| int8-sdnq-hadamard | interval2 | 2.126 / 43.12 | 5.641 / 43.08 | +| int8-sdnq-hadamard | seg2-stride4 | 2.125 / 45.19 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 2.451 / 36.56 | 6.322 / 36.51 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 3.867 / 15.11 | 11.501 / 15.06 | +| fp8-torchao | layer | 3.579 / 15.28 | 10.822 / 15.24 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kolors + +Example: `kolors.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.635 / 7.22 | 0.628 / 7.17 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.118 / 5.36 | 1.065 / 5.31 | +| bf16 | interval2 | 1.105 / 5.36 | 1.068 / 5.31 | +| bf16 | seg2-stride4 | 1.110 / 5.36 | 1.072 / 5.31 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.860 / 5.68 | 1.726 / 5.63 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.967 / 3.38 | 2.803 / 3.33 | +| int8-sdnq-hadamard | interval2 | 2.770 / 3.32 | 2.655 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4 | 2.805 / 3.32 | 2.742 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.629 / 8.79 | 1.637 / 8.75 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.044 / 3.50 | 3.013 / 3.45 | +| fp8-torchao | interval2 | 3.040 / 3.50 | 3.019 / 3.45 | +| fp8-torchao | seg2-stride4 | 2.988 / 3.50 | 2.935 / 3.45 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Krea 2 + +Ejemplo: `krea2.peft-lora`. Resolucion de entrenamiento: recorte cuadrado de 512 px. Ajuste de validacion del ejemplo: 1024x1024; la validacion estuvo desactivada en el benchmark. + +La tabla principal usa regional compilation. Eso ayuda al step speed de Krea2, pero no da una comparacion limpia de VRAM: el compiled graph/workspace mantiene el pico cerca del pico sin checkpointing en varios modos. Una corrida de control bf16 con regional compilation desactivado mostro que el checkpointing esta conectado y tiene la forma esperada de memoria/velocidad: + +La fila `activation-offload` aqui significa full-block checkpointing mas attention activation offload. Frente a full-block `layer` checkpointing solo, attention offload no redujo el peak VRAM de Krea2 en este shape; principalmente agrego CPU transfer overhead. + +| Mode | H100 no-compile | L40S no-compile | +| --- | ---: | ---: | +| none | 0.272 / 40.09 | 0.661 / 40.01 | +| layer | 0.371 / 30.06 | 0.919 / 30.01 | +| seg2-stride4 | 0.317 / 34.75 | 0.788 / 34.70 | +| activation-offload | 0.657 / 30.50 | 1.341 / 30.30 | + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.275 / 40.06 | 0.662 / 40.01 | +| bf16 | activation-offload | 0.416 / 34.62 | 0.822 / 34.57 | +| bf16 | layer | 0.268 / 40.06 | 0.665 / 40.01 | +| bf16 | interval2 | 0.274 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4 | 0.279 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4-offload | 0.404 / 34.62 | 0.819 / 34.57 | +| int8-sdnq-hadamard | none | 0.462 / 27.01 | 0.807 / 26.96 | +| int8-sdnq-hadamard | activation-offload | 0.773 / 21.57 | 1.012 / 21.53 | +| int8-sdnq-hadamard | layer | 0.473 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | interval2 | 0.474 / 27.01 | 0.804 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4 | 0.472 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4-offload | 0.744 / 21.57 | 1.007 / 21.53 | +| fp8-torchao | none | 0.689 / 51.63 | OOM | +| fp8-torchao | activation-offload | 0.975 / 37.77 | 2.058 / 37.73 | +| fp8-torchao | layer | 0.689 / 51.63 | OOM | +| fp8-torchao | interval2 | 0.684 / 51.63 | OOM | +| fp8-torchao | seg2-stride4 | 0.674 / 51.63 | OOM | +| fp8-torchao | seg2-stride4-offload | 0.965 / 37.77 | 2.053 / 37.73 | + +### LongCat Image + +Ejemplo: `longcat-image.peft-lora`. Resolucion de entrenamiento: 512 px cuadrada; resolucion de validacion: 1024x1024. Las filas usan `attention_mechanism=native-flash`. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.193 / 16.47 | 0.262 / 16.42 | +| bf16 | activation-offload | 0.544 / 12.73 | 0.543 / 12.69 | +| bf16 | layer | 0.327 / 12.38 | 0.370 / 12.34 | +| bf16 | interval2 | 0.257 / 14.38 | 0.313 / 14.34 | +| bf16 | seg2-stride4 | 0.263 / 14.36 | 0.316 / 14.31 | +| bf16 | seg2-stride4-offload | 0.446 / 13.42 | 0.492 / 13.38 | +| int8-sdnq-hadamard | none | 0.578 / 12.54 | 0.537 / 12.45 | +| int8-sdnq-hadamard | activation-offload | 1.184 / 7.43 | 1.185 / 7.39 | +| int8-sdnq-hadamard | layer | 0.901 / 7.19 | 0.911 / 7.09 | +| int8-sdnq-hadamard | interval2 | 0.718 / 9.73 | 0.695 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4 | 0.735 / 9.72 | 0.717 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.056 / 8.49 | 1.011 / 8.45 | +| fp8-torchao | none | 0.602 / 25.19 | 0.834 / 25.14 | +| fp8-torchao | activation-offload | 1.662 / 7.75 | 1.844 / 7.70 | +| fp8-torchao | layer | 0.984 / 7.57 | 1.080 / 7.53 | +| fp8-torchao | interval2 | 0.750 / 16.15 | 0.938 / 16.10 | +| fp8-torchao | seg2-stride4 | 0.760 / 16.13 | 0.961 / 16.09 | +| fp8-torchao | seg2-stride4-offload | 1.287 / 13.02 | 1.653 / 12.98 | + +### LongCat Video + +Ejemplo: `longcat-video.peft-lora+ramtorch`. Resolucion: 832x480, 81f. Las filas usan `attention_mechanism=native-flash`. + +LongCat Video usa muchas activations con esta forma. Full per-block checkpointing es la fila practica. Las filas partial checkpoint (`interval2`, `seg2-stride4`) no caben aqui, incluso con attention activation offload en la fila strided. Attention activation offload simple cabe para bf16 y SDNQ, pero es mucho mas lento que full checkpointing. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM (77.86 GiB) | OOM (43.30 GiB) | +| bf16 | activation-offload | 25.774 / 37.36 | 49.149 / 37.14 | +| bf16 | layer | 7.448 / 23.73 | 24.866 / 23.68 | +| bf16 | interval2 | OOM (76.41 GiB) | OOM (42.80 GiB) | +| bf16 | seg2-stride4 | OOM (76.42 GiB) | OOM (43.06 GiB) | +| bf16 | seg2-stride4-offload | OOM (76.72 GiB) | OOM (42.29 GiB) | +| int8-sdnq-hadamard | none | OOM (77.40 GiB) | OOM (43.59 GiB) | +| int8-sdnq-hadamard | activation-offload | 30.887 / 35.28 | 61.270 / 35.24 | +| int8-sdnq-hadamard | layer | 8.444 / 21.60 | 25.164 / 21.55 | +| int8-sdnq-hadamard | interval2 | OOM (76.01 GiB) | OOM (42.47 GiB) | +| int8-sdnq-hadamard | seg2-stride4 | OOM (77.01 GiB) | OOM (43.02 GiB) | +| int8-sdnq-hadamard | seg2-stride4-offload | OOM (76.42 GiB) | OOM (42.57 GiB) | +| fp8-torchao | none | OOM (75.87 GiB) | OOM (41.28 GiB) | +| fp8-torchao | activation-offload | 30.163 / 47.88 | OOM (40.11 GiB) | +| fp8-torchao | layer | 8.343 / 34.16 | 24.659 / 34.07 | +| fp8-torchao | interval2 | OOM (74.63 GiB) | OOM (40.85 GiB) | +| fp8-torchao | seg2-stride4 | OOM (75.37 GiB) | OOM (41.19 GiB) | +| fp8-torchao | seg2-stride4-offload | OOM (75.55 GiB) | OOM (40.94 GiB) | + +### LTXVideo 0.9.5 + +Example: `ltxvideo-0.9.5-t2v.peft-lora`. Resolution: 768x512, 49f. + +Los numeros son segundos warm por step / GiB pico. El promedio de la ejecucion completa incluye setup y compile, y queda en los artifacts del sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.274 / 8.77 | 0.275 / 8.72 | +| bf16 | layer | 0.462 / 4.21 | 0.459 / 4.16 | +| bf16 | interval2 | 0.449 / 4.30 | 0.433 / 4.25 | +| bf16 | seg2-stride4 | 0.359 / 6.40 | 0.357 / 6.35 | +| int8-sdnq-hadamard | none | 0.688 / 7.05 | 0.640 / 6.90 | +| int8-sdnq-hadamard | layer | 1.094 / 2.59 | 1.081 / 2.44 | +| int8-sdnq-hadamard | interval2 | 1.112 / 2.57 | 1.073 / 2.53 | +| int8-sdnq-hadamard | seg2-stride4 | 0.887 / 4.61 | 0.817 / 4.56 | +| fp8-torchao | none | 0.655 / 17.45 | 0.735 / 17.41 | +| fp8-torchao | layer | 1.226 / 2.94 | 1.206 / 2.89 | +| fp8-torchao | interval2 | 1.259 / 3.40 | 1.233 / 3.35 | +| fp8-torchao | seg2-stride4 | 0.933 / 9.96 | 0.901 / 9.91 | +| fp8wo-torchao | none | 0.328 / 10.06 | 0.325 / 10.01 | +| fp8wo-torchao | layer | 0.567 / 2.64 | 0.540 / 2.59 | +| fp8wo-torchao | interval2 | 0.555 / 2.84 | 0.531 / 2.79 | +| fp8wo-torchao | seg2-stride4 | 0.443 / 6.20 | 0.432 / 6.15 | + +Las filas de attention activation offload no estan soportadas para LTXVideo 0.9 en este sweep. + +### LTXVideo2 2.3 + +Example: `ltxvideo2-2.3-dev-720p-single-gpu.peft-lora+sdnq-hadamard`. Resolution: 1280x704, 49f. + +Note: LTXVideo2 2.3 should be read from the no-regional-compile rows in this sweep; regional compile raised memory pressure for this model. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.350 / 58.36 | OOM | +| bf16 | layer | 3.993 / 47.95 | OOM | +| bf16 | interval2 | 3.977 / 48.83 | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 5.554 / 75.64 | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 10.102 / 38.20 | 9.852 / 38.15 | +| int8-sdnq-hadamard | layer | 7.753 / 27.78 | 7.579 / 27.73 | +| int8-sdnq-hadamard | interval2 | 7.733 / 28.66 | 7.288 / 28.61 | +| int8-sdnq-hadamard | seg2-stride4 | 6.522 / 61.50 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 8.659 / 55.48 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | failed | 22.917 / 38.57 | +| fp8-torchao | layer | 8.580 / 30.27 | 10.381 / 30.22 | +| fp8-torchao | interval2 | 8.660 / 33.71 | 10.661 / 33.66 | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | failed | OOM | + +### Lumina2 + +Example: `lumina2.peft-lora`. Resolution: 512x512. + +Nota: Lumina2 ahora usa la ruta segmented whole-block. `interval2` checkpointa cada segmento de dos blocks; `seg2-stride4` checkpointa dos blocks, deja que los dos siguientes conserven activations y repite. Attention activation offload no fue parte de esta corrida de Lumina2. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.235 / 15.99 | 0.332 / 15.94 | +| bf16 | layer | 0.384 / 6.60 | 0.457 / 6.56 | +| bf16 | interval2 | 0.356 / 6.87 | 0.424 / 6.82 | +| bf16 | seg2-stride4 | 0.295 / 11.43 | 0.377 / 11.38 | +| int8-sdnq-hadamard | none | 0.584 / 13.59 | 0.541 / 13.55 | +| int8-sdnq-hadamard | layer | 0.899 / 4.21 | 0.827 / 4.16 | +| int8-sdnq-hadamard | interval2 | 0.865 / 4.48 | 0.835 / 4.43 | +| int8-sdnq-hadamard | seg2-stride4 | 0.719 / 9.03 | 0.707 / 8.99 | +| fp8-torchao | none | 0.598 / 28.03 | 0.763 / 27.98 | +| fp8-torchao | layer | 0.950 / 5.93 | 0.967 / 5.88 | +| fp8-torchao | interval2 | 0.938 / 6.70 | 0.974 / 6.66 | +| fp8-torchao | seg2-stride4 | 0.765 / 17.36 | 0.901 / 17.32 | +| fp8wo-torchao | none | 0.273 / 17.97 | 0.389 / 17.93 | +| fp8wo-torchao | layer | 0.452 / 4.99 | 0.525 / 4.94 | +| fp8wo-torchao | interval2 | 0.427 / 5.40 | 0.522 / 5.35 | +| fp8wo-torchao | seg2-stride4 | 0.360 / 11.68 | 0.466 / 11.64 | + +### MageFlow + +Example: `mageflow-image-24g.peft-lora`. Resolution: 1024x1024. + +Nota: la ruta de imagen variable de MageFlow a 1024px se beneficia sobre todo de attention activation offload y FP8 weight-only. Los modos de block checkpointing son validos, pero no redujeron el pico de residencia medido en este sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 7.735 / 36.74 | 7.658 / 36.69 | +| bf16 | activation-offload | 8.902 / 23.05 | 9.477 / 23.00 | +| bf16 | layer | 7.805 / 36.74 | 7.504 / 36.69 | +| bf16 | interval2 | 8.036 / 36.74 | 7.762 / 36.69 | +| bf16 | seg2-stride4 | 7.882 / 36.74 | 7.531 / 36.69 | +| bf16 | seg2-stride4-offload | 8.833 / 23.05 | 9.288 / 23.01 | +| int8-sdnq-hadamard | none | 81.016 / 37.38 | 94.991 / 37.34 | +| fp8wo-torchao | none | 5.454 / 36.86 | 5.772 / 36.82 | +| fp8wo-torchao | activation-offload | 6.295 / 23.18 | 6.738 / 23.14 | +| fp8wo-torchao | seg2-stride4 | 5.542 / 36.86 | 5.595 / 36.82 | + +### OmniGen + +Example: `omnigen.lycoris-lokr`. Resolution: 1024x1024. + +Nota: OmniGen usa prompts como token IDs en vez de embeddings de texto cacheados. Estas filas miden los caminos soportados sin checkpointing y con checkpointing torch de bloque completo; los controles interval, segmented-stride y attention-offload no estan implementados para esta familia en este barrido. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.425 / 14.20 | 0.293 / 14.24 | +| bf16 | layer | 0.597 / 10.13 | 0.389 / 10.09 | +| int8-sdnq-hadamard | none | 1.523 / 11.00 | 1.388 / 11.06 | +| int8-sdnq-hadamard | layer | 1.312 / 6.75 | 1.064 / 6.70 | +| fp8-torchao | none | 0.690 / 19.25 | 0.608 / 19.30 | +| fp8-torchao | layer | 1.069 / 7.09 | 0.824 / 7.04 | +| fp8wo-torchao | none | 0.454 / 17.71 | 0.377 / 17.73 | +| fp8wo-torchao | layer | 0.646 / 6.98 | 0.534 / 6.94 | + +### PixArt + +Example: `pixart.lycoris-lokr`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.700 / 41.73 | 1.734 / 41.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.433 / 5.63 | 2.346 / 5.58 | +| bf16 | interval2 | 2.440 / 6.01 | 2.348 / 5.96 | +| bf16 | seg2-stride4 | 2.092 / 23.90 | 2.072 / 23.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.905 / 47.58 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.738 / 6.07 | 2.937 / 6.03 | +| int8-sdnq-hadamard | interval2 | 2.734 / 6.03 | 2.943 / 5.99 | +| int8-sdnq-hadamard | seg2-stride4 | 2.336 / 26.85 | 2.596 / 26.81 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.254 / 8.06 | 4.646 / 8.01 | +| fp8-torchao | interval2 | 3.260 / 9.66 | 4.649 / 9.61 | +| fp8-torchao | seg2-stride4 | 2.827 / 63.27 | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Qwen Image + +Example: `qwen_image.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.202 / 41.04 | 3.377 / 40.99 | +| bf16 | interval2 | 1.201 / 41.04 | 3.382 / 40.99 | +| bf16 | seg2-stride4 | 1.205 / 41.03 | 3.385 / 40.99 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.675 / 63.48 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.640 / 24.09 | 3.928 / 24.05 | +| int8-sdnq-hadamard | interval2 | 2.722 / 24.09 | 3.918 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4 | 2.663 / 24.09 | 3.919 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.088 / 25.34 | 6.172 / 25.29 | +| fp8-torchao | interval2 | 3.095 / 25.34 | 6.173 / 25.29 | +| fp8-torchao | seg2-stride4 | 3.125 / 25.34 | 6.141 / 25.29 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Sana + +Example: `sana.lycoris-lokr`. Resolution: 1024x1024. + +Note: Sana has interval checkpointing; stride is not a separate segmented schedule for this family in the measured rows. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 23.71 | 0.597 / 23.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.529 / 23.72 | 0.596 / 23.66 | +| bf16 | interval2 | 0.530 / 23.72 | 0.598 / 23.66 | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.554 / 22.92 | 0.590 / 22.88 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.554 / 22.92 | 0.589 / 22.88 | +| int8-sdnq-hadamard | interval2 | 0.556 / 22.92 | 0.591 / 22.88 | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.633 / 33.67 | 0.753 / 33.62 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.633 / 33.67 | 0.755 / 33.62 | +| fp8-torchao | interval2 | 0.631 / 33.67 | 0.759 / 33.62 | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SanaVideo + +Example: `sanavideo-2b-480p.peft-lora`. Resolution: 832x480, 49f. + +Note: SanaVideo usa linear attention, asi que attention activation offload sigue unsupported. Segmented whole-block checkpointing esta soportado en la ruta standard. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.599 / 59.15 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.597 / 59.15 | OOM | +| bf16 | interval2 | not run | OOM | +| bf16 | seg2-stride4 | not run | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | interval2 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD 1.x + +Example: `sd1x-dreamshaper.peft-lora`. Resolution: 512x512. + +Nota: SD1x usa la ruta UNet de diffusers. El checkpointing por capa funciona, pero los controles interval y segmented stride no estan conectados para esta familia. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.181 / 2.87 | 0.176 / 2.83 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.305 / 1.98 | 0.292 / 1.93 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.446 / 3.07 | 0.431 / 3.04 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.701 / 1.79 | 0.666 / 1.74 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.410 / 4.23 | 0.401 / 4.18 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.785 / 1.96 | 0.756 / 1.91 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD3 + +Example: `sd3.peft-lora`. Resolution: 1024x1024. + +Nota: SD3 usa checkpointing segmentado contiguo real en la ruta transformer simple. Attention activation offload esta soportado; reduce mucho la VRAM, pero cuesta throughput. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 34.59 | 1.189 / 34.53 | +| bf16 | activation-offload | 1.335 / 9.68 | 2.850 / 9.63 | +| bf16 | layer | 0.721 / 7.67 | 1.607 / 7.62 | +| bf16 | interval2 | 0.723 / 8.93 | 1.606 / 8.88 | +| bf16 | seg2-stride4 | 0.620 / 20.14 | 1.398 / 20.09 | +| bf16 | seg2-stride4-offload | 1.229 / 12.21 | 2.639 / 12.16 | +| int8-sdnq-hadamard | none | 0.858 / 33.66 | 1.381 / 33.61 | +| int8-sdnq-hadamard | activation-offload | 1.845 / 8.90 | 3.297 / 8.85 | +| int8-sdnq-hadamard | layer | 1.265 / 6.58 | 1.857 / 6.53 | +| int8-sdnq-hadamard | interval2 | 1.264 / 7.30 | 1.858 / 7.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.049 / 19.02 | 1.625 / 18.98 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.527 / 12.42 | 3.023 / 12.37 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 4.707 / 10.27 | 9.814 / 10.23 | +| fp8-torchao | layer | 1.575 / 9.13 | 3.279 / 9.09 | +| fp8-torchao | interval2 | 1.576 / 12.62 | 3.282 / 12.58 | +| fp8-torchao | seg2-stride4 | 1.376 / 45.70 | OOM | +| fp8-torchao | seg2-stride4-offload | 4.184 / 26.17 | 8.714 / 26.12 | +| fp8wo-torchao | none | 0.567 / 35.21 | 1.252 / 35.17 | +| fp8wo-torchao | activation-offload | 1.392 / 8.71 | 2.921 / 8.67 | +| fp8wo-torchao | layer | 0.795 / 5.69 | 1.736 / 5.65 | +| fp8wo-torchao | interval2 | 0.797 / 7.08 | 1.734 / 7.03 | +| fp8wo-torchao | seg2-stride4 | 0.679 / 19.43 | 1.490 / 19.38 | +| fp8wo-torchao | seg2-stride4-offload | 1.257 / 12.00 | 2.676 / 11.96 | + +### SDXL + +Example: `sdxl.lycoris-lokr`. Resolution: 1024x1024. + +Note: SDXL has real layer checkpointing. Interval and stride rows are included as coverage data, not as segmented-support recommendations. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.606 / 13.03 | 0.585 / 12.98 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.080 / 6.53 | 1.029 / 6.48 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.741 / 13.72 | 1.643 / 13.68 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.820 / 4.64 | 2.647 / 4.59 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.608 / 26.22 | 1.582 / 26.16 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.939 / 5.10 | 2.890 / 5.04 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Stable Cascade + +Example: `cascade-stage-c.lycoris-lokr`. Resolution: 1024x1024. + +Nota: stage C usa la ruta prior en precision completa. Estas filas se ejecutaron con `mixed_precision=no` y `base_model_precision=no_change`; las filas de precision base cuantizada no son significativas para este modelo. Los modos interval y stride operan sobre la secuencia de micro-bloques Res/Timestep/Attention del UNet. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.884 / 51.52 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.179 / 22.68 | 2.135 / 22.61 | +| bf16 | interval2 | 1.032 / 36.99 | 1.871 / 36.92 | +| bf16 | seg2-stride4 | 1.032 / 37.20 | 1.870 / 37.13 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | unsupported | unsupported | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | unsupported | unsupported | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Wan 2.1 T2V 1.3B + +Example: `wan2.1-t2v-1.3b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 1.3B should be read from the no-regional-compile/RamTorch rows; regional compile was not a useful throughput setting in this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.407 / 71.90 | OOM | +| bf16 | activation-offload | 3.472 / 8.78 | 7.179 / 8.66 | +| bf16 | layer | 2.099 / 4.73 | 4.459 / 4.68 | +| bf16 | interval2 | 2.139 / 6.32 | 4.514 / 6.27 | +| bf16 | seg2-stride4 | 1.806 / 39.25 | 3.921 / 39.21 | +| bf16 | seg2-stride4-offload | 2.993 / 22.66 | 6.493 / 22.61 | +| int8-sdnq-hadamard | none | 1.850 / 71.91 | OOM | +| int8-sdnq-hadamard | activation-offload | 4.204 / 8.72 | 7.387 / 8.68 | +| int8-sdnq-hadamard | layer | 2.790 / 4.70 | 4.989 / 4.65 | +| int8-sdnq-hadamard | interval2 | 2.874 / 6.29 | 5.093 / 6.24 | +| int8-sdnq-hadamard | seg2-stride4 | 2.558 / 39.27 | 4.393 / 39.22 | +| int8-sdnq-hadamard | seg2-stride4-offload | 3.711 / 22.67 | 6.695 / 22.63 | +| fp8-torchao | none | 1.727 / 73.57 | OOM | +| fp8-torchao | activation-offload | 4.061 / 10.08 | 7.404 / 9.96 | +| fp8-torchao | layer | 2.607 / 5.98 | 4.888 / 5.93 | +| fp8-torchao | interval2 | 2.744 / 7.57 | 4.916 / 7.52 | +| fp8-torchao | seg2-stride4 | 2.246 / 40.55 | 4.245 / 40.50 | +| fp8-torchao | seg2-stride4-offload | 3.602 / 24.02 | 6.683 / 23.91 | + +### Wan 2.1 T2V 14B + +Example: `wan2.1-t2v-14b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 14B is mainly a fit test for activation savings. Status-only cells are still useful because they show which combinations reached the memory limit. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 13.144 / 36.80 | OOM | +| bf16 | layer | 7.162 / 16.28 | 21.770 / 16.23 | +| bf16 | interval2 | 7.172 / 19.62 | 21.777 / 19.58 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | failed | failed | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | failed | failed | +| fp8-torchao | seg2-stride4 | failed | failed | +| fp8-torchao | seg2-stride4-offload | failed | failed | + +### Wan S2V + +Example: `wan-s2v-14b-480p.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan S2V is included as coverage data for the video/audio path. Treat failed cells as implementation coverage gaps. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | failed | failed | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Z-Image Turbo + +Example: `z-image-turbo.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.243 / 21.25 | 0.316 / 21.21 | +| bf16 | activation-offload | 0.837 / 13.24 | 0.805 / 13.19 | +| bf16 | layer | 0.479 / 12.87 | 0.493 / 12.83 | +| bf16 | interval2 | 0.452 / 13.04 | 0.477 / 12.99 | +| bf16 | seg2-stride4 | 0.349 / 16.88 | 0.400 / 16.83 | +| bf16 | seg2-stride4-offload | 0.681 / 15.03 | 0.736 / 14.99 | +| int8-sdnq-hadamard | none | 0.645 / 15.60 | 0.615 / 15.55 | +| int8-sdnq-hadamard | activation-offload | 1.439 / 7.61 | 1.382 / 7.56 | +| int8-sdnq-hadamard | layer | 1.046 / 7.25 | 1.021 / 7.20 | +| int8-sdnq-hadamard | interval2 | 1.074 / 7.41 | 0.996 / 7.36 | +| int8-sdnq-hadamard | seg2-stride4 | 0.867 / 11.25 | 0.841 / 11.20 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.202 / 9.39 | 1.162 / 9.35 | +| fp8-torchao | none | 1.232 / 37.50 | 1.476 / 37.46 | +| fp8-torchao | activation-offload | 3.623 / 7.97 | 3.564 / 7.93 | +| fp8-torchao | layer | 2.319 / 7.93 | 2.344 / 7.88 | +| fp8-torchao | interval2 | 2.336 / 8.80 | 2.309 / 8.75 | +| fp8-torchao | seg2-stride4 | 1.843 / 22.55 | 1.930 / 22.50 | +| fp8-torchao | seg2-stride4-offload | 2.947 / 15.57 | 3.243 / 15.52 | + +### ZLab I1 + +Example: `zlab-i1.peft-lora`. Resolution: 1024x1024. + +Nota: ZLab I1 lleva sus skip tensors tipo U-Net dentro del estado segmented checkpoint. Attention activation offload no esta conectado para esta familia. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.462 / 22.21 | 0.865 / 22.16 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.693 / 7.79 | 1.148 / 7.75 | +| bf16 | interval2 | 0.676 / 8.30 | 1.152 / 8.25 | +| bf16 | seg2-stride4 | 0.567 / 14.97 | 1.014 / 14.92 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.861 / 19.21 | 0.926 / 19.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.385 / 4.78 | 1.265 / 4.74 | +| int8-sdnq-hadamard | interval2 | 1.298 / 5.30 | 1.277 / 5.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.073 / 11.98 | 1.098 / 11.93 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8wo-torchao | none | 0.504 / 25.08 | 0.930 / 25.02 | +| fp8wo-torchao | activation-offload | unsupported | unsupported | +| fp8wo-torchao | layer | 0.772 / 5.12 | 1.280 / 5.07 | +| fp8wo-torchao | interval2 | 0.759 / 5.84 | 1.290 / 5.79 | +| fp8wo-torchao | seg2-stride4 | 0.633 / 15.12 | 1.115 / 15.08 | +| fp8wo-torchao | seg2-stride4-offload | unsupported | unsupported | + + diff --git a/documentation/experimental/SEGMENTED_CHECKPOINTING.hi.md b/documentation/experimental/SEGMENTED_CHECKPOINTING.hi.md new file mode 100644 index 000000000..359e1f1d4 --- /dev/null +++ b/documentation/experimental/SEGMENTED_CHECKPOINTING.hi.md @@ -0,0 +1,1007 @@ +# Segmented Checkpointing + +Segmented checkpointing हर block को checkpoint करने और कोई checkpointing न करने के बीच का mode है। + +यह PyTorch activation checkpointing backend इस्तेमाल करता है। SimpleTuner contiguous transformer blocks के group को एक checkpoint call में चलाता है, फिर returned hidden state अगले group को देता है। Wider groups backward में कम recompute करते हैं, लेकिन ज्यादा activations alive रखते हैं। + +CPU offload और FFN-only checkpointing के लिए [Unsloth-style checkpointing](UNSLOTH_CHECKPOINTING.hi.md#controls) देखें। छोटी rule of thumb [Decision Rule](UNSLOTH_CHECKPOINTING.hi.md#decision-rule) में है। + +## Controls + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2 +} +``` + +Supported whole-block paths पर `gradient_checkpointing_interval` segment width है। `2` का मतलब blocks `0-1`, `2-3`, `4-5` वगैरह checkpoint होंगे। + +ज्यादा fine VRAM control के लिए stride जोड़ें: + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2, + "gradient_checkpointing_segment_stride": 4 +} +``` + +यह blocks `0-1` checkpoint करता है, `2-3` normal चलाता है, `4-5` checkpoint करता है, `6-7` normal चलाता है, और repeat करता है। stride कम से कम interval जितना होना चाहिए; overlapping schedules valid नहीं हैं। + +Supported segmented whole-block paths: Flux.1, Flux.2, HunyuanVideo, Krea 2, LongCat Image, LongCat Video, LTXVideo 0.9, LTXVideo2, Lumina2, MageFlow, PixArt, SD3, SanaVideo, Z-Image, ZLab I1, और Wan। + +Stable Cascade stage C भी interval और stride support करता है, लेकिन schedule transformer whole-block groups की जगह UNet के Res/Timestep/Attention micro-block sequence पर apply होता है। + +कुछ model families पुराने semantics use करते हैं: + +| Family | `gradient_checkpointing_interval` | `gradient_checkpointing_segment_stride` | +| --- | --- | --- | +| Sana | हर N-th block checkpoint | Ignore | +| Stable Cascade stage C | interval से UNet micro-blocks checkpoint | stride checkpointed और non-checkpointed UNet micro-block windows alternate करता है | +| SD1x, SDXL | segmented whole-block support नहीं | Ignore | + +जहां stride ignore होता है, वहां stride rows compare न करें। अगर numbers identical दिखें, तो usually option apply नहीं हुआ। + +## कब इस्तेमाल करें + +जब normal per-block checkpointing fit हो जाए लेकिन step time बहुत महंगा हो, तब इसे इस्तेमाल करें। `2` से शुरू करें। अगर VRAM allow करे, बहुत deep models पर `2` with stride `4` try करें। + +जब peak मुख्य रूप से trainable weights, optimizer state, validation, VAE caching, block swapping, या routing से हो, तो इससे मदद की उम्मीद न करें। जब model feature को per-block control चाहिए, SimpleTuner safer per-block path पर वापस जाता है। + +`dynamo_use_regional_compilation` universal win नहीं है। कुछ image models पर यह helpful या neutral था, लेकिन नीचे Wan/RamTorch और LTXVideo2 profiles में खराब fit था। + +## Benchmarks + +Real SimpleTuner examples से single-GPU H100 और L40S pods पर measure किया गया। validation और checkpoint saves disabled थे, cache preparation exclude था, और post-warmup timing available होने पर train loop के first-step compile/setup को भी exclude किया गया। + +हर measured cell `post-warmup sec/step / peak VRAM GiB` है। status-only cells का मतलब है: `OOM` GPU memory खत्म होना, `failed` measured training steps तक न पहुंचना, `unsupported` उस family में option wired न होना, और `not run` उस combination का sweep में न होना। + +पहले एक ही family के अंदर modes compare करें। Cross-family comparison rough है क्योंकि resolution, frame count, attention backend, model depth, trainable adapter type, और dataset shape अलग हो सकते हैं। + +नीचे की matrix इस sweep की source of truth है। Model-specific notes caveats बताते हैं जब कोई row recommendation के बजाय coverage data हो। + + + +### Family Sweep Results + +### ACE Step 1.5 + +Example: `ace_step-v1-5.peft-lora`. Resolution: 512. + +Note: This sweep did not produce a usable ACE Step throughput row. The status-only entries below should be treated as coverage gaps, not as a recommendation. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | OOM | OOM | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | OOM | OOM | +| int8-sdnq-hadamard | interval2 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Anima पर AnyFlow Distillation + +Example: `anima-anyflow.peft-lora`। Resolution: 1024x1024। यह row Anima के साथ AnyFlow distillation मापती है, plain Anima LoRA example नहीं। plain 1024x1024 Anima image training के लिए `anima.peft-lora` use करें। + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.022 / 17.26 | 0.719 / 17.21 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.232 / 5.60 | 0.903 / 5.56 | +| bf16 | interval2 | 1.252 / 5.60 | 0.897 / 5.56 | +| bf16 | seg2-stride4 | 1.244 / 5.60 | 0.898 / 5.56 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 5.417 / 18.61 | 4.974 / 18.57 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.562 / 4.36 | 4.019 / 4.31 | +| int8-sdnq-hadamard | interval2 | 3.723 / 4.36 | 3.196 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4 | 3.658 / 4.36 | 3.140 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.242 / 45.71 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.810 / 5.51 | 2.576 / 5.46 | +| fp8-torchao | interval2 | 2.846 / 5.51 | 2.581 / 5.46 | +| fp8-torchao | seg2-stride4 | 2.766 / 5.51 | 2.567 / 5.46 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### AuraFlow + +Example: `auraflow.peft-lora`. Resolution: 1024x1024. + +Note: AuraFlow SDNQ और TorchAO quantization support करता है। Quantized `none` rows नीचे हैं; quantized checkpoint rows में numbers डालने से पहले fresh full-length benchmark coverage चाहिए। + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.180 / 19.19 | 0.233 / 19.12 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.824 / 13.37 | 0.833 / 13.32 | +| bf16 | interval2 | 1.764 / 16.21 | 0.877 / 16.14 | +| bf16 | seg2-stride4 | 1.771 / 16.21 | 0.887 / 16.14 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.642 / 12.88 | 0.610 / 12.87 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | not run | not run | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.621 / 24.53 | 0.757 / 24.44 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | not run | not run | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Boogu Image + +Example: `boogu-image-v0.1.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.694 / 59.14 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.907 / 23.44 | 2.641 / 23.39 | +| bf16 | interval2 | 0.912 / 23.44 | 2.649 / 23.39 | +| bf16 | seg2-stride4 | 0.911 / 23.44 | 2.648 / 23.39 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.488 / 53.24 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.878 / 15.20 | 3.309 / 15.15 | +| int8-sdnq-hadamard | interval2 | 1.630 / 34.12 | 2.577 / 34.07 | +| int8-sdnq-hadamard | seg2-stride4 | 1.656 / 34.11 | 2.574 / 34.06 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.713 / 18.48 | 3.778 / 18.44 | +| fp8-torchao | interval2 | 1.731 / 18.48 | 3.777 / 18.44 | +| fp8-torchao | seg2-stride4 | 1.721 / 18.48 | 3.779 / 18.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Chroma + +Example: `chroma.peft-lora`. Resolution: 1024x1024. + +Note: Checkpointed Chroma rows use `attention_mechanism=native-efficient`, which was the stable attention path for this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.454 / 26.18 | 0.559 / 26.13 | +| bf16 | activation-offload | 4.873 / 18.74 | 4.793 / 18.69 | +| bf16 | layer | 1.276 / 17.67 | 1.430 / 17.63 | +| bf16 | interval2 | 1.204 / 21.80 | 1.349 / 21.75 | +| bf16 | seg2-stride4 | 1.200 / 21.78 | 1.382 / 21.74 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.083 / 21.42 | 1.061 / 21.37 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.714 / 10.41 | 1.646 / 10.36 | +| int8-sdnq-hadamard | interval2 | 1.443 / 15.72 | 1.391 / 15.68 | +| int8-sdnq-hadamard | seg2-stride4 | 1.428 / 15.71 | 1.323 / 15.67 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.122 / 45.44 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.821 / 10.64 | 2.104 / 10.60 | +| fp8-torchao | interval2 | 1.447 / 27.49 | 1.871 / 27.44 | +| fp8-torchao | seg2-stride4 | 1.431 / 27.49 | 1.877 / 27.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 2 Image + +Example: `cosmos2image.lycoris-lokr`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.336 / 8.05 | 0.316 / 8.00 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.567 / 4.09 | 0.559 / 4.04 | +| bf16 | interval2 | 0.595 / 4.09 | 0.544 / 4.04 | +| bf16 | seg2-stride4 | 0.598 / 4.09 | 0.546 / 4.04 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.831 / 6.56 | 0.783 / 6.56 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.315 / 2.44 | 1.288 / 2.39 | +| int8-sdnq-hadamard | interval2 | 1.346 / 2.44 | 1.321 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4 | 1.413 / 2.44 | 1.274 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.850 / 13.30 | 0.840 / 13.25 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.692 / 2.70 | 1.560 / 2.65 | +| fp8-torchao | interval2 | 1.626 / 2.70 | 1.544 / 2.65 | +| fp8-torchao | seg2-stride4 | 1.607 / 2.70 | 1.555 / 2.65 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 3 + +Example: `cosmos3-edge-image-24g.lycoris-lokr`. Resolution: 1024 px. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 2.747 / 8.90 | 2.904 / 8.86 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.602 / 8.90 | 2.606 / 8.86 | +| bf16 | interval2 | 2.658 / 8.90 | 2.567 / 8.86 | +| bf16 | seg2-stride4 | 2.628 / 8.90 | 2.899 / 8.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 17.112 / 6.21 | 17.965 / 6.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 3.238 / 6.21 | 2.891 / 6.16 | +| int8-sdnq-hadamard | interval2 | 3.253 / 6.21 | 2.916 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4 | 3.254 / 6.21 | 2.923 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.849 / 17.84 | 1.559 / 17.80 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.869 / 17.84 | 1.599 / 17.79 | +| fp8-torchao | interval2 | 1.855 / 17.84 | 1.600 / 17.80 | +| fp8-torchao | seg2-stride4 | 1.859 / 17.84 | 1.523 / 17.80 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### ERNIE 4.5 Image + +Example: `ernie.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.711 / 15.61 | 1.282 / 15.56 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.110 / 4.94 | 1.840 / 4.90 | +| bf16 | interval2 | 0.876 / 10.16 | 1.536 / 10.12 | +| bf16 | seg2-stride4 | 0.874 / 10.16 | 1.532 / 10.12 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 2.782 / 13.75 | 2.722 / 13.70 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.262 / 2.98 | 4.393 / 2.94 | +| int8-sdnq-hadamard | interval2 | 3.547 / 8.26 | 3.380 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4 | 3.366 / 8.26 | 3.457 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.432 / 14.03 | 2.303 / 13.98 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.920 / 3.26 | 4.008 / 3.22 | +| fp8-torchao | interval2 | 3.316 / 8.54 | 2.973 / 8.49 | +| fp8-torchao | seg2-stride4 | 2.994 / 8.54 | 3.046 / 8.49 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HeartMula + +Example: `heartmula.peft-lora`. Audio-token training. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HiDream + +Example: `hidream.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.515 / 44.58 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.928 / 33.57 | 1.043 / 33.52 | +| bf16 | interval2 | 0.971 / 33.57 | 1.002 / 33.52 | +| bf16 | seg2-stride4 | 0.952 / 33.57 | 1.010 / 33.52 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | not measured | not measured | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.497 / 17.99 | 2.058 / 17.93 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | not measured | not measured | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 4.920 / 18.10 | 4.338 / 18.05 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +SDNQ Hadamard numbers use `sdnq_compile_mode=eager`. The compiled SDNQ path quantizes HiDream, but this sweep spent the first training step in Inductor dequantizer compilation, so it is not listed as a throughput row. + +### HunyuanVideo + +Example: `hunyuanvideo-1.5-t2v.peft-lora`. Training shape: 480 pixel-area video buckets, 48 frames, batch 2. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | not run | not run | +| bf16 | layer | 7.682 / 26.35 | 22.816 / 26.30 | +| bf16 | interval2 | 7.398 / 26.11 | 22.772 / 26.06 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 10.679 / 58.37 | not run | +| int8-sdnq-hadamard | none | not run | not run | +| int8-sdnq-hadamard | activation-offload | not run | not run | +| int8-sdnq-hadamard | layer | 11.765 / 25.96 | 34.464 / 25.92 | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | not run | not run | +| fp8-torchao | none | not run | not run | +| fp8-torchao | activation-offload | not run | not run | +| fp8-torchao | layer | 10.516 / 33.55 | 32.003 / 33.53 | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | not run | not run | + +HunyuanVideo इस training shape पर activation-heavy है। Per-block और interval-2 checkpointing साफ चलते हैं; checkpointing बंद करने पर 80 GB H100 भी पर्याप्त नहीं है। `seg2-stride4` इस sweep में attention activation offload के साथ ही चला, इसलिए वह speed recommendation नहीं बल्कि memory fallback है। SDNQ Hadamard काम करता है, लेकिन variable conditioning shapes measurement window में भी dynamic kernel compilation trigger करते हैं। + +### Ideogram 4.0 + +Example: `ideogram-fp8.peft-lora`. Resolution: 1024x1024. `fp8` flavour Ideogram 4 के native weight-only fp8 checkpoint का उपयोग करता है (`base_model_precision=no_change`). + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| fp8-native | none | failed | OOM | +| fp8-native | activation-offload | unsupported | unsupported | +| fp8-native | layer | 1.033 / 12.57 | 3.101 / 11.82 | +| fp8-native | interval2 | 1.030 / 12.33 | 3.098 / 11.82 | +| fp8-native | seg2-stride4 | 1.031 / 12.33 | 3.088 / 11.82 | +| fp8-native | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.702 / 61.78 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.033 / 12.33 | 3.030 / 11.82 | +| int8-sdnq-hadamard | interval2 | 1.032 / 12.33 | 3.034 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4 | 1.028 / 12.33 | 3.035 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | + +### Kandinsky 5 Image + +Example: `kandinsky5-image-6b-t2i.lycoris-lokr`. Resolution: 1024x1024. + +Note: batch 3 at 1024x1024 needs full checkpointing on both cards. SDNQ with Hadamard is the best low-VRAM row; H100 can also use partial checkpointing with SDNQ, but only near the top of the card. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.956 / 25.52 | 10.189 / 25.42 | +| bf16 | layer | 6.590 / 25.58 | 9.458 / 25.55 | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 7.186 / 20.00 | 10.141 / 19.95 | +| int8-sdnq-hadamard | layer | 6.830 / 20.12 | 9.362 / 20.08 | +| int8-sdnq-hadamard | interval2 | 5.746 / 75.67 | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 6.057 / 71.46 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 8.319 / 24.29 | 14.716 / 24.24 | +| fp8-torchao | layer | 7.976 / 24.40 | 13.949 / 24.35 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kandinsky 5 Video + +Example: `kandinsky5-video-2b-t2v.peft-lora`. Resolution: 768x512, 81f. + +Kandinsky 5 video is activation-heavy at this frame count. Full block checkpointing is the practical baseline on both cards. On H100, `interval2` and `seg2-stride4` are faster when they fit; on L40S, SDNQ `interval2` is the only partial-checkpoint row here that fits without attention activation offload. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 2.580 / 9.49 | 7.136 / 9.45 | +| bf16 | layer | 2.267 / 9.83 | 6.379 / 9.79 | +| bf16 | interval2 | 1.967 / 44.57 | OOM | +| bf16 | seg2-stride4 | 1.971 / 46.62 | OOM | +| bf16 | seg2-stride4-offload | 2.275 / 37.99 | 6.249 / 37.94 | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 2.844 / 8.07 | 7.234 / 8.02 | +| int8-sdnq-hadamard | layer | 2.460 / 8.40 | 6.509 / 8.35 | +| int8-sdnq-hadamard | interval2 | 2.126 / 43.12 | 5.641 / 43.08 | +| int8-sdnq-hadamard | seg2-stride4 | 2.125 / 45.19 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 2.451 / 36.56 | 6.322 / 36.51 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 3.867 / 15.11 | 11.501 / 15.06 | +| fp8-torchao | layer | 3.579 / 15.28 | 10.822 / 15.24 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kolors + +Example: `kolors.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.635 / 7.22 | 0.628 / 7.17 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.118 / 5.36 | 1.065 / 5.31 | +| bf16 | interval2 | 1.105 / 5.36 | 1.068 / 5.31 | +| bf16 | seg2-stride4 | 1.110 / 5.36 | 1.072 / 5.31 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.860 / 5.68 | 1.726 / 5.63 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.967 / 3.38 | 2.803 / 3.33 | +| int8-sdnq-hadamard | interval2 | 2.770 / 3.32 | 2.655 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4 | 2.805 / 3.32 | 2.742 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.629 / 8.79 | 1.637 / 8.75 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.044 / 3.50 | 3.013 / 3.45 | +| fp8-torchao | interval2 | 3.040 / 3.50 | 3.019 / 3.45 | +| fp8-torchao | seg2-stride4 | 2.988 / 3.50 | 2.935 / 3.45 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Krea 2 + +Example: `krea2.peft-lora`. Training resolution: 512 px square crop. Example validation setting: 1024x1024; benchmark में validation disabled था. + +Main table regional compilation use करती है। यह Krea2 step speed के लिए अच्छा है, लेकिन VRAM comparison साफ नहीं रहता: compiled graph/workspace कई checkpoint modes में peak को uncheckpointed peak के करीब रखता है। regional compilation बंद करके bf16 control run ने दिखाया कि checkpointing wired है और memory/speed shape expected है: + +यहां `activation-offload` row का मतलब full-block checkpointing plus attention activation offload है। सिर्फ full-block `layer` checkpointing के मुकाबले, इस Krea2 shape में attention offload ने peak VRAM घटाया नहीं; इसने mostly CPU transfer overhead जोड़ा। + +| Mode | H100 no-compile | L40S no-compile | +| --- | ---: | ---: | +| none | 0.272 / 40.09 | 0.661 / 40.01 | +| layer | 0.371 / 30.06 | 0.919 / 30.01 | +| seg2-stride4 | 0.317 / 34.75 | 0.788 / 34.70 | +| activation-offload | 0.657 / 30.50 | 1.341 / 30.30 | + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.275 / 40.06 | 0.662 / 40.01 | +| bf16 | activation-offload | 0.416 / 34.62 | 0.822 / 34.57 | +| bf16 | layer | 0.268 / 40.06 | 0.665 / 40.01 | +| bf16 | interval2 | 0.274 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4 | 0.279 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4-offload | 0.404 / 34.62 | 0.819 / 34.57 | +| int8-sdnq-hadamard | none | 0.462 / 27.01 | 0.807 / 26.96 | +| int8-sdnq-hadamard | activation-offload | 0.773 / 21.57 | 1.012 / 21.53 | +| int8-sdnq-hadamard | layer | 0.473 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | interval2 | 0.474 / 27.01 | 0.804 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4 | 0.472 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4-offload | 0.744 / 21.57 | 1.007 / 21.53 | +| fp8-torchao | none | 0.689 / 51.63 | OOM | +| fp8-torchao | activation-offload | 0.975 / 37.77 | 2.058 / 37.73 | +| fp8-torchao | layer | 0.689 / 51.63 | OOM | +| fp8-torchao | interval2 | 0.684 / 51.63 | OOM | +| fp8-torchao | seg2-stride4 | 0.674 / 51.63 | OOM | +| fp8-torchao | seg2-stride4-offload | 0.965 / 37.77 | 2.053 / 37.73 | + +### LongCat Image + +Example: `longcat-image.peft-lora`. Training resolution: 512 px square; validation resolution: 1024x1024. Rows use `attention_mechanism=native-flash`. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.193 / 16.47 | 0.262 / 16.42 | +| bf16 | activation-offload | 0.544 / 12.73 | 0.543 / 12.69 | +| bf16 | layer | 0.327 / 12.38 | 0.370 / 12.34 | +| bf16 | interval2 | 0.257 / 14.38 | 0.313 / 14.34 | +| bf16 | seg2-stride4 | 0.263 / 14.36 | 0.316 / 14.31 | +| bf16 | seg2-stride4-offload | 0.446 / 13.42 | 0.492 / 13.38 | +| int8-sdnq-hadamard | none | 0.578 / 12.54 | 0.537 / 12.45 | +| int8-sdnq-hadamard | activation-offload | 1.184 / 7.43 | 1.185 / 7.39 | +| int8-sdnq-hadamard | layer | 0.901 / 7.19 | 0.911 / 7.09 | +| int8-sdnq-hadamard | interval2 | 0.718 / 9.73 | 0.695 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4 | 0.735 / 9.72 | 0.717 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.056 / 8.49 | 1.011 / 8.45 | +| fp8-torchao | none | 0.602 / 25.19 | 0.834 / 25.14 | +| fp8-torchao | activation-offload | 1.662 / 7.75 | 1.844 / 7.70 | +| fp8-torchao | layer | 0.984 / 7.57 | 1.080 / 7.53 | +| fp8-torchao | interval2 | 0.750 / 16.15 | 0.938 / 16.10 | +| fp8-torchao | seg2-stride4 | 0.760 / 16.13 | 0.961 / 16.09 | +| fp8-torchao | seg2-stride4-offload | 1.287 / 13.02 | 1.653 / 12.98 | + +### LongCat Video + +Example: `longcat-video.peft-lora+ramtorch`. Resolution: 832x480, 81f. Rows `attention_mechanism=native-flash` use करते हैं। + +LongCat Video इस shape पर activation-heavy है। Full per-block checkpointing practical row है। Partial checkpoint rows (`interval2`, `seg2-stride4`) यहाँ fit नहीं होते, strided row पर attention activation offload enabled होने पर भी नहीं। Plain attention activation offload bf16 और SDNQ में fit होता है, लेकिन full checkpointing से काफी धीमा है। + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM (77.86 GiB) | OOM (43.30 GiB) | +| bf16 | activation-offload | 25.774 / 37.36 | 49.149 / 37.14 | +| bf16 | layer | 7.448 / 23.73 | 24.866 / 23.68 | +| bf16 | interval2 | OOM (76.41 GiB) | OOM (42.80 GiB) | +| bf16 | seg2-stride4 | OOM (76.42 GiB) | OOM (43.06 GiB) | +| bf16 | seg2-stride4-offload | OOM (76.72 GiB) | OOM (42.29 GiB) | +| int8-sdnq-hadamard | none | OOM (77.40 GiB) | OOM (43.59 GiB) | +| int8-sdnq-hadamard | activation-offload | 30.887 / 35.28 | 61.270 / 35.24 | +| int8-sdnq-hadamard | layer | 8.444 / 21.60 | 25.164 / 21.55 | +| int8-sdnq-hadamard | interval2 | OOM (76.01 GiB) | OOM (42.47 GiB) | +| int8-sdnq-hadamard | seg2-stride4 | OOM (77.01 GiB) | OOM (43.02 GiB) | +| int8-sdnq-hadamard | seg2-stride4-offload | OOM (76.42 GiB) | OOM (42.57 GiB) | +| fp8-torchao | none | OOM (75.87 GiB) | OOM (41.28 GiB) | +| fp8-torchao | activation-offload | 30.163 / 47.88 | OOM (40.11 GiB) | +| fp8-torchao | layer | 8.343 / 34.16 | 24.659 / 34.07 | +| fp8-torchao | interval2 | OOM (74.63 GiB) | OOM (40.85 GiB) | +| fp8-torchao | seg2-stride4 | OOM (75.37 GiB) | OOM (41.19 GiB) | +| fp8-torchao | seg2-stride4-offload | OOM (75.55 GiB) | OOM (40.94 GiB) | + +### LTXVideo 0.9.5 + +Example: `ltxvideo-0.9.5-t2v.peft-lora`. Resolution: 768x512, 49f. + +Numbers warm seconds/step / peak GiB हैं। full-run average setup और compile overhead include करता है, इसलिए वह sweep artifacts में रखा गया है। + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.274 / 8.77 | 0.275 / 8.72 | +| bf16 | layer | 0.462 / 4.21 | 0.459 / 4.16 | +| bf16 | interval2 | 0.449 / 4.30 | 0.433 / 4.25 | +| bf16 | seg2-stride4 | 0.359 / 6.40 | 0.357 / 6.35 | +| int8-sdnq-hadamard | none | 0.688 / 7.05 | 0.640 / 6.90 | +| int8-sdnq-hadamard | layer | 1.094 / 2.59 | 1.081 / 2.44 | +| int8-sdnq-hadamard | interval2 | 1.112 / 2.57 | 1.073 / 2.53 | +| int8-sdnq-hadamard | seg2-stride4 | 0.887 / 4.61 | 0.817 / 4.56 | +| fp8-torchao | none | 0.655 / 17.45 | 0.735 / 17.41 | +| fp8-torchao | layer | 1.226 / 2.94 | 1.206 / 2.89 | +| fp8-torchao | interval2 | 1.259 / 3.40 | 1.233 / 3.35 | +| fp8-torchao | seg2-stride4 | 0.933 / 9.96 | 0.901 / 9.91 | +| fp8wo-torchao | none | 0.328 / 10.06 | 0.325 / 10.01 | +| fp8wo-torchao | layer | 0.567 / 2.64 | 0.540 / 2.59 | +| fp8wo-torchao | interval2 | 0.555 / 2.84 | 0.531 / 2.79 | +| fp8wo-torchao | seg2-stride4 | 0.443 / 6.20 | 0.432 / 6.15 | + +इस sweep में LTXVideo 0.9 के लिए attention activation offload rows supported नहीं हैं। + +### LTXVideo2 2.3 + +Example: `ltxvideo2-2.3-dev-720p-single-gpu.peft-lora+sdnq-hadamard`. Resolution: 1280x704, 49f. + +Note: LTXVideo2 2.3 should be read from the no-regional-compile rows in this sweep; regional compile raised memory pressure for this model. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.350 / 58.36 | OOM | +| bf16 | layer | 3.993 / 47.95 | OOM | +| bf16 | interval2 | 3.977 / 48.83 | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 5.554 / 75.64 | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 10.102 / 38.20 | 9.852 / 38.15 | +| int8-sdnq-hadamard | layer | 7.753 / 27.78 | 7.579 / 27.73 | +| int8-sdnq-hadamard | interval2 | 7.733 / 28.66 | 7.288 / 28.61 | +| int8-sdnq-hadamard | seg2-stride4 | 6.522 / 61.50 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 8.659 / 55.48 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | failed | 22.917 / 38.57 | +| fp8-torchao | layer | 8.580 / 30.27 | 10.381 / 30.22 | +| fp8-torchao | interval2 | 8.660 / 33.71 | 10.661 / 33.66 | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | failed | OOM | + +### Lumina2 + +Example: `lumina2.peft-lora`. Resolution: 512x512. + +Note: Lumina2 अब segmented whole-block path इस्तेमाल करता है। `interval2` हर two-block segment को checkpoint करता है; `seg2-stride4` दो blocks checkpoint करता है, अगले दो blocks को activations रखने देता है, फिर repeat करता है। Attention activation offload इस Lumina2 run में शामिल नहीं था। + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.235 / 15.99 | 0.332 / 15.94 | +| bf16 | layer | 0.384 / 6.60 | 0.457 / 6.56 | +| bf16 | interval2 | 0.356 / 6.87 | 0.424 / 6.82 | +| bf16 | seg2-stride4 | 0.295 / 11.43 | 0.377 / 11.38 | +| int8-sdnq-hadamard | none | 0.584 / 13.59 | 0.541 / 13.55 | +| int8-sdnq-hadamard | layer | 0.899 / 4.21 | 0.827 / 4.16 | +| int8-sdnq-hadamard | interval2 | 0.865 / 4.48 | 0.835 / 4.43 | +| int8-sdnq-hadamard | seg2-stride4 | 0.719 / 9.03 | 0.707 / 8.99 | +| fp8-torchao | none | 0.598 / 28.03 | 0.763 / 27.98 | +| fp8-torchao | layer | 0.950 / 5.93 | 0.967 / 5.88 | +| fp8-torchao | interval2 | 0.938 / 6.70 | 0.974 / 6.66 | +| fp8-torchao | seg2-stride4 | 0.765 / 17.36 | 0.901 / 17.32 | +| fp8wo-torchao | none | 0.273 / 17.97 | 0.389 / 17.93 | +| fp8wo-torchao | layer | 0.452 / 4.99 | 0.525 / 4.94 | +| fp8wo-torchao | interval2 | 0.427 / 5.40 | 0.522 / 5.35 | +| fp8wo-torchao | seg2-stride4 | 0.360 / 11.68 | 0.466 / 11.64 | + +### MageFlow + +Example: `mageflow-image-24g.peft-lora`. Resolution: 1024x1024. + +Note: MageFlow के 1024px variable-shape image path में सबसे ज्यादा फायदा attention activation offload और weight-only FP8 से मिला। Block checkpointing modes valid हैं, लेकिन इस sweep में measured peak residency कम नहीं हुई। + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 7.735 / 36.74 | 7.658 / 36.69 | +| bf16 | activation-offload | 8.902 / 23.05 | 9.477 / 23.00 | +| bf16 | layer | 7.805 / 36.74 | 7.504 / 36.69 | +| bf16 | interval2 | 8.036 / 36.74 | 7.762 / 36.69 | +| bf16 | seg2-stride4 | 7.882 / 36.74 | 7.531 / 36.69 | +| bf16 | seg2-stride4-offload | 8.833 / 23.05 | 9.288 / 23.01 | +| int8-sdnq-hadamard | none | 81.016 / 37.38 | 94.991 / 37.34 | +| fp8wo-torchao | none | 5.454 / 36.86 | 5.772 / 36.82 | +| fp8wo-torchao | activation-offload | 6.295 / 23.18 | 6.738 / 23.14 | +| fp8wo-torchao | seg2-stride4 | 5.542 / 36.86 | 5.595 / 36.82 | + +### OmniGen + +Example: `omnigen.lycoris-lokr`. Resolution: 1024x1024. + +Note: OmniGen cached text embeddings की जगह token-ID prompts use करता है। ये rows supported no-checkpointing और full-block torch checkpointing paths को measure करती हैं; interval, segmented-stride, और attention-offload controls इस family के लिए इस sweep में implemented नहीं हैं। + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.425 / 14.20 | 0.293 / 14.24 | +| bf16 | layer | 0.597 / 10.13 | 0.389 / 10.09 | +| int8-sdnq-hadamard | none | 1.523 / 11.00 | 1.388 / 11.06 | +| int8-sdnq-hadamard | layer | 1.312 / 6.75 | 1.064 / 6.70 | +| fp8-torchao | none | 0.690 / 19.25 | 0.608 / 19.30 | +| fp8-torchao | layer | 1.069 / 7.09 | 0.824 / 7.04 | +| fp8wo-torchao | none | 0.454 / 17.71 | 0.377 / 17.73 | +| fp8wo-torchao | layer | 0.646 / 6.98 | 0.534 / 6.94 | + +### PixArt + +Example: `pixart.lycoris-lokr`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.700 / 41.73 | 1.734 / 41.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.433 / 5.63 | 2.346 / 5.58 | +| bf16 | interval2 | 2.440 / 6.01 | 2.348 / 5.96 | +| bf16 | seg2-stride4 | 2.092 / 23.90 | 2.072 / 23.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.905 / 47.58 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.738 / 6.07 | 2.937 / 6.03 | +| int8-sdnq-hadamard | interval2 | 2.734 / 6.03 | 2.943 / 5.99 | +| int8-sdnq-hadamard | seg2-stride4 | 2.336 / 26.85 | 2.596 / 26.81 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.254 / 8.06 | 4.646 / 8.01 | +| fp8-torchao | interval2 | 3.260 / 9.66 | 4.649 / 9.61 | +| fp8-torchao | seg2-stride4 | 2.827 / 63.27 | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Qwen Image + +Example: `qwen_image.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.202 / 41.04 | 3.377 / 40.99 | +| bf16 | interval2 | 1.201 / 41.04 | 3.382 / 40.99 | +| bf16 | seg2-stride4 | 1.205 / 41.03 | 3.385 / 40.99 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.675 / 63.48 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.640 / 24.09 | 3.928 / 24.05 | +| int8-sdnq-hadamard | interval2 | 2.722 / 24.09 | 3.918 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4 | 2.663 / 24.09 | 3.919 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.088 / 25.34 | 6.172 / 25.29 | +| fp8-torchao | interval2 | 3.095 / 25.34 | 6.173 / 25.29 | +| fp8-torchao | seg2-stride4 | 3.125 / 25.34 | 6.141 / 25.29 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Sana + +Example: `sana.lycoris-lokr`. Resolution: 1024x1024. + +Note: Sana has interval checkpointing; stride is not a separate segmented schedule for this family in the measured rows. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 23.71 | 0.597 / 23.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.529 / 23.72 | 0.596 / 23.66 | +| bf16 | interval2 | 0.530 / 23.72 | 0.598 / 23.66 | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.554 / 22.92 | 0.590 / 22.88 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.554 / 22.92 | 0.589 / 22.88 | +| int8-sdnq-hadamard | interval2 | 0.556 / 22.92 | 0.591 / 22.88 | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.633 / 33.67 | 0.753 / 33.62 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.633 / 33.67 | 0.755 / 33.62 | +| fp8-torchao | interval2 | 0.631 / 33.67 | 0.759 / 33.62 | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SanaVideo + +Example: `sanavideo-2b-480p.peft-lora`. Resolution: 832x480, 49f. + +Note: SanaVideo linear attention use करता है, इसलिए attention activation offload unsupported रहता है। Standard path में segmented whole-block checkpointing supported है। + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.599 / 59.15 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.597 / 59.15 | OOM | +| bf16 | interval2 | not run | OOM | +| bf16 | seg2-stride4 | not run | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | interval2 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD 1.x + +Example: `sd1x-dreamshaper.peft-lora`. Resolution: 512x512. + +Note: SD1x diffusers UNet path use करता है. Regular layer checkpointing supported है, लेकिन interval और segmented stride controls इस family में wired नहीं हैं. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.181 / 2.87 | 0.176 / 2.83 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.305 / 1.98 | 0.292 / 1.93 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.446 / 3.07 | 0.431 / 3.04 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.701 / 1.79 | 0.666 / 1.74 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.410 / 4.23 | 0.401 / 4.18 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.785 / 1.96 | 0.756 / 1.91 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD3 + +Example: `sd3.peft-lora`. Resolution: 1024x1024. + +Note: SD3 plain transformer path पर real contiguous segmented checkpointing use करता है. Attention activation offload supported है; यह VRAM काफी घटाता है, लेकिन throughput खर्च करता है. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 34.59 | 1.189 / 34.53 | +| bf16 | activation-offload | 1.335 / 9.68 | 2.850 / 9.63 | +| bf16 | layer | 0.721 / 7.67 | 1.607 / 7.62 | +| bf16 | interval2 | 0.723 / 8.93 | 1.606 / 8.88 | +| bf16 | seg2-stride4 | 0.620 / 20.14 | 1.398 / 20.09 | +| bf16 | seg2-stride4-offload | 1.229 / 12.21 | 2.639 / 12.16 | +| int8-sdnq-hadamard | none | 0.858 / 33.66 | 1.381 / 33.61 | +| int8-sdnq-hadamard | activation-offload | 1.845 / 8.90 | 3.297 / 8.85 | +| int8-sdnq-hadamard | layer | 1.265 / 6.58 | 1.857 / 6.53 | +| int8-sdnq-hadamard | interval2 | 1.264 / 7.30 | 1.858 / 7.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.049 / 19.02 | 1.625 / 18.98 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.527 / 12.42 | 3.023 / 12.37 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 4.707 / 10.27 | 9.814 / 10.23 | +| fp8-torchao | layer | 1.575 / 9.13 | 3.279 / 9.09 | +| fp8-torchao | interval2 | 1.576 / 12.62 | 3.282 / 12.58 | +| fp8-torchao | seg2-stride4 | 1.376 / 45.70 | OOM | +| fp8-torchao | seg2-stride4-offload | 4.184 / 26.17 | 8.714 / 26.12 | +| fp8wo-torchao | none | 0.567 / 35.21 | 1.252 / 35.17 | +| fp8wo-torchao | activation-offload | 1.392 / 8.71 | 2.921 / 8.67 | +| fp8wo-torchao | layer | 0.795 / 5.69 | 1.736 / 5.65 | +| fp8wo-torchao | interval2 | 0.797 / 7.08 | 1.734 / 7.03 | +| fp8wo-torchao | seg2-stride4 | 0.679 / 19.43 | 1.490 / 19.38 | +| fp8wo-torchao | seg2-stride4-offload | 1.257 / 12.00 | 2.676 / 11.96 | + +### SDXL + +Example: `sdxl.lycoris-lokr`. Resolution: 1024x1024. + +Note: SDXL has real layer checkpointing. Interval and stride rows are included as coverage data, not as segmented-support recommendations. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.606 / 13.03 | 0.585 / 12.98 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.080 / 6.53 | 1.029 / 6.48 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.741 / 13.72 | 1.643 / 13.68 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.820 / 4.64 | 2.647 / 4.59 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.608 / 26.22 | 1.582 / 26.16 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.939 / 5.10 | 2.890 / 5.04 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Stable Cascade + +Example: `cascade-stage-c.lycoris-lokr`. Resolution: 1024x1024. + +Note: stage C full-precision prior path use करता है। ये rows `mixed_precision=no` और `base_model_precision=no_change` के साथ run हुईं; quantized base precision rows इस model के लिए meaningful नहीं हैं। interval और stride modes UNet के Res/Timestep/Attention micro-block sequence पर काम करते हैं। + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.884 / 51.52 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.179 / 22.68 | 2.135 / 22.61 | +| bf16 | interval2 | 1.032 / 36.99 | 1.871 / 36.92 | +| bf16 | seg2-stride4 | 1.032 / 37.20 | 1.870 / 37.13 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | unsupported | unsupported | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | unsupported | unsupported | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Wan 2.1 T2V 1.3B + +Example: `wan2.1-t2v-1.3b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 1.3B should be read from the no-regional-compile/RamTorch rows; regional compile was not a useful throughput setting in this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.407 / 71.90 | OOM | +| bf16 | activation-offload | 3.472 / 8.78 | 7.179 / 8.66 | +| bf16 | layer | 2.099 / 4.73 | 4.459 / 4.68 | +| bf16 | interval2 | 2.139 / 6.32 | 4.514 / 6.27 | +| bf16 | seg2-stride4 | 1.806 / 39.25 | 3.921 / 39.21 | +| bf16 | seg2-stride4-offload | 2.993 / 22.66 | 6.493 / 22.61 | +| int8-sdnq-hadamard | none | 1.850 / 71.91 | OOM | +| int8-sdnq-hadamard | activation-offload | 4.204 / 8.72 | 7.387 / 8.68 | +| int8-sdnq-hadamard | layer | 2.790 / 4.70 | 4.989 / 4.65 | +| int8-sdnq-hadamard | interval2 | 2.874 / 6.29 | 5.093 / 6.24 | +| int8-sdnq-hadamard | seg2-stride4 | 2.558 / 39.27 | 4.393 / 39.22 | +| int8-sdnq-hadamard | seg2-stride4-offload | 3.711 / 22.67 | 6.695 / 22.63 | +| fp8-torchao | none | 1.727 / 73.57 | OOM | +| fp8-torchao | activation-offload | 4.061 / 10.08 | 7.404 / 9.96 | +| fp8-torchao | layer | 2.607 / 5.98 | 4.888 / 5.93 | +| fp8-torchao | interval2 | 2.744 / 7.57 | 4.916 / 7.52 | +| fp8-torchao | seg2-stride4 | 2.246 / 40.55 | 4.245 / 40.50 | +| fp8-torchao | seg2-stride4-offload | 3.602 / 24.02 | 6.683 / 23.91 | + +### Wan 2.1 T2V 14B + +Example: `wan2.1-t2v-14b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 14B is mainly a fit test for activation savings. Status-only cells are still useful because they show which combinations reached the memory limit. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 13.144 / 36.80 | OOM | +| bf16 | layer | 7.162 / 16.28 | 21.770 / 16.23 | +| bf16 | interval2 | 7.172 / 19.62 | 21.777 / 19.58 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | failed | failed | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | failed | failed | +| fp8-torchao | seg2-stride4 | failed | failed | +| fp8-torchao | seg2-stride4-offload | failed | failed | + +### Wan S2V + +Example: `wan-s2v-14b-480p.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan S2V is included as coverage data for the video/audio path. Treat failed cells as implementation coverage gaps. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | failed | failed | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Z-Image Turbo + +Example: `z-image-turbo.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.243 / 21.25 | 0.316 / 21.21 | +| bf16 | activation-offload | 0.837 / 13.24 | 0.805 / 13.19 | +| bf16 | layer | 0.479 / 12.87 | 0.493 / 12.83 | +| bf16 | interval2 | 0.452 / 13.04 | 0.477 / 12.99 | +| bf16 | seg2-stride4 | 0.349 / 16.88 | 0.400 / 16.83 | +| bf16 | seg2-stride4-offload | 0.681 / 15.03 | 0.736 / 14.99 | +| int8-sdnq-hadamard | none | 0.645 / 15.60 | 0.615 / 15.55 | +| int8-sdnq-hadamard | activation-offload | 1.439 / 7.61 | 1.382 / 7.56 | +| int8-sdnq-hadamard | layer | 1.046 / 7.25 | 1.021 / 7.20 | +| int8-sdnq-hadamard | interval2 | 1.074 / 7.41 | 0.996 / 7.36 | +| int8-sdnq-hadamard | seg2-stride4 | 0.867 / 11.25 | 0.841 / 11.20 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.202 / 9.39 | 1.162 / 9.35 | +| fp8-torchao | none | 1.232 / 37.50 | 1.476 / 37.46 | +| fp8-torchao | activation-offload | 3.623 / 7.97 | 3.564 / 7.93 | +| fp8-torchao | layer | 2.319 / 7.93 | 2.344 / 7.88 | +| fp8-torchao | interval2 | 2.336 / 8.80 | 2.309 / 8.75 | +| fp8-torchao | seg2-stride4 | 1.843 / 22.55 | 1.930 / 22.50 | +| fp8-torchao | seg2-stride4-offload | 2.947 / 15.57 | 3.243 / 15.52 | + +### ZLab I1 + +Example: `zlab-i1.peft-lora`. Resolution: 1024x1024. + +Note: ZLab I1 अपने U-Net-style skip tensors को segmented checkpoint state में carry करता है। इस family में attention activation offload wired नहीं है। + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.462 / 22.21 | 0.865 / 22.16 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.693 / 7.79 | 1.148 / 7.75 | +| bf16 | interval2 | 0.676 / 8.30 | 1.152 / 8.25 | +| bf16 | seg2-stride4 | 0.567 / 14.97 | 1.014 / 14.92 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.861 / 19.21 | 0.926 / 19.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.385 / 4.78 | 1.265 / 4.74 | +| int8-sdnq-hadamard | interval2 | 1.298 / 5.30 | 1.277 / 5.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.073 / 11.98 | 1.098 / 11.93 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8wo-torchao | none | 0.504 / 25.08 | 0.930 / 25.02 | +| fp8wo-torchao | activation-offload | unsupported | unsupported | +| fp8wo-torchao | layer | 0.772 / 5.12 | 1.280 / 5.07 | +| fp8wo-torchao | interval2 | 0.759 / 5.84 | 1.290 / 5.79 | +| fp8wo-torchao | seg2-stride4 | 0.633 / 15.12 | 1.115 / 15.08 | +| fp8wo-torchao | seg2-stride4-offload | unsupported | unsupported | + + diff --git a/documentation/experimental/SEGMENTED_CHECKPOINTING.ja.md b/documentation/experimental/SEGMENTED_CHECKPOINTING.ja.md new file mode 100644 index 000000000..b53b9fbcf --- /dev/null +++ b/documentation/experimental/SEGMENTED_CHECKPOINTING.ja.md @@ -0,0 +1,1007 @@ +# Segmented Checkpointing + +Segmented checkpointing は、全 block を checkpoint する設定と checkpoint なしの中間にある設定です。 + +PyTorch の activation checkpointing backend を使います。SimpleTuner は連続した transformer blocks を 1 回の checkpoint 呼び出しで実行し、返された hidden state を次の group に渡します。group を広くすると backward の再計算は減りますが、保持する activations は増えます。 + +CPU offload と FFN-only checkpointing は [Unsloth-style checkpointing](UNSLOTH_CHECKPOINTING.ja.md#controls) を参照してください。短い判断基準は [Decision Rule](UNSLOTH_CHECKPOINTING.ja.md#decision-rule) にあります。 + +## Controls + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2 +} +``` + +対応している whole-block path では、`gradient_checkpointing_interval` が segment width です。`2` は blocks `0-1`、`2-3`、`4-5` を checkpoint します。 + +より細かく VRAM を制御するには stride を追加します: + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2, + "gradient_checkpointing_segment_stride": 4 +} +``` + +これは blocks `0-1` を checkpoint し、`2-3` は通常実行、`4-5` を checkpoint、`6-7` は通常実行、という形で繰り返します。stride は interval 以上である必要があり、重複 schedule は無効です。 + +Segmented whole-block path 対応: Flux.1、Flux.2、HunyuanVideo、Krea 2、LongCat Image、LongCat Video、LTXVideo 0.9、LTXVideo2、Lumina2、MageFlow、PixArt、SD3、SanaVideo、Z-Image、ZLab I1、Wan。 + +Stable Cascade stage C も interval と stride に対応していますが、schedule は transformer whole-block group ではなく UNet の Res/Timestep/Attention micro-block sequence に適用されます。 + +一部の family は古い semantics です: + +| Family | `gradient_checkpointing_interval` | `gradient_checkpointing_segment_stride` | +| --- | --- | --- | +| Sana | N block ごとに checkpoint | 無視 | +| Stable Cascade stage C | interval ごとに UNet micro-block を checkpoint | stride は checkpointed/non-checkpointed UNet micro-block window を交互にする | +| SD1x, SDXL | segmented whole-block 非対応 | 無視 | + +stride が無視される family では stride 行を比較しないでください。数字が同じなら、多くの場合は option が効いていないだけです。 + +## 使う場面 + +通常の per-block checkpointing では fit するが step time が高すぎる場合に使います。まず `2` から始めてください。VRAM に余裕がある深いモデルでは `2` と stride `4` を試します。 + +peak の主因が trainable weights、optimizer state、validation、VAE caching、block swapping、routing の場合は効果を期待しないでください。モデル機能が per-block 制御を必要とする場合、SimpleTuner はより安全な per-block path に戻ります。 + +`dynamo_use_regional_compilation` は万能ではありません。一部の image model では有利または中立でしたが、下の Wan/RamTorch と LTXVideo2 profile では悪い結果でした。compile 設定も benchmark 条件として扱ってください。 + +## Benchmarks + +実際の SimpleTuner examples を使い、single-GPU H100/L40S pods で測定しました。validation と checkpoint saves は無効、cache preparation は除外、post-warmup timing がある場合は train loop 内の first-step compile/setup も除外しています。 + +測定済み cell は `post-warmup sec/step / peak VRAM GiB` です。status-only cell の意味は、`OOM` が GPU memory 不足、`failed` が measured training steps まで到達しなかった run、`unsupported` がその family に option が wired されていない状態、`not run` が sweep にその組み合わせがなかった状態です。 + +まず同じ family 内で mode を比較してください。family 間の比較は、resolution、frame count、attention backend、model depth、trainable adapter type、dataset shape が違うため、おおまかな目安です。 + +下の matrix がこの sweep の source of truth です。model-specific notes は、row が recommendation ではなく coverage data の場合に caveat を示します。 + + + +### Family Sweep Results + +### ACE Step 1.5 + +Example: `ace_step-v1-5.peft-lora`. Resolution: 512. + +Note: This sweep did not produce a usable ACE Step throughput row. The status-only entries below should be treated as coverage gaps, not as a recommendation. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | OOM | OOM | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | OOM | OOM | +| int8-sdnq-hadamard | interval2 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Anima 上の AnyFlow 蒸留 + +例: `anima-anyflow.peft-lora`。解像度: 1024x1024。この行は Anima を使った AnyFlow 蒸留の測定で、通常の Anima LoRA 例ではありません。通常の 1024x1024 Anima 画像 training には `anima.peft-lora` を使ってください。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.022 / 17.26 | 0.719 / 17.21 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.232 / 5.60 | 0.903 / 5.56 | +| bf16 | interval2 | 1.252 / 5.60 | 0.897 / 5.56 | +| bf16 | seg2-stride4 | 1.244 / 5.60 | 0.898 / 5.56 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 5.417 / 18.61 | 4.974 / 18.57 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.562 / 4.36 | 4.019 / 4.31 | +| int8-sdnq-hadamard | interval2 | 3.723 / 4.36 | 3.196 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4 | 3.658 / 4.36 | 3.140 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.242 / 45.71 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.810 / 5.51 | 2.576 / 5.46 | +| fp8-torchao | interval2 | 2.846 / 5.51 | 2.581 / 5.46 | +| fp8-torchao | seg2-stride4 | 2.766 / 5.51 | 2.567 / 5.46 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### AuraFlow + +Example: `auraflow.peft-lora`. Resolution: 1024x1024. + +注記: AuraFlow は SDNQ と TorchAO 量子化に対応しています。量子化した `none` 行は下にあります。量子化 checkpoint 行は、新しい full-length benchmark で確認してから数値を入れます。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.180 / 19.19 | 0.233 / 19.12 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.824 / 13.37 | 0.833 / 13.32 | +| bf16 | interval2 | 1.764 / 16.21 | 0.877 / 16.14 | +| bf16 | seg2-stride4 | 1.771 / 16.21 | 0.887 / 16.14 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.642 / 12.88 | 0.610 / 12.87 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | not run | not run | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.621 / 24.53 | 0.757 / 24.44 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | not run | not run | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Boogu Image + +Example: `boogu-image-v0.1.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.694 / 59.14 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.907 / 23.44 | 2.641 / 23.39 | +| bf16 | interval2 | 0.912 / 23.44 | 2.649 / 23.39 | +| bf16 | seg2-stride4 | 0.911 / 23.44 | 2.648 / 23.39 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.488 / 53.24 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.878 / 15.20 | 3.309 / 15.15 | +| int8-sdnq-hadamard | interval2 | 1.630 / 34.12 | 2.577 / 34.07 | +| int8-sdnq-hadamard | seg2-stride4 | 1.656 / 34.11 | 2.574 / 34.06 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.713 / 18.48 | 3.778 / 18.44 | +| fp8-torchao | interval2 | 1.731 / 18.48 | 3.777 / 18.44 | +| fp8-torchao | seg2-stride4 | 1.721 / 18.48 | 3.779 / 18.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Chroma + +Example: `chroma.peft-lora`. Resolution: 1024x1024. + +Note: Checkpointed Chroma rows use `attention_mechanism=native-efficient`, which was the stable attention path for this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.454 / 26.18 | 0.559 / 26.13 | +| bf16 | activation-offload | 4.873 / 18.74 | 4.793 / 18.69 | +| bf16 | layer | 1.276 / 17.67 | 1.430 / 17.63 | +| bf16 | interval2 | 1.204 / 21.80 | 1.349 / 21.75 | +| bf16 | seg2-stride4 | 1.200 / 21.78 | 1.382 / 21.74 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.083 / 21.42 | 1.061 / 21.37 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.714 / 10.41 | 1.646 / 10.36 | +| int8-sdnq-hadamard | interval2 | 1.443 / 15.72 | 1.391 / 15.68 | +| int8-sdnq-hadamard | seg2-stride4 | 1.428 / 15.71 | 1.323 / 15.67 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.122 / 45.44 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.821 / 10.64 | 2.104 / 10.60 | +| fp8-torchao | interval2 | 1.447 / 27.49 | 1.871 / 27.44 | +| fp8-torchao | seg2-stride4 | 1.431 / 27.49 | 1.877 / 27.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 2 Image + +Example: `cosmos2image.lycoris-lokr`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.336 / 8.05 | 0.316 / 8.00 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.567 / 4.09 | 0.559 / 4.04 | +| bf16 | interval2 | 0.595 / 4.09 | 0.544 / 4.04 | +| bf16 | seg2-stride4 | 0.598 / 4.09 | 0.546 / 4.04 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.831 / 6.56 | 0.783 / 6.56 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.315 / 2.44 | 1.288 / 2.39 | +| int8-sdnq-hadamard | interval2 | 1.346 / 2.44 | 1.321 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4 | 1.413 / 2.44 | 1.274 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.850 / 13.30 | 0.840 / 13.25 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.692 / 2.70 | 1.560 / 2.65 | +| fp8-torchao | interval2 | 1.626 / 2.70 | 1.544 / 2.65 | +| fp8-torchao | seg2-stride4 | 1.607 / 2.70 | 1.555 / 2.65 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 3 + +Example: `cosmos3-edge-image-24g.lycoris-lokr`. Resolution: 1024 px. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 2.747 / 8.90 | 2.904 / 8.86 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.602 / 8.90 | 2.606 / 8.86 | +| bf16 | interval2 | 2.658 / 8.90 | 2.567 / 8.86 | +| bf16 | seg2-stride4 | 2.628 / 8.90 | 2.899 / 8.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 17.112 / 6.21 | 17.965 / 6.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 3.238 / 6.21 | 2.891 / 6.16 | +| int8-sdnq-hadamard | interval2 | 3.253 / 6.21 | 2.916 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4 | 3.254 / 6.21 | 2.923 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.849 / 17.84 | 1.559 / 17.80 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.869 / 17.84 | 1.599 / 17.79 | +| fp8-torchao | interval2 | 1.855 / 17.84 | 1.600 / 17.80 | +| fp8-torchao | seg2-stride4 | 1.859 / 17.84 | 1.523 / 17.80 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### ERNIE 4.5 Image + +Example: `ernie.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.711 / 15.61 | 1.282 / 15.56 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.110 / 4.94 | 1.840 / 4.90 | +| bf16 | interval2 | 0.876 / 10.16 | 1.536 / 10.12 | +| bf16 | seg2-stride4 | 0.874 / 10.16 | 1.532 / 10.12 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 2.782 / 13.75 | 2.722 / 13.70 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.262 / 2.98 | 4.393 / 2.94 | +| int8-sdnq-hadamard | interval2 | 3.547 / 8.26 | 3.380 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4 | 3.366 / 8.26 | 3.457 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.432 / 14.03 | 2.303 / 13.98 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.920 / 3.26 | 4.008 / 3.22 | +| fp8-torchao | interval2 | 3.316 / 8.54 | 2.973 / 8.49 | +| fp8-torchao | seg2-stride4 | 2.994 / 8.54 | 3.046 / 8.49 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HeartMula + +Example: `heartmula.peft-lora`. Audio-token training. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HiDream + +Example: `hidream.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.515 / 44.58 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.928 / 33.57 | 1.043 / 33.52 | +| bf16 | interval2 | 0.971 / 33.57 | 1.002 / 33.52 | +| bf16 | seg2-stride4 | 0.952 / 33.57 | 1.010 / 33.52 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | not measured | not measured | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.497 / 17.99 | 2.058 / 17.93 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | not measured | not measured | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 4.920 / 18.10 | 4.338 / 18.05 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +SDNQ Hadamard numbers use `sdnq_compile_mode=eager`. The compiled SDNQ path quantizes HiDream, but this sweep spent the first training step in Inductor dequantizer compilation, so it is not listed as a throughput row. + +### HunyuanVideo + +Example: `hunyuanvideo-1.5-t2v.peft-lora`. Training shape: 480 pixel-area video buckets, 48 frames, batch 2. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | not run | not run | +| bf16 | layer | 7.682 / 26.35 | 22.816 / 26.30 | +| bf16 | interval2 | 7.398 / 26.11 | 22.772 / 26.06 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 10.679 / 58.37 | not run | +| int8-sdnq-hadamard | none | not run | not run | +| int8-sdnq-hadamard | activation-offload | not run | not run | +| int8-sdnq-hadamard | layer | 11.765 / 25.96 | 34.464 / 25.92 | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | not run | not run | +| fp8-torchao | none | not run | not run | +| fp8-torchao | activation-offload | not run | not run | +| fp8-torchao | layer | 10.516 / 33.55 | 32.003 / 33.53 | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | not run | not run | + +HunyuanVideo はこの training shape では activation が重いです。Per-block と interval-2 checkpointing は安定して入り、checkpointing なしは 80 GB H100 でも入りません。`seg2-stride4` は attention activation offload を有効にした場合だけ入りましたが、速度向けではなくメモリ用の fallback です。SDNQ Hadamard は動きますが、conditioning shape が変わるため測定中にも dynamic kernel compilation が残ります。 + +### Ideogram 4.0 + +Example: `ideogram-fp8.peft-lora`. Resolution: 1024x1024. `fp8` flavour は Ideogram 4 の native weight-only fp8 checkpoint を使います (`base_model_precision=no_change`)。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| fp8-native | none | failed | OOM | +| fp8-native | activation-offload | unsupported | unsupported | +| fp8-native | layer | 1.033 / 12.57 | 3.101 / 11.82 | +| fp8-native | interval2 | 1.030 / 12.33 | 3.098 / 11.82 | +| fp8-native | seg2-stride4 | 1.031 / 12.33 | 3.088 / 11.82 | +| fp8-native | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.702 / 61.78 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.033 / 12.33 | 3.030 / 11.82 | +| int8-sdnq-hadamard | interval2 | 1.032 / 12.33 | 3.034 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4 | 1.028 / 12.33 | 3.035 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | + +### Kandinsky 5 Image + +Example: `kandinsky5-image-6b-t2i.lycoris-lokr`. Resolution: 1024x1024. + +Note: batch 3 at 1024x1024 needs full checkpointing on both cards. SDNQ with Hadamard is the best low-VRAM row; H100 can also use partial checkpointing with SDNQ, but only near the top of the card. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.956 / 25.52 | 10.189 / 25.42 | +| bf16 | layer | 6.590 / 25.58 | 9.458 / 25.55 | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 7.186 / 20.00 | 10.141 / 19.95 | +| int8-sdnq-hadamard | layer | 6.830 / 20.12 | 9.362 / 20.08 | +| int8-sdnq-hadamard | interval2 | 5.746 / 75.67 | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 6.057 / 71.46 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 8.319 / 24.29 | 14.716 / 24.24 | +| fp8-torchao | layer | 7.976 / 24.40 | 13.949 / 24.35 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kandinsky 5 Video + +Example: `kandinsky5-video-2b-t2v.peft-lora`. Resolution: 768x512, 81f. + +Kandinsky 5 video is activation-heavy at this frame count. Full block checkpointing is the practical baseline on both cards. On H100, `interval2` and `seg2-stride4` are faster when they fit; on L40S, SDNQ `interval2` is the only partial-checkpoint row here that fits without attention activation offload. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 2.580 / 9.49 | 7.136 / 9.45 | +| bf16 | layer | 2.267 / 9.83 | 6.379 / 9.79 | +| bf16 | interval2 | 1.967 / 44.57 | OOM | +| bf16 | seg2-stride4 | 1.971 / 46.62 | OOM | +| bf16 | seg2-stride4-offload | 2.275 / 37.99 | 6.249 / 37.94 | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 2.844 / 8.07 | 7.234 / 8.02 | +| int8-sdnq-hadamard | layer | 2.460 / 8.40 | 6.509 / 8.35 | +| int8-sdnq-hadamard | interval2 | 2.126 / 43.12 | 5.641 / 43.08 | +| int8-sdnq-hadamard | seg2-stride4 | 2.125 / 45.19 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 2.451 / 36.56 | 6.322 / 36.51 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 3.867 / 15.11 | 11.501 / 15.06 | +| fp8-torchao | layer | 3.579 / 15.28 | 10.822 / 15.24 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kolors + +Example: `kolors.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.635 / 7.22 | 0.628 / 7.17 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.118 / 5.36 | 1.065 / 5.31 | +| bf16 | interval2 | 1.105 / 5.36 | 1.068 / 5.31 | +| bf16 | seg2-stride4 | 1.110 / 5.36 | 1.072 / 5.31 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.860 / 5.68 | 1.726 / 5.63 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.967 / 3.38 | 2.803 / 3.33 | +| int8-sdnq-hadamard | interval2 | 2.770 / 3.32 | 2.655 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4 | 2.805 / 3.32 | 2.742 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.629 / 8.79 | 1.637 / 8.75 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.044 / 3.50 | 3.013 / 3.45 | +| fp8-torchao | interval2 | 3.040 / 3.50 | 3.019 / 3.45 | +| fp8-torchao | seg2-stride4 | 2.988 / 3.50 | 2.935 / 3.45 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Krea 2 + +例: `krea2.peft-lora`。学習解像度: 512 px square crop。例の validation 設定: 1024x1024。benchmark では validation は無効です。 + +メイン表は regional compilation を使っています。Krea2 の step speed には有利ですが、VRAM 比較としてはきれいではありません。compiled graph/workspace により、いくつかの checkpoint mode で peak が未 checkpoint の peak に近く残ります。regional compilation を無効にした bf16 control run では、checkpointing が正しく接続され、期待通りの memory/speed 形になりました: + +ここでの `activation-offload` 行は、full-block checkpointing に attention activation offload を足したものです。full-block `layer` checkpointing 単体と比べると、この Krea2 shape では attention offload は peak VRAM を下げず、主に CPU transfer overhead を増やしました。 + +| Mode | H100 no-compile | L40S no-compile | +| --- | ---: | ---: | +| none | 0.272 / 40.09 | 0.661 / 40.01 | +| layer | 0.371 / 30.06 | 0.919 / 30.01 | +| seg2-stride4 | 0.317 / 34.75 | 0.788 / 34.70 | +| activation-offload | 0.657 / 30.50 | 1.341 / 30.30 | + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.275 / 40.06 | 0.662 / 40.01 | +| bf16 | activation-offload | 0.416 / 34.62 | 0.822 / 34.57 | +| bf16 | layer | 0.268 / 40.06 | 0.665 / 40.01 | +| bf16 | interval2 | 0.274 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4 | 0.279 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4-offload | 0.404 / 34.62 | 0.819 / 34.57 | +| int8-sdnq-hadamard | none | 0.462 / 27.01 | 0.807 / 26.96 | +| int8-sdnq-hadamard | activation-offload | 0.773 / 21.57 | 1.012 / 21.53 | +| int8-sdnq-hadamard | layer | 0.473 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | interval2 | 0.474 / 27.01 | 0.804 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4 | 0.472 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4-offload | 0.744 / 21.57 | 1.007 / 21.53 | +| fp8-torchao | none | 0.689 / 51.63 | OOM | +| fp8-torchao | activation-offload | 0.975 / 37.77 | 2.058 / 37.73 | +| fp8-torchao | layer | 0.689 / 51.63 | OOM | +| fp8-torchao | interval2 | 0.684 / 51.63 | OOM | +| fp8-torchao | seg2-stride4 | 0.674 / 51.63 | OOM | +| fp8-torchao | seg2-stride4-offload | 0.965 / 37.77 | 2.053 / 37.73 | + +### LongCat Image + +例: `longcat-image.peft-lora`。Training resolution: 512 px square; validation resolution: 1024x1024. Rows use `attention_mechanism=native-flash`. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.193 / 16.47 | 0.262 / 16.42 | +| bf16 | activation-offload | 0.544 / 12.73 | 0.543 / 12.69 | +| bf16 | layer | 0.327 / 12.38 | 0.370 / 12.34 | +| bf16 | interval2 | 0.257 / 14.38 | 0.313 / 14.34 | +| bf16 | seg2-stride4 | 0.263 / 14.36 | 0.316 / 14.31 | +| bf16 | seg2-stride4-offload | 0.446 / 13.42 | 0.492 / 13.38 | +| int8-sdnq-hadamard | none | 0.578 / 12.54 | 0.537 / 12.45 | +| int8-sdnq-hadamard | activation-offload | 1.184 / 7.43 | 1.185 / 7.39 | +| int8-sdnq-hadamard | layer | 0.901 / 7.19 | 0.911 / 7.09 | +| int8-sdnq-hadamard | interval2 | 0.718 / 9.73 | 0.695 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4 | 0.735 / 9.72 | 0.717 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.056 / 8.49 | 1.011 / 8.45 | +| fp8-torchao | none | 0.602 / 25.19 | 0.834 / 25.14 | +| fp8-torchao | activation-offload | 1.662 / 7.75 | 1.844 / 7.70 | +| fp8-torchao | layer | 0.984 / 7.57 | 1.080 / 7.53 | +| fp8-torchao | interval2 | 0.750 / 16.15 | 0.938 / 16.10 | +| fp8-torchao | seg2-stride4 | 0.760 / 16.13 | 0.961 / 16.09 | +| fp8-torchao | seg2-stride4-offload | 1.287 / 13.02 | 1.653 / 12.98 | + +### LongCat Video + +例: `longcat-video.peft-lora+ramtorch`。Resolution: 832x480, 81f。各行は `attention_mechanism=native-flash` を使用。 + +LongCat Video はこの shape では activation が重いです。Full per-block checkpointing が実用的な行です。Partial checkpoint 行（`interval2`、`seg2-stride4`）はここでは入りません。strided 行に attention activation offload を足しても不足します。通常の attention activation offload は bf16 と SDNQ では入りますが、full checkpointing よりかなり遅くなります。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM (77.86 GiB) | OOM (43.30 GiB) | +| bf16 | activation-offload | 25.774 / 37.36 | 49.149 / 37.14 | +| bf16 | layer | 7.448 / 23.73 | 24.866 / 23.68 | +| bf16 | interval2 | OOM (76.41 GiB) | OOM (42.80 GiB) | +| bf16 | seg2-stride4 | OOM (76.42 GiB) | OOM (43.06 GiB) | +| bf16 | seg2-stride4-offload | OOM (76.72 GiB) | OOM (42.29 GiB) | +| int8-sdnq-hadamard | none | OOM (77.40 GiB) | OOM (43.59 GiB) | +| int8-sdnq-hadamard | activation-offload | 30.887 / 35.28 | 61.270 / 35.24 | +| int8-sdnq-hadamard | layer | 8.444 / 21.60 | 25.164 / 21.55 | +| int8-sdnq-hadamard | interval2 | OOM (76.01 GiB) | OOM (42.47 GiB) | +| int8-sdnq-hadamard | seg2-stride4 | OOM (77.01 GiB) | OOM (43.02 GiB) | +| int8-sdnq-hadamard | seg2-stride4-offload | OOM (76.42 GiB) | OOM (42.57 GiB) | +| fp8-torchao | none | OOM (75.87 GiB) | OOM (41.28 GiB) | +| fp8-torchao | activation-offload | 30.163 / 47.88 | OOM (40.11 GiB) | +| fp8-torchao | layer | 8.343 / 34.16 | 24.659 / 34.07 | +| fp8-torchao | interval2 | OOM (74.63 GiB) | OOM (40.85 GiB) | +| fp8-torchao | seg2-stride4 | OOM (75.37 GiB) | OOM (41.19 GiB) | +| fp8-torchao | seg2-stride4-offload | OOM (75.55 GiB) | OOM (40.94 GiB) | + +### LTXVideo 0.9.5 + +Example: `ltxvideo-0.9.5-t2v.peft-lora`. Resolution: 768x512, 49f. + +数字は warm seconds/step / peak GiB です。full-run average は setup と compile overhead を含むため、sweep artifacts に記録しています。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.274 / 8.77 | 0.275 / 8.72 | +| bf16 | layer | 0.462 / 4.21 | 0.459 / 4.16 | +| bf16 | interval2 | 0.449 / 4.30 | 0.433 / 4.25 | +| bf16 | seg2-stride4 | 0.359 / 6.40 | 0.357 / 6.35 | +| int8-sdnq-hadamard | none | 0.688 / 7.05 | 0.640 / 6.90 | +| int8-sdnq-hadamard | layer | 1.094 / 2.59 | 1.081 / 2.44 | +| int8-sdnq-hadamard | interval2 | 1.112 / 2.57 | 1.073 / 2.53 | +| int8-sdnq-hadamard | seg2-stride4 | 0.887 / 4.61 | 0.817 / 4.56 | +| fp8-torchao | none | 0.655 / 17.45 | 0.735 / 17.41 | +| fp8-torchao | layer | 1.226 / 2.94 | 1.206 / 2.89 | +| fp8-torchao | interval2 | 1.259 / 3.40 | 1.233 / 3.35 | +| fp8-torchao | seg2-stride4 | 0.933 / 9.96 | 0.901 / 9.91 | +| fp8wo-torchao | none | 0.328 / 10.06 | 0.325 / 10.01 | +| fp8wo-torchao | layer | 0.567 / 2.64 | 0.540 / 2.59 | +| fp8wo-torchao | interval2 | 0.555 / 2.84 | 0.531 / 2.79 | +| fp8wo-torchao | seg2-stride4 | 0.443 / 6.20 | 0.432 / 6.15 | + +Attention activation offload rows は、この sweep の LTXVideo 0.9 では非対応です。 + +### LTXVideo2 2.3 + +Example: `ltxvideo2-2.3-dev-720p-single-gpu.peft-lora+sdnq-hadamard`. Resolution: 1280x704, 49f. + +Note: LTXVideo2 2.3 should be read from the no-regional-compile rows in this sweep; regional compile raised memory pressure for this model. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.350 / 58.36 | OOM | +| bf16 | layer | 3.993 / 47.95 | OOM | +| bf16 | interval2 | 3.977 / 48.83 | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 5.554 / 75.64 | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 10.102 / 38.20 | 9.852 / 38.15 | +| int8-sdnq-hadamard | layer | 7.753 / 27.78 | 7.579 / 27.73 | +| int8-sdnq-hadamard | interval2 | 7.733 / 28.66 | 7.288 / 28.61 | +| int8-sdnq-hadamard | seg2-stride4 | 6.522 / 61.50 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 8.659 / 55.48 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | failed | 22.917 / 38.57 | +| fp8-torchao | layer | 8.580 / 30.27 | 10.381 / 30.22 | +| fp8-torchao | interval2 | 8.660 / 33.71 | 10.661 / 33.66 | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | failed | OOM | + +### Lumina2 + +Example: `lumina2.peft-lora`. Resolution: 512x512. + +注記: Lumina2 は segmented whole-block path を使うようになりました。`interval2` は 2-block segment ごとに checkpoint し、`seg2-stride4` は 2 blocks を checkpoint、次の 2 blocks は activations を保持して実行し、それを繰り返します。Attention activation offload はこの Lumina2 run には含めていません。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.235 / 15.99 | 0.332 / 15.94 | +| bf16 | layer | 0.384 / 6.60 | 0.457 / 6.56 | +| bf16 | interval2 | 0.356 / 6.87 | 0.424 / 6.82 | +| bf16 | seg2-stride4 | 0.295 / 11.43 | 0.377 / 11.38 | +| int8-sdnq-hadamard | none | 0.584 / 13.59 | 0.541 / 13.55 | +| int8-sdnq-hadamard | layer | 0.899 / 4.21 | 0.827 / 4.16 | +| int8-sdnq-hadamard | interval2 | 0.865 / 4.48 | 0.835 / 4.43 | +| int8-sdnq-hadamard | seg2-stride4 | 0.719 / 9.03 | 0.707 / 8.99 | +| fp8-torchao | none | 0.598 / 28.03 | 0.763 / 27.98 | +| fp8-torchao | layer | 0.950 / 5.93 | 0.967 / 5.88 | +| fp8-torchao | interval2 | 0.938 / 6.70 | 0.974 / 6.66 | +| fp8-torchao | seg2-stride4 | 0.765 / 17.36 | 0.901 / 17.32 | +| fp8wo-torchao | none | 0.273 / 17.97 | 0.389 / 17.93 | +| fp8wo-torchao | layer | 0.452 / 4.99 | 0.525 / 4.94 | +| fp8wo-torchao | interval2 | 0.427 / 5.40 | 0.522 / 5.35 | +| fp8wo-torchao | seg2-stride4 | 0.360 / 11.68 | 0.466 / 11.64 | + +### MageFlow + +Example: `mageflow-image-24g.peft-lora`. Resolution: 1024x1024. + +Note: MageFlow の 1024px variable-shape image path では、主に attention activation offload と weight-only FP8 が効きました。Block checkpointing mode は有効ですが、この sweep では measured peak residency は下がりませんでした。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 7.735 / 36.74 | 7.658 / 36.69 | +| bf16 | activation-offload | 8.902 / 23.05 | 9.477 / 23.00 | +| bf16 | layer | 7.805 / 36.74 | 7.504 / 36.69 | +| bf16 | interval2 | 8.036 / 36.74 | 7.762 / 36.69 | +| bf16 | seg2-stride4 | 7.882 / 36.74 | 7.531 / 36.69 | +| bf16 | seg2-stride4-offload | 8.833 / 23.05 | 9.288 / 23.01 | +| int8-sdnq-hadamard | none | 81.016 / 37.38 | 94.991 / 37.34 | +| fp8wo-torchao | none | 5.454 / 36.86 | 5.772 / 36.82 | +| fp8wo-torchao | activation-offload | 6.295 / 23.18 | 6.738 / 23.14 | +| fp8wo-torchao | seg2-stride4 | 5.542 / 36.86 | 5.595 / 36.82 | + +### OmniGen + +Example: `omnigen.lycoris-lokr`. Resolution: 1024x1024. + +Note: OmniGen は cached text embeddings ではなく token-ID prompt を使います。この行は、対応済みの no-checkpointing と full-block torch checkpointing path を測定したものです。interval、segmented-stride、attention-offload controls は、この family ではこの sweep 時点で未実装です。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.425 / 14.20 | 0.293 / 14.24 | +| bf16 | layer | 0.597 / 10.13 | 0.389 / 10.09 | +| int8-sdnq-hadamard | none | 1.523 / 11.00 | 1.388 / 11.06 | +| int8-sdnq-hadamard | layer | 1.312 / 6.75 | 1.064 / 6.70 | +| fp8-torchao | none | 0.690 / 19.25 | 0.608 / 19.30 | +| fp8-torchao | layer | 1.069 / 7.09 | 0.824 / 7.04 | +| fp8wo-torchao | none | 0.454 / 17.71 | 0.377 / 17.73 | +| fp8wo-torchao | layer | 0.646 / 6.98 | 0.534 / 6.94 | + +### PixArt + +Example: `pixart.lycoris-lokr`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.700 / 41.73 | 1.734 / 41.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.433 / 5.63 | 2.346 / 5.58 | +| bf16 | interval2 | 2.440 / 6.01 | 2.348 / 5.96 | +| bf16 | seg2-stride4 | 2.092 / 23.90 | 2.072 / 23.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.905 / 47.58 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.738 / 6.07 | 2.937 / 6.03 | +| int8-sdnq-hadamard | interval2 | 2.734 / 6.03 | 2.943 / 5.99 | +| int8-sdnq-hadamard | seg2-stride4 | 2.336 / 26.85 | 2.596 / 26.81 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.254 / 8.06 | 4.646 / 8.01 | +| fp8-torchao | interval2 | 3.260 / 9.66 | 4.649 / 9.61 | +| fp8-torchao | seg2-stride4 | 2.827 / 63.27 | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Qwen Image + +Example: `qwen_image.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.202 / 41.04 | 3.377 / 40.99 | +| bf16 | interval2 | 1.201 / 41.04 | 3.382 / 40.99 | +| bf16 | seg2-stride4 | 1.205 / 41.03 | 3.385 / 40.99 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.675 / 63.48 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.640 / 24.09 | 3.928 / 24.05 | +| int8-sdnq-hadamard | interval2 | 2.722 / 24.09 | 3.918 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4 | 2.663 / 24.09 | 3.919 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.088 / 25.34 | 6.172 / 25.29 | +| fp8-torchao | interval2 | 3.095 / 25.34 | 6.173 / 25.29 | +| fp8-torchao | seg2-stride4 | 3.125 / 25.34 | 6.141 / 25.29 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Sana + +Example: `sana.lycoris-lokr`. Resolution: 1024x1024. + +Note: Sana has interval checkpointing; stride is not a separate segmented schedule for this family in the measured rows. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 23.71 | 0.597 / 23.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.529 / 23.72 | 0.596 / 23.66 | +| bf16 | interval2 | 0.530 / 23.72 | 0.598 / 23.66 | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.554 / 22.92 | 0.590 / 22.88 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.554 / 22.92 | 0.589 / 22.88 | +| int8-sdnq-hadamard | interval2 | 0.556 / 22.92 | 0.591 / 22.88 | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.633 / 33.67 | 0.753 / 33.62 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.633 / 33.67 | 0.755 / 33.62 | +| fp8-torchao | interval2 | 0.631 / 33.67 | 0.759 / 33.62 | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SanaVideo + +Example: `sanavideo-2b-480p.peft-lora`. Resolution: 832x480, 49f. + +Note: SanaVideo は linear attention を使うため、attention activation offload は未対応です。標準 path では segmented whole-block checkpointing に対応しています。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.599 / 59.15 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.597 / 59.15 | OOM | +| bf16 | interval2 | not run | OOM | +| bf16 | seg2-stride4 | not run | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | interval2 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD 1.x + +Example: `sd1x-dreamshaper.peft-lora`. Resolution: 512x512. + +Note: SD1x は diffusers UNet path を使います。通常の layer checkpointing は supported ですが、interval と segmented stride controls はこの family では未接続です。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.181 / 2.87 | 0.176 / 2.83 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.305 / 1.98 | 0.292 / 1.93 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.446 / 3.07 | 0.431 / 3.04 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.701 / 1.79 | 0.666 / 1.74 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.410 / 4.23 | 0.401 / 4.18 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.785 / 1.96 | 0.756 / 1.91 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD3 + +Example: `sd3.peft-lora`. Resolution: 1024x1024. + +Note: SD3 は plain transformer path で real contiguous segmented checkpointing を使います。Attention activation offload も supported です。VRAM は大きく下がりますが、throughput は落ちます。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 34.59 | 1.189 / 34.53 | +| bf16 | activation-offload | 1.335 / 9.68 | 2.850 / 9.63 | +| bf16 | layer | 0.721 / 7.67 | 1.607 / 7.62 | +| bf16 | interval2 | 0.723 / 8.93 | 1.606 / 8.88 | +| bf16 | seg2-stride4 | 0.620 / 20.14 | 1.398 / 20.09 | +| bf16 | seg2-stride4-offload | 1.229 / 12.21 | 2.639 / 12.16 | +| int8-sdnq-hadamard | none | 0.858 / 33.66 | 1.381 / 33.61 | +| int8-sdnq-hadamard | activation-offload | 1.845 / 8.90 | 3.297 / 8.85 | +| int8-sdnq-hadamard | layer | 1.265 / 6.58 | 1.857 / 6.53 | +| int8-sdnq-hadamard | interval2 | 1.264 / 7.30 | 1.858 / 7.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.049 / 19.02 | 1.625 / 18.98 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.527 / 12.42 | 3.023 / 12.37 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 4.707 / 10.27 | 9.814 / 10.23 | +| fp8-torchao | layer | 1.575 / 9.13 | 3.279 / 9.09 | +| fp8-torchao | interval2 | 1.576 / 12.62 | 3.282 / 12.58 | +| fp8-torchao | seg2-stride4 | 1.376 / 45.70 | OOM | +| fp8-torchao | seg2-stride4-offload | 4.184 / 26.17 | 8.714 / 26.12 | +| fp8wo-torchao | none | 0.567 / 35.21 | 1.252 / 35.17 | +| fp8wo-torchao | activation-offload | 1.392 / 8.71 | 2.921 / 8.67 | +| fp8wo-torchao | layer | 0.795 / 5.69 | 1.736 / 5.65 | +| fp8wo-torchao | interval2 | 0.797 / 7.08 | 1.734 / 7.03 | +| fp8wo-torchao | seg2-stride4 | 0.679 / 19.43 | 1.490 / 19.38 | +| fp8wo-torchao | seg2-stride4-offload | 1.257 / 12.00 | 2.676 / 11.96 | + +### SDXL + +Example: `sdxl.lycoris-lokr`. Resolution: 1024x1024. + +Note: SDXL has real layer checkpointing. Interval and stride rows are included as coverage data, not as segmented-support recommendations. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.606 / 13.03 | 0.585 / 12.98 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.080 / 6.53 | 1.029 / 6.48 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.741 / 13.72 | 1.643 / 13.68 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.820 / 4.64 | 2.647 / 4.59 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.608 / 26.22 | 1.582 / 26.16 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.939 / 5.10 | 2.890 / 5.04 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Stable Cascade + +Example: `cascade-stage-c.lycoris-lokr`. Resolution: 1024x1024. + +注記: stage C は full precision の prior path です。これらの行は `mixed_precision=no` と `base_model_precision=no_change` で実行しました。量子化 base precision 行はこのモデルでは有用な測定ではありません。interval と stride は UNet の Res/Timestep/Attention micro-block sequence に適用されます。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.884 / 51.52 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.179 / 22.68 | 2.135 / 22.61 | +| bf16 | interval2 | 1.032 / 36.99 | 1.871 / 36.92 | +| bf16 | seg2-stride4 | 1.032 / 37.20 | 1.870 / 37.13 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | unsupported | unsupported | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | unsupported | unsupported | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Wan 2.1 T2V 1.3B + +Example: `wan2.1-t2v-1.3b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 1.3B should be read from the no-regional-compile/RamTorch rows; regional compile was not a useful throughput setting in this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.407 / 71.90 | OOM | +| bf16 | activation-offload | 3.472 / 8.78 | 7.179 / 8.66 | +| bf16 | layer | 2.099 / 4.73 | 4.459 / 4.68 | +| bf16 | interval2 | 2.139 / 6.32 | 4.514 / 6.27 | +| bf16 | seg2-stride4 | 1.806 / 39.25 | 3.921 / 39.21 | +| bf16 | seg2-stride4-offload | 2.993 / 22.66 | 6.493 / 22.61 | +| int8-sdnq-hadamard | none | 1.850 / 71.91 | OOM | +| int8-sdnq-hadamard | activation-offload | 4.204 / 8.72 | 7.387 / 8.68 | +| int8-sdnq-hadamard | layer | 2.790 / 4.70 | 4.989 / 4.65 | +| int8-sdnq-hadamard | interval2 | 2.874 / 6.29 | 5.093 / 6.24 | +| int8-sdnq-hadamard | seg2-stride4 | 2.558 / 39.27 | 4.393 / 39.22 | +| int8-sdnq-hadamard | seg2-stride4-offload | 3.711 / 22.67 | 6.695 / 22.63 | +| fp8-torchao | none | 1.727 / 73.57 | OOM | +| fp8-torchao | activation-offload | 4.061 / 10.08 | 7.404 / 9.96 | +| fp8-torchao | layer | 2.607 / 5.98 | 4.888 / 5.93 | +| fp8-torchao | interval2 | 2.744 / 7.57 | 4.916 / 7.52 | +| fp8-torchao | seg2-stride4 | 2.246 / 40.55 | 4.245 / 40.50 | +| fp8-torchao | seg2-stride4-offload | 3.602 / 24.02 | 6.683 / 23.91 | + +### Wan 2.1 T2V 14B + +Example: `wan2.1-t2v-14b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 14B is mainly a fit test for activation savings. Status-only cells are still useful because they show which combinations reached the memory limit. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 13.144 / 36.80 | OOM | +| bf16 | layer | 7.162 / 16.28 | 21.770 / 16.23 | +| bf16 | interval2 | 7.172 / 19.62 | 21.777 / 19.58 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | failed | failed | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | failed | failed | +| fp8-torchao | seg2-stride4 | failed | failed | +| fp8-torchao | seg2-stride4-offload | failed | failed | + +### Wan S2V + +Example: `wan-s2v-14b-480p.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan S2V is included as coverage data for the video/audio path. Treat failed cells as implementation coverage gaps. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | failed | failed | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Z-Image Turbo + +Example: `z-image-turbo.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.243 / 21.25 | 0.316 / 21.21 | +| bf16 | activation-offload | 0.837 / 13.24 | 0.805 / 13.19 | +| bf16 | layer | 0.479 / 12.87 | 0.493 / 12.83 | +| bf16 | interval2 | 0.452 / 13.04 | 0.477 / 12.99 | +| bf16 | seg2-stride4 | 0.349 / 16.88 | 0.400 / 16.83 | +| bf16 | seg2-stride4-offload | 0.681 / 15.03 | 0.736 / 14.99 | +| int8-sdnq-hadamard | none | 0.645 / 15.60 | 0.615 / 15.55 | +| int8-sdnq-hadamard | activation-offload | 1.439 / 7.61 | 1.382 / 7.56 | +| int8-sdnq-hadamard | layer | 1.046 / 7.25 | 1.021 / 7.20 | +| int8-sdnq-hadamard | interval2 | 1.074 / 7.41 | 0.996 / 7.36 | +| int8-sdnq-hadamard | seg2-stride4 | 0.867 / 11.25 | 0.841 / 11.20 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.202 / 9.39 | 1.162 / 9.35 | +| fp8-torchao | none | 1.232 / 37.50 | 1.476 / 37.46 | +| fp8-torchao | activation-offload | 3.623 / 7.97 | 3.564 / 7.93 | +| fp8-torchao | layer | 2.319 / 7.93 | 2.344 / 7.88 | +| fp8-torchao | interval2 | 2.336 / 8.80 | 2.309 / 8.75 | +| fp8-torchao | seg2-stride4 | 1.843 / 22.55 | 1.930 / 22.50 | +| fp8-torchao | seg2-stride4-offload | 2.947 / 15.57 | 3.243 / 15.52 | + +### ZLab I1 + +Example: `zlab-i1.peft-lora`. Resolution: 1024x1024. + +注記: ZLab I1 は U-Net 風の skip tensors を segmented checkpoint state に入れて運びます。この family では attention activation offload は未接続です。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.462 / 22.21 | 0.865 / 22.16 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.693 / 7.79 | 1.148 / 7.75 | +| bf16 | interval2 | 0.676 / 8.30 | 1.152 / 8.25 | +| bf16 | seg2-stride4 | 0.567 / 14.97 | 1.014 / 14.92 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.861 / 19.21 | 0.926 / 19.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.385 / 4.78 | 1.265 / 4.74 | +| int8-sdnq-hadamard | interval2 | 1.298 / 5.30 | 1.277 / 5.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.073 / 11.98 | 1.098 / 11.93 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8wo-torchao | none | 0.504 / 25.08 | 0.930 / 25.02 | +| fp8wo-torchao | activation-offload | unsupported | unsupported | +| fp8wo-torchao | layer | 0.772 / 5.12 | 1.280 / 5.07 | +| fp8wo-torchao | interval2 | 0.759 / 5.84 | 1.290 / 5.79 | +| fp8wo-torchao | seg2-stride4 | 0.633 / 15.12 | 1.115 / 15.08 | +| fp8wo-torchao | seg2-stride4-offload | unsupported | unsupported | + + diff --git a/documentation/experimental/SEGMENTED_CHECKPOINTING.md b/documentation/experimental/SEGMENTED_CHECKPOINTING.md new file mode 100644 index 000000000..ff697a741 --- /dev/null +++ b/documentation/experimental/SEGMENTED_CHECKPOINTING.md @@ -0,0 +1,1007 @@ +# Segmented Checkpointing + +Segmented checkpointing is the middle gear between checkpointing every block and checkpointing nothing. + +It uses PyTorch's activation checkpointing backend. SimpleTuner runs a contiguous group of transformer blocks under one checkpoint call, then carries the returned hidden state into the next group. Wider groups recompute less in backward, but keep more activations alive. + +For CPU offload and FFN-only checkpointing, use [Unsloth-style checkpointing](UNSLOTH_CHECKPOINTING.md#controls). The short rule of thumb still lives in [Decision Rule](UNSLOTH_CHECKPOINTING.md#decision-rule). + +## Controls + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2 +} +``` + +On supported whole-block paths, `gradient_checkpointing_interval` is the segment width. `2` means checkpoint blocks `0-1`, `2-3`, `4-5`, and so on. + +For finer VRAM control, add a stride: + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2, + "gradient_checkpointing_segment_stride": 4 +} +``` + +That checkpoints blocks `0-1`, runs `2-3` normally, checkpoints `4-5`, runs `6-7` normally, and repeats. The stride must be at least the interval; overlapping schedules are not valid. + +Supported segmented whole-block paths: Flux.1, Flux.2, HunyuanVideo, Krea 2, LongCat Image, LongCat Video, LTXVideo 0.9, LTXVideo2, Lumina2, MageFlow, PixArt, SD3, SanaVideo, Z-Image, ZLab I1, and Wan. + +Stable Cascade stage C also supports interval and stride control, but it applies the schedule to the UNet Res/Timestep/Attention micro-block sequence instead of transformer whole-block groups. + +Some model families use model-specific interval semantics: + +| Family | `gradient_checkpointing_interval` | `gradient_checkpointing_segment_stride` | +| --- | --- | --- | +| Sana | Checkpoint every N-th block | Ignored | +| Stable Cascade stage C | Checkpoint UNet micro-blocks by interval | Stride alternates checkpointed and non-checkpointed UNet micro-block windows | +| SD1x, SDXL | No segmented whole-block support | Ignored | + +Do not compare stride rows for families where stride is ignored. If the benchmark numbers look identical there, that is usually the option being ignored, not a useful performance result. + +## When To Use It + +Use it after normal per-block checkpointing fits but costs too much step time. Start with `2`. If VRAM allows, try `2` with stride `4` on very deep models. + +Do not expect it to help when the peak is mostly trainable weights, optimizer state, validation, VAE caching, block swapping, or routing. SimpleTuner falls back to the safer per-block path when a model feature needs per-block control. + +`dynamo_use_regional_compilation` is not a universal win. It helped or stayed neutral on several image-model runs, but it was a bad fit for the Wan/RamTorch and LTXVideo2 profiles below. Treat compile settings as part of the benchmark, not as background noise. + +## Benchmarks + +Measured with real SimpleTuner examples on single-GPU H100 and L40S pods. Validation and checkpoint saves were disabled, cache preparation was excluded, and first-step compile/setup inside the train loop is excluded when post-warmup timing is available. + +Each measured cell is `post-warmup sec/step / peak VRAM GiB`. Status-only cells mean: `OOM` ran out of GPU memory, `failed` did not reach measured training steps, `unsupported` means that option was not wired for that family, and `not run` means the sweep did not include that combination. + +Compare modes within a family first. Cross-family comparisons are rough because resolution, frame count, attention backend, model depth, trainable adapter type, and dataset shape differ. + +The matrix below is the source of truth for this sweep. Model-specific notes call out caveats when a row is coverage data rather than a recommendation. + + + +### Family Sweep Results + +### ACE Step 1.5 + +Example: `ace_step-v1-5.peft-lora`. Resolution: 512. + +Note: This sweep did not produce a usable ACE Step throughput row. The status-only entries below should be treated as coverage gaps, not as a recommendation. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | OOM | OOM | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | OOM | OOM | +| int8-sdnq-hadamard | interval2 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### AnyFlow Distillation on Anima + +Example: `anima-anyflow.peft-lora`. Resolution: 1024x1024. This row measures AnyFlow distillation using Anima, not the plain Anima LoRA example. Use `anima.peft-lora` for plain 1024x1024 Anima image training. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.022 / 17.26 | 0.719 / 17.21 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.232 / 5.60 | 0.903 / 5.56 | +| bf16 | interval2 | 1.252 / 5.60 | 0.897 / 5.56 | +| bf16 | seg2-stride4 | 1.244 / 5.60 | 0.898 / 5.56 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 5.417 / 18.61 | 4.974 / 18.57 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.562 / 4.36 | 4.019 / 4.31 | +| int8-sdnq-hadamard | interval2 | 3.723 / 4.36 | 3.196 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4 | 3.658 / 4.36 | 3.140 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.242 / 45.71 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.810 / 5.51 | 2.576 / 5.46 | +| fp8-torchao | interval2 | 2.846 / 5.51 | 2.581 / 5.46 | +| fp8-torchao | seg2-stride4 | 2.766 / 5.51 | 2.567 / 5.46 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### AuraFlow + +Example: `auraflow.peft-lora`. Resolution: 1024x1024. + +Note: AuraFlow supports SDNQ and TorchAO quantization. Quantized `none` rows are included below; quantized checkpoint rows need fresh full-length benchmark coverage before they get numbers here. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.180 / 19.19 | 0.233 / 19.12 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.824 / 13.37 | 0.833 / 13.32 | +| bf16 | interval2 | 1.764 / 16.21 | 0.877 / 16.14 | +| bf16 | seg2-stride4 | 1.771 / 16.21 | 0.887 / 16.14 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.642 / 12.88 | 0.610 / 12.87 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | not run | not run | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.621 / 24.53 | 0.757 / 24.44 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | not run | not run | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Boogu Image + +Example: `boogu-image-v0.1.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.694 / 59.14 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.907 / 23.44 | 2.641 / 23.39 | +| bf16 | interval2 | 0.912 / 23.44 | 2.649 / 23.39 | +| bf16 | seg2-stride4 | 0.911 / 23.44 | 2.648 / 23.39 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.488 / 53.24 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.878 / 15.20 | 3.309 / 15.15 | +| int8-sdnq-hadamard | interval2 | 1.630 / 34.12 | 2.577 / 34.07 | +| int8-sdnq-hadamard | seg2-stride4 | 1.656 / 34.11 | 2.574 / 34.06 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.713 / 18.48 | 3.778 / 18.44 | +| fp8-torchao | interval2 | 1.731 / 18.48 | 3.777 / 18.44 | +| fp8-torchao | seg2-stride4 | 1.721 / 18.48 | 3.779 / 18.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Chroma + +Example: `chroma.peft-lora`. Resolution: 1024x1024. + +Note: Checkpointed Chroma rows use `attention_mechanism=native-efficient`, which was the stable attention path for this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.454 / 26.18 | 0.559 / 26.13 | +| bf16 | activation-offload | 4.873 / 18.74 | 4.793 / 18.69 | +| bf16 | layer | 1.276 / 17.67 | 1.430 / 17.63 | +| bf16 | interval2 | 1.204 / 21.80 | 1.349 / 21.75 | +| bf16 | seg2-stride4 | 1.200 / 21.78 | 1.382 / 21.74 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.083 / 21.42 | 1.061 / 21.37 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.714 / 10.41 | 1.646 / 10.36 | +| int8-sdnq-hadamard | interval2 | 1.443 / 15.72 | 1.391 / 15.68 | +| int8-sdnq-hadamard | seg2-stride4 | 1.428 / 15.71 | 1.323 / 15.67 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.122 / 45.44 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.821 / 10.64 | 2.104 / 10.60 | +| fp8-torchao | interval2 | 1.447 / 27.49 | 1.871 / 27.44 | +| fp8-torchao | seg2-stride4 | 1.431 / 27.49 | 1.877 / 27.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 2 Image + +Example: `cosmos2image.lycoris-lokr`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.336 / 8.05 | 0.316 / 8.00 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.567 / 4.09 | 0.559 / 4.04 | +| bf16 | interval2 | 0.595 / 4.09 | 0.544 / 4.04 | +| bf16 | seg2-stride4 | 0.598 / 4.09 | 0.546 / 4.04 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.831 / 6.56 | 0.783 / 6.56 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.315 / 2.44 | 1.288 / 2.39 | +| int8-sdnq-hadamard | interval2 | 1.346 / 2.44 | 1.321 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4 | 1.413 / 2.44 | 1.274 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.850 / 13.30 | 0.840 / 13.25 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.692 / 2.70 | 1.560 / 2.65 | +| fp8-torchao | interval2 | 1.626 / 2.70 | 1.544 / 2.65 | +| fp8-torchao | seg2-stride4 | 1.607 / 2.70 | 1.555 / 2.65 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 3 + +Example: `cosmos3-edge-image-24g.lycoris-lokr`. Resolution: 1024 px. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 2.747 / 8.90 | 2.904 / 8.86 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.602 / 8.90 | 2.606 / 8.86 | +| bf16 | interval2 | 2.658 / 8.90 | 2.567 / 8.86 | +| bf16 | seg2-stride4 | 2.628 / 8.90 | 2.899 / 8.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 17.112 / 6.21 | 17.965 / 6.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 3.238 / 6.21 | 2.891 / 6.16 | +| int8-sdnq-hadamard | interval2 | 3.253 / 6.21 | 2.916 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4 | 3.254 / 6.21 | 2.923 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.849 / 17.84 | 1.559 / 17.80 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.869 / 17.84 | 1.599 / 17.79 | +| fp8-torchao | interval2 | 1.855 / 17.84 | 1.600 / 17.80 | +| fp8-torchao | seg2-stride4 | 1.859 / 17.84 | 1.523 / 17.80 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### ERNIE 4.5 Image + +Example: `ernie.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.711 / 15.61 | 1.282 / 15.56 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.110 / 4.94 | 1.840 / 4.90 | +| bf16 | interval2 | 0.876 / 10.16 | 1.536 / 10.12 | +| bf16 | seg2-stride4 | 0.874 / 10.16 | 1.532 / 10.12 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 2.782 / 13.75 | 2.722 / 13.70 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.262 / 2.98 | 4.393 / 2.94 | +| int8-sdnq-hadamard | interval2 | 3.547 / 8.26 | 3.380 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4 | 3.366 / 8.26 | 3.457 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.432 / 14.03 | 2.303 / 13.98 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.920 / 3.26 | 4.008 / 3.22 | +| fp8-torchao | interval2 | 3.316 / 8.54 | 2.973 / 8.49 | +| fp8-torchao | seg2-stride4 | 2.994 / 8.54 | 3.046 / 8.49 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HeartMula + +Example: `heartmula.peft-lora`. Audio-token training. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HiDream + +Example: `hidream.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.515 / 44.58 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.928 / 33.57 | 1.043 / 33.52 | +| bf16 | interval2 | 0.971 / 33.57 | 1.002 / 33.52 | +| bf16 | seg2-stride4 | 0.952 / 33.57 | 1.010 / 33.52 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | not measured | not measured | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.497 / 17.99 | 2.058 / 17.93 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | not measured | not measured | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 4.920 / 18.10 | 4.338 / 18.05 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +SDNQ Hadamard numbers use `sdnq_compile_mode=eager`. The compiled SDNQ path quantizes HiDream, but this sweep spent the first training step in Inductor dequantizer compilation, so it is not listed as a throughput row. + +### HunyuanVideo + +Example: `hunyuanvideo-1.5-t2v.peft-lora`. Training shape: 480 pixel-area video buckets, 48 frames, batch 2. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | not run | not run | +| bf16 | layer | 7.682 / 26.35 | 22.816 / 26.30 | +| bf16 | interval2 | 7.398 / 26.11 | 22.772 / 26.06 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 10.679 / 58.37 | not run | +| int8-sdnq-hadamard | none | not run | not run | +| int8-sdnq-hadamard | activation-offload | not run | not run | +| int8-sdnq-hadamard | layer | 11.765 / 25.96 | 34.464 / 25.92 | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | not run | not run | +| fp8-torchao | none | not run | not run | +| fp8-torchao | activation-offload | not run | not run | +| fp8-torchao | layer | 10.516 / 33.55 | 32.003 / 33.53 | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | not run | not run | + +HunyuanVideo is activation-heavy at this training shape. Per-block and interval-2 checkpointing both fit cleanly; leaving checkpointing off does not fit on an 80 GB H100. `seg2-stride4` only fit in this sweep when attention activation offload was enabled, and that row is a fallback rather than a speed recommendation. SDNQ Hadamard works, but variable conditioning shapes still trigger dynamic-kernel compilation in the measured window. + +### Ideogram 4.0 + +Example: `ideogram-fp8.peft-lora`. Resolution: 1024x1024. The fp8 flavour uses Ideogram 4's native weight-only fp8 checkpoint (`base_model_precision=no_change`). + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| fp8-native | none | failed | OOM | +| fp8-native | activation-offload | unsupported | unsupported | +| fp8-native | layer | 1.033 / 12.57 | 3.101 / 11.82 | +| fp8-native | interval2 | 1.030 / 12.33 | 3.098 / 11.82 | +| fp8-native | seg2-stride4 | 1.031 / 12.33 | 3.088 / 11.82 | +| fp8-native | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.702 / 61.78 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.033 / 12.33 | 3.030 / 11.82 | +| int8-sdnq-hadamard | interval2 | 1.032 / 12.33 | 3.034 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4 | 1.028 / 12.33 | 3.035 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | + +### Kandinsky 5 Image + +Example: `kandinsky5-image-6b-t2i.lycoris-lokr`. Resolution: 1024x1024. + +Note: batch 3 at 1024x1024 needs full checkpointing on both cards. SDNQ with Hadamard is the best low-VRAM row; H100 can also use partial checkpointing with SDNQ, but only near the top of the card. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.956 / 25.52 | 10.189 / 25.42 | +| bf16 | layer | 6.590 / 25.58 | 9.458 / 25.55 | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 7.186 / 20.00 | 10.141 / 19.95 | +| int8-sdnq-hadamard | layer | 6.830 / 20.12 | 9.362 / 20.08 | +| int8-sdnq-hadamard | interval2 | 5.746 / 75.67 | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 6.057 / 71.46 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 8.319 / 24.29 | 14.716 / 24.24 | +| fp8-torchao | layer | 7.976 / 24.40 | 13.949 / 24.35 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kandinsky 5 Video + +Example: `kandinsky5-video-2b-t2v.peft-lora`. Resolution: 768x512, 81f. + +Kandinsky 5 video is activation-heavy at this frame count. Full block checkpointing is the practical baseline on both cards. On H100, `interval2` and `seg2-stride4` are faster when they fit; on L40S, SDNQ `interval2` is the only partial-checkpoint row here that fits without attention activation offload. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 2.580 / 9.49 | 7.136 / 9.45 | +| bf16 | layer | 2.267 / 9.83 | 6.379 / 9.79 | +| bf16 | interval2 | 1.967 / 44.57 | OOM | +| bf16 | seg2-stride4 | 1.971 / 46.62 | OOM | +| bf16 | seg2-stride4-offload | 2.275 / 37.99 | 6.249 / 37.94 | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 2.844 / 8.07 | 7.234 / 8.02 | +| int8-sdnq-hadamard | layer | 2.460 / 8.40 | 6.509 / 8.35 | +| int8-sdnq-hadamard | interval2 | 2.126 / 43.12 | 5.641 / 43.08 | +| int8-sdnq-hadamard | seg2-stride4 | 2.125 / 45.19 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 2.451 / 36.56 | 6.322 / 36.51 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 3.867 / 15.11 | 11.501 / 15.06 | +| fp8-torchao | layer | 3.579 / 15.28 | 10.822 / 15.24 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kolors + +Example: `kolors.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.635 / 7.22 | 0.628 / 7.17 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.118 / 5.36 | 1.065 / 5.31 | +| bf16 | interval2 | 1.105 / 5.36 | 1.068 / 5.31 | +| bf16 | seg2-stride4 | 1.110 / 5.36 | 1.072 / 5.31 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.860 / 5.68 | 1.726 / 5.63 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.967 / 3.38 | 2.803 / 3.33 | +| int8-sdnq-hadamard | interval2 | 2.770 / 3.32 | 2.655 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4 | 2.805 / 3.32 | 2.742 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.629 / 8.79 | 1.637 / 8.75 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.044 / 3.50 | 3.013 / 3.45 | +| fp8-torchao | interval2 | 3.040 / 3.50 | 3.019 / 3.45 | +| fp8-torchao | seg2-stride4 | 2.988 / 3.50 | 2.935 / 3.45 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Krea 2 + +Example: `krea2.peft-lora`. Training resolution: 512 px square crop. Example validation setting: 1024x1024; validation was disabled for the benchmark. + +The main table uses regional compilation, which is good for Krea2 step speed but not a clean VRAM comparison: the compiled graph/workspace keeps the peak close to the uncheckpointed peak for several checkpoint modes. A bf16 control run with regional compilation disabled showed checkpointing is wired and has the expected memory/speed shape: + +The `activation-offload` row here means full-block checkpointing plus attention activation offload. Against full-block `layer` checkpointing alone, attention offload did not reduce Krea2 peak VRAM in this shape; it mostly added CPU transfer overhead. + +| Mode | H100 no-compile | L40S no-compile | +| --- | ---: | ---: | +| none | 0.272 / 40.09 | 0.661 / 40.01 | +| layer | 0.371 / 30.06 | 0.919 / 30.01 | +| seg2-stride4 | 0.317 / 34.75 | 0.788 / 34.70 | +| activation-offload | 0.657 / 30.50 | 1.341 / 30.30 | + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.275 / 40.06 | 0.662 / 40.01 | +| bf16 | activation-offload | 0.416 / 34.62 | 0.822 / 34.57 | +| bf16 | layer | 0.268 / 40.06 | 0.665 / 40.01 | +| bf16 | interval2 | 0.274 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4 | 0.279 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4-offload | 0.404 / 34.62 | 0.819 / 34.57 | +| int8-sdnq-hadamard | none | 0.462 / 27.01 | 0.807 / 26.96 | +| int8-sdnq-hadamard | activation-offload | 0.773 / 21.57 | 1.012 / 21.53 | +| int8-sdnq-hadamard | layer | 0.473 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | interval2 | 0.474 / 27.01 | 0.804 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4 | 0.472 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4-offload | 0.744 / 21.57 | 1.007 / 21.53 | +| fp8-torchao | none | 0.689 / 51.63 | OOM | +| fp8-torchao | activation-offload | 0.975 / 37.77 | 2.058 / 37.73 | +| fp8-torchao | layer | 0.689 / 51.63 | OOM | +| fp8-torchao | interval2 | 0.684 / 51.63 | OOM | +| fp8-torchao | seg2-stride4 | 0.674 / 51.63 | OOM | +| fp8-torchao | seg2-stride4-offload | 0.965 / 37.77 | 2.053 / 37.73 | + +### LongCat Image + +Example: `longcat-image.peft-lora`. Training resolution: 512 px square; validation resolution: 1024x1024. Rows use `attention_mechanism=native-flash`. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.193 / 16.47 | 0.262 / 16.42 | +| bf16 | activation-offload | 0.544 / 12.73 | 0.543 / 12.69 | +| bf16 | layer | 0.327 / 12.38 | 0.370 / 12.34 | +| bf16 | interval2 | 0.257 / 14.38 | 0.313 / 14.34 | +| bf16 | seg2-stride4 | 0.263 / 14.36 | 0.316 / 14.31 | +| bf16 | seg2-stride4-offload | 0.446 / 13.42 | 0.492 / 13.38 | +| int8-sdnq-hadamard | none | 0.578 / 12.54 | 0.537 / 12.45 | +| int8-sdnq-hadamard | activation-offload | 1.184 / 7.43 | 1.185 / 7.39 | +| int8-sdnq-hadamard | layer | 0.901 / 7.19 | 0.911 / 7.09 | +| int8-sdnq-hadamard | interval2 | 0.718 / 9.73 | 0.695 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4 | 0.735 / 9.72 | 0.717 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.056 / 8.49 | 1.011 / 8.45 | +| fp8-torchao | none | 0.602 / 25.19 | 0.834 / 25.14 | +| fp8-torchao | activation-offload | 1.662 / 7.75 | 1.844 / 7.70 | +| fp8-torchao | layer | 0.984 / 7.57 | 1.080 / 7.53 | +| fp8-torchao | interval2 | 0.750 / 16.15 | 0.938 / 16.10 | +| fp8-torchao | seg2-stride4 | 0.760 / 16.13 | 0.961 / 16.09 | +| fp8-torchao | seg2-stride4-offload | 1.287 / 13.02 | 1.653 / 12.98 | + +### LongCat Video + +Example: `longcat-video.peft-lora+ramtorch`. Resolution: 832x480, 81f. Rows use `attention_mechanism=native-flash`. + +LongCat Video is activation-heavy at this shape. Full per-block checkpointing is the practical row. The partial checkpoint rows (`interval2`, `seg2-stride4`) do not fit here, even when attention activation offload is enabled for the strided row. Plain attention activation offload fits for bf16 and SDNQ, but it is much slower than full checkpointing. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM (77.86 GiB) | OOM (43.30 GiB) | +| bf16 | activation-offload | 25.774 / 37.36 | 49.149 / 37.14 | +| bf16 | layer | 7.448 / 23.73 | 24.866 / 23.68 | +| bf16 | interval2 | OOM (76.41 GiB) | OOM (42.80 GiB) | +| bf16 | seg2-stride4 | OOM (76.42 GiB) | OOM (43.06 GiB) | +| bf16 | seg2-stride4-offload | OOM (76.72 GiB) | OOM (42.29 GiB) | +| int8-sdnq-hadamard | none | OOM (77.40 GiB) | OOM (43.59 GiB) | +| int8-sdnq-hadamard | activation-offload | 30.887 / 35.28 | 61.270 / 35.24 | +| int8-sdnq-hadamard | layer | 8.444 / 21.60 | 25.164 / 21.55 | +| int8-sdnq-hadamard | interval2 | OOM (76.01 GiB) | OOM (42.47 GiB) | +| int8-sdnq-hadamard | seg2-stride4 | OOM (77.01 GiB) | OOM (43.02 GiB) | +| int8-sdnq-hadamard | seg2-stride4-offload | OOM (76.42 GiB) | OOM (42.57 GiB) | +| fp8-torchao | none | OOM (75.87 GiB) | OOM (41.28 GiB) | +| fp8-torchao | activation-offload | 30.163 / 47.88 | OOM (40.11 GiB) | +| fp8-torchao | layer | 8.343 / 34.16 | 24.659 / 34.07 | +| fp8-torchao | interval2 | OOM (74.63 GiB) | OOM (40.85 GiB) | +| fp8-torchao | seg2-stride4 | OOM (75.37 GiB) | OOM (41.19 GiB) | +| fp8-torchao | seg2-stride4-offload | OOM (75.55 GiB) | OOM (40.94 GiB) | + +### LTXVideo 0.9.5 + +Example: `ltxvideo-0.9.5-t2v.peft-lora`. Resolution: 768x512, 49f. + +Numbers are warm seconds per step / peak GiB. The full-run average includes setup and compile overhead and is recorded in the sweep artifacts. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.274 / 8.77 | 0.275 / 8.72 | +| bf16 | layer | 0.462 / 4.21 | 0.459 / 4.16 | +| bf16 | interval2 | 0.449 / 4.30 | 0.433 / 4.25 | +| bf16 | seg2-stride4 | 0.359 / 6.40 | 0.357 / 6.35 | +| int8-sdnq-hadamard | none | 0.688 / 7.05 | 0.640 / 6.90 | +| int8-sdnq-hadamard | layer | 1.094 / 2.59 | 1.081 / 2.44 | +| int8-sdnq-hadamard | interval2 | 1.112 / 2.57 | 1.073 / 2.53 | +| int8-sdnq-hadamard | seg2-stride4 | 0.887 / 4.61 | 0.817 / 4.56 | +| fp8-torchao | none | 0.655 / 17.45 | 0.735 / 17.41 | +| fp8-torchao | layer | 1.226 / 2.94 | 1.206 / 2.89 | +| fp8-torchao | interval2 | 1.259 / 3.40 | 1.233 / 3.35 | +| fp8-torchao | seg2-stride4 | 0.933 / 9.96 | 0.901 / 9.91 | +| fp8wo-torchao | none | 0.328 / 10.06 | 0.325 / 10.01 | +| fp8wo-torchao | layer | 0.567 / 2.64 | 0.540 / 2.59 | +| fp8wo-torchao | interval2 | 0.555 / 2.84 | 0.531 / 2.79 | +| fp8wo-torchao | seg2-stride4 | 0.443 / 6.20 | 0.432 / 6.15 | + +Attention activation offload rows are unsupported for LTXVideo 0.9 in this sweep. + +### LTXVideo2 2.3 + +Example: `ltxvideo2-2.3-dev-720p-single-gpu.peft-lora+sdnq-hadamard`. Resolution: 1280x704, 49f. + +Note: LTXVideo2 2.3 should be read from the no-regional-compile rows in this sweep; regional compile raised memory pressure for this model. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.350 / 58.36 | OOM | +| bf16 | layer | 3.993 / 47.95 | OOM | +| bf16 | interval2 | 3.977 / 48.83 | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 5.554 / 75.64 | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 10.102 / 38.20 | 9.852 / 38.15 | +| int8-sdnq-hadamard | layer | 7.753 / 27.78 | 7.579 / 27.73 | +| int8-sdnq-hadamard | interval2 | 7.733 / 28.66 | 7.288 / 28.61 | +| int8-sdnq-hadamard | seg2-stride4 | 6.522 / 61.50 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 8.659 / 55.48 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | failed | 22.917 / 38.57 | +| fp8-torchao | layer | 8.580 / 30.27 | 10.381 / 30.22 | +| fp8-torchao | interval2 | 8.660 / 33.71 | 10.661 / 33.66 | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | failed | OOM | + +### Lumina2 + +Example: `lumina2.peft-lora`. Resolution: 512x512. + +Note: Lumina2 now uses the segmented whole-block path. `interval2` checkpoints every two-block segment; `seg2-stride4` checkpoints two blocks, lets the next two keep activations, then repeats. Attention activation offload was not part of this Lumina2 run. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.235 / 15.99 | 0.332 / 15.94 | +| bf16 | layer | 0.384 / 6.60 | 0.457 / 6.56 | +| bf16 | interval2 | 0.356 / 6.87 | 0.424 / 6.82 | +| bf16 | seg2-stride4 | 0.295 / 11.43 | 0.377 / 11.38 | +| int8-sdnq-hadamard | none | 0.584 / 13.59 | 0.541 / 13.55 | +| int8-sdnq-hadamard | layer | 0.899 / 4.21 | 0.827 / 4.16 | +| int8-sdnq-hadamard | interval2 | 0.865 / 4.48 | 0.835 / 4.43 | +| int8-sdnq-hadamard | seg2-stride4 | 0.719 / 9.03 | 0.707 / 8.99 | +| fp8-torchao | none | 0.598 / 28.03 | 0.763 / 27.98 | +| fp8-torchao | layer | 0.950 / 5.93 | 0.967 / 5.88 | +| fp8-torchao | interval2 | 0.938 / 6.70 | 0.974 / 6.66 | +| fp8-torchao | seg2-stride4 | 0.765 / 17.36 | 0.901 / 17.32 | +| fp8wo-torchao | none | 0.273 / 17.97 | 0.389 / 17.93 | +| fp8wo-torchao | layer | 0.452 / 4.99 | 0.525 / 4.94 | +| fp8wo-torchao | interval2 | 0.427 / 5.40 | 0.522 / 5.35 | +| fp8wo-torchao | seg2-stride4 | 0.360 / 11.68 | 0.466 / 11.64 | + +### MageFlow + +Example: `mageflow-image-24g.peft-lora`. Resolution: 1024x1024. + +Note: MageFlow's 1024px variable-shape image path is mostly helped by attention activation offload and weight-only FP8. Block checkpointing modes are valid, but did not lower measured peak residency in this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 7.735 / 36.74 | 7.658 / 36.69 | +| bf16 | activation-offload | 8.902 / 23.05 | 9.477 / 23.00 | +| bf16 | layer | 7.805 / 36.74 | 7.504 / 36.69 | +| bf16 | interval2 | 8.036 / 36.74 | 7.762 / 36.69 | +| bf16 | seg2-stride4 | 7.882 / 36.74 | 7.531 / 36.69 | +| bf16 | seg2-stride4-offload | 8.833 / 23.05 | 9.288 / 23.01 | +| int8-sdnq-hadamard | none | 81.016 / 37.38 | 94.991 / 37.34 | +| fp8wo-torchao | none | 5.454 / 36.86 | 5.772 / 36.82 | +| fp8wo-torchao | activation-offload | 6.295 / 23.18 | 6.738 / 23.14 | +| fp8wo-torchao | seg2-stride4 | 5.542 / 36.86 | 5.595 / 36.82 | + +### OmniGen + +Example: `omnigen.lycoris-lokr`. Resolution: 1024x1024. + +Note: OmniGen uses token-ID prompts instead of cached text embeddings. These rows measure the supported no-checkpoint and full-block torch checkpointing paths; interval, segmented-stride, and attention-offload controls are not implemented for this family in this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.425 / 14.20 | 0.293 / 14.24 | +| bf16 | layer | 0.597 / 10.13 | 0.389 / 10.09 | +| int8-sdnq-hadamard | none | 1.523 / 11.00 | 1.388 / 11.06 | +| int8-sdnq-hadamard | layer | 1.312 / 6.75 | 1.064 / 6.70 | +| fp8-torchao | none | 0.690 / 19.25 | 0.608 / 19.30 | +| fp8-torchao | layer | 1.069 / 7.09 | 0.824 / 7.04 | +| fp8wo-torchao | none | 0.454 / 17.71 | 0.377 / 17.73 | +| fp8wo-torchao | layer | 0.646 / 6.98 | 0.534 / 6.94 | + +### PixArt + +Example: `pixart.lycoris-lokr`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.700 / 41.73 | 1.734 / 41.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.433 / 5.63 | 2.346 / 5.58 | +| bf16 | interval2 | 2.440 / 6.01 | 2.348 / 5.96 | +| bf16 | seg2-stride4 | 2.092 / 23.90 | 2.072 / 23.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.905 / 47.58 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.738 / 6.07 | 2.937 / 6.03 | +| int8-sdnq-hadamard | interval2 | 2.734 / 6.03 | 2.943 / 5.99 | +| int8-sdnq-hadamard | seg2-stride4 | 2.336 / 26.85 | 2.596 / 26.81 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.254 / 8.06 | 4.646 / 8.01 | +| fp8-torchao | interval2 | 3.260 / 9.66 | 4.649 / 9.61 | +| fp8-torchao | seg2-stride4 | 2.827 / 63.27 | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Qwen Image + +Example: `qwen_image.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.202 / 41.04 | 3.377 / 40.99 | +| bf16 | interval2 | 1.201 / 41.04 | 3.382 / 40.99 | +| bf16 | seg2-stride4 | 1.205 / 41.03 | 3.385 / 40.99 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.675 / 63.48 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.640 / 24.09 | 3.928 / 24.05 | +| int8-sdnq-hadamard | interval2 | 2.722 / 24.09 | 3.918 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4 | 2.663 / 24.09 | 3.919 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.088 / 25.34 | 6.172 / 25.29 | +| fp8-torchao | interval2 | 3.095 / 25.34 | 6.173 / 25.29 | +| fp8-torchao | seg2-stride4 | 3.125 / 25.34 | 6.141 / 25.29 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Sana + +Example: `sana.lycoris-lokr`. Resolution: 1024x1024. + +Note: Sana has interval checkpointing; stride is not a separate segmented schedule for this family in the measured rows. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 23.71 | 0.597 / 23.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.529 / 23.72 | 0.596 / 23.66 | +| bf16 | interval2 | 0.530 / 23.72 | 0.598 / 23.66 | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.554 / 22.92 | 0.590 / 22.88 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.554 / 22.92 | 0.589 / 22.88 | +| int8-sdnq-hadamard | interval2 | 0.556 / 22.92 | 0.591 / 22.88 | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.633 / 33.67 | 0.753 / 33.62 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.633 / 33.67 | 0.755 / 33.62 | +| fp8-torchao | interval2 | 0.631 / 33.67 | 0.759 / 33.62 | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SanaVideo + +Example: `sanavideo-2b-480p.peft-lora`. Resolution: 832x480, 49f. + +Note: SanaVideo uses linear attention, so attention activation offload remains unsupported. Segmented whole-block checkpointing is supported for the standard path. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.599 / 59.15 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.597 / 59.15 | OOM | +| bf16 | interval2 | not run | OOM | +| bf16 | seg2-stride4 | not run | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | interval2 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD 1.x + +Example: `sd1x-dreamshaper.peft-lora`. Resolution: 512x512. + +Note: SD1x uses the diffusers UNet path. Regular layer checkpointing is supported, but interval and segmented stride controls are not wired for this family. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.181 / 2.87 | 0.176 / 2.83 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.305 / 1.98 | 0.292 / 1.93 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.446 / 3.07 | 0.431 / 3.04 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.701 / 1.79 | 0.666 / 1.74 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.410 / 4.23 | 0.401 / 4.18 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.785 / 1.96 | 0.756 / 1.91 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD3 + +Example: `sd3.peft-lora`. Resolution: 1024x1024. + +Note: SD3 uses true contiguous segmented checkpointing on the plain transformer path. Attention activation offload is supported; it cuts VRAM hard, but costs throughput. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 34.59 | 1.189 / 34.53 | +| bf16 | activation-offload | 1.335 / 9.68 | 2.850 / 9.63 | +| bf16 | layer | 0.721 / 7.67 | 1.607 / 7.62 | +| bf16 | interval2 | 0.723 / 8.93 | 1.606 / 8.88 | +| bf16 | seg2-stride4 | 0.620 / 20.14 | 1.398 / 20.09 | +| bf16 | seg2-stride4-offload | 1.229 / 12.21 | 2.639 / 12.16 | +| int8-sdnq-hadamard | none | 0.858 / 33.66 | 1.381 / 33.61 | +| int8-sdnq-hadamard | activation-offload | 1.845 / 8.90 | 3.297 / 8.85 | +| int8-sdnq-hadamard | layer | 1.265 / 6.58 | 1.857 / 6.53 | +| int8-sdnq-hadamard | interval2 | 1.264 / 7.30 | 1.858 / 7.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.049 / 19.02 | 1.625 / 18.98 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.527 / 12.42 | 3.023 / 12.37 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 4.707 / 10.27 | 9.814 / 10.23 | +| fp8-torchao | layer | 1.575 / 9.13 | 3.279 / 9.09 | +| fp8-torchao | interval2 | 1.576 / 12.62 | 3.282 / 12.58 | +| fp8-torchao | seg2-stride4 | 1.376 / 45.70 | OOM | +| fp8-torchao | seg2-stride4-offload | 4.184 / 26.17 | 8.714 / 26.12 | +| fp8wo-torchao | none | 0.567 / 35.21 | 1.252 / 35.17 | +| fp8wo-torchao | activation-offload | 1.392 / 8.71 | 2.921 / 8.67 | +| fp8wo-torchao | layer | 0.795 / 5.69 | 1.736 / 5.65 | +| fp8wo-torchao | interval2 | 0.797 / 7.08 | 1.734 / 7.03 | +| fp8wo-torchao | seg2-stride4 | 0.679 / 19.43 | 1.490 / 19.38 | +| fp8wo-torchao | seg2-stride4-offload | 1.257 / 12.00 | 2.676 / 11.96 | + +### SDXL + +Example: `sdxl.lycoris-lokr`. Resolution: 1024x1024. + +Note: SDXL has real layer checkpointing. Interval and stride rows are included as coverage data, not as segmented-support recommendations. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.606 / 13.03 | 0.585 / 12.98 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.080 / 6.53 | 1.029 / 6.48 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.741 / 13.72 | 1.643 / 13.68 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.820 / 4.64 | 2.647 / 4.59 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.608 / 26.22 | 1.582 / 26.16 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.939 / 5.10 | 2.890 / 5.04 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Stable Cascade + +Example: `cascade-stage-c.lycoris-lokr`. Resolution: 1024x1024. + +Note: Stage C is a full-precision prior path. These rows ran with `mixed_precision=no` and `base_model_precision=no_change`; quantized base precision rows are not meaningful for this model. The interval and stride modes operate over the UNet's Res/Timestep/Attention micro-block sequence. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.884 / 51.52 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.179 / 22.68 | 2.135 / 22.61 | +| bf16 | interval2 | 1.032 / 36.99 | 1.871 / 36.92 | +| bf16 | seg2-stride4 | 1.032 / 37.20 | 1.870 / 37.13 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | unsupported | unsupported | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | unsupported | unsupported | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Wan 2.1 T2V 1.3B + +Example: `wan2.1-t2v-1.3b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 1.3B should be read from the no-regional-compile/RamTorch rows; regional compile was not a useful throughput setting in this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.407 / 71.90 | OOM | +| bf16 | activation-offload | 3.472 / 8.78 | 7.179 / 8.66 | +| bf16 | layer | 2.099 / 4.73 | 4.459 / 4.68 | +| bf16 | interval2 | 2.139 / 6.32 | 4.514 / 6.27 | +| bf16 | seg2-stride4 | 1.806 / 39.25 | 3.921 / 39.21 | +| bf16 | seg2-stride4-offload | 2.993 / 22.66 | 6.493 / 22.61 | +| int8-sdnq-hadamard | none | 1.850 / 71.91 | OOM | +| int8-sdnq-hadamard | activation-offload | 4.204 / 8.72 | 7.387 / 8.68 | +| int8-sdnq-hadamard | layer | 2.790 / 4.70 | 4.989 / 4.65 | +| int8-sdnq-hadamard | interval2 | 2.874 / 6.29 | 5.093 / 6.24 | +| int8-sdnq-hadamard | seg2-stride4 | 2.558 / 39.27 | 4.393 / 39.22 | +| int8-sdnq-hadamard | seg2-stride4-offload | 3.711 / 22.67 | 6.695 / 22.63 | +| fp8-torchao | none | 1.727 / 73.57 | OOM | +| fp8-torchao | activation-offload | 4.061 / 10.08 | 7.404 / 9.96 | +| fp8-torchao | layer | 2.607 / 5.98 | 4.888 / 5.93 | +| fp8-torchao | interval2 | 2.744 / 7.57 | 4.916 / 7.52 | +| fp8-torchao | seg2-stride4 | 2.246 / 40.55 | 4.245 / 40.50 | +| fp8-torchao | seg2-stride4-offload | 3.602 / 24.02 | 6.683 / 23.91 | + +### Wan 2.1 T2V 14B + +Example: `wan2.1-t2v-14b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 14B is mainly a fit test for activation savings. Status-only cells are still useful because they show which combinations reached the memory limit. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 13.144 / 36.80 | OOM | +| bf16 | layer | 7.162 / 16.28 | 21.770 / 16.23 | +| bf16 | interval2 | 7.172 / 19.62 | 21.777 / 19.58 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | failed | failed | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | failed | failed | +| fp8-torchao | seg2-stride4 | failed | failed | +| fp8-torchao | seg2-stride4-offload | failed | failed | + +### Wan S2V + +Example: `wan-s2v-14b-480p.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan S2V is included as coverage data for the video/audio path. Treat failed cells as implementation coverage gaps. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | failed | failed | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Z-Image Turbo + +Example: `z-image-turbo.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.243 / 21.25 | 0.316 / 21.21 | +| bf16 | activation-offload | 0.837 / 13.24 | 0.805 / 13.19 | +| bf16 | layer | 0.479 / 12.87 | 0.493 / 12.83 | +| bf16 | interval2 | 0.452 / 13.04 | 0.477 / 12.99 | +| bf16 | seg2-stride4 | 0.349 / 16.88 | 0.400 / 16.83 | +| bf16 | seg2-stride4-offload | 0.681 / 15.03 | 0.736 / 14.99 | +| int8-sdnq-hadamard | none | 0.645 / 15.60 | 0.615 / 15.55 | +| int8-sdnq-hadamard | activation-offload | 1.439 / 7.61 | 1.382 / 7.56 | +| int8-sdnq-hadamard | layer | 1.046 / 7.25 | 1.021 / 7.20 | +| int8-sdnq-hadamard | interval2 | 1.074 / 7.41 | 0.996 / 7.36 | +| int8-sdnq-hadamard | seg2-stride4 | 0.867 / 11.25 | 0.841 / 11.20 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.202 / 9.39 | 1.162 / 9.35 | +| fp8-torchao | none | 1.232 / 37.50 | 1.476 / 37.46 | +| fp8-torchao | activation-offload | 3.623 / 7.97 | 3.564 / 7.93 | +| fp8-torchao | layer | 2.319 / 7.93 | 2.344 / 7.88 | +| fp8-torchao | interval2 | 2.336 / 8.80 | 2.309 / 8.75 | +| fp8-torchao | seg2-stride4 | 1.843 / 22.55 | 1.930 / 22.50 | +| fp8-torchao | seg2-stride4-offload | 2.947 / 15.57 | 3.243 / 15.52 | + +### ZLab I1 + +Example: `zlab-i1.peft-lora`. Resolution: 1024x1024. + +Note: ZLab I1 carries its U-Net-style skip tensors through the segmented checkpoint state. Attention activation offload is not wired for this family. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.462 / 22.21 | 0.865 / 22.16 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.693 / 7.79 | 1.148 / 7.75 | +| bf16 | interval2 | 0.676 / 8.30 | 1.152 / 8.25 | +| bf16 | seg2-stride4 | 0.567 / 14.97 | 1.014 / 14.92 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.861 / 19.21 | 0.926 / 19.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.385 / 4.78 | 1.265 / 4.74 | +| int8-sdnq-hadamard | interval2 | 1.298 / 5.30 | 1.277 / 5.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.073 / 11.98 | 1.098 / 11.93 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8wo-torchao | none | 0.504 / 25.08 | 0.930 / 25.02 | +| fp8wo-torchao | activation-offload | unsupported | unsupported | +| fp8wo-torchao | layer | 0.772 / 5.12 | 1.280 / 5.07 | +| fp8wo-torchao | interval2 | 0.759 / 5.84 | 1.290 / 5.79 | +| fp8wo-torchao | seg2-stride4 | 0.633 / 15.12 | 1.115 / 15.08 | +| fp8wo-torchao | seg2-stride4-offload | unsupported | unsupported | + + diff --git a/documentation/experimental/SEGMENTED_CHECKPOINTING.pt-BR.md b/documentation/experimental/SEGMENTED_CHECKPOINTING.pt-BR.md new file mode 100644 index 000000000..da5b4cf20 --- /dev/null +++ b/documentation/experimental/SEGMENTED_CHECKPOINTING.pt-BR.md @@ -0,0 +1,1007 @@ +# Checkpointing Segmentado + +Checkpointing segmentado e o meio-termo entre checkpoint em todo block e nenhum checkpoint. + +Ele usa o backend de activation checkpointing do PyTorch. O SimpleTuner executa um grupo contiguo de transformer blocks em uma chamada de checkpoint e passa o hidden state retornado para o proximo grupo. Grupos mais largos recomputam menos no backward, mas mantem mais activations vivas. + +Para CPU offload e FFN-only checkpointing, veja [Unsloth-style checkpointing](UNSLOTH_CHECKPOINTING.pt-BR.md#controls). A regra curta continua em [Decision Rule](UNSLOTH_CHECKPOINTING.pt-BR.md#decision-rule). + +## Controles + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2 +} +``` + +Nos caminhos whole-block suportados, `gradient_checkpointing_interval` e a largura do segment. `2` significa checkpoint dos blocks `0-1`, `2-3`, `4-5` e assim por diante. + +Para controle mais fino de VRAM, adicione stride: + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2, + "gradient_checkpointing_segment_stride": 4 +} +``` + +Isso faz checkpoint dos blocks `0-1`, executa `2-3` normalmente, faz checkpoint de `4-5`, executa `6-7` normalmente e repete. O stride deve ser pelo menos o interval; schedules sobrepostos nao sao validos. + +Caminhos segmented whole-block suportados: Flux.1, Flux.2, HunyuanVideo, Krea 2, LongCat Image, LongCat Video, LTXVideo 0.9, LTXVideo2, Lumina2, MageFlow, PixArt, SD3, SanaVideo, Z-Image, ZLab I1 e Wan. + +Stable Cascade stage C tambem suporta interval e stride, mas aplica o schedule a sequencia de micro-blocos Res/Timestep/Attention do UNet em vez de grupos transformer whole-block. + +Algumas familias usam semantica de interval especifica do modelo: + +| Family | `gradient_checkpointing_interval` | `gradient_checkpointing_segment_stride` | +| --- | --- | --- | +| Sana | Checkpoint a cada N-th block | Ignorado | +| Stable Cascade stage C | Checkpoint de micro-blocos UNet por interval | Stride alterna janelas UNet checkpointed e non-checkpointed | +| SD1x, SDXL | Sem suporte segmented whole-block | Ignorado | + +Nao compare linhas stride quando stride e ignorado. Se os numeros forem identicos, normalmente a option nao foi aplicada. + +## Quando Usar + +Use depois que o checkpointing normal por block couber, mas custar tempo demais por step. Comece com `2`. Se houver VRAM, tente `2` com stride `4` em modelos muito profundos. + +Nao espere ajuda quando o peak vem principalmente de pesos treinaveis, optimizer state, validation, cache de VAE, block swapping ou routing. O SimpleTuner volta para o caminho per-block mais seguro quando uma feature do modelo precisa desse controle. + +`dynamo_use_regional_compilation` nao e ganho universal. Ele ajudou ou foi neutro em alguns image models, mas foi ruim nos perfis Wan/RamTorch e LTXVideo2 abaixo. + +## Benchmarks + +Medido com exemplos reais do SimpleTuner em pods single-GPU H100 e L40S. Validation e checkpoint saves ficaram desativados, cache preparation foi excluida, e o compile/setup do primeiro step dentro do train loop e excluido quando ha timing post-warmup. + +Cada celula medida e `post-warmup sec/step / peak VRAM GiB`. Celulas somente com status significam: `OOM` ficou sem memoria de GPU, `failed` nao chegou aos training steps medidos, `unsupported` significa que a opcao nao estava conectada para essa familia, e `not run` significa que o sweep nao incluiu essa combinacao. + +Compare modos dentro da mesma familia primeiro. Comparacoes entre familias sao aproximadas porque resolution, frame count, attention backend, model depth, trainable adapter type e dataset shape mudam. + +A matriz abaixo e a fonte de verdade para este sweep. Notas especificas por modelo marcam caveats quando uma linha e coverage data em vez de recomendacao. + + + +### Resultados Por Familia + +### ACE Step 1.5 + +Example: `ace_step-v1-5.peft-lora`. Resolution: 512. + +Note: This sweep did not produce a usable ACE Step throughput row. The status-only entries below should be treated as coverage gaps, not as a recommendation. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | OOM | OOM | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | OOM | OOM | +| int8-sdnq-hadamard | interval2 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Destilacao AnyFlow no Anima + +Exemplo: `anima-anyflow.peft-lora`. Resolucao: 1024x1024. Esta linha mede destilacao AnyFlow usando Anima, nao o exemplo LoRA de Anima puro. Use `anima.peft-lora` para treinamento de imagem Anima puro em 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.022 / 17.26 | 0.719 / 17.21 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.232 / 5.60 | 0.903 / 5.56 | +| bf16 | interval2 | 1.252 / 5.60 | 0.897 / 5.56 | +| bf16 | seg2-stride4 | 1.244 / 5.60 | 0.898 / 5.56 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 5.417 / 18.61 | 4.974 / 18.57 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.562 / 4.36 | 4.019 / 4.31 | +| int8-sdnq-hadamard | interval2 | 3.723 / 4.36 | 3.196 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4 | 3.658 / 4.36 | 3.140 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.242 / 45.71 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.810 / 5.51 | 2.576 / 5.46 | +| fp8-torchao | interval2 | 2.846 / 5.51 | 2.581 / 5.46 | +| fp8-torchao | seg2-stride4 | 2.766 / 5.51 | 2.567 / 5.46 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### AuraFlow + +Example: `auraflow.peft-lora`. Resolution: 1024x1024. + +Nota: AuraFlow suporta quantizacao com SDNQ e TorchAO. As linhas quantizadas sem checkpointing estao abaixo; as linhas quantizadas com checkpointing precisam de uma nova medicao completa antes de receber numeros aqui. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.180 / 19.19 | 0.233 / 19.12 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.824 / 13.37 | 0.833 / 13.32 | +| bf16 | interval2 | 1.764 / 16.21 | 0.877 / 16.14 | +| bf16 | seg2-stride4 | 1.771 / 16.21 | 0.887 / 16.14 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.642 / 12.88 | 0.610 / 12.87 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | not run | not run | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.621 / 24.53 | 0.757 / 24.44 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | not run | not run | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Boogu Image + +Example: `boogu-image-v0.1.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.694 / 59.14 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.907 / 23.44 | 2.641 / 23.39 | +| bf16 | interval2 | 0.912 / 23.44 | 2.649 / 23.39 | +| bf16 | seg2-stride4 | 0.911 / 23.44 | 2.648 / 23.39 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.488 / 53.24 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.878 / 15.20 | 3.309 / 15.15 | +| int8-sdnq-hadamard | interval2 | 1.630 / 34.12 | 2.577 / 34.07 | +| int8-sdnq-hadamard | seg2-stride4 | 1.656 / 34.11 | 2.574 / 34.06 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.713 / 18.48 | 3.778 / 18.44 | +| fp8-torchao | interval2 | 1.731 / 18.48 | 3.777 / 18.44 | +| fp8-torchao | seg2-stride4 | 1.721 / 18.48 | 3.779 / 18.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Chroma + +Example: `chroma.peft-lora`. Resolution: 1024x1024. + +Note: Checkpointed Chroma rows use `attention_mechanism=native-efficient`, which was the stable attention path for this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.454 / 26.18 | 0.559 / 26.13 | +| bf16 | activation-offload | 4.873 / 18.74 | 4.793 / 18.69 | +| bf16 | layer | 1.276 / 17.67 | 1.430 / 17.63 | +| bf16 | interval2 | 1.204 / 21.80 | 1.349 / 21.75 | +| bf16 | seg2-stride4 | 1.200 / 21.78 | 1.382 / 21.74 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.083 / 21.42 | 1.061 / 21.37 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.714 / 10.41 | 1.646 / 10.36 | +| int8-sdnq-hadamard | interval2 | 1.443 / 15.72 | 1.391 / 15.68 | +| int8-sdnq-hadamard | seg2-stride4 | 1.428 / 15.71 | 1.323 / 15.67 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.122 / 45.44 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.821 / 10.64 | 2.104 / 10.60 | +| fp8-torchao | interval2 | 1.447 / 27.49 | 1.871 / 27.44 | +| fp8-torchao | seg2-stride4 | 1.431 / 27.49 | 1.877 / 27.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 2 Image + +Example: `cosmos2image.lycoris-lokr`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.336 / 8.05 | 0.316 / 8.00 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.567 / 4.09 | 0.559 / 4.04 | +| bf16 | interval2 | 0.595 / 4.09 | 0.544 / 4.04 | +| bf16 | seg2-stride4 | 0.598 / 4.09 | 0.546 / 4.04 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.831 / 6.56 | 0.783 / 6.56 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.315 / 2.44 | 1.288 / 2.39 | +| int8-sdnq-hadamard | interval2 | 1.346 / 2.44 | 1.321 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4 | 1.413 / 2.44 | 1.274 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.850 / 13.30 | 0.840 / 13.25 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.692 / 2.70 | 1.560 / 2.65 | +| fp8-torchao | interval2 | 1.626 / 2.70 | 1.544 / 2.65 | +| fp8-torchao | seg2-stride4 | 1.607 / 2.70 | 1.555 / 2.65 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 3 + +Example: `cosmos3-edge-image-24g.lycoris-lokr`. Resolution: 1024 px. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 2.747 / 8.90 | 2.904 / 8.86 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.602 / 8.90 | 2.606 / 8.86 | +| bf16 | interval2 | 2.658 / 8.90 | 2.567 / 8.86 | +| bf16 | seg2-stride4 | 2.628 / 8.90 | 2.899 / 8.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 17.112 / 6.21 | 17.965 / 6.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 3.238 / 6.21 | 2.891 / 6.16 | +| int8-sdnq-hadamard | interval2 | 3.253 / 6.21 | 2.916 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4 | 3.254 / 6.21 | 2.923 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.849 / 17.84 | 1.559 / 17.80 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.869 / 17.84 | 1.599 / 17.79 | +| fp8-torchao | interval2 | 1.855 / 17.84 | 1.600 / 17.80 | +| fp8-torchao | seg2-stride4 | 1.859 / 17.84 | 1.523 / 17.80 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### ERNIE 4.5 Image + +Example: `ernie.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.711 / 15.61 | 1.282 / 15.56 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.110 / 4.94 | 1.840 / 4.90 | +| bf16 | interval2 | 0.876 / 10.16 | 1.536 / 10.12 | +| bf16 | seg2-stride4 | 0.874 / 10.16 | 1.532 / 10.12 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 2.782 / 13.75 | 2.722 / 13.70 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.262 / 2.98 | 4.393 / 2.94 | +| int8-sdnq-hadamard | interval2 | 3.547 / 8.26 | 3.380 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4 | 3.366 / 8.26 | 3.457 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.432 / 14.03 | 2.303 / 13.98 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.920 / 3.26 | 4.008 / 3.22 | +| fp8-torchao | interval2 | 3.316 / 8.54 | 2.973 / 8.49 | +| fp8-torchao | seg2-stride4 | 2.994 / 8.54 | 3.046 / 8.49 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HeartMula + +Example: `heartmula.peft-lora`. Treinamento com tokens de áudio. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HiDream + +Example: `hidream.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.515 / 44.58 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.928 / 33.57 | 1.043 / 33.52 | +| bf16 | interval2 | 0.971 / 33.57 | 1.002 / 33.52 | +| bf16 | seg2-stride4 | 0.952 / 33.57 | 1.010 / 33.52 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | not measured | not measured | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.497 / 17.99 | 2.058 / 17.93 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | not measured | not measured | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 4.920 / 18.10 | 4.338 / 18.05 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +SDNQ Hadamard numbers use `sdnq_compile_mode=eager`. The compiled SDNQ path quantizes HiDream, but this sweep spent the first training step in Inductor dequantizer compilation, so it is not listed as a throughput row. + +### HunyuanVideo + +Exemplo: `hunyuanvideo-1.5-t2v.peft-lora`. Forma de training: video buckets de 480 pixel-area, 48 frames, batch 2. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | not run | not run | +| bf16 | layer | 7.682 / 26.35 | 22.816 / 26.30 | +| bf16 | interval2 | 7.398 / 26.11 | 22.772 / 26.06 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 10.679 / 58.37 | not run | +| int8-sdnq-hadamard | none | not run | not run | +| int8-sdnq-hadamard | activation-offload | not run | not run | +| int8-sdnq-hadamard | layer | 11.765 / 25.96 | 34.464 / 25.92 | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | not run | not run | +| fp8-torchao | none | not run | not run | +| fp8-torchao | activation-offload | not run | not run | +| fp8-torchao | layer | 10.516 / 33.55 | 32.003 / 33.53 | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | not run | not run | + +HunyuanVideo usa muitas activations nessa forma de training. Per-block e interval-2 cabem bem; sem checkpointing nao cabe em um H100 de 80 GB. `seg2-stride4` so coube neste sweep com attention activation offload, e essa linha e uma saida para memoria, nao uma recomendacao de velocidade. SDNQ Hadamard funciona, mas formas variaveis de conditioning ainda disparam compilacao de kernels dinamicos durante a medicao. + +### Ideogram 4.0 + +Exemplo: `ideogram-fp8.peft-lora`. Resolucao: 1024x1024. O flavour fp8 usa o checkpoint fp8 nativo weight-only do Ideogram 4 (`base_model_precision=no_change`). + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| fp8-native | none | failed | OOM | +| fp8-native | activation-offload | unsupported | unsupported | +| fp8-native | layer | 1.033 / 12.57 | 3.101 / 11.82 | +| fp8-native | interval2 | 1.030 / 12.33 | 3.098 / 11.82 | +| fp8-native | seg2-stride4 | 1.031 / 12.33 | 3.088 / 11.82 | +| fp8-native | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.702 / 61.78 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.033 / 12.33 | 3.030 / 11.82 | +| int8-sdnq-hadamard | interval2 | 1.032 / 12.33 | 3.034 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4 | 1.028 / 12.33 | 3.035 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | + +### Kandinsky 5 Image + +Example: `kandinsky5-image-6b-t2i.lycoris-lokr`. Resolution: 1024x1024. + +Note: batch 3 at 1024x1024 needs full checkpointing on both cards. SDNQ with Hadamard is the best low-VRAM row; H100 can also use partial checkpointing with SDNQ, but only near the top of the card. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.956 / 25.52 | 10.189 / 25.42 | +| bf16 | layer | 6.590 / 25.58 | 9.458 / 25.55 | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 7.186 / 20.00 | 10.141 / 19.95 | +| int8-sdnq-hadamard | layer | 6.830 / 20.12 | 9.362 / 20.08 | +| int8-sdnq-hadamard | interval2 | 5.746 / 75.67 | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 6.057 / 71.46 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 8.319 / 24.29 | 14.716 / 24.24 | +| fp8-torchao | layer | 7.976 / 24.40 | 13.949 / 24.35 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kandinsky 5 Video + +Example: `kandinsky5-video-2b-t2v.peft-lora`. Resolution: 768x512, 81f. + +Kandinsky 5 video is activation-heavy at this frame count. Full block checkpointing is the practical baseline on both cards. On H100, `interval2` and `seg2-stride4` are faster when they fit; on L40S, SDNQ `interval2` is the only partial-checkpoint row here that fits without attention activation offload. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 2.580 / 9.49 | 7.136 / 9.45 | +| bf16 | layer | 2.267 / 9.83 | 6.379 / 9.79 | +| bf16 | interval2 | 1.967 / 44.57 | OOM | +| bf16 | seg2-stride4 | 1.971 / 46.62 | OOM | +| bf16 | seg2-stride4-offload | 2.275 / 37.99 | 6.249 / 37.94 | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 2.844 / 8.07 | 7.234 / 8.02 | +| int8-sdnq-hadamard | layer | 2.460 / 8.40 | 6.509 / 8.35 | +| int8-sdnq-hadamard | interval2 | 2.126 / 43.12 | 5.641 / 43.08 | +| int8-sdnq-hadamard | seg2-stride4 | 2.125 / 45.19 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 2.451 / 36.56 | 6.322 / 36.51 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 3.867 / 15.11 | 11.501 / 15.06 | +| fp8-torchao | layer | 3.579 / 15.28 | 10.822 / 15.24 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kolors + +Example: `kolors.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.635 / 7.22 | 0.628 / 7.17 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.118 / 5.36 | 1.065 / 5.31 | +| bf16 | interval2 | 1.105 / 5.36 | 1.068 / 5.31 | +| bf16 | seg2-stride4 | 1.110 / 5.36 | 1.072 / 5.31 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.860 / 5.68 | 1.726 / 5.63 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.967 / 3.38 | 2.803 / 3.33 | +| int8-sdnq-hadamard | interval2 | 2.770 / 3.32 | 2.655 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4 | 2.805 / 3.32 | 2.742 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.629 / 8.79 | 1.637 / 8.75 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.044 / 3.50 | 3.013 / 3.45 | +| fp8-torchao | interval2 | 3.040 / 3.50 | 3.019 / 3.45 | +| fp8-torchao | seg2-stride4 | 2.988 / 3.50 | 2.935 / 3.45 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Krea 2 + +Exemplo: `krea2.peft-lora`. Resolucao de treino: crop quadrado de 512 px. Configuracao de validacao do exemplo: 1024x1024; a validacao ficou desativada no benchmark. + +A tabela principal usa regional compilation. Isso ajuda o step speed do Krea2, mas nao e uma comparacao limpa de VRAM: o compiled graph/workspace mantem o pico perto do pico sem checkpointing em varios modos. Uma execucao de controle bf16 com regional compilation desativado mostrou que o checkpointing esta conectado e tem o formato esperado de memoria/velocidade: + +A linha `activation-offload` aqui significa full-block checkpointing mais attention activation offload. Contra full-block `layer` checkpointing sozinho, attention offload nao reduziu o peak VRAM do Krea2 neste shape; principalmente adicionou CPU transfer overhead. + +| Mode | H100 no-compile | L40S no-compile | +| --- | ---: | ---: | +| none | 0.272 / 40.09 | 0.661 / 40.01 | +| layer | 0.371 / 30.06 | 0.919 / 30.01 | +| seg2-stride4 | 0.317 / 34.75 | 0.788 / 34.70 | +| activation-offload | 0.657 / 30.50 | 1.341 / 30.30 | + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.275 / 40.06 | 0.662 / 40.01 | +| bf16 | activation-offload | 0.416 / 34.62 | 0.822 / 34.57 | +| bf16 | layer | 0.268 / 40.06 | 0.665 / 40.01 | +| bf16 | interval2 | 0.274 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4 | 0.279 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4-offload | 0.404 / 34.62 | 0.819 / 34.57 | +| int8-sdnq-hadamard | none | 0.462 / 27.01 | 0.807 / 26.96 | +| int8-sdnq-hadamard | activation-offload | 0.773 / 21.57 | 1.012 / 21.53 | +| int8-sdnq-hadamard | layer | 0.473 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | interval2 | 0.474 / 27.01 | 0.804 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4 | 0.472 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4-offload | 0.744 / 21.57 | 1.007 / 21.53 | +| fp8-torchao | none | 0.689 / 51.63 | OOM | +| fp8-torchao | activation-offload | 0.975 / 37.77 | 2.058 / 37.73 | +| fp8-torchao | layer | 0.689 / 51.63 | OOM | +| fp8-torchao | interval2 | 0.684 / 51.63 | OOM | +| fp8-torchao | seg2-stride4 | 0.674 / 51.63 | OOM | +| fp8-torchao | seg2-stride4-offload | 0.965 / 37.77 | 2.053 / 37.73 | + +### LongCat Image + +Exemplo: `longcat-image.peft-lora`. Resolucao de treino: 512 px quadrada; resolucao de validacao: 1024x1024. As linhas usam `attention_mechanism=native-flash`. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.193 / 16.47 | 0.262 / 16.42 | +| bf16 | activation-offload | 0.544 / 12.73 | 0.543 / 12.69 | +| bf16 | layer | 0.327 / 12.38 | 0.370 / 12.34 | +| bf16 | interval2 | 0.257 / 14.38 | 0.313 / 14.34 | +| bf16 | seg2-stride4 | 0.263 / 14.36 | 0.316 / 14.31 | +| bf16 | seg2-stride4-offload | 0.446 / 13.42 | 0.492 / 13.38 | +| int8-sdnq-hadamard | none | 0.578 / 12.54 | 0.537 / 12.45 | +| int8-sdnq-hadamard | activation-offload | 1.184 / 7.43 | 1.185 / 7.39 | +| int8-sdnq-hadamard | layer | 0.901 / 7.19 | 0.911 / 7.09 | +| int8-sdnq-hadamard | interval2 | 0.718 / 9.73 | 0.695 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4 | 0.735 / 9.72 | 0.717 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.056 / 8.49 | 1.011 / 8.45 | +| fp8-torchao | none | 0.602 / 25.19 | 0.834 / 25.14 | +| fp8-torchao | activation-offload | 1.662 / 7.75 | 1.844 / 7.70 | +| fp8-torchao | layer | 0.984 / 7.57 | 1.080 / 7.53 | +| fp8-torchao | interval2 | 0.750 / 16.15 | 0.938 / 16.10 | +| fp8-torchao | seg2-stride4 | 0.760 / 16.13 | 0.961 / 16.09 | +| fp8-torchao | seg2-stride4-offload | 1.287 / 13.02 | 1.653 / 12.98 | + +### LongCat Video + +Exemplo: `longcat-video.peft-lora+ramtorch`. Resolucao: 832x480, 81f. As linhas usam `attention_mechanism=native-flash`. + +LongCat Video usa muitas activations neste shape. Full per-block checkpointing e a linha pratica. As linhas partial checkpoint (`interval2`, `seg2-stride4`) nao cabem aqui, mesmo com attention activation offload na linha strided. Attention activation offload simples cabe para bf16 e SDNQ, mas e muito mais lento que full checkpointing. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM (77.86 GiB) | OOM (43.30 GiB) | +| bf16 | activation-offload | 25.774 / 37.36 | 49.149 / 37.14 | +| bf16 | layer | 7.448 / 23.73 | 24.866 / 23.68 | +| bf16 | interval2 | OOM (76.41 GiB) | OOM (42.80 GiB) | +| bf16 | seg2-stride4 | OOM (76.42 GiB) | OOM (43.06 GiB) | +| bf16 | seg2-stride4-offload | OOM (76.72 GiB) | OOM (42.29 GiB) | +| int8-sdnq-hadamard | none | OOM (77.40 GiB) | OOM (43.59 GiB) | +| int8-sdnq-hadamard | activation-offload | 30.887 / 35.28 | 61.270 / 35.24 | +| int8-sdnq-hadamard | layer | 8.444 / 21.60 | 25.164 / 21.55 | +| int8-sdnq-hadamard | interval2 | OOM (76.01 GiB) | OOM (42.47 GiB) | +| int8-sdnq-hadamard | seg2-stride4 | OOM (77.01 GiB) | OOM (43.02 GiB) | +| int8-sdnq-hadamard | seg2-stride4-offload | OOM (76.42 GiB) | OOM (42.57 GiB) | +| fp8-torchao | none | OOM (75.87 GiB) | OOM (41.28 GiB) | +| fp8-torchao | activation-offload | 30.163 / 47.88 | OOM (40.11 GiB) | +| fp8-torchao | layer | 8.343 / 34.16 | 24.659 / 34.07 | +| fp8-torchao | interval2 | OOM (74.63 GiB) | OOM (40.85 GiB) | +| fp8-torchao | seg2-stride4 | OOM (75.37 GiB) | OOM (41.19 GiB) | +| fp8-torchao | seg2-stride4-offload | OOM (75.55 GiB) | OOM (40.94 GiB) | + +### LTXVideo 0.9.5 + +Example: `ltxvideo-0.9.5-t2v.peft-lora`. Resolution: 768x512, 49f. + +Os numeros sao segundos warm por step / GiB pico. A media da execucao completa inclui setup e compile, e fica nos artifacts do sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.274 / 8.77 | 0.275 / 8.72 | +| bf16 | layer | 0.462 / 4.21 | 0.459 / 4.16 | +| bf16 | interval2 | 0.449 / 4.30 | 0.433 / 4.25 | +| bf16 | seg2-stride4 | 0.359 / 6.40 | 0.357 / 6.35 | +| int8-sdnq-hadamard | none | 0.688 / 7.05 | 0.640 / 6.90 | +| int8-sdnq-hadamard | layer | 1.094 / 2.59 | 1.081 / 2.44 | +| int8-sdnq-hadamard | interval2 | 1.112 / 2.57 | 1.073 / 2.53 | +| int8-sdnq-hadamard | seg2-stride4 | 0.887 / 4.61 | 0.817 / 4.56 | +| fp8-torchao | none | 0.655 / 17.45 | 0.735 / 17.41 | +| fp8-torchao | layer | 1.226 / 2.94 | 1.206 / 2.89 | +| fp8-torchao | interval2 | 1.259 / 3.40 | 1.233 / 3.35 | +| fp8-torchao | seg2-stride4 | 0.933 / 9.96 | 0.901 / 9.91 | +| fp8wo-torchao | none | 0.328 / 10.06 | 0.325 / 10.01 | +| fp8wo-torchao | layer | 0.567 / 2.64 | 0.540 / 2.59 | +| fp8wo-torchao | interval2 | 0.555 / 2.84 | 0.531 / 2.79 | +| fp8wo-torchao | seg2-stride4 | 0.443 / 6.20 | 0.432 / 6.15 | + +As linhas de attention activation offload nao sao suportadas para LTXVideo 0.9 neste sweep. + +### LTXVideo2 2.3 + +Example: `ltxvideo2-2.3-dev-720p-single-gpu.peft-lora+sdnq-hadamard`. Resolution: 1280x704, 49f. + +Note: LTXVideo2 2.3 should be read from the no-regional-compile rows in this sweep; regional compile raised memory pressure for this model. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.350 / 58.36 | OOM | +| bf16 | layer | 3.993 / 47.95 | OOM | +| bf16 | interval2 | 3.977 / 48.83 | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 5.554 / 75.64 | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 10.102 / 38.20 | 9.852 / 38.15 | +| int8-sdnq-hadamard | layer | 7.753 / 27.78 | 7.579 / 27.73 | +| int8-sdnq-hadamard | interval2 | 7.733 / 28.66 | 7.288 / 28.61 | +| int8-sdnq-hadamard | seg2-stride4 | 6.522 / 61.50 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 8.659 / 55.48 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | failed | 22.917 / 38.57 | +| fp8-torchao | layer | 8.580 / 30.27 | 10.381 / 30.22 | +| fp8-torchao | interval2 | 8.660 / 33.71 | 10.661 / 33.66 | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | failed | OOM | + +### Lumina2 + +Example: `lumina2.peft-lora`. Resolution: 512x512. + +Nota: Lumina2 agora usa o caminho segmented whole-block. `interval2` faz checkpoint de cada segmento de dois blocks; `seg2-stride4` checkpointa dois blocks, deixa os dois seguintes manterem activations e repete. Attention activation offload nao fez parte desta rodada de Lumina2. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.235 / 15.99 | 0.332 / 15.94 | +| bf16 | layer | 0.384 / 6.60 | 0.457 / 6.56 | +| bf16 | interval2 | 0.356 / 6.87 | 0.424 / 6.82 | +| bf16 | seg2-stride4 | 0.295 / 11.43 | 0.377 / 11.38 | +| int8-sdnq-hadamard | none | 0.584 / 13.59 | 0.541 / 13.55 | +| int8-sdnq-hadamard | layer | 0.899 / 4.21 | 0.827 / 4.16 | +| int8-sdnq-hadamard | interval2 | 0.865 / 4.48 | 0.835 / 4.43 | +| int8-sdnq-hadamard | seg2-stride4 | 0.719 / 9.03 | 0.707 / 8.99 | +| fp8-torchao | none | 0.598 / 28.03 | 0.763 / 27.98 | +| fp8-torchao | layer | 0.950 / 5.93 | 0.967 / 5.88 | +| fp8-torchao | interval2 | 0.938 / 6.70 | 0.974 / 6.66 | +| fp8-torchao | seg2-stride4 | 0.765 / 17.36 | 0.901 / 17.32 | +| fp8wo-torchao | none | 0.273 / 17.97 | 0.389 / 17.93 | +| fp8wo-torchao | layer | 0.452 / 4.99 | 0.525 / 4.94 | +| fp8wo-torchao | interval2 | 0.427 / 5.40 | 0.522 / 5.35 | +| fp8wo-torchao | seg2-stride4 | 0.360 / 11.68 | 0.466 / 11.64 | + +### MageFlow + +Example: `mageflow-image-24g.peft-lora`. Resolution: 1024x1024. + +Nota: o caminho de imagem com shapes variaveis do MageFlow em 1024px se beneficia principalmente de attention activation offload e FP8 weight-only. Os modos de block checkpointing sao validos, mas nao reduziram o pico de residencia medido neste sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 7.735 / 36.74 | 7.658 / 36.69 | +| bf16 | activation-offload | 8.902 / 23.05 | 9.477 / 23.00 | +| bf16 | layer | 7.805 / 36.74 | 7.504 / 36.69 | +| bf16 | interval2 | 8.036 / 36.74 | 7.762 / 36.69 | +| bf16 | seg2-stride4 | 7.882 / 36.74 | 7.531 / 36.69 | +| bf16 | seg2-stride4-offload | 8.833 / 23.05 | 9.288 / 23.01 | +| int8-sdnq-hadamard | none | 81.016 / 37.38 | 94.991 / 37.34 | +| fp8wo-torchao | none | 5.454 / 36.86 | 5.772 / 36.82 | +| fp8wo-torchao | activation-offload | 6.295 / 23.18 | 6.738 / 23.14 | +| fp8wo-torchao | seg2-stride4 | 5.542 / 36.86 | 5.595 / 36.82 | + +### OmniGen + +Example: `omnigen.lycoris-lokr`. Resolution: 1024x1024. + +Nota: OmniGen usa prompts como token IDs em vez de embeddings de texto em cache. Estas linhas medem os caminhos suportados sem checkpointing e com checkpointing torch de bloco completo; os controles interval, segmented-stride e attention-offload nao estao implementados para esta familia neste sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.425 / 14.20 | 0.293 / 14.24 | +| bf16 | layer | 0.597 / 10.13 | 0.389 / 10.09 | +| int8-sdnq-hadamard | none | 1.523 / 11.00 | 1.388 / 11.06 | +| int8-sdnq-hadamard | layer | 1.312 / 6.75 | 1.064 / 6.70 | +| fp8-torchao | none | 0.690 / 19.25 | 0.608 / 19.30 | +| fp8-torchao | layer | 1.069 / 7.09 | 0.824 / 7.04 | +| fp8wo-torchao | none | 0.454 / 17.71 | 0.377 / 17.73 | +| fp8wo-torchao | layer | 0.646 / 6.98 | 0.534 / 6.94 | + +### PixArt + +Example: `pixart.lycoris-lokr`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.700 / 41.73 | 1.734 / 41.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.433 / 5.63 | 2.346 / 5.58 | +| bf16 | interval2 | 2.440 / 6.01 | 2.348 / 5.96 | +| bf16 | seg2-stride4 | 2.092 / 23.90 | 2.072 / 23.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.905 / 47.58 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.738 / 6.07 | 2.937 / 6.03 | +| int8-sdnq-hadamard | interval2 | 2.734 / 6.03 | 2.943 / 5.99 | +| int8-sdnq-hadamard | seg2-stride4 | 2.336 / 26.85 | 2.596 / 26.81 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.254 / 8.06 | 4.646 / 8.01 | +| fp8-torchao | interval2 | 3.260 / 9.66 | 4.649 / 9.61 | +| fp8-torchao | seg2-stride4 | 2.827 / 63.27 | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Qwen Image + +Example: `qwen_image.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.202 / 41.04 | 3.377 / 40.99 | +| bf16 | interval2 | 1.201 / 41.04 | 3.382 / 40.99 | +| bf16 | seg2-stride4 | 1.205 / 41.03 | 3.385 / 40.99 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.675 / 63.48 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.640 / 24.09 | 3.928 / 24.05 | +| int8-sdnq-hadamard | interval2 | 2.722 / 24.09 | 3.918 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4 | 2.663 / 24.09 | 3.919 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.088 / 25.34 | 6.172 / 25.29 | +| fp8-torchao | interval2 | 3.095 / 25.34 | 6.173 / 25.29 | +| fp8-torchao | seg2-stride4 | 3.125 / 25.34 | 6.141 / 25.29 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Sana + +Example: `sana.lycoris-lokr`. Resolution: 1024x1024. + +Note: Sana has interval checkpointing; stride is not a separate segmented schedule for this family in the measured rows. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 23.71 | 0.597 / 23.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.529 / 23.72 | 0.596 / 23.66 | +| bf16 | interval2 | 0.530 / 23.72 | 0.598 / 23.66 | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.554 / 22.92 | 0.590 / 22.88 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.554 / 22.92 | 0.589 / 22.88 | +| int8-sdnq-hadamard | interval2 | 0.556 / 22.92 | 0.591 / 22.88 | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.633 / 33.67 | 0.753 / 33.62 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.633 / 33.67 | 0.755 / 33.62 | +| fp8-torchao | interval2 | 0.631 / 33.67 | 0.759 / 33.62 | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SanaVideo + +Example: `sanavideo-2b-480p.peft-lora`. Resolution: 832x480, 49f. + +Note: SanaVideo usa linear attention, entao attention activation offload continua unsupported. Segmented whole-block checkpointing e suportado no caminho standard. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.599 / 59.15 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.597 / 59.15 | OOM | +| bf16 | interval2 | not run | OOM | +| bf16 | seg2-stride4 | not run | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | interval2 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD 1.x + +Example: `sd1x-dreamshaper.peft-lora`. Resolution: 512x512. + +Nota: SD1x usa o caminho UNet do diffusers. O checkpointing por camada funciona, mas os controles interval e segmented stride nao estao conectados para esta familia. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.181 / 2.87 | 0.176 / 2.83 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.305 / 1.98 | 0.292 / 1.93 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.446 / 3.07 | 0.431 / 3.04 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.701 / 1.79 | 0.666 / 1.74 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.410 / 4.23 | 0.401 / 4.18 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.785 / 1.96 | 0.756 / 1.91 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD3 + +Example: `sd3.peft-lora`. Resolution: 1024x1024. + +Nota: SD3 usa checkpointing segmentado contiguo real no caminho transformer simples. Attention activation offload e suportado; reduz bastante a VRAM, mas custa throughput. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 34.59 | 1.189 / 34.53 | +| bf16 | activation-offload | 1.335 / 9.68 | 2.850 / 9.63 | +| bf16 | layer | 0.721 / 7.67 | 1.607 / 7.62 | +| bf16 | interval2 | 0.723 / 8.93 | 1.606 / 8.88 | +| bf16 | seg2-stride4 | 0.620 / 20.14 | 1.398 / 20.09 | +| bf16 | seg2-stride4-offload | 1.229 / 12.21 | 2.639 / 12.16 | +| int8-sdnq-hadamard | none | 0.858 / 33.66 | 1.381 / 33.61 | +| int8-sdnq-hadamard | activation-offload | 1.845 / 8.90 | 3.297 / 8.85 | +| int8-sdnq-hadamard | layer | 1.265 / 6.58 | 1.857 / 6.53 | +| int8-sdnq-hadamard | interval2 | 1.264 / 7.30 | 1.858 / 7.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.049 / 19.02 | 1.625 / 18.98 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.527 / 12.42 | 3.023 / 12.37 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 4.707 / 10.27 | 9.814 / 10.23 | +| fp8-torchao | layer | 1.575 / 9.13 | 3.279 / 9.09 | +| fp8-torchao | interval2 | 1.576 / 12.62 | 3.282 / 12.58 | +| fp8-torchao | seg2-stride4 | 1.376 / 45.70 | OOM | +| fp8-torchao | seg2-stride4-offload | 4.184 / 26.17 | 8.714 / 26.12 | +| fp8wo-torchao | none | 0.567 / 35.21 | 1.252 / 35.17 | +| fp8wo-torchao | activation-offload | 1.392 / 8.71 | 2.921 / 8.67 | +| fp8wo-torchao | layer | 0.795 / 5.69 | 1.736 / 5.65 | +| fp8wo-torchao | interval2 | 0.797 / 7.08 | 1.734 / 7.03 | +| fp8wo-torchao | seg2-stride4 | 0.679 / 19.43 | 1.490 / 19.38 | +| fp8wo-torchao | seg2-stride4-offload | 1.257 / 12.00 | 2.676 / 11.96 | + +### SDXL + +Example: `sdxl.lycoris-lokr`. Resolution: 1024x1024. + +Note: SDXL has real layer checkpointing. Interval and stride rows are included as coverage data, not as segmented-support recommendations. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.606 / 13.03 | 0.585 / 12.98 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.080 / 6.53 | 1.029 / 6.48 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.741 / 13.72 | 1.643 / 13.68 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.820 / 4.64 | 2.647 / 4.59 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.608 / 26.22 | 1.582 / 26.16 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.939 / 5.10 | 2.890 / 5.04 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Stable Cascade + +Example: `cascade-stage-c.lycoris-lokr`. Resolution: 1024x1024. + +Nota: stage C usa o prior em precision completa. Estas linhas foram executadas com `mixed_precision=no` e `base_model_precision=no_change`; linhas de precision base quantizada nao sao significativas para este modelo. Os modos interval e stride operam sobre a sequencia de micro-blocos Res/Timestep/Attention do UNet. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.884 / 51.52 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.179 / 22.68 | 2.135 / 22.61 | +| bf16 | interval2 | 1.032 / 36.99 | 1.871 / 36.92 | +| bf16 | seg2-stride4 | 1.032 / 37.20 | 1.870 / 37.13 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | unsupported | unsupported | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | unsupported | unsupported | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Wan 2.1 T2V 1.3B + +Example: `wan2.1-t2v-1.3b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 1.3B should be read from the no-regional-compile/RamTorch rows; regional compile was not a useful throughput setting in this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.407 / 71.90 | OOM | +| bf16 | activation-offload | 3.472 / 8.78 | 7.179 / 8.66 | +| bf16 | layer | 2.099 / 4.73 | 4.459 / 4.68 | +| bf16 | interval2 | 2.139 / 6.32 | 4.514 / 6.27 | +| bf16 | seg2-stride4 | 1.806 / 39.25 | 3.921 / 39.21 | +| bf16 | seg2-stride4-offload | 2.993 / 22.66 | 6.493 / 22.61 | +| int8-sdnq-hadamard | none | 1.850 / 71.91 | OOM | +| int8-sdnq-hadamard | activation-offload | 4.204 / 8.72 | 7.387 / 8.68 | +| int8-sdnq-hadamard | layer | 2.790 / 4.70 | 4.989 / 4.65 | +| int8-sdnq-hadamard | interval2 | 2.874 / 6.29 | 5.093 / 6.24 | +| int8-sdnq-hadamard | seg2-stride4 | 2.558 / 39.27 | 4.393 / 39.22 | +| int8-sdnq-hadamard | seg2-stride4-offload | 3.711 / 22.67 | 6.695 / 22.63 | +| fp8-torchao | none | 1.727 / 73.57 | OOM | +| fp8-torchao | activation-offload | 4.061 / 10.08 | 7.404 / 9.96 | +| fp8-torchao | layer | 2.607 / 5.98 | 4.888 / 5.93 | +| fp8-torchao | interval2 | 2.744 / 7.57 | 4.916 / 7.52 | +| fp8-torchao | seg2-stride4 | 2.246 / 40.55 | 4.245 / 40.50 | +| fp8-torchao | seg2-stride4-offload | 3.602 / 24.02 | 6.683 / 23.91 | + +### Wan 2.1 T2V 14B + +Example: `wan2.1-t2v-14b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 14B is mainly a fit test for activation savings. Status-only cells are still useful because they show which combinations reached the memory limit. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 13.144 / 36.80 | OOM | +| bf16 | layer | 7.162 / 16.28 | 21.770 / 16.23 | +| bf16 | interval2 | 7.172 / 19.62 | 21.777 / 19.58 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | failed | failed | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | failed | failed | +| fp8-torchao | seg2-stride4 | failed | failed | +| fp8-torchao | seg2-stride4-offload | failed | failed | + +### Wan S2V + +Example: `wan-s2v-14b-480p.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan S2V is included as coverage data for the video/audio path. Treat failed cells as implementation coverage gaps. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | failed | failed | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Z-Image Turbo + +Example: `z-image-turbo.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.243 / 21.25 | 0.316 / 21.21 | +| bf16 | activation-offload | 0.837 / 13.24 | 0.805 / 13.19 | +| bf16 | layer | 0.479 / 12.87 | 0.493 / 12.83 | +| bf16 | interval2 | 0.452 / 13.04 | 0.477 / 12.99 | +| bf16 | seg2-stride4 | 0.349 / 16.88 | 0.400 / 16.83 | +| bf16 | seg2-stride4-offload | 0.681 / 15.03 | 0.736 / 14.99 | +| int8-sdnq-hadamard | none | 0.645 / 15.60 | 0.615 / 15.55 | +| int8-sdnq-hadamard | activation-offload | 1.439 / 7.61 | 1.382 / 7.56 | +| int8-sdnq-hadamard | layer | 1.046 / 7.25 | 1.021 / 7.20 | +| int8-sdnq-hadamard | interval2 | 1.074 / 7.41 | 0.996 / 7.36 | +| int8-sdnq-hadamard | seg2-stride4 | 0.867 / 11.25 | 0.841 / 11.20 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.202 / 9.39 | 1.162 / 9.35 | +| fp8-torchao | none | 1.232 / 37.50 | 1.476 / 37.46 | +| fp8-torchao | activation-offload | 3.623 / 7.97 | 3.564 / 7.93 | +| fp8-torchao | layer | 2.319 / 7.93 | 2.344 / 7.88 | +| fp8-torchao | interval2 | 2.336 / 8.80 | 2.309 / 8.75 | +| fp8-torchao | seg2-stride4 | 1.843 / 22.55 | 1.930 / 22.50 | +| fp8-torchao | seg2-stride4-offload | 2.947 / 15.57 | 3.243 / 15.52 | + +### ZLab I1 + +Example: `zlab-i1.peft-lora`. Resolution: 1024x1024. + +Nota: ZLab I1 carrega seus skip tensors estilo U-Net pelo estado segmented checkpoint. Attention activation offload nao esta conectado para esta familia. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.462 / 22.21 | 0.865 / 22.16 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.693 / 7.79 | 1.148 / 7.75 | +| bf16 | interval2 | 0.676 / 8.30 | 1.152 / 8.25 | +| bf16 | seg2-stride4 | 0.567 / 14.97 | 1.014 / 14.92 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.861 / 19.21 | 0.926 / 19.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.385 / 4.78 | 1.265 / 4.74 | +| int8-sdnq-hadamard | interval2 | 1.298 / 5.30 | 1.277 / 5.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.073 / 11.98 | 1.098 / 11.93 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8wo-torchao | none | 0.504 / 25.08 | 0.930 / 25.02 | +| fp8wo-torchao | activation-offload | unsupported | unsupported | +| fp8wo-torchao | layer | 0.772 / 5.12 | 1.280 / 5.07 | +| fp8wo-torchao | interval2 | 0.759 / 5.84 | 1.290 / 5.79 | +| fp8wo-torchao | seg2-stride4 | 0.633 / 15.12 | 1.115 / 15.08 | +| fp8wo-torchao | seg2-stride4-offload | unsupported | unsupported | + + diff --git a/documentation/experimental/SEGMENTED_CHECKPOINTING.zh.md b/documentation/experimental/SEGMENTED_CHECKPOINTING.zh.md new file mode 100644 index 000000000..020dfd52b --- /dev/null +++ b/documentation/experimental/SEGMENTED_CHECKPOINTING.zh.md @@ -0,0 +1,1007 @@ +# 分段 Checkpointing + +分段 checkpointing 介于每个 block 都 checkpoint 和完全不 checkpoint 之间。 + +它使用 PyTorch activation checkpointing backend。SimpleTuner 会把连续的 transformer blocks 放进一次 checkpoint 调用，然后把返回的 hidden state 传给下一组。组越宽，backward 里的重算越少，但保留的 activations 越多。 + +CPU offload 和 FFN-only checkpointing 请看 [Unsloth-style checkpointing](UNSLOTH_CHECKPOINTING.zh.md#controls)。简短判断规则仍在 [Decision Rule](UNSLOTH_CHECKPOINTING.zh.md#decision-rule)。 + +## 控制项 + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2 +} +``` + +在支持 whole-block 的路径上，`gradient_checkpointing_interval` 是 segment 宽度。`2` 表示 checkpoint blocks `0-1`、`2-3`、`4-5`，依此类推。 + +如果需要更细的 VRAM 控制，可以加 stride： + +```json +{ + "gradient_checkpointing": true, + "gradient_checkpointing_backend": "torch", + "gradient_checkpointing_interval": 2, + "gradient_checkpointing_segment_stride": 4 +} +``` + +这会 checkpoint blocks `0-1`，正常运行 `2-3`，checkpoint `4-5`，正常运行 `6-7`，然后重复。stride 必须大于等于 interval；不支持重叠 schedule。 + +支持 segmented whole-block 的路径：Flux.1、Flux.2、HunyuanVideo、Krea 2、LongCat Image、LongCat Video、LTXVideo 0.9、LTXVideo2、Lumina2、MageFlow、PixArt、SD3、SanaVideo、Z-Image、ZLab I1 和 Wan。 + +Stable Cascade stage C 也支持 interval 和 stride，但 schedule 作用在 UNet 的 Res/Timestep/Attention micro-block 序列上，而不是 transformer whole-block group。 + +一些 model family 使用旧语义： + +| Family | `gradient_checkpointing_interval` | `gradient_checkpointing_segment_stride` | +| --- | --- | --- | +| Sana | 每 N 个 block checkpoint 一次 | 忽略 | +| Stable Cascade stage C | 按 interval checkpoint UNet micro-block | stride 在 checkpointed 和非 checkpointed UNet micro-block window 之间交替 | +| SD1x, SDXL | 不支持 segmented whole-block | 忽略 | + +不要比较 stride 被忽略的 family 的 stride 行。如果数字完全一样，通常是 option 没生效，不是有用的性能结果。 + +## 什么时候使用 + +当普通 per-block checkpointing 能 fit 但 step time 太高时使用。先从 `2` 开始。如果 VRAM 允许，在很深的模型上试 `2` + stride `4`。 + +当 peak 主要来自可训练权重、optimizer state、validation、VAE cache、block swapping 或 routing 时，不要期待它有帮助。当模型特性需要 per-block 控制时，SimpleTuner 会回到更稳的 per-block 路径。 + +`dynamo_use_regional_compilation` 不是通用加速。它在一些 image model 上有帮助或基本中性，但在下面的 Wan/RamTorch 和 LTXVideo2 profile 里效果很差。compile 设置也要当作 benchmark 条件。 + +## Benchmarks + +使用真实 SimpleTuner examples，在单卡 H100 和 L40S pods 上测量。validation 和 checkpoint saves 已关闭，cache preparation 不计入；存在 post-warmup timing 时，也会排除 train loop 内第一个 step 的 compile/setup 时间。 + +每个实测单元格都是 `post-warmup sec/step / peak VRAM GiB`。只有状态的单元格含义是：`OOM` 表示 GPU 显存不足，`failed` 表示没有到达可统计的训练 step，`unsupported` 表示该 family 没有接入这个选项，`not run` 表示 sweep 没有覆盖这个组合。 + +优先在同一个 family 内比较不同 mode。跨 family 比较只能作为粗略参考，因为 resolution、frame count、attention backend、model depth、trainable adapter type 和 dataset shape 都可能不同。 + +下面的矩阵是这个 sweep 的唯一数据源。model-specific notes 会说明某些行只是覆盖情况数据，而不是推荐配置。 + + + +### Family Sweep Results + +### ACE Step 1.5 + +Example: `ace_step-v1-5.peft-lora`. Resolution: 512. + +Note: This sweep did not produce a usable ACE Step throughput row. The status-only entries below should be treated as coverage gaps, not as a recommendation. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | OOM | OOM | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | OOM | OOM | +| int8-sdnq-hadamard | interval2 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### 基于 Anima 的 AnyFlow 蒸馏 + +示例：`anima-anyflow.peft-lora`。分辨率：1024x1024。此行测量的是使用 Anima 的 AnyFlow 蒸馏，不是普通 Anima LoRA 示例。普通 1024x1024 Anima 图像训练请使用 `anima.peft-lora`。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.022 / 17.26 | 0.719 / 17.21 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.232 / 5.60 | 0.903 / 5.56 | +| bf16 | interval2 | 1.252 / 5.60 | 0.897 / 5.56 | +| bf16 | seg2-stride4 | 1.244 / 5.60 | 0.898 / 5.56 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 5.417 / 18.61 | 4.974 / 18.57 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.562 / 4.36 | 4.019 / 4.31 | +| int8-sdnq-hadamard | interval2 | 3.723 / 4.36 | 3.196 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4 | 3.658 / 4.36 | 3.140 / 4.31 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.242 / 45.71 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.810 / 5.51 | 2.576 / 5.46 | +| fp8-torchao | interval2 | 2.846 / 5.51 | 2.581 / 5.46 | +| fp8-torchao | seg2-stride4 | 2.766 / 5.51 | 2.567 / 5.46 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### AuraFlow + +Example: `auraflow.peft-lora`. Resolution: 1024x1024. + +说明：AuraFlow 支持 SDNQ 和 TorchAO 量化。下表包含量化的 `none` 行；量化 checkpoint 行需要新的完整 benchmark 覆盖后再填写数值。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.180 / 19.19 | 0.233 / 19.12 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.824 / 13.37 | 0.833 / 13.32 | +| bf16 | interval2 | 1.764 / 16.21 | 0.877 / 16.14 | +| bf16 | seg2-stride4 | 1.771 / 16.21 | 0.887 / 16.14 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.642 / 12.88 | 0.610 / 12.87 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | not run | not run | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.621 / 24.53 | 0.757 / 24.44 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | not run | not run | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Boogu Image + +Example: `boogu-image-v0.1.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.694 / 59.14 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.907 / 23.44 | 2.641 / 23.39 | +| bf16 | interval2 | 0.912 / 23.44 | 2.649 / 23.39 | +| bf16 | seg2-stride4 | 0.911 / 23.44 | 2.648 / 23.39 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.488 / 53.24 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.878 / 15.20 | 3.309 / 15.15 | +| int8-sdnq-hadamard | interval2 | 1.630 / 34.12 | 2.577 / 34.07 | +| int8-sdnq-hadamard | seg2-stride4 | 1.656 / 34.11 | 2.574 / 34.06 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.713 / 18.48 | 3.778 / 18.44 | +| fp8-torchao | interval2 | 1.731 / 18.48 | 3.777 / 18.44 | +| fp8-torchao | seg2-stride4 | 1.721 / 18.48 | 3.779 / 18.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Chroma + +Example: `chroma.peft-lora`. Resolution: 1024x1024. + +Note: Checkpointed Chroma rows use `attention_mechanism=native-efficient`, which was the stable attention path for this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.454 / 26.18 | 0.559 / 26.13 | +| bf16 | activation-offload | 4.873 / 18.74 | 4.793 / 18.69 | +| bf16 | layer | 1.276 / 17.67 | 1.430 / 17.63 | +| bf16 | interval2 | 1.204 / 21.80 | 1.349 / 21.75 | +| bf16 | seg2-stride4 | 1.200 / 21.78 | 1.382 / 21.74 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.083 / 21.42 | 1.061 / 21.37 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.714 / 10.41 | 1.646 / 10.36 | +| int8-sdnq-hadamard | interval2 | 1.443 / 15.72 | 1.391 / 15.68 | +| int8-sdnq-hadamard | seg2-stride4 | 1.428 / 15.71 | 1.323 / 15.67 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.122 / 45.44 | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.821 / 10.64 | 2.104 / 10.60 | +| fp8-torchao | interval2 | 1.447 / 27.49 | 1.871 / 27.44 | +| fp8-torchao | seg2-stride4 | 1.431 / 27.49 | 1.877 / 27.44 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 2 Image + +Example: `cosmos2image.lycoris-lokr`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.336 / 8.05 | 0.316 / 8.00 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.567 / 4.09 | 0.559 / 4.04 | +| bf16 | interval2 | 0.595 / 4.09 | 0.544 / 4.04 | +| bf16 | seg2-stride4 | 0.598 / 4.09 | 0.546 / 4.04 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.831 / 6.56 | 0.783 / 6.56 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.315 / 2.44 | 1.288 / 2.39 | +| int8-sdnq-hadamard | interval2 | 1.346 / 2.44 | 1.321 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4 | 1.413 / 2.44 | 1.274 / 2.39 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.850 / 13.30 | 0.840 / 13.25 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.692 / 2.70 | 1.560 / 2.65 | +| fp8-torchao | interval2 | 1.626 / 2.70 | 1.544 / 2.65 | +| fp8-torchao | seg2-stride4 | 1.607 / 2.70 | 1.555 / 2.65 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Cosmos 3 + +Example: `cosmos3-edge-image-24g.lycoris-lokr`. Resolution: 1024 px. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 2.747 / 8.90 | 2.904 / 8.86 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.602 / 8.90 | 2.606 / 8.86 | +| bf16 | interval2 | 2.658 / 8.90 | 2.567 / 8.86 | +| bf16 | seg2-stride4 | 2.628 / 8.90 | 2.899 / 8.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 17.112 / 6.21 | 17.965 / 6.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 3.238 / 6.21 | 2.891 / 6.16 | +| int8-sdnq-hadamard | interval2 | 3.253 / 6.21 | 2.916 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4 | 3.254 / 6.21 | 2.923 / 6.16 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.849 / 17.84 | 1.559 / 17.80 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 1.869 / 17.84 | 1.599 / 17.79 | +| fp8-torchao | interval2 | 1.855 / 17.84 | 1.600 / 17.80 | +| fp8-torchao | seg2-stride4 | 1.859 / 17.84 | 1.523 / 17.80 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### ERNIE 4.5 Image + +Example: `ernie.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.711 / 15.61 | 1.282 / 15.56 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.110 / 4.94 | 1.840 / 4.90 | +| bf16 | interval2 | 0.876 / 10.16 | 1.536 / 10.12 | +| bf16 | seg2-stride4 | 0.874 / 10.16 | 1.532 / 10.12 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 2.782 / 13.75 | 2.722 / 13.70 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 4.262 / 2.98 | 4.393 / 2.94 | +| int8-sdnq-hadamard | interval2 | 3.547 / 8.26 | 3.380 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4 | 3.366 / 8.26 | 3.457 / 8.21 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 2.432 / 14.03 | 2.303 / 13.98 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.920 / 3.26 | 4.008 / 3.22 | +| fp8-torchao | interval2 | 3.316 / 8.54 | 2.973 / 8.49 | +| fp8-torchao | seg2-stride4 | 2.994 / 8.54 | 3.046 / 8.49 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HeartMula + +Example: `heartmula.peft-lora`. 音频 token 训练。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### HiDream + +Example: `hidream.peft-lora`. Resolution: 512x512. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.515 / 44.58 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.928 / 33.57 | 1.043 / 33.52 | +| bf16 | interval2 | 0.971 / 33.57 | 1.002 / 33.52 | +| bf16 | seg2-stride4 | 0.952 / 33.57 | 1.010 / 33.52 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | not measured | not measured | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.497 / 17.99 | 2.058 / 17.93 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | not measured | not measured | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 4.920 / 18.10 | 4.338 / 18.05 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +SDNQ Hadamard numbers use `sdnq_compile_mode=eager`. The compiled SDNQ path quantizes HiDream, but this sweep spent the first training step in Inductor dequantizer compilation, so it is not listed as a throughput row. + +### HunyuanVideo + +Example: `hunyuanvideo-1.5-t2v.peft-lora`. Training shape: 480 pixel-area video buckets, 48 frames, batch 2. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | not run | not run | +| bf16 | layer | 7.682 / 26.35 | 22.816 / 26.30 | +| bf16 | interval2 | 7.398 / 26.11 | 22.772 / 26.06 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 10.679 / 58.37 | not run | +| int8-sdnq-hadamard | none | not run | not run | +| int8-sdnq-hadamard | activation-offload | not run | not run | +| int8-sdnq-hadamard | layer | 11.765 / 25.96 | 34.464 / 25.92 | +| int8-sdnq-hadamard | interval2 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4 | not run | not run | +| int8-sdnq-hadamard | seg2-stride4-offload | not run | not run | +| fp8-torchao | none | not run | not run | +| fp8-torchao | activation-offload | not run | not run | +| fp8-torchao | layer | 10.516 / 33.55 | 32.003 / 33.53 | +| fp8-torchao | interval2 | not run | not run | +| fp8-torchao | seg2-stride4 | not run | not run | +| fp8-torchao | seg2-stride4-offload | not run | not run | + +HunyuanVideo 在这个 training shape 下 activation 压力很高。Per-block 和 interval-2 checkpointing 都能稳定运行；不开 checkpointing 时 80 GB H100 也放不下。`seg2-stride4` 在本轮 sweep 中只有启用 attention activation offload 才能跑通，这一行更适合作为内存 fallback，而不是速度推荐。SDNQ Hadamard 可以工作，但可变 conditioning shape 仍会在测量窗口内触发 dynamic kernel compilation。 + +### Ideogram 4.0 + +示例：`ideogram-fp8.peft-lora`。分辨率：1024x1024。`fp8` flavour 使用 Ideogram 4 的原生 weight-only fp8 checkpoint（`base_model_precision=no_change`）。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| fp8-native | none | failed | OOM | +| fp8-native | activation-offload | unsupported | unsupported | +| fp8-native | layer | 1.033 / 12.57 | 3.101 / 11.82 | +| fp8-native | interval2 | 1.030 / 12.33 | 3.098 / 11.82 | +| fp8-native | seg2-stride4 | 1.031 / 12.33 | 3.088 / 11.82 | +| fp8-native | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.702 / 61.78 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.033 / 12.33 | 3.030 / 11.82 | +| int8-sdnq-hadamard | interval2 | 1.032 / 12.33 | 3.034 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4 | 1.028 / 12.33 | 3.035 / 11.82 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | + +### Kandinsky 5 Image + +Example: `kandinsky5-image-6b-t2i.lycoris-lokr`. Resolution: 1024x1024. + +Note: batch 3 at 1024x1024 needs full checkpointing on both cards. SDNQ with Hadamard is the best low-VRAM row; H100 can also use partial checkpointing with SDNQ, but only near the top of the card. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.956 / 25.52 | 10.189 / 25.42 | +| bf16 | layer | 6.590 / 25.58 | 9.458 / 25.55 | +| bf16 | interval2 | OOM | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 7.186 / 20.00 | 10.141 / 19.95 | +| int8-sdnq-hadamard | layer | 6.830 / 20.12 | 9.362 / 20.08 | +| int8-sdnq-hadamard | interval2 | 5.746 / 75.67 | OOM | +| int8-sdnq-hadamard | seg2-stride4 | OOM | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 6.057 / 71.46 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 8.319 / 24.29 | 14.716 / 24.24 | +| fp8-torchao | layer | 7.976 / 24.40 | 13.949 / 24.35 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kandinsky 5 Video + +Example: `kandinsky5-video-2b-t2v.peft-lora`. Resolution: 768x512, 81f. + +Kandinsky 5 video is activation-heavy at this frame count. Full block checkpointing is the practical baseline on both cards. On H100, `interval2` and `seg2-stride4` are faster when they fit; on L40S, SDNQ `interval2` is the only partial-checkpoint row here that fits without attention activation offload. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 2.580 / 9.49 | 7.136 / 9.45 | +| bf16 | layer | 2.267 / 9.83 | 6.379 / 9.79 | +| bf16 | interval2 | 1.967 / 44.57 | OOM | +| bf16 | seg2-stride4 | 1.971 / 46.62 | OOM | +| bf16 | seg2-stride4-offload | 2.275 / 37.99 | 6.249 / 37.94 | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 2.844 / 8.07 | 7.234 / 8.02 | +| int8-sdnq-hadamard | layer | 2.460 / 8.40 | 6.509 / 8.35 | +| int8-sdnq-hadamard | interval2 | 2.126 / 43.12 | 5.641 / 43.08 | +| int8-sdnq-hadamard | seg2-stride4 | 2.125 / 45.19 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 2.451 / 36.56 | 6.322 / 36.51 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 3.867 / 15.11 | 11.501 / 15.06 | +| fp8-torchao | layer | 3.579 / 15.28 | 10.822 / 15.24 | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | OOM | OOM | + +### Kolors + +Example: `kolors.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.635 / 7.22 | 0.628 / 7.17 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.118 / 5.36 | 1.065 / 5.31 | +| bf16 | interval2 | 1.105 / 5.36 | 1.068 / 5.31 | +| bf16 | seg2-stride4 | 1.110 / 5.36 | 1.072 / 5.31 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.860 / 5.68 | 1.726 / 5.63 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.967 / 3.38 | 2.803 / 3.33 | +| int8-sdnq-hadamard | interval2 | 2.770 / 3.32 | 2.655 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4 | 2.805 / 3.32 | 2.742 / 3.27 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.629 / 8.79 | 1.637 / 8.75 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.044 / 3.50 | 3.013 / 3.45 | +| fp8-torchao | interval2 | 3.040 / 3.50 | 3.019 / 3.45 | +| fp8-torchao | seg2-stride4 | 2.988 / 3.50 | 2.935 / 3.45 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Krea 2 + +示例：`krea2.peft-lora`。训练分辨率：512 px 方形裁剪。示例验证设置：1024x1024；benchmark 中已禁用 validation。 + +主表使用 regional compilation；它对 Krea2 step speed 有帮助，但不是干净的 VRAM 对比。compiled graph/workspace 会让几个 checkpoint 模式的峰值接近未 checkpoint 的峰值。关闭 regional compilation 的 bf16 control run 显示 checkpointing 已接通，并呈现预期的 memory/speed 形态： + +这里的 `activation-offload` 行表示 full-block checkpointing 加 attention activation offload。和单独的 full-block `layer` checkpointing 相比，attention offload 在这个 Krea2 shape 下没有降低 peak VRAM，主要增加了 CPU transfer overhead。 + +| Mode | H100 no-compile | L40S no-compile | +| --- | ---: | ---: | +| none | 0.272 / 40.09 | 0.661 / 40.01 | +| layer | 0.371 / 30.06 | 0.919 / 30.01 | +| seg2-stride4 | 0.317 / 34.75 | 0.788 / 34.70 | +| activation-offload | 0.657 / 30.50 | 1.341 / 30.30 | + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.275 / 40.06 | 0.662 / 40.01 | +| bf16 | activation-offload | 0.416 / 34.62 | 0.822 / 34.57 | +| bf16 | layer | 0.268 / 40.06 | 0.665 / 40.01 | +| bf16 | interval2 | 0.274 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4 | 0.279 / 40.06 | 0.663 / 40.01 | +| bf16 | seg2-stride4-offload | 0.404 / 34.62 | 0.819 / 34.57 | +| int8-sdnq-hadamard | none | 0.462 / 27.01 | 0.807 / 26.96 | +| int8-sdnq-hadamard | activation-offload | 0.773 / 21.57 | 1.012 / 21.53 | +| int8-sdnq-hadamard | layer | 0.473 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | interval2 | 0.474 / 27.01 | 0.804 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4 | 0.472 / 27.01 | 0.802 / 26.96 | +| int8-sdnq-hadamard | seg2-stride4-offload | 0.744 / 21.57 | 1.007 / 21.53 | +| fp8-torchao | none | 0.689 / 51.63 | OOM | +| fp8-torchao | activation-offload | 0.975 / 37.77 | 2.058 / 37.73 | +| fp8-torchao | layer | 0.689 / 51.63 | OOM | +| fp8-torchao | interval2 | 0.684 / 51.63 | OOM | +| fp8-torchao | seg2-stride4 | 0.674 / 51.63 | OOM | +| fp8-torchao | seg2-stride4-offload | 0.965 / 37.77 | 2.053 / 37.73 | + +### LongCat Image + +示例：`longcat-image.peft-lora`。训练分辨率：512 px 正方形；验证分辨率：1024x1024。各行使用 `attention_mechanism=native-flash`。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.193 / 16.47 | 0.262 / 16.42 | +| bf16 | activation-offload | 0.544 / 12.73 | 0.543 / 12.69 | +| bf16 | layer | 0.327 / 12.38 | 0.370 / 12.34 | +| bf16 | interval2 | 0.257 / 14.38 | 0.313 / 14.34 | +| bf16 | seg2-stride4 | 0.263 / 14.36 | 0.316 / 14.31 | +| bf16 | seg2-stride4-offload | 0.446 / 13.42 | 0.492 / 13.38 | +| int8-sdnq-hadamard | none | 0.578 / 12.54 | 0.537 / 12.45 | +| int8-sdnq-hadamard | activation-offload | 1.184 / 7.43 | 1.185 / 7.39 | +| int8-sdnq-hadamard | layer | 0.901 / 7.19 | 0.911 / 7.09 | +| int8-sdnq-hadamard | interval2 | 0.718 / 9.73 | 0.695 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4 | 0.735 / 9.72 | 0.717 / 9.68 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.056 / 8.49 | 1.011 / 8.45 | +| fp8-torchao | none | 0.602 / 25.19 | 0.834 / 25.14 | +| fp8-torchao | activation-offload | 1.662 / 7.75 | 1.844 / 7.70 | +| fp8-torchao | layer | 0.984 / 7.57 | 1.080 / 7.53 | +| fp8-torchao | interval2 | 0.750 / 16.15 | 0.938 / 16.10 | +| fp8-torchao | seg2-stride4 | 0.760 / 16.13 | 0.961 / 16.09 | +| fp8-torchao | seg2-stride4-offload | 1.287 / 13.02 | 1.653 / 12.98 | + +### LongCat Video + +示例：`longcat-video.peft-lora+ramtorch`。分辨率：832x480，81f。各行使用 `attention_mechanism=native-flash`。 + +LongCat Video 在这个 shape 下 activation 压力很高。Full per-block checkpointing 是实际可用的基线。Partial checkpoint 行（`interval2`、`seg2-stride4`）在这里放不下；即使 strided 行启用 attention activation offload 也不够。单独的 attention activation offload 可让 bf16 和 SDNQ 跑通，但明显慢于 full checkpointing。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM (77.86 GiB) | OOM (43.30 GiB) | +| bf16 | activation-offload | 25.774 / 37.36 | 49.149 / 37.14 | +| bf16 | layer | 7.448 / 23.73 | 24.866 / 23.68 | +| bf16 | interval2 | OOM (76.41 GiB) | OOM (42.80 GiB) | +| bf16 | seg2-stride4 | OOM (76.42 GiB) | OOM (43.06 GiB) | +| bf16 | seg2-stride4-offload | OOM (76.72 GiB) | OOM (42.29 GiB) | +| int8-sdnq-hadamard | none | OOM (77.40 GiB) | OOM (43.59 GiB) | +| int8-sdnq-hadamard | activation-offload | 30.887 / 35.28 | 61.270 / 35.24 | +| int8-sdnq-hadamard | layer | 8.444 / 21.60 | 25.164 / 21.55 | +| int8-sdnq-hadamard | interval2 | OOM (76.01 GiB) | OOM (42.47 GiB) | +| int8-sdnq-hadamard | seg2-stride4 | OOM (77.01 GiB) | OOM (43.02 GiB) | +| int8-sdnq-hadamard | seg2-stride4-offload | OOM (76.42 GiB) | OOM (42.57 GiB) | +| fp8-torchao | none | OOM (75.87 GiB) | OOM (41.28 GiB) | +| fp8-torchao | activation-offload | 30.163 / 47.88 | OOM (40.11 GiB) | +| fp8-torchao | layer | 8.343 / 34.16 | 24.659 / 34.07 | +| fp8-torchao | interval2 | OOM (74.63 GiB) | OOM (40.85 GiB) | +| fp8-torchao | seg2-stride4 | OOM (75.37 GiB) | OOM (41.19 GiB) | +| fp8-torchao | seg2-stride4-offload | OOM (75.55 GiB) | OOM (40.94 GiB) | + +### LTXVideo 0.9.5 + +Example: `ltxvideo-0.9.5-t2v.peft-lora`. Resolution: 768x512, 49f. + +数字是 warm seconds/step / peak GiB。完整运行平均值包含 setup 和 compile overhead，保存在 sweep artifacts 里。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.274 / 8.77 | 0.275 / 8.72 | +| bf16 | layer | 0.462 / 4.21 | 0.459 / 4.16 | +| bf16 | interval2 | 0.449 / 4.30 | 0.433 / 4.25 | +| bf16 | seg2-stride4 | 0.359 / 6.40 | 0.357 / 6.35 | +| int8-sdnq-hadamard | none | 0.688 / 7.05 | 0.640 / 6.90 | +| int8-sdnq-hadamard | layer | 1.094 / 2.59 | 1.081 / 2.44 | +| int8-sdnq-hadamard | interval2 | 1.112 / 2.57 | 1.073 / 2.53 | +| int8-sdnq-hadamard | seg2-stride4 | 0.887 / 4.61 | 0.817 / 4.56 | +| fp8-torchao | none | 0.655 / 17.45 | 0.735 / 17.41 | +| fp8-torchao | layer | 1.226 / 2.94 | 1.206 / 2.89 | +| fp8-torchao | interval2 | 1.259 / 3.40 | 1.233 / 3.35 | +| fp8-torchao | seg2-stride4 | 0.933 / 9.96 | 0.901 / 9.91 | +| fp8wo-torchao | none | 0.328 / 10.06 | 0.325 / 10.01 | +| fp8wo-torchao | layer | 0.567 / 2.64 | 0.540 / 2.59 | +| fp8wo-torchao | interval2 | 0.555 / 2.84 | 0.531 / 2.79 | +| fp8wo-torchao | seg2-stride4 | 0.443 / 6.20 | 0.432 / 6.15 | + +这个 sweep 中，LTXVideo 0.9 不支持 attention activation offload rows。 + +### LTXVideo2 2.3 + +Example: `ltxvideo2-2.3-dev-720p-single-gpu.peft-lora+sdnq-hadamard`. Resolution: 1280x704, 49f. + +Note: LTXVideo2 2.3 should be read from the no-regional-compile rows in this sweep; regional compile raised memory pressure for this model. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 6.350 / 58.36 | OOM | +| bf16 | layer | 3.993 / 47.95 | OOM | +| bf16 | interval2 | 3.977 / 48.83 | OOM | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | 5.554 / 75.64 | OOM | +| int8-sdnq-hadamard | none | OOM | OOM | +| int8-sdnq-hadamard | activation-offload | 10.102 / 38.20 | 9.852 / 38.15 | +| int8-sdnq-hadamard | layer | 7.753 / 27.78 | 7.579 / 27.73 | +| int8-sdnq-hadamard | interval2 | 7.733 / 28.66 | 7.288 / 28.61 | +| int8-sdnq-hadamard | seg2-stride4 | 6.522 / 61.50 | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | 8.659 / 55.48 | OOM | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | failed | 22.917 / 38.57 | +| fp8-torchao | layer | 8.580 / 30.27 | 10.381 / 30.22 | +| fp8-torchao | interval2 | 8.660 / 33.71 | 10.661 / 33.66 | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | failed | OOM | + +### Lumina2 + +Example: `lumina2.peft-lora`. Resolution: 512x512. + +说明：Lumina2 现在使用 segmented whole-block 路径。`interval2` 会 checkpoint 每个 two-block segment；`seg2-stride4` 会 checkpoint 两个 blocks，让接下来的两个 blocks 保留 activations，然后重复。Attention activation offload 不在这次 Lumina2 run 中。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.235 / 15.99 | 0.332 / 15.94 | +| bf16 | layer | 0.384 / 6.60 | 0.457 / 6.56 | +| bf16 | interval2 | 0.356 / 6.87 | 0.424 / 6.82 | +| bf16 | seg2-stride4 | 0.295 / 11.43 | 0.377 / 11.38 | +| int8-sdnq-hadamard | none | 0.584 / 13.59 | 0.541 / 13.55 | +| int8-sdnq-hadamard | layer | 0.899 / 4.21 | 0.827 / 4.16 | +| int8-sdnq-hadamard | interval2 | 0.865 / 4.48 | 0.835 / 4.43 | +| int8-sdnq-hadamard | seg2-stride4 | 0.719 / 9.03 | 0.707 / 8.99 | +| fp8-torchao | none | 0.598 / 28.03 | 0.763 / 27.98 | +| fp8-torchao | layer | 0.950 / 5.93 | 0.967 / 5.88 | +| fp8-torchao | interval2 | 0.938 / 6.70 | 0.974 / 6.66 | +| fp8-torchao | seg2-stride4 | 0.765 / 17.36 | 0.901 / 17.32 | +| fp8wo-torchao | none | 0.273 / 17.97 | 0.389 / 17.93 | +| fp8wo-torchao | layer | 0.452 / 4.99 | 0.525 / 4.94 | +| fp8wo-torchao | interval2 | 0.427 / 5.40 | 0.522 / 5.35 | +| fp8wo-torchao | seg2-stride4 | 0.360 / 11.68 | 0.466 / 11.64 | + +### MageFlow + +Example: `mageflow-image-24g.peft-lora`. Resolution: 1024x1024. + +说明：MageFlow 的 1024px 可变形状图像路径主要受益于 attention activation offload 和 weight-only FP8。Block checkpointing 模式是可用的，但在本轮 sweep 中没有降低测得的峰值驻留显存。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 7.735 / 36.74 | 7.658 / 36.69 | +| bf16 | activation-offload | 8.902 / 23.05 | 9.477 / 23.00 | +| bf16 | layer | 7.805 / 36.74 | 7.504 / 36.69 | +| bf16 | interval2 | 8.036 / 36.74 | 7.762 / 36.69 | +| bf16 | seg2-stride4 | 7.882 / 36.74 | 7.531 / 36.69 | +| bf16 | seg2-stride4-offload | 8.833 / 23.05 | 9.288 / 23.01 | +| int8-sdnq-hadamard | none | 81.016 / 37.38 | 94.991 / 37.34 | +| fp8wo-torchao | none | 5.454 / 36.86 | 5.772 / 36.82 | +| fp8wo-torchao | activation-offload | 6.295 / 23.18 | 6.738 / 23.14 | +| fp8wo-torchao | seg2-stride4 | 5.542 / 36.86 | 5.595 / 36.82 | + +### OmniGen + +Example: `omnigen.lycoris-lokr`. Resolution: 1024x1024. + +说明：OmniGen 使用 token-ID prompt，而不是缓存的文本 embedding。这里测的是已支持的无 checkpoint 和整 block torch checkpointing 路径；interval、segmented-stride 和 attention-offload 控制在本次 sweep 中尚未为这个 family 实现。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.425 / 14.20 | 0.293 / 14.24 | +| bf16 | layer | 0.597 / 10.13 | 0.389 / 10.09 | +| int8-sdnq-hadamard | none | 1.523 / 11.00 | 1.388 / 11.06 | +| int8-sdnq-hadamard | layer | 1.312 / 6.75 | 1.064 / 6.70 | +| fp8-torchao | none | 0.690 / 19.25 | 0.608 / 19.30 | +| fp8-torchao | layer | 1.069 / 7.09 | 0.824 / 7.04 | +| fp8wo-torchao | none | 0.454 / 17.71 | 0.377 / 17.73 | +| fp8wo-torchao | layer | 0.646 / 6.98 | 0.534 / 6.94 | + +### PixArt + +Example: `pixart.lycoris-lokr`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.700 / 41.73 | 1.734 / 41.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 2.433 / 5.63 | 2.346 / 5.58 | +| bf16 | interval2 | 2.440 / 6.01 | 2.348 / 5.96 | +| bf16 | seg2-stride4 | 2.092 / 23.90 | 2.072 / 23.86 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.905 / 47.58 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.738 / 6.07 | 2.937 / 6.03 | +| int8-sdnq-hadamard | interval2 | 2.734 / 6.03 | 2.943 / 5.99 | +| int8-sdnq-hadamard | seg2-stride4 | 2.336 / 26.85 | 2.596 / 26.81 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.254 / 8.06 | 4.646 / 8.01 | +| fp8-torchao | interval2 | 3.260 / 9.66 | 4.649 / 9.61 | +| fp8-torchao | seg2-stride4 | 2.827 / 63.27 | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Qwen Image + +Example: `qwen_image.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.202 / 41.04 | 3.377 / 40.99 | +| bf16 | interval2 | 1.201 / 41.04 | 3.382 / 40.99 | +| bf16 | seg2-stride4 | 1.205 / 41.03 | 3.385 / 40.99 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.675 / 63.48 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.640 / 24.09 | 3.928 / 24.05 | +| int8-sdnq-hadamard | interval2 | 2.722 / 24.09 | 3.918 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4 | 2.663 / 24.09 | 3.919 / 24.05 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 3.088 / 25.34 | 6.172 / 25.29 | +| fp8-torchao | interval2 | 3.095 / 25.34 | 6.173 / 25.29 | +| fp8-torchao | seg2-stride4 | 3.125 / 25.34 | 6.141 / 25.29 | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Sana + +Example: `sana.lycoris-lokr`. Resolution: 1024x1024. + +Note: Sana has interval checkpointing; stride is not a separate segmented schedule for this family in the measured rows. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 23.71 | 0.597 / 23.67 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.529 / 23.72 | 0.596 / 23.66 | +| bf16 | interval2 | 0.530 / 23.72 | 0.598 / 23.66 | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.554 / 22.92 | 0.590 / 22.88 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.554 / 22.92 | 0.589 / 22.88 | +| int8-sdnq-hadamard | interval2 | 0.556 / 22.92 | 0.591 / 22.88 | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.633 / 33.67 | 0.753 / 33.62 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.633 / 33.67 | 0.755 / 33.62 | +| fp8-torchao | interval2 | 0.631 / 33.67 | 0.759 / 33.62 | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SanaVideo + +Example: `sanavideo-2b-480p.peft-lora`. Resolution: 832x480, 49f. + +Note: SanaVideo 使用 linear attention，所以 attention activation offload 仍不支持。标准路径支持 segmented whole-block checkpointing。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.599 / 59.15 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.597 / 59.15 | OOM | +| bf16 | interval2 | not run | OOM | +| bf16 | seg2-stride4 | not run | OOM | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.641 / 58.36 | OOM | +| int8-sdnq-hadamard | interval2 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4 | not run | OOM | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | OOM | OOM | +| fp8-torchao | interval2 | OOM | OOM | +| fp8-torchao | seg2-stride4 | OOM | OOM | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD 1.x + +Example: `sd1x-dreamshaper.peft-lora`. Resolution: 512x512. + +说明：SD1x 使用 diffusers UNet path。普通 layer checkpointing 受支持，但 interval 和 segmented stride controls 没有接入这个 family。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.181 / 2.87 | 0.176 / 2.83 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.305 / 1.98 | 0.292 / 1.93 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.446 / 3.07 | 0.431 / 3.04 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 0.701 / 1.79 | 0.666 / 1.74 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 0.410 / 4.23 | 0.401 / 4.18 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 0.785 / 1.96 | 0.756 / 1.91 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### SD3 + +Example: `sd3.peft-lora`. Resolution: 1024x1024. + +说明：SD3 在普通 transformer path 上使用真正的连续 segmented checkpointing。Attention activation offload 已支持；它会大幅降低 VRAM，但会牺牲 throughput。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.529 / 34.59 | 1.189 / 34.53 | +| bf16 | activation-offload | 1.335 / 9.68 | 2.850 / 9.63 | +| bf16 | layer | 0.721 / 7.67 | 1.607 / 7.62 | +| bf16 | interval2 | 0.723 / 8.93 | 1.606 / 8.88 | +| bf16 | seg2-stride4 | 0.620 / 20.14 | 1.398 / 20.09 | +| bf16 | seg2-stride4-offload | 1.229 / 12.21 | 2.639 / 12.16 | +| int8-sdnq-hadamard | none | 0.858 / 33.66 | 1.381 / 33.61 | +| int8-sdnq-hadamard | activation-offload | 1.845 / 8.90 | 3.297 / 8.85 | +| int8-sdnq-hadamard | layer | 1.265 / 6.58 | 1.857 / 6.53 | +| int8-sdnq-hadamard | interval2 | 1.264 / 7.30 | 1.858 / 7.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.049 / 19.02 | 1.625 / 18.98 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.527 / 12.42 | 3.023 / 12.37 | +| fp8-torchao | none | OOM | OOM | +| fp8-torchao | activation-offload | 4.707 / 10.27 | 9.814 / 10.23 | +| fp8-torchao | layer | 1.575 / 9.13 | 3.279 / 9.09 | +| fp8-torchao | interval2 | 1.576 / 12.62 | 3.282 / 12.58 | +| fp8-torchao | seg2-stride4 | 1.376 / 45.70 | OOM | +| fp8-torchao | seg2-stride4-offload | 4.184 / 26.17 | 8.714 / 26.12 | +| fp8wo-torchao | none | 0.567 / 35.21 | 1.252 / 35.17 | +| fp8wo-torchao | activation-offload | 1.392 / 8.71 | 2.921 / 8.67 | +| fp8wo-torchao | layer | 0.795 / 5.69 | 1.736 / 5.65 | +| fp8wo-torchao | interval2 | 0.797 / 7.08 | 1.734 / 7.03 | +| fp8wo-torchao | seg2-stride4 | 0.679 / 19.43 | 1.490 / 19.38 | +| fp8wo-torchao | seg2-stride4-offload | 1.257 / 12.00 | 2.676 / 11.96 | + +### SDXL + +Example: `sdxl.lycoris-lokr`. Resolution: 1024x1024. + +Note: SDXL has real layer checkpointing. Interval and stride rows are included as coverage data, not as segmented-support recommendations. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.606 / 13.03 | 0.585 / 12.98 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.080 / 6.53 | 1.029 / 6.48 | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 1.741 / 13.72 | 1.643 / 13.68 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 2.820 / 4.64 | 2.647 / 4.59 | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | 1.608 / 26.22 | 1.582 / 26.16 | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | 2.939 / 5.10 | 2.890 / 5.04 | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Stable Cascade + +Example: `cascade-stage-c.lycoris-lokr`. Resolution: 1024x1024. + +说明：stage C 使用全精度 prior 路径。这些行使用 `mixed_precision=no` 和 `base_model_precision=no_change`；量化 base precision 行对这个模型没有实际意义。interval 和 stride 模式作用在 UNet 的 Res/Timestep/Attention micro-block 序列上。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.884 / 51.52 | OOM | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 1.179 / 22.68 | 2.135 / 22.61 | +| bf16 | interval2 | 1.032 / 36.99 | 1.871 / 36.92 | +| bf16 | seg2-stride4 | 1.032 / 37.20 | 1.870 / 37.13 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | unsupported | unsupported | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | unsupported | unsupported | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Wan 2.1 T2V 1.3B + +Example: `wan2.1-t2v-1.3b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 1.3B should be read from the no-regional-compile/RamTorch rows; regional compile was not a useful throughput setting in this sweep. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 1.407 / 71.90 | OOM | +| bf16 | activation-offload | 3.472 / 8.78 | 7.179 / 8.66 | +| bf16 | layer | 2.099 / 4.73 | 4.459 / 4.68 | +| bf16 | interval2 | 2.139 / 6.32 | 4.514 / 6.27 | +| bf16 | seg2-stride4 | 1.806 / 39.25 | 3.921 / 39.21 | +| bf16 | seg2-stride4-offload | 2.993 / 22.66 | 6.493 / 22.61 | +| int8-sdnq-hadamard | none | 1.850 / 71.91 | OOM | +| int8-sdnq-hadamard | activation-offload | 4.204 / 8.72 | 7.387 / 8.68 | +| int8-sdnq-hadamard | layer | 2.790 / 4.70 | 4.989 / 4.65 | +| int8-sdnq-hadamard | interval2 | 2.874 / 6.29 | 5.093 / 6.24 | +| int8-sdnq-hadamard | seg2-stride4 | 2.558 / 39.27 | 4.393 / 39.22 | +| int8-sdnq-hadamard | seg2-stride4-offload | 3.711 / 22.67 | 6.695 / 22.63 | +| fp8-torchao | none | 1.727 / 73.57 | OOM | +| fp8-torchao | activation-offload | 4.061 / 10.08 | 7.404 / 9.96 | +| fp8-torchao | layer | 2.607 / 5.98 | 4.888 / 5.93 | +| fp8-torchao | interval2 | 2.744 / 7.57 | 4.916 / 7.52 | +| fp8-torchao | seg2-stride4 | 2.246 / 40.55 | 4.245 / 40.50 | +| fp8-torchao | seg2-stride4-offload | 3.602 / 24.02 | 6.683 / 23.91 | + +### Wan 2.1 T2V 14B + +Example: `wan2.1-t2v-14b-480p-single-gpu.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan 14B is mainly a fit test for activation savings. Status-only cells are still useful because they show which combinations reached the memory limit. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | OOM | OOM | +| bf16 | activation-offload | 13.144 / 36.80 | OOM | +| bf16 | layer | 7.162 / 16.28 | 21.770 / 16.23 | +| bf16 | interval2 | 7.172 / 19.62 | 21.777 / 19.58 | +| bf16 | seg2-stride4 | OOM | OOM | +| bf16 | seg2-stride4-offload | OOM | OOM | +| int8-sdnq-hadamard | none | unsupported | unsupported | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | unsupported | unsupported | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | failed | failed | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | failed | failed | +| fp8-torchao | seg2-stride4 | failed | failed | +| fp8-torchao | seg2-stride4-offload | failed | failed | + +### Wan S2V + +Example: `wan-s2v-14b-480p.peft-lora+ramtorch`. Resolution: 832x480, 81f. + +Note: Wan S2V is included as coverage data for the video/audio path. Treat failed cells as implementation coverage gaps. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | failed | failed | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | failed | failed | +| bf16 | interval2 | unsupported | unsupported | +| bf16 | seg2-stride4 | unsupported | unsupported | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | failed | failed | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | failed | failed | +| int8-sdnq-hadamard | interval2 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4 | unsupported | unsupported | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8-torchao | none | failed | failed | +| fp8-torchao | activation-offload | unsupported | unsupported | +| fp8-torchao | layer | failed | failed | +| fp8-torchao | interval2 | unsupported | unsupported | +| fp8-torchao | seg2-stride4 | unsupported | unsupported | +| fp8-torchao | seg2-stride4-offload | unsupported | unsupported | + +### Z-Image Turbo + +Example: `z-image-turbo.peft-lora`. Resolution: 1024x1024. + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.243 / 21.25 | 0.316 / 21.21 | +| bf16 | activation-offload | 0.837 / 13.24 | 0.805 / 13.19 | +| bf16 | layer | 0.479 / 12.87 | 0.493 / 12.83 | +| bf16 | interval2 | 0.452 / 13.04 | 0.477 / 12.99 | +| bf16 | seg2-stride4 | 0.349 / 16.88 | 0.400 / 16.83 | +| bf16 | seg2-stride4-offload | 0.681 / 15.03 | 0.736 / 14.99 | +| int8-sdnq-hadamard | none | 0.645 / 15.60 | 0.615 / 15.55 | +| int8-sdnq-hadamard | activation-offload | 1.439 / 7.61 | 1.382 / 7.56 | +| int8-sdnq-hadamard | layer | 1.046 / 7.25 | 1.021 / 7.20 | +| int8-sdnq-hadamard | interval2 | 1.074 / 7.41 | 0.996 / 7.36 | +| int8-sdnq-hadamard | seg2-stride4 | 0.867 / 11.25 | 0.841 / 11.20 | +| int8-sdnq-hadamard | seg2-stride4-offload | 1.202 / 9.39 | 1.162 / 9.35 | +| fp8-torchao | none | 1.232 / 37.50 | 1.476 / 37.46 | +| fp8-torchao | activation-offload | 3.623 / 7.97 | 3.564 / 7.93 | +| fp8-torchao | layer | 2.319 / 7.93 | 2.344 / 7.88 | +| fp8-torchao | interval2 | 2.336 / 8.80 | 2.309 / 8.75 | +| fp8-torchao | seg2-stride4 | 1.843 / 22.55 | 1.930 / 22.50 | +| fp8-torchao | seg2-stride4-offload | 2.947 / 15.57 | 3.243 / 15.52 | + +### ZLab I1 + +Example: `zlab-i1.peft-lora`. Resolution: 1024x1024. + +说明：ZLab I1 会把 U-Net 式 skip tensors 放进 segmented checkpoint state。该 family 尚未接入 attention activation offload。 + +| Precision | Mode | H100 | L40S | +| --- | --- | ---: | ---: | +| bf16 | none | 0.462 / 22.21 | 0.865 / 22.16 | +| bf16 | activation-offload | unsupported | unsupported | +| bf16 | layer | 0.693 / 7.79 | 1.148 / 7.75 | +| bf16 | interval2 | 0.676 / 8.30 | 1.152 / 8.25 | +| bf16 | seg2-stride4 | 0.567 / 14.97 | 1.014 / 14.92 | +| bf16 | seg2-stride4-offload | unsupported | unsupported | +| int8-sdnq-hadamard | none | 0.861 / 19.21 | 0.926 / 19.16 | +| int8-sdnq-hadamard | activation-offload | unsupported | unsupported | +| int8-sdnq-hadamard | layer | 1.385 / 4.78 | 1.265 / 4.74 | +| int8-sdnq-hadamard | interval2 | 1.298 / 5.30 | 1.277 / 5.26 | +| int8-sdnq-hadamard | seg2-stride4 | 1.073 / 11.98 | 1.098 / 11.93 | +| int8-sdnq-hadamard | seg2-stride4-offload | unsupported | unsupported | +| fp8wo-torchao | none | 0.504 / 25.08 | 0.930 / 25.02 | +| fp8wo-torchao | activation-offload | unsupported | unsupported | +| fp8wo-torchao | layer | 0.772 / 5.12 | 1.280 / 5.07 | +| fp8wo-torchao | interval2 | 0.759 / 5.84 | 1.290 / 5.79 | +| fp8wo-torchao | seg2-stride4 | 0.633 / 15.12 | 1.115 / 15.08 | +| fp8wo-torchao | seg2-stride4-offload | unsupported | unsupported | + + diff --git a/documentation/experimental/UNSLOTH_CHECKPOINTING.es.md b/documentation/experimental/UNSLOTH_CHECKPOINTING.es.md index 9f0911f9d..99f7564c0 100644 --- a/documentation/experimental/UNSLOTH_CHECKPOINTING.es.md +++ b/documentation/experimental/UNSLOTH_CHECKPOINTING.es.md @@ -34,7 +34,13 @@ En familias compatibles también puedes checkpointar menos bloques: `gradient_checkpointing_interval: 2` checkpointa chunks contiguos de dos bloques en rutas de bloque completo compatibles. Valores más altos recomputan menos y dejan más activaciones en VRAM. -`torch-ffn` y `unsloth-ffn` soportan actualmente bloques estilo Flux.1 y MageFlow. Otras familias fallan claramente hasta que sus bloques expongan el mismo límite seguro. +En esas rutas segmentadas, `gradient_checkpointing_segment_stride` también funciona con `unsloth`. Trátalo como una palanca para caber, no para acelerar: los bloques saltados se quedan en GPU, mientras los bloques checkpointed siguen usando CPU offload para tensores guardados. Para el resumen con torch y benchmarks por modelo, consulta [Segmented Checkpointing](SEGMENTED_CHECKPOINTING.md). + +`gradient_checkpointing_offload_attention` es independiente del backend. En bloques compatibles con separación attention/FFN, descarga las activaciones guardadas del lado attention. Puede ejecutarse solo o combinarse con `torch`, `torch-ffn`, `unsloth` o `unsloth-ffn` cuando el modelo soporte ese backend. + +`gradient_checkpointing_offload_pin_memory_max_buckets` controla el pooling de CPU pinned para tensores guardados offloaded. El valor predeterminado es `12` buckets de tensor distintos; usa `0` para usar solo memoria CPU normal. + +`torch-ffn` y `unsloth-ffn` soportan actualmente Chroma, Flux, Krea 2, LTXVideo2, MageFlow, Wan y Z-Image. Otras familias fallan claramente hasta que sus bloques expongan el mismo límite seguro. ## Qué intercambia diff --git a/documentation/experimental/UNSLOTH_CHECKPOINTING.hi.md b/documentation/experimental/UNSLOTH_CHECKPOINTING.hi.md index f79c3c79a..07007efe1 100644 --- a/documentation/experimental/UNSLOTH_CHECKPOINTING.hi.md +++ b/documentation/experimental/UNSLOTH_CHECKPOINTING.hi.md @@ -34,7 +34,13 @@ Supported model families में आप कम blocks भी checkpoint कर `gradient_checkpointing_interval: 2` supported whole-block paths पर contiguous दो-block chunks checkpoint करता है। Higher values कम recompute करती हैं और VRAM में ज्यादा activations रखती हैं। -`torch-ffn` और `unsloth-ffn` अभी Flux.1-style blocks और MageFlow support करते हैं। बाकी model families साफ error देंगी जब तक उनके blocks वही safe boundary expose नहीं करते। +इन segmented paths पर `gradient_checkpointing_segment_stride` भी `unsloth` के साथ काम करता है। इसे fit lever मानें, speed lever नहीं: skipped blocks GPU पर रहते हैं, और checkpointed blocks saved tensors के लिए CPU offload use करते हैं। Torch-only overview और model benchmarks के लिए [Segmented Checkpointing](SEGMENTED_CHECKPOINTING.md) देखें। + +`gradient_checkpointing_offload_attention` backend से अलग option है। Supported attention/FFN split blocks पर यह attention-side saved activations offload करता है। यह अकेले चल सकता है या model उस backend को support करे तो `torch`, `torch-ffn`, `unsloth`, या `unsloth-ffn` के साथ combine हो सकता है। + +`gradient_checkpointing_offload_pin_memory_max_buckets` offloaded saved tensors के लिए pinned CPU pooling control करता है। default `12` distinct tensor buckets है; normal CPU memory ही इस्तेमाल करने के लिए इसे `0` करें। + +`torch-ffn` और `unsloth-ffn` अभी Chroma, Flux, Krea 2, LTXVideo2, MageFlow, Wan, और Z-Image support करते हैं। बाकी model families साफ error देंगी जब तक उनके blocks वही safe boundary expose नहीं करते। ## Tradeoff diff --git a/documentation/experimental/UNSLOTH_CHECKPOINTING.ja.md b/documentation/experimental/UNSLOTH_CHECKPOINTING.ja.md index 75fc87577..c44502197 100644 --- a/documentation/experimental/UNSLOTH_CHECKPOINTING.ja.md +++ b/documentation/experimental/UNSLOTH_CHECKPOINTING.ja.md @@ -34,7 +34,13 @@ `gradient_checkpointing_interval: 2` は対応する whole-block path で連続した 2-block chunk を checkpoint します。値を大きくすると再計算は減り、VRAM に残る activation は増えます。 -`torch-ffn` と `unsloth-ffn` は現在 Flux.1-style blocks と MageFlow に対応しています。他のモデル族は、同じ安全な境界を expose するまで明示的に失敗します。 +これらの segmented path では、`gradient_checkpointing_segment_stride` も `unsloth` で使えます。速度目的ではなく fit lever として扱ってください。Skip された blocks は GPU に残り、checkpoint された blocks は保存 tensor に CPU offload を使います。Torch-only の概要とモデル別 benchmark は [Segmented Checkpointing](SEGMENTED_CHECKPOINTING.md) を参照してください。 + +`gradient_checkpointing_offload_attention` は backend とは別の option です。対応する attention/FFN split blocks では、attention 側の保存 activations を offload します。単体でも実行でき、モデルがその backend をサポートする場合は `torch`、`torch-ffn`、`unsloth`、`unsloth-ffn` と組み合わせられます。 + +`gradient_checkpointing_offload_pin_memory_max_buckets` は offload された保存 tensor の pinned CPU pooling を制御します。デフォルトは `12` 個の distinct tensor buckets です。`0` にすると通常の CPU memory だけを使います。 + +`torch-ffn` と `unsloth-ffn` は現在 Chroma、Flux、Krea 2、LTXVideo2、MageFlow、Wan、Z-Image に対応しています。他のモデル族は、同じ安全な境界を expose するまで明示的に失敗します。 ## 何を交換するか diff --git a/documentation/experimental/UNSLOTH_CHECKPOINTING.md b/documentation/experimental/UNSLOTH_CHECKPOINTING.md index 82d08ff0e..170aeb9d1 100644 --- a/documentation/experimental/UNSLOTH_CHECKPOINTING.md +++ b/documentation/experimental/UNSLOTH_CHECKPOINTING.md @@ -34,7 +34,13 @@ For supported model families, you can also checkpoint fewer blocks: `gradient_checkpointing_interval: 2` checkpoints contiguous two-block chunks on supported whole-block paths. Higher values spend less time recomputing and keep more activations in VRAM. -`torch-ffn` and `unsloth-ffn` currently support Flux.1-style blocks and MageFlow. Other model families fail clearly until their block internals expose the same safe boundary. +On those segmented paths, `gradient_checkpointing_segment_stride` also works with `unsloth`. Treat it as a fit lever, not a speed lever: the skipped blocks stay on GPU, while checkpointed blocks still use CPU offload for saved tensors. For the torch-only overview and model benchmarks, see [Segmented Checkpointing](SEGMENTED_CHECKPOINTING.md). + +`gradient_checkpointing_offload_attention` is separate from the backend. On supported attention/FFN split blocks, it offloads attention-side saved activations. It can run by itself or combine with `torch`, `torch-ffn`, `unsloth`, or `unsloth-ffn` when that backend is supported by the model. + +`gradient_checkpointing_offload_pin_memory_max_buckets` controls pinned CPU pooling for offloaded saved tensors. The default is `12` distinct tensor buckets; set it to `0` to use normal CPU memory only. + +`torch-ffn` and `unsloth-ffn` currently support Chroma, Flux, Krea 2, LTXVideo2, MageFlow, Wan, and Z-Image. Other model families fail clearly until their block internals expose the same safe boundary. ## What It Trades diff --git a/documentation/experimental/UNSLOTH_CHECKPOINTING.pt-BR.md b/documentation/experimental/UNSLOTH_CHECKPOINTING.pt-BR.md index 7791b3537..f2594e44c 100644 --- a/documentation/experimental/UNSLOTH_CHECKPOINTING.pt-BR.md +++ b/documentation/experimental/UNSLOTH_CHECKPOINTING.pt-BR.md @@ -34,7 +34,13 @@ Em famílias compatíveis, você também pode checkpointar menos blocos: `gradient_checkpointing_interval: 2` faz checkpoint de chunks contiguos de dois blocos em caminhos whole-block compatíveis. Valores maiores recomputam menos e mantêm mais activations na VRAM. -`torch-ffn` e `unsloth-ffn` atualmente suportam blocos estilo Flux.1 e MageFlow. Outras famílias falham claramente até seus blocos exporem a mesma fronteira segura. +Nesses caminhos segmentados, `gradient_checkpointing_segment_stride` também funciona com `unsloth`. Trate como alavanca para caber, não para acelerar: os blocos pulados ficam na GPU, enquanto os blocos checkpointed ainda usam CPU offload para tensores salvos. Para o resumo torch-only e benchmarks por modelo, veja [Segmented Checkpointing](SEGMENTED_CHECKPOINTING.md). + +`gradient_checkpointing_offload_attention` e independente do backend. Em blocos compatíveis com separacao attention/FFN, faz offload das activations salvas do lado attention. Pode rodar sozinho ou ser combinado com `torch`, `torch-ffn`, `unsloth` ou `unsloth-ffn` quando o modelo suportar esse backend. + +`gradient_checkpointing_offload_pin_memory_max_buckets` controla o pooling de CPU pinned para tensores salvos offloaded. O padrao e `12` buckets de tensor distintos; use `0` para usar apenas memoria CPU normal. + +`torch-ffn` e `unsloth-ffn` atualmente suportam Chroma, Flux, Krea 2, LTXVideo2, MageFlow, Wan e Z-Image. Outras famílias falham claramente até seus blocos exporem a mesma fronteira segura. ## O tradeoff diff --git a/documentation/experimental/UNSLOTH_CHECKPOINTING.zh.md b/documentation/experimental/UNSLOTH_CHECKPOINTING.zh.md index b71c4dc5f..3391eb691 100644 --- a/documentation/experimental/UNSLOTH_CHECKPOINTING.zh.md +++ b/documentation/experimental/UNSLOTH_CHECKPOINTING.zh.md @@ -34,7 +34,13 @@ `gradient_checkpointing_interval: 2` 会在受支持的 whole-block 路径上 checkpoint 连续的两个 block chunk。值越大，重算越少，VRAM 里保留的 activation 越多。 -`torch-ffn` 和 `unsloth-ffn` 目前支持 Flux.1 风格 blocks 和 MageFlow。其他模型族会明确报错，直到它们的 block 暴露同样安全的边界。 +在这些分段路径上，`gradient_checkpointing_segment_stride` 也可以和 `unsloth` 一起使用。把它当作 fit lever，而不是 speed lever：跳过的 blocks 仍留在 GPU，checkpointed blocks 仍会把保存 tensor CPU offload。Torch-only 概览和模型 benchmark 见 [Segmented Checkpointing](SEGMENTED_CHECKPOINTING.md)。 + +`gradient_checkpointing_offload_attention` 独立于 backend。在支持 attention/FFN 分离的 blocks 上，它会 offload attention 侧保存的 activations。它可以单独运行；当模型支持所选 backend 时，也可与 `torch`、`torch-ffn`、`unsloth` 或 `unsloth-ffn` 组合。 + +`gradient_checkpointing_offload_pin_memory_max_buckets` 控制 offloaded saved tensors 的 pinned CPU pooling。默认是 `12` 个不同 tensor buckets；设为 `0` 时只使用普通 CPU memory。 + +`torch-ffn` 和 `unsloth-ffn` 目前支持 Chroma、Flux、Krea 2、LTXVideo2、MageFlow、Wan 和 Z-Image。其他模型族会明确报错，直到它们的 block 暴露同样安全的边界。 ## 它交换了什么 diff --git a/documentation/index.es.md b/documentation/index.es.md index 532579eae..546744159 100644 --- a/documentation/index.es.md +++ b/documentation/index.es.md @@ -72,9 +72,9 @@ --- - Funciones de investigación como AnyFlow, cuantización SDNQ Hadamard estilo ConvRot, checkpointing estilo Unsloth, Prompt2Effect, Self-Flow, Flow-DPO, LayerSync, Diff2Flow, Metal Flash Attention y Video CREPA + Funciones de investigación como AnyFlow, cuantización SDNQ Hadamard estilo ConvRot, checkpointing segmentado, checkpointing estilo Unsloth, Prompt2Effect, Self-Flow, Flow-DPO, LayerSync, Diff2Flow, Metal Flash Attention y Video CREPA - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/documentation/index.hi.md b/documentation/index.hi.md index c725ff7e7..df6665f2c 100644 --- a/documentation/index.hi.md +++ b/documentation/index.hi.md @@ -72,9 +72,9 @@ --- - AnyFlow, ConvRot-style SDNQ Hadamard quantization, Unsloth-style checkpointing, Prompt2Effect, Self-Flow, Flow-DPO, LayerSync, Diff2Flow, Metal Flash Attention और Video CREPA जैसी research features + AnyFlow, ConvRot-style SDNQ Hadamard quantization, segmented checkpointing, Unsloth-style checkpointing, Prompt2Effect, Self-Flow, Flow-DPO, LayerSync, Diff2Flow, Metal Flash Attention और Video CREPA जैसी research features - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/documentation/index.ja.md b/documentation/index.ja.md index 75b99cffb..4d4f3d807 100644 --- a/documentation/index.ja.md +++ b/documentation/index.ja.md @@ -72,9 +72,9 @@ --- - AnyFlow、ConvRot 形式の SDNQ Hadamard 量子化、Unsloth 形式 checkpointing、Prompt2Effect、Self-Flow、Flow-DPO、LayerSync、Diff2Flow、Metal Flash Attention、Video CREPA などの研究向け機能 + AnyFlow、ConvRot 形式の SDNQ Hadamard 量子化、segmented checkpointing、Unsloth 形式 checkpointing、Prompt2Effect、Self-Flow、Flow-DPO、LayerSync、Diff2Flow、Metal Flash Attention、Video CREPA などの研究向け機能 - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/documentation/index.md b/documentation/index.md index a45ac1695..c2c2df4da 100644 --- a/documentation/index.md +++ b/documentation/index.md @@ -72,9 +72,9 @@ --- - Research features such as AnyFlow, ConvRot-style SDNQ Hadamard quantization, Unsloth-style checkpointing, Prompt2Effect, Self-Flow, Flow-DPO, LayerSync, Diff2Flow, Metal Flash Attention, and Video CREPA + Research features such as AnyFlow, ConvRot-style SDNQ Hadamard quantization, segmented checkpointing, Unsloth-style checkpointing, Prompt2Effect, Self-Flow, Flow-DPO, LayerSync, Diff2Flow, Metal Flash Attention, and Video CREPA - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/documentation/index.pt-BR.md b/documentation/index.pt-BR.md index 222169c21..6b3502dd8 100644 --- a/documentation/index.pt-BR.md +++ b/documentation/index.pt-BR.md @@ -72,9 +72,9 @@ --- - Recursos de pesquisa como AnyFlow, quantização SDNQ Hadamard no estilo ConvRot, checkpointing estilo Unsloth, Prompt2Effect, Self-Flow, Flow-DPO, LayerSync, Diff2Flow, Metal Flash Attention e Video CREPA + Recursos de pesquisa como AnyFlow, quantização SDNQ Hadamard no estilo ConvRot, checkpointing segmentado, checkpointing estilo Unsloth, Prompt2Effect, Self-Flow, Flow-DPO, LayerSync, Diff2Flow, Metal Flash Attention e Video CREPA - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/documentation/index.zh.md b/documentation/index.zh.md index 522cff98e..d97af565c 100644 --- a/documentation/index.zh.md +++ b/documentation/index.zh.md @@ -72,9 +72,9 @@ --- - AnyFlow、ConvRot 风格的 SDNQ Hadamard 量化、Unsloth 风格 checkpointing、Prompt2Effect、Self-Flow、Flow-DPO、LayerSync、Diff2Flow、Metal Flash Attention、Video CREPA 等研究功能 + AnyFlow、ConvRot 风格的 SDNQ Hadamard 量化、segmented checkpointing、Unsloth 风格 checkpointing、Prompt2Effect、Self-Flow、Flow-DPO、LayerSync、Diff2Flow、Metal Flash Attention、Video CREPA 等研究功能 - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/mkdocs.yml b/mkdocs.yml index 787f36654..8a4980ebb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -357,6 +357,7 @@ nav: - Metal Flash Attention: experimental/METAL_FLASH_ATTENTION.md - Prompt2Effect: experimental/PROMPT2EFFECT.md - Self-Flow: experimental/SELF_FLOW.md + - Segmented Checkpointing: experimental/SEGMENTED_CHECKPOINTING.md - Scheduled Sampling: experimental/SCHEDULED_SAMPLING.md - T-LoRA: experimental/T_LORA.md - Unsloth-Style Checkpointing: experimental/UNSLOTH_CHECKPOINTING.md diff --git a/simpletuner/helpers/configuration/env_file.py b/simpletuner/helpers/configuration/env_file.py index 635bb0a1c..a88812a56 100644 --- a/simpletuner/helpers/configuration/env_file.py +++ b/simpletuner/helpers/configuration/env_file.py @@ -35,6 +35,10 @@ "QWEN_TEXT_ENCODER_MODEL_NAME_OR_PATH": "--qwen_text_encoder_model_name_or_path", "TRAIN_BATCH_SIZE": "--train_batch_size", "USE_GRADIENT_CHECKPOINTING": "--gradient_checkpointing", + "GRADIENT_CHECKPOINTING_OFFLOAD_ATTENTION": "--gradient_checkpointing_offload_attention", + "GRADIENT_CHECKPOINTING_OFFLOAD_PREFETCH": "--gradient_checkpointing_offload_prefetch", + "GRADIENT_CHECKPOINTING_OFFLOAD_PIN_MEMORY_MAX_BUCKETS": "--gradient_checkpointing_offload_pin_memory_max_buckets", + "GRADIENT_CHECKPOINTING_SEGMENT_STRIDE": "--gradient_checkpointing_segment_stride", "ENABLE_CHUNKED_FEED_FORWARD": "--enable_chunked_feed_forward", "FEED_FORWARD_CHUNK_SIZE": "--feed_forward_chunk_size", "CAPTION_DROPOUT_PROBABILITY": "--caption_dropout_probability", diff --git a/simpletuner/helpers/models/common.py b/simpletuner/helpers/models/common.py index df365cd7b..97a47423d 100644 --- a/simpletuner/helpers/models/common.py +++ b/simpletuner/helpers/models/common.py @@ -3266,16 +3266,59 @@ def _load_model(load_kwargs: dict): if checkpoint_backend in {"unsloth", "unsloth-ffn"}: logger.info("Using Unsloth-style gradient checkpointing (CPU offload)") + offload_attention = bool(getattr(self.config, "gradient_checkpointing_offload_attention", False)) + raw_pin_bucket_count = getattr(self.config, "gradient_checkpointing_offload_pin_memory_max_buckets", 12) + offload_pin_memory_max_buckets = 12 if raw_pin_bucket_count in (None, "", "None") else int(raw_pin_bucket_count) + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + set_activation_offload_pin_memory_max_buckets, + set_activation_offload_prefetch_enabled, + ) + + set_activation_offload_pin_memory_max_buckets(offload_pin_memory_max_buckets) + set_activation_offload_prefetch_enabled(bool(getattr(self.config, "gradient_checkpointing_offload_prefetch", False))) + if offload_attention: + trained_component = self.unwrap_model(model=self.model) if self.model is not None else None + if trained_component is None or not getattr(trained_component, "_supports_attention_activation_offload", False): + raise ValueError( + "--gradient_checkpointing_offload_attention requires a model with an attention/FFN checkpointing " + f"boundary, but {self.config.model_family} does not expose it." + ) + logger.info("Using attention activation offload for gradient checkpointing") + if self.config.gradient_checkpointing_interval is not None and self.config.gradient_checkpointing_interval > 1: if self.model is not None and hasattr(self.model, "set_gradient_checkpointing_interval"): logger.info("Setting gradient checkpointing interval..") self.unwrap_model(model=self.model).set_gradient_checkpointing_interval( int(self.config.gradient_checkpointing_interval) ) + else: + logger.warning( + "--gradient_checkpointing_interval=%s was requested, but %s does not expose interval " + "checkpointing support; the value will be ignored.", + self.config.gradient_checkpointing_interval, + self.config.model_family, + ) + + gradient_checkpointing_segment_stride = getattr(self.config, "gradient_checkpointing_segment_stride", None) + if gradient_checkpointing_segment_stride is not None: + if self.model is not None and hasattr(self.model, "set_gradient_checkpointing_segment_stride"): + logger.info("Setting gradient checkpointing segment stride..") + self.unwrap_model(model=self.model).set_gradient_checkpointing_segment_stride( + int(gradient_checkpointing_segment_stride) + ) + else: + logger.warning( + "--gradient_checkpointing_segment_stride=%s was requested, but %s does not expose segmented " + "checkpoint stride support; the value will be ignored.", + gradient_checkpointing_segment_stride, + self.config.model_family, + ) # Set gradient checkpointing backend on model if supported if self.model is not None and hasattr(self.model, "set_gradient_checkpointing_backend"): self.unwrap_model(model=self.model).set_gradient_checkpointing_backend(checkpoint_backend) + if self.model is not None and hasattr(self.model, "set_gradient_checkpointing_offload_attention"): + self.unwrap_model(model=self.model).set_gradient_checkpointing_offload_attention(offload_attention) self.fuse_qkv_projections() self.post_model_load_setup() diff --git a/simpletuner/helpers/musubi_block_swap.py b/simpletuner/helpers/musubi_block_swap.py index fba81e601..a467e93df 100644 --- a/simpletuner/helpers/musubi_block_swap.py +++ b/simpletuner/helpers/musubi_block_swap.py @@ -31,10 +31,28 @@ def _is_quanto_tensor(tensor) -> bool: return module_name.startswith("optimum.quanto.") and hasattr(tensor, "_data") +def _is_sdnq_tensor(tensor) -> bool: + return ( + type(tensor).__module__.startswith("sdnq.") + or type(tensor).__name__ == "SDNQTensor" + or (hasattr(tensor, "sdnq_dequantizer") and hasattr(tensor, "weight") and hasattr(tensor, "scale")) + ) + + +def _is_sdnq_module(module: nn.Module) -> bool: + return type(module).__module__.startswith("sdnq.") or type(module).__name__.startswith("SDNQ") + + def _tensor_on_device(tensor, device: torch.device) -> bool: if not _same_device(tensor.device, device): return False if not _is_quanto_tensor(tensor): + if not _is_sdnq_tensor(tensor): + return True + for attr in ("weight", "scale", "zero_point", "svd_up", "svd_down"): + value = getattr(tensor, attr, None) + if value is not None and hasattr(value, "device") and not _same_device(value.device, device): + return False return True for attr in ("_data", "_scale", "_shift", "_scale_shift"): value = getattr(tensor, attr, None) @@ -55,6 +73,32 @@ def _module_has_quanto_tensor(module: nn.Module) -> bool: ) +def _module_has_sdnq_payload(module: nn.Module) -> bool: + return ( + any(_is_sdnq_module(child) for child in module.modules()) + or any(_is_sdnq_tensor(tensor) for tensor in module.parameters()) + or any(_is_sdnq_tensor(tensor) for tensor in module.buffers()) + ) + + +def _module_has_trainable_local_state(module: nn.Module) -> bool: + return any(param is not None and param.requires_grad for param in module._parameters.values()) + + +def _module_has_local_quantized_payload(module: nn.Module) -> bool: + return ( + any( + param is not None and (_is_quanto_tensor(param) or _is_sdnq_tensor(param)) + for param in module._parameters.values() + ) + or any( + buffer is not None and (_is_quanto_tensor(buffer) or _is_sdnq_tensor(buffer)) + for buffer in module._buffers.values() + ) + or _is_sdnq_module(module) + ) + + def _move_quanto_tensor_to_device(tensor, device: torch.device): if not _same_device(tensor.device, device): tensor.data = tensor.data.to(device, non_blocking=True) @@ -69,12 +113,28 @@ def _move_quanto_tensor_to_device(tensor, device: torch.device): setattr(tensor, attr, value.to(device, non_blocking=True)) -def _move_module_without_swapping_quanto_params(module: nn.Module, device: torch.device): +def _move_sdnq_tensor_to_device(tensor, device: torch.device): + moved = tensor.to(device, non_blocking=True) + for attr in ("weight", "scale", "zero_point", "svd_up", "svd_down"): + value = getattr(moved, attr, None) + if value is None: + value = getattr(tensor, attr, None) + if value is not None and hasattr(value, "device") and not _same_device(value.device, device): + value = value.to(device, non_blocking=True) + if value is not None and hasattr(value, "device"): + setattr(tensor, attr, value) + if not _same_device(tensor.device, device): + tensor.data = moved.data + + +def _move_module_without_swapping_quantized_params(module: nn.Module, device: torch.device): for child in module.children(): - _move_module_without_swapping_quanto_params(child, device) + _move_module_without_swapping_quantized_params(child, device) - keep_local_trainable_state = device.type == "cpu" and any( - param is not None and param.requires_grad for param in module._parameters.values() + keep_local_trainable_state = ( + device.type == "cpu" + and not _module_has_local_quantized_payload(module) + and any(param is not None and param.requires_grad for param in module._parameters.values()) ) for key, param in module._parameters.items(): @@ -84,6 +144,8 @@ def _move_module_without_swapping_quanto_params(module: nn.Module, device: torch continue if _is_quanto_tensor(param): _move_quanto_tensor_to_device(param, device) + elif _is_sdnq_tensor(param): + _move_sdnq_tensor_to_device(param, device) elif not _same_device(param.device, device): param.data = param.data.to(device, non_blocking=True) if param.grad is not None and not _same_device(param.grad.device, device): @@ -96,6 +158,8 @@ def _move_module_without_swapping_quanto_params(module: nn.Module, device: torch continue if _is_quanto_tensor(buffer): _move_quanto_tensor_to_device(buffer, device) + elif _is_sdnq_tensor(buffer): + _move_sdnq_tensor_to_device(buffer, device) elif not _same_device(buffer.device, device): module._buffers[key] = buffer.to(device, non_blocking=True) @@ -221,8 +285,10 @@ def _move_module(self, module: nn.Module, device: torch.device): if _module_on_device(module, device): return with torch.no_grad(): - if _module_has_quanto_tensor(module): - _move_module_without_swapping_quanto_params(module, device) + if _module_has_quanto_tensor(module) or _module_has_sdnq_payload(module): + _move_module_without_swapping_quantized_params(module, device) + elif device.type == "cpu" and any(_module_has_trainable_local_state(child) for child in module.modules()): + _move_module_without_swapping_quantized_params(module, device) else: module.to(device) diff --git a/simpletuner/helpers/ramtorch/modules/linear.py b/simpletuner/helpers/ramtorch/modules/linear.py index 7d843c575..0ac8e517c 100644 --- a/simpletuner/helpers/ramtorch/modules/linear.py +++ b/simpletuner/helpers/ramtorch/modules/linear.py @@ -9,6 +9,7 @@ - Scenarios where GPU memory is limited but CPU memory is abundant """ +import os from collections import OrderedDict import torch @@ -22,6 +23,20 @@ _DEVICE_STATE = {} +def _env_int(name: str, default: int) -> int: + value = os.environ.get(name) + if value in (None, ""): + return default + try: + return int(value) + except ValueError: + return default + + +def _forward_prefetch_stream_count() -> int: + return max(_env_int("SIMPLETUNER_RAMTORCH_FORWARD_PREFETCH_STREAMS", 4), 1) + + def _to_cpu_pinned(tensor: torch.Tensor, *, dtype: torch.dtype | None = None) -> torch.Tensor: if dtype is not None and tensor.dtype != dtype: tensor = tensor.to(dtype=dtype) @@ -52,6 +67,10 @@ def _get_device_state(device=None): _DEVICE_STATE[device] = { # streams & events "transfer_stream": torch.cuda.Stream(device=device), + "forward_transfer_streams": [ + torch.cuda.Stream(device=device) for _ in range(_forward_prefetch_stream_count()) + ], + "forward_transfer_stream_clk": 0, "transfer_grad_stream": torch.cuda.Stream(device=device), "transfer_backward_finished_event": torch.cuda.Event(), "transfer_weight_backward_start_event": torch.cuda.Event(), @@ -78,6 +97,16 @@ def _get_device_state(device=None): return _DEVICE_STATE[device] +def _next_forward_transfer_stream(state, device): + streams = state.get("forward_transfer_streams") + if not streams: + streams = [state.get("transfer_stream") or torch.cuda.Stream(device=device)] + state["forward_transfer_streams"] = streams + selected = int(state.get("forward_transfer_stream_clk", 0)) % len(streams) + state["forward_transfer_stream_clk"] = selected + 1 + return streams[selected] + + def _prefetch_key(weight_cpu, bias_cpu): return (id(weight_cpu), id(bias_cpu) if bias_cpu is not None else None) @@ -94,6 +123,11 @@ def _resident_bytes(entry): return int(entry.get("bytes", 0)) +def _clear_linear_forward_residency(weight_cpu) -> None: + if hasattr(weight_cpu, "_ramtorch_last_forward_residency"): + delattr(weight_cpu, "_ramtorch_last_forward_residency") + + def _dense_weight_for_grad(weight, *, device=None, dtype=None): if hasattr(weight, "dequantize"): weight = weight.dequantize() @@ -104,6 +138,24 @@ def _dense_weight_for_grad(weight, *, device=None, dtype=None): return weight +def _dense_weight_for_linear(weight, *, dtype=None): + if hasattr(weight, "dequantize"): + weight = weight.dequantize() + if dtype is not None and getattr(weight, "dtype", None) != dtype: + weight = weight.to(dtype) + return weight + + +def _record_stream_if_supported(tensor: torch.Tensor | None, stream) -> None: + if tensor is None: + return + try: + tensor.record_stream(stream) + except (AssertionError, NotImplementedError) as exc: + if "record_stream" not in str(exc): + raise + + def _evict_backward_resident_entry(state, key, device): entry = state["forward_backward_residency"].pop(key, None) if entry is None: @@ -160,32 +212,21 @@ def prefetch_linear_forward(weight_cpu, bias_cpu, device="cuda"): if not allowed: return False - selected_buffer = state["forward_clk"] - state["forward_clk"] ^= 1 - state["forward_buffer_generations"][selected_buffer] += 1 - buffer_generation = state["forward_buffer_generations"][selected_buffer] - - transfer_stream = state["transfer_stream"] - release_event = state["forward_buffer_release_events"][selected_buffer] + transfer_stream = _next_forward_transfer_stream(state, device_obj) with torch.cuda.stream(transfer_stream): - if release_event is not None: - transfer_stream.wait_event(release_event) - with record_function("forward_weight_bias_prefetch"): - state["w_buffers"][selected_buffer] = weight_cpu.to(device_obj, non_blocking=True) - state["b_buffers"][selected_buffer] = ( - bias_cpu.to(device_obj, non_blocking=True) if bias_cpu is not None else None - ) + weight_gpu = weight_cpu.to(device_obj, non_blocking=True) + bias_gpu = bias_cpu.to(device_obj, non_blocking=True) if bias_cpu is not None else None event = torch.cuda.Event() event.record() state["forward_prefetches"][key] = { - "buffer": selected_buffer, "event": event, - "generation": buffer_generation, "versions": versions, + "weight": weight_gpu, + "bias": bias_gpu, } ramtorch_profile.record_prefetch_enqueued( "linear", @@ -207,7 +248,11 @@ def _consume_linear_forward_prefetch(weight_cpu, bias_cpu, device): if entry["versions"] != _prefetch_versions(weight_cpu, bias_cpu): ramtorch_profile.record_prefetch_stale("linear", key, device) return None - if state["forward_buffer_generations"][entry["buffer"]] != entry.get("generation"): + buffer_index = entry.get("buffer") + if buffer_index is not None and state["forward_buffer_generations"][buffer_index] != entry.get("generation"): + ramtorch_profile.record_prefetch_stale("linear", key, device) + return None + if buffer_index is None and entry.get("weight") is None: ramtorch_profile.record_prefetch_stale("linear", key, device) return None ramtorch_profile.record_prefetch_consumed("linear", key, device) @@ -270,11 +315,12 @@ def preserve_linear_forward_for_backward( state = _get_device_state(device_obj) buffer_index = last_forward.get("buffer") - if ( - buffer_index is None - or state["forward_buffer_generations"][buffer_index] != last_forward.get("generation") - or last_forward.get("weight") is None - ): + if last_forward.get("weight") is None: + _clear_linear_forward_residency(weight_cpu) + ramtorch_profile.record_backward_preserve_attempt(device_obj, (weight_cpu, bias_cpu), retained=False) + return False + if buffer_index is not None and state["forward_buffer_generations"][buffer_index] != last_forward.get("generation"): + _clear_linear_forward_residency(weight_cpu) ramtorch_profile.record_backward_preserve_attempt(device_obj, (weight_cpu, bias_cpu), retained=False) return False @@ -306,6 +352,7 @@ def preserve_linear_forward_for_backward( max_bytes=max(int(max_bytes), 0), ) retained = key in residency + _clear_linear_forward_residency(weight_cpu) ramtorch_profile.record_backward_preserve_attempt( device_obj, (weight_cpu, bias_cpu), @@ -469,35 +516,45 @@ def forward(ctx, x, weight_cpu, bias_cpu, device="cuda"): transfer = _consume_linear_forward_prefetch(weight_cpu, bias_cpu, device_obj) if transfer is None: transfer = _transfer_linear_forward(weight_cpu, bias_cpu, device_obj) - selected_buffer = transfer["buffer"] + selected_buffer = transfer.get("buffer") + weight_forward = transfer.get("weight") + bias_forward = transfer.get("bias") + if weight_forward is None: + weight_forward = w_buffers[selected_buffer] + bias_forward = b_buffers[selected_buffer] with torch.cuda.device(device_obj): compute_stream = torch.cuda.current_stream(device_obj) with record_function("forward_linear_compute"): # for profiling and easy debugging # make compute stream wait for this transfer compute_stream.wait_event(transfer["event"]) + _record_stream_if_supported(weight_forward, compute_stream) + if bias_forward is not None: + _record_stream_if_supported(bias_forward, compute_stream) # Manual casting when autocast is enabled if autocast_enabled: x_compute = x.to(autocast_dtype) - w_compute = w_buffers[selected_buffer].to(autocast_dtype) - b_compute = ( - b_buffers[selected_buffer].to(autocast_dtype) if b_buffers[selected_buffer] is not None else None - ) + w_compute = _dense_weight_for_linear(weight_forward, dtype=autocast_dtype) + b_compute = bias_forward.to(autocast_dtype) if bias_forward is not None else None out = F.linear(x_compute, w_compute, b_compute) else: - out = F.linear(x, w_buffers[selected_buffer], b_buffers[selected_buffer]) + w_compute = _dense_weight_for_linear(weight_forward, dtype=x.dtype) + out = F.linear(x, w_compute, bias_forward) release_event = torch.cuda.Event() release_event.record(compute_stream) - state["forward_buffer_release_events"][selected_buffer] = release_event + if selected_buffer is not None: + state["forward_buffer_release_events"][selected_buffer] = release_event weight_cpu._ramtorch_last_forward_residency = { "key": _prefetch_key(weight_cpu, bias_cpu), "versions": _prefetch_versions(weight_cpu, bias_cpu), "buffer": selected_buffer, - "generation": state["forward_buffer_generations"][selected_buffer], + "generation": ( + state["forward_buffer_generations"][selected_buffer] if selected_buffer is not None else None + ), "event": release_event, - "weight": w_buffers[selected_buffer], - "bias": b_buffers[selected_buffer], + "weight": weight_forward, + "bias": bias_forward, } # save for backward @@ -894,5 +951,9 @@ def preserve_forward_for_backward(self, *, max_entries: int = 2, max_bytes: int max_bytes=max_bytes, ) + def discard_forward_for_backward(self): + """Release the most recent forward GPU weight reference when it will not be reused.""" + _clear_linear_forward_residency(self.weight) + Linear = CPUBouncingLinear diff --git a/simpletuner/helpers/ramtorch/profiling.py b/simpletuner/helpers/ramtorch/profiling.py index a03ba5385..a00694ea5 100644 --- a/simpletuner/helpers/ramtorch/profiling.py +++ b/simpletuner/helpers/ramtorch/profiling.py @@ -80,6 +80,12 @@ def reset_for_new_run() -> None: _STATS = _initial_stats() _OUTSTANDING.clear() _STEP_DURATIONS.clear() + if torch.cuda.is_available(): + for index in range(torch.cuda.device_count()): + try: + torch.cuda.reset_peak_memory_stats(torch.device("cuda", index)) + except Exception: + continue def profile_enabled() -> bool: @@ -492,6 +498,18 @@ def snapshot() -> dict[str, Any]: } data["train_step_durations"] = _step_duration_summary() data["peak_memory"] = _peak_memory() + try: + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + get_activation_offload_pin_memory_stats, + get_activation_offload_prefetch_stats, + ) + + data["activation_offload"] = { + "pin_memory": get_activation_offload_pin_memory_stats(), + "prefetch": get_activation_offload_prefetch_stats(), + } + except Exception as exc: + data["activation_offload"] = {"error": str(exc)} data["finished_at_unix"] = time.time() return data diff --git a/simpletuner/helpers/ramtorch_extensions.py b/simpletuner/helpers/ramtorch_extensions.py index 6baad628f..b63f74464 100644 --- a/simpletuner/helpers/ramtorch_extensions.py +++ b/simpletuner/helpers/ramtorch_extensions.py @@ -23,6 +23,10 @@ _DEVICE_STATE = {} +def _forward_prefetch_stream_count() -> int: + return max(_env_int("SIMPLETUNER_RAMTORCH_FORWARD_PREFETCH_STREAMS", 4), 1) + + def _to_cpu_pinned(tensor: torch.Tensor, *, dtype: torch.dtype | None = None) -> torch.Tensor: if dtype is not None and tensor.dtype != dtype: tensor = tensor.to(dtype=dtype) @@ -47,6 +51,16 @@ def _tensor_versions(tensors): return tuple(_tensor_version(tensor) for tensor in tensors) +def _record_stream_if_supported(tensor: torch.Tensor | None, stream) -> None: + if tensor is None: + return + try: + tensor.record_stream(stream) + except (AssertionError, NotImplementedError) as exc: + if "record_stream" not in str(exc): + raise + + def _prefetch_forward_tensors(module: nn.Module, *tensors: torch.Tensor | None) -> bool: device = _device_obj(module.device) if device.type != "cuda": @@ -74,7 +88,7 @@ def _prefetch_forward_tensors(module: nn.Module, *tensors: torch.Tensor | None) return False state = _get_device_state(device) - transfer_stream = state["transfer_stream"] + transfer_stream = _next_transfer_stream(state, device) with torch.cuda.stream(transfer_stream): with record_function("forward_weight_bias_prefetch"): copied = tuple(tensor.to(device, non_blocking=True) if tensor is not None else None for tensor in tensors) @@ -110,7 +124,11 @@ def _consume_forward_prefetch(module: nn.Module, *tensors: torch.Tensor | None): return None with torch.cuda.device(device): - torch.cuda.current_stream(device).wait_event(prefetched["event"]) + current_stream = torch.cuda.current_stream(device) + current_stream.wait_event(prefetched["event"]) + for tensor in prefetched["tensors"]: + if tensor is not None: + _record_stream_if_supported(tensor, current_stream) ramtorch_profile.record_prefetch_consumed("extensions", key, device) return prefetched["tensors"] @@ -126,14 +144,18 @@ def _transfer_forward_tensors(module: nn.Module, *tensors: torch.Tensor | None): ramtorch_profile.record_fallback_forward_transfer(device, tensors) state = _get_device_state(device) - transfer_stream = state["transfer_stream"] + transfer_stream = _next_transfer_stream(state, device) with torch.cuda.stream(transfer_stream): with record_function("forward_weight_bias_transfer"): copied = tuple(tensor.to(device, non_blocking=True) if tensor is not None else None for tensor in tensors) with torch.cuda.device(device): - torch.cuda.current_stream(device).wait_stream(transfer_stream) + current_stream = torch.cuda.current_stream(device) + current_stream.wait_stream(transfer_stream) + for tensor in copied: + if tensor is not None: + _record_stream_if_supported(tensor, current_stream) return copied @@ -197,13 +219,24 @@ def _get_device_state(device): if device not in _DEVICE_STATE: with torch.cuda.device(device): _DEVICE_STATE[device] = { - "transfer_stream": torch.cuda.Stream(device=device), + "transfer_streams": [torch.cuda.Stream(device=device) for _ in range(_forward_prefetch_stream_count())], + "transfer_stream_clk": 0, "buffers": {}, "clock": 0, } return _DEVICE_STATE[device] +def _next_transfer_stream(state, device): + streams = state.get("transfer_streams") + if not streams: + streams = [state.get("transfer_stream") or torch.cuda.Stream(device=device)] + state["transfer_streams"] = streams + selected = int(state.get("transfer_stream_clk", 0)) % len(streams) + state["transfer_stream_clk"] = selected + 1 + return streams[selected] + + class CPUBouncingEmbedding(nn.Module): """ Embedding layer with CPU-stored weights that bounce to GPU on demand. @@ -1152,33 +1185,36 @@ def add_ramtorch_prefetch_hooks(module: nn.Module, component_label: str | None = hooks.append(module.register_forward_pre_hook(lambda _mod, _inp: runtime.begin_forward())) hooks.append(module.register_forward_hook(lambda _mod, _inp, _out: runtime.end_forward())) + def preserve_or_discard(current_id: str, current_module: nn.Module) -> None: + preserve_fn = getattr(current_module, "preserve_forward_for_backward", None) + if runtime.should_preserve_for_backward(current_id) and callable(preserve_fn): + preserve_fn( + max_entries=runtime.backward_preserve_max_entries, + max_bytes=runtime.backward_preserve_max_bytes, + ) + return + + discard_fn = getattr(current_module, "discard_forward_for_backward", None) + if callable(discard_fn): + discard_fn() + for module_id, current in ramtorch_modules: def pre_hook(_mod, _inp, current_id=module_id): runtime.module_entered(current_id) - return None - - def hook(_mod, _inp, _out, current_id=module_id, current_module=current): successor_id, target, source = runtime.choose_successor(current_id) if target is None or successor_id is None: ramtorch_profile.record_hook_prefetch_skipped_learned_order() - preserve_fn = getattr(current_module, "preserve_forward_for_backward", None) - if runtime.should_preserve_for_backward(current_id) and callable(preserve_fn): - preserve_fn( - max_entries=runtime.backward_preserve_max_entries, - max_bytes=runtime.backward_preserve_max_bytes, - ) return None + runtime.predicted(current_id, successor_id) ramtorch_profile.record_prefetch_successor_source(source or "traversal") success = target.prefetch_forward() ramtorch_profile.record_hook_prefetch(bool(success)) - preserve_fn = getattr(current_module, "preserve_forward_for_backward", None) - if runtime.should_preserve_for_backward(current_id) and callable(preserve_fn): - preserve_fn( - max_entries=runtime.backward_preserve_max_entries, - max_bytes=runtime.backward_preserve_max_bytes, - ) + return None + + def hook(_mod, _inp, _out, current_id=module_id, current_module=current): + preserve_or_discard(current_id, current_module) return None hooks.append(current.register_forward_pre_hook(pre_hook)) diff --git a/simpletuner/helpers/training/default_settings/safety_check.py b/simpletuner/helpers/training/default_settings/safety_check.py index b17494ef0..d0d374c55 100644 --- a/simpletuner/helpers/training/default_settings/safety_check.py +++ b/simpletuner/helpers/training/default_settings/safety_check.py @@ -150,6 +150,29 @@ def safety_check(args, accelerator): "cosmos3", "mageflow", ] + gradient_checkpointing_segment_stride_supported_models = [] + attention_activation_offload_supported_models = [] + if getattr(args, "gradient_checkpointing_offload_attention", False): + if args.model_family.lower() not in attention_activation_offload_supported_models: + raise ValueError( + "--gradient_checkpointing_offload_attention is only supported on model families with a clean " + f"attention/FFN checkpointing boundary. Currently supported models: {attention_activation_offload_supported_models}" + ) + elif getattr(args, "gradient_checkpointing_offload_prefetch", False): + logger.warning( + "Gradient checkpointing activation prefetch requires --gradient_checkpointing_offload_attention; disabling prefetch." + ) + args.gradient_checkpointing_offload_prefetch = False + raw_offload_pin_bucket_count = getattr(args, "gradient_checkpointing_offload_pin_memory_max_buckets", 12) + if raw_offload_pin_bucket_count in (None, "", "None"): + args.gradient_checkpointing_offload_pin_memory_max_buckets = 12 + else: + try: + args.gradient_checkpointing_offload_pin_memory_max_buckets = int(raw_offload_pin_bucket_count) + except (TypeError, ValueError): + raise ValueError("Gradient checkpointing offload pinned bucket count must be a non-negative integer.") + if args.gradient_checkpointing_offload_pin_memory_max_buckets < 0: + raise ValueError("Gradient checkpointing offload pinned bucket count must be non-negative.") if args.gradient_checkpointing_interval == 1: args.gradient_checkpointing_interval = None if args.gradient_checkpointing_interval is not None: @@ -160,6 +183,28 @@ def safety_check(args, accelerator): args.gradient_checkpointing_interval = None if args.gradient_checkpointing_interval == 0: raise ValueError("Gradient checkpointing interval must be greater than 0. Please set it to a positive integer.") + if getattr(args, "gradient_checkpointing_segment_stride", None) in ("", "None"): + args.gradient_checkpointing_segment_stride = None + if getattr(args, "gradient_checkpointing_segment_stride", None) is not None: + try: + args.gradient_checkpointing_segment_stride = int(args.gradient_checkpointing_segment_stride) + except (TypeError, ValueError): + raise ValueError("Gradient checkpointing segment stride must be a positive integer.") + if args.model_family.lower() not in gradient_checkpointing_segment_stride_supported_models: + logger.warning( + f"Gradient checkpointing segment stride is not supported with {args.model_family} models. " + f"Currently supported models: {gradient_checkpointing_segment_stride_supported_models}" + ) + args.gradient_checkpointing_segment_stride = None + elif args.gradient_checkpointing_segment_stride <= 0: + raise ValueError("Gradient checkpointing segment stride must be greater than 0.") + elif args.gradient_checkpointing_interval is None: + logger.warning( + "Gradient checkpointing segment stride requires --gradient_checkpointing_interval greater than 1; ignoring segment stride." + ) + args.gradient_checkpointing_segment_stride = None + elif args.gradient_checkpointing_segment_stride < args.gradient_checkpointing_interval: + raise ValueError("Gradient checkpointing segment stride must be at least the checkpointing interval.") def _normalize_interval(raw_value, cast): if raw_value in (None, "", "None"): diff --git a/simpletuner/helpers/training/gradient_checkpointing_interval.py b/simpletuner/helpers/training/gradient_checkpointing_interval.py index de90a7333..b468a6d4a 100644 --- a/simpletuner/helpers/training/gradient_checkpointing_interval.py +++ b/simpletuner/helpers/training/gradient_checkpointing_interval.py @@ -49,6 +49,29 @@ def get_checkpoint_function(): return torch.utils.checkpoint.checkpoint +def should_checkpoint_block( + block_index: int, + gradient_checkpointing: bool, + interval: int | None = None, + segment_stride: int | None = None, +) -> bool: + """Return whether an individual block should be checkpointed. + + ``interval=None`` means every block, matching the legacy layer checkpointing + behavior. With a stride, checkpoint the first ``interval`` blocks in each + stride window, e.g. interval=2 stride=4 checkpoints 0,1,4,5,... + """ + if not gradient_checkpointing: + return False + if interval is None or interval <= 1: + return True + if segment_stride is None: + return block_index % interval == 0 + if segment_stride < interval: + raise ValueError("segment_stride must be at least interval") + return block_index % segment_stride < interval + + def checkpoint_sequential_state( blocks: Sequence[Any], segment_size: int, @@ -56,6 +79,7 @@ def checkpoint_sequential_state( run_block: Callable[..., tuple[Any, ...] | Any], checkpoint_fn: Callable[..., Any], checkpoint_kwargs: dict[str, Any] | None = None, + segment_stride: int | None = None, ) -> tuple[Any, ...]: """Checkpoint contiguous chunks of a stateful block sequence. @@ -64,11 +88,15 @@ def checkpoint_sequential_state( """ if segment_size < 1: raise ValueError("segment_size must be greater than 0") + if segment_stride is None: + segment_stride = segment_size + if segment_stride < segment_size: + raise ValueError("segment_stride must be at least segment_size") current_state = state if isinstance(state, tuple) else (state,) checkpoint_kwargs = dict(checkpoint_kwargs or {}) - for segment_start in range(0, len(blocks), segment_size): + for segment_start in range(0, len(blocks), segment_stride): segment_blocks = tuple(blocks[segment_start : segment_start + segment_size]) def run_segment(*segment_state, _segment_start=segment_start, _segment_blocks=segment_blocks): @@ -83,4 +111,10 @@ def run_segment(*segment_state, _segment_start=segment_start, _segment_blocks=se result = checkpoint_fn(run_segment, *current_state, **checkpoint_kwargs) current_state = result if isinstance(result, tuple) else (result,) + gap_start = segment_start + len(segment_blocks) + gap_end = min(segment_start + segment_stride, len(blocks)) + for block_index in range(gap_start, gap_end): + result = run_block(block_index, blocks[block_index], *current_state) + current_state = result if isinstance(result, tuple) else (result,) + return current_state diff --git a/simpletuner/helpers/training/offloaded_gradient_checkpointer.py b/simpletuner/helpers/training/offloaded_gradient_checkpointer.py index 2b7b65cc9..7be079ab1 100644 --- a/simpletuner/helpers/training/offloaded_gradient_checkpointer.py +++ b/simpletuner/helpers/training/offloaded_gradient_checkpointer.py @@ -1,22 +1,724 @@ -""" -Unsloth-style gradient checkpointing using CPU offload. +"""CPU saved-tensor offload helpers for activation memory pressure.""" -Uses PyTorch's saved_tensors_hooks to intercept tensor saves during checkpoint, -offloading them to CPU asynchronously and restoring during backward pass. -This trades PCIe bandwidth for GPU memory savings. -""" +import sys +from collections import OrderedDict, defaultdict +from contextlib import nullcontext +from dataclasses import dataclass import torch from diffusers.utils.torch_utils import is_torch_version from torch.utils.checkpoint import checkpoint as torch_checkpoint +_DEFAULT_PIN_MEMORY_MAX_BUCKETS = 12 +_ACTIVATION_PREFETCH_MIN_OBSERVATIONS = 2 +_ACTIVATION_PREFETCH_MIN_CONFIDENCE = 0.80 +_ACTIVATION_PREFETCH_AUTOTUNE_MIN_SAMPLES = 8 +_ACTIVATION_PREFETCH_AUTOTUNE_MARGIN = 0.98 +_DEFAULT_D2H_COPY_STREAMS = 2 +_DEFAULT_H2D_PREFETCH_STREAMS = 4 + + +@dataclass(frozen=True) +class _PinnedBucketKey: + size: tuple[int, ...] + stride: tuple[int, ...] + dtype: torch.dtype + layout: torch.layout + + +@dataclass(frozen=True) +class _RestoreView: + size: tuple[int, ...] + stride: tuple[int, ...] + + +@dataclass +class _OffloadedActivationRecord: + logical_id: str + predictor_id: str + generation: int + tensor: torch.Tensor + original_device: torch.device + pool_key: "_PinnedBucketKey | None" + restore_view: _RestoreView | None + ready_event: torch.cuda.Event | None + prefetched_tensor: torch.Tensor | None = None + prefetch_event: torch.cuda.Event | None = None + consumed: bool = False + cpu_released: bool = False + + +@dataclass +class _PinnedBucketStats: + accesses: int = 0 + resident_accesses: int = 0 + pinned_checkouts: int = 0 + buffer_reuses: int = 0 + allocations: int = 0 + admissions: int = 0 + evictions: int = 0 + cap_misses: int = 0 + disabled_misses: int = 0 + allocation_failures: int = 0 + releases: int = 0 + dropped_releases: int = 0 + last_access: int = 0 + last_admission: int = 0 + last_eviction: int = 0 + + @property + def pinned_checkout_rate(self) -> float: + return self.pinned_checkouts / self.accesses if self.accesses else 0.0 + + @property + def buffer_reuse_rate(self) -> float: + return self.buffer_reuses / self.pinned_checkouts if self.pinned_checkouts else 0.0 + + +class _PinnedMemoryPool: + def __init__(self, max_buckets: int = _DEFAULT_PIN_MEMORY_MAX_BUCKETS): + self.max_buckets = max(0, int(max_buckets)) + self.available: OrderedDict[_PinnedBucketKey, list[torch.Tensor]] = OrderedDict() + self.pending: dict[_PinnedBucketKey, list[tuple[torch.cuda.Event, torch.Tensor]]] = defaultdict(list) + self.stats: dict[_PinnedBucketKey, _PinnedBucketStats] = {} + self.total_accesses = 0 + self.total_pinned_checkouts = 0 + self.total_buffer_reuses = 0 + self.total_allocations = 0 + self.total_cap_misses = 0 + self.total_evictions = 0 + + def set_max_buckets(self, max_buckets: int) -> None: + self.max_buckets = max(0, int(max_buckets)) + while len(self.available) > self.max_buckets: + victim, _ = next(iter(self.available.items())) + self._evict(victim) + + def key_for(self, tensor: torch.Tensor) -> _PinnedBucketKey | None: + if tensor.layout != torch.strided: + return None + return _PinnedBucketKey(tuple(tensor.size()), tuple(tensor.stride()), tensor.dtype, tensor.layout) + + def checkout(self, key: _PinnedBucketKey) -> torch.Tensor | None: + stats = self._record_access(key) + if self.max_buckets <= 0: + stats.disabled_misses += 1 + return None + self._drain_completed(key) + if key not in self.available: + if len(self.available) >= self.max_buckets: + if not self._admit_by_eviction(key): + stats.cap_misses += 1 + self.total_cap_misses += 1 + return None + self.available[key] = [] + stats.admissions += 1 + stats.last_admission = self.total_accesses + else: + stats.resident_accesses += 1 + self.available.move_to_end(key) + if self.available[key]: + stats.buffer_reuses += 1 + stats.pinned_checkouts += 1 + self.total_buffer_reuses += 1 + self.total_pinned_checkouts += 1 + return self.available[key].pop() + try: + tensor = self._allocate(key) + except RuntimeError: + stats.allocation_failures += 1 + self.discard_empty_bucket(key) + return None + stats.allocations += 1 + stats.pinned_checkouts += 1 + self.total_allocations += 1 + self.total_pinned_checkouts += 1 + return tensor + + def release_after_cuda_copy(self, key: _PinnedBucketKey, tensor: torch.Tensor, device: torch.device) -> None: + stats = self._stats_for(key) + if key not in self.available: + stats.dropped_releases += 1 + return + stats.releases += 1 + if device.type == "cuda" and torch.cuda.is_available(): + event = torch.cuda.Event() + event.record(torch.cuda.current_stream(device)) + self._release_after_event(key, tensor, event) + return + self.available[key].append(tensor) + + def release_after_record_event( + self, key: _PinnedBucketKey, tensor: torch.Tensor, event: torch.cuda.Event | None + ) -> None: + stats = self._stats_for(key) + if key not in self.available: + stats.dropped_releases += 1 + return + stats.releases += 1 + self._release_after_event(key, tensor, event) + + def _release_after_event(self, key: _PinnedBucketKey, tensor: torch.Tensor, event: torch.cuda.Event | None) -> None: + if event is not None and torch.cuda.is_available(): + self.pending[key].append((event, tensor)) + return + self.available[key].append(tensor) + + def discard_empty_bucket(self, key: _PinnedBucketKey) -> None: + if not self.available.get(key) and not self.pending.get(key): + self.available.pop(key, None) + + def _drain_completed(self, key: _PinnedBucketKey) -> None: + pending = self.pending.get(key) + if not pending: + return + remaining = [] + for event, tensor in pending: + if event.query(): + self.available.setdefault(key, []).append(tensor) + else: + remaining.append((event, tensor)) + if remaining: + self.pending[key] = remaining + else: + self.pending.pop(key, None) + + def _allocate(self, key: _PinnedBucketKey) -> torch.Tensor: + return torch.empty_strided(key.size, key.stride, dtype=key.dtype, layout=key.layout, device="cpu", pin_memory=True) + + def _record_access(self, key: _PinnedBucketKey) -> _PinnedBucketStats: + stats = self._stats_for(key) + self.total_accesses += 1 + stats.accesses += 1 + stats.last_access = self.total_accesses + return stats + + def _stats_for(self, key: _PinnedBucketKey) -> _PinnedBucketStats: + stats = self.stats.get(key) + if stats is None: + stats = _PinnedBucketStats() + self.stats[key] = stats + return stats + + def _admit_by_eviction(self, candidate: _PinnedBucketKey) -> bool: + evictable = [key for key in self.available if not self.pending.get(key)] + if not evictable: + return False + candidate_score = self._admission_score(candidate) + victim = min(evictable, key=self._admission_score) + if candidate_score <= self._admission_score(victim): + return False + self._evict(victim) + return True + + def _admission_score(self, key: _PinnedBucketKey) -> tuple[int, float, int]: + stats = self._stats_for(key) + return (stats.accesses, stats.pinned_checkout_rate, stats.last_access) + + def _evict(self, key: _PinnedBucketKey) -> None: + self.available.pop(key, None) + stats = self._stats_for(key) + stats.evictions += 1 + stats.last_eviction = self.total_accesses + self.total_evictions += 1 + + def snapshot(self) -> dict: + buckets = [] + resident = set(self.available) + pending_counts = {key: len(value) for key, value in self.pending.items()} + for key, stats in self.stats.items(): + buckets.append( + { + "size": key.size, + "stride": key.stride, + "dtype": str(key.dtype), + "layout": str(key.layout), + "resident": key in resident, + "available_buffers": len(self.available.get(key, ())), + "pending_buffers": pending_counts.get(key, 0), + "accesses": stats.accesses, + "resident_accesses": stats.resident_accesses, + "pinned_checkouts": stats.pinned_checkouts, + "buffer_reuses": stats.buffer_reuses, + "allocations": stats.allocations, + "admissions": stats.admissions, + "evictions": stats.evictions, + "cap_misses": stats.cap_misses, + "disabled_misses": stats.disabled_misses, + "allocation_failures": stats.allocation_failures, + "releases": stats.releases, + "dropped_releases": stats.dropped_releases, + "pinned_checkout_rate": stats.pinned_checkout_rate, + "buffer_reuse_rate": stats.buffer_reuse_rate, + "last_access": stats.last_access, + "last_admission": stats.last_admission, + "last_eviction": stats.last_eviction, + } + ) + buckets.sort(key=lambda item: (item["accesses"], item["pinned_checkouts"], item["last_access"]), reverse=True) + return { + "max_buckets": self.max_buckets, + "resident_buckets": len(self.available), + "tracked_buckets": len(self.stats), + "total_accesses": self.total_accesses, + "total_pinned_checkouts": self.total_pinned_checkouts, + "total_buffer_reuses": self.total_buffer_reuses, + "total_allocations": self.total_allocations, + "total_cap_misses": self.total_cap_misses, + "total_evictions": self.total_evictions, + "pinned_checkout_rate": self.total_pinned_checkouts / self.total_accesses if self.total_accesses else 0.0, + "buffer_reuse_rate": ( + self.total_buffer_reuses / self.total_pinned_checkouts if self.total_pinned_checkouts else 0.0 + ), + "buckets": buckets, + } + + def reset_stats(self) -> None: + self.stats.clear() + self.total_accesses = 0 + self.total_pinned_checkouts = 0 + self.total_buffer_reuses = 0 + self.total_allocations = 0 + self.total_cap_misses = 0 + self.total_evictions = 0 + + +_PINNED_MEMORY_POOL = _PinnedMemoryPool() +_ACTIVATION_PREFETCH_ENABLED = False +_ACTIVATION_PREFETCH_AUTOTUNE_ENABLED = True + + +class _CudaStreamPool: + def __init__(self, width: int): + self.width = max(1, int(width)) + self.streams: dict[int, list[torch.cuda.Stream]] = {} + self.next_index: dict[int, int] = defaultdict(int) + self.uses: dict[int, list[int]] = {} + + def set_width(self, width: int) -> None: + self.width = max(1, int(width)) + self.streams.clear() + self.next_index.clear() + self.uses.clear() + + def next(self, device: torch.device) -> torch.cuda.Stream: + device_index = _device_index(device) + streams = self.streams.get(device_index) + if streams is None or len(streams) != self.width: + streams = [torch.cuda.Stream(device=device_index) for _ in range(self.width)] + self.streams[device_index] = streams + self.next_index[device_index] = 0 + self.uses[device_index] = [0 for _ in range(self.width)] + stream_index = self.next_index[device_index] + self.next_index[device_index] = (stream_index + 1) % self.width + self.uses[device_index][stream_index] += 1 + return streams[stream_index] + + def snapshot(self) -> dict: + return { + "width": self.width, + "devices": { + device_index: { + "allocated_streams": len(self.streams.get(device_index, ())), + "uses": list(self.uses.get(device_index, ())), + "total_uses": sum(self.uses.get(device_index, ())), + } + for device_index in sorted(set(self.streams) | set(self.uses)) + }, + } + + def reset_stats(self) -> None: + for device_index, uses in list(self.uses.items()): + self.uses[device_index] = [0 for _ in uses] + + +_D2H_COPY_STREAMS = _CudaStreamPool(_DEFAULT_D2H_COPY_STREAMS) +_H2D_PREFETCH_STREAMS = _CudaStreamPool(_DEFAULT_H2D_PREFETCH_STREAMS) + + +class _ActivationOffloadPrefetchRuntime: + def __init__(self): + self.records: dict[tuple[int, str], list[_OffloadedActivationRecord]] = defaultdict(list) + self.transitions: dict[str, dict[str, int]] = defaultdict(dict) + self.successors: dict[str, str] = {} + self.disabled: dict[str, dict] = {} + self.previous_by_generation: dict[int, str] = {} + self.current_generation = 0 + self.saw_unpack_since_pack = False + self.total_packs = 0 + self.total_unpacks = 0 + self.prefetch_attempts = 0 + self.prefetch_hits = 0 + self.prefetch_misses = 0 + self.prefetch_enqueued = 0 + self.prefetch_skipped = 0 + self.prefetch_stale = 0 + self.transition_updates = 0 + self.jit_restore_ms: list[float] = [] + self.prefetch_wait_ms: list[float] = [] + self.autotune_disabled = False + self.autotune_decision: str | None = None + + def next_generation_for_pack(self) -> int: + if self.saw_unpack_since_pack: + self._retire_generation(self.current_generation) + self.current_generation += 1 + self.saw_unpack_since_pack = False + self.previous_by_generation.pop(self.current_generation, None) + return self.current_generation + + def register(self, record: _OffloadedActivationRecord) -> None: + self.total_packs += 1 + self.records[(record.generation, record.predictor_id)].append(record) + + def consume(self, record: _OffloadedActivationRecord) -> None: + self.total_unpacks += 1 + self.saw_unpack_since_pack = True + self._remove_record(record) + previous_id = self.previous_by_generation.get(record.generation) + if previous_id is not None: + self._record_transition(previous_id, record.predictor_id) + self.previous_by_generation[record.generation] = record.predictor_id + successor_id = self.successors.get(record.predictor_id) + if successor_id and self.prefetch_allowed(): + self.prefetch_attempts += 1 + if not self.prefetch(record.generation, successor_id): + self.prefetch_misses += 1 + + def prefetch(self, generation: int, logical_id: str) -> bool: + candidates = self.records.get((generation, logical_id), ()) + for candidate in reversed(candidates): + if candidate.consumed or candidate.prefetched_tensor is not None: + continue + if candidate.original_device.type != "cuda" or not torch.cuda.is_available(): + self.prefetch_skipped += 1 + return False + if candidate.pool_key is None or not candidate.tensor.is_pinned(): + self.prefetch_skipped += 1 + return False + try: + copy_stream = _h2d_prefetch_stream_for(candidate.original_device) + with torch.cuda.stream(copy_stream): + if candidate.ready_event is not None: + copy_stream.wait_event(candidate.ready_event) + restored = candidate.tensor.to(candidate.original_device, non_blocking=True) + event = torch.cuda.Event() + event.record(copy_stream) + candidate.prefetched_tensor = restored + candidate.prefetch_event = event + self.prefetch_enqueued += 1 + return True + except RuntimeError: + candidate.prefetched_tensor = None + candidate.prefetch_event = None + self.prefetch_stale += 1 + return False + self.prefetch_skipped += 1 + return False + + def _remove_record(self, record: _OffloadedActivationRecord) -> None: + key = (record.generation, record.predictor_id) + entries = self.records.get(key) + if not entries: + return + try: + entries.remove(record) + except ValueError: + return + if not entries: + self.records.pop(key, None) + + def _retire_generation(self, generation: int) -> None: + stale_keys = [key for key in self.records if key[0] == generation] + for key in stale_keys: + entries = self.records.pop(key, ()) + for record in entries: + self._release_unused_record(record) + + def _release_unused_record(self, record: _OffloadedActivationRecord) -> None: + if record.cpu_released or record.pool_key is None: + return + release_event = record.prefetch_event or record.ready_event + _PINNED_MEMORY_POOL.release_after_record_event(record.pool_key, record.tensor, release_event) + record.cpu_released = True + record.prefetched_tensor = None + record.prefetch_event = None + + def _record_transition(self, previous_id: str, actual_id: str) -> None: + if self.disabled.get(previous_id): + return + counts = self.transitions[previous_id] + counts[actual_id] = int(counts.get(actual_id, 0)) + 1 + total = sum(int(value) for value in counts.values()) + best_id, best_count = max(counts.items(), key=lambda item: int(item[1])) + if total < _ACTIVATION_PREFETCH_MIN_OBSERVATIONS: + return + confidence = best_count / total if total else 0.0 + old_successor = self.successors.get(previous_id) + if confidence >= _ACTIVATION_PREFETCH_MIN_CONFIDENCE: + self.successors[previous_id] = best_id + if old_successor != best_id: + self.transition_updates += 1 + else: + self.successors.pop(previous_id, None) + + def record_hit(self) -> None: + self.prefetch_hits += 1 + + def prefetch_allowed(self) -> bool: + return _ACTIVATION_PREFETCH_ENABLED and not self.autotune_disabled + + @staticmethod + def _median(values: list[float]) -> float: + ordered = sorted(values) + midpoint = len(ordered) // 2 + if len(ordered) % 2: + return ordered[midpoint] + return (ordered[midpoint - 1] + ordered[midpoint]) / 2.0 + + def record_jit_restore_ms(self, value: float) -> None: + if not _ACTIVATION_PREFETCH_AUTOTUNE_ENABLED or self.autotune_decision is not None: + return + self.jit_restore_ms.append(float(value)) + + def record_prefetch_wait_ms(self, value: float) -> None: + if not _ACTIVATION_PREFETCH_AUTOTUNE_ENABLED or self.autotune_decision is not None: + return + self.prefetch_wait_ms.append(float(value)) + self._maybe_autotune() + + def _maybe_autotune(self) -> None: + if len(self.jit_restore_ms) < _ACTIVATION_PREFETCH_AUTOTUNE_MIN_SAMPLES: + return + if len(self.prefetch_wait_ms) < _ACTIVATION_PREFETCH_AUTOTUNE_MIN_SAMPLES: + return + jit_median = self._median(self.jit_restore_ms) + prefetch_median = self._median(self.prefetch_wait_ms) + if prefetch_median < jit_median * _ACTIVATION_PREFETCH_AUTOTUNE_MARGIN: + self.autotune_decision = "prefetch" + return + self.autotune_disabled = True + self.autotune_decision = "jit" + + def snapshot(self) -> dict: + active_records = sum(len(entries) for entries in self.records.values()) + return { + "enabled": _ACTIVATION_PREFETCH_ENABLED, + "active_records": active_records, + "generations_tracked": len({generation for generation, _logical_id in self.records}), + "total_packs": self.total_packs, + "total_unpacks": self.total_unpacks, + "prefetch_attempts": self.prefetch_attempts, + "prefetch_hits": self.prefetch_hits, + "prefetch_misses": self.prefetch_misses, + "prefetch_enqueued": self.prefetch_enqueued, + "prefetch_skipped": self.prefetch_skipped, + "prefetch_stale": self.prefetch_stale, + "transition_updates": self.transition_updates, + "learned_successors": len(self.successors), + "hit_rate": self.prefetch_hits / self.prefetch_attempts if self.prefetch_attempts else 0.0, + "autotune_enabled": _ACTIVATION_PREFETCH_AUTOTUNE_ENABLED, + "autotune_disabled": self.autotune_disabled, + "autotune_decision": self.autotune_decision, + "jit_restore_samples": len(self.jit_restore_ms), + "prefetch_wait_samples": len(self.prefetch_wait_ms), + "jit_restore_median_ms": self._median(self.jit_restore_ms) if self.jit_restore_ms else None, + "prefetch_wait_median_ms": self._median(self.prefetch_wait_ms) if self.prefetch_wait_ms else None, + "copy_streams": get_activation_offload_copy_stream_stats(), + } + + def reset(self) -> None: + for entries in list(self.records.values()): + for record in entries: + self._release_unused_record(record) + self.records.clear() + self.transitions.clear() + self.successors.clear() + self.disabled.clear() + self.previous_by_generation.clear() + self.current_generation = 0 + self.saw_unpack_since_pack = False + self.total_packs = 0 + self.total_unpacks = 0 + self.prefetch_attempts = 0 + self.prefetch_hits = 0 + self.prefetch_misses = 0 + self.prefetch_enqueued = 0 + self.prefetch_skipped = 0 + self.prefetch_stale = 0 + self.transition_updates = 0 + self.jit_restore_ms.clear() + self.prefetch_wait_ms.clear() + self.autotune_disabled = False + self.autotune_decision = None + + +_ACTIVATION_PREFETCH_RUNTIME = _ActivationOffloadPrefetchRuntime() + + +def _device_index(device: torch.device) -> int: + return device.index if device.index is not None else torch.cuda.current_device() + + +def _d2h_copy_stream_for(device: torch.device) -> torch.cuda.Stream: + return _D2H_COPY_STREAMS.next(device) + + +def _h2d_prefetch_stream_for(device: torch.device) -> torch.cuda.Stream: + return _H2D_PREFETCH_STREAMS.next(device) + + +def set_activation_offload_d2h_copy_stream_count(count: int) -> None: + """Set the number of round-robin CUDA streams used for GPU-to-CPU offload copies.""" + _D2H_COPY_STREAMS.set_width(count) + + +def get_activation_offload_d2h_copy_stream_count() -> int: + return _D2H_COPY_STREAMS.width + + +def set_activation_offload_h2d_prefetch_stream_count(count: int) -> None: + """Set the number of round-robin CUDA streams used for CPU-to-GPU prefetch copies.""" + _H2D_PREFETCH_STREAMS.set_width(count) + + +def get_activation_offload_h2d_prefetch_stream_count() -> int: + return _H2D_PREFETCH_STREAMS.width + + +def get_activation_offload_copy_stream_stats() -> dict: + return { + "d2h": _D2H_COPY_STREAMS.snapshot(), + "h2d_prefetch": _H2D_PREFETCH_STREAMS.snapshot(), + } + + +def reset_activation_offload_copy_stream_stats() -> None: + _D2H_COPY_STREAMS.reset_stats() + _H2D_PREFETCH_STREAMS.reset_stats() + + +def set_activation_offload_pin_memory_max_buckets(max_buckets: int) -> None: + """Set the max number of distinct pinned CPU tensor buckets used for activation offload.""" + _PINNED_MEMORY_POOL.set_max_buckets(max_buckets) + + +def get_activation_offload_pin_memory_max_buckets() -> int: + return _PINNED_MEMORY_POOL.max_buckets + + +def get_activation_offload_pin_memory_stats() -> dict: + """Return lifetime pinned CPU bucket statistics for activation offload.""" + return _PINNED_MEMORY_POOL.snapshot() + + +def reset_activation_offload_pin_memory_stats() -> None: + """Reset pinned CPU bucket statistics without clearing resident pinned buffers.""" + _PINNED_MEMORY_POOL.reset_stats() + + +def set_activation_offload_prefetch_enabled(enabled: bool) -> None: + """Enable learned H2D prefetching for labeled activation offload contexts.""" + global _ACTIVATION_PREFETCH_ENABLED + _ACTIVATION_PREFETCH_ENABLED = bool(enabled) + + +def get_activation_offload_prefetch_enabled() -> bool: + return _ACTIVATION_PREFETCH_ENABLED + + +def set_activation_offload_prefetch_autotune_enabled(enabled: bool) -> None: + """Enable first-run latency gating for activation prefetch.""" + global _ACTIVATION_PREFETCH_AUTOTUNE_ENABLED + _ACTIVATION_PREFETCH_AUTOTUNE_ENABLED = bool(enabled) + + +def get_activation_offload_prefetch_autotune_enabled() -> bool: + return _ACTIVATION_PREFETCH_AUTOTUNE_ENABLED + + +def get_activation_offload_prefetch_stats() -> dict: + return _ACTIVATION_PREFETCH_RUNTIME.snapshot() + + +def reset_activation_offload_prefetch_stats() -> None: + _ACTIVATION_PREFETCH_RUNTIME.reset() + reset_activation_offload_copy_stream_stats() + + +def set_activation_offload_prefetch_runtime_disabled(disabled: bool, *, decision: str | None = None) -> None: + """Force the learned prefetch runtime on or off without changing payload labeling.""" + _ACTIVATION_PREFETCH_RUNTIME.autotune_disabled = bool(disabled) + _ACTIVATION_PREFETCH_RUNTIME.autotune_decision = decision + + +def mark_activation_offload_prefetch_autotune_decision(decision: str) -> None: + """Record the end-to-end autotune decision made by the trainer.""" + normalised = str(decision).lower() + if normalised not in {"prefetch", "jit"}: + raise ValueError("activation offload prefetch decision must be 'prefetch' or 'jit'") + set_activation_offload_prefetch_runtime_disabled(normalised == "jit", decision=normalised) + + +def activation_offload_prefetch_autotune_decision() -> str | None: + return _ACTIVATION_PREFETCH_RUNTIME.autotune_decision + class CPUOffloadHooks: """Context manager hooks that offload saved tensors to CPU during checkpointing.""" - def __init__(self): - # No global device state; device is tracked per saved tensor via pack/unpack payloads. - pass + def __init__( + self, + *, + offload_leaf_tensors: bool = False, + pin_memory: bool = True, + label: str | None = None, + prefetch: bool | None = None, + ): + self.offload_leaf_tensors = offload_leaf_tensors + self.pin_memory = pin_memory + self.label = str(label) if label else None + self.prefetch = _ACTIVATION_PREFETCH_ENABLED if prefetch is None else bool(prefetch) + self.pack_index = 0 + + @staticmethod + def _flat_storage_view(tensor: torch.Tensor) -> torch.Tensor | None: + if tensor.layout != torch.strided: + return None + span = 1 + sum((size - 1) * stride for size, stride in zip(tensor.shape, tensor.stride(), strict=True)) + if span != tensor.numel(): + return None + return torch.as_strided(tensor, (tensor.numel(),), (1,), tensor.storage_offset()) + + def _transfer_view(self, tensor: torch.Tensor) -> tuple[torch.Tensor, _RestoreView | None]: + flat_view = self._flat_storage_view(tensor) + if flat_view is None: + return tensor, None + return flat_view, _RestoreView(tuple(tensor.size()), tuple(tensor.stride())) + + def _copy_to_cpu( + self, tensor: torch.Tensor + ) -> tuple[torch.Tensor, _PinnedBucketKey | None, _RestoreView | None, torch.cuda.Event | None]: + detached = tensor.detach() + transfer_tensor, restore_view = self._transfer_view(detached) + if self.pin_memory: + key = _PINNED_MEMORY_POOL.key_for(transfer_tensor) + if key is not None: + cpu_tensor = _PINNED_MEMORY_POOL.checkout(key) + if cpu_tensor is not None: + try: + copy_stream = _d2h_copy_stream_for(tensor.device) + current_stream = torch.cuda.current_stream(tensor.device) + with torch.cuda.stream(copy_stream): + copy_stream.wait_stream(current_stream) + transfer_tensor.record_stream(copy_stream) + cpu_tensor.copy_(transfer_tensor, non_blocking=True) + ready_event = torch.cuda.Event() + ready_event.record(copy_stream) + return cpu_tensor, key, restore_view, ready_event + except RuntimeError: + _PINNED_MEMORY_POOL.discard_empty_bucket(key) + return transfer_tensor.to("cpu", non_blocking=True), None, restore_view, None def pack(self, tensor: torch.Tensor): """Called when a tensor is saved for backward - offload to CPU. @@ -25,33 +727,147 @@ def pack(self, tensor: torch.Tensor): restore only tensors that were actually offloaded, and to their correct original devices. """ - if tensor.device.type == "cuda": - cpu_tensor = tensor.to("cpu", non_blocking=True) - return cpu_tensor, tensor.device - # Tensor is already on CPU (or a non-CUDA device); mark as not offloaded. + if tensor.device.type == "cuda" and (self.offload_leaf_tensors or not tensor.is_leaf): + cpu_tensor, pool_key, restore_view, ready_event = self._copy_to_cpu(tensor) + if self.prefetch and self.label is not None: + generation = _ACTIVATION_PREFETCH_RUNTIME.next_generation_for_pack() + predictor_id = f"{self.label}:{self.pack_index}" + logical_id = f"{generation}:{predictor_id}:{id(cpu_tensor)}" + self.pack_index += 1 + record = _OffloadedActivationRecord( + logical_id=logical_id, + predictor_id=predictor_id, + generation=generation, + tensor=cpu_tensor, + original_device=tensor.device, + pool_key=pool_key, + restore_view=restore_view, + ready_event=ready_event, + ) + _ACTIVATION_PREFETCH_RUNTIME.register(record) + return record + return cpu_tensor, tensor.device, pool_key, restore_view, ready_event return tensor, None def unpack(self, payload) -> torch.Tensor: """Called when a tensor is needed for backward - restore to original device if needed.""" - # Expect payload of the form (cpu_tensor, original_device). + if isinstance(payload, _OffloadedActivationRecord): + return self._unpack_record(payload) + + # Expect payload of the form (cpu_tensor, original_device[, pool_key, restore_view]). try: - tensor, original_device = payload + tensor, original_device, *rest = payload except (TypeError, ValueError): # Fallback: if payload is not in the expected form, return it as-is. return payload if original_device is not None and tensor.device.type == "cpu": - return tensor.to(original_device, non_blocking=True) + current_stream = None + ready_event = rest[2] if len(rest) > 2 else None + if ready_event is not None and original_device.type == "cuda": + current_stream = torch.cuda.current_stream(original_device) + current_stream.wait_event(ready_event) + restored = tensor.to(original_device, non_blocking=True) + if original_device.type == "cuda": + current_stream = current_stream or torch.cuda.current_stream(original_device) + restored.record_stream(current_stream) + pool_key = rest[0] if rest else None + restore_view = rest[1] if len(rest) > 1 else None + if pool_key is not None: + _PINNED_MEMORY_POOL.release_after_cuda_copy(pool_key, tensor, original_device) + if restore_view is not None: + restored = torch.as_strided(restored, restore_view.size, restore_view.stride, 0) + return restored return tensor + def _release_cpu_tensor(self, record: _OffloadedActivationRecord) -> None: + if record.cpu_released or record.pool_key is None: + return + _PINNED_MEMORY_POOL.release_after_cuda_copy(record.pool_key, record.tensor, record.original_device) + record.cpu_released = True + + def _restore_view_if_needed(self, tensor: torch.Tensor, restore_view: _RestoreView | None) -> torch.Tensor: + if restore_view is None: + return tensor + return torch.as_strided(tensor, restore_view.size, restore_view.stride, 0) + + def _unpack_record(self, record: _OffloadedActivationRecord) -> torch.Tensor: + record.consumed = True + if record.prefetched_tensor is not None: + elapsed_ms = None + if record.prefetch_event is not None and record.original_device.type == "cuda": + current_stream = torch.cuda.current_stream(record.original_device) + if _ACTIVATION_PREFETCH_AUTOTUNE_ENABLED and _ACTIVATION_PREFETCH_RUNTIME.autotune_decision is None: + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record(current_stream) + current_stream.wait_event(record.prefetch_event) + end_event.record(current_stream) + end_event.synchronize() + elapsed_ms = start_event.elapsed_time(end_event) + else: + current_stream.wait_event(record.prefetch_event) + record.prefetched_tensor.record_stream(current_stream) + restored = record.prefetched_tensor + _ACTIVATION_PREFETCH_RUNTIME.record_hit() + if elapsed_ms is not None: + _ACTIVATION_PREFETCH_RUNTIME.record_prefetch_wait_ms(elapsed_ms) + self._release_cpu_tensor(record) + _ACTIVATION_PREFETCH_RUNTIME.consume(record) + return self._restore_view_if_needed(restored, record.restore_view) + + if record.ready_event is not None and record.original_device.type == "cuda": + torch.cuda.current_stream(record.original_device).wait_event(record.ready_event) + elapsed_ms = None + if ( + _ACTIVATION_PREFETCH_AUTOTUNE_ENABLED + and _ACTIVATION_PREFETCH_RUNTIME.autotune_decision is None + and record.original_device.type == "cuda" + ): + current_stream = torch.cuda.current_stream(record.original_device) + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record(current_stream) + restored = record.tensor.to(record.original_device, non_blocking=True) + end_event.record(current_stream) + end_event.synchronize() + elapsed_ms = start_event.elapsed_time(end_event) + else: + restored = record.tensor.to(record.original_device, non_blocking=True) + if record.original_device.type == "cuda": + restored.record_stream(torch.cuda.current_stream(record.original_device)) + if elapsed_ms is not None: + _ACTIVATION_PREFETCH_RUNTIME.record_jit_restore_ms(elapsed_ms) + self._release_cpu_tensor(record) + _ACTIVATION_PREFETCH_RUNTIME.consume(record) + return self._restore_view_if_needed(restored, record.restore_view) + + +def activation_offload_context(enabled: bool = True, *, label: str | None = None, prefetch: bool | None = None): + """Offload non-leaf CUDA tensors saved for backward inside the context.""" + if not enabled: + return nullcontext() + if label is not None: + frame = sys._getframe(1) + label = f"{label}:{frame.f_code.co_name}:{frame.f_lineno}" + hooks = CPUOffloadHooks(offload_leaf_tensors=False, label=label, prefetch=prefetch) + return torch.autograd.graph.saved_tensors_hooks(hooks.pack, hooks.unpack) + + +def activation_offload(function, *args, **kwargs): + """Run a function normally while offloading saved non-leaf CUDA tensors.""" + kwargs.pop("use_reentrant", None) + with activation_offload_context(): + return function(*args, **kwargs) + def offloaded_checkpoint(function, *args, use_reentrant: bool = False, **kwargs): """ Drop-in replacement for torch.utils.checkpoint.checkpoint using CPU offload. - Instead of recomputing activations during backward pass (standard checkpointing), - this offloads saved tensors to CPU asynchronously and restores them when needed. - This approach trades PCIe bandwidth for GPU memory savings. + This still uses PyTorch checkpoint rematerialization. Saved tensor hooks + offload tensors that autograd saves inside the checkpointed region and + restore them when needed. Args: function: The forward function to checkpoint diff --git a/simpletuner/helpers/training/trainer.py b/simpletuner/helpers/training/trainer.py index 549276b4d..9fa816c49 100644 --- a/simpletuner/helpers/training/trainer.py +++ b/simpletuner/helpers/training/trainer.py @@ -3386,6 +3386,9 @@ def enable_gradient_checkpointing(self): logger.debug("Enabling gradient checkpointing.") if hasattr(self.model.get_trained_component(), "enable_gradient_checkpointing"): unwrap_model(self.accelerator, self.model.get_trained_component()).enable_gradient_checkpointing() + model_level_enable = getattr(self.model, "enable_gradient_checkpointing", None) + if callable(model_level_enable): + model_level_enable() if hasattr(self.config, "train_text_encoder") and self.config.train_text_encoder: for text_encoder in self.model.text_encoders: if text_encoder is not None: @@ -3398,6 +3401,9 @@ def disable_gradient_checkpointing(self): unwrap_model( self.accelerator, self.model.get_trained_component(base_model=True) ).disable_gradient_checkpointing() + model_level_disable = getattr(self.model, "disable_gradient_checkpointing", None) + if callable(model_level_disable): + model_level_disable() if self.config.controlnet: unwrap_model(self.accelerator, self.model.get_trained_component()).disable_gradient_checkpointing() if hasattr(self.config, "train_text_encoder") and self.config.train_text_encoder: @@ -4172,6 +4178,9 @@ def init_prepare_models(self, lr_scheduler): attach_shared_ramtorch_parameters as attach_shared_ramtorch_parameters, ) + moved = ramtorch_utils.move_embeddings_to_device(primary_model, self.accelerator.device) + if moved: + logger.info("Moved %s non-RamTorch CPU parameters/buffers before DDP prepare.", moved) ignored = ramtorch_utils.mark_ddp_ignore_params(primary_model) if ignored: logger.info("Marking %s RamTorch parameters to ignore for DDP.", ignored) @@ -5641,6 +5650,119 @@ def model_predict( return model_pred + def _compute_model_prediction_loss(self, prepared_batch: dict) -> tuple[torch.Tensor, dict, torch.Tensor, dict, dict]: + model_pred = self.model_predict( + prepared_batch=prepared_batch, + ) + loss, loss_logs = self.model.loss_with_logs( + prepared_batch=prepared_batch, + model_output=model_pred, + apply_conditioning_mask=True, + ) + diffusion_loss = loss.clone() + loss, aux_loss_logs = self.model.auxiliary_loss( + prepared_batch=prepared_batch, + model_output=model_pred, + loss=loss, + ) + distill_logs = {} + if self.config.distillation_method is not None: + loss, distill_logs = self.distiller.compute_distill_loss(prepared_batch, model_pred, loss) + loss, gen_logs = self.distiller.generator_loss_step(prepared_batch, model_pred, loss) + distill_logs.update(gen_logs) + return loss, loss_logs, diffusion_loss, aux_loss_logs, distill_logs + + def _discard_probe_gradients(self) -> None: + trained_component = self.model.get_trained_component(unwrap_model=False) + if trained_component is not None: + trained_component.zero_grad(set_to_none=True) + if getattr(self, "optimizer", None) is not None: + self.optimizer.zero_grad(set_to_none=True) + if getattr(self, "sidecar_optimizer", None) is not None: + self.sidecar_optimizer.zero_grad(set_to_none=True) + + def _activation_offload_prefetch_autotune_probe(self, prepared_batch: dict, *, label: str) -> float: + if torch.cuda.is_available(): + torch.cuda.synchronize() + start_time = time.perf_counter() + loss, _loss_logs, _diffusion_loss, _aux_loss_logs, _distill_logs = self._compute_model_prediction_loss( + dict(prepared_batch) + ) + self.accelerator.backward(loss) + if torch.cuda.is_available(): + torch.cuda.synchronize() + elapsed = time.perf_counter() - start_time + self._discard_probe_gradients() + logger.debug("Activation offload prefetch autotune %s probe: %.4fs", label, elapsed) + return elapsed + + def _maybe_autotune_activation_offload_prefetch(self, prepared_batch: dict) -> None: + if not getattr(self.config, "gradient_checkpointing_offload_prefetch", False): + return + if not getattr(self.config, "gradient_checkpointing_offload_attention", False): + logger.info("Skipping activation offload prefetch autotune because attention activation offload is disabled.") + return + if getattr(self, "_activation_offload_prefetch_autotuned", False): + return + if self.config.disable_accelerator or not torch.cuda.is_available(): + logger.info("Skipping activation offload prefetch autotune because CUDA training is unavailable.") + return + if self.config.distillation_method is not None: + logger.warning("Skipping activation offload prefetch autotune for distillation runs.") + self._activation_offload_prefetch_autotuned = True + return + + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + get_activation_offload_prefetch_autotune_enabled, + mark_activation_offload_prefetch_autotune_decision, + reset_activation_offload_prefetch_stats, + set_activation_offload_prefetch_autotune_enabled, + set_activation_offload_prefetch_enabled, + set_activation_offload_prefetch_runtime_disabled, + ) + + self._activation_offload_prefetch_autotuned = True + previous_autotune_enabled = get_activation_offload_prefetch_autotune_enabled() + cpu_rng_state = torch.get_rng_state() + cuda_rng_states = torch.cuda.get_rng_state_all() + + def restore_rng_state() -> None: + torch.set_rng_state(cpu_rng_state) + torch.cuda.set_rng_state_all(cuda_rng_states) + + logger.info("Autotuning attention activation prefetch with a real model_predict/backward probe.") + try: + set_activation_offload_prefetch_enabled(True) + set_activation_offload_prefetch_autotune_enabled(False) + reset_activation_offload_prefetch_stats() + + set_activation_offload_prefetch_runtime_disabled(True, decision=None) + restore_rng_state() + self._activation_offload_prefetch_autotune_probe(prepared_batch, label="learn") + + restore_rng_state() + jit_seconds = self._activation_offload_prefetch_autotune_probe(prepared_batch, label="jit") + + set_activation_offload_prefetch_runtime_disabled(False, decision=None) + restore_rng_state() + prefetch_seconds = self._activation_offload_prefetch_autotune_probe(prepared_batch, label="prefetch") + + decision = "prefetch" if prefetch_seconds < jit_seconds * 0.98 else "jit" + mark_activation_offload_prefetch_autotune_decision(decision) + logger.info( + "Activation offload prefetch autotune selected %s (jit=%.4fs, prefetch=%.4fs).", + decision, + jit_seconds, + prefetch_seconds, + ) + except Exception as exc: + logger.warning("Activation offload prefetch autotune failed; disabling prefetch. Error: %s", exc) + mark_activation_offload_prefetch_autotune_decision("jit") + self._discard_probe_gradients() + finally: + restore_rng_state() + set_activation_offload_prefetch_autotune_enabled(previous_autotune_enabled) + def _prepare_custom_timestep_batch(self, prepared_batch: dict, custom_timesteps): overridden_batch = dict(prepared_batch) reference_timesteps = prepared_batch.get("timesteps") @@ -6423,26 +6545,12 @@ def train(self): module.scaling[key] = val * strength slider_original_scaling[layer_id] = (module, saved) + self._maybe_autotune_activation_offload_prefetch(prepared_batch) + training_logger.debug("Predicting.") - model_pred = self.model_predict( - prepared_batch=prepared_batch, - ) - loss, loss_logs = self.model.loss_with_logs( - prepared_batch=prepared_batch, - model_output=model_pred, - apply_conditioning_mask=True, - ) - diffusion_loss = loss.clone() - loss, aux_loss_logs = self.model.auxiliary_loss( - prepared_batch=prepared_batch, - model_output=model_pred, - loss=loss, + loss, loss_logs, diffusion_loss, aux_loss_logs, distill_logs = self._compute_model_prediction_loss( + prepared_batch ) - distill_logs = {} - if self.config.distillation_method is not None: - loss, distill_logs = self.distiller.compute_distill_loss(prepared_batch, model_pred, loss) - loss, gen_logs = self.distiller.generator_loss_step(prepared_batch, model_pred, loss) - distill_logs.update(gen_logs) parent_loss = None if is_regularisation_data: parent_loss = loss diff --git a/simpletuner/helpers/utils/ramtorch.py b/simpletuner/helpers/utils/ramtorch.py index 7d81f4540..44b9f11a3 100644 --- a/simpletuner/helpers/utils/ramtorch.py +++ b/simpletuner/helpers/utils/ramtorch.py @@ -778,20 +778,25 @@ def move_non_linear_layers_to_device(module: nn.Module, device: object) -> int: def mark_ddp_ignore_params(module: nn.Module) -> int: """ - Mark RamTorch parameters on a module to be ignored by DistributedDataParallel. + Mark RamTorch/CPU-resident parameters on a module to be ignored by + DistributedDataParallel. Returns: - Number of parameters added to the ignore list. + Number of parameters and buffers added to the ignore list. """ - ramtorch_names = [name for name, param in module.named_parameters() if getattr(param, "is_ramtorch", False)] - ramtorch_names.extend(name for name, buffer in module.named_buffers() if getattr(buffer, "is_ramtorch", False)) - if not ramtorch_names: - device_types = {param.device.type for _, param in module.named_parameters() if param.device is not None} - if "cuda" in device_types and "cpu" in device_types: - ramtorch_names = [name for name, param in module.named_parameters() if param.device.type == "cpu"] - if not ramtorch_names: + named_params = list(module.named_parameters()) + named_buffers = list(module.named_buffers()) + ignore_names = {name for name, param in named_params if getattr(param, "is_ramtorch", False)} + ignore_names.update(name for name, buffer in named_buffers if getattr(buffer, "is_ramtorch", False)) + + device_types = {param.device.type for _, param in named_params if param.device is not None} + should_ignore_cpu_residents = bool(ignore_names) or {"cpu", "cuda"}.issubset(device_types) + if should_ignore_cpu_residents: + ignore_names.update(name for name, param in named_params if param.device.type == "cpu" and not param.requires_grad) + ignore_names.update(name for name, buffer in named_buffers if buffer.device.type == "cpu") + if not ignore_names: return 0 existing = getattr(module, "_ddp_params_and_buffers_to_ignore", set()) - module._ddp_params_and_buffers_to_ignore = set(existing) | set(ramtorch_names) - return len(ramtorch_names) + module._ddp_params_and_buffers_to_ignore = set(existing) | ignore_names + return len(ignore_names) diff --git a/simpletuner/simpletuner_sdk/server/services/field_registry/sections/model.py b/simpletuner/simpletuner_sdk/server/services/field_registry/sections/model.py index 82f4baa6a..70283c63a 100644 --- a/simpletuner/simpletuner_sdk/server/services/field_registry/sections/model.py +++ b/simpletuner/simpletuner_sdk/server/services/field_registry/sections/model.py @@ -490,14 +490,34 @@ def _quant_label(value: str) -> str: default_value=None, validation_rules=[ValidationRule(ValidationRuleType.MIN, value=1, message="Interval must be at least 1")], dependencies=[FieldDependency(field="gradient_checkpointing", operator="equals", value=True, action="enable")], - help_text="Checkpoint contiguous chunks of N transformer blocks (leave blank for per-block checkpointing)", - tooltip="Higher values reduce recompute overhead but keep more activations in VRAM. Flux and MageFlow use contiguous chunked checkpointing on whole-block paths.", + help_text="Adjust checkpoint spacing or chunk size for supported transformer blocks (leave blank for per-block checkpointing)", + tooltip="Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image, and Wan whole-block paths use contiguous chunks of N blocks. Other families may checkpoint every N-th block. Higher values can reduce recompute but keep more activations in VRAM.", importance=ImportanceLevel.ADVANCED, order=16, documentation="OPTIONS.md#--gradient_checkpointing_interval", ) ) + # Gradient Checkpointing Segment Stride + registry._add_field( + ConfigField( + name="gradient_checkpointing_segment_stride", + arg_name="--gradient_checkpointing_segment_stride", + ui_label="Gradient Checkpointing Segment Stride", + field_type=FieldType.NUMBER, + tab="model", + section="memory_optimization", + default_value=None, + validation_rules=[ValidationRule(ValidationRuleType.MIN, value=1, message="Stride must be at least 1")], + dependencies=[FieldDependency(field="gradient_checkpointing", operator="equals", value=True, action="enable")], + help_text="Start a checkpointed segment every N blocks on supported segmented paths", + tooltip="Use with interval as segment width. interval=2 and stride=4 checkpoints two blocks, runs the next two blocks normally, and repeats on supported whole-block paths.", + importance=ImportanceLevel.ADVANCED, + order=17, + documentation="OPTIONS.md#--gradient_checkpointing_segment_stride", + ) + ) + # Offload During Startup registry._add_field( ConfigField( @@ -511,7 +531,7 @@ def _quant_label(value: str) -> str: help_text="Offload text encoders to CPU during VAE caching", tooltip="Useful for large models that OOM during startup. May significantly increase startup time.", importance=ImportanceLevel.ADVANCED, - order=17, + order=18, documentation="OPTIONS.md#--offload_during_startup", ) ) diff --git a/simpletuner/simpletuner_sdk/server/services/field_registry/sections/training.py b/simpletuner/simpletuner_sdk/server/services/field_registry/sections/training.py index 97334d492..780589f47 100644 --- a/simpletuner/simpletuner_sdk/server/services/field_registry/sections/training.py +++ b/simpletuner/simpletuner_sdk/server/services/field_registry/sections/training.py @@ -258,6 +258,68 @@ def register_training_fields(registry: "FieldRegistry") -> None: ) ) + registry._add_field( + ConfigField( + name="gradient_checkpointing_offload_attention", + arg_name="--gradient_checkpointing_offload_attention", + ui_label="Offload Attention Activations", + field_type=FieldType.CHECKBOX, + tab="model", + section="memory_optimization", + default_value=False, + help_text="Move attention-side saved activations to CPU instead of keeping them in VRAM", + tooltip="Works on supported attention/FFN split blocks. Best paired with FFN-only checkpointing when attention transfer is cheaper than recompute.", + importance=ImportanceLevel.ADVANCED, + order=3, + documentation="OPTIONS.md#--gradient_checkpointing_offload_attention", + ) + ) + + registry._add_field( + ConfigField( + name="gradient_checkpointing_offload_pin_memory_max_buckets", + arg_name="--gradient_checkpointing_offload_pin_memory_max_buckets", + ui_label="Offload Pinned Memory Buckets", + field_type=FieldType.NUMBER, + tab="model", + section="memory_optimization", + default_value=12, + validation_rules=[ValidationRule(ValidationRuleType.MIN, value=0, message="Bucket count must be non-negative")], + dependencies=[ + FieldDependency( + field="gradient_checkpointing_offload_attention", operator="equals", value=True, action="show" + ) + ], + help_text="Maximum number of distinct pinned CPU tensor buckets for activation offload", + tooltip="Set 0 to disable pinned-memory pooling. Once the cap is reached, unseen tensor shapes use normal CPU memory.", + importance=ImportanceLevel.ADVANCED, + order=4, + documentation="OPTIONS.md#--gradient_checkpointing_offload_pin_memory_max_buckets", + ) + ) + + registry._add_field( + ConfigField( + name="gradient_checkpointing_offload_prefetch", + arg_name="--gradient_checkpointing_offload_prefetch", + ui_label="Prefetch Offloaded Activations", + field_type=FieldType.CHECKBOX, + tab="model", + section="memory_optimization", + default_value=False, + dependencies=[ + FieldDependency( + field="gradient_checkpointing_offload_attention", operator="equals", value=True, action="show" + ) + ], + help_text="Learn activation restore order and prefetch CPU-offloaded tensors back to GPU", + tooltip="Experimental. Can overlap H2D restore with backward compute after the runtime has observed a stable unpack order.", + importance=ImportanceLevel.ADVANCED, + order=5, + documentation="OPTIONS.md#--gradient_checkpointing_offload_prefetch", + ) + ) + # Group Offloading registry._add_field( ConfigField( diff --git a/tests/test_gradient_checkpointing_backend.py b/tests/test_gradient_checkpointing_backend.py index 70039d5b9..c7f6618cd 100644 --- a/tests/test_gradient_checkpointing_backend.py +++ b/tests/test_gradient_checkpointing_backend.py @@ -3,6 +3,8 @@ """ import unittest +from types import SimpleNamespace +from unittest import mock import torch import torch.nn as nn @@ -109,12 +111,12 @@ def test_cpu_offload_hooks_pack_unpack(self): hooks = CPUOffloadHooks() if torch.cuda.is_available(): - tensor = torch.randn(4, 4, device="cuda") + tensor = torch.randn(4, 4, device="cuda", requires_grad=True) * 2 packed = hooks.pack(tensor) - # Pack returns (cpu_tensor, original_device) tuple + # Pack returns (cpu_tensor, original_device[, pool_key]) tuple self.assertIsInstance(packed, tuple) - self.assertEqual(len(packed), 2) - cpu_tensor, original_device = packed + self.assertGreaterEqual(len(packed), 2) + cpu_tensor, original_device = packed[:2] self.assertEqual(cpu_tensor.device.type, "cpu") self.assertEqual(original_device.type, "cuda") @@ -129,10 +131,472 @@ def test_cpu_offload_hooks_pack_unpack(self): self.assertEqual(cpu_tensor.device.type, "cpu") self.assertIsNone(original_device) + def test_activation_offload_does_not_recompute_forward(self): + """Activation offload preserves the original forward graph instead of rematerializing it.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import activation_offload + + class CountingModule(nn.Module): + def __init__(self): + super().__init__() + self.calls = 0 + self.linear = nn.Linear(4, 4) + + def forward(self, x): + self.calls += 1 + return torch.relu(self.linear(x)) + + module = CountingModule() + x = torch.randn(2, 4, requires_grad=True) + activation_offload(module, x).sum().backward() + + self.assertEqual(module.calls, 1) + + def test_activation_offload_pin_memory_bucket_setting(self): + """Pinned bucket limit is configurable.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + get_activation_offload_pin_memory_max_buckets, + set_activation_offload_pin_memory_max_buckets, + ) + + original = get_activation_offload_pin_memory_max_buckets() + try: + set_activation_offload_pin_memory_max_buckets(7) + self.assertEqual(get_activation_offload_pin_memory_max_buckets(), 7) + set_activation_offload_pin_memory_max_buckets(0) + self.assertEqual(get_activation_offload_pin_memory_max_buckets(), 0) + finally: + set_activation_offload_pin_memory_max_buckets(original) + + def test_activation_offload_copy_stream_counts_are_configurable(self): + """Activation offload copy stream pools expose bounded tunable widths.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + get_activation_offload_copy_stream_stats, + get_activation_offload_d2h_copy_stream_count, + get_activation_offload_h2d_prefetch_stream_count, + set_activation_offload_d2h_copy_stream_count, + set_activation_offload_h2d_prefetch_stream_count, + ) + + original_d2h = get_activation_offload_d2h_copy_stream_count() + original_h2d = get_activation_offload_h2d_prefetch_stream_count() + try: + set_activation_offload_d2h_copy_stream_count(3) + set_activation_offload_h2d_prefetch_stream_count(5) + + self.assertEqual(get_activation_offload_d2h_copy_stream_count(), 3) + self.assertEqual(get_activation_offload_h2d_prefetch_stream_count(), 5) + stats = get_activation_offload_copy_stream_stats() + self.assertEqual(stats["d2h"]["width"], 3) + self.assertEqual(stats["h2d_prefetch"]["width"], 5) + + set_activation_offload_d2h_copy_stream_count(0) + set_activation_offload_h2d_prefetch_stream_count(-1) + self.assertEqual(get_activation_offload_d2h_copy_stream_count(), 1) + self.assertEqual(get_activation_offload_h2d_prefetch_stream_count(), 1) + finally: + set_activation_offload_d2h_copy_stream_count(original_d2h) + set_activation_offload_h2d_prefetch_stream_count(original_h2d) + + def test_pinned_memory_pool_tracks_reuse_stats(self): + """Pinned pool stats track allocation, release, and buffer reuse per bucket.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import _PinnedMemoryPool + + pool = _PinnedMemoryPool(max_buckets=1) + pool._allocate = lambda key: torch.empty_strided(key.size, key.stride, dtype=key.dtype, layout=key.layout) + key = pool.key_for(torch.empty(2, 3)) + + first = pool.checkout(key) + pool.release_after_cuda_copy(key, first, torch.device("cpu")) + second = pool.checkout(key) + + self.assertIs(first, second) + snapshot = pool.snapshot() + bucket = snapshot["buckets"][0] + self.assertEqual(snapshot["total_accesses"], 2) + self.assertEqual(snapshot["total_allocations"], 1) + self.assertEqual(snapshot["total_buffer_reuses"], 1) + self.assertEqual(bucket["accesses"], 2) + self.assertEqual(bucket["allocations"], 1) + self.assertEqual(bucket["buffer_reuses"], 1) + self.assertEqual(bucket["releases"], 1) + + def test_pinned_memory_pool_eviction_uses_persisted_stats(self): + """Repeated non-resident shapes can evict colder resident buckets without losing old counters.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import _PinnedMemoryPool + + pool = _PinnedMemoryPool(max_buckets=1) + pool._allocate = lambda key: torch.empty_strided(key.size, key.stride, dtype=key.dtype, layout=key.layout) + key_a = pool.key_for(torch.empty(2, 3)) + key_b = pool.key_for(torch.empty(4, 5)) + + first = pool.checkout(key_a) + pool.release_after_cuda_copy(key_a, first, torch.device("cpu")) + + self.assertIsNone(pool.checkout(key_b)) + second = pool.checkout(key_b) + + self.assertIsNotNone(second) + snapshot = pool.snapshot() + buckets = {(bucket["size"], bucket["stride"]): bucket for bucket in snapshot["buckets"]} + bucket_a = buckets[((2, 3), (3, 1))] + bucket_b = buckets[((4, 5), (5, 1))] + + self.assertFalse(bucket_a["resident"]) + self.assertEqual(bucket_a["evictions"], 1) + self.assertEqual(bucket_a["accesses"], 1) + self.assertTrue(bucket_b["resident"]) + self.assertEqual(bucket_b["accesses"], 2) + self.assertEqual(bucket_b["cap_misses"], 1) + self.assertEqual(bucket_b["admissions"], 1) + self.assertEqual(snapshot["tracked_buckets"], 2) + self.assertEqual(snapshot["total_evictions"], 1) + + def test_cpu_offload_dense_noncontiguous_views_use_flat_transfer(self): + """Dense views can transfer as flat storage and restore their original logical stride.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import CPUOffloadHooks + + hooks = CPUOffloadHooks() + base = torch.arange(12).view(3, 4) + transposed = base.t() + + flat = hooks._flat_storage_view(transposed) + + self.assertIsNotNone(flat) + self.assertEqual(tuple(flat.shape), (12,)) + self.assertEqual(tuple(flat.stride()), (1,)) + self.assertEqual(flat.storage_offset(), transposed.storage_offset()) + + transfer_tensor, restore_view = hooks._transfer_view(transposed) + restored = hooks.unpack((transfer_tensor.clone(), torch.device("cpu"), None, restore_view)) + + self.assertEqual(tuple(transfer_tensor.shape), (12,)) + self.assertEqual(tuple(restored.shape), tuple(transposed.shape)) + self.assertEqual(tuple(restored.stride()), tuple(transposed.stride())) + self.assertTrue(torch.equal(restored, transposed)) + + def test_cpu_offload_sparse_storage_views_keep_original_layout(self): + """Views with holes in storage are not flattened because storage-order transfer would lose layout.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import CPUOffloadHooks + + hooks = CPUOffloadHooks() + sparse_view = torch.arange(12).view(3, 4)[:, ::2] + + flat = hooks._flat_storage_view(sparse_view) + transfer_tensor, restore_view = hooks._transfer_view(sparse_view) + + self.assertIsNone(flat) + self.assertIs(transfer_tensor, sparse_view) + self.assertIsNone(restore_view) + + def test_cpu_offload_flat_transfer_key_ignores_dense_view_stride(self): + """Pinned bucket keys are based on transfer shape, not dense view logical stride.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import CPUOffloadHooks, _PinnedMemoryPool + + hooks = CPUOffloadHooks() + pool = _PinnedMemoryPool(max_buckets=2) + base = torch.empty(3, 4) + transposed = base.t() + + base_transfer, base_restore = hooks._transfer_view(base) + transposed_transfer, transposed_restore = hooks._transfer_view(transposed) + base_key = pool.key_for(base_transfer) + transposed_key = pool.key_for(transposed_transfer) + + self.assertEqual(base_key, transposed_key) + self.assertEqual(base_key.size, (12,)) + self.assertEqual(base_key.stride, (1,)) + self.assertEqual(base_restore.size, (3, 4)) + self.assertEqual(transposed_restore.size, (4, 3)) + self.assertEqual(transposed_restore.stride, (1, 4)) + + @unittest.skipIf(not torch.cuda.is_available(), "CUDA required for pinned offload test") + def test_activation_offload_pin_memory_bucket_limit(self): + """New shapes fall back to pageable CPU once the pinned bucket cap is reached.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + CPUOffloadHooks, + get_activation_offload_pin_memory_max_buckets, + get_activation_offload_pin_memory_stats, + reset_activation_offload_pin_memory_stats, + set_activation_offload_pin_memory_max_buckets, + ) + + original = get_activation_offload_pin_memory_max_buckets() + try: + set_activation_offload_pin_memory_max_buckets(0) + reset_activation_offload_pin_memory_stats() + set_activation_offload_pin_memory_max_buckets(1) + hooks = CPUOffloadHooks() + + first = hooks.pack(torch.randn(4, 4, device="cuda", requires_grad=True) * 2) + second = hooks.pack(torch.randn(8, 8, device="cuda", requires_grad=True) * 2) + + self.assertTrue(first[0].is_pinned()) + self.assertIsNotNone(first[2]) + self.assertIsNone(second[2]) + self.assertEqual(get_activation_offload_pin_memory_stats()["total_cap_misses"], 1) + finally: + set_activation_offload_pin_memory_max_buckets(original) + + @unittest.skipIf(not torch.cuda.is_available(), "CUDA required for copy stream offload test") + def test_cpu_offload_uses_copy_stream_ready_event(self): + """Pinned CUDA offload returns a ready event and restores dense view strides.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import CPUOffloadHooks + + hooks = CPUOffloadHooks() + tensor = (torch.arange(12, device="cuda", dtype=torch.float32, requires_grad=True) * 2).view(3, 4).t() + + packed = hooks.pack(tensor) + + self.assertGreaterEqual(len(packed), 5) + self.assertIsNotNone(packed[2]) + self.assertIsNotNone(packed[3]) + self.assertIsNotNone(packed[4]) + + unpacked = hooks.unpack(packed) + torch.cuda.synchronize() + + self.assertEqual(unpacked.device.type, "cuda") + self.assertEqual(tuple(unpacked.shape), tuple(tensor.shape)) + self.assertEqual(tuple(unpacked.stride()), tuple(tensor.stride())) + self.assertTrue(torch.equal(unpacked, tensor.detach())) + + @unittest.skipIf(not torch.cuda.is_available(), "CUDA required for copy stream pool test") + def test_activation_offload_copy_stream_pools_round_robin_by_direction(self): + """D2H offload and H2D prefetch use independent round-robin stream pools.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + _ACTIVATION_PREFETCH_RUNTIME, + _PINNED_MEMORY_POOL, + CPUOffloadHooks, + _OffloadedActivationRecord, + get_activation_offload_copy_stream_stats, + get_activation_offload_d2h_copy_stream_count, + get_activation_offload_h2d_prefetch_stream_count, + reset_activation_offload_copy_stream_stats, + reset_activation_offload_prefetch_stats, + set_activation_offload_d2h_copy_stream_count, + set_activation_offload_h2d_prefetch_stream_count, + ) + + original_d2h = get_activation_offload_d2h_copy_stream_count() + original_h2d = get_activation_offload_h2d_prefetch_stream_count() + try: + set_activation_offload_d2h_copy_stream_count(2) + set_activation_offload_h2d_prefetch_stream_count(2) + reset_activation_offload_prefetch_stats() + reset_activation_offload_copy_stream_stats() + + hooks = CPUOffloadHooks() + hooks.pack(torch.randn(32, 32, device="cuda", requires_grad=True) * 2) + hooks.pack(torch.randn(32, 32, device="cuda", requires_grad=True) * 3) + + pinned_a = torch.empty(32, 32, pin_memory=True) + pinned_b = torch.empty(32, 32, pin_memory=True) + pool_key = _PINNED_MEMORY_POOL.key_for(pinned_a) + for logical_id, tensor in (("a", pinned_a), ("b", pinned_b)): + _ACTIVATION_PREFETCH_RUNTIME.register( + _OffloadedActivationRecord( + logical_id=logical_id, + predictor_id=logical_id, + generation=0, + tensor=tensor, + original_device=torch.device("cuda"), + pool_key=pool_key, + restore_view=None, + ready_event=None, + ) + ) + self.assertTrue(_ACTIVATION_PREFETCH_RUNTIME.prefetch(0, logical_id)) + + torch.cuda.synchronize() + stats = get_activation_offload_copy_stream_stats() + d2h_devices = list(stats["d2h"]["devices"].values()) + h2d_devices = list(stats["h2d_prefetch"]["devices"].values()) + + self.assertEqual(d2h_devices[0]["uses"], [1, 1]) + self.assertEqual(h2d_devices[0]["uses"], [1, 1]) + finally: + reset_activation_offload_prefetch_stats() + set_activation_offload_d2h_copy_stream_count(original_d2h) + set_activation_offload_h2d_prefetch_stream_count(original_h2d) + + def test_activation_offload_prefetch_learns_stable_successors(self): + """Activation prefetch learns on stable predictor ids instead of unique payload ids.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + _ACTIVATION_PREFETCH_RUNTIME, + _OffloadedActivationRecord, + get_activation_offload_prefetch_enabled, + reset_activation_offload_prefetch_stats, + set_activation_offload_prefetch_enabled, + ) + + reset_activation_offload_prefetch_stats() + original_enabled = get_activation_offload_prefetch_enabled() + try: + set_activation_offload_prefetch_enabled(False) + for generation in range(2): + first = _OffloadedActivationRecord( + logical_id=f"{generation}:a:payload", + predictor_id="block.attn:0", + generation=generation, + tensor=torch.empty(1), + original_device=torch.device("cpu"), + pool_key=None, + restore_view=None, + ready_event=None, + ) + second = _OffloadedActivationRecord( + logical_id=f"{generation}:b:payload", + predictor_id="block.attn:1", + generation=generation, + tensor=torch.empty(1), + original_device=torch.device("cpu"), + pool_key=None, + restore_view=None, + ready_event=None, + ) + _ACTIVATION_PREFETCH_RUNTIME.register(first) + _ACTIVATION_PREFETCH_RUNTIME.register(second) + _ACTIVATION_PREFETCH_RUNTIME.consume(first) + _ACTIVATION_PREFETCH_RUNTIME.consume(second) + + stats = _ACTIVATION_PREFETCH_RUNTIME.snapshot() + self.assertEqual(_ACTIVATION_PREFETCH_RUNTIME.successors["block.attn:0"], "block.attn:1") + self.assertEqual(stats["learned_successors"], 1) + self.assertEqual(stats["transition_updates"], 1) + finally: + set_activation_offload_prefetch_enabled(original_enabled) + reset_activation_offload_prefetch_stats() + + def test_activation_offload_prefetch_retires_unconsumed_generation_records(self): + """Unpacked-only subsets should not strand pinned buffers in the prefetch runtime.""" + from simpletuner.helpers.training import offloaded_gradient_checkpointer as offload + + original_pool = offload._PINNED_MEMORY_POOL + original_runtime = offload._ACTIVATION_PREFETCH_RUNTIME + pool = offload._PinnedMemoryPool(max_buckets=1) + pool._allocate = lambda key: torch.empty_strided(key.size, key.stride, dtype=key.dtype, layout=key.layout) + runtime = offload._ActivationOffloadPrefetchRuntime() + try: + offload._PINNED_MEMORY_POOL = pool + offload._ACTIVATION_PREFETCH_RUNTIME = runtime + key = pool.key_for(torch.empty(2, 3)) + cpu_tensor = pool.checkout(key) + record = offload._OffloadedActivationRecord( + logical_id="0:unused:payload", + predictor_id="unused", + generation=0, + tensor=cpu_tensor, + original_device=torch.device("cuda" if torch.cuda.is_available() else "cpu"), + pool_key=key, + restore_view=None, + ready_event=None, + ) + + runtime.register(record) + runtime.saw_unpack_since_pack = True + + self.assertEqual(runtime.snapshot()["active_records"], 1) + self.assertEqual(runtime.next_generation_for_pack(), 1) + + snapshot = runtime.snapshot() + pool_snapshot = pool.snapshot() + self.assertEqual(snapshot["active_records"], 0) + self.assertTrue(record.cpu_released) + self.assertEqual(pool_snapshot["buckets"][0]["releases"], 1) + self.assertEqual(pool_snapshot["buckets"][0]["available_buffers"], 1) + finally: + offload._PINNED_MEMORY_POOL = original_pool + offload._ACTIVATION_PREFETCH_RUNTIME = original_runtime + + def test_activation_offload_prefetch_autotune_disables_worse_prefetch(self): + """Autotune disables prefetch when measured waits are not better than JIT restore.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + _ACTIVATION_PREFETCH_RUNTIME, + reset_activation_offload_prefetch_stats, + ) + + reset_activation_offload_prefetch_stats() + try: + for _ in range(8): + _ACTIVATION_PREFETCH_RUNTIME.record_jit_restore_ms(1.0) + _ACTIVATION_PREFETCH_RUNTIME.record_prefetch_wait_ms(1.1) + + stats = _ACTIVATION_PREFETCH_RUNTIME.snapshot() + self.assertTrue(stats["autotune_disabled"]) + self.assertEqual(stats["autotune_decision"], "jit") + finally: + reset_activation_offload_prefetch_stats() + class TestGradientCheckpointingBackend(unittest.TestCase): """Tests for the gradient checkpointing backend module.""" + def test_trainer_prefetch_autotune_uses_model_predict_and_discards_gradients(self): + """Trainer-level prefetch autotune probes the real prediction/loss path without retaining grads.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + activation_offload_prefetch_autotune_decision, + reset_activation_offload_prefetch_stats, + ) + from simpletuner.helpers.training.trainer import Trainer + + class FakeAccelerator: + def __init__(self): + self.backward_calls = 0 + + def backward(self, loss): + self.backward_calls += 1 + loss.backward() + + class FakeModel: + def __init__(self, component): + self.component = component + + def get_trained_component(self, unwrap_model=False): + return self.component + + def loss_with_logs(self, prepared_batch, model_output, apply_conditioning_mask=True): + return model_output.square().mean(), {} + + def auxiliary_loss(self, prepared_batch, model_output, loss): + return loss, {} + + trainer = Trainer.__new__(Trainer) + trainer.config = SimpleNamespace( + gradient_checkpointing_offload_prefetch=True, + gradient_checkpointing_offload_attention=True, + disable_accelerator=False, + distillation_method=None, + ) + trainer.probe_component = nn.Linear(2, 2) + trainer.model = FakeModel(trainer.probe_component) + trainer.optimizer = torch.optim.SGD(trainer.probe_component.parameters(), lr=0.1) + trainer.sidecar_optimizer = None + trainer.accelerator = FakeAccelerator() + trainer.predict_calls = 0 + + def model_predict(prepared_batch): + trainer.predict_calls += 1 + return trainer.probe_component(prepared_batch["x"]) + + trainer.model_predict = model_predict + prepared_batch = {"x": torch.ones(1, 2)} + + reset_activation_offload_prefetch_stats() + with ( + mock.patch("simpletuner.helpers.training.trainer.torch.cuda.is_available", return_value=True), + mock.patch("simpletuner.helpers.training.trainer.torch.cuda.synchronize"), + mock.patch("simpletuner.helpers.training.trainer.torch.cuda.get_rng_state_all", return_value=[]), + mock.patch("simpletuner.helpers.training.trainer.torch.cuda.set_rng_state_all"), + ): + trainer._maybe_autotune_activation_offload_prefetch(prepared_batch) + + self.assertEqual(trainer.predict_calls, 3) + self.assertEqual(trainer.accelerator.backward_calls, 3) + self.assertIsNone(trainer.probe_component.weight.grad) + self.assertIsNone(trainer.probe_component.bias.grad) + self.assertIn(activation_offload_prefetch_autotune_decision(), {"jit", "prefetch"}) + reset_activation_offload_prefetch_stats() + def test_set_checkpoint_backend(self): """Test that checkpoint backend can be set.""" from simpletuner.helpers.training.gradient_checkpointing_interval import ( @@ -339,6 +803,47 @@ def run_block(index, block, x): self.assertEqual(result.item(), 15) self.assertEqual(calls, ["checkpoint", 0, 1, "checkpoint", 2, 3, "checkpoint", 4]) + def test_checkpoint_sequential_state_segment_stride_runs_gaps_without_checkpoint(self): + """Test that segment_stride leaves deterministic eager gaps between chunks.""" + from simpletuner.helpers.training.gradient_checkpointing_interval import checkpoint_sequential_state + + calls = [] + + def checkpoint_fn(function, *args, **_kwargs): + calls.append("checkpoint") + return function(*args) + + def run_block(index, block, x): + calls.append(index) + return x + block + + (result,) = checkpoint_sequential_state( + [1, 2, 3, 4, 5, 6], + 2, + (torch.tensor(0),), + run_block, + checkpoint_fn, + {"use_reentrant": False}, + segment_stride=4, + ) + + self.assertEqual(result.item(), 21) + self.assertEqual(calls, ["checkpoint", 0, 1, 2, 3, "checkpoint", 4, 5]) + + def test_checkpoint_sequential_state_rejects_overlapping_stride(self): + """Test that overlapping segment schedules are rejected.""" + from simpletuner.helpers.training.gradient_checkpointing_interval import checkpoint_sequential_state + + with self.assertRaisesRegex(ValueError, "segment_stride"): + checkpoint_sequential_state( + [1, 2], + 2, + (torch.tensor(0),), + lambda _index, block, x: x + block, + lambda function, *args, **_kwargs: function(*args), + segment_stride=1, + ) + class TestConfigFieldIntegration(unittest.TestCase): """Tests for the configuration field integration.""" @@ -357,6 +862,42 @@ def test_gradient_checkpointing_backend_field_exists(self): self.assertIn({"value": "unsloth", "label": "Unsloth layer (CPU offload)"}, field.choices) self.assertIn({"value": "unsloth-ffn", "label": "Unsloth FFN-only (CPU offload)"}, field.choices) + def test_gradient_checkpointing_offload_attention_field_exists(self): + """Test that the attention activation offload field is registered.""" + from simpletuner.simpletuner_sdk.server.services.field_registry import FieldRegistry + + registry = FieldRegistry() + field = registry.get_field("gradient_checkpointing_offload_attention") + + self.assertIsNotNone(field) + self.assertEqual(field.default_value, False) + self.assertEqual(field.arg_name, "--gradient_checkpointing_offload_attention") + self.assertEqual(field.dependencies, []) + + def test_gradient_checkpointing_offload_pin_memory_max_buckets_field_exists(self): + """Test that the attention offload pinned bucket field is registered.""" + from simpletuner.simpletuner_sdk.server.services.field_registry import FieldRegistry + + registry = FieldRegistry() + field = registry.get_field("gradient_checkpointing_offload_pin_memory_max_buckets") + + self.assertIsNotNone(field) + self.assertEqual(field.default_value, 12) + self.assertEqual(field.arg_name, "--gradient_checkpointing_offload_pin_memory_max_buckets") + self.assertEqual(len(field.dependencies), 1) + self.assertEqual(field.dependencies[0].field, "gradient_checkpointing_offload_attention") + + def test_gradient_checkpointing_offload_prefetch_field_exists(self): + """Test that the attention offload prefetch field is registered.""" + from simpletuner.simpletuner_sdk.server.services.field_registry import FieldRegistry + + registry = FieldRegistry() + field = registry.get_field("gradient_checkpointing_offload_prefetch") + + self.assertIsNotNone(field) + self.assertEqual(field.default_value, False) + self.assertEqual(field.arg_name, "--gradient_checkpointing_offload_prefetch") + def test_gradient_checkpointing_backend_validation(self): """Test that invalid backend values are rejected.""" from simpletuner.simpletuner_sdk.server.services.field_registry import FieldRegistry @@ -377,87 +918,16 @@ def test_gradient_checkpointing_backend_validation(self): self.assertIn("unsloth", choices_rule.value) self.assertIn("unsloth-ffn", choices_rule.value) + def test_gradient_checkpointing_segment_stride_field_exists(self): + """Test that the segmented checkpointing stride field is registered.""" + from simpletuner.simpletuner_sdk.server.services.field_registry import FieldRegistry -class TestTransformerBackendAttribute(unittest.TestCase): - """Tests that transformer models have the backend attribute and setter.""" - - def test_flux_transformer_has_backend_attribute(self): - """Test that FluxTransformer2DModel has gradient_checkpointing_backend.""" - from simpletuner.helpers.models.flux.transformer import FluxTransformer2DModel - - self.assertTrue(hasattr(FluxTransformer2DModel, "set_gradient_checkpointing_backend")) - self.assertTrue(getattr(FluxTransformer2DModel, "_supports_ffn_gradient_checkpointing", False)) - - def test_flux_blocks_support_ffn_checkpoint_scope(self): - """Test that Flux blocks preserve output values with FFN-only checkpointing.""" - from simpletuner.helpers.models.flux.transformer import FluxSingleTransformerBlock, FluxTransformerBlock - - double_block = FluxTransformerBlock(dim=16, num_attention_heads=2, attention_head_dim=8).train() - hidden = torch.randn(2, 4, 16, requires_grad=True) - encoder_hidden = torch.randn(2, 3, 16, requires_grad=True) - temb = torch.randn(2, 16) - - expected_encoder, expected_hidden = double_block(hidden, encoder_hidden, temb) - actual_encoder, actual_hidden = double_block( - hidden, - encoder_hidden, - temb, - checkpoint_ffn=True, - checkpoint_fn=torch.utils.checkpoint.checkpoint, - ) - self.assertTrue(torch.allclose(expected_encoder, actual_encoder, atol=1e-6)) - self.assertTrue(torch.allclose(expected_hidden, actual_hidden, atol=1e-6)) - - single_block = FluxSingleTransformerBlock(dim=16, num_attention_heads=2, attention_head_dim=8).train() - hidden = torch.randn(2, 7, 16, requires_grad=True) - temb = torch.randn(2, 16) - - expected_hidden = single_block(hidden, temb) - actual_hidden = single_block( - hidden, - temb, - checkpoint_ffn=True, - checkpoint_fn=torch.utils.checkpoint.checkpoint, - ) - self.assertTrue(torch.allclose(expected_hidden, actual_hidden, atol=1e-6)) - - def test_sana_transformer_has_backend_attribute(self): - """Test that SanaTransformer2DModel has gradient_checkpointing_backend.""" - from simpletuner.helpers.models.sana.transformer import SanaTransformer2DModel - - self.assertTrue(hasattr(SanaTransformer2DModel, "set_gradient_checkpointing_backend")) - - def test_sd3_transformer_has_backend_attribute(self): - """Test that SD3Transformer2DModel has gradient_checkpointing_backend.""" - from simpletuner.helpers.models.sd3.transformer import SD3Transformer2DModel - - self.assertTrue(hasattr(SD3Transformer2DModel, "set_gradient_checkpointing_backend")) - - def test_chroma_transformer_has_backend_attribute(self): - """Test that ChromaTransformer2DModel has gradient_checkpointing_backend.""" - from simpletuner.helpers.models.chroma.transformer import ChromaTransformer2DModel - - self.assertTrue(hasattr(ChromaTransformer2DModel, "set_gradient_checkpointing_backend")) - - def test_auraflow_transformer_has_backend_attribute(self): - """Test that AuraFlowTransformer2DModel has gradient_checkpointing_backend.""" - from simpletuner.helpers.models.auraflow.transformer import AuraFlowTransformer2DModel - - self.assertTrue(hasattr(AuraFlowTransformer2DModel, "set_gradient_checkpointing_backend")) - - def test_mageflow_transformer_has_backend_attribute(self): - """Test that MageFlowTransformer2DModel has gradient_checkpointing_backend.""" - from simpletuner.helpers.models.mageflow.transformer import MageFlowTransformer2DModel - - self.assertTrue(hasattr(MageFlowTransformer2DModel, "set_gradient_checkpointing_backend")) - self.assertTrue(hasattr(MageFlowTransformer2DModel, "set_gradient_checkpointing_interval")) - self.assertTrue(getattr(MageFlowTransformer2DModel, "_supports_ffn_gradient_checkpointing", False)) - - def test_qwen_image_transformer_has_backend_attribute(self): - """Test that QwenImageTransformer2DModel has gradient_checkpointing_backend.""" - from simpletuner.helpers.models.qwen_image.transformer import QwenImageTransformer2DModel + registry = FieldRegistry() + field = registry.get_field("gradient_checkpointing_segment_stride") - self.assertTrue(hasattr(QwenImageTransformer2DModel, "set_gradient_checkpointing_backend")) + self.assertIsNotNone(field) + self.assertEqual(field.default_value, None) + self.assertEqual(field.arg_name, "--gradient_checkpointing_segment_stride") if __name__ == "__main__": diff --git a/tests/test_musubi_block_swap.py b/tests/test_musubi_block_swap.py index a6e112173..bff4108e3 100644 --- a/tests/test_musubi_block_swap.py +++ b/tests/test_musubi_block_swap.py @@ -51,6 +51,40 @@ def test_quanto_qlinear_streams_without_apply_swap(self): self.assertEqual(qlinear.weight._data.device.type, "cpu") self.assertEqual(qlinear.weight._scale.device.type, "cpu") + def test_sdnq_module_streams_without_apply_swap(self): + device = self._accelerator_device() + + class FakeSDNQLinear(nn.Linear): + pass + + FakeSDNQLinear.__module__ = "sdnq.training.layers.linear" + block = nn.Sequential(FakeSDNQLinear(4, 4), nn.SiLU(), FakeSDNQLinear(4, 4)) + for param in block.parameters(): + param.requires_grad_(False) + param.sdnq_dequantizer = object() + param.weight = param.detach().clone() + param.scale = torch.ones(param.shape[0], 1, device=param.device) + block.to(device) + + manager = MusubiBlockSwapManager( + block_indices=[0], + offload_device=torch.device("cpu"), + logger=logging.getLogger(__name__), + ) + + with patch.object(FakeSDNQLinear, "_apply", side_effect=RuntimeError("_apply(): Couldn't swap SDNQLinear.weight")): + manager.stream_out(block) + self.assertTrue(_module_on_device(block, torch.device("cpu"))) + self.assertEqual(block[0].weight.device.type, "cpu") + self.assertEqual(block[0].weight.scale.device.type, "cpu") + + manager.stream_in(block, device) + self.assertTrue(_module_on_device(block, device)) + self.assertEqual(block[0].weight.device.type, device.type) + self.assertEqual(block[0].weight.scale.device.type, device.type) + output = block(torch.randn(2, 4, device=device)) + self.assertEqual(output.device.type, device.type) + def test_stream_out_keeps_trainable_params_on_accelerator(self): device = self._accelerator_device() block = nn.Sequential(nn.Linear(4, 4), nn.SiLU(), nn.Linear(4, 4)) diff --git a/tests/test_ramtorch.py b/tests/test_ramtorch.py index 2b457ab26..238c20cbc 100644 --- a/tests/test_ramtorch.py +++ b/tests/test_ramtorch.py @@ -192,6 +192,20 @@ def __init__(self): self.assertEqual(model.ramtorch_buffer.device.type, "cpu") self.assertTrue(getattr(model.ramtorch_buffer, "is_ramtorch", False)) + def test_ramtorch_profile_reset_resets_cuda_peak_memory_stats(self): + from simpletuner.helpers.ramtorch import profiling as ramtorch_profile + + with ( + patch("torch.cuda.is_available", return_value=True), + patch("torch.cuda.device_count", return_value=2), + patch("torch.cuda.reset_peak_memory_stats") as reset_peak_memory_stats, + ): + ramtorch_profile.reset_for_new_run() + + self.assertEqual(reset_peak_memory_stats.call_count, 2) + self.assertEqual(str(reset_peak_memory_stats.call_args_list[0].args[0]), "cuda:0") + self.assertEqual(str(reset_peak_memory_stats.call_args_list[1].args[0]), "cuda:1") + def test_torchao_int8_ramtorch_backward_uses_dense_weight_view(self): try: from torchao.prototype.quantized_training import int8_weight_only_quantized_training @@ -313,10 +327,18 @@ def __init__(self): ignored = ramtorch_utils.mark_ddp_ignore_params(model) self.assertEqual(ignored, 3) ignore_set = getattr(model, "_ddp_params_and_buffers_to_ignore", set()) + self.assertNotIn("linear_a.weight", ignore_set) + self.assertNotIn("linear_a.bias", ignore_set) self.assertIn("linear_b.weight", ignore_set) self.assertIn("linear_b.bias", ignore_set) self.assertIn("ramtorch_buffer", ignore_set) + def test_mark_ddp_ignore_params_skips_plain_cpu_modules(self): + model = nn.Sequential(nn.Linear(2, 2), nn.LayerNorm(2)) + ignored = ramtorch_utils.mark_ddp_ignore_params(model) + self.assertEqual(ignored, 0) + self.assertFalse(hasattr(model, "_ddp_params_and_buffers_to_ignore")) + def test_prefetch_hooks_follow_ramtorch_module_order(self): from simpletuner.helpers.ramtorch_extensions import add_ramtorch_prefetch_hooks from simpletuner.helpers.training.state_tracker import StateTracker @@ -622,6 +644,53 @@ def forward(self, x): with patch.dict("os.environ", {"SIMPLETUNER_RAMTORCH_PREFETCH_POLICY": "sync"}): self.assertEqual(add_ramtorch_prefetch_hooks(model), []) + def test_prefetch_hooks_start_successor_before_current_forward(self): + from simpletuner.helpers.ramtorch_extensions import add_ramtorch_prefetch_hooks + from simpletuner.helpers.training.state_tracker import StateTracker + + StateTracker.reset_ramtorch_prefetch_orders() + + class _PrefetchModule(nn.Module): + is_ramtorch = True + + def __init__(self, label, calls): + super().__init__() + self.label = label + self.calls = calls + + def prefetch_forward(self): + self.calls.append(("prefetch", self.label)) + return True + + def forward(self, x): + self.calls.append(("forward", self.label)) + return x + + calls = [] + model = nn.Sequential( + _PrefetchModule("first", calls), + _PrefetchModule("second", calls), + _PrefetchModule("third", calls), + ) + + hooks = add_ramtorch_prefetch_hooks(model, component_label="prehook-timing-test") + try: + model(torch.ones(1)) + finally: + for hook in hooks: + hook.remove() + + self.assertEqual( + calls, + [ + ("prefetch", "second"), + ("forward", "first"), + ("prefetch", "third"), + ("forward", "second"), + ("forward", "third"), + ], + ) + def test_bundled_linear_skips_frozen_weight_gradients(self): from simpletuner.helpers.ramtorch.modules.linear import Linear From 615ba824afe8b69debdda1980a506ced606b9fea Mon Sep 17 00:00:00 2001 From: bghira Date: Mon, 3 Aug 2026 03:58:22 -0600 Subject: [PATCH 2/2] Address segmented checkpointing base review feedback --- documentation/OPTIONS.es.md | 4 +- documentation/OPTIONS.hi.md | 4 +- documentation/OPTIONS.ja.md | 4 +- documentation/OPTIONS.md | 4 +- documentation/OPTIONS.pt-BR.md | 4 +- documentation/OPTIONS.zh.md | 4 +- simpletuner/helpers/models/common.py | 6 ++- .../training/default_settings/safety_check.py | 14 ++----- .../offloaded_gradient_checkpointer.py | 26 +++++++++--- simpletuner/helpers/utils/ramtorch.py | 7 ++-- tests/test_gradient_checkpointing_backend.py | 40 +++++++++++++++++-- tests/test_ramtorch.py | 1 + 12 files changed, 82 insertions(+), 36 deletions(-) diff --git a/documentation/OPTIONS.es.md b/documentation/OPTIONS.es.md index 8c4024c34..61bb37a60 100644 --- a/documentation/OPTIONS.es.md +++ b/documentation/OPTIONS.es.md @@ -289,7 +289,7 @@ Donde `foo` es tu entorno de configuración; o simplemente usa `config/config.js - **Qué**: Inicia un segmento con checkpoint cada *n* bloques en rutas segmented whole-block compatibles. - **Ejemplo**: Con `--gradient_checkpointing_interval=2` y `--gradient_checkpointing_segment_stride=4`, SimpleTuner checkpointa dos bloques, ejecuta los dos siguientes normalmente y repite. -- **Nota**: Compatible con rutas segmentadas de Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image y Wan. El stride debe ser al menos igual al interval. Consulta [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md). +- **Nota**: Solo tiene efecto en familias de modelos que exponen soporte segmented whole-block en la version instalada de SimpleTuner. Las familias no soportadas registran una advertencia e ignoran el valor. El stride debe ser al menos igual al interval. Consulta [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md). ### `--gradient_checkpointing_backend` @@ -305,7 +305,7 @@ Donde `foo` es tu entorno de configuración; o simplemente usa `config/config.js - **Qué**: Hace offload a CPU de las activaciones guardadas del lado attention en modelos con un límite attention/FFN limpio. - **Por qué**: Cuando transferir es más barato que recomputar attention, reduce VRAM sin pagar todo el coste de rematerializar attention. -- **Nota**: Puede activarse por si solo. Tambien puede combinarse con cualquier checkpoint backend que soporte el modelo. Attention offload actualmente soporta Chroma, Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Wan y Z-Image; las familias no soportadas fallan de forma explícita. +- **Nota**: Puede activarse por si solo. Tambien puede combinarse con cualquier checkpoint backend que soporte el modelo. Solo tiene efecto en familias de modelos que exponen una frontera attention/FFN limpia en la version instalada de SimpleTuner; las familias no soportadas fallan de forma explicita. ### `--gradient_checkpointing_offload_pin_memory_max_buckets` diff --git a/documentation/OPTIONS.hi.md b/documentation/OPTIONS.hi.md index a4f55e186..c9377a630 100644 --- a/documentation/OPTIONS.hi.md +++ b/documentation/OPTIONS.hi.md @@ -289,7 +289,7 @@ simpletuner configure config/foo/config.json - **What**: Supported segmented whole-block paths पर हर *n* blocks में checkpointed segment शुरू करें। - **Example**: `--gradient_checkpointing_interval=2` और `--gradient_checkpointing_segment_stride=4` के साथ SimpleTuner दो blocks checkpoint करता है, अगले दो blocks normally चलाता है, और repeat करता है। -- **Note**: Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image, और Wan segmented paths पर supported। stride interval से कम नहीं हो सकता। [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) देखें। +- **Note**: यह केवल उन model families पर प्रभावी होता है जो installed SimpleTuner version में segmented whole-block support expose करती हैं। Unsupported families warning log करती हैं और value ignore करती हैं। stride interval से कम नहीं हो सकता। [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) देखें। ### `--gradient_checkpointing_backend` @@ -305,7 +305,7 @@ simpletuner configure config/foo/config.json - **What**: साफ attention/FFN boundary वाले models पर attention-side saved activations CPU पर offload करें। - **Why**: जब transfer attention recompute से सस्ता हो, तो full attention rematerialization cost दिए बिना VRAM घट सकती है। -- **Note**: इसे अकेले enable किया जा सकता है। Model जिस भी checkpoint backend को support करे, उसके साथ भी combine किया जा सकता है। Attention offload अभी Chroma, Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Wan, और Z-Image support करता है; unsupported model families साफ error देती हैं। +- **Note**: इसे अकेले enable किया जा सकता है। Model जिस भी checkpoint backend को support करे, उसके साथ भी combine किया जा सकता है। यह केवल उन model families पर प्रभावी होता है जो installed SimpleTuner version में साफ attention/FFN boundary expose करती हैं; unsupported model families साफ error देती हैं। ### `--gradient_checkpointing_offload_pin_memory_max_buckets` diff --git a/documentation/OPTIONS.ja.md b/documentation/OPTIONS.ja.md index d7f24dc47..d3f435367 100644 --- a/documentation/OPTIONS.ja.md +++ b/documentation/OPTIONS.ja.md @@ -290,7 +290,7 @@ simpletuner configure config/foo/config.json - **内容**: 対応する segmented whole-block path で、*n* block ごとに checkpointed segment を開始します。 - **例**: `--gradient_checkpointing_interval=2` と `--gradient_checkpointing_segment_stride=4` では、SimpleTuner は 2 block を checkpoint し、次の 2 block を通常実行し、それを繰り返します。 -- **注記**: Flux、Flux.2、Krea 2、LTXVideo2、MageFlow、Z-Image、Wan の segmented path で対応しています。stride は interval 以上である必要があります。[Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) を参照してください。 +- **注記**: インストール済み SimpleTuner のバージョンで segmented whole-block support を公開している model family でのみ有効です。未対応 family では warning を記録し、値を無視します。stride は interval 以上である必要があります。[Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) を参照してください。 ### `--gradient_checkpointing_backend` @@ -306,7 +306,7 @@ simpletuner configure config/foo/config.json - **内容**: 明確な attention/FFN 境界があるモデルで、attention 側の保存 activations を CPU に offload します。 - **理由**: attention を再計算するより転送が安い場合、完全な attention rematerialization cost を払わずに VRAM を減らせます。 -- **注記**: 単体でも有効化できます。そのモデルがサポートする任意の checkpoint backend とも組み合わせられます。Attention offload は現在 Chroma、Flux、Flux.2、Krea 2、LTXVideo2、MageFlow、Wan、Z-Image に対応し、未対応モデルでは明示的に失敗します。 +- **注記**: 単体でも有効化できます。そのモデルがサポートする任意の checkpoint backend とも組み合わせられます。インストール済み SimpleTuner のバージョンで明確な attention/FFN boundary を公開している model family でのみ有効です。未対応モデルでは明示的に失敗します。 ### `--gradient_checkpointing_offload_pin_memory_max_buckets` diff --git a/documentation/OPTIONS.md b/documentation/OPTIONS.md index 087476bae..71308a0f3 100644 --- a/documentation/OPTIONS.md +++ b/documentation/OPTIONS.md @@ -296,7 +296,7 @@ Where `foo` is your config environment - or just use `config/config.json` if you - **What**: Start a checkpointed segment every *n* blocks on supported segmented whole-block paths. - **Example**: With `--gradient_checkpointing_interval=2` and `--gradient_checkpointing_segment_stride=4`, SimpleTuner checkpoints two blocks, runs the next two blocks normally, and repeats. -- **Note**: Supported by Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image, and Wan segmented paths. The stride must be at least the interval. See [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md). +- **Note**: Only takes effect on model families that expose segmented whole-block support in the installed SimpleTuner version. Unsupported families log a warning and ignore the value. The stride must be at least the interval. See [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md). ### `--gradient_checkpointing_backend` @@ -312,7 +312,7 @@ Where `foo` is your config environment - or just use `config/config.json` if you - **What**: Offload attention-side saved activations to CPU on models with a clean attention/FFN boundary. - **Why**: When transfer is cheaper than recomputing attention, this can reduce VRAM without paying the full attention rematerialization cost. -- **Note**: This can be enabled by itself. It can also be combined with any checkpoint backend that the model supports. Attention offload is currently supported by Chroma, Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Wan, and Z-Image; unsupported model families fail loudly. +- **Note**: This can be enabled by itself. It can also be combined with any checkpoint backend that the model supports. It only takes effect on model families that expose a clean attention/FFN boundary in the installed SimpleTuner version; unsupported model families fail loudly. ### `--gradient_checkpointing_offload_pin_memory_max_buckets` diff --git a/documentation/OPTIONS.pt-BR.md b/documentation/OPTIONS.pt-BR.md index a7355132c..10c5e132a 100644 --- a/documentation/OPTIONS.pt-BR.md +++ b/documentation/OPTIONS.pt-BR.md @@ -289,7 +289,7 @@ Onde `foo` e seu ambiente de config — ou use `config/config.json` se nao estiv - **O que**: Inicia um segmento com checkpoint a cada *n* blocos nos caminhos segmented whole-block suportados. - **Exemplo**: Com `--gradient_checkpointing_interval=2` e `--gradient_checkpointing_segment_stride=4`, o SimpleTuner faz checkpoint de dois blocos, executa os dois blocos seguintes normalmente e repete. -- **Nota**: Suportado pelos caminhos segmentados de Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Z-Image e Wan. O stride deve ser pelo menos igual ao interval. Veja [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md). +- **Nota**: So tem efeito em familias de modelos que expoem suporte segmented whole-block na versao instalada do SimpleTuner. Familias nao suportadas registram um aviso e ignoram o valor. O stride deve ser pelo menos igual ao interval. Veja [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md). ### `--gradient_checkpointing_backend` @@ -305,7 +305,7 @@ Onde `foo` e seu ambiente de config — ou use `config/config.json` se nao estiv - **O que**: Faz offload para CPU das activations salvas do lado attention em modelos com fronteira attention/FFN limpa. - **Por que**: Quando transferencia e mais barata que recomputar attention, reduz VRAM sem pagar todo o custo de rematerializar attention. -- **Nota**: Pode ser ativado sozinho. Tambem pode ser combinado com qualquer checkpoint backend que o modelo suportar. Attention offload atualmente suporta Chroma, Flux, Flux.2, Krea 2, LTXVideo2, MageFlow, Wan e Z-Image; familias nao suportadas falham explicitamente. +- **Nota**: Pode ser ativado sozinho. Tambem pode ser combinado com qualquer checkpoint backend que o modelo suportar. So tem efeito em familias de modelos que expoem uma fronteira attention/FFN limpa na versao instalada do SimpleTuner; familias nao suportadas falham explicitamente. ### `--gradient_checkpointing_offload_pin_memory_max_buckets` diff --git a/documentation/OPTIONS.zh.md b/documentation/OPTIONS.zh.md index 66461d886..a72e43fd4 100644 --- a/documentation/OPTIONS.zh.md +++ b/documentation/OPTIONS.zh.md @@ -290,7 +290,7 @@ simpletuner configure config/foo/config.json - **内容**：在支持 segmented whole-block 路径的模型上，每隔 *n* 个 block 启动一个 checkpointed segment。 - **示例**：设置 `--gradient_checkpointing_interval=2` 且 `--gradient_checkpointing_segment_stride=4` 时，SimpleTuner 会 checkpoint 两个 block，然后正常运行接下来的两个 block，如此重复。 -- **说明**：Flux、Flux.2、Krea 2、LTXVideo2、MageFlow、Z-Image 和 Wan segmented 路径支持此选项。stride 必须大于或等于 interval。参见 [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md)。 +- **说明**：仅在已安装的 SimpleTuner 版本中暴露 segmented whole-block 支持的模型族上生效。不支持的模型族会记录 warning 并忽略该值。stride 必须大于或等于 interval。参见 [Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md)。 ### `--gradient_checkpointing_backend` @@ -306,7 +306,7 @@ simpletuner configure config/foo/config.json - **内容**：在有清晰 attention/FFN 边界的模型上，把 attention 侧保存的 activations 卸载到 CPU。 - **原因**：当传输比重算 attention 更便宜时，可降低 VRAM，且不需要支付完整 attention 重算成本。 -- **说明**：它可以单独启用。也可与模型支持的任意 checkpoint backend 组合。Attention offload 目前支持 Chroma、Flux、Flux.2、Krea 2、LTXVideo2、MageFlow、Wan 和 Z-Image；不支持的模型会直接报错。 +- **说明**：它可以单独启用。也可与模型支持的任意 checkpoint backend 组合。仅在已安装的 SimpleTuner 版本中暴露清晰 attention/FFN 边界的模型族上生效；不支持的模型会直接报错。 ### `--gradient_checkpointing_offload_pin_memory_max_buckets` diff --git a/simpletuner/helpers/models/common.py b/simpletuner/helpers/models/common.py index 97a47423d..f887cd41e 100644 --- a/simpletuner/helpers/models/common.py +++ b/simpletuner/helpers/models/common.py @@ -3267,13 +3267,15 @@ def _load_model(load_kwargs: dict): logger.info("Using Unsloth-style gradient checkpointing (CPU offload)") offload_attention = bool(getattr(self.config, "gradient_checkpointing_offload_attention", False)) - raw_pin_bucket_count = getattr(self.config, "gradient_checkpointing_offload_pin_memory_max_buckets", 12) - offload_pin_memory_max_buckets = 12 if raw_pin_bucket_count in (None, "", "None") else int(raw_pin_bucket_count) from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + normalize_activation_offload_pin_memory_max_buckets, set_activation_offload_pin_memory_max_buckets, set_activation_offload_prefetch_enabled, ) + offload_pin_memory_max_buckets = normalize_activation_offload_pin_memory_max_buckets( + getattr(self.config, "gradient_checkpointing_offload_pin_memory_max_buckets", 12) + ) set_activation_offload_pin_memory_max_buckets(offload_pin_memory_max_buckets) set_activation_offload_prefetch_enabled(bool(getattr(self.config, "gradient_checkpointing_offload_prefetch", False))) if offload_attention: diff --git a/simpletuner/helpers/training/default_settings/safety_check.py b/simpletuner/helpers/training/default_settings/safety_check.py index d0d374c55..a03c23b11 100644 --- a/simpletuner/helpers/training/default_settings/safety_check.py +++ b/simpletuner/helpers/training/default_settings/safety_check.py @@ -8,6 +8,7 @@ from simpletuner.helpers.training.attention_backend import AttentionBackendMode from simpletuner.helpers.training.multi_process import _get_rank as get_rank +from simpletuner.helpers.training.offloaded_gradient_checkpointer import normalize_activation_offload_pin_memory_max_buckets logger = logging.getLogger(__name__) from simpletuner.helpers.training.multi_process import should_log @@ -163,16 +164,9 @@ def safety_check(args, accelerator): "Gradient checkpointing activation prefetch requires --gradient_checkpointing_offload_attention; disabling prefetch." ) args.gradient_checkpointing_offload_prefetch = False - raw_offload_pin_bucket_count = getattr(args, "gradient_checkpointing_offload_pin_memory_max_buckets", 12) - if raw_offload_pin_bucket_count in (None, "", "None"): - args.gradient_checkpointing_offload_pin_memory_max_buckets = 12 - else: - try: - args.gradient_checkpointing_offload_pin_memory_max_buckets = int(raw_offload_pin_bucket_count) - except (TypeError, ValueError): - raise ValueError("Gradient checkpointing offload pinned bucket count must be a non-negative integer.") - if args.gradient_checkpointing_offload_pin_memory_max_buckets < 0: - raise ValueError("Gradient checkpointing offload pinned bucket count must be non-negative.") + args.gradient_checkpointing_offload_pin_memory_max_buckets = normalize_activation_offload_pin_memory_max_buckets( + getattr(args, "gradient_checkpointing_offload_pin_memory_max_buckets", 12) + ) if args.gradient_checkpointing_interval == 1: args.gradient_checkpointing_interval = None if args.gradient_checkpointing_interval is not None: diff --git a/simpletuner/helpers/training/offloaded_gradient_checkpointer.py b/simpletuner/helpers/training/offloaded_gradient_checkpointer.py index 7be079ab1..aab525323 100644 --- a/simpletuner/helpers/training/offloaded_gradient_checkpointer.py +++ b/simpletuner/helpers/training/offloaded_gradient_checkpointer.py @@ -30,6 +30,7 @@ class _PinnedBucketKey: class _RestoreView: size: tuple[int, ...] stride: tuple[int, ...] + storage_offset: int @dataclass @@ -385,8 +386,8 @@ def consume(self, record: _OffloadedActivationRecord) -> None: if not self.prefetch(record.generation, successor_id): self.prefetch_misses += 1 - def prefetch(self, generation: int, logical_id: str) -> bool: - candidates = self.records.get((generation, logical_id), ()) + def prefetch(self, generation: int, predictor_id: str) -> bool: + candidates = self.records.get((generation, predictor_id), ()) for candidate in reversed(candidates): if candidate.consumed or candidate.prefetched_tensor is not None: continue @@ -607,6 +608,20 @@ def get_activation_offload_pin_memory_max_buckets() -> int: return _PINNED_MEMORY_POOL.max_buckets +def normalize_activation_offload_pin_memory_max_buckets( + raw_value: object, default: int = _DEFAULT_PIN_MEMORY_MAX_BUCKETS +) -> int: + if raw_value in (None, "", "None"): + return default + try: + max_buckets = int(raw_value) + except (TypeError, ValueError) as exc: + raise ValueError("Gradient checkpointing offload pinned bucket count must be a non-negative integer.") from exc + if max_buckets < 0: + raise ValueError("Gradient checkpointing offload pinned bucket count must be non-negative.") + return max_buckets + + def get_activation_offload_pin_memory_stats() -> dict: """Return lifetime pinned CPU bucket statistics for activation offload.""" return _PINNED_MEMORY_POOL.snapshot() @@ -694,7 +709,8 @@ def _transfer_view(self, tensor: torch.Tensor) -> tuple[torch.Tensor, _RestoreVi flat_view = self._flat_storage_view(tensor) if flat_view is None: return tensor, None - return flat_view, _RestoreView(tuple(tensor.size()), tuple(tensor.stride())) + restore_storage_offset = tensor.storage_offset() - flat_view.storage_offset() + return flat_view, _RestoreView(tuple(tensor.size()), tuple(tensor.stride()), restore_storage_offset) def _copy_to_cpu( self, tensor: torch.Tensor @@ -776,7 +792,7 @@ def unpack(self, payload) -> torch.Tensor: if pool_key is not None: _PINNED_MEMORY_POOL.release_after_cuda_copy(pool_key, tensor, original_device) if restore_view is not None: - restored = torch.as_strided(restored, restore_view.size, restore_view.stride, 0) + restored = torch.as_strided(restored, restore_view.size, restore_view.stride, restore_view.storage_offset) return restored return tensor @@ -789,7 +805,7 @@ def _release_cpu_tensor(self, record: _OffloadedActivationRecord) -> None: def _restore_view_if_needed(self, tensor: torch.Tensor, restore_view: _RestoreView | None) -> torch.Tensor: if restore_view is None: return tensor - return torch.as_strided(tensor, restore_view.size, restore_view.stride, 0) + return torch.as_strided(tensor, restore_view.size, restore_view.stride, restore_view.storage_offset) def _unpack_record(self, record: _OffloadedActivationRecord) -> torch.Tensor: record.consumed = True diff --git a/simpletuner/helpers/utils/ramtorch.py b/simpletuner/helpers/utils/ramtorch.py index 44b9f11a3..4539467c1 100644 --- a/simpletuner/helpers/utils/ramtorch.py +++ b/simpletuner/helpers/utils/ramtorch.py @@ -797,6 +797,7 @@ def mark_ddp_ignore_params(module: nn.Module) -> int: if not ignore_names: return 0 - existing = getattr(module, "_ddp_params_and_buffers_to_ignore", set()) - module._ddp_params_and_buffers_to_ignore = set(existing) | ignore_names - return len(ignore_names) + existing = set(getattr(module, "_ddp_params_and_buffers_to_ignore", set())) + newly_added = ignore_names - existing + module._ddp_params_and_buffers_to_ignore = existing | ignore_names + return len(newly_added) diff --git a/tests/test_gradient_checkpointing_backend.py b/tests/test_gradient_checkpointing_backend.py index c7f6618cd..527e2381c 100644 --- a/tests/test_gradient_checkpointing_backend.py +++ b/tests/test_gradient_checkpointing_backend.py @@ -167,6 +167,22 @@ def test_activation_offload_pin_memory_bucket_setting(self): finally: set_activation_offload_pin_memory_max_buckets(original) + def test_activation_offload_pin_memory_bucket_count_normalization(self): + """Pinned bucket config values produce explicit validation errors.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( + normalize_activation_offload_pin_memory_max_buckets, + ) + + self.assertEqual(normalize_activation_offload_pin_memory_max_buckets(None), 12) + self.assertEqual(normalize_activation_offload_pin_memory_max_buckets(""), 12) + self.assertEqual(normalize_activation_offload_pin_memory_max_buckets("0"), 0) + self.assertEqual(normalize_activation_offload_pin_memory_max_buckets(3), 3) + + with self.assertRaisesRegex(ValueError, "non-negative integer"): + normalize_activation_offload_pin_memory_max_buckets("many") + with self.assertRaisesRegex(ValueError, "non-negative"): + normalize_activation_offload_pin_memory_max_buckets(-1) + def test_activation_offload_copy_stream_counts_are_configurable(self): """Activation offload copy stream pools expose bounded tunable widths.""" from simpletuner.helpers.training.offloaded_gradient_checkpointer import ( @@ -274,6 +290,22 @@ def test_cpu_offload_dense_noncontiguous_views_use_flat_transfer(self): self.assertEqual(tuple(restored.stride()), tuple(transposed.stride())) self.assertTrue(torch.equal(restored, transposed)) + def test_cpu_offload_dense_offset_views_restore_values(self): + """Dense views with non-zero base offsets restore from the copied transfer span.""" + from simpletuner.helpers.training.offloaded_gradient_checkpointer import CPUOffloadHooks + + hooks = CPUOffloadHooks() + base = torch.arange(20) + offset_view = torch.as_strided(base, (3, 4), (1, 3), 2) + + self.assertEqual(offset_view.storage_offset(), 2) + transfer_tensor, restore_view = hooks._transfer_view(offset_view) + restored = hooks.unpack((transfer_tensor.clone(), torch.device("cpu"), None, restore_view)) + + self.assertEqual(tuple(restored.shape), tuple(offset_view.shape)) + self.assertEqual(tuple(restored.stride()), tuple(offset_view.stride())) + self.assertTrue(torch.equal(restored, offset_view)) + def test_cpu_offload_sparse_storage_views_keep_original_layout(self): """Views with holes in storage are not flattened because storage-order transfer would lose layout.""" from simpletuner.helpers.training.offloaded_gradient_checkpointer import CPUOffloadHooks @@ -392,11 +424,11 @@ def test_activation_offload_copy_stream_pools_round_robin_by_direction(self): pinned_a = torch.empty(32, 32, pin_memory=True) pinned_b = torch.empty(32, 32, pin_memory=True) pool_key = _PINNED_MEMORY_POOL.key_for(pinned_a) - for logical_id, tensor in (("a", pinned_a), ("b", pinned_b)): + for predictor_id, tensor in (("a", pinned_a), ("b", pinned_b)): _ACTIVATION_PREFETCH_RUNTIME.register( _OffloadedActivationRecord( - logical_id=logical_id, - predictor_id=logical_id, + logical_id=predictor_id, + predictor_id=predictor_id, generation=0, tensor=tensor, original_device=torch.device("cuda"), @@ -405,7 +437,7 @@ def test_activation_offload_copy_stream_pools_round_robin_by_direction(self): ready_event=None, ) ) - self.assertTrue(_ACTIVATION_PREFETCH_RUNTIME.prefetch(0, logical_id)) + self.assertTrue(_ACTIVATION_PREFETCH_RUNTIME.prefetch(0, predictor_id)) torch.cuda.synchronize() stats = get_activation_offload_copy_stream_stats() diff --git a/tests/test_ramtorch.py b/tests/test_ramtorch.py index 238c20cbc..9ec6db2fb 100644 --- a/tests/test_ramtorch.py +++ b/tests/test_ramtorch.py @@ -332,6 +332,7 @@ def __init__(self): self.assertIn("linear_b.weight", ignore_set) self.assertIn("linear_b.bias", ignore_set) self.assertIn("ramtorch_buffer", ignore_set) + self.assertEqual(ramtorch_utils.mark_ddp_ignore_params(model), 0) def test_mark_ddp_ignore_params_skips_plain_cpu_modules(self): model = nn.Sequential(nn.Linear(2, 2), nn.LayerNorm(2))