From 41c53b8eceaafef339e8984e861f16f749e85f38 Mon Sep 17 00:00:00 2001 From: Minghong Sun <118113656+Raidriar7170@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:02:00 +0800 Subject: [PATCH 1/5] Harden private A100 SFT smoke preflight --- configs/sft-a100-smoke.example.json | 39 + ...7-13-enable-private-a100-sft-smoke-v1.html | 93 + .../.openspec.yaml | 2 + .../design.md | 72 + .../proposal.md | 35 + .../specs/supervised-contract-tuning/spec.md | 108 + .../enable-private-a100-sft-smoke-v1/tasks.md | 75 + ...baf843a527b0fb96469273c93b66ce20b62db3c.py | 2693 +++++++++++++++++ src/voice2task/cli/train.py | 75 +- .../contract_compiler_causal_audit.py | 4 + src/voice2task/training.py | 1514 ++++++++- tests/test_a100_sft_smoke.py | 234 +- tests/test_contract_compiler_causal_audit.py | 22 +- tests/test_private_a100_sft_preflight.py | 2241 ++++++++++++++ 14 files changed, 7105 insertions(+), 102 deletions(-) create mode 100644 configs/sft-a100-smoke.example.json create mode 100644 docs/human-briefs/2026-07-13-enable-private-a100-sft-smoke-v1.html create mode 100644 openspec/changes/enable-private-a100-sft-smoke-v1/.openspec.yaml create mode 100644 openspec/changes/enable-private-a100-sft-smoke-v1/design.md create mode 100644 openspec/changes/enable-private-a100-sft-smoke-v1/proposal.md create mode 100644 openspec/changes/enable-private-a100-sft-smoke-v1/specs/supervised-contract-tuning/spec.md create mode 100644 openspec/changes/enable-private-a100-sft-smoke-v1/tasks.md create mode 100644 reports/public-sample/contract-compiler-v2-causal-boundary/source-snapshots/training.978e2df42be7b1e020c5215febaf843a527b0fb96469273c93b66ce20b62db3c.py create mode 100644 tests/test_private_a100_sft_preflight.py diff --git a/configs/sft-a100-smoke.example.json b/configs/sft-a100-smoke.example.json new file mode 100644 index 0000000..a25879c --- /dev/null +++ b/configs/sft-a100-smoke.example.json @@ -0,0 +1,39 @@ +{ + "base_model_public_id": "Qwen/Qwen2.5-7B-Instruct", + "base_model_runtime_path": "", + "allow_heavy_training": false, + "local_files_only": true, + "trust_remote_code": false, + "dtype": "bfloat16", + "torch_dtype": "bfloat16", + "dataset_split": "train", + "dataset_manifest_id": "public-sample-20260619T090925Z", + "max_train_rows": 2, + "max_steps": 1, + "per_device_train_batch_size": 1, + "gradient_accumulation_steps": 1, + "max_seq_length": 1024, + "bf16": true, + "fp16": false, + "tf32": true, + "gradient_checkpointing": true, + "use_cache": false, + "low_cpu_mem_usage": true, + "save_strategy": "no", + "logging_steps": 1, + "seed": 42, + "report_to": [], + "min_free_disk_gib": 20, + "output_root": "", + "lora": { + "r": 8, + "alpha": 16, + "dropout": 0.05, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj" + ] + } +} diff --git a/docs/human-briefs/2026-07-13-enable-private-a100-sft-smoke-v1.html b/docs/human-briefs/2026-07-13-enable-private-a100-sft-smoke-v1.html new file mode 100644 index 0000000..3ad330c --- /dev/null +++ b/docs/human-briefs/2026-07-13-enable-private-a100-sft-smoke-v1.html @@ -0,0 +1,93 @@ + + + + + + Voice2Task Private A100 SFT Smoke V1 · 人类简报 + + + +
+

OpenSpec · enable-private-a100-sft-smoke-v1

+

真实训练之前，先把边界变成可证明的

+

本阶段只允许一次 Qwen2.5-7B-Instruct LoRA SFT 基础设施 smoke：formal public sample 的 train split、最多 2 条、恰好 1 optimizer step、单张显式 GPU。

+

FINAL_REVIEW_READY · PREFLIGHT_BLOCKED · A100_UNREACHABLE · REAL_SMOKE_NOT_RUN

+
+
+
+

为什么要修

+

旧路径可能在未配置输出根、未规范化路径或阻断结果退出码为 0 的情况下继续。新合同把输出目录、共享 preflight、CLI 退出码和 7B 本地加载串成同一个 fail-closed 门。

+
+
+

本阶段交付

+
    +
  • 绝对、已存在、非 symlink 的 output root；候选必须是规范化后的全新严格子目录，任何已存在候选（包括空目录）都拒绝。runner 排他创建并再次校验路径 hash。
    • +
  • sft-preflight 与真实 run 共用同一 core；core 同时产生 public report 与 immutable private context。真实 run 只消费已绑定 config、manifest、SFT、rows、model inventory 和 output facts，并在加载前重新 hash。
    • +
  • 真实 run 在任何 legacy config/metadata/dataset 读取前先进入 exception-safe shared core；阻断结果不写 candidate，ready metadata 只来自 report/context，并绑定模型 probe 已验证的同一份 facts。
    • +
  • 稳定 blocker code、单一 stdout JSON、真实非成功退出码。
    • +
  • max_seq_length 只接受 1–4096 的非布尔整数；public config facts 使用白名单。共享 core 的意外异常只返回 PREFLIGHT_INTERNAL_ERROR，不回显私有值或原始异常。
    • +
  • 所有真实训练预算采用严格 JSON 整数语义：rows 只能是整数 1/2，step、batch、accumulation 只能是整数 1；seed 必须为整数，logging_steps 必须为正整数。true 与 1.0 不再被隐式转换。
    • +
  • 严格 A100 与 Qwen2.5-7B geometry/weight inventory、BF16、单 GPU、1 step、1–2 rows 的 LoRA 参数与审计 metadata；成功还要求 finite non-bool loss、adapter config+weights，并扫描整个 run tree 排除完整模型权重。
    • +
  • DPO real mode 从调用方提供的 manifest checkout 推导 repo root，不依赖进程 cwd；无法确认 checkout 时在 config/metadata/dependency/runner 之前 fail closed。本阶段仍不授权或执行 DPO。
    • +
+
+
+

硬停止

+

不会自动完整训练，不会运行 DPO、GRPO、prediction、evaluation，不访问 lockbox，不声明模型改善，不提交私有 config、adapter、日志、缓存或私有路径。

+
+
+

clean-evaluation 事实不变

+
+
acquisition_source_status
UNAVAILABLE
+
authoritatively_bound_binding_count
0
+
human_acceptance_status
NOT_RECORDED
+
protocol_freeze_status
NOT_FROZEN
+
clean_population_status
NOT_MATERIALIZED
+
freeze_authorized
false
+
execution_readiness
false
+
+
+
+

执行门

+

只有 ignored private config 已存在、共享 preflight 返回退出码 0 且 ready=true、并确认一张空闲 A100 后，才允许恰好一次真实 smoke。任一条件缺失都必须以 blocker 停止，不得用 fixture、mock、dry-run 或小模型替代成功。

+
+
+

验证结果

+
    +
  • 最终 fresh reviewer module：123 passed；Reviewer 最终合并聚焦回归：214 passed，Final Verdict 为 Pass，无 Must Fix 或 Should Fix。
    • +
  • 提交前独立全量 pytest：1381 passed, 4 failed。四个失败均为 active OpenSpec change 触发的既有生命周期/冻结边界 guard；未增加永久 allowlist，也未绕过。
    • +
  • 提交前独立 ruff、严格 OpenSpec validation（16 passed, 0 failed）和 git diff --check 通过。
    • +
  • 用户指定 targeted mypy 命令只报告两个既有 imported-file diagnostic（slot_error_analysis.py:574、copy_backed_shadow_interface.py:357）；本次触及的 training.py 与 cli/train.py 没有诊断。
    • +
  • truth-surface check 仅因本 change 仍处于 active 生命周期而非绿；该状态保留为真实结果。
    • +
+
+
+

真实执行状态

+

ignored private config 不存在。clean-HEAD 上实际 sft-preflight 退出码为 1，stdout 为单个 JSON document、stderr 为 0 字节，唯一 blocker 为 CONFIG_FILE_MISSING。既有授权 A100 alias 的只读占用探测另行重试三次，均连接超时，因此没有取得 GPU 或本地 7B 模型事实，也没有执行真实 1-step smoke。

+

没有 adapter、训练日志、缓存、prediction 或 evaluation 产物进入 Git；没有完整训练、DPO、GRPO、lockbox 访问或模型改进声明。

+
+
+

更新时间：2026-07-13 · 当前为 final-review-ready 的代码交付；共享 preflight 的唯一 blocker 为 CONFIG_FILE_MISSING，A100 alias 同时不可达。

+ + diff --git a/openspec/changes/enable-private-a100-sft-smoke-v1/.openspec.yaml b/openspec/changes/enable-private-a100-sft-smoke-v1/.openspec.yaml new file mode 100644 index 0000000..b119b63 --- /dev/null +++ b/openspec/changes/enable-private-a100-sft-smoke-v1/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-13 diff --git a/openspec/changes/enable-private-a100-sft-smoke-v1/design.md b/openspec/changes/enable-private-a100-sft-smoke-v1/design.md new file mode 100644 index 0000000..5f7616f --- /dev/null +++ b/openspec/changes/enable-private-a100-sft-smoke-v1/design.md @@ -0,0 +1,72 @@ +## Context + +The existing SFT entrypoint has three fail-open boundaries: output paths are checked before canonicalization and may be unrooted, blocked CLI results can exit successfully, and a real run can bypass a separately performed readiness check. This change is an infrastructure-only prerequisite for one private, explicitly selected A100, one or two formal-public train rows, and exactly one optimizer step. It does not change the clean-evaluation truth surface or authorize broader model work. + +## Goals / Non-Goals + +**Goals:** + +- Make output placement fail closed against missing roots, relative paths, traversal, symlinks, non-empty destinations, tracked repository paths, and preflight-to-load drift. +- Expose one shared, machine-readable, public-safe SFT preflight used by both the CLI and the real training path before model-weight loading. +- Wire a local-only Qwen2.5-7B-Instruct BF16 LoRA smoke configuration through model loading, training arguments, metadata, and postconditions. +- Return one JSON document and an exit status that accurately distinguishes success from blocked, skipped, unavailable, or failed execution. + +**Non-Goals:** + +- Full or 282-row SFT, DPO, GRPO, prediction, evaluation, lockbox access, public adapter release, deployment, or any model-improvement/readiness claim. +- Relaxing or rewriting any clean-evaluation blocker. +- Discovering private model or output paths outside the explicitly supplied ignored config. + +## Decisions + +### 1. One structured preflight report is the execution gate + +`_run_sft_preflight_core` is the exception-safe shared gate and returns two products from one execution: a schema-versioned public JSON-compatible report and, only when ready, a frozen private execution context. The context binds canonical config, manifest, SFT, selected rows, the exact model facts returned by the validated probe, and output facts by hashes and immutable JSON snapshots. The public `run_sft_preflight` wrapper discards the private context. Real `run_sft(..., run_training=True)` calls this shared gate before any legacy config, metadata, or dataset read; blocked execution returns a minimal no-write result, while ready metadata is constructed only from the bound report and context. The real runner rehashes config, manifest, the complete SFT file, and model inventory immediately before model/tokenizer loading. Dataset selection and model-inventory binding do not run on second, weaker paths around preflight. + +Alternative considered: keep a lightweight CLI preflight and separate runtime guards. Rejected because the paths can drift and the real run could become less strict than the audit command. + +### 2. Output authorization is based on canonical filesystem identity + +The output root must already exist, be absolute, and not itself be a symlink. The absolute candidate is rejected if it equals the root, if any existing component is a symlink, if its resolved location is not a strict descendant of the resolved root, if it already exists (including an empty directory), or if it falls within the repository. Writability and free space are checked on the nearest existing parent. The real runner claims the final directory with exclusive creation, then accepts only the expected existing-empty result from the same policy and the bound path hash. DPO real mode derives its repository root from the supplied manifest checkout with a cwd-independent fail-closed helper, passes that root explicitly to its runner, uses the same generic gate before metadata/dependency work, and repeats it in the runner. This is defensive hardening, not authorization to execute DPO. + +Alternative considered: lexical `Path.relative_to`. Rejected because it does not defend against `..`, symlink redirection, or later path substitution. + +### 3. Blockers are stable codes; diagnostics are sanitized sections + +Exceptions and machine-specific values are converted to enumerated blocker codes. The shared top-level gate catches unexpected internal exceptions and returns the complete blocked schema with only `PREFLIGHT_INTERNAL_ERROR`; it never serializes the exception. Public config facts are an explicit whitelist of approved constants, booleans, bounded numbers, and hashes. Public JSON includes hashes, booleans, counts, versions, GPU model/capability/memory, and file-name/size inventories, but not private paths, hostname, IP, GPU UUID, secrets, environment values, arbitrary config strings, or raw exception text. + +Alternative considered: surface exception strings for convenience. Rejected because they are unstable and can leak private runtime details. + +### 4. Dataset and objective validation precede model weights + +The canonical formal manifest and complete SFT JSONL are hashed. After binding the current manifest ID and exact SFT entry, parsing reads from the beginning, requires `split=train` for every selected record, stops immediately after the configured one or two rows, rejects duplicates/empty selections and `train_source_ids`, and hashes the ordered selected IDs. Smoke budget scalars use exact JSON integer semantics: `max_train_rows` is a non-boolean integer in `{1, 2}`; `max_steps`, `per_device_train_batch_size`, and `gradient_accumulation_steps` are non-boolean integer `1`; `max_seq_length` is a non-boolean integer from 1 through 4096; `seed` is a non-boolean integer; and `logging_steps` is a positive non-boolean integer. Those exact serialized rows are carried in the private context. A local tokenizer constructs the same assistant-only records used by training and verifies mask, target, length, and tensor-shape invariants before the 7B model is loaded. + +### 5. The trainer owns one explicitly visible GPU + +The caller must set `CUDA_VISIBLE_DEVICES`, exactly one CUDA device must be visible, the device name must identify an A100, BF16 must be supported, compute capability must meet the BF16 threshold, and memory must be at least 35 GiB. The private model must match Qwen2.5-7B-Instruct geometry and architecture, expose at least 12 GiB of weight inventory, use `dtype=torch_dtype=bfloat16`, `trust_remote_code=false`, local-only loading, and a bounded q/k/v/o LoRA target set. Model loading does not use `device_map="auto"`; Trainer/Accelerate performs placement. Gradient checkpointing forces `use_cache=false`. A missing pad token uses only the tokenizer EOS token after validation. + +### 6. Smoke completion is a narrow postcondition + +`SMOKE_COMPLETED` requires exit success, `training_completed`, one true integer optimizer step, an observed row count exactly equal to configured `max_train_rows`, a finite non-boolean numeric loss, and both non-empty `adapter_config.json` and adapter weights. The entire run tree is scanned for full base-model files, indexes, or shards rather than checking only the adapter directory. Metadata records the bounded budget and public-safe provenance. Any mismatch is a failed run, not a partial success. + +## Risks / Trade-offs + +- [Filesystem state changes after preflight] -> Rehash bound inputs, repeat policy, claim the final output directory exclusively, and post-claim revalidate before local tokenizer/model loading. Parent-component mutation after the final check would require a directory-fd based redesign to eliminate completely. +- [Tokenizer loading itself can be expensive] -> Keep it local-only and load it before model weights because objective validity depends on the real tokenizer. +- [Git status includes generated private ignored config] -> Inspect tracked changes only and record commit SHA; ignored files never make readiness dirty. +- [Package/GPU APIs vary between versions] -> Keep dependency and GPU probes injectable for unit tests and convert failures to stable blockers. +- [A smoke can be mistaken for evaluation] -> Preserve all seven clean-evaluation blockers and use only `SMOKE_COMPLETED`. + +## Migration Plan + +1. Add failing unit/CLI tests for the new contracts. +2. Add the shared preflight and output policy, then wire the CLI and runtime path. +3. Add bounded Qwen2.5-7B runtime options and the disabled public example. +4. Run focused and repository-wide static/test/OpenSpec/truth checks. +5. Run a private preflight only if an explicitly supplied ignored config exists; run the single authorized smoke only if it returns `ready=true` on an idle explicitly selected A100. + +Rollback is a code revert. Private outputs and configs remain ignored and are never migration inputs. + +## Open Questions + +None. The requested budget, model identity, data split, blocker vocabulary, and prohibited actions are explicit. diff --git a/openspec/changes/enable-private-a100-sft-smoke-v1/proposal.md b/openspec/changes/enable-private-a100-sft-smoke-v1/proposal.md new file mode 100644 index 0000000..cf63224 --- /dev/null +++ b/openspec/changes/enable-private-a100-sft-smoke-v1/proposal.md @@ -0,0 +1,35 @@ +## Why + +The current SFT entrypoint can accept an unconfigured or path-escaping output directory, can return exit code 0 for blocked work, and has no single auditable preflight that proves a real private Qwen2.5-7B one-step run is bounded before model weights are loaded. This change closes those fail-open boundaries while preserving every existing clean-evaluation blocker and claim limitation. + +## What Changes + +- **BREAKING**: Require every `sft --run-training` invocation to use an existing absolute non-symlink `output_root` and an absolute child `output_dir` that does not already exist, remains under the resolved root, and stays outside repository paths. +- Add stable, sanitized output-policy blocker codes, check the nearest existing parent, claim the final directory exclusively, and repeat the same policy immediately before model loading so preflight-to-run path drift fails closed. Apply the same generic real-training output gate to DPO without authorizing DPO execution. +- Add `voice2task-train sft-preflight` with one shared internal core that returns a public report plus a private immutable execution context covering Git state, bounded config, dependencies, explicit one-GPU BF16 runtime, formal public train-row selection, assistant-only objective construction, local-only model identity/fingerprints, and output capacity. Real SFT consumes only the bound context and rehashes mutable inputs before model loading. +- Make blocked, skipped, unavailable, and runtime-failed CLI results non-zero while preserving one JSON document on stdout and no competing result JSON on stderr. +- Wire private Qwen2.5-7B-Instruct BF16/local-only LoRA SFT options through tokenizer, model loading, `TrainingArguments`, gradient checkpointing, adapter writing, and metadata, with exact Qwen2.5-7B geometry, A100, minimum weight-inventory size, `trust_remote_code=false`, and bounded LoRA checks. +- Add a public-safe disabled-by-default example config; only an ignored private config may contain real model/output paths and enable heavy training. +- Permit at most one explicitly selected A100, one or two formal-public train rows, and exactly one optimizer step for `SMOKE_COMPLETED`; do not authorize broader execution. +- Preserve the clean-evaluation truth surface exactly: `acquisition_source_status=UNAVAILABLE`, `authoritatively_bound_binding_count=0`, `human_acceptance_status=NOT_RECORDED`, `protocol_freeze_status=NOT_FROZEN`, `clean_population_status=NOT_MATERIALIZED`, `freeze_authorized=false`, and `execution_readiness=false`. +- Add no-network/no-GPU unit tests for output traversal/symlinks/TOCTOU, direct-runner/DPO bypasses, CLI exit semantics, dependency/GPU/model/data/objective/output preflight blockers, immutable context drift, strict smoke postconditions, and model/training option propagation. + +Non-goals are full or 282-row SFT, DPO, first-phase GRPO, prediction, evaluation, lockbox or clean-population access, generic chat fine-tuning, skill routing, GUI action-policy learning, public full-corpus release, checkpoint or adapter release, model-improvement claims, clean-evaluation readiness, production readiness, merge, deploy, or release. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `supervised-contract-tuning`: Strengthen the real SFT execution contract with a shared fail-closed private A100 smoke preflight, safe output policy, bounded Qwen2.5-7B runtime options, stable CLI outcomes, and truthful smoke metadata. + +## Impact + +- Code: `src/voice2task/training.py`, `src/voice2task/cli/train.py`. +- Configuration: new public-safe `configs/sft-a100-smoke.example.json`; an optional ignored private override remains outside Git. +- Tests: focused training/preflight/CLI tests that never require CUDA, load real model weights, or access the network. +- Runtime dependencies: checks now include Python, `torch`, `accelerate`, `datasets`, `peft`, `transformers`, `trl`, and `pip check`; no dependency is added to the project core install. +- Evidence boundary: infrastructure smoke success proves only a real gradient update, one optimizer step, adapter write, and training-path viability. It does not change clean-evaluation readiness or any model-quality claim. diff --git a/openspec/changes/enable-private-a100-sft-smoke-v1/specs/supervised-contract-tuning/spec.md b/openspec/changes/enable-private-a100-sft-smoke-v1/specs/supervised-contract-tuning/spec.md new file mode 100644 index 0000000..96bd13d --- /dev/null +++ b/openspec/changes/enable-private-a100-sft-smoke-v1/specs/supervised-contract-tuning/spec.md @@ -0,0 +1,108 @@ +## MODIFIED Requirements + +### Requirement: Run A100 public-sample SFT smoke +The system SHALL provide a bounded, opt-in, local-only Qwen2.5-7B-Instruct LoRA SFT smoke workflow that uses an explicitly selected single GPU, trains one or two rows from the current formal public sample `train` split for exactly one optimizer step, and writes private outputs only beneath an approved output root. + +#### Scenario: Launch smoke with explicit opt-in +- **WHEN** a developer launches SFT with `--run-training`, a private ignored config whose `allow_heavy_training` is `true`, and a shared preflight result whose `ready` is `true` +- **THEN** the system repeats the shared preflight before model weights load, uses only the configured local Qwen2.5-7B-Instruct runtime, and records public-safe Git, config, dataset, model, dependency, GPU, output, objective, and actual-budget metadata + +#### Scenario: Bound successful execution +- **WHEN** the authorized smoke completes +- **THEN** it reports `SMOKE_COMPLETED`, exactly one observed optimizer step, one or two training rows, finite loss, and non-empty adapter files without saving or copying full base-model weights + +#### Scenario: Reject accidental heavy training +- **WHEN** a developer launches the training path without `--run-training`, with `allow_heavy_training` false, or with any budget outside the approved smoke bounds +- **THEN** the system does not load model weights or start training and returns a non-success training status with a stable blocker where applicable + +#### Scenario: Keep remote evidence private by default +- **WHEN** a preflight or smoke produces private paths, configs, adapters, caches, or raw logs +- **THEN** those values and artifacts remain out of Git and public JSON exposes only approved hashes, counts, versions, GPU facts, and file-name/size inventory + +#### Scenario: Preserve clean-evaluation blockers +- **WHEN** infrastructure preflight or the authorized smoke succeeds +- **THEN** the system leaves `acquisition_source_status=UNAVAILABLE`, `authoritatively_bound_binding_count=0`, `human_acceptance_status=NOT_RECORDED`, `protocol_freeze_status=NOT_FROZEN`, `clean_population_status=NOT_MATERIALIZED`, `freeze_authorized=false`, and `execution_readiness=false` + +## ADDED Requirements + +### Requirement: Enforce canonical SFT output boundaries +Every real SFT run MUST require an existing absolute non-symlink output root and an absolute, not-yet-existing, strict descendant output directory after canonical resolution. The system MUST reject traversal, any existing symlink component, final-directory symlinks, root-external destinations, every existing candidate including an empty directory, and repository locations with stable blocker codes. It MUST check the nearest existing parent, claim the final directory exclusively, and revalidate the bound path before model and tokenizer loading. + +#### Scenario: Reject an unsafe output path +- **WHEN** the root is missing, relative, symlinked, or the candidate is relative, equal to the root, escaping, symlinked, already exists, or is inside the repository +- **THEN** the system returns an output-policy blocker without creating or writing metadata into the unsafe candidate + +#### Scenario: Accept a new canonical child +- **WHEN** the existing absolute root is not a symlink and the absolute candidate is a new writable strict descendant with no symlink components and sufficient free space +- **THEN** output policy reports the canonical path hash as ready without exposing the path + +#### Scenario: Detect preflight-to-load drift +- **WHEN** filesystem state makes a previously accepted output path unsafe before tokenizer or model loading +- **THEN** the repeated shared policy blocks execution before model weights load + +### Requirement: Provide one shared SFT preflight +The system MUST expose `sft-preflight` and MUST use the same internal preflight core from real SFT execution before loading model weights. That core MUST produce the public report plus an immutable private execution context binding canonical inputs, selected rows, model inventory, and output facts. Real training MUST consume the bound context rather than reselecting data from caller arguments and MUST rehash mutable inputs before model loading. The single public JSON result MUST use schema `voice2task-sft-preflight-v1`, report `ready`, `status`, `blockers`, and the `git`, `config`, `dataset`, `model`, `runtime`, `gpu`, `output`, and `objective` sections, and MUST not access the network. + +#### Scenario: Report a ready bounded smoke +- **WHEN** Git tracked state is clean and includes the change, the ignored non-symlink private config is exactly smoke-bounded, dependencies and `pip check` pass, one explicit BF16-capable >=35 GiB A100 is visible, exact local Qwen2.5-7B model/tokenizer identity and >=12 GiB weight inventory pass, selected formal train rows and assistant-only labels pass, and output policy passes +- **THEN** preflight returns exit code 0 with `ready=true`, an empty blocker list, stable fingerprints, hashes, versions, counts, and public-safe facts + +#### Scenario: Report blocked readiness safely +- **WHEN** any required Git, config, dependency, GPU, model, dataset, objective, or output condition fails +- **THEN** preflight returns non-zero with `ready=false` and only stable enumerated blocker codes, without raw exceptions or private runtime values + +#### Scenario: Fail closed on an unexpected preflight exception +- **WHEN** any internal shared-preflight operation raises an unexpected exception +- **THEN** both `sft-preflight` and real SFT return the complete blocked preflight schema with only `PREFLIGHT_INTERNAL_ERROR`, do not serialize exception text or private values, and do not write the candidate output + +#### Scenario: Consume only the bound ready context +- **WHEN** real SFT reaches a ready shared preflight result +- **THEN** it builds execution metadata only from the bound report and immutable context, without rereading legacy config, manifest, or dataset inputs around the gate, and binds the exact model facts produced by the validated model probe + +### Requirement: Keep smoke row selection and objective exact +The shared preflight MUST require the canonical current formal manifest and exact SFT entry, hash both complete files, then parse only the first exactly `max_train_rows` ordered `split=train` records where that value is a non-boolean integer in `{1, 2}` and stop. It MUST reject JSON booleans and floats for `max_train_rows`, `max_steps`, `per_device_train_batch_size`, and `gradient_accumulation_steps`; the latter three MUST be the non-boolean integer `1`. It MUST also require `seed` to be a non-boolean integer and `logging_steps` to be a positive non-boolean integer before those values reach real `TrainingArguments`. It MUST reject empty or duplicate row IDs and every implicit selector such as `train_source_ids`, bind those exact rows into the private execution context, and validate every selected row with the same real-tokenizer assistant-only record builder used for training. + +#### Scenario: Validate assistant-only records +- **WHEN** a selected row is encoded +- **THEN** `max_seq_length` has already been validated as a non-boolean integer from 1 through 4096, prompt labels are all `-100`, at least one assistant target label is trainable, `input_ids`, `attention_mask`, and `labels` lengths match, and length does not exceed `max_seq_length` + +#### Scenario: Reject data or selection drift +- **WHEN** the manifest/SFT hash or manifest ID does not match its declared value, train selection changes, rows are empty or duplicated, labels are invalid, or sequence length is exceeded +- **THEN** preflight fails with a stable dataset or objective blocker before model weights load + +#### Scenario: Reject numeric type coercion +- **WHEN** a smoke budget, seed, or logging field uses a JSON boolean or float where an integer is required, or `logging_steps` is not positive +- **THEN** preflight returns `CONFIG_NOT_SMOKE_BOUNDED`; a malformed `max_train_rows` additionally returns `TRAIN_ROW_SELECTION_INVALID` and selects zero rows + +### Requirement: Load the private 7B runtime without downloads +The real SFT path MUST load tokenizer and model from the configured existing private `base_model_runtime_path`, require public identity and exact geometry for `Qwen/Qwen2.5-7B-Instruct`, require at least 12 GiB of local weight inventory, pass `local_files_only=true`, `trust_remote_code=false`, BF16 dtype, and `low_cpu_mem_usage`, and MUST NOT use `device_map="auto"`. It MUST propagate `bf16`, `fp16`, `tf32`, `gradient_checkpointing`, `use_cache`, `seed`, `max_steps`, `max_train_rows`, batch size, accumulation, save strategy, and logging steps into training behavior and metadata. + +#### Scenario: Configure the bounded local model +- **WHEN** the ready smoke loads the Qwen2.5-7B-Instruct runtime +- **THEN** both tokenizer and model use local-only loading, the model uses BF16 and low-CPU-memory loading, Trainer/Accelerate owns placement on the one visible GPU, gradient checkpointing sets `model.config.use_cache=false`, and a missing pad token uses only a validated EOS fallback + +#### Scenario: Reject unresolved local model +- **WHEN** the private model path is absent, public identity differs, local config/tokenizer cannot load, or required stable inventory cannot be produced +- **THEN** preflight returns a stable model blocker without downloading or loading model weights + +### Requirement: Apply the real-training output gate to DPO +The system MUST derive DPO's repository root from the supplied manifest checkout without depending on the process working directory, fail closed when that checkout cannot be established, apply the same canonical output-root policy to DPO `--run-training` before metadata or dependency work, pass the derived root explicitly to the runner, and exclusively claim and revalidate the output directory again before imports or model loading. This defensive gate MUST NOT be interpreted as authorization to execute DPO in this change. + +#### Scenario: Block a DPO output bypass +- **WHEN** DPO real mode receives a missing, escaping, symlinked, existing, or repository-local output destination +- **THEN** it returns a stable output-policy failure before dependency imports or model loading and does not write into the unsafe destination + +#### Scenario: Resolve DPO policy from the manifest checkout +- **WHEN** DPO real mode is invoked from an unrelated process working directory +- **THEN** output policy still uses the repository containing the supplied manifest; if that repository cannot be resolved, execution fails closed before config, metadata, dependency, or runner work + +### Requirement: Return truthful CLI exit status and one JSON result +The training CLI MUST write exactly one JSON document to stdout. Dry-run success, ready preflight, and `training_status=training_completed` MUST exit 0. Skipped-by-config, unavailable, output-policy-blocked, preflight-blocked, and runtime-exception results MUST exit non-zero. Stderr MUST NOT contain a second result JSON. + +#### Scenario: Map successful results +- **WHEN** dry-run succeeds, preflight is ready, or training completes +- **THEN** the CLI emits one JSON document and exits 0 + +#### Scenario: Map non-success results +- **WHEN** training is skipped, unavailable, output-policy-blocked, preflight-blocked, or raises at runtime +- **THEN** the CLI emits one sanitized JSON document and exits non-zero without a competing result document on stderr diff --git a/openspec/changes/enable-private-a100-sft-smoke-v1/tasks.md b/openspec/changes/enable-private-a100-sft-smoke-v1/tasks.md new file mode 100644 index 0000000..2fa00c9 --- /dev/null +++ b/openspec/changes/enable-private-a100-sft-smoke-v1/tasks.md @@ -0,0 +1,75 @@ +## 1. Contract and Test Harness + +- [x] 1.1 Add focused failing output-policy tests for missing/relative roots, relative/equal/escaping candidates, parent/final symlinks, non-empty targets, valid new child, tracked paths, and pre-load drift +- [x] 1.2 Add focused failing CLI tests for one-JSON stdout and zero/non-zero exit mapping across dry-run, preflight, completed, skipped, unavailable, output-blocked, preflight-blocked, and runtime-exception states +- [x] 1.3 Add focused failing preflight tests for config, dependency, GPU, model, dataset, objective, Git, and output ready/blocked outcomes without network, GPU, or real model loading + +## 2. Shared Preflight and Output Policy + +- [x] 2.1 Implement canonical fail-closed output policy with stable sanitized blockers and a no-write unsafe path contract +- [x] 2.2 Implement schema-versioned shared `run_sft_preflight` covering Git, config, dependencies, GPU, dataset, objective, local model, and output facts +- [x] 2.3 Reuse the shared preflight and repeat output policy immediately before tokenizer/model loading to close preflight-to-load drift + +## 3. CLI and Runtime Wiring + +- [x] 3.1 Add `sft-preflight` arguments, one-JSON rendering, sanitized runtime failure handling, and truthful exit-code mapping +- [x] 3.2 Wire local-only Qwen2.5-7B BF16/low-memory loading, EOS pad fallback, gradient checkpointing/use-cache policy, bounded TrainingArguments, and metadata +- [x] 3.3 Enforce smoke postconditions and `SMOKE_COMPLETED` without full-weight, prediction, evaluation, or clean-readiness claims +- [x] 3.4 Add disabled public-safe `configs/sft-a100-smoke.example.json` and verify any private config remains ignored and untracked + +## 4. Verification and Gated Execution + +- [x] 4.1 Run focused RED/GREEN tests and the full `PYTHONPATH=src pytest -q` suite + - Reviewer module: `95 passed`; existing A100 module: `52 passed`; combined training-focused regression: `176 passed`; CLI matrix: `12 passed`. + - Full suite: `1353 passed, 4 failed`. The four failures are preserved lifecycle/frozen-boundary guards caused solely by this active OpenSpec change; they were not bypassed. +- [x] 4.2 Run `ruff`, targeted `mypy`, strict OpenSpec validation, truth-surface check, public-leak/ignore checks, and `git diff --check` + - `ruff`, strict OpenSpec validation (`16 passed, 0 failed`), public-leak/ignore checks, and `git diff --check` passed. + - Targeted `mypy` reports only the two pre-existing imported-file diagnostics; the isolated touched-file result matches the four-error origin/main baseline. + - Truth-surface check remains non-green only because `enable-private-a100-sft-smoke-v1` is correctly active. +- [x] 4.3 Run private `sft-preflight` only if the authorized ignored config exists; run exactly one real one-step smoke only after `ready=true` on one explicitly selected idle A100, otherwise record the exact blocker and stop + - The ignored private config is absent. On a clean committed HEAD, actual CLI preflight exited `1` with one JSON document, empty stderr, and the sole blocker `CONFIG_FILE_MISSING`. + - A read-only occupancy probe against the existing authorized A100 alias timed out three times, so no GPU or local-model facts were obtained and no training followed. +- [x] 4.4 Update the Human Brief with final verification and execution state while preserving all clean-evaluation blockers + +## 5. Reviewer Hardening + +- [x] 5.1 Return one public preflight report plus an immutable private execution context; bind exact rows and rehash config, manifest, SFT, and model inventory before loading +- [x] 5.2 Tighten canonical formal-data, A100/Qwen geometry, private-config, output claim/recheck, and DPO output-bypass contracts +- [x] 5.3 Tighten public-safe exception handling and smoke postconditions for finite loss, exact rows, adapter config/weights, and whole-run full-weight scanning +- [x] 5.4 Add direct bypass/regression coverage and update OpenSpec/Human Brief truth surfaces without bypassing active-change lifecycle guards + - Reviewer module: `95 passed`; existing A100 module: `52 passed`; combined training-focused regression: `176 passed`; CLI boundary: `12 passed`. + - Second-review-ready final rerun: full pytest `1353 passed, 4 failed` (only preserved active-change lifecycle/frozen-boundary guards); Ruff passed; strict OpenSpec `16 passed, 0 failed`; `git diff --check` passed; targeted Mypy contains zero touched-file diagnostics and only two pre-existing imported-module errors. + +## 6. Final Reviewer Closure + +- [x] 6.1 Make real SFT invoke the safe shared core before all legacy reads, return a minimal no-write blocked result, and build ready metadata only from the immutable context +- [x] 6.2 Bind the exact model facts returned by the validated probe, derive DPO repository identity from the supplied manifest checkout, and pass that identity explicitly to the runner +- [x] 6.3 Validate `max_seq_length` as a non-boolean integer in `1..4096`, whitelist public config facts, and convert every unexpected shared-core exception to `PREFLIGHT_INTERNAL_ERROR` +- [x] 6.4 Add direct regressions for legacy-read bypass, model-fact mutation, unrelated DPO cwd/fail-closed checkout, malformed private config values, and both public/real preflight exception paths + - Fresh focused result: reviewer module `108 passed`; reviewer + existing A100 + formatting regression `189 passed`. +- [x] 6.5 Run final Ruff, targeted Mypy, strict OpenSpec validation, and `git diff --check`; then update the Human Brief to final-review-ready without rerunning the repository-wide suite + - Ruff passed; strict OpenSpec validation passed `16/16`; `git diff --check` passed. + - The exact targeted Mypy command reports only the two pre-existing imported-module diagnostics in `slot_error_analysis.py:574` and `copy_backed_shadow_interface.py:357`; neither touched training file has a diagnostic. + - Per final-review scope, the repository-wide pytest suite was not rerun; the prior second-review result remains recorded separately and is not presented as fresh third-review evidence. + +## 7. Exact Integer Budget Closure + +- [x] 7.1 Add parameterized RED regressions for JSON booleans and floats in `max_train_rows`, `max_steps`, per-device batch size, and gradient accumulation, plus unsafe `seed` and `logging_steps` encodings +- [x] 7.2 Require exact non-boolean integers for all smoke budget fields, an integer `seed`, and positive integer `logging_steps`; malformed row limits select zero rows +- [x] 7.3 Run the final focused and static verification without full-suite, GPU, commit, or archive actions + - Fresh reviewer module: `123 passed`; final Reviewer combined regression: `214 passed`; final verdict `Pass` with no Must Fix or Should Fix. + - Ruff passed; strict OpenSpec validation passed `16/16`; `git diff --check` passed. + - The exact targeted Mypy command still reports only the two pre-existing imported-module diagnostics; neither touched training file has a diagnostic. + +## 8. Independent Pre-Commit Verification + +- [x] 8.1 Refresh `origin/main` and confirm the bounded change still starts at `4c99e8d6cf1fe2782b2760e36437ff5fae164dff` +- [x] 8.2 Run the repository-wide suite after the exact-integer closure + - Fresh result: `1381 passed, 4 failed`; the four failures are only the preserved active-change lifecycle/frozen-boundary guards. +- [x] 8.3 Run all requested static and truth-surface commands + - Ruff, strict OpenSpec validation (`16/16`), and `git diff --check` passed. + - Truth-surface check exits `1` only because this bounded change remains active. + - Exact targeted Mypy exits `1` only for the two pre-existing imported-module diagnostics; neither touched training file has a diagnostic. +- [x] 8.4 Exercise the real execution gate from a clean committed HEAD and stop fail-closed + - Shared CLI preflight: exit `1`, exactly one stdout JSON document, empty stderr, sole blocker `CONFIG_FILE_MISSING`. + - Authorized A100 alias: three read-only occupancy probes timed out; no remote writes, model-path search, training, prediction, or evaluation occurred. diff --git a/reports/public-sample/contract-compiler-v2-causal-boundary/source-snapshots/training.978e2df42be7b1e020c5215febaf843a527b0fb96469273c93b66ce20b62db3c.py b/reports/public-sample/contract-compiler-v2-causal-boundary/source-snapshots/training.978e2df42be7b1e020c5215febaf843a527b0fb96469273c93b66ce20b62db3c.py new file mode 100644 index 0000000..48e5b71 --- /dev/null +++ b/reports/public-sample/contract-compiler-v2-causal-boundary/source-snapshots/training.978e2df42be7b1e020c5215febaf843a527b0fb96469273c93b66ce20b62db3c.py @@ -0,0 +1,2693 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import inspect +import json +import re +import sys +from collections.abc import Callable +from datetime import datetime, timezone +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any, cast + +from voice2task.copy_backed_prediction_shadow_hook import ( + PredictionShadowHookOutcome, + load_prediction_shadow_policy_snapshot, + prediction_shadow_config_error_code, + run_prediction_shadow_hook, + shadow_config_from_mapping, + sidecar_path_conflicts, + summarize_prediction_shadow_outcomes, +) +from voice2task.formatting import ( + FORMATTING_POLICY, + UNIFIED_GOLD_FREE_PROMPT_POLICY_ID, + PredictionInput, + format_dpo_pair, + format_schema_retry_prompt_text, + format_sft_prediction_prompt, + format_sft_training_text, + prediction_output_boundary_summary, + prediction_prompt_constraint_summary, + prompt_constraint_summary, + schema_retry_template_boundary_summary, +) +from voice2task.io import read_json, read_jsonl, write_json +from voice2task.schemas import ( + PRIVATE_IP_RE, + PRIVATE_PATH_RE, + ROUTES, + SECRET_RE, + TASK_TYPES, + DPOPair, + SFTDatasetRow, + as_contract, + canonical_contract_json, + validate_contract_status, +) + + +def _load_config(config_path: Path) -> dict[str, Any]: + if config_path.suffix != ".json": + raise ValueError("bootstrap training configs are JSON in this phase") + return read_json(config_path) + + +def _resolve_manifest_file(manifest_path: Path, value: Any) -> Path | None: + if not isinstance(value, str) or not value: + return None + candidate = Path(value) + if candidate.is_absolute() and candidate.exists(): + return candidate + relative_candidate = manifest_path.parent / candidate.name + if relative_candidate.exists(): + return relative_candidate + if candidate.exists(): + return candidate + return None + + +def _manifest_load_summary(manifest_path: Path, stage: str) -> dict[str, Any]: + manifest = read_json(manifest_path) + files = manifest.get("files", {}) + if not isinstance(files, dict): + files = {} + dataset_key = "dpo" if stage == "dpo" else "sft" + dataset_path = _resolve_manifest_file(manifest_path, files.get(dataset_key)) + loaded_rows = len(read_jsonl(dataset_path)) if dataset_path is not None else 0 + return { + "manifest_id": str(manifest.get("manifest_id", manifest_path.stem)), + "manifest_counts": manifest.get("counts", {}), + "dataset_key": dataset_key, + "dataset_path": dataset_path.as_posix() if dataset_path is not None else None, + "loaded_rows": loaded_rows, + } + + +def _heavy_training_allowed(config: dict[str, Any]) -> bool: + return bool(config.get("allow_heavy_training")) + + +def _sanitized_package_versions() -> dict[str, str]: + versions = {"python": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"} + for package in ("accelerate", "datasets", "peft", "transformers", "trl"): + try: + versions[package] = version(package) + except PackageNotFoundError: + versions[package] = "not_installed" + return versions + + +def _gpu_selection_policy(config: dict[str, Any]) -> dict[str, str]: + policy = config.get("gpu_selection_policy") + if not isinstance(policy, str) or not policy: + policy = "not_selected_locally_select_idle_gpu_for_a100" + identifier_policy = config.get("gpu_identifier_policy") + if not isinstance(identifier_policy, str) or not identifier_policy: + identifier_policy = "policy_only_no_host_ip_or_gpu_uuid" + return { + "policy": policy, + "identifier_policy": identifier_policy, + } + + +PRIVATE_PATH_PREFIXES = ("/mnt/data/", "/Users/", "/root/", "/tmp/", "/private/") +_MNT_DATA_PREFIX = "/" + "mnt/data" +PRIVATE_DECODED_PATH_RE = re.compile(rf"({_MNT_DATA_PREFIX}/[^\s\"')]+)") +PRIVATE_METADATA_PATH_RE = re.compile(r"(/(?:mnt/data|Users|root|tmp|private)/[^\s\"')]+)") +MARKDOWN_FENCE_SUPPRESSION_TOKEN_SOURCES = ("```", "```json", "```JSON") + + +def _public_display_path(value: Path | str, placeholder: str) -> str: + raw = value.as_posix() if isinstance(value, Path) else str(value) + if any(raw.startswith(prefix) for prefix in PRIVATE_PATH_PREFIXES): + return placeholder + return raw + + +def _public_display_model(value: Any) -> str: + if isinstance(value, str) and value: + return _public_display_path(value, "") + return "unknown" + + +def _public_base_model(config: dict[str, Any]) -> str: + return str(config.get("base_model_public_id") or config.get("base_model") or "unknown") + + +def _runtime_base_model(config: dict[str, Any]) -> str: + return str(config.get("base_model_runtime_path") or config.get("base_model")) + + +def _public_display_artifact_path(value: Path, placeholder: str) -> str: + raw = value.as_posix() + if any(raw.startswith(prefix) for prefix in PRIVATE_PATH_PREFIXES): + return placeholder + return raw + + +def _sanitize_training_metadata_value(value: Any) -> Any: + if isinstance(value, str): + sanitized = PRIVATE_METADATA_PATH_RE.sub("", value) + sanitized = PRIVATE_PATH_RE.sub("", sanitized) + sanitized = PRIVATE_IP_RE.sub("", sanitized) + return SECRET_RE.sub("", sanitized) + if isinstance(value, dict): + return { + str(_sanitize_training_metadata_value(str(key))): _sanitize_training_metadata_value(item) + for key, item in value.items() + } + if isinstance(value, list): + return [_sanitize_training_metadata_value(item) for item in value] + return value + + +def _manifest_metadata_without_dataset_load(manifest_path: Path) -> dict[str, Any]: + manifest = read_json(manifest_path) + return { + "manifest_id": str(manifest.get("manifest_id", manifest_path.stem)), + "manifest_counts": manifest.get("counts", {}), + "manifest_public_safe": bool(manifest.get("public_safe", False)), + } + + +def _runtime_private_fields(config: dict[str, Any]) -> dict[str, str]: + fields: dict[str, str] = {} + for key in ( + "adapter_path", + "evidence_output_dir", + "output_root", + "runtime_check_output_dir", + "private_override_path", + ): + value = config.get(key) + if isinstance(value, str) and value: + fields[key] = value + return fields + + +def _unresolved_runtime_fields(config: dict[str, Any]) -> list[str]: + return sorted( + key + for key, value in _runtime_private_fields(config).items() + if "<" in value or ">" in value + ) + + +def _runtime_output_root_policy(config: dict[str, Any], unresolved_fields: list[str]) -> dict[str, Any]: + raw_output_root = config.get("output_root") + policy = str(config.get("output_root_policy", config.get("a100_project_root_policy", "")) or "") + if "output_root" in unresolved_fields: + status = "blocked_unresolved_template" + elif isinstance(raw_output_root, str) and raw_output_root: + status = "resolved_private_override_not_run" + else: + status = "missing_output_root" + return { + "status": status, + "approved_policy": policy or "must_resolve_to_approved_private_a100_project_root", + "output_root": _sanitize_training_metadata_value(raw_output_root or ""), + "public_template_output_root": "", + } + + +def _runtime_check_output_root_policy( + config: dict[str, Any], + output_path: Path, + unresolved_fields: list[str], +) -> dict[str, Any]: + runtime_root = config.get("runtime_check_output_dir") or config.get("output_root") + policy = str(config.get("output_root_policy", config.get("a100_project_root_policy", "")) or "") + if "output_root" in unresolved_fields or "runtime_check_output_dir" in unresolved_fields: + status = "blocked_unresolved_template" + elif _output_file_within_configured_runtime_root(config, output_path): + status = "approved_private_root" + else: + status = "blocked_output_outside_approved_root" + return { + "status": status, + "approved_policy": policy or "must_resolve_to_approved_private_a100_project_root", + "runtime_check_output_dir": _sanitize_training_metadata_value( + runtime_root or "" + ), + "requested_output": _sanitize_training_metadata_value(output_path.as_posix()), + "public_template_output_root": "", + } + + +def _output_file_within_configured_runtime_root(config: dict[str, Any], output_path: Path) -> bool: + root_value = config.get("runtime_check_output_dir") or config.get("output_root") + if not isinstance(root_value, str) or not root_value: + return False + if "<" in root_value or ">" in root_value: + return False + candidate = output_path.expanduser() + root = Path(root_value).expanduser() + if not candidate.is_absolute() or not root.is_absolute(): + return False + candidate = candidate.resolve(strict=False) + root = root.resolve(strict=False) + try: + candidate.relative_to(root) + except ValueError: + return False + return True + + +def _runtime_check_status( + *, + unresolved_fields: list[str], + config_allows_runtime_check: bool, +) -> str: + if unresolved_fields: + return "blocked_unresolved_private_override" + if not config_allows_runtime_check: + return "skipped_no_runtime_opt_in" + return "prepared_private_override_resolved_not_run" + + +def prepare_sft_runtime_label_provenance( + config_path: Path, + manifest_path: Path, + *, + metadata_path: Path | None = None, +) -> dict[str, Any]: + config = _load_config(config_path) + manifest_summary = _manifest_metadata_without_dataset_load(manifest_path) + unresolved_fields = _unresolved_runtime_fields(config) + config_allows_runtime_check = bool(config.get("allow_runtime_label_provenance_check", False)) + private_override_required = bool(config.get("private_override_required", True)) + private_override_resolved = private_override_required and not unresolved_fields + runtime_check_status = _runtime_check_status( + unresolved_fields=unresolved_fields, + config_allows_runtime_check=config_allows_runtime_check, + ) + evidence_gaps = [ + "runtime_check_not_executed", + "real_training_labels_not_inspected", + "real_training_label_provenance_missing", + ] + if unresolved_fields: + evidence_gaps.append("private_override_unresolved") + if not config_allows_runtime_check: + evidence_gaps.append("runtime_opt_in_missing") + metadata = { + "evidence_kind": "sft_runtime_label_provenance_prep", + "stage": "sft_runtime_label_provenance_prep", + "generated_at": datetime.now(timezone.utc).isoformat(), + "config_path": _sanitize_training_metadata_value(config_path.as_posix()), + "dataset_manifest_path": _sanitize_training_metadata_value(manifest_path.as_posix()), + "dataset_manifest_id": manifest_summary["manifest_id"], + "manifest_counts": manifest_summary["manifest_counts"], + "manifest_public_safe": manifest_summary["manifest_public_safe"], + "runtime_check_status": runtime_check_status, + "runtime_gate": { + "cli_requested_runtime_check": False, + "config_allow_runtime_label_provenance_check": config_allows_runtime_check, + "private_override_resolved": private_override_resolved, + "will_run_runtime_label_provenance_check": False, + }, + "private_override": { + "required": private_override_required, + "status": "resolved" if private_override_resolved else "unresolved", + "unresolved_fields": unresolved_fields, + "requirements": _sanitize_training_metadata_value(config.get("private_override_requirements", [])), + "public_placeholder": "", + }, + "output_root_policy": _runtime_output_root_policy(config, unresolved_fields), + "dependency_policy": { + "policy": str(config.get("dependency_policy", "prep_only_no_train_dependency_import_no_model_download")), + "train_dependencies_imported": False, + "model_download_allowed": False, + "private_adapter_load_allowed": False, + "a100_connection_allowed": False, + }, + "label_provenance_intent": { + "intent": str( + config.get( + "label_provenance_intent", + "inspect_real_tokenizer_collator_labels_later", + ) + ), + "private_labels_inspected": False, + "runtime_path": "future_authorized_private_tokenizer_collator_check", + }, + "label_tensor_available": False, + "true_label_mask_status": "unavailable", + "inspection_status": "runtime_check_not_executed", + "evidence_gaps": _deduped_gaps(evidence_gaps), + "prior_artifacts": _sanitize_training_metadata_value(config.get("prior_artifacts", {})), + "claims": { + "runtime_readiness_proves_contract_learning": False, + "checkpoint_release": False, + "adapter_release": False, + "held_out_generalization_claim": False, + "production_readiness_claim": False, + "live_browser_benchmark_claim": False, + }, + "artifact_policy": { + "raw_rendered_prompts_written": False, + "raw_logs_copied_to_git": False, + "checkpoints_or_adapters_copied_to_git": False, + "private_paths_omitted": True, + }, + "metadata_path": metadata_path.as_posix() if metadata_path is not None else "not_written", + "notes": ( + "Preparation metadata only; no A100/private adapter execution occurred, no model was downloaded, " + "and no true runtime labels were inspected." + ), + } + sanitized = _sanitize_training_metadata_value(metadata) + if not isinstance(sanitized, dict): + raise AssertionError("runtime label provenance prep metadata must be a mapping") + return cast(dict[str, Any], sanitized) + + +def _heavy_training_gate(config: dict[str, Any], dry_run: bool) -> dict[str, bool]: + cli_run_training = not dry_run + config_allows = _heavy_training_allowed(config) + return { + "cli_run_training": cli_run_training, + "config_allow_heavy_training": config_allows, + "will_run_heavy_training": cli_run_training and config_allows, + } + + +def _output_dir_within_configured_root(config: dict[str, Any], output_dir: Path) -> bool: + output_root = config.get("output_root") + if not isinstance(output_root, str) or not output_root: + return True + if "<" in output_root or ">" in output_root: + return False + candidate = output_dir.expanduser() + root = Path(output_root).expanduser() + if not candidate.is_absolute() or not root.is_absolute(): + return False + try: + candidate.relative_to(root) + except ValueError: + return False + return True + + +def _output_paths( + *, + config: dict[str, Any], + output_dir: Path, + adapter_path: Path, + metadata_path: Path, +) -> dict[str, Any]: + paths: dict[str, Any] = { + "run_output_dir": output_dir.as_posix(), + "adapter_path": adapter_path.as_posix(), + "metadata_path": metadata_path.as_posix(), + } + for config_key, output_key in ( + ("output_root", "configured_output_root"), + ("output_dir", "configured_output_dir"), + ("adapter_output_dir", "configured_adapter_output_dir"), + ("evidence_output_dir", "configured_evidence_output_dir"), + ): + value = config.get(config_key) + if isinstance(value, str) and value: + paths[output_key] = value + return paths + + +def _command_summary( + *, + stage: str, + config_path: Path, + manifest_path: Path, + output_dir: Path, + dry_run: bool, +) -> dict[str, Any]: + return { + "entrypoint": f"voice2task-train {stage}", + "config": config_path.as_posix(), + "manifest": manifest_path.as_posix(), + "output_dir": output_dir.as_posix(), + "mode": "dry_run" if dry_run else "run_training", + "requires_cli_run_training": not dry_run, + "requires_config_allow_heavy_training": True, + } + + +def _loss_mask_policy(stage: str) -> dict[str, Any]: + if stage == "sft": + return { + "policy": "assistant_only_completion_only", + "prompt_label_id": -100, + "assistant_target": "browser_task_contract_json", + "trainer_integration": "trl_sfttrainer_pretokenized_input_ids_attention_mask_labels", + "full_text_causal_lm_labels": False, + } + return {"policy": "dpo_pairwise_preference_loss"} + + +def _training_stack(stage: str) -> str: + if stage == "sft": + return "transformers+peft+trl+pretokenized_assistant_only_labels" + return "transformers+peft+trl" + + +def _metadata_common( + *, + stage: str, + config_path: Path, + manifest_path: Path, + output_dir: Path, + dry_run: bool, +) -> dict[str, Any]: + config = _load_config(config_path) + load_summary = _manifest_load_summary(manifest_path, stage) + adapter_path = output_dir / "adapter" + metadata_path = output_dir / "adapter_metadata.json" + mode_flag = "--dry-run" if dry_run else "--run-training" + heavy_training_gate = _heavy_training_gate(config, dry_run) + return { + "stage": stage, + "stack": _training_stack(stage), + "base_model": _public_base_model(config), + "adapter_path": adapter_path.as_posix(), + "dataset_manifest_id": load_summary["manifest_id"], + "dataset_manifest_path": manifest_path.as_posix(), + "dataset_load": load_summary, + "hyperparameters": config, + "dry_run": dry_run, + "release_status": "not_released", + "adapter_release_status": "not_released", + "training_status": "dry_run" if dry_run else "pending_heavy_training", + "formatting_policy": dict(FORMATTING_POLICY), + "loss_mask_policy": _loss_mask_policy(stage), + "created_at": datetime.now(timezone.utc).isoformat(), + "package_versions": _sanitized_package_versions(), + "gpu_selection_policy": _gpu_selection_policy(config), + "heavy_training_gate": heavy_training_gate, + "output_paths": _output_paths( + config=config, + output_dir=output_dir, + adapter_path=adapter_path, + metadata_path=metadata_path, + ), + "command_summary": _command_summary( + stage=stage, + config_path=config_path, + manifest_path=manifest_path, + output_dir=output_dir, + dry_run=dry_run, + ), + "training_command": ( + f"voice2task-train {stage} --config {config_path.as_posix()} " + f"--manifest {manifest_path.as_posix()} --output-dir {output_dir.as_posix()} {mode_flag}" + ), + "metadata_path": metadata_path.as_posix(), + "notes": "Dry-run metadata only; no model download or heavy training was executed." + if dry_run + else "Training entrypoint checked; install train extras and provide runtime resources for heavy execution.", + } + + +def _train_dependencies_available() -> bool: + return all(importlib.util.find_spec(module) is not None for module in ("datasets", "peft", "transformers", "trl")) + + +def _runtime_label_dependencies_available() -> bool: + return importlib.util.find_spec("transformers") is not None or globals().get("AutoTokenizer") is not None + + +def _write_training_plan(metadata: dict[str, Any], stage: str) -> dict[str, Any]: + metadata["release_status"] = "not_released" + metadata["training_status"] = "training_unavailable" + metadata["trainer_available"] = False + metadata["notes"] = ( + f"{stage.upper()} real-training entrypoint is wired, but train extras are not installed in this runtime. " + "Install the train dependency group before running heavy training." + ) + write_json(Path(metadata["metadata_path"]), metadata) + return metadata + + +def _write_training_skipped(metadata: dict[str, Any], stage: str) -> dict[str, Any]: + metadata["release_status"] = "not_released" + metadata["training_status"] = "training_skipped_by_config" + metadata["trainer_available"] = False + metadata["notes"] = ( + f"{stage.upper()} real-training entrypoint is available, but this config does not set " + "`allow_heavy_training: true`. This prevents accidental model downloads during bootstrap validation." + ) + write_json(Path(metadata["metadata_path"]), metadata) + return metadata + + +def _write_training_blocked_by_output_policy(metadata: dict[str, Any], stage: str) -> dict[str, Any]: + config_output_root = metadata.get("hyperparameters", {}).get("output_root") + unresolved_template = isinstance(config_output_root, str) and ( + "<" in config_output_root or ">" in config_output_root + ) + metadata["release_status"] = "not_released" + metadata["training_status"] = "training_blocked_by_output_policy" + metadata["trainer_available"] = False + metadata["heavy_training_gate"]["will_run_heavy_training"] = False + if unresolved_template: + metadata["notes"] = ( + f"{stage.upper()} heavy training was blocked because config output_root is an unresolved output_root " + "template. Create a private A100 override that resolves it before running heavy training." + ) + else: + metadata["notes"] = ( + f"{stage.upper()} heavy training was blocked because the requested output directory is outside " + "configured output_root. A100 smoke outputs must stay under the configured project directory." + ) + write_json(Path(metadata["metadata_path"]), metadata) + return metadata + + +def _training_error_category(exc: Exception) -> str: + message = str(exc).lower() + if ( + "network is unreachable" in message + or "couldn't connect" in message + or "localentrynotfound" in message + or "huggingface.co" in message + ): + return "model_download_unavailable" + return "training_failed" + + +def _write_training_failed(metadata: dict[str, Any], stage: str, exc: Exception) -> None: + metadata["release_status"] = "not_released" + metadata["training_status"] = "training_failed" + metadata["trainer_available"] = True + metadata["heavy_training_gate"]["will_run_heavy_training"] = False + metadata["error_category"] = _training_error_category(exc) + metadata["error_summary"] = "Training failed before completion; raw logs remain private." + metadata["notes"] = ( + f"{stage.upper()} heavy training failed before completion. Sanitized metadata was written; " + "raw remote logs, caches, checkpoints, and adapters remain outside git." + ) + write_json(Path(metadata["metadata_path"]), metadata) + + +def _lora_config(config: dict[str, Any]) -> Any: + from peft import LoraConfig # type: ignore[import-not-found, unused-ignore] + + lora = config.get("lora", {}) + return LoraConfig( + r=int(lora.get("r", 8)), + lora_alpha=int(lora.get("alpha", 16)), + lora_dropout=float(lora.get("dropout", 0.05)), + target_modules=list(lora.get("target_modules", ["q_proj", "v_proj"])), + task_type="CAUSAL_LM", + ) + + +def _training_arguments(config: dict[str, Any], output_dir: Path) -> Any: + from transformers import TrainingArguments + + return TrainingArguments( + output_dir=output_dir.as_posix(), + num_train_epochs=float(config.get("num_train_epochs", 1)), + max_steps=int(config.get("max_steps", -1)), + per_device_train_batch_size=int(config.get("per_device_train_batch_size", 1)), + gradient_accumulation_steps=int(config.get("gradient_accumulation_steps", 1)), + learning_rate=float(config.get("learning_rate", 5e-5)), + warmup_ratio=float(config.get("warmup_ratio", 0.0)), + logging_steps=int(config.get("logging_steps", 1)), + save_strategy=str(config.get("save_strategy", "no")), + seed=int(config.get("seed", 42)), + report_to=[], + ) + + +def _load_sft_training_rows(manifest_path: Path, split: str) -> list[SFTDatasetRow]: + summary = _manifest_load_summary(manifest_path, "sft") + dataset_path = summary["dataset_path"] + if dataset_path is None: + return [] + return [row for row in (SFTDatasetRow(**record) for record in read_jsonl(Path(dataset_path))) if row.split == split] + + +def _configured_sft_training_row_limit(config: dict[str, Any]) -> int | None: + value = config.get("max_train_rows") + if value is None: + return None + row_limit = int(value) + if row_limit < 1: + raise ValueError("max_train_rows must be at least 1 when configured") + return row_limit + + +def _configured_sft_training_source_ids(config: dict[str, Any]) -> list[str] | None: + value = config.get("train_source_ids") + if value is None: + return None + if not isinstance(value, list) or not value or not all(isinstance(item, str) and item for item in value): + raise ValueError("train_source_ids must be a non-empty list of source_id strings when configured") + return list(value) + + +def _sft_row_source_id(row: SFTDatasetRow) -> str: + source_id = row.provenance.get("source_id") if isinstance(row.provenance, dict) else None + return str(source_id or row.id) + + +def _source_id_counts(rows: list[SFTDatasetRow]) -> dict[str, int]: + counts: dict[str, int] = {} + for row in rows: + source_id = _sft_row_source_id(row) + counts[source_id] = counts.get(source_id, 0) + 1 + return dict(sorted(counts.items())) + + +def _limited_sft_training_rows( + rows: list[SFTDatasetRow], + config: dict[str, Any], +) -> tuple[list[SFTDatasetRow], int | None, list[str] | None, int, int]: + source_ids = _configured_sft_training_source_ids(config) + rows_before_source_filter = len(rows) + if source_ids is not None: + source_id_set = set(source_ids) + rows = [row for row in rows if _sft_row_source_id(row) in source_id_set] + rows_before_limit = len(rows) + row_limit = _configured_sft_training_row_limit(config) + if row_limit is None: + return rows, None, source_ids, rows_before_source_filter, rows_before_limit + return rows[:row_limit], row_limit, source_ids, rows_before_source_filter, rows_before_limit + + +def _record_sft_training_row_selection( + metadata: dict[str, Any], + *, + split: str, + rows: list[SFTDatasetRow], + row_limit: int | None, + source_ids: list[str] | None, + loaded_rows_before_limit: int, + loaded_rows_before_source_filter: int, +) -> None: + row_ids = [row.id for row in rows] + metadata["training_split"] = split + metadata["training_source_ids"] = source_ids + metadata["training_row_limit"] = row_limit + metadata["training_rows_used"] = len(rows) + metadata["training_row_ids"] = row_ids + metadata["training_source_id_counts"] = _source_id_counts(rows) + metadata["training_rows_before_source_filter"] = loaded_rows_before_source_filter + dataset_load = metadata.setdefault("dataset_load", {}) + if isinstance(dataset_load, dict): + dataset_load["training_split"] = split + dataset_load["training_source_ids"] = source_ids + dataset_load["training_row_limit"] = row_limit + dataset_load["training_rows_used"] = len(rows) + dataset_load["training_row_ids"] = row_ids + dataset_load["training_source_id_counts"] = _source_id_counts(rows) + dataset_load["training_rows_before_source_filter"] = loaded_rows_before_source_filter + dataset_load["loaded_rows_before_training_row_limit"] = loaded_rows_before_limit + + +def _record_sft_training_selection_from_config( + metadata: dict[str, Any], + config: dict[str, Any], + manifest_path: Path, +) -> list[SFTDatasetRow]: + split = str(config.get("dataset_split", "train")) + all_rows = _load_sft_training_rows(manifest_path, split=split) + rows, row_limit, source_ids, rows_before_source_filter, rows_before_limit = _limited_sft_training_rows( + all_rows, config + ) + _record_sft_training_row_selection( + metadata, + split=split, + rows=rows, + row_limit=row_limit, + source_ids=source_ids, + loaded_rows_before_limit=rows_before_limit, + loaded_rows_before_source_filter=rows_before_source_filter, + ) + return rows + + +def _target_token_count(records: list[dict[str, Any]]) -> int: + total = 0 + for record in records: + labels = _token_list(record.get("labels")) + total += sum(1 for label in labels if label != -100) + return total + + +def _safe_training_metric_value(value: Any) -> int | float | str | bool | None: + if value is None or isinstance(value, str | bool): + return value + if isinstance(value, int | float): + return value + return str(value) + + +def _observed_optimizer_steps(trainer: Any, train_result: Any) -> int | None: + state = getattr(trainer, "state", None) + value = getattr(state, "global_step", None) + if value is None: + value = getattr(train_result, "global_step", None) + if value is None: + metrics = getattr(train_result, "metrics", None) + if isinstance(metrics, dict): + value = metrics.get("global_step") + if isinstance(value, int | float): + return int(value) + return None + + +def _record_sft_training_budget_metadata( + metadata: dict[str, Any], + *, + config: dict[str, Any], + train_row_count: int, + records: list[dict[str, Any]], + trainer: Any, + train_result: Any, +) -> None: + effective_batch_size = int(config.get("per_device_train_batch_size", 1)) * int( + config.get("gradient_accumulation_steps", 1) + ) + configured_max_steps = int(config.get("max_steps", -1)) + observed_steps = _observed_optimizer_steps(trainer, train_result) + step_budget = observed_steps if observed_steps is not None else configured_max_steps + if step_budget > 0: + theoretical_examples_seen = step_budget * effective_batch_size + else: + theoretical_examples_seen = int(round(train_row_count * float(config.get("num_train_epochs", 1)))) + target_tokens_per_single_pass = _target_token_count(records) + target_tokens_seen_estimate = ( + int(round(target_tokens_per_single_pass * theoretical_examples_seen / train_row_count)) + if train_row_count + else 0 + ) + metrics = getattr(train_result, "metrics", None) + metadata["training_budget"] = { + "configured_max_steps": configured_max_steps, + "observed_optimizer_steps": observed_steps, + "num_train_epochs": float(config.get("num_train_epochs", 1)), + "per_device_train_batch_size": int(config.get("per_device_train_batch_size", 1)), + "gradient_accumulation_steps": int(config.get("gradient_accumulation_steps", 1)), + "effective_batch_size": effective_batch_size, + "scheduler_max_steps": configured_max_steps if configured_max_steps > 0 else None, + "train_row_count": train_row_count, + "theoretical_examples_seen": theoretical_examples_seen, + "target_tokens_per_single_pass": target_tokens_per_single_pass, + "target_tokens_seen_estimate": target_tokens_seen_estimate, + "target_tokens_seen_status": "estimated_from_label_tokens_and_step_budget", + "step_matching_unit": "optimizer_steps", + "step_matched_not_token_matched": True, + } + metadata["observed_optimizer_steps"] = observed_steps + metadata["target_tokens_seen"] = target_tokens_seen_estimate + metadata["target_tokens_seen_status"] = "estimated_from_label_tokens_and_step_budget" + if isinstance(metrics, dict): + metadata["train_result_metrics"] = { + str(key): _safe_training_metric_value(value) for key, value in sorted(metrics.items()) + } + + +def _load_sft_prediction_rows(manifest_path: Path, split: str) -> list[SFTDatasetRow]: + summary = _manifest_load_summary(manifest_path, "sft") + dataset_path = summary["dataset_path"] + if dataset_path is None: + return [] + rows = [SFTDatasetRow(**record) for record in read_jsonl(Path(dataset_path))] + if split == "all": + return rows + return [row for row in rows if row.split == split] + + +def _configured_sft_prediction_row_limit(config: dict[str, Any]) -> int | None: + value = config.get("max_prediction_rows") + if value is None: + return None + row_limit = int(value) + if row_limit < 1: + raise ValueError("max_prediction_rows must be at least 1 when configured") + return row_limit + + +def _limited_sft_prediction_rows( + rows: list[SFTDatasetRow], + config: dict[str, Any], +) -> tuple[list[SFTDatasetRow], int | None]: + row_limit = _configured_sft_prediction_row_limit(config) + if row_limit is None: + return rows, None + return rows[:row_limit], row_limit + + +def _record_sft_prediction_row_selection( + metadata: dict[str, Any], + *, + rows: list[SFTDatasetRow], + row_limit: int | None, + loaded_rows_before_limit: int, +) -> None: + metadata["prediction_row_limit"] = row_limit + metadata["prediction_row_ids"] = [row.id for row in rows] + metadata["prediction_rows_before_limit"] = loaded_rows_before_limit + + +def _prediction_gate(config: dict[str, Any], dry_run: bool, fixture_mode: bool) -> dict[str, bool]: + config_allows = bool(config.get("allow_private_prediction")) + adapter_configured = isinstance(config.get("adapter_path"), str) and bool(str(config.get("adapter_path")).strip()) + return { + "cli_run_prediction": not dry_run, + "fixture_mode": fixture_mode, + "config_allow_private_prediction": config_allows, + "adapter_configured": adapter_configured, + "will_run_private_prediction": (not dry_run) and (not fixture_mode) and config_allows and adapter_configured, + } + + +def _decoding_policy(config: dict[str, Any]) -> dict[str, Any]: + schema_retry_enabled = bool(config.get("schema_retry_enabled", True)) + return { + "strategy": "greedy", + "do_sample": False, + "max_new_tokens": int(config.get("max_new_tokens", 256)), + "markdown_fence_suppression_enabled": True, + "markdown_fence_suppression_strategy": "bad_words_ids", + "markdown_fence_suppression_token_sources": list(MARKDOWN_FENCE_SUPPRESSION_TOKEN_SOURCES), + "raw_decoded_sidecar_written": False, + "generation_trace_sidecar_written": False, + "schema_repair_applied": False, + "schema_guard_enabled": True, + "schema_retry_enabled": schema_retry_enabled, + "schema_retry_max_attempts": 1 if schema_retry_enabled else 0, + } + + +def _prediction_sidecar_paths(output_path: Path) -> dict[str, Path]: + return { + "prompt_snapshot": output_path.parent / "prompt_snapshot.json", + "raw_decoded_summary": output_path.parent / "raw_decoded_summary.jsonl", + "generation_trace": output_path.parent / "generation_trace.jsonl", + } + + +def _public_sidecar_paths(sidecar_paths: dict[str, Path]) -> dict[str, str]: + placeholders = { + "prompt_snapshot": "", + "raw_decoded_summary": "", + "generation_trace": "", + } + return { + name: _public_display_artifact_path(path, placeholders.get(name, "")) + for name, path in sidecar_paths.items() + } + + +def _diagnostic_artifact_paths(output_path: Path, *, overfit_diagnostic: bool) -> dict[str, str]: + if not overfit_diagnostic: + return {} + return { + "objective_inspection": _public_display_artifact_path( + output_path.parent / "objective_inspection.json", + "", + ), + "leak_scan": _public_display_artifact_path( + output_path.parent / "leak_scan_result.json", + "", + ), + } + + +def _prediction_metadata_common( + *, + config_path: Path, + manifest_path: Path, + output_path: Path, + dry_run: bool, + fixture_mode: bool, +) -> dict[str, Any]: + config = _load_config(config_path) + load_summary = _manifest_load_summary(manifest_path, "sft") + sidecar_paths = _prediction_sidecar_paths(output_path) + return { + "stage": "sft_prediction", + "stack": "transformers+peft+trl", + "base_model": _public_display_model(_public_base_model(config)), + "model_source": config.get("model_source", "unknown"), + "dataset_manifest_id": load_summary["manifest_id"], + "dataset_manifest_path": _public_display_path(manifest_path, "data/public-samples/manifest_public_sample.json"), + "prediction_output_path": _public_display_path(output_path, ""), + "prediction_split": str(config.get("prediction_split", "all")), + "overfit_diagnostic": bool(config.get("overfit_diagnostic", False)), + "generalization_claim": bool(config.get("generalization_claim", False)), + "prediction_source_kind": "none", + "prediction_status": "pending", + "prediction_count": 0, + "release_status": "not_released", + "adapter_release_status": "not_released", + "formatting_policy": dict(FORMATTING_POLICY), + "prompt_policy": UNIFIED_GOLD_FREE_PROMPT_POLICY_ID, + "prompt_constraints": prediction_prompt_constraint_summary(), + "prediction_output_boundary": prediction_output_boundary_summary(), + "retry_prompt_constraints": schema_retry_prompt_constraint_summary(), + "retry_template_boundary": schema_retry_template_boundary_summary(), + "decoding_policy": _decoding_policy(config), + "sidecars": _public_sidecar_paths(sidecar_paths), + "diagnostic_artifacts": _diagnostic_artifact_paths( + output_path, + overfit_diagnostic=bool(config.get("overfit_diagnostic", False)), + ), + "metadata_path": _public_display_artifact_path( + output_path.parent / "prediction_metadata.json", + "", + ), + "created_at": datetime.now(timezone.utc).isoformat(), + "prediction_gate": _prediction_gate(config, dry_run, fixture_mode), + "command_summary": { + "entrypoint": "voice2task-train sft-predict", + "config": _public_display_path(config_path, ""), + "manifest": _public_display_path(manifest_path, "data/public-samples/manifest_public_sample.json"), + "output": _public_display_path(output_path, ""), + "mode": "fixture_mode" if fixture_mode else ("dry_run" if dry_run else "run_prediction"), + "requires_cli_run_prediction": not dry_run, + "requires_config_allow_private_prediction": True, + "prompt_policy": UNIFIED_GOLD_FREE_PROMPT_POLICY_ID, + }, + "notes": "Prediction metadata only; no private adapter artifacts were loaded.", + } + + +def _write_fixture_predictions(rows: list[SFTDatasetRow], output_path: Path) -> int: + output_path.parent.mkdir(parents=True, exist_ok=True) + records = [ + { + "id": row.id, + "prediction": as_contract(row.target_contract).to_dict(), + "prediction_source_kind": "public_sample_contract_fixture", + "provenance": {"public_safe": True, "source_id": row.provenance.get("source_id", row.id)}, + } + for row in rows + ] + with output_path.open("w", encoding="utf-8") as handle: + for record in records: + handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + return len(records) + + +def _run_copy_backed_prediction_shadow_hook( + *, + config: dict[str, Any], + config_path: Path, + rows: list[SFTDatasetRow], + output_path: Path, +) -> dict[str, Any] | None: + if "copy_backed_shadow" not in config: + return None + hook_config = shadow_config_from_mapping( + config.get("copy_backed_shadow"), + config_dir=config_path.parent, + output_dir=output_path.parent, + ) + if not hook_config.enabled: + return summarize_prediction_shadow_outcomes([], enabled=False) + row_by_id = {row.id: row for row in rows} + outcomes: list[PredictionShadowHookOutcome] = [] + reserved_artifact_paths = [ + output_path, + output_path.parent / "prediction_metadata.json", + *_prediction_sidecar_paths(output_path).values(), + ] + path_conflict = sidecar_path_conflicts(hook_config.sidecar_output_path, reserved_artifact_paths) + policy_snapshot = None + policy_error_code = None + if prediction_shadow_config_error_code(hook_config) is None and not path_conflict: + try: + policy_snapshot = load_prediction_shadow_policy_snapshot(hook_config) + except Exception: + policy_error_code = "policy_load_or_validation_failed" + for record in read_jsonl(output_path): + row_id = str(record.get("id", "")) + source_row = row_by_id.get(row_id) + outcomes.append( + run_prediction_shadow_hook( + source_text=source_row.input_text if source_row is not None else None, + prediction=record.get("prediction"), + config=hook_config, + request_id=row_id, + policy_snapshot=policy_snapshot, + policy_error_code=policy_error_code, + sidecar_path_conflict=path_conflict, + ) + ) + return summarize_prediction_shadow_outcomes(outcomes, enabled=True, policy_snapshot=policy_snapshot) + + +def _sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _sanitize_preview(text: str, limit: int = 240) -> str: + return _sanitize_decoded_prediction_text(text)[:limit] + + +def _prompt_snapshot_row(row: SFTDatasetRow, prompt: str) -> dict[str, Any]: + sanitized_prompt = _sanitize_decoded_prediction_text(prompt) + return { + "id": row.id, + "prompt_sha256": _sha256_text(sanitized_prompt), + "prompt_char_count": len(sanitized_prompt), + "prompt_preview": sanitized_prompt[:240], + "prompt_constraints": prompt_constraint_summary(sanitized_prompt), + "input_text_preview": _sanitize_preview(row.input_text, limit=120), + "provenance": {"public_safe": True, "source_id": row.provenance.get("source_id", row.id)}, + } + + +def _write_prompt_snapshot( + rows: list[dict[str, Any]], + path: Path, + *, + prediction_split: str, + decoding_policy: dict[str, Any], +) -> None: + write_json( + path, + { + "artifact_kind": "sft_prediction_prompt_snapshot", + "prediction_split": prediction_split, + "formatting_policy": dict(FORMATTING_POLICY), + "prompt_policy": UNIFIED_GOLD_FREE_PROMPT_POLICY_ID, + "prompt_constraints": prediction_prompt_constraint_summary(), + "prediction_output_boundary": prediction_output_boundary_summary(), + "retry_prompt_constraints": schema_retry_prompt_constraint_summary(), + "retry_template_boundary": schema_retry_template_boundary_summary(), + "decoding_policy": dict(decoding_policy), + "rows": rows, + "claims": { + "prompt_snapshot_only": True, + "contains_gold_contract": False, + "public_safe": True, + }, + }, + ) + + +def _decoded_parse_status(decoded: str) -> str: + stripped = decoded.strip() + if not stripped: + return "empty" + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + parsed = None + else: + return "json_object" if isinstance(parsed, dict) else "json_non_object" + object_start = stripped.find("{") + object_end = stripped.rfind("}") + if object_start >= 0 and object_end > object_start: + try: + parsed = json.loads(stripped[object_start : object_end + 1]) + except json.JSONDecodeError: + return "non_json" + return "json_fragment_object" if isinstance(parsed, dict) else "json_fragment_non_object" + return "non_json" + + +def _decoded_attempt_summary(decoded: str) -> dict[str, Any]: + sanitized = _sanitize_decoded_prediction_text(decoded) + return { + "parse_status": _decoded_parse_status(sanitized), + "decoded_sha256": _sha256_text(sanitized), + "decoded_char_count": len(sanitized), + "decoded_prefix": sanitized[:240], + "decoded_suffix": sanitized[-240:], + "private_values_sanitized": sanitized != decoded, + } + + +def _raw_decoded_summary_row( + row_id: str, + decoded: str, + *, + schema_guard: dict[str, Any] | None = None, + retry_decoded: str | None = None, +) -> dict[str, Any]: + raw_attempt = _decoded_attempt_summary(decoded) + row = { + "id": row_id, + **raw_attempt, + "raw_attempt": raw_attempt, + "retry_attempt": _decoded_attempt_summary(retry_decoded) if retry_decoded is not None else None, + "schema_repair_applied": False, + } + if schema_guard is not None: + row["schema_guard"] = schema_guard + return row + + +def _write_jsonl_records(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + + +def _sidecar_written_decoding_policy(config: dict[str, Any]) -> dict[str, Any]: + policy = _decoding_policy(config) + policy["raw_decoded_sidecar_written"] = True + policy["generation_trace_sidecar_written"] = True + return policy + + +def _token_list(value: Any) -> list[Any]: + if hasattr(value, "tolist"): + value = value.tolist() + if isinstance(value, tuple): + value = list(value) + if isinstance(value, list): + if len(value) == 1 and isinstance(value[0], (list, tuple)): + return list(value[0]) + return value + return [] + + +def _generation_trace_row( + *, + row_id: str, + attempt: str, + prediction_source_kind: str, + generated_tokens: Any, + max_new_tokens: int, + eos_token_id: Any, + finish_state: str | None = None, +) -> dict[str, Any]: + tokens = _token_list(generated_tokens) + eos_seen = eos_token_id is not None and eos_token_id in tokens + max_new_tokens_hit = max_new_tokens > 0 and len(tokens) >= max_new_tokens + resolved_finish_state = finish_state or ("eos_observed" if eos_seen else "no_eos_observed") + finish_state_basis = "explicit_fixture_status" if finish_state else "tokenizer_eos_membership" + if finish_state == "fixture_no_generation": + stop_reason_evidence = "fixture_no_generation" + elif eos_seen: + stop_reason_evidence = "tokenizer_eos_observed" + elif max_new_tokens_hit: + stop_reason_evidence = "max_new_tokens_reached_without_tokenizer_eos" + else: + stop_reason_evidence = "not_recorded_below_max_without_tokenizer_eos" + return { + "id": row_id, + "attempt": attempt, + "prediction_source_kind": prediction_source_kind, + "strategy": "greedy", + "do_sample": False, + "max_new_tokens": max_new_tokens, + "generated_token_count": len(tokens), + "max_new_tokens_hit": max_new_tokens_hit, + "eos_token_id_available": eos_token_id is not None, + "eos_token_seen": eos_seen, + "finish_state": resolved_finish_state, + "finish_state_basis": finish_state_basis, + "stop_reason_evidence": stop_reason_evidence, + "actual_stop_reason_recorded": False, + "actual_stop_reason": None, + } + + +def _write_fixture_sidecars( + *, + rows: list[SFTDatasetRow], + output_path: Path, + sidecar_paths: dict[str, Path], + prediction_split: str, + max_new_tokens: int, + decoding_policy: dict[str, Any], +) -> None: + prompt_rows: list[dict[str, Any]] = [] + raw_rows: list[dict[str, Any]] = [] + trace_rows: list[dict[str, Any]] = [] + for row in rows: + prompt = format_sft_prediction_prompt(PredictionInput.from_sft_row(row), tokenizer=None) + prompt_rows.append(_prompt_snapshot_row(row, prompt)) + decoded = json.dumps(as_contract(row.target_contract).to_dict(), ensure_ascii=False, sort_keys=True) + raw_rows.append(_raw_decoded_summary_row(row.id, decoded)) + trace_rows.append( + _generation_trace_row( + row_id=row.id, + attempt="raw_attempt", + prediction_source_kind="public_sample_contract_fixture", + generated_tokens=[], + max_new_tokens=max_new_tokens, + eos_token_id=None, + finish_state="fixture_no_generation", + ) + ) + _write_prompt_snapshot( + prompt_rows, + sidecar_paths["prompt_snapshot"], + prediction_split=prediction_split, + decoding_policy=decoding_policy, + ) + _write_jsonl_records(sidecar_paths["raw_decoded_summary"], raw_rows) + _write_jsonl_records(sidecar_paths["generation_trace"], trace_rows) + + +def _mark_sidecars_written(metadata: dict[str, Any]) -> None: + metadata["decoding_policy"]["raw_decoded_sidecar_written"] = True + metadata["decoding_policy"]["generation_trace_sidecar_written"] = True + + +def _write_prediction_metadata(output_path: Path, metadata: dict[str, Any]) -> None: + write_json(output_path.parent / "prediction_metadata.json", metadata) + + +def _write_private_prediction_unavailable(metadata: dict[str, Any]) -> dict[str, Any]: + metadata["prediction_status"] = "prediction_unavailable_private_runtime" + metadata["prediction_source_kind"] = "private_adapter_not_run_locally" + metadata["prediction_gate"]["will_run_private_prediction"] = False + metadata["notes"] = ( + "Private trained-adapter prediction requires the A100 runtime, train dependencies, and a private adapter " + "path. No public predictions were written by this local command." + ) + return metadata + + +def _prediction_dependencies_available() -> bool: + return all(importlib.util.find_spec(module) is not None for module in ("peft", "torch", "transformers")) + + +def _extract_json_object(text: str) -> Any: + stripped = text.strip() + if stripped: + try: + return _sanitize_prediction_value(json.loads(stripped)) + except json.JSONDecodeError: + pass + object_start = stripped.find("{") + object_end = stripped.rfind("}") + if object_start >= 0 and object_end > object_start: + try: + parsed = json.loads(stripped[object_start : object_end + 1]) + except json.JSONDecodeError: + pass + else: + return _sanitize_prediction_value(parsed) + return _sanitize_decoded_prediction_text(stripped) + + +def _extract_strict_json_object(text: str) -> Any: + stripped = text.strip() + if not stripped: + return "" + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + return _sanitize_decoded_prediction_text(stripped) + return _sanitize_prediction_value(parsed) + + +def _required_field_missing(prediction: Any) -> list[str]: + required = { + "task_type", + "route", + "safety", + "confirmation_required", + "slots", + "normalized_command", + "language", + "contract_version", + } + if not isinstance(prediction, dict): + return sorted(required) + return sorted(required - set(prediction)) + + +def _schema_guard_status(prediction: Any) -> dict[str, Any]: + return validate_contract_status(prediction) + + +def _schema_retry_prompt(prediction_input: PredictionInput, raw_prediction: Any, guard_status: dict[str, Any]) -> str: + if not isinstance(prediction_input, PredictionInput): + raise TypeError("schema retry prompt rendering requires PredictionInput") + missing = guard_status.get("missing_required_fields", []) + missing_text = ", ".join(str(field) for field in missing) if missing else "unknown" + extra = guard_status.get("extra_top_level_fields", []) + extra_text = ", ".join(str(field) for field in extra) if extra else "none" + validation_error = str(guard_status.get("validation_error") or "unknown") + raw_summary = json.dumps(_sanitize_prediction_value(raw_prediction), ensure_ascii=False, sort_keys=True) + canonical_skeleton = json.dumps( + { + "task_type": "search", + "route": "search_web", + "safety": {"allow": True, "reason": "public_readonly"}, + "confirmation_required": False, + "slots": {}, + "normalized_command": prediction_input.input_text, + "language": "zh-CN", + "contract_version": "v1", + }, + ensure_ascii=False, + ) + return "\n".join( + [ + "你刚才输出的 JSON 不是合法 Browser Task Contract。", + f"缺失字段: {missing_text}。", + f"额外顶层字段: {extra_text}。", + f"schema 违规摘要: {validation_error}。", + "如果存在额外顶层字段，必须删除；root object 只能包含规定的 8 个顶层字段。", + f"合法 task_type enum: {', '.join(sorted(TASK_TYPES))}。", + f"合法 route enum: {', '.join(sorted(ROUTES))}。", + "请重新输出一个完整 Browser Task Contract JSON object，必须包含全部 8 个顶层字段：", + "task_type, route, safety, confirmation_required, slots, normalized_command, language, contract_version。", + f"Canonical required skeleton: {canonical_skeleton}", + "safety 必须是 object，且包含 boolean safety.allow 和非空字符串 safety.reason。", + "route 是 enum，不是 URL/path；不要输出 /weather、https://...、www... 或文件路径。", + "task_type 不能使用 search_web、open_url、query_weather_request，也不能使用 app/action name。", + "public-readonly search: task_type 必须是 search，不能是 search_web；route 必须是 search_web。", + "只输出一个 minified JSON object；全部 8 个顶层字段必须都在同一个 root object 内。", + "Retry response must be exactly one JSON object and nothing else.", + "No text outside the root JSON object; no preamble, wrapper, suffix, or trailing analysis.", + "Return a machine-readable only retry response; do not include human-facing commentary.", + "Retry template mode: machine_contract_regeneration.", + "Treat this as a machine-only retry turn, not a conversational assistant answer.", + "Assistant output boundary: assistant JSON payload only.", + "Strict whole-object parser boundary: wrapped fragments remain invalid.", + "不要在 normalized_command 之前提前关闭 root object。", + "第一个非空字符必须是 `{`；最后一个非空字符必须是 `}`。", + "不要 Markdown/code fences/prose；不要解释、不要自然语言前后缀。", + "不要输出任何前缀或后缀文本；不要以“这是”或“以下”开头；不要使用 Here is。", + "不要使用自然语言 wrapper/preamble，例如“这是”、“以下”或 Here is。", + "不要在 JSON 后添加解释、分析或用户输入复述；不要输出第二个 JSON object。", + "否则 strict parser 会拒绝 retry attempt。", + f"用户输入: {prediction_input.input_text}", + f"上一轮输出摘要: {raw_summary[:500]}", + ] + ) + + +def schema_retry_prompt_constraint_summary(prompt: str | None = None) -> dict[str, bool]: + if prompt is None: + row = SFTDatasetRow( + id="retry-constraint-summary", + split="test", + input_text="帮我搜索北京明天的天气", + target_contract={ + "task_type": "search", + "route": "search_web", + "safety": {"allow": True, "reason": "public_readonly"}, + "confirmation_required": False, + "slots": {"query": "北京 明天 天气"}, + "normalized_command": "搜索北京明天天气", + "language": "zh-CN", + "contract_version": "v1", + }, + provenance={"source_id": "retry-constraint-summary", "public_safe": True}, + ) + raw_prediction = { + "route": "search_web", + "safety": {"allow": True, "reason": "public_readonly"}, + "confirmation_required": False, + "slots": {"query": "北京明天天气"}, + "normalized_command": "搜索北京明天天气", + "language": "zh-CN", + "contract_version": "v1", + } + prompt = _schema_retry_prompt( + PredictionInput.from_sft_row(row), + raw_prediction, + _schema_guard_status(raw_prediction), + ) + return { + "minified_json_only_visible": "只输出一个 minified JSON object" in prompt, + "single_root_json_object_visible": "同一个 root object" in prompt, + "first_last_brace_visible": "第一个非空字符必须是 `{`" in prompt + and "最后一个非空字符必须是 `}`" in prompt, + "no_markdown_prose_visible": "不要 Markdown/code fences/prose" in prompt, + "no_prefix_suffix_text_visible": "不要输出任何前缀或后缀文本" in prompt, + "no_zh_this_following_prefix_visible": "不要以“这是”或“以下”开头" in prompt, + "no_here_is_visible": "不要使用 Here is" in prompt, + "no_trailing_analysis_visible": "不要在 JSON 后添加解释、分析或用户输入复述" in prompt, + "no_second_json_object_visible": "不要输出第二个 JSON object" in prompt, + "exact_json_only_output_visible": "Retry response must be exactly one JSON object and nothing else" in prompt, + "no_text_outside_root_json_object_visible": "No text outside the root JSON object" in prompt, + "no_natural_language_wrapper_or_preamble_visible": "不要使用自然语言 wrapper/preamble" in prompt, + "machine_readable_only_retry_response_visible": "machine-readable only retry response" in prompt, + "strict_parser_rejection_warning_visible": "否则 strict parser 会拒绝 retry attempt" in prompt, + "task_type_search_not_search_web_visible": "task_type 必须是 search，不能是 search_web" in prompt, + } + + +def _encode_suppression_sequence(tokenizer: Any, text: str) -> list[int]: + encode = getattr(tokenizer, "encode", None) + if callable(encode): + try: + encoded = encode(text, add_special_tokens=False) + except TypeError: + encoded = encode(text) + if isinstance(encoded, list): + return [int(token_id) for token_id in encoded if isinstance(token_id, int)] + if callable(tokenizer): + try: + encoded_mapping = tokenizer(text, add_special_tokens=False) + except TypeError: + return [] + input_ids = encoded_mapping.get("input_ids") if isinstance(encoded_mapping, dict) else None + if isinstance(input_ids, list): + return [int(token_id) for token_id in input_ids if isinstance(token_id, int)] + return [] + + +def _markdown_fence_bad_words_ids(tokenizer: Any) -> list[list[int]]: + sequences: list[list[int]] = [] + seen: set[tuple[int, ...]] = set() + for source in MARKDOWN_FENCE_SUPPRESSION_TOKEN_SOURCES: + token_ids = _encode_suppression_sequence(tokenizer, source) + sequence_key = tuple(token_ids) + if len(token_ids) == 1 and sequence_key not in seen: + sequences.append(token_ids) + seen.add(sequence_key) + return sequences + + +def _decode_prediction_attempt( + *, + model: Any, + tokenizer: Any, + prompt: str, + max_new_tokens: int, + torch_module: Any, +) -> tuple[str, Any, Any]: + inputs: Any = tokenizer(prompt, return_tensors="pt").to(model.device) + bad_words_ids = _markdown_fence_bad_words_ids(tokenizer) + generation_kwargs: dict[str, Any] = { + **inputs, + "max_new_tokens": max_new_tokens, + "do_sample": False, + "pad_token_id": tokenizer.eos_token_id, + } + if bad_words_ids: + generation_kwargs["bad_words_ids"] = bad_words_ids + with torch_module.no_grad(): + generated: Any = model.generate(**generation_kwargs) + new_tokens = generated[0][inputs["input_ids"].shape[-1] :] + decoded_value = tokenizer.decode(new_tokens, skip_special_tokens=True) + decoded = decoded_value if isinstance(decoded_value, str) else str(decoded_value) + return decoded, new_tokens, inputs + + +def _merge_and_unload_if_available(model: Any) -> Any: + merge_and_unload = getattr(model, "merge_and_unload", None) + if callable(merge_and_unload): + return merge_and_unload() + return model + + +def _build_schema_guard( + *, + raw_status: dict[str, Any], + retry_enabled: bool, + retry_attempted: bool, + retry_status: dict[str, Any] | None, +) -> dict[str, Any]: + retry_schema_valid = None if retry_status is None else retry_status["schema_valid"] + if raw_status["schema_valid"]: + validated_source = "raw_attempt" + validated_valid = True + elif retry_status is not None and retry_status["schema_valid"]: + validated_source = "retry_attempt" + validated_valid = True + else: + validated_source = "none" + validated_valid = False + return { + "raw_attempt_schema_valid": raw_status["schema_valid"], + "raw_attempt_validation_error": raw_status["validation_error"], + "raw_attempt_missing_required_fields": raw_status["missing_required_fields"], + "retry_enabled": retry_enabled, + "retry_attempted": retry_attempted, + "retry_attempt_schema_valid": retry_schema_valid, + "retry_attempt_validation_error": None if retry_status is None else retry_status["validation_error"], + "validated_output_schema_valid": validated_valid, + "validated_output_source": validated_source, + } + + +def _sanitize_prediction_object(value: dict[str, Any]) -> dict[str, Any]: + return { + _sanitize_decoded_prediction_text(key): _sanitize_prediction_value(item) + for key, item in value.items() + } + + +def _sanitize_prediction_value(value: Any) -> Any: + if isinstance(value, str): + return _sanitize_decoded_prediction_text(value) + if isinstance(value, dict): + return _sanitize_prediction_object(value) + if isinstance(value, list): + return [_sanitize_prediction_value(item) for item in value] + return value + + +def _sanitize_decoded_prediction_text(text: str) -> str: + sanitized = PRIVATE_METADATA_PATH_RE.sub("", text) + sanitized = PRIVATE_PATH_RE.sub("", sanitized) + sanitized = PRIVATE_IP_RE.sub("", sanitized) + sanitized = SECRET_RE.sub("", sanitized) + return sanitized + + +def _run_real_sft_prediction( + config: dict[str, Any], + rows: list[SFTDatasetRow], + output_path: Path, + *, + sidecar_paths: dict[str, Path] | None = None, +) -> int: + import torch + from peft import PeftModel # type: ignore[import-not-found, unused-ignore] + from transformers import AutoModelForCausalLM, AutoTokenizer + + base_model = _runtime_base_model(config) + adapter_path = str(config["adapter_path"]) + tokenizer: Any = AutoTokenizer.from_pretrained(base_model, trust_remote_code=True) + dtype = torch.float16 if torch.cuda.is_available() else torch.float32 + sft_adapter_path = config.get("sft_adapter_path") + if sft_adapter_path: + model: Any = AutoModelForCausalLM.from_pretrained( + base_model, torch_dtype=dtype, trust_remote_code=True, + ) + model = PeftModel.from_pretrained(model, sft_adapter_path) + model = _merge_and_unload_if_available(model) + if hasattr(model, "peft_config"): + del model.peft_config + model = PeftModel.from_pretrained(model, adapter_path) + model = _merge_and_unload_if_available(model) + model = model.to("cuda" if torch.cuda.is_available() else "cpu") + else: + model = AutoModelForCausalLM.from_pretrained( + base_model, device_map="auto", torch_dtype=dtype, trust_remote_code=True, + ) + model = PeftModel.from_pretrained(model, adapter_path) + model = _merge_and_unload_if_available(model) + model.eval() + max_new_tokens = int(config.get("max_new_tokens", 256)) + schema_retry_enabled = bool(config.get("schema_retry_enabled", True)) + prompt_rows: list[dict[str, Any]] = [] + raw_rows: list[dict[str, Any]] = [] + trace_rows: list[dict[str, Any]] = [] + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8") as handle: + for row in rows: + prediction_input = PredictionInput.from_sft_row(row) + prompt = format_sft_prediction_prompt(prediction_input, tokenizer=tokenizer) + prompt_rows.append(_prompt_snapshot_row(row, prompt)) + decoded, new_tokens, _ = _decode_prediction_attempt( + model=model, + tokenizer=tokenizer, + prompt=prompt, + max_new_tokens=max_new_tokens, + torch_module=torch, + ) + raw_prediction = _extract_strict_json_object(decoded) + raw_status = _schema_guard_status(raw_prediction) + retry_status: dict[str, Any] | None = None + retry_prediction: Any = None + retry_decoded: str | None = None + retry_new_tokens: Any = None + retry_attempted = False + if schema_retry_enabled and not raw_status["schema_valid"]: + retry_attempted = True + retry_instruction = _schema_retry_prompt(prediction_input, raw_prediction, raw_status) + retry_prompt = format_schema_retry_prompt_text(retry_instruction, tokenizer=tokenizer) + retry_decoded, retry_new_tokens, _ = _decode_prediction_attempt( + model=model, + tokenizer=tokenizer, + prompt=retry_prompt, + max_new_tokens=max_new_tokens, + torch_module=torch, + ) + retry_prediction = _extract_strict_json_object(retry_decoded) + retry_status = _schema_guard_status(retry_prediction) + schema_guard = _build_schema_guard( + raw_status=raw_status, + retry_enabled=schema_retry_enabled, + retry_attempted=retry_attempted, + retry_status=retry_status, + ) + final_prediction = ( + retry_prediction if schema_guard["validated_output_source"] == "retry_attempt" else raw_prediction + ) + raw_rows.append( + _raw_decoded_summary_row( + row.id, + decoded, + schema_guard=schema_guard, + retry_decoded=retry_decoded, + ) + ) + trace_rows.append( + _generation_trace_row( + row_id=row.id, + attempt="raw_attempt", + prediction_source_kind="private_a100_adapter", + generated_tokens=new_tokens, + max_new_tokens=max_new_tokens, + eos_token_id=getattr(tokenizer, "eos_token_id", None), + ) + ) + if retry_attempted: + trace_rows.append( + _generation_trace_row( + row_id=row.id, + attempt="retry_attempt", + prediction_source_kind="private_a100_adapter", + generated_tokens=retry_new_tokens, + max_new_tokens=max_new_tokens, + eos_token_id=getattr(tokenizer, "eos_token_id", None), + ) + ) + record = { + "id": row.id, + "prediction": final_prediction, + "schema_guard": schema_guard, + "prediction_source_kind": "private_a100_adapter", + "provenance": {"public_safe": True, "source_id": row.provenance.get("source_id", row.id)}, + } + handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + if sidecar_paths is not None: + _write_prompt_snapshot( + prompt_rows, + sidecar_paths["prompt_snapshot"], + prediction_split=str(config.get("prediction_split", "all")), + decoding_policy=_sidecar_written_decoding_policy(config), + ) + _write_jsonl_records(sidecar_paths["raw_decoded_summary"], raw_rows) + _write_jsonl_records(sidecar_paths["generation_trace"], trace_rows) + return len(rows) + + +def _run_real_prediction_with_optional_sidecars( + config: dict[str, Any], + rows: list[SFTDatasetRow], + output_path: Path, + sidecar_paths: dict[str, Path], +) -> int: + return _run_real_sft_prediction(config, rows, output_path, sidecar_paths=sidecar_paths) + + +def run_sft_prediction_export( + config_path: Path, + manifest_path: Path, + output_path: Path, + *, + dry_run: bool = True, + fixture_mode: bool = False, +) -> dict[str, Any]: + config = _load_config(config_path) + sidecar_paths = _prediction_sidecar_paths(output_path) + metadata = _prediction_metadata_common( + config_path=config_path, + manifest_path=manifest_path, + output_path=output_path, + dry_run=dry_run, + fixture_mode=fixture_mode, + ) + if dry_run and not fixture_mode: + metadata["prediction_status"] = "prediction_skipped_no_opt_in" + return metadata + raw_rows = _load_sft_prediction_rows(manifest_path, split=str(config.get("prediction_split", "all"))) + rows, row_limit = _limited_sft_prediction_rows(raw_rows, config) + _record_sft_prediction_row_selection( + metadata, + rows=rows, + row_limit=row_limit, + loaded_rows_before_limit=len(raw_rows), + ) + if fixture_mode: + metadata["prediction_count"] = _write_fixture_predictions(rows, output_path) + _mark_sidecars_written(metadata) + _write_fixture_sidecars( + rows=rows, + output_path=output_path, + sidecar_paths=sidecar_paths, + prediction_split=str(config.get("prediction_split", "all")), + max_new_tokens=int(config.get("max_new_tokens", 256)), + decoding_policy=metadata["decoding_policy"], + ) + copy_shadow = _run_copy_backed_prediction_shadow_hook( + config=config, + config_path=config_path, + rows=rows, + output_path=output_path, + ) + if copy_shadow is not None: + metadata["copy_backed_shadow"] = copy_shadow + metadata["prediction_status"] = "fixture_predictions_written" + metadata["prediction_source_kind"] = "public_sample_contract_fixture" + metadata["notes"] = ( + "Fixture-mode predictions mirror public-sample target contracts to validate the evidence pipeline. " + "No private adapter artifacts were loaded." + ) + _write_prediction_metadata(output_path, metadata) + return metadata + adapter_path = config.get("adapter_path") + if not isinstance(adapter_path, str) or not adapter_path.strip(): + metadata["prediction_status"] = "prediction_blocked_missing_adapter" + metadata["prediction_source_kind"] = "none" + metadata["prediction_gate"]["will_run_private_prediction"] = False + metadata["notes"] = "Private prediction was blocked because no adapter_path was configured." + return metadata + if not bool(config.get("allow_private_prediction")): + metadata["prediction_status"] = "prediction_blocked_by_config" + metadata["prediction_source_kind"] = "none" + metadata["prediction_gate"]["will_run_private_prediction"] = False + metadata["notes"] = "Private prediction was blocked because allow_private_prediction is not true." + return metadata + if "<" in adapter_path or ">" in adapter_path: + metadata["prediction_status"] = "prediction_blocked_by_adapter_template" + metadata["prediction_source_kind"] = "none" + metadata["prediction_gate"]["will_run_private_prediction"] = False + metadata["notes"] = "Private prediction was blocked because adapter_path is an unresolved template." + return metadata + if not _prediction_dependencies_available(): + return _write_private_prediction_unavailable(metadata) + metadata["prediction_count"] = _run_real_prediction_with_optional_sidecars(config, rows, output_path, sidecar_paths) + _mark_sidecars_written(metadata) + copy_shadow = _run_copy_backed_prediction_shadow_hook( + config=config, + config_path=config_path, + rows=rows, + output_path=output_path, + ) + if copy_shadow is not None: + metadata["copy_backed_shadow"] = copy_shadow + metadata["prediction_status"] = "private_adapter_predictions_written" + metadata["prediction_source_kind"] = "private_a100_adapter" + metadata["notes"] = ( + "Private A100 adapter predictions were written as sanitized public-sample contract prediction rows. " + "No checkpoints, adapters, raw logs, or private paths were copied into the prediction artifact." + ) + _write_prediction_metadata(output_path, metadata) + return metadata + + +def _loss_interpretation() -> dict[str, bool]: + return { + "loss_improvement_alone_proves_contract_learning": False, + "requires_assistant_loss_evidence": True, + } + + +def _label_provenance( + value: dict[str, Any] | str | None, + *, + source_kind: str, + real_training_path: bool, +) -> dict[str, Any]: + if isinstance(value, dict): + provenance = dict(value) + elif isinstance(value, str) and value: + provenance = {"source_kind": value} + else: + provenance = {"source_kind": source_kind} + provenance.setdefault("source_kind", source_kind) + provenance.setdefault("real_training_path", real_training_path) + return provenance + + +_REAL_LABEL_SOURCES = {"real_training_labels", "actual_training_labels", "trl_collator_labels"} +_NON_REAL_LABEL_SOURCE_KINDS = { + "fixture", + "fixture_collator", + "simulated", + "simulated_collator", + "unavailable", + "unspecified", +} + + +def _deduped_gaps(values: list[str]) -> list[str]: + return list(dict.fromkeys(values)) + + +def _real_training_label_provenance( + *, + label_source: str, + collator_status: str, + provenance: dict[str, Any], +) -> bool: + source_kind = str(provenance.get("source_kind", "unspecified")) + return ( + label_source in _REAL_LABEL_SOURCES + and collator_status == "labels_inspected" + and provenance.get("real_training_path") is True + and source_kind not in _NON_REAL_LABEL_SOURCE_KINDS + ) + + +def _fixture_or_simulated_label_provenance(label_source: str, provenance: dict[str, Any]) -> bool: + source_kind = str(provenance.get("source_kind", "unspecified")) + return source_kind in {"fixture", "fixture_collator", "simulated", "simulated_collator"} or label_source in { + "fixture_labels", + "fixture_collator_labels", + "simulated_labels", + "simulated_collator_labels", + } + + +def _true_label_mask_status( + *, + label_source: str, + provenance: dict[str, Any], + real_training_path: bool, +) -> str: + if real_training_path: + return "inspectable" + if _fixture_or_simulated_label_provenance(label_source, provenance): + return "fixture_only" + return "unavailable" + + +def _inspectable_label_evidence_gaps( + *, + label_source: str, + provenance: dict[str, Any], + explicit_provenance_supplied: bool, + real_training_path: bool, +) -> list[str]: + if real_training_path: + return [] + gaps = ["real_training_labels_not_inspected", "real_training_label_provenance_missing"] + if _fixture_or_simulated_label_provenance(label_source, provenance): + gaps.append("fixture_labels_not_real_training_proof") + if not explicit_provenance_supplied: + gaps.append("label_provenance_unspecified") + if provenance.get("real_training_path") is not True: + gaps.append("label_provenance_not_real_training_path") + return _deduped_gaps(gaps) + + +def _objective_unavailable( + reason: str, + *, + inspection_status: str = "dependency_unavailable", + dependency_unavailable: bool = True, + tokenizer_status: str = "unavailable", + tokenizer_template_status: str = "unavailable", + collator_status: str = "unavailable", + evidence_gaps: list[str] | None = None, +) -> dict[str, Any]: + resolved_gaps = list(evidence_gaps or []) + for gap in ("real_training_labels_not_inspected", "real_training_label_provenance_missing"): + if gap not in resolved_gaps: + resolved_gaps.append(gap) + return { + "inspection_status": inspection_status, + "dependency_unavailable": dependency_unavailable, + "unavailable_reason": reason, + "tokenizer_status": tokenizer_status, + "tokenizer_template_status": tokenizer_template_status, + "collator_status": collator_status, + "label_source": "unavailable", + "label_provenance": _label_provenance(None, source_kind="unavailable", real_training_path=False), + "label_tensor_available": False, + "true_label_mask_status": "unavailable", + "prompt_token_count": None, + "assistant_token_count": None, + "prompt_tokens_masked": None, + "assistant_tokens_carry_loss": None, + "evidence_gaps": resolved_gaps, + "loss_interpretation": _loss_interpretation(), + } + + +def _flatten_offsets(value: Any) -> list[tuple[int, int]]: + offsets = _token_list(value) + normalized: list[tuple[int, int]] = [] + for item in offsets: + if isinstance(item, (list, tuple)) and len(item) >= 2: + normalized.append((int(item[0]), int(item[1]))) + return normalized + + +def _mapping_value(value: Any, key: str) -> Any: + if isinstance(value, dict): + return value.get(key) + return getattr(value, key, None) + + +def _tokenizer_template_status(tokenizer: Any | None) -> str: + if tokenizer is None: + return "unavailable" + chat_template = getattr(tokenizer, "chat_template", None) + if isinstance(chat_template, str) and chat_template.strip(): + return "template_available" + if callable(getattr(tokenizer, "apply_chat_template", None)): + return "template_callable" + return "fallback" + + +def _collator_labels( + *, + collator: Any | None, + encoded: Any, + assistant_start: int, + assistant_end: int, + offsets: list[tuple[int, int]], + row_id: str, +) -> tuple[list[Any], str, str, list[str]]: + if collator is None: + labels, evidence_gaps = _assistant_only_labels_from_encoded( + encoded=encoded, + offsets=offsets, + assistant_start=assistant_start, + assistant_end=assistant_end, + ) + if labels and not evidence_gaps: + return labels, "assistant_only_constructed_labels", "assistant_only_labels_constructed", [] + return [], "unavailable", "assistant_only_labels_unavailable", evidence_gaps + if not callable(collator): + return [], "unavailable", "not_callable", ["collator_not_callable"] + feature = dict(encoded) if isinstance(encoded, dict) else {"input_ids": _mapping_value(encoded, "input_ids")} + feature["label_provenance_row_id"] = row_id + feature["label_provenance_assistant_start"] = assistant_start + feature["label_provenance_assistant_end"] = assistant_end + try: + batch = collator([feature]) + except Exception: + return [], "unavailable", "error", ["collator_label_extraction_failed"] + labels = _token_list(_mapping_value(batch, "labels")) + if labels: + return labels, "trl_collator_labels", "labels_inspected", [] + return [], "unavailable", "labels_missing", ["label_tensor_unavailable"] + + +def _assistant_target_span(training_text: str, assistant_text: str) -> tuple[str, int | None, int | None]: + if not assistant_text: + return "assistant_span_unavailable", None, None + starts: list[int] = [] + next_start = training_text.find(assistant_text) + while next_start >= 0: + starts.append(next_start) + next_start = training_text.find(assistant_text, next_start + len(assistant_text)) + if not starts: + return "assistant_span_unavailable", None, None + if len(starts) > 1: + return "assistant_span_ambiguous", None, None + start = starts[0] + return "available", start, start + len(assistant_text) + + +def _assistant_only_labels_from_encoded( + *, + encoded: Any, + offsets: list[tuple[int, int]], + assistant_start: int, + assistant_end: int, +) -> tuple[list[Any], list[str]]: + input_ids = _token_list(_mapping_value(encoded, "input_ids")) + if not input_ids: + return [], ["input_ids_unavailable"] + if not offsets or len(input_ids) != len(offsets): + gaps = ["token_offsets_unavailable"] if not offsets else ["label_token_offset_length_mismatch"] + return [], gaps + + assistant_indices: set[int] = set() + boundary_overlap = False + for index, (start, end) in enumerate(offsets): + if start == end: + continue + if (start < assistant_start < end) or (start < assistant_end < end): + boundary_overlap = True + continue + if start >= assistant_start and end <= assistant_end: + assistant_indices.add(index) + + if boundary_overlap: + return [], ["assistant_span_token_boundary_unavailable"] + if not assistant_indices: + return [], ["assistant_target_tokens_unavailable"] + + return [ + token_id if index in assistant_indices else -100 + for index, token_id in enumerate(input_ids) + ], [] + + +def _assistant_only_training_record( + row: SFTDatasetRow, + tokenizer: Any, + *, + max_seq_length: int | None = None, +) -> dict[str, list[Any]]: + training_text = format_sft_training_text(row, tokenizer=tokenizer) + assistant_text = canonical_contract_json(row.target_contract) + span_status, assistant_start, assistant_end = _assistant_target_span(training_text, assistant_text) + if span_status != "available" or assistant_start is None or assistant_end is None: + raise ValueError(f"assistant-only SFT labels unavailable: {span_status}") + + encoded = tokenizer(training_text, return_offsets_mapping=True, add_special_tokens=False) + input_ids = _token_list(_mapping_value(encoded, "input_ids")) + offsets = _flatten_offsets(_mapping_value(encoded, "offset_mapping")) + labels, evidence_gaps = _assistant_only_labels_from_encoded( + encoded=encoded, + offsets=offsets, + assistant_start=assistant_start, + assistant_end=assistant_end, + ) + if not labels or evidence_gaps: + gaps = ",".join(evidence_gaps) if evidence_gaps else "label_tensor_unavailable" + raise ValueError(f"assistant-only SFT labels unavailable: {gaps}") + if max_seq_length is not None and len(input_ids) > max_seq_length: + raise ValueError("assistant-only SFT labels unavailable: max_seq_length_exceeded") + + attention_mask = _token_list(_mapping_value(encoded, "attention_mask")) + if len(attention_mask) != len(input_ids): + attention_mask = [1 for _ in input_ids] + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "labels": labels, + } + + +class _AssistantOnlyCausalLmDataCollator: + def __init__(self, tokenizer: Any, *, tensorize: bool = True) -> None: + self._tokenizer = tokenizer + self._tensorize = tensorize + + def __call__(self, features: list[dict[str, Any]]) -> dict[str, Any]: + pad_token_id = getattr(self._tokenizer, "pad_token_id", None) + if pad_token_id is None: + pad_token_id = getattr(self._tokenizer, "eos_token_id", 0) + max_length = max(len(_token_list(feature.get("input_ids"))) for feature in features) + batch: dict[str, list[list[Any]]] = {"input_ids": [], "attention_mask": [], "labels": []} + for feature in features: + input_ids = _token_list(feature.get("input_ids")) + seq_len = len(input_ids) + attention_mask = _token_list(feature.get("attention_mask"))[:seq_len] + labels = _token_list(feature.get("labels"))[:seq_len] + if len(attention_mask) < seq_len: + attention_mask = attention_mask + [1] * (seq_len - len(attention_mask)) + if len(labels) < seq_len: + labels = labels + [-100] * (seq_len - len(labels)) + pad_length = max_length - seq_len + batch["input_ids"].append(input_ids + [pad_token_id] * pad_length) + batch["attention_mask"].append(attention_mask + [0] * pad_length) + batch["labels"].append(labels + [-100] * pad_length) + if not self._tensorize: + return batch + import torch + + return {key: torch.tensor(value, dtype=torch.long) for key, value in batch.items()} + + +def _sft_trainer_tokenizer_kwargs(trainer_class: Any, tokenizer: Any) -> dict[str, Any]: + try: + parameters = inspect.signature(trainer_class.__init__).parameters + except (TypeError, ValueError): + return {"processing_class": tokenizer} + if "processing_class" in parameters: + return {"processing_class": tokenizer} + if "tokenizer" in parameters: + return {"tokenizer": tokenizer} + return {} + + +def inspect_sft_objective( + row: SFTDatasetRow, + *, + tokenizer: Any | None = None, + collator: Any | None = None, + label_source: str | None = None, + label_provenance: dict[str, Any] | str | None = None, +) -> dict[str, Any]: + if tokenizer is None: + if not _train_dependencies_available(): + return _objective_unavailable("train dependencies or tokenizer are not available in this runtime") + return { + **_objective_unavailable( + "tokenizer was not supplied for local non-heavy inspection", + inspection_status="tokenizer_unavailable", + tokenizer_status="unavailable", + tokenizer_template_status="unavailable", + collator_status="unavailable", + ), + "inspection_status": "tokenizer_unavailable", + } + + template_status = _tokenizer_template_status(tokenizer) + training_text = format_sft_training_text(row, tokenizer=tokenizer) + assistant_text = canonical_contract_json(row.target_contract) + span_status, assistant_start, assistant_end = _assistant_target_span(training_text, assistant_text) + if span_status != "available" or assistant_start is None or assistant_end is None: + return _objective_unavailable( + "assistant target span was not found in rendered training text", + inspection_status=span_status, + dependency_unavailable=False, + tokenizer_status="available", + tokenizer_template_status=template_status, + collator_status="unavailable" if collator is None else "not_inspected", + evidence_gaps=[span_status], + ) + + encoded = tokenizer(training_text, return_offsets_mapping=True, add_special_tokens=False) + offsets = _flatten_offsets(_mapping_value(encoded, "offset_mapping")) + labels, inferred_label_source, collator_status, label_evidence_gaps = _collator_labels( + collator=collator, + encoded=encoded, + assistant_start=assistant_start, + assistant_end=assistant_end, + offsets=offsets, + row_id=row.id, + ) + if not labels or not offsets or len(labels) != len(offsets): + gaps = ["label_tensor_unavailable", *label_evidence_gaps] + if not offsets: + gaps.append("token_offsets_unavailable") + return _objective_unavailable( + "labels or token offsets were not available from the inspected local path", + inspection_status="labels_unavailable", + dependency_unavailable=False, + tokenizer_status="available", + tokenizer_template_status=template_status, + collator_status=collator_status, + evidence_gaps=gaps, + ) + + prompt_indices = [index for index, (_, end) in enumerate(offsets) if end <= assistant_start] + assistant_indices = [ + index + for index, (start, end) in enumerate(offsets) + if start >= assistant_start and end <= assistant_end and start != end + ] + prompt_tokens_masked = bool(prompt_indices) and all(labels[index] == -100 for index in prompt_indices) + assistant_tokens_carry_loss = bool(assistant_indices) and any(labels[index] != -100 for index in assistant_indices) + resolved_label_source = label_source or inferred_label_source + resolved_label_provenance = _label_provenance( + label_provenance, + source_kind="unspecified" if label_provenance is None else "training_runtime", + real_training_path=False, + ) + real_training_path = _real_training_label_provenance( + label_source=resolved_label_source, + collator_status=collator_status, + provenance=resolved_label_provenance, + ) + true_label_mask_status = _true_label_mask_status( + label_source=resolved_label_source, + provenance=resolved_label_provenance, + real_training_path=real_training_path, + ) + evidence_gaps = _inspectable_label_evidence_gaps( + label_source=resolved_label_source, + provenance=resolved_label_provenance, + explicit_provenance_supplied=label_provenance is not None, + real_training_path=real_training_path, + ) + return { + "inspection_status": "inspectable", + "dependency_unavailable": False, + "row_id": row.id, + "tokenizer_status": "available", + "tokenizer_template_status": template_status, + "collator_status": collator_status, + "label_source": resolved_label_source, + "label_provenance": resolved_label_provenance, + "label_tensor_available": True, + "true_label_mask_status": true_label_mask_status, + "prompt_token_count": len(prompt_indices), + "assistant_token_count": len(assistant_indices), + "prompt_tokens_masked": prompt_tokens_masked, + "assistant_tokens_carry_loss": assistant_tokens_carry_loss, + "evidence_gaps": evidence_gaps, + "loss_interpretation": _loss_interpretation(), + } + + +def inspect_sft_objective_from_manifest(manifest_path: Path, *, split: str = "train") -> dict[str, Any]: + rows = _load_sft_training_rows(manifest_path, split=split) + if not rows: + result = _objective_unavailable(f"no SFT rows found for split={split}") + result["inspection_status"] = "row_unavailable" + return result + return inspect_sft_objective(rows[0], tokenizer=None) + + +def _runtime_label_provenance_claims() -> dict[str, bool]: + return { + "checkpoint_release": False, + "adapter_release": False, + "held_out_generalization_claim": False, + "production_readiness_claim": False, + "live_browser_benchmark_claim": False, + "model_recovery_claim": False, + } + + +def _runtime_label_provenance_artifact_policy() -> dict[str, bool]: + return { + "raw_rendered_prompts_written": False, + "raw_logs_copied_to_git": False, + "checkpoints_or_adapters_copied_to_git": False, + "private_paths_omitted": True, + } + + +def _inspect_runtime_sft_objective(row: SFTDatasetRow, config: dict[str, Any]) -> dict[str, Any]: + if not _runtime_label_dependencies_available(): + return _objective_unavailable("runtime tokenizer dependency is not available in this runtime") + base_model = config.get("base_model") + if not isinstance(base_model, str) or not base_model: + if isinstance(config.get("base_model_public_id"), str) and config.get("base_model_public_id"): + return _objective_unavailable( + "runtime base model must be a repo-external private local path for this check", + inspection_status="tokenizer_unavailable", + tokenizer_status="unavailable", + tokenizer_template_status="unavailable", + collator_status="unavailable", + evidence_gaps=["runtime_base_model_not_private_local_path"], + ) + return _objective_unavailable( + "runtime base model was not configured", + inspection_status="tokenizer_unavailable", + tokenizer_status="unavailable", + tokenizer_template_status="unavailable", + collator_status="unavailable", + evidence_gaps=["runtime_base_model_missing"], + ) + if "<" in base_model or ">" in base_model or not Path(base_model).expanduser().is_absolute(): + return _objective_unavailable( + "runtime base model must be a repo-external private local path for this check", + inspection_status="tokenizer_unavailable", + tokenizer_status="unavailable", + tokenizer_template_status="unavailable", + collator_status="unavailable", + evidence_gaps=["runtime_base_model_not_private_local_path"], + ) + try: + tokenizer_factory = cast(Any, globals().get("AutoTokenizer")) + if tokenizer_factory is None: + from transformers import AutoTokenizer + + tokenizer_factory = AutoTokenizer + + tokenizer = tokenizer_factory.from_pretrained(base_model, trust_remote_code=True, local_files_only=True) + except Exception: + return _objective_unavailable( + "runtime tokenizer could not be loaded; raw model/runtime error details remain private", + inspection_status="tokenizer_unavailable", + tokenizer_status="unavailable", + tokenizer_template_status="unavailable", + collator_status="unavailable", + evidence_gaps=["runtime_tokenizer_load_failed"], + ) + try: + max_seq_length = int(config.get("max_seq_length", 1024)) + training_record = _assistant_only_training_record(row, tokenizer, max_seq_length=max_seq_length) + training_collator = _AssistantOnlyCausalLmDataCollator(tokenizer, tensorize=False) + + def runtime_training_collator(features: list[dict[str, Any]]) -> dict[str, Any]: + return training_collator([training_record]) + + return inspect_sft_objective( + row, + tokenizer=tokenizer, + collator=runtime_training_collator, + label_source="actual_training_labels", + label_provenance={"source_kind": "private_training_runtime", "real_training_path": True}, + ) + except Exception: + return _objective_unavailable( + "runtime label inspection failed; raw tokenizer/collator error details remain private", + inspection_status="labels_unavailable", + dependency_unavailable=False, + tokenizer_status="available", + tokenizer_template_status=_tokenizer_template_status(tokenizer), + collator_status="unavailable", + evidence_gaps=["runtime_label_inspection_failed"], + ) + + +def _runtime_label_evidence_status(objective_inspection: dict[str, Any]) -> str: + provenance = objective_inspection.get("label_provenance") + source_kind = str(provenance.get("source_kind", "")) if isinstance(provenance, dict) else str(provenance or "") + real_training_path = isinstance(provenance, dict) and provenance.get("real_training_path") is True + if objective_inspection.get("true_label_mask_status") == "fixture_only" or source_kind in { + "fixture", + "fixture_collator", + "simulated", + "simulated_collator", + }: + return "fixture_only" + if ( + objective_inspection.get("inspection_status") == "inspectable" + and objective_inspection.get("label_tensor_available") is True + and objective_inspection.get("true_label_mask_status") == "inspectable" + and real_training_path + ): + return "labels_inspected" + if objective_inspection.get("label_tensor_available") is True: + return "labels_available_but_not_real_training_proof" + return "labels_unavailable" + + +def _runtime_label_metadata_base( + *, + config_path: Path, + manifest_path: Path, + output_path: Path, + split: str, + status: str, + config: dict[str, Any], + manifest_summary: dict[str, Any], + unresolved_fields: list[str], + run_runtime_check: bool, + output_root_policy: dict[str, Any], + evidence_gaps: list[str] | None = None, +) -> dict[str, Any]: + config_allows_runtime_check = bool(config.get("allow_runtime_label_provenance_check", False)) + private_override_required = bool(config.get("private_override_required", True)) + private_override_resolved = not unresolved_fields + return { + "evidence_kind": "sft_runtime_label_provenance_observed", + "stage": "sft_runtime_label_provenance_observed", + "generated_at": datetime.now(timezone.utc).isoformat(), + "config_path": _sanitize_training_metadata_value(config_path.as_posix()), + "dataset_manifest_path": _sanitize_training_metadata_value(manifest_path.as_posix()), + "dataset_manifest_id": manifest_summary["manifest_id"], + "manifest_counts": manifest_summary["manifest_counts"], + "manifest_public_safe": manifest_summary["manifest_public_safe"], + "split": split, + "metadata_path": _sanitize_training_metadata_value(output_path.as_posix()), + "runtime_source_kind": "private_a100_runtime", + "runtime_check_status": status, + "evidence_status": status, + "runtime_gate": { + "cli_requested_runtime_check": run_runtime_check, + "config_allow_runtime_label_provenance_check": config_allows_runtime_check, + "private_override_resolved": private_override_resolved, + "will_run_runtime_label_provenance_check": ( + run_runtime_check + and config_allows_runtime_check + and private_override_resolved + and output_root_policy.get("status") == "approved_private_root" + ), + }, + "private_override": { + "required": private_override_required, + "status": "resolved" if private_override_resolved else "unresolved", + "unresolved_fields": unresolved_fields, + "requirements": _sanitize_training_metadata_value(config.get("private_override_requirements", [])), + "public_placeholder": "", + }, + "output_root_policy": output_root_policy, + "dependency_policy": { + "policy": str(config.get("dependency_policy", "runtime_check_no_public_model_download_in_local_tests")), + "model_download_allowed": False, + "private_adapter_load_allowed": False, + "raw_private_logs_copied_to_git": False, + }, + "package_versions": _sanitized_package_versions(), + "label_tensor_available": False, + "true_label_mask_status": "unavailable", + "inspection_status": status, + "label_source": "unavailable", + "label_source_kind": "unavailable", + "label_provenance": {"source_kind": "unavailable", "real_training_path": False}, + "prompt_tokens_masked": None, + "assistant_tokens_carry_loss": None, + "evidence_gaps": _deduped_gaps(evidence_gaps or []), + "prior_artifacts": _sanitize_training_metadata_value(config.get("prior_artifacts", {})), + "release_status": "not_released", + "claims": _runtime_label_provenance_claims(), + "artifact_policy": _runtime_label_provenance_artifact_policy(), + } + + +def _maybe_write_runtime_label_metadata(output_path: Path, metadata: dict[str, Any]) -> None: + output_root_policy = metadata.get("output_root_policy", {}) + if not isinstance(output_root_policy, dict) or output_root_policy.get("status") != "approved_private_root": + return + write_json(output_path, metadata) + + +def run_sft_runtime_label_provenance_check( + config_path: Path, + manifest_path: Path, + *, + split: str = "train", + output_path: Path, + run_runtime_check: bool = False, + objective_inspector: Callable[[SFTDatasetRow, dict[str, Any]], dict[str, Any]] | None = None, +) -> dict[str, Any]: + config = _load_config(config_path) + manifest_summary = _manifest_metadata_without_dataset_load(manifest_path) + unresolved_fields = _unresolved_runtime_fields(config) + output_root_policy = _runtime_check_output_root_policy(config, output_path, unresolved_fields) + config_allows_runtime_check = bool(config.get("allow_runtime_label_provenance_check", False)) + + status = "executed_runtime_label_provenance_check" + gaps: list[str] = [] + if not run_runtime_check or not config_allows_runtime_check: + status = "skipped_no_runtime_opt_in" + gaps.extend(["runtime_check_not_executed", "runtime_opt_in_missing"]) + elif unresolved_fields: + status = "blocked_unresolved_private_override" + gaps.extend(["runtime_check_not_executed", "private_override_unresolved"]) + elif output_root_policy["status"] != "approved_private_root": + status = "blocked_output_outside_approved_root" + gaps.extend(["runtime_check_not_executed", "runtime_output_outside_approved_root"]) + + metadata = _runtime_label_metadata_base( + config_path=config_path, + manifest_path=manifest_path, + output_path=output_path, + split=split, + status=status, + config=config, + manifest_summary=manifest_summary, + unresolved_fields=unresolved_fields, + run_runtime_check=run_runtime_check, + output_root_policy=output_root_policy, + evidence_gaps=gaps, + ) + if status != "executed_runtime_label_provenance_check": + sanitized = _sanitize_training_metadata_value(metadata) + if not isinstance(sanitized, dict): + raise AssertionError("runtime label provenance metadata must be a mapping") + result = cast(dict[str, Any], sanitized) + _maybe_write_runtime_label_metadata(output_path, result) + return result + + rows = _load_sft_training_rows(manifest_path, split=split) + if not rows: + metadata["runtime_check_status"] = "blocked_no_sft_rows" + metadata["evidence_status"] = "blocked_no_sft_rows" + metadata["inspection_status"] = "row_unavailable" + metadata["evidence_gaps"] = ["runtime_check_not_executed", "sft_rows_unavailable"] + sanitized = _sanitize_training_metadata_value(metadata) + if not isinstance(sanitized, dict): + raise AssertionError("runtime label provenance metadata must be a mapping") + result = cast(dict[str, Any], sanitized) + _maybe_write_runtime_label_metadata(output_path, result) + return result + + inspector = objective_inspector or _inspect_runtime_sft_objective + objective_inspection = inspector(rows[0], config) + provenance = objective_inspection.get("label_provenance") + if not isinstance(provenance, dict): + provenance = _label_provenance( + provenance if isinstance(provenance, str) else None, + source_kind="unavailable", + real_training_path=False, + ) + evidence_status = _runtime_label_evidence_status(objective_inspection) + metadata.update( + { + "evidence_status": evidence_status, + "inspection_status": objective_inspection.get("inspection_status", "unknown"), + "tokenizer_status": objective_inspection.get("tokenizer_status", "unknown"), + "tokenizer_template_status": objective_inspection.get("tokenizer_template_status", "unknown"), + "collator_status": objective_inspection.get("collator_status", "unknown"), + "label_source": objective_inspection.get("label_source", "unavailable"), + "label_source_kind": str(provenance.get("source_kind", "unavailable")), + "label_provenance": dict(provenance), + "label_tensor_available": bool(objective_inspection.get("label_tensor_available", False)), + "true_label_mask_status": objective_inspection.get("true_label_mask_status", "unavailable"), + "prompt_token_count": objective_inspection.get("prompt_token_count"), + "assistant_token_count": objective_inspection.get("assistant_token_count"), + "prompt_tokens_masked": objective_inspection.get("prompt_tokens_masked"), + "assistant_tokens_carry_loss": objective_inspection.get("assistant_tokens_carry_loss"), + "evidence_gaps": _deduped_gaps( + [str(gap) for gap in objective_inspection.get("evidence_gaps", []) if gap] + ), + "loss_interpretation": _sanitize_training_metadata_value( + objective_inspection.get("loss_interpretation", _loss_interpretation()) + ), + "notes": ( + "Runtime label provenance metadata records objective-path evidence only. It is not a checkpoint " + "release, adapter release, held-out generalization claim, production-readiness claim, or " + "live-browser benchmark improvement claim." + ), + } + ) + sanitized = _sanitize_training_metadata_value(metadata) + if not isinstance(sanitized, dict): + raise AssertionError("runtime label provenance metadata must be a mapping") + result = cast(dict[str, Any], sanitized) + write_json(output_path, result) + return result + + +def _load_dpo_training_pairs(manifest_path: Path, split: str) -> list[DPOPair]: + summary = _manifest_load_summary(manifest_path, "dpo") + dataset_path = summary["dataset_path"] + if dataset_path is None: + return [] + return [pair for pair in (DPOPair(**record) for record in read_jsonl(Path(dataset_path))) if pair.split == split] + + +def _run_real_sft(metadata: dict[str, Any], config: dict[str, Any], manifest_path: Path, output_dir: Path) -> None: + from datasets import Dataset # type: ignore[import-not-found, unused-ignore] + from transformers import AutoModelForCausalLM, AutoTokenizer + from trl import SFTTrainer # type: ignore[import-not-found, unused-ignore] + + rows = _record_sft_training_selection_from_config(metadata, config, manifest_path) + base_model = _runtime_base_model(config) + tokenizer = AutoTokenizer.from_pretrained(base_model) + max_seq_length = int(config.get("max_seq_length", 1024)) + records = [_assistant_only_training_record(row, tokenizer, max_seq_length=max_seq_length) for row in rows] + dataset = Dataset.from_list(records) + model = AutoModelForCausalLM.from_pretrained(base_model) + trainer = SFTTrainer( + model=model, + train_dataset=dataset, + args=_training_arguments(config, output_dir), + peft_config=_lora_config(config), + data_collator=_AssistantOnlyCausalLmDataCollator(tokenizer), + **_sft_trainer_tokenizer_kwargs(SFTTrainer, tokenizer), + ) + train_result = trainer.train() + _record_sft_training_budget_metadata( + metadata, + config=config, + train_row_count=len(rows), + records=records, + trainer=trainer, + train_result=train_result, + ) + trainer.model.save_pretrained(metadata["adapter_path"]) + + +def _run_real_dpo(metadata: dict[str, Any], config: dict[str, Any], manifest_path: Path, output_dir: Path) -> None: + from datasets import Dataset # type: ignore[import-not-found, unused-ignore] + from peft import PeftModel # type: ignore[import-not-found, unused-ignore] + from transformers import AutoModelForCausalLM, AutoTokenizer + from trl import DPOConfig, DPOTrainer # type: ignore[import-not-found, unused-ignore] + + pairs = _load_dpo_training_pairs(manifest_path, split=str(config.get("dataset_split", "train"))) + dataset = Dataset.from_list( + [ + { + "prompt": json.dumps(format_dpo_pair(pair)["prompt"], ensure_ascii=False), + "chosen": format_dpo_pair(pair)["chosen"], + "rejected": format_dpo_pair(pair)["rejected"], + } + for pair in pairs + ] + ) + tokenizer = AutoTokenizer.from_pretrained(config["base_model"], trust_remote_code=True) + sft_adapter_path = config.get("sft_adapter_path") + if sft_adapter_path: + base_model = AutoModelForCausalLM.from_pretrained(config["base_model"], trust_remote_code=True) + model = PeftModel.from_pretrained(base_model, sft_adapter_path) + model = model.merge_and_unload() + else: + model = AutoModelForCausalLM.from_pretrained(config["base_model"], trust_remote_code=True) + + dpo_config = DPOConfig( + output_dir=str(output_dir), + per_device_train_batch_size=int(config.get("per_device_train_batch_size", 2)), + gradient_accumulation_steps=int(config.get("gradient_accumulation_steps", 4)), + num_train_epochs=int(config.get("num_train_epochs", 2)), + learning_rate=float(config.get("learning_rate", 5e-5)), + warmup_ratio=float(config.get("warmup_ratio", 0.1)), + beta=float(config.get("beta", 0.1)), + max_length=int(config.get("max_seq_length", 1024)), + logging_steps=int(config.get("logging_steps", 1)), + save_strategy=str(config.get("save_strategy", "no")), + report_to=[], + remove_unused_columns=False, + ) + trainer = DPOTrainer( + model=model, + args=dpo_config, + train_dataset=dataset, + processing_class=tokenizer, + peft_config=_lora_config(config), + ) + trainer.train() + trainer.model.save_pretrained(metadata["adapter_path"]) + + +def run_sft(config_path: Path, manifest_path: Path, output_dir: Path, dry_run: bool = True) -> dict[str, Any]: + config = _load_config(config_path) + metadata = _metadata_common( + stage="sft", + config_path=config_path, + manifest_path=manifest_path, + output_dir=output_dir, + dry_run=dry_run, + ) + _record_sft_training_selection_from_config(metadata, config, manifest_path) + if not dry_run and not _heavy_training_allowed(config): + return _write_training_skipped(metadata, "sft") + if not dry_run and not _output_dir_within_configured_root(config, output_dir): + return _write_training_blocked_by_output_policy(metadata, "sft") + if not dry_run and not _train_dependencies_available(): + return _write_training_plan(metadata, "sft") + metadata["trainer_available"] = not dry_run + if not dry_run: + try: + _run_real_sft(metadata, config, manifest_path, output_dir) + except Exception as exc: + _write_training_failed(metadata, "sft", exc) + raise + metadata["release_status"] = "not_released" + metadata["training_status"] = "training_completed" + metadata["notes"] = "SFT training ran locally; adapter metadata is not a public checkpoint release." + write_json(Path(metadata["metadata_path"]), metadata) + return metadata + + +def run_dpo(config_path: Path, manifest_path: Path, output_dir: Path, dry_run: bool = True) -> dict[str, Any]: + config = _load_config(config_path) + metadata = _metadata_common( + stage="dpo", + config_path=config_path, + manifest_path=manifest_path, + output_dir=output_dir, + dry_run=dry_run, + ) + metadata["sft_model_ref"] = config.get("sft_model_ref") + metadata["dpo_initialization"] = "base_model_lora" + if not dry_run and not _heavy_training_allowed(config): + return _write_training_skipped(metadata, "dpo") + if not dry_run and not _output_dir_within_configured_root(config, output_dir): + return _write_training_blocked_by_output_policy(metadata, "dpo") + if not dry_run and not _train_dependencies_available(): + return _write_training_plan(metadata, "dpo") + metadata["trainer_available"] = not dry_run + if not dry_run: + try: + _run_real_dpo(metadata, config, manifest_path, output_dir) + except Exception as exc: + _write_training_failed(metadata, "dpo", exc) + raise + metadata["release_status"] = "not_released" + metadata["training_status"] = "training_completed" + metadata["notes"] = "DPO training ran locally; adapter metadata is not a public checkpoint release." + write_json(Path(metadata["metadata_path"]), metadata) + return metadata diff --git a/src/voice2task/cli/train.py b/src/voice2task/cli/train.py index 426a236..f8105e4 100644 --- a/src/voice2task/cli/train.py +++ b/src/voice2task/cli/train.py @@ -8,9 +8,11 @@ from voice2task.training import ( inspect_sft_objective_from_manifest, prepare_sft_runtime_label_provenance, + public_training_metadata, run_dpo, run_sft, run_sft_prediction_export, + run_sft_preflight, run_sft_runtime_label_provenance_check, ) @@ -26,6 +28,10 @@ def build_parser() -> argparse.ArgumentParser: mode = subparser.add_mutually_exclusive_group() mode.add_argument("--dry-run", dest="dry_run", action="store_true", default=True) mode.add_argument("--run-training", dest="dry_run", action="store_false") + preflight = subcommands.add_parser("sft-preflight") + preflight.add_argument("--config", type=Path, required=True) + preflight.add_argument("--manifest", type=Path, required=True) + preflight.add_argument("--output-dir", type=Path, required=True) prediction = subcommands.add_parser("sft-predict") prediction.add_argument("--config", type=Path, required=True) prediction.add_argument("--manifest", type=Path, required=True) @@ -53,32 +59,45 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) - if args.command == "sft": - metadata = run_sft(args.config, args.manifest, args.output_dir, dry_run=args.dry_run) - elif args.command == "dpo": - metadata = run_dpo(args.config, args.manifest, args.output_dir, dry_run=args.dry_run) - elif args.command == "sft-predict": - metadata = run_sft_prediction_export( - args.config, - args.manifest, - args.output, - dry_run=args.dry_run, - fixture_mode=args.fixture_mode, - ) - elif args.command == "sft-inspect-objective": - metadata = inspect_sft_objective_from_manifest(args.manifest, split=args.split) - elif args.command == "sft-prepare-runtime-label-provenance": - metadata = prepare_sft_runtime_label_provenance(args.config, args.manifest, metadata_path=args.output) - elif args.command == "sft-runtime-label-provenance-check": - metadata = run_sft_runtime_label_provenance_check( - args.config, - args.manifest, - split=args.split, - output_path=args.output, - run_runtime_check=args.run_runtime_check, - ) - else: - raise AssertionError(f"unhandled command: {args.command}") + try: + if args.command == "sft": + metadata = run_sft(args.config, args.manifest, args.output_dir, dry_run=args.dry_run) + elif args.command == "sft-preflight": + metadata = run_sft_preflight(args.config, args.manifest, args.output_dir) + elif args.command == "dpo": + metadata = run_dpo(args.config, args.manifest, args.output_dir, dry_run=args.dry_run) + elif args.command == "sft-predict": + metadata = run_sft_prediction_export( + args.config, + args.manifest, + args.output, + dry_run=args.dry_run, + fixture_mode=args.fixture_mode, + ) + elif args.command == "sft-inspect-objective": + metadata = inspect_sft_objective_from_manifest(args.manifest, split=args.split) + elif args.command == "sft-prepare-runtime-label-provenance": + metadata = prepare_sft_runtime_label_provenance(args.config, args.manifest, metadata_path=args.output) + elif args.command == "sft-runtime-label-provenance-check": + metadata = run_sft_runtime_label_provenance_check( + args.config, + args.manifest, + split=args.split, + output_path=args.output, + run_runtime_check=args.run_runtime_check, + ) + else: + raise AssertionError(f"unhandled command: {args.command}") + except Exception: + metadata = { + "schema_version": "voice2task-training-result-v1", + "training_status": "training_failed", + "blockers": ["TRAINING_RUNTIME_ERROR"], + } + print(json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True)) + return 1 + if args.command in {"sft", "dpo"}: + metadata = public_training_metadata(metadata) if args.command == "sft-inspect-objective" and args.output: write_json(args.output, metadata) elif args.command == "sft-prepare-runtime-label-provenance": @@ -88,6 +107,10 @@ def main(argv: list[str] | None = None) -> int: print(json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True)) else: print(json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True)) + if args.command == "sft-preflight": + return 0 if metadata.get("ready") is True else 1 + if args.command in {"sft", "dpo"}: + return 0 if metadata.get("training_status") in {"dry_run", "training_completed"} else 1 return 0 diff --git a/src/voice2task/contract_compiler_causal_audit.py b/src/voice2task/contract_compiler_causal_audit.py index 7f082ea..05714d8 100644 --- a/src/voice2task/contract_compiler_causal_audit.py +++ b/src/voice2task/contract_compiler_causal_audit.py @@ -72,6 +72,10 @@ } HISTORICAL_SOURCE_SNAPSHOT_OVERRIDES = { + "src/voice2task/training.py": ( + "reports/public-sample/contract-compiler-v2-causal-boundary/source-snapshots/" + "training.978e2df42be7b1e020c5215febaf843a527b0fb96469273c93b66ce20b62db3c.py" + ), "CONTEXT.md": ( "reports/public-sample/contract-compiler-v2-causal-boundary/source-snapshots/" "CONTEXT.2ffc67d81be8b3e482555efd23db5b0bf60239eb4ef4d9e24514cae24ea1009f.md" diff --git a/src/voice2task/training.py b/src/voice2task/training.py index 48e5b71..fd3a7f2 100644 --- a/src/voice2task/training.py +++ b/src/voice2task/training.py @@ -1,12 +1,18 @@ from __future__ import annotations import hashlib +import importlib import importlib.util import inspect import json +import math +import os import re +import shutil +import subprocess import sys from collections.abc import Callable +from dataclasses import dataclass from datetime import datetime, timezone from importlib.metadata import PackageNotFoundError, version from pathlib import Path @@ -118,6 +124,13 @@ def _gpu_selection_policy(config: dict[str, Any]) -> dict[str, str]: PRIVATE_DECODED_PATH_RE = re.compile(rf"({_MNT_DATA_PREFIX}/[^\s\"')]+)") PRIVATE_METADATA_PATH_RE = re.compile(r"(/(?:mnt/data|Users|root|tmp|private)/[^\s\"')]+)") MARKDOWN_FENCE_SUPPRESSION_TOKEN_SOURCES = ("```", "```json", "```JSON") +SFT_MINIMUM_PYTHON = (3, 10) +FORMAL_PUBLIC_MANIFEST_RELATIVE_PATH = Path( + "data/public-samples/manifest_public_sample.json" +) +FORMAL_PUBLIC_SFT_RELATIVE_PATH = Path("data/public-samples/sft_public_sample.jsonl") +FORMAL_PUBLIC_MANIFEST_ID = "public-sample-20260619T090925Z" +PRIVATE_SFT_CONFIG_DIRECTORY = Path("data/local-private/runtime") def _public_display_path(value: Path | str, placeholder: str) -> str: @@ -164,6 +177,13 @@ def _sanitize_training_metadata_value(value: Any) -> Any: return value +def public_training_metadata(metadata: dict[str, Any]) -> dict[str, Any]: + sanitized = _sanitize_training_metadata_value(metadata) + if not isinstance(sanitized, dict): + raise AssertionError("training metadata must remain a mapping after sanitization") + return cast(dict[str, Any], sanitized) + + def _manifest_metadata_without_dataset_load(manifest_path: Path) -> dict[str, Any]: manifest = read_json(manifest_path) return { @@ -376,14 +396,24 @@ def _heavy_training_gate(config: dict[str, Any], dry_run: bool) -> dict[str, boo def _output_dir_within_configured_root(config: dict[str, Any], output_dir: Path) -> bool: - output_root = config.get("output_root") - if not isinstance(output_root, str) or not output_root: - return True - if "<" in output_root or ">" in output_root: - return False - candidate = output_dir.expanduser() - root = Path(output_root).expanduser() - if not candidate.is_absolute() or not root.is_absolute(): + return bool(validate_sft_output_policy(config, output_dir)["ready"]) + + +def _path_has_symlink_below(root: Path, candidate: Path) -> bool: + try: + relative = candidate.relative_to(root) + except ValueError: + return candidate.is_symlink() + current = root + for part in relative.parts: + current = current / part + if current.is_symlink(): + return True + return False + + +def _strict_descendant(candidate: Path, root: Path) -> bool: + if candidate == root: return False try: candidate.relative_to(root) @@ -392,6 +422,1039 @@ def _output_dir_within_configured_root(config: dict[str, Any], output_dir: Path) return True +def _nearest_existing_parent(candidate: Path) -> Path: + current = candidate.parent + while not current.exists(): + current = current.parent + return current + + +class SFTOutputPolicyError(RuntimeError): + def __init__(self, blockers: list[str]) -> None: + self.blockers = list(dict.fromkeys(blockers)) + super().__init__(",".join(self.blockers)) + + +def validate_sft_output_policy( + config: dict[str, Any], + output_dir: Path, + *, + repo_root: Path | None = None, +) -> dict[str, Any]: + """Return a public-safe, no-write decision for a real SFT output directory.""" + blockers: list[str] = [] + try: + configured_minimum_free_disk_gib = float(config.get("min_free_disk_gib", 20)) + except (TypeError, ValueError): + configured_minimum_free_disk_gib = 20.0 + raw_root = config.get("output_root") + if not isinstance(raw_root, str) or not raw_root or "<" in raw_root or ">" in raw_root: + blockers.append("OUTPUT_ROOT_MISSING") + return { + "ready": False, + "blockers": blockers, + "root_path_sha256": None, + "output_path_sha256": None, + "writable": False, + "free_disk_gib": None, + "minimum_free_disk_gib": configured_minimum_free_disk_gib, + } + + root = Path(raw_root).expanduser() + candidate = output_dir.expanduser() + if not root.is_absolute(): + blockers.append("OUTPUT_ROOT_NOT_ABSOLUTE") + if not candidate.is_absolute(): + blockers.append("OUTPUT_DIR_NOT_ABSOLUTE") + if blockers: + return { + "ready": False, + "blockers": blockers, + "root_path_sha256": None, + "output_path_sha256": None, + "writable": False, + "free_disk_gib": None, + "minimum_free_disk_gib": configured_minimum_free_disk_gib, + } + + try: + if root.is_symlink(): + blockers.append("OUTPUT_ROOT_SYMLINK") + if not root.exists() or not root.is_dir(): + blockers.append("OUTPUT_ROOT_NOT_DIRECTORY") + except OSError: + blockers.append("OUTPUT_FILESYSTEM_UNAVAILABLE") + if blockers: + return { + "ready": False, + "blockers": blockers, + "root_path_sha256": None, + "output_path_sha256": None, + "writable": False, + "free_disk_gib": None, + "minimum_free_disk_gib": configured_minimum_free_disk_gib, + } + + try: + root_resolved = root.resolve(strict=True) + candidate_resolved = candidate.resolve(strict=False) + if _path_has_symlink_below(root, candidate): + blockers.append("OUTPUT_PATH_SYMLINK") + candidate_is_descendant = _strict_descendant(candidate_resolved, root_resolved) + if not candidate_is_descendant: + blockers.append("OUTPUT_PATH_OUTSIDE_ROOT") + if candidate_is_descendant and candidate.exists() and not candidate.is_symlink(): + if not candidate.is_dir() or any(candidate.iterdir()): + blockers.append("OUTPUT_DIRECTORY_NOT_EMPTY") + else: + blockers.append("OUTPUT_DIRECTORY_EXISTS") + + if repo_root is not None: + resolved_repo = repo_root.resolve(strict=False) + if _strict_descendant(candidate_resolved, resolved_repo) or candidate_resolved == resolved_repo: + blockers.append("OUTPUT_PATH_GIT_TRACKED") + except (OSError, RuntimeError): + return { + "ready": False, + "blockers": ["OUTPUT_FILESYSTEM_UNAVAILABLE"], + "root_path_sha256": None, + "output_path_sha256": None, + "writable": False, + "free_disk_gib": None, + "minimum_free_disk_gib": configured_minimum_free_disk_gib, + } + + writable = False + free_disk_gib: float | None = None + try: + writable_parent = _nearest_existing_parent(candidate_resolved) + writable = os.access(writable_parent, os.W_OK) + except OSError: + writable_parent = root_resolved + blockers.append("OUTPUT_FILESYSTEM_UNAVAILABLE") + if "OUTPUT_FILESYSTEM_UNAVAILABLE" not in blockers and not writable: + blockers.append("OUTPUT_NOT_WRITABLE") + try: + minimum_free_disk_gib = configured_minimum_free_disk_gib + free_disk_gib = shutil.disk_usage(writable_parent).free / float(1024**3) + except (OSError, TypeError, ValueError): + minimum_free_disk_gib = 20.0 + blockers.append("OUTPUT_DISK_CHECK_FAILED") + if free_disk_gib is not None and free_disk_gib < minimum_free_disk_gib: + blockers.append("OUTPUT_DISK_INSUFFICIENT") + + return { + "ready": not blockers, + "blockers": list(dict.fromkeys(blockers)), + "root_path_sha256": _sha256_text(root_resolved.as_posix()), + "output_path_sha256": _sha256_text(candidate_resolved.as_posix()), + "writable": writable, + "free_disk_gib": round(free_disk_gib, 3) if free_disk_gib is not None else None, + "minimum_free_disk_gib": minimum_free_disk_gib, + } + + +def _claim_sft_output_directory( + config: dict[str, Any], + output_dir: Path, + *, + repo_root: Path | None = None, + expected_output_path_sha256: str | None = None, +) -> None: + """Exclusively claim and revalidate a previously unused SFT output directory.""" + initial = validate_sft_output_policy(config, output_dir, repo_root=repo_root) + if initial.get("ready") is not True: + raise SFTOutputPolicyError([str(code) for code in initial.get("blockers", [])]) + expected_hash = expected_output_path_sha256 or initial.get("output_path_sha256") + try: + output_dir.mkdir(parents=True, exist_ok=False) + except FileExistsError as exc: + raise SFTOutputPolicyError(["OUTPUT_DIRECTORY_EXISTS"]) from exc + except OSError as exc: + raise SFTOutputPolicyError(["OUTPUT_CREATE_FAILED"]) from exc + + claimed = validate_sft_output_policy(config, output_dir, repo_root=repo_root) + claimed_blockers = [str(code) for code in claimed.get("blockers", [])] + if claimed_blockers != ["OUTPUT_DIRECTORY_EXISTS"]: + raise SFTOutputPolicyError(claimed_blockers or ["OUTPUT_PATH_CHANGED"]) + if not isinstance(expected_hash, str) or claimed.get("output_path_sha256") != expected_hash: + raise SFTOutputPolicyError(["OUTPUT_PATH_CHANGED"]) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _git_repository_root() -> Path | None: + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + value = result.stdout.strip() + return Path(value) if value else None + + +def _git_repository_root_for_manifest(manifest_path: Path) -> Path | None: + anchor = manifest_path.expanduser().parent + try: + result = subprocess.run( + ["git", "-C", anchor.as_posix(), "rev-parse", "--show-toplevel"], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, RuntimeError, subprocess.SubprocessError): + return None + value = result.stdout.strip() if result.returncode == 0 else "" + if not value: + return None + try: + repo_root = Path(value).expanduser().resolve(strict=True) + manifest_path.expanduser().resolve(strict=False).relative_to(repo_root) + except (OSError, RuntimeError, ValueError): + return None + return repo_root + + +def _probe_sft_git(root: Path) -> tuple[dict[str, Any], list[str]]: + facts: dict[str, Any] = {"commit_sha": None, "tracked_worktree_clean": False} + try: + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=root, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + status = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=no"], + cwd=root, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, RuntimeError, subprocess.SubprocessError): + return facts, ["GIT_PROBE_FAILED"] + blockers: list[str] = [] + sha = commit.stdout.strip() if commit.returncode == 0 else "" + if not re.fullmatch(r"[0-9a-f]{40}", sha): + blockers.append("GIT_COMMIT_UNAVAILABLE") + else: + facts["commit_sha"] = sha + clean = status.returncode == 0 and not status.stdout.strip() + facts["tracked_worktree_clean"] = clean + if not clean: + blockers.append("WORKTREE_NOT_CLEAN") + return facts, blockers + + +def _installed_module_version(module_name: str, module: Any) -> str: + module_version = getattr(module, "__version__", None) + if isinstance(module_version, str) and module_version: + return module_version + try: + return version(module_name) + except PackageNotFoundError: + return "unknown" + + +def _probe_sft_dependencies() -> tuple[dict[str, Any], list[str]]: + required = ("torch", "accelerate", "datasets", "peft", "transformers", "trl") + missing: list[str] = [] + versions: dict[str, str] = {} + for module_name in required: + try: + module = importlib.import_module(module_name) + except Exception: + missing.append(module_name) + continue + versions[module_name] = _installed_module_version(module_name, module) + + python_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" + facts: dict[str, Any] = { + "python": python_version, + "python_requirement": ">=3.10", + "versions": versions, + "missing": missing, + "pip_check": "not_run", + } + blockers: list[str] = [] + if sys.version_info[:2] < SFT_MINIMUM_PYTHON: + blockers.append("DEPENDENCY_CONFLICT") + if missing: + blockers.append("DEPENDENCY_MISSING") + try: + pip_check = subprocess.run( + [sys.executable, "-m", "pip", "check"], + check=False, + capture_output=True, + text=True, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + pip_check = None + if pip_check is None or pip_check.returncode != 0: + facts["pip_check"] = "conflict" + blockers.append("DEPENDENCY_CONFLICT") + else: + facts["pip_check"] = "ok" + return facts, list(dict.fromkeys(blockers)) + + +def _probe_sft_gpu() -> tuple[dict[str, Any], list[str]]: + selected = os.environ.get("CUDA_VISIBLE_DEVICES") + selected_parts = [part.strip() for part in selected.split(",") if part.strip()] if selected else [] + facts: dict[str, Any] = { + "explicit_selection": bool(selected_parts), + "visible_device_count": 0, + "name": None, + "compute_capability": None, + "total_memory_gib": None, + "minimum_memory_gib": 35.0, + "cuda_version": None, + "bf16_supported": False, + } + blockers: list[str] = [] + if len(selected_parts) != 1 or selected_parts == ["-1"]: + blockers.append("GPU_SELECTION_NOT_EXPLICIT" if not selected_parts else "GPU_SELECTION_NOT_SINGLE") + try: + torch = importlib.import_module("torch") + except Exception: + return facts, list(dict.fromkeys([*blockers, "CUDA_UNAVAILABLE"])) + cuda = getattr(torch, "cuda", None) + if cuda is None: + return facts, list(dict.fromkeys([*blockers, "CUDA_UNAVAILABLE"])) + try: + cuda_available = bool(cuda.is_available()) + except Exception: + return facts, list(dict.fromkeys([*blockers, "CUDA_PROBE_FAILED"])) + if not cuda_available: + return facts, list(dict.fromkeys([*blockers, "CUDA_UNAVAILABLE"])) + try: + visible_count = int(cuda.device_count()) + except Exception: + return facts, list(dict.fromkeys([*blockers, "CUDA_PROBE_FAILED"])) + facts["visible_device_count"] = visible_count + if visible_count != 1: + blockers.append("GPU_SELECTION_NOT_SINGLE") + return facts, list(dict.fromkeys(blockers)) + + try: + properties = cuda.get_device_properties(0) + capability = tuple(int(value) for value in cuda.get_device_capability(0)) + total_memory_gib = int(getattr(properties, "total_memory", 0)) / float(1024**3) + bf16_supported = bool(cuda.is_bf16_supported()) + gpu_name = str(cuda.get_device_name(0)) + except Exception: + return facts, list(dict.fromkeys([*blockers, "CUDA_PROBE_FAILED"])) + facts.update( + { + "name": gpu_name, + "compute_capability": f"{capability[0]}.{capability[1]}", + "total_memory_gib": round(total_memory_gib, 3), + "cuda_version": str(getattr(getattr(torch, "version", None), "cuda", None) or "unknown"), + "bf16_supported": bf16_supported, + } + ) + if capability < (8, 0) or not bf16_supported: + blockers.append("BF16_UNSUPPORTED") + if total_memory_gib < 35.0: + blockers.append("GPU_MEMORY_INSUFFICIENT") + if re.search(r"\bA100\b", gpu_name, flags=re.IGNORECASE) is None: + blockers.append("GPU_NOT_A100") + return facts, list(dict.fromkeys(blockers)) + + +def _smoke_config_facts(config_path: Path, config: dict[str, Any]) -> tuple[dict[str, Any], list[str]]: + blockers: list[str] = [] + if config.get("allow_heavy_training") is not True: + blockers.append("CONFIG_HEAVY_TRAINING_NOT_ALLOWED") + lora = config.get("lora") + lora_valid = False + if isinstance(lora, dict): + rank = lora.get("r") + alpha = lora.get("alpha") + dropout = lora.get("dropout") + target_modules = lora.get("target_modules") + lora_valid = ( + type(rank) is int + and 1 <= rank <= 64 + and type(alpha) is int + and 1 <= alpha <= 128 + and isinstance(dropout, int | float) + and not isinstance(dropout, bool) + and 0.0 <= float(dropout) <= 0.2 + and isinstance(target_modules, list) + and len(target_modules) == 4 + and set(target_modules) == {"q_proj", "k_proj", "v_proj", "o_proj"} + ) + minimum_free_disk_gib = config.get("min_free_disk_gib") + normalized_minimum_free_disk_gib = ( + float(minimum_free_disk_gib) + if isinstance(minimum_free_disk_gib, int | float) and not isinstance(minimum_free_disk_gib, bool) + else None + ) + disk_floor_valid = ( + normalized_minimum_free_disk_gib is not None and normalized_minimum_free_disk_gib >= 20.0 + ) + max_train_rows = config.get("max_train_rows") + max_train_rows_valid = type(max_train_rows) is int and max_train_rows in (1, 2) + max_steps = config.get("max_steps") + max_steps_valid = type(max_steps) is int and max_steps == 1 + per_device_train_batch_size = config.get("per_device_train_batch_size") + per_device_train_batch_size_valid = ( + type(per_device_train_batch_size) is int and per_device_train_batch_size == 1 + ) + gradient_accumulation_steps = config.get("gradient_accumulation_steps") + gradient_accumulation_steps_valid = ( + type(gradient_accumulation_steps) is int and gradient_accumulation_steps == 1 + ) + max_seq_length = config.get("max_seq_length") + max_seq_length_valid = type(max_seq_length) is int and 1 <= max_seq_length <= 4096 + seed = config.get("seed") + seed_valid = type(seed) is int + logging_steps = config.get("logging_steps") + logging_steps_valid = type(logging_steps) is int and logging_steps > 0 + bounded = ( + config.get("dataset_split") == "train" + and max_train_rows_valid + and max_steps_valid + and per_device_train_batch_size_valid + and gradient_accumulation_steps_valid + and max_seq_length_valid + and seed_valid + and logging_steps_valid + and config.get("save_strategy") == "no" + and config.get("report_to") == [] + and config.get("base_model_public_id") == "Qwen/Qwen2.5-7B-Instruct" + and config.get("local_files_only") is True + and config.get("bf16") is True + and config.get("fp16") is False + and config.get("tf32") is True + and config.get("gradient_checkpointing") is True + and config.get("use_cache") is False + and config.get("low_cpu_mem_usage") is True + and config.get("trust_remote_code") is False + and config.get("dtype") == "bfloat16" + and config.get("torch_dtype") == "bfloat16" + and disk_floor_valid + and lora_valid + ) + if not bounded: + blockers.append("CONFIG_NOT_SMOKE_BOUNDED") + return { + "config_sha256": _sha256_file(config_path), + "allow_heavy_training": config.get("allow_heavy_training") is True, + "base_model_public_id": ( + "Qwen/Qwen2.5-7B-Instruct" + if config.get("base_model_public_id") == "Qwen/Qwen2.5-7B-Instruct" + else None + ), + "dataset_split": "train" if config.get("dataset_split") == "train" else None, + "max_train_rows": max_train_rows if max_train_rows_valid else None, + "max_steps": max_steps if max_steps_valid else None, + "per_device_train_batch_size": ( + per_device_train_batch_size if per_device_train_batch_size_valid else None + ), + "gradient_accumulation_steps": ( + gradient_accumulation_steps if gradient_accumulation_steps_valid else None + ), + "max_seq_length": max_seq_length if max_seq_length_valid else None, + "seed": seed if seed_valid else None, + "logging_steps": logging_steps if logging_steps_valid else None, + "save_strategy": "no" if config.get("save_strategy") == "no" else None, + "report_to_empty": config.get("report_to") == [], + "local_files_only": config.get("local_files_only") is True, + "bf16": config.get("bf16") is True, + "fp16": config.get("fp16") is True, + "tf32": config.get("tf32") is True, + "gradient_checkpointing": config.get("gradient_checkpointing") is True, + "use_cache": config.get("use_cache") is True, + "low_cpu_mem_usage": config.get("low_cpu_mem_usage") is True, + "trust_remote_code": config.get("trust_remote_code") is True, + "dtype_is_bfloat16": config.get("dtype") == "bfloat16", + "torch_dtype_is_bfloat16": config.get("torch_dtype") == "bfloat16", + "minimum_free_disk_gib": ( + normalized_minimum_free_disk_gib if disk_floor_valid else None + ), + "lora_policy_valid": lora_valid, + }, blockers + + +def _load_selected_smoke_rows( + manifest_path: Path, + config: dict[str, Any], +) -> tuple[dict[str, Any], list[SFTDatasetRow], list[str]]: + empty_facts: dict[str, Any] = { + "manifest_file": manifest_path.name, + "manifest_sha256": None, + "manifest_id": None, + "sft_file": None, + "sft_sha256": None, + "selected_split": "train", + "selected_row_count": 0, + "selected_row_ids_sha256": None, + "non_train_rows_selected": 0, + } + row_limit = config.get("max_train_rows") + row_limit_valid = type(row_limit) is int and row_limit in (1, 2) + blockers: list[str] = [] + if not row_limit_valid or config.get("train_source_ids") is not None: + blockers.append("TRAIN_ROW_SELECTION_INVALID") + repo_root = _repo_root_from_canonical_manifest(manifest_path) + if repo_root is None: + return empty_facts, [], ["MANIFEST_PATH_NOT_CANONICAL"] + sft_path = repo_root / FORMAL_PUBLIC_SFT_RELATIVE_PATH + try: + manifest = read_json(manifest_path) + manifest_hash = _sha256_file(manifest_path) + sft_hash = _sha256_file(sft_path) + selected_rows: list[SFTDatasetRow] = [] + if row_limit_valid: + with sft_path.open("r", encoding="utf-8") as handle: + for line in handle: + if not line.strip(): + continue + record = json.loads(line) + if not isinstance(record, dict) or record.get("split") != "train": + blockers.append("TRAIN_ROW_SELECTION_INVALID") + break + selected_rows.append(SFTDatasetRow(**record)) + if len(selected_rows) == row_limit: + break + except (OSError, RuntimeError, TypeError, ValueError, json.JSONDecodeError): + return empty_facts, [], ["DATASET_HASH_OR_ID_MISMATCH"] + + if not selected_rows: + blockers.append("TRAIN_SPLIT_EMPTY") + elif len(selected_rows) != row_limit: + blockers.append("TRAIN_ROW_SELECTION_INVALID") + selected_ids = [row.id for row in selected_rows] + if len(selected_ids) != len(set(selected_ids)): + blockers.append("TRAIN_ROW_SELECTION_INVALID") + selected_hash = _sha256_text(json.dumps(selected_ids, ensure_ascii=False, separators=(",", ":"))) + manifest_id = str(manifest.get("manifest_id", "")) + if config.get("dataset_manifest_id") != FORMAL_PUBLIC_MANIFEST_ID: + blockers.append("DATASET_HASH_OR_ID_MISMATCH") + for expected_key, actual in ( + ("expected_manifest_sha256", manifest_hash), + ("expected_sft_sha256", sft_hash), + ("expected_train_row_selection_sha256", selected_hash), + ): + expected = config.get(expected_key) + if expected is not None and expected != actual: + blocker = ( + "DATASET_HASH_OR_ID_MISMATCH" + if "row_selection" not in expected_key + else "TRAIN_ROW_SELECTION_INVALID" + ) + blockers.append(blocker) + facts = { + "manifest_file": manifest_path.name, + "manifest_sha256": manifest_hash, + "manifest_id": manifest_id, + "sft_file": sft_path.name, + "sft_sha256": sft_hash, + "selected_split": "train", + "selected_row_count": len(selected_rows), + "selected_row_ids_sha256": selected_hash, + "non_train_rows_selected": 0, + } + return facts, selected_rows, list(dict.fromkeys(blockers)) + + +def _stable_local_model_inventory(model_root: Path) -> tuple[dict[str, str], list[dict[str, Any]]]: + fingerprint_names = ( + "config.json", + "generation_config.json", + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "vocab.json", + "merges.txt", + "model.safetensors.index.json", + "pytorch_model.bin.index.json", + ) + fingerprints = { + name: _sha256_file(model_root / name) + for name in fingerprint_names + if (model_root / name).is_file() + } + weight_files = sorted( + path for pattern in ("*.safetensors", "*.bin") for path in model_root.glob(pattern) if path.is_file() + ) + inventory = [{"name": path.name, "size": path.stat().st_size} for path in weight_files] + return fingerprints, inventory + + +def _probe_sft_model_and_objective( + config: dict[str, Any], + rows: list[SFTDatasetRow], +) -> tuple[dict[str, Any], dict[str, Any], list[str]]: + model_facts: dict[str, Any] = { + "public_id": "Qwen/Qwen2.5-7B-Instruct", + "local_files_only": True, + "stable_fingerprints": {}, + "weight_inventory": [], + "total_weight_bytes": 0, + "minimum_weight_bytes": 12 * 1024**3, + "geometry_matches_qwen2_5_7b": False, + "snapshot_revision_sha256": None, + } + objective_facts: dict[str, Any] = { + "records_checked": 0, + "prompt_labels_masked": False, + "assistant_target_present": False, + "max_sequence_length": int(config.get("max_seq_length", 0) or 0), + } + runtime_value = config.get("base_model_runtime_path") + if not isinstance(runtime_value, str) or not runtime_value or "<" in runtime_value or ">" in runtime_value: + return model_facts, objective_facts, ["MODEL_PATH_UNRESOLVED"] + model_root = Path(runtime_value).expanduser() + if not model_root.is_absolute() or not model_root.is_dir(): + return model_facts, objective_facts, ["MODEL_PATH_UNRESOLVED"] + try: + from transformers import AutoConfig, AutoTokenizer + + trust_remote_code = bool(config.get("trust_remote_code", False)) + local_config = AutoConfig.from_pretrained( + model_root.as_posix(), + local_files_only=True, + trust_remote_code=trust_remote_code, + ) + tokenizer = AutoTokenizer.from_pretrained( + model_root.as_posix(), + local_files_only=True, + trust_remote_code=trust_remote_code, + ) + fingerprints, inventory = _stable_local_model_inventory(model_root) + if "config.json" not in fingerprints or not inventory: + return model_facts, objective_facts, ["MODEL_LOCAL_LOAD_FAILED"] + architectures = getattr(local_config, "architectures", None) + geometry_matches = ( + getattr(local_config, "model_type", None) == "qwen2" + and isinstance(architectures, list) + and architectures == ["Qwen2ForCausalLM"] + and getattr(local_config, "hidden_size", None) == 3584 + and getattr(local_config, "intermediate_size", None) == 18944 + and getattr(local_config, "num_hidden_layers", None) == 28 + and getattr(local_config, "num_attention_heads", None) == 28 + and getattr(local_config, "num_key_value_heads", None) == 4 + and getattr(local_config, "vocab_size", None) == 152064 + ) + total_weight_bytes = sum(int(item["size"]) for item in inventory) + model_facts.update( + { + "stable_fingerprints": fingerprints, + "weight_inventory": inventory, + "total_weight_bytes": total_weight_bytes, + "geometry_matches_qwen2_5_7b": geometry_matches, + "snapshot_revision_sha256": _sha256_text(model_root.resolve().name), + } + ) + model_identity_blockers: list[str] = [] + if not geometry_matches: + model_identity_blockers.append("MODEL_IDENTITY_MISMATCH") + if total_weight_bytes < 12 * 1024**3: + model_identity_blockers.append("MODEL_WEIGHT_INSUFFICIENT") + if model_identity_blockers: + return model_facts, objective_facts, model_identity_blockers + max_seq_length = int(config.get("max_seq_length", 1024)) + records = [_assistant_only_training_record(row, tokenizer, max_seq_length=max_seq_length) for row in rows] + except Exception as exc: + blocker = "MAX_SEQUENCE_LENGTH_EXCEEDED" if "max_seq_length_exceeded" in str(exc) else "MODEL_LOCAL_LOAD_FAILED" + if "assistant-only SFT labels unavailable" in str(exc) and blocker != "MAX_SEQUENCE_LENGTH_EXCEEDED": + blocker = "ASSISTANT_ONLY_LABELS_INVALID" + return model_facts, objective_facts, [blocker] + + all_lengths_match = all( + len(record["input_ids"]) == len(record["attention_mask"]) == len(record["labels"]) + for record in records + ) + prompt_masked = all(any(label == -100 for label in record["labels"]) for record in records) + assistant_present = all(any(label != -100 for label in record["labels"]) for record in records) + if not records or not all_lengths_match or not prompt_masked or not assistant_present: + return model_facts, objective_facts, ["ASSISTANT_ONLY_LABELS_INVALID"] + objective_facts.update( + { + "records_checked": len(records), + "prompt_labels_masked": True, + "assistant_target_present": True, + "maximum_observed_tokens": max(len(record["input_ids"]) for record in records), + } + ) + return model_facts, objective_facts, [] + + +@dataclass(frozen=True) +class _SFTPreflightExecutionContext: + config_path: Path + config_sha256: str + config_json: str + repo_root: Path + manifest_path: Path + manifest_sha256: str + manifest_id: str + sft_path: Path + sft_sha256: str + selected_rows_json: tuple[str, ...] + selected_row_ids_sha256: str + model_root: Path + model_inventory_sha256: str + output_root: Path + output_dir: Path + output_facts_json: str + + def config_snapshot(self) -> dict[str, Any]: + parsed = json.loads(self.config_json) + if not isinstance(parsed, dict): + raise ValueError("bound SFT config snapshot must remain an object") + return cast(dict[str, Any], parsed) + + def selected_rows(self) -> tuple[SFTDatasetRow, ...]: + return tuple(SFTDatasetRow(**json.loads(record)) for record in self.selected_rows_json) + + +class SFTPreflightDriftError(RuntimeError): + def __init__(self, blockers: list[str]) -> None: + self.blockers = list(dict.fromkeys(blockers)) + super().__init__(",".join(self.blockers)) + + +def _preflight_input_drift_blockers( + execution_context: _SFTPreflightExecutionContext, +) -> list[str]: + blockers: list[str] = [] + try: + config_matches = ( + execution_context.config_path.is_file() + and not execution_context.config_path.is_symlink() + and _sha256_file(execution_context.config_path) == execution_context.config_sha256 + ) + except (OSError, RuntimeError): + config_matches = False + if not config_matches: + blockers.append("CONFIG_DRIFT_DETECTED") + try: + if _sha256_file(execution_context.manifest_path) != execution_context.manifest_sha256: + blockers.append("MANIFEST_DRIFT_DETECTED") + except (OSError, RuntimeError): + blockers.append("MANIFEST_DRIFT_DETECTED") + try: + if _sha256_file(execution_context.sft_path) != execution_context.sft_sha256: + blockers.append("SFT_DRIFT_DETECTED") + except (OSError, RuntimeError): + blockers.append("SFT_DRIFT_DETECTED") + try: + fingerprints, inventory = _stable_local_model_inventory(execution_context.model_root) + current_inventory_sha256 = _sha256_text( + json.dumps( + {"fingerprints": fingerprints, "weights": inventory}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + ) + if current_inventory_sha256 != execution_context.model_inventory_sha256: + blockers.append("MODEL_DRIFT_DETECTED") + except (OSError, RuntimeError): + blockers.append("MODEL_DRIFT_DETECTED") + return list(dict.fromkeys(blockers)) + + +def _blocked_sft_preflight_result(blockers: list[str]) -> dict[str, Any]: + return { + "schema_version": "voice2task-sft-preflight-v1", + "ready": False, + "status": "blocked", + "blockers": list(dict.fromkeys(blockers)), + "git": {}, + "config": {}, + "dataset": {}, + "model": {}, + "runtime": {}, + "gpu": {}, + "output": {}, + "objective": {}, + } + + +def _repo_root_from_canonical_manifest(manifest_path: Path) -> Path | None: + try: + resolved_manifest = manifest_path.expanduser().resolve(strict=True) + except (OSError, RuntimeError): + return None + relative_parts = FORMAL_PUBLIC_MANIFEST_RELATIVE_PATH.parts + if resolved_manifest.parts[-len(relative_parts) :] != relative_parts: + return None + repo_root = resolved_manifest.parents[len(relative_parts) - 1] + expected_manifest = repo_root / FORMAL_PUBLIC_MANIFEST_RELATIVE_PATH + try: + if expected_manifest.resolve(strict=True) != resolved_manifest: + return None + except (OSError, RuntimeError): + return None + return repo_root + + +def _git_config_path_state( + config_path: Path, + repo_root: Path, +) -> tuple[bool, bool, bool]: + try: + relative_path = config_path.relative_to(repo_root).as_posix() + ignored = subprocess.run( + ["git", "check-ignore", "--quiet", "--", relative_path], + cwd=repo_root, + check=False, + capture_output=True, + timeout=30, + ) + tracked = subprocess.run( + ["git", "ls-files", "--error-unmatch", "--", relative_path], + cwd=repo_root, + check=False, + capture_output=True, + timeout=30, + ) + except (OSError, RuntimeError, subprocess.SubprocessError, ValueError): + return False, False, False + ignored_known = ignored.returncode in (0, 1) + tracked_known = tracked.returncode in (0, 1) + return ignored.returncode == 0, tracked.returncode == 0, ignored_known and tracked_known + + +def _probe_private_sft_config( + config_path: Path, + repo_root: Path, +) -> tuple[dict[str, Any], list[str]]: + candidate = config_path.expanduser() + if not candidate.is_absolute(): + candidate = repo_root / candidate + private_root = repo_root / PRIVATE_SFT_CONFIG_DIRECTORY + facts: dict[str, Any] = { + "under_private_runtime": False, + "nonsymlink": False, + "git_ignored": False, + "git_tracked": False, + } + try: + candidate.relative_to(private_root) + except ValueError: + return facts, ["CONFIG_PATH_NOT_PRIVATE"] + + current = repo_root + symlink_detected = False + try: + for part in candidate.relative_to(repo_root).parts: + current = current / part + if current.is_symlink(): + symlink_detected = True + break + except (OSError, RuntimeError, ValueError): + return facts, ["CONFIG_PATH_NOT_PRIVATE"] + if symlink_detected: + return facts, ["CONFIG_PATH_SYMLINK"] + try: + resolved_candidate = candidate.resolve(strict=True) + resolved_private_root = private_root.resolve(strict=True) + resolved_candidate.relative_to(resolved_private_root) + except (OSError, RuntimeError, ValueError): + return facts, ["CONFIG_PATH_NOT_PRIVATE"] + + ignored, tracked, available = _git_config_path_state(resolved_candidate, repo_root) + facts.update( + { + "under_private_runtime": True, + "nonsymlink": True, + "git_ignored": ignored, + "git_tracked": tracked, + } + ) + blockers: list[str] = [] + if not available: + blockers.append("CONFIG_GIT_POLICY_UNAVAILABLE") + else: + if not ignored: + blockers.append("CONFIG_FILE_NOT_IGNORED") + if tracked: + blockers.append("CONFIG_FILE_TRACKED") + return facts, blockers + + +def _run_sft_preflight_core_unchecked( + config_path: Path, + manifest_path: Path, + output_dir: Path, +) -> tuple[dict[str, Any], _SFTPreflightExecutionContext | None]: + if not config_path.is_file(): + return _blocked_sft_preflight_result(["CONFIG_FILE_MISSING"]), None + repo_root = _repo_root_from_canonical_manifest(manifest_path) + if repo_root is None: + return _blocked_sft_preflight_result(["MANIFEST_PATH_NOT_CANONICAL"]), None + private_config_facts, private_config_blockers = _probe_private_sft_config( + config_path, + repo_root, + ) + if private_config_blockers: + result = _blocked_sft_preflight_result(private_config_blockers) + result["config"] = {"private_file": private_config_facts} + return result, None + try: + manifest = read_json(manifest_path) + except (OSError, ValueError, json.JSONDecodeError): + return _blocked_sft_preflight_result(["DATASET_HASH_OR_ID_MISMATCH"]), None + files = manifest.get("files") + if ( + manifest.get("manifest_id") != FORMAL_PUBLIC_MANIFEST_ID + or not isinstance(files, dict) + or files.get("sft") != FORMAL_PUBLIC_SFT_RELATIVE_PATH.as_posix() + ): + return _blocked_sft_preflight_result(["DATASET_HASH_OR_ID_MISMATCH"]), None + try: + config = _load_config(config_path) + except (OSError, TypeError, ValueError): + return _blocked_sft_preflight_result(["CONFIG_LOAD_FAILED"]), None + config_facts, config_blockers = _smoke_config_facts(config_path, config) + config_facts["private_file"] = private_config_facts + git_facts, git_blockers = _probe_sft_git(repo_root) + runtime_facts, dependency_blockers = _probe_sft_dependencies() + gpu_facts, gpu_blockers = _probe_sft_gpu() + dataset_facts, selected_rows, dataset_blockers = _load_selected_smoke_rows(manifest_path, config) + + runtime_value = config.get("base_model_runtime_path") + model_path_valid = ( + isinstance(runtime_value, str) + and bool(runtime_value) + and "<" not in runtime_value + and ">" not in runtime_value + and Path(runtime_value).expanduser().is_absolute() + and Path(runtime_value).expanduser().is_dir() + ) + model_blockers: list[str] = [] + model_facts: dict[str, Any] = { + "public_id": "Qwen/Qwen2.5-7B-Instruct", + "local_files_only": config.get("local_files_only") is True, + "stable_fingerprints": {}, + "weight_inventory": [], + } + objective_facts: dict[str, Any] = {"records_checked": 0} + if not model_path_valid: + model_blockers.append("MODEL_PATH_UNRESOLVED") + elif not config_blockers and not dependency_blockers and not dataset_blockers and selected_rows: + model_facts, objective_facts, model_blockers = _probe_sft_model_and_objective(config, selected_rows) + + output_facts = validate_sft_output_policy(config, output_dir, repo_root=repo_root) + blockers = list( + dict.fromkeys( + [ + *config_blockers, + *git_blockers, + *dependency_blockers, + *gpu_blockers, + *dataset_blockers, + *model_blockers, + *[str(code) for code in output_facts["blockers"]], + ] + ) + ) + ready = not blockers + public_result = { + "schema_version": "voice2task-sft-preflight-v1", + "ready": ready, + "status": "ready" if ready else "blocked", + "blockers": blockers, + "git": git_facts, + "config": config_facts, + "dataset": dataset_facts, + "model": model_facts, + "runtime": runtime_facts, + "gpu": gpu_facts, + "output": output_facts, + "objective": objective_facts, + } + if not ready or repo_root is None: + return public_result, None + + sft_path = _resolve_manifest_file( + manifest_path, + files.get("sft") if isinstance(files, dict) else None, + ) + runtime_value = config.get("base_model_runtime_path") + raw_output_root = config.get("output_root") + if ( + sft_path is None + or not isinstance(runtime_value, str) + or not isinstance(raw_output_root, str) + ): + return public_result, None + model_fingerprints = model_facts.get("stable_fingerprints") + model_inventory = model_facts.get("weight_inventory") + if not isinstance(model_fingerprints, dict) or not isinstance(model_inventory, list): + return public_result, None + context = _SFTPreflightExecutionContext( + config_path=config_path.resolve(strict=True), + config_sha256=str(config_facts["config_sha256"]), + config_json=json.dumps(config, ensure_ascii=False, sort_keys=True, separators=(",", ":")), + repo_root=repo_root.resolve(strict=True), + manifest_path=manifest_path.resolve(strict=True), + manifest_sha256=str(dataset_facts["manifest_sha256"]), + manifest_id=str(dataset_facts["manifest_id"]), + sft_path=sft_path.resolve(strict=True), + sft_sha256=str(dataset_facts["sft_sha256"]), + selected_rows_json=tuple( + json.dumps(row.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + for row in selected_rows + ), + selected_row_ids_sha256=str(dataset_facts["selected_row_ids_sha256"]), + model_root=Path(runtime_value).expanduser().resolve(strict=True), + model_inventory_sha256=_sha256_text( + json.dumps( + {"fingerprints": model_fingerprints, "weights": model_inventory}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + ), + output_root=Path(raw_output_root).expanduser().resolve(strict=True), + output_dir=output_dir.expanduser().resolve(strict=False), + output_facts_json=json.dumps( + output_facts, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ), + ) + return public_result, context + + +def _run_sft_preflight_core( + config_path: Path, + manifest_path: Path, + output_dir: Path, +) -> tuple[dict[str, Any], _SFTPreflightExecutionContext | None]: + try: + return _run_sft_preflight_core_unchecked(config_path, manifest_path, output_dir) + except Exception: + return _blocked_sft_preflight_result(["PREFLIGHT_INTERNAL_ERROR"]), None + + +def run_sft_preflight(config_path: Path, manifest_path: Path, output_dir: Path) -> dict[str, Any]: + public_result, _ = _run_sft_preflight_core(config_path, manifest_path, output_dir) + return public_result + + def _output_paths( *, config: dict[str, Any], @@ -510,6 +1573,71 @@ def _metadata_common( } +def _metadata_from_sft_execution_context( + preflight: dict[str, Any], + execution_context: _SFTPreflightExecutionContext, +) -> dict[str, Any]: + config = execution_context.config_snapshot() + output_dir = execution_context.output_dir + adapter_path = output_dir / "adapter" + metadata_path = output_dir / "adapter_metadata.json" + runtime_facts = preflight.get("runtime") + versions = runtime_facts.get("versions") if isinstance(runtime_facts, dict) else None + package_versions = dict(versions) if isinstance(versions, dict) else {} + if isinstance(runtime_facts, dict) and isinstance(runtime_facts.get("python"), str): + package_versions["python"] = str(runtime_facts["python"]) + dataset_load = { + "manifest_id": execution_context.manifest_id, + "manifest_sha256": execution_context.manifest_sha256, + "dataset_key": "sft", + "dataset_path": execution_context.sft_path.as_posix(), + "sft_sha256": execution_context.sft_sha256, + "loaded_rows": len(execution_context.selected_rows_json), + "selected_row_ids_sha256": execution_context.selected_row_ids_sha256, + } + return { + "stage": "sft", + "stack": _training_stack("sft"), + "base_model": _public_base_model(config), + "adapter_path": adapter_path.as_posix(), + "dataset_manifest_id": execution_context.manifest_id, + "dataset_manifest_path": execution_context.manifest_path.as_posix(), + "dataset_load": dataset_load, + "hyperparameters": config, + "dry_run": False, + "release_status": "not_released", + "adapter_release_status": "not_released", + "training_status": "pending_heavy_training", + "formatting_policy": dict(FORMATTING_POLICY), + "loss_mask_policy": _loss_mask_policy("sft"), + "created_at": datetime.now(timezone.utc).isoformat(), + "package_versions": package_versions, + "gpu_selection_policy": _gpu_selection_policy(config), + "heavy_training_gate": _heavy_training_gate(config, False), + "output_paths": _output_paths( + config=config, + output_dir=output_dir, + adapter_path=adapter_path, + metadata_path=metadata_path, + ), + "command_summary": _command_summary( + stage="sft", + config_path=execution_context.config_path, + manifest_path=execution_context.manifest_path, + output_dir=output_dir, + dry_run=False, + ), + "training_command": ( + f"voice2task-train sft --config {execution_context.config_path.as_posix()} " + f"--manifest {execution_context.manifest_path.as_posix()} " + f"--output-dir {output_dir.as_posix()} --run-training" + ), + "metadata_path": metadata_path.as_posix(), + "preflight": preflight, + "notes": "Training entrypoint passed shared preflight; real execution remains smoke-bounded.", + } + + def _train_dependencies_available() -> bool: return all(importlib.util.find_spec(module) is not None for module in ("datasets", "peft", "transformers", "trl")) @@ -565,6 +1693,139 @@ def _write_training_blocked_by_output_policy(metadata: dict[str, Any], stage: st return metadata +def _blocked_sft_result( + metadata: dict[str, Any], + *, + status: str, + blockers: list[str], +) -> dict[str, Any]: + metadata["release_status"] = "not_released" + metadata["training_status"] = status + metadata["trainer_available"] = False + metadata["blockers"] = list(dict.fromkeys(blockers)) + metadata["heavy_training_gate"]["will_run_heavy_training"] = False + metadata["notes"] = "Real SFT execution was blocked before model weights loaded." + return metadata + + +def _minimal_blocked_sft_preflight_result( + preflight: dict[str, Any], + *, + status: str, + blockers: list[str], +) -> dict[str, Any]: + config_facts = preflight.get("config") + config_allows = isinstance(config_facts, dict) and config_facts.get("allow_heavy_training") is True + return { + "schema_version": "voice2task-training-result-v1", + "stage": "sft", + "dry_run": False, + "release_status": "not_released", + "adapter_release_status": "not_released", + "training_status": status, + "trainer_available": False, + "blockers": list(dict.fromkeys(blockers)), + "preflight": preflight, + "heavy_training_gate": { + "cli_run_training": True, + "config_allow_heavy_training": config_allows, + "will_run_heavy_training": False, + }, + "notes": "Real SFT execution was blocked by shared preflight before legacy metadata reads.", + } + + +def _blocked_training_output_policy_result(stage: str, blockers: list[str]) -> dict[str, Any]: + return { + "schema_version": "voice2task-training-result-v1", + "stage": stage, + "dry_run": False, + "release_status": "not_released", + "adapter_release_status": "not_released", + "training_status": "training_blocked_by_output_policy", + "trainer_available": False, + "blockers": list(dict.fromkeys(blockers)), + "heavy_training_gate": { + "cli_run_training": True, + "config_allow_heavy_training": False, + "will_run_heavy_training": False, + }, + "notes": "Training was blocked by the output policy before dependency imports or model loading.", + } + + +def _clean_evaluation_truth_surface() -> dict[str, Any]: + return { + "acquisition_source_status": "UNAVAILABLE", + "authoritatively_bound_binding_count": 0, + "human_acceptance_status": "NOT_RECORDED", + "protocol_freeze_status": "NOT_FROZEN", + "clean_population_status": "NOT_MATERIALIZED", + "freeze_authorized": False, + "execution_readiness": False, + } + + +def _sft_smoke_postconditions(metadata: dict[str, Any]) -> list[str]: + blockers: list[str] = [] + observed_steps = metadata.get("observed_optimizer_steps") + if not isinstance(observed_steps, int) or isinstance(observed_steps, bool) or observed_steps != 1: + blockers.append("OPTIMIZER_STEP_MISMATCH") + configured = metadata.get("hyperparameters") + configured_rows = configured.get("max_train_rows") if isinstance(configured, dict) else None + rows_used = metadata.get("training_rows_used") + if ( + not isinstance(configured_rows, int) + or isinstance(configured_rows, bool) + or configured_rows not in (1, 2) + or rows_used != configured_rows + ): + blockers.append("TRAIN_ROW_SELECTION_INVALID") + metrics = metadata.get("train_result_metrics") + loss = metrics.get("train_loss") if isinstance(metrics, dict) else None + if isinstance(loss, bool) or not isinstance(loss, int | float) or not math.isfinite(float(loss)): + blockers.append("TRAINING_LOSS_INVALID") + + adapter_path = Path(str(metadata.get("adapter_path", ""))) + try: + adapter_files = sorted( + path for path in adapter_path.rglob("*") if path.is_file() and path.stat().st_size > 0 + ) + metadata["adapter_files"] = [ + {"name": path.relative_to(adapter_path).as_posix(), "size": path.stat().st_size} + for path in adapter_files + ] + except OSError: + adapter_files = [] + metadata["adapter_files"] = [] + adapter_names = {path.name for path in adapter_files} + has_adapter_config = "adapter_config.json" in adapter_names + has_adapter_weights = bool(adapter_names & {"adapter_model.safetensors", "adapter_model.bin"}) + if not has_adapter_config or not has_adapter_weights: + blockers.append("ADAPTER_OUTPUT_INVALID") + metadata_path = Path(str(metadata.get("metadata_path", ""))) + run_root = metadata_path.parent + try: + run_files = [path for path in run_root.rglob("*") if path.is_file()] + except OSError: + run_files = [] + blockers.append("ADAPTER_OUTPUT_INVALID") + full_weight_names = { + "model.safetensors", + "model.safetensors.index.json", + "pytorch_model.bin", + "pytorch_model.bin.index.json", + } + if any( + path.name in full_weight_names + or path.name.startswith("model-") + or path.name.startswith("pytorch_model-") + for path in run_files + ): + blockers.append("FULL_MODEL_WEIGHTS_DETECTED") + return list(dict.fromkeys(blockers)) + + def _training_error_category(exc: Exception) -> str: message = str(exc).lower() if ( @@ -618,7 +1879,11 @@ def _training_arguments(config: dict[str, Any], output_dir: Path) -> Any: logging_steps=int(config.get("logging_steps", 1)), save_strategy=str(config.get("save_strategy", "no")), seed=int(config.get("seed", 42)), - report_to=[], + report_to=list(config.get("report_to", [])), + bf16=bool(config.get("bf16", False)), + fp16=bool(config.get("fp16", False)), + tf32=bool(config.get("tf32", False)), + gradient_checkpointing=bool(config.get("gradient_checkpointing", False)), ) @@ -730,6 +1995,25 @@ def _record_sft_training_selection_from_config( return rows +def _record_bound_sft_training_selection( + metadata: dict[str, Any], + config: dict[str, Any], + execution_context: _SFTPreflightExecutionContext, +) -> list[SFTDatasetRow]: + rows = list(execution_context.selected_rows()) + row_limit = int(config["max_train_rows"]) + _record_sft_training_row_selection( + metadata, + split="train", + rows=rows, + row_limit=row_limit, + source_ids=None, + loaded_rows_before_limit=len(rows), + loaded_rows_before_source_filter=len(rows), + ) + return rows + + def _target_token_count(records: list[dict[str, Any]]) -> int: total = 0 for record in records: @@ -2549,18 +3833,82 @@ def _load_dpo_training_pairs(manifest_path: Path, split: str) -> list[DPOPair]: return [pair for pair in (DPOPair(**record) for record in read_jsonl(Path(dataset_path))) if pair.split == split] -def _run_real_sft(metadata: dict[str, Any], config: dict[str, Any], manifest_path: Path, output_dir: Path) -> None: +def _run_real_sft( + metadata: dict[str, Any], + config: dict[str, Any], + manifest_path: Path, + output_dir: Path, + *, + execution_context: _SFTPreflightExecutionContext | object | None = None, +) -> None: + if not isinstance(execution_context, _SFTPreflightExecutionContext): + raise RuntimeError("PREFLIGHT_CONTEXT_MISSING") + drift_blockers = _preflight_input_drift_blockers(execution_context) + if drift_blockers: + raise SFTPreflightDriftError(drift_blockers) + config = execution_context.config_snapshot() + output_dir = execution_context.output_dir + _claim_sft_output_directory( + config, + output_dir, + repo_root=execution_context.repo_root, + expected_output_path_sha256=json.loads(execution_context.output_facts_json).get("output_path_sha256"), + ) + from datasets import Dataset # type: ignore[import-not-found, unused-ignore] from transformers import AutoModelForCausalLM, AutoTokenizer from trl import SFTTrainer # type: ignore[import-not-found, unused-ignore] - rows = _record_sft_training_selection_from_config(metadata, config, manifest_path) - base_model = _runtime_base_model(config) - tokenizer = AutoTokenizer.from_pretrained(base_model) + rows = _record_bound_sft_training_selection(metadata, config, execution_context) + base_model = execution_context.model_root.as_posix() + local_files_only = bool(config.get("local_files_only", True)) + trust_remote_code = bool(config.get("trust_remote_code", False)) + tokenizer = AutoTokenizer.from_pretrained( + base_model, + local_files_only=local_files_only, + trust_remote_code=trust_remote_code, + ) + if getattr(tokenizer, "pad_token_id", None) is None: + if getattr(tokenizer, "eos_token_id", None) is None or getattr(tokenizer, "eos_token", None) is None: + raise RuntimeError("TOKENIZER_PAD_TOKEN_UNAVAILABLE") + tokenizer.pad_token = tokenizer.eos_token max_seq_length = int(config.get("max_seq_length", 1024)) records = [_assistant_only_training_record(row, tokenizer, max_seq_length=max_seq_length) for row in rows] dataset = Dataset.from_list(records) - model = AutoModelForCausalLM.from_pretrained(base_model) + import torch + + model = AutoModelForCausalLM.from_pretrained( + base_model, + local_files_only=local_files_only, + trust_remote_code=trust_remote_code, + torch_dtype=torch.bfloat16, + low_cpu_mem_usage=bool(config.get("low_cpu_mem_usage", True)), + ) + if bool(config.get("gradient_checkpointing", False)): + gradient_checkpointing_enable = getattr(model, "gradient_checkpointing_enable", None) + if callable(gradient_checkpointing_enable): + gradient_checkpointing_enable() + model.config.use_cache = False + else: + model.config.use_cache = bool(config.get("use_cache", True)) + metadata["runtime_options"] = { + "torch_dtype": "bfloat16", + "bf16": bool(config.get("bf16", False)), + "fp16": bool(config.get("fp16", False)), + "tf32": bool(config.get("tf32", False)), + "gradient_checkpointing": bool(config.get("gradient_checkpointing", False)), + "use_cache": bool(getattr(model.config, "use_cache", False)), + "low_cpu_mem_usage": bool(config.get("low_cpu_mem_usage", True)), + "local_files_only": local_files_only, + "trust_remote_code": trust_remote_code, + "seed": int(config.get("seed", 42)), + "max_steps": int(config.get("max_steps", -1)), + "max_train_rows": config.get("max_train_rows"), + "per_device_train_batch_size": int(config.get("per_device_train_batch_size", 1)), + "gradient_accumulation_steps": int(config.get("gradient_accumulation_steps", 1)), + "save_strategy": str(config.get("save_strategy", "no")), + "logging_steps": int(config.get("logging_steps", 1)), + } trainer = SFTTrainer( model=model, train_dataset=dataset, @@ -2581,7 +3929,16 @@ def _run_real_sft(metadata: dict[str, Any], config: dict[str, Any], manifest_pat trainer.model.save_pretrained(metadata["adapter_path"]) -def _run_real_dpo(metadata: dict[str, Any], config: dict[str, Any], manifest_path: Path, output_dir: Path) -> None: +def _run_real_dpo( + metadata: dict[str, Any], + config: dict[str, Any], + manifest_path: Path, + output_dir: Path, + *, + repo_root: Path, +) -> None: + _claim_sft_output_directory(config, output_dir, repo_root=repo_root) + from datasets import Dataset # type: ignore[import-not-found, unused-ignore] from peft import PeftModel # type: ignore[import-not-found, unused-ignore] from transformers import AutoModelForCausalLM, AutoTokenizer @@ -2633,37 +3990,108 @@ def _run_real_dpo(metadata: dict[str, Any], config: dict[str, Any], manifest_pat def run_sft(config_path: Path, manifest_path: Path, output_dir: Path, dry_run: bool = True) -> dict[str, Any]: - config = _load_config(config_path) - metadata = _metadata_common( - stage="sft", - config_path=config_path, - manifest_path=manifest_path, - output_dir=output_dir, - dry_run=dry_run, + if dry_run: + config = _load_config(config_path) + metadata = _metadata_common( + stage="sft", + config_path=config_path, + manifest_path=manifest_path, + output_dir=output_dir, + dry_run=True, + ) + _record_sft_training_selection_from_config(metadata, config, manifest_path) + metadata["trainer_available"] = False + write_json(Path(metadata["metadata_path"]), metadata) + return metadata + + preflight, execution_context = _run_sft_preflight_core(config_path, manifest_path, output_dir) + preflight_blockers = [str(code) for code in preflight.get("blockers", [])] + if preflight.get("ready") is not True: + if "CONFIG_HEAVY_TRAINING_NOT_ALLOWED" in preflight_blockers: + status = "training_skipped_by_config" + elif any(code.startswith("OUTPUT_") for code in preflight_blockers): + status = "training_blocked_by_output_policy" + elif any(code.startswith("DEPENDENCY_") for code in preflight_blockers): + status = "training_unavailable" + else: + status = "training_blocked_by_preflight" + return _minimal_blocked_sft_preflight_result(preflight, status=status, blockers=preflight_blockers) + + if not isinstance(execution_context, _SFTPreflightExecutionContext): + return _minimal_blocked_sft_preflight_result( + preflight, + status="training_blocked_by_preflight", + blockers=["PREFLIGHT_CONTEXT_MISSING"], + ) + config = execution_context.config_snapshot() + metadata = _metadata_from_sft_execution_context(preflight, execution_context) + final_output_policy = validate_sft_output_policy( + config, + execution_context.output_dir, + repo_root=execution_context.repo_root, ) - _record_sft_training_selection_from_config(metadata, config, manifest_path) - if not dry_run and not _heavy_training_allowed(config): - return _write_training_skipped(metadata, "sft") - if not dry_run and not _output_dir_within_configured_root(config, output_dir): - return _write_training_blocked_by_output_policy(metadata, "sft") - if not dry_run and not _train_dependencies_available(): - return _write_training_plan(metadata, "sft") - metadata["trainer_available"] = not dry_run - if not dry_run: - try: - _run_real_sft(metadata, config, manifest_path, output_dir) - except Exception as exc: - _write_training_failed(metadata, "sft", exc) - raise + if final_output_policy.get("ready") is not True: + return _blocked_sft_result( + metadata, + status="training_blocked_by_output_policy", + blockers=[str(code) for code in final_output_policy.get("blockers", [])], + ) + metadata["trainer_available"] = True + try: + _run_real_sft( + metadata, + config, + execution_context.manifest_path, + execution_context.output_dir, + execution_context=execution_context, + ) + except SFTOutputPolicyError as exc: + return _blocked_sft_result( + metadata, + status="training_blocked_by_output_policy", + blockers=exc.blockers, + ) + except SFTPreflightDriftError as exc: + return _blocked_sft_result( + metadata, + status="training_blocked_by_preflight", + blockers=exc.blockers, + ) + except Exception as exc: + _write_training_failed(metadata, "sft", exc) + raise + postcondition_blockers = _sft_smoke_postconditions(metadata) + metadata["clean_evaluation"] = _clean_evaluation_truth_surface() + if postcondition_blockers: + metadata["training_status"] = "training_failed" + metadata["blockers"] = postcondition_blockers + metadata["smoke_status"] = "SMOKE_FAILED" + metadata["notes"] = "SFT smoke ran but failed bounded completion postconditions." + else: metadata["release_status"] = "not_released" metadata["training_status"] = "training_completed" - metadata["notes"] = "SFT training ran locally; adapter metadata is not a public checkpoint release." + metadata["smoke_status"] = "SMOKE_COMPLETED" + metadata["notes"] = ( + "One-step SFT infrastructure smoke completed; this is not a model-improvement or readiness claim." + ) write_json(Path(metadata["metadata_path"]), metadata) return metadata def run_dpo(config_path: Path, manifest_path: Path, output_dir: Path, dry_run: bool = True) -> dict[str, Any]: + repo_root: Path | None = None + if not dry_run: + repo_root = _git_repository_root_for_manifest(manifest_path) + if repo_root is None: + return _blocked_training_output_policy_result("dpo", ["OUTPUT_REPOSITORY_UNAVAILABLE"]) config = _load_config(config_path) + if not dry_run: + output_policy = validate_sft_output_policy(config, output_dir, repo_root=repo_root) + if output_policy.get("ready") is not True: + return _blocked_training_output_policy_result( + "dpo", + [str(code) for code in output_policy.get("blockers", [])], + ) metadata = _metadata_common( stage="dpo", config_path=config_path, @@ -2675,14 +4103,20 @@ def run_dpo(config_path: Path, manifest_path: Path, output_dir: Path, dry_run: b metadata["dpo_initialization"] = "base_model_lora" if not dry_run and not _heavy_training_allowed(config): return _write_training_skipped(metadata, "dpo") - if not dry_run and not _output_dir_within_configured_root(config, output_dir): - return _write_training_blocked_by_output_policy(metadata, "dpo") if not dry_run and not _train_dependencies_available(): return _write_training_plan(metadata, "dpo") metadata["trainer_available"] = not dry_run if not dry_run: try: - _run_real_dpo(metadata, config, manifest_path, output_dir) + if repo_root is None: + raise SFTOutputPolicyError(["OUTPUT_REPOSITORY_UNAVAILABLE"]) + _run_real_dpo(metadata, config, manifest_path, output_dir, repo_root=repo_root) + except SFTOutputPolicyError as exc: + return _blocked_sft_result( + metadata, + status="training_blocked_by_output_policy", + blockers=exc.blockers, + ) except Exception as exc: _write_training_failed(metadata, "dpo", exc) raise diff --git a/tests/test_a100_sft_smoke.py b/tests/test_a100_sft_smoke.py index 5cee50d..1a9ca24 100644 --- a/tests/test_a100_sft_smoke.py +++ b/tests/test_a100_sft_smoke.py @@ -164,6 +164,10 @@ def _write_runtime_label_provenance_config( class _RuntimeInspectableTokenizer: chat_template = "private-runtime-template" + pad_token_id = 0 + pad_token = "" + eos_token_id = 1 + eos_token = "" def __call__(self, text: str, **kwargs: Any) -> dict[str, Any]: tokens = [ord(char) for char in text] @@ -1565,18 +1569,85 @@ def test_runtime_label_provenance_train_cli_does_not_write_unresolved_status_out assert output.as_posix() not in stdout +def _install_bounded_run_sft_preflight_stub(monkeypatch: Any) -> None: + def preflight( + config_path: Path, + manifest_path: Path, + output_dir: Path, + ) -> tuple[dict[str, Any], training._SFTPreflightExecutionContext | None]: # noqa: SLF001 + config = json.loads(config_path.read_text(encoding="utf-8")) + if config.get("allow_heavy_training") is not True: + return ( + { + "schema_version": "voice2task-sft-preflight-v1", + "ready": False, + "status": "blocked", + "blockers": ["CONFIG_HEAVY_TRAINING_NOT_ALLOWED"], + "config": {"allow_heavy_training": False}, + }, + None, + ) + output_policy = training.validate_sft_output_policy( + config, + output_dir, + repo_root=config_path.parent / "not-the-output-repo", + ) + if output_policy["ready"] is not True: + return ( + { + "schema_version": "voice2task-sft-preflight-v1", + "ready": False, + "status": "blocked", + "blockers": output_policy["blockers"], + "config": {"allow_heavy_training": True}, + "output": output_policy, + }, + None, + ) + rows = training._load_sft_training_rows(manifest_path, split="train") # noqa: SLF001 + execution_context = _bound_sft_execution_context( + config_path.parent, + config, + manifest_path, + output_dir, + rows, + ) + return ( + { + "schema_version": "voice2task-sft-preflight-v1", + "ready": True, + "status": "ready", + "blockers": [], + "config": {"allow_heavy_training": True}, + "runtime": { + "python": "test", + "versions": { + name: "test" + for name in ("accelerate", "datasets", "peft", "transformers", "trl") + }, + }, + "output": output_policy, + }, + execution_context, + ) + + monkeypatch.setattr(training, "_run_sft_preflight_core", preflight) + + def test_sft_heavy_training_requires_cli_and_config_opt_ins(monkeypatch: Any, tmp_path: Path) -> None: manifest = _write_manifest(tmp_path) calls: list[Path] = [] monkeypatch.setattr(training, "_train_dependencies_available", lambda: True) + _install_bounded_run_sft_preflight_stub(monkeypatch) monkeypatch.setattr( training, "_run_real_sft", - lambda metadata, config, manifest_path, output_dir: calls.append(output_dir), + lambda metadata, config, manifest_path, output_dir, *, execution_context: calls.append(output_dir), ) allowed_root = tmp_path / "remote-root" + allowed_root.mkdir() config_allows = _write_config(tmp_path, allow_heavy_training=True, output_root=allowed_root.as_posix()) dry_run_meta = run_sft( config_path=config_allows, @@ -1623,13 +1694,15 @@ def test_sft_heavy_training_requires_cli_and_config_opt_ins(monkeypatch: Any, tm def test_sft_a100_run_training_blocks_output_outside_configured_root(monkeypatch: Any, tmp_path: Path) -> None: manifest = _write_manifest(tmp_path) allowed_root = tmp_path / "allowed-root" + allowed_root.mkdir() config = _write_config(tmp_path, allow_heavy_training=True, output_root=allowed_root.as_posix()) calls: list[Path] = [] monkeypatch.setattr(training, "_train_dependencies_available", lambda: True) + _install_bounded_run_sft_preflight_stub(monkeypatch) monkeypatch.setattr( training, "_run_real_sft", - lambda metadata, config, manifest_path, output_dir: calls.append(output_dir), + lambda metadata, config, manifest_path, output_dir, *, execution_context: calls.append(output_dir), ) metadata = run_sft( @@ -1643,7 +1716,7 @@ def test_sft_a100_run_training_blocks_output_outside_configured_root(monkeypatch assert metadata["release_status"] == "not_released" assert metadata["training_status"] == "training_blocked_by_output_policy" assert metadata["heavy_training_gate"]["will_run_heavy_training"] is False - assert "outside configured output_root" in metadata["notes"] + assert "OUTPUT_PATH_OUTSIDE_ROOT" in metadata["blockers"] def test_sft_a100_run_training_blocks_unresolved_public_template_output_root( @@ -1654,10 +1727,11 @@ def test_sft_a100_run_training_blocks_unresolved_public_template_output_root( config = _write_config(tmp_path, allow_heavy_training=True, output_root="") calls: list[Path] = [] monkeypatch.setattr(training, "_train_dependencies_available", lambda: True) + _install_bounded_run_sft_preflight_stub(monkeypatch) monkeypatch.setattr( training, "_run_real_sft", - lambda metadata, config, manifest_path, output_dir: calls.append(output_dir), + lambda metadata, config, manifest_path, output_dir, *, execution_context: calls.append(output_dir), ) metadata = run_sft( @@ -1671,7 +1745,7 @@ def test_sft_a100_run_training_blocks_unresolved_public_template_output_root( assert metadata["release_status"] == "not_released" assert metadata["training_status"] == "training_blocked_by_output_policy" assert metadata["heavy_training_gate"]["will_run_heavy_training"] is False - assert "unresolved output_root template" in metadata["notes"] + assert "OUTPUT_ROOT_MISSING" in metadata["blockers"] def _install_fake_training_modules( @@ -1689,19 +1763,22 @@ def from_list(records: list[dict[str, list[int]]]) -> list[dict[str, list[int]]] return records class FakeModel: + def __init__(self) -> None: + self.config = types.SimpleNamespace(use_cache=True) + def save_pretrained(self, path: str) -> None: saved_paths.append(path) class FakeAutoModelForCausalLM: @staticmethod - def from_pretrained(base_model: str) -> FakeModel: - assert base_model == "fake-qwen" + def from_pretrained(base_model: str, **kwargs: Any) -> FakeModel: + assert Path(base_model).name == "fake-qwen" return FakeModel() class FakeAutoTokenizer: @staticmethod - def from_pretrained(base_model: str) -> _RuntimeInspectableTokenizer: - assert base_model == "fake-qwen" + def from_pretrained(base_model: str, **kwargs: Any) -> _RuntimeInspectableTokenizer: + assert Path(base_model).name == "fake-qwen" return _RuntimeInspectableTokenizer() class FakeTrainingArguments: @@ -1734,6 +1811,58 @@ def train(self) -> None: monkeypatch.setitem(sys.modules, "trl", fake_trl) +def _bound_sft_execution_context( + tmp_path: Path, + config: dict[str, Any], + manifest: Path, + output_dir: Path, + rows: list[training.SFTDatasetRow], +) -> Any: + bound_config = dict(config) + bound_config.setdefault("max_train_rows", len(rows)) + model_root = tmp_path / "fake-qwen" + model_root.mkdir(exist_ok=True) + bound_config["base_model_runtime_path"] = model_root.as_posix() + config_path = tmp_path / "bound-private-config.json" + config_path.write_text(json.dumps(bound_config, sort_keys=True), encoding="utf-8") + fingerprints, inventory = training._stable_local_model_inventory(model_root) # noqa: SLF001 + output_facts = training.validate_sft_output_policy( + bound_config, + output_dir, + repo_root=tmp_path / "not-the-output-repo", + ) + assert output_facts["ready"] is True + return training._SFTPreflightExecutionContext( # noqa: SLF001 + config_path=config_path, + config_sha256=training._sha256_file(config_path), # noqa: SLF001 + config_json=json.dumps(bound_config, ensure_ascii=False, sort_keys=True, separators=(",", ":")), + repo_root=tmp_path / "not-the-output-repo", + manifest_path=manifest, + manifest_sha256=training._sha256_file(manifest), # noqa: SLF001 + manifest_id=str(json.loads(manifest.read_text(encoding="utf-8"))["manifest_id"]), + sft_path=manifest, + sft_sha256=training._sha256_file(manifest), # noqa: SLF001 + selected_rows_json=tuple( + json.dumps(row.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) for row in rows + ), + selected_row_ids_sha256=training._sha256_text( # noqa: SLF001 + json.dumps([row.id for row in rows], separators=(",", ":")) + ), + model_root=model_root, + model_inventory_sha256=training._sha256_text( # noqa: SLF001 + json.dumps( + {"fingerprints": fingerprints, "weights": inventory}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + ), + output_root=tmp_path, + output_dir=output_dir, + output_facts_json=json.dumps(output_facts, sort_keys=True, separators=(",", ":")), + ) + + def test_real_sft_heavy_path_keeps_new_trl_sfttrainer_with_assistant_only_labels( monkeypatch: Any, tmp_path: Path, @@ -1780,16 +1909,22 @@ def train(self) -> None: saved_paths=saved_paths, ) + config = { + "base_model": "fake-qwen", + "output_root": tmp_path.as_posix(), + "min_free_disk_gib": 0, + "dataset_split": "train", + "max_seq_length": SHARED_PREFIX_RUNTIME_SFT_MAX_SEQ_LENGTH, + "lora": {"r": 8, "alpha": 16, "dropout": 0.05, "target_modules": ["q_proj"]}, + } + output_dir = tmp_path / "run" + execution_context = _bound_sft_execution_context(tmp_path, config, manifest, output_dir, rows) training._run_real_sft( # noqa: SLF001 {"adapter_path": (tmp_path / "adapter").as_posix()}, - { - "base_model": "fake-qwen", - "dataset_split": "train", - "max_seq_length": SHARED_PREFIX_RUNTIME_SFT_MAX_SEQ_LENGTH, - "lora": {"r": 8, "alpha": 16, "dropout": 0.05, "target_modules": ["q_proj"]}, - }, + config, manifest, - tmp_path / "run", + output_dir, + execution_context=execution_context, ) assert trainer_calls == [] @@ -1814,6 +1949,7 @@ def test_real_sft_heavy_path_supports_old_trl_sfttrainer_tokenizer_signature( tmp_path: Path, ) -> None: manifest = _write_manifest(tmp_path) + rows = training._load_sft_training_rows(manifest, split="train") # noqa: SLF001 dataset_records: list[list[dict[str, list[int]]]] = [] sft_trainer_calls: list[dict[str, Any]] = [] trainer_calls: list[dict[str, Any]] = [] @@ -1853,16 +1989,22 @@ def train(self) -> None: saved_paths=saved_paths, ) + config = { + "base_model": "fake-qwen", + "output_root": tmp_path.as_posix(), + "min_free_disk_gib": 0, + "dataset_split": "train", + "max_seq_length": SHARED_PREFIX_RUNTIME_SFT_MAX_SEQ_LENGTH, + "lora": {"r": 8, "alpha": 16, "dropout": 0.05, "target_modules": ["q_proj"]}, + } + output_dir = tmp_path / "run" + execution_context = _bound_sft_execution_context(tmp_path, config, manifest, output_dir, rows) training._run_real_sft( # noqa: SLF001 {"adapter_path": (tmp_path / "adapter").as_posix()}, - { - "base_model": "fake-qwen", - "dataset_split": "train", - "max_seq_length": SHARED_PREFIX_RUNTIME_SFT_MAX_SEQ_LENGTH, - "lora": {"r": 8, "alpha": 16, "dropout": 0.05, "target_modules": ["q_proj"]}, - }, + config, manifest, - tmp_path / "run", + output_dir, + execution_context=execution_context, ) assert trainer_calls == [] @@ -1877,6 +2019,7 @@ def test_real_sft_heavy_path_limits_tiny_overfit_rows_and_records_metadata( tmp_path: Path, ) -> None: manifest = _write_multirow_manifest(tmp_path, train_rows=4) + rows = training._load_sft_training_rows(manifest, split="train")[:2] # noqa: SLF001 dataset_records: list[list[dict[str, list[int]]]] = [] sft_trainer_calls: list[dict[str, Any]] = [] trainer_calls: list[dict[str, Any]] = [] @@ -1916,18 +2059,24 @@ def train(self) -> None: saved_paths=saved_paths, ) + config = { + "base_model": "fake-qwen", + "output_root": tmp_path.as_posix(), + "min_free_disk_gib": 0, + "dataset_split": "train", + "max_train_rows": 2, + "max_seq_length": SHARED_PREFIX_RUNTIME_SFT_MAX_SEQ_LENGTH, + "lora": {"r": 8, "alpha": 16, "dropout": 0.05, "target_modules": ["q_proj"]}, + } + output_dir = tmp_path / "run" + execution_context = _bound_sft_execution_context(tmp_path, config, manifest, output_dir, rows) metadata = {"adapter_path": (tmp_path / "adapter").as_posix(), "dataset_load": {}} training._run_real_sft( # noqa: SLF001 metadata, - { - "base_model": "fake-qwen", - "dataset_split": "train", - "max_train_rows": 2, - "max_seq_length": SHARED_PREFIX_RUNTIME_SFT_MAX_SEQ_LENGTH, - "lora": {"r": 8, "alpha": 16, "dropout": 0.05, "target_modules": ["q_proj"]}, - }, + config, manifest, - tmp_path / "run", + output_dir, + execution_context=execution_context, ) assert trainer_calls == [] @@ -1939,16 +2088,22 @@ def train(self) -> None: assert metadata["training_row_ids"] == ["sft-1", "sft-2"] assert metadata["dataset_load"]["training_rows_used"] == 2 assert metadata["dataset_load"]["training_row_ids"] == ["sft-1", "sft-2"] - assert metadata["dataset_load"]["loaded_rows_before_training_row_limit"] == 4 + assert metadata["dataset_load"]["loaded_rows_before_training_row_limit"] == 2 def test_sft_metadata_contains_public_safe_a100_smoke_fields(monkeypatch: Any, tmp_path: Path) -> None: manifest = _write_manifest(tmp_path) allowed_root = tmp_path / "remote-root" + allowed_root.mkdir() config = _write_config(tmp_path, allow_heavy_training=True, output_root=allowed_root.as_posix()) output_dir = allowed_root / "runs" / "run" monkeypatch.setattr(training, "_train_dependencies_available", lambda: True) - monkeypatch.setattr(training, "_run_real_sft", lambda metadata, config, manifest_path, output_dir: None) + _install_bounded_run_sft_preflight_stub(monkeypatch) + monkeypatch.setattr( + training, + "_run_real_sft", + lambda metadata, config, manifest_path, output_dir, *, execution_context: None, + ) metadata = run_sft(config_path=config, manifest_path=manifest, output_dir=output_dir, dry_run=False) @@ -1973,11 +2128,20 @@ def test_sft_metadata_contains_public_safe_a100_smoke_fields(monkeypatch: Any, t def test_sft_training_failure_writes_sanitized_metadata(monkeypatch: Any, tmp_path: Path) -> None: manifest = _write_manifest(tmp_path) allowed_root = tmp_path / "remote-root" + allowed_root.mkdir() config = _write_config(tmp_path, allow_heavy_training=True, output_root=allowed_root.as_posix()) output_dir = allowed_root / "runs" / "run" monkeypatch.setattr(training, "_train_dependencies_available", lambda: True) - - def fail_training(metadata: dict[str, Any], config: dict[str, Any], manifest_path: Path, output_dir: Path) -> None: + _install_bounded_run_sft_preflight_stub(monkeypatch) + + def fail_training( + metadata: dict[str, Any], + config: dict[str, Any], + manifest_path: Path, + output_dir: Path, + *, + execution_context: object, + ) -> None: raise RuntimeError("Network is unreachable while reading /" + "Users/person/token.txt") monkeypatch.setattr(training, "_run_real_sft", fail_training) diff --git a/tests/test_contract_compiler_causal_audit.py b/tests/test_contract_compiler_causal_audit.py index c810c6e..7939e0e 100644 --- a/tests/test_contract_compiler_causal_audit.py +++ b/tests/test_contract_compiler_causal_audit.py @@ -22,6 +22,11 @@ "reports/public-sample/contract-compiler-v2-causal-boundary/" f"source-snapshots/CONTEXT.{HISTORICAL_CONTEXT_SHA256}.md" ) +HISTORICAL_TRAINING_SHA256 = "978e2df42be7b1e020c5215febaf843a527b0fb96469273c93b66ce20b62db3c" +HISTORICAL_TRAINING_SNAPSHOT = ( + "reports/public-sample/contract-compiler-v2-causal-boundary/" + f"source-snapshots/training.{HISTORICAL_TRAINING_SHA256}.py" +) def _audit() -> dict[str, object]: @@ -637,7 +642,10 @@ def test_historical_context_snapshot_is_exact_and_live_context_has_advanced() -> overrides = getattr(audit_module, "HISTORICAL_SOURCE_SNAPSHOT_OVERRIDES", None) resolver = getattr(audit_module, "_resolve_audit_source_path", None) - assert overrides == {"CONTEXT.md": HISTORICAL_CONTEXT_SNAPSHOT} + assert overrides == { + "src/voice2task/training.py": HISTORICAL_TRAINING_SNAPSHOT, + "CONTEXT.md": HISTORICAL_CONTEXT_SNAPSHOT, + } assert callable(resolver) snapshot = resolver(REPO_ROOT, "CONTEXT.md") assert snapshot == REPO_ROOT / HISTORICAL_CONTEXT_SNAPSHOT @@ -649,6 +657,18 @@ def test_historical_context_snapshot_is_exact_and_live_context_has_advanced() -> ) +def test_historical_training_snapshot_is_exact_and_live_training_has_advanced() -> None: + snapshot = audit_module._resolve_audit_source_path(REPO_ROOT, "src/voice2task/training.py") + + assert snapshot == REPO_ROOT / HISTORICAL_TRAINING_SNAPSHOT + snapshot_bytes = snapshot.read_bytes() + assert len(snapshot_bytes) == 112_979 + assert hashlib.sha256(snapshot_bytes).hexdigest() == HISTORICAL_TRAINING_SHA256 + assert hashlib.sha256( + (REPO_ROOT / "src/voice2task/training.py").read_bytes() + ).hexdigest() != HISTORICAL_TRAINING_SHA256 + + def test_historical_builder_emits_logical_context_path_and_phase_time_hash() -> None: audit = _audit() context_records = [ diff --git a/tests/test_private_a100_sft_preflight.py b/tests/test_private_a100_sft_preflight.py new file mode 100644 index 0000000..4eacb09 --- /dev/null +++ b/tests/test_private_a100_sft_preflight.py @@ -0,0 +1,2241 @@ +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path +from typing import Any + +import pytest + +from voice2task import training +from voice2task.cli import train as train_cli + + +def _policy(config: dict[str, Any], output_dir: Path, *, repo_root: Path | None = None) -> dict[str, Any]: + return training.validate_sft_output_policy(config, output_dir, repo_root=repo_root) + + +def test_output_policy_requires_output_root(tmp_path: Path) -> None: + result = _policy({}, tmp_path / "run") + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_ROOT_MISSING"] + + +def test_output_policy_rejects_relative_output_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + Path("root").mkdir() + + result = _policy({"output_root": "root"}, (tmp_path / "root" / "run")) + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_ROOT_NOT_ABSOLUTE"] + + +def test_output_policy_rejects_relative_output_dir(tmp_path: Path) -> None: + root = tmp_path / "root" + root.mkdir() + + result = _policy({"output_root": root.as_posix()}, Path("run")) + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_DIR_NOT_ABSOLUTE"] + + +def test_output_policy_rejects_output_dir_equal_to_root(tmp_path: Path) -> None: + root = tmp_path / "root" + root.mkdir() + + result = _policy({"output_root": root.as_posix()}, root) + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_PATH_OUTSIDE_ROOT"] + + +def test_output_policy_rejects_dotdot_escape(tmp_path: Path) -> None: + root = tmp_path / "root" + root.mkdir() + + result = _policy({"output_root": root.as_posix()}, root / ".." / "outside") + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_PATH_OUTSIDE_ROOT"] + + +def test_output_policy_rejects_parent_symlink_to_outside(tmp_path: Path) -> None: + root = tmp_path / "root" + outside = tmp_path / "outside" + root.mkdir() + outside.mkdir() + (root / "linked").symlink_to(outside, target_is_directory=True) + + result = _policy({"output_root": root.as_posix()}, root / "linked" / "run") + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_PATH_SYMLINK", "OUTPUT_PATH_OUTSIDE_ROOT"] + + +def test_output_policy_rejects_final_output_dir_symlink(tmp_path: Path) -> None: + root = tmp_path / "root" + target = root / "target" + root.mkdir() + target.mkdir() + output_dir = root / "run" + output_dir.symlink_to(target, target_is_directory=True) + + result = _policy({"output_root": root.as_posix()}, output_dir) + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_PATH_SYMLINK"] + + +def test_output_policy_rejects_nonempty_existing_output_dir(tmp_path: Path) -> None: + root = tmp_path / "root" + output_dir = root / "run" + output_dir.mkdir(parents=True) + (output_dir / "adapter_model.safetensors").write_bytes(b"existing") + + result = _policy({"output_root": root.as_posix()}, output_dir) + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_DIRECTORY_NOT_EMPTY"] + + +def test_output_policy_rejects_existing_empty_output_dir(tmp_path: Path) -> None: + root = tmp_path / "root" + output_dir = root / "run" + output_dir.mkdir(parents=True) + + result = _policy({"output_root": root.as_posix(), "min_free_disk_gib": 0}, output_dir) + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_DIRECTORY_EXISTS"] + + +def test_output_policy_accepts_new_absolute_child_without_creating_it(tmp_path: Path) -> None: + root = tmp_path / "root" + root.mkdir() + output_dir = root / "runs" / "smoke" + + result = _policy({"output_root": root.as_posix(), "min_free_disk_gib": 0}, output_dir) + + assert result["ready"] is True + assert result["blockers"] == [] + assert result["root_path_sha256"] == training._sha256_text(root.resolve().as_posix()) # noqa: SLF001 + assert result["output_path_sha256"] == training._sha256_text(output_dir.resolve().as_posix()) # noqa: SLF001 + assert not output_dir.exists() + serialized = json.dumps(result, sort_keys=True) + assert root.as_posix() not in serialized + assert output_dir.as_posix() not in serialized + + +def test_output_policy_checks_nearest_existing_parent_writability( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "root" + nearest_parent = root / "runs" + nearest_parent.mkdir(parents=True) + output_dir = nearest_parent / "smoke" + checked_paths: list[Path] = [] + + def fake_access(path: Path, mode: int) -> bool: + assert mode == training.os.W_OK + checked_paths.append(Path(path)) + return Path(path) != nearest_parent + + monkeypatch.setattr(training.os, "access", fake_access) + + result = _policy({"output_root": root.as_posix(), "min_free_disk_gib": 0}, output_dir) + + assert result["ready"] is False + assert "OUTPUT_NOT_WRITABLE" in result["blockers"] + assert nearest_parent in checked_paths + + +@pytest.mark.parametrize( + ("probe", "expected_blocker"), + [ + ("access", "OUTPUT_FILESYSTEM_UNAVAILABLE"), + ("disk", "OUTPUT_DISK_CHECK_FAILED"), + ], +) +def test_output_policy_converts_probe_errors_to_private_safe_blockers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + probe: str, + expected_blocker: str, +) -> None: + root = tmp_path / "root" + root.mkdir() + private_detail = "/private/model/secret-token" + if probe == "access": + monkeypatch.setattr(training.os, "access", lambda *args: (_ for _ in ()).throw(OSError(private_detail))) + else: + monkeypatch.setattr( + training.shutil, + "disk_usage", + lambda *args: (_ for _ in ()).throw(OSError(private_detail)), + ) + + result = _policy( + {"output_root": root.as_posix(), "min_free_disk_gib": 20}, + root / "smoke", + ) + + assert result["ready"] is False + assert expected_blocker in result["blockers"] + assert private_detail not in json.dumps(result, sort_keys=True) + + +def test_output_policy_converts_path_resolution_error_to_private_safe_blocker( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "root" + root.mkdir() + original_resolve = Path.resolve + private_detail = "/private/model/secret-token" + + def selective_resolve(path: Path, *args: Any, **kwargs: Any) -> Path: + if path == root: + raise OSError(private_detail) + return original_resolve(path, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", selective_resolve) + + result = _policy({"output_root": root.as_posix()}, root / "smoke") + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_FILESYSTEM_UNAVAILABLE"] + assert private_detail not in json.dumps(result, sort_keys=True) + + +def test_output_policy_rejects_repository_location(tmp_path: Path) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + output_dir = repo_root / "runs" / "smoke" + + result = _policy( + {"output_root": repo_root.as_posix(), "min_free_disk_gib": 0}, + output_dir, + repo_root=repo_root, + ) + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_PATH_GIT_TRACKED"] + + +def test_output_claim_fails_closed_if_candidate_appears_after_preflight( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "root" + root.mkdir() + output_dir = root / "smoke" + original_mkdir = Path.mkdir + + def racing_mkdir(path: Path, *args: Any, **kwargs: Any) -> None: + if path == output_dir: + original_mkdir(path, *args, **kwargs) + raise FileExistsError(path) + original_mkdir(path, *args, **kwargs) + + monkeypatch.setattr(Path, "mkdir", racing_mkdir) + + with pytest.raises(training.SFTOutputPolicyError) as exc_info: + training._claim_sft_output_directory( # noqa: SLF001 + {"output_root": root.as_posix(), "min_free_disk_gib": 0}, + output_dir, + ) + + assert exc_info.value.blockers == ["OUTPUT_DIRECTORY_EXISTS"] + + +def test_output_policy_rejects_symlink_output_root(tmp_path: Path) -> None: + target = tmp_path / "target" + target.mkdir() + root = tmp_path / "root" + root.symlink_to(target, target_is_directory=True) + + result = _policy({"output_root": root.as_posix()}, root / "run") + + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_ROOT_SYMLINK"] + + +def test_dpo_run_training_checks_output_policy_before_metadata_or_dependencies( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "root" + root.mkdir() + config = tmp_path / "dpo.json" + config.write_text( + json.dumps({"allow_heavy_training": True, "output_root": root.as_posix()}), + encoding="utf-8", + ) + manifest = tmp_path / "manifest.json" + manifest.write_text("{}\n", encoding="utf-8") + repo_root = tmp_path / "checkout" + repo_root.mkdir() + monkeypatch.setattr(training, "_git_repository_root_for_manifest", lambda path: repo_root) + output_dir = tmp_path / "outside" / "dpo" + monkeypatch.setattr( + training, + "_metadata_common", + lambda **kwargs: (_ for _ in ()).throw(AssertionError("metadata path must not run before output gate")), + ) + monkeypatch.setattr( + training, + "_train_dependencies_available", + lambda: (_ for _ in ()).throw(AssertionError("dependency probe must not run before output gate")), + ) + + result = training.run_dpo(config, manifest, output_dir, dry_run=False) + + assert result["training_status"] == "training_blocked_by_output_policy" + assert result["blockers"] == ["OUTPUT_PATH_OUTSIDE_ROOT"] + assert not output_dir.exists() + + +def test_real_dpo_rechecks_output_policy_before_import_or_model_load( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "root" + root.mkdir() + output_dir = tmp_path / "outside" / "dpo" + imports: list[str] = [] + + class FailModule(types.ModuleType): + def __getattr__(self, name: str) -> Any: + imports.append(f"{self.__name__}.{name}") + raise AssertionError("training dependency must not import before the output gate") + + for module_name in ("datasets", "peft", "transformers", "trl"): + monkeypatch.setitem(sys.modules, module_name, FailModule(module_name)) + + with pytest.raises(training.SFTOutputPolicyError) as exc_info: + training._run_real_dpo( # noqa: SLF001 + {"adapter_path": (output_dir / "adapter").as_posix()}, + {"output_root": root.as_posix(), "min_free_disk_gib": 0}, + tmp_path / "manifest.json", + output_dir, + repo_root=tmp_path / "checkout", + ) + + assert exc_info.value.blockers == ["OUTPUT_PATH_OUTSIDE_ROOT"] + assert imports == [] + assert not output_dir.exists() + + +def test_dpo_real_mode_derives_repo_from_manifest_when_process_cwd_is_unrelated( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo_root = tmp_path / "checkout" + manifest = repo_root / "data" / "public-samples" / "manifest_public_sample.json" + manifest.parent.mkdir(parents=True) + manifest.write_text('{"manifest_id":"test"}\n', encoding="utf-8") + output_root = tmp_path / "private-output" + output_root.mkdir() + output_dir = output_root / "dpo-smoke" + config = repo_root / "dpo.json" + config.write_text( + json.dumps({"allow_heavy_training": True, "output_root": output_root.as_posix()}), + encoding="utf-8", + ) + unrelated = tmp_path / "unrelated-cwd" + unrelated.mkdir() + monkeypatch.chdir(unrelated) + git_commands: list[list[str]] = [] + + def fake_git(command: list[str], **kwargs: Any) -> Any: + git_commands.append(command) + assert command == ["git", "-C", manifest.parent.as_posix(), "rev-parse", "--show-toplevel"] + return types.SimpleNamespace(returncode=0, stdout=repo_root.as_posix() + "\n") + + monkeypatch.setattr(training.subprocess, "run", fake_git) + monkeypatch.setattr(training, "_train_dependencies_available", lambda: True) + received_roots: list[Path] = [] + + def run_real( + metadata: dict[str, Any], + config_snapshot: dict[str, Any], + manifest_path: Path, + run_output_dir: Path, + *, + repo_root: Path, + ) -> None: + received_roots.append(repo_root) + + monkeypatch.setattr(training, "_run_real_dpo", run_real) + + result = training.run_dpo(config, manifest, output_dir, dry_run=False) + + assert result["training_status"] == "training_completed" + assert git_commands + assert received_roots == [repo_root] + + +def test_dpo_real_mode_fails_closed_without_manifest_checkout_before_any_write( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config = tmp_path / "dpo.json" + config.write_text("{}\n", encoding="utf-8") + manifest = tmp_path / "manifest.json" + manifest.write_text("{}\n", encoding="utf-8") + output_dir = tmp_path / "output" / "dpo" + monkeypatch.setattr(training, "_git_repository_root_for_manifest", lambda path: None) + + def forbidden(*args: Any, **kwargs: Any) -> None: + raise AssertionError("repo-root failure must precede config, metadata, dependencies, and runner") + + monkeypatch.setattr(training, "_load_config", forbidden) + monkeypatch.setattr(training, "_metadata_common", forbidden) + monkeypatch.setattr(training, "_train_dependencies_available", forbidden) + monkeypatch.setattr(training, "_run_real_dpo", forbidden) + + result = training.run_dpo(config, manifest, output_dir, dry_run=False) + + assert result["training_status"] == "training_blocked_by_output_policy" + assert result["blockers"] == ["OUTPUT_REPOSITORY_UNAVAILABLE"] + assert not output_dir.exists() + + +def _write_preflight_fixture(tmp_path: Path) -> tuple[Path, Path, Path]: + output_root = tmp_path.parent / f"{tmp_path.name}-private-output" + output_root.mkdir() + model_root = tmp_path / "private-model" + model_root.mkdir() + (model_root / "config.json").write_text('{"model_type":"qwen2"}\n', encoding="utf-8") + (model_root / "tokenizer_config.json").write_text("{}\n", encoding="utf-8") + (model_root / "model.safetensors.index.json").write_text("{}\n", encoding="utf-8") + (model_root / "model-00001-of-00001.safetensors").write_bytes(b"weights") + + public_dir = tmp_path / "data" / "public-samples" + public_dir.mkdir(parents=True) + rows_path = public_dir / "sft_public_sample.jsonl" + rows = [ + { + "id": f"sft-{index}", + "split": "train", + "input_text": f"搜索天气 {index}", + "target_contract": { + "task_type": "search", + "route": "search_web", + "safety": {"allow": True, "reason": "public_readonly"}, + "confirmation_required": False, + "slots": {"query": f"天气 {index}"}, + "normalized_command": f"搜索天气 {index}", + "language": "zh-CN", + "contract_version": "v1", + }, + "provenance": {"source_id": f"seed-{index}", "public_safe": True}, + } + for index in (1, 2) + ] + rows_path.write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), encoding="utf-8") + manifest = public_dir / "manifest_public_sample.json" + manifest.write_text( + json.dumps( + { + "manifest_id": "public-sample-20260619T090925Z", + "public_safe": True, + "files": {"sft": "data/public-samples/sft_public_sample.jsonl"}, + "split_counts": {"train": 2, "dev": 0, "test": 0}, + } + ), + encoding="utf-8", + ) + config = tmp_path / "data" / "local-private" / "runtime" / "sft-a100-smoke.json" + config.parent.mkdir(parents=True) + config.write_text( + json.dumps( + { + "base_model_public_id": "Qwen/Qwen2.5-7B-Instruct", + "base_model_runtime_path": model_root.as_posix(), + "allow_heavy_training": True, + "local_files_only": True, + "trust_remote_code": False, + "dtype": "bfloat16", + "torch_dtype": "bfloat16", + "dataset_split": "train", + "dataset_manifest_id": "public-sample-20260619T090925Z", + "max_train_rows": 2, + "max_steps": 1, + "per_device_train_batch_size": 1, + "gradient_accumulation_steps": 1, + "max_seq_length": 1024, + "bf16": True, + "fp16": False, + "tf32": True, + "gradient_checkpointing": True, + "use_cache": False, + "low_cpu_mem_usage": True, + "save_strategy": "no", + "logging_steps": 1, + "seed": 42, + "report_to": [], + "output_root": output_root.as_posix(), + "min_free_disk_gib": 20, + "lora": { + "r": 8, + "alpha": 16, + "dropout": 0.05, + "target_modules": ["q_proj", "k_proj", "v_proj", "o_proj"], + }, + } + ), + encoding="utf-8", + ) + return config, manifest, output_root / "sft-smoke-test" + + +def _install_ready_preflight_probes(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + training, + "_probe_private_sft_config", + lambda config_path, repo_root: ( + {"under_private_runtime": True, "git_ignored": True, "git_tracked": False}, + [], + ), + raising=False, + ) + monkeypatch.setattr( + training, + "_probe_sft_git", + lambda repo_root: ({"commit_sha": "a" * 40, "tracked_worktree_clean": True}, []), + raising=False, + ) + monkeypatch.setattr( + training, + "_probe_sft_dependencies", + lambda: ( + { + "python": "3.12.0", + "versions": { + name: "test" for name in ("torch", "accelerate", "datasets", "peft", "transformers", "trl") + }, + "pip_check": "ok", + }, + [], + ), + raising=False, + ) + monkeypatch.setattr( + training, + "_probe_sft_gpu", + lambda: ( + { + "explicit_selection": True, + "visible_device_count": 1, + "name": "NVIDIA A100-SXM4-80GB", + "compute_capability": "8.0", + "total_memory_gib": 79.15, + "cuda_version": "12.4", + "bf16_supported": True, + }, + [], + ), + raising=False, + ) + def ready_model_probe( + config: dict[str, Any], + rows: list[Any], + ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: + model_root = Path(config["base_model_runtime_path"]) + fingerprints, inventory = training._stable_local_model_inventory(model_root) # noqa: SLF001 + return ( + { + "public_id": "Qwen/Qwen2.5-7B-Instruct", + "local_files_only": True, + "stable_fingerprints": fingerprints, + "weight_inventory": inventory, + }, + { + "records_checked": len(rows), + "prompt_labels_masked": True, + "assistant_target_present": True, + "max_sequence_length": 1024, + }, + [], + ) + + monkeypatch.setattr(training, "_probe_sft_model_and_objective", ready_model_probe, raising=False) + + +def test_sft_preflight_ready_schema_is_complete_and_private_safe( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["schema_version"] == "voice2task-sft-preflight-v1" + assert result["ready"] is True + assert result["status"] == "ready" + assert result["blockers"] == [] + assert set(result) >= { + "git", + "config", + "dataset", + "model", + "runtime", + "gpu", + "output", + "objective", + } + assert result["dataset"]["selected_row_count"] == 2 + assert len(result["dataset"]["manifest_sha256"]) == 64 + assert len(result["dataset"]["sft_sha256"]) == 64 + assert len(result["dataset"]["selected_row_ids_sha256"]) == 64 + serialized = json.dumps(result, sort_keys=True) + assert tmp_path.as_posix() not in serialized + assert "private-model" not in serialized + assert "private-output" not in serialized + + +@pytest.mark.parametrize("entrypoint", ["preflight", "real_sft"]) +def test_shared_sft_preflight_wrapper_converts_unexpected_exception_to_private_safe_blocker( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + entrypoint: str, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + + def broken_dependency_probe() -> tuple[dict[str, Any], list[str]]: + raise RuntimeError("/private/secret-dependency-probe") + + monkeypatch.setattr(training, "_probe_sft_dependencies", broken_dependency_probe) + + if entrypoint == "preflight": + result = training.run_sft_preflight(config, manifest, output_dir) + preflight = result + else: + result = training.run_sft(config, manifest, output_dir, dry_run=False) + assert result["training_status"] == "training_blocked_by_preflight" + assert result["blockers"] == ["PREFLIGHT_INTERNAL_ERROR"] + preflight = result["preflight"] + + assert preflight == { + "schema_version": "voice2task-sft-preflight-v1", + "ready": False, + "status": "blocked", + "blockers": ["PREFLIGHT_INTERNAL_ERROR"], + "git": {}, + "config": {}, + "dataset": {}, + "model": {}, + "runtime": {}, + "gpu": {}, + "output": {}, + "objective": {}, + } + serialized = json.dumps(result, sort_keys=True) + assert "/private/secret-dependency-probe" not in serialized + assert tmp_path.as_posix() not in serialized + assert not output_dir.exists() + + +@pytest.mark.parametrize( + ("case", "expected_blocker"), + [ + ("outside", "CONFIG_PATH_NOT_PRIVATE"), + ("symlink", "CONFIG_PATH_SYMLINK"), + ("not_ignored", "CONFIG_FILE_NOT_IGNORED"), + ("tracked", "CONFIG_FILE_TRACKED"), + ], +) +def test_private_config_policy_fails_closed_for_each_bypass( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + case: str, + expected_blocker: str, +) -> None: + config_path, manifest, _ = _write_preflight_fixture(tmp_path) + repo_root = manifest.parents[2] + candidate = config_path + if case == "outside": + candidate = tmp_path.parent / f"{tmp_path.name}-outside-config.json" + candidate.write_bytes(config_path.read_bytes()) + elif case == "symlink": + target = tmp_path.parent / f"{tmp_path.name}-config-target.json" + target.write_bytes(config_path.read_bytes()) + config_path.unlink() + config_path.symlink_to(target) + + if case == "not_ignored": + git_state = (False, False, True) + elif case == "tracked": + git_state = (True, True, True) + else: + git_state = (True, False, True) + monkeypatch.setattr( + training, + "_git_config_path_state", + lambda *args: git_state, + raising=False, + ) + + _, blockers = training._probe_private_sft_config(candidate, repo_root) # noqa: SLF001 + + assert blockers == [expected_blocker] + + +@pytest.mark.parametrize( + ("mutations", "blocker"), + [ + ({"allow_heavy_training": False}, "CONFIG_HEAVY_TRAINING_NOT_ALLOWED"), + ({"dataset_split": "dev"}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_train_rows": 3}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_train_rows": True}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_train_rows": 1.0}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_steps": 2}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_steps": True}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_steps": 1.0}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"per_device_train_batch_size": 2}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"per_device_train_batch_size": True}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"per_device_train_batch_size": 1.0}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"gradient_accumulation_steps": 2}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"gradient_accumulation_steps": True}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"gradient_accumulation_steps": 1.0}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_seq_length": True}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_seq_length": 0}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_seq_length": 4097}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_seq_length": 1024.0}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"max_seq_length": "/private/secret-sequence-policy"}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"seed": True}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"seed": 42.0}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"logging_steps": True}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"logging_steps": 1.0}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"logging_steps": 0}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"save_strategy": "steps"}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"report_to": ["wandb"]}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"base_model_public_id": "Qwen/Qwen2.5-0.5B-Instruct"}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"local_files_only": False}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"bf16": False}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"fp16": True}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"tf32": False}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"gradient_checkpointing": False}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"use_cache": True}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"trust_remote_code": True}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"dtype": "float16"}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"torch_dtype": "float16"}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"min_free_disk_gib": 19}, "CONFIG_NOT_SMOKE_BOUNDED"), + ({"lora": {"r": 0, "alpha": 16, "dropout": 0.05, "target_modules": ["q_proj"]}}, "CONFIG_NOT_SMOKE_BOUNDED"), + ( + { + "lora": { + "r": 8, + "alpha": 16, + "dropout": 0.05, + "target_modules": ["lm_head"], + } + }, + "CONFIG_NOT_SMOKE_BOUNDED", + ), + ], +) +def test_sft_preflight_blocks_non_smoke_config( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mutations: dict[str, Any], + blocker: str, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + payload = json.loads(config.read_text(encoding="utf-8")) + payload.update(mutations) + config.write_text(json.dumps(payload), encoding="utf-8") + _install_ready_preflight_probes(monkeypatch) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is False + assert blocker in result["blockers"] + assert "/private/secret-sequence-policy" not in json.dumps(result, sort_keys=True) + + +@pytest.mark.parametrize("invalid_row_limit", [True, 1.0]) +def test_sft_preflight_does_not_select_rows_for_non_integer_max_train_rows( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + invalid_row_limit: Any, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + payload = json.loads(config.read_text(encoding="utf-8")) + payload["max_train_rows"] = invalid_row_limit + config.write_text(json.dumps(payload), encoding="utf-8") + _install_ready_preflight_probes(monkeypatch) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is False + assert "CONFIG_NOT_SMOKE_BOUNDED" in result["blockers"] + assert "TRAIN_ROW_SELECTION_INVALID" in result["blockers"] + assert result["dataset"]["selected_row_count"] == 0 + + +def test_sft_preflight_public_config_facts_never_echo_unapproved_private_strings( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + payload = json.loads(config.read_text(encoding="utf-8")) + payload.update( + { + "base_model_public_id": "/private/secret-model-identity", + "dataset_split": "/private/secret-split", + "save_strategy": "/private/secret-save-strategy", + } + ) + config.write_text(json.dumps(payload), encoding="utf-8") + _install_ready_preflight_probes(monkeypatch) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is False + assert "CONFIG_NOT_SMOKE_BOUNDED" in result["blockers"] + serialized = json.dumps(result, sort_keys=True) + assert "/private/secret" not in serialized + assert result["config"]["base_model_public_id"] is None + assert result["config"]["dataset_split"] is None + assert result["config"]["save_strategy"] is None + + +@pytest.mark.parametrize("missing", ["torch", "accelerate"]) +def test_sft_preflight_blocks_missing_torch_or_accelerate( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + missing: str, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + monkeypatch.setattr( + training, + "_probe_sft_dependencies", + lambda: ({"missing": [missing], "versions": {}}, ["DEPENDENCY_MISSING"]), + ) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is False + assert result["blockers"] == ["DEPENDENCY_MISSING"] + + +def test_git_probe_uses_bound_repo_and_sanitizes_subprocess_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[Path] = [] + + def broken_git(*args: Any, **kwargs: Any) -> None: + calls.append(Path(kwargs["cwd"])) + raise OSError("private git transport detail") + + monkeypatch.setattr(training.subprocess, "run", broken_git) + + facts, blockers = training._probe_sft_git(tmp_path) # noqa: SLF001 + + assert calls == [tmp_path] + assert facts == {"commit_sha": None, "tracked_worktree_clean": False} + assert blockers == ["GIT_PROBE_FAILED"] + + +@pytest.mark.parametrize( + "blocker", + [ + "CUDA_UNAVAILABLE", + "BF16_UNSUPPORTED", + "GPU_MEMORY_INSUFFICIENT", + "GPU_SELECTION_NOT_EXPLICIT", + ], +) +def test_sft_preflight_blocks_invalid_gpu_probe( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + blocker: str, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + monkeypatch.setattr(training, "_probe_sft_gpu", lambda: ({"available": False}, [blocker])) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is False + assert result["blockers"] == [blocker] + + +def test_gpu_probe_rejects_non_a100_even_when_other_cuda_facts_pass( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") + + class FakeCuda: + @staticmethod + def is_available() -> bool: + return True + + @staticmethod + def device_count() -> int: + return 1 + + @staticmethod + def get_device_properties(index: int) -> Any: + assert index == 0 + return types.SimpleNamespace(total_memory=80 * 1024**3) + + @staticmethod + def get_device_capability(index: int) -> tuple[int, int]: + assert index == 0 + return (8, 9) + + @staticmethod + def is_bf16_supported() -> bool: + return True + + @staticmethod + def get_device_name(index: int) -> str: + assert index == 0 + return "NVIDIA GeForce RTX 4090" + + monkeypatch.setitem( + sys.modules, + "torch", + types.SimpleNamespace(cuda=FakeCuda(), version=types.SimpleNamespace(cuda="12.4")), + ) + + facts, blockers = training._probe_sft_gpu() # noqa: SLF001 + + assert facts["name"] == "NVIDIA GeForce RTX 4090" + assert blockers == ["GPU_NOT_A100"] + + +def test_gpu_probe_converts_cuda_api_exception_to_stable_blocker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") + + class BrokenCuda: + @staticmethod + def is_available() -> bool: + return True + + @staticmethod + def device_count() -> int: + raise RuntimeError("private CUDA driver detail") + + monkeypatch.setitem( + sys.modules, + "torch", + types.SimpleNamespace(cuda=BrokenCuda(), version=types.SimpleNamespace(cuda="12.4")), + ) + + facts, blockers = training._probe_sft_gpu() # noqa: SLF001 + + assert facts["visible_device_count"] == 0 + assert blockers == ["CUDA_PROBE_FAILED"] + + +def test_sft_preflight_blocks_missing_private_model_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + payload = json.loads(config.read_text(encoding="utf-8")) + payload["base_model_runtime_path"] = (tmp_path / "missing-model").as_posix() + config.write_text(json.dumps(payload), encoding="utf-8") + _install_ready_preflight_probes(monkeypatch) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is False + assert "MODEL_PATH_UNRESOLVED" in result["blockers"] + assert "missing-model" not in json.dumps(result) + + +def test_sft_preflight_blocks_manifest_id_mismatch(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + payload = json.loads(config.read_text(encoding="utf-8")) + payload["dataset_manifest_id"] = "different-id" + config.write_text(json.dumps(payload), encoding="utf-8") + _install_ready_preflight_probes(monkeypatch) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is False + assert "DATASET_HASH_OR_ID_MISMATCH" in result["blockers"] + + +def test_sft_preflight_rejects_noncanonical_manifest_before_dataset_read( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, canonical_manifest, output_dir = _write_preflight_fixture(tmp_path) + alternate_manifest = tmp_path / "manifest_public_sample.json" + alternate_manifest.write_bytes(canonical_manifest.read_bytes()) + _install_ready_preflight_probes(monkeypatch) + + def forbidden_dataset_read(*args: Any, **kwargs: Any) -> None: + raise AssertionError("noncanonical manifest must be rejected before SFT access") + + monkeypatch.setattr(training, "_load_selected_smoke_rows", forbidden_dataset_read) + + result = training.run_sft_preflight(config, alternate_manifest, output_dir) + + assert result["ready"] is False + assert result["blockers"] == ["MANIFEST_PATH_NOT_CANONICAL"] + + +def test_sft_preflight_rejects_noncanonical_sft_binding_before_row_parse( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + payload = json.loads(manifest.read_text(encoding="utf-8")) + payload["files"]["sft"] = "data/public-samples/alternate-sft.jsonl" + manifest.write_text(json.dumps(payload), encoding="utf-8") + _install_ready_preflight_probes(monkeypatch) + + def forbidden_row_parse(*args: Any, **kwargs: Any) -> None: + raise AssertionError("noncanonical SFT binding must be rejected before row parsing") + + monkeypatch.setattr(training, "_load_selected_smoke_rows", forbidden_row_parse) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is False + assert result["blockers"] == ["DATASET_HASH_OR_ID_MISMATCH"] + + +def test_sft_preflight_selection_hash_changes_with_train_rows( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + first = training.run_sft_preflight(config, manifest, output_dir) + rows_path = manifest.parent / "sft_public_sample.jsonl" + rows = [json.loads(line) for line in rows_path.read_text(encoding="utf-8").splitlines()] + rows.reverse() + rows_path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + + second = training.run_sft_preflight(config, manifest, output_dir) + + assert first["dataset"]["selected_row_ids_sha256"] != second["dataset"]["selected_row_ids_sha256"] + assert first["dataset"]["sft_sha256"] != second["dataset"]["sft_sha256"] + + +def test_sft_preflight_stops_parsing_after_exact_bounded_train_rows( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + rows_path = manifest.parent / "sft_public_sample.jsonl" + with rows_path.open("a", encoding="utf-8") as handle: + handle.write("this later dev/test payload must never be parsed\n") + _install_ready_preflight_probes(monkeypatch) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is True + assert result["dataset"]["selected_row_count"] == 2 + assert result["dataset"]["non_train_rows_selected"] == 0 + + +@pytest.mark.parametrize( + "blocker", + ["ASSISTANT_ONLY_LABELS_INVALID", "MAX_SEQUENCE_LENGTH_EXCEEDED"], +) +def test_sft_preflight_blocks_invalid_objective( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + blocker: str, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + monkeypatch.setattr( + training, + "_probe_sft_model_and_objective", + lambda config, rows: ({"local_files_only": True}, {"records_checked": 0}, [blocker]), + ) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is False + assert result["blockers"] == [blocker] + + +@pytest.mark.parametrize( + ("training_status", "expected_exit"), + [ + ("dry_run", 0), + ("training_completed", 0), + ("training_skipped_by_config", 1), + ("training_unavailable", 1), + ("training_blocked_by_output_policy", 1), + ("training_blocked_by_preflight", 1), + ("training_failed", 1), + ], +) +def test_sft_cli_maps_training_status_to_exit_code_and_one_json( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + training_status: str, + expected_exit: int, +) -> None: + monkeypatch.setattr(train_cli, "run_sft", lambda *args, **kwargs: {"training_status": training_status}) + + exit_code = train_cli.main( + [ + "sft", + "--config", + "config.json", + "--manifest", + "manifest.json", + "--output-dir", + "/approved/run", + "--dry-run", + ] + ) + + captured = capsys.readouterr() + assert exit_code == expected_exit + assert json.loads(captured.out) == {"training_status": training_status} + assert captured.err == "" + + +@pytest.mark.parametrize(("ready", "expected_exit"), [(True, 0), (False, 1)]) +def test_sft_preflight_cli_maps_ready_to_exit_code_and_one_json( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ready: bool, + expected_exit: int, +) -> None: + result = { + "schema_version": "voice2task-sft-preflight-v1", + "ready": ready, + "status": "ready" if ready else "blocked", + "blockers": [] if ready else ["CUDA_UNAVAILABLE"], + } + monkeypatch.setattr(train_cli, "run_sft_preflight", lambda *args, **kwargs: result, raising=False) + + exit_code = train_cli.main( + [ + "sft-preflight", + "--config", + "config.json", + "--manifest", + "manifest.json", + "--output-dir", + "/approved/run", + ] + ) + + captured = capsys.readouterr() + assert exit_code == expected_exit + assert json.loads(captured.out) == result + assert captured.err == "" + + +def test_sft_preflight_cli_missing_config_is_stable_blocked_preflight_json( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + missing_config = tmp_path / "missing-private-config.json" + + exit_code = train_cli.main( + [ + "sft-preflight", + "--config", + missing_config.as_posix(), + "--manifest", + (tmp_path / "manifest.json").as_posix(), + "--output-dir", + (tmp_path / "approved" / "run").as_posix(), + ] + ) + + captured = capsys.readouterr() + assert exit_code == 1 + assert json.loads(captured.out) == { + "schema_version": "voice2task-sft-preflight-v1", + "ready": False, + "status": "blocked", + "blockers": ["CONFIG_FILE_MISSING"], + "git": {}, + "config": {}, + "dataset": {}, + "model": {}, + "runtime": {}, + "gpu": {}, + "output": {}, + "objective": {}, + } + assert captured.err == "" + assert missing_config.as_posix() not in captured.out + + +def test_sft_preflight_malformed_config_is_sanitized_blocked_schema( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + config.write_text("{not-json", encoding="utf-8") + _install_ready_preflight_probes(monkeypatch) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["schema_version"] == "voice2task-sft-preflight-v1" + assert result["ready"] is False + assert result["status"] == "blocked" + assert result["blockers"] == ["CONFIG_LOAD_FAILED"] + serialized = json.dumps(result) + assert config.as_posix() not in serialized + assert "not-json" not in serialized + + +def test_sft_cli_runtime_exception_is_sanitized_nonzero_single_json( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def fail(*args: Any, **kwargs: Any) -> dict[str, Any]: + raise RuntimeError("private failure at /" + "Users/person/model with token=secret") + + monkeypatch.setattr(train_cli, "run_sft", fail) + + exit_code = train_cli.main( + [ + "sft", + "--config", + "config.json", + "--manifest", + "manifest.json", + "--output-dir", + "/approved/run", + "--run-training", + ] + ) + + captured = capsys.readouterr() + result = json.loads(captured.out) + assert exit_code == 1 + assert result == { + "schema_version": "voice2task-training-result-v1", + "training_status": "training_failed", + "blockers": ["TRAINING_RUNTIME_ERROR"], + } + assert captured.err == "" + assert "Users" not in captured.out + assert "secret" not in captured.out + + +def test_sft_cli_sanitizes_private_paths_in_normal_blocked_result( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + train_cli, + "run_sft", + lambda *args, **kwargs: { + "training_status": "training_blocked_by_preflight", + "blockers": ["CUDA_UNAVAILABLE"], + "hyperparameters": { + "base_model_runtime_path": "/" + "Users/person/private-model", + "output_root": "/" + "mnt/data/person/private-output", + }, + }, + ) + + exit_code = train_cli.main( + [ + "sft", + "--config", + "config.json", + "--manifest", + "manifest.json", + "--output-dir", + "/approved/run", + "--run-training", + ] + ) + + captured = capsys.readouterr() + result = json.loads(captured.out) + assert exit_code == 1 + assert result["hyperparameters"] == { + "base_model_runtime_path": "", + "output_root": "", + } + assert "Users/person" not in captured.out + assert "mnt/data/person" not in captured.out + assert captured.err == "" + + +def _ready_preflight_result() -> dict[str, Any]: + return { + "schema_version": "voice2task-sft-preflight-v1", + "ready": True, + "status": "ready", + "blockers": [], + "git": {"commit_sha": "a" * 40, "tracked_worktree_clean": True}, + "config": {"config_sha256": "b" * 64}, + "dataset": { + "manifest_sha256": "c" * 64, + "sft_sha256": "d" * 64, + "selected_row_ids_sha256": "e" * 64, + "selected_row_count": 2, + }, + "model": {"public_id": "Qwen/Qwen2.5-7B-Instruct"}, + "runtime": {"versions": {"torch": "test"}}, + "gpu": {"name": "NVIDIA A100-SXM4-80GB"}, + "output": {"ready": True, "blockers": [], "output_path_sha256": "f" * 64}, + "objective": {"records_checked": 2}, + } + + +def test_run_sft_returns_preflight_blocker_without_writing_candidate( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + blocked = _ready_preflight_result() + blocked.update({"ready": False, "status": "blocked", "blockers": ["CUDA_UNAVAILABLE"]}) + monkeypatch.setattr( + training, + "_run_sft_preflight_core", + lambda *args: (blocked, None), + ) + model_loads: list[str] = [] + monkeypatch.setattr(training, "_run_real_sft", lambda *args: model_loads.append("loaded")) + + result = training.run_sft(config, manifest, output_dir, dry_run=False) + + assert result["training_status"] == "training_blocked_by_preflight" + assert result["blockers"] == ["CUDA_UNAVAILABLE"] + assert model_loads == [] + assert not output_dir.exists() + + +def test_run_sft_blocked_real_mode_calls_preflight_before_all_legacy_reads( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + blocked = _ready_preflight_result() + blocked.update({"ready": False, "status": "blocked", "blockers": ["CUDA_UNAVAILABLE"]}) + calls: list[str] = [] + + def shared_core(*args: Any) -> tuple[dict[str, Any], None]: + calls.append("preflight") + return blocked, None + + def forbidden_read(*args: Any, **kwargs: Any) -> None: + raise AssertionError("real SFT must not use legacy config/metadata/dataset reads around preflight") + + monkeypatch.setattr(training, "_run_sft_preflight_core", shared_core) + monkeypatch.setattr(training, "_load_config", forbidden_read) + monkeypatch.setattr(training, "_metadata_common", forbidden_read) + monkeypatch.setattr(training, "_manifest_load_summary", forbidden_read) + monkeypatch.setattr(training, "read_jsonl", forbidden_read) + + result = training.run_sft(config, manifest, output_dir, dry_run=False) + + assert calls == ["preflight"] + assert result["training_status"] == "training_blocked_by_preflight" + assert result["blockers"] == ["CUDA_UNAVAILABLE"] + assert result["preflight"] == blocked + assert not output_dir.exists() + + +def test_run_sft_ready_real_mode_builds_metadata_only_from_bound_context( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert execution_context is not None + calls: list[str] = [] + + def shared_core(*args: Any) -> tuple[dict[str, Any], object]: + calls.append("preflight") + return preflight, execution_context + + def forbidden_read(*args: Any, **kwargs: Any) -> None: + raise AssertionError("ready real SFT must use only bound preflight context") + + monkeypatch.setattr(training, "_run_sft_preflight_core", shared_core) + monkeypatch.setattr(training, "_load_config", forbidden_read) + monkeypatch.setattr(training, "_metadata_common", forbidden_read) + monkeypatch.setattr(training, "_manifest_load_summary", forbidden_read) + monkeypatch.setattr(training, "read_jsonl", forbidden_read) + + def run_real( + metadata: dict[str, Any], + config_snapshot: dict[str, Any], + bound_manifest: Path, + bound_output: Path, + *, + execution_context: object, + ) -> None: + assert config_snapshot == execution_context.config_snapshot() + assert bound_manifest == execution_context.manifest_path + assert bound_output == execution_context.output_dir + adapter = bound_output / "adapter" + adapter.mkdir(parents=True) + (adapter / "adapter_config.json").write_text("{}\n", encoding="utf-8") + (adapter / "adapter_model.safetensors").write_bytes(b"adapter") + metadata.update( + { + "observed_optimizer_steps": 1, + "training_rows_used": 2, + "train_result_metrics": {"train_loss": 1.25}, + } + ) + + monkeypatch.setattr(training, "_run_real_sft", run_real) + + result = training.run_sft(config, manifest, output_dir, dry_run=False) + + assert calls == ["preflight"] + assert result["training_status"] == "training_completed" + assert result["dataset_manifest_id"] == execution_context.manifest_id + assert result["dataset_load"]["loaded_rows"] == 2 + assert result["preflight"] == preflight + + +def test_run_sft_does_not_read_dataset_before_blocked_preflight( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + blocked = _ready_preflight_result() + blocked.update({"ready": False, "status": "blocked", "blockers": ["CUDA_UNAVAILABLE"]}) + monkeypatch.setattr( + training, + "_run_sft_preflight_core", + lambda *args: (blocked, None), + ) + + def forbidden_legacy_dataset_read(*args: Any, **kwargs: Any) -> None: + raise AssertionError("real run must not select dataset rows before preflight") + + monkeypatch.setattr( + training, + "_record_sft_training_selection_from_config", + forbidden_legacy_dataset_read, + ) + + result = training.run_sft(config, manifest, output_dir, dry_run=False) + + assert result["training_status"] == "training_blocked_by_preflight" + assert result["blockers"] == ["CUDA_UNAVAILABLE"] + assert not output_dir.exists() + + +def test_run_sft_uses_shared_preflight_core_and_passes_bound_context_to_runner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + preflight, bound_context = training._run_sft_preflight_core( # noqa: SLF001 + config, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert bound_context is not None + core_calls: list[tuple[Path, Path, Path]] = [] + runner_contexts: list[object] = [] + + def shared_core( + config_path: Path, + manifest_path: Path, + candidate: Path, + ) -> tuple[dict[str, Any], object]: + core_calls.append((config_path, manifest_path, candidate)) + return preflight, bound_context + + monkeypatch.setattr(training, "_run_sft_preflight_core", shared_core, raising=False) + + def forbidden_public_wrapper(*args: Any, **kwargs: Any) -> dict[str, Any]: + raise AssertionError("real execution must call the shared preflight core") + + monkeypatch.setattr(training, "run_sft_preflight", forbidden_public_wrapper) + monkeypatch.setattr( + training, + "validate_sft_output_policy", + lambda *args, **kwargs: {"ready": True, "blockers": [], "output_path_sha256": "f" * 64}, + ) + + def run_real( + metadata: dict[str, Any], + config_snapshot: dict[str, Any], + manifest_path: Path, + run_output_dir: Path, + *, + execution_context: object, + ) -> None: + runner_contexts.append(execution_context) + adapter = run_output_dir / "adapter" + adapter.mkdir(parents=True) + (adapter / "adapter_config.json").write_text("{}\n", encoding="utf-8") + (adapter / "adapter_model.safetensors").write_bytes(b"adapter") + metadata.update( + { + "observed_optimizer_steps": 1, + "training_rows_used": 2, + "train_result_metrics": {"train_loss": 1.25}, + } + ) + + monkeypatch.setattr(training, "_run_real_sft", run_real) + + result = training.run_sft(config, manifest, output_dir, dry_run=False) + + assert result["training_status"] == "training_completed" + assert core_calls == [(config, manifest, output_dir)] + assert runner_contexts == [bound_context] + + +def test_run_sft_rechecks_output_policy_after_ready_preflight_and_blocks_drift( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert execution_context is not None + monkeypatch.setattr( + training, + "_run_sft_preflight_core", + lambda *args: (preflight, execution_context), + ) + policy_calls: list[Path] = [] + + def drifted_policy(config: dict[str, Any], candidate: Path, **kwargs: Any) -> dict[str, Any]: + policy_calls.append(candidate) + return { + "ready": False, + "blockers": ["OUTPUT_PATH_SYMLINK"], + "root_path_sha256": "a" * 64, + "output_path_sha256": "b" * 64, + } + + monkeypatch.setattr(training, "validate_sft_output_policy", drifted_policy) + model_loads: list[str] = [] + monkeypatch.setattr(training, "_run_real_sft", lambda *args: model_loads.append("loaded")) + + result = training.run_sft(config, manifest, output_dir, dry_run=False) + + assert result["training_status"] == "training_blocked_by_output_policy" + assert result["blockers"] == ["OUTPUT_PATH_SYMLINK"] + assert policy_calls == [output_dir] + assert model_loads == [] + assert not output_dir.exists() + + +def test_run_sft_does_not_write_when_final_in_runner_output_check_detects_drift( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert execution_context is not None + monkeypatch.setattr( + training, + "_run_sft_preflight_core", + lambda *args: (preflight, execution_context), + ) + policies = iter( + [ + {"ready": True, "blockers": [], "output_path_sha256": "a" * 64}, + {"ready": False, "blockers": ["OUTPUT_PATH_SYMLINK"], "output_path_sha256": "a" * 64}, + ] + ) + monkeypatch.setattr(training, "validate_sft_output_policy", lambda *args, **kwargs: next(policies)) + + result = training.run_sft(config, manifest, output_dir, dry_run=False) + + assert result["training_status"] == "training_blocked_by_output_policy" + assert result["blockers"] == ["OUTPUT_PATH_SYMLINK"] + assert not output_dir.exists() + + +def test_run_sft_returns_stable_blocker_when_bound_preflight_input_drifts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert execution_context is not None + monkeypatch.setattr( + training, + "_run_sft_preflight_core", + lambda *args: (preflight, execution_context), + ) + monkeypatch.setattr( + training, + "validate_sft_output_policy", + lambda *args, **kwargs: {"ready": True, "blockers": []}, + ) + + def drifted_runner(*args: Any, **kwargs: Any) -> None: + raise training.SFTPreflightDriftError(["CONFIG_DRIFT_DETECTED"]) + + monkeypatch.setattr(training, "_run_real_sft", drifted_runner) + + result = training.run_sft(config, manifest, output_dir, dry_run=False) + + assert result["training_status"] == "training_blocked_by_preflight" + assert result["blockers"] == ["CONFIG_DRIFT_DETECTED"] + assert not output_dir.exists() + + +def test_run_sft_smoke_completion_preserves_clean_evaluation_blockers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert execution_context is not None + monkeypatch.setattr( + training, + "_run_sft_preflight_core", + lambda *args: (preflight, execution_context), + ) + monkeypatch.setattr( + training, + "validate_sft_output_policy", + lambda *args, **kwargs: {"ready": True, "blockers": [], "output_path_sha256": "f" * 64}, + ) + + def run_real( + metadata: dict[str, Any], + config: dict[str, Any], + manifest_path: Path, + run_output_dir: Path, + *, + execution_context: object, + ) -> None: + adapter = run_output_dir / "adapter" + adapter.mkdir(parents=True) + (adapter / "adapter_config.json").write_text("{}\n", encoding="utf-8") + (adapter / "adapter_model.safetensors").write_bytes(b"adapter") + metadata.update( + { + "observed_optimizer_steps": 1, + "training_rows_used": 2, + "train_result_metrics": {"train_loss": 1.25}, + "training_budget": {"configured_max_steps": 1, "observed_optimizer_steps": 1}, + } + ) + + monkeypatch.setattr(training, "_run_real_sft", run_real) + + result = training.run_sft(config, manifest, output_dir, dry_run=False) + + assert result["training_status"] == "training_completed" + assert result["smoke_status"] == "SMOKE_COMPLETED" + assert result["observed_optimizer_steps"] == 1 + assert result["training_rows_used"] == 2 + assert result["adapter_files"] == [ + {"name": "adapter_config.json", "size": 3}, + {"name": "adapter_model.safetensors", "size": 7}, + ] + assert result["clean_evaluation"] == { + "acquisition_source_status": "UNAVAILABLE", + "authoritatively_bound_binding_count": 0, + "human_acceptance_status": "NOT_RECORDED", + "protocol_freeze_status": "NOT_FROZEN", + "clean_population_status": "NOT_MATERIALIZED", + "freeze_authorized": False, + "execution_readiness": False, + } + assert result["preflight"] == preflight + + +@pytest.mark.parametrize( + ("case", "expected_blocker"), + [ + ("boolean_loss", "TRAINING_LOSS_INVALID"), + ("missing_adapter_config", "ADAPTER_OUTPUT_INVALID"), + ("full_weight_outside_adapter", "FULL_MODEL_WEIGHTS_DETECTED"), + ("row_count_mismatch", "TRAIN_ROW_SELECTION_INVALID"), + ], +) +def test_sft_smoke_postconditions_fail_closed_on_invalid_artifacts_and_budget( + tmp_path: Path, + case: str, + expected_blocker: str, +) -> None: + run_dir = tmp_path / "run" + adapter = run_dir / "adapter" + adapter.mkdir(parents=True) + (adapter / "adapter_config.json").write_text("{}\n", encoding="utf-8") + (adapter / "adapter_model.safetensors").write_bytes(b"adapter") + metadata: dict[str, Any] = { + "adapter_path": adapter.as_posix(), + "metadata_path": (run_dir / "adapter_metadata.json").as_posix(), + "hyperparameters": {"max_train_rows": 2}, + "observed_optimizer_steps": 1, + "training_rows_used": 2, + "train_result_metrics": {"train_loss": 1.25}, + } + + if case == "boolean_loss": + metadata["train_result_metrics"] = {"train_loss": True} + elif case == "missing_adapter_config": + (adapter / "adapter_config.json").unlink() + elif case == "full_weight_outside_adapter": + checkpoint = run_dir / "checkpoint-1" + checkpoint.mkdir() + (checkpoint / "model-00001-of-00002.safetensors").write_bytes(b"base") + else: + metadata["training_rows_used"] = 1 + + blockers = training._sft_smoke_postconditions(metadata) # noqa: SLF001 + + assert expected_blocker in blockers + + +def test_real_sft_rechecks_output_before_tokenizer_or_model_loading( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path, manifest, output_dir = _write_preflight_fixture(tmp_path) + config = json.loads(config_path.read_text(encoding="utf-8")) + _install_ready_preflight_probes(monkeypatch) + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config_path, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert execution_context is not None + loads: list[str] = [] + monkeypatch.setattr( + training, + "validate_sft_output_policy", + lambda *args, **kwargs: { + "ready": False, + "blockers": ["OUTPUT_PATH_SYMLINK"], + "output_path_sha256": "a" * 64, + }, + ) + + class FailIfLoaded: + @staticmethod + def from_pretrained(*args: Any, **kwargs: Any) -> None: + loads.append("loaded") + + monkeypatch.setitem( + sys.modules, + "transformers", + types.SimpleNamespace(AutoModelForCausalLM=FailIfLoaded, AutoTokenizer=FailIfLoaded), + ) + monkeypatch.setitem(sys.modules, "datasets", types.SimpleNamespace(Dataset=object)) + monkeypatch.setitem(sys.modules, "trl", types.SimpleNamespace(SFTTrainer=object)) + + with pytest.raises(training.SFTOutputPolicyError) as exc_info: + training._run_real_sft( # noqa: SLF001 + {"adapter_path": (output_dir / "adapter").as_posix(), "dataset_load": {}}, + config, + manifest, + output_dir, + execution_context=execution_context, + ) + + assert exc_info.value.blockers == ["OUTPUT_PATH_SYMLINK"] + assert loads == [] + assert not output_dir.exists() + + +def test_real_sft_blocks_config_drift_before_any_model_load( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config_path, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert execution_context is not None + + payload = json.loads(config_path.read_text(encoding="utf-8")) + payload["max_steps"] = 2 + config_path.write_text(json.dumps(payload), encoding="utf-8") + model_loads: list[str] = [] + + class FailIfLoaded: + @staticmethod + def from_pretrained(*args: Any, **kwargs: Any) -> None: + model_loads.append("loaded") + raise AssertionError("model/tokenizer load must not occur after preflight drift") + + monkeypatch.setitem( + sys.modules, + "transformers", + types.SimpleNamespace(AutoModelForCausalLM=FailIfLoaded, AutoTokenizer=FailIfLoaded), + ) + monkeypatch.setitem(sys.modules, "datasets", types.SimpleNamespace(Dataset=object)) + monkeypatch.setitem(sys.modules, "trl", types.SimpleNamespace(SFTTrainer=object)) + + with pytest.raises(RuntimeError) as exc_info: + training._run_real_sft( # noqa: SLF001 + {"adapter_path": (output_dir / "adapter").as_posix(), "dataset_load": {}}, + payload, + manifest, + output_dir, + execution_context=execution_context, + ) + + assert getattr(exc_info.value, "blockers", None) == ["CONFIG_DRIFT_DETECTED"] + assert model_loads == [] + assert not output_dir.exists() + + +@pytest.mark.parametrize( + ("mutated_input", "expected_blocker"), + [ + ("manifest", "MANIFEST_DRIFT_DETECTED"), + ("sft", "SFT_DRIFT_DETECTED"), + ("model", "MODEL_DRIFT_DETECTED"), + ], +) +def test_bound_preflight_context_detects_each_mutable_input_drift( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mutated_input: str, + expected_blocker: str, +) -> None: + config_path, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config_path, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert execution_context is not None + + if mutated_input == "manifest": + with manifest.open("a", encoding="utf-8") as handle: + handle.write("\n") + elif mutated_input == "sft": + with (manifest.parent / "sft_public_sample.jsonl").open("a", encoding="utf-8") as handle: + handle.write("{}\n") + else: + model_path = Path(json.loads(config_path.read_text(encoding="utf-8"))["base_model_runtime_path"]) + with (model_path / "model-00001-of-00001.safetensors").open("ab") as handle: + handle.write(b"drift") + + assert training._preflight_input_drift_blockers(execution_context) == [ # noqa: SLF001 + expected_blocker + ] + + +def test_preflight_context_binds_model_inventory_returned_by_validated_probe( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + config = json.loads(config_path.read_text(encoding="utf-8")) + model_root = Path(config["base_model_runtime_path"]) + weight_path = model_root / "model-00001-of-00001.safetensors" + original_size = weight_path.stat().st_size + + def probe_then_swap( + config_snapshot: dict[str, Any], + rows: list[Any], + ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: + fingerprints, inventory = training._stable_local_model_inventory(model_root) # noqa: SLF001 + weight_path.write_bytes(weight_path.read_bytes() + b"swapped-after-probe") + return ( + { + "public_id": "Qwen/Qwen2.5-7B-Instruct", + "local_files_only": True, + "stable_fingerprints": fingerprints, + "weight_inventory": inventory, + "total_weight_bytes": sum(int(item["size"]) for item in inventory), + "minimum_weight_bytes": 12 * 1024**3, + "geometry_matches_qwen2_5_7b": True, + "snapshot_revision_sha256": "f" * 64, + }, + { + "records_checked": len(rows), + "prompt_labels_masked": True, + "assistant_target_present": True, + "max_sequence_length": 1024, + }, + [], + ) + + monkeypatch.setattr(training, "_probe_sft_model_and_objective", probe_then_swap) + + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config_path, + manifest, + output_dir, + ) + + assert preflight["ready"] is True + assert preflight["model"]["weight_inventory"][0]["size"] == original_size + assert execution_context is not None + assert training._preflight_input_drift_blockers(execution_context) == [ # noqa: SLF001 + "MODEL_DRIFT_DETECTED" + ] + + +def test_real_sft_passes_local_bf16_options_and_bounded_training_arguments( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path, manifest, output_dir = _write_preflight_fixture(tmp_path) + config = json.loads(config_path.read_text(encoding="utf-8")) + _install_ready_preflight_probes(monkeypatch) + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config_path, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert execution_context is not None + calls: dict[str, Any] = {"order": []} + fake_bfloat16 = object() + + class FakeTokenizer: + pad_token_id = None + pad_token = None + eos_token_id = 2 + eos_token = "" + + tokenizer = FakeTokenizer() + + class FakeAutoTokenizer: + @staticmethod + def from_pretrained(path: str, **kwargs: Any) -> FakeTokenizer: + calls["order"].append("tokenizer") + calls["tokenizer"] = {"path": path, **kwargs} + return tokenizer + + class FakeModel: + def __init__(self) -> None: + self.config = types.SimpleNamespace(use_cache=True) + self.gradient_checkpointing_enabled = False + + def gradient_checkpointing_enable(self) -> None: + self.gradient_checkpointing_enabled = True + + def save_pretrained(self, path: str) -> None: + adapter = Path(path) + adapter.mkdir(parents=True) + (adapter / "adapter_model.safetensors").write_bytes(b"adapter") + + model = FakeModel() + + class FakeAutoModelForCausalLM: + @staticmethod + def from_pretrained(path: str, **kwargs: Any) -> FakeModel: + calls["order"].append("model") + calls["model"] = {"path": path, **kwargs} + return model + + class FakeDataset: + @staticmethod + def from_list(records: list[dict[str, list[int]]]) -> list[dict[str, list[int]]]: + calls["records"] = records + return records + + class FakeTrainingArguments: + def __init__(self, **kwargs: Any) -> None: + calls["training_arguments"] = kwargs + + class FakeSFTTrainer: + def __init__( + self, + *, + model: Any, + processing_class: Any, + train_dataset: Any, + args: Any, + peft_config: Any, + data_collator: Any, + ) -> None: + self.model = model + self.state = types.SimpleNamespace(global_step=1) + + def train(self) -> Any: + return types.SimpleNamespace(metrics={"train_loss": 0.5}) + + monkeypatch.setitem(sys.modules, "torch", types.SimpleNamespace(bfloat16=fake_bfloat16)) + monkeypatch.setitem(sys.modules, "datasets", types.SimpleNamespace(Dataset=FakeDataset)) + monkeypatch.setitem( + sys.modules, + "transformers", + types.SimpleNamespace( + AutoModelForCausalLM=FakeAutoModelForCausalLM, + AutoTokenizer=FakeAutoTokenizer, + TrainingArguments=FakeTrainingArguments, + ), + ) + monkeypatch.setitem(sys.modules, "trl", types.SimpleNamespace(SFTTrainer=FakeSFTTrainer)) + monkeypatch.setitem(sys.modules, "peft", types.SimpleNamespace(LoraConfig=lambda **kwargs: kwargs)) + output_policy_results = iter( + [ + { + "ready": True, + "blockers": [], + "output_path_sha256": json.loads(execution_context.output_facts_json)["output_path_sha256"], + }, + { + "ready": False, + "blockers": ["OUTPUT_DIRECTORY_EXISTS"], + "output_path_sha256": json.loads(execution_context.output_facts_json)["output_path_sha256"], + }, + ] + ) + monkeypatch.setattr( + training, + "validate_sft_output_policy", + lambda *args, **kwargs: calls["order"].append("policy") or next(output_policy_results), + ) + monkeypatch.setattr( + training, + "_assistant_only_training_record", + lambda row, tokenizer, max_seq_length: { + "input_ids": [1, 2], + "attention_mask": [1, 1], + "labels": [-100, 2], + }, + ) + monkeypatch.setattr( + training, + "_record_sft_training_selection_from_config", + lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("runner must consume rows from bound preflight context") + ), + ) + metadata: dict[str, Any] = {"adapter_path": (output_dir / "adapter").as_posix(), "dataset_load": {}} + + training._run_real_sft( # noqa: SLF001 + metadata, + config, + manifest, + output_dir, + execution_context=execution_context, + ) + + assert calls["order"] == ["policy", "policy", "tokenizer", "model"] + assert calls["tokenizer"] == { + "path": config["base_model_runtime_path"], + "local_files_only": True, + "trust_remote_code": False, + } + assert calls["model"]["path"] == config["base_model_runtime_path"] + assert calls["model"]["local_files_only"] is True + assert calls["model"]["trust_remote_code"] is False + assert calls["model"]["torch_dtype"] is fake_bfloat16 + assert calls["model"]["low_cpu_mem_usage"] is True + assert "device_map" not in calls["model"] + assert tokenizer.pad_token == tokenizer.eos_token + assert model.gradient_checkpointing_enabled is True + assert model.config.use_cache is False + expected_training_arguments = { + "max_steps": 1, + "per_device_train_batch_size": 1, + "gradient_accumulation_steps": 1, + "save_strategy": "no", + "logging_steps": 1, + "seed": 42, + "bf16": True, + "fp16": False, + "tf32": True, + "gradient_checkpointing": True, + "report_to": [], + } + assert {key: calls["training_arguments"][key] for key in expected_training_arguments} == expected_training_arguments + assert metadata["observed_optimizer_steps"] == 1 + assert metadata["training_rows_used"] == 2 + assert metadata["train_result_metrics"]["train_loss"] == 0.5 + + +def test_model_objective_probe_passes_local_files_only_to_config_and_tokenizer( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path, manifest, _ = _write_preflight_fixture(tmp_path) + config = json.loads(config_path.read_text(encoding="utf-8")) + rows = training._load_sft_training_rows(manifest, split="train") # noqa: SLF001 + model_root = Path(config["base_model_runtime_path"]) + (model_root / "model-00001-of-00001.safetensors").open("r+b").truncate(12 * 1024**3) + calls: list[dict[str, Any]] = [] + + class FakeAutoConfig: + @staticmethod + def from_pretrained(path: str, **kwargs: Any) -> Any: + calls.append({"kind": "config", "path": path, **kwargs}) + return types.SimpleNamespace( + model_type="qwen2", + architectures=["Qwen2ForCausalLM"], + hidden_size=3584, + intermediate_size=18944, + num_hidden_layers=28, + num_attention_heads=28, + num_key_value_heads=4, + vocab_size=152064, + ) + + class FakeAutoTokenizer: + @staticmethod + def from_pretrained(path: str, **kwargs: Any) -> object: + calls.append({"kind": "tokenizer", "path": path, **kwargs}) + return object() + + monkeypatch.setitem( + sys.modules, + "transformers", + types.SimpleNamespace(AutoConfig=FakeAutoConfig, AutoTokenizer=FakeAutoTokenizer), + ) + monkeypatch.setattr( + training, + "_assistant_only_training_record", + lambda row, tokenizer, max_seq_length: { + "input_ids": [1, 2], + "attention_mask": [1, 1], + "labels": [-100, 2], + }, + ) + + model, objective, blockers = training._probe_sft_model_and_objective(config, rows) # noqa: SLF001 + + assert blockers == [] + assert [call["kind"] for call in calls] == ["config", "tokenizer"] + assert all(call["local_files_only"] is True for call in calls) + assert all(call["trust_remote_code"] is False for call in calls) + assert model["stable_fingerprints"] + assert model["weight_inventory"] == [ + {"name": "model-00001-of-00001.safetensors", "size": 12 * 1024**3} + ] + assert objective["records_checked"] == 2 + + +def test_model_probe_rejects_generic_qwen2_geometry_and_tiny_weight_inventory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path, manifest, _ = _write_preflight_fixture(tmp_path) + config = json.loads(config_path.read_text(encoding="utf-8")) + rows = training._load_sft_training_rows(manifest, split="train") # noqa: SLF001 + + class GenericQwenConfig: + @staticmethod + def from_pretrained(path: str, **kwargs: Any) -> Any: + return types.SimpleNamespace(model_type="qwen2") + + class FakeTokenizer: + @staticmethod + def from_pretrained(path: str, **kwargs: Any) -> object: + return object() + + monkeypatch.setitem( + sys.modules, + "transformers", + types.SimpleNamespace(AutoConfig=GenericQwenConfig, AutoTokenizer=FakeTokenizer), + ) + monkeypatch.setattr( + training, + "_assistant_only_training_record", + lambda row, tokenizer, max_seq_length: { + "input_ids": [1, 2], + "attention_mask": [1, 1], + "labels": [-100, 2], + }, + ) + + _, _, blockers = training._probe_sft_model_and_objective(config, rows) # noqa: SLF001 + + assert blockers == ["MODEL_IDENTITY_MISMATCH", "MODEL_WEIGHT_INSUFFICIENT"] + + +def test_public_a100_smoke_example_is_disabled_bounded_and_path_safe() -> None: + path = Path(__file__).resolve().parents[1] / "configs" / "sft-a100-smoke.example.json" + + assert path.is_file() + config = json.loads(path.read_text(encoding="utf-8")) + assert config == { + "base_model_public_id": "Qwen/Qwen2.5-7B-Instruct", + "base_model_runtime_path": "", + "allow_heavy_training": False, + "local_files_only": True, + "trust_remote_code": False, + "dtype": "bfloat16", + "torch_dtype": "bfloat16", + "dataset_split": "train", + "dataset_manifest_id": "public-sample-20260619T090925Z", + "max_train_rows": 2, + "max_steps": 1, + "per_device_train_batch_size": 1, + "gradient_accumulation_steps": 1, + "max_seq_length": 1024, + "bf16": True, + "fp16": False, + "tf32": True, + "gradient_checkpointing": True, + "use_cache": False, + "low_cpu_mem_usage": True, + "save_strategy": "no", + "logging_steps": 1, + "seed": 42, + "report_to": [], + "min_free_disk_gib": 20, + "output_root": "", + "lora": { + "r": 8, + "alpha": 16, + "dropout": 0.05, + "target_modules": ["q_proj", "k_proj", "v_proj", "o_proj"], + }, + } + assert "/mnt/data/" not in path.read_text(encoding="utf-8") + assert "/Users/" not in path.read_text(encoding="utf-8") From 3a4bf38a574ed81534eaea1103b491b137da3049 Mon Sep 17 00:00:00 2001 From: Minghong Sun <118113656+Raidriar7170@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:28:31 +0800 Subject: [PATCH 2/5] Harden private A100 SFT smoke preflight --- ...7-14-enable-private-a100-sft-smoke-v1.html | 10 + .../.openspec.yaml | 0 .../design.md | 17 +- .../proposal.md | 3 +- .../specs/supervised-contract-tuning/spec.md | 26 +- .../tasks.md | 13 + .../specs/supervised-contract-tuning/spec.md | 118 ++- src/voice2task/cli/train.py | 4 +- src/voice2task/training.py | 786 +++++++++++++++++- tests/test_a100_sft_review_hardening.py | 471 +++++++++++ tests/test_a100_sft_smoke.py | 7 + tests/test_private_a100_sft_preflight.py | 178 +++- 12 files changed, 1567 insertions(+), 66 deletions(-) create mode 100644 docs/human-briefs/2026-07-14-enable-private-a100-sft-smoke-v1.html rename openspec/changes/{enable-private-a100-sft-smoke-v1 => archive/2026-07-14-enable-private-a100-sft-smoke-v1}/.openspec.yaml (100%) rename openspec/changes/{enable-private-a100-sft-smoke-v1 => archive/2026-07-14-enable-private-a100-sft-smoke-v1}/design.md (64%) rename openspec/changes/{enable-private-a100-sft-smoke-v1 => archive/2026-07-14-enable-private-a100-sft-smoke-v1}/proposal.md (84%) rename openspec/changes/{enable-private-a100-sft-smoke-v1 => archive/2026-07-14-enable-private-a100-sft-smoke-v1}/specs/supervised-contract-tuning/spec.md (78%) rename openspec/changes/{enable-private-a100-sft-smoke-v1 => archive/2026-07-14-enable-private-a100-sft-smoke-v1}/tasks.md (83%) create mode 100644 tests/test_a100_sft_review_hardening.py diff --git a/docs/human-briefs/2026-07-14-enable-private-a100-sft-smoke-v1.html b/docs/human-briefs/2026-07-14-enable-private-a100-sft-smoke-v1.html new file mode 100644 index 0000000..c20cf16 --- /dev/null +++ b/docs/human-briefs/2026-07-14-enable-private-a100-sft-smoke-v1.html @@ -0,0 +1,10 @@ + +Private A100 SFT Smoke V1 · Archive Closeout +

OpenSpec · enable-private-a100-sft-smoke-v1

审阅收口：把真实更新证据和运行时边界做成硬门

ARCHIVED_CODE_GREEN · REAL_SMOKE_NOT_RUN · execution_readiness=false

+

本轮只修复审阅指出的五项：public stdout 严格白名单、descriptor-relative 输出 claim、空闲显存与占用门、offset 驱动的 assistant-only 标签验证、adapter 参数真实变化证据。

+

不执行完整训练、DPO、GRPO、prediction、evaluation 或 lockbox 访问；未通过所有共享 preflight 与 postconditions 时不得使用 SMOKE_COMPLETED。

+

clean-evaluation 保持：acquisition unavailable、binding count 0、human acceptance not recorded、protocol not frozen、clean population not materialized、freeze unauthorized、execution readiness false。

+

输出 claim 的威胁模型已由用户明确收窄：代码继续防御所有检查点可观察的 drift、symlink、inode exchange 与 concurrent leaf；不声称最后一次 identity checkpoint 到 mkdirat syscall 之间对同 UID 恶意 rename 具备内核原子性。部署前必须保证该窗口内 output root/parent namespace 不被同 UID 对手恶意改名。

+

本轮新回归先得到 17 failed；最终 focused suite 为 203 passed，完整 pytest 为 1413 passed。Ruff、OpenSpec strict（15/15）、truth-surface、git diff check 均通过。Targeted mypy 仅保留两个未触及文件中的既有诊断，本次训练文件零诊断。

+

主 supervised-contract-tuning spec 已同步，change 已归档到 openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/，active changes 为 0。未运行真实训练。

+ diff --git a/openspec/changes/enable-private-a100-sft-smoke-v1/.openspec.yaml b/openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/.openspec.yaml similarity index 100% rename from openspec/changes/enable-private-a100-sft-smoke-v1/.openspec.yaml rename to openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/.openspec.yaml diff --git a/openspec/changes/enable-private-a100-sft-smoke-v1/design.md b/openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/design.md similarity index 64% rename from openspec/changes/enable-private-a100-sft-smoke-v1/design.md rename to openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/design.md index 5f7616f..e13008d 100644 --- a/openspec/changes/enable-private-a100-sft-smoke-v1/design.md +++ b/openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/design.md @@ -27,13 +27,15 @@ Alternative considered: keep a lightweight CLI preflight and separate runtime gu ### 2. Output authorization is based on canonical filesystem identity -The output root must already exist, be absolute, and not itself be a symlink. The absolute candidate is rejected if it equals the root, if any existing component is a symlink, if its resolved location is not a strict descendant of the resolved root, if it already exists (including an empty directory), or if it falls within the repository. Writability and free space are checked on the nearest existing parent. The real runner claims the final directory with exclusive creation, then accepts only the expected existing-empty result from the same policy and the bound path hash. DPO real mode derives its repository root from the supplied manifest checkout with a cwd-independent fail-closed helper, passes that root explicitly to its runner, uses the same generic gate before metadata/dependency work, and repeats it in the runner. This is defensive hardening, not authorization to execute DPO. +The output root and candidate parent must already exist, be absolute directories, and not be symlinks. Preflight binds both objects' `st_dev`, `st_ino`, `st_uid`, `st_gid`, and `st_mode` in the private execution context. The runner opens root-to-parent components descriptor-relatively with directory/no-follow flags and creates only the final leaf with `os.mkdir(..., mode=0o700, dir_fd=parent_fd)`. It compares `fstat` identity before and after claim and never performs pathname cleanup once identity becomes uncertain. DPO uses the same claim primitive defensively without authorizing execution. + +Threat-model clarification authorized by the user: these checks defend every drift, symlink, inode-exchange, and concurrent-leaf condition observable at a checkpoint. They do not claim that userspace identity comparison and the following `mkdirat` syscall are one kernel-atomic operation. Production use therefore requires the output root and parent namespace not be maliciously renamed by another same-UID process during that final boundary. Alternative considered: lexical `Path.relative_to`. Rejected because it does not defend against `..`, symlink redirection, or later path substitution. -### 3. Blockers are stable codes; diagnostics are sanitized sections +### 3. Public results are strict allowlists; blockers are stable codes -Exceptions and machine-specific values are converted to enumerated blocker codes. The shared top-level gate catches unexpected internal exceptions and returns the complete blocked schema with only `PREFLIGHT_INTERNAL_ERROR`; it never serializes the exception. Public config facts are an explicit whitelist of approved constants, booleans, bounded numbers, and hashes. Public JSON includes hashes, booleans, counts, versions, GPU model/capability/memory, and file-name/size inventories, but not private paths, hostname, IP, GPU UUID, secrets, environment values, arbitrary config strings, or raw exception text. +Exceptions and machine-specific values are converted to enumerated blocker codes. CLI training output is rebuilt from an exact top-level allowlist and a separately allowlisted adapter-file inventory; it never recursively sanitizes or copies the private metadata object. Public JSON may include only approved schema/status, blockers, preflight, budget/metrics, adapter filename-size-hash evidence, and the unchanged clean-evaluation facts. Private execution metadata remains only in the ignored output directory. Alternative considered: surface exception strings for convenience. Rejected because they are unstable and can leak private runtime details. @@ -41,17 +43,18 @@ Alternative considered: surface exception strings for convenience. Rejected beca The canonical formal manifest and complete SFT JSONL are hashed. After binding the current manifest ID and exact SFT entry, parsing reads from the beginning, requires `split=train` for every selected record, stops immediately after the configured one or two rows, rejects duplicates/empty selections and `train_source_ids`, and hashes the ordered selected IDs. Smoke budget scalars use exact JSON integer semantics: `max_train_rows` is a non-boolean integer in `{1, 2}`; `max_steps`, `per_device_train_batch_size`, and `gradient_accumulation_steps` are non-boolean integer `1`; `max_seq_length` is a non-boolean integer from 1 through 4096; `seed` is a non-boolean integer; and `logging_steps` is a positive non-boolean integer. Those exact serialized rows are carried in the private context. A local tokenizer constructs the same assistant-only records used by training and verifies mask, target, length, and tensor-shape invariants before the 7B model is loaded. -### 5. The trainer owns one explicitly visible GPU +### 5. The trainer owns one explicitly visible idle GPU -The caller must set `CUDA_VISIBLE_DEVICES`, exactly one CUDA device must be visible, the device name must identify an A100, BF16 must be supported, compute capability must meet the BF16 threshold, and memory must be at least 35 GiB. The private model must match Qwen2.5-7B-Instruct geometry and architecture, expose at least 12 GiB of weight inventory, use `dtype=torch_dtype=bfloat16`, `trust_remote_code=false`, local-only loading, and a bounded q/k/v/o LoRA target set. Model loading does not use `device_map="auto"`; Trainer/Accelerate performs placement. Gradient checkpointing forces `use_cache=false`. A missing pad token uses only the tokenizer EOS token after validation. +The caller must set `CUDA_VISIBLE_DEVICES`, exactly one device must be visible, the name must identify an A100, BF16 and compute capability must pass, total and current free memory must each be at least 35 GiB, and a sanitized compute-process probe must count zero processes. The same probe runs again immediately before weights load. Public facts expose only aggregate count and readiness, never process identity. ### 6. Smoke completion is a narrow postcondition -`SMOKE_COMPLETED` requires exit success, `training_completed`, one true integer optimizer step, an observed row count exactly equal to configured `max_train_rows`, a finite non-boolean numeric loss, and both non-empty `adapter_config.json` and adapter weights. The entire run tree is scanned for full base-model files, indexes, or shards rather than checking only the adapter directory. Metadata records the bounded budget and public-safe provenance. Any mismatch is a failed run, not a partial success. +`SMOKE_COMPLETED` additionally requires a positive trainable-parameter count, positive adapter-tensor count, stable before/after adapter-state digests, at least one changed adapter tensor, and finite values for every adapter tensor. Adapter files carry filename, size, and SHA-256. A step count and files alone never prove an update. ## Risks / Trade-offs -- [Filesystem state changes after preflight] -> Rehash bound inputs, repeat policy, claim the final output directory exclusively, and post-claim revalidate before local tokenizer/model loading. Parent-component mutation after the final check would require a directory-fd based redesign to eliminate completely. +- [Filesystem state changes after preflight] -> Bind root/parent identities and use descriptor-relative no-follow traversal plus leaf-only mkdir; fail closed without pathname cleanup on uncertainty. +- [Same-UID rename at the final syscall boundary] -> This narrow interval is an explicit deployment precondition, not a claimed atomic defense; prevent malicious same-UID namespace renames during claim. - [Tokenizer loading itself can be expensive] -> Keep it local-only and load it before model weights because objective validity depends on the real tokenizer. - [Git status includes generated private ignored config] -> Inspect tracked changes only and record commit SHA; ignored files never make readiness dirty. - [Package/GPU APIs vary between versions] -> Keep dependency and GPU probes injectable for unit tests and convert failures to stable blockers. diff --git a/openspec/changes/enable-private-a100-sft-smoke-v1/proposal.md b/openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/proposal.md similarity index 84% rename from openspec/changes/enable-private-a100-sft-smoke-v1/proposal.md rename to openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/proposal.md index cf63224..d028661 100644 --- a/openspec/changes/enable-private-a100-sft-smoke-v1/proposal.md +++ b/openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/proposal.md @@ -5,7 +5,7 @@ The current SFT entrypoint can accept an unconfigured or path-escaping output di ## What Changes - **BREAKING**: Require every `sft --run-training` invocation to use an existing absolute non-symlink `output_root` and an absolute child `output_dir` that does not already exist, remains under the resolved root, and stays outside repository paths. -- Add stable, sanitized output-policy blocker codes, check the nearest existing parent, claim the final directory exclusively, and repeat the same policy immediately before model loading so preflight-to-run path drift fails closed. Apply the same generic real-training output gate to DPO without authorizing DPO execution. +- Add stable, sanitized output-policy blocker codes, require the exact `output_dir.parent` to already exist and check that exact parent, claim the final directory exclusively, and repeat the same policy immediately before model loading so preflight-to-run path drift fails closed. Apply the same generic real-training output gate to DPO without authorizing DPO execution. - Add `voice2task-train sft-preflight` with one shared internal core that returns a public report plus a private immutable execution context covering Git state, bounded config, dependencies, explicit one-GPU BF16 runtime, formal public train-row selection, assistant-only objective construction, local-only model identity/fingerprints, and output capacity. Real SFT consumes only the bound context and rehashes mutable inputs before model loading. - Make blocked, skipped, unavailable, and runtime-failed CLI results non-zero while preserving one JSON document on stdout and no competing result JSON on stderr. - Wire private Qwen2.5-7B-Instruct BF16/local-only LoRA SFT options through tokenizer, model loading, `TrainingArguments`, gradient checkpointing, adapter writing, and metadata, with exact Qwen2.5-7B geometry, A100, minimum weight-inventory size, `trust_remote_code=false`, and bounded LoRA checks. @@ -13,6 +13,7 @@ The current SFT entrypoint can accept an unconfigured or path-escaping output di - Permit at most one explicitly selected A100, one or two formal-public train rows, and exactly one optimizer step for `SMOKE_COMPLETED`; do not authorize broader execution. - Preserve the clean-evaluation truth surface exactly: `acquisition_source_status=UNAVAILABLE`, `authoritatively_bound_binding_count=0`, `human_acceptance_status=NOT_RECORDED`, `protocol_freeze_status=NOT_FROZEN`, `clean_population_status=NOT_MATERIALIZED`, `freeze_authorized=false`, and `execution_readiness=false`. - Add no-network/no-GPU unit tests for output traversal/symlinks/TOCTOU, direct-runner/DPO bypasses, CLI exit semantics, dependency/GPU/model/data/objective/output preflight blockers, immutable context drift, strict smoke postconditions, and model/training option propagation. +- Replace path-pattern redaction with an explicit public-result allowlist, bind filesystem identities for descriptor-relative output claims, require an idle A100 with at least 35 GiB free memory, prove prompt labels are fully masked from tokenizer offsets, and require measured adapter tensor changes before smoke completion. Non-goals are full or 282-row SFT, DPO, first-phase GRPO, prediction, evaluation, lockbox or clean-population access, generic chat fine-tuning, skill routing, GUI action-policy learning, public full-corpus release, checkpoint or adapter release, model-improvement claims, clean-evaluation readiness, production readiness, merge, deploy, or release. diff --git a/openspec/changes/enable-private-a100-sft-smoke-v1/specs/supervised-contract-tuning/spec.md b/openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/specs/supervised-contract-tuning/spec.md similarity index 78% rename from openspec/changes/enable-private-a100-sft-smoke-v1/specs/supervised-contract-tuning/spec.md rename to openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/specs/supervised-contract-tuning/spec.md index 96bd13d..84a404b 100644 --- a/openspec/changes/enable-private-a100-sft-smoke-v1/specs/supervised-contract-tuning/spec.md +++ b/openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/specs/supervised-contract-tuning/spec.md @@ -17,7 +17,7 @@ The system SHALL provide a bounded, opt-in, local-only Qwen2.5-7B-Instruct LoRA #### Scenario: Keep remote evidence private by default - **WHEN** a preflight or smoke produces private paths, configs, adapters, caches, or raw logs -- **THEN** those values and artifacts remain out of Git and public JSON exposes only approved hashes, counts, versions, GPU facts, and file-name/size inventory +- **THEN** those values and artifacts remain out of Git and CLI JSON is rebuilt from the exact approved result-field allowlist, with adapter evidence limited to filename, size, and SHA-256 #### Scenario: Preserve clean-evaluation blockers - **WHEN** infrastructure preflight or the authorized smoke succeeds @@ -26,7 +26,9 @@ The system SHALL provide a bounded, opt-in, local-only Qwen2.5-7B-Instruct LoRA ## ADDED Requirements ### Requirement: Enforce canonical SFT output boundaries -Every real SFT run MUST require an existing absolute non-symlink output root and an absolute, not-yet-existing, strict descendant output directory after canonical resolution. The system MUST reject traversal, any existing symlink component, final-directory symlinks, root-external destinations, every existing candidate including an empty directory, and repository locations with stable blocker codes. It MUST check the nearest existing parent, claim the final directory exclusively, and revalidate the bound path before model and tokenizer loading. +Every real SFT run MUST require an existing absolute non-symlink output root and an absolute, not-yet-existing, strict descendant output directory after canonical resolution. The system MUST reject traversal, any existing symlink component, final-directory symlinks, root-external destinations, every existing candidate including an empty directory, and repository locations with stable blocker codes. The exact candidate parent MUST already exist and pass the output checks before the system claims the final directory exclusively and revalidates the bound path before model and tokenizer loading. + +This contract defends output drift, symlink substitution, inode exchange, and concurrent final-leaf creation that is observable at its identity checkpoints. Its deployment precondition is that no malicious same-UID process renames the output-root or candidate-parent namespace during the final identity-checkpoint-to-`mkdirat` syscall window. The contract does not claim kernel-atomic protection against such a same-UID rename inside that narrow window. #### Scenario: Reject an unsafe output path - **WHEN** the root is missing, relative, symlinked, or the candidate is relative, equal to the root, escaping, symlinked, already exists, or is inside the repository @@ -40,16 +42,24 @@ Every real SFT run MUST require an existing absolute non-symlink output root and - **WHEN** filesystem state makes a previously accepted output path unsafe before tokenizer or model loading - **THEN** the repeated shared policy blocks execution before model weights load +#### Scenario: Claim the leaf without following exchanged paths +- **WHEN** preflight accepts an existing root and parent +- **THEN** the private context binds their filesystem identities and the runner traverses directory descriptors without following symlinks, creates only the final leaf with mode `0700`, and fails closed on any root/parent identity change or concurrent leaf creation without pathname cleanup + +#### Scenario: Enforce the deployment namespace precondition +- **WHEN** a deployment permits another same-UID process to rename the accepted output root or parent during the final identity-checkpoint-to-`mkdirat` boundary +- **THEN** that deployment is outside this claim contract and MUST prevent that adversarial namespace mutation before authorizing a real smoke + ### Requirement: Provide one shared SFT preflight The system MUST expose `sft-preflight` and MUST use the same internal preflight core from real SFT execution before loading model weights. That core MUST produce the public report plus an immutable private execution context binding canonical inputs, selected rows, model inventory, and output facts. Real training MUST consume the bound context rather than reselecting data from caller arguments and MUST rehash mutable inputs before model loading. The single public JSON result MUST use schema `voice2task-sft-preflight-v1`, report `ready`, `status`, `blockers`, and the `git`, `config`, `dataset`, `model`, `runtime`, `gpu`, `output`, and `objective` sections, and MUST not access the network. #### Scenario: Report a ready bounded smoke -- **WHEN** Git tracked state is clean and includes the change, the ignored non-symlink private config is exactly smoke-bounded, dependencies and `pip check` pass, one explicit BF16-capable >=35 GiB A100 is visible, exact local Qwen2.5-7B model/tokenizer identity and >=12 GiB weight inventory pass, selected formal train rows and assistant-only labels pass, and output policy passes +- **WHEN** Git tracked state is clean and includes the change, the ignored non-symlink private config is exactly smoke-bounded, dependencies and `pip check` pass, one explicit idle BF16-capable A100 has at least 35 GiB total and free memory and zero compute processes, exact local Qwen2.5-7B model/tokenizer identity and >=12 GiB weight inventory pass, selected formal train rows and assistant-only labels pass, and output policy passes - **THEN** preflight returns exit code 0 with `ready=true`, an empty blocker list, stable fingerprints, hashes, versions, counts, and public-safe facts #### Scenario: Report blocked readiness safely - **WHEN** any required Git, config, dependency, GPU, model, dataset, objective, or output condition fails -- **THEN** preflight returns non-zero with `ready=false` and only stable enumerated blocker codes, without raw exceptions or private runtime values +- **THEN** preflight returns non-zero with `ready=false` and only stable enumerated blocker codes, including `GPU_FREE_MEMORY_INSUFFICIENT`, `GPU_BUSY`, or `GPU_OCCUPANCY_PROBE_FAILED` for the corresponding GPU state, without raw exceptions or private runtime values #### Scenario: Fail closed on an unexpected preflight exception - **WHEN** any internal shared-preflight operation raises an unexpected exception @@ -63,8 +73,12 @@ The system MUST expose `sft-preflight` and MUST use the same internal preflight The shared preflight MUST require the canonical current formal manifest and exact SFT entry, hash both complete files, then parse only the first exactly `max_train_rows` ordered `split=train` records where that value is a non-boolean integer in `{1, 2}` and stop. It MUST reject JSON booleans and floats for `max_train_rows`, `max_steps`, `per_device_train_batch_size`, and `gradient_accumulation_steps`; the latter three MUST be the non-boolean integer `1`. It MUST also require `seed` to be a non-boolean integer and `logging_steps` to be a positive non-boolean integer before those values reach real `TrainingArguments`. It MUST reject empty or duplicate row IDs and every implicit selector such as `train_source_ids`, bind those exact rows into the private execution context, and validate every selected row with the same real-tokenizer assistant-only record builder used for training. #### Scenario: Validate assistant-only records -- **WHEN** a selected row is encoded -- **THEN** `max_seq_length` has already been validated as a non-boolean integer from 1 through 4096, prompt labels are all `-100`, at least one assistant target label is trainable, `input_ids`, `attention_mask`, and `labels` lengths match, and length does not exceed `max_seq_length` +- **WHEN** a selected row is encoded with tokenizer offsets +- **THEN** every token outside the contiguous assistant target region has label `-100`, every assistant label matches its input token, at least one assistant token carries loss, all tensor lengths match, and length does not exceed the validated `max_seq_length` + +#### Scenario: Prove a real adapter update +- **WHEN** one-step training returns +- **THEN** smoke completion additionally requires positive trainable/adapter tensor counts, different stable before/after adapter-state digests, at least one changed adapter tensor, finite adapter values, and hashed non-empty adapter files #### Scenario: Reject data or selection drift - **WHEN** the manifest/SFT hash or manifest ID does not match its declared value, train selection changes, rows are empty or duplicated, labels are invalid, or sequence length is exceeded diff --git a/openspec/changes/enable-private-a100-sft-smoke-v1/tasks.md b/openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/tasks.md similarity index 83% rename from openspec/changes/enable-private-a100-sft-smoke-v1/tasks.md rename to openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/tasks.md index 2fa00c9..e3598b4 100644 --- a/openspec/changes/enable-private-a100-sft-smoke-v1/tasks.md +++ b/openspec/changes/archive/2026-07-14-enable-private-a100-sft-smoke-v1/tasks.md @@ -73,3 +73,16 @@ - [x] 8.4 Exercise the real execution gate from a clean committed HEAD and stop fail-closed - Shared CLI preflight: exit `1`, exactly one stdout JSON document, empty stderr, sole blocker `CONFIG_FILE_MISSING`. - Authorized A100 alias: three read-only occupancy probes timed out; no remote writes, model-path search, training, prediction, or evaluation occurred. + +## 9. Changes-Requested Hardening + +- [x] 9.1 Replace regex-based public training metadata with a strict allowlist result builder and prove arbitrary absolute private paths never reach CLI stdout +- [x] 9.2 Bind output-root and parent filesystem identities, require the parent to pre-exist, and claim the leaf descriptor-relatively without pathname cleanup after uncertainty +- [x] 9.3 Add free-memory and zero-compute-process GPU readiness facts and repeat the same occupancy probe before model-weight loading +- [x] 9.4 Validate assistant-only labels against the tokenizer offset-derived assistant token region and reject any prompt-label leakage +- [x] 9.5 Record before/after adapter-state digests, changed finite adapter tensor counts, and SHA-256 adapter-file inventory as required smoke postconditions +- [x] 9.6 Add no-GPU/no-model RED/GREEN regressions, update the Human Brief, and run focused verification without training, archive, commit, push, or PR actions + - RED: `17 failed` in the new review-hardening module before production changes. + - GREEN: combined focused suite `200 passed`; focused Ruff passed; strict change validation passed. + - Targeted Mypy reports only the two pre-existing imported-module diagnostics and no touched-file diagnostic. +- [x] 9.7 Record the user-authorized output-claim threat-model clarification: checkpoint-observable races remain defended, while malicious same-UID namespace rename inside the final identity-checkpoint-to-`mkdirat` boundary is a deployment precondition rather than a syscall-atomicity claim diff --git a/openspec/specs/supervised-contract-tuning/spec.md b/openspec/specs/supervised-contract-tuning/spec.md index 4f5c9fa..59aca7b 100644 --- a/openspec/specs/supervised-contract-tuning/spec.md +++ b/openspec/specs/supervised-contract-tuning/spec.md @@ -32,19 +32,123 @@ The system SHALL export SFT adapter metadata without implying a public checkpoin - **THEN** the system writes an adapter summary containing base model, adapter path, dataset manifest ID, metrics path, and release status ### Requirement: Run A100 public-sample SFT smoke -The system SHALL provide a bounded, opt-in A100 SFT smoke workflow that trains the existing Qwen-family LoRA SFT path on the committed public sample and writes all remote outputs under the approved private A100 project root. +The system SHALL provide a bounded, opt-in, local-only Qwen2.5-7B-Instruct LoRA SFT smoke workflow that uses an explicitly selected single GPU, trains one or two rows from the current formal public sample `train` split for exactly one optimizer step, and writes private outputs only beneath an approved output root. #### Scenario: Launch smoke with explicit opt-in -- **WHEN** a developer launches the A100 SFT smoke with `--run-training` and a config whose `allow_heavy_training` is `true` -- **THEN** the system uses the configured public-sample manifest, writes adapter/checkpoint outputs under the configured A100 project directory, and records the base model, dataset manifest ID, hyperparameters, package versions, output paths, and training command in adapter metadata +- **WHEN** a developer launches SFT with `--run-training`, a private ignored config whose `allow_heavy_training` is `true`, and a shared preflight result whose `ready` is `true` +- **THEN** the system repeats the shared preflight before model weights load, uses only the configured local Qwen2.5-7B-Instruct runtime, and records public-safe Git, config, dataset, model, dependency, GPU, output, objective, and actual-budget metadata + +#### Scenario: Bound successful execution +- **WHEN** the authorized smoke completes +- **THEN** it reports `SMOKE_COMPLETED`, exactly one observed optimizer step, one or two training rows, finite loss, and non-empty adapter files without saving or copying full base-model weights #### Scenario: Reject accidental heavy training -- **WHEN** a developer launches the A100 SFT smoke without `--run-training` or with `allow_heavy_training` unset or false -- **THEN** the system does not download models or start training and instead emits dry-run metadata that clearly states no heavy training occurred +- **WHEN** a developer launches the training path without `--run-training`, with `allow_heavy_training` false, or with any budget outside the approved smoke bounds +- **THEN** the system does not load model weights or start training and returns a non-success training status with a stable blocker where applicable #### Scenario: Keep remote evidence private by default -- **WHEN** the A100 smoke run completes -- **THEN** raw checkpoints, adapters, caches, and logs remain out of git unless a later explicit release change approves a sanitized artifact +- **WHEN** a preflight or smoke produces private paths, configs, adapters, caches, or raw logs +- **THEN** those values and artifacts remain out of Git and CLI JSON is rebuilt from the exact approved result-field allowlist, with adapter evidence limited to filename, size, and SHA-256 + +#### Scenario: Preserve clean-evaluation blockers +- **WHEN** infrastructure preflight or the authorized smoke succeeds +- **THEN** the system leaves `acquisition_source_status=UNAVAILABLE`, `authoritatively_bound_binding_count=0`, `human_acceptance_status=NOT_RECORDED`, `protocol_freeze_status=NOT_FROZEN`, `clean_population_status=NOT_MATERIALIZED`, `freeze_authorized=false`, and `execution_readiness=false` + +### Requirement: Enforce canonical SFT output boundaries +Every real SFT run MUST require an existing absolute non-symlink output root and an absolute, not-yet-existing, strict descendant output directory after canonical resolution. The system MUST reject traversal, any existing symlink component, final-directory symlinks, root-external destinations, every existing candidate including an empty directory, and repository locations with stable blocker codes. The exact candidate parent MUST already exist and pass the output checks before the system claims the final directory exclusively and revalidates the bound path before model and tokenizer loading. + +This contract defends output drift, symlink substitution, inode exchange, and concurrent final-leaf creation that is observable at its identity checkpoints. Its deployment precondition is that no malicious same-UID process renames the output-root or candidate-parent namespace during the final identity-checkpoint-to-`mkdirat` syscall window. The contract does not claim kernel-atomic protection against such a same-UID rename inside that narrow window. + +#### Scenario: Reject an unsafe output path +- **WHEN** the root is missing, relative, symlinked, or the candidate is relative, equal to the root, escaping, symlinked, already exists, or is inside the repository +- **THEN** the system returns an output-policy blocker without creating or writing metadata into the unsafe candidate + +#### Scenario: Accept a new canonical child +- **WHEN** the existing absolute root is not a symlink and the absolute candidate is a new writable strict descendant with no symlink components and sufficient free space +- **THEN** output policy reports the canonical path hash as ready without exposing the path + +#### Scenario: Detect preflight-to-load drift +- **WHEN** filesystem state makes a previously accepted output path unsafe before tokenizer or model loading +- **THEN** the repeated shared policy blocks execution before model weights load + +#### Scenario: Claim the leaf without following exchanged paths +- **WHEN** preflight accepts an existing root and parent +- **THEN** the private context binds their filesystem identities and the runner traverses directory descriptors without following symlinks, creates only the final leaf with mode `0700`, and fails closed on any root/parent identity change or concurrent leaf creation without pathname cleanup + +#### Scenario: Enforce the deployment namespace precondition +- **WHEN** a deployment permits another same-UID process to rename the accepted output root or parent during the final identity-checkpoint-to-`mkdirat` boundary +- **THEN** that deployment is outside this claim contract and MUST prevent that adversarial namespace mutation before authorizing a real smoke + +### Requirement: Provide one shared SFT preflight +The system MUST expose `sft-preflight` and MUST use the same internal preflight core from real SFT execution before loading model weights. That core MUST produce the public report plus an immutable private execution context binding canonical inputs, selected rows, model inventory, and output facts. Real training MUST consume the bound context rather than reselecting data from caller arguments and MUST rehash mutable inputs before model loading. The single public JSON result MUST use schema `voice2task-sft-preflight-v1`, report `ready`, `status`, `blockers`, and the `git`, `config`, `dataset`, `model`, `runtime`, `gpu`, `output`, and `objective` sections, and MUST not access the network. + +#### Scenario: Report a ready bounded smoke +- **WHEN** Git tracked state is clean and includes the change, the ignored non-symlink private config is exactly smoke-bounded, dependencies and `pip check` pass, one explicit idle BF16-capable A100 has at least 35 GiB total and free memory and zero compute processes, exact local Qwen2.5-7B model/tokenizer identity and >=12 GiB weight inventory pass, selected formal train rows and assistant-only labels pass, and output policy passes +- **THEN** preflight returns exit code 0 with `ready=true`, an empty blocker list, stable fingerprints, hashes, versions, counts, and public-safe facts + +#### Scenario: Report blocked readiness safely +- **WHEN** any required Git, config, dependency, GPU, model, dataset, objective, or output condition fails +- **THEN** preflight returns non-zero with `ready=false` and only stable enumerated blocker codes, including `GPU_FREE_MEMORY_INSUFFICIENT`, `GPU_BUSY`, or `GPU_OCCUPANCY_PROBE_FAILED` for the corresponding GPU state, without raw exceptions or private runtime values + +#### Scenario: Fail closed on an unexpected preflight exception +- **WHEN** any internal shared-preflight operation raises an unexpected exception +- **THEN** both `sft-preflight` and real SFT return the complete blocked preflight schema with only `PREFLIGHT_INTERNAL_ERROR`, do not serialize exception text or private values, and do not write the candidate output + +#### Scenario: Consume only the bound ready context +- **WHEN** real SFT reaches a ready shared preflight result +- **THEN** it builds execution metadata only from the bound report and immutable context, without rereading legacy config, manifest, or dataset inputs around the gate, and binds the exact model facts produced by the validated model probe + +### Requirement: Keep smoke row selection and objective exact +The shared preflight MUST require the canonical current formal manifest and exact SFT entry, hash both complete files, then parse only the first exactly `max_train_rows` ordered `split=train` records where that value is a non-boolean integer in `{1, 2}` and stop. It MUST reject JSON booleans and floats for `max_train_rows`, `max_steps`, `per_device_train_batch_size`, and `gradient_accumulation_steps`; the latter three MUST be the non-boolean integer `1`. It MUST also require `seed` to be a non-boolean integer and `logging_steps` to be a positive non-boolean integer before those values reach real `TrainingArguments`. It MUST reject empty or duplicate row IDs and every implicit selector such as `train_source_ids`, bind those exact rows into the private execution context, and validate every selected row with the same real-tokenizer assistant-only record builder used for training. + +#### Scenario: Validate assistant-only records +- **WHEN** a selected row is encoded with tokenizer offsets +- **THEN** every token outside the contiguous assistant target region has label `-100`, every assistant label matches its input token, at least one assistant token carries loss, all tensor lengths match, and length does not exceed the validated `max_seq_length` + +#### Scenario: Prove a real adapter update +- **WHEN** one-step training returns +- **THEN** smoke completion additionally requires positive trainable/adapter tensor counts, different stable before/after adapter-state digests, at least one changed adapter tensor, finite adapter values, and hashed non-empty adapter files + +#### Scenario: Reject data or selection drift +- **WHEN** the manifest/SFT hash or manifest ID does not match its declared value, train selection changes, rows are empty or duplicated, labels are invalid, or sequence length is exceeded +- **THEN** preflight fails with a stable dataset or objective blocker before model weights load + +#### Scenario: Reject numeric type coercion +- **WHEN** a smoke budget, seed, or logging field uses a JSON boolean or float where an integer is required, or `logging_steps` is not positive +- **THEN** preflight returns `CONFIG_NOT_SMOKE_BOUNDED`; a malformed `max_train_rows` additionally returns `TRAIN_ROW_SELECTION_INVALID` and selects zero rows + +### Requirement: Load the private 7B runtime without downloads +The real SFT path MUST load tokenizer and model from the configured existing private `base_model_runtime_path`, require public identity and exact geometry for `Qwen/Qwen2.5-7B-Instruct`, require at least 12 GiB of local weight inventory, pass `local_files_only=true`, `trust_remote_code=false`, BF16 dtype, and `low_cpu_mem_usage`, and MUST NOT use `device_map="auto"`. It MUST propagate `bf16`, `fp16`, `tf32`, `gradient_checkpointing`, `use_cache`, `seed`, `max_steps`, `max_train_rows`, batch size, accumulation, save strategy, and logging steps into training behavior and metadata. + +#### Scenario: Configure the bounded local model +- **WHEN** the ready smoke loads the Qwen2.5-7B-Instruct runtime +- **THEN** both tokenizer and model use local-only loading, the model uses BF16 and low-CPU-memory loading, Trainer/Accelerate owns placement on the one visible GPU, gradient checkpointing sets `model.config.use_cache=false`, and a missing pad token uses only a validated EOS fallback + +#### Scenario: Reject unresolved local model +- **WHEN** the private model path is absent, public identity differs, local config/tokenizer cannot load, or required stable inventory cannot be produced +- **THEN** preflight returns a stable model blocker without downloading or loading model weights + +### Requirement: Apply the real-training output gate to DPO +The system MUST derive DPO's repository root from the supplied manifest checkout without depending on the process working directory, fail closed when that checkout cannot be established, apply the same canonical output-root policy to DPO `--run-training` before metadata or dependency work, pass the derived root explicitly to the runner, and exclusively claim and revalidate the output directory again before imports or model loading. This defensive gate MUST NOT be interpreted as authorization to execute DPO in this change. + +#### Scenario: Block a DPO output bypass +- **WHEN** DPO real mode receives a missing, escaping, symlinked, existing, or repository-local output destination +- **THEN** it returns a stable output-policy failure before dependency imports or model loading and does not write into the unsafe destination + +#### Scenario: Resolve DPO policy from the manifest checkout +- **WHEN** DPO real mode is invoked from an unrelated process working directory +- **THEN** output policy still uses the repository containing the supplied manifest; if that repository cannot be resolved, execution fails closed before config, metadata, dependency, or runner work + +### Requirement: Return truthful CLI exit status and one JSON result +The training CLI MUST write exactly one JSON document to stdout. Dry-run success, ready preflight, and `training_status=training_completed` MUST exit 0. Skipped-by-config, unavailable, output-policy-blocked, preflight-blocked, and runtime-exception results MUST exit non-zero. Stderr MUST NOT contain a second result JSON. + +#### Scenario: Map successful results +- **WHEN** dry-run succeeds, preflight is ready, or training completes +- **THEN** the CLI emits one JSON document and exits 0 + +#### Scenario: Map non-success results +- **WHEN** training is skipped, unavailable, output-policy-blocked, preflight-blocked, or raises at runtime +- **THEN** the CLI emits one sanitized JSON document and exits non-zero without a competing result document on stderr ### Requirement: Export A100 trained-path public-sample predictions The system SHALL provide a bounded prediction export workflow for the A100 public-sample SFT adapter path that emits only sanitized public-sample prediction rows for contract evaluation. diff --git a/src/voice2task/cli/train.py b/src/voice2task/cli/train.py index f8105e4..09ca77b 100644 --- a/src/voice2task/cli/train.py +++ b/src/voice2task/cli/train.py @@ -8,7 +8,7 @@ from voice2task.training import ( inspect_sft_objective_from_manifest, prepare_sft_runtime_label_provenance, - public_training_metadata, + public_training_result, run_dpo, run_sft, run_sft_prediction_export, @@ -97,7 +97,7 @@ def main(argv: list[str] | None = None) -> int: print(json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True)) return 1 if args.command in {"sft", "dpo"}: - metadata = public_training_metadata(metadata) + metadata = public_training_result(metadata) if args.command == "sft-inspect-objective" and args.output: write_json(args.output, metadata) elif args.command == "sft-prepare-runtime-label-provenance": diff --git a/src/voice2task/training.py b/src/voice2task/training.py index fd3a7f2..edd5312 100644 --- a/src/voice2task/training.py +++ b/src/voice2task/training.py @@ -9,6 +9,7 @@ import os import re import shutil +import stat import subprocess import sys from collections.abc import Callable @@ -184,6 +185,386 @@ def public_training_metadata(metadata: dict[str, Any]) -> dict[str, Any]: return cast(dict[str, Any], sanitized) +PUBLIC_TRAINING_RESULT_FIELDS = ( + "schema_version", + "stage", + "training_status", + "smoke_status", + "blockers", + "preflight", + "observed_optimizer_steps", + "training_rows_used", + "training_budget", + "train_result_metrics", + "adapter_files", + "clean_evaluation", +) + +PUBLIC_TRAINING_BUDGET_FIELDS = ( + "configured_max_steps", + "observed_optimizer_steps", + "num_train_epochs", + "per_device_train_batch_size", + "gradient_accumulation_steps", + "effective_batch_size", + "scheduler_max_steps", + "train_row_count", + "theoretical_examples_seen", + "target_tokens_per_single_pass", + "target_tokens_seen_estimate", + "target_tokens_seen_status", + "step_matching_unit", + "step_matched_not_token_matched", +) +PUBLIC_TRAINING_METRIC_FIELDS = ( + "train_runtime", + "train_samples_per_second", + "train_steps_per_second", + "train_loss", + "epoch", + "global_step", + "trainable_parameter_count", + "adapter_tensor_count", + "adapter_state_digest_before", + "adapter_state_digest_after", + "changed_adapter_tensor_count", + "all_adapter_tensors_finite", +) + +PUBLIC_SFT_PREFLIGHT_SECTION_FIELDS: dict[str, tuple[str, ...]] = { + "git": ("commit_sha", "tracked_worktree_clean"), + "config": ( + "config_sha256", + "allow_heavy_training", + "base_model_public_id", + "dataset_split", + "max_train_rows", + "max_steps", + "per_device_train_batch_size", + "gradient_accumulation_steps", + "max_seq_length", + "seed", + "logging_steps", + "save_strategy", + "report_to_empty", + "local_files_only", + "bf16", + "fp16", + "tf32", + "gradient_checkpointing", + "use_cache", + "low_cpu_mem_usage", + "trust_remote_code", + "dtype_is_bfloat16", + "torch_dtype_is_bfloat16", + "minimum_free_disk_gib", + "lora_policy_valid", + "private_file", + ), + "dataset": ( + "manifest_file", + "manifest_sha256", + "manifest_id", + "sft_file", + "sft_sha256", + "selected_split", + "selected_row_count", + "selected_row_ids_sha256", + "non_train_rows_selected", + ), + "model": ( + "public_id", + "local_files_only", + "stable_fingerprints", + "weight_inventory", + "total_weight_bytes", + "minimum_weight_bytes", + "geometry_matches_qwen2_5_7b", + "snapshot_revision_sha256", + ), + "runtime": ("python", "python_requirement", "versions", "missing", "pip_check"), + "gpu": ( + "explicit_selection", + "visible_device_count", + "name", + "compute_capability", + "total_memory_gib", + "minimum_memory_gib", + "free_memory_gib", + "minimum_free_memory_gib", + "compute_process_count", + "idle_verified", + "cuda_version", + "bf16_supported", + ), + "output": ( + "ready", + "blockers", + "root_path_sha256", + "output_path_sha256", + "writable", + "free_disk_gib", + "minimum_free_disk_gib", + ), + "objective": ( + "records_checked", + "prompt_labels_masked", + "assistant_target_present", + "max_sequence_length", + "maximum_observed_tokens", + ), +} + +_INVALID_PUBLIC_VALUE = object() + + +def _public_preflight_scalar(section: str, key: str, value: Any) -> Any: + if not isinstance(value, str): + if isinstance(value, float) and not math.isfinite(value): + return _INVALID_PUBLIC_VALUE + return value if value is None or isinstance(value, int | float | bool) else _INVALID_PUBLIC_VALUE + exact_values: dict[tuple[str, str], set[str]] = { + ("config", "base_model_public_id"): {"Qwen/Qwen2.5-7B-Instruct"}, + ("config", "dataset_split"): {"train"}, + ("config", "save_strategy"): {"no"}, + ("dataset", "manifest_file"): {"manifest_public_sample.json"}, + ("dataset", "manifest_id"): {FORMAL_PUBLIC_MANIFEST_ID}, + ("dataset", "sft_file"): {"sft_public_sample.jsonl"}, + ("dataset", "selected_split"): {"train"}, + ("model", "public_id"): {"Qwen/Qwen2.5-7B-Instruct"}, + ("runtime", "python_requirement"): {">=3.10"}, + ("runtime", "pip_check"): {"ok", "conflict", "not_run"}, + } + allowed = exact_values.get((section, key)) + if allowed is not None: + return value if value in allowed else _INVALID_PUBLIC_VALUE + if key == "commit_sha": + return value if re.fullmatch(r"[0-9a-f]{40}", value) else _INVALID_PUBLIC_VALUE + if key.endswith("sha256"): + return value if re.fullmatch(r"[0-9a-f]{64}", value) else _INVALID_PUBLIC_VALUE + if section == "runtime" and key == "python": + return value if re.fullmatch(r"\d+\.\d+\.\d+", value) else _INVALID_PUBLIC_VALUE + if section == "gpu" and key in {"compute_capability", "cuda_version"}: + return value if re.fullmatch(r"(?:\d+(?:\.\d+)+|unknown)", value) else _INVALID_PUBLIC_VALUE + if section == "gpu" and key == "name": + return ( + value + if re.fullmatch(r"[A-Za-z0-9 ._()+-]{1,100}", value) + and re.search(r"\bA100\b", value, flags=re.IGNORECASE) + else _INVALID_PUBLIC_VALUE + ) + return _INVALID_PUBLIC_VALUE + + +def _public_sft_preflight_result(value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + return {} + result: dict[str, Any] = {} + if value.get("schema_version") == "voice2task-sft-preflight-v1": + result["schema_version"] = "voice2task-sft-preflight-v1" + if isinstance(value.get("ready"), bool): + result["ready"] = value["ready"] + if value.get("status") in {"ready", "blocked"}: + result["status"] = value["status"] + blockers = value.get("blockers") + if isinstance(blockers, list): + result["blockers"] = [ + code + for code in blockers + if isinstance(code, str) and re.fullmatch(r"[A-Z][A-Z0-9_]*", code) + ] + for section, allowed_fields in PUBLIC_SFT_PREFLIGHT_SECTION_FIELDS.items(): + raw_section = value.get(section) + if not isinstance(raw_section, dict): + continue + public_section: dict[str, Any] = {} + for key in allowed_fields: + if key not in raw_section: + continue + raw_value = raw_section[key] + if key in {"private_file", "versions", "missing", "stable_fingerprints", "weight_inventory", "blockers"}: + public_section[key] = raw_value + continue + public_value = _public_preflight_scalar(section, key, raw_value) + if public_value is not _INVALID_PUBLIC_VALUE: + public_section[key] = public_value + if section == "config" and isinstance(public_section.get("private_file"), dict): + private_file = cast(dict[str, Any], public_section["private_file"]) + public_section["private_file"] = { + key: private_file[key] + for key in ("under_private_runtime", "nonsymlink", "git_ignored", "git_tracked") + if key in private_file and isinstance(private_file[key], bool) + } + elif section == "runtime": + versions = public_section.get("versions") + if isinstance(versions, dict): + public_section["versions"] = { + key: versions[key] + for key in ("torch", "accelerate", "datasets", "peft", "transformers", "trl") + if key in versions + and isinstance(versions[key], str) + and re.fullmatch(r"[A-Za-z0-9.+_-]{1,80}", versions[key]) + } + missing = public_section.get("missing") + if isinstance(missing, list): + allowed_dependencies = {"torch", "accelerate", "datasets", "peft", "transformers", "trl"} + public_section["missing"] = [name for name in missing if name in allowed_dependencies] + elif section == "model": + fingerprints = public_section.get("stable_fingerprints") + if isinstance(fingerprints, dict): + allowed_fingerprints = { + "config.json", + "generation_config.json", + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "vocab.json", + "merges.txt", + "model.safetensors.index.json", + "pytorch_model.bin.index.json", + } + public_section["stable_fingerprints"] = { + key: fingerprints[key] + for key in allowed_fingerprints + if key in fingerprints + and isinstance(fingerprints[key], str) + and re.fullmatch(r"[0-9a-f]{64}", fingerprints[key]) + } + inventory = public_section.get("weight_inventory") + if isinstance(inventory, list): + public_inventory: list[dict[str, Any]] = [] + for item in inventory: + if not isinstance(item, dict): + continue + name = item.get("name") + size = item.get("size") + if ( + isinstance(name, str) + and name + and Path(name).name == name + and type(size) is int + and size >= 0 + ): + public_inventory.append({"name": name, "size": size}) + public_section["weight_inventory"] = public_inventory + elif section == "output" and isinstance(public_section.get("blockers"), list): + public_section["blockers"] = [ + code + for code in cast(list[Any], public_section["blockers"]) + if isinstance(code, str) and re.fullmatch(r"[A-Z][A-Z0-9_]*", code) + ] + result[section] = public_section + return result + + +def public_training_result(metadata: dict[str, Any]) -> dict[str, Any]: + """Build CLI output from an exact allowlist; never copy private metadata wholesale.""" + result: dict[str, Any] = {} + top_level_enums: dict[str, set[str]] = { + "schema_version": {"voice2task-training-result-v1"}, + "stage": {"sft", "dpo"}, + "training_status": { + "dry_run", + "training_completed", + "training_failed", + "training_skipped_by_config", + "training_unavailable", + "training_blocked_by_output_policy", + "training_blocked_by_preflight", + }, + "smoke_status": {"SMOKE_COMPLETED", "SMOKE_FAILED"}, + } + for key, allowed in top_level_enums.items(): + if metadata.get(key) in allowed: + result[key] = metadata[key] + for key in ("observed_optimizer_steps", "training_rows_used"): + if type(metadata.get(key)) is int: + result[key] = metadata[key] + if "preflight" in metadata: + result["preflight"] = _public_sft_preflight_result(metadata["preflight"]) + blockers = metadata.get("blockers") + if isinstance(blockers, list): + result["blockers"] = [ + code + for code in blockers + if isinstance(code, str) and re.fullmatch(r"[A-Z][A-Z0-9_]*", code) + ] + raw_budget = metadata.get("training_budget") + if isinstance(raw_budget, dict): + result["training_budget"] = { + key: raw_budget[key] + for key in PUBLIC_TRAINING_BUDGET_FIELDS + if key in raw_budget + and ( + raw_budget[key] is None + or isinstance(raw_budget[key], int | bool) + or ( + isinstance(raw_budget[key], float) + and math.isfinite(raw_budget[key]) + ) + or raw_budget[key] + in { + "estimated_from_label_tokens_and_step_budget", + "optimizer_steps", + } + ) + } + raw_metrics = metadata.get("train_result_metrics") + public_metrics: dict[str, Any] = {} + if isinstance(raw_metrics, dict): + for key in PUBLIC_TRAINING_METRIC_FIELDS: + value = raw_metrics.get(key) + if isinstance(value, int | bool) or ( + isinstance(value, float) and math.isfinite(value) + ) or ( + key.startswith("adapter_state_digest_") + and isinstance(value, str) + and re.fullmatch(r"[0-9a-f]{64}", value) + ): + public_metrics[key] = value + for key in PUBLIC_TRAINING_METRIC_FIELDS[6:]: + if key in metadata: + value = metadata[key] + if isinstance(value, int | bool) or ( + key.startswith("adapter_state_digest_") + and isinstance(value, str) + and re.fullmatch(r"[0-9a-f]{64}", value) + ): + public_metrics[key] = value + if public_metrics or isinstance(raw_metrics, dict): + result["train_result_metrics"] = public_metrics + raw_adapter_files = metadata.get("adapter_files") + if isinstance(raw_adapter_files, list): + adapter_files: list[dict[str, Any]] = [] + for item in raw_adapter_files: + if not isinstance(item, dict): + continue + raw_name = item.get("name") + if ( + not isinstance(raw_name, str) + or not raw_name + or Path(raw_name).is_absolute() + or ".." in Path(raw_name).parts + ): + continue + public_item = { + key: item[key] + for key in ("name", "size", "sha256") + if key in item + } + if ( + type(public_item.get("size")) is int + and public_item["size"] >= 0 + and isinstance(public_item.get("sha256"), str) + and re.fullmatch(r"[0-9a-f]{64}", public_item["sha256"]) + ): + adapter_files.append(public_item) + result["adapter_files"] = adapter_files + if "clean_evaluation" in metadata: + result["clean_evaluation"] = _clean_evaluation_truth_surface() + return result + + def _manifest_metadata_without_dataset_load(manifest_path: Path) -> dict[str, Any]: manifest = read_json(manifest_path) return { @@ -435,6 +816,83 @@ def __init__(self, blockers: list[str]) -> None: super().__init__(",".join(self.blockers)) +@dataclass(frozen=True) +class _OutputIdentity: + st_dev: int + st_ino: int + st_uid: int + st_gid: int + st_mode: int + + +@dataclass(frozen=True) +class _BoundOutputIdentities: + root: _OutputIdentity + parent: _OutputIdentity + + +def _identity_from_stat(value: os.stat_result) -> _OutputIdentity: + return _OutputIdentity( + st_dev=int(value.st_dev), + st_ino=int(value.st_ino), + st_uid=int(value.st_uid), + st_gid=int(value.st_gid), + st_mode=int(value.st_mode), + ) + + +def _bind_output_identities(root: Path, parent: Path) -> _BoundOutputIdentities: + try: + root_stat = root.lstat() + parent_stat = parent.lstat() + except OSError as exc: + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_UNAVAILABLE"]) from exc + if root.is_symlink() or parent.is_symlink(): + raise SFTOutputPolicyError(["OUTPUT_PATH_SYMLINK"]) + if not stat.S_ISDIR(root_stat.st_mode) or not stat.S_ISDIR(parent_stat.st_mode): + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_UNAVAILABLE"]) + return _BoundOutputIdentities( + root=_identity_from_stat(root_stat), + parent=_identity_from_stat(parent_stat), + ) + + +def _open_bound_output_parent( + root: Path, + parent: Path, + expected: _BoundOutputIdentities, +) -> tuple[int, int]: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + flags |= getattr(os, "O_CLOEXEC", 0) + root_fd = -1 + parent_fd = -1 + try: + root_fd = os.open(root, flags) + if _identity_from_stat(os.fstat(root_fd)) != expected.root: + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_CHANGED"]) + relative_parent = parent.relative_to(root) + parent_fd = os.dup(root_fd) + for component in relative_parent.parts: + next_fd = os.open(component, flags, dir_fd=parent_fd) + os.close(parent_fd) + parent_fd = next_fd + if _identity_from_stat(os.fstat(parent_fd)) != expected.parent: + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_CHANGED"]) + return root_fd, parent_fd + except SFTOutputPolicyError: + if parent_fd >= 0: + os.close(parent_fd) + if root_fd >= 0: + os.close(root_fd) + raise + except (OSError, ValueError) as exc: + if parent_fd >= 0: + os.close(parent_fd) + if root_fd >= 0: + os.close(root_fd) + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_CHANGED"]) from exc + + def validate_sft_output_policy( config: dict[str, Any], output_dir: Path, @@ -503,6 +961,11 @@ def validate_sft_output_policy( candidate_is_descendant = _strict_descendant(candidate_resolved, root_resolved) if not candidate_is_descendant: blockers.append("OUTPUT_PATH_OUTSIDE_ROOT") + parent = candidate.parent + if candidate_is_descendant and (not parent.exists() or not parent.is_dir()): + blockers.append("OUTPUT_PARENT_MISSING") + elif candidate_is_descendant and parent.is_symlink(): + blockers.append("OUTPUT_PATH_SYMLINK") if candidate_is_descendant and candidate.exists() and not candidate.is_symlink(): if not candidate.is_dir() or any(candidate.iterdir()): blockers.append("OUTPUT_DIRECTORY_NOT_EMPTY") @@ -526,8 +989,24 @@ def validate_sft_output_policy( writable = False free_disk_gib: float | None = None + structural_blockers = { + "OUTPUT_PARENT_MISSING", + "OUTPUT_PATH_OUTSIDE_ROOT", + "OUTPUT_PATH_SYMLINK", + "OUTPUT_PATH_GIT_TRACKED", + } + if any(code in structural_blockers for code in blockers): + return { + "ready": False, + "blockers": list(dict.fromkeys(blockers)), + "root_path_sha256": _sha256_text(root_resolved.as_posix()), + "output_path_sha256": _sha256_text(candidate_resolved.as_posix()), + "writable": False, + "free_disk_gib": None, + "minimum_free_disk_gib": configured_minimum_free_disk_gib, + } try: - writable_parent = _nearest_existing_parent(candidate_resolved) + writable_parent = candidate_resolved.parent writable = os.access(writable_parent, os.W_OK) except OSError: writable_parent = root_resolved @@ -560,24 +1039,70 @@ def _claim_sft_output_directory( *, repo_root: Path | None = None, expected_output_path_sha256: str | None = None, + expected_identities: _BoundOutputIdentities | None = None, ) -> None: - """Exclusively claim and revalidate a previously unused SFT output directory.""" + """Claim one leaf under a namespace kept stable across the final mkdirat boundary. + + Identity checkpoints fail closed on observable drift and never trigger pathname + cleanup after uncertainty. This does not claim syscall-atomic protection from a + malicious same-UID rename between the final checkpoint and mkdirat; deployment + must prevent that namespace mutation during this narrow claim window. + """ initial = validate_sft_output_policy(config, output_dir, repo_root=repo_root) if initial.get("ready") is not True: + if expected_identities is not None: + initial_blockers = [str(code) for code in initial.get("blockers", [])] + if initial_blockers in (["OUTPUT_DIRECTORY_EXISTS"], ["OUTPUT_DIRECTORY_NOT_EMPTY"]): + raise SFTOutputPolicyError(["OUTPUT_DIRECTORY_EXISTS"]) + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_CHANGED"]) raise SFTOutputPolicyError([str(code) for code in initial.get("blockers", [])]) expected_hash = expected_output_path_sha256 or initial.get("output_path_sha256") + raw_root = config.get("output_root") + if not isinstance(raw_root, str): + raise SFTOutputPolicyError(["OUTPUT_ROOT_MISSING"]) + root = Path(raw_root).expanduser().resolve(strict=True) + parent = output_dir.expanduser().parent.resolve(strict=True) + identities = expected_identities or _bind_output_identities(root, parent) + root_fd = -1 + parent_fd = -1 try: - output_dir.mkdir(parents=True, exist_ok=False) + root_fd, parent_fd = _open_bound_output_parent(root, parent, identities) + try: + path_identities = _bind_output_identities(root, parent) + except SFTOutputPolicyError as exc: + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_CHANGED"]) from exc + if ( + _identity_from_stat(os.fstat(root_fd)) != identities.root + or _identity_from_stat(os.fstat(parent_fd)) != identities.parent + or path_identities != identities + ): + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_CHANGED"]) + os.mkdir(output_dir.name, mode=0o700, dir_fd=parent_fd) + if ( + _identity_from_stat(os.fstat(root_fd)) != identities.root + or _identity_from_stat(os.fstat(parent_fd)) != identities.parent + ): + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_CHANGED"]) except FileExistsError as exc: raise SFTOutputPolicyError(["OUTPUT_DIRECTORY_EXISTS"]) from exc + except SFTOutputPolicyError: + raise except OSError as exc: raise SFTOutputPolicyError(["OUTPUT_CREATE_FAILED"]) from exc + finally: + if parent_fd >= 0: + os.close(parent_fd) + if root_fd >= 0: + os.close(root_fd) - claimed = validate_sft_output_policy(config, output_dir, repo_root=repo_root) - claimed_blockers = [str(code) for code in claimed.get("blockers", [])] - if claimed_blockers != ["OUTPUT_DIRECTORY_EXISTS"]: - raise SFTOutputPolicyError(claimed_blockers or ["OUTPUT_PATH_CHANGED"]) - if not isinstance(expected_hash, str) or claimed.get("output_path_sha256") != expected_hash: + try: + current = _bind_output_identities(root, parent) + except SFTOutputPolicyError as exc: + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_CHANGED"]) from exc + if current != identities: + raise SFTOutputPolicyError(["OUTPUT_IDENTITY_CHANGED"]) + current_hash = _sha256_text(output_dir.expanduser().resolve(strict=True).as_posix()) + if not isinstance(expected_hash, str) or current_hash != expected_hash: raise SFTOutputPolicyError(["OUTPUT_PATH_CHANGED"]) @@ -722,6 +1247,10 @@ def _probe_sft_gpu() -> tuple[dict[str, Any], list[str]]: "compute_capability": None, "total_memory_gib": None, "minimum_memory_gib": 35.0, + "free_memory_gib": None, + "minimum_free_memory_gib": 35.0, + "compute_process_count": None, + "idle_verified": False, "cuda_version": None, "bf16_supported": False, } @@ -754,6 +1283,8 @@ def _probe_sft_gpu() -> tuple[dict[str, Any], list[str]]: properties = cuda.get_device_properties(0) capability = tuple(int(value) for value in cuda.get_device_capability(0)) total_memory_gib = int(getattr(properties, "total_memory", 0)) / float(1024**3) + free_memory_bytes, _ = cuda.mem_get_info(0) + free_memory_gib = int(free_memory_bytes) / float(1024**3) bf16_supported = bool(cuda.is_bf16_supported()) gpu_name = str(cuda.get_device_name(0)) except Exception: @@ -763,6 +1294,7 @@ def _probe_sft_gpu() -> tuple[dict[str, Any], list[str]]: "name": gpu_name, "compute_capability": f"{capability[0]}.{capability[1]}", "total_memory_gib": round(total_memory_gib, 3), + "free_memory_gib": round(free_memory_gib, 3), "cuda_version": str(getattr(getattr(torch, "version", None), "cuda", None) or "unknown"), "bf16_supported": bf16_supported, } @@ -771,8 +1303,41 @@ def _probe_sft_gpu() -> tuple[dict[str, Any], list[str]]: blockers.append("BF16_UNSUPPORTED") if total_memory_gib < 35.0: blockers.append("GPU_MEMORY_INSUFFICIENT") + if free_memory_gib < 35.0: + blockers.append("GPU_FREE_MEMORY_INSUFFICIENT") if re.search(r"\bA100\b", gpu_name, flags=re.IGNORECASE) is None: blockers.append("GPU_NOT_A100") + try: + occupancy = subprocess.run( + [ + "nvidia-smi", + f"--id={selected_parts[0]}", + "--query-compute-apps=pid", + "--format=csv,noheader,nounits", + ], + check=False, + capture_output=True, + text=True, + timeout=15, + ) + if occupancy.returncode != 0: + raise RuntimeError("occupancy probe failed") + process_ids: set[int] = set() + for line in occupancy.stdout.splitlines(): + value = line.strip() + if not value: + continue + if not value.isdigit(): + raise RuntimeError("occupancy output invalid") + process_id = int(value) + if process_id != os.getpid(): + process_ids.add(process_id) + facts["compute_process_count"] = len(process_ids) + facts["idle_verified"] = not process_ids + if process_ids: + blockers.append("GPU_BUSY") + except (OSError, RuntimeError, subprocess.SubprocessError): + blockers.append("GPU_OCCUPANCY_PROBE_FAILED") return facts, list(dict.fromkeys(blockers)) @@ -1078,13 +1643,7 @@ def _probe_sft_model_and_objective( blocker = "ASSISTANT_ONLY_LABELS_INVALID" return model_facts, objective_facts, [blocker] - all_lengths_match = all( - len(record["input_ids"]) == len(record["attention_mask"]) == len(record["labels"]) - for record in records - ) - prompt_masked = all(any(label == -100 for label in record["labels"]) for record in records) - assistant_present = all(any(label != -100 for label in record["labels"]) for record in records) - if not records or not all_lengths_match or not prompt_masked or not assistant_present: + if not records or not all(_assistant_only_record_is_valid(record) for record in records): return model_facts, objective_facts, ["ASSISTANT_ONLY_LABELS_INVALID"] objective_facts.update( { @@ -1114,6 +1673,7 @@ class _SFTPreflightExecutionContext: model_inventory_sha256: str output_root: Path output_dir: Path + output_identities: _BoundOutputIdentities output_facts_json: str def config_snapshot(self) -> dict[str, Any]: @@ -1403,6 +1963,9 @@ def _run_sft_preflight_core_unchecked( model_inventory = model_facts.get("weight_inventory") if not isinstance(model_fingerprints, dict) or not isinstance(model_inventory, list): return public_result, None + resolved_output_root = Path(raw_output_root).expanduser().resolve(strict=True) + resolved_output_dir = output_dir.expanduser().resolve(strict=False) + output_identities = _bind_output_identities(resolved_output_root, resolved_output_dir.parent) context = _SFTPreflightExecutionContext( config_path=config_path.resolve(strict=True), config_sha256=str(config_facts["config_sha256"]), @@ -1427,8 +1990,9 @@ def _run_sft_preflight_core_unchecked( separators=(",", ":"), ) ), - output_root=Path(raw_output_root).expanduser().resolve(strict=True), - output_dir=output_dir.expanduser().resolve(strict=False), + output_root=resolved_output_root, + output_dir=resolved_output_dir, + output_identities=output_identities, output_facts_json=json.dumps( output_facts, ensure_ascii=False, @@ -1766,6 +2330,74 @@ def _clean_evaluation_truth_surface() -> dict[str, Any]: } +@dataclass(frozen=True) +class _AdapterStateSnapshot: + tensors: tuple[tuple[str, int, str], ...] + trainable_parameter_count: int + all_finite: bool + digest: str + + +def _capture_adapter_state(model: Any) -> _AdapterStateSnapshot: + import torch + + tensors: list[tuple[str, int, str]] = [] + digest = hashlib.sha256() + trainable_parameter_count = 0 + all_finite = True + named_parameters = getattr(model, "named_parameters", None) + if not callable(named_parameters): + return _AdapterStateSnapshot((), 0, False, hashlib.sha256().hexdigest()) + for name, parameter in sorted(named_parameters(), key=lambda item: item[0]): + if not bool(getattr(parameter, "requires_grad", False)): + continue + detached = parameter.detach().cpu().contiguous() + trainable_parameter_count += int(detached.numel()) + normalized_name = str(name).lower() + if "lora_" not in normalized_name and "adapter" not in normalized_name: + continue + raw = detached.view(torch.uint8).numpy().tobytes() + tensor_hash = hashlib.sha256(raw).hexdigest() + parameter_count = int(detached.numel()) + all_finite = all_finite and bool(torch.isfinite(detached.float()).all().item()) + tensors.append((str(name), parameter_count, tensor_hash)) + digest.update(str(name).encode("utf-8")) + digest.update(b"\0") + digest.update(str(tuple(detached.shape)).encode("ascii")) + digest.update(b"\0") + digest.update(str(detached.dtype).encode("ascii")) + digest.update(b"\0") + digest.update(raw) + return _AdapterStateSnapshot( + tensors=tuple(tensors), + trainable_parameter_count=trainable_parameter_count, + all_finite=all_finite, + digest=digest.hexdigest(), + ) + + +def _adapter_update_evidence( + before: _AdapterStateSnapshot, + after: _AdapterStateSnapshot, +) -> dict[str, Any]: + before_hashes = {name: tensor_hash for name, _, tensor_hash in before.tensors} + after_hashes = {name: tensor_hash for name, _, tensor_hash in after.tensors} + changed = sum( + 1 + for name, tensor_hash in after_hashes.items() + if before_hashes.get(name) != tensor_hash + ) + same_tensor_set = before_hashes.keys() == after_hashes.keys() + return { + "trainable_parameter_count": after.trainable_parameter_count, + "adapter_tensor_count": len(after.tensors), + "adapter_state_digest_before": before.digest, + "adapter_state_digest_after": after.digest, + "changed_adapter_tensor_count": changed if same_tensor_set else 0, + "all_adapter_tensors_finite": before.all_finite and after.all_finite, + } + + def _sft_smoke_postconditions(metadata: dict[str, Any]) -> list[str]: blockers: list[str] = [] observed_steps = metadata.get("observed_optimizer_steps") @@ -1786,13 +2418,39 @@ def _sft_smoke_postconditions(metadata: dict[str, Any]) -> list[str]: if isinstance(loss, bool) or not isinstance(loss, int | float) or not math.isfinite(float(loss)): blockers.append("TRAINING_LOSS_INVALID") + trainable_parameter_count = metadata.get("trainable_parameter_count") + adapter_tensor_count = metadata.get("adapter_tensor_count") + before_digest = metadata.get("adapter_state_digest_before") + after_digest = metadata.get("adapter_state_digest_after") + changed_adapter_tensor_count = metadata.get("changed_adapter_tensor_count") + all_adapter_tensors_finite = metadata.get("all_adapter_tensors_finite") + if ( + type(trainable_parameter_count) is not int + or trainable_parameter_count <= 0 + or type(adapter_tensor_count) is not int + or adapter_tensor_count <= 0 + or not isinstance(before_digest, str) + or len(before_digest) != 64 + or not isinstance(after_digest, str) + or len(after_digest) != 64 + or before_digest == after_digest + or type(changed_adapter_tensor_count) is not int + or changed_adapter_tensor_count <= 0 + or all_adapter_tensors_finite is not True + ): + blockers.append("ADAPTER_UPDATE_NOT_OBSERVED") + adapter_path = Path(str(metadata.get("adapter_path", ""))) try: adapter_files = sorted( path for path in adapter_path.rglob("*") if path.is_file() and path.stat().st_size > 0 ) metadata["adapter_files"] = [ - {"name": path.relative_to(adapter_path).as_posix(), "size": path.stat().st_size} + { + "name": path.relative_to(adapter_path).as_posix(), + "size": path.stat().st_size, + "sha256": _sha256_file(path), + } for path in adapter_files ] except OSError: @@ -3283,20 +3941,14 @@ def _assistant_target_span(training_text: str, assistant_text: str) -> tuple[str return "available", start, start + len(assistant_text) -def _assistant_only_labels_from_encoded( +def _assistant_token_indices_from_offsets( *, - encoded: Any, offsets: list[tuple[int, int]], assistant_start: int, assistant_end: int, -) -> tuple[list[Any], list[str]]: - input_ids = _token_list(_mapping_value(encoded, "input_ids")) - if not input_ids: - return [], ["input_ids_unavailable"] - if not offsets or len(input_ids) != len(offsets): - gaps = ["token_offsets_unavailable"] if not offsets else ["label_token_offset_length_mismatch"] - return [], gaps - +) -> tuple[list[int], list[str]]: + if not offsets: + return [], ["token_offsets_unavailable"] assistant_indices: set[int] = set() boundary_overlap = False for index, (start, end) in enumerate(offsets): @@ -3312,13 +3964,65 @@ def _assistant_only_labels_from_encoded( return [], ["assistant_span_token_boundary_unavailable"] if not assistant_indices: return [], ["assistant_target_tokens_unavailable"] + return sorted(assistant_indices), [] + + +def _assistant_only_labels_from_encoded( + *, + encoded: Any, + offsets: list[tuple[int, int]], + assistant_start: int, + assistant_end: int, +) -> tuple[list[Any], list[str]]: + input_ids = _token_list(_mapping_value(encoded, "input_ids")) + if not input_ids: + return [], ["input_ids_unavailable"] + if not offsets or len(input_ids) != len(offsets): + gaps = ["token_offsets_unavailable"] if not offsets else ["label_token_offset_length_mismatch"] + return [], gaps + assistant_indices, evidence_gaps = _assistant_token_indices_from_offsets( + offsets=offsets, + assistant_start=assistant_start, + assistant_end=assistant_end, + ) + if evidence_gaps: + return [], evidence_gaps + assistant_index_set = set(assistant_indices) return [ - token_id if index in assistant_indices else -100 + token_id if index in assistant_index_set else -100 for index, token_id in enumerate(input_ids) ], [] +def _assistant_only_record_is_valid(record: dict[str, Any]) -> bool: + input_ids = _token_list(record.get("input_ids")) + attention_mask = _token_list(record.get("attention_mask")) + labels = _token_list(record.get("labels")) + raw_indices = record.get("assistant_token_indices") + if not isinstance(raw_indices, list) or not raw_indices: + return False + if not all(type(index) is int for index in raw_indices): + return False + assistant_indices = cast(list[int], raw_indices) + if assistant_indices != sorted(set(assistant_indices)): + return False + if assistant_indices != list(range(assistant_indices[0], assistant_indices[-1] + 1)): + return False + if not input_ids or len(input_ids) != len(attention_mask) or len(input_ids) != len(labels): + return False + if assistant_indices[0] < 0 or assistant_indices[-1] >= len(input_ids): + return False + assistant_set = set(assistant_indices) + for index, (token_id, label) in enumerate(zip(input_ids, labels, strict=True)): + if index in assistant_set: + if label == -100 or label != token_id: + return False + elif label != -100: + return False + return True + + def _assistant_only_training_record( row: SFTDatasetRow, tokenizer: Any, @@ -3349,11 +4053,23 @@ def _assistant_only_training_record( attention_mask = _token_list(_mapping_value(encoded, "attention_mask")) if len(attention_mask) != len(input_ids): attention_mask = [1 for _ in input_ids] - return { + assistant_indices, index_gaps = _assistant_token_indices_from_offsets( + offsets=offsets, + assistant_start=assistant_start, + assistant_end=assistant_end, + ) + if index_gaps: + gaps = ",".join(index_gaps) + raise ValueError(f"assistant-only SFT labels unavailable: {gaps}") + record = { "input_ids": input_ids, "attention_mask": attention_mask, "labels": labels, + "assistant_token_indices": assistant_indices, } + if not _assistant_only_record_is_valid(record): + raise ValueError("assistant-only SFT labels unavailable: assistant_label_region_invalid") + return record class _AssistantOnlyCausalLmDataCollator: @@ -3846,6 +4562,9 @@ def _run_real_sft( drift_blockers = _preflight_input_drift_blockers(execution_context) if drift_blockers: raise SFTPreflightDriftError(drift_blockers) + _, gpu_blockers = _probe_sft_gpu() + if gpu_blockers: + raise SFTPreflightDriftError(gpu_blockers) config = execution_context.config_snapshot() output_dir = execution_context.output_dir _claim_sft_output_directory( @@ -3853,6 +4572,7 @@ def _run_real_sft( output_dir, repo_root=execution_context.repo_root, expected_output_path_sha256=json.loads(execution_context.output_facts_json).get("output_path_sha256"), + expected_identities=execution_context.output_identities, ) from datasets import Dataset # type: ignore[import-not-found, unused-ignore] @@ -3877,6 +4597,9 @@ def _run_real_sft( dataset = Dataset.from_list(records) import torch + _, immediate_gpu_blockers = _probe_sft_gpu() + if immediate_gpu_blockers: + raise SFTPreflightDriftError(immediate_gpu_blockers) model = AutoModelForCausalLM.from_pretrained( base_model, local_files_only=local_files_only, @@ -3917,7 +4640,10 @@ def _run_real_sft( data_collator=_AssistantOnlyCausalLmDataCollator(tokenizer), **_sft_trainer_tokenizer_kwargs(SFTTrainer, tokenizer), ) + adapter_state_before = _capture_adapter_state(trainer.model) train_result = trainer.train() + adapter_state_after = _capture_adapter_state(trainer.model) + metadata.update(_adapter_update_evidence(adapter_state_before, adapter_state_after)) _record_sft_training_budget_metadata( metadata, config=config, diff --git a/tests/test_a100_sft_review_hardening.py b/tests/test_a100_sft_review_hardening.py new file mode 100644 index 0000000..2a03737 --- /dev/null +++ b/tests/test_a100_sft_review_hardening.py @@ -0,0 +1,471 @@ +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path +from typing import Any + +import pytest + +from voice2task import training +from voice2task.cli import train as train_cli + +PUBLIC_RESULT_KEYS = { + "schema_version", + "stage", + "training_status", + "smoke_status", + "blockers", + "preflight", + "observed_optimizer_steps", + "training_rows_used", + "training_budget", + "train_result_metrics", + "adapter_files", + "clean_evaluation", +} + + +@pytest.mark.parametrize( + "private_path", + [ + "/home/alice/private/model", + "/data/secret/output", + "/workspace/company/config.json", + "/opt/private/adapter", + "/srv/custom-absolute/private-run", + ], +) +def test_public_training_result_is_exact_allowlist_for_arbitrary_absolute_paths(private_path: str) -> None: + metadata = { + "schema_version": "voice2task-training-result-v1", + "stage": "sft", + "training_status": "training_completed", + "smoke_status": "SMOKE_COMPLETED", + "blockers": [], + "preflight": {"ready": True, "status": "ready", "blockers": []}, + "observed_optimizer_steps": 1, + "training_rows_used": 2, + "training_budget": {"configured_max_steps": 1}, + "train_result_metrics": {"train_loss": 1.0}, + "adapter_files": [ + {"name": "adapter_model.safetensors", "size": 7, "sha256": "a" * 64, "path": private_path} + ], + "clean_evaluation": {"execution_readiness": False}, + "hyperparameters": {"base_model_runtime_path": private_path}, + "base_model_runtime_path": private_path, + "output_root": private_path, + "adapter_path": private_path, + "metadata_path": private_path, + "dataset_path": private_path, + "output_paths": [private_path], + "training_command": ["--output-dir", private_path], + "command_summary": f"run --output-dir {private_path}", + "private_config": {"path": private_path}, + } + + result = training.public_training_result(metadata) + + assert set(result) == PUBLIC_RESULT_KEYS + assert result["adapter_files"] == [ + {"name": "adapter_model.safetensors", "size": 7, "sha256": "a" * 64} + ] + assert private_path not in json.dumps(result, sort_keys=True) + + +@pytest.mark.parametrize( + "private_path", + ["/home/a/private", "/data/private", "/workspace/private", "/opt/private", "/custom/private/root"], +) +def test_training_cli_stdout_uses_public_result_allowlist( + monkeypatch: pytest.MonkeyPatch, + capsys: Any, + private_path: str, +) -> None: + monkeypatch.setattr( + train_cli, + "run_sft", + lambda *args, **kwargs: { + "schema_version": "voice2task-training-result-v1", + "stage": "sft", + "training_status": "training_completed", + "blockers": [], + "adapter_files": [], + "hyperparameters": {"output_root": private_path}, + "metadata_path": f"{private_path}/adapter_metadata.json", + }, + ) + + exit_code = train_cli.main( + ["sft", "--config", "config.json", "--manifest", "manifest.json", "--output-dir", "run"] + ) + + captured = capsys.readouterr() + result = json.loads(captured.out) + assert exit_code == 0 + assert set(result).issubset(PUBLIC_RESULT_KEYS) + assert private_path not in captured.out + assert captured.err == "" + + +def test_public_training_result_rebuilds_preflight_without_unknown_nested_private_fields() -> None: + private_path = "/workspace/acme/secret-model" + metadata = { + "training_status": "training_blocked_by_preflight", + "blockers": ["MODEL_PATH_UNRESOLVED"], + "preflight": { + "schema_version": "voice2task-sft-preflight-v1", + "ready": False, + "status": "blocked", + "blockers": ["MODEL_PATH_UNRESOLVED"], + "model": { + "public_id": "Qwen/Qwen2.5-7B-Instruct", + "local_files_only": True, + "private_runtime_path": private_path, + "unknown": {"deep_secret": private_path}, + }, + "unknown_section": {"path": private_path}, + }, + } + result = training.public_training_result(metadata) + + assert private_path not in json.dumps(result, sort_keys=True) + assert result["preflight"] == { + "schema_version": "voice2task-sft-preflight-v1", + "ready": False, + "status": "blocked", + "blockers": ["MODEL_PATH_UNRESOLVED"], + "model": { + "public_id": "Qwen/Qwen2.5-7B-Instruct", + "local_files_only": True, + }, + } + + +def test_public_training_result_rejects_private_paths_in_every_allowed_string_container() -> None: + private_path = "/workspace/acme/secret-model" + metadata = { + "schema_version": private_path, + "stage": private_path, + "training_status": private_path, + "smoke_status": private_path, + "preflight": { + "schema_version": private_path, + "ready": False, + "status": private_path, + "blockers": [], + "git": {"commit_sha": private_path}, + "config": { + "config_sha256": private_path, + "base_model_public_id": private_path, + "dataset_split": private_path, + "save_strategy": private_path, + }, + "dataset": { + "manifest_file": private_path, + "manifest_sha256": private_path, + "manifest_id": private_path, + "sft_file": private_path, + "sft_sha256": private_path, + "selected_split": private_path, + "selected_row_ids_sha256": private_path, + }, + "model": { + "public_id": private_path, + "stable_fingerprints": {"config.json": private_path}, + "weight_inventory": [{"name": private_path, "size": 12}], + "snapshot_revision_sha256": private_path, + }, + "runtime": { + "python": private_path, + "python_requirement": private_path, + "versions": {"torch": private_path}, + "pip_check": private_path, + }, + "gpu": { + "name": private_path, + "compute_capability": private_path, + "cuda_version": private_path, + }, + "output": { + "root_path_sha256": private_path, + "output_path_sha256": private_path, + }, + }, + } + + result = training.public_training_result(metadata) + + assert private_path not in json.dumps(result, sort_keys=True) + + +def test_output_policy_requires_existing_parent(tmp_path: Path) -> None: + root = tmp_path / "root" + root.mkdir() + result = training.validate_sft_output_policy( + {"output_root": root.as_posix(), "min_free_disk_gib": 0}, + root / "missing-parent" / "run", + ) + assert result["ready"] is False + assert result["blockers"] == ["OUTPUT_PARENT_MISSING"] + + +def test_output_claim_rejects_concurrent_final_leaf_creation(tmp_path: Path) -> None: + root = tmp_path / "root" + parent = root / "runs" + parent.mkdir(parents=True) + output = parent / "smoke" + config = {"output_root": root.as_posix(), "min_free_disk_gib": 0} + identities = training._bind_output_identities(root, parent) # noqa: SLF001 + output.mkdir() + + with pytest.raises(training.SFTOutputPolicyError) as exc_info: + training._claim_sft_output_directory( # noqa: SLF001 + config, + output, + expected_identities=identities, + ) + assert exc_info.value.blockers == ["OUTPUT_DIRECTORY_EXISTS"] + + +def test_output_claim_blocks_root_same_path_inode_exchange_without_external_mutation(tmp_path: Path) -> None: + root = tmp_path / "root" + parent = root / "runs" + parent.mkdir(parents=True) + output = parent / "smoke" + config = {"output_root": root.as_posix(), "min_free_disk_gib": 0} + identities = training._bind_output_identities(root, parent) # noqa: SLF001 + old_root = tmp_path / "old-root" + root.rename(old_root) + root.mkdir() + (root / "runs").mkdir() + + with pytest.raises(training.SFTOutputPolicyError) as exc_info: + training._claim_sft_output_directory(config, output, expected_identities=identities) # noqa: SLF001 + assert exc_info.value.blockers == ["OUTPUT_IDENTITY_CHANGED"] + assert not output.exists() + + +def test_output_claim_blocks_parent_exchange_to_external_symlink_with_zero_external_mutation(tmp_path: Path) -> None: + root = tmp_path / "root" + parent = root / "runs" + outside = tmp_path / "outside" + parent.mkdir(parents=True) + outside.mkdir() + output = parent / "smoke" + config = {"output_root": root.as_posix(), "min_free_disk_gib": 0} + identities = training._bind_output_identities(root, parent) # noqa: SLF001 + parent.rename(root / "old-runs") + parent.symlink_to(outside, target_is_directory=True) + + with pytest.raises(training.SFTOutputPolicyError) as exc_info: + training._claim_sft_output_directory(config, output, expected_identities=identities) # noqa: SLF001 + assert exc_info.value.blockers == ["OUTPUT_IDENTITY_CHANGED"] + assert list(outside.iterdir()) == [] + + +def test_missing_parent_becoming_external_symlink_remains_blocked_with_zero_external_mutation(tmp_path: Path) -> None: + root = tmp_path / "root" + outside = tmp_path / "outside" + root.mkdir() + outside.mkdir() + parent = root / "missing" + output = parent / "smoke" + config = {"output_root": root.as_posix(), "min_free_disk_gib": 0} + assert training.validate_sft_output_policy(config, output)["blockers"] == ["OUTPUT_PARENT_MISSING"] + parent.symlink_to(outside, target_is_directory=True) + with pytest.raises(training.SFTOutputPolicyError): + training._claim_sft_output_directory(config, output) # noqa: SLF001 + assert list(outside.iterdir()) == [] + + +def test_output_claim_blocks_existing_parent_inode_exchange(tmp_path: Path) -> None: + root = tmp_path / "root" + parent = root / "runs" + parent.mkdir(parents=True) + output = parent / "smoke" + config = {"output_root": root.as_posix(), "min_free_disk_gib": 0} + identities = training._bind_output_identities(root, parent) # noqa: SLF001 + parent.rename(root / "old-runs") + parent.mkdir() + with pytest.raises(training.SFTOutputPolicyError) as exc_info: + training._claim_sft_output_directory(config, output, expected_identities=identities) # noqa: SLF001 + assert exc_info.value.blockers == ["OUTPUT_IDENTITY_CHANGED"] + assert not output.exists() + + +def test_output_claim_checks_parent_path_identity_after_open_before_leaf_mkdir( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "root" + parent = root / "runs" + outside = tmp_path / "outside" + parent.mkdir(parents=True) + outside.mkdir() + moved_parent = outside / "moved-runs" + output = parent / "smoke" + config = {"output_root": root.as_posix(), "min_free_disk_gib": 0} + identities = training._bind_output_identities(root, parent) # noqa: SLF001 + original_open = training._open_bound_output_parent # noqa: SLF001 + + def open_then_move(*args: Any, **kwargs: Any) -> tuple[int, int]: + descriptors = original_open(*args, **kwargs) + parent.rename(moved_parent) + return descriptors + + monkeypatch.setattr(training, "_open_bound_output_parent", open_then_move) + + with pytest.raises(training.SFTOutputPolicyError) as exc_info: + training._claim_sft_output_directory( # noqa: SLF001 + config, + output, + expected_identities=identities, + ) + assert exc_info.value.blockers == ["OUTPUT_IDENTITY_CHANGED"] + assert not (moved_parent / "smoke").exists() + + +def _install_fake_torch(monkeypatch: pytest.MonkeyPatch, *, free_gib: float = 80.0) -> None: + class FakeCuda: + @staticmethod + def is_available() -> bool: + return True + + @staticmethod + def device_count() -> int: + return 1 + + @staticmethod + def get_device_properties(index: int) -> Any: + return types.SimpleNamespace(total_memory=80 * 1024**3) + + @staticmethod + def get_device_capability(index: int) -> tuple[int, int]: + return (8, 0) + + @staticmethod + def is_bf16_supported() -> bool: + return True + + @staticmethod + def get_device_name(index: int) -> str: + return "NVIDIA A100-SXM4-80GB" + + @staticmethod + def mem_get_info(index: int) -> tuple[int, int]: + assert index == 0 + return (int(free_gib * 1024**3), 80 * 1024**3) + + monkeypatch.setitem( + sys.modules, + "torch", + types.SimpleNamespace(cuda=FakeCuda(), version=types.SimpleNamespace(cuda="12.4")), + ) + + +def test_gpu_probe_reports_free_memory_and_idle_aggregate(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3") + _install_fake_torch(monkeypatch) + monkeypatch.setattr( + training.subprocess, + "run", + lambda *args, **kwargs: types.SimpleNamespace(returncode=0, stdout="", stderr=""), + ) + facts, blockers = training._probe_sft_gpu() # noqa: SLF001 + assert blockers == [] + assert facts["free_memory_gib"] == 80.0 + assert facts["minimum_free_memory_gib"] == 35.0 + assert facts["compute_process_count"] == 0 + assert facts["idle_verified"] is True + serialized = json.dumps(facts) + for forbidden in ("pid", "username", "uuid", "hostname", "command"): + assert forbidden not in serialized.lower() + + +@pytest.mark.parametrize( + ("free_gib", "process_output", "expected"), + [ + (34.0, "", "GPU_FREE_MEMORY_INSUFFICIENT"), + (80.0, "12345\n", "GPU_BUSY"), + ], +) +def test_gpu_probe_blocks_insufficient_free_memory_or_busy_device( + monkeypatch: pytest.MonkeyPatch, + free_gib: float, + process_output: str, + expected: str, +) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") + _install_fake_torch(monkeypatch, free_gib=free_gib) + monkeypatch.setattr( + training.subprocess, + "run", + lambda *args, **kwargs: types.SimpleNamespace(returncode=0, stdout=process_output, stderr=""), + ) + _, blockers = training._probe_sft_gpu() # noqa: SLF001 + assert expected in blockers + + +def test_gpu_probe_fails_closed_when_occupancy_probe_fails(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") + _install_fake_torch(monkeypatch) + monkeypatch.setattr( + training.subprocess, + "run", + lambda *args, **kwargs: types.SimpleNamespace(returncode=1, stdout="private pid 123", stderr="secret"), + ) + facts, blockers = training._probe_sft_gpu() # noqa: SLF001 + assert blockers == ["GPU_OCCUPANCY_PROBE_FAILED"] + assert facts["compute_process_count"] is None + assert facts["idle_verified"] is False + assert "123" not in json.dumps(facts) + + +def test_assistant_label_validator_rejects_single_prompt_token_leak() -> None: + record = { + "input_ids": [10, 11, 12, 13], + "attention_mask": [1, 1, 1, 1], + "labels": [-100, 11, 12, 13], + "assistant_token_indices": [2, 3], + } + assert training._assistant_only_record_is_valid(record) is False # noqa: SLF001 + + +def test_adapter_state_evidence_requires_real_finite_change() -> None: + torch = pytest.importorskip("torch") + class TinyAdapter(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.adapter_weight = torch.nn.Parameter(torch.zeros(1, 2)) + + model = TinyAdapter() + before = training._capture_adapter_state(model) # noqa: SLF001 + with torch.no_grad(): + model.adapter_weight.add_(1.0) + after = training._capture_adapter_state(model) # noqa: SLF001 + evidence = training._adapter_update_evidence(before, after) # noqa: SLF001 + assert evidence["trainable_parameter_count"] == 2 + assert evidence["adapter_tensor_count"] == 1 + assert evidence["adapter_state_digest_before"] != evidence["adapter_state_digest_after"] + assert evidence["changed_adapter_tensor_count"] == 1 + assert evidence["all_adapter_tensors_finite"] is True + + +def test_smoke_postconditions_require_adapter_update_evidence(tmp_path: Path) -> None: + adapter = tmp_path / "run" / "adapter" + adapter.mkdir(parents=True) + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + (adapter / "adapter_model.safetensors").write_bytes(b"adapter") + metadata = { + "adapter_path": adapter.as_posix(), + "metadata_path": (tmp_path / "run" / "adapter_metadata.json").as_posix(), + "hyperparameters": {"max_train_rows": 1}, + "observed_optimizer_steps": 1, + "training_rows_used": 1, + "train_result_metrics": {"train_loss": 1.0}, + } + blockers = training._sft_smoke_postconditions(metadata) # noqa: SLF001 + assert "ADAPTER_UPDATE_NOT_OBSERVED" in blockers + assert all("sha256" in item for item in metadata["adapter_files"]) diff --git a/tests/test_a100_sft_smoke.py b/tests/test_a100_sft_smoke.py index 1a9ca24..43652b6 100644 --- a/tests/test_a100_sft_smoke.py +++ b/tests/test_a100_sft_smoke.py @@ -1648,6 +1648,7 @@ def test_sft_heavy_training_requires_cli_and_config_opt_ins(monkeypatch: Any, tm allowed_root = tmp_path / "remote-root" allowed_root.mkdir() + (allowed_root / "runs").mkdir() config_allows = _write_config(tmp_path, allow_heavy_training=True, output_root=allowed_root.as_posix()) dry_run_meta = run_sft( config_path=config_allows, @@ -1859,6 +1860,7 @@ def _bound_sft_execution_context( ), output_root=tmp_path, output_dir=output_dir, + output_identities=training._bind_output_identities(tmp_path, output_dir.parent), # noqa: SLF001 output_facts_json=json.dumps(output_facts, sort_keys=True, separators=(",", ":")), ) @@ -1867,6 +1869,7 @@ def test_real_sft_heavy_path_keeps_new_trl_sfttrainer_with_assistant_only_labels monkeypatch: Any, tmp_path: Path, ) -> None: + monkeypatch.setattr(training, "_probe_sft_gpu", lambda: ({"idle_verified": True}, [])) manifest = _write_manifest(tmp_path) rows = training._load_sft_training_rows(manifest, split="train") # noqa: SLF001 target = canonical_contract_json(rows[0].target_contract) @@ -1948,6 +1951,7 @@ def test_real_sft_heavy_path_supports_old_trl_sfttrainer_tokenizer_signature( monkeypatch: Any, tmp_path: Path, ) -> None: + monkeypatch.setattr(training, "_probe_sft_gpu", lambda: ({"idle_verified": True}, [])) manifest = _write_manifest(tmp_path) rows = training._load_sft_training_rows(manifest, split="train") # noqa: SLF001 dataset_records: list[list[dict[str, list[int]]]] = [] @@ -2018,6 +2022,7 @@ def test_real_sft_heavy_path_limits_tiny_overfit_rows_and_records_metadata( monkeypatch: Any, tmp_path: Path, ) -> None: + monkeypatch.setattr(training, "_probe_sft_gpu", lambda: ({"idle_verified": True}, [])) manifest = _write_multirow_manifest(tmp_path, train_rows=4) rows = training._load_sft_training_rows(manifest, split="train")[:2] # noqa: SLF001 dataset_records: list[list[dict[str, list[int]]]] = [] @@ -2095,6 +2100,7 @@ def test_sft_metadata_contains_public_safe_a100_smoke_fields(monkeypatch: Any, t manifest = _write_manifest(tmp_path) allowed_root = tmp_path / "remote-root" allowed_root.mkdir() + (allowed_root / "runs").mkdir() config = _write_config(tmp_path, allow_heavy_training=True, output_root=allowed_root.as_posix()) output_dir = allowed_root / "runs" / "run" monkeypatch.setattr(training, "_train_dependencies_available", lambda: True) @@ -2129,6 +2135,7 @@ def test_sft_training_failure_writes_sanitized_metadata(monkeypatch: Any, tmp_pa manifest = _write_manifest(tmp_path) allowed_root = tmp_path / "remote-root" allowed_root.mkdir() + (allowed_root / "runs").mkdir() config = _write_config(tmp_path, allow_heavy_training=True, output_root=allowed_root.as_posix()) output_dir = allowed_root / "runs" / "run" monkeypatch.setattr(training, "_train_dependencies_available", lambda: True) diff --git a/tests/test_private_a100_sft_preflight.py b/tests/test_private_a100_sft_preflight.py index 4eacb09..6391bd1 100644 --- a/tests/test_private_a100_sft_preflight.py +++ b/tests/test_private_a100_sft_preflight.py @@ -116,6 +116,7 @@ def test_output_policy_rejects_existing_empty_output_dir(tmp_path: Path) -> None def test_output_policy_accepts_new_absolute_child_without_creating_it(tmp_path: Path) -> None: root = tmp_path / "root" root.mkdir() + (root / "runs").mkdir() output_dir = root / "runs" / "smoke" result = _policy({"output_root": root.as_posix(), "min_free_disk_gib": 0}, output_dir) @@ -215,6 +216,7 @@ def selective_resolve(path: Path, *args: Any, **kwargs: Any) -> Path: def test_output_policy_rejects_repository_location(tmp_path: Path) -> None: repo_root = tmp_path / "repo" repo_root.mkdir() + (repo_root / "runs").mkdir() output_dir = repo_root / "runs" / "smoke" result = _policy( @@ -234,15 +236,15 @@ def test_output_claim_fails_closed_if_candidate_appears_after_preflight( root = tmp_path / "root" root.mkdir() output_dir = root / "smoke" - original_mkdir = Path.mkdir + original_mkdir = training.os.mkdir - def racing_mkdir(path: Path, *args: Any, **kwargs: Any) -> None: - if path == output_dir: + def racing_mkdir(path: Any, *args: Any, **kwargs: Any) -> None: + if path == output_dir.name and kwargs.get("dir_fd") is not None: original_mkdir(path, *args, **kwargs) raise FileExistsError(path) original_mkdir(path, *args, **kwargs) - monkeypatch.setattr(Path, "mkdir", racing_mkdir) + monkeypatch.setattr(training.os, "mkdir", racing_mkdir) with pytest.raises(training.SFTOutputPolicyError) as exc_info: training._claim_sft_output_directory( # noqa: SLF001 @@ -905,11 +907,21 @@ def get_device_name(index: int) -> str: assert index == 0 return "NVIDIA GeForce RTX 4090" + @staticmethod + def mem_get_info(index: int) -> tuple[int, int]: + assert index == 0 + return (80 * 1024**3, 80 * 1024**3) + monkeypatch.setitem( sys.modules, "torch", types.SimpleNamespace(cuda=FakeCuda(), version=types.SimpleNamespace(cuda="12.4")), ) + monkeypatch.setattr( + training.subprocess, + "run", + lambda *args, **kwargs: types.SimpleNamespace(returncode=0, stdout="", stderr=""), + ) facts, blockers = training._probe_sft_gpu() # noqa: SLF001 @@ -1269,10 +1281,7 @@ def test_sft_cli_sanitizes_private_paths_in_normal_blocked_result( captured = capsys.readouterr() result = json.loads(captured.out) assert exit_code == 1 - assert result["hyperparameters"] == { - "base_model_runtime_path": "", - "output_root": "", - } + assert "hyperparameters" not in result assert "Users/person" not in captured.out assert "mnt/data/person" not in captured.out assert captured.err == "" @@ -1402,6 +1411,12 @@ def run_real( "observed_optimizer_steps": 1, "training_rows_used": 2, "train_result_metrics": {"train_loss": 1.25}, + "trainable_parameter_count": 8, + "adapter_tensor_count": 2, + "adapter_state_digest_before": "a" * 64, + "adapter_state_digest_after": "b" * 64, + "changed_adapter_tensor_count": 1, + "all_adapter_tensors_finite": True, } ) @@ -1499,6 +1514,12 @@ def run_real( "observed_optimizer_steps": 1, "training_rows_used": 2, "train_result_metrics": {"train_loss": 1.25}, + "trainable_parameter_count": 8, + "adapter_tensor_count": 2, + "adapter_state_digest_before": "a" * 64, + "adapter_state_digest_after": "b" * 64, + "changed_adapter_tensor_count": 1, + "all_adapter_tensors_finite": True, } ) @@ -1582,7 +1603,7 @@ def test_run_sft_does_not_write_when_final_in_runner_output_check_detects_drift( result = training.run_sft(config, manifest, output_dir, dry_run=False) assert result["training_status"] == "training_blocked_by_output_policy" - assert result["blockers"] == ["OUTPUT_PATH_SYMLINK"] + assert result["blockers"] == ["OUTPUT_IDENTITY_CHANGED"] assert not output_dir.exists() @@ -1664,6 +1685,12 @@ def run_real( "training_rows_used": 2, "train_result_metrics": {"train_loss": 1.25}, "training_budget": {"configured_max_steps": 1, "observed_optimizer_steps": 1}, + "trainable_parameter_count": 8, + "adapter_tensor_count": 2, + "adapter_state_digest_before": "a" * 64, + "adapter_state_digest_after": "b" * 64, + "changed_adapter_tensor_count": 1, + "all_adapter_tensors_finite": True, } ) @@ -1676,8 +1703,20 @@ def run_real( assert result["observed_optimizer_steps"] == 1 assert result["training_rows_used"] == 2 assert result["adapter_files"] == [ - {"name": "adapter_config.json", "size": 3}, - {"name": "adapter_model.safetensors", "size": 7}, + { + "name": "adapter_config.json", + "size": 3, + "sha256": training._sha256_file( # noqa: SLF001 + output_dir / "adapter" / "adapter_config.json" + ), + }, + { + "name": "adapter_model.safetensors", + "size": 7, + "sha256": training._sha256_file( # noqa: SLF001 + output_dir / "adapter" / "adapter_model.safetensors" + ), + }, ] assert result["clean_evaluation"] == { "acquisition_source_status": "UNAVAILABLE", @@ -1749,6 +1788,7 @@ def test_real_sft_rechecks_output_before_tokenizer_or_model_loading( ) assert preflight["ready"] is True assert execution_context is not None + monkeypatch.setattr(training, "_probe_sft_gpu", lambda: ({"idle_verified": True}, [])) loads: list[str] = [] monkeypatch.setattr( training, @@ -1782,7 +1822,7 @@ def from_pretrained(*args: Any, **kwargs: Any) -> None: execution_context=execution_context, ) - assert exc_info.value.blockers == ["OUTPUT_PATH_SYMLINK"] + assert exc_info.value.blockers == ["OUTPUT_IDENTITY_CHANGED"] assert loads == [] assert not output_dir.exists() @@ -1942,6 +1982,11 @@ def test_real_sft_passes_local_bf16_options_and_bounded_training_arguments( assert preflight["ready"] is True assert execution_context is not None calls: dict[str, Any] = {"order": []} + monkeypatch.setattr( + training, + "_probe_sft_gpu", + lambda: calls["order"].append("gpu") or ({"idle_verified": True}, []), + ) fake_bfloat16 = object() class FakeTokenizer: @@ -2047,6 +2092,7 @@ def train(self) -> Any: "input_ids": [1, 2], "attention_mask": [1, 1], "labels": [-100, 2], + "assistant_token_indices": [1], }, ) monkeypatch.setattr( @@ -2066,7 +2112,7 @@ def train(self) -> Any: execution_context=execution_context, ) - assert calls["order"] == ["policy", "policy", "tokenizer", "model"] + assert calls["order"] == ["gpu", "policy", "tokenizer", "gpu", "model"] assert calls["tokenizer"] == { "path": config["base_model_runtime_path"], "local_files_only": True, @@ -2144,6 +2190,7 @@ def from_pretrained(path: str, **kwargs: Any) -> object: "input_ids": [1, 2], "attention_mask": [1, 1], "labels": [-100, 2], + "assistant_token_indices": [1], }, ) @@ -2160,6 +2207,110 @@ def from_pretrained(path: str, **kwargs: Any) -> object: assert objective["records_checked"] == 2 +def test_sft_preflight_rejects_one_prompt_label_leak_even_with_other_masked_tokens( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config, manifest, output_dir = _write_preflight_fixture(tmp_path) + actual_model_probe = training._probe_sft_model_and_objective # noqa: SLF001 + _install_ready_preflight_probes(monkeypatch) + monkeypatch.setattr(training, "_probe_sft_model_and_objective", actual_model_probe) + config_payload = json.loads(config.read_text(encoding="utf-8")) + model_root = Path(config_payload["base_model_runtime_path"]) + (model_root / "model-00001-of-00001.safetensors").open("r+b").truncate(12 * 1024**3) + + class FakeAutoConfig: + @staticmethod + def from_pretrained(path: str, **kwargs: Any) -> Any: + return types.SimpleNamespace( + model_type="qwen2", + architectures=["Qwen2ForCausalLM"], + hidden_size=3584, + intermediate_size=18944, + num_hidden_layers=28, + num_attention_heads=28, + num_key_value_heads=4, + vocab_size=152064, + ) + + class FakeTokenizer: + def __call__(self, text: str, **kwargs: Any) -> dict[str, Any]: + return { + "input_ids": [10, 11, 12, 13], + "attention_mask": [1, 1, 1, 1], + "offset_mapping": [(0, 1), (1, 2), (2, 3), (3, 4)], + } + + class FakeAutoTokenizer: + @staticmethod + def from_pretrained(path: str, **kwargs: Any) -> FakeTokenizer: + return FakeTokenizer() + + monkeypatch.setitem( + sys.modules, + "transformers", + types.SimpleNamespace(AutoConfig=FakeAutoConfig, AutoTokenizer=FakeAutoTokenizer), + ) + monkeypatch.setattr(training, "format_sft_training_text", lambda row, tokenizer: "abcd") + monkeypatch.setattr(training, "canonical_contract_json", lambda target: "cd") + monkeypatch.setattr( + training, + "_assistant_only_labels_from_encoded", + lambda **kwargs: ([-100, 11, 12, 13], []), + ) + + result = training.run_sft_preflight(config, manifest, output_dir) + + assert result["ready"] is False + assert result["blockers"] == ["ASSISTANT_ONLY_LABELS_INVALID"] + + +def test_real_sft_rechecks_gpu_idle_state_before_model_weight_load( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path, manifest, output_dir = _write_preflight_fixture(tmp_path) + _install_ready_preflight_probes(monkeypatch) + preflight, execution_context = training._run_sft_preflight_core( # noqa: SLF001 + config_path, + manifest, + output_dir, + ) + assert preflight["ready"] is True + assert execution_context is not None + monkeypatch.setattr( + training, + "_probe_sft_gpu", + lambda: ( + {"free_memory_gib": 34.0, "compute_process_count": 0, "idle_verified": False}, + ["GPU_FREE_MEMORY_INSUFFICIENT"], + ), + ) + model_loads: list[str] = [] + + class FailIfLoaded: + @staticmethod + def from_pretrained(*args: Any, **kwargs: Any) -> None: + model_loads.append("loaded") + + monkeypatch.setitem( + sys.modules, + "transformers", + types.SimpleNamespace(AutoModelForCausalLM=FailIfLoaded, AutoTokenizer=FailIfLoaded), + ) + with pytest.raises(training.SFTPreflightDriftError) as exc_info: + training._run_real_sft( # noqa: SLF001 + {"adapter_path": (output_dir / "adapter").as_posix(), "dataset_load": {}}, + execution_context.config_snapshot(), + manifest, + output_dir, + execution_context=execution_context, + ) + assert exc_info.value.blockers == ["GPU_FREE_MEMORY_INSUFFICIENT"] + assert model_loads == [] + assert not output_dir.exists() + + def test_model_probe_rejects_generic_qwen2_geometry_and_tiny_weight_inventory( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -2190,6 +2341,7 @@ def from_pretrained(path: str, **kwargs: Any) -> object: "input_ids": [1, 2], "attention_mask": [1, 1], "labels": [-100, 2], + "assistant_token_indices": [1], }, ) From f214b9be551e364742c2b4a07c341f5062dca552 Mon Sep 17 00:00:00 2001 From: Minghong Sun <118113656+Raidriar7170@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:05:18 +0800 Subject: [PATCH 3/5] Fix container-safe SFT GPU occupancy probe --- ...tainer-pid-aware-sft-gpu-occupancy-v1.html | 26 + .../.openspec.yaml | 2 + .../README.md | 3 + .../design.md | 62 ++ .../proposal.md | 30 + .../specs/supervised-contract-tuning/spec.md | 32 + .../tasks.md | 28 + .../specs/supervised-contract-tuning/spec.md | 16 +- src/voice2task/_sft_gpu_probe_helper.py | 111 ++++ src/voice2task/training.py | 273 +++++++-- tests/test_a100_sft_review_hardening.py | 84 +-- tests/test_private_a100_sft_preflight.py | 77 +-- tests/test_sft_gpu_occupancy_helper.py | 548 ++++++++++++++++++ 13 files changed, 1120 insertions(+), 172 deletions(-) create mode 100644 docs/human-briefs/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1.html create mode 100644 openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/.openspec.yaml create mode 100644 openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/README.md create mode 100644 openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/design.md create mode 100644 openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/proposal.md create mode 100644 openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/specs/supervised-contract-tuning/spec.md create mode 100644 openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/tasks.md create mode 100644 src/voice2task/_sft_gpu_probe_helper.py create mode 100644 tests/test_sft_gpu_occupancy_helper.py diff --git a/docs/human-briefs/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1.html b/docs/human-briefs/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1.html new file mode 100644 index 0000000..b5cdc6c --- /dev/null +++ b/docs/human-briefs/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1.html @@ -0,0 +1,26 @@ + + + + + + Container PID-aware SFT GPU Occupancy · 人类简报 + + + +

OpenSpec · fix-container-pid-aware-sft-gpu-occupancy-v1

+

让空闲 GPU 检查不再依赖容器 PID 映射

+

LOCAL_IMPLEMENTATION · FOCUSED_GREEN · REMOTE_NOT_RUN

+

父进程先确认显式选择的 GPU 没有 compute process，再以 sys.executable -I、绝对 helper 脚本和固定最小环境启动短生命周期内部 Python helper，采集 CUDA、A100、BF16、总显存和可用显存事实。helper 不继承 proxy、token、PYTHONPATH 或动态加载注入变量，本身不预建 socket，并在导入 torch 前安装阻断 CPython 真实 socket creation 与网络事件的 audit hook。helper 退出并释放自己的 CUDA context 后，父进程再次确认 occupancy 为空。

+

每次 shared preflight 都完整重复 occupancy → helper → occupancy，不推断、不学习、不缓存或公开任何 PID。任一已确认占用返回 GPU_BUSY；采样、helper timeout/nonzero 或协议异常返回 GPU_OCCUPANCY_PROBE_FAILED。

+

该合同只证明 pre/post 两个 occupancy 样本为空，并记录 helper 期间的可用显存；不声称两个样本之间持续绝对没有外部进程。

+

torch 导入失败或 CUDA 不可用返回 CUDA_UNAVAILABLE；CUDA API 异常返回 CUDA_PROBE_FAILED。helper 只输出一个严格 typed JSON，所有失败均不包含原始异常或私有运行值。

+

本阶段没有运行真实训练、完整 SFT、DPO、GRPO、prediction 或 evaluation；没有访问 lockbox，没有远端操作，也没有模型改进、clean-evaluation ready 或 production-ready 声明。

+

clean-evaluation 七项事实保持阻断，execution_readiness=false。

+ + diff --git a/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/.openspec.yaml b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/.openspec.yaml new file mode 100644 index 0000000..64105fc --- /dev/null +++ b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-14 diff --git a/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/README.md b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/README.md new file mode 100644 index 0000000..5b714cc --- /dev/null +++ b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/README.md @@ -0,0 +1,3 @@ +# fix-container-pid-aware-sft-gpu-occupancy-v1 + +Fail-closed container PID namespace handling for shared A100 SFT GPU occupancy probes. diff --git a/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/design.md b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/design.md new file mode 100644 index 0000000..449ac56 --- /dev/null +++ b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/design.md @@ -0,0 +1,62 @@ +## Context + +The shared SFT preflight checks aggregate GPU compute occupancy after calling CUDA APIs such as `torch.cuda.mem_get_info(0)`. In the authorized A100 container, `nvidia-smi` reports the host PID for the probe's newly-created CUDA context while Python exposes a container-local PID, so filtering only `os.getpid()` produces a false `GPU_BUSY`. The gate must require empty occupancy at its two defined sample points and sufficient free memory during the helper probe, and must never expose process identity. It does not prove continuous absence of external processes between samples. + +## Goals / Non-Goals + +**Goals:** + +- Avoid treating the ephemeral helper's released CUDA context as persistent occupancy across host/container PID namespaces. +- Keep both occupancy samples and the helper bound to one explicit `CUDA_VISIBLE_DEVICES` selector on every repeated shared preflight. +- Fail closed on occupancy observed in either sample or on occupancy/helper probe failure. +- Preserve public facts as counts and booleans only. + +**Non-Goals:** + +- Relaxing the requirement for one explicit A100, at least 35 GiB total memory, two empty occupancy samples, and at least 35 GiB helper-period free memory. +- Trusting process names, usernames, GPU UUIDs, commands, or caller-provided PID mappings. +- Changing training budgets, datasets, models, clean-evaluation facts, or any evaluation path. +- Starting full training, DPO, GRPO, prediction, or evaluation. +- Claiming that sampling proves continuous absence of an external process between the two sample points. + +## Decisions + +### Isolate CUDA facts in an ephemeral helper process + +The parent process samples aggregate compute occupancy before launching a dedicated internal helper. If any process is present, it returns `GPU_BUSY` without initializing CUDA. The helper imports torch, collects the existing availability, device, BF16, total-memory, and `mem_get_info(0)` facts, emits exactly one strict typed JSON response to its parent, and exits. A torch import failure or `cuda.is_available() == false` reports `CUDA_UNAVAILABLE`; an exception from a CUDA API reports `CUDA_PROBE_FAILED`; helper timeout, non-zero exit, malformed JSON, or protocol drift reports `GPU_OCCUPANCY_PROBE_FAILED`. Raw exception text is never serialized. Only after the helper has exited and released its CUDA context does the parent sample occupancy again. Both samples must be valid and empty. This sequence is used on every shared-preflight call, including the check immediately before model loading, so the long-lived training process does not acquire a CUDA context merely to inspect the gate. + +Alternative considered: compare with `os.getpid()` or `/proc/self/status` namespace PIDs. The authorized container demonstrated that neither value matches the NVML PID, so this is not portable enough. Alternative considered: infer that the unique PID appearing after CUDA initialization belongs to the probe and cache it. That can accidentally whitelist an unrelated process observed at that instant, so no identity is inferred or cached. Alternative considered: remove the occupancy check, accept one arbitrary PID, or require a host PID namespace deployment. These either weaken the two-sample gate or are unavailable in the authorized runtime. + +### Keep process identity private + +The public GPU object continues to expose only `compute_process_count` and `idle_verified`. Raw occupancy output and helper protocol details never enter blockers, logs, metadata, or CLI JSON. + +### Launch the helper in an isolated offline environment + +The parent invokes the helper as `sys.executable -I ` with a newly constructed allowlisted environment. It passes only the exact `CUDA_VISIBLE_DEVICES` selector, user-site isolation, offline flags, and fixed locale; it does not copy caller proxy, token, `PYTHONPATH`, `PYTHONHOME`, site-customization, or dynamic-loader injection variables. Before importing torch, the helper installs a Python audit hook that rejects real CPython socket creation plus connect, bind, address-resolution, and send audit events. The helper does not pre-create a socket, so listen, connect, connect-ex, send-to, and send-message operations cannot acquire a socket before this boundary. A blocked network event becomes only the typed `CUDA_PROBE_FAILED` result. + +### Isolate dependency repair from repository behavior + +Runtime dependency conflicts are repaired only in a dedicated ignored venv under the authorized project directory. No system/shared Python packages are changed, and no dependency artifact is committed. + +## Risks / Trade-offs + +- [An external process runs only while the helper is alive and exits before the post-sample] -> Preserve the free-memory probe inside that interval and repeat the complete pre/helper/post gate immediately before model loading. Sampling cannot prove absence between observations, but no external identity is ever trusted or whitelisted. +- [The helper emits malformed data, times out, or exits non-zero] -> Return only `GPU_OCCUPANCY_PROBE_FAILED`; do not fall back to an in-process CUDA probe. +- [The helper cannot import torch or a CUDA API raises] -> Preserve the typed distinction as `CUDA_UNAVAILABLE` or `CUDA_PROBE_FAILED`, respectively, without exception text. +- [Caller environment attempts Python/module shadowing, credential propagation, proxy access, or loader injection] -> Use isolated execution, an absolute script, a minimal environment allowlist, and the pre-import network audit hook. +- [The explicit selector changes] -> Pass the exact single selector to both occupancy samples and the helper; validate the helper's visible-device count remains one. + +## Migration Plan + +1. Add red regression tests for isolated helper ordering, repeat probes, external occupancy, malformed helper results, and selector propagation. +2. Implement the minimal private helper protocol and parent-side fail-closed before/after sampling. +3. Run focused and full repository verification, sync the delta spec, and archive the change. +4. Push the new PR head, deploy that exact commit to the private A100 checkout, and run shared preflight. +5. Run exactly one bounded smoke only if preflight returns exit code 0, `ready=true`, and no blockers. + +Rollback is the single implementation commit; removing it restores the previous conservative false blocker and cannot start training. + +## Open Questions + +None. A real smoke remains conditional on all shared preflight gates. diff --git a/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/proposal.md b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/proposal.md new file mode 100644 index 0000000..d06bc84 --- /dev/null +++ b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/proposal.md @@ -0,0 +1,30 @@ +## Why + +The shared A100 preflight currently compares NVML host PIDs with the container-local `os.getpid()`. After the probe creates its own CUDA context, that namespace mismatch can misclassify the probe itself as an external compute process and block a GPU whose required pre-probe occupancy sample was empty with `GPU_BUSY`. + +## What Changes + +- Avoid a host/container PID comparison by collecting CUDA facts in an ephemeral helper between two occupancy samples on the explicitly selected GPU. +- Require both occupancy samples to be valid and empty, and require the helper-period free-memory threshold; fail closed when any observed sample or probe fails. +- Keep PIDs and all other process identity details out of public preflight facts and CLI output. +- Add regression coverage for repeated preflight, external occupancy, ambiguous races, and GPU-selection isolation. +- Keep all existing A100, BF16, total/free-memory, explicit-selection, and real-smoke gates unchanged. +- Do not claim continuous absence of external processes between the two occupancy samples; the contract is the two samples plus helper-period free memory. +- Do not run full training, DPO, GRPO, prediction, evaluation, or make model-improvement claims. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `supervised-contract-tuning`: Strengthen the shared SFT GPU occupancy contract for container PID namespaces while preserving fail-closed external occupancy detection. + +## Impact + +- Affected implementation: `src/voice2task/training.py` shared SFT GPU preflight. +- Affected tests: private A100 preflight and review-hardening GPU occupancy tests. +- Affected specification: `supervised-contract-tuning`. +- No public API expansion, dependency change, dataset change, or evaluation-readiness change. diff --git a/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/specs/supervised-contract-tuning/spec.md b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/specs/supervised-contract-tuning/spec.md new file mode 100644 index 0000000..3afb557 --- /dev/null +++ b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/specs/supervised-contract-tuning/spec.md @@ -0,0 +1,32 @@ +## MODIFIED Requirements + +### Requirement: Provide one shared SFT preflight +The system MUST expose `sft-preflight` and MUST use the same internal preflight core from real SFT execution before loading model weights. That core MUST produce the public report plus an immutable private execution context binding canonical inputs, selected rows, model inventory, and output facts. Real training MUST consume the bound context rather than reselecting data from caller arguments and MUST rehash mutable inputs before model loading. The single public JSON result MUST use schema `voice2task-sft-preflight-v1`, report `ready`, `status`, `blockers`, and the `git`, `config`, `dataset`, `model`, `runtime`, `gpu`, `output`, and `objective` sections, and MUST not access the network. In a container PID namespace, the GPU hardware probe MUST run in an ephemeral helper process between two parent occupancy samples so the helper releases its own CUDA context before the second sample; both parent samples MUST be valid and contain zero compute processes, the helper-period free-memory threshold MUST pass, and the system MUST NOT infer or serialize any process identity. This sampling contract MUST NOT be represented as proof that no external process existed continuously between the two samples. + +#### Scenario: Report a ready bounded smoke +- **WHEN** Git tracked state is clean and includes the change, the ignored non-symlink private config is exactly smoke-bounded, dependencies and `pip check` pass, one explicit BF16-capable A100 has at least 35 GiB total memory, both defined occupancy samples contain zero compute processes, the helper observes at least 35 GiB free memory, exact local Qwen2.5-7B model/tokenizer identity and >=12 GiB weight inventory pass, selected formal train rows and assistant-only labels pass, and output policy passes +- **THEN** preflight returns exit code 0 with `ready=true`, an empty blocker list, stable fingerprints, hashes, versions, counts, and public-safe facts + +#### Scenario: Isolate the probe context across PID namespaces +- **WHEN** the selected GPU has no compute process before the helper starts, the helper successfully collects CUDA facts and exits, and the selected GPU again has no compute process after the helper exits +- **THEN** the shared probe reports `compute_process_count=0` and `idle_verified=true` without comparing, inferring, caching, or emitting any PID + +#### Scenario: Preserve typed CUDA failures and fail closed on helper uncertainty +- **WHEN** occupancy exists in either parent sample, an occupancy sample fails, torch cannot be imported, a CUDA API raises, or the helper times out, exits non-zero, or emits an invalid result +- **THEN** preflight returns non-zero with `GPU_BUSY` for confirmed occupancy, `CUDA_UNAVAILABLE` for torch import or unavailable CUDA, `CUDA_PROBE_FAILED` for a CUDA API failure, or `GPU_OCCUPANCY_PROBE_FAILED` for occupancy/helper process or protocol uncertainty, without exposing process identities or raw error text + +#### Scenario: Isolate the helper from caller environment and network +- **WHEN** the parent starts the ephemeral CUDA helper +- **THEN** it uses `sys.executable -I` with the absolute helper script and a new allowlisted environment containing only the exact selector, user-site isolation, offline flags, and fixed locale, and before importing torch the socket-free helper installs a Python audit hook that blocks real socket creation and network audit events + +#### Scenario: Report blocked readiness safely +- **WHEN** any required Git, config, dependency, GPU, model, dataset, objective, or output condition fails +- **THEN** preflight returns non-zero with `ready=false` and only stable enumerated blocker codes, including `GPU_FREE_MEMORY_INSUFFICIENT`, `GPU_BUSY`, or `GPU_OCCUPANCY_PROBE_FAILED` for the corresponding GPU state, without raw exceptions or private runtime values + +#### Scenario: Fail closed on an unexpected preflight exception +- **WHEN** any internal shared-preflight operation raises an unexpected exception +- **THEN** both `sft-preflight` and real SFT return the complete blocked preflight schema with only `PREFLIGHT_INTERNAL_ERROR`, do not serialize exception text or private values, and do not write the candidate output + +#### Scenario: Consume only the bound ready context +- **WHEN** real SFT reaches a ready shared preflight result +- **THEN** it builds execution metadata only from the bound report and immutable context, without rereading legacy config, manifest, or dataset inputs around the gate, and binds the exact model facts produced by the validated model probe diff --git a/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/tasks.md b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/tasks.md new file mode 100644 index 0000000..5136a42 --- /dev/null +++ b/openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1/tasks.md @@ -0,0 +1,28 @@ +## 1. Regression Contract + +- [x] 1.1 Add red tests for helper isolation, pre/post occupancy ordering, repeated probes, and exact selector propagation. +- [x] 1.2 Add red tests that fail closed for pre-existing/post-helper occupancy, helper timeout/failure/malformed output, and never expose process identity. +- [x] 1.3 Add reviewer-requested red tests for typed CUDA failures, strict helper JSON, isolated absolute-script launch, minimal environment, direct fake-torch collection, and the pre-import network audit hook. +- [x] 1.4 Add a second-review red test that installs the real hook only in an isolated child and verifies actual socket creation, listen, connect, connect-ex, send-to, send-message, and DNS APIs cannot proceed; add cross-field protocol cases. + +## 2. Minimal Implementation + +- [x] 2.1 Implement the private ephemeral GPU fact helper without network access or public path/process disclosure. +- [x] 2.2 Replace in-process CUDA occupancy filtering with parent pre-sample, helper execution, and parent post-sample while preserving all existing A100/BF16/memory gates. +- [x] 2.3 Generate the required Chinese Human Brief with exact training and evaluation non-goals. +- [x] 2.4 Preserve `CUDA_UNAVAILABLE` for torch import/unavailable CUDA and `CUDA_PROBE_FAILED` for CUDA API exceptions while keeping helper process/protocol uncertainty as `GPU_OCCUPANCY_PROBE_FAILED`. +- [x] 2.5 Run the helper with `sys.executable -I`, an absolute script, a fixed minimal offline environment, and a network-denying audit hook installed before torch import. +- [x] 2.6 Block real CPython `socket.__new__` before torch import and narrow all readiness wording to the two empty occupancy samples plus helper-period free memory. + +## 3. Verification And Lifecycle + +- [x] 3.1 Run focused GPU/preflight tests and targeted mypy on touched Python files. + - RED: the original helper/orchestration suite initially failed `15/15`; the strengthened post-helper sample assertion then failed `2/2`; first-review hardening initially failed `16/30`; the real-socket second review failed `2/38` before `socket.__new__` was blocked. + - GREEN: focused GPU/preflight/A100 regression passed `241/241`; touched Ruff and `git diff --check` passed. + - Targeted Mypy reported only the two pre-existing imported-module diagnostics in `slot_error_analysis.py:574` and `copy_backed_shadow_interface.py:357`; neither touched Python file had a diagnostic. +- [x] 3.2 Run pre-archive full pytest, ruff, strict OpenSpec validation, truth-surface checker, and `git diff --check`. + - Pre-archive full pytest passed `1424/1428`; the four expected failures and truth-surface exit `1` were solely the repository's active-change-empty boundary. Ruff, strict validation, and `git diff --check` passed; no allowlist was added. +- [x] 3.3 Sync the delta to `supervised-contract-tuning`, archive the change, confirm active changes=0, then rerun full verification. + - Archived at `openspec/changes/archive/2026-07-14-fix-container-pid-aware-sft-gpu-occupancy-v1`; active changes are `0`. + - Post-archive verification: full pytest `1451 passed`; Ruff, strict OpenSpec validation, truth-surface checker, and `git diff --check` passed. + - Exact targeted Mypy reported only the two pre-existing imported-module diagnostics in `slot_error_analysis.py:574` and `copy_backed_shadow_interface.py:357`; no touched Python file had a diagnostic. diff --git a/openspec/specs/supervised-contract-tuning/spec.md b/openspec/specs/supervised-contract-tuning/spec.md index 59aca7b..3930b92 100644 --- a/openspec/specs/supervised-contract-tuning/spec.md +++ b/openspec/specs/supervised-contract-tuning/spec.md @@ -80,12 +80,24 @@ This contract defends output drift, symlink substitution, inode exchange, and co - **THEN** that deployment is outside this claim contract and MUST prevent that adversarial namespace mutation before authorizing a real smoke ### Requirement: Provide one shared SFT preflight -The system MUST expose `sft-preflight` and MUST use the same internal preflight core from real SFT execution before loading model weights. That core MUST produce the public report plus an immutable private execution context binding canonical inputs, selected rows, model inventory, and output facts. Real training MUST consume the bound context rather than reselecting data from caller arguments and MUST rehash mutable inputs before model loading. The single public JSON result MUST use schema `voice2task-sft-preflight-v1`, report `ready`, `status`, `blockers`, and the `git`, `config`, `dataset`, `model`, `runtime`, `gpu`, `output`, and `objective` sections, and MUST not access the network. +The system MUST expose `sft-preflight` and MUST use the same internal preflight core from real SFT execution before loading model weights. That core MUST produce the public report plus an immutable private execution context binding canonical inputs, selected rows, model inventory, and output facts. Real training MUST consume the bound context rather than reselecting data from caller arguments and MUST rehash mutable inputs before model loading. The single public JSON result MUST use schema `voice2task-sft-preflight-v1`, report `ready`, `status`, `blockers`, and the `git`, `config`, `dataset`, `model`, `runtime`, `gpu`, `output`, and `objective` sections, and MUST not access the network. In a container PID namespace, the GPU hardware probe MUST run in an ephemeral helper process between two parent occupancy samples so the helper releases its own CUDA context before the second sample; both parent samples MUST be valid and contain zero compute processes, the helper-period free-memory threshold MUST pass, and the system MUST NOT infer or serialize any process identity. This sampling contract MUST NOT be represented as proof that no external process existed continuously between the two samples. #### Scenario: Report a ready bounded smoke -- **WHEN** Git tracked state is clean and includes the change, the ignored non-symlink private config is exactly smoke-bounded, dependencies and `pip check` pass, one explicit idle BF16-capable A100 has at least 35 GiB total and free memory and zero compute processes, exact local Qwen2.5-7B model/tokenizer identity and >=12 GiB weight inventory pass, selected formal train rows and assistant-only labels pass, and output policy passes +- **WHEN** Git tracked state is clean and includes the change, the ignored non-symlink private config is exactly smoke-bounded, dependencies and `pip check` pass, one explicit BF16-capable A100 has at least 35 GiB total memory, both defined occupancy samples contain zero compute processes, the helper observes at least 35 GiB free memory, exact local Qwen2.5-7B model/tokenizer identity and >=12 GiB weight inventory pass, selected formal train rows and assistant-only labels pass, and output policy passes - **THEN** preflight returns exit code 0 with `ready=true`, an empty blocker list, stable fingerprints, hashes, versions, counts, and public-safe facts +#### Scenario: Isolate the probe context across PID namespaces +- **WHEN** the selected GPU has no compute process before the helper starts, the helper successfully collects CUDA facts and exits, and the selected GPU again has no compute process after the helper exits +- **THEN** the shared probe reports `compute_process_count=0` and `idle_verified=true` without comparing, inferring, caching, or emitting any PID + +#### Scenario: Preserve typed CUDA failures and fail closed on helper uncertainty +- **WHEN** occupancy exists in either parent sample, an occupancy sample fails, torch cannot be imported, a CUDA API raises, or the helper times out, exits non-zero, or emits an invalid result +- **THEN** preflight returns non-zero with `GPU_BUSY` for confirmed occupancy, `CUDA_UNAVAILABLE` for torch import or unavailable CUDA, `CUDA_PROBE_FAILED` for a CUDA API failure, or `GPU_OCCUPANCY_PROBE_FAILED` for occupancy/helper process or protocol uncertainty, without exposing process identities or raw error text + +#### Scenario: Isolate the helper from caller environment and network +- **WHEN** the parent starts the ephemeral CUDA helper +- **THEN** it uses `sys.executable -I` with the absolute helper script and a new allowlisted environment containing only the exact selector, user-site isolation, offline flags, and fixed locale, and before importing torch the socket-free helper installs a Python audit hook that blocks real socket creation and network audit events + #### Scenario: Report blocked readiness safely - **WHEN** any required Git, config, dependency, GPU, model, dataset, objective, or output condition fails - **THEN** preflight returns non-zero with `ready=false` and only stable enumerated blocker codes, including `GPU_FREE_MEMORY_INSUFFICIENT`, `GPU_BUSY`, or `GPU_OCCUPANCY_PROBE_FAILED` for the corresponding GPU state, without raw exceptions or private runtime values diff --git a/src/voice2task/_sft_gpu_probe_helper.py b/src/voice2task/_sft_gpu_probe_helper.py new file mode 100644 index 0000000..46f5713 --- /dev/null +++ b/src/voice2task/_sft_gpu_probe_helper.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import importlib +import json +import sys +from typing import Any + +SCHEMA_VERSION = "voice2task-sft-gpu-helper-v2" +_NETWORK_AUDIT_EVENTS = { + "socket.__new__", + "socket.bind", + "socket.connect", + "socket.getaddrinfo", + "socket.gethostbyaddr", + "socket.gethostbyname", + "socket.gethostbyname_ex", + "socket.sendto", +} + + +def _empty_gpu_facts( + *, + cuda_available: bool = False, + visible_device_count: int = 0, + cuda_version: str = "unknown", +) -> dict[str, Any]: + return { + "cuda_available": cuda_available, + "visible_device_count": visible_device_count, + "name": None, + "compute_capability": None, + "total_memory_gib": None, + "free_memory_gib": None, + "cuda_version": cuda_version, + "bf16_supported": False, + } + + +def _result(status: str, facts: dict[str, Any]) -> dict[str, Any]: + return {"schema_version": SCHEMA_VERSION, "status": status, **facts} + + +def _network_audit_hook(event: str, args: tuple[Any, ...]) -> None: + del args + if event in _NETWORK_AUDIT_EVENTS: + raise PermissionError("NETWORK_ACCESS_BLOCKED") + + +def collect_gpu_result(torch_module: Any | None = None) -> dict[str, Any]: + if torch_module is None: + try: + torch_module = importlib.import_module("torch") + except PermissionError: + return _result("CUDA_PROBE_FAILED", _empty_gpu_facts()) + except Exception: + return _result("CUDA_UNAVAILABLE", _empty_gpu_facts()) + + try: + cuda = torch_module.cuda + cuda_version = str(getattr(torch_module.version, "cuda", None) or "unknown") + if not bool(cuda.is_available()): + return _result( + "CUDA_UNAVAILABLE", + _empty_gpu_facts(cuda_version=cuda_version), + ) + visible_device_count = int(cuda.device_count()) + if visible_device_count != 1: + return _result( + "GPU_SELECTION_NOT_SINGLE", + _empty_gpu_facts( + cuda_available=True, + visible_device_count=visible_device_count, + cuda_version=cuda_version, + ), + ) + properties = cuda.get_device_properties(0) + capability = tuple(int(value) for value in cuda.get_device_capability(0)) + free_memory_bytes, _ = cuda.mem_get_info(0) + facts = { + "cuda_available": True, + "visible_device_count": 1, + "name": str(cuda.get_device_name(0)), + "compute_capability": f"{capability[0]}.{capability[1]}", + "total_memory_gib": round( + int(getattr(properties, "total_memory", 0)) / float(1024**3), + 3, + ), + "free_memory_gib": round(int(free_memory_bytes) / float(1024**3), 3), + "cuda_version": cuda_version, + "bf16_supported": bool(cuda.is_bf16_supported()), + } + except Exception: + return _result("CUDA_PROBE_FAILED", _empty_gpu_facts()) + return _result("OK", facts) + + +def main_result() -> dict[str, Any]: + try: + sys.addaudithook(_network_audit_hook) + except Exception: + return _result("CUDA_PROBE_FAILED", _empty_gpu_facts()) + return collect_gpu_result() + + +def main() -> int: + print(json.dumps(main_result(), ensure_ascii=True, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/voice2task/training.py b/src/voice2task/training.py index edd5312..559a74b 100644 --- a/src/voice2task/training.py +++ b/src/voice2task/training.py @@ -1237,11 +1237,28 @@ def _probe_sft_dependencies() -> tuple[dict[str, Any], list[str]]: return facts, list(dict.fromkeys(blockers)) -def _probe_sft_gpu() -> tuple[dict[str, Any], list[str]]: - selected = os.environ.get("CUDA_VISIBLE_DEVICES") - selected_parts = [part.strip() for part in selected.split(",") if part.strip()] if selected else [] - facts: dict[str, Any] = { - "explicit_selection": bool(selected_parts), +SFT_GPU_HELPER_SCHEMA_VERSION = "voice2task-sft-gpu-helper-v2" +SFT_GPU_HELPER_STATUSES = { + "OK", + "CUDA_UNAVAILABLE", + "CUDA_PROBE_FAILED", + "GPU_SELECTION_NOT_SINGLE", +} +SFT_GPU_HELPER_FACT_FIELDS = { + "cuda_available", + "visible_device_count", + "name", + "compute_capability", + "total_memory_gib", + "free_memory_gib", + "cuda_version", + "bf16_supported", +} + + +def _empty_sft_gpu_facts(*, explicit_selection: bool) -> dict[str, Any]: + return { + "explicit_selection": explicit_selection, "visible_device_count": 0, "name": None, "compute_capability": None, @@ -1254,48 +1271,211 @@ def _probe_sft_gpu() -> tuple[dict[str, Any], list[str]]: "cuda_version": None, "bf16_supported": False, } + + +def _sample_sft_gpu_compute_process_count(selector: str) -> int: + try: + occupancy = subprocess.run( + [ + "nvidia-smi", + f"--id={selector}", + "--query-compute-apps=pid", + "--format=csv,noheader,nounits", + ], + check=False, + capture_output=True, + text=True, + timeout=15, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") from exc + if occupancy.returncode != 0: + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + process_count = 0 + for line in occupancy.stdout.splitlines(): + value = line.strip() + if not value: + continue + if not value.isdigit(): + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + process_count += 1 + return process_count + + +def _validated_sft_gpu_helper_result(value: Any) -> dict[str, Any]: + expected_fields = {*SFT_GPU_HELPER_FACT_FIELDS, "status"} + if not isinstance(value, dict) or set(value) != expected_fields: + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + status = value.get("status") + cuda_available = value.get("cuda_available") + visible_device_count = value.get("visible_device_count") + bf16_supported = value.get("bf16_supported") + cuda_version = value.get("cuda_version") + if ( + not isinstance(status, str) + or status not in SFT_GPU_HELPER_STATUSES + or not isinstance(cuda_available, bool) + or type(visible_device_count) is not int + or visible_device_count < 0 + or not isinstance(bf16_supported, bool) + or not isinstance(cuda_version, str) + or not re.fullmatch(r"(?:\d+(?:\.\d+)+|unknown)", cuda_version) + ): + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + for key in ("total_memory_gib", "free_memory_gib"): + number = value.get(key) + if number is not None and ( + isinstance(number, bool) + or not isinstance(number, int | float) + or not math.isfinite(float(number)) + or float(number) < 0 + ): + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + name = value.get("name") + capability = value.get("compute_capability") + if name is not None and ( + not isinstance(name, str) or re.fullmatch(r"[A-Za-z0-9 ._()+-]{1,100}", name) is None + ): + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + if capability is not None and ( + not isinstance(capability, str) or re.fullmatch(r"\d+\.\d+", capability) is None + ): + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + device_fact_values = ( + name, + capability, + value.get("total_memory_gib"), + value.get("free_memory_gib"), + ) + if status == "OK": + if not cuda_available or visible_device_count != 1 or any( + item is None for item in device_fact_values + ): + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + elif status == "GPU_SELECTION_NOT_SINGLE": + if ( + not cuda_available + or visible_device_count == 1 + or any(item is not None for item in device_fact_values) + or bf16_supported + ): + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + elif ( + cuda_available + or visible_device_count != 0 + or any(item is not None for item in device_fact_values) + or bf16_supported + ): + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + return {"status": status, **{key: value[key] for key in SFT_GPU_HELPER_FACT_FIELDS}} + + +def _run_sft_gpu_fact_helper(selector: str) -> dict[str, Any]: + environment = { + "CUDA_VISIBLE_DEVICES": selector, + "PYTHONNOUSERSITE": "1", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "HF_DATASETS_OFFLINE": "1", + "LC_ALL": "C", + "LANG": "C", + } + try: + python_executable = Path(sys.executable).expanduser() + if not python_executable.is_absolute() or not python_executable.is_file(): + raise OSError + helper_path = Path(__file__).with_name("_sft_gpu_probe_helper.py").resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") from exc + try: + completed = subprocess.run( + [python_executable.as_posix(), "-I", helper_path.as_posix()], + check=False, + capture_output=True, + text=True, + timeout=30, + env=environment, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") from exc + if completed.returncode != 0: + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + try: + payload = json.loads(completed.stdout) + except (TypeError, json.JSONDecodeError) as exc: + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") from exc + if not isinstance(payload, dict) or payload.get("schema_version") != SFT_GPU_HELPER_SCHEMA_VERSION: + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + if set(payload) != {*SFT_GPU_HELPER_FACT_FIELDS, "schema_version", "status"}: + raise RuntimeError("GPU_OCCUPANCY_PROBE_FAILED") + return _validated_sft_gpu_helper_result( + { + "status": payload["status"], + **{key: payload[key] for key in SFT_GPU_HELPER_FACT_FIELDS}, + } + ) + + +def _probe_sft_gpu() -> tuple[dict[str, Any], list[str]]: + selected = os.environ.get("CUDA_VISIBLE_DEVICES") + selected_parts = [part.strip() for part in selected.split(",") if part.strip()] if selected else [] + facts = _empty_sft_gpu_facts(explicit_selection=bool(selected_parts)) blockers: list[str] = [] if len(selected_parts) != 1 or selected_parts == ["-1"]: blockers.append("GPU_SELECTION_NOT_EXPLICIT" if not selected_parts else "GPU_SELECTION_NOT_SINGLE") + return facts, blockers + selector = selected_parts[0] try: - torch = importlib.import_module("torch") + pre_count = _sample_sft_gpu_compute_process_count(selector) except Exception: - return facts, list(dict.fromkeys([*blockers, "CUDA_UNAVAILABLE"])) - cuda = getattr(torch, "cuda", None) - if cuda is None: - return facts, list(dict.fromkeys([*blockers, "CUDA_UNAVAILABLE"])) + return facts, ["GPU_OCCUPANCY_PROBE_FAILED"] + facts["compute_process_count"] = pre_count + if pre_count: + return facts, ["GPU_BUSY"] + helper_result: dict[str, Any] | None = None + helper_failed = False try: - cuda_available = bool(cuda.is_available()) + helper_result = _validated_sft_gpu_helper_result(_run_sft_gpu_fact_helper(selector)) except Exception: - return facts, list(dict.fromkeys([*blockers, "CUDA_PROBE_FAILED"])) - if not cuda_available: - return facts, list(dict.fromkeys([*blockers, "CUDA_UNAVAILABLE"])) + helper_failed = True try: - visible_count = int(cuda.device_count()) + post_count = _sample_sft_gpu_compute_process_count(selector) except Exception: - return facts, list(dict.fromkeys([*blockers, "CUDA_PROBE_FAILED"])) + return facts, ["GPU_OCCUPANCY_PROBE_FAILED"] + facts["compute_process_count"] = post_count + if post_count: + return facts, ["GPU_BUSY"] + if helper_failed or helper_result is None: + return facts, ["GPU_OCCUPANCY_PROBE_FAILED"] + facts["idle_verified"] = True + helper_status = str(helper_result["status"]) + cuda_available = bool(helper_result["cuda_available"]) + visible_count = int(helper_result["visible_device_count"]) facts["visible_device_count"] = visible_count - if visible_count != 1: - blockers.append("GPU_SELECTION_NOT_SINGLE") - return facts, list(dict.fromkeys(blockers)) - + if helper_status == "CUDA_UNAVAILABLE": + return facts, ["CUDA_UNAVAILABLE"] + if helper_status == "CUDA_PROBE_FAILED": + return facts, ["CUDA_PROBE_FAILED"] + if helper_status == "GPU_SELECTION_NOT_SINGLE": + return facts, ["GPU_SELECTION_NOT_SINGLE"] + if helper_status != "OK" or not cuda_available or visible_count != 1: + return facts, ["GPU_OCCUPANCY_PROBE_FAILED"] + gpu_name = str(helper_result["name"]) + capability_text = str(helper_result["compute_capability"]) try: - properties = cuda.get_device_properties(0) - capability = tuple(int(value) for value in cuda.get_device_capability(0)) - total_memory_gib = int(getattr(properties, "total_memory", 0)) / float(1024**3) - free_memory_bytes, _ = cuda.mem_get_info(0) - free_memory_gib = int(free_memory_bytes) / float(1024**3) - bf16_supported = bool(cuda.is_bf16_supported()) - gpu_name = str(cuda.get_device_name(0)) - except Exception: - return facts, list(dict.fromkeys([*blockers, "CUDA_PROBE_FAILED"])) + capability = tuple(int(value) for value in capability_text.split(".")) + total_memory_gib = float(helper_result["total_memory_gib"]) + free_memory_gib = float(helper_result["free_memory_gib"]) + except (TypeError, ValueError): + return facts, ["GPU_OCCUPANCY_PROBE_FAILED"] + bf16_supported = bool(helper_result["bf16_supported"]) facts.update( { "name": gpu_name, - "compute_capability": f"{capability[0]}.{capability[1]}", + "compute_capability": capability_text, "total_memory_gib": round(total_memory_gib, 3), "free_memory_gib": round(free_memory_gib, 3), - "cuda_version": str(getattr(getattr(torch, "version", None), "cuda", None) or "unknown"), + "cuda_version": str(helper_result["cuda_version"]), "bf16_supported": bf16_supported, } ) @@ -1307,37 +1487,6 @@ def _probe_sft_gpu() -> tuple[dict[str, Any], list[str]]: blockers.append("GPU_FREE_MEMORY_INSUFFICIENT") if re.search(r"\bA100\b", gpu_name, flags=re.IGNORECASE) is None: blockers.append("GPU_NOT_A100") - try: - occupancy = subprocess.run( - [ - "nvidia-smi", - f"--id={selected_parts[0]}", - "--query-compute-apps=pid", - "--format=csv,noheader,nounits", - ], - check=False, - capture_output=True, - text=True, - timeout=15, - ) - if occupancy.returncode != 0: - raise RuntimeError("occupancy probe failed") - process_ids: set[int] = set() - for line in occupancy.stdout.splitlines(): - value = line.strip() - if not value: - continue - if not value.isdigit(): - raise RuntimeError("occupancy output invalid") - process_id = int(value) - if process_id != os.getpid(): - process_ids.add(process_id) - facts["compute_process_count"] = len(process_ids) - facts["idle_verified"] = not process_ids - if process_ids: - blockers.append("GPU_BUSY") - except (OSError, RuntimeError, subprocess.SubprocessError): - blockers.append("GPU_OCCUPANCY_PROBE_FAILED") return facts, list(dict.fromkeys(blockers)) diff --git a/tests/test_a100_sft_review_hardening.py b/tests/test_a100_sft_review_hardening.py index 2a03737..4ec314f 100644 --- a/tests/test_a100_sft_review_hardening.py +++ b/tests/test_a100_sft_review_hardening.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import sys import types from pathlib import Path from typing import Any @@ -327,51 +326,23 @@ def open_then_move(*args: Any, **kwargs: Any) -> tuple[int, int]: assert not (moved_parent / "smoke").exists() -def _install_fake_torch(monkeypatch: pytest.MonkeyPatch, *, free_gib: float = 80.0) -> None: - class FakeCuda: - @staticmethod - def is_available() -> bool: - return True - - @staticmethod - def device_count() -> int: - return 1 - - @staticmethod - def get_device_properties(index: int) -> Any: - return types.SimpleNamespace(total_memory=80 * 1024**3) - - @staticmethod - def get_device_capability(index: int) -> tuple[int, int]: - return (8, 0) - - @staticmethod - def is_bf16_supported() -> bool: - return True - - @staticmethod - def get_device_name(index: int) -> str: - return "NVIDIA A100-SXM4-80GB" - - @staticmethod - def mem_get_info(index: int) -> tuple[int, int]: - assert index == 0 - return (int(free_gib * 1024**3), 80 * 1024**3) - - monkeypatch.setitem( - sys.modules, - "torch", - types.SimpleNamespace(cuda=FakeCuda(), version=types.SimpleNamespace(cuda="12.4")), - ) - - def test_gpu_probe_reports_free_memory_and_idle_aggregate(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3") - _install_fake_torch(monkeypatch) + monkeypatch.setattr(training, "_sample_sft_gpu_compute_process_count", lambda selector: 0) monkeypatch.setattr( - training.subprocess, - "run", - lambda *args, **kwargs: types.SimpleNamespace(returncode=0, stdout="", stderr=""), + training, + "_run_sft_gpu_fact_helper", + lambda selector: { + "status": "OK", + "cuda_available": True, + "visible_device_count": 1, + "name": "NVIDIA A100-SXM4-80GB", + "compute_capability": "8.0", + "total_memory_gib": 80.0, + "free_memory_gib": 80.0, + "cuda_version": "12.4", + "bf16_supported": True, + }, ) facts, blockers = training._probe_sft_gpu() # noqa: SLF001 assert blockers == [] @@ -385,24 +356,34 @@ def test_gpu_probe_reports_free_memory_and_idle_aggregate(monkeypatch: pytest.Mo @pytest.mark.parametrize( - ("free_gib", "process_output", "expected"), + ("free_gib", "process_count", "expected"), [ - (34.0, "", "GPU_FREE_MEMORY_INSUFFICIENT"), - (80.0, "12345\n", "GPU_BUSY"), + (34.0, 0, "GPU_FREE_MEMORY_INSUFFICIENT"), + (80.0, 1, "GPU_BUSY"), ], ) def test_gpu_probe_blocks_insufficient_free_memory_or_busy_device( monkeypatch: pytest.MonkeyPatch, free_gib: float, - process_output: str, + process_count: int, expected: str, ) -> None: monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") - _install_fake_torch(monkeypatch, free_gib=free_gib) + monkeypatch.setattr(training, "_sample_sft_gpu_compute_process_count", lambda selector: process_count) monkeypatch.setattr( - training.subprocess, - "run", - lambda *args, **kwargs: types.SimpleNamespace(returncode=0, stdout=process_output, stderr=""), + training, + "_run_sft_gpu_fact_helper", + lambda selector: { + "status": "OK", + "cuda_available": True, + "visible_device_count": 1, + "name": "NVIDIA A100-SXM4-80GB", + "compute_capability": "8.0", + "total_memory_gib": 80.0, + "free_memory_gib": free_gib, + "cuda_version": "12.4", + "bf16_supported": True, + }, ) _, blockers = training._probe_sft_gpu() # noqa: SLF001 assert expected in blockers @@ -410,7 +391,6 @@ def test_gpu_probe_blocks_insufficient_free_memory_or_busy_device( def test_gpu_probe_fails_closed_when_occupancy_probe_fails(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") - _install_fake_torch(monkeypatch) monkeypatch.setattr( training.subprocess, "run", diff --git a/tests/test_private_a100_sft_preflight.py b/tests/test_private_a100_sft_preflight.py index 6391bd1..6894963 100644 --- a/tests/test_private_a100_sft_preflight.py +++ b/tests/test_private_a100_sft_preflight.py @@ -879,48 +879,21 @@ def test_gpu_probe_rejects_non_a100_even_when_other_cuda_facts_pass( ) -> None: monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") - class FakeCuda: - @staticmethod - def is_available() -> bool: - return True - - @staticmethod - def device_count() -> int: - return 1 - - @staticmethod - def get_device_properties(index: int) -> Any: - assert index == 0 - return types.SimpleNamespace(total_memory=80 * 1024**3) - - @staticmethod - def get_device_capability(index: int) -> tuple[int, int]: - assert index == 0 - return (8, 9) - - @staticmethod - def is_bf16_supported() -> bool: - return True - - @staticmethod - def get_device_name(index: int) -> str: - assert index == 0 - return "NVIDIA GeForce RTX 4090" - - @staticmethod - def mem_get_info(index: int) -> tuple[int, int]: - assert index == 0 - return (80 * 1024**3, 80 * 1024**3) - - monkeypatch.setitem( - sys.modules, - "torch", - types.SimpleNamespace(cuda=FakeCuda(), version=types.SimpleNamespace(cuda="12.4")), - ) + monkeypatch.setattr(training, "_sample_sft_gpu_compute_process_count", lambda selector: 0) monkeypatch.setattr( - training.subprocess, - "run", - lambda *args, **kwargs: types.SimpleNamespace(returncode=0, stdout="", stderr=""), + training, + "_run_sft_gpu_fact_helper", + lambda selector: { + "status": "OK", + "cuda_available": True, + "visible_device_count": 1, + "name": "NVIDIA GeForce RTX 4090", + "compute_capability": "8.9", + "total_memory_gib": 80.0, + "free_memory_gib": 80.0, + "cuda_version": "12.4", + "bf16_supported": True, + }, ) facts, blockers = training._probe_sft_gpu() # noqa: SLF001 @@ -929,30 +902,22 @@ def mem_get_info(index: int) -> tuple[int, int]: assert blockers == ["GPU_NOT_A100"] -def test_gpu_probe_converts_cuda_api_exception_to_stable_blocker( +def test_gpu_probe_converts_helper_process_failure_to_stable_blocker( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") - class BrokenCuda: - @staticmethod - def is_available() -> bool: - return True - - @staticmethod - def device_count() -> int: - raise RuntimeError("private CUDA driver detail") - - monkeypatch.setitem( - sys.modules, - "torch", - types.SimpleNamespace(cuda=BrokenCuda(), version=types.SimpleNamespace(cuda="12.4")), + monkeypatch.setattr(training, "_sample_sft_gpu_compute_process_count", lambda selector: 0) + monkeypatch.setattr( + training, + "_run_sft_gpu_fact_helper", + lambda selector: (_ for _ in ()).throw(RuntimeError("private CUDA driver detail")), ) facts, blockers = training._probe_sft_gpu() # noqa: SLF001 assert facts["visible_device_count"] == 0 - assert blockers == ["CUDA_PROBE_FAILED"] + assert blockers == ["GPU_OCCUPANCY_PROBE_FAILED"] def test_sft_preflight_blocks_missing_private_model_path( diff --git a/tests/test_sft_gpu_occupancy_helper.py b/tests/test_sft_gpu_occupancy_helper.py new file mode 100644 index 0000000..cfd9d0d --- /dev/null +++ b/tests/test_sft_gpu_occupancy_helper.py @@ -0,0 +1,548 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +import types +from pathlib import Path +from typing import Any + +import pytest + +from voice2task import _sft_gpu_probe_helper as helper +from voice2task import training + + +def _ready_helper_facts(*, free_memory_gib: float = 80.0) -> dict[str, Any]: + return { + "cuda_available": True, + "visible_device_count": 1, + "name": "NVIDIA A100-SXM4-80GB", + "compute_capability": "8.0", + "total_memory_gib": 80.0, + "free_memory_gib": free_memory_gib, + "cuda_version": "12.4", + "bf16_supported": True, + } + + +def _helper_result(status: str = "OK", **overrides: Any) -> dict[str, Any]: + return {"status": status, **_ready_helper_facts(), **overrides} + + +def _fake_torch( + *, + available: bool = True, + visible_count: int = 1, + api_failure: Exception | None = None, +) -> Any: + def availability() -> bool: + if api_failure is not None: + raise api_failure + return available + + cuda = types.SimpleNamespace( + is_available=availability, + device_count=lambda: visible_count, + get_device_properties=lambda index: types.SimpleNamespace(total_memory=80 * 1024**3), + get_device_capability=lambda index: (8, 0), + mem_get_info=lambda index: (79 * 1024**3, 80 * 1024**3), + get_device_name=lambda index: "NVIDIA A100-SXM4-80GB", + is_bf16_supported=lambda: True, + ) + return types.SimpleNamespace(cuda=cuda, version=types.SimpleNamespace(cuda="12.4")) + + +def test_helper_collect_reports_typed_cuda_unavailable_without_raw_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + helper.importlib, + "import_module", + lambda name: (_ for _ in ()).throw(ImportError("secret /workspace/torch")), + ) + + result = helper.collect_gpu_result() + + assert result["status"] == "CUDA_UNAVAILABLE" + assert result["cuda_available"] is False + assert "secret" not in json.dumps(result, sort_keys=True) + assert "/workspace" not in json.dumps(result, sort_keys=True) + + +def test_helper_collect_reports_typed_cuda_unavailable_for_false_cuda() -> None: + result = helper.collect_gpu_result(_fake_torch(available=False)) + + assert result["status"] == "CUDA_UNAVAILABLE" + assert result["cuda_available"] is False + assert result["visible_device_count"] == 0 + + +def test_helper_collect_reports_typed_single_selection_failure() -> None: + result = helper.collect_gpu_result(_fake_torch(visible_count=2)) + + assert result["status"] == "GPU_SELECTION_NOT_SINGLE" + assert result["cuda_available"] is True + assert result["visible_device_count"] == 2 + + +def test_helper_collect_reports_valid_a100_facts() -> None: + result = helper.collect_gpu_result(_fake_torch()) + + assert result == { + "schema_version": "voice2task-sft-gpu-helper-v2", + "status": "OK", + **_ready_helper_facts(free_memory_gib=79.0), + } + + +def test_helper_collect_reports_typed_cuda_probe_failure_without_raw_error() -> None: + result = helper.collect_gpu_result( + _fake_torch(api_failure=RuntimeError("driver secret /data/private")) + ) + + assert result["status"] == "CUDA_PROBE_FAILED" + assert "driver secret" not in json.dumps(result, sort_keys=True) + assert "/data/private" not in json.dumps(result, sort_keys=True) + + +@pytest.mark.parametrize( + "event", + [ + "socket.__new__", + "socket.connect", + "socket.bind", + "socket.getaddrinfo", + ], +) +def test_helper_network_audit_hook_blocks_network_events(event: str) -> None: + with pytest.raises(PermissionError, match="^NETWORK_ACCESS_BLOCKED$"): + helper._network_audit_hook(event, ()) # noqa: SLF001 + + +def test_helper_installs_network_audit_hook_before_importing_torch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + installed: list[Any] = [] + monkeypatch.setattr(helper.sys, "addaudithook", installed.append) + + def fake_import(name: str) -> Any: + assert installed == [helper._network_audit_hook] # noqa: SLF001 + installed[0]("socket.connect", ()) + return _fake_torch() + + monkeypatch.setattr(helper.importlib, "import_module", fake_import) + + result = helper.main_result() + + assert result["status"] == "CUDA_PROBE_FAILED" + assert "NETWORK_ACCESS_BLOCKED" not in json.dumps(result, sort_keys=True) + + +def test_helper_network_audit_hook_blocks_real_socket_apis_in_isolated_child() -> None: + helper_path = Path(helper.__file__).resolve(strict=True) + script = """ +import importlib.util +import json +import socket +import sys + +spec = importlib.util.spec_from_file_location("isolated_gpu_helper", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(module) +sys.addaudithook(module._network_audit_hook) + +def with_tcp(action): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + return action(sock) + finally: + sock.close() + +def with_udp(action): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + return action(sock) + finally: + sock.close() + +operations = { + "creation": lambda: socket.socket(socket.AF_INET, socket.SOCK_STREAM).close(), + "listen": lambda: with_tcp(lambda sock: (sock.bind(("127.0.0.1", 0)), sock.listen())), + "connect": lambda: with_tcp(lambda sock: sock.connect(("127.0.0.1", 9))), + "connect_ex": lambda: with_tcp(lambda sock: sock.connect_ex(("127.0.0.1", 9))), + "sendto": lambda: with_udp(lambda sock: sock.sendto(b"x", ("127.0.0.1", 9))), + "sendmsg": lambda: with_udp(lambda sock: sock.sendmsg([b"x"], [], 0, ("127.0.0.1", 9))), + "dns": lambda: socket.getaddrinfo("localhost", 80), +} +results = {} +for name, operation in operations.items(): + try: + operation() + except PermissionError as exc: + results[name] = "BLOCKED" if str(exc) == "NETWORK_ACCESS_BLOCKED" else "WRONG_ERROR" + except Exception: + results[name] = "NOT_AUDIT_BLOCKED" + else: + results[name] = "NOT_BLOCKED" +print(json.dumps(results, sort_keys=True)) +""" + completed = subprocess.run( + [sys.executable, "-I", "-c", script, helper_path.as_posix()], + check=False, + capture_output=True, + text=True, + timeout=30, + env={"LC_ALL": "C", "LANG": "C"}, + ) + + assert completed.returncode == 0 + assert completed.stderr == "" + assert json.loads(completed.stdout) == { + "connect": "BLOCKED", + "connect_ex": "BLOCKED", + "creation": "BLOCKED", + "dns": "BLOCKED", + "listen": "BLOCKED", + "sendmsg": "BLOCKED", + "sendto": "BLOCKED", + } + + +def test_helper_real_child_emits_exactly_one_strict_typed_json_document() -> None: + helper_path = Path(helper.__file__).resolve(strict=True) + completed = subprocess.run( + [sys.executable, "-I", helper_path.as_posix()], + check=False, + capture_output=True, + text=True, + timeout=30, + env={ + "CUDA_VISIBLE_DEVICES": "-1", + "PYTHONNOUSERSITE": "1", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "HF_DATASETS_OFFLINE": "1", + "LC_ALL": "C", + "LANG": "C", + }, + ) + + assert completed.returncode == 0 + assert completed.stderr == "" + assert completed.stdout.count("\n") == 1 + payload = json.loads(completed.stdout) + assert set(payload) == { + "schema_version", + "status", + *training.SFT_GPU_HELPER_FACT_FIELDS, + } + assert payload["schema_version"] == "voice2task-sft-gpu-helper-v2" + assert payload["status"] == "CUDA_UNAVAILABLE" + assert payload["cuda_available"] is False + + +def test_gpu_probe_orders_pre_occupancy_helper_and_post_occupancy_with_exact_selector( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3") + events: list[tuple[str, str]] = [] + monkeypatch.setattr( + training, + "_sample_sft_gpu_compute_process_count", + lambda selector: events.append(("occupancy", selector)) or 0, + raising=False, + ) + monkeypatch.setattr( + training, + "_run_sft_gpu_fact_helper", + lambda selector: events.append(("helper", selector)) or _helper_result(), + raising=False, + ) + + facts, blockers = training._probe_sft_gpu() # noqa: SLF001 + + assert blockers == [] + assert events == [("occupancy", "3"), ("helper", "3"), ("occupancy", "3")] + assert facts["compute_process_count"] == 0 + assert facts["idle_verified"] is True + + +def test_gpu_probe_repeats_full_sequence_without_pid_inference_or_cache( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "GPU-selector") + events: list[str] = [] + monkeypatch.setattr( + training, + "_sample_sft_gpu_compute_process_count", + lambda selector: events.append(f"occupancy:{selector}") or 0, + raising=False, + ) + monkeypatch.setattr( + training, + "_run_sft_gpu_fact_helper", + lambda selector: events.append(f"helper:{selector}") or _helper_result(), + raising=False, + ) + + first = training._probe_sft_gpu() # noqa: SLF001 + second = training._probe_sft_gpu() # noqa: SLF001 + + assert first[1] == second[1] == [] + assert events == [ + "occupancy:GPU-selector", + "helper:GPU-selector", + "occupancy:GPU-selector", + "occupancy:GPU-selector", + "helper:GPU-selector", + "occupancy:GPU-selector", + ] + serialized = json.dumps([first, second], sort_keys=True).lower() + for forbidden in ("pid", "username", "uuid", "hostname", "command", "selector"): + assert forbidden not in serialized + + +def test_gpu_probe_blocks_preexisting_occupancy_without_starting_helper( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") + helper_calls: list[str] = [] + monkeypatch.setattr(training, "_sample_sft_gpu_compute_process_count", lambda selector: 1, raising=False) + monkeypatch.setattr( + training, + "_run_sft_gpu_fact_helper", + lambda selector: helper_calls.append(selector), + raising=False, + ) + + facts, blockers = training._probe_sft_gpu() # noqa: SLF001 + + assert blockers == ["GPU_BUSY"] + assert helper_calls == [] + assert facts["compute_process_count"] == 1 + assert facts["idle_verified"] is False + + +def test_gpu_probe_blocks_post_helper_occupancy(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") + samples = iter([0, 1]) + monkeypatch.setattr( + training, + "_sample_sft_gpu_compute_process_count", + lambda selector: next(samples), + raising=False, + ) + monkeypatch.setattr(training, "_run_sft_gpu_fact_helper", lambda selector: _helper_result(), raising=False) + + facts, blockers = training._probe_sft_gpu() # noqa: SLF001 + + assert blockers == ["GPU_BUSY"] + assert facts["compute_process_count"] == 1 + assert facts["idle_verified"] is False + + +@pytest.mark.parametrize( + "failure", + [ + TimeoutError("private timeout detail"), + RuntimeError("private helper detail /workspace/secret"), + ], +) +def test_gpu_probe_converts_helper_failure_to_private_safe_blocker( + monkeypatch: pytest.MonkeyPatch, + failure: Exception, +) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") + sample_calls: list[str] = [] + monkeypatch.setattr( + training, + "_sample_sft_gpu_compute_process_count", + lambda selector: sample_calls.append(selector) or 0, + raising=False, + ) + monkeypatch.setattr( + training, + "_run_sft_gpu_fact_helper", + lambda selector: (_ for _ in ()).throw(failure), + raising=False, + ) + + facts, blockers = training._probe_sft_gpu() # noqa: SLF001 + + assert blockers == ["GPU_OCCUPANCY_PROBE_FAILED"] + assert sample_calls == ["0", "0"] + serialized = json.dumps(facts, sort_keys=True) + assert "private" not in serialized + assert "/workspace" not in serialized + + +@pytest.mark.parametrize( + ("status", "expected_blocker"), + [ + ("CUDA_UNAVAILABLE", "CUDA_UNAVAILABLE"), + ("CUDA_PROBE_FAILED", "CUDA_PROBE_FAILED"), + ], +) +def test_gpu_probe_preserves_typed_helper_failure( + monkeypatch: pytest.MonkeyPatch, + status: str, + expected_blocker: str, +) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") + monkeypatch.setattr(training, "_sample_sft_gpu_compute_process_count", lambda selector: 0) + monkeypatch.setattr( + training, + "_run_sft_gpu_fact_helper", + lambda selector: _helper_result( + status, + cuda_available=False, + visible_device_count=0, + name=None, + compute_capability=None, + total_memory_gib=None, + free_memory_gib=None, + cuda_version="unknown", + bf16_supported=False, + ), + ) + + facts, blockers = training._probe_sft_gpu() # noqa: SLF001 + + assert blockers == [expected_blocker] + assert facts["idle_verified"] is True + + +@pytest.mark.parametrize( + "malformed", + [ + {}, + {"cuda_available": True}, + {**_helper_result(), "visible_device_count": "1"}, + {**_helper_result(), "private_path": "/workspace/secret"}, + _helper_result("UNKNOWN_STATUS"), + ], +) +def test_gpu_probe_fails_closed_on_malformed_helper_result( + monkeypatch: pytest.MonkeyPatch, + malformed: dict[str, Any], +) -> None: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0") + monkeypatch.setattr(training, "_sample_sft_gpu_compute_process_count", lambda selector: 0, raising=False) + monkeypatch.setattr(training, "_run_sft_gpu_fact_helper", lambda selector: malformed, raising=False) + + facts, blockers = training._probe_sft_gpu() # noqa: SLF001 + + assert blockers == ["GPU_OCCUPANCY_PROBE_FAILED"] + assert "/workspace" not in json.dumps(facts, sort_keys=True) + + +@pytest.mark.parametrize( + "malformed", + [ + _helper_result("OK", cuda_available=False), + _helper_result("OK", visible_device_count=2), + _helper_result("OK", name=None), + _helper_result("CUDA_UNAVAILABLE", cuda_available=True), + _helper_result("CUDA_PROBE_FAILED", name="NVIDIA A100-SXM4-80GB"), + _helper_result("GPU_SELECTION_NOT_SINGLE", visible_device_count=1), + _helper_result("GPU_SELECTION_NOT_SINGLE", visible_device_count=2, bf16_supported=True), + ], +) +def test_gpu_helper_protocol_rejects_cross_field_inconsistency( + malformed: dict[str, Any], +) -> None: + with pytest.raises(RuntimeError, match="^GPU_OCCUPANCY_PROBE_FAILED$"): + training._validated_sft_gpu_helper_result(malformed) # noqa: SLF001 + + +def test_gpu_fact_helper_runner_uses_exact_selector_and_private_one_json_protocol( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, Any]] = [] + payload = { + "schema_version": "voice2task-sft-gpu-helper-v2", + **_helper_result(), + } + + def fake_run(command: list[str], **kwargs: Any) -> Any: + calls.append({"command": command, **kwargs}) + return types.SimpleNamespace(returncode=0, stdout=json.dumps(payload), stderr="") + + monkeypatch.setattr(training.subprocess, "run", fake_run) + + result = training._run_sft_gpu_fact_helper("7") # noqa: SLF001 + + assert result == _helper_result() + assert calls[0]["command"] == [ + training.sys.executable, + "-I", + Path(helper.__file__).resolve(strict=True).as_posix(), + ] + assert calls[0]["env"] == { + "CUDA_VISIBLE_DEVICES": "7", + "PYTHONNOUSERSITE": "1", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "HF_DATASETS_OFFLINE": "1", + "LC_ALL": "C", + "LANG": "C", + } + for forbidden in ( + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "TOKEN", + "PYTHONPATH", + "PYTHONHOME", + "LD_PRELOAD", + "DYLD_INSERT_LIBRARIES", + ): + assert forbidden not in calls[0]["env"] + assert os.path.isabs(calls[0]["command"][0]) + assert os.path.isabs(calls[0]["command"][2]) + assert calls[0]["capture_output"] is True + assert calls[0]["text"] is True + + +@pytest.mark.parametrize( + "result", + [ + types.SimpleNamespace(returncode=1, stdout="", stderr="private error"), + types.SimpleNamespace(returncode=0, stdout="not-json", stderr=""), + types.SimpleNamespace(returncode=0, stdout="{}", stderr=""), + types.SimpleNamespace( + returncode=0, + stdout=json.dumps( + { + "schema_version": "voice2task-sft-gpu-helper-v2", + **_helper_result("UNKNOWN_STATUS"), + } + ), + stderr="", + ), + ], +) +def test_gpu_fact_helper_runner_rejects_nonzero_or_malformed_output( + monkeypatch: pytest.MonkeyPatch, + result: Any, +) -> None: + monkeypatch.setattr(training.subprocess, "run", lambda *args, **kwargs: result) + + with pytest.raises(RuntimeError, match="GPU_OCCUPANCY_PROBE_FAILED"): + training._run_sft_gpu_fact_helper("0") # noqa: SLF001 + + +def test_gpu_fact_helper_runner_converts_timeout_without_raw_detail( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + training.subprocess, + "run", + lambda *args, **kwargs: (_ for _ in ()).throw(subprocess.TimeoutExpired("/private/helper", 1)), + ) + + with pytest.raises(RuntimeError, match="^GPU_OCCUPANCY_PROBE_FAILED$"): + training._run_sft_gpu_fact_helper("0") # noqa: SLF001 From df25648d1d837d7aa4af7757b9aea68783d2a5b6 Mon Sep 17 00:00:00 2001 From: Minghong Sun <118113656+Raidriar7170@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:22:25 +0800 Subject: [PATCH 4/5] fix training CLI single-JSON output --- ...07-15-fix-real-sft-cli-single-json-v1.html | 60 +++++++++++ .../.openspec.yaml | 2 + .../design.md | 45 +++++++++ .../proposal.md | 29 ++++++ .../specs/supervised-contract-tuning/spec.md | 20 ++++ .../tasks.md | 35 +++++++ .../specs/supervised-contract-tuning/spec.md | 10 +- src/voice2task/cli/train.py | 89 +++++++++++------ tests/test_a100_sft_review_hardening.py | 99 +++++++++++++++++++ 9 files changed, 360 insertions(+), 29 deletions(-) create mode 100644 docs/human-briefs/2026-07-15-fix-real-sft-cli-single-json-v1.html create mode 100644 openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/.openspec.yaml create mode 100644 openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/design.md create mode 100644 openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/proposal.md create mode 100644 openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/specs/supervised-contract-tuning/spec.md create mode 100644 openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/tasks.md diff --git a/docs/human-briefs/2026-07-15-fix-real-sft-cli-single-json-v1.html b/docs/human-briefs/2026-07-15-fix-real-sft-cli-single-json-v1.html new file mode 100644 index 0000000..9e73073 --- /dev/null +++ b/docs/human-briefs/2026-07-15-fix-real-sft-cli-single-json-v1.html @@ -0,0 +1,60 @@ + + + + + + 真实 SFT CLI 单一 JSON 修复 · Human Brief + + + +

OpenSpec · fix-real-sft-cli-single-json-v1

+

真实 SFT CLI stdout 单一 JSON 修复

+

ARCHIVED_CODE_GREEN · PRIOR_SMOKE_RUNTIME_POSTCONDITION_FAILED · SECOND_SMOKE_NOT_AUTHORIZED · execution_readiness=false

+ +

第一次获授权的真实 1-step SFT 已证明 adapter 参数发生有限、非零且有限值的更新，但进程 stdout 在最终 JSON 前包含 Trainer 的两行 Python 字典输出，因此完整 stdout 不是单一 JSON document。该运行不得对外描述为 SMOKE_COMPLETED，也不得据此进入阶段二。

+ +

本轮边界

+
    +
  • 只修复 CLI 输出隔离与回归测试；不修改模型、数据、训练预算、preflight 或 adapter 证据。
    • +
  • 统一 backend dispatch 内的 Python stdout 被逐行加 voice2task-backend: 前缀并流式转发到 stderr；最终成功或失败 JSON 保留在隔离范围外。
    • +
  • 不再次运行真实 smoke，不运行完整 SFT、DPO、GRPO、prediction、evaluation，不访问 test、lockbox 或 clean-evaluation population。
    • +
+ +

TDD 与当前证据

+
    +
  • RED：3 个回归按预期失败，分别复现 noisy SFT success、noisy runtime exception、mock-only DPO dispatch 的 stdout 污染。
    • +
  • GREEN：3 个新增回归通过；A100 review-hardening 与 private-preflight 组合回归为 154 passed。
    • +
  • 独立只读审阅：Pass，Must Fix 0，Should Fix 0；pre-archive 完整测试为 1450 passed，另有 4 个只由 active change 触发的既有生命周期门失败，未增加 allowlist。
    • +
  • Ruff、strict OpenSpec 16/16、formal public data 696 SFT / 2100 DPO、88 个数据与 schema 回归、public leak scan 和 diff check 均通过；targeted Mypy 仅保留两个未触及 imported module 的既有诊断。
    • +
  • 归档后：active changes=0，完整 pytest 1454 passed，strict OpenSpec 15/15，truth-surface、5/5 本地链接、private ignore/untracked/status guard 与 Stop hook 双路径 exit 0 均通过。
    • +
  • 测试不下载模型、不要求 GPU，也不执行训练或 DPO。
    • +
+ +

不变的 clean-evaluation 事实

+
    +
  • acquisition_source_status=UNAVAILABLE
    • +
  • authoritatively_bound_binding_count=0
    • +
  • human_acceptance_status=NOT_RECORDED
    • +
  • protocol_freeze_status=NOT_FROZEN
    • +
  • clean_population_status=NOT_MATERIALIZED
    • +
  • freeze_authorized=false；execution_readiness=false
    • +
+ +

审阅入口

+ + +
本 bounded code fix 已审阅并归档，但先前真实 smoke 仍是 Runtime Postcondition Failed。任何第二次真实 smoke 都需要新的明确授权；本 brief 不构成训练、评测、发布、生产就绪或模型改进声明。
+ + diff --git a/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/.openspec.yaml b/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/.openspec.yaml new file mode 100644 index 0000000..4f63482 --- /dev/null +++ b/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-15 diff --git a/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/design.md b/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/design.md new file mode 100644 index 0000000..d00af11 --- /dev/null +++ b/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/design.md @@ -0,0 +1,45 @@ +## Context + +The CLI currently dispatches directly to training functions and prints the returned public result afterward. During the first authorized real one-step SFT process, the Transformers/TRL `PrinterCallback` wrote two Python mapping representations to stdout before the final result JSON. The process exited zero and the returned adapter metadata contained real update evidence, but the complete stdout was not one JSON document, so the runtime postcondition failed. + +The contract belongs at the CLI process boundary: output can be emitted while loading dependencies, constructing trainers, invoking callbacks, or training. Isolating only `trainer.train()` would leave other downstream sources able to contaminate stdout. The CLI runs one synchronous command per process, and long-running output must stream instead of accumulating in memory. + +## Goals / Non-Goals + +**Goals:** + +- Reserve CLI stdout for exactly one final sanitized result JSON on both success and failure. +- Preserve downstream Python stdout as diagnostic stderr with a non-JSON line prefix. +- Keep the existing public result allowlist and typed exit-code mapping unchanged. +- Cover noisy SFT success, noisy runtime failure, and the shared DPO dispatch boundary without executing training. + +**Non-Goals:** + +- No second real smoke, model loading, GPU work, full SFT, DPO, GRPO, prediction, evaluation, or lockbox access. +- No change to training budgets, model/data selection, clean-evaluation truth, adapter evidence, or output-directory policy. +- No interception of native code or subprocesses that bypass Python `sys.stdout` and write directly to file descriptor 1. + +## Decisions + +### Isolate at the unified CLI dispatch boundary + +All backend command dispatch will run inside `contextlib.redirect_stdout(...)`; final success and sanitized exception JSON printing will remain outside that context. This covers SFT, DPO, preflight, and other CLI backends consistently, and guarantees stdout restoration before the exception handler writes the only failure result. + +Alternative considered: redirect only around SFT `trainer.train()`. Rejected because model/tokenizer loading, trainer construction, callbacks, and DPO can also print before or outside that call. + +### Stream diagnostics to a prefixed stderr adapter + +The redirect target will forward data to the current stderr stream and add a stable non-JSON prefix at each logical line start. This preserves progress without retaining potentially long training logs and prevents a backend-emitted JSON line from being mistaken for the CLI result document. + +Alternative considered: redirect directly to stderr. Rejected because a downstream valid JSON object could then be misread as a second result. Alternative considered: capture in `StringIO`. Rejected because training logs are unbounded relative to this smoke contract. + +### Test the observed failure shape without loading models + +Regression tests will monkeypatch backend functions to print the two observed Python mapping forms before returning or raising. They will parse the complete stdout with `json.loads`, verify exit status/result status, and verify stderr diagnostics are prefixed and contain no independently parseable training-result document. DPO coverage uses the same mock-only dispatch test and does not authorize DPO execution. + +## Risks / Trade-offs + +- [Risk] `redirect_stdout` changes process-global `sys.stdout` and is not thread-local. → The CLI is a synchronous one-command process; document and test this boundary without exposing it as a concurrent library API. +- [Risk] A custom stream adapter may receive partial writes. → Track line-start state across writes and test multiple lines plus a final line without a newline. +- [Risk] Native code can write directly to file descriptor 1. → This bounded fix addresses the observed Python callback path; no claim is made about OS-level descriptor capture. +- [Risk] Prefixing changes diagnostic formatting. → Use one stable public-safe prefix and keep all result semantics exclusively on stdout. diff --git a/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/proposal.md b/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/proposal.md new file mode 100644 index 0000000..2568bd4 --- /dev/null +++ b/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/proposal.md @@ -0,0 +1,29 @@ +## Why + +The first authorized real one-step SFT process exposed that Transformers/TRL callback output can reach CLI stdout before the final result, so a successful process can violate the existing one-JSON protocol even though the final metadata is valid. The CLI boundary must own stdout and route dependency progress elsewhere before any additional real smoke can be considered. + +## What Changes + +- Isolate training-library stdout while the CLI invokes SFT so stdout contains only the final sanitized result JSON. +- Route ordinary training progress to stderr without emitting a second result JSON. +- Preserve the existing typed exit-code policy and public-result allowlist for success and runtime exceptions. +- Add a regression test reproducing the real two-line Python mapping output observed from `Trainer.train()` and proving the complete stdout parses as exactly one JSON document. +- Record the fix in the supervised contract-tuning specification and an auditable Human Brief. +- Explicitly do not authorize or perform a second real smoke, full training, prediction, evaluation, DPO, GRPO, lockbox access, or any model-improvement claim. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `supervised-contract-tuning`: Strengthen the existing training CLI one-JSON requirement so third-party training progress cannot contaminate stdout during real execution. + +## Impact + +- Affected code: `src/voice2task/cli/train.py` and focused CLI tests. +- Affected contract: the existing stdout/stderr and exit-status requirement in `openspec/specs/supervised-contract-tuning/spec.md`. +- No model, dataset, dependency, output-directory, clean-evaluation, or training-budget behavior changes. +- No generic chat fine-tuning, skill routing, GUI action policy learning, first-phase GRPO, public release of the full local corpus, released checkpoint claim, or live-browser benchmark claim. diff --git a/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/specs/supervised-contract-tuning/spec.md b/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/specs/supervised-contract-tuning/spec.md new file mode 100644 index 0000000..75938f6 --- /dev/null +++ b/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/specs/supervised-contract-tuning/spec.md @@ -0,0 +1,20 @@ +## MODIFIED Requirements + +### Requirement: Return truthful CLI exit status and one JSON result +The training CLI MUST reserve stdout for exactly one final sanitized JSON document. All Python-level stdout emitted by downstream command execution, including model or tokenizer loading, trainer construction, training callbacks, and trainer progress, MUST be streamed to stderr as prefixed diagnostics that cannot be parsed as a second result JSON. Dry-run success, ready preflight, and `training_status=training_completed` MUST exit 0. Skipped-by-config, unavailable, output-policy-blocked, preflight-blocked, and runtime-exception results MUST exit non-zero. Stderr MUST NOT contain a second result JSON. + +#### Scenario: Map successful results +- **WHEN** dry-run succeeds, preflight is ready, or training completes +- **THEN** the CLI emits one JSON document and exits 0 + +#### Scenario: Map non-success results +- **WHEN** training is skipped, unavailable, output-policy-blocked, preflight-blocked, or raises at runtime +- **THEN** the CLI emits one sanitized JSON document and exits non-zero without a competing result document on stderr + +#### Scenario: Isolate downstream training progress +- **WHEN** an SFT or DPO backend prints Python objects, progress, or JSON-shaped text to stdout before returning a result +- **THEN** the CLI streams that output to stderr with a non-JSON diagnostic prefix and keeps stdout parseable as exactly one final result JSON + +#### Scenario: Restore result stdout after a noisy runtime exception +- **WHEN** a backend writes progress to stdout and then raises at runtime +- **THEN** the CLI restores result stdout, emits exactly one sanitized failure JSON there, exits non-zero, and leaves only prefixed diagnostics on stderr diff --git a/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/tasks.md b/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/tasks.md new file mode 100644 index 0000000..f3e42ba --- /dev/null +++ b/openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1/tasks.md @@ -0,0 +1,35 @@ +## 1. Reproduce the Real CLI Failure + +- [x] 1.1 Add an SFT CLI regression that emits the two observed Trainer mapping lines before a completed result, then prove the unmodified CLI stdout is not one JSON document +- [x] 1.2 Add noisy runtime-exception and DPO dispatch regressions that require one result JSON on stdout, non-zero/zero typed exits, and only prefixed non-result diagnostics on stderr + +## 2. Implement the CLI Output Boundary + +- [x] 2.1 Add a line-aware streaming stderr diagnostic adapter and redirect all backend Python stdout at the unified CLI dispatch boundary +- [x] 2.2 Keep final success/failure JSON rendering outside the redirect context and preserve the existing public allowlist and exit-code mapping +- [x] 2.3 Run the focused CLI tests through RED/GREEN and the related A100/preflight regression modules without model downloads, GPU access, or training + - RED: focused noisy SFT success, noisy exception, and mock-only DPO dispatch failed `3/3` because backend output contaminated stdout. + - GREEN: the focused tests passed `3/3`; full review-hardening plus private-preflight regression passed `154/154`. + +## 3. Audit and Documentation + +- [x] 3.1 Generate `docs/human-briefs/2026-07-15-fix-real-sft-cli-single-json-v1.html` with the observed failure, bounded fix, validation evidence, and explicit no-rerun/no-stage-two boundary +- [x] 3.2 Run an independent read-only review and resolve every Must Fix while preserving clean-evaluation truth and the prior runtime-postcondition-failed status + - Independent verdict: Pass; Must Fix none, Should Fix none, Re-plan not needed. The review also exercised a real `tqdm(..., file=sys.stdout)` compatibility path without training. + +## 4. Pre-Archive Verification + +- [x] 4.1 Run focused and full `PYTHONPATH=src pytest -q`, `PYTHONPATH=src ruff check .`, targeted Mypy for touched training files, `OPENSPEC_TELEMETRY=0 openspec validate --all --strict`, and `git diff --check`; record any active-change-only lifecycle failure without adding an allowlist + - Focused CLI regression: 154 passed. Full pre-archive suite: 1450 passed and 4 active-change-only lifecycle/frozen-boundary guard failures; no allowlist was added. Ruff, strict OpenSpec 16/16, and diff check passed. Targeted Mypy has no touched-file diagnostic and only the two recorded imported-module baseline diagnostics. +- [x] 4.2 Run the read-only formal public-data/schema and DPO checks: `PYTHONPATH=src python -m voice2task.cli.data validate --sft data/public-samples/sft_public_sample.jsonl --dpo data/public-samples/dpo_public_sample.jsonl --manifest data/public-samples/manifest_public_sample.json --public`, `PYTHONPATH=src python -m voice2task.cli.data dpo-check --dpo data/public-samples/dpo_public_sample.jsonl`, and `PYTHONPATH=src pytest -q tests/test_dataset_builder.py tests/test_schemas.py tests/test_dpo_validation.py tests/test_evaluator_reports.py` + - Formal validation passed with 696 SFT rows and 2100 DPO pairs; DPO check read 2100 pairs; the focused dataset/schema/DPO/evaluator suite passed 88 tests. +- [x] 4.3 Run a public leak scan over all changed public artifacts and confirm no private config, runtime path, host fact, log, adapter, cache, or remote connection detail is tracked + - The explicit changed-artifact scan returned zero findings; scoped tracked/status checks for `data/local-private` returned empty. + +## 5. Lifecycle Closeout + +- [x] 5.1 Sync the delta into `supervised-contract-tuning`, archive the change, confirm active changes are zero, then rerun full pytest, Ruff, targeted Mypy, strict OpenSpec validation, `PYTHONPATH=src python scripts/check_current_truth_surface.py`, public leak/ignore/status guards, and `git diff --check` + - The complete delta requirement was synchronized byte-for-byte and the change was archived at `openspec/changes/archive/2026-07-15-fix-real-sft-cli-single-json-v1`; active changes are zero. + - Post-archive full pytest passed 1454 tests. Ruff, strict OpenSpec 15/15, current truth-surface, five Human Brief links, public leak scan, private ignore/untracked/status guards, and `git diff --check` passed. + - Targeted Mypy has zero diagnostics in the touched training files and only the two recorded imported-module baseline diagnostics. Direct Stop-hook probes returned exit code 0 for both valid input and the fatal-parse catch path. + - No second real smoke, full training, DPO, GRPO, prediction, evaluation, lockbox access, GPU execution, or model-improvement claim occurred. diff --git a/openspec/specs/supervised-contract-tuning/spec.md b/openspec/specs/supervised-contract-tuning/spec.md index 3930b92..78477fe 100644 --- a/openspec/specs/supervised-contract-tuning/spec.md +++ b/openspec/specs/supervised-contract-tuning/spec.md @@ -152,7 +152,7 @@ The system MUST derive DPO's repository root from the supplied manifest checkout - **THEN** output policy still uses the repository containing the supplied manifest; if that repository cannot be resolved, execution fails closed before config, metadata, dependency, or runner work ### Requirement: Return truthful CLI exit status and one JSON result -The training CLI MUST write exactly one JSON document to stdout. Dry-run success, ready preflight, and `training_status=training_completed` MUST exit 0. Skipped-by-config, unavailable, output-policy-blocked, preflight-blocked, and runtime-exception results MUST exit non-zero. Stderr MUST NOT contain a second result JSON. +The training CLI MUST reserve stdout for exactly one final sanitized JSON document. All Python-level stdout emitted by downstream command execution, including model or tokenizer loading, trainer construction, training callbacks, and trainer progress, MUST be streamed to stderr as prefixed diagnostics that cannot be parsed as a second result JSON. Dry-run success, ready preflight, and `training_status=training_completed` MUST exit 0. Skipped-by-config, unavailable, output-policy-blocked, preflight-blocked, and runtime-exception results MUST exit non-zero. Stderr MUST NOT contain a second result JSON. #### Scenario: Map successful results - **WHEN** dry-run succeeds, preflight is ready, or training completes @@ -162,6 +162,14 @@ The training CLI MUST write exactly one JSON document to stdout. Dry-run success - **WHEN** training is skipped, unavailable, output-policy-blocked, preflight-blocked, or raises at runtime - **THEN** the CLI emits one sanitized JSON document and exits non-zero without a competing result document on stderr +#### Scenario: Isolate downstream training progress +- **WHEN** an SFT or DPO backend prints Python objects, progress, or JSON-shaped text to stdout before returning a result +- **THEN** the CLI streams that output to stderr with a non-JSON diagnostic prefix and keeps stdout parseable as exactly one final result JSON + +#### Scenario: Restore result stdout after a noisy runtime exception +- **WHEN** a backend writes progress to stdout and then raises at runtime +- **THEN** the CLI restores result stdout, emits exactly one sanitized failure JSON there, exits non-zero, and leaves only prefixed diagnostics on stderr + ### Requirement: Export A100 trained-path public-sample predictions The system SHALL provide a bounded prediction export workflow for the A100 public-sample SFT adapter path that emits only sanitized public-sample prediction rows for contract evaluation. diff --git a/src/voice2task/cli/train.py b/src/voice2task/cli/train.py index 09ca77b..ab43ab2 100644 --- a/src/voice2task/cli/train.py +++ b/src/voice2task/cli/train.py @@ -1,8 +1,12 @@ from __future__ import annotations import argparse +import io import json +import sys +from contextlib import redirect_stdout from pathlib import Path +from typing import TextIO, cast from voice2task.io import write_json from voice2task.training import ( @@ -16,6 +20,29 @@ run_sft_runtime_label_provenance_check, ) +BACKEND_DIAGNOSTIC_PREFIX = "voice2task-backend: " + + +class _PrefixedDiagnosticStream(io.TextIOBase): + """Stream backend stdout to stderr with one prefix per logical line.""" + + def __init__(self, sink: TextIO) -> None: + self._sink = sink + self._at_line_start = True + + def write(self, text: str) -> int: + if not isinstance(text, str): + raise TypeError("diagnostic stream accepts text only") + for part in text.splitlines(keepends=True): + if self._at_line_start: + self._sink.write(BACKEND_DIAGNOSTIC_PREFIX) + self._sink.write(part) + self._at_line_start = part.endswith(("\n", "\r")) + return len(text) + + def flush(self) -> None: + self._sink.flush() + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="voice2task-train") @@ -60,34 +87,40 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: - if args.command == "sft": - metadata = run_sft(args.config, args.manifest, args.output_dir, dry_run=args.dry_run) - elif args.command == "sft-preflight": - metadata = run_sft_preflight(args.config, args.manifest, args.output_dir) - elif args.command == "dpo": - metadata = run_dpo(args.config, args.manifest, args.output_dir, dry_run=args.dry_run) - elif args.command == "sft-predict": - metadata = run_sft_prediction_export( - args.config, - args.manifest, - args.output, - dry_run=args.dry_run, - fixture_mode=args.fixture_mode, - ) - elif args.command == "sft-inspect-objective": - metadata = inspect_sft_objective_from_manifest(args.manifest, split=args.split) - elif args.command == "sft-prepare-runtime-label-provenance": - metadata = prepare_sft_runtime_label_provenance(args.config, args.manifest, metadata_path=args.output) - elif args.command == "sft-runtime-label-provenance-check": - metadata = run_sft_runtime_label_provenance_check( - args.config, - args.manifest, - split=args.split, - output_path=args.output, - run_runtime_check=args.run_runtime_check, - ) - else: - raise AssertionError(f"unhandled command: {args.command}") + diagnostic_stream = cast(TextIO, _PrefixedDiagnosticStream(sys.stderr)) + with redirect_stdout(diagnostic_stream): + if args.command == "sft": + metadata = run_sft(args.config, args.manifest, args.output_dir, dry_run=args.dry_run) + elif args.command == "sft-preflight": + metadata = run_sft_preflight(args.config, args.manifest, args.output_dir) + elif args.command == "dpo": + metadata = run_dpo(args.config, args.manifest, args.output_dir, dry_run=args.dry_run) + elif args.command == "sft-predict": + metadata = run_sft_prediction_export( + args.config, + args.manifest, + args.output, + dry_run=args.dry_run, + fixture_mode=args.fixture_mode, + ) + elif args.command == "sft-inspect-objective": + metadata = inspect_sft_objective_from_manifest(args.manifest, split=args.split) + elif args.command == "sft-prepare-runtime-label-provenance": + metadata = prepare_sft_runtime_label_provenance( + args.config, + args.manifest, + metadata_path=args.output, + ) + elif args.command == "sft-runtime-label-provenance-check": + metadata = run_sft_runtime_label_provenance_check( + args.config, + args.manifest, + split=args.split, + output_path=args.output, + run_runtime_check=args.run_runtime_check, + ) + else: + raise AssertionError(f"unhandled command: {args.command}") except Exception: metadata = { "schema_version": "voice2task-training-result-v1", diff --git a/tests/test_a100_sft_review_hardening.py b/tests/test_a100_sft_review_hardening.py index 4ec314f..7d0be86 100644 --- a/tests/test_a100_sft_review_hardening.py +++ b/tests/test_a100_sft_review_hardening.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import sys import types from pathlib import Path from typing import Any @@ -24,6 +25,16 @@ "adapter_files", "clean_evaluation", } +BACKEND_DIAGNOSTIC_PREFIX = "voice2task-backend: " + + +def _assert_only_prefixed_nonresult_diagnostics(stderr: str) -> None: + lines = stderr.splitlines() + assert lines + assert all(line.startswith(BACKEND_DIAGNOSTIC_PREFIX) for line in lines) + for line in lines: + with pytest.raises(json.JSONDecodeError): + json.loads(line) @pytest.mark.parametrize( @@ -108,6 +119,94 @@ def test_training_cli_stdout_uses_public_result_allowlist( assert captured.err == "" +def test_sft_cli_isolates_observed_trainer_mapping_lines_and_json_shaped_backend_output( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + observed_lines = [ + "{'loss': 2.3265, 'grad_norm': 1.0, 'learning_rate': 0.0, 'epoch': 1.0}", + ( + "{'train_runtime': 1.0, 'train_samples_per_second': 1.0, " + "'train_steps_per_second': 1.0, 'train_loss': 2.3265, 'epoch': 1.0}" + ), + ] + + def noisy_sft(*args: Any, **kwargs: Any) -> dict[str, Any]: + print(*observed_lines, sep="\n") + sys.stdout.write('{"backend_progress":') + sys.stdout.write('"not-a-result"}\n') + return { + "schema_version": "voice2task-training-result-v1", + "stage": "sft", + "training_status": "training_completed", + "blockers": [], + } + + monkeypatch.setattr(train_cli, "run_sft", noisy_sft) + + exit_code = train_cli.main( + ["sft", "--config", "config.json", "--manifest", "manifest.json", "--output-dir", "run"] + ) + + captured = capsys.readouterr() + result = json.loads(captured.out) + assert exit_code == 0 + assert result["training_status"] == "training_completed" + for line in observed_lines: + assert f"{BACKEND_DIAGNOSTIC_PREFIX}{line}" in captured.err + assert f'{BACKEND_DIAGNOSTIC_PREFIX}{{"backend_progress":"not-a-result"}}' in captured.err + _assert_only_prefixed_nonresult_diagnostics(captured.err) + + +def test_sft_cli_restores_result_stdout_after_noisy_runtime_exception( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def noisy_failure(*args: Any, **kwargs: Any) -> dict[str, Any]: + print("{'loss': 9.9}") + sys.stdout.write('{"private_backend_error":"must-not-be-result"}') + raise RuntimeError("private /workspace/model exception") + + monkeypatch.setattr(train_cli, "run_sft", noisy_failure) + + exit_code = train_cli.main( + ["sft", "--config", "config.json", "--manifest", "manifest.json", "--output-dir", "run"] + ) + + captured = capsys.readouterr() + assert exit_code == 1 + assert json.loads(captured.out) == { + "blockers": ["TRAINING_RUNTIME_ERROR"], + "schema_version": "voice2task-training-result-v1", + "training_status": "training_failed", + } + assert "private /workspace/model exception" not in captured.out + captured.err + _assert_only_prefixed_nonresult_diagnostics(captured.err) + + +def test_dpo_cli_uses_shared_stdout_isolation_without_running_dpo( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def noisy_dpo(*args: Any, **kwargs: Any) -> dict[str, Any]: + sys.stdout.write('{"dpo_backend":') + sys.stdout.write('"mock-only"}\n') + return {"stage": "dpo", "training_status": "dry_run", "blockers": []} + + monkeypatch.setattr(train_cli, "run_dpo", noisy_dpo) + + exit_code = train_cli.main( + ["dpo", "--config", "config.json", "--manifest", "manifest.json", "--output-dir", "run"] + ) + + captured = capsys.readouterr() + result = json.loads(captured.out) + assert exit_code == 0 + assert result["training_status"] == "dry_run" + assert captured.err == f'{BACKEND_DIAGNOSTIC_PREFIX}{{"dpo_backend":"mock-only"}}\n' + _assert_only_prefixed_nonresult_diagnostics(captured.err) + + def test_public_training_result_rebuilds_preflight_without_unknown_nested_private_fields() -> None: private_path = "/workspace/acme/secret-model" metadata = { From d86475b14f23e1b65182160853bceae44140042e Mon Sep 17 00:00:00 2001 From: Minghong Sun <118113656+Raidriar7170@users.noreply.github.com> Date: Wed, 15 Jul 2026 12:28:38 +0800 Subject: [PATCH 5/5] Record successful real A100 SFT smoke rerun --- ...-real-a100-sft-smoke-after-cli-fix-v1.html | 78 +++++++++++++++++++ .../.openspec.yaml | 2 + .../design.md | 78 +++++++++++++++++++ .../proposal.md | 32 ++++++++ .../specs/supervised-contract-tuning/spec.md | 45 +++++++++++ .../tasks.md | 41 ++++++++++ 6 files changed, 276 insertions(+) create mode 100644 docs/human-briefs/2026-07-15-rerun-real-a100-sft-smoke-after-cli-fix-v1.html create mode 100644 openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/.openspec.yaml create mode 100644 openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/design.md create mode 100644 openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/proposal.md create mode 100644 openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/specs/supervised-contract-tuning/spec.md create mode 100644 openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/tasks.md diff --git a/docs/human-briefs/2026-07-15-rerun-real-a100-sft-smoke-after-cli-fix-v1.html b/docs/human-briefs/2026-07-15-rerun-real-a100-sft-smoke-after-cli-fix-v1.html new file mode 100644 index 0000000..b961a36 --- /dev/null +++ b/docs/human-briefs/2026-07-15-rerun-real-a100-sft-smoke-after-cli-fix-v1.html @@ -0,0 +1,78 @@ + + + + + + 阶段 A：修复后真实 A100 SFT smoke 重跑 + + + +

阶段 A：修复后真实 A100 SFT smoke 重跑

+

当前结论：固定提交上的全部 gate 与唯一一次真实 smoke 均已通过。launch count 精确为 1，没有重试；这只证明 bounded real training path 与 adapter update 可用，不是模型指标提升或 clean-evaluation readiness。

+ +

固定边界

+ + + + + + + +
变更rerun-real-a100-sft-smoke-after-cli-fix-v1
可执行提交df25648d1d837d7aa4af7757b9aea68783d2a5b6
模型Qwen2.5-7B-Instruct，本地文件，禁止下载
数据current formal manifest 的 train split，1–2 rows
预算exactly 1 optimizer step；单张显式选择且空闲的 A100
执行次数最多一次；失败、异常或 postcondition 不满足均不得重试
+ +

必须先通过的 gate

+
    +
  • 远端 tracked checkout clean，HEAD 精确等于固定提交，且 stdout isolation 修复存在。
    • +
  • private config 为 ignored、untracked、非 symlink。
    • +
  • 本地 Qwen2.5-7B-Instruct identity 与 inventory/hash 通过。
    • +
  • 显式单卡可见、完全空闲且 free-memory gate 通过。
    • +
  • 全新 output leaf 尚未创建，output identity policy 通过。
    • +
  • shared preflight exit 0、ready=true、无 blockers，并绑定精确数据与预算。
    • +
+ +

成功判定

+

必须同时满足：exit_code=0；stdout 恰为一个 JSON document；stderr 无竞争性结果 JSON；training_status=training_completed；smoke_status=SMOKE_COMPLETED；observed_optimizer_steps=1；training_rows_used 为 1 或 2；changed_adapter_tensor_count>0；all_adapter_tensors_finite=true；adapter config 与 weights 存在且非空；整个 run tree 不含完整 base model weights。

+ +

真实执行证据

+ + + + + + + + + + +
部署远端 tracked checkout clean，HEAD 精确为固定提交；stdout redirect 与 diagnostic prefix 修复均存在。
Private configgit_ignored=true、git_tracked=false、nonsymlink=true、mode 0600。
Model identityQwen/Qwen2.5-7B-Instruct，geometry 匹配，local-only；snapshot revision SHA-256 2a1aaaf30e79bc9817f385e41260f822b4533f570a62f7253afc536e8f724363；weight inventory 15,231,271,888 bytes。
GPU gate显式单张 A100-SXM4-80GB；visible count 1、compute process count 0、idle verified；free 78.734 GiB，阈值 35 GiB。
Data / budgetformal manifest public-sample-20260619T090925Z，train，2 rows，exactly 1 optimizer step。
Preflightexit 0；stdout 单 JSON、stderr 0 bytes；ready=true、blockers=[]、output ready。
Launch protocollaunch marker attempt=1，phase launch count 1；exit 0；stdout 6,758 bytes 且完整解析为单 JSON；stderr 1,573 bytes，未发现 whole-document 或 line-level JSON，也无竞争性 result fields。
Training resulttraining_completed、SMOKE_COMPLETED、optimizer steps 1、rows 2、changed adapter tensors 112/224、all finite；before/after digest 不同。
Artifactsadapter config 1,029 bytes；adapter safetensors 20,214,760 bytes；run tree 4 files / 20,234,578 bytes；base shard match 0、suspicious full-weight files 0。
+ +

证据与隐私

+

原始日志、私有绝对路径、配置内容、raw rows、模型文件、cache 与 adapter 只留在 ignored/private 远端证据区。仓库只记录固定提交、稳定状态、脱敏 hash/count、验证结果与真实 stop verdict；本地未提交 OpenSpec 不混入远端可执行 checkout。

+ +

明确不做

+

不运行 prediction、evaluation、DPO、GRPO、public test、lockbox 或 clean-evaluation；不 commit、push、更新或合并 PR #3，不 archive、release、production deploy；不声称模型提升、held-out/generalization gain、生产就绪或 live-browser benchmark 改善。

+ +

验证

+
    +
  • CLI/preflight/A100 smoke focused tests：206 passed。
    • +
  • public dataset-builder、schema、DPO-pair no-training contract tests：30 passed。
    • +
  • 本 change 与 Human Brief public leak scan：0 findings。
    • +
  • openspec validate --all --strict：16/16 passed。
    • +
  • git diff --check：passed；禁止 claim marker scan：passed。
    • +
  • git status --short：仅新增本 Human Brief 与 bounded OpenSpec change，均为 untracked；无代码修改。
    • +
+ +

下一步

+

验证已通过。保持变更 active，等待只读 Reviewer 与人类决定是否另行授权更新 PR #3。不得把后续 search-query 指标实验提交进 PR #3。

+ + diff --git a/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/.openspec.yaml b/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/.openspec.yaml new file mode 100644 index 0000000..4f63482 --- /dev/null +++ b/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-15 diff --git a/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/design.md b/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/design.md new file mode 100644 index 0000000..755f70a --- /dev/null +++ b/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/design.md @@ -0,0 +1,78 @@ +## Context + +The real A100 SFT path and all bounded smoke guards already exist on the PR #3 branch. The first authorized launch completed backend work but failed the public CLI protocol because downstream progress reached stdout. The fixed branch is now pinned to `df25648d1d837d7aa4af7757b9aea68783d2a5b6`; this change is an execution-and-evidence phase, not a new training implementation or metric experiment. + +The execution uses a private ignored runtime on the authorized A100 machine. Public repository artifacts may contain only sanitized hashes, counts, stable status values, and verification results. They must not contain the remote host, private absolute paths, config contents, raw training rows, raw logs, model files, adapter bytes, credentials, or connection details. + +## Goals / Non-Goals + +**Goals:** + +- Prove the exact fixed commit is deployed cleanly without copying this uncommitted OpenSpec change into the remote checkout. +- Re-establish every immutable gate before the only authorized launch. +- Execute at most one real Qwen2.5-7B-Instruct LoRA SFT smoke using one or two current-formal-manifest `train` rows and exactly one optimizer step on one explicitly selected idle A100. +- Independently verify the process, one-document CLI protocol, training completion, measured adapter update, adapter artifacts, and absence of full base-model weights. +- Preserve private evidence remotely while recording only sanitized, reviewable evidence in Git. + +**Non-Goals:** + +- Prediction, evaluation, DPO, GRPO, public test, lockbox, clean-evaluation population access, or any second launch. +- Hyperparameter, prompt, decoder, evaluator, curriculum, or metric experimentation. +- Generic chat fine-tuning, skill routing, GUI action-policy learning, first-phase GRPO, or public release of the full local corpus. +- Commit, push, PR mutation, merge, archive, release, deploy to production, checkpoint publication, model-improvement claim, live-browser benchmark claim, or readiness claim. + +## Decisions + +### 1. The exact committed branch is the only executable source + +The remote repository MUST be clean and pinned to the requested commit. The stdout fix MUST be demonstrated from that checkout before preflight. Local uncommitted OpenSpec/Human Brief files are never transferred into the runtime checkout. + +Alternative considered: copy the current worktree wholesale. Rejected because it would make the execution source differ from the fixed PR head and mix derived evidence files into runtime provenance. + +### 2. Gates are sequential and launch authorization is consumed once + +Before launch, verify the private config is ignored, untracked, and not a symlink; the local model has the required Qwen2.5-7B-Instruct identity and stable inventory/hash; the explicitly selected single visible A100 is idle and passes the configured free-memory threshold; the fresh output leaf is not pre-created and passes identity policy; and the shared preflight exits 0 with `ready=true` and no blockers. A private launch marker and count are written before invocation. Any failed gate consumes no launch but terminates the phase; after invocation, any failure terminates the phase and the launch is never repeated. + +Alternative considered: repair a failed gate or run automatically. Rejected because the authorization is for one audited attempt after a complete ready gate, not an adaptive retry loop. + +### 3. The CLI process is captured as three independent evidence channels + +Capture stdout, stderr, and exit code separately in the private evidence directory. Parse stdout as one complete JSON document with no prefix, suffix, or second document. Treat stderr as diagnostics only and reject any competing result JSON. Verify the result object and filesystem artifacts independently rather than trusting a single status field. + +Alternative considered: redirect stderr into stdout for convenience. Rejected because it destroys the protocol boundary that this rerun is intended to validate. + +### 4. Smoke completion requires all runtime and filesystem postconditions + +Success requires `exit_code=0`, one stdout JSON document, no competing stderr result, `training_status=training_completed`, `smoke_status=SMOKE_COMPLETED`, one observed optimizer step, one or two training rows, a positive changed-adapter-tensor count, finite adapter tensors, non-empty adapter config and adapter weight files, and no complete base-model weights anywhere in the run tree. The final check scans the complete fresh output tree, not only the adapter directory. + +Alternative considered: accept trainer completion plus saved files. Rejected because it would not prove a parameter update, numerical finiteness, CLI correctness, or absence of copied base weights. + +### 5. Public evidence is derived and sanitized + +OpenSpec tasks and the Human Brief record the fixed commit, stable gate names, booleans, hashes where safe, aggregate counts, result fields, validation commands, and the stop verdict. Raw private evidence remains ignored on the remote machine and is referenced to the controlling agent only, not embedded in repository files. + +Alternative considered: commit raw logs for auditability. Rejected because raw logs and paths can disclose private model/runtime details and training data. + +## Risks / Trade-offs + +- [A gate changes after preflight] → The real runner repeats bound checks before model loading; no manual bypass is permitted. +- [Another process occupies the GPU] → Select exactly one device that is fully idle at gate time and rely on the shared repeated occupancy guard; stop on drift. +- [The output directory is created concurrently] → Require a fresh absent leaf and the existing exclusive output-claim policy; stop on identity or existence drift. +- [Training succeeds but the CLI protocol is invalid] → Treat protocol failure as total smoke failure and do not retry. +- [A partial adapter exists after failure] → Preserve it only as private failure evidence; do not relabel it as a successful smoke or a control adapter. +- [Sanitization weakens reproducibility] → Preserve the complete immutable raw evidence privately and expose stable hashes/counts/statuses publicly. + +## Migration Plan + +1. Validate this bounded change locally. +2. Deploy only the exact committed fixed head to the existing private remote repository and prove the checkout is clean. +3. Run the ordered read-only gates and shared preflight. +4. If and only if every gate passes, create the launch marker and execute the single authorized smoke once. +5. Verify all postconditions independently and update the OpenSpec tasks/Human Brief with sanitized evidence. +6. Stop without archive, commit, push, PR mutation, merge, release, or any additional run. + +Rollback is evidence-only: repository files can be reverted locally, while any private failed output remains isolated and untracked for audit. A failed or interrupted launch is not retried under this authorization. + +## Open Questions + +None. Model, data split, row count, optimizer-step budget, GPU count, gate order, success criteria, privacy boundary, and prohibited actions are fixed by the request. diff --git a/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/proposal.md b/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/proposal.md new file mode 100644 index 0000000..3481bee --- /dev/null +++ b/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/proposal.md @@ -0,0 +1,32 @@ +## Why + +The prior authorized real one-step SFT process reached training but its CLI stdout was contaminated by downstream progress output, so it did not produce a legally auditable smoke result. Commit `df25648d1d837d7aa4af7757b9aea68783d2a5b6` contains the reviewed stdout isolation fix; this change authorizes one and only one fail-closed rerun to determine whether the bounded real A100 smoke now satisfies the complete contract. + +## What Changes + +- Verify the remote checkout is clean and pinned to `df25648d1d837d7aa4af7757b9aea68783d2a5b6`, including the stdout-isolation fix. +- Re-establish every private-config, local-model, explicit-idle-A100, free-memory, fresh-output, and shared-preflight gate before execution. +- Authorize exactly one real local-only Qwen2.5-7B-Instruct LoRA SFT smoke over one or two rows from the current formal manifest `train` split for exactly one optimizer step. +- Accept success only when the process, CLI protocol, training metadata, measured adapter update, adapter artifacts, and no-full-base-weight postconditions all pass together. +- Stop on the first failed gate or postcondition and never retry automatically. +- Record only sanitized execution evidence in OpenSpec and the Human Brief; keep private paths, config, raw rows, logs, adapters, model files, host details, and credentials outside Git. +- Explicitly do not run prediction, evaluation, DPO, GRPO, public test, lockbox, or clean-evaluation work, and do not merge, archive, release, deploy, or claim model improvement. + +Non-goals also include generic chat fine-tuning, skill routing, GUI action-policy learning, first-phase GRPO, public release of the full local corpus, a released checkpoint, or any live-browser benchmark claim. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `supervised-contract-tuning`: Add the operational acceptance contract for a single post-fix real A100 smoke rerun, including exact launch authorization, postconditions, evidence privacy, and no-retry behavior. + +## Impact + +- OpenSpec and derived Human Brief receive sanitized gate and result evidence. +- The private remote runtime may create one fresh ignored output tree containing the adapter and raw execution evidence only after all preflight gates pass. +- No production code, public dataset, model weights, evaluation population, prompt/evaluator policy, or clean-evaluation truth field is changed. +- PR #3 may later receive the sanitized evidence through a separately authorized update; this change does not authorize commit, push, PR mutation, merge, or archive. diff --git a/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/specs/supervised-contract-tuning/spec.md b/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/specs/supervised-contract-tuning/spec.md new file mode 100644 index 0000000..6bdc181 --- /dev/null +++ b/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/specs/supervised-contract-tuning/spec.md @@ -0,0 +1,45 @@ +## ADDED Requirements + +### Requirement: Rerun one real A100 SFT smoke after the CLI stdout fix +The system MUST permit at most one real SFT launch from clean commit `df25648d1d837d7aa4af7757b9aea68783d2a5b6`, using only the local Qwen2.5-7B-Instruct runtime, one or two ordered rows from the current formal manifest `train` split, exactly one optimizer step, and one explicitly selected idle A100. The launch MUST NOT perform prediction, evaluation, DPO, GRPO, public test, lockbox, or clean-evaluation population access. + +#### Scenario: Establish the executable commit +- **WHEN** the bounded rerun is prepared +- **THEN** the remote tracked checkout is clean, its HEAD equals the fixed commit, the stdout-isolation fix is present, and no local uncommitted planning artifact is deployed with it + +#### Scenario: Reject execution outside the fixed scope +- **WHEN** model identity, split, selected row count, optimizer-step budget, visible GPU count, or requested action differs from the authorized values +- **THEN** the system stops before launch and does not substitute another model, dataset, budget, GPU arrangement, or action + +### Requirement: Re-establish every real-smoke gate before launch +Before the launch, the system MUST prove that the private config is ignored, untracked, and not a symlink; the local model identity and inventory/hash match Qwen2.5-7B-Instruct; exactly one explicitly selected A100 is visible, fully idle, and passes the configured free-memory gate; the requested output leaf is fresh, absent, and passes the bound output-identity policy; and the shared preflight exits 0 with `ready=true`, no blockers, and the exact bounded configuration. A launch marker and launch count MUST make the single authorization auditable. + +#### Scenario: All gates pass +- **WHEN** every config, model, GPU, output, Git, dependency, data, objective, and shared-preflight condition passes +- **THEN** the system records the one authorized launch marker and invokes the real SFT process once + +#### Scenario: Any gate fails +- **WHEN** any required gate is false, unavailable, inconsistent, or changes before invocation +- **THEN** the system records the stable blocker, performs no training launch, and stops without automatic repair, fallback, or retry + +### Requirement: Accept smoke success only from the complete evidence conjunction +The run MUST be successful only when all of the following are independently true: process exit code is 0; stdout is exactly one complete JSON document; stderr contains no competing result JSON; `training_status=training_completed`; `smoke_status=SMOKE_COMPLETED`; `observed_optimizer_steps=1`; `training_rows_used` is 1 or 2; `changed_adapter_tensor_count>0`; `all_adapter_tensors_finite=true`; adapter configuration and adapter weights exist and are non-empty; and the complete run tree contains no full base-model weight file or full base-model weight inventory. + +#### Scenario: Complete success evidence +- **WHEN** the single process returns and every process, protocol, metadata, update, artifact, and no-base-weight postcondition passes +- **THEN** the bounded change records the result as a completed real one-step infrastructure smoke with sanitized evidence + +#### Scenario: Any postcondition fails +- **WHEN** the process is non-zero, stdout is not one JSON document, stderr contains a competing result, any required field is absent or wrong, the adapter update/artifact evidence is insufficient, or the run tree contains full base-model weights +- **THEN** the bounded change records the exact failed postcondition, does not report smoke success, and stops without a second launch + +### Requirement: Keep real-smoke evidence private and claims narrow +Raw config, private paths, host details, raw selected rows, logs, cache, local model files, and adapter artifacts MUST remain in ignored private storage. Repository evidence MUST be limited to sanitized commit identity, stable hashes and counts, gate outcomes, result fields, validation results, and the truthful stop verdict. Smoke success MUST NOT change any clean-evaluation truth field or imply model quality, held-out gain, generalization gain, production readiness, released weights, or live-browser benchmark improvement. + +#### Scenario: Record sanitized success or blocker evidence +- **WHEN** the phase stops after a gate or after the single launch +- **THEN** OpenSpec and the Human Brief contain enough sanitized evidence to audit the decision without disclosing private runtime values or training content + +#### Scenario: Preserve lifecycle and claim boundaries +- **WHEN** the execution evidence is ready for human review +- **THEN** the system leaves the change active, does not commit, push, mutate or merge PR #3, and does not archive, release, deploy, run metric experiments, or make a model-improvement/readiness claim diff --git a/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/tasks.md b/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/tasks.md new file mode 100644 index 0000000..14108ac --- /dev/null +++ b/openspec/changes/rerun-real-a100-sft-smoke-after-cli-fix-v1/tasks.md @@ -0,0 +1,41 @@ +## 1. Bounded Change and Source Identity + +- [x] 1.1 Confirm the isolated local worktree is clean at `df25648d1d837d7aa4af7757b9aea68783d2a5b6` before creating this change +- [x] 1.2 Create proposal, design, delta spec, task checklist, and the initial Chinese Human Brief with the one-launch, no-retry, privacy, and prohibited-action boundaries +- [x] 1.3 Validate the change artifacts strictly before any remote deployment or preflight + - Fresh `openspec validate rerun-real-a100-sft-smoke-after-cli-fix-v1 --strict`: passed. + +## 2. Exact Remote Deployment and Ordered Gates + +- [x] 2.1 Deploy only committed head `df25648d1d837d7aa4af7757b9aea68783d2a5b6` to the existing private runtime repository; prove remote HEAD and tracked cleanliness without copying local uncommitted artifacts + - Normal fetch did not complete; a verified 39 KiB incremental Git bundle containing only the committed branch delta from the already present base was used. Remote HEAD equals the fixed commit and tracked change count is zero. +- [x] 2.2 Prove the CLI stdout-isolation fix is present and the private config is ignored, untracked, and not a symlink + - The fixed checkout contains both the stdout redirect and diagnostic-prefix markers. Private config facts are `git_ignored=true`, `git_tracked=false`, `nonsymlink=true`, with mode `0600`. +- [x] 2.3 Prove the local-only Qwen2.5-7B-Instruct identity/inventory hash, select exactly one fully idle A100 explicitly, and pass its free-memory gate + - Model public identity is `Qwen/Qwen2.5-7B-Instruct`; geometry matches, local-only mode is true, snapshot revision SHA-256 is `2a1aaaf30e79bc9817f385e41260f822b4533f570a62f7253afc536e8f724363`, and weight inventory totals 15,231,271,888 bytes. One A100-SXM4-80GB was selected explicitly with zero compute processes and 78.734 GiB free against the 35 GiB threshold. +- [x] 2.4 Prove the fresh output leaf is absent and passes output identity policy, then run the shared preflight once and require exit 0, `ready=true`, no blockers, the current formal manifest `train` split, one or two rows, and exactly one optimizer step + - The output leaf was absent and its root was an absolute existing non-symlink directory. Shared preflight exited 0 with exactly one stdout JSON document, empty stderr, `ready=true`, `blockers=[]`, output ready, current formal manifest `public-sample-20260619T090925Z` / `train`, 2 selected rows, and `max_steps=1`. + +## 3. Single Authorized Smoke and Independent Verification + +- [x] 3.1 If and only if every gate passes, write the private launch marker/count and invoke the real SFT smoke exactly once; otherwise record the blocker and stop + - The private atomic launch marker records `attempt=1`; the phase launch-directory count is exactly 1 and no second invocation occurred. +- [x] 3.2 Capture stdout, stderr, and exit code independently; prove exit 0, stdout is exactly one JSON document, and stderr has no competing result JSON + - Exit code is 0. Stdout is 6,758 bytes and parses in full as exactly one JSON document. Stderr is 1,573 bytes; whole-document and line-level JSON parsing found zero JSON documents, and result-field scans found no competing result. +- [x] 3.3 Prove `training_status=training_completed`, `smoke_status=SMOKE_COMPLETED`, `observed_optimizer_steps=1`, `training_rows_used` is 1 or 2, `changed_adapter_tensor_count>0`, and `all_adapter_tensors_finite=true` + - Observed values are `training_completed`, `SMOKE_COMPLETED`, 1 optimizer step, 2 rows, 112 changed adapter tensors out of 224, and all adapter tensors finite. Before/after adapter digests differ. +- [x] 3.4 Prove non-empty adapter configuration and adapter weights exist, scan the whole run tree for absence of full base-model weights, and prove this phase recorded only one launch + - The adapter config is 1,029 bytes and the adapter safetensors file is 20,214,760 bytes. The full run tree contains 4 files totaling 20,234,578 bytes; exact base-shard matches and suspicious full-weight files are both zero. Launch count remains 1. + +## 4. Sanitized Evidence and Validation + +- [x] 4.1 Keep raw logs, paths, config, rows, cache, model files, and adapter artifacts private; update OpenSpec and the Human Brief only with sanitized gate/result evidence and the truthful completed-or-blocked verdict + - Repository evidence contains only public commit/model/dataset identities, hashes, aggregate counts, status fields, and validation results. Raw runtime evidence and adapter files remain ignored/private on the authorized machine. +- [x] 4.2 Run focused CLI/preflight regression tests for the fixed stdout and smoke contract + - Fresh focused result: `206 passed` across the review-hardening, private-preflight, and A100 SFT smoke modules. +- [x] 4.3 Run fresh public dataset-builder, schema-metric, and DPO-pair contract checks only through their existing no-training test entry points; record them as N/A only if the repository exposes no applicable entry point for this evidence-only change + - Fresh no-training contract result: `30 passed` across `test_dataset_builder.py`, `test_schemas.py`, and `test_dpo_validation.py`. +- [x] 4.4 Run the existing public-leak/ignore checks, `openspec validate --all --strict`, and `git diff --check`; report `git status --short` + - Leak scan of this change and Human Brief passed with zero findings; remote ignore/untracked/non-symlink checks passed; strict OpenSpec validation passed 16/16; `git diff --check` passed. Status contains only the new Human Brief and new bounded change as untracked paths. +- [x] 4.5 Stop with the OpenSpec change active and without archive, commit, push, PR mutation, merge, release, deploy, prediction, evaluation, DPO, GRPO, test/lockbox access, clean-evaluation work, or a model-improvement/readiness claim + - The change remains active. No protected integration or prohibited execution occurred; forbidden claim-marker scan passed.