From 4ca6ae53d34ccf51d65f62d33af4a675103ef405 Mon Sep 17 00:00:00 2001 From: chaudhryumer Date: Tue, 18 Aug 2026 10:02:39 -0700 Subject: [PATCH 1/3] feat: update solver code and finalize gitignore --- .gitignore | 4 ++ config.json | 6 +-- docs/EVAL_RESULTS.md | 22 ++-------- docs/TRAINING_RESULTS.md | 29 ++++++-------- eval/run_eval.py | 18 ++++++--- inference/beam_search.py | 21 ++++++++-- inference/solve.py | 16 ++++---- model/transformer.py | 2 +- train.py | 87 ++++++++++++++++++++++++---------------- 9 files changed, 115 insertions(+), 90 deletions(-) diff --git a/.gitignore b/.gitignore index 767a40f..b459cc7 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ logs/ # Model Checkpoints (avoid committing large binary weights) checkpoints/ + +*.pt + +*.pt diff --git a/config.json b/config.json index e9611cd..aea5165 100644 --- a/config.json +++ b/config.json @@ -2,10 +2,10 @@ "learning_rate": 0.0001, "warmup_steps": 1000, "batch_size": 32, - "max_steps": 3500, + "max_steps": 500, "hidden_dim": 256, - "max_len": 32, - "epochs": 20, + "max_len": 48, + "epochs": 5, "grad_clip_max_norm": 1.0, "early_stopping": { "patience": 15, diff --git a/docs/EVAL_RESULTS.md b/docs/EVAL_RESULTS.md index 0a9145d..e54d4e7 100644 --- a/docs/EVAL_RESULTS.md +++ b/docs/EVAL_RESULTS.md @@ -1,26 +1,12 @@ # Evaluation Results -> **STALE -- BLOCKED ON DEV 3.** Per `Objectives v3`, these numbers predate -> every fix in that document (train.py crash fix, max_len 32->48, pre-flight -> check) and must not be treated as current model quality. No -> `checkpoints/final/best.pt` exists in this environment yet (`checkpoints/` -> is gitignored and train.py currently cannot complete a run -- see -> Dev 3 objective 1, the `model()` call with undefined `src_seq`/`tgt_in`). -> -> Dev 2's regeneration objective is unblocked only once Dev 3 lands: -> 1. the crash fix, 2. `max_len` raised to >=48, 3. a completed training run. -> -> Once that checkpoint lands, regenerate this file with: -> `python eval/run_eval.py` -> Do not hand-edit the table below in the meantime. - **Checkpoint Evaluated:** `checkpoints\final\best.pt` | Operation | Total Problems | Exact Match (Accuracy) | Verification Rate | |---|---|---|---| -| diff | 80 | 19/80 (23.8%) | 19/80 (23.8%) | +| diff | 80 | 0/80 (0.0%) | 0/80 (0.0%) | | gradient | 50 | 0/50 (0.0%) | 0/50 (0.0%) | -| integrate | 60 | 37/60 (61.7%) | 37/60 (61.7%) | -| partial | 60 | 9/60 (15.0%) | 9/60 (15.0%) | +| integrate | 60 | 0/60 (0.0%) | 0/60 (0.0%) | +| partial | 60 | 0/60 (0.0%) | 0/60 (0.0%) | | tangent_line | 50 | 0/50 (0.0%) | 0/50 (0.0%) | -| **Overall** | **300** | **65/300 (21.7%)** | **65/300 (21.7%)** | +| **Overall** | **300** | **0/300 (0.0%)** | **0/300 (0.0%)** | diff --git a/docs/TRAINING_RESULTS.md b/docs/TRAINING_RESULTS.md index 8564b85..839b8b5 100644 --- a/docs/TRAINING_RESULTS.md +++ b/docs/TRAINING_RESULTS.md @@ -1,33 +1,28 @@ # Training Results -**Git Commit Hash:** `d26bbe19f4b0b694d6bca560b51cd60e42a36cbc` -**Best Validation Loss:** 0.0167 -**Total Epochs Run:** 10 +**Git Commit Hash:** `c16c7c9963834c864b98c44d18f77acae8cf757d` +**Best Validation Loss:** 0.0149 +**Total Epochs Run:** 5 ## Per-Epoch Metrics | Epoch | Train Loss | Val Loss | Per-Token Acc | Val Seq Acc | Saved | |-------|-----------|----------|---------------|-------------|-------| -| 1 | 0.2888 | 0.0200 | 0.9856 | 0.7683 | Yes | -| 2 | 0.0191 | 0.0170 | 0.9898 | 0.8346 | Yes | -| 3 | 0.0179 | 0.0170 | 0.9898 | 0.8345 | No | -| 4 | 0.0176 | 0.0202 | 0.9892 | 0.8252 | No | -| 5 | 0.0173 | 0.0167 | 0.9899 | 0.8363 | Yes | -| 6 | 0.0172 | 0.0171 | 0.9898 | 0.8346 | No | -| 7 | 0.0170 | 0.0168 | 0.9899 | 0.8363 | No | -| 8 | 0.0170 | 0.0173 | 0.9897 | 0.8335 | No | -| 9 | 0.0171 | 0.0167 | 0.9898 | 0.8356 | No | -| 10 | 0.0168 | 0.0169 | 0.9898 | 0.8356 | No | +| 1 | 1.2289 | 0.2352 | 0.9384 | 0.2080 | Yes | +| 2 | 0.0951 | 0.0417 | 0.9849 | 0.8426 | Yes | +| 3 | 0.0312 | 0.0212 | 0.9928 | 0.9174 | Yes | +| 4 | 0.0207 | 0.0169 | 0.9940 | 0.9272 | Yes | +| 5 | 0.0178 | 0.0149 | 0.9948 | 0.9391 | Yes | ## Configuration Snapshot -- **Architecture:** SimpleCalculusModel (standard nn.Transformer encoder-decoder) - **Learning Rate:** 0.0001 - **Warmup Steps:** 1000 - **Batch Size:** 32 - **Hidden Dim:** 256 -- **Max Steps/Epoch:** 3500 -- **Early Stopping:** patience=12, min_delta=0.0002 +- **Max Len:** 48 +- **Max Steps/Epoch:** 500 +- **Early Stopping:** patience=15, min_delta=0.0002 - **Vocab Size:** 124 - **Gradient Clipping:** max_norm=1.0 -- **Rule Prediction:** Folded into output sequence as leading RULE:xxx token +- **Rule Prediction:** Multi-head prediction output (decoder_logits, rule_logits, verifier_logits) diff --git a/eval/run_eval.py b/eval/run_eval.py index e167ef8..eeff660 100644 --- a/eval/run_eval.py +++ b/eval/run_eval.py @@ -3,6 +3,11 @@ import sys import glob from pathlib import Path +import torch + +# Performance optimizations for CPU execution +torch.set_grad_enabled(False) +torch.set_num_threads(4) # Adjust based on your CPU physical core count # Ensure project root is in path ROOT = Path(__file__).resolve().parents[1] @@ -18,8 +23,9 @@ def main(): print(f"Error: checkpoint {checkpoint_path} does not exist.") sys.exit(1) - print("Loading neural model...") - solver = CalculusSolverInference(model_path=str(checkpoint_path)) + print("Loading neural model (with beam_size=1 for fast evaluation)...") + # Set beam_size=1 to avoid freezing and speed up inference significantly + solver = CalculusSolverInference(model_path=str(checkpoint_path), beam_size=1) benchmark_dir = ROOT / "eval" / "benchmarks" benchmark_files = glob.glob(str(benchmark_dir / "*.json")) @@ -49,7 +55,8 @@ def main(): verified_count = 0 op_total = len(problems) - for p in problems: + print(f"Evaluating {op_name} ({op_total} problems)...") + for i, p in enumerate(problems): expr = p["expr"] target = p["target"] @@ -63,7 +70,7 @@ def main(): if res.get("verified", False): verified_count += 1 except Exception as e: - print(f"Error evaluating problem: {e}") + print(f"Error evaluating problem {i} in {op_name}: {e}") accuracy = exact_match_count / op_total if op_total > 0 else 0.0 ver_rate = verified_count / op_total if op_total > 0 else 0.0 @@ -81,10 +88,11 @@ def main(): # Write report eval_results_path = ROOT / "docs" / "EVAL_RESULTS.md" + eval_results_path.parent.mkdir(parents=True, exist_ok=True) with open(eval_results_path, "w", encoding="utf-8") as f: f.write("\n".join(report_lines) + "\n") print(f"Saved evaluation results to {eval_results_path}") if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/inference/beam_search.py b/inference/beam_search.py index 16a0c11..e928901 100644 --- a/inference/beam_search.py +++ b/inference/beam_search.py @@ -20,8 +20,15 @@ def beam_search( max_len: int = 32, node_pool: Optional[NodeValidityPool] = None, ) -> Dict[str, Any]: - """Simplified beam search for SimpleCalculusModel -- one model call - per step (src_seq, tgt_in_seq), no rule_embeddings, no tree kwargs.""" + """Beam search for CalculusSolverModel (tree-based, model/transformer.py). + + FIX: model() returns a 3-tuple (decoder_logits, rule_logits, + verifier_logits), not a single tensor. Only decoder_logits is used for + next-token selection here. The previous version indexed the raw + 3-tuple directly (logits[0, -1, :]), which raised "tuple indices must + be integers or slices, not tuple" on every single call, regardless of + model quality. Fixed by unpacking model_output[0] before indexing. + """ device = src_tokens.device vocab = vocab_map["token_to_id"] id_to_token = vocab_map["id_to_token"] @@ -53,8 +60,14 @@ def beam_search( ) tgt = torch.tensor([current_tokens], device=device) - logits = model(src_tokens, tgt) - next_logits = logits[0, -1, :] + + # FIX: unpack the tuple safely -- works whether model() returns + # a single tensor or a (decoder_logits, rule_logits, + # verifier_logits) tuple, so future model interface changes + # won't silently reintroduce this same crash. + model_output = model(src_tokens, tgt) + decoder_logits = model_output[0] if isinstance(model_output, tuple) else model_output + next_logits = decoder_logits[0, -1, :] mask = node_pool.mask(validity_tokens, all_candidate_tokens) invalid_mask = torch.tensor([not v for v in mask], device=device) diff --git a/inference/solve.py b/inference/solve.py index 18187b9..62d852d 100644 --- a/inference/solve.py +++ b/inference/solve.py @@ -138,18 +138,18 @@ def solve(self, input_env: Dict[str, Any]) -> Dict[str, Any]: padded_tokens = token_ids + [self.pad_id] * (self.max_len - len(token_ids)) src_tokens = torch.tensor([padded_tokens], dtype=torch.long, device=self.device) - src_positions = torch.zeros( - (1, self.max_len, 3), dtype=torch.float32, device=self.device - ) - parent_child_pairs = torch.zeros( - (1, self.max_len, self.max_len), dtype=torch.float32, device=self.device - ) + # FIX: this environment's inference/beam_search.py has the + # simplified signature (model, src_tokens, vocab_map, beam_size, + # max_len, node_pool) -- it does not accept src_positions or + # parent_child_pairs as external arguments. model/transformer.py's + # CalculusSolverModel.forward() builds these zero-tensors + # internally, so they were never needed here; passing them caused + # "beam_search() got an unexpected keyword argument 'src_positions'" + # on every single solve() call. result = beam_search( model=self.model, src_tokens=src_tokens, - src_positions=src_positions, - parent_child_pairs=parent_child_pairs, vocab_map=self.vocab_map, beam_size=self.beam_size, max_len=self.max_len, diff --git a/model/transformer.py b/model/transformer.py index 8497a45..908cc39 100644 --- a/model/transformer.py +++ b/model/transformer.py @@ -58,7 +58,7 @@ def __init__( templates=templates ) - def forward(self, src_seq, tgt_in_seq): + def forward(self, src_seq, tgt_in_seq, true_rule_ids=None): device = src_seq.device batch_size, seq_len = src_seq.size() diff --git a/train.py b/train.py index e60fc01..c57088c 100644 --- a/train.py +++ b/train.py @@ -12,7 +12,6 @@ from tokenizer.slang_serializer import serialize_slang_math from solver_model import CalculusSolverModel -from checkpoint_provenance import stamp_and_record with open("config.json", "r") as cfg_file: config = json.load(cfg_file) @@ -31,8 +30,6 @@ def flatten_vocab(raw_vocab): """ Same flattening rule as inference/beam_search.flatten_vocab on org main: merge every sub-dict, skip keys starting with '_' (e.g. _comment, _version). - Keeping this identical to beam_search's version on purpose, so training-time - token IDs and inference-time token IDs can never drift apart again. """ flat = {} for key, value in raw_vocab.items(): @@ -49,14 +46,15 @@ def flatten_vocab(raw_vocab): vocab_mapping = flatten_vocab(_raw_vocab) # IDs are NOT contiguous (gaps by design — see docs/KNOWN_ISSUES.md, STRUCT:OPEN @ 23). -# len(vocab_mapping) undercounts; embedding table must cover the highest real ID. REAL_VOCAB_SIZE = max(vocab_mapping.values()) + 1 # Rule labels/tokens, derived from vocab's rule_tokens, ordered by ID. _rule_items = sorted(_raw_vocab.get("rule_tokens", {}).items(), key=lambda kv: kv[1]) RULE_LABELS = [name.split("RULE:", 1)[1] for name, _ in _rule_items] +RULE_TOKEN_STRINGS = [name for name, _ in _rule_items] -MAX_LEN = config.get("max_len", 32) +MAX_LEN = config.get("max_len", 48) +PAD_ID = vocab_mapping["[PAD]"] CHECKPOINT_DIR = Path("checkpoints/final") FINAL_CHECKPOINT_PATH = CHECKPOINT_DIR / "best.pt" @@ -73,9 +71,10 @@ def __init__(self, file_path, max_len=MAX_LEN): def __len__(self): return len(self.data) - def _tokenize(self, envelope, add_boundaries=False): - # serialize_slang_math returns a single List[str] — no parent/child tuple. + def _tokenize(self, envelope, extra_prefix_tokens=None, add_boundaries=False): tokens = serialize_slang_math(envelope) + if extra_prefix_tokens: + tokens = list(extra_prefix_tokens) + tokens if add_boundaries: tokens = ["[BOS]"] + tokens + ["[EOS]"] @@ -105,8 +104,8 @@ def __getitem__(self, idx): prefix = [rule_token] if rule_token else [] src_ids = self._tokenize(item["src_tokens"], add_boundaries=False) - tgt_in_ids = self._tokenize(item["tgt_input_tokens"], add_boundaries=True) - tgt_out_ids = self._tokenize(item["tgt_output_tokens"], add_boundaries=True) + tgt_in_ids = self._tokenize(item["tgt_input_tokens"], extra_prefix_tokens=prefix, add_boundaries=True) + tgt_out_ids = self._tokenize(item["tgt_output_tokens"], extra_prefix_tokens=prefix, add_boundaries=True) return { "src_seq": src_ids, "tgt_in_seq": tgt_in_ids, @@ -116,7 +115,31 @@ def __getitem__(self, idx): } -def evaluate_validation(model, val_loader, criterion_sequence, criterion_rule, criterion_verify): +def preflight_check_max_len(dataset_path, max_len): + worst = 0 + worst_field = None + seen = 0 + with open(dataset_path, encoding="utf-8") as f: + for line in f: + row = json.loads(line) + for field in ("src_tokens", "tgt_input_tokens", "tgt_output_tokens"): + n = len(serialize_slang_math(row[field])) + 2 + if n > worst: + worst = n + worst_field = field + seen += 1 + print(f"[Pre-flight] Scanned {seen} rows in {dataset_path}. " + f"Max token length observed: {worst} (field: {worst_field}). " + f"Configured max_len: {max_len}.") + if worst > max_len: + print(f"[Pre-flight] WARNING: real max length ({worst}) exceeds " + f"max_len ({max_len}) -- sequences WILL be silently truncated. " + f"Raise max_len in config.json before continuing.") + else: + print(f"[Pre-flight] OK: max_len has {max_len - worst} tokens of headroom.") + + +def evaluate_validation(model, val_loader, criterion): model.eval() total_loss = 0.0 total_correct_seq = 0 @@ -130,27 +153,25 @@ def evaluate_validation(model, val_loader, criterion_sequence, criterion_rule, c src_seq = batch["src_seq"] tgt_in = batch["tgt_in_seq"][:, :-1] tgt_out = batch["tgt_out_seq"][:, 1:] + rule_id = batch["rule_id"] - logits = model(src_seq, tgt_in) - loss = criterion( - logits.reshape(-1, REAL_VOCAB_SIZE), tgt_out.reshape(-1) - ) + # REPLACED CODE: updated to multi-output model forward pass + decoder_logits, rule_logits, verifier_logits = model(src_seq, tgt_in, true_rule_ids=rule_id) + loss = criterion(decoder_logits.reshape(-1, REAL_VOCAB_SIZE), tgt_out.reshape(-1)) total_loss += loss.item() - preds = logits.argmax(dim=-1) + preds = decoder_logits.argmax(dim=-1) mask = tgt_out != PAD_ID - # Per-token accuracy logic correct_token_mask = (preds == tgt_out) & mask total_correct_tokens += correct_token_mask.sum().item() total_valid_tokens += mask.sum().item() - # Exact sequence match logic correct_seq = ((preds == tgt_out) | ~mask).all(dim=1) total_correct_seq += correct_seq.sum().item() total_seq += tgt_out.size(0) steps += 1 - + if steps == 0: return 0.0, 0.0, 0.0 @@ -198,11 +219,12 @@ def write_training_results(metrics_log, best_val_loss, git_commit_hash): f"- **Warmup Steps:** {config.get('warmup_steps', 1000)}", f"- **Batch Size:** {config.get('batch_size')}", f"- **Hidden Dim:** {config.get('hidden_dim')}", + f"- **Max Len:** {MAX_LEN}", f"- **Max Steps/Epoch:** {config.get('max_steps')}", f"- **Early Stopping:** patience={config.get('early_stopping', {}).get('patience', 'N/A')}, min_delta={config.get('early_stopping', {}).get('min_delta', 'N/A')}", f"- **Vocab Size:** {REAL_VOCAB_SIZE}", f"- **Gradient Clipping:** max_norm={config.get('grad_clip_max_norm', 1.0)}", - f"- **Rule Prediction:** Folded into output sequence as leading RULE:xxx token", + f"- **Rule Prediction:** Multi-head prediction output (decoder_logits, rule_logits, verifier_logits)", "", ]) @@ -213,13 +235,15 @@ def write_training_results(metrics_log, best_val_loss, git_commit_hash): def run_training_pipeline(): commit_hash = get_git_commit_hash() - print(f"--- Training SimpleCalculusModel (commit: {commit_hash}, vocab: {REAL_VOCAB_SIZE}) ---") + print(f"--- Training CalculusSolverModel (commit: {commit_hash}, vocab: {REAL_VOCAB_SIZE}) ---") train_file = Path("data/splits/train.jsonl") if not train_file.exists(): print("CRITICAL: Train split missing! Run problem_generator.py first.") sys.exit(1) + preflight_check_max_len(train_file, MAX_LEN) + print("[DEBUG] Loading train dataset into memory...", flush=True) train_dataset = SlangDatasetLoader(train_file) print(f"[DEBUG] Train dataset loaded: {len(train_dataset)} examples", flush=True) @@ -244,7 +268,6 @@ def run_training_pipeline(): base_lr = config["learning_rate"] optimizer = torch.optim.Adam(model.parameters(), lr=base_lr) - # Linear Warmup Scheduler setup warmup_steps = config.get("warmup_steps", 1000) def lr_lambda(current_step): if current_step < warmup_steps: @@ -259,7 +282,7 @@ def lr_lambda(current_step): best_val_loss = float("inf") patience_counter = 0 metrics_log = [] - + early_stopping_cfg = config.get("early_stopping", False) if isinstance(early_stopping_cfg, dict): patience = early_stopping_cfg.get("patience", 3) @@ -285,7 +308,7 @@ def lr_lambda(current_step): model.train() epoch_loss = 0.0 steps_run = 0 - + for step, batch in enumerate(train_loader): if step >= config.get("max_steps", 1500): break @@ -293,15 +316,14 @@ def lr_lambda(current_step): print(f"[DEBUG] epoch {epoch} step {step} - batch received, running forward/backward...", flush=True) optimizer.zero_grad() - batch_size, seq_len = batch["src_seq"].shape - - decoder_logits, rule_logits, verifier_logits = model( - batch["src_seq"], - batch["tgt_in_seq"], - ) + src_seq = batch["src_seq"] + tgt_in = batch["tgt_in_seq"][:, :-1] + tgt_out = batch["tgt_out_seq"][:, 1:] + rule_id = batch["rule_id"] - logits = model(src_seq, tgt_in) - loss = criterion(logits.reshape(-1, REAL_VOCAB_SIZE), tgt_out.reshape(-1)) + # REPLACED CODE: updated to multi-output model forward pass + decoder_logits, rule_logits, verifier_logits = model(src_seq, tgt_in, true_rule_ids=rule_id) + loss = criterion(decoder_logits.reshape(-1, REAL_VOCAB_SIZE), tgt_out.reshape(-1)) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=grad_clip_max_norm) @@ -316,7 +338,6 @@ def lr_lambda(current_step): current_lr = scheduler.get_last_lr()[0] print(f"Epoch {epoch}/{epochs} - Train Loss: {avg_train_loss:.4f} (LR: {current_lr:.6f})") - # ── Validation + best-checkpoint logic ──────────────────────────────── epoch_metrics = { "epoch": epoch, "train_loss": avg_train_loss, @@ -337,7 +358,6 @@ def lr_lambda(current_step): epoch_metrics["val_token_acc"] = val_token_acc epoch_metrics["val_seq_acc"] = val_seq_acc - # Best-checkpoint logic: only save when val loss improves if val_loss < best_val_loss - (min_delta if use_early_stopping else 0): best_val_loss = val_loss patience_counter = 0 @@ -353,7 +373,6 @@ def lr_lambda(current_step): metrics_log.append(epoch_metrics) break else: - # No validation set — save every epoch CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True) torch.save(model.state_dict(), str(FINAL_CHECKPOINT_PATH)) print(f"Checkpoint saved to {FINAL_CHECKPOINT_PATH}") From 750cc2acf377ceb168de40dab3eac02c8605fe42 Mon Sep 17 00:00:00 2001 From: chaudhryumer Date: Wed, 19 Aug 2026 01:34:39 -0700 Subject: [PATCH 2/3] fix: resolve merge conflict and update inference scripts --- eval/run_eval.py | 8 ++++---- inference/solve.py | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/eval/run_eval.py b/eval/run_eval.py index eeff660..623ad6e 100644 --- a/eval/run_eval.py +++ b/eval/run_eval.py @@ -23,9 +23,9 @@ def main(): print(f"Error: checkpoint {checkpoint_path} does not exist.") sys.exit(1) - print("Loading neural model (with beam_size=1 for fast evaluation)...") - # Set beam_size=1 to avoid freezing and speed up inference significantly - solver = CalculusSolverInference(model_path=str(checkpoint_path), beam_size=1) + print("Loading neural model (with beam_size=2 for fast evaluation)...") + # Set beam_size=2 to avoid freezing and speed up inference significantly + solver = CalculusSolverInference(model_path=str(checkpoint_path), beam_size=2) benchmark_dir = ROOT / "eval" / "benchmarks" benchmark_files = glob.glob(str(benchmark_dir / "*.json")) @@ -95,4 +95,4 @@ def main(): print(f"Saved evaluation results to {eval_results_path}") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/inference/solve.py b/inference/solve.py index 62d852d..3edfff3 100644 --- a/inference/solve.py +++ b/inference/solve.py @@ -174,6 +174,19 @@ def solve(self, input_env: Dict[str, Any]) -> Dict[str, Any]: if output_token_strings and output_token_strings[0] == "[BOS]": output_token_strings = output_token_strings[1:] + # FIX: this model folds rule prediction into the output sequence as + # a leading RULE:xxx token (see docs/KNOWN_ISSUES.md, "Rule + # prediction folded into output sequence"). It is not part of the + # SLaNg AST grammar the verifier deserializes. Without this strip, + # deserialization failed with "Unexpected token while parsing node + # at index 0: RULE:partial_derivative" (or any other rule label) + # on every single call, regardless of whether the rest of the + # generated sequence was correct. + predicted_rule = None + if output_token_strings and output_token_strings[0].startswith("RULE:"): + predicted_rule = output_token_strings[0] + output_token_strings = output_token_strings[1:] + verifier_result = self._verify_output(input_env, output_token_strings) if verifier_result.get("status") in ("solved", "unverified", "unsolvable"): result["status"] = verifier_result["status"] @@ -189,7 +202,7 @@ def solve(self, input_env: Dict[str, Any]) -> Dict[str, Any]: "status": result["status"], "verified": result["verified"], "confidence": result["confidence"], - "rule": result.get("root_rule_label"), + "rule": predicted_rule, "output": result["output"], "warning": result.get("warning"), } From 83e1a7266b605161bd202b58cb29a8dc55dc4be2 Mon Sep 17 00:00:00 2001 From: chaudhryumer Date: Wed, 19 Aug 2026 01:44:36 -0700 Subject: [PATCH 3/3] fix: resolve merge conflicts and align rule label validation --- model/transformer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/model/transformer.py b/model/transformer.py index 2749e2a..df53bd4 100644 --- a/model/transformer.py +++ b/model/transformer.py @@ -34,9 +34,15 @@ def __init__( position_dim=position_dim, ) - # Dynamic rule label mapping (resolves RULE_i placeholder issue) + # 1. Validate custom rule_labels count if provided + if rule_labels is not None and len(rule_labels) != num_rules: + raise ValueError( + f"Expected {num_rules} rule labels, but got {len(rule_labels)}." + ) + + # 2. Dynamic fallback with underscore formatting (resolves RULE_i test assertion) if rule_labels is None: - rule_labels = [f"RULE:{i}" for i in range(num_rules)] + rule_labels = [f"RULE_{i}" for i in range(num_rules)] self.rule_head = RuleHead( hidden_dim=hidden_dim,