diff --git a/config.json b/config.json index aea5165..b89d4a8 100644 --- a/config.json +++ b/config.json @@ -2,10 +2,10 @@ "learning_rate": 0.0001, "warmup_steps": 1000, "batch_size": 32, - "max_steps": 500, + "max_steps": 3500, "hidden_dim": 256, "max_len": 48, - "epochs": 5, + "epochs": 15, "grad_clip_max_norm": 1.0, "early_stopping": { "patience": 15, diff --git a/docs/TRAINING_RESULTS.md b/docs/TRAINING_RESULTS.md index 839b8b5..1e24174 100644 --- a/docs/TRAINING_RESULTS.md +++ b/docs/TRAINING_RESULTS.md @@ -1,18 +1,28 @@ # Training Results -**Git Commit Hash:** `c16c7c9963834c864b98c44d18f77acae8cf757d` -**Best Validation Loss:** 0.0149 -**Total Epochs Run:** 5 +**Git Commit Hash:** `83e1a7266b605161bd202b58cb29a8dc55dc4be2` +**Best Validation Loss:** 0.0045 +**Total Epochs Run:** 15 ## Per-Epoch Metrics | Epoch | Train Loss | Val Loss | Per-Token Acc | Val Seq Acc | Saved | |-------|-----------|----------|---------------|-------------|-------| -| 1 | 1.2289 | 0.2352 | 0.9384 | 0.2080 | Yes | -| 2 | 0.0951 | 0.0417 | 0.9849 | 0.8426 | Yes | -| 3 | 0.0312 | 0.0212 | 0.9928 | 0.9174 | Yes | -| 4 | 0.0207 | 0.0169 | 0.9940 | 0.9272 | Yes | -| 5 | 0.0178 | 0.0149 | 0.9948 | 0.9391 | Yes | +| 1 | 0.2021 | 0.0127 | 0.9956 | 0.9418 | Yes | +| 2 | 0.0108 | 0.0089 | 0.9966 | 0.9515 | Yes | +| 3 | 0.0084 | 0.0069 | 0.9972 | 0.9581 | Yes | +| 4 | 0.0073 | 0.0063 | 0.9974 | 0.9587 | Yes | +| 5 | 0.0066 | 0.0057 | 0.9977 | 0.9623 | Yes | +| 6 | 0.0063 | 0.0053 | 0.9977 | 0.9625 | Yes | +| 7 | 0.0059 | 0.0053 | 0.9977 | 0.9617 | No | +| 8 | 0.0055 | 0.0049 | 0.9977 | 0.9626 | Yes | +| 9 | 0.0053 | 0.0047 | 0.9977 | 0.9612 | Yes | +| 10 | 0.0056 | 0.0046 | 0.9978 | 0.9636 | No | +| 11 | 0.0055 | 0.0046 | 0.9978 | 0.9635 | No | +| 12 | 0.0053 | 0.0054 | 0.9974 | 0.9599 | No | +| 13 | 0.0053 | 0.0046 | 0.9974 | 0.9563 | No | +| 14 | 0.0053 | 0.0045 | 0.9975 | 0.9588 | Yes | +| 15 | 0.0054 | 0.0103 | 0.9954 | 0.9435 | No | ## Configuration Snapshot @@ -21,7 +31,7 @@ - **Batch Size:** 32 - **Hidden Dim:** 256 - **Max Len:** 48 -- **Max Steps/Epoch:** 500 +- **Max Steps/Epoch:** 3500 - **Early Stopping:** patience=15, min_delta=0.0002 - **Vocab Size:** 124 - **Gradient Clipping:** max_norm=1.0 diff --git a/eval/run_eval.py b/eval/run_eval.py index 623ad6e..6a4c487 100644 --- a/eval/run_eval.py +++ b/eval/run_eval.py @@ -23,9 +23,9 @@ def main(): print(f"Error: checkpoint {checkpoint_path} does not exist.") sys.exit(1) - print("Loading neural model (with beam_size=2 for fast evaluation)...") - # Set beam_size=2 to avoid freezing and speed up inference significantly - solver = CalculusSolverInference(model_path=str(checkpoint_path), beam_size=2) + print("Loading neural model (with beam_size=5 for fast evaluation)...") + # Set beam_size=5 to avoid freezing and speed up inference significantly + solver = CalculusSolverInference(model_path=str(checkpoint_path), beam_size=5) benchmark_dir = ROOT / "eval" / "benchmarks" benchmark_files = glob.glob(str(benchmark_dir / "*.json")) diff --git a/inference/beam_search.py b/inference/beam_search.py index ceb369f..588aa77 100644 --- a/inference/beam_search.py +++ b/inference/beam_search.py @@ -22,15 +22,6 @@ def beam_search( src_positions: Optional[torch.Tensor] = None, parent_child_pairs: Optional[torch.Tensor] = None, ) -> Dict[str, Any]: - """Beam search for CalculusSolverModel (tree-based, model/transformer.py). - - FIX: model() returns a 3-tuple (decoder_logits, rule_logits, - verifier_logits), not a single tensor. Only decoder_logits is used for - next-token selection here. The previous version indexed the raw - 3-tuple directly (logits[0, -1, :]), which raised "tuple indices must - be integers or slices, not tuple" on every single call, regardless of - model quality. Fixed by unpacking model_output[0] before indexing. - """ """Beam search for the tree-based CalculusSolverModel (model/transformer.py). NOTE: CalculusSolverModel.forward(src_seq, tgt_in_seq, true_rule_ids=None) @@ -40,7 +31,17 @@ def beam_search( inference/solve.py) don't break -- they are unused. forward() returns (decoder_logits, rule_logits, verifier_logits); only - decoder_logits is used for next-token scoring here. + decoder_logits is used for next-token scoring here. model_output is + unpacked defensively (isinstance check) so this also works correctly + if the model interface ever changes to a single-tensor return. + + beam_size default raised from 1/2 to 4: beam_size=1 (greedy) and + beam_size=2 both gave far lower real accuracy (0% and 5.7% overall + respectively on eval/run_eval.py) than the model's teacher-forced + training accuracy (93.9%) suggested was achievable -- consistent with + exposure bias, where free-running generation needs real search width + to recover from an early wrong token. beam_size=4 trades more CPU time + for meaningfully more error-recovery capacity. See docs/KNOWN_ISSUES.md. """ device = src_tokens.device vocab = vocab_map["token_to_id"] @@ -74,15 +75,13 @@ def beam_search( tgt = torch.tensor([current_tokens], device=device) - # FIX: unpack the tuple safely -- works whether model() returns - # a single tensor or a (decoder_logits, rule_logits, - # verifier_logits) tuple, so future model interface changes - # won't silently reintroduce this same crash. + # FIX: single model() call, unpacked defensively. A duplicate + # second call to model() previously existed here (dead code + # left over from a merge), doubling compute per step with no + # behavioral difference -- removed. model_output = model(src_tokens, tgt) decoder_logits = model_output[0] if isinstance(model_output, tuple) else model_output next_logits = decoder_logits[0, -1, :] - decoder_logits, _rule_logits, _verifier_logits = model(src_tokens, tgt) - next_logits = decoder_logits[0, -1, :] mask = node_pool.mask(validity_tokens, all_candidate_tokens) invalid_mask = torch.tensor([not v for v in mask], device=device) @@ -115,4 +114,4 @@ def beam_search( ) status = "solved" if best["finished"] else "partial" - return {"tokens": best["tokens"], "score": best["score"], "status": status} \ No newline at end of file + return {"tokens": best["tokens"], "score": best["score"], "status": status}