harness/crossmodel.py

"""Cross-model generalization summary (Phase 3).

Reads the per-model acceptance result JSONs (gpt-oss Phase-2 headline + any
Phase-3 models) and emits:
  * results/phase3_crossmodel.json  -- machine-readable comparison
  * paper/figures/crossmodel.{pdf,png} -- grouped bar of personal vs static MAT
  * a LaTeX-ready table printed to stdout

No model is re-run here; this only aggregates existing result files. Missing
model files are skipped (so it works whether or not Nemotron served).
"""
from __future__ import annotations

import json
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
RESULTS = ROOT / "results"
FIGS = ROOT / "figures"

# (display name, results json). row order = table order (gpt-oss first).
MODELS = [
    ("gpt-oss-120b", RESULTS / "phase2_accept_results.json"),
    ("gemma-4-31B-it", RESULTS / "phase3_gemma_accept_results.json"),
    ("Nemotron-3-Super-120B", RESULTS / "phase3_nemotron_accept_results.json"),
]


def _row(path: Path):
    d = json.loads(path.read_text())
    ow = d["overall_post_warmup"]
    per = d["summary"]  # per-session, for tail gap
    sessions = sorted(int(s) for s in per["personal_memory"].keys())
    last = str(sessions[-1])
    def mat(arm, block=ow, key=None):
        return block[arm]["MAT"] if key is None else block[arm][key]["MAT"]
    stat, pers, nomem = mat("static_global"), mat("personal_memory"), mat("no_memory")
    gap = 100.0 * (pers - stat) / stat
    tail_stat = per["static_global"][last]["MAT"]
    tail_pers = per["personal_memory"][last]["MAT"]
    tail_gap = 100.0 * (tail_pers - tail_stat) / tail_stat
    return {
        "no_memory": round(nomem, 2),
        "static_global": round(stat, 2),
        "personal_memory": round(pers, 2),
        "gap_pct": round(gap, 1),
        "tail_gap_pct": round(tail_gap, 1),
        "personal_seed_std": ow["personal_memory"].get("MAT_seed_std"),
        "n": ow["personal_memory"]["n"],
    }


def main():
    out = {}
    for name, path in MODELS:
        if path.exists():
            out[name] = _row(path)
            print(f"[ok] {name}: {out[name]}")
        else:
            print(f"[skip] {name}: {path.name} not found")
    (RESULTS / "phase3_crossmodel.json").write_text(json.dumps(out, indent=2))

    # LaTeX table body
    print("\n% --- LaTeX table rows (personal vs static vs none, +gap) ---")
    for name, r in out.items():
        print(f"{name} & {r['no_memory']:.2f} & {r['static_global']:.2f} & "
              f"{r['personal_memory']:.2f} & $+{r['gap_pct']:.0f}\\%$ & "
              f"$+{r['tail_gap_pct']:.0f}\\%$ \\\\")

    # grouped bar figure
    try:
        import matplotlib
        matplotlib.use("Agg")
        import matplotlib.pyplot as plt
        import numpy as np
        names = list(out.keys())
        x = np.arange(len(names))
        w = 0.26
        nomem = [out[n]["no_memory"] for n in names]
        stat = [out[n]["static_global"] for n in names]
        pers = [out[n]["personal_memory"] for n in names]
        fig, ax = plt.subplots(figsize=(7.2, 3.6))
        ax.bar(x - w, nomem, w, label="No memory", color="#9e9e9e")
        ax.bar(x, stat, w, label="Static datastore", color="#4C72B0")
        ax.bar(x + w, pers, w, label="Personal evicting (ours)", color="#C44E52")
        ymax = max(pers) * 1.30  # headroom for labels + legend
        ax.set_ylim(0, ymax)
        for xi, n in zip(x, names):
            ax.text(xi + w, out[n]["personal_memory"] + ymax * 0.015,
                    f"+{out[n]['gap_pct']:.0f}%", ha="center", fontsize=8,
                    color="#C44E52", fontweight="bold")
        ax.set_xticks(x)
        ax.set_xticklabels(names, fontsize=9)
        ax.set_ylabel("Mean accepted tokens (post-warmup)")
        ax.set_title("Personalized evicting memory generalizes across served models")
        ax.legend(fontsize=8, loc="upper center", ncol=3, frameon=False,
                  bbox_to_anchor=(0.5, 1.0))
        ax.grid(axis="y", alpha=0.3)
        fig.tight_layout()
        FIGS.mkdir(parents=True, exist_ok=True)
        fig.savefig(FIGS / "crossmodel.pdf")
        fig.savefig(FIGS / "crossmodel.png", dpi=150)
        print(f"\n[fig] wrote {FIGS/'crossmodel.pdf'}")
    except Exception as e:  # noqa: BLE001
        print(f"[fig] skipped: {e}")


if __name__ == "__main__":
    main()