software/scripts/evaluate_verse_sample.py

1"""Evaluate a specific verse sample with the project's evaluator.23Writes per-verse dimension scores to a JSON file WITHOUT touching the4canonical chapter eval files — the harness used for all A/B experiments5(register calibration, morphology, translator-model comparisons).67Usage:8    uv run python scripts/evaluate_verse_sample.py <out.json> book:ch:v [...]9"""10import json, subprocess, sys11sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent.parent))12from src.context.builder import ContextBuilder13from src.context.glossary import GlossaryManager14from src.data.loader import DataLoader15from src.data.storage import Storage16from src.evaluation.evaluator import (17    EVALUATION_SYSTEM_PROMPT, TranslationEvaluator, parse_evaluation_response,18)19from src.models.verse import VerseReference20from src.pipeline.translator import DEFAULT_MODEL2122if len(sys.argv) < 3:23    print(__doc__)24    raise SystemExit(2)25out_path, refs = sys.argv[1], " ".join(sys.argv[2:]).split()26loader = DataLoader("data")27glossary = GlossaryManager(loader.load_glossary())28builder = ContextBuilder(loader, glossary)29storage = Storage("data")30evaluator = TranslationEvaluator(data_dir="data")3132results = {}33for r in refs:34    book, ch, v = r.split(":")35    ref = VerseReference(book=book, chapter=int(ch), verse=int(v))36    t = storage.load_translation(ref)37    if not t:38        print(f"{r}: no translation"); continue39    ctx = builder.build_context(ref)40    prompt = evaluator.build_verse_prompt(t.text, ctx)41    full = f"{EVALUATION_SYSTEM_PROMPT}\n\n{prompt}"42    p = subprocess.run(["claude", "--print", "-p", full, "--model", DEFAULT_MODEL],43                       capture_output=True, text=True, timeout=300)44    if p.returncode != 0:45        print(f"{r}: eval failed: {p.stderr[:100]}"); continue46    try:47        scores = parse_evaluation_response(p.stdout.strip())48    except ValueError as e:49        print(f"{r}: parse failed: {e}"); continue50    results[r] = {51        "text": t.text,52        "scores": {k: {"score": s.score, "reasoning": s.reasoning} for k, s in scores.items()},53        "avg": sum(s.score for s in scores.values()) / len(scores),54    }55    print(f"{r}: avg={results[r]['avg']:.1f}")56json.dump(results, open(out_path, "w"), ensure_ascii=False, indent=1)57print(f"saved {len(results)} to {out_path}")58