software/scripts/evaluate_translations.py

1"""Script to evaluate existing translations.23This script generates evaluation prompts for translations.4Since the user has Claude Max subscription, the evaluations5can be performed directly by Claude in conversation.67Usage:8    # Generate evaluation prompt for a verse9    uv run python scripts/evaluate_translations.py titus 1 11011    # Generate prompts for entire chapter12    uv run python scripts/evaluate_translations.py titus 11314    # Show evaluation summary for a book15    uv run python scripts/evaluate_translations.py --summary titus1617    # Show flagged verses18    uv run python scripts/evaluate_translations.py --flagged titus19"""2021from __future__ import annotations2223import argparse24import json25import sys26from pathlib import Path2728from src.context.builder import ContextBuilder29from src.context.glossary import GlossaryManager30from src.data.loader import DataLoader31from src.evaluation.evaluator import TranslationEvaluator, build_evaluation_prompt32from src.evaluation.storage import EvaluationStorage33from src.models.verse import VerseReference343536def load_translation(data_dir: Path, book: str, chapter: int, verse: int) -> str | None:37    """Load a translation from the output files."""38    chapter_file = data_dir / "output" / book / f"chapter_{chapter:02d}.json"39    if not chapter_file.exists():40        return None4142    with chapter_file.open("r", encoding="utf-8") as f:43        data = json.load(f)4445    verses = data.get("verses", {})46    verse_data = verses.get(str(verse))47    if verse_data:48        return verse_data.get("text")49    return None505152def generate_verse_prompt(53    data_dir: Path, book: str, chapter: int, verse: int54) -> str | None:55    """Generate an evaluation prompt for a single verse."""56    loader = DataLoader(data_dir)57    glossary = GlossaryManager(loader.load_glossary())58    context_builder = ContextBuilder(loader, glossary)5960    # Load the translation61    translation = load_translation(data_dir, book, chapter, verse)62    if not translation:63        print(f"No translation found for {book}:{chapter}:{verse}", file=sys.stderr)64        return None6566    # Build context67    ref = VerseReference(book=book, chapter=chapter, verse=verse)68    context = context_builder.build_context(ref)6970    # Build prompt71    evaluator = TranslationEvaluator()72    return evaluator.build_verse_prompt(translation, context)737475def generate_chapter_prompts(data_dir: Path, book: str, chapter: int) -> list[dict]:76    """Generate evaluation prompts for all verses in a chapter."""77    loader = DataLoader(data_dir)7879    # Get verse count80    chapter_file = data_dir / "output" / book / f"chapter_{chapter:02d}.json"81    if not chapter_file.exists():82        print(f"No translations found for {book} chapter {chapter}", file=sys.stderr)83        return []8485    with chapter_file.open("r", encoding="utf-8") as f:86        data = json.load(f)8788    verses = data.get("verses", {})89    prompts = []9091    for verse_num_str in sorted(verses.keys(), key=int):92        verse_num = int(verse_num_str)93        translation = verses[verse_num_str].get("text")94        if translation:95            prompt = generate_verse_prompt(data_dir, book, chapter, verse_num)96            if prompt:97                prompts.append(98                    {99                        "reference": f"{book}:{chapter}:{verse_num}",100                        "translation": translation,101                        "prompt": prompt,102                    }103                )104105    return prompts106107108def show_summary(data_dir: Path, book: str) -> None:109    """Show evaluation summary for a book."""110    storage = EvaluationStorage(data_dir)111    summary = storage.get_book_summary(book)112113    if not summary:114        print(f"No evaluations found for {book}")115        return116117    print(f"\n{'='*60}")118    print(f"Evaluation Summary: {book.upper()}")119    print(f"{'='*60}")120    print(f"Chapters evaluated: {summary.total_chapters}")121    print(f"Verses evaluated: {summary.total_verses}")122    print(f"Overall score: {summary.overall_score}%")123    print(f"\nDimension Scores:")124    for dim, score in sorted(summary.aggregate_scores.items()):125        print(f"  {dim}: {score}%")126    print(f"\nFlagged verses: {len(summary.flagged_verses)}")127    if summary.flagged_verses:128        for ref in summary.flagged_verses[:10]:  # Show first 10129            print(f"  - {ref}")130        if len(summary.flagged_verses) > 10:131            print(f"  ... and {len(summary.flagged_verses) - 10} more")132133134def show_flagged(data_dir: Path, book: str) -> None:135    """Show all flagged verses for a book."""136    storage = EvaluationStorage(data_dir)137    flagged = storage.get_flagged_verses(book)138139    if not flagged:140        print(f"No flagged verses for {book}")141        return142143    print(f"\nFlagged verses in {book}:")144    for ref in flagged:145        # Load evaluation to get flags146        chapter_eval = storage.load_chapter_evaluation(ref.book, ref.chapter)147        if chapter_eval and ref.verse in chapter_eval.verses:148            verse_eval = chapter_eval.verses[ref.verse]149            flags = ", ".join(verse_eval.flags)150            print(f"  {ref.book}:{ref.chapter}:{ref.verse} - [{flags}]")151152153def main() -> None:154    """Main entry point."""155    parser = argparse.ArgumentParser(156        description="Generate evaluation prompts for translations"157    )158    parser.add_argument("book", nargs="?", help="Book name (e.g., 'titus')")159    parser.add_argument("chapter", nargs="?", type=int, help="Chapter number")160    parser.add_argument("verse", nargs="?", type=int, help="Verse number (optional)")161    parser.add_argument(162        "--summary", action="store_true", help="Show evaluation summary for book"163    )164    parser.add_argument(165        "--flagged", action="store_true", help="Show flagged verses for book"166    )167    parser.add_argument(168        "--data-dir", default="data", help="Data directory (default: data)"169    )170171    args = parser.parse_args()172    data_dir = Path(args.data_dir)173174    if args.summary:175        if not args.book:176            print("Error: book name required for --summary", file=sys.stderr)177            sys.exit(1)178        show_summary(data_dir, args.book)179        return180181    if args.flagged:182        if not args.book:183            print("Error: book name required for --flagged", file=sys.stderr)184            sys.exit(1)185        show_flagged(data_dir, args.book)186        return187188    if not args.book or not args.chapter:189        parser.print_help()190        sys.exit(1)191192    if args.verse:193        # Single verse194        prompt = generate_verse_prompt(data_dir, args.book, args.chapter, args.verse)195        if prompt:196            print(prompt)197    else:198        # Entire chapter199        prompts = generate_chapter_prompts(data_dir, args.book, args.chapter)200        print(json.dumps(prompts, ensure_ascii=False, indent=2))201202203if __name__ == "__main__":204    main()205