"""Script to evaluate existing translations. This script generates evaluation prompts for translations. Since the user has Claude Max subscription, the evaluations can be performed directly by Claude in conversation. Usage: # Generate evaluation prompt for a verse uv run python scripts/evaluate_translations.py titus 1 1 # Generate prompts for entire chapter uv run python scripts/evaluate_translations.py titus 1 # Show evaluation summary for a book uv run python scripts/evaluate_translations.py --summary titus # Show flagged verses uv run python scripts/evaluate_translations.py --flagged titus """ from __future__ import annotations import argparse import json import sys from pathlib import Path from src.context.builder import ContextBuilder from src.context.glossary import GlossaryManager from src.data.loader import DataLoader from src.evaluation.evaluator import TranslationEvaluator, build_evaluation_prompt from src.evaluation.storage import EvaluationStorage from src.models.verse import VerseReference def load_translation(data_dir: Path, book: str, chapter: int, verse: int) -> str | None: """Load a translation from the output files.""" chapter_file = data_dir / "output" / book / f"chapter_{chapter:02d}.json" if not chapter_file.exists(): return None with chapter_file.open("r", encoding="utf-8") as f: data = json.load(f) verses = data.get("verses", {}) verse_data = verses.get(str(verse)) if verse_data: return verse_data.get("text") return None def generate_verse_prompt( data_dir: Path, book: str, chapter: int, verse: int ) -> str | None: """Generate an evaluation prompt for a single verse.""" loader = DataLoader(data_dir) glossary = GlossaryManager(loader.load_glossary()) context_builder = ContextBuilder(loader, glossary) # Load the translation translation = load_translation(data_dir, book, chapter, verse) if not translation: print(f"No translation found for {book}:{chapter}:{verse}", file=sys.stderr) return None # Build context ref = VerseReference(book=book, chapter=chapter, verse=verse) context = context_builder.build_context(ref) # Build prompt evaluator = TranslationEvaluator() return evaluator.build_verse_prompt(translation, context) def generate_chapter_prompts(data_dir: Path, book: str, chapter: int) -> list[dict]: """Generate evaluation prompts for all verses in a chapter.""" loader = DataLoader(data_dir) # Get verse count chapter_file = data_dir / "output" / book / f"chapter_{chapter:02d}.json" if not chapter_file.exists(): print(f"No translations found for {book} chapter {chapter}", file=sys.stderr) return [] with chapter_file.open("r", encoding="utf-8") as f: data = json.load(f) verses = data.get("verses", {}) prompts = [] for verse_num_str in sorted(verses.keys(), key=int): verse_num = int(verse_num_str) translation = verses[verse_num_str].get("text") if translation: prompt = generate_verse_prompt(data_dir, book, chapter, verse_num) if prompt: prompts.append( { "reference": f"{book}:{chapter}:{verse_num}", "translation": translation, "prompt": prompt, } ) return prompts def show_summary(data_dir: Path, book: str) -> None: """Show evaluation summary for a book.""" storage = EvaluationStorage(data_dir) summary = storage.get_book_summary(book) if not summary: print(f"No evaluations found for {book}") return print(f"\n{'='*60}") print(f"Evaluation Summary: {book.upper()}") print(f"{'='*60}") print(f"Chapters evaluated: {summary.total_chapters}") print(f"Verses evaluated: {summary.total_verses}") print(f"Overall score: {summary.overall_score}%") print(f"\nDimension Scores:") for dim, score in sorted(summary.aggregate_scores.items()): print(f" {dim}: {score}%") print(f"\nFlagged verses: {len(summary.flagged_verses)}") if summary.flagged_verses: for ref in summary.flagged_verses[:10]: # Show first 10 print(f" - {ref}") if len(summary.flagged_verses) > 10: print(f" ... and {len(summary.flagged_verses) - 10} more") def show_flagged(data_dir: Path, book: str) -> None: """Show all flagged verses for a book.""" storage = EvaluationStorage(data_dir) flagged = storage.get_flagged_verses(book) if not flagged: print(f"No flagged verses for {book}") return print(f"\nFlagged verses in {book}:") for ref in flagged: # Load evaluation to get flags chapter_eval = storage.load_chapter_evaluation(ref.book, ref.chapter) if chapter_eval and ref.verse in chapter_eval.verses: verse_eval = chapter_eval.verses[ref.verse] flags = ", ".join(verse_eval.flags) print(f" {ref.book}:{ref.chapter}:{ref.verse} - [{flags}]") def main() -> None: """Main entry point.""" parser = argparse.ArgumentParser( description="Generate evaluation prompts for translations" ) parser.add_argument("book", nargs="?", help="Book name (e.g., 'titus')") parser.add_argument("chapter", nargs="?", type=int, help="Chapter number") parser.add_argument("verse", nargs="?", type=int, help="Verse number (optional)") parser.add_argument( "--summary", action="store_true", help="Show evaluation summary for book" ) parser.add_argument( "--flagged", action="store_true", help="Show flagged verses for book" ) parser.add_argument( "--data-dir", default="data", help="Data directory (default: data)" ) args = parser.parse_args() data_dir = Path(args.data_dir) if args.summary: if not args.book: print("Error: book name required for --summary", file=sys.stderr) sys.exit(1) show_summary(data_dir, args.book) return if args.flagged: if not args.book: print("Error: book name required for --flagged", file=sys.stderr) sys.exit(1) show_flagged(data_dir, args.book) return if not args.book or not args.chapter: parser.print_help() sys.exit(1) if args.verse: # Single verse prompt = generate_verse_prompt(data_dir, args.book, args.chapter, args.verse) if prompt: print(prompt) else: # Entire chapter prompts = generate_chapter_prompts(data_dir, args.book, args.chapter) print(json.dumps(prompts, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()