software/src/evaluation/evaluator.py

1"""Translation evaluator using LLM-based assessment."""23from __future__ import annotations45import json6from datetime import datetime7from pathlib import Path8from typing import TYPE_CHECKING, Any910from src.models.evaluation import (11    DIMENSIONS,12    ChapterEvaluation,13    EvaluationScore,14    VerseEvaluation,15)1617if TYPE_CHECKING:18    from src.context.builder import TranslationContext192021def _load_style_config(data_dir: Path | str = "data") -> dict[str, Any]:22    """Load style configuration from JSON file.2324    Args:25        data_dir: Path to data directory.2627    Returns:28        Style configuration dictionary.29    """30    style_path = Path(data_dir) / "config" / "style.json"31    if style_path.exists():32        with style_path.open("r", encoding="utf-8") as f:33            return json.load(f)34    return {}353637def _build_style_criteria(style_config: dict[str, Any]) -> str:38    """Build style-based evaluation criteria from config.3940    Args:41        style_config: Style configuration dictionary.4243    Returns:44        Formatted style criteria text for the evaluation prompt.45    """46    target_style = style_config.get("target_style_reference", "現代のウェブ記事・ナレーションのような平易な日本語")47    formality = style_config.get("formality_level", "standard")48    avoid_archaic = style_config.get("avoid_archaic", ["なり", "たり", "候"])49    equivalence = style_config.get("equivalence_approach", "dynamic")5051    # Map formality level to description52    formality_desc = {53        "casual": "casual, conversational Japanese",54        "standard": "standard, neutral Japanese appropriate for general audiences",55        "polite": "warm, friendly です/ます narration (リビングバイブル-like)",56        "formal": "formal, elevated Japanese with honorific language",57        "liturgical": "liturgical, solemn Japanese suitable for worship",58    }.get(formality, "standard Japanese")5960    # Map equivalence approach to description61    equivalence_desc = {62        "formal": "formal equivalence (word-for-word, preserving source structure)",63        "dynamic": "dynamic equivalence (thought-for-thought, natural target language)",64        "balanced": "balanced approach between formal and dynamic equivalence",65    }.get(equivalence, "dynamic equivalence")6667    archaic_list = ", ".join(avoid_archaic) if avoid_archaic else "none specified"6869    readability = style_config.get("readability_level", "web_native")70    audience_line = ""71    if readability == "web_native":72        audience_line = (73            "\n   Would a Japanese reader in their 20s who mostly reads online"74            " media (news apps, blogs, popular creators) find it effortless —"75            " short sentences, everyday vocabulary, no stiff literary wording?"76        )7778    return f"""7. STYLE ADHERENCE (style_adherence)79   Does it match the target style reference: {target_style}?80   Does it follow {equivalence_desc}?{audience_line}81828. FORMALITY LEVEL (formality_level)83   Does it use {formality_desc}?84   Is the register appropriate and consistent?85869. ARCHAIC AVOIDANCE (archaic_avoidance)87   Does it avoid archaic forms? (avoid: {archaic_list})88   Is modern Japanese used appropriately?"""899091EVALUATION_SYSTEM_PROMPT = """You are an expert evaluator of Japanese Bible translations.92You will assess translations across multiple quality dimensions.9394For each dimension, provide:951. A score from 0-100 (percentage)962. A confidence level from 0-100 (how certain you are)973. A brief reasoning (1-2 sentences)9899Be rigorous but fair. A score of 100 means perfect, 80-90 is excellent,10070-80 is good, 60-70 is acceptable, below 60 needs improvement."""101102103def build_evaluation_prompt(104    translation: str,105    greek_text: str | None,106    kjv_text: str | None,107    reference_japanese: str | None,108    glossary_terms: list[str] | None = None,109    style_config: dict[str, Any] | None = None,110    hebrew_text: str | None = None,111    surrounding_context: str | None = None,112) -> str:113    """Build the evaluation prompt for a verse.114115    Args:116        translation: The Japanese translation to evaluate.117        greek_text: Original Greek source text (NT).118        kjv_text: KJV English reference translation.119        reference_japanese: Reference Japanese translation (e.g., Kougo).120        glossary_terms: Glossary guidance entries (term → Japanese, with notes).121        style_config: Style configuration from style.json.122        hebrew_text: Original Hebrew source text (OT).123        surrounding_context: Neighboring verses, for judging cohesion.124125    Returns:126        Formatted evaluation prompt string.127    """128    sections = []129130    if greek_text:131        sections.append(f"ORIGINAL TEXT (Greek):\n{greek_text}")132    elif hebrew_text:133        sections.append(f"ORIGINAL TEXT (Hebrew):\n{hebrew_text}")134135    sections.append(f"TRANSLATION TO EVALUATE:\n{translation}")136137    if kjv_text:138        sections.append(f"REFERENCE (KJV English):\n{kjv_text}")139140    if reference_japanese:141        sections.append(f"REFERENCE (Japanese Kougo-yaku):\n{reference_japanese}")142143    if surrounding_context:144        sections.append(145            "SURROUNDING PASSAGE (judge whether the translation reads as a natural,"146            " register-consistent continuation of this passage. Lines marked"147            " 'reference version' are an older translation shown for flow only —"148            " do NOT penalize terminology differences from them; the glossary"149            " governs terminology):\n"150            f"{surrounding_context}"151        )152153    if glossary_terms:154        sections.append(155            "GLOSSARY GUIDANCE (contextual notes take precedence over the bare"156            " default rendering):\n" + "\n".join(f"- {g}" for g in glossary_terms)157        )158159    # Build style criteria section160    style_criteria = _build_style_criteria(style_config or {})161162    sections.append(f"""163EVALUATE on these dimensions:1641651. THEOLOGICAL ACCURACY (theological_accuracy)166   Is the theological meaning preserved without error?167   Are doctrinal concepts accurately conveyed?1681692. SOURCE FIDELITY (source_fidelity)170   How faithfully does it follow the original source text (Greek/Hebrew)?171   Are key words and phrases properly translated?1721733. NATURALNESS (naturalness)174   Does it sound like natural modern Japanese?175   Would a native speaker find it comfortable to read?176   Does it read as a smooth continuation of the surrounding passage?1771784. READABILITY (readability)179   Is it easy to understand for contemporary readers?180   Is the sentence structure clear?1811825. GLOSSARY CONSISTENCY (glossary_consistency)183   Are standard theological terms used correctly?184   Is terminology consistent with the glossary?1851866. LITERARY QUALITY (literary_quality)187   Is the prose/poetic quality preserved?188   Does it maintain appropriate tone and style?189190{style_criteria}191192Respond ONLY with valid JSON in this exact format:193{{194  "theological_accuracy": {{"score": N, "confidence": N, "reasoning": "..."}},195  "source_fidelity": {{"score": N, "confidence": N, "reasoning": "..."}},196  "naturalness": {{"score": N, "confidence": N, "reasoning": "..."}},197  "readability": {{"score": N, "confidence": N, "reasoning": "..."}},198  "glossary_consistency": {{"score": N, "confidence": N, "reasoning": "..."}},199  "literary_quality": {{"score": N, "confidence": N, "reasoning": "..."}},200  "style_adherence": {{"score": N, "confidence": N, "reasoning": "..."}},201  "formality_level": {{"score": N, "confidence": N, "reasoning": "..."}},202  "archaic_avoidance": {{"score": N, "confidence": N, "reasoning": "..."}}203}}""")204205    return "\n\n".join(sections)206207208def parse_evaluation_response(response: str) -> dict[str, EvaluationScore]:209    """Parse LLM evaluation response into EvaluationScore objects.210211    Args:212        response: JSON string from LLM evaluation.213214    Returns:215        Dictionary mapping dimension names to EvaluationScore objects.216217    Raises:218        ValueError: If response cannot be parsed.219    """220    # Try to extract JSON from response (handle markdown code blocks)221    json_str = response.strip()222    if "```json" in json_str:223        json_str = json_str.split("```json")[1].split("```")[0].strip()224    elif "```" in json_str:225        json_str = json_str.split("```")[1].split("```")[0].strip()226227    try:228        data = json.loads(json_str)229    except json.JSONDecodeError as e:230        raise ValueError(f"Failed to parse evaluation response as JSON: {e}") from e231232    scores = {}233    for dimension in DIMENSIONS:234        if dimension not in data:235            raise ValueError(f"Missing dimension in response: {dimension}")236237        dim_data = data[dimension]238        scores[dimension] = EvaluationScore(239            dimension=dimension,240            score=float(dim_data["score"]),241            confidence=float(dim_data["confidence"]),242            reasoning=str(dim_data["reasoning"]),243        )244245    return scores246247248class TranslationEvaluator:249    """Evaluates translations using LLM-based assessment.250251    This class builds evaluation prompts and processes responses.252    The actual LLM calls can be made externally (e.g., via Claude API253    or directly in conversation with Claude).254    """255256    def __init__(257        self,258        model_name: str = "claude-opus-4-8",259        data_dir: Path | str = "data",260    ) -> None:261        """Initialize the evaluator.262263        Args:264            model_name: Name of the model used for evaluation.265            data_dir: Path to data directory for loading style config.266        """267        self.model_name = model_name268        self.data_dir = Path(data_dir)269        self._style_config: dict[str, Any] | None = None270271    @property272    def style_config(self) -> dict[str, Any]:273        """Get the style configuration, loading if needed."""274        if self._style_config is None:275            self._style_config = _load_style_config(self.data_dir)276        return self._style_config277278    def build_verse_prompt(279        self,280        translation: str,281        context: TranslationContext,282    ) -> str:283        """Build evaluation prompt for a verse with full context.284285        Args:286            translation: The Japanese translation to evaluate.287            context: TranslationContext with source texts and glossary.288289        Returns:290            Formatted evaluation prompt.291        """292        glossary_terms = []293        for term in context.glossary_terms:294            entry = f"{term.original} → {term.japanese}"295            if term.notes:296                entry += f" ({term.notes})"297            glossary_terms.append(entry)298299        surrounding = self._format_surrounding_context(context)300301        return build_evaluation_prompt(302            translation=translation,303            greek_text=context.target_verse.greek_text,304            kjv_text=context.target_verse.english_kjv,305            reference_japanese=(306                context.target_verse.japanese_kougo307                or context.target_verse.japanese_taisho308            ),309            glossary_terms=glossary_terms if glossary_terms else None,310            style_config=self.style_config,311            hebrew_text=context.target_verse.hebrew_text,312            surrounding_context=surrounding,313        )314315    def _format_surrounding_context(self, context: TranslationContext) -> str | None:316        """Format neighboring verses for cohesion judging.317318        Prefers this project's own saved translations of the neighbors (so319        internal consistency is judged against the translation under test,320        not against a reference version); falls back to Kougo/Taisho/KJV.321        """322        from src.data.storage import Storage323324        storage = Storage(self.data_dir)325        lines = []326        for label, verses in (327            ("before", context.preceding_verses),328            ("after", context.following_verses),329        ):330            for v in verses:331                own = storage.load_translation(v.reference)332                if own and own.text:333                    lines.append(334                        f"[{label} {v.reference.verse} — this translation] {own.text}"335                    )336                    continue337                text = v.japanese_kougo or v.japanese_taisho or v.english_kjv338                if text:339                    lines.append(340                        f"[{label} {v.reference.verse} — reference version] {text}"341                    )342        return "\n".join(lines) if lines else None343344    def create_verse_evaluation(345        self,346        verse_number: int,347        scores: dict[str, EvaluationScore],348        notes: str | None = None,349    ) -> VerseEvaluation:350        """Create a VerseEvaluation from scores.351352        Args:353            verse_number: The verse number.354            scores: Dictionary of dimension scores.355            notes: Optional notes.356357        Returns:358            VerseEvaluation object with computed flags.359        """360        evaluation = VerseEvaluation(361            verse_number=verse_number,362            evaluator_model=self.model_name,363            timestamp=datetime.utcnow(),364            scores=scores,365            flags=[],366            notes=notes,367        )368        # Compute and set flags369        evaluation.flags = evaluation.compute_flags()370        return evaluation371372    def create_chapter_evaluation(373        self,374        book: str,375        chapter: int,376        verse_evaluations: dict[int, VerseEvaluation],377    ) -> ChapterEvaluation:378        """Create a ChapterEvaluation from verse evaluations.379380        Args:381            book: Book name.382            chapter: Chapter number.383            verse_evaluations: Dictionary mapping verse numbers to evaluations.384385        Returns:386            ChapterEvaluation object with aggregated scores.387        """388        return ChapterEvaluation(389            book=book,390            chapter=chapter,391            evaluator_model=self.model_name,392            evaluated_at=datetime.utcnow(),393            verses=verse_evaluations,394        )395