"""Translation evaluator using LLM-based assessment.""" from __future__ import annotations import json from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any from src.models.evaluation import ( DIMENSIONS, ChapterEvaluation, EvaluationScore, VerseEvaluation, ) if TYPE_CHECKING: from src.context.builder import TranslationContext def _load_style_config(data_dir: Path | str = "data") -> dict[str, Any]: """Load style configuration from JSON file. Args: data_dir: Path to data directory. Returns: Style configuration dictionary. """ style_path = Path(data_dir) / "config" / "style.json" if style_path.exists(): with style_path.open("r", encoding="utf-8") as f: return json.load(f) return {} def _build_style_criteria(style_config: dict[str, Any]) -> str: """Build style-based evaluation criteria from config. Args: style_config: Style configuration dictionary. Returns: Formatted style criteria text for the evaluation prompt. """ target_style = style_config.get("target_style_reference", "現代のウェブ記事・ナレーションのような平易な日本語") formality = style_config.get("formality_level", "standard") avoid_archaic = style_config.get("avoid_archaic", ["なり", "たり", "候"]) equivalence = style_config.get("equivalence_approach", "dynamic") # Map formality level to description formality_desc = { "casual": "casual, conversational Japanese", "standard": "standard, neutral Japanese appropriate for general audiences", "polite": "warm, friendly です/ます narration (リビングバイブル-like)", "formal": "formal, elevated Japanese with honorific language", "liturgical": "liturgical, solemn Japanese suitable for worship", }.get(formality, "standard Japanese") # Map equivalence approach to description equivalence_desc = { "formal": "formal equivalence (word-for-word, preserving source structure)", "dynamic": "dynamic equivalence (thought-for-thought, natural target language)", "balanced": "balanced approach between formal and dynamic equivalence", }.get(equivalence, "dynamic equivalence") archaic_list = ", ".join(avoid_archaic) if avoid_archaic else "none specified" readability = style_config.get("readability_level", "web_native") audience_line = "" if readability == "web_native": audience_line = ( "\n Would a Japanese reader in their 20s who mostly reads online" " media (news apps, blogs, popular creators) find it effortless —" " short sentences, everyday vocabulary, no stiff literary wording?" ) return f"""7. STYLE ADHERENCE (style_adherence) Does it match the target style reference: {target_style}? Does it follow {equivalence_desc}?{audience_line} 8. FORMALITY LEVEL (formality_level) Does it use {formality_desc}? Is the register appropriate and consistent? 9. ARCHAIC AVOIDANCE (archaic_avoidance) Does it avoid archaic forms? (avoid: {archaic_list}) Is modern Japanese used appropriately?""" EVALUATION_SYSTEM_PROMPT = """You are an expert evaluator of Japanese Bible translations. You will assess translations across multiple quality dimensions. For each dimension, provide: 1. A score from 0-100 (percentage) 2. A confidence level from 0-100 (how certain you are) 3. A brief reasoning (1-2 sentences) Be rigorous but fair. A score of 100 means perfect, 80-90 is excellent, 70-80 is good, 60-70 is acceptable, below 60 needs improvement.""" def build_evaluation_prompt( translation: str, greek_text: str | None, kjv_text: str | None, reference_japanese: str | None, glossary_terms: list[str] | None = None, style_config: dict[str, Any] | None = None, hebrew_text: str | None = None, surrounding_context: str | None = None, ) -> str: """Build the evaluation prompt for a verse. Args: translation: The Japanese translation to evaluate. greek_text: Original Greek source text (NT). kjv_text: KJV English reference translation. reference_japanese: Reference Japanese translation (e.g., Kougo). glossary_terms: Glossary guidance entries (term → Japanese, with notes). style_config: Style configuration from style.json. hebrew_text: Original Hebrew source text (OT). surrounding_context: Neighboring verses, for judging cohesion. Returns: Formatted evaluation prompt string. """ sections = [] if greek_text: sections.append(f"ORIGINAL TEXT (Greek):\n{greek_text}") elif hebrew_text: sections.append(f"ORIGINAL TEXT (Hebrew):\n{hebrew_text}") sections.append(f"TRANSLATION TO EVALUATE:\n{translation}") if kjv_text: sections.append(f"REFERENCE (KJV English):\n{kjv_text}") if reference_japanese: sections.append(f"REFERENCE (Japanese Kougo-yaku):\n{reference_japanese}") if surrounding_context: sections.append( "SURROUNDING PASSAGE (judge whether the translation reads as a natural," " register-consistent continuation of this passage. Lines marked" " 'reference version' are an older translation shown for flow only —" " do NOT penalize terminology differences from them; the glossary" " governs terminology):\n" f"{surrounding_context}" ) if glossary_terms: sections.append( "GLOSSARY GUIDANCE (contextual notes take precedence over the bare" " default rendering):\n" + "\n".join(f"- {g}" for g in glossary_terms) ) # Build style criteria section style_criteria = _build_style_criteria(style_config or {}) sections.append(f""" EVALUATE on these dimensions: 1. THEOLOGICAL ACCURACY (theological_accuracy) Is the theological meaning preserved without error? Are doctrinal concepts accurately conveyed? 2. SOURCE FIDELITY (source_fidelity) How faithfully does it follow the original source text (Greek/Hebrew)? Are key words and phrases properly translated? 3. NATURALNESS (naturalness) Does it sound like natural modern Japanese? Would a native speaker find it comfortable to read? Does it read as a smooth continuation of the surrounding passage? 4. READABILITY (readability) Is it easy to understand for contemporary readers? Is the sentence structure clear? 5. GLOSSARY CONSISTENCY (glossary_consistency) Are standard theological terms used correctly? Is terminology consistent with the glossary? 6. LITERARY QUALITY (literary_quality) Is the prose/poetic quality preserved? Does it maintain appropriate tone and style? {style_criteria} Respond ONLY with valid JSON in this exact format: {{ "theological_accuracy": {{"score": N, "confidence": N, "reasoning": "..."}}, "source_fidelity": {{"score": N, "confidence": N, "reasoning": "..."}}, "naturalness": {{"score": N, "confidence": N, "reasoning": "..."}}, "readability": {{"score": N, "confidence": N, "reasoning": "..."}}, "glossary_consistency": {{"score": N, "confidence": N, "reasoning": "..."}}, "literary_quality": {{"score": N, "confidence": N, "reasoning": "..."}}, "style_adherence": {{"score": N, "confidence": N, "reasoning": "..."}}, "formality_level": {{"score": N, "confidence": N, "reasoning": "..."}}, "archaic_avoidance": {{"score": N, "confidence": N, "reasoning": "..."}} }}""") return "\n\n".join(sections) def parse_evaluation_response(response: str) -> dict[str, EvaluationScore]: """Parse LLM evaluation response into EvaluationScore objects. Args: response: JSON string from LLM evaluation. Returns: Dictionary mapping dimension names to EvaluationScore objects. Raises: ValueError: If response cannot be parsed. """ # Try to extract JSON from response (handle markdown code blocks) json_str = response.strip() if "```json" in json_str: json_str = json_str.split("```json")[1].split("```")[0].strip() elif "```" in json_str: json_str = json_str.split("```")[1].split("```")[0].strip() try: data = json.loads(json_str) except json.JSONDecodeError as e: raise ValueError(f"Failed to parse evaluation response as JSON: {e}") from e scores = {} for dimension in DIMENSIONS: if dimension not in data: raise ValueError(f"Missing dimension in response: {dimension}") dim_data = data[dimension] scores[dimension] = EvaluationScore( dimension=dimension, score=float(dim_data["score"]), confidence=float(dim_data["confidence"]), reasoning=str(dim_data["reasoning"]), ) return scores class TranslationEvaluator: """Evaluates translations using LLM-based assessment. This class builds evaluation prompts and processes responses. The actual LLM calls can be made externally (e.g., via Claude API or directly in conversation with Claude). """ def __init__( self, model_name: str = "claude-opus-4-8", data_dir: Path | str = "data", ) -> None: """Initialize the evaluator. Args: model_name: Name of the model used for evaluation. data_dir: Path to data directory for loading style config. """ self.model_name = model_name self.data_dir = Path(data_dir) self._style_config: dict[str, Any] | None = None @property def style_config(self) -> dict[str, Any]: """Get the style configuration, loading if needed.""" if self._style_config is None: self._style_config = _load_style_config(self.data_dir) return self._style_config def build_verse_prompt( self, translation: str, context: TranslationContext, ) -> str: """Build evaluation prompt for a verse with full context. Args: translation: The Japanese translation to evaluate. context: TranslationContext with source texts and glossary. Returns: Formatted evaluation prompt. """ glossary_terms = [] for term in context.glossary_terms: entry = f"{term.original} → {term.japanese}" if term.notes: entry += f" ({term.notes})" glossary_terms.append(entry) surrounding = self._format_surrounding_context(context) return build_evaluation_prompt( translation=translation, greek_text=context.target_verse.greek_text, kjv_text=context.target_verse.english_kjv, reference_japanese=( context.target_verse.japanese_kougo or context.target_verse.japanese_taisho ), glossary_terms=glossary_terms if glossary_terms else None, style_config=self.style_config, hebrew_text=context.target_verse.hebrew_text, surrounding_context=surrounding, ) def _format_surrounding_context(self, context: TranslationContext) -> str | None: """Format neighboring verses for cohesion judging. Prefers this project's own saved translations of the neighbors (so internal consistency is judged against the translation under test, not against a reference version); falls back to Kougo/Taisho/KJV. """ from src.data.storage import Storage storage = Storage(self.data_dir) lines = [] for label, verses in ( ("before", context.preceding_verses), ("after", context.following_verses), ): for v in verses: own = storage.load_translation(v.reference) if own and own.text: lines.append( f"[{label} {v.reference.verse} — this translation] {own.text}" ) continue text = v.japanese_kougo or v.japanese_taisho or v.english_kjv if text: lines.append( f"[{label} {v.reference.verse} — reference version] {text}" ) return "\n".join(lines) if lines else None def create_verse_evaluation( self, verse_number: int, scores: dict[str, EvaluationScore], notes: str | None = None, ) -> VerseEvaluation: """Create a VerseEvaluation from scores. Args: verse_number: The verse number. scores: Dictionary of dimension scores. notes: Optional notes. Returns: VerseEvaluation object with computed flags. """ evaluation = VerseEvaluation( verse_number=verse_number, evaluator_model=self.model_name, timestamp=datetime.utcnow(), scores=scores, flags=[], notes=notes, ) # Compute and set flags evaluation.flags = evaluation.compute_flags() return evaluation def create_chapter_evaluation( self, book: str, chapter: int, verse_evaluations: dict[int, VerseEvaluation], ) -> ChapterEvaluation: """Create a ChapterEvaluation from verse evaluations. Args: book: Book name. chapter: Chapter number. verse_evaluations: Dictionary mapping verse numbers to evaluations. Returns: ChapterEvaluation object with aggregated scores. """ return ChapterEvaluation( book=book, chapter=chapter, evaluator_model=self.model_name, evaluated_at=datetime.utcnow(), verses=verse_evaluations, )