software/src/pipeline/translator.py

1"""Translator module using Claude CLI."""23from __future__ import annotations45import asyncio6import json7import logging8import os9import re10import subprocess11import time12from collections.abc import Callable13from typing import TYPE_CHECKING, Any1415from src.context.builder import ContextBuilder16from src.data.storage import Storage17from src.exceptions import (18    ChunkParsingError,19    ClaudeAPIError,20    EvaluationError,21    QualityThresholdError,22    RateLimitError,23    classify_claude_error,24)25from src.models.pericope import ChunkTranslationContext, ChunkTranslationOutput, Pericope26from src.models.verse import TranslationOutput, VerseReference27from src.pipeline.prompt import PromptBuilder28from src.pipeline.request_archive import RequestArchive29from src.utils.furigana import parse_furigana30from src.utils.retry import RetryConfig, RetryState3132if TYPE_CHECKING:33    from src.context.translation_memory import TranslationMemoryManager34    from src.evaluation.evaluator import TranslationEvaluator35    from src.evaluation.storage import EvaluationStorage36    from src.models.evaluation import VerseEvaluation37    from src.models.verse import TranslationContext38    from src.orchestration.provenance import TranslationProvenance3940logger = logging.getLogger(__name__)4142# Find Claude binary43CLAUDE_BINARY = os.environ.get("CLAUDE_BINARY", "claude")444546def _clean_env() -> dict[str, str]:47    """Return a copy of the environment without CLAUDECODE to avoid nested session errors."""48    env = os.environ.copy()49    env.pop("CLAUDECODE", None)50    return env5152DEFAULT_MODEL = os.environ.get("TRANSLATE_BIBLE_MODEL", "claude-opus-4-8")535455def _sanitize_translation_output(raw: str) -> str:56    """Strip meta-commentary from a verse translation response.5758    Models occasionally emit self-corrections ("Wait—let me provide the59    correct translation.") or markdown fences around the answer. A verse60    translation is a single Japanese block, so: drop code fences, drop61    predominantly-Latin blocks, and if several Japanese blocks remain62    (a self-correction), keep the last one.63    """64    text = raw.strip()65    if not text:66        return text6768    lines = [ln for ln in text.split("\n") if not ln.strip().startswith("```")]6970    blocks: list[list[str]] = [[]]71    for ln in lines:72        if ln.strip():73            blocks[-1].append(ln)74        elif blocks[-1]:75            blocks.append([])76    block_texts = ["\n".join(b).strip() for b in blocks if b]7778    def is_japanese(block: str) -> bool:79        cjk = sum(1 for ch in block if "぀" <= ch <= "鿿" or "！" <= ch <= "｠")80        latin = sum(1 for ch in block if ch.isascii() and ch.isalpha())81        return cjk > latin8283    japanese_blocks = [b for b in block_texts if is_japanese(b)]84    if not japanese_blocks:85        return text8687    result = japanese_blocks[-1]88    result = re.sub(r"^(翻訳|訳文|Translation)\s*[:：]\s*", "", result)89    if result != text:90        logger.warning("Sanitized translation output (removed meta-commentary)")91    return result9293# Default retry configuration94DEFAULT_RETRY_CONFIG = RetryConfig(95    max_attempts=3,96    base_delay=2.0,97    max_delay=60.0,98    exponential_base=2.0,99    retryable_exceptions=(RateLimitError, ConnectionError, TimeoutError),100)101102# Quality thresholds for evaluation gating103DEFAULT_QUALITY_THRESHOLD = 70.0104THEOLOGICAL_ACCURACY_THRESHOLD = 75.0105106107class Translator:108    """Translates Bible verses using Claude CLI."""109110    def __init__(111        self,112        context_builder: ContextBuilder,113        prompt_builder: PromptBuilder,114        storage: Storage,115        model: str = DEFAULT_MODEL,116        rate_limit_delay: float = 1.2,117        translation_memory: TranslationMemoryManager | None = None,118        auto_evaluate: bool = True,119        evaluator: TranslationEvaluator | None = None,120        evaluation_storage: EvaluationStorage | None = None,121        retry_config: RetryConfig | None = None,122        quality_threshold: float = DEFAULT_QUALITY_THRESHOLD,123        enforce_quality_gate: bool = False,124        subprocess_timeout: float | None = None,125        enable_proto_archive: bool = False,126    ) -> None:127        """Initialize the translator.128129        Args:130            context_builder: Context builder for creating translation context.131            prompt_builder: Prompt builder for formatting prompts.132            storage: Storage for saving translations.133            model: Claude model to use.134            rate_limit_delay: Delay between API calls in seconds.135            translation_memory: Optional TM manager for reuse.136            auto_evaluate: Whether to automatically evaluate translations.137            evaluator: Evaluator to use for auto-evaluation.138            evaluation_storage: Storage for evaluation results.139            retry_config: Configuration for retry with backoff.140            quality_threshold: Minimum score for evaluation gating.141            enforce_quality_gate: If True, reject low-quality translations.142            subprocess_timeout: Timeout for Claude CLI calls in seconds.143                Defaults to the TRANSLATE_BIBLE_TIMEOUT env var, or 120.144                Long pericope chunks (10+ verses) can need 300s+.145            enable_proto_archive: Whether to archive translation inputs/outputs.146        """147        self.context_builder = context_builder148        self.prompt_builder = prompt_builder149        self.storage = storage150        self.model = model151        self.request_archive = RequestArchive(storage.data_dir / "request-archive")152        self.rate_limit_delay = rate_limit_delay153        self._translation_memory = translation_memory154        self._auto_evaluate = auto_evaluate155        self._evaluator = evaluator156        self._evaluation_storage = evaluation_storage157        self._retry_config = retry_config or DEFAULT_RETRY_CONFIG158        self._quality_threshold = quality_threshold159        self._enforce_quality_gate = enforce_quality_gate160        if subprocess_timeout is None:161            subprocess_timeout = float(os.environ.get("TRANSLATE_BIBLE_TIMEOUT", "120"))162        self._subprocess_timeout = subprocess_timeout163        self._enable_archive = enable_proto_archive164        # Enable archive on storage if requested165        if enable_proto_archive:166            self.storage.enable_archive = True167168    @property169    def enable_proto_archive(self) -> bool:170        """Get proto archive enabled status."""171        return self._enable_archive172173    @enable_proto_archive.setter174    def enable_proto_archive(self, value: bool) -> None:175        """Set proto archive enabled status."""176        self._enable_archive = value177        self.storage.enable_archive = value178179    @property180    def translation_memory(self) -> TranslationMemoryManager | None:181        """Get the translation memory manager."""182        return self._translation_memory183184    @translation_memory.setter185    def translation_memory(self, value: TranslationMemoryManager | None) -> None:186        """Set the translation memory manager."""187        self._translation_memory = value188189    @property190    def auto_evaluate(self) -> bool:191        """Get auto-evaluation setting."""192        return self._auto_evaluate193194    @auto_evaluate.setter195    def auto_evaluate(self, value: bool) -> None:196        """Set auto-evaluation setting."""197        self._auto_evaluate = value198199    def _call_claude(self, system_prompt: str, user_prompt: str) -> str:200        """Call Claude CLI with the given prompts.201202        Includes retry logic with exponential backoff for transient errors.203204        Args:205            system_prompt: System prompt for Claude.206            user_prompt: User prompt for Claude.207208        Returns:209            The response text from Claude.210211        Raises:212            ClaudeAPIError: If the Claude CLI call fails after all retries.213        """214        retry_state = RetryState(self._retry_config)215216        while True:217            request_id = self.request_archive.start(self.model, system_prompt, user_prompt)218            try:219                result = subprocess.run(220                    [221                        CLAUDE_BINARY,222                        "--print",223                        "-p",224                        user_prompt,225                        "--system-prompt",226                        system_prompt,227                        "--model",228                        self.model,229                    ],230                    capture_output=True,231                    text=True,232                    check=False,233                    timeout=self._subprocess_timeout,234                    env=_clean_env(),235                )236237                self.request_archive.finish(request_id, result.stdout, result.stderr, result.returncode)238                if result.returncode != 0:239                    error = classify_claude_error(result.stderr, result.returncode)240                    raise error241242                return result.stdout.strip()243244            except subprocess.TimeoutExpired as e:245                self.request_archive.finish(request_id, error="timeout")246                error = ClaudeAPIError(247                    f"Claude CLI timed out after {self._subprocess_timeout}s",248                    stderr=str(e),249                    return_code=-1,250                )251                if retry_state.should_retry(error):252                    logger.warning(253                        "Claude CLI timeout, attempt %d/%d. Retrying...",254                        retry_state.attempt + 1,255                        self._retry_config.max_attempts,256                    )257                    retry_state.wait()258                else:259                    raise error from e260261            except FileNotFoundError as e:262                self.request_archive.finish(request_id, error="cli_not_found")263                # Claude CLI not installed264                from src.exceptions import CLINotFoundError265                raise CLINotFoundError() from e266267            except ClaudeAPIError as e:268                if retry_state.should_retry(e):269                    logger.warning(270                        "Claude API error, attempt %d/%d: %s. Retrying...",271                        retry_state.attempt + 1,272                        self._retry_config.max_attempts,273                        str(e),274                    )275                    retry_state.wait()276                else:277                    raise278279    async def _call_claude_async(self, system_prompt: str, user_prompt: str) -> str:280        """Call Claude CLI asynchronously with the given prompts.281282        Includes retry logic with exponential backoff for transient errors.283284        Args:285            system_prompt: System prompt for Claude.286            user_prompt: User prompt for Claude.287288        Returns:289            The response text from Claude.290291        Raises:292            ClaudeAPIError: If the Claude CLI call fails after all retries.293        """294        retry_state = RetryState(self._retry_config)295296        while True:297            request_id = self.request_archive.start(self.model, system_prompt, user_prompt)298            process = None299            try:300                process = await asyncio.wait_for(301                    asyncio.create_subprocess_exec(302                        CLAUDE_BINARY,303                        "--print",304                        "-p",305                        user_prompt,306                        "--system-prompt",307                        system_prompt,308                        "--model",309                        self.model,310                        stdout=asyncio.subprocess.PIPE,311                        stderr=asyncio.subprocess.PIPE,312                        env=_clean_env(),313                    ),314                    timeout=self._subprocess_timeout,315                )316                stdout, stderr = await asyncio.wait_for(317                    process.communicate(),318                    timeout=self._subprocess_timeout,319                )320321                self.request_archive.finish(request_id, stdout.decode(), stderr.decode(), process.returncode)322                if process.returncode != 0:323                    error = classify_claude_error(stderr.decode(), process.returncode or 1)324                    raise error325326                return stdout.decode().strip()327328            except asyncio.TimeoutError as e:329                if process is not None and process.returncode is None:330                    process.kill()331                    await process.wait()332                self.request_archive.finish(request_id, error="timeout")333                error = ClaudeAPIError(334                    f"Claude CLI timed out after {self._subprocess_timeout}s",335                    stderr="Async timeout",336                    return_code=-1,337                )338                if retry_state.should_retry(error):339                    logger.warning(340                        "Claude CLI async timeout, attempt %d/%d. Retrying...",341                        retry_state.attempt + 1,342                        self._retry_config.max_attempts,343                    )344                    await retry_state.async_wait()345                else:346                    raise error from e347348            except FileNotFoundError as e:349                self.request_archive.finish(request_id, error="cli_not_found")350                from src.exceptions import CLINotFoundError351                raise CLINotFoundError() from e352353            except ClaudeAPIError as e:354                if retry_state.should_retry(e):355                    logger.warning(356                        "Claude API error (async), attempt %d/%d: %s. Retrying...",357                        retry_state.attempt + 1,358                        self._retry_config.max_attempts,359                        str(e),360                    )361                    await retry_state.async_wait()362                else:363                    raise364365    def _get_tm_matches(self, context: TranslationContext) -> list | None:366        """Get TM matches for the context if TM is enabled.367368        Args:369            context: Translation context.370371        Returns:372            List of TM matches or None if TM not available.373        """374        if self._translation_memory is None:375            return None376377        # Get source text for matching378        source_text = (379            context.target_verse.greek_text380            or context.target_verse.hebrew_text381            or ""382        )383        if not source_text:384            return None385386        matches = self._translation_memory.find_matches(387            source_text, threshold=0.7, max_results=3388        )389        return matches if matches else None390391    def _add_to_tm(392        self, context: TranslationContext, translation_text: str393    ) -> None:394        """Add completed translation to translation memory.395396        Args:397            context: Translation context.398            translation_text: The translated text.399        """400        if self._translation_memory is None:401            return402403        source_text = (404            context.target_verse.greek_text405            or context.target_verse.hebrew_text406        )407        if not source_text:408            return409410        self._translation_memory.add_from_translation(411            source_text=source_text,412            target_text=translation_text,413            reference=context.target_verse.reference,414            metadata={"model": self.model},415        )416417    def _evaluate_translation(418        self,419        output: TranslationOutput,420        context: TranslationContext,421    ) -> VerseEvaluation | None:422        """Evaluate a translation if auto-evaluate is enabled.423424        Args:425            output: The translation output.426            context: The translation context.427428        Returns:429            VerseEvaluation or None if evaluation failed/disabled.430431        Raises:432            EvaluationError: If enforce_quality_gate is True and evaluation fails.433        """434        if not self._auto_evaluate or not self._evaluator:435            return None436437        ref = output.reference438        ref_str = f"{ref.book} {ref.chapter}:{ref.verse}"439440        try:441            from src.evaluation.evaluator import (442                EVALUATION_SYSTEM_PROMPT,443                parse_evaluation_response,444            )445446            # Build evaluation prompt447            eval_prompt = self._evaluator.build_verse_prompt(448                output.text, context449            )450451            # Call Claude for evaluation452            response = self._call_claude(EVALUATION_SYSTEM_PROMPT, eval_prompt)453454            # Parse response455            scores = parse_evaluation_response(response)456457            # Create evaluation458            evaluation = self._evaluator.create_verse_evaluation(459                verse_number=output.reference.verse,460                scores=scores,461            )462463            # Log evaluation result464            logger.info(465                "Evaluated %s: overall=%.1f%%, flags=%s",466                ref_str,467                evaluation.overall_score,468                evaluation.flags or "none",469            )470471            return evaluation472473        except Exception as e:474            # Log the error with details475            logger.error(476                "Evaluation failed for %s: %s",477                ref_str,478                str(e),479            )480481            # If quality gate is enforced, raise the error482            if self._enforce_quality_gate:483                raise EvaluationError(484                    f"Evaluation failed for {ref_str}: {e}",485                    verse_ref=ref_str,486                ) from e487488            # Otherwise, return None (don't block translation)489            return None490491    def _check_quality_gate(492        self,493        evaluation: VerseEvaluation,494        reference: VerseReference,495    ) -> None:496        """Check if translation passes quality threshold.497498        Args:499            evaluation: The verse evaluation.500            reference: The verse reference.501502        Raises:503            QualityThresholdError: If translation fails quality gate.504        """505        if not self._enforce_quality_gate:506            return507508        ref_str = f"{reference.book} {reference.chapter}:{reference.verse}"509510        # Check overall score511        if evaluation.overall_score < self._quality_threshold:512            raise QualityThresholdError(513                f"Translation for {ref_str} failed quality gate: "514                f"{evaluation.overall_score:.1f}% < {self._quality_threshold}%",515                score=evaluation.overall_score,516                threshold=self._quality_threshold,517            )518519        # Check theological accuracy specifically520        if "theological_accuracy" in evaluation.scores:521            ta_score = evaluation.scores["theological_accuracy"].score522            if ta_score < THEOLOGICAL_ACCURACY_THRESHOLD:523                raise QualityThresholdError(524                    f"Translation for {ref_str} failed theological accuracy: "525                    f"{ta_score:.1f}% < {THEOLOGICAL_ACCURACY_THRESHOLD}%",526                    score=ta_score,527                    threshold=THEOLOGICAL_ACCURACY_THRESHOLD,528                    dimension="theological_accuracy",529                )530531    def _save_evaluation(532        self,533        evaluation: VerseEvaluation,534        reference: VerseReference,535    ) -> None:536        """Save evaluation result if storage is configured.537538        Args:539            evaluation: The evaluation to save.540            reference: The verse reference.541        """542        if not self._evaluation_storage:543            return544545        self._evaluation_storage.save_verse_evaluation(546            reference.book,547            reference.chapter,548            evaluation,549        )550551    def translate_verse(552        self,553        reference: VerseReference,554        skip_existing: bool = True,555        evaluate: bool | None = None,556    ) -> TranslationOutput:557        """Translate a single verse.558559        Args:560            reference: Verse reference to translate.561            skip_existing: If True, return existing translation if available.562            evaluate: Override auto_evaluate setting for this call.563564        Returns:565            TranslationOutput with the translation.566        """567        # Check if already translated568        if skip_existing and self.storage.is_verse_translated(reference):569            existing = self.storage.load_translation(reference)570            if existing:571                return existing572573        # Build context and prompt574        context = self.context_builder.build_context(reference)575576        # Get TM matches577        tm_matches = self._get_tm_matches(context)578579        # Build prompt (with TM matches if available)580        prompt = self.prompt_builder.build_prompt(context, tm_matches=tm_matches)581        system_prompt = self.prompt_builder.get_system_prompt()582583        # Call Claude CLI584        translation_text = _sanitize_translation_output(585            self._call_claude(system_prompt, prompt)586        )587588        # Parse furigana from Claude's response589        plain_text, annotations = parse_furigana(translation_text)590591        # Create output592        output = TranslationOutput(593            reference=reference,594            text=plain_text,595            text_with_furigana=translation_text,596            furigana_annotations=[597                {598                    "start_offset": a.start_offset,599                    "end_offset": a.end_offset,600                    "base_text": a.base_text,601                    "reading": a.reading,602                }603                for a in annotations604            ],605            model=self.model,606            sources_used=self._get_sources_used(context),607            glossary_terms_applied=[t.original for t in context.glossary_terms],608        )609610        # Save translation611        self.storage.save_translation(output)612613        # Add to TM614        self._add_to_tm(context, plain_text)615616        # Evaluate if enabled617        should_evaluate = evaluate if evaluate is not None else self._auto_evaluate618        if should_evaluate:619            evaluation = self._evaluate_translation(output, context)620            if evaluation:621                self._save_evaluation(evaluation, reference)622                # Check quality gate623                self._check_quality_gate(evaluation, reference)624625        return output626627    def translate_verse_with_provenance(  # pragma: no cover628        self,629        reference: VerseReference,630        skip_existing: bool = True,631    ) -> tuple[TranslationOutput, "TranslationProvenance"]:632        """Translate a verse and capture full provenance.633634        Args:635            reference: Verse reference to translate.636            skip_existing: If True, return existing translation if available.637638        Returns:639            Tuple of (TranslationOutput, TranslationProvenance).640        """641        from datetime import datetime642643        from src.orchestration.provenance import (644            TranslationProvenance,645            compute_glossary_hash,646            compute_style_hash,647        )648649        # Check if already translated650        if skip_existing and self.storage.is_verse_translated(reference):651            existing = self.storage.load_translation(reference)652            if existing:653                # Try to load existing provenance654                provenance = self.storage.load_provenance(reference)655                if provenance:656                    return existing, provenance657                # Create minimal provenance for existing translation658                provenance = TranslationProvenance(model=existing.model)659                return existing, provenance660661        # Track timing662        started_at = datetime.utcnow()663664        # Build context665        context = self.context_builder.build_context(reference)666667        # Get TM matches668        tm_matches = self._get_tm_matches(context) or []669670        # Build prompts671        prompt = self.prompt_builder.build_prompt(context, tm_matches=tm_matches)672        system_prompt = self.prompt_builder.get_system_prompt()673674        # Build provenance675        provenance = TranslationProvenance(676            model=self.model,677            prompt_version="v1.0",678            system_prompt=system_prompt,679            user_prompt=prompt,680            started_at=started_at,681            attempt_number=1,682        )683684        # Capture source texts685        verse = context.target_verse686        if verse.greek_text:687            provenance.source_texts["greek"] = verse.greek_text688        if verse.hebrew_text:689            provenance.source_texts["hebrew"] = verse.hebrew_text690        if verse.english_kjv:691            provenance.source_texts["kjv"] = verse.english_kjv692        if verse.english_asv:693            provenance.source_texts["asv"] = verse.english_asv694        if verse.japanese_meiji:695            provenance.source_texts["meiji"] = verse.japanese_meiji696        if verse.japanese_taisho:697            provenance.source_texts["taisho"] = verse.japanese_taisho698699        # Compute source hashes700        provenance.compute_source_hashes()701702        # Capture context verses703        provenance.preceding_verses = [704            str(v.reference) for v in context.preceding_verses705        ]706        provenance.following_verses = [707            str(v.reference) for v in context.following_verses708        ]709710        # Capture chapter summary711        if context.chapter_summary:712            provenance.chapter_summary = context.chapter_summary.summary713714        # Capture book metadata715        if context.book_metadata:716            provenance.book_metadata = {717                "name": context.book_metadata.name_english,718                "testament": context.book_metadata.testament,719                "themes": context.book_metadata.themes,720            }721722        # Capture glossary hash723        provenance.glossary_hash = compute_glossary_hash(context.glossary_terms)724725        # Capture style hash726        if self.prompt_builder._style:727            style_dict = self.prompt_builder._style.to_dict()728            provenance.style_config_hash = compute_style_hash(style_dict)729730        # Capture TM matches731        if tm_matches:732            provenance.tm_matches_used = [733                {734                    "source": m.segment.source_text,735                    "target": m.segment.target_text,736                    "similarity": m.similarity,737                }738                for m in tm_matches739            ]740741        # Create input proto for archive (before translation)742        input_proto = None743        if self._enable_archive:744            from src.protos.converters import (745                translation_context_to_prompt_input,746                tm_match_to_proto,747            )748            from src.protos.serialization import compute_proto_hash749750            input_proto = translation_context_to_prompt_input(751                context, system_prompt, prompt, self.model,752            )753            # Add TM matches to input proto754            if tm_matches and hasattr(tm_matches[0], "segment"):755                input_proto.tm_matches = [tm_match_to_proto(m) for m in tm_matches]756757        # Call Claude758        translation_text = self._call_claude(system_prompt, prompt)759760        # Complete timing761        completed_at = datetime.utcnow()762        provenance.completed_at = completed_at763764        # Parse furigana from Claude's response765        plain_text, annotations = parse_furigana(translation_text)766767        # Create output768        output = TranslationOutput(769            reference=reference,770            text=plain_text,771            text_with_furigana=translation_text,772            furigana_annotations=[773                {774                    "start_offset": a.start_offset,775                    "end_offset": a.end_offset,776                    "base_text": a.base_text,777                    "reading": a.reading,778                }779                for a in annotations780            ],781            model=self.model,782            sources_used=list(provenance.source_texts.keys()),783            glossary_terms_applied=[t.original for t in context.glossary_terms],784        )785786        # Save translation with provenance787        self.storage.save_translation_with_provenance(output, provenance)788789        # Save proto archive if enabled790        if self._enable_archive and input_proto is not None:791            from src.protos.converters import (792                provenance_to_proto,793                translation_result_to_proto,794            )795            from src.protos.serialization import compute_proto_hash796797            output_proto = translation_result_to_proto(798                output, translation_text, annotations,799                input_hash=compute_proto_hash(input_proto)800            )801            output_proto.provenance = provenance_to_proto(provenance)802            self.storage.save_translation_archive(reference, input_proto, output_proto)803804        # Add to TM805        self._add_to_tm(context, plain_text)806807        return output, provenance808809    async def translate_verse_async(810        self, reference: VerseReference, skip_existing: bool = True811    ) -> TranslationOutput:812        """Translate a single verse asynchronously.813814        Args:815            reference: Verse reference to translate.816            skip_existing: If True, return existing translation if available.817818        Returns:819            TranslationOutput with the translation.820        """821        # Check if already translated822        if skip_existing and self.storage.is_verse_translated(reference):823            existing = self.storage.load_translation(reference)824            if existing:825                return existing826827        # Build context and prompt828        context = self.context_builder.build_context(reference)829830        # Get TM matches831        tm_matches = self._get_tm_matches(context)832833        # Build prompt834        prompt = self.prompt_builder.build_prompt(context, tm_matches=tm_matches)835        system_prompt = self.prompt_builder.get_system_prompt()836837        # Call Claude CLI838        translation_text = await self._call_claude_async(system_prompt, prompt)839840        # Parse furigana from Claude's response841        plain_text, annotations = parse_furigana(translation_text)842843        # Create output844        output = TranslationOutput(845            reference=reference,846            text=plain_text,847            text_with_furigana=translation_text,848            furigana_annotations=[849                {850                    "start_offset": a.start_offset,851                    "end_offset": a.end_offset,852                    "base_text": a.base_text,853                    "reading": a.reading,854                }855                for a in annotations856            ],857            model=self.model,858            sources_used=self._get_sources_used(context),859            glossary_terms_applied=[t.original for t in context.glossary_terms],860        )861862        # Save translation863        self.storage.save_translation(output)864865        # Add to TM866        self._add_to_tm(context, plain_text)867868        return output869870    def translate_chapter(871        self,872        book: str,873        chapter: int,874        skip_existing: bool = True,875        progress_callback: Callable[..., Any] | None = None,876        evaluate: bool | None = None,877    ) -> list[TranslationOutput]:878        """Translate all verses in a chapter.879880        Args:881            book: Book name.882            chapter: Chapter number.883            skip_existing: If True, skip already translated verses.884            progress_callback: Optional callback for progress updates.885            evaluate: Override auto_evaluate setting for this call.886887        Returns:888            List of TranslationOutput for each verse.889        """890        # Get all verses in the chapter891        verses = self.context_builder.loader.load_chapter_verses(book, chapter)892        outputs = []893894        for i, verse in enumerate(verses):895            if progress_callback:896                progress_callback(i + 1, len(verses), verse.reference)897898            output = self.translate_verse(899                verse.reference, skip_existing, evaluate=evaluate900            )901            outputs.append(output)902903            # Rate limiting904            if i < len(verses) - 1:905                time.sleep(self.rate_limit_delay)906907        # Save TM after chapter908        if self._translation_memory is not None:909            self._translation_memory.save()910911        return outputs912913    async def translate_chapter_async(914        self,915        book: str,916        chapter: int,917        skip_existing: bool = True,918        max_concurrent: int = 5,919    ) -> list[TranslationOutput]:920        """Translate all verses in a chapter asynchronously.921922        Args:923            book: Book name.924            chapter: Chapter number.925            skip_existing: If True, skip already translated verses.926            max_concurrent: Maximum concurrent CLI calls.927928        Returns:929            List of TranslationOutput for each verse.930        """931        # Get all verses in the chapter932        verses = self.context_builder.loader.load_chapter_verses(book, chapter)933934        # Create semaphore for rate limiting935        semaphore = asyncio.Semaphore(max_concurrent)936937        async def translate_with_limit(ref: VerseReference) -> TranslationOutput:938            async with semaphore:939                result = await self.translate_verse_async(ref, skip_existing)940                await asyncio.sleep(self.rate_limit_delay)941                return result942943        tasks = [translate_with_limit(v.reference) for v in verses]944        outputs = await asyncio.gather(*tasks)945946        # Save TM after chapter947        if self._translation_memory is not None:948            self._translation_memory.save()949950        return list(outputs)951952    def _get_sources_used(self, context: TranslationContext) -> list[str]:953        """Get list of source text types used."""954        sources = []955        verse = context.target_verse956957        if verse.greek_text:958            sources.append("greek")959        if verse.hebrew_text:960            sources.append("hebrew")961        if verse.japanese_meiji:962            sources.append("meiji")963        if verse.japanese_taisho:964            sources.append("taisho")965        if verse.japanese_kougo:966            sources.append("kougo")967        if verse.english_kjv:968            sources.append("kjv")969        if verse.english_asv:970            sources.append("asv")971972        return sources973974    def translate_chunk(975        self,976        book: str,977        pericope: Pericope,978        skip_existing: bool = True,979    ) -> ChunkTranslationOutput:980        """Translate an entire pericope/chunk as a cohesive unit.981982        Args:983            book: Book name.984            pericope: The pericope to translate.985            skip_existing: If True, skip verses already translated.986987        Returns:988            ChunkTranslationOutput with verse-aligned translations.989        """990        # Build chunk context991        context = self.context_builder.build_chunk_context(book, pericope)992993        # Check if all verses already translated994        if skip_existing:995            existing = self._get_existing_chunk_translations(book, pericope)996            if len(existing) == len(pericope.verses):997                return ChunkTranslationOutput(998                    pericope_id=pericope.id,999                    book=book,1000                    translations=existing,1001                    model=self.model,1002                    coherence_note="Loaded from existing translations",1003                )10041005        # Get TM matches for the combined source text1006        tm_matches = self._get_chunk_tm_matches(context)10071008        # Seam continuity: surface our own translations of the immediately1009        # preceding verses so the new chunk continues them seamlessly.1010        preceding_translations = []1011        for v in context.preceding_verses:1012            existing_t = self.storage.load_translation(v.reference)1013            if existing_t and existing_t.text:1014                preceding_translations.append((v.reference, existing_t.text))10151016        # Build chunk prompt1017        prompt = self.prompt_builder.build_chunk_prompt(1018            context,1019            tm_matches=tm_matches,1020            preceding_translations=preceding_translations or None,1021        )1022        system_prompt = self.prompt_builder.get_chunk_system_prompt()10231024        # Call Claude CLI1025        response = self._call_claude(system_prompt, prompt)10261027        # Parse JSON response1028        translations = self._parse_chunk_response(response, pericope.verses)10291030        # Create output1031        output = ChunkTranslationOutput(1032            pericope_id=pericope.id,1033            book=book,1034            translations=translations,1035            model=self.model,1036        )10371038        # Save individual verse translations1039        self._save_chunk_translations(book, translations)10401041        # Add to translation memory1042        self._add_chunk_to_tm(context, translations)10431044        return output10451046    def translate_chunk_by_id(1047        self,1048        pericope_id: str,1049        skip_existing: bool = True,1050    ) -> ChunkTranslationOutput | None:1051        """Translate a pericope by its ID.10521053        Args:1054            pericope_id: The pericope identifier.1055            skip_existing: If True, skip verses already translated.10561057        Returns:1058            ChunkTranslationOutput or None if pericope not found.1059        """1060        result = self.context_builder.pericope_manager.get_pericope_by_id(pericope_id)1061        if result is None:1062            return None10631064        book, pericope = result1065        return self.translate_chunk(book, pericope, skip_existing)10661067    def translate_chapter_by_pericopes(1068        self,1069        book: str,1070        chapter: int,1071        skip_existing: bool = True,1072        progress_callback: Callable[..., Any] | None = None,1073    ) -> list[ChunkTranslationOutput]:1074        """Translate a chapter using pericope-based chunking.10751076        Args:1077            book: Book name.1078            chapter: Chapter number.1079            skip_existing: If True, skip already translated pericopes.1080            progress_callback: Optional callback for progress updates.10811082        Returns:1083            List of ChunkTranslationOutput for each pericope.1084        """1085        pericopes = self.context_builder.pericope_manager.get_chapter_pericopes(1086            book, chapter1087        )10881089        if not pericopes:1090            # Fall back to verse-by-verse translation1091            outputs = self.translate_chapter(1092                book, chapter, skip_existing, progress_callback1093            )1094            # Convert to chunk format1095            return [1096                ChunkTranslationOutput(1097                    pericope_id=f"{book}-{chapter}-fallback",1098                    book=book,1099                    translations={1100                        f"{o.reference.chapter}:{o.reference.verse}": o.text1101                        for o in outputs1102                    },1103                    model=self.model,1104                    coherence_note="Verse-by-verse fallback (no pericope data)",1105                )1106            ]11071108        outputs = []1109        for i, pericope in enumerate(pericopes):1110            if progress_callback:1111                progress_callback(i + 1, len(pericopes), pericope)11121113            output = self.translate_chunk(book, pericope, skip_existing)1114            outputs.append(output)11151116            # Rate limiting between pericopes1117            if i < len(pericopes) - 1:1118                time.sleep(self.rate_limit_delay)11191120        # Save TM after chapter1121        if self._translation_memory is not None:1122            self._translation_memory.save()11231124        return outputs11251126    def _get_existing_chunk_translations(1127        self, book: str, pericope: Pericope1128    ) -> dict[str, str]:1129        """Get existing translations for verses in a pericope.11301131        Args:1132            book: Book name.1133            pericope: The pericope to check.11341135        Returns:1136            Dict mapping verse ref to translation text.1137        """1138        existing = {}1139        for verse_ref in pericope.verses:1140            chapter, verse = map(int, verse_ref.split(":"))1141            ref = VerseReference(book=book, chapter=chapter, verse=verse)1142            if self.storage.is_verse_translated(ref):1143                translation = self.storage.load_translation(ref)1144                if translation:1145                    existing[verse_ref] = translation.text1146        return existing11471148    def _get_chunk_tm_matches(1149        self, context: ChunkTranslationContext1150    ) -> list | None:1151        """Get TM matches for chunk context.11521153        Args:1154            context: Chunk translation context.11551156        Returns:1157            List of TM matches or None.1158        """1159        if self._translation_memory is None:1160            return None11611162        # Use combined source text for matching1163        combined = context.get_combined_source_text()1164        if not combined:1165            return None11661167        # Get matches for the first verse as representative1168        first_verse = context.verses[0] if context.verses else None1169        if not first_verse:  # pragma: no cover - combined text implies a verse1170            return None11711172        source = first_verse.greek_text or first_verse.hebrew_text or ""1173        if not source:1174            return None11751176        matches = self._translation_memory.find_matches(1177            source, threshold=0.7, max_results=31178        )1179        return matches if matches else None11801181    def _parse_chunk_response(1182        self, response: str, expected_verses: list[str]1183    ) -> dict[str, str]:1184        """Parse JSON response from chunk translation.11851186        Uses robust JSON extraction that handles:1187        - Markdown code blocks (```json ... ```)1188        - Nested JSON structures1189        - Text before/after the JSON1190        - Unicode characters in values11911192        Args:1193            response: Raw response from Claude.1194            expected_verses: List of expected verse refs.11951196        Returns:1197            Dict mapping verse ref to translation.11981199        Raises:1200            ChunkParsingError: If response cannot be parsed.1201        """1202        original_response = response1203        response = response.strip()12041205        # Strategy 1: Remove markdown code blocks if present1206        if "```" in response:1207            response = self._extract_from_code_block(response)12081209        # Strategy 2: Try direct JSON parse first (most common case)1210        translations = self._try_parse_json(response)1211        if translations is not None:1212            return self._validate_translations(translations, expected_verses, original_response)12131214        # Strategy 3: Find JSON object using bracket matching1215        json_str = self._extract_json_by_brackets(response)1216        if json_str:1217            translations = self._try_parse_json(json_str)1218            if translations is not None:1219                return self._validate_translations(translations, expected_verses, original_response)12201221        # Strategy 4: Try to find JSON anywhere in the response1222        for match in re.finditer(r'\{', response):1223            start = match.start()1224            json_str = self._extract_json_from_position(response, start)1225            if json_str:1226                translations = self._try_parse_json(json_str)1227                if translations is not None:1228                    return self._validate_translations(translations, expected_verses, original_response)12291230        # All strategies failed1231        raise ChunkParsingError(1232            "Failed to parse chunk translation response as JSON",1233            response=original_response,1234            expected_verses=expected_verses,1235        )12361237    def _extract_from_code_block(self, response: str) -> str:1238        """Extract content from markdown code block.12391240        Args:1241            response: Response potentially containing code blocks.12421243        Returns:1244            Content inside code block, or original if not found.1245        """1246        lines = response.split("\n")1247        json_lines = []1248        in_block = False12491250        for line in lines:1251            stripped = line.strip()1252            if stripped.startswith("```") and not in_block:1253                in_block = True1254                continue1255            elif stripped.startswith("```") and in_block:1256                break1257            elif in_block:1258                json_lines.append(line)12591260        if json_lines:1261            return "\n".join(json_lines)1262        return response12631264    def _try_parse_json(self, text: str) -> dict | None:1265        """Try to parse text as JSON dict.12661267        Args:1268            text: Text to parse.12691270        Returns:1271            Parsed dict or None if parsing failed.1272        """1273        try:1274            result = json.loads(text)1275            if isinstance(result, dict):1276                return result1277        except json.JSONDecodeError:1278            pass1279        return None12801281    def _extract_json_by_brackets(self, text: str) -> str | None:1282        """Extract JSON object by matching brackets.12831284        Handles nested objects correctly.12851286        Args:1287            text: Text containing JSON.12881289        Returns:1290            Extracted JSON string or None if not found.1291        """1292        start = text.find("{")1293        if start == -1:1294            return None12951296        depth = 01297        in_string = False1298        escape_next = False12991300        for i, char in enumerate(text[start:], start):1301            if escape_next:1302                escape_next = False1303                continue13041305            if char == "\\":1306                escape_next = True1307                continue13081309            if char == '"' and not escape_next:1310                in_string = not in_string1311                continue13121313            if in_string:1314                continue13151316            if char == "{":1317                depth += 11318            elif char == "}":1319                depth -= 11320                if depth == 0:1321                    return text[start : i + 1]13221323        return None13241325    def _extract_json_from_position(self, text: str, start: int) -> str | None:1326        """Extract JSON object starting at given position.13271328        Args:1329            text: Full text.1330            start: Starting position of '{'.13311332        Returns:1333            Extracted JSON string or None if invalid.1334        """1335        if start >= len(text) or text[start] != "{":1336            return None13371338        depth = 01339        in_string = False1340        escape_next = False13411342        for i, char in enumerate(text[start:], start):1343            if escape_next:1344                escape_next = False1345                continue13461347            if char == "\\":1348                escape_next = True1349                continue13501351            if char == '"' and not escape_next:1352                in_string = not in_string1353                continue13541355            if in_string:1356                continue13571358            if char == "{":1359                depth += 11360            elif char == "}":1361                depth -= 11362                if depth == 0:1363                    return text[start : i + 1]13641365        return None13661367    def _validate_translations(1368        self,1369        translations: dict,1370        expected_verses: list[str],1371        original_response: str,1372    ) -> dict[str, str]:1373        """Validate parsed translations have all expected verses.13741375        Args:1376            translations: Parsed translation dict.1377            expected_verses: List of expected verse refs.1378            original_response: Original response for error context.13791380        Returns:1381            Validated translations dict.13821383        Raises:1384            ChunkParsingError: If validation fails.1385        """1386        if not isinstance(translations, dict):1387            raise ChunkParsingError(1388                f"Expected dict, got {type(translations).__name__}",1389                response=original_response,1390                expected_verses=expected_verses,1391            )13921393        # Validate all expected verses are present1394        missing = set(expected_verses) - set(translations.keys())1395        if missing:1396            raise ChunkParsingError(1397                f"Missing translations for verses: {sorted(missing)}",1398                response=original_response,1399                expected_verses=expected_verses,1400            )14011402        # Keep only the verses this pericope asked for. The prompt shows the1403        # preceding and following verses with their references, and a model1404        # that "helpfully" translates one of them would otherwise overwrite a1405        # verse in the neighbouring pericope; a stray non-verse key ("note")1406        # would crash the save loop after half the chunk was written.1407        extra = set(translations) - set(expected_verses)1408        if extra:1409            logger.warning(1410                "Discarding %d unrequested key(s) from chunk response: %s",1411                len(extra),1412                sorted(map(str, extra))[:5],1413            )14141415        # Validate all values are strings1416        for ref in expected_verses:1417            text = translations[ref]1418            if not isinstance(text, str):1419                raise ChunkParsingError(1420                    f"Translation for {ref} is not a string: {type(text).__name__}",1421                    response=original_response,1422                    expected_verses=expected_verses,1423                )14241425        return {ref: translations[ref] for ref in expected_verses}14261427    def _save_chunk_translations(1428        self, book: str, translations: dict[str, str]1429    ) -> None:1430        """Save individual verse translations from chunk output.14311432        Args:1433            book: Book name.1434            translations: Dict mapping verse ref to translation.1435        """1436        for verse_ref, text in translations.items():1437            chapter, verse = map(int, verse_ref.split(":"))1438            ref = VerseReference(book=book, chapter=chapter, verse=verse)14391440            # Parse furigana from translation text1441            plain_text, annotations = parse_furigana(text)14421443            output = TranslationOutput(1444                reference=ref,1445                text=plain_text,1446                text_with_furigana=text,1447                furigana_annotations=[1448                    {1449                        "start_offset": a.start_offset,1450                        "end_offset": a.end_offset,1451                        "base_text": a.base_text,1452                        "reading": a.reading,1453                    }1454                    for a in annotations1455                ],1456                model=self.model,1457                sources_used=["chunk_translation"],1458            )1459            self.storage.save_translation(output)14601461    def _add_chunk_to_tm(1462        self,1463        context: ChunkTranslationContext,1464        translations: dict[str, str],1465    ) -> None:1466        """Add chunk translations to translation memory.14671468        Args:1469            context: Chunk translation context.1470            translations: Dict mapping verse ref to translation.1471        """1472        if self._translation_memory is None:1473            return14741475        for verse in context.verses:1476            ref = f"{verse.reference.chapter}:{verse.reference.verse}"1477            source = verse.greek_text or verse.hebrew_text1478            target = translations.get(ref)14791480            if source and target:1481                self._translation_memory.add_from_translation(1482                    source_text=source,1483                    target_text=target,1484                    reference=verse.reference,1485                    metadata={"model": self.model, "pericope_id": context.pericope.id},1486                )1487