translator.py

1"""Translator module using Claude CLI."""23from __future__ import annotations45import asyncio6import json7import logging8import os9import re10import subprocess11import time12from collections.abc import Callable13from typing import TYPE_CHECKING, Any1415from src.context.builder import ContextBuilder16from src.data.storage import Storage17from src.exceptions import (18    ChunkParsingError,19    ClaudeAPIError,20    EvaluationError,21    QualityThresholdError,22    RateLimitError,23    classify_claude_error,24)25from src.models.pericope import ChunkTranslationContext, ChunkTranslationOutput, Pericope26from src.models.verse import TranslationOutput, VerseReference27from src.pipeline.prompt import PromptBuilder28from src.utils.furigana import parse_furigana29from src.utils.retry import RetryConfig, RetryState3031if TYPE_CHECKING:32    from src.context.translation_memory import TranslationMemoryManager33    from src.evaluation.evaluator import TranslationEvaluator34    from src.evaluation.storage import EvaluationStorage35    from src.models.evaluation import VerseEvaluation36    from src.models.verse import TranslationContext37    from src.orchestration.provenance import TranslationProvenance3839logger = logging.getLogger(__name__)4041# Find Claude binary42CLAUDE_BINARY = os.environ.get("CLAUDE_BINARY", "claude")434445def _clean_env() -> dict[str, str]:46    """Return a copy of the environment without CLAUDECODE to avoid nested session errors."""47    env = os.environ.copy()48    env.pop("CLAUDECODE", None)49    return env5051DEFAULT_MODEL = os.environ.get("TRANSLATE_BIBLE_MODEL", "claude-opus-4-8")525354def _sanitize_translation_output(raw: str) -> str:55    """Strip meta-commentary from a verse translation response.5657    Models occasionally emit self-corrections ("Wait—let me provide the58    correct translation.") or markdown fences around the answer. A verse59    translation is a single Japanese block, so: drop code fences, drop60    predominantly-Latin blocks, and if several Japanese blocks remain61    (a self-correction), keep the last one.62    """63    text = raw.strip()64    if not text:65        return text6667    lines = [ln for ln in text.split("\n") if not ln.strip().startswith("```")]6869    blocks: list[list[str]] = [[]]70    for ln in lines:71        if ln.strip():72            blocks[-1].append(ln)73        elif blocks[-1]:74            blocks.append([])75    block_texts = ["\n".join(b).strip() for b in blocks if b]7677    def is_japanese(block: str) -> bool:78        cjk = sum(1 for ch in block if "぀" <= ch <= "鿿" or "！" <= ch <= "｠")79        latin = sum(1 for ch in block if ch.isascii() and ch.isalpha())80        return cjk > latin8182    japanese_blocks = [b for b in block_texts if is_japanese(b)]83    if not japanese_blocks:84        return text8586    result = japanese_blocks[-1]87    result = re.sub(r"^(翻訳|訳文|Translation)\s*[:：]\s*", "", result)88    if result != text:89        logger.warning("Sanitized translation output (removed meta-commentary)")90    return result9192# Default retry configuration93DEFAULT_RETRY_CONFIG = RetryConfig(94    max_attempts=3,95    base_delay=2.0,96    max_delay=60.0,97    exponential_base=2.0,98    retryable_exceptions=(RateLimitError, ConnectionError, TimeoutError),99)100101# Quality thresholds for evaluation gating102DEFAULT_QUALITY_THRESHOLD = 70.0103THEOLOGICAL_ACCURACY_THRESHOLD = 75.0104105106class Translator:107    """Translates Bible verses using Claude CLI."""108109    def __init__(110        self,111        context_builder: ContextBuilder,112        prompt_builder: PromptBuilder,113        storage: Storage,114        model: str = DEFAULT_MODEL,115        rate_limit_delay: float = 1.2,116        translation_memory: TranslationMemoryManager | None = None,117        auto_evaluate: bool = True,118        evaluator: TranslationEvaluator | None = None,119        evaluation_storage: EvaluationStorage | None = None,120        retry_config: RetryConfig | None = None,121        quality_threshold: float = DEFAULT_QUALITY_THRESHOLD,122        enforce_quality_gate: bool = False,123        subprocess_timeout: float | None = None,124        enable_proto_archive: bool = False,125    ) -> None:126        """Initialize the translator.127128        Args:129            context_builder: Context builder for creating translation context.130            prompt_builder: Prompt builder for formatting prompts.131            storage: Storage for saving translations.132            model: Claude model to use.133            rate_limit_delay: Delay between API calls in seconds.134            translation_memory: Optional TM manager for reuse.135            auto_evaluate: Whether to automatically evaluate translations.136            evaluator: Evaluator to use for auto-evaluation.137            evaluation_storage: Storage for evaluation results.138            retry_config: Configuration for retry with backoff.139            quality_threshold: Minimum score for evaluation gating.140            enforce_quality_gate: If True, reject low-quality translations.141            subprocess_timeout: Timeout for Claude CLI calls in seconds.142                Defaults to the TRANSLATE_BIBLE_TIMEOUT env var, or 120.143                Long pericope chunks (10+ verses) can need 300s+.144            enable_proto_archive: Whether to archive translation inputs/outputs.145        """146        self.context_builder = context_builder147        self.prompt_builder = prompt_builder148        self.storage = storage149        self.model = model150        self.rate_limit_delay = rate_limit_delay151        self._translation_memory = translation_memory152        self._auto_evaluate = auto_evaluate153        self._evaluator = evaluator154        self._evaluation_storage = evaluation_storage155        self._retry_config = retry_config or DEFAULT_RETRY_CONFIG156        self._quality_threshold = quality_threshold157        self._enforce_quality_gate = enforce_quality_gate158        if subprocess_timeout is None:159            subprocess_timeout = float(os.environ.get("TRANSLATE_BIBLE_TIMEOUT", "120"))160        self._subprocess_timeout = subprocess_timeout161        self._enable_archive = enable_proto_archive162        # Enable archive on storage if requested163        if enable_proto_archive:164            self.storage.enable_archive = True165166    @property167    def enable_proto_archive(self) -> bool:168        """Get proto archive enabled status."""169        return self._enable_archive170171    @enable_proto_archive.setter172    def enable_proto_archive(self, value: bool) -> None:173        """Set proto archive enabled status."""174        self._enable_archive = value175        self.storage.enable_archive = value176177    @property178    def translation_memory(self) -> TranslationMemoryManager | None:179        """Get the translation memory manager."""180        return self._translation_memory181182    @translation_memory.setter183    def translation_memory(self, value: TranslationMemoryManager | None) -> None:184        """Set the translation memory manager."""185        self._translation_memory = value186187    @property188    def auto_evaluate(self) -> bool:189        """Get auto-evaluation setting."""190        return self._auto_evaluate191192    @auto_evaluate.setter193    def auto_evaluate(self, value: bool) -> None:194        """Set auto-evaluation setting."""195        self._auto_evaluate = value196197    def _call_claude(self, system_prompt: str, user_prompt: str) -> str:198        """Call Claude CLI with the given prompts.199200        Includes retry logic with exponential backoff for transient errors.201202        Args:203            system_prompt: System prompt for Claude.204            user_prompt: User prompt for Claude.205206        Returns:207            The response text from Claude.208209        Raises:210            ClaudeAPIError: If the Claude CLI call fails after all retries.211        """212        retry_state = RetryState(self._retry_config)213214        while True:215            try:216                result = subprocess.run(217                    [218                        CLAUDE_BINARY,219                        "--print",220                        "-p",221                        user_prompt,222                        "--system-prompt",223                        system_prompt,224                        "--model",225                        self.model,226                    ],227                    capture_output=True,228                    text=True,229                    check=False,230                    timeout=self._subprocess_timeout,231                    env=_clean_env(),232                )233234                if result.returncode != 0:235                    error = classify_claude_error(result.stderr, result.returncode)236                    raise error237238                return result.stdout.strip()239240            except subprocess.TimeoutExpired as e:241                error = ClaudeAPIError(242                    f"Claude CLI timed out after {self._subprocess_timeout}s",243                    stderr=str(e),244                    return_code=-1,245                )246                if retry_state.should_retry(error):247                    logger.warning(248                        "Claude CLI timeout, attempt %d/%d. Retrying...",249                        retry_state.attempt + 1,250                        self._retry_config.max_attempts,251                    )252                    retry_state.wait()253                else:254                    raise error from e255256            except FileNotFoundError as e:257                # Claude CLI not installed258                from src.exceptions import CLINotFoundError259                raise CLINotFoundError() from e260261            except ClaudeAPIError as e:262                if retry_state.should_retry(e):263                    logger.warning(264                        "Claude API error, attempt %d/%d: %s. Retrying...",265                        retry_state.attempt + 1,266                        self._retry_config.max_attempts,267                        str(e),268                    )269                    retry_state.wait()270                else:271                    raise272273    async def _call_claude_async(self, system_prompt: str, user_prompt: str) -> str:274        """Call Claude CLI asynchronously with the given prompts.275276        Includes retry logic with exponential backoff for transient errors.277278        Args:279            system_prompt: System prompt for Claude.280            user_prompt: User prompt for Claude.281282        Returns:283            The response text from Claude.284285        Raises:286            ClaudeAPIError: If the Claude CLI call fails after all retries.287        """288        retry_state = RetryState(self._retry_config)289290        while True:291            try:292                process = await asyncio.wait_for(293                    asyncio.create_subprocess_exec(294                        CLAUDE_BINARY,295                        "--print",296                        "-p",297                        user_prompt,298                        "--system-prompt",299                        system_prompt,300                        "--model",301                        self.model,302                        stdout=asyncio.subprocess.PIPE,303                        stderr=asyncio.subprocess.PIPE,304                        env=_clean_env(),305                    ),306                    timeout=self._subprocess_timeout,307                )308                stdout, stderr = await asyncio.wait_for(309                    process.communicate(),310                    timeout=self._subprocess_timeout,311                )312313                if process.returncode != 0:314                    error = classify_claude_error(stderr.decode(), process.returncode or 1)315                    raise error316317                return stdout.decode().strip()318319            except asyncio.TimeoutError as e:320                error = ClaudeAPIError(321                    f"Claude CLI timed out after {self._subprocess_timeout}s",322                    stderr="Async timeout",323                    return_code=-1,324                )325                if retry_state.should_retry(error):326                    logger.warning(327                        "Claude CLI async timeout, attempt %d/%d. Retrying...",328                        retry_state.attempt + 1,329                        self._retry_config.max_attempts,330                    )331                    await retry_state.async_wait()332                else:333                    raise error from e334335            except FileNotFoundError as e:336                from src.exceptions import CLINotFoundError337                raise CLINotFoundError() from e338339            except ClaudeAPIError as e:340                if retry_state.should_retry(e):341                    logger.warning(342                        "Claude API error (async), attempt %d/%d: %s. Retrying...",343                        retry_state.attempt + 1,344                        self._retry_config.max_attempts,345                        str(e),346                    )347                    await retry_state.async_wait()348                else:349                    raise350351    def _get_tm_matches(self, context: TranslationContext) -> list | None:352        """Get TM matches for the context if TM is enabled.353354        Args:355            context: Translation context.356357        Returns:358            List of TM matches or None if TM not available.359        """360        if self._translation_memory is None:361            return None362363        # Get source text for matching364        source_text = (365            context.target_verse.greek_text366            or context.target_verse.hebrew_text367            or ""368        )369        if not source_text:370            return None371372        matches = self._translation_memory.find_matches(373            source_text, threshold=0.7, max_results=3374        )375        return matches if matches else None376377    def _add_to_tm(378        self, context: TranslationContext, translation_text: str379    ) -> None:380        """Add completed translation to translation memory.381382        Args:383            context: Translation context.384            translation_text: The translated text.385        """386        if self._translation_memory is None:387            return388389        source_text = (390            context.target_verse.greek_text391            or context.target_verse.hebrew_text392        )393        if not source_text:394            return395396        self._translation_memory.add_from_translation(397            source_text=source_text,398            target_text=translation_text,399            reference=context.target_verse.reference,400            metadata={"model": self.model},401        )402403    def _evaluate_translation(404        self,405        output: TranslationOutput,406        context: TranslationContext,407    ) -> VerseEvaluation | None:408        """Evaluate a translation if auto-evaluate is enabled.409410        Args:411            output: The translation output.412            context: The translation context.413414        Returns:415            VerseEvaluation or None if evaluation failed/disabled.416417        Raises:418            EvaluationError: If enforce_quality_gate is True and evaluation fails.419        """420        if not self._auto_evaluate or not self._evaluator:421            return None422423        ref = output.reference424        ref_str = f"{ref.book} {ref.chapter}:{ref.verse}"425426        try:427            from src.evaluation.evaluator import (428                EVALUATION_SYSTEM_PROMPT,429                parse_evaluation_response,430            )431432            # Build evaluation prompt433            eval_prompt = self._evaluator.build_verse_prompt(434                output.text, context435            )436437            # Call Claude for evaluation438            response = self._call_claude(EVALUATION_SYSTEM_PROMPT, eval_prompt)439440            # Parse response441            scores = parse_evaluation_response(response)442443            # Create evaluation444            evaluation = self._evaluator.create_verse_evaluation(445                verse_number=output.reference.verse,446                scores=scores,447            )448449            # Log evaluation result450            logger.info(451                "Evaluated %s: overall=%.1f%%, flags=%s",452                ref_str,453                evaluation.overall_score,454                evaluation.flags or "none",455            )456457            return evaluation458459        except Exception as e:460            # Log the error with details461            logger.error(462                "Evaluation failed for %s: %s",463                ref_str,464                str(e),465            )466467            # If quality gate is enforced, raise the error468            if self._enforce_quality_gate:469                raise EvaluationError(470                    f"Evaluation failed for {ref_str}: {e}",471                    verse_ref=ref_str,472                ) from e473474            # Otherwise, return None (don't block translation)475            return None476477    def _check_quality_gate(478        self,479        evaluation: VerseEvaluation,480        reference: VerseReference,481    ) -> None:482        """Check if translation passes quality threshold.483484        Args:485            evaluation: The verse evaluation.486            reference: The verse reference.487488        Raises:489            QualityThresholdError: If translation fails quality gate.490        """491        if not self._enforce_quality_gate:492            return493494        ref_str = f"{reference.book} {reference.chapter}:{reference.verse}"495496        # Check overall score497        if evaluation.overall_score < self._quality_threshold:498            raise QualityThresholdError(499                f"Translation for {ref_str} failed quality gate: "500                f"{evaluation.overall_score:.1f}% < {self._quality_threshold}%",501                score=evaluation.overall_score,502                threshold=self._quality_threshold,503            )504505        # Check theological accuracy specifically506        if "theological_accuracy" in evaluation.scores:507            ta_score = evaluation.scores["theological_accuracy"].score508            if ta_score < THEOLOGICAL_ACCURACY_THRESHOLD:509                raise QualityThresholdError(510                    f"Translation for {ref_str} failed theological accuracy: "511                    f"{ta_score:.1f}% < {THEOLOGICAL_ACCURACY_THRESHOLD}%",512                    score=ta_score,513                    threshold=THEOLOGICAL_ACCURACY_THRESHOLD,514                    dimension="theological_accuracy",515                )516517    def _save_evaluation(518        self,519        evaluation: VerseEvaluation,520        reference: VerseReference,521    ) -> None:522        """Save evaluation result if storage is configured.523524        Args:525            evaluation: The evaluation to save.526            reference: The verse reference.527        """528        if not self._evaluation_storage:529            return530531        self._evaluation_storage.save_verse_evaluation(532            reference.book,533            reference.chapter,534            evaluation,535        )536537    def translate_verse(538        self,539        reference: VerseReference,540        skip_existing: bool = True,541        evaluate: bool | None = None,542    ) -> TranslationOutput:543        """Translate a single verse.544545        Args:546            reference: Verse reference to translate.547            skip_existing: If True, return existing translation if available.548            evaluate: Override auto_evaluate setting for this call.549550        Returns:551            TranslationOutput with the translation.552        """553        # Check if already translated554        if skip_existing and self.storage.is_verse_translated(reference):555            existing = self.storage.load_translation(reference)556            if existing:557                return existing558559        # Build context and prompt560        context = self.context_builder.build_context(reference)561562        # Get TM matches563        tm_matches = self._get_tm_matches(context)564565        # Build prompt (with TM matches if available)566        prompt = self.prompt_builder.build_prompt(context, tm_matches=tm_matches)567        system_prompt = self.prompt_builder.get_system_prompt()568569        # Call Claude CLI570        translation_text = _sanitize_translation_output(571            self._call_claude(system_prompt, prompt)572        )573574        # Parse furigana from Claude's response575        plain_text, annotations = parse_furigana(translation_text)576577        # Create output578        output = TranslationOutput(579            reference=reference,580            text=plain_text,581            text_with_furigana=translation_text,582            furigana_annotations=[583                {584                    "start_offset": a.start_offset,585                    "end_offset": a.end_offset,586                    "base_text": a.base_text,587                    "reading": a.reading,588                }589                for a in annotations590            ],591            model=self.model,592            sources_used=self._get_sources_used(context),593            glossary_terms_applied=[t.original for t in context.glossary_terms],594        )595596        # Save translation597        self.storage.save_translation(output)598599        # Add to TM600        self._add_to_tm(context, plain_text)601602        # Evaluate if enabled603        should_evaluate = evaluate if evaluate is not None else self._auto_evaluate604        if should_evaluate:605            evaluation = self._evaluate_translation(output, context)606            if evaluation:607                self._save_evaluation(evaluation, reference)608                # Check quality gate609                self._check_quality_gate(evaluation, reference)610611        return output612613    def translate_verse_with_provenance(  # pragma: no cover614        self,615        reference: VerseReference,616        skip_existing: bool = True,617    ) -> tuple[TranslationOutput, "TranslationProvenance"]:618        """Translate a verse and capture full provenance.619620        Args:621            reference: Verse reference to translate.622            skip_existing: If True, return existing translation if available.623624        Returns:625            Tuple of (TranslationOutput, TranslationProvenance).626        """627        from datetime import datetime628629        from src.orchestration.provenance import (630            TranslationProvenance,631            compute_glossary_hash,632            compute_style_hash,633        )634635        # Check if already translated636        if skip_existing and self.storage.is_verse_translated(reference):637            existing = self.storage.load_translation(reference)638            if existing:639                # Try to load existing provenance640                provenance = self.storage.load_provenance(reference)641                if provenance:642                    return existing, provenance643                # Create minimal provenance for existing translation644                provenance = TranslationProvenance(model=existing.model)645                return existing, provenance646647        # Track timing648        started_at = datetime.utcnow()649650        # Build context651        context = self.context_builder.build_context(reference)652653        # Get TM matches654        tm_matches = self._get_tm_matches(context) or []655656        # Build prompts657        prompt = self.prompt_builder.build_prompt(context, tm_matches=tm_matches)658        system_prompt = self.prompt_builder.get_system_prompt()659660        # Build provenance661        provenance = TranslationProvenance(662            model=self.model,663            prompt_version="v1.0",664            system_prompt=system_prompt,665            user_prompt=prompt,666            started_at=started_at,667            attempt_number=1,668        )669670        # Capture source texts671        verse = context.target_verse672        if verse.greek_text:673            provenance.source_texts["greek"] = verse.greek_text674        if verse.hebrew_text:675            provenance.source_texts["hebrew"] = verse.hebrew_text676        if verse.english_kjv:677            provenance.source_texts["kjv"] = verse.english_kjv678        if verse.english_asv:679            provenance.source_texts["asv"] = verse.english_asv680        if verse.japanese_meiji:681            provenance.source_texts["meiji"] = verse.japanese_meiji682        if verse.japanese_taisho:683            provenance.source_texts["taisho"] = verse.japanese_taisho684685        # Compute source hashes686        provenance.compute_source_hashes()687688        # Capture context verses689        provenance.preceding_verses = [690            str(v.reference) for v in context.preceding_verses691        ]692        provenance.following_verses = [693            str(v.reference) for v in context.following_verses694        ]695696        # Capture chapter summary697        if context.chapter_summary:698            provenance.chapter_summary = context.chapter_summary.summary699700        # Capture book metadata701        if context.book_metadata:702            provenance.book_metadata = {703                "name": context.book_metadata.name_english,704                "testament": context.book_metadata.testament,705                "themes": context.book_metadata.themes,706            }707708        # Capture glossary hash709        provenance.glossary_hash = compute_glossary_hash(context.glossary_terms)710711        # Capture style hash712        if self.prompt_builder._style:713            style_dict = self.prompt_builder._style.to_dict()714            provenance.style_config_hash = compute_style_hash(style_dict)715716        # Capture TM matches717        if tm_matches:718            provenance.tm_matches_used = [719                {720                    "source": m.segment.source_text,721                    "target": m.segment.target_text,722                    "similarity": m.similarity,723                }724                for m in tm_matches725            ]726727        # Create input proto for archive (before translation)728        input_proto = None729        if self._enable_archive:730            from src.protos.converters import (731                translation_context_to_prompt_input,732                tm_match_to_proto,733            )734            from src.protos.serialization import compute_proto_hash735736            input_proto = translation_context_to_prompt_input(737                context, system_prompt, prompt, self.model,738            )739            # Add TM matches to input proto740            if tm_matches and hasattr(tm_matches[0], "segment"):741                input_proto.tm_matches = [tm_match_to_proto(m) for m in tm_matches]742743        # Call Claude744        translation_text = self._call_claude(system_prompt, prompt)745746        # Complete timing747        completed_at = datetime.utcnow()748        provenance.completed_at = completed_at749750        # Parse furigana from Claude's response751        plain_text, annotations = parse_furigana(translation_text)752753        # Create output754        output = TranslationOutput(755            reference=reference,756            text=plain_text,757            text_with_furigana=translation_text,758            furigana_annotations=[759                {760                    "start_offset": a.start_offset,761                    "end_offset": a.end_offset,762                    "base_text": a.base_text,763                    "reading": a.reading,764                }765                for a in annotations766            ],767            model=self.model,768            sources_used=list(provenance.source_texts.keys()),769            glossary_terms_applied=[t.original for t in context.glossary_terms],770        )771772        # Save translation with provenance773        self.storage.save_translation_with_provenance(output, provenance)774775        # Save proto archive if enabled776        if self._enable_archive and input_proto is not None:777            from src.protos.converters import (778                provenance_to_proto,779                translation_result_to_proto,780            )781            from src.protos.serialization import compute_proto_hash782783            output_proto = translation_result_to_proto(784                output, translation_text, annotations,785                input_hash=compute_proto_hash(input_proto)786            )787            output_proto.provenance = provenance_to_proto(provenance)788            self.storage.save_translation_archive(reference, input_proto, output_proto)789790        # Add to TM791        self._add_to_tm(context, plain_text)792793        return output, provenance794795    async def translate_verse_async(796        self, reference: VerseReference, skip_existing: bool = True797    ) -> TranslationOutput:798        """Translate a single verse asynchronously.799800        Args:801            reference: Verse reference to translate.802            skip_existing: If True, return existing translation if available.803804        Returns:805            TranslationOutput with the translation.806        """807        # Check if already translated808        if skip_existing and self.storage.is_verse_translated(reference):809            existing = self.storage.load_translation(reference)810            if existing:811                return existing812813        # Build context and prompt814        context = self.context_builder.build_context(reference)815816        # Get TM matches817        tm_matches = self._get_tm_matches(context)818819        # Build prompt820        prompt = self.prompt_builder.build_prompt(context, tm_matches=tm_matches)821        system_prompt = self.prompt_builder.get_system_prompt()822823        # Call Claude CLI824        translation_text = await self._call_claude_async(system_prompt, prompt)825826        # Parse furigana from Claude's response827        plain_text, annotations = parse_furigana(translation_text)828829        # Create output830        output = TranslationOutput(831            reference=reference,832            text=plain_text,833            text_with_furigana=translation_text,834            furigana_annotations=[835                {836                    "start_offset": a.start_offset,837                    "end_offset": a.end_offset,838                    "base_text": a.base_text,839                    "reading": a.reading,840                }841                for a in annotations842            ],843            model=self.model,844            sources_used=self._get_sources_used(context),845            glossary_terms_applied=[t.original for t in context.glossary_terms],846        )847848        # Save translation849        self.storage.save_translation(output)850851        # Add to TM852        self._add_to_tm(context, plain_text)853854        return output855856    def translate_chapter(857        self,858        book: str,859        chapter: int,860        skip_existing: bool = True,861        progress_callback: Callable[..., Any] | None = None,862        evaluate: bool | None = None,863    ) -> list[TranslationOutput]:864        """Translate all verses in a chapter.865866        Args:867            book: Book name.868            chapter: Chapter number.869            skip_existing: If True, skip already translated verses.870            progress_callback: Optional callback for progress updates.871            evaluate: Override auto_evaluate setting for this call.872873        Returns:874            List of TranslationOutput for each verse.875        """876        # Get all verses in the chapter877        verses = self.context_builder.loader.load_chapter_verses(book, chapter)878        outputs = []879880        for i, verse in enumerate(verses):881            if progress_callback:882                progress_callback(i + 1, len(verses), verse.reference)883884            output = self.translate_verse(885                verse.reference, skip_existing, evaluate=evaluate886            )887            outputs.append(output)888889            # Rate limiting890            if i < len(verses) - 1:891                time.sleep(self.rate_limit_delay)892893        # Save TM after chapter894        if self._translation_memory is not None:895            self._translation_memory.save()896897        return outputs898899    async def translate_chapter_async(900        self,901        book: str,902        chapter: int,903        skip_existing: bool = True,904        max_concurrent: int = 5,905    ) -> list[TranslationOutput]:906        """Translate all verses in a chapter asynchronously.907908        Args:909            book: Book name.910            chapter: Chapter number.911            skip_existing: If True, skip already translated verses.912            max_concurrent: Maximum concurrent CLI calls.913914        Returns:915            List of TranslationOutput for each verse.916        """917        # Get all verses in the chapter918        verses = self.context_builder.loader.load_chapter_verses(book, chapter)919920        # Create semaphore for rate limiting921        semaphore = asyncio.Semaphore(max_concurrent)922923        async def translate_with_limit(ref: VerseReference) -> TranslationOutput:924            async with semaphore:925                result = await self.translate_verse_async(ref, skip_existing)926                await asyncio.sleep(self.rate_limit_delay)927                return result928929        tasks = [translate_with_limit(v.reference) for v in verses]930        outputs = await asyncio.gather(*tasks)931932        # Save TM after chapter933        if self._translation_memory is not None:934            self._translation_memory.save()935936        return list(outputs)937938    def _get_sources_used(self, context: TranslationContext) -> list[str]:939        """Get list of source text types used."""940        sources = []941        verse = context.target_verse942943        if verse.greek_text:944            sources.append("greek")945        if verse.hebrew_text:946            sources.append("hebrew")947        if verse.japanese_meiji:948            sources.append("meiji")949        if verse.japanese_taisho:950            sources.append("taisho")951        if verse.japanese_kougo:952            sources.append("kougo")953        if verse.english_kjv:954            sources.append("kjv")955        if verse.english_asv:956            sources.append("asv")957958        return sources959960    def translate_chunk(961        self,962        book: str,963        pericope: Pericope,964        skip_existing: bool = True,965    ) -> ChunkTranslationOutput:966        """Translate an entire pericope/chunk as a cohesive unit.967968        Args:969            book: Book name.970            pericope: The pericope to translate.971            skip_existing: If True, skip verses already translated.972973        Returns:974            ChunkTranslationOutput with verse-aligned translations.975        """976        # Build chunk context977        context = self.context_builder.build_chunk_context(book, pericope)978979        # Check if all verses already translated980        if skip_existing:981            existing = self._get_existing_chunk_translations(book, pericope)982            if len(existing) == len(pericope.verses):983                return ChunkTranslationOutput(984                    pericope_id=pericope.id,985                    book=book,986                    translations=existing,987                    model=self.model,988                    coherence_note="Loaded from existing translations",989                )990991        # Get TM matches for the combined source text992        tm_matches = self._get_chunk_tm_matches(context)993994        # Seam continuity: surface our own translations of the immediately995        # preceding verses so the new chunk continues them seamlessly.996        preceding_translations = []997        for v in context.preceding_verses:998            existing_t = self.storage.load_translation(v.reference)999            if existing_t and existing_t.text:1000                preceding_translations.append((v.reference, existing_t.text))10011002        # Build chunk prompt1003        prompt = self.prompt_builder.build_chunk_prompt(1004            context,1005            tm_matches=tm_matches,1006            preceding_translations=preceding_translations or None,1007        )1008        system_prompt = self.prompt_builder.get_chunk_system_prompt()10091010        # Call Claude CLI1011        response = self._call_claude(system_prompt, prompt)10121013        # Parse JSON response1014        translations = self._parse_chunk_response(response, pericope.verses)10151016        # Create output1017        output = ChunkTranslationOutput(1018            pericope_id=pericope.id,1019            book=book,1020            translations=translations,1021            model=self.model,1022        )10231024        # Save individual verse translations1025        self._save_chunk_translations(book, translations)10261027        # Add to translation memory1028        self._add_chunk_to_tm(context, translations)10291030        return output10311032    def translate_chunk_by_id(1033        self,1034        pericope_id: str,1035        skip_existing: bool = True,1036    ) -> ChunkTranslationOutput | None:1037        """Translate a pericope by its ID.10381039        Args:1040            pericope_id: The pericope identifier.1041            skip_existing: If True, skip verses already translated.10421043        Returns:1044            ChunkTranslationOutput or None if pericope not found.1045        """1046        result = self.context_builder.pericope_manager.get_pericope_by_id(pericope_id)1047        if result is None:1048            return None10491050        book, pericope = result1051        return self.translate_chunk(book, pericope, skip_existing)10521053    def translate_chapter_by_pericopes(1054        self,1055        book: str,1056        chapter: int,1057        skip_existing: bool = True,1058        progress_callback: Callable[..., Any] | None = None,1059    ) -> list[ChunkTranslationOutput]:1060        """Translate a chapter using pericope-based chunking.10611062        Args:1063            book: Book name.1064            chapter: Chapter number.1065            skip_existing: If True, skip already translated pericopes.1066            progress_callback: Optional callback for progress updates.10671068        Returns:1069            List of ChunkTranslationOutput for each pericope.1070        """1071        pericopes = self.context_builder.pericope_manager.get_chapter_pericopes(1072            book, chapter1073        )10741075        if not pericopes:1076            # Fall back to verse-by-verse translation1077            outputs = self.translate_chapter(1078                book, chapter, skip_existing, progress_callback1079            )1080            # Convert to chunk format1081            return [1082                ChunkTranslationOutput(1083                    pericope_id=f"{book}-{chapter}-fallback",1084                    book=book,1085                    translations={1086                        f"{o.reference.chapter}:{o.reference.verse}": o.text1087                        for o in outputs1088                    },1089                    model=self.model,1090                    coherence_note="Verse-by-verse fallback (no pericope data)",1091                )1092            ]10931094        outputs = []1095        for i, pericope in enumerate(pericopes):1096            if progress_callback:1097                progress_callback(i + 1, len(pericopes), pericope)10981099            output = self.translate_chunk(book, pericope, skip_existing)1100            outputs.append(output)11011102            # Rate limiting between pericopes1103            if i < len(pericopes) - 1:1104                time.sleep(self.rate_limit_delay)11051106        # Save TM after chapter1107        if self._translation_memory is not None:1108            self._translation_memory.save()11091110        return outputs11111112    def _get_existing_chunk_translations(1113        self, book: str, pericope: Pericope1114    ) -> dict[str, str]:1115        """Get existing translations for verses in a pericope.11161117        Args:1118            book: Book name.1119            pericope: The pericope to check.11201121        Returns:1122            Dict mapping verse ref to translation text.1123        """1124        existing = {}1125        for verse_ref in pericope.verses:1126            chapter, verse = map(int, verse_ref.split(":"))1127            ref = VerseReference(book=book, chapter=chapter, verse=verse)1128            if self.storage.is_verse_translated(ref):1129                translation = self.storage.load_translation(ref)1130                if translation:1131                    existing[verse_ref] = translation.text1132        return existing11331134    def _get_chunk_tm_matches(1135        self, context: ChunkTranslationContext1136    ) -> list | None:1137        """Get TM matches for chunk context.11381139        Args:1140            context: Chunk translation context.11411142        Returns:1143            List of TM matches or None.1144        """1145        if self._translation_memory is None:1146            return None11471148        # Use combined source text for matching1149        combined = context.get_combined_source_text()1150        if not combined:1151            return None11521153        # Get matches for the first verse as representative1154        first_verse = context.verses[0] if context.verses else None1155        if not first_verse:  # pragma: no cover - combined text implies a verse1156            return None11571158        source = first_verse.greek_text or first_verse.hebrew_text or ""1159        if not source:1160            return None11611162        matches = self._translation_memory.find_matches(1163            source, threshold=0.7, max_results=31164        )1165        return matches if matches else None11661167    def _parse_chunk_response(1168        self, response: str, expected_verses: list[str]1169    ) -> dict[str, str]:1170        """Parse JSON response from chunk translation.11711172        Uses robust JSON extraction that handles:1173        - Markdown code blocks (```json ... ```)1174        - Nested JSON structures1175        - Text before/after the JSON1176        - Unicode characters in values11771178        Args:1179            response: Raw response from Claude.1180            expected_verses: List of expected verse refs.11811182        Returns:1183            Dict mapping verse ref to translation.11841185        Raises:1186            ChunkParsingError: If response cannot be parsed.1187        """1188        original_response = response1189        response = response.strip()11901191        # Strategy 1: Remove markdown code blocks if present1192        if "```" in response:1193            response = self._extract_from_code_block(response)11941195        # Strategy 2: Try direct JSON parse first (most common case)1196        translations = self._try_parse_json(response)1197        if translations is not None:1198            return self._validate_translations(translations, expected_verses, original_response)11991200        # Strategy 3: Find JSON object using bracket matching1201        json_str = self._extract_json_by_brackets(response)1202        if json_str:1203            translations = self._try_parse_json(json_str)1204            if translations is not None:1205                return self._validate_translations(translations, expected_verses, original_response)12061207        # Strategy 4: Try to find JSON anywhere in the response1208        for match in re.finditer(r'\{', response):1209            start = match.start()1210            json_str = self._extract_json_from_position(response, start)1211            if json_str:1212                translations = self._try_parse_json(json_str)1213                if translations is not None:1214                    return self._validate_translations(translations, expected_verses, original_response)12151216        # All strategies failed1217        raise ChunkParsingError(1218            "Failed to parse chunk translation response as JSON",1219            response=original_response,1220            expected_verses=expected_verses,1221        )12221223    def _extract_from_code_block(self, response: str) -> str:1224        """Extract content from markdown code block.12251226        Args:1227            response: Response potentially containing code blocks.12281229        Returns:1230            Content inside code block, or original if not found.1231        """1232        lines = response.split("\n")1233        json_lines = []1234        in_block = False12351236        for line in lines:1237            stripped = line.strip()1238            if stripped.startswith("```") and not in_block:1239                in_block = True1240                continue1241            elif stripped.startswith("```") and in_block:1242                break1243            elif in_block:1244                json_lines.append(line)12451246        if json_lines:1247            return "\n".join(json_lines)1248        return response12491250    def _try_parse_json(self, text: str) -> dict | None:1251        """Try to parse text as JSON dict.12521253        Args:1254            text: Text to parse.12551256        Returns:1257            Parsed dict or None if parsing failed.1258        """1259        try:1260            result = json.loads(text)1261            if isinstance(result, dict):1262                return result1263        except json.JSONDecodeError:1264            pass1265        return None12661267    def _extract_json_by_brackets(self, text: str) -> str | None:1268        """Extract JSON object by matching brackets.12691270        Handles nested objects correctly.12711272        Args:1273            text: Text containing JSON.12741275        Returns:1276            Extracted JSON string or None if not found.1277        """1278        start = text.find("{")1279        if start == -1:1280            return None12811282        depth = 01283        in_string = False1284        escape_next = False12851286        for i, char in enumerate(text[start:], start):1287            if escape_next:1288                escape_next = False1289                continue12901291            if char == "\\":1292                escape_next = True1293                continue12941295            if char == '"' and not escape_next:1296                in_string = not in_string1297                continue12981299            if in_string:1300                continue13011302            if char == "{":1303                depth += 11304            elif char == "}":1305                depth -= 11306                if depth == 0:1307                    return text[start : i + 1]13081309        return None13101311    def _extract_json_from_position(self, text: str, start: int) -> str | None:1312        """Extract JSON object starting at given position.13131314        Args:1315            text: Full text.1316            start: Starting position of '{'.13171318        Returns:1319            Extracted JSON string or None if invalid.1320        """1321        if start >= len(text) or text[start] != "{":1322            return None13231324        depth = 01325        in_string = False1326        escape_next = False13271328        for i, char in enumerate(text[start:], start):1329            if escape_next:1330                escape_next = False1331                continue13321333            if char == "\\":1334                escape_next = True1335                continue13361337            if char == '"' and not escape_next:1338                in_string = not in_string1339                continue13401341            if in_string:1342                continue13431344            if char == "{":1345                depth += 11346            elif char == "}":1347                depth -= 11348                if depth == 0:1349                    return text[start : i + 1]13501351        return None13521353    def _validate_translations(1354        self,1355        translations: dict,1356        expected_verses: list[str],1357        original_response: str,1358    ) -> dict[str, str]:1359        """Validate parsed translations have all expected verses.13601361        Args:1362            translations: Parsed translation dict.1363            expected_verses: List of expected verse refs.1364            original_response: Original response for error context.13651366        Returns:1367            Validated translations dict.13681369        Raises:1370            ChunkParsingError: If validation fails.1371        """1372        if not isinstance(translations, dict):1373            raise ChunkParsingError(1374                f"Expected dict, got {type(translations).__name__}",1375                response=original_response,1376                expected_verses=expected_verses,1377            )13781379        # Validate all expected verses are present1380        missing = set(expected_verses) - set(translations.keys())1381        if missing:1382            raise ChunkParsingError(1383                f"Missing translations for verses: {sorted(missing)}",1384                response=original_response,1385                expected_verses=expected_verses,1386            )13871388        # Keep only the verses this pericope asked for. The prompt shows the1389        # preceding and following verses with their references, and a model1390        # that "helpfully" translates one of them would otherwise overwrite a1391        # verse in the neighbouring pericope; a stray non-verse key ("note")1392        # would crash the save loop after half the chunk was written.1393        extra = set(translations) - set(expected_verses)1394        if extra:1395            logger.warning(1396                "Discarding %d unrequested key(s) from chunk response: %s",1397                len(extra),1398                sorted(map(str, extra))[:5],1399            )14001401        # Validate all values are strings1402        for ref in expected_verses:1403            text = translations[ref]1404            if not isinstance(text, str):1405                raise ChunkParsingError(1406                    f"Translation for {ref} is not a string: {type(text).__name__}",1407                    response=original_response,1408                    expected_verses=expected_verses,1409                )14101411        return {ref: translations[ref] for ref in expected_verses}14121413    def _save_chunk_translations(1414        self, book: str, translations: dict[str, str]1415    ) -> None:1416        """Save individual verse translations from chunk output.14171418        Args:1419            book: Book name.1420            translations: Dict mapping verse ref to translation.1421        """1422        for verse_ref, text in translations.items():1423            chapter, verse = map(int, verse_ref.split(":"))1424            ref = VerseReference(book=book, chapter=chapter, verse=verse)14251426            # Parse furigana from translation text1427            plain_text, annotations = parse_furigana(text)14281429            output = TranslationOutput(1430                reference=ref,1431                text=plain_text,1432                text_with_furigana=text,1433                furigana_annotations=[1434                    {1435                        "start_offset": a.start_offset,1436                        "end_offset": a.end_offset,1437                        "base_text": a.base_text,1438                        "reading": a.reading,1439                    }1440                    for a in annotations1441                ],1442                model=self.model,1443                sources_used=["chunk_translation"],1444            )1445            self.storage.save_translation(output)14461447    def _add_chunk_to_tm(1448        self,1449        context: ChunkTranslationContext,1450        translations: dict[str, str],1451    ) -> None:1452        """Add chunk translations to translation memory.14531454        Args:1455            context: Chunk translation context.1456            translations: Dict mapping verse ref to translation.1457        """1458        if self._translation_memory is None:1459            return14601461        for verse in context.verses:1462            ref = f"{verse.reference.chapter}:{verse.reference.verse}"1463            source = verse.greek_text or verse.hebrew_text1464            target = translations.get(ref)14651466            if source and target:1467                self._translation_memory.add_from_translation(1468                    source_text=source,1469                    target_text=target,1470                    reference=verse.reference,1471                    metadata={"model": self.model, "pericope_id": context.pericope.id},1472                )1473