software/tests/test_auto_evaluation.py

1"""Tests for auto-evaluation and TM integration in translator."""23import json4import pytest5from pathlib import Path6from unittest.mock import MagicMock, patch78from src.context.builder import ContextBuilder9from src.context.glossary import GlossaryManager10from src.context.translation_memory import TranslationMemoryManager11from src.data.loader import DataLoader12from src.data.storage import Storage13from src.evaluation.evaluator import TranslationEvaluator14from src.evaluation.storage import EvaluationStorage15from src.models.evaluation import EvaluationScore, VerseEvaluation, DIMENSIONS16from src.models.verse import (17    GlossaryTerm,18    TranslationContext,19    TranslationOutput,20    Verse,21    VerseReference,22)23from src.pipeline.prompt import PromptBuilder24from src.pipeline.translator import Translator252627@pytest.fixture28def temp_data_dir(tmp_path: Path) -> Path:29    """Create temporary data directory with test data."""30    sources_dir = tmp_path / "sources"31    (sources_dir / "greek").mkdir(parents=True)32    (sources_dir / "japanese" / "meiji").mkdir(parents=True)33    (sources_dir / "japanese" / "taisho").mkdir(parents=True)34    (sources_dir / "english" / "kjv").mkdir(parents=True)35    (sources_dir / "english" / "asv").mkdir(parents=True)36    (tmp_path / "metadata" / "john").mkdir(parents=True)37    (tmp_path / "output" / "john").mkdir(parents=True)38    (tmp_path / "memory").mkdir(parents=True)39    (tmp_path / "config").mkdir(parents=True)4041    # Greek data42    greek_data = {43        "1:1": {"text": "Ἐν ἀρχῇ ἦν ὁ λόγος", "translit": "En archē ēn ho logos"},44        "1:2": {"text": "οὗτος ἦν ἐν ἀρχῇ", "translit": "houtos ēn en archē"},45    }46    with (sources_dir / "greek" / "john.json").open("w") as f:47        json.dump(greek_data, f)4849    # KJV data50    kjv_data = {51        "1:1": "In the beginning was the Word",52        "1:2": "The same was in the beginning with God",53    }54    with (sources_dir / "english" / "kjv" / "john.json").open("w") as f:55        json.dump(kjv_data, f)5657    # Glossary58    glossary = {"terms": []}59    with (tmp_path / "metadata" / "glossary.json").open("w") as f:60        json.dump(glossary, f)6162    # Style config63    style_config = {64        "equivalence_approach": "dynamic",65        "gender_language": "moderate_inclusive",66        "readability_level": "general_public",67        "theological_tradition": "evangelical",68        "textual_basis": "MorphGNT/SBLGNT",69        "term_consistency": "balanced",70        "formality_level": "standard",71    }72    with (tmp_path / "config" / "style.json").open("w") as f:73        json.dump(style_config, f)7475    # Empty TM76    tm_data = {"version": "1.0", "updated_at": None, "segments": []}77    with (tmp_path / "memory" / "segments.json").open("w") as f:78        json.dump(tm_data, f)7980    return tmp_path818283@pytest.fixture84def translator_with_tm(temp_data_dir: Path) -> Translator:85    """Create translator with TM enabled."""86    loader = DataLoader(temp_data_dir)87    storage = Storage(temp_data_dir)88    glossary = GlossaryManager(loader.load_glossary())89    tm = TranslationMemoryManager(temp_data_dir / "memory" / "segments.json")9091    context_builder = ContextBuilder(loader, glossary)92    prompt_builder = PromptBuilder(glossary, translation_memory=tm)9394    return Translator(95        context_builder, prompt_builder, storage, translation_memory=tm96    )979899@pytest.fixture100def translator_with_evaluation(temp_data_dir: Path) -> Translator:101    """Create translator with auto-evaluation enabled."""102    loader = DataLoader(temp_data_dir)103    storage = Storage(temp_data_dir)104    glossary = GlossaryManager(loader.load_glossary())105106    context_builder = ContextBuilder(loader, glossary)107    prompt_builder = PromptBuilder(glossary)108    evaluator = TranslationEvaluator()109    eval_storage = EvaluationStorage(temp_data_dir)110111    return Translator(112        context_builder,113        prompt_builder,114        storage,115        auto_evaluate=True,116        evaluator=evaluator,117        evaluation_storage=eval_storage,118    )119120121class TestTranslatorTMIntegration:122    """Tests for TM integration in Translator."""123124    def test_translation_memory_property(self, translator_with_tm: Translator) -> None:125        """Test translation_memory property."""126        assert translator_with_tm.translation_memory is not None127        assert isinstance(128            translator_with_tm.translation_memory, TranslationMemoryManager129        )130131    def test_translation_memory_setter(self, temp_data_dir: Path) -> None:132        """Test setting translation_memory."""133        loader = DataLoader(temp_data_dir)134        storage = Storage(temp_data_dir)135        glossary = GlossaryManager()136        context_builder = ContextBuilder(loader, glossary)137        prompt_builder = PromptBuilder(glossary)138139        translator = Translator(context_builder, prompt_builder, storage)140        assert translator.translation_memory is None141142        tm = TranslationMemoryManager(temp_data_dir / "memory" / "segments.json")143        translator.translation_memory = tm144        assert translator.translation_memory is tm145146    def test_get_tm_matches_no_tm(self, temp_data_dir: Path) -> None:147        """Test _get_tm_matches with no TM configured."""148        loader = DataLoader(temp_data_dir)149        storage = Storage(temp_data_dir)150        glossary = GlossaryManager()151        context_builder = ContextBuilder(loader, glossary)152        prompt_builder = PromptBuilder(glossary)153154        translator = Translator(context_builder, prompt_builder, storage)155156        ref = VerseReference(book="john", chapter=1, verse=1)157        context = context_builder.build_context(ref)158        matches = translator._get_tm_matches(context)159        assert matches is None160161    def test_get_tm_matches_no_source_text(162        self, translator_with_tm: Translator163    ) -> None:164        """Test _get_tm_matches with no source text."""165        verse = Verse(reference=VerseReference(book="test", chapter=1, verse=1))166        context = TranslationContext(target_verse=verse)167        matches = translator_with_tm._get_tm_matches(context)168        assert matches is None169170    def test_get_tm_matches_with_source(171        self, translator_with_tm: Translator172    ) -> None:173        """Test _get_tm_matches finds matches."""174        # Add a segment to TM175        ref = VerseReference(book="john", chapter=1, verse=1)176        translator_with_tm.translation_memory.add_from_translation(177            source_text="Ἐν ἀρχῇ ἦν ὁ λόγος",178            target_text="初めに言葉があった",179            reference=ref,180        )181182        # Create context with same source text183        verse = Verse(184            reference=ref,185            greek_text="Ἐν ἀρχῇ ἦν ὁ λόγος",186        )187        context = TranslationContext(target_verse=verse)188        matches = translator_with_tm._get_tm_matches(context)189        assert matches is not None190        assert len(matches) == 1191192    def test_add_to_tm_no_tm(self, temp_data_dir: Path) -> None:193        """Test _add_to_tm with no TM configured."""194        loader = DataLoader(temp_data_dir)195        storage = Storage(temp_data_dir)196        glossary = GlossaryManager()197        context_builder = ContextBuilder(loader, glossary)198        prompt_builder = PromptBuilder(glossary)199200        translator = Translator(context_builder, prompt_builder, storage)201202        ref = VerseReference(book="john", chapter=1, verse=1)203        verse = Verse(reference=ref, greek_text="Test text")204        context = TranslationContext(target_verse=verse)205206        # Should not raise207        translator._add_to_tm(context, "翻訳")208209    def test_add_to_tm_no_source(self, translator_with_tm: Translator) -> None:210        """Test _add_to_tm with no source text."""211        ref = VerseReference(book="john", chapter=1, verse=1)212        verse = Verse(reference=ref)213        context = TranslationContext(target_verse=verse)214215        initial_count = len(translator_with_tm.translation_memory)216        translator_with_tm._add_to_tm(context, "翻訳")217        assert len(translator_with_tm.translation_memory) == initial_count218219    def test_add_to_tm_with_source(self, translator_with_tm: Translator) -> None:220        """Test _add_to_tm adds segment."""221        ref = VerseReference(book="john", chapter=1, verse=1)222        verse = Verse(reference=ref, greek_text="Test source for TM")223        context = TranslationContext(target_verse=verse)224225        # Force load the TM226        translator_with_tm.translation_memory.load()227        initial_count = len(translator_with_tm.translation_memory)228        translator_with_tm._add_to_tm(context, "翻訳")229        assert len(translator_with_tm.translation_memory) == initial_count + 1230231232class TestTranslatorAutoEvaluation:233    """Tests for auto-evaluation in Translator."""234235    def test_auto_evaluate_property(236        self, translator_with_evaluation: Translator237    ) -> None:238        """Test auto_evaluate property."""239        assert translator_with_evaluation.auto_evaluate is True240241    def test_auto_evaluate_setter(self, temp_data_dir: Path) -> None:242        """Test setting auto_evaluate."""243        loader = DataLoader(temp_data_dir)244        storage = Storage(temp_data_dir)245        glossary = GlossaryManager()246        context_builder = ContextBuilder(loader, glossary)247        prompt_builder = PromptBuilder(glossary)248249        # Default is now True, explicitly test with False250        translator = Translator(251            context_builder, prompt_builder, storage, auto_evaluate=False252        )253        assert translator.auto_evaluate is False254255        translator.auto_evaluate = True256        assert translator.auto_evaluate is True257258    def test_evaluate_translation_disabled(self, temp_data_dir: Path) -> None:259        """Test _evaluate_translation when disabled."""260        loader = DataLoader(temp_data_dir)261        storage = Storage(temp_data_dir)262        glossary = GlossaryManager()263        context_builder = ContextBuilder(loader, glossary)264        prompt_builder = PromptBuilder(glossary)265266        translator = Translator(267            context_builder, prompt_builder, storage, auto_evaluate=False268        )269270        ref = VerseReference(book="john", chapter=1, verse=1)271        output = TranslationOutput(272            reference=ref, text="テスト", model="test-model"273        )274        verse = Verse(reference=ref)275        context = TranslationContext(target_verse=verse)276277        result = translator._evaluate_translation(output, context)278        assert result is None279280    def test_evaluate_translation_no_evaluator(281        self, temp_data_dir: Path282    ) -> None:283        """Test _evaluate_translation without evaluator."""284        loader = DataLoader(temp_data_dir)285        storage = Storage(temp_data_dir)286        glossary = GlossaryManager()287        context_builder = ContextBuilder(loader, glossary)288        prompt_builder = PromptBuilder(glossary)289290        translator = Translator(291            context_builder, prompt_builder, storage, auto_evaluate=True292        )293294        ref = VerseReference(book="john", chapter=1, verse=1)295        output = TranslationOutput(296            reference=ref, text="テスト", model="test-model"297        )298        verse = Verse(reference=ref)299        context = TranslationContext(target_verse=verse)300301        result = translator._evaluate_translation(output, context)302        assert result is None303304    @patch("src.pipeline.translator.Translator._call_claude")305    def test_evaluate_translation_success(306        self, mock_claude: MagicMock, translator_with_evaluation: Translator307    ) -> None:308        """Test _evaluate_translation success path."""309        # Mock response310        mock_response = json.dumps({311            dim: {"score": 85, "confidence": 90, "reasoning": "Good"}312            for dim in DIMENSIONS313        })314        mock_claude.return_value = mock_response315316        ref = VerseReference(book="john", chapter=1, verse=1)317        output = TranslationOutput(318            reference=ref, text="初めに言葉があった", model="test-model"319        )320        verse = Verse(reference=ref, greek_text="Ἐν ἀρχῇ")321        context = TranslationContext(target_verse=verse)322323        result = translator_with_evaluation._evaluate_translation(output, context)324        assert result is not None325        assert isinstance(result, VerseEvaluation)326327    @patch("src.pipeline.translator.Translator._call_claude")328    def test_evaluate_translation_exception(329        self, mock_claude: MagicMock, translator_with_evaluation: Translator330    ) -> None:331        """Test _evaluate_translation handles exceptions."""332        mock_claude.side_effect = RuntimeError("Claude failed")333334        ref = VerseReference(book="john", chapter=1, verse=1)335        output = TranslationOutput(336            reference=ref, text="テスト", model="test-model"337        )338        verse = Verse(reference=ref, greek_text="Test")339        context = TranslationContext(target_verse=verse)340341        # Should not raise, returns None342        result = translator_with_evaluation._evaluate_translation(output, context)343        assert result is None344345    def test_save_evaluation_no_storage(self, temp_data_dir: Path) -> None:346        """Test _save_evaluation without storage."""347        loader = DataLoader(temp_data_dir)348        storage = Storage(temp_data_dir)349        glossary = GlossaryManager()350        context_builder = ContextBuilder(loader, glossary)351        prompt_builder = PromptBuilder(glossary)352353        translator = Translator(context_builder, prompt_builder, storage)354355        ref = VerseReference(book="john", chapter=1, verse=1)356        scores = {357            dim: EvaluationScore(358                dimension=dim, score=80, confidence=90, reasoning="Good"359            )360            for dim in DIMENSIONS361        }362        evaluation = VerseEvaluation(363            verse_number=1,364            evaluator_model="test",365            scores=scores,366            flags=[],367        )368369        # Should not raise370        translator._save_evaluation(evaluation, ref)371372    def test_save_evaluation_with_storage(373        self, temp_data_dir: Path374    ) -> None:375        """Test _save_evaluation with storage."""376        loader = DataLoader(temp_data_dir)377        storage = Storage(temp_data_dir)378        glossary = GlossaryManager()379        context_builder = ContextBuilder(loader, glossary)380        prompt_builder = PromptBuilder(glossary)381        evaluator = TranslationEvaluator()382        eval_storage = EvaluationStorage(temp_data_dir)383384        translator = Translator(385            context_builder,386            prompt_builder,387            storage,388            auto_evaluate=True,389            evaluator=evaluator,390            evaluation_storage=eval_storage,391        )392393        ref = VerseReference(book="john", chapter=1, verse=1)394        scores = {395            dim: EvaluationScore(396                dimension=dim, score=80, confidence=90, reasoning="Good"397            )398            for dim in DIMENSIONS399        }400        evaluation = VerseEvaluation(401            verse_number=1,402            evaluator_model="test",403            scores=scores,404            flags=[],405        )406407        # Should not raise408        translator._save_evaluation(evaluation, ref)409410411class TestTranslatorTranslateVerse:412    """Tests for translate_verse with TM and evaluation."""413414    @patch("src.pipeline.translator.Translator._call_claude")415    def test_translate_verse_adds_to_tm(416        self, mock_claude: MagicMock, translator_with_tm: Translator417    ) -> None:418        """Test translate_verse adds to TM."""419        mock_claude.return_value = "初めに言葉があった"420421        ref = VerseReference(book="john", chapter=1, verse=1)422        initial_count = len(translator_with_tm.translation_memory)423424        translator_with_tm.translate_verse(ref, skip_existing=False)425426        assert len(translator_with_tm.translation_memory) == initial_count + 1427428    @patch("src.pipeline.translator.Translator._call_claude")429    def test_translate_verse_with_evaluate_override(430        self, mock_claude: MagicMock, translator_with_evaluation: Translator431    ) -> None:432        """Test translate_verse with evaluate override."""433        mock_claude.return_value = "初めに言葉があった"434435        ref = VerseReference(book="john", chapter=1, verse=1)436437        # Disable evaluation for this call438        translator_with_evaluation.translate_verse(439            ref, skip_existing=False, evaluate=False440        )441442        # Should only call Claude once for translation (not evaluation)443        assert mock_claude.call_count == 1444445446    @patch("src.pipeline.translator.Translator._call_claude")447    def test_translate_verse_with_successful_evaluation(448        self, mock_claude: MagicMock, translator_with_evaluation: Translator449    ) -> None:450        """Test translate_verse with successful evaluation flow."""451        # First call for translation, second for evaluation452        eval_response = json.dumps({453            dim: {"score": 85, "confidence": 90, "reasoning": "Good"}454            for dim in DIMENSIONS455        })456        mock_claude.side_effect = ["初めに言葉があった", eval_response]457458        ref = VerseReference(book="john", chapter=1, verse=1)459460        # Enable evaluation for this call461        translator_with_evaluation.translate_verse(462            ref, skip_existing=False, evaluate=True463        )464465        # Should call Claude twice (translation + evaluation)466        assert mock_claude.call_count == 2467468469class TestTranslatorTranslateChapter:470    """Tests for translate_chapter with TM."""471472    @patch("src.pipeline.translator.Translator._call_claude")473    def test_translate_chapter_saves_tm(474        self, mock_claude: MagicMock, translator_with_tm: Translator475    ) -> None:476        """Test translate_chapter saves TM after completion."""477        mock_claude.return_value = "翻訳テキスト"478479        # Monkey patch save to track calls480        save_called = []481        original_save = translator_with_tm.translation_memory.save482483        def track_save():484            save_called.append(True)485            original_save()486487        translator_with_tm.translation_memory.save = track_save488489        translator_with_tm.translate_chapter("john", 1, skip_existing=False)490491        # Save should have been called492        assert len(save_called) > 0493