"""Tests for auto-evaluation and TM integration in translator.""" import json import pytest from pathlib import Path from unittest.mock import MagicMock, patch from src.context.builder import ContextBuilder from src.context.glossary import GlossaryManager from src.context.translation_memory import TranslationMemoryManager from src.data.loader import DataLoader from src.data.storage import Storage from src.evaluation.evaluator import TranslationEvaluator from src.evaluation.storage import EvaluationStorage from src.models.evaluation import EvaluationScore, VerseEvaluation, DIMENSIONS from src.models.verse import ( GlossaryTerm, TranslationContext, TranslationOutput, Verse, VerseReference, ) from src.pipeline.prompt import PromptBuilder from src.pipeline.translator import Translator @pytest.fixture def temp_data_dir(tmp_path: Path) -> Path: """Create temporary data directory with test data.""" sources_dir = tmp_path / "sources" (sources_dir / "greek").mkdir(parents=True) (sources_dir / "japanese" / "meiji").mkdir(parents=True) (sources_dir / "japanese" / "taisho").mkdir(parents=True) (sources_dir / "english" / "kjv").mkdir(parents=True) (sources_dir / "english" / "asv").mkdir(parents=True) (tmp_path / "metadata" / "john").mkdir(parents=True) (tmp_path / "output" / "john").mkdir(parents=True) (tmp_path / "memory").mkdir(parents=True) (tmp_path / "config").mkdir(parents=True) # Greek data greek_data = { "1:1": {"text": "Ἐν ἀρχῇ ἦν ὁ λόγος", "translit": "En archē ēn ho logos"}, "1:2": {"text": "οὗτος ἦν ἐν ἀρχῇ", "translit": "houtos ēn en archē"}, } with (sources_dir / "greek" / "john.json").open("w") as f: json.dump(greek_data, f) # KJV data kjv_data = { "1:1": "In the beginning was the Word", "1:2": "The same was in the beginning with God", } with (sources_dir / "english" / "kjv" / "john.json").open("w") as f: json.dump(kjv_data, f) # Glossary glossary = {"terms": []} with (tmp_path / "metadata" / "glossary.json").open("w") as f: json.dump(glossary, f) # Style config style_config = { "equivalence_approach": "dynamic", "gender_language": "moderate_inclusive", "readability_level": "general_public", "theological_tradition": "evangelical", "textual_basis": "MorphGNT/SBLGNT", "term_consistency": "balanced", "formality_level": "standard", } with (tmp_path / "config" / "style.json").open("w") as f: json.dump(style_config, f) # Empty TM tm_data = {"version": "1.0", "updated_at": None, "segments": []} with (tmp_path / "memory" / "segments.json").open("w") as f: json.dump(tm_data, f) return tmp_path @pytest.fixture def translator_with_tm(temp_data_dir: Path) -> Translator: """Create translator with TM enabled.""" loader = DataLoader(temp_data_dir) storage = Storage(temp_data_dir) glossary = GlossaryManager(loader.load_glossary()) tm = TranslationMemoryManager(temp_data_dir / "memory" / "segments.json") context_builder = ContextBuilder(loader, glossary) prompt_builder = PromptBuilder(glossary, translation_memory=tm) return Translator( context_builder, prompt_builder, storage, translation_memory=tm ) @pytest.fixture def translator_with_evaluation(temp_data_dir: Path) -> Translator: """Create translator with auto-evaluation enabled.""" loader = DataLoader(temp_data_dir) storage = Storage(temp_data_dir) glossary = GlossaryManager(loader.load_glossary()) context_builder = ContextBuilder(loader, glossary) prompt_builder = PromptBuilder(glossary) evaluator = TranslationEvaluator() eval_storage = EvaluationStorage(temp_data_dir) return Translator( context_builder, prompt_builder, storage, auto_evaluate=True, evaluator=evaluator, evaluation_storage=eval_storage, ) class TestTranslatorTMIntegration: """Tests for TM integration in Translator.""" def test_translation_memory_property(self, translator_with_tm: Translator) -> None: """Test translation_memory property.""" assert translator_with_tm.translation_memory is not None assert isinstance( translator_with_tm.translation_memory, TranslationMemoryManager ) def test_translation_memory_setter(self, temp_data_dir: Path) -> None: """Test setting translation_memory.""" loader = DataLoader(temp_data_dir) storage = Storage(temp_data_dir) glossary = GlossaryManager() context_builder = ContextBuilder(loader, glossary) prompt_builder = PromptBuilder(glossary) translator = Translator(context_builder, prompt_builder, storage) assert translator.translation_memory is None tm = TranslationMemoryManager(temp_data_dir / "memory" / "segments.json") translator.translation_memory = tm assert translator.translation_memory is tm def test_get_tm_matches_no_tm(self, temp_data_dir: Path) -> None: """Test _get_tm_matches with no TM configured.""" loader = DataLoader(temp_data_dir) storage = Storage(temp_data_dir) glossary = GlossaryManager() context_builder = ContextBuilder(loader, glossary) prompt_builder = PromptBuilder(glossary) translator = Translator(context_builder, prompt_builder, storage) ref = VerseReference(book="john", chapter=1, verse=1) context = context_builder.build_context(ref) matches = translator._get_tm_matches(context) assert matches is None def test_get_tm_matches_no_source_text( self, translator_with_tm: Translator ) -> None: """Test _get_tm_matches with no source text.""" verse = Verse(reference=VerseReference(book="test", chapter=1, verse=1)) context = TranslationContext(target_verse=verse) matches = translator_with_tm._get_tm_matches(context) assert matches is None def test_get_tm_matches_with_source( self, translator_with_tm: Translator ) -> None: """Test _get_tm_matches finds matches.""" # Add a segment to TM ref = VerseReference(book="john", chapter=1, verse=1) translator_with_tm.translation_memory.add_from_translation( source_text="Ἐν ἀρχῇ ἦν ὁ λόγος", target_text="初めに言葉があった", reference=ref, ) # Create context with same source text verse = Verse( reference=ref, greek_text="Ἐν ἀρχῇ ἦν ὁ λόγος", ) context = TranslationContext(target_verse=verse) matches = translator_with_tm._get_tm_matches(context) assert matches is not None assert len(matches) == 1 def test_add_to_tm_no_tm(self, temp_data_dir: Path) -> None: """Test _add_to_tm with no TM configured.""" loader = DataLoader(temp_data_dir) storage = Storage(temp_data_dir) glossary = GlossaryManager() context_builder = ContextBuilder(loader, glossary) prompt_builder = PromptBuilder(glossary) translator = Translator(context_builder, prompt_builder, storage) ref = VerseReference(book="john", chapter=1, verse=1) verse = Verse(reference=ref, greek_text="Test text") context = TranslationContext(target_verse=verse) # Should not raise translator._add_to_tm(context, "翻訳") def test_add_to_tm_no_source(self, translator_with_tm: Translator) -> None: """Test _add_to_tm with no source text.""" ref = VerseReference(book="john", chapter=1, verse=1) verse = Verse(reference=ref) context = TranslationContext(target_verse=verse) initial_count = len(translator_with_tm.translation_memory) translator_with_tm._add_to_tm(context, "翻訳") assert len(translator_with_tm.translation_memory) == initial_count def test_add_to_tm_with_source(self, translator_with_tm: Translator) -> None: """Test _add_to_tm adds segment.""" ref = VerseReference(book="john", chapter=1, verse=1) verse = Verse(reference=ref, greek_text="Test source for TM") context = TranslationContext(target_verse=verse) # Force load the TM translator_with_tm.translation_memory.load() initial_count = len(translator_with_tm.translation_memory) translator_with_tm._add_to_tm(context, "翻訳") assert len(translator_with_tm.translation_memory) == initial_count + 1 class TestTranslatorAutoEvaluation: """Tests for auto-evaluation in Translator.""" def test_auto_evaluate_property( self, translator_with_evaluation: Translator ) -> None: """Test auto_evaluate property.""" assert translator_with_evaluation.auto_evaluate is True def test_auto_evaluate_setter(self, temp_data_dir: Path) -> None: """Test setting auto_evaluate.""" loader = DataLoader(temp_data_dir) storage = Storage(temp_data_dir) glossary = GlossaryManager() context_builder = ContextBuilder(loader, glossary) prompt_builder = PromptBuilder(glossary) # Default is now True, explicitly test with False translator = Translator( context_builder, prompt_builder, storage, auto_evaluate=False ) assert translator.auto_evaluate is False translator.auto_evaluate = True assert translator.auto_evaluate is True def test_evaluate_translation_disabled(self, temp_data_dir: Path) -> None: """Test _evaluate_translation when disabled.""" loader = DataLoader(temp_data_dir) storage = Storage(temp_data_dir) glossary = GlossaryManager() context_builder = ContextBuilder(loader, glossary) prompt_builder = PromptBuilder(glossary) translator = Translator( context_builder, prompt_builder, storage, auto_evaluate=False ) ref = VerseReference(book="john", chapter=1, verse=1) output = TranslationOutput( reference=ref, text="テスト", model="test-model" ) verse = Verse(reference=ref) context = TranslationContext(target_verse=verse) result = translator._evaluate_translation(output, context) assert result is None def test_evaluate_translation_no_evaluator( self, temp_data_dir: Path ) -> None: """Test _evaluate_translation without evaluator.""" loader = DataLoader(temp_data_dir) storage = Storage(temp_data_dir) glossary = GlossaryManager() context_builder = ContextBuilder(loader, glossary) prompt_builder = PromptBuilder(glossary) translator = Translator( context_builder, prompt_builder, storage, auto_evaluate=True ) ref = VerseReference(book="john", chapter=1, verse=1) output = TranslationOutput( reference=ref, text="テスト", model="test-model" ) verse = Verse(reference=ref) context = TranslationContext(target_verse=verse) result = translator._evaluate_translation(output, context) assert result is None @patch("src.pipeline.translator.Translator._call_claude") def test_evaluate_translation_success( self, mock_claude: MagicMock, translator_with_evaluation: Translator ) -> None: """Test _evaluate_translation success path.""" # Mock response mock_response = json.dumps({ dim: {"score": 85, "confidence": 90, "reasoning": "Good"} for dim in DIMENSIONS }) mock_claude.return_value = mock_response ref = VerseReference(book="john", chapter=1, verse=1) output = TranslationOutput( reference=ref, text="初めに言葉があった", model="test-model" ) verse = Verse(reference=ref, greek_text="Ἐν ἀρχῇ") context = TranslationContext(target_verse=verse) result = translator_with_evaluation._evaluate_translation(output, context) assert result is not None assert isinstance(result, VerseEvaluation) @patch("src.pipeline.translator.Translator._call_claude") def test_evaluate_translation_exception( self, mock_claude: MagicMock, translator_with_evaluation: Translator ) -> None: """Test _evaluate_translation handles exceptions.""" mock_claude.side_effect = RuntimeError("Claude failed") ref = VerseReference(book="john", chapter=1, verse=1) output = TranslationOutput( reference=ref, text="テスト", model="test-model" ) verse = Verse(reference=ref, greek_text="Test") context = TranslationContext(target_verse=verse) # Should not raise, returns None result = translator_with_evaluation._evaluate_translation(output, context) assert result is None def test_save_evaluation_no_storage(self, temp_data_dir: Path) -> None: """Test _save_evaluation without storage.""" loader = DataLoader(temp_data_dir) storage = Storage(temp_data_dir) glossary = GlossaryManager() context_builder = ContextBuilder(loader, glossary) prompt_builder = PromptBuilder(glossary) translator = Translator(context_builder, prompt_builder, storage) ref = VerseReference(book="john", chapter=1, verse=1) scores = { dim: EvaluationScore( dimension=dim, score=80, confidence=90, reasoning="Good" ) for dim in DIMENSIONS } evaluation = VerseEvaluation( verse_number=1, evaluator_model="test", scores=scores, flags=[], ) # Should not raise translator._save_evaluation(evaluation, ref) def test_save_evaluation_with_storage( self, temp_data_dir: Path ) -> None: """Test _save_evaluation with storage.""" loader = DataLoader(temp_data_dir) storage = Storage(temp_data_dir) glossary = GlossaryManager() context_builder = ContextBuilder(loader, glossary) prompt_builder = PromptBuilder(glossary) evaluator = TranslationEvaluator() eval_storage = EvaluationStorage(temp_data_dir) translator = Translator( context_builder, prompt_builder, storage, auto_evaluate=True, evaluator=evaluator, evaluation_storage=eval_storage, ) ref = VerseReference(book="john", chapter=1, verse=1) scores = { dim: EvaluationScore( dimension=dim, score=80, confidence=90, reasoning="Good" ) for dim in DIMENSIONS } evaluation = VerseEvaluation( verse_number=1, evaluator_model="test", scores=scores, flags=[], ) # Should not raise translator._save_evaluation(evaluation, ref) class TestTranslatorTranslateVerse: """Tests for translate_verse with TM and evaluation.""" @patch("src.pipeline.translator.Translator._call_claude") def test_translate_verse_adds_to_tm( self, mock_claude: MagicMock, translator_with_tm: Translator ) -> None: """Test translate_verse adds to TM.""" mock_claude.return_value = "初めに言葉があった" ref = VerseReference(book="john", chapter=1, verse=1) initial_count = len(translator_with_tm.translation_memory) translator_with_tm.translate_verse(ref, skip_existing=False) assert len(translator_with_tm.translation_memory) == initial_count + 1 @patch("src.pipeline.translator.Translator._call_claude") def test_translate_verse_with_evaluate_override( self, mock_claude: MagicMock, translator_with_evaluation: Translator ) -> None: """Test translate_verse with evaluate override.""" mock_claude.return_value = "初めに言葉があった" ref = VerseReference(book="john", chapter=1, verse=1) # Disable evaluation for this call translator_with_evaluation.translate_verse( ref, skip_existing=False, evaluate=False ) # Should only call Claude once for translation (not evaluation) assert mock_claude.call_count == 1 @patch("src.pipeline.translator.Translator._call_claude") def test_translate_verse_with_successful_evaluation( self, mock_claude: MagicMock, translator_with_evaluation: Translator ) -> None: """Test translate_verse with successful evaluation flow.""" # First call for translation, second for evaluation eval_response = json.dumps({ dim: {"score": 85, "confidence": 90, "reasoning": "Good"} for dim in DIMENSIONS }) mock_claude.side_effect = ["初めに言葉があった", eval_response] ref = VerseReference(book="john", chapter=1, verse=1) # Enable evaluation for this call translator_with_evaluation.translate_verse( ref, skip_existing=False, evaluate=True ) # Should call Claude twice (translation + evaluation) assert mock_claude.call_count == 2 class TestTranslatorTranslateChapter: """Tests for translate_chapter with TM.""" @patch("src.pipeline.translator.Translator._call_claude") def test_translate_chapter_saves_tm( self, mock_claude: MagicMock, translator_with_tm: Translator ) -> None: """Test translate_chapter saves TM after completion.""" mock_claude.return_value = "翻訳テキスト" # Monkey patch save to track calls save_called = [] original_save = translator_with_tm.translation_memory.save def track_save(): save_called.append(True) original_save() translator_with_tm.translation_memory.save = track_save translator_with_tm.translate_chapter("john", 1, skip_existing=False) # Save should have been called assert len(save_called) > 0