OPEN SOURCE
software/tests/test_translation_memory.py
このサイトで読み、必要なファイルを保存できます。
日常訳はAIを用いて作成された翻訳です。人間の聖書学・原語の専門家による全訳の検証は確認されていません。既存の翻訳や原語資料と照らし合わせてお読みください。
1"""Tests for Translation Memory models and manager."""23import json4import pytest5from datetime import datetime6from pathlib import Path78from src.context.translation_memory import TranslationMemoryManager9from src.models.translation_memory import (10MatchType,11SegmentLevel,12TMMatch,13TMSegment,14TMStatistics,15calculate_similarity,16normalize_text,17_levenshtein_distance,18)19from src.models.verse import VerseReference202122class TestNormalizeText:23"""Tests for text normalization."""2425def test_lowercase(self) -> None:26"""Test lowercasing."""27assert normalize_text("ABC") == "abc"2829def test_remove_punctuation(self) -> None:30"""Test punctuation removal."""31result = normalize_text("Hello, World!")32assert "," not in result33assert "!" not in result3435def test_collapse_whitespace(self) -> None:36"""Test whitespace collapsing."""37result = normalize_text("hello world")38assert result == "hello world"3940def test_unicode_normalization(self) -> None:41"""Test Unicode normalization."""42# Greek text should be normalized43text = "λόγος"44result = normalize_text(text)45assert result == "λόγος"4647def test_preserve_greek_characters(self) -> None:48"""Test Greek characters are preserved."""49text = "Ἐν ἀρχῇ ἦν ὁ λόγος"50result = normalize_text(text)51assert "λόγος" in result525354class TestCalculateSimilarity:55"""Tests for similarity calculation."""5657def test_exact_match(self) -> None:58"""Test exact match returns 1.0."""59assert calculate_similarity("hello", "hello") == 1.06061def test_completely_different(self) -> None:62"""Test completely different returns low score."""63score = calculate_similarity("abc", "xyz")64assert score < 0.56566def test_similar_strings(self) -> None:67"""Test similar strings return high score."""68score = calculate_similarity("hello", "hallo")69assert score > 0.77071def test_empty_strings(self) -> None:72"""Test empty string handling."""73assert calculate_similarity("", "") == 1.074assert calculate_similarity("hello", "") == 0.075assert calculate_similarity("", "hello") == 0.07677def test_symmetric(self) -> None:78"""Test similarity is symmetric."""79s1 = calculate_similarity("hello", "hallo")80s2 = calculate_similarity("hallo", "hello")81assert s1 == s2828384class TestLevenshteinDistance:85"""Tests for Levenshtein distance."""8687def test_identical(self) -> None:88"""Test identical strings have distance 0."""89assert _levenshtein_distance("hello", "hello") == 09091def test_single_insertion(self) -> None:92"""Test single insertion."""93assert _levenshtein_distance("hello", "hellos") == 19495def test_single_deletion(self) -> None:96"""Test single deletion."""97assert _levenshtein_distance("hello", "hell") == 19899def test_single_substitution(self) -> None:100"""Test single substitution."""101assert _levenshtein_distance("hello", "hallo") == 1102103def test_empty_to_string(self) -> None:104"""Test empty to string."""105assert _levenshtein_distance("", "hello") == 5106107def test_string_to_empty(self) -> None:108"""Test string to empty."""109assert _levenshtein_distance("hello", "") == 5110111112class TestSegmentLevel:113"""Tests for SegmentLevel enum."""114115def test_values(self) -> None:116"""Test enum values."""117assert SegmentLevel.VERSE.value == "verse"118assert SegmentLevel.PHRASE.value == "phrase"119120121class TestMatchType:122"""Tests for MatchType enum."""123124def test_values(self) -> None:125"""Test enum values."""126assert MatchType.EXACT.value == "exact"127assert MatchType.FUZZY.value == "fuzzy"128assert MatchType.PHRASE.value == "phrase"129130131class TestTMSegment:132"""Tests for TMSegment model."""133134def test_create_basic(self) -> None:135"""Test basic segment creation."""136ref = VerseReference(book="john", chapter=1, verse=1)137segment = TMSegment(138source_text="Ἐν ἀρχῇ ἦν ὁ λόγος",139source_normalized="ἐν ἀρχῇ ἦν ὁ λόγος",140target_text="初めに言葉があった",141reference=ref,142)143assert segment.source_text == "Ἐν ἀρχῇ ἦν ὁ λόγος"144assert segment.level == SegmentLevel.VERSE145assert segment.reuse_count == 0146147def test_create_factory(self) -> None:148"""Test factory method with auto-normalization."""149ref = VerseReference(book="john", chapter=1, verse=1)150segment = TMSegment.create(151source_text="Ἐν ἀρχῇ ἦν ὁ λόγος",152target_text="初めに言葉があった",153reference=ref,154)155# Normalized should be lowercase without extra punctuation156assert segment.source_normalized == normalize_text("Ἐν ἀρχῇ ἦν ὁ λόγος")157158def test_create_with_metadata(self) -> None:159"""Test creation with metadata."""160ref = VerseReference(book="john", chapter=1, verse=1)161segment = TMSegment.create(162source_text="Test",163target_text="テスト",164reference=ref,165metadata={"model": "claude-sonnet", "version": "1.0"},166)167assert segment.metadata["model"] == "claude-sonnet"168169def test_auto_generated_id(self) -> None:170"""Test ID is auto-generated."""171ref = VerseReference(book="john", chapter=1, verse=1)172segment = TMSegment.create(173source_text="Test",174target_text="テスト",175reference=ref,176)177assert segment.id is not None178assert len(segment.id) > 0179180def test_increment_reuse(self) -> None:181"""Test reuse counter increment."""182ref = VerseReference(book="john", chapter=1, verse=1)183segment = TMSegment.create(184source_text="Test",185target_text="テスト",186reference=ref,187)188assert segment.reuse_count == 0189segment.increment_reuse()190assert segment.reuse_count == 1191segment.increment_reuse()192assert segment.reuse_count == 2193194195class TestTMMatch:196"""Tests for TMMatch model."""197198@pytest.fixture199def sample_segment(self) -> TMSegment:200"""Create a sample segment."""201ref = VerseReference(book="john", chapter=1, verse=1)202return TMSegment.create(203source_text="Test",204target_text="テスト",205reference=ref,206)207208def test_create_match(self, sample_segment: TMSegment) -> None:209"""Test creating a match."""210match = TMMatch(211segment=sample_segment,212similarity=0.95,213match_type=MatchType.FUZZY,214)215assert match.similarity == 0.95216assert match.match_type == MatchType.FUZZY217218def test_is_exact(self, sample_segment: TMSegment) -> None:219"""Test is_exact property."""220exact = TMMatch(221segment=sample_segment,222similarity=1.0,223match_type=MatchType.EXACT,224)225fuzzy = TMMatch(226segment=sample_segment,227similarity=0.9,228match_type=MatchType.FUZZY,229)230assert exact.is_exact is True231assert fuzzy.is_exact is False232233def test_is_usable(self, sample_segment: TMSegment) -> None:234"""Test is_usable property."""235high = TMMatch(236segment=sample_segment,237similarity=0.8,238match_type=MatchType.FUZZY,239)240low = TMMatch(241segment=sample_segment,242similarity=0.5,243match_type=MatchType.FUZZY,244)245assert high.is_usable is True246assert low.is_usable is False247248249class TestTMStatistics:250"""Tests for TMStatistics model."""251252def test_from_empty_segments(self) -> None:253"""Test statistics from empty list."""254stats = TMStatistics.from_segments([])255assert stats.total_segments == 0256assert stats.average_reuse == 0.0257258def test_from_segments(self) -> None:259"""Test statistics calculation."""260segments = [261TMSegment.create(262source_text=f"Test {i}",263target_text=f"テスト {i}",264reference=VerseReference(book="john", chapter=1, verse=i),265level=SegmentLevel.VERSE if i % 2 == 0 else SegmentLevel.PHRASE,266)267for i in range(1, 6)268]269# Set some reuse counts270segments[0].reuse_count = 5271segments[1].reuse_count = 3272273stats = TMStatistics.from_segments(segments)274assert stats.total_segments == 5275# range(1,6) gives [1,2,3,4,5]. i%2==0 for i=2,4. So verse_segments = 2276assert stats.verse_segments == 2277assert stats.phrase_segments == 3278assert stats.total_reuses == 8279assert "john" in stats.books_covered280281282class TestTranslationMemoryManager:283"""Tests for TranslationMemoryManager."""284285@pytest.fixture286def manager(self, tmp_path: Path) -> TranslationMemoryManager:287"""Create a manager with temporary storage."""288return TranslationMemoryManager(tmp_path / "segments.json")289290@pytest.fixture291def sample_segment(self) -> TMSegment:292"""Create a sample segment."""293ref = VerseReference(book="john", chapter=1, verse=1)294return TMSegment.create(295source_text="Ἐν ἀρχῇ ἦν ὁ λόγος",296target_text="初めに言葉があった",297reference=ref,298)299300def test_init_default_path(self) -> None:301"""The default path is taken from the module constant.302303The constant is redirected by an autouse fixture during tests, so this304asserts the wiring rather than a literal production path — which is305the point: no test may reach the real store.306"""307import src.context.translation_memory as tm_module308309manager = TranslationMemoryManager()310assert manager.storage_path == tm_module.DEFAULT_STORAGE_PATH311assert "data/memory/segments.json" not in str(manager.storage_path)312313def test_init_custom_path(self, tmp_path: Path) -> None:314"""Test custom storage path."""315path = tmp_path / "custom.json"316manager = TranslationMemoryManager(path)317assert manager.storage_path == path318319def test_add_segment(320self, manager: TranslationMemoryManager, sample_segment: TMSegment321) -> None:322"""Test adding a segment."""323manager.add_segment(sample_segment)324assert len(manager) == 1325326def test_add_from_translation(self, manager: TranslationMemoryManager) -> None:327"""Test creating segment from translation."""328ref = VerseReference(book="john", chapter=1, verse=1)329segment = manager.add_from_translation(330source_text="Test source",331target_text="Test target",332reference=ref,333metadata={"model": "test"},334)335assert len(manager) == 1336assert segment.metadata["model"] == "test"337338def test_get_segment(339self, manager: TranslationMemoryManager, sample_segment: TMSegment340) -> None:341"""Test getting a segment by ID."""342manager.add_segment(sample_segment)343retrieved = manager.get_segment(sample_segment.id)344assert retrieved is not None345assert retrieved.id == sample_segment.id346347def test_get_segment_not_found(self, manager: TranslationMemoryManager) -> None:348"""Test getting nonexistent segment."""349result = manager.get_segment("nonexistent")350assert result is None351352def test_get_by_reference(353self, manager: TranslationMemoryManager, sample_segment: TMSegment354) -> None:355"""Test getting segments by reference."""356manager.add_segment(sample_segment)357results = manager.get_by_reference(sample_segment.reference)358assert len(results) == 1359assert results[0].id == sample_segment.id360361def test_get_by_reference_empty(self, manager: TranslationMemoryManager) -> None:362"""Test getting segments for unknown reference."""363ref = VerseReference(book="john", chapter=99, verse=99)364results = manager.get_by_reference(ref)365assert len(results) == 0366367def test_find_matches_exact(368self, manager: TranslationMemoryManager, sample_segment: TMSegment369) -> None:370"""Test finding exact match."""371manager.add_segment(sample_segment)372matches = manager.find_matches(sample_segment.source_text)373assert len(matches) == 1374assert matches[0].is_exact375376def test_find_matches_fuzzy(self, manager: TranslationMemoryManager) -> None:377"""Test finding fuzzy matches."""378ref = VerseReference(book="john", chapter=1, verse=1)379segment = TMSegment.create(380source_text="hello world",381target_text="こんにちは世界",382reference=ref,383)384manager.add_segment(segment)385386matches = manager.find_matches("hello worlds", threshold=0.8)387assert len(matches) == 1388assert matches[0].match_type == MatchType.FUZZY389390def test_find_matches_threshold(self, manager: TranslationMemoryManager) -> None:391"""Test threshold filtering."""392ref = VerseReference(book="john", chapter=1, verse=1)393segment = TMSegment.create(394source_text="hello world",395target_text="こんにちは世界",396reference=ref,397)398manager.add_segment(segment)399400# High threshold should filter out401matches = manager.find_matches("completely different text", threshold=0.9)402assert len(matches) == 0403404def test_find_matches_max_results(self, manager: TranslationMemoryManager) -> None:405"""Test max results limit."""406ref = VerseReference(book="john", chapter=1, verse=1)407for i in range(10):408manager.add_segment(409TMSegment.create(410source_text=f"test {i}",411target_text=f"テスト {i}",412reference=ref,413)414)415416matches = manager.find_matches("test 5", threshold=0.5, max_results=3)417assert len(matches) <= 3418419def test_find_exact_match(420self, manager: TranslationMemoryManager, sample_segment: TMSegment421) -> None:422"""Test find_exact_match helper."""423manager.add_segment(sample_segment)424425result = manager.find_exact_match(sample_segment.source_text)426assert result is not None427assert result.id == sample_segment.id428429def test_find_exact_match_not_found(430self, manager: TranslationMemoryManager431) -> None:432"""Test find_exact_match when no exact match."""433result = manager.find_exact_match("nonexistent text")434assert result is None435436def test_increment_reuse(437self, manager: TranslationMemoryManager, sample_segment: TMSegment438) -> None:439"""Test incrementing reuse count."""440manager.add_segment(sample_segment)441assert sample_segment.reuse_count == 0442443result = manager.increment_reuse(sample_segment.id)444assert result is True445assert manager.get_segment(sample_segment.id).reuse_count == 1446447def test_increment_reuse_not_found(448self, manager: TranslationMemoryManager449) -> None:450"""Test increment for nonexistent segment."""451result = manager.increment_reuse("nonexistent")452assert result is False453454def test_get_statistics(455self, manager: TranslationMemoryManager, sample_segment: TMSegment456) -> None:457"""Test getting statistics."""458manager.add_segment(sample_segment)459stats = manager.get_statistics()460assert stats.total_segments == 1461assert "john" in stats.books_covered462463def test_get_all_segments(464self, manager: TranslationMemoryManager, sample_segment: TMSegment465) -> None:466"""Test getting all segments."""467manager.add_segment(sample_segment)468all_segs = manager.get_all_segments()469assert len(all_segs) == 1470471def test_remove_segment(472self, manager: TranslationMemoryManager, sample_segment: TMSegment473) -> None:474"""Test removing a segment."""475manager.add_segment(sample_segment)476assert len(manager) == 1477478result = manager.remove_segment(sample_segment.id)479assert result is True480assert len(manager) == 0481482def test_remove_segment_not_found(483self, manager: TranslationMemoryManager484) -> None:485"""Test removing nonexistent segment."""486result = manager.remove_segment("nonexistent")487assert result is False488489def test_clear(490self, manager: TranslationMemoryManager, sample_segment: TMSegment491) -> None:492"""Test clearing all segments."""493manager.add_segment(sample_segment)494manager.clear()495assert len(manager) == 0496497def test_save_and_load(498self, manager: TranslationMemoryManager, sample_segment: TMSegment499) -> None:500"""Test save and load roundtrip."""501manager.add_segment(sample_segment)502manager.save()503504# Create new manager and load505manager2 = TranslationMemoryManager(manager.storage_path)506manager2.load()507508assert len(manager2) == 1509loaded = manager2.get_segment(sample_segment.id)510assert loaded is not None511assert loaded.source_text == sample_segment.source_text512assert loaded.target_text == sample_segment.target_text513514def test_save_creates_directory(self, tmp_path: Path) -> None:515"""Test save creates parent directories."""516path = tmp_path / "nested" / "dir" / "segments.json"517manager = TranslationMemoryManager(path)518manager.add_from_translation(519source_text="Test",520target_text="テスト",521reference=VerseReference(book="john", chapter=1, verse=1),522)523manager.save()524assert path.exists()525526def test_load_nonexistent(self, tmp_path: Path) -> None:527"""Test loading from nonexistent file."""528path = tmp_path / "nonexistent.json"529manager = TranslationMemoryManager(path)530manager.load()531assert len(manager) == 0532533def test_reload(534self, manager: TranslationMemoryManager, sample_segment: TMSegment535) -> None:536"""Test reload discards in-memory changes."""537# Save initial state538manager.add_segment(sample_segment)539manager.save()540541# Add more in memory542manager.add_from_translation(543source_text="New",544target_text="新規",545reference=VerseReference(book="john", chapter=1, verse=2),546)547assert len(manager) == 2548549# Reload should restore to saved state550manager.reload()551assert len(manager) == 1552553def test_format_matches_for_prompt(self, manager: TranslationMemoryManager) -> None:554"""Test formatting matches for prompt."""555ref = VerseReference(book="john", chapter=1, verse=1)556segment = TMSegment.create(557source_text="Test",558target_text="テスト",559reference=ref,560)561match = TMMatch(segment=segment, similarity=0.9, match_type=MatchType.FUZZY)562563formatted = manager.format_matches_for_prompt([match])564assert "90%" in formatted565assert "john:1:1" in formatted566assert "テスト" in formatted567568def test_format_matches_empty(self, manager: TranslationMemoryManager) -> None:569"""Test formatting empty matches."""570formatted = manager.format_matches_for_prompt([])571assert "No similar translations" in formatted572573def test_format_matches_limit(self, manager: TranslationMemoryManager) -> None:574"""Test match display limit."""575matches = []576for i in range(5):577ref = VerseReference(book="john", chapter=1, verse=i + 1)578segment = TMSegment.create(579source_text=f"Test {i}",580target_text=f"テスト {i}",581reference=ref,582)583matches.append(584TMMatch(segment=segment, similarity=0.9 - i * 0.05, match_type=MatchType.FUZZY)585)586587formatted = manager.format_matches_for_prompt(matches)588# Should show "and X more"589assert "more" in formatted590591592class TestPromptBuilderWithTM:593"""Tests for PromptBuilder with Translation Memory integration."""594595def test_build_prompt_with_tm_matches(self) -> None:596"""Test prompt includes TM matches."""597from src.context.glossary import GlossaryManager598from src.pipeline.prompt import PromptBuilder599from src.models.verse import (600TranslationContext,601Verse,602VerseReference,603)604605glossary = GlossaryManager()606builder = PromptBuilder(glossary)607608ref = VerseReference(book="john", chapter=1, verse=1)609verse = Verse(610reference=ref,611greek_text="Ἐν ἀρχῇ ἦν ὁ λόγος",612)613context = TranslationContext(target_verse=verse)614615# Create TM match616segment = TMSegment.create(617source_text="Ἐν ἀρχῇ",618target_text="初めに",619reference=VerseReference(book="john", chapter=1, verse=1),620)621match = TMMatch(segment=segment, similarity=0.8, match_type=MatchType.FUZZY)622623prompt = builder.build_prompt(context, tm_matches=[match])624assert "TRANSLATION MEMORY" in prompt625assert "80%" in prompt626assert "初めに" in prompt627628def test_build_prompt_without_tm(self) -> None:629"""Test prompt without TM matches."""630from src.context.glossary import GlossaryManager631from src.pipeline.prompt import PromptBuilder632from src.models.verse import (633TranslationContext,634Verse,635VerseReference,636)637638glossary = GlossaryManager()639builder = PromptBuilder(glossary)640641ref = VerseReference(book="john", chapter=1, verse=1)642verse = Verse(reference=ref, greek_text="Ἐν ἀρχῇ")643context = TranslationContext(target_verse=verse)644645prompt = builder.build_prompt(context)646assert "TRANSLATION MEMORY" not in prompt647648649class TestConcurrentSavesNeverLoseWork:650"""Parallel book workers must not roll back each other's translations.651652Each worker holds its own manager over one shared file, loaded when its653book started and saved hours later. A plain overwrite meant the last654writer discarded every other book's segments — days of paid translation655gone, silently, with the run still reporting success.656"""657658@staticmethod659def _ref(book: str, chapter: int, verse: int) -> VerseReference:660"""Build a verse reference.661662Args:663book: Book name.664chapter: Chapter number.665verse: Verse number.666667Returns:668The VerseReference.669"""670return VerseReference(book=book, chapter=chapter, verse=verse)671672def test_a_later_save_keeps_an_earlier_workers_segments(self, tmp_path):673path = tmp_path / "segments.json"674675first = TranslationMemoryManager(path)676second = TranslationMemoryManager(path)677first.load()678second.load()679680first.add_from_translation("source A", "訳A", self._ref("john", 1, 1))681second.add_from_translation("source B", "訳B", self._ref("mark", 2, 2))682683first.save()684second.save()685686final = TranslationMemoryManager(path)687final.load()688assert sorted(s.source_text for s in final.get_all_segments()) == [689"source A",690"source B",691]692693def test_many_concurrent_saves_lose_nothing(self, tmp_path):694import threading695696path = tmp_path / "segments.json"697managers = []698for index in range(8):699manager = TranslationMemoryManager(path)700manager.load()701manager.add_from_translation(702f"source {index}", f"訳 {index}", self._ref("john", 1, index + 1)703)704managers.append(manager)705706threads = [threading.Thread(target=m.save) for m in managers]707for thread in threads:708thread.start()709for thread in threads:710thread.join()711712final = TranslationMemoryManager(path)713final.load()714assert len(final) == 8715716def test_a_retranslation_is_added_rather_than_replacing_the_earlier_one(self, tmp_path):717# Segment ids are random UUIDs here, so re-translating the same source718# yields a second segment rather than overwriting the first. That is719# duplication, not loss — and loss is the property under test. Merging720# must never be the thing that drops either rendering.721path = tmp_path / "segments.json"722723first = TranslationMemoryManager(path)724first.add_from_translation("same source", "古い訳", self._ref("john", 1, 1))725first.save()726727second = TranslationMemoryManager(path)728second.add_from_translation("same source", "新しい訳", self._ref("john", 1, 1))729second.save()730731final = TranslationMemoryManager(path)732final.load()733assert sorted(s.target_text for s in final.get_all_segments()) == ["古い訳", "新しい訳"]734735def test_saving_the_same_manager_twice_does_not_duplicate(self, tmp_path):736path = tmp_path / "segments.json"737738manager = TranslationMemoryManager(path)739manager.add_from_translation("source", "訳", self._ref("john", 1, 1))740manager.save()741manager.save()742743final = TranslationMemoryManager(path)744final.load()745assert len(final) == 1746747def test_the_reference_index_survives_a_merge(self, tmp_path):748path = tmp_path / "segments.json"749750first = TranslationMemoryManager(path)751first.add_from_translation("source A", "訳A", self._ref("john", 1, 1))752first.save()753754second = TranslationMemoryManager(path)755second.load()756second.add_from_translation("source B", "訳B", self._ref("mark", 2, 2))757second.save()758759assert second.get_by_reference(self._ref("john", 1, 1))760assert second.get_by_reference(self._ref("mark", 2, 2))761762763class TestCorruptStoreIsPreserved:764def test_an_unreadable_store_is_quarantined_not_overwritten(self, tmp_path, caplog):765# Silently starting fresh meant the next save replaced days of766# accumulated translation with whatever the process happened to hold.767path = tmp_path / "segments.json"768path.write_text("{ this is not json", encoding="utf-8")769770manager = TranslationMemoryManager(path)771manager.load()772manager.add_from_translation(773"after corruption", "訳", VerseReference(book="luke", chapter=1, verse=1)774)775manager.save()776777quarantined = list(tmp_path.glob("segments.corrupt.*.json"))778assert len(quarantined) == 1779assert quarantined[0].read_text(encoding="utf-8") == "{ this is not json"780781final = TranslationMemoryManager(path)782final.load()783assert len(final) == 1784785def test_a_missing_store_is_not_an_error(self, tmp_path):786manager = TranslationMemoryManager(tmp_path / "absent.json")787manager.load()788assert len(manager) == 0789790def test_no_stray_temp_file_survives_a_save(self, tmp_path):791path = tmp_path / "segments.json"792manager = TranslationMemoryManager(path)793manager.add_from_translation(794"source", "訳", VerseReference(book="john", chapter=1, verse=1)795)796manager.save()797assert not list(tmp_path.glob("*.tmp"))798