software/tests/test_data.py

1"""Tests for data loading and storage."""23import json4import pytest5from datetime import datetime6from pathlib import Path78from src.data.loader import DataLoader9from src.data.storage import Storage10from src.models.verse import (11    TranslationOutput,12    VerseReference,13)141516@pytest.fixture17def temp_data_dir(tmp_path: Path) -> Path:18    """Create a temporary data directory with test data."""19    # Create directory structure20    sources_dir = tmp_path / "sources"21    (sources_dir / "greek").mkdir(parents=True)22    (sources_dir / "japanese" / "meiji").mkdir(parents=True)23    (sources_dir / "japanese" / "taisho").mkdir(parents=True)24    (sources_dir / "english" / "kjv").mkdir(parents=True)25    (sources_dir / "english" / "asv").mkdir(parents=True)26    (tmp_path / "metadata" / "john").mkdir(parents=True)27    (tmp_path / "output").mkdir(parents=True)28    (tmp_path / "progress").mkdir(parents=True)2930    # Create test Greek data31    greek_data = {32        "1:1": {"text": "Ἐν ἀρχῇ ἦν ὁ λόγος", "translit": "En archē ēn ho logos"},33        "1:2": {"text": "οὗτος ἦν ἐν ἀρχῇ", "translit": "houtos ēn en archē"},34        "1:3": "Verse 3 text only",35    }36    with (sources_dir / "greek" / "john.json").open("w") as f:37        json.dump(greek_data, f)3839    # Create test KJV data40    kjv_data = {41        "1:1": "In the beginning was the Word",42        "1:2": "The same was in the beginning with God",43    }44    with (sources_dir / "english" / "kjv" / "john.json").open("w") as f:45        json.dump(kjv_data, f)4647    # Create test Meiji data48    meiji_data = {49        "1:1": "太初に言あり",50    }51    with (sources_dir / "japanese" / "meiji" / "john.json").open("w") as f:52        json.dump(meiji_data, f)5354    # Create test metadata55    book_metadata = {56        "book": "john",57        "name_japanese": "ヨハネによる福音書",58        "name_english": "Gospel of John",59        "testament": "NT",60        "author": "John the Apostle",61        "themes": ["love", "light"],62        "description": "The fourth Gospel",63        "chapter_count": 21,64        "verse_count": 879,65    }66    with (tmp_path / "metadata" / "john" / "book_context.json").open("w") as f:67        json.dump(book_metadata, f)6869    # Create chapter summaries70    chapter_summaries = {71        "1": {72            "summary": "The prologue",73            "key_events": ["Word becomes flesh"],74            "key_characters": ["Jesus"],75            "key_themes": ["Light"],76        }77    }78    with (tmp_path / "metadata" / "john" / "chapter_summaries.json").open("w") as f:79        json.dump(chapter_summaries, f)8081    # Create glossary82    glossary = {83        "terms": [84            {85                "original": "λόγος",86                "transliteration": "logos",87                "japanese": "言葉",88                "english": "word",89                "notes": "The Word",90            }91        ]92    }93    with (tmp_path / "metadata" / "glossary.json").open("w") as f:94        json.dump(glossary, f)9596    return tmp_path979899class TestDataLoader:100    """Tests for DataLoader."""101102    def test_init(self, temp_data_dir: Path) -> None:103        """Test initializing data loader."""104        loader = DataLoader(temp_data_dir)105        assert loader.data_dir == temp_data_dir106107    def test_load_greek_book(self, temp_data_dir: Path) -> None:108        """Test loading Greek text."""109        loader = DataLoader(temp_data_dir)110        greek = loader.load_greek_book("john")111        assert "1:1" in greek112113    def test_load_greek_book_not_found(self, temp_data_dir: Path) -> None:114        """Test loading non-existent Greek text."""115        loader = DataLoader(temp_data_dir)116        greek = loader.load_greek_book("nonexistent")117        assert greek == {}118119    def test_load_japanese_meiji(self, temp_data_dir: Path) -> None:120        """Test loading Meiji translation."""121        loader = DataLoader(temp_data_dir)122        meiji = loader.load_japanese_meiji("john")123        assert "1:1" in meiji124125    def test_load_japanese_taisho(self, temp_data_dir: Path) -> None:126        """Test loading Taisho translation."""127        loader = DataLoader(temp_data_dir)128        taisho = loader.load_japanese_taisho("john")129        assert taisho == {}  # No test data for taisho130131    def test_load_english_kjv(self, temp_data_dir: Path) -> None:132        """Test loading KJV translation."""133        loader = DataLoader(temp_data_dir)134        kjv = loader.load_english_kjv("john")135        assert "1:1" in kjv136137    def test_load_english_asv(self, temp_data_dir: Path) -> None:138        """Test loading ASV translation."""139        loader = DataLoader(temp_data_dir)140        asv = loader.load_english_asv("john")141        assert asv == {}  # No test data for ASV142143    def test_load_verse(self, temp_data_dir: Path) -> None:144        """Test loading a verse with all sources."""145        loader = DataLoader(temp_data_dir)146        ref = VerseReference(book="john", chapter=1, verse=1)147        verse = loader.load_verse(ref)148        assert verse.greek_text == "Ἐν ἀρχῇ ἦν ὁ λόγος"149        assert verse.greek_translit == "En archē ēn ho logos"150        assert verse.english_kjv == "In the beginning was the Word"151        assert verse.japanese_meiji == "太初に言あり"152153    def test_load_verse_with_string_greek(self, temp_data_dir: Path) -> None:154        """Test loading verse with string-only Greek data."""155        loader = DataLoader(temp_data_dir)156        ref = VerseReference(book="john", chapter=1, verse=3)157        verse = loader.load_verse(ref)158        assert verse.greek_text == "Verse 3 text only"159        assert verse.greek_translit is None160161    def test_load_chapter_verses(self, temp_data_dir: Path) -> None:162        """Test loading all verses in a chapter."""163        loader = DataLoader(temp_data_dir)164        verses = loader.load_chapter_verses("john", 1)165        assert len(verses) >= 2  # At least 2 verses in test data166167    def test_load_book_metadata(self, temp_data_dir: Path) -> None:168        """Test loading book metadata."""169        loader = DataLoader(temp_data_dir)170        meta = loader.load_book_metadata("john")171        assert meta is not None172        assert meta.name_japanese == "ヨハネによる福音書"173174    def test_load_book_metadata_not_found(self, temp_data_dir: Path) -> None:175        """Test loading non-existent metadata."""176        loader = DataLoader(temp_data_dir)177        meta = loader.load_book_metadata("nonexistent")178        assert meta is None179180    def test_load_chapter_summary(self, temp_data_dir: Path) -> None:181        """Test loading chapter summary."""182        loader = DataLoader(temp_data_dir)183        summary = loader.load_chapter_summary("john", 1)184        assert summary is not None185        assert summary.summary == "The prologue"186187    def test_load_chapter_summary_not_found(self, temp_data_dir: Path) -> None:188        """Test loading non-existent chapter summary."""189        loader = DataLoader(temp_data_dir)190        summary = loader.load_chapter_summary("john", 99)191        assert summary is None192193    def test_load_glossary(self, temp_data_dir: Path) -> None:194        """Test loading glossary."""195        loader = DataLoader(temp_data_dir)196        glossary = loader.load_glossary()197        assert len(glossary) == 1198        assert glossary[0].original == "λόγος"199200    def test_load_glossary_empty(self, tmp_path: Path) -> None:201        """Test loading empty glossary."""202        (tmp_path / "sources").mkdir()203        (tmp_path / "metadata").mkdir()204        loader = DataLoader(tmp_path)205        glossary = loader.load_glossary()206        assert glossary == []207208    def test_get_book_chapters(self, temp_data_dir: Path) -> None:209        """Test getting chapter list."""210        loader = DataLoader(temp_data_dir)211        chapters = loader.get_book_chapters("john")212        assert 1 in chapters213214    def test_get_book_chapters_handles_invalid_keys(self, temp_data_dir: Path) -> None:215        """Test get_book_chapters handles non-numeric keys."""216        # Add invalid key to Greek data217        greek_file = temp_data_dir / "sources" / "greek" / "john.json"218        with greek_file.open("r") as f:219            data = json.load(f)220        data["invalid"] = "test"221        with greek_file.open("w") as f:222            json.dump(data, f)223224        loader = DataLoader(temp_data_dir)225        chapters = loader.get_book_chapters("john")226        assert 1 in chapters227228    def test_load_hebrew_book(self, temp_data_dir: Path) -> None:229        """Test loading Hebrew text."""230        # Create Hebrew source directory and file231        hebrew_dir = temp_data_dir / "sources" / "hebrew"232        hebrew_dir.mkdir(parents=True, exist_ok=True)233        hebrew_data = {234            "1:1": {"text": "בְּרֵאשִׁית", "translit": "bereshit"},235        }236        with (hebrew_dir / "genesis.json").open("w") as f:237            json.dump(hebrew_data, f)238239        loader = DataLoader(temp_data_dir)240        hebrew = loader.load_hebrew_book("genesis")241        assert "1:1" in hebrew242243    def test_load_verse_with_hebrew(self, temp_data_dir: Path) -> None:244        """Test loading a verse with Hebrew text."""245        # Create Hebrew source246        hebrew_dir = temp_data_dir / "sources" / "hebrew"247        hebrew_dir.mkdir(parents=True, exist_ok=True)248        hebrew_data = {249            "1:1": {"text": "בְּרֵאשִׁית", "translit": "bereshit"},250        }251        with (hebrew_dir / "genesis.json").open("w") as f:252            json.dump(hebrew_data, f)253254        loader = DataLoader(temp_data_dir)255        ref = VerseReference(book="genesis", chapter=1, verse=1)256        verse = loader.load_verse(ref)257        assert verse.hebrew_text == "בְּרֵאשִׁית"258        assert verse.hebrew_translit == "bereshit"259260    def test_load_verse_with_hebrew_string_only(self, temp_data_dir: Path) -> None:261        """Test loading verse with string-only Hebrew data."""262        hebrew_dir = temp_data_dir / "sources" / "hebrew"263        hebrew_dir.mkdir(parents=True, exist_ok=True)264        hebrew_data = {265            "1:1": "בְּרֵאשִׁית",  # String instead of dict266        }267        with (hebrew_dir / "genesis.json").open("w") as f:268            json.dump(hebrew_data, f)269270        loader = DataLoader(temp_data_dir)271        ref = VerseReference(book="genesis", chapter=1, verse=1)272        verse = loader.load_verse(ref)273        assert verse.hebrew_text == "בְּרֵאשִׁית"274        assert verse.hebrew_translit is None275276277class TestStorage:278    """Tests for Storage."""279280    def test_init(self, temp_data_dir: Path) -> None:281        """Test initializing storage."""282        storage = Storage(temp_data_dir)283        assert storage.data_dir == temp_data_dir284285    def test_save_and_load_translation(self, temp_data_dir: Path) -> None:286        """Test saving and loading a translation."""287        storage = Storage(temp_data_dir)288        ref = VerseReference(book="john", chapter=1, verse=1)289        output = TranslationOutput(290            reference=ref,291            text="初めに言葉があった。",292            model="gpt-4o",293            sources_used=["greek", "kjv"],294            glossary_terms_applied=["λόγος"],295        )296        storage.save_translation(output)297298        loaded = storage.load_translation(ref)299        assert loaded is not None300        assert loaded.text == "初めに言葉があった。"301        assert loaded.model == "gpt-4o"302303    def test_load_translation_not_found(self, temp_data_dir: Path) -> None:304        """Test loading non-existent translation."""305        storage = Storage(temp_data_dir)306        ref = VerseReference(book="john", chapter=99, verse=99)307        loaded = storage.load_translation(ref)308        assert loaded is None309310    def test_load_chapter_translations(self, temp_data_dir: Path) -> None:311        """Test loading all translations for a chapter."""312        storage = Storage(temp_data_dir)313314        # Save a few translations315        for verse_num in [1, 2]:316            ref = VerseReference(book="john", chapter=1, verse=verse_num)317            output = TranslationOutput(318                reference=ref,319                text=f"Translation {verse_num}",320                model="test",321            )322            storage.save_translation(output)323324        translations = storage.load_chapter_translations("john", 1)325        assert len(translations) == 2326        assert 1 in translations327        assert 2 in translations328329    def test_is_verse_translated(self, temp_data_dir: Path) -> None:330        """Test checking if verse is translated."""331        storage = Storage(temp_data_dir)332        ref = VerseReference(book="john", chapter=1, verse=1)333334        assert storage.is_verse_translated(ref) is False335336        output = TranslationOutput(reference=ref, text="Test", model="test")337        storage.save_translation(output)338339        assert storage.is_verse_translated(ref) is True340341    def test_get_progress(self, temp_data_dir: Path) -> None:342        """Test getting progress."""343        storage = Storage(temp_data_dir)344        ref = VerseReference(book="john", chapter=1, verse=1)345        output = TranslationOutput(reference=ref, text="Test", model="test")346        storage.save_translation(output)347348        progress = storage.get_progress("john")349        assert "completed" in progress350        assert "1" in progress["completed"]351352    def test_get_completion_stats(self, temp_data_dir: Path) -> None:353        """Test getting completion stats."""354        storage = Storage(temp_data_dir)355356        # Save some translations357        for verse in [1, 2, 3]:358            ref = VerseReference(book="john", chapter=1, verse=verse)359            output = TranslationOutput(reference=ref, text="Test", model="test")360            storage.save_translation(output)361362        stats = storage.get_completion_stats("john", 100)363        assert stats["completed"] == 3364        assert stats["total"] == 100365        assert stats["percentage"] == 3.0366367    def test_get_completion_stats_zero_total(self, temp_data_dir: Path) -> None:368        """Test completion stats with zero total."""369        storage = Storage(temp_data_dir)370        stats = storage.get_completion_stats("john", 0)371        assert stats["percentage"] == 0372373    def test_save_and_load_translation_with_furigana(374        self, temp_data_dir: Path375    ) -> None:376        """Test saving and loading a translation with furigana."""377        storage = Storage(temp_data_dir)378        ref = VerseReference(book="john", chapter=1, verse=1)379        annotations = [380            {381                "start_offset": 0,382                "end_offset": 1,383                "base_text": "神",384                "reading": "かみ",385            },386            {387                "start_offset": 2,388                "end_offset": 3,389                "base_text": "愛",390                "reading": "あい",391            },392        ]393        output = TranslationOutput(394            reference=ref,395            text="神は愛です",396            text_with_furigana="神[かみ]は愛[あい]です",397            furigana_annotations=annotations,398            model="gpt-4o",399        )400        storage.save_translation(output)401402        loaded = storage.load_translation(ref)403        assert loaded is not None404        assert loaded.text == "神は愛です"405        assert loaded.text_with_furigana == "神[かみ]は愛[あい]です"406        assert len(loaded.furigana_annotations) == 2407        assert loaded.furigana_annotations[0]["base_text"] == "神"408        assert loaded.furigana_annotations[1]["reading"] == "あい"409410    def test_load_translation_backward_compatibility(411        self, temp_data_dir: Path412    ) -> None:413        """Test loading translation without furigana fields (backward compatibility)."""414        storage = Storage(temp_data_dir)415        ref = VerseReference(book="john", chapter=1, verse=1)416417        # Manually create a chapter file without furigana fields418        chapter_file = storage.output_dir / "john" / "chapter_01.json"419        chapter_file.parent.mkdir(parents=True, exist_ok=True)420        chapter_data = {421            "book": "john",422            "chapter": 1,423            "verses": {424                "1": {425                    "text": "初めに言葉があった。",426                    "model": "test-model",427                    "timestamp": "2024-01-01T00:00:00",428                    "prompt_version": "v1.0",429                    "sources_used": [],430                    "glossary_terms_applied": [],431                }432            },433        }434        with chapter_file.open("w", encoding="utf-8") as f:435            json.dump(chapter_data, f)436437        loaded = storage.load_translation(ref)438        assert loaded is not None439        assert loaded.text == "初めに言葉があった。"440        assert loaded.text_with_furigana == ""441        assert loaded.furigana_annotations == []442443    def test_load_chapter_translations_with_furigana(444        self, temp_data_dir: Path445    ) -> None:446        """Test loading chapter translations includes furigana fields."""447        storage = Storage(temp_data_dir)448449        # Save translations with furigana450        for verse_num in [1, 2]:451            ref = VerseReference(book="john", chapter=1, verse=verse_num)452            output = TranslationOutput(453                reference=ref,454                text=f"翻訳{verse_num}",455                text_with_furigana=f"翻訳[ほんやく]{verse_num}",456                furigana_annotations=[457                    {458                        "start_offset": 0,459                        "end_offset": 2,460                        "base_text": "翻訳",461                        "reading": "ほんやく",462                    }463                ],464                model="test",465            )466            storage.save_translation(output)467468        translations = storage.load_chapter_translations("john", 1)469        assert len(translations) == 2470        assert translations[1].text_with_furigana == "翻訳[ほんやく]1"471        assert len(translations[1].furigana_annotations) == 1472473474class TestStorageArchive:475    """Tests for archive storage functionality."""476477    def test_init_with_archive_disabled(self, temp_data_dir: Path) -> None:478        """Test storage init with archive disabled by default."""479        storage = Storage(temp_data_dir)480        assert storage.enable_archive is False481        assert not storage.archive_dir.exists()482483    def test_init_with_archive_enabled(self, temp_data_dir: Path) -> None:484        """Test storage init with archive enabled."""485        storage = Storage(temp_data_dir, enable_archive=True)486        assert storage.enable_archive is True487        assert storage.archive_dir.exists()488489    def test_enable_archive_property(self, temp_data_dir: Path) -> None:490        """Test enable_archive property setter."""491        storage = Storage(temp_data_dir)492        assert storage.enable_archive is False493494        storage.enable_archive = True495        assert storage.enable_archive is True496        assert storage.archive_dir.exists()497498    def test_save_translation_archive(self, temp_data_dir: Path) -> None:499        """Test saving translation archive."""500        from src.protos.converters import (501            TranslationPromptInputProto,502            TranslationResultProto,503            VerseProto,504            VerseReferenceProto,505        )506507        storage = Storage(temp_data_dir, enable_archive=True)508        ref = VerseReference(book="john", chapter=1, verse=1)509510        input_proto = TranslationPromptInputProto(511            target_verse=VerseProto(512                reference=VerseReferenceProto(book="john", chapter=1, verse=1),513                greek_text="Ἐν ἀρχῇ",514            ),515            system_prompt="System",516            user_prompt="User",517            model="test-model",518        )519        output_proto = TranslationResultProto(520            reference=VerseReferenceProto(book="john", chapter=1, verse=1),521            text="初めに",522            model="test-model",523        )524525        archive_path = storage.save_translation_archive(ref, input_proto, output_proto)526527        assert archive_path.exists()528        assert (archive_path / "verse_01_input.msgpack").exists()529        assert (archive_path / "verse_01_output.msgpack").exists()530531    def test_load_translation_archive(self, temp_data_dir: Path) -> None:532        """Test loading translation archive."""533        from src.protos.converters import (534            TranslationPromptInputProto,535            TranslationResultProto,536            VerseProto,537            VerseReferenceProto,538        )539540        storage = Storage(temp_data_dir, enable_archive=True)541        ref = VerseReference(book="john", chapter=1, verse=1)542543        input_proto = TranslationPromptInputProto(544            target_verse=VerseProto(545                reference=VerseReferenceProto(book="john", chapter=1, verse=1),546                greek_text="Ἐν ἀρχῇ ἦν ὁ λόγος",547            ),548            system_prompt="You are a translator",549            user_prompt="Translate this",550            model="claude-3-opus",551        )552        output_proto = TranslationResultProto(553            reference=VerseReferenceProto(book="john", chapter=1, verse=1),554            text="初めに言葉があった",555            model="claude-3-opus",556            input_hash="abc123",557        )558559        storage.save_translation_archive(ref, input_proto, output_proto)560561        result = storage.load_translation_archive(ref)562        assert result is not None563        loaded_input, loaded_output = result564565        assert loaded_input.target_verse.greek_text == "Ἐν ἀρχῇ ἦν ὁ λόγος"566        assert loaded_input.system_prompt == "You are a translator"567        assert loaded_output.text == "初めに言葉があった"568        assert loaded_output.input_hash == "abc123"569570    def test_load_translation_archive_not_found(self, temp_data_dir: Path) -> None:571        """Test loading non-existent archive returns None."""572        storage = Storage(temp_data_dir, enable_archive=True)573        ref = VerseReference(book="john", chapter=99, verse=99)574575        result = storage.load_translation_archive(ref)576        assert result is None577578    def test_archive_directory_structure(self, temp_data_dir: Path) -> None:579        """Test archive creates proper directory structure."""580        from src.protos.converters import (581            TranslationPromptInputProto,582            TranslationResultProto,583            VerseReferenceProto,584        )585586        storage = Storage(temp_data_dir, enable_archive=True)587588        # Save multiple verses589        for chapter in [1, 2]:590            for verse in [1, 2, 3]:591                ref = VerseReference(book="john", chapter=chapter, verse=verse)592                input_proto = TranslationPromptInputProto(model="test")593                output_proto = TranslationResultProto(594                    reference=VerseReferenceProto(595                        book="john", chapter=chapter, verse=verse596                    ),597                    text=f"Translation {chapter}:{verse}",598                    model="test",599                )600                storage.save_translation_archive(ref, input_proto, output_proto)601602        # Check directory structure603        assert (storage.archive_dir / "john").exists()604        assert (storage.archive_dir / "john" / "chapter_01").exists()605        assert (storage.archive_dir / "john" / "chapter_02").exists()606607        # Check files608        ch1_path = storage.archive_dir / "john" / "chapter_01"609        assert (ch1_path / "verse_01_input.msgpack").exists()610        assert (ch1_path / "verse_02_input.msgpack").exists()611        assert (ch1_path / "verse_03_input.msgpack").exists()612613    def test_archive_roundtrip_with_provenance(self, temp_data_dir: Path) -> None:614        """Test archive roundtrip includes provenance data."""615        from datetime import datetime, timezone616617        from src.protos.converters import (618            TranslationPromptInputProto,619            TranslationProvenanceProto,620            TranslationResultProto,621            VerseReferenceProto,622        )623624        storage = Storage(temp_data_dir, enable_archive=True)625        ref = VerseReference(book="matthew", chapter=5, verse=3)626627        provenance = TranslationProvenanceProto(628            source_texts={"greek": "Μακάριοι"},629            model="claude-opus",630            prompt_version="v2.0",631            started_at=datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc),632            completed_at=datetime(2024, 1, 1, 12, 0, 30, tzinfo=timezone.utc),633        )634635        input_proto = TranslationPromptInputProto(636            system_prompt="Translate",637            user_prompt="Verse",638            model="claude-opus",639        )640        output_proto = TranslationResultProto(641            reference=VerseReferenceProto(book="matthew", chapter=5, verse=3),642            text="心の貧しい人々は幸いです",643            model="claude-opus",644            provenance=provenance,645        )646647        storage.save_translation_archive(ref, input_proto, output_proto)648649        result = storage.load_translation_archive(ref)650        assert result is not None651        _, loaded_output = result652653        assert loaded_output.provenance is not None654        assert loaded_output.provenance.model == "claude-opus"655        assert loaded_output.provenance.source_texts == {"greek": "Μακάριοι"}656