software/tests/test_evaluation.py

1"""Tests for the evaluation module."""23from __future__ import annotations45import json6import tempfile7from datetime import datetime8from pathlib import Path910import pytest1112from src.models.evaluation import (13    ANY_DIMENSION_THRESHOLD,14    CONFIDENCE_THRESHOLD,15    DIMENSIONS,16    SOURCE_FIDELITY_THRESHOLD,17    THEOLOGICAL_ACCURACY_THRESHOLD,18    BookEvaluationSummary,19    ChapterEvaluation,20    EvaluationScore,21    VerseEvaluation,22)23from src.evaluation.evaluator import (24    TranslationEvaluator,25    build_evaluation_prompt,26    parse_evaluation_response,27)28from src.evaluation.storage import EvaluationStorage293031class TestEvaluationScore:32    """Tests for EvaluationScore model."""3334    def test_create_score(self) -> None:35        """Test creating an evaluation score."""36        score = EvaluationScore(37            dimension="theological_accuracy",38            score=95.0,39            confidence=90.0,40            reasoning="Theological meaning is well preserved.",41        )42        assert score.dimension == "theological_accuracy"43        assert score.score == 95.044        assert score.confidence == 90.045        assert "preserved" in score.reasoning4647    def test_score_bounds(self) -> None:48        """Test score validation bounds."""49        # Valid scores50        score = EvaluationScore(51            dimension="test", score=0.0, confidence=100.0, reasoning="test"52        )53        assert score.score == 0.054        assert score.confidence == 100.05556        # Invalid scores should raise57        with pytest.raises(ValueError):58            EvaluationScore(59                dimension="test", score=-1.0, confidence=50.0, reasoning="test"60            )6162        with pytest.raises(ValueError):63            EvaluationScore(64                dimension="test", score=101.0, confidence=50.0, reasoning="test"65            )666768class TestVerseEvaluation:69    """Tests for VerseEvaluation model."""7071    def test_create_verse_evaluation(self) -> None:72        """Test creating a verse evaluation."""73        scores = {74            "theological_accuracy": EvaluationScore(75                dimension="theological_accuracy",76                score=95.0,77                confidence=90.0,78                reasoning="Good",79            ),80            "source_fidelity": EvaluationScore(81                dimension="source_fidelity",82                score=88.0,83                confidence=85.0,84                reasoning="Good",85            ),86        }87        evaluation = VerseEvaluation(88            verse_number=1,89            evaluator_model="test-model",90            scores=scores,91        )92        assert evaluation.verse_number == 193        assert evaluation.evaluator_model == "test-model"94        assert len(evaluation.scores) == 29596    def test_overall_score_calculation(self) -> None:97        """Test overall score is calculated correctly."""98        scores = {99            "dim1": EvaluationScore(100                dimension="dim1", score=80.0, confidence=90.0, reasoning="test"101            ),102            "dim2": EvaluationScore(103                dimension="dim2", score=90.0, confidence=90.0, reasoning="test"104            ),105        }106        evaluation = VerseEvaluation(107            verse_number=1, evaluator_model="test", scores=scores108        )109        assert evaluation.overall_score == 85.0110111    def test_compute_flags_theological_concern(self) -> None:112        """Test flagging for theological accuracy below threshold."""113        scores = {114            "theological_accuracy": EvaluationScore(115                dimension="theological_accuracy",116                score=THEOLOGICAL_ACCURACY_THRESHOLD - 1,117                confidence=90.0,118                reasoning="test",119            ),120        }121        evaluation = VerseEvaluation(122            verse_number=1, evaluator_model="test", scores=scores123        )124        flags = evaluation.compute_flags()125        assert "theological_concern" in flags126127    def test_compute_flags_low_confidence(self) -> None:128        """Test flagging for low confidence on theology."""129        scores = {130            "theological_accuracy": EvaluationScore(131                dimension="theological_accuracy",132                score=90.0,133                confidence=CONFIDENCE_THRESHOLD - 1,134                reasoning="test",135            ),136        }137        evaluation = VerseEvaluation(138            verse_number=1, evaluator_model="test", scores=scores139        )140        flags = evaluation.compute_flags()141        assert "low_confidence_theology" in flags142143    def test_compute_flags_source_fidelity(self) -> None:144        """Test flagging for source fidelity below threshold."""145        scores = {146            "source_fidelity": EvaluationScore(147                dimension="source_fidelity",148                score=SOURCE_FIDELITY_THRESHOLD - 1,149                confidence=90.0,150                reasoning="test",151            ),152        }153        evaluation = VerseEvaluation(154            verse_number=1, evaluator_model="test", scores=scores155        )156        flags = evaluation.compute_flags()157        assert "source_fidelity_concern" in flags158159    def test_compute_flags_any_low_dimension(self) -> None:160        """Test flagging for any dimension below threshold."""161        scores = {162            "naturalness": EvaluationScore(163                dimension="naturalness",164                score=ANY_DIMENSION_THRESHOLD - 1,165                confidence=90.0,166                reasoning="test",167            ),168        }169        evaluation = VerseEvaluation(170            verse_number=1, evaluator_model="test", scores=scores171        )172        flags = evaluation.compute_flags()173        assert "low_naturalness" in flags174175    def test_no_flags_for_good_scores(self) -> None:176        """Test no flags when all scores are good."""177        scores = {178            dim: EvaluationScore(179                dimension=dim, score=90.0, confidence=90.0, reasoning="test"180            )181            for dim in DIMENSIONS182        }183        evaluation = VerseEvaluation(184            verse_number=1, evaluator_model="test", scores=scores185        )186        flags = evaluation.compute_flags()187        assert len(flags) == 0188189190class TestChapterEvaluation:191    """Tests for ChapterEvaluation model."""192193    def test_create_chapter_evaluation(self) -> None:194        """Test creating a chapter evaluation."""195        evaluation = ChapterEvaluation(196            book="titus",197            chapter=1,198            evaluator_model="test-model",199        )200        assert evaluation.book == "titus"201        assert evaluation.chapter == 1202        assert len(evaluation.verses) == 0203204    def test_aggregate_scores(self) -> None:205        """Test aggregate scores calculation."""206        scores1 = {207            "dim1": EvaluationScore(208                dimension="dim1", score=80.0, confidence=90.0, reasoning="test"209            ),210        }211        scores2 = {212            "dim1": EvaluationScore(213                dimension="dim1", score=90.0, confidence=90.0, reasoning="test"214            ),215        }216        verse1 = VerseEvaluation(verse_number=1, evaluator_model="test", scores=scores1)217        verse2 = VerseEvaluation(verse_number=2, evaluator_model="test", scores=scores2)218219        chapter = ChapterEvaluation(220            book="test",221            chapter=1,222            evaluator_model="test",223            verses={1: verse1, 2: verse2},224        )225        assert chapter.aggregate_scores["dim1"] == 85.0226227    def test_flagged_verses(self) -> None:228        """Test flagged verses list."""229        scores_bad = {230            "theological_accuracy": EvaluationScore(231                dimension="theological_accuracy",232                score=50.0,233                confidence=90.0,234                reasoning="test",235            ),236        }237        scores_good = {238            "theological_accuracy": EvaluationScore(239                dimension="theological_accuracy",240                score=95.0,241                confidence=90.0,242                reasoning="test",243            ),244        }245246        verse1 = VerseEvaluation(247            verse_number=1, evaluator_model="test", scores=scores_bad248        )249        verse1.flags = verse1.compute_flags()250251        verse2 = VerseEvaluation(252            verse_number=2, evaluator_model="test", scores=scores_good253        )254        verse2.flags = verse2.compute_flags()255256        chapter = ChapterEvaluation(257            book="test",258            chapter=1,259            evaluator_model="test",260            verses={1: verse1, 2: verse2},261        )262        assert 1 in chapter.flagged_verses263        assert 2 not in chapter.flagged_verses264        assert chapter.flagged_count == 1265266267class TestBuildEvaluationPrompt:268    """Tests for build_evaluation_prompt function."""269270    def test_basic_prompt(self) -> None:271        """Test building a basic evaluation prompt."""272        prompt = build_evaluation_prompt(273            translation="初めに言葉があった。",274            greek_text="Ἐν ἀρχῇ ἦν ὁ λόγος",275            kjv_text="In the beginning was the Word",276            reference_japanese=None,277            glossary_terms=None,278        )279        assert "初めに言葉があった" in prompt280        assert "Ἐν ἀρχῇ" in prompt281        assert "In the beginning" in prompt282        assert "theological_accuracy" in prompt.lower()283284    def test_prompt_with_glossary(self) -> None:285        """Test prompt includes glossary terms."""286        prompt = build_evaluation_prompt(287            translation="test",288            greek_text="test",289            kjv_text=None,290            reference_japanese=None,291            glossary_terms=["言葉", "神"],292        )293        assert "言葉" in prompt294        assert "神" in prompt295296    def test_prompt_with_reference_japanese(self) -> None:297        """Test prompt includes reference Japanese."""298        prompt = build_evaluation_prompt(299            translation="test",300            greek_text=None,301            kjv_text=None,302            reference_japanese="参考日本語訳",303            glossary_terms=None,304        )305        assert "参考日本語訳" in prompt306307308class TestParseEvaluationResponse:309    """Tests for parse_evaluation_response function."""310311    def test_parse_valid_json(self) -> None:312        """Test parsing valid JSON response."""313        response = json.dumps(314            {315                dim: {"score": 90, "confidence": 85, "reasoning": "Good"}316                for dim in DIMENSIONS317            }318        )319        scores = parse_evaluation_response(response)320        assert len(scores) == len(DIMENSIONS)321        assert all(dim in scores for dim in DIMENSIONS)322        assert scores["theological_accuracy"].score == 90323324    def test_parse_json_in_code_block(self) -> None:325        """Test parsing JSON wrapped in code block."""326        json_content = json.dumps(327            {328                dim: {"score": 90, "confidence": 85, "reasoning": "Good"}329                for dim in DIMENSIONS330            }331        )332        response = f"```json\n{json_content}\n```"333        scores = parse_evaluation_response(response)334        assert len(scores) == len(DIMENSIONS)335336    def test_parse_invalid_json(self) -> None:337        """Test parsing invalid JSON raises error."""338        with pytest.raises(ValueError, match="Failed to parse"):339            parse_evaluation_response("not valid json")340341    def test_parse_missing_dimension(self) -> None:342        """Test parsing response with missing dimension raises error."""343        response = json.dumps(344            {"theological_accuracy": {"score": 90, "confidence": 85, "reasoning": "Good"}}345        )346        with pytest.raises(ValueError, match="Missing dimension"):347            parse_evaluation_response(response)348349350class TestTranslationEvaluator:351    """Tests for TranslationEvaluator class."""352353    def test_create_evaluator(self) -> None:354        """Test creating an evaluator."""355        evaluator = TranslationEvaluator(model_name="test-model")356        assert evaluator.model_name == "test-model"357358    def test_create_verse_evaluation(self) -> None:359        """Test creating a verse evaluation from scores."""360        evaluator = TranslationEvaluator()361        scores = {362            dim: EvaluationScore(363                dimension=dim, score=90.0, confidence=85.0, reasoning="Good"364            )365            for dim in DIMENSIONS366        }367        evaluation = evaluator.create_verse_evaluation(368            verse_number=1, scores=scores, notes="Test note"369        )370        assert evaluation.verse_number == 1371        assert evaluation.notes == "Test note"372        assert len(evaluation.flags) == 0  # Good scores, no flags373374    def test_create_chapter_evaluation(self) -> None:375        """Test creating a chapter evaluation."""376        evaluator = TranslationEvaluator()377        scores = {378            dim: EvaluationScore(379                dimension=dim, score=90.0, confidence=85.0, reasoning="Good"380            )381            for dim in DIMENSIONS382        }383        verse_eval = evaluator.create_verse_evaluation(verse_number=1, scores=scores)384385        chapter_eval = evaluator.create_chapter_evaluation(386            book="titus", chapter=1, verse_evaluations={1: verse_eval}387        )388        assert chapter_eval.book == "titus"389        assert chapter_eval.chapter == 1390        assert len(chapter_eval.verses) == 1391392393class TestEvaluationStorage:394    """Tests for EvaluationStorage class."""395396    def test_save_and_load_chapter_evaluation(self) -> None:397        """Test saving and loading a chapter evaluation."""398        with tempfile.TemporaryDirectory() as tmpdir:399            storage = EvaluationStorage(Path(tmpdir))400401            # Create evaluation402            scores = {403                dim: EvaluationScore(404                    dimension=dim, score=90.0, confidence=85.0, reasoning="Good"405                )406                for dim in DIMENSIONS407            }408            verse_eval = VerseEvaluation(409                verse_number=1, evaluator_model="test", scores=scores410            )411            chapter_eval = ChapterEvaluation(412                book="titus",413                chapter=1,414                evaluator_model="test",415                verses={1: verse_eval},416            )417418            # Save419            path = storage.save_chapter_evaluation(chapter_eval)420            assert path.exists()421422            # Load423            loaded = storage.load_chapter_evaluation("titus", 1)424            assert loaded is not None425            assert loaded.book == "titus"426            assert loaded.chapter == 1427            assert len(loaded.verses) == 1428            assert loaded.verses[1].scores["theological_accuracy"].score == 90.0429430    def test_load_nonexistent(self) -> None:431        """Test loading non-existent evaluation returns None."""432        with tempfile.TemporaryDirectory() as tmpdir:433            storage = EvaluationStorage(Path(tmpdir))434            loaded = storage.load_chapter_evaluation("nonexistent", 1)435            assert loaded is None436437    def test_is_chapter_evaluated(self) -> None:438        """Test checking if chapter is evaluated."""439        with tempfile.TemporaryDirectory() as tmpdir:440            storage = EvaluationStorage(Path(tmpdir))441442            assert not storage.is_chapter_evaluated("titus", 1)443444            # Save evaluation445            chapter_eval = ChapterEvaluation(446                book="titus", chapter=1, evaluator_model="test", verses={}447            )448            storage.save_chapter_evaluation(chapter_eval)449450            assert storage.is_chapter_evaluated("titus", 1)451452    def test_get_flagged_verses(self) -> None:453        """Test getting flagged verses for a book."""454        with tempfile.TemporaryDirectory() as tmpdir:455            storage = EvaluationStorage(Path(tmpdir))456457            # Create evaluation with flagged verse458            scores = {459                "theological_accuracy": EvaluationScore(460                    dimension="theological_accuracy",461                    score=50.0,462                    confidence=90.0,463                    reasoning="Issues",464                ),465            }466            verse_eval = VerseEvaluation(467                verse_number=5, evaluator_model="test", scores=scores468            )469            verse_eval.flags = verse_eval.compute_flags()470471            chapter_eval = ChapterEvaluation(472                book="titus",473                chapter=1,474                evaluator_model="test",475                verses={5: verse_eval},476            )477            storage.save_chapter_evaluation(chapter_eval)478479            flagged = storage.get_flagged_verses("titus")480            assert len(flagged) == 1481            assert flagged[0].chapter == 1482            assert flagged[0].verse == 5483484    def test_get_book_summary(self) -> None:485        """Test getting book evaluation summary."""486        with tempfile.TemporaryDirectory() as tmpdir:487            storage = EvaluationStorage(Path(tmpdir))488489            # Save two chapter evaluations490            for chapter in [1, 2]:491                scores = {492                    dim: EvaluationScore(493                        dimension=dim,494                        score=85.0 + chapter,495                        confidence=90.0,496                        reasoning="Good",497                    )498                    for dim in DIMENSIONS499                }500                verse_eval = VerseEvaluation(501                    verse_number=1, evaluator_model="test", scores=scores502                )503                chapter_eval = ChapterEvaluation(504                    book="titus",505                    chapter=chapter,506                    evaluator_model="test",507                    verses={1: verse_eval},508                )509                storage.save_chapter_evaluation(chapter_eval)510511            summary = storage.get_book_summary("titus")512            assert summary is not None513            assert summary.book == "titus"514            assert summary.total_chapters == 2515            assert summary.total_verses == 2516            assert summary.overall_score > 0517518    def test_get_book_summary_no_evaluations(self) -> None:519        """Test getting summary when no evaluations exist."""520        with tempfile.TemporaryDirectory() as tmpdir:521            storage = EvaluationStorage(Path(tmpdir))522            summary = storage.get_book_summary("nonexistent")523            assert summary is None524525    def test_get_book_summary_dir_exists_no_eval_files(self) -> None:526        """Test getting summary when book dir exists but has no eval files."""527        with tempfile.TemporaryDirectory() as tmpdir:528            storage = EvaluationStorage(Path(tmpdir))529            # Create the book directory but don't add any eval files530            book_dir = Path(tmpdir) / "output" / "titus"531            book_dir.mkdir(parents=True)532            # Add a non-eval file533            (book_dir / "chapter_01.json").write_text("{}")534535            summary = storage.get_book_summary("titus")536            assert summary is None537538    def test_get_book_summary_with_flagged_verses(self) -> None:539        """Test getting book summary with flagged verses."""540        with tempfile.TemporaryDirectory() as tmpdir:541            storage = EvaluationStorage(Path(tmpdir))542543            # Create evaluation with low theological accuracy (will be flagged)544            scores = {545                dim: EvaluationScore(546                    dimension=dim,547                    score=95.0 if dim != "theological_accuracy" else 75.0,548                    confidence=90.0,549                    reasoning="Test",550                )551                for dim in DIMENSIONS552            }553            verse_eval = VerseEvaluation(554                verse_number=5,555                evaluator_model="test",556                scores=scores,557                flags=["theological_concern"],  # Explicitly set flag558            )559            chapter_eval = ChapterEvaluation(560                book="titus",561                chapter=1,562                evaluator_model="test",563                verses={5: verse_eval},564            )565            storage.save_chapter_evaluation(chapter_eval)566567            summary = storage.get_book_summary("titus")568            assert summary is not None569            assert len(summary.flagged_verses) == 1570            assert "titus:1:5" in summary.flagged_verses571572573class TestBookEvaluationSummary:574    """Tests for BookEvaluationSummary model."""575576    def test_create_summary(self) -> None:577        """Test creating a book evaluation summary."""578        summary = BookEvaluationSummary(579            book="titus",580            total_chapters=3,581            total_verses=46,582            aggregate_scores={583                "theological_accuracy": 92.5,584                "source_fidelity": 88.0,585            },586            overall_score=90.3,587            flagged_verses=["titus:1:5", "titus:2:3"],588        )589        assert summary.book == "titus"590        assert summary.total_chapters == 3591        assert summary.total_verses == 46592        assert len(summary.flagged_verses) == 2593594595class TestParseEvaluationResponseExtended:596    """Extended tests for parse_evaluation_response function."""597598    def test_parse_json_in_plain_code_block(self) -> None:599        """Test parsing JSON wrapped in plain code block (no json tag)."""600        json_content = json.dumps(601            {602                dim: {"score": 90, "confidence": 85, "reasoning": "Good"}603                for dim in DIMENSIONS604            }605        )606        response = f"```\n{json_content}\n```"607        scores = parse_evaluation_response(response)608        assert len(scores) == len(DIMENSIONS)609610611class TestEvaluationStorageExtended:612    """Extended tests for EvaluationStorage class."""613614    def test_serialize_datetime(self) -> None:615        """Test datetime serialization helper."""616        with tempfile.TemporaryDirectory() as tmpdir:617            storage = EvaluationStorage(Path(tmpdir))618            dt = datetime(2024, 1, 15, 10, 30, 0)619            result = storage._serialize_datetime(dt)620            assert result == "2024-01-15T10:30:00"621622    def test_serialize_datetime_invalid_type(self) -> None:623        """Test datetime serialization with invalid type."""624        with tempfile.TemporaryDirectory() as tmpdir:625            storage = EvaluationStorage(Path(tmpdir))626            with pytest.raises(TypeError, match="not JSON serializable"):627                storage._serialize_datetime("not a datetime")628629    def test_get_flagged_verses_empty_book(self) -> None:630        """Test getting flagged verses when book directory doesn't exist."""631        with tempfile.TemporaryDirectory() as tmpdir:632            storage = EvaluationStorage(Path(tmpdir))633            flagged = storage.get_flagged_verses("nonexistent")634            assert flagged == []635636    def test_load_chapter_with_missing_fields(self) -> None:637        """Test loading chapter with some optional fields missing."""638        with tempfile.TemporaryDirectory() as tmpdir:639            storage = EvaluationStorage(Path(tmpdir))640641            # Create minimal evaluation file642            output_dir = Path(tmpdir) / "output" / "titus"643            output_dir.mkdir(parents=True)644            eval_file = output_dir / "chapter_01_eval.json"645646            data = {647                "book": "titus",648                "chapter": 1,649                "verses": {650                    "1": {651                        "scores": {652                            dim: {"score": 90, "confidence": 85, "reasoning": "Good"}653                            for dim in DIMENSIONS654                        },655                        "overall_score": 90.0,656                        "flags": [],657                    }658                },659            }660            with eval_file.open("w") as f:661                json.dump(data, f)662663            # Load should work even with missing optional fields664            loaded = storage.load_chapter_evaluation("titus", 1)665            assert loaded is not None666            assert loaded.book == "titus"667668669class TestTranslationEvaluatorExtended:670    """Extended tests for TranslationEvaluator class."""671672    def test_build_verse_prompt_with_context(self) -> None:673        """Test building verse prompt with full translation context."""674        from src.models.verse import (675            GlossaryTerm,676            TranslationContext,677            Verse,678            VerseReference,679        )680681        # Create a mock context682        ref = VerseReference(book="titus", chapter=1, verse=1)683        verse = Verse(684            reference=ref,685            greek_text="Παῦλος δοῦλος θεοῦ",686            greek_translit=None,687            hebrew_text=None,688            hebrew_translit=None,689            japanese_meiji=None,690            japanese_taisho="神の僕パウロ",691            english_kjv="Paul, a servant of God",692            english_asv=None,693        )694        glossary = [695            GlossaryTerm(696                original="θεός",697                transliteration="theos",698                japanese="神",699                english="God",700                notes=None,701            )702        ]703        context = TranslationContext(704            target_verse=verse,705            preceding_verses=[],706            following_verses=[],707            chapter_summary=None,708            book_metadata=None,709            glossary_terms=glossary,710        )711712        evaluator = TranslationEvaluator()713        prompt = evaluator.build_verse_prompt("神の僕パウロから", context)714715        assert "神の僕パウロから" in prompt716        assert "Παῦλος" in prompt717        assert "神" in prompt  # Glossary term718719    def test_build_verse_prompt_empty_glossary(self) -> None:720        """Test building verse prompt with empty glossary."""721        from src.models.verse import TranslationContext, Verse, VerseReference722723        ref = VerseReference(book="titus", chapter=1, verse=1)724        verse = Verse(725            reference=ref,726            greek_text="Test Greek",727            greek_translit=None,728            hebrew_text=None,729            hebrew_translit=None,730            japanese_meiji=None,731            japanese_taisho=None,732            english_kjv="Test English",733            english_asv=None,734        )735        context = TranslationContext(736            target_verse=verse,737            preceding_verses=[],738            following_verses=[],739            chapter_summary=None,740            book_metadata=None,741            glossary_terms=[],  # Empty glossary742        )743744        evaluator = TranslationEvaluator()745        prompt = evaluator.build_verse_prompt("テスト翻訳", context)746747        assert "テスト翻訳" in prompt748        assert "GLOSSARY TERMS" not in prompt  # Should not include glossary section749