software/src/models/evaluation.py

1"""Data models for translation evaluation."""23from __future__ import annotations45from datetime import datetime6from typing import TYPE_CHECKING78from pydantic import BaseModel, Field, computed_field910if TYPE_CHECKING:11    pass1213# Evaluation dimension names14DIMENSIONS = [15    "theological_accuracy",16    "source_fidelity",17    "naturalness",18    "readability",19    "glossary_consistency",20    "literary_quality",21    # Style-based dimensions22    "style_adherence",23    "formality_level",24    "archaic_avoidance",25]2627# Thresholds for flagging28THEOLOGICAL_ACCURACY_THRESHOLD = 80.029SOURCE_FIDELITY_THRESHOLD = 70.030ANY_DIMENSION_THRESHOLD = 60.031CONFIDENCE_THRESHOLD = 70.0323334class EvaluationScore(BaseModel):35    """Score for a single evaluation dimension."""3637    dimension: str = Field(..., description="Name of the evaluation dimension")38    score: float = Field(..., ge=0, le=100, description="Score as percentage (0-100)")39    confidence: float = Field(40        ..., ge=0, le=100, description="Confidence in the score (0-100)"41    )42    reasoning: str = Field(..., description="Brief explanation for the score")434445class VerseEvaluation(BaseModel):46    """Complete evaluation for a single verse."""4748    verse_number: int = Field(..., ge=1, description="Verse number")49    evaluator_model: str = Field(..., description="Model used for evaluation")50    timestamp: datetime = Field(51        default_factory=datetime.utcnow, description="When evaluation was performed"52    )53    scores: dict[str, EvaluationScore] = Field(54        ..., description="Scores by dimension name"55    )56    flags: list[str] = Field(57        default_factory=list, description="Flags for issues found"58    )59    notes: str | None = Field(None, description="Additional notes")6061    @computed_field62    @property63    def overall_score(self) -> float:64        """Calculate weighted average of all dimension scores."""65        if not self.scores:66            return 0.067        total = sum(s.score for s in self.scores.values())68        return round(total / len(self.scores), 1)6970    def compute_flags(self) -> list[str]:71        """Compute flags based on scores and thresholds."""72        flags = []7374        # Check theological accuracy75        if "theological_accuracy" in self.scores:76            ta = self.scores["theological_accuracy"]77            if ta.score < THEOLOGICAL_ACCURACY_THRESHOLD:78                flags.append("theological_concern")79            if ta.confidence < CONFIDENCE_THRESHOLD:80                flags.append("low_confidence_theology")8182        # Check source fidelity83        if "source_fidelity" in self.scores:84            sf = self.scores["source_fidelity"]85            if sf.score < SOURCE_FIDELITY_THRESHOLD:86                flags.append("source_fidelity_concern")8788        # Check any dimension below threshold89        for dim, score in self.scores.items():90            if score.score < ANY_DIMENSION_THRESHOLD:91                flags.append(f"low_{dim}")9293        return flags949596class ChapterEvaluation(BaseModel):97    """Aggregated evaluation for a chapter."""9899    book: str = Field(..., description="Book name")100    chapter: int = Field(..., ge=1, description="Chapter number")101    evaluator_model: str = Field(..., description="Model used for evaluation")102    evaluated_at: datetime = Field(103        default_factory=datetime.utcnow, description="When evaluation was performed"104    )105    verses: dict[int, VerseEvaluation] = Field(106        default_factory=dict, description="Evaluations by verse number"107    )108109    @computed_field110    @property111    def aggregate_scores(self) -> dict[str, float]:112        """Calculate average scores per dimension across all verses."""113        if not self.verses:114            return {}115116        dimension_totals: dict[str, list[float]] = {}117        for verse_eval in self.verses.values():118            for dim, score in verse_eval.scores.items():119                if dim not in dimension_totals:120                    dimension_totals[dim] = []121                dimension_totals[dim].append(score.score)122123        return {124            dim: round(sum(scores) / len(scores), 1)125            for dim, scores in dimension_totals.items()126        }127128    @computed_field129    @property130    def overall_score(self) -> float:131        """Calculate overall average score for the chapter."""132        if not self.verses:133            return 0.0134        total = sum(v.overall_score for v in self.verses.values())135        return round(total / len(self.verses), 1)136137    @computed_field138    @property139    def flagged_verses(self) -> list[int]:140        """Get list of verse numbers that have flags."""141        return [v_num for v_num, v_eval in self.verses.items() if v_eval.flags]142143    @computed_field144    @property145    def flagged_count(self) -> int:146        """Count of flagged verses."""147        return len(self.flagged_verses)148149150class BookEvaluationSummary(BaseModel):151    """Summary evaluation for an entire book."""152153    book: str = Field(..., description="Book name")154    total_chapters: int = Field(..., description="Total chapters evaluated")155    total_verses: int = Field(..., description="Total verses evaluated")156    aggregate_scores: dict[str, float] = Field(157        ..., description="Average scores per dimension"158    )159    overall_score: float = Field(..., description="Overall average score")160    flagged_verses: list[str] = Field(161        ..., description="List of flagged verse references (book:chapter:verse)"162    )163    evaluated_at: datetime = Field(164        default_factory=datetime.utcnow, description="When summary was generated"165    )166