"""Data models for translation evaluation.""" from __future__ import annotations from datetime import datetime from typing import TYPE_CHECKING from pydantic import BaseModel, Field, computed_field if TYPE_CHECKING: pass # Evaluation dimension names DIMENSIONS = [ "theological_accuracy", "source_fidelity", "naturalness", "readability", "glossary_consistency", "literary_quality", # Style-based dimensions "style_adherence", "formality_level", "archaic_avoidance", ] # Thresholds for flagging THEOLOGICAL_ACCURACY_THRESHOLD = 80.0 SOURCE_FIDELITY_THRESHOLD = 70.0 ANY_DIMENSION_THRESHOLD = 60.0 CONFIDENCE_THRESHOLD = 70.0 class EvaluationScore(BaseModel): """Score for a single evaluation dimension.""" dimension: str = Field(..., description="Name of the evaluation dimension") score: float = Field(..., ge=0, le=100, description="Score as percentage (0-100)") confidence: float = Field( ..., ge=0, le=100, description="Confidence in the score (0-100)" ) reasoning: str = Field(..., description="Brief explanation for the score") class VerseEvaluation(BaseModel): """Complete evaluation for a single verse.""" verse_number: int = Field(..., ge=1, description="Verse number") evaluator_model: str = Field(..., description="Model used for evaluation") timestamp: datetime = Field( default_factory=datetime.utcnow, description="When evaluation was performed" ) scores: dict[str, EvaluationScore] = Field( ..., description="Scores by dimension name" ) flags: list[str] = Field( default_factory=list, description="Flags for issues found" ) notes: str | None = Field(None, description="Additional notes") @computed_field @property def overall_score(self) -> float: """Calculate weighted average of all dimension scores.""" if not self.scores: return 0.0 total = sum(s.score for s in self.scores.values()) return round(total / len(self.scores), 1) def compute_flags(self) -> list[str]: """Compute flags based on scores and thresholds.""" flags = [] # Check theological accuracy if "theological_accuracy" in self.scores: ta = self.scores["theological_accuracy"] if ta.score < THEOLOGICAL_ACCURACY_THRESHOLD: flags.append("theological_concern") if ta.confidence < CONFIDENCE_THRESHOLD: flags.append("low_confidence_theology") # Check source fidelity if "source_fidelity" in self.scores: sf = self.scores["source_fidelity"] if sf.score < SOURCE_FIDELITY_THRESHOLD: flags.append("source_fidelity_concern") # Check any dimension below threshold for dim, score in self.scores.items(): if score.score < ANY_DIMENSION_THRESHOLD: flags.append(f"low_{dim}") return flags class ChapterEvaluation(BaseModel): """Aggregated evaluation for a chapter.""" book: str = Field(..., description="Book name") chapter: int = Field(..., ge=1, description="Chapter number") evaluator_model: str = Field(..., description="Model used for evaluation") evaluated_at: datetime = Field( default_factory=datetime.utcnow, description="When evaluation was performed" ) verses: dict[int, VerseEvaluation] = Field( default_factory=dict, description="Evaluations by verse number" ) @computed_field @property def aggregate_scores(self) -> dict[str, float]: """Calculate average scores per dimension across all verses.""" if not self.verses: return {} dimension_totals: dict[str, list[float]] = {} for verse_eval in self.verses.values(): for dim, score in verse_eval.scores.items(): if dim not in dimension_totals: dimension_totals[dim] = [] dimension_totals[dim].append(score.score) return { dim: round(sum(scores) / len(scores), 1) for dim, scores in dimension_totals.items() } @computed_field @property def overall_score(self) -> float: """Calculate overall average score for the chapter.""" if not self.verses: return 0.0 total = sum(v.overall_score for v in self.verses.values()) return round(total / len(self.verses), 1) @computed_field @property def flagged_verses(self) -> list[int]: """Get list of verse numbers that have flags.""" return [v_num for v_num, v_eval in self.verses.items() if v_eval.flags] @computed_field @property def flagged_count(self) -> int: """Count of flagged verses.""" return len(self.flagged_verses) class BookEvaluationSummary(BaseModel): """Summary evaluation for an entire book.""" book: str = Field(..., description="Book name") total_chapters: int = Field(..., description="Total chapters evaluated") total_verses: int = Field(..., description="Total verses evaluated") aggregate_scores: dict[str, float] = Field( ..., description="Average scores per dimension" ) overall_score: float = Field(..., description="Overall average score") flagged_verses: list[str] = Field( ..., description="List of flagged verse references (book:chapter:verse)" ) evaluated_at: datetime = Field( default_factory=datetime.utcnow, description="When summary was generated" )