software/src/models/translation_memory.py

1"""Translation Memory models for segment storage and matching."""23from __future__ import annotations45import re6import unicodedata7from datetime import datetime8from enum import Enum9from uuid import uuid41011from pydantic import BaseModel, Field1213from src.models.verse import VerseReference141516class SegmentLevel(str, Enum):17    """Level of granularity for a translation segment."""1819    VERSE = "verse"  # Full verse translation20    PHRASE = "phrase"  # Sub-verse phrase212223class TMSegment(BaseModel):24    """A translation memory segment storing a source-target pair."""2526    id: str = Field(default_factory=lambda: str(uuid4()))27    source_text: str = Field(..., description="Original Greek/Hebrew text")28    source_normalized: str = Field(29        ..., description="Normalized form for matching"30    )31    target_text: str = Field(..., description="Japanese translation")32    reference: VerseReference = Field(33        ..., description="Source verse reference"34    )35    level: SegmentLevel = Field(36        default=SegmentLevel.VERSE, description="Segment granularity"37    )38    created_at: datetime = Field(default_factory=datetime.utcnow)39    reuse_count: int = Field(40        default=0, ge=0, description="Number of times this segment was reused"41    )42    metadata: dict = Field(43        default_factory=dict,44        description="Additional metadata (model, prompt_version, etc.)",45    )4647    @classmethod48    def create(49        cls,50        source_text: str,51        target_text: str,52        reference: VerseReference,53        level: SegmentLevel = SegmentLevel.VERSE,54        metadata: dict | None = None,55    ) -> TMSegment:56        """Create a new segment with automatic normalization.5758        Args:59            source_text: Original source text.60            target_text: Translated target text.61            reference: Verse reference.62            level: Segment granularity level.63            metadata: Additional metadata.6465        Returns:66            New TMSegment instance.67        """68        return cls(69            source_text=source_text,70            source_normalized=normalize_text(source_text),71            target_text=target_text,72            reference=reference,73            level=level,74            metadata=metadata or {},75        )7677    def increment_reuse(self) -> None:78        """Increment the reuse counter."""79        self.reuse_count += 1808182class MatchType(str, Enum):83    """Type of translation memory match."""8485    EXACT = "exact"  # 100% match86    FUZZY = "fuzzy"  # Partial match (configurable threshold)87    PHRASE = "phrase"  # Phrase-level match within verse888990class TMMatch(BaseModel):91    """A match result from translation memory lookup."""9293    segment: TMSegment = Field(..., description="The matched segment")94    similarity: float = Field(95        ..., ge=0.0, le=1.0, description="Match similarity score"96    )97    match_type: MatchType = Field(..., description="Type of match")9899    @property100    def is_exact(self) -> bool:101        """Check if this is an exact match."""102        return self.match_type == MatchType.EXACT103104    @property105    def is_usable(self) -> bool:106        """Check if this match is usable (above minimum threshold)."""107        return self.similarity >= 0.7108109110def normalize_text(text: str) -> str:111    """Normalize text for matching.112113    Normalizes Unicode, removes punctuation, lowercases,114    and collapses whitespace.115116    Args:117        text: Text to normalize.118119    Returns:120        Normalized text string.121    """122    # Unicode normalization (NFC)123    text = unicodedata.normalize("NFC", text)124125    # Lowercase126    text = text.lower()127128    # Remove punctuation (keep letters, numbers, spaces)129    text = re.sub(r"[^\w\s]", "", text)130131    # Collapse whitespace132    text = " ".join(text.split())133134    return text135136137def calculate_similarity(text1: str, text2: str) -> float:138    """Calculate similarity between two texts using Levenshtein ratio.139140    Args:141        text1: First text (normalized).142        text2: Second text (normalized).143144    Returns:145        Similarity score between 0.0 and 1.0.146    """147    if text1 == text2:148        return 1.0149150    if not text1 or not text2:151        return 0.0152153    # Use character-level Levenshtein distance ratio154    len1, len2 = len(text1), len(text2)155    max_len = max(len1, len2)156157    # Unreachable: two empty strings are equal and one empty string returned158    # 0.0 above. Kept as a guard against a zero division if that changes.159    if max_len == 0:  # pragma: no cover160        return 1.0161162    # Simple Levenshtein implementation163    distance = _levenshtein_distance(text1, text2)164    return 1.0 - (distance / max_len)165166167def _levenshtein_distance(s1: str, s2: str) -> int:168    """Calculate Levenshtein distance between two strings.169170    Args:171        s1: First string.172        s2: Second string.173174    Returns:175        Edit distance between the strings.176    """177    if len(s1) < len(s2):178        return _levenshtein_distance(s2, s1)179180    if len(s2) == 0:181        return len(s1)182183    previous_row = range(len(s2) + 1)184    for i, c1 in enumerate(s1):185        current_row = [i + 1]186        for j, c2 in enumerate(s2):187            insertions = previous_row[j + 1] + 1188            deletions = current_row[j] + 1189            substitutions = previous_row[j] + (c1 != c2)190            current_row.append(min(insertions, deletions, substitutions))191        previous_row = current_row192193    return previous_row[-1]194195196class TMStatistics(BaseModel):197    """Statistics about the translation memory."""198199    total_segments: int = 0200    verse_segments: int = 0201    phrase_segments: int = 0202    total_reuses: int = 0203    average_reuse: float = 0.0204    books_covered: list[str] = Field(default_factory=list)205    most_reused_segments: list[str] = Field(default_factory=list)206207    @classmethod208    def from_segments(cls, segments: list[TMSegment]) -> TMStatistics:209        """Calculate statistics from a list of segments.210211        Args:212            segments: List of TM segments.213214        Returns:215            TMStatistics instance.216        """217        if not segments:218            return cls()219220        verse_count = sum(1 for s in segments if s.level == SegmentLevel.VERSE)221        phrase_count = sum(1 for s in segments if s.level == SegmentLevel.PHRASE)222        total_reuses = sum(s.reuse_count for s in segments)223        avg_reuse = total_reuses / len(segments) if segments else 0.0224225        books = sorted(set(s.reference.book for s in segments))226227        # Get most reused segments (top 5)228        sorted_by_reuse = sorted(segments, key=lambda s: s.reuse_count, reverse=True)229        most_reused = [s.id for s in sorted_by_reuse[:5] if s.reuse_count > 0]230231        return cls(232            total_segments=len(segments),233            verse_segments=verse_count,234            phrase_segments=phrase_count,235            total_reuses=total_reuses,236            average_reuse=avg_reuse,237            books_covered=books,238            most_reused_segments=most_reused,239        )240