"""Translation Memory models for segment storage and matching.""" from __future__ import annotations import re import unicodedata from datetime import datetime from enum import Enum from uuid import uuid4 from pydantic import BaseModel, Field from src.models.verse import VerseReference class SegmentLevel(str, Enum): """Level of granularity for a translation segment.""" VERSE = "verse" # Full verse translation PHRASE = "phrase" # Sub-verse phrase class TMSegment(BaseModel): """A translation memory segment storing a source-target pair.""" id: str = Field(default_factory=lambda: str(uuid4())) source_text: str = Field(..., description="Original Greek/Hebrew text") source_normalized: str = Field( ..., description="Normalized form for matching" ) target_text: str = Field(..., description="Japanese translation") reference: VerseReference = Field( ..., description="Source verse reference" ) level: SegmentLevel = Field( default=SegmentLevel.VERSE, description="Segment granularity" ) created_at: datetime = Field(default_factory=datetime.utcnow) reuse_count: int = Field( default=0, ge=0, description="Number of times this segment was reused" ) metadata: dict = Field( default_factory=dict, description="Additional metadata (model, prompt_version, etc.)", ) @classmethod def create( cls, source_text: str, target_text: str, reference: VerseReference, level: SegmentLevel = SegmentLevel.VERSE, metadata: dict | None = None, ) -> TMSegment: """Create a new segment with automatic normalization. Args: source_text: Original source text. target_text: Translated target text. reference: Verse reference. level: Segment granularity level. metadata: Additional metadata. Returns: New TMSegment instance. """ return cls( source_text=source_text, source_normalized=normalize_text(source_text), target_text=target_text, reference=reference, level=level, metadata=metadata or {}, ) def increment_reuse(self) -> None: """Increment the reuse counter.""" self.reuse_count += 1 class MatchType(str, Enum): """Type of translation memory match.""" EXACT = "exact" # 100% match FUZZY = "fuzzy" # Partial match (configurable threshold) PHRASE = "phrase" # Phrase-level match within verse class TMMatch(BaseModel): """A match result from translation memory lookup.""" segment: TMSegment = Field(..., description="The matched segment") similarity: float = Field( ..., ge=0.0, le=1.0, description="Match similarity score" ) match_type: MatchType = Field(..., description="Type of match") @property def is_exact(self) -> bool: """Check if this is an exact match.""" return self.match_type == MatchType.EXACT @property def is_usable(self) -> bool: """Check if this match is usable (above minimum threshold).""" return self.similarity >= 0.7 def normalize_text(text: str) -> str: """Normalize text for matching. Normalizes Unicode, removes punctuation, lowercases, and collapses whitespace. Args: text: Text to normalize. Returns: Normalized text string. """ # Unicode normalization (NFC) text = unicodedata.normalize("NFC", text) # Lowercase text = text.lower() # Remove punctuation (keep letters, numbers, spaces) text = re.sub(r"[^\w\s]", "", text) # Collapse whitespace text = " ".join(text.split()) return text def calculate_similarity(text1: str, text2: str) -> float: """Calculate similarity between two texts using Levenshtein ratio. Args: text1: First text (normalized). text2: Second text (normalized). Returns: Similarity score between 0.0 and 1.0. """ if text1 == text2: return 1.0 if not text1 or not text2: return 0.0 # Use character-level Levenshtein distance ratio len1, len2 = len(text1), len(text2) max_len = max(len1, len2) # Unreachable: two empty strings are equal and one empty string returned # 0.0 above. Kept as a guard against a zero division if that changes. if max_len == 0: # pragma: no cover return 1.0 # Simple Levenshtein implementation distance = _levenshtein_distance(text1, text2) return 1.0 - (distance / max_len) def _levenshtein_distance(s1: str, s2: str) -> int: """Calculate Levenshtein distance between two strings. Args: s1: First string. s2: Second string. Returns: Edit distance between the strings. """ if len(s1) < len(s2): return _levenshtein_distance(s2, s1) if len(s2) == 0: return len(s1) previous_row = range(len(s2) + 1) for i, c1 in enumerate(s1): current_row = [i + 1] for j, c2 in enumerate(s2): insertions = previous_row[j + 1] + 1 deletions = current_row[j] + 1 substitutions = previous_row[j] + (c1 != c2) current_row.append(min(insertions, deletions, substitutions)) previous_row = current_row return previous_row[-1] class TMStatistics(BaseModel): """Statistics about the translation memory.""" total_segments: int = 0 verse_segments: int = 0 phrase_segments: int = 0 total_reuses: int = 0 average_reuse: float = 0.0 books_covered: list[str] = Field(default_factory=list) most_reused_segments: list[str] = Field(default_factory=list) @classmethod def from_segments(cls, segments: list[TMSegment]) -> TMStatistics: """Calculate statistics from a list of segments. Args: segments: List of TM segments. Returns: TMStatistics instance. """ if not segments: return cls() verse_count = sum(1 for s in segments if s.level == SegmentLevel.VERSE) phrase_count = sum(1 for s in segments if s.level == SegmentLevel.PHRASE) total_reuses = sum(s.reuse_count for s in segments) avg_reuse = total_reuses / len(segments) if segments else 0.0 books = sorted(set(s.reference.book for s in segments)) # Get most reused segments (top 5) sorted_by_reuse = sorted(segments, key=lambda s: s.reuse_count, reverse=True) most_reused = [s.id for s in sorted_by_reuse[:5] if s.reuse_count > 0] return cls( total_segments=len(segments), verse_segments=verse_count, phrase_segments=phrase_count, total_reuses=total_reuses, average_reuse=avg_reuse, books_covered=books, most_reused_segments=most_reused, )