"""Translation Memory management for segment storage and retrieval.""" from __future__ import annotations import fcntl import json import logging from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path from typing import Iterator from src.models.translation_memory import ( MatchType, SegmentLevel, TMMatch, TMSegment, TMStatistics, calculate_similarity, normalize_text, ) from src.models.verse import VerseReference logger = logging.getLogger(__name__) #: Default store location. Kept as a module constant so the test suite can #: redirect it — a test that constructs a manager with no path would otherwise #: read and write the real, irreplaceable production memory. DEFAULT_STORAGE_PATH = Path("data/memory/segments.json") class TranslationMemoryManager: """Manages translation memory segments for reuse. Provides storage, retrieval, and fuzzy matching of previously translated segments. """ def __init__(self, storage_path: Path | str | None = None) -> None: """Initialize the translation memory manager. Args: storage_path: Path to the storage file. If None, uses default. """ if storage_path is None: self._storage_path = DEFAULT_STORAGE_PATH else: self._storage_path = Path(storage_path) self._segments: dict[str, TMSegment] = {} self._by_reference: dict[str, list[str]] = {} # ref_str -> [segment_ids] self._loaded = False @property def storage_path(self) -> Path: """Get the storage file path.""" return self._storage_path def _ensure_loaded(self) -> None: """Ensure segments are loaded from storage.""" if not self._loaded: self.load() def add_segment(self, segment: TMSegment) -> None: """Add a segment to the translation memory. Args: segment: The segment to add. """ self._ensure_loaded() self._segments[segment.id] = segment # Index by reference ref_str = str(segment.reference) if ref_str not in self._by_reference: self._by_reference[ref_str] = [] if segment.id not in self._by_reference[ref_str]: self._by_reference[ref_str].append(segment.id) def add_from_translation( self, source_text: str, target_text: str, reference: VerseReference, level: SegmentLevel = SegmentLevel.VERSE, metadata: dict | None = None, ) -> TMSegment: """Create and add a segment from a translation result. Args: source_text: Original source text. target_text: Translated text. reference: Verse reference. level: Segment level. metadata: Additional metadata. Returns: The created TMSegment. """ segment = TMSegment.create( source_text=source_text, target_text=target_text, reference=reference, level=level, metadata=metadata, ) self.add_segment(segment) return segment def get_segment(self, segment_id: str) -> TMSegment | None: """Get a segment by ID. Args: segment_id: The segment ID. Returns: The segment if found, None otherwise. """ self._ensure_loaded() return self._segments.get(segment_id) def get_by_reference(self, reference: VerseReference) -> list[TMSegment]: """Get all segments for a verse reference. Args: reference: The verse reference. Returns: List of segments for that reference. """ self._ensure_loaded() ref_str = str(reference) segment_ids = self._by_reference.get(ref_str, []) return [self._segments[sid] for sid in segment_ids if sid in self._segments] def find_matches( self, source_text: str, threshold: float = 0.7, max_results: int = 5, ) -> list[TMMatch]: """Find matching segments for a source text. Args: source_text: The source text to match. threshold: Minimum similarity threshold (0.0-1.0). max_results: Maximum number of results to return. Returns: List of TMMatch objects sorted by similarity (highest first). """ self._ensure_loaded() if not self._segments: return [] normalized = normalize_text(source_text) matches = [] for segment in self._segments.values(): similarity = calculate_similarity(normalized, segment.source_normalized) if similarity >= threshold: if similarity == 1.0: match_type = MatchType.EXACT elif segment.level == SegmentLevel.PHRASE: match_type = MatchType.PHRASE else: match_type = MatchType.FUZZY matches.append( TMMatch( segment=segment, similarity=similarity, match_type=match_type, ) ) # Sort by similarity (highest first) matches.sort(key=lambda m: m.similarity, reverse=True) return matches[:max_results] def find_exact_match(self, source_text: str) -> TMSegment | None: """Find an exact match for a source text. Args: source_text: The source text to match. Returns: The matching segment if found, None otherwise. """ matches = self.find_matches(source_text, threshold=1.0, max_results=1) if matches and matches[0].is_exact: return matches[0].segment return None def increment_reuse(self, segment_id: str) -> bool: """Increment the reuse count for a segment. Args: segment_id: The segment ID. Returns: True if successful, False if segment not found. """ self._ensure_loaded() segment = self._segments.get(segment_id) if segment: segment.increment_reuse() return True return False def get_statistics(self) -> TMStatistics: """Get statistics about the translation memory. Returns: TMStatistics object. """ self._ensure_loaded() return TMStatistics.from_segments(list(self._segments.values())) def get_all_segments(self) -> list[TMSegment]: """Get all segments in the translation memory. Returns: List of all TMSegment objects. """ self._ensure_loaded() return list(self._segments.values()) def remove_segment(self, segment_id: str) -> bool: """Remove a segment from the translation memory. Args: segment_id: The segment ID to remove. Returns: True if removed, False if not found. """ self._ensure_loaded() segment = self._segments.pop(segment_id, None) if segment: ref_str = str(segment.reference) if ref_str in self._by_reference: self._by_reference[ref_str] = [ sid for sid in self._by_reference[ref_str] if sid != segment_id ] return True return False def clear(self) -> None: """Clear all segments from memory (not from storage).""" self._segments.clear() self._by_reference.clear() def save(self) -> None: """Merge in-memory segments into the stored file, atomically. Parallel book workers each hold their own manager over the same file, each loaded when its book started and each saving hours later. A plain overwrite therefore discards every other book's segments — the last writer wins and days of paid translation vanish silently. So the write is both: - **atomic** (temp file then rename), so a crash cannot leave a torn file that poisons every later run; and - **merging** (re-read under an exclusive lock, combine, then write), so a slow worker cannot roll back a fast one's work. On conflict the in-memory segment wins, being the more recent translation of that identical source text. """ self._ensure_loaded() self._storage_path.parent.mkdir(parents=True, exist_ok=True) with self._exclusive_lock(): merged = self._read_disk_segments() merged.update(self._segments) data = { "version": "1.0", "updated_at": datetime.now(timezone.utc).isoformat(), "segments": [ { "id": s.id, "source_text": s.source_text, "source_normalized": s.source_normalized, "target_text": s.target_text, "reference": str(s.reference), "level": s.level.value, "created_at": s.created_at.isoformat(), "reuse_count": s.reuse_count, "metadata": s.metadata, } for s in merged.values() ], } import tempfile tmp_fd, tmp_path = tempfile.mkstemp( dir=self._storage_path.parent, suffix=".tmp", ) try: with open(tmp_fd, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) Path(tmp_path).replace(self._storage_path) except BaseException: Path(tmp_path).unlink(missing_ok=True) raise self._segments = merged self._rebuild_reference_index() @contextmanager def _exclusive_lock(self) -> Iterator[None]: """Hold an exclusive lock over the store for the duration of the block. Uses a sidecar lock file rather than the store itself, so the atomic replace cannot swap the locked file out from under a held lock. Yields: None, once the lock is held. """ lock_path = self._storage_path.with_suffix( self._storage_path.suffix + ".lock" ) lock_path.parent.mkdir(parents=True, exist_ok=True) handle = lock_path.open("w") try: fcntl.flock(handle.fileno(), fcntl.LOCK_EX) yield finally: fcntl.flock(handle.fileno(), fcntl.LOCK_UN) handle.close() def _rebuild_reference_index(self) -> None: """Rebuild the reference index from the current segments.""" self._by_reference.clear() for segment in self._segments.values(): ref_str = str(segment.reference) self._by_reference.setdefault(ref_str, []).append(segment.id) def _quarantine_corrupt_file(self) -> Path: """Move an unreadable store aside so a later save cannot overwrite it. Returns: The path the corrupt file was moved to. """ stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") target = self._storage_path.with_name( f"{self._storage_path.stem}.corrupt.{stamp}.json" ) self._storage_path.replace(target) return target def _read_disk_segments(self) -> dict[str, TMSegment]: """Read stored segments without disturbing in-memory state. A corrupt file is quarantined rather than ignored. Ignoring it meant the next save silently replaced days of accumulated translation with whatever the current process happened to hold. Returns: The segments currently on disk, keyed by id. """ if not self._storage_path.exists(): return {} try: with self._storage_path.open("r", encoding="utf-8") as f: data = json.load(f) if not isinstance(data, dict): # Valid JSON of the wrong shape is just as unreadable; without # this it crashed load/save with an AttributeError instead. raise json.JSONDecodeError("not a JSON object", "", 0) except json.JSONDecodeError: quarantined = self._quarantine_corrupt_file() logger.error( "Translation memory file was unreadable; preserved at %s. " "Starting from empty rather than overwriting it.", quarantined, ) return {} segments: dict[str, TMSegment] = {} for seg_data in data.get("segments", []): reference = VerseReference.from_string(seg_data["reference"]) segment = TMSegment( id=seg_data["id"], source_text=seg_data["source_text"], source_normalized=seg_data["source_normalized"], target_text=seg_data["target_text"], reference=reference, level=SegmentLevel(seg_data["level"]), created_at=datetime.fromisoformat(seg_data["created_at"]), reuse_count=seg_data.get("reuse_count", 0), metadata=seg_data.get("metadata", {}), ) segments[segment.id] = segment return segments def load(self) -> None: """Load segments from the storage file.""" self._loaded = True self._segments = self._read_disk_segments() self._rebuild_reference_index() def reload(self) -> None: """Force reload from storage, discarding in-memory changes.""" self._loaded = False self.load() def __len__(self) -> int: """Get the number of segments in memory.""" self._ensure_loaded() return len(self._segments) def format_matches_for_prompt( self, matches: list[TMMatch], max_display: int = 3 ) -> str: """Format TM matches for inclusion in a translation prompt. Args: matches: List of TM matches. max_display: Maximum number of matches to display. Returns: Formatted string for prompt inclusion. """ if not matches: return "No similar translations found in memory." lines = ["Similar translations from memory:"] for i, match in enumerate(matches[:max_display]): similarity_pct = int(match.similarity * 100) seg = match.segment lines.append( f" [{similarity_pct}% match] {seg.reference}: {seg.target_text}" ) if len(matches) > max_display: lines.append(f" ... and {len(matches) - max_display} more") return "\n".join(lines)