alignment.py

1"""Apply reviewed reference routes to a release candidate, retaining every conflict.23This never edits the translation harness's source-numbered output. An audit is4bound to a complete input snapshot; no lexical similarity score authorizes edits.5"""6from __future__ import annotations78import hashlib9import json10import re11from collections import Counter, defaultdict12from pathlib import Path1314from src.output.originals import load_site151617def digest(value):18    return hashlib.sha256(value.encode()).hexdigest()192021def read_output(directory):22    records, files = {}, {}23    for path in sorted(Path(directory).glob('*/chapter_*.json')):24        content = path.read_bytes()25        files[str(path.relative_to(directory))] = hashlib.sha256(content).hexdigest()26        data = json.loads(content)27        for verse, record in data['verses'].items():28            key = f'{path.parent.name}:{data["chapter"]}:{int(verse)}'29            if key in records or not record['text'].strip():30                raise ValueError(f'Duplicate or empty translation: {key}')31            records[key] = record['text']32    if not records:33        raise ValueError('No translations found')34    return records, files353637def ref_order(ref):38    book, ch, verse = ref.split(':')39    return book, int(ch), int(verse)404142def candidate_hashes(choices):43    return sorted(({'source': item['source'], 'text_sha256': digest(item['text'])}44                   for item in choices), key=lambda item: ref_order(item['source']))454647def validate_repair(target, repair, choices):48    if repair['expected_candidates'] != candidate_hashes(choices):49        raise ValueError(f'Repair candidates changed: {target}')50    text = repair['text']51    if not isinstance(text, str) or not text.strip() or re.search(r'[\x00-\x1f\x7f<>\ufffd]|```', text):52        raise ValueError(f'Invalid repaired text: {target}')53    if not repair.get('reason') or not repair.get('evidence_sha256') or not repair.get('verified_by'):54        raise ValueError(f'Repair lacks review evidence: {target}')55    return text565758def route_record(source, text, review):59    """Return lossless character slices, or hold a record requiring more review."""60    if review is None:61        return [(source, text)]62    if review['text_sha256'] != digest(text):63        raise ValueError(f'Reviewed text changed: {source}')64    routes = review.get('routes')65    if routes is None:66        decision = review['decision']67        if decision == 'same':68            return [(source, text)]69        if decision == 'mapped' and len(review['mapped_targets']) == 1:70            return [(review['mapped_targets'][0], text)]71        if decision not in ('mapped', 'mixed', 'unclear'):72            raise ValueError(f'Unknown review decision: {source}')73        return []74    cursor, result = 0, []75    for route in routes:76        start, end = route['start'], route['end']77        if start != cursor or not start < end <= len(text):78            raise ValueError(f'Character loss or duplication: {source}')79        result.append((route['target'], text[start:end]))80        cursor = end81    if cursor != len(text):82        raise ValueError(f'Unassigned characters: {source}')83    return result848586def align(records, site_refs, review):87    candidates, held = defaultdict(list), {}88    reviewed = review['records']89    if reviewed.keys() - records.keys():90        raise ValueError('Review contains unknown source references')91    for source, text in records.items():92        routes = route_record(source, text, reviewed.get(source))93        if not routes:94            held[source] = {'text': text, 'review': reviewed[source]}95        for target, part in routes:96            book, ch, verse = ref_order(target)97            if target not in site_refs and not (verse == 0 and f'{book}:{ch}:1' in site_refs):98                raise ValueError(f'Route outside site: {source} -> {target}')99            candidates[target].append({'source': source, 'text': part})100    resolved, conflicts, dispositions = {}, {}, {}101    repairs = review.get('target_repairs', {})102    for target in repairs:103        if target not in site_refs:104            raise ValueError(f'Repair outside site: {target}')105        candidates.setdefault(target, [])106    resolutions = review.get('resolutions', {})107    if resolutions.keys() - candidates.keys():108        raise ValueError('Resolution has no candidates')109    for target, choices in sorted(candidates.items(), key=lambda item: ref_order(item[0])):110        choices.sort(key=lambda item: ref_order(item['source']))111        if target in repairs:112            repair = repairs[target]113            text = validate_repair(target, repair, choices)114            dispositions[target] = {'reason': repair['reason'], 'all_candidates': choices,115                                    'repair': repair}116            resolved[target] = {'text': text, 'sources': [item['source'] for item in choices],117                                'restored': not choices, 'reviewed_repair': True}118            continue119        resolution = resolutions.get(target)120        if resolution:121            actual = [item['source'] for item in choices]122            if sorted(actual) != sorted(resolution['expected_candidates']):123                raise ValueError(f'Resolution candidates changed: {target}')124            by_source = {item['source']: item for item in choices}125            selected = resolution['sources']126            if len(set(actual)) != len(actual) or not selected or len(set(selected)) != len(selected) or set(selected) - by_source.keys():127                raise ValueError(f'Invalid selection: {target}')128            parts = [by_source[source] for source in selected]129            dispositions[target] = {'reason': resolution['reason'], 'selected': selected, 'all_candidates': choices}130        elif target.endswith(':0'):131            parts = choices  # Consecutive, separately numbered superscription lines.132        elif len({item['text'] for item in choices}) == 1:133            parts = choices[:1]134            if len(choices) > 1:135                dispositions[target] = {'reason': 'Identical text at the same destination', 'selected': [parts[0]['source']], 'all_candidates': choices}136        else:137            conflicts[target] = choices138            continue139        resolved[target] = {'text': ''.join(item['text'] for item in parts), 'sources': [item['source'] for item in parts]}140    missing = sorted(site_refs - candidates.keys(), key=ref_order)141    extras = {}142    for source in sorted(records.keys() - site_refs, key=ref_order):143        targets = [t for t, choices in candidates.items() if any(item['source'] == source for item in choices)]144        extras[source] = {'text': records[source], 'targets': targets,145                          'basis': reviewed[source]['basis'], 'decision': reviewed[source]['decision'],146                          'status': 'held' if source in held else ('conflict' if any(t in conflicts for t in targets) else 'resolved'),147                          'destinations': {t: {'candidates': candidates[t], 'selected': resolved.get(t)} for t in targets}}148    changes = {key: value for key, value in resolved.items() if value['sources'] != [key] or value['text'] != records.get(key)}149    return {'schema': 1, 'translation_name': '日常訳第1版',150            'numbering_ready': not (missing or conflicts or held),151            'summary': {'source_records': len(records), 'site_references': len(site_refs),152                        'reviewed_sensitive_records': len(reviewed), 'reviewers': dict(Counter(r['reviewer'] for r in reviewed.values())),153                        'resolved_site_references': len(site_refs & resolved.keys()), 'aligned_changes': len(changes),154                        'missing_passages': len(missing), 'conflicting_destinations': len(conflicts),155                        'held_records': len(held), 'extra_references': len(extras)},156            'aligned_changes': changes, 'missing_passages': missing, 'conflicts': conflicts,157            'held_records': held, 'resolutions': dispositions, 'extra_references': extras}158159160def build_audit(source, review_path, out):161    source, review_path, out = Path(source), Path(review_path), Path(out).resolve()162    if out.is_relative_to(source.resolve()) or out == review_path.resolve():163        raise ValueError('Audit must not overwrite its inputs')164    records, files = read_output(source)165    review = json.loads(review_path.read_text())166    if files != review['input_files']:167        raise ValueError('Translation snapshot changed; refresh the affected reviews')168    site, _ = load_site()169    site_refs = {':'.join(map(str, key)) for key in site}170    # Bind the reference text too, since numbering alone cannot prove alignment.171    site_hash = digest(json.dumps({':'.join(map(str, k)): v for k, v in site.items()}, ensure_ascii=False, sort_keys=True))172    if review['site_sha256'] != site_hash:173        raise ValueError('Site reference text changed; refresh the review')174    for relative, expected in review.get('repair_source_files', {}).items():175        path = (source.parent / relative).resolve()176        if not path.is_relative_to((source.parent / 'sources').resolve()):177            raise ValueError('Repair source path outside sources directory')178        if hashlib.sha256(path.read_bytes()).hexdigest() != expected:179            raise ValueError(f'Repair source text changed: {relative}')180    result = align(records, site_refs, review)181    result['missing_reference_text'] = {ref: site[ref_order(ref)] for ref in result['missing_passages']}182    result['input_files'] = files183    result['site_sha256'] = site_hash184    result['review_sha256'] = hashlib.sha256(review_path.read_bytes()).hexdigest()185    result['usage'] = review['usage']186    out.parent.mkdir(parents=True, exist_ok=True)187    out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')188    return result189190191def export_aligned(source, audit_path, out):192    """Write a complete, separately numbered release; never mutate raw records."""193    source, audit_path, out = Path(source), Path(audit_path), Path(out).resolve()194    if out.is_relative_to(source.resolve()) or source.resolve().is_relative_to(out):195        raise ValueError('Release must be separate from raw translation output')196    if out.exists() and any(out.iterdir()):197        raise ValueError('Release destination must be empty')198    records, files = read_output(source)199    audit = json.loads(audit_path.read_text())200    if not audit['numbering_ready'] or files != audit['input_files'] or any(201        audit.get(key) for key in ('missing_passages', 'conflicts', 'held_records')202    ):203        raise ValueError('Numbering audit incomplete or source snapshot changed')204    site, codes = load_site()205    site_hash = digest(json.dumps({':'.join(map(str, k)): v for k, v in site.items()}, ensure_ascii=False, sort_keys=True))206    if audit['site_sha256'] != site_hash:207        raise ValueError('Release reference text changed')208    books = {book: {'schema': 1, 'book': code, 'translation': '日常訳第1版', 'chapters': {}} for book, code in codes.items()}209    changes = audit['aligned_changes']210    for key in sorted(site):211        book, ch, verse = key212        ref = ':'.join(map(str, key))213        text = changes[ref]['text'] if ref in changes else records.get(ref)214        if not text:215            raise ValueError(f'Missing release verse: {ref}')216        chapter = books[book]['chapters'].setdefault(str(ch), {'heading': '', 'verses': {}})217        chapter['verses'][str(verse)] = text218    for ref, change in changes.items():219        book, ch, verse = ref_order(ref)220        if verse == 0:221            books[book]['chapters'][str(ch)]['heading'] = change['text']222    out.mkdir(parents=True, exist_ok=True)223    hashes = {}224    for book, payload in books.items():225        filename = codes[book] + '.json'226        text = json.dumps(payload, ensure_ascii=False, separators=(',', ':')) + '\n'227        (out / filename).write_text(text)228        hashes[filename] = digest(text)229    manifest = {'schema': 1, 'translation': '日常訳第1版', 'verses': len(site),230                'chapters': len({k[:2] for k in site}), 'books': len(books), 'files': hashes,231                'audit_sha256': hashlib.sha256(audit_path.read_bytes()).hexdigest()}232    (out / 'manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')233    return manifest234